Ceph

"waiting_on_backfill": 0,
"backfill_pos": "0\/\/0\/\/-1",
"backfill_info": { "begin": "0\/\/0\/\/-1",
"end": "0\/\/0\/\/-1",
"objects": []},
"peer_backfill_info": { "begin": "0\/\/0\/\/-1",
"end": "0\/\/0\/\/-1",
"objects": []},
"backfills_in_flight": [],
"pull_from_peer": [],
"pushing": []},
"scrub": { "scrubber.epoch_start": "0",
"scrubber.active": 0,
"scrubber.block_writes": 0,
"scrubber.finalizing": 0,
"scrubber.waiting_on": 0,
"scrubber.waiting_on_whom": []}}, { "name": "Started",
"enter_time": "2013-04-11 14:57:02.207148"}]}

2013-04-11T14:57:32.542 INFO:teuthology.task.peer:pg is {u'last_scrub': u"0'0", u'last_clean_scrub_stamp': u'0.000000', u'log_start': u"0'0", u'last_active': u'2013-04-11 14:57:03.235413', u'log_size': 0, u'last_deep_scrub': u"0'0", u'parent_split_bits': 0, u'stats_invalid': u'0', u'ondisk_log_size': 0, u'mapping_epoch': 5, u'state': u'active+recovery_wait', u'version': u"6'1", u'pgid': u'1.b', u'parent': u'0.0', u'reported': u"3'21", u'last_epoch_clean': 1, u'last_deep_scrub_stamp': u'0.000000', u'stat_cat_sum': {}, u'last_fresh': u'2013-04-11 14:57:03.235413', u'last_change': u'2013-04-11 14:57:03.235413', u'created': 1, u'up': [0, 2], u'stat_sum': {u'num_objects_unfound': 0, u'num_objects_missing_on_primary': 0, u'num_write': 1, u'num_object_clones': 0, u'num_objects': 1, u'num_object_copies': 2, u'num_bytes': 22, u'num_read_kb': 0, u'num_bytes_recovered': 0, u'num_scrub_errors': 0, u'num_read': 0, u'num_keys_recovered': 0, u'num_write_kb': 1, u'num_objects_degraded': 2, u'num_objects_recovered': 0}, u'acting': [0, 2], u'last_clean': u'2013-04-11 14:56:48.140787', u'last_unstale': u'2013-04-11 14:57:03.235413', u'last_scrub_stamp': u'0.000000', u'ondisk_log_start': u"0'0"}, query json is {u'info': {u'last_backfill': u'MAX', u'dne': 0, u'pgid': u'1.b', u'log_tail': u"0'0", u'last_update': u"6'1", u'purged_snaps': u'[]', u'last_complete': u"6'1", u'incomplete': 0, u'stats': {u'last_scrub': u"0'0", u'last_clean_scrub_stamp': u'0.000000', u'log_start': u"0'0", u'last_active': u'2013-04-11 14:57:03.235413', u'log_size': 0, u'last_deep_scrub': u"0'0", u'parent_split_bits': 0, u'stats_invalid': u'0', u'ondisk_log_size': 0, u'mapping_epoch': 5, u'state': u'active+recovery_wait', u'version': u"6'1", u'parent': u'0.0', u'reported': u"3'21", u'last_epoch_clean': 1, u'last_deep_scrub_stamp': u'0.000000', u'stat_cat_sum': {}, u'last_fresh': u'2013-04-11 14:57:03.235413', u'last_change': u'2013-04-11 14:57:03.235413', u'created': 1, u'up': [0, 2], u'stat_sum': {u'num_objects_unfound': 0, u'num_objects_missing_on_primary': 0, u'num_write': 1, u'num_object_clones': 0, u'num_objects': 1, u'num_object_copies': 0, u'num_bytes': 22, u'num_read_kb': 0, u'num_bytes_recovered': 0, u'num_scrub_errors': 0, u'num_read': 0, u'num_keys_recovered': 0, u'num_write_kb': 1, u'num_objects_degraded': 0, u'num_objects_recovered': 0}, u'acting': [0, 2], u'last_clean': u'2013-04-11 14:56:48.140787', u'last_unstale': u'2013-04-11 14:57:03.235413', u'last_scrub_stamp': u'0.000000', u'ondisk_log_start': u"0'0"}, u'last_epoch_started': 9, u'empty': 0, u'history': {u'last_scrub': u"0'0", u'epoch_created': 1, u'last_clean_scrub_stamp': u'0.000000', u'last_deep_scrub_stamp': u'0.000000', u'same_interval_since': 8, u'same_primary_since': 3, u'last_epoch_split': 0, u'same_up_since': 8, u'last_deep_scrub': u"0'0", u'last_epoch_clean': 6, u'last_epoch_started': 9, u'last_scrub_stamp': u'0.000000'}}, u'up': [0, 2], u'state': u'active+recovering', u'epoch': 9, u'recovery_state': [{u'recovery_progress': {u'pushing': [], u'backfill_info': {u'begin': u'0//0//-1', u'objects': [], u'end': u'0//0//-1'}, u'pull_from_peer': [], u'peer_backfill_info': {u'begin': u'0//0//-1', u'objects': [], u'end': u'0//0//-1'}, u'waiting_on_backfill': 0, u'backfills_in_flight': [], u'backfill_target': -1, u'backfill_pos': u'0//0//-1'}, u'scrub': {u'scrubber.block_writes': 0, u'scrubber.finalizing': 0, u'scrubber.active': 0, u'scrubber.epoch_start': u'0', u'scrubber.waiting_on_whom': [], u'scrubber.waiting_on': 0}, u'enter_time': u'2013-04-11 14:57:03.221842', u'name': u'Started/Primary/Active', u'might_have_unfound': []}, {u'enter_time': u'2013-04-11 14:57:02.207148', u'name': u'Started'}], u'acting': [0, 2]}
2013-04-11T14:57:32.543 ERROR:teuthology.run_tasks:Saw exception from tasks
Traceback (most recent call last):
File "/var/lib/teuthworker/teuthology-master/teuthology/run_tasks.py", line 25, in run_tasks
manager = _run_one_task(taskname, ctx=ctx, config=config)
File "/var/lib/teuthworker/teuthology-master/teuthology/run_tasks.py", line 14, in _run_one_task
return fn(*kwargs)
File "/var/lib/teuthworker/teuthology-master/teuthology/task/peer.py", line 93, in task
assert normalize_state(j['state']) == normalize_state(pg['state'])
AssertionError
2013-04-11T14:57:32.621 DEBUG:teuthology.run_tasks:Unwinding manager <contextlib.GeneratorContextManager object at 0x1795990>
2013-04-11T14:57:32.622 ERROR:teuthology.contextutil:Saw exception from nested tasks
Traceback (most recent call last):
File "/var/lib/teuthworker/teuthology-master/teuthology/contextutil.py", line 27, in nested
yield vars
File "/var/lib/teuthworker/teuthology-master/teuthology/task/ceph.py", line 1112, in task
yield
File "/var/lib/teuthworker/teuthology-master/teuthology/run_tasks.py", line 25, in run_tasks
manager = _run_one_task(taskname, ctx=ctx, config=config)
File "/var/lib/teuthworker/teuthology-master/teuthology/run_tasks.py", line 14, in _run_one_task
return fn(*kwargs)
File "/var/lib/teuthworker/teuthology-master/teuthology/task/peer.py", line 93, in task
assert normalize_state(j['state']) == normalize_state(pg['state'])
AssertionError
2013-04-11T14:57:32.623 INFO:teuthology.task.ceph:Shutting down mds daemons...
2013-04-11T14:57:32.623 DEBUG:teuthology.task.ceph.mds.a:waiting for process to exit
2013-04-11T14:57:32.635 INFO:teuthology.task.ceph.mds.a:Stopped
2013-04-11T14:57:32.635 INFO:teuthology.task.ceph:Shutting down osd daemons...
2013-04-11T14:57:32.635 ERROR:teuthology.task.ceph.osd.1:tried to stop a non-running daemon
2013-04-11T14:57:32.636 DEBUG:teuthology.task.ceph.osd.0:waiting for process to exit
2013-04-11T14:57:32.683 INFO:teuthology.task.ceph.osd.0:Stopped
2013-04-11T14:57:32.684 DEBUG:teuthology.task.ceph.osd.2:waiting for process to exit