Project

General

Profile

Download (47.2 KB)

Bug #9752 » log.txt

pg query with duplicates - Loïc Dachary, 10/22/2014 10:49 AM

 
{ "state": "incomplete",
"epoch": 8984,
"up": [
13,
15],
"acting": [
13,
15],
"info": { "pgid": "3.222",
"last_update": "2886'32405",
"last_complete": "2886'32405",
"log_tail": "1193'29405",
"last_user_version": 0,
"last_backfill": "0\/\/0\/\/-1",
"purged_snaps": "[]",
"history": { "epoch_created": 188,
"last_epoch_started": 6394,
"last_epoch_clean": 4671,
"last_epoch_split": 0,
"same_up_since": 8737,
"same_interval_since": 8737,
"same_primary_since": 8727,
"last_scrub": "2886'32405",
"last_scrub_stamp": "2014-10-20 09:55:32.038134",
"last_deep_scrub": "2886'32405",
"last_deep_scrub_stamp": "2014-10-20 09:55:32.038134",
"last_clean_scrub_stamp": "2014-10-20 09:55:32.038134"},
"stats": { "version": "2886'32405",
"reported_seq": "678",
"reported_epoch": "8984",
"state": "incomplete",
"last_fresh": "2014-10-21 18:39:01.949044",
"last_change": "2014-10-21 12:18:49.582389",
"last_active": "0.000000",
"last_clean": "0.000000",
"last_became_active": "0.000000",
"last_unstale": "2014-10-21 18:39:01.949044",
"mapping_epoch": 8729,
"log_start": "1193'29405",
"ondisk_log_start": "1193'29405",
"created": 188,
"last_epoch_clean": 4671,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "2886'32405",
"last_scrub_stamp": "2014-10-20 09:55:32.038134",
"last_deep_scrub": "2886'32405",
"last_deep_scrub_stamp": "2014-10-20 09:55:32.038134",
"last_clean_scrub_stamp": "2014-10-20 09:55:32.038134",
"log_size": 3000,
"ondisk_log_size": 3000,
"stats_invalid": "0",
"stat_sum": { "num_bytes": 0,
"num_objects": 0,
"num_object_clones": 0,
"num_object_copies": 0,
"num_objects_missing_on_primary": 0,
"num_objects_degraded": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 0,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 0,
"num_write_kb": 0,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0},
"stat_cat_sum": {},
"up": [
13,
15],
"acting": [
13,
15],
"up_primary": 13,
"acting_primary": 13},
"empty": 0,
"dne": 0,
"incomplete": 1,
"last_epoch_started": 0,
"hit_set_history": { "current_last_update": "0'0",
"current_last_stamp": "0.000000",
"current_info": { "begin": "0.000000",
"end": "0.000000",
"version": "0'0"},
"history": []}},
"peer_info": [
{ "peer": "10",
"pgid": "3.222",
"last_update": "0'0",
"last_complete": "0'0",
"log_tail": "0'0",
"last_user_version": 0,
"last_backfill": "MAX",
"purged_snaps": "[]",
"history": { "epoch_created": 0,
"last_epoch_started": 0,
"last_epoch_clean": 0,
"last_epoch_split": 0,
"same_up_since": 0,
"same_interval_since": 0,
"same_primary_since": 0,
"last_scrub": "0'0",
"last_scrub_stamp": "0.000000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "0.000000",
"last_clean_scrub_stamp": "0.000000"},
"stats": { "version": "0'0",
"reported_seq": "0",
"reported_epoch": "0",
"state": "inactive",
"last_fresh": "0.000000",
"last_change": "0.000000",
"last_active": "0.000000",
"last_clean": "0.000000",
"last_became_active": "0.000000",
"last_unstale": "0.000000",
"mapping_epoch": 0,
"log_start": "0'0",
"ondisk_log_start": "0'0",
"created": 0,
"last_epoch_clean": 0,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "0'0",
"last_scrub_stamp": "0.000000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "0.000000",
"last_clean_scrub_stamp": "0.000000",
"log_size": 0,
"ondisk_log_size": 0,
"stats_invalid": "0",
"stat_sum": { "num_bytes": 0,
"num_objects": 0,
"num_object_clones": 0,
"num_object_copies": 0,
"num_objects_missing_on_primary": 0,
"num_objects_degraded": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 0,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 0,
"num_write_kb": 0,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0},
"stat_cat_sum": {},
"up": [],
"acting": [],
"up_primary": -1,
"acting_primary": -1},
"empty": 1,
"dne": 1,
"incomplete": 0,
"last_epoch_started": 0,
"hit_set_history": { "current_last_update": "0'0",
"current_last_stamp": "0.000000",
"current_info": { "begin": "0.000000",
"end": "0.000000",
"version": "0'0"},
"history": []}},
{ "peer": "15",
"pgid": "3.222",
"last_update": "2886'32405",
"last_complete": "2886'32405",
"log_tail": "1193'29405",
"last_user_version": 0,
"last_backfill": "0\/\/0\/\/-1",
"purged_snaps": "[]",
"history": { "epoch_created": 188,
"last_epoch_started": 6394,
"last_epoch_clean": 4671,
"last_epoch_split": 0,
"same_up_since": 8737,
"same_interval_since": 8737,
"same_primary_since": 8727,
"last_scrub": "2886'32405",
"last_scrub_stamp": "2014-10-20 09:55:32.038134",
"last_deep_scrub": "2886'32405",
"last_deep_scrub_stamp": "2014-10-20 09:55:32.038134",
"last_clean_scrub_stamp": "2014-10-20 09:55:32.038134"},
"stats": { "version": "2886'32405",
"reported_seq": "11",
"reported_epoch": "8726",
"state": "down+peering",
"last_fresh": "2014-10-21 12:16:29.464666",
"last_change": "2014-10-21 12:16:17.520959",
"last_active": "0.000000",
"last_clean": "0.000000",
"last_became_active": "0.000000",
"last_unstale": "2014-10-21 12:16:29.464666",
"mapping_epoch": 8729,
"log_start": "1193'29405",
"ondisk_log_start": "1193'29405",
"created": 188,
"last_epoch_clean": 4671,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "2886'32405",
"last_scrub_stamp": "2014-10-20 09:55:32.038134",
"last_deep_scrub": "2886'32405",
"last_deep_scrub_stamp": "2014-10-20 09:55:32.038134",
"last_clean_scrub_stamp": "2014-10-20 09:55:32.038134",
"log_size": 3000,
"ondisk_log_size": 3000,
"stats_invalid": "0",
"stat_sum": { "num_bytes": 0,
"num_objects": 0,
"num_object_clones": 0,
"num_object_copies": 0,
"num_objects_missing_on_primary": 0,
"num_objects_degraded": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 0,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 0,
"num_write_kb": 0,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0},
"stat_cat_sum": {},
"up": [
13,
15],
"acting": [
13,
15],
"up_primary": 13,
"acting_primary": 13},
"empty": 0,
"dne": 0,
"incomplete": 1,
"last_epoch_started": 0,
"hit_set_history": { "current_last_update": "0'0",
"current_last_stamp": "0.000000",
"current_info": { "begin": "0.000000",
"end": "0.000000",
"version": "0'0"},
"history": []}},
{ "peer": "17",
"pgid": "3.222",
"last_update": "2886'32405",
"last_complete": "2886'32405",
"log_tail": "1193'29405",
"last_user_version": 0,
"last_backfill": "0\/\/0\/\/-1",
"purged_snaps": "[]",
"history": { "epoch_created": 188,
"last_epoch_started": 4689,
"last_epoch_clean": 4671,
"last_epoch_split": 0,
"same_up_since": 8737,
"same_interval_since": 8737,
"same_primary_since": 8727,
"last_scrub": "2886'32405",
"last_scrub_stamp": "2014-10-20 09:55:32.038134",
"last_deep_scrub": "2886'32405",
"last_deep_scrub_stamp": "2014-10-20 09:55:32.038134",
"last_clean_scrub_stamp": "2014-10-20 09:55:32.038134"},
"stats": { "version": "0'0",
"reported_seq": "0",
"reported_epoch": "0",
"state": "inactive",
"last_fresh": "0.000000",
"last_change": "0.000000",
"last_active": "0.000000",
"last_clean": "0.000000",
"last_became_active": "0.000000",
"last_unstale": "0.000000",
"mapping_epoch": 8729,
"log_start": "0'0",
"ondisk_log_start": "0'0",
"created": 0,
"last_epoch_clean": 0,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "0'0",
"last_scrub_stamp": "0.000000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "0.000000",
"last_clean_scrub_stamp": "0.000000",
"log_size": 0,
"ondisk_log_size": 0,
"stats_invalid": "0",
"stat_sum": { "num_bytes": 0,
"num_objects": 0,
"num_object_clones": 0,
"num_object_copies": 0,
"num_objects_missing_on_primary": 0,
"num_objects_degraded": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 0,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 0,
"num_write_kb": 0,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0},
"stat_cat_sum": {},
"up": [
13,
15],
"acting": [
13,
15],
"up_primary": 13,
"acting_primary": 13},
"empty": 0,
"dne": 0,
"incomplete": 1,
"last_epoch_started": 0,
"hit_set_history": { "current_last_update": "0'0",
"current_last_stamp": "0.000000",
"current_info": { "begin": "0.000000",
"end": "0.000000",
"version": "0'0"},
"history": []}},
{ "peer": "25",
"pgid": "3.222",
"last_update": "2886'32405",
"last_complete": "2886'32405",
"log_tail": "1193'29405",
"last_user_version": 0,
"last_backfill": "MAX",
"purged_snaps": "[]",
"history": { "epoch_created": 188,
"last_epoch_started": 6394,
"last_epoch_clean": 4671,
"last_epoch_split": 0,
"same_up_since": 8737,
"same_interval_since": 8737,
"same_primary_since": 8727,
"last_scrub": "2886'32405",
"last_scrub_stamp": "2014-10-20 09:55:32.038134",
"last_deep_scrub": "2886'32405",
"last_deep_scrub_stamp": "2014-10-20 09:55:32.038134",
"last_clean_scrub_stamp": "2014-10-20 09:55:32.038134"},
"stats": { "version": "2886'32405",
"reported_seq": "131515",
"reported_epoch": "8191",
"state": "peering",
"last_fresh": "2014-10-21 09:49:51.160398",
"last_change": "2014-10-21 09:49:51.160398",
"last_active": "2014-10-20 10:37:20.378869",
"last_clean": "2014-10-20 10:37:20.378869",
"last_became_active": "0.000000",
"last_unstale": "2014-10-21 09:49:51.160398",
"mapping_epoch": 8729,
"log_start": "1193'29405",
"ondisk_log_start": "1193'29405",
"created": 188,
"last_epoch_clean": 4671,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "2886'32405",
"last_scrub_stamp": "2014-10-20 09:55:32.038134",
"last_deep_scrub": "2886'32405",
"last_deep_scrub_stamp": "2014-10-20 09:55:32.038134",
"last_clean_scrub_stamp": "2014-10-20 09:55:32.038134",
"log_size": 3000,
"ondisk_log_size": 3000,
"stats_invalid": "0",
"stat_sum": { "num_bytes": 7080120320,
"num_objects": 1697,
"num_object_clones": 0,
"num_object_copies": 3394,
"num_objects_missing_on_primary": 0,
"num_objects_degraded": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 1697,
"num_whiteouts": 0,
"num_read": 72828,
"num_read_kb": 8794722,
"num_write": 32405,
"num_write_kb": 11424120,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 1687,
"num_bytes_recovered": 7038177280,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0},
"stat_cat_sum": {},
"up": [
13,
15],
"acting": [
13,
15],
"up_primary": 13,
"acting_primary": 13},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 6394,
"hit_set_history": { "current_last_update": "0'0",
"current_last_stamp": "0.000000",
"current_info": { "begin": "0.000000",
"end": "0.000000",
"version": "0'0"},
"history": []}}],
"recovery_state": [
{ "name": "Started\/Primary\/Peering",
"enter_time": "2014-10-21 12:18:48.482666",
"past_intervals": [
{ "first": 4663,
"last": 4685,
"maybe_went_rw": 1,
"up": [],
"acting": [
10,
25,
10,
-1]},
{ "first": 4686,
"last": 4698,
"maybe_went_rw": 1,
"up": [
17],
"acting": [
10,
25,
10,
17]},
{ "first": 4699,
"last": 4712,
"maybe_went_rw": 0,
"up": [
17],
"acting": [
10,
10,
17]},
{ "first": 4713,
"last": 4715,
"maybe_went_rw": 1,
"up": [
17],
"acting": [
10,
25,
10,
17]},
{ "first": 4716,
"last": 5617,
"maybe_went_rw": 1,
"up": [
13,
15],
"acting": [
10,
25,
10,
13]},
{ "first": 5618,
"last": 6387,
"maybe_went_rw": 0,
"up": [
13,
15],
"acting": [
25,
25,
13]},
{ "first": 6388,
"last": 7117,
"maybe_went_rw": 1,
"up": [
13,
15],
"acting": [
10,
25,
10,
13]},
{ "first": 7118,
"last": 8193,
"maybe_went_rw": 0,
"up": [
13,
15],
"acting": [
25,
25,
13]},
{ "first": 8194,
"last": 8194,
"maybe_went_rw": 0,
"up": [
13,
15],
"acting": [
13,
15,
13,
13]},
{ "first": 8195,
"last": 8195,
"maybe_went_rw": 0,
"up": [
13],
"acting": [
13,
13,
13]},
{ "first": 8196,
"last": 8196,
"maybe_went_rw": 0,
"up": [
13,
15],
"acting": [
13,
15,
13,
13]},
{ "first": 8197,
"last": 8197,
"maybe_went_rw": 0,
"up": [
15],
"acting": [
15,
15,
15]},
{ "first": 8198,
"last": 8718,
"maybe_went_rw": 1,
"up": [
13,
15],
"acting": [
13,
15,
13,
13]},
{ "first": 8719,
"last": 8726,
"maybe_went_rw": 0,
"up": [
15],
"acting": [
15,
15,
15]},
{ "first": 8727,
"last": 8728,
"maybe_went_rw": 1,
"up": [
13,
15],
"acting": [
13,
15,
13,
13]},
{ "first": 8729,
"last": 8736,
"maybe_went_rw": 0,
"up": [
13],
"acting": [
13,
13,
13]}],
"probing_osds": [
"10",
"13",
"15",
"25"],
"down_osds_we_would_probe": [],
"peering_blocked_by": []},
{ "name": "Started",
"enter_time": "2014-10-21 12:18:48.482562"}],
"agent_state": {}}ceph@agreus:/etc/ceph$ ceph -f json-pretty pg 3.222 query | less

{ "state": "incomplete",
"epoch": 8984,
"up": [
13,
15],
"acting": [
13,
15],
"info": { "pgid": "3.222",
"last_update": "2886'32405",
"last_complete": "2886'32405",
"log_tail": "1193'29405",
"last_user_version": 0,
"last_backfill": "0\/\/0\/\/-1",
"purged_snaps": "[]",
"history": { "epoch_created": 188,
"last_update": "2886'32405",
"last_complete": "2886'32405",
"log_tail": "1193'29405",
"last_user_version": 0,
"last_backfill": "0\/\/0\/\/-1",
"purged_snaps": "[]",
"history": { "epoch_created": 188,
"last_epoch_started": 6394,
"last_epoch_clean": 4671,
"last_epoch_split": 0,
"same_up_since": 8737,
"same_interval_since": 8737,
"same_primary_since": 8727,
"last_scrub": "2886'32405",
"last_scrub_stamp": "2014-10-20 09:55:32.038134",
"last_deep_scrub": "2886'32405",
"last_deep_scrub_stamp": "2014-10-20 09:55:32.038134",
"last_clean_scrub_stamp": "2014-10-20 09:55:32.038134"},
"stats": { "version": "2886'32405",
"reported_seq": "678",
"reported_epoch": "8984",
"state": "incomplete",
"last_fresh": "2014-10-21 18:39:01.949044",
"last_change": "2014-10-21 12:18:49.582389",
"last_active": "0.000000",
"last_clean": "0.000000",
"last_became_active": "0.000000",
"last_unstale": "2014-10-21 18:39:01.949044",
"mapping_epoch": 8729,
"log_start": "1193'29405",
"ondisk_log_start": "1193'29405",
"created": 188,
"last_epoch_clean": 4671,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "2886'32405",
"last_scrub_stamp": "2014-10-20 09:55:32.038134",
"last_deep_scrub": "2886'32405",
"last_deep_scrub_stamp": "2014-10-20 09:55:32.038134",
"last_clean_scrub_stamp": "2014-10-20 09:55:32.038134",
"log_size": 3000,
"ondisk_log_size": 3000,
"stats_invalid": "0",
"stat_sum": { "num_bytes": 0,
"num_objects": 0,
"num_object_clones": 0,
"num_object_copies": 0,
"num_objects_missing_on_primary": 0,
"num_objects_degraded": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 0,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 0,
"num_write_kb": 0,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0},
"stat_cat_sum": {},
"up": [
13,
15],
"acting": [
13,
15],
"up_primary": 13,
"acting_primary": 13},
"empty": 0,
"dne": 0,
"incomplete": 1,
"last_epoch_started": 0,
"hit_set_history": { "current_last_update": "0'0",
"current_last_stamp": "0.000000",
"current_info": { "begin": "0.000000",
"end": "0.000000",
"version": "0'0"},
"history": []}},
"peer_info": [
{ "peer": "10",
"pgid": "3.222",
"last_update": "0'0",
"last_complete": "0'0",
"log_tail": "0'0",
"last_user_version": 0,
"last_backfill": "MAX",
"purged_snaps": "[]",
"history": { "epoch_created": 0,
"last_epoch_started": 0,
"last_epoch_clean": 0,
"last_epoch_split": 0,
"same_up_since": 0,
"same_interval_since": 0,
"same_primary_since": 0,
"last_scrub": "0'0",
"last_scrub_stamp": "0.000000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "0.000000",
"last_clean_scrub_stamp": "0.000000"},
"stats": { "version": "0'0",
"reported_seq": "0",
"reported_epoch": "0",
"state": "inactive",
"last_fresh": "0.000000",
"last_change": "0.000000",
"last_active": "0.000000",
"last_clean": "0.000000",
"last_became_active": "0.000000",
"last_unstale": "0.000000",
"mapping_epoch": 0,
"log_start": "0'0",
"ondisk_log_start": "0'0",
"created": 0,
"last_epoch_clean": 0,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "0'0",
"last_scrub_stamp": "0.000000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "0.000000",
"last_clean_scrub_stamp": "0.000000",
"log_size": 0,
"ondisk_log_size": 0,
"stats_invalid": "0",
"stat_sum": { "num_bytes": 0,
"num_objects": 0,
"num_object_clones": 0,
"num_object_copies": 0,
"num_objects_missing_on_primary": 0,
"num_objects_degraded": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 0,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 0,
"last_deep_scrub": "2886'32405",
"last_deep_scrub_stamp": "2014-10-20 09:55:32.038134",
"last_clean_scrub_stamp": "2014-10-20 09:55:32.038134",
"log_size": 3000,
"ondisk_log_size": 3000,
"stats_invalid": "0",
"stat_sum": { "num_bytes": 0,
"num_objects": 0,
"num_object_clones": 0,
"num_object_copies": 0,
"num_objects_missing_on_primary": 0,
"num_objects_degraded": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 0,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 0,
"num_write_kb": 0,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0},
"stat_cat_sum": {},
"up": [
13,
15],
"acting": [
13,
15],
"up_primary": 13,
"acting_primary": 13},
"empty": 0,
"dne": 0,
"incomplete": 1,
"last_epoch_started": 0,
"hit_set_history": { "current_last_update": "0'0",
"current_last_stamp": "0.000000",
"current_info": { "begin": "0.000000",
"end": "0.000000",
"version": "0'0"},
"history": []}},
"peer_info": [
{ "peer": "10",
"pgid": "3.222",
"last_update": "0'0",
"last_complete": "0'0",
"log_tail": "0'0",
"last_user_version": 0,
"last_backfill": "MAX",
"purged_snaps": "[]",
"history": { "epoch_created": 0,
"last_epoch_started": 0,
"last_epoch_clean": 0,
"last_epoch_split": 0,
"same_up_since": 0,
"same_interval_since": 0,
"same_primary_since": 0,
"last_scrub": "0'0",
"last_scrub_stamp": "0.000000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "0.000000",
"last_clean_scrub_stamp": "0.000000"},
"stats": { "version": "0'0",
"reported_seq": "0",
"reported_epoch": "0",
"state": "inactive",
"last_fresh": "0.000000",
"last_change": "0.000000",
"last_active": "0.000000",
"last_clean": "0.000000",
"last_became_active": "0.000000",
"last_unstale": "0.000000",
"mapping_epoch": 0,
"log_start": "0'0",
"ondisk_log_start": "0'0",
"created": 0,
"last_epoch_clean": 0,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "0'0",
"last_scrub_stamp": "0.000000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "0.000000",
"last_clean_scrub_stamp": "0.000000",
"log_size": 0,
"ondisk_log_size": 0,
"stats_invalid": "0",
"stat_sum": { "num_bytes": 0,
"num_objects": 0,
"num_object_clones": 0,
"num_object_copies": 0,
"num_objects_missing_on_primary": 0,
"num_objects_degraded": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 0,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 0,
"num_write_kb": 0,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0},
"stat_cat_sum": {},
"up": [],
"acting": [],
"up_primary": -1,
"acting_primary": -1},
"empty": 1,
"dne": 1,
"incomplete": 0,
"last_epoch_started": 0,
"hit_set_history": { "current_last_update": "0'0",
"current_last_stamp": "0.000000",
"current_info": { "begin": "0.000000",
"end": "0.000000",
"version": "0'0"},
"history": []}},
{ "peer": "15",
"pgid": "3.222",
"last_update": "2886'32405",
"last_complete": "2886'32405",
"log_tail": "1193'29405",
"last_user_version": 0,
"last_backfill": "0\/\/0\/\/-1",
"purged_snaps": "[]",
"history": { "epoch_created": 188,
"last_epoch_started": 6394,
"last_epoch_clean": 4671,
"last_epoch_split": 0,
"same_up_since": 8737,
"same_interval_since": 8737,
"same_primary_since": 8727,
"last_scrub": "2886'32405",
"last_scrub_stamp": "2014-10-20 09:55:32.038134",
"last_deep_scrub": "2886'32405",
"last_deep_scrub_stamp": "2014-10-20 09:55:32.038134",
"last_clean_scrub_stamp": "2014-10-20 09:55:32.038134"},
"stats": { "version": "2886'32405",
"reported_seq": "11",
"reported_epoch": "8726",
"state": "down+peering",
"last_fresh": "2014-10-21 12:16:29.464666",
"last_change": "2014-10-21 12:16:17.520959",
"last_active": "0.000000",
"last_clean": "0.000000",
"last_became_active": "0.000000",
"last_unstale": "2014-10-21 12:16:29.464666",
"mapping_epoch": 8729,
"log_start": "1193'29405",
"ondisk_log_start": "1193'29405",
"created": 188,
"last_epoch_clean": 4671,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "2886'32405",
"last_scrub_stamp": "2014-10-20 09:55:32.038134",
"last_deep_scrub": "2886'32405",
"last_deep_scrub_stamp": "2014-10-20 09:55:32.038134",
"last_clean_scrub_stamp": "2014-10-20 09:55:32.038134",
"log_size": 3000,
"ondisk_log_size": 3000,
"stats_invalid": "0",
"stat_sum": { "num_bytes": 0,
"num_objects": 0,
"num_object_clones": 0,
"num_object_copies": 0,
"num_objects_missing_on_primary": 0,
"num_objects_degraded": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 0,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 0,
"num_write_kb": 0,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0},
"stat_cat_sum": {},
"up": [
13,
15],
"acting": [
13,
15],
"up_primary": 13,
"acting_primary": 13},
"empty": 0,
"dne": 0,
"incomplete": 1,
"last_epoch_started": 0,
"hit_set_history": { "current_last_update": "0'0",
"current_last_stamp": "0.000000",
"current_info": { "begin": "0.000000",
"end": "0.000000",
"version": "0'0"},
"history": []}},
{ "peer": "17",
"pgid": "3.222",
"last_update": "2886'32405",
"last_complete": "2886'32405",
"log_tail": "1193'29405",
"last_user_version": 0,
"last_backfill": "0\/\/0\/\/-1",
"purged_snaps": "[]",
"history": { "epoch_created": 188,
"last_epoch_started": 4689,
"last_epoch_clean": 4671,
"last_epoch_split": 0,
"same_up_since": 8737,
"same_interval_since": 8737,
"same_primary_since": 8727,
"last_scrub": "2886'32405",
"last_scrub_stamp": "2014-10-20 09:55:32.038134",
"last_deep_scrub": "2886'32405",
"last_deep_scrub_stamp": "2014-10-20 09:55:32.038134",
"last_clean_scrub_stamp": "2014-10-20 09:55:32.038134"},
"stats": { "version": "0'0",
"reported_seq": "0",
"reported_epoch": "0",
"state": "inactive",
"last_fresh": "0.000000",
"last_change": "0.000000",
"last_active": "0.000000",
"last_clean": "0.000000",
"last_became_active": "0.000000",
"last_unstale": "0.000000",
"mapping_epoch": 8729,
"log_start": "0'0",
"ondisk_log_start": "0'0",
"created": 0,
"last_epoch_clean": 0,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "0'0",
"last_scrub_stamp": "0.000000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "0.000000",
"last_clean_scrub_stamp": "0.000000",
"log_size": 0,
"ondisk_log_size": 0,
"stats_invalid": "0",
"stat_sum": { "num_bytes": 0,
"num_objects": 0,
"num_object_clones": 0,
"num_object_copies": 0,
"num_objects_missing_on_primary": 0,
"num_objects_degraded": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 0,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 0,
"num_write_kb": 0,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0},
"stat_cat_sum": {},
"up": [
13,
15],
"acting": [
13,
15],
"up_primary": 13,
"acting_primary": 13},
"empty": 0,
"dne": 0,
"incomplete": 1,
"last_epoch_started": 0,
"hit_set_history": { "current_last_update": "0'0",
"current_last_stamp": "0.000000",
"current_info": { "begin": "0.000000",
"end": "0.000000",
"version": "0'0"},
"history": []}},
{ "peer": "25",
"pgid": "3.222",
"last_update": "2886'32405",
"last_complete": "2886'32405",
"log_tail": "1193'29405",
"last_user_version": 0,
"last_backfill": "MAX",
"purged_snaps": "[]",
"history": { "epoch_created": 188,
"last_epoch_started": 6394,
"last_epoch_clean": 4671,
"last_epoch_split": 0,
"same_up_since": 8737,
"same_interval_since": 8737,
"same_primary_since": 8727,
"last_scrub": "2886'32405",
"last_scrub_stamp": "2014-10-20 09:55:32.038134",
"last_deep_scrub": "2886'32405",
"last_deep_scrub_stamp": "2014-10-20 09:55:32.038134",
"last_clean_scrub_stamp": "2014-10-20 09:55:32.038134"},
"stats": { "version": "2886'32405",
"reported_seq": "131515",
"reported_epoch": "8191",
"state": "peering",
"last_fresh": "2014-10-21 09:49:51.160398",
"last_change": "2014-10-21 09:49:51.160398",
"last_active": "2014-10-20 10:37:20.378869",
"last_clean": "2014-10-20 10:37:20.378869",
"last_became_active": "0.000000",
"last_unstale": "2014-10-21 09:49:51.160398",
"mapping_epoch": 8729,
"log_start": "1193'29405",
"ondisk_log_start": "1193'29405",
"created": 188,
"last_epoch_clean": 4671,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "2886'32405",
"last_scrub_stamp": "2014-10-20 09:55:32.038134",
"last_deep_scrub": "2886'32405",
"last_deep_scrub_stamp": "2014-10-20 09:55:32.038134",
"last_clean_scrub_stamp": "2014-10-20 09:55:32.038134",
"log_size": 3000,
"ondisk_log_size": 3000,
"stats_invalid": "0",
"stat_sum": { "num_bytes": 7080120320,
"num_objects": 1697,
"num_object_clones": 0,
"num_object_copies": 3394,
"num_objects_missing_on_primary": 0,
"num_objects_degraded": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 1697,
"num_whiteouts": 0,
"num_read": 72828,
"num_read_kb": 8794722,
"num_write": 32405,
"num_write_kb": 11424120,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 1687,
"num_bytes_recovered": 7038177280,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0},
"stat_cat_sum": {},
"up": [
13,
15],
"acting": [
13,
15],
"up_primary": 13,
"acting_primary": 13},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 6394,
"hit_set_history": { "current_last_update": "0'0",
"current_last_stamp": "0.000000",
"current_info": { "begin": "0.000000",
"end": "0.000000",
"version": "0'0"},
"history": []}}],
"recovery_state": [
{ "name": "Started\/Primary\/Peering",
"enter_time": "2014-10-21 12:18:48.482666",
"past_intervals": [
{ "first": 4663,
"last": 4685,
"maybe_went_rw": 1,
"up": [],
"acting": [
10,
25,
10,
-1]},
{ "first": 4686,
"last": 4698,
"maybe_went_rw": 1,
"up": [
17],
"acting": [
10,
25,
10,
17]},
{ "first": 4699,
"last": 4712,
"maybe_went_rw": 0,
"up": [
17],
"acting": [
10,
10,
17]},
{ "first": 4713,
"last": 4715,
"maybe_went_rw": 1,
"up": [
17],
"acting": [
10,
25,
10,
17]},
{ "first": 4716,
"last": 5617,
"maybe_went_rw": 1,
"up": [
13,
15],
"acting": [
10,
25,
10,
13]},
{ "first": 5618,
"last": 6387,
"maybe_went_rw": 0,
"up": [
13,
15],
"acting": [
25,
25,
13]},
{ "first": 6388,
"last": 7117,
"maybe_went_rw": 1,
"up": [
13,
15],
"acting": [
10,
25,
10,
13]},
{ "first": 7118,
"last": 8193,
"maybe_went_rw": 0,
"up": [
13,
15],
"acting": [
25,
25,
13]},
{ "first": 8194,
"last": 8194,
"maybe_went_rw": 0,
"up": [
13,
15],
"acting": [
13,
15,
13,
13]},
{ "first": 8195,
"last": 8195,
"maybe_went_rw": 0,
"up": [
13],
"acting": [
13,
13,
13]},
{ "first": 8196,
"last": 8196,
"maybe_went_rw": 0,
"up": [
13,
15],
"acting": [
13,
15,
13,
13]},
{ "first": 8197,
"last": 8197,
"maybe_went_rw": 0,
"up": [
15],
"acting": [
15,
15,
15]},
{ "first": 8198,
"last": 8718,
"maybe_went_rw": 1,
"up": [
13,
15],
"acting": [
13,
15,
13,
13]},
{ "first": 8719,
"last": 8726,
"maybe_went_rw": 0,
"up": [
15],
"acting": [
15,
15,
15]},
{ "first": 8727,
"last": 8728,
"maybe_went_rw": 1,
"up": [
13,
15],
"acting": [
13,
15,
13,
13]},
{ "first": 8729,
"last": 8736,
"maybe_went_rw": 0,
"up": [
13],
"acting": [
13,
13,
13]}],
"probing_osds": [
"10",
"13",
"15",
"25"],
"down_osds_we_would_probe": [],
"peering_blocked_by": []},
{ "name": "Started",
"enter_time": "2014-10-21 12:18:48.482562"}],
"agent_state": {}}
(1-1/9)