2014-12-08 18:13:53.031099 7f291ac78900 10 osd.70 71581 pgid 19.6e coll 19.6e_head 2014-12-08 18:13:53.031314 7f291ac78900 20 osd.70 0 get_map 71456 - loading and decoding 0x4457000 2014-12-08 18:13:53.043827 7f291ac78900 10 osd.70 0 add_map_bl 71456 60749 bytes 2014-12-08 18:13:53.044362 7f291ac78900 10 osd.70 71581 _open_lock_pg 19.6e 2014-12-08 18:13:53.044379 7f291ac78900 10 osd.70 71581 _get_pool 19 2014-12-08 18:13:53.044433 7f291ac78900 5 osd.70 pg_epoch: 71456 pg[19.6e(unlocked)] enter Initial 2014-12-08 18:13:53.044448 7f291ac78900 20 osd.70 pg_epoch: 71456 pg[19.6e(unlocked)] enter NotTrimming 2014-12-08 18:13:53.044689 7f291ac78900 10 read_log 2014-12-08 18:13:53.044851 7f291ac78900 20 read_log 50495'48906593 (50495'48906384) modify 54b74ee/rbd_data.12896844d9a421.000000000001a8b8/head//19 by client.1214845.0:4802589437 2014-11-24 12:54:18.683780 2014-12-08 18:13:53.044868 7f291ac78900 20 read_log 50495'48906594 (50495'48906593) modify 54b74ee/rbd_data.12896844d9a421.000000000001a8b8/head//19 by client.1214845.0:4802589438 2014-11-24 12:54:18.683883 2014-12-08 18:13:53.044911 7f291ac78900 20 read_log 50495'48906595 (50495'48906569) modify c83f0bee/rbd_data.12896844d9a421.00000000000194e4/head//19 by client.1214845.0:4802589490 2014-11-24 12:54:18.723121 2014-12-08 18:13:53.044927 7f291ac78900 20 read_log 50495'48906596 (50495'48906415) modify fee4cdee/rbd_data.12896844d9a421.0000000000019cd1/head//19 by client.1214845.0:4802590141 2014-11-24 12:54:19.047180 2014-12-08 18:13:53.044963 7f291ac78900 20 read_log 50495'48906597 (50495'48906596) modify fee4cdee/rbd_data.12896844d9a421.0000000000019cd1/head//19 by client.1214845.0:4802590142 2014-11-24 12:54:19.047259 2014-12-08 18:13:53.044972 7f291ac78900 20 read_log 50495'48906598 (50495'48906181) modify 87d6e76e/rbd_data.12896844d9a421.0000000000018afe/head//19 by client.1214845.0:4802590567 2014-11-24 12:54:19.250604 2014-12-08 18:13:53.044981 7f291ac78900 20 read_log 50495'48906599 (50495'48906598) modify 87d6e76e/rbd_data.12896844d9a421.0000000000018afe/head//19 by client.1214845.0:4802590568 2014-11-24 12:54:19.250677 2014-12-08 18:13:53.044990 7f291ac78900 20 read_log 50495'48906600 (50495'48906599) modify 87d6e76e/rbd_data.12896844d9a421.0000000000018afe/head//19 by client.1214845.0:4802590569 2014-11-24 12:54:19.250751 2014-12-08 18:13:53.044999 7f291ac78900 20 read_log 50495'48906601 (50495'48906166) modify 82dd80ee/rbd_data.12896844d9a421.000000000001a4df/head//19 by client.1214845.0:4802590592 2014-11-24 12:54:19.253041 2014-12-08 18:13:53.045008 7f291ac78900 20 read_log 50495'48906602 (50495'48906601) modify 82dd80ee/rbd_data.12896844d9a421.000000000001a4df/head//19 by client.1214845.0:4802590593 2014-11-24 12:54:19.253107 2014-12-08 18:13:53.045016 7f291ac78900 20 read_log 50495'48906603 (50495'48906469) modify 2dbd756e/rbd_data.12896844d9a421.000000000001cf6b/head//19 by client.1214845.0:4802590622 2014-11-24 12:54:19.272292 2014-12-08 18:13:53.045025 7f291ac78900 20 read_log 50495'48906604 (50495'48906498) modify 100c05ee/rbd_data.12896844d9a421.000000000001a8ca/head//19 by client.1214845.0:4802591042 2014-11-24 12:54:19.490890 2014-12-08 18:13:53.045034 7f291ac78900 20 read_log 50495'48906605 (50495'48906604) modify 100c05ee/rbd_data.12896844d9a421.000000000001a8ca/head//19 by client.1214845.0:4802591043 2014-11-24 12:54:19.490981 2014-12-08 18:13:53.045042 7f291ac78900 20 read_log 50495'48906606 (50495'48906459) modify 148e96e/rbd_data.12896844d9a421.00000000000184b4/head//19 by client.1214845.0:4802591172 2014-11-24 12:54:19.532481 2014-12-08 18:13:53.045067 7f291ac78900 20 read_log 50495'48906607 (50495'48906606) modify 148e96e/rbd_data.12896844d9a421.00000000000184b4/head//19 by client.1214845.0:4802591174 2014-11-24 12:54:19.532646 2014-12-08 18:13:53.045076 7f291ac78900 20 read_log 50495'48906608 (50495'48906607) modify 148e96e/rbd_data.12896844d9a421.00000000000184b4/head//19 by client.1214845.0:4802591175 2014-11-24 12:54:19.532790 2014-12-08 18:13:53.045085 7f291ac78900 20 read_log 50495'48906609 (50495'48906585) modify 9e148dee/rbd_data.12896844d9a421.000000000001a65e/head//19 by client.1214845.0:4802591487 2014-11-24 12:54:19.666505 2014-12-08 18:13:53.045093 7f291ac78900 20 read_log 50495'48906610 (50495'48906609) modify 9e148dee/rbd_data.12896844d9a421.000000000001a65e/head//19 by client.1214845.0:4802591488 2014-11-24 12:54:19.666578 2014-12-08 18:13:53.045102 7f291ac78900 20 read_log 50495'48906611 (50495'48906610) modify 9e148dee/rbd_data.12896844d9a421.000000000001a65e/head//19 by client.1214845.0:4802591489 2014-11-24 12:54:19.666778 -- 2014-12-08 18:13:53.094748 7f291ac78900 10 osd.70 pg_epoch: 71456 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=0 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_loaded 2014-12-08 18:13:53.094766 7f291ac78900 5 osd.70 pg_epoch: 71456 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=0 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] exit Initial 0.050334 0 0.000000 2014-12-08 18:13:53.094777 7f291ac78900 5 osd.70 pg_epoch: 71456 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=0 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] enter Reset 2014-12-08 18:13:53.094783 7f291ac78900 20 osd.70 pg_epoch: 71456 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=0 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] set_last_peering_reset 71456 2014-12-08 18:13:53.094790 7f291ac78900 10 osd.70 pg_epoch: 71456 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] Clearing blocked outgoing recovery messages 2014-12-08 18:13:53.094798 7f291ac78900 10 osd.70 pg_epoch: 71456 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] Not blocking outgoing recovery messages 2014-12-08 18:13:53.094804 7f291ac78900 10 osd.70 71581 load_pgs loaded pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] log((50495'48906592,51072'48910307], crt=51071'48910301) 2014-12-08 18:13:53.094814 7f291ac78900 10 osd.70 71581 pgid 19.71 coll 19.71_head 2014-12-08 18:13:53.094925 7f291ac78900 10 osd.70 71581 _open_lock_pg 19.71 2014-12-08 18:13:53.094933 7f291ac78900 10 osd.70 71581 _get_pool 19 2014-12-08 18:13:53.094963 7f291ac78900 5 osd.70 pg_epoch: 71435 pg[19.71(unlocked)] enter Initial 2014-12-08 18:13:53.094974 7f291ac78900 20 osd.70 pg_epoch: 71435 pg[19.71(unlocked)] enter NotTrimming 2014-12-08 18:13:53.095048 7f291ac78900 10 read_log 2014-12-08 18:13:53.096421 7f291ac78900 20 read_log 70963'39951248 (70963'39951247) modify 56a8be71/rbd_data.6240a2667d211e.0000000000000175/head//19 by client.6526574.0:148554 2014-12-03 20:04:45.554380 2014-12-08 18:13:53.096438 7f291ac78900 20 read_log 70963'39951249 (70963'39951248) modify 56a8be71/rbd_data.6240a2667d211e.0000000000000175/head//19 by client.6526574.0:148559 2014-12-03 20:04:45.602376 2014-12-08 18:13:53.096455 7f291ac78900 20 read_log 70963'39951250 (70963'39951249) modify 56a8be71/rbd_data.6240a2667d211e.0000000000000175/head//19 by client.6526574.0:148564 2014-12-03 20:04:45.650812 2014-12-08 18:13:53.096467 7f291ac78900 20 read_log 70963'39951251 (70963'39951250) modify 56a8be71/rbd_data.6240a2667d211e.0000000000000175/head//19 by client.6526574.0:148569 2014-12-03 20:04:45.722570 2014-12-08 18:13:53.096483 7f291ac78900 20 read_log 70963'39951252 (70963'39951251) modify 56a8be71/rbd_data.6240a2667d211e.0000000000000175/head//19 by client.6526574.0:148574 2014-12-03 20:04:45.767044 2014-12-08 18:13:53.096499 7f291ac78900 20 read_log 70963'39951253 (70963'39951252) modify 56a8be71/rbd_data.6240a2667d211e.0000000000000175/head//19 by client.6526574.0:148579 2014-12-03 20:04:45.801471 2014-12-08 18:13:53.096526 7f291ac78900 20 read_log 70963'39951254 (70963'39951253) modify 56a8be71/rbd_data.6240a2667d211e.0000000000000175/head//19 by client.6526574.0:148584 2014-12-03 20:04:45.836752 2014-12-08 18:13:53.096543 7f291ac78900 20 read_log 70963'39951255 (70963'39951254) modify 56a8be71/rbd_data.6240a2667d211e.0000000000000175/head//19 by client.6526574.0:148589 2014-12-03 20:04:45.881899 2014-12-08 18:13:53.096573 7f291ac78900 20 read_log 70963'39951256 (70963'39951255) modify 56a8be71/rbd_data.6240a2667d211e.0000000000000175/head//19 by client.6526574.0:148594 2014-12-03 20:04:45.922103 2014-12-08 18:13:53.096589 7f291ac78900 20 read_log 70963'39951257 (70963'39951256) modify 56a8be71/rbd_data.6240a2667d211e.0000000000000175/head//19 by client.6526574.0:148599 2014-12-03 20:04:45.966544 2014-12-08 18:13:53.096604 7f291ac78900 20 read_log 70963'39951258 (70963'39951257) modify 56a8be71/rbd_data.6240a2667d211e.0000000000000175/head//19 by client.6526574.0:148604 2014-12-03 20:04:46.016796 2014-12-08 18:13:53.096619 7f291ac78900 20 read_log 70963'39951259 (70963'39951258) modify 56a8be71/rbd_data.6240a2667d211e.0000000000000175/head//19 by client.6526574.0:148609 2014-12-03 20:04:46.069626 2014-12-08 18:13:53.096634 7f291ac78900 20 read_log 70963'39951260 (70963'39951259) modify 56a8be71/rbd_data.6240a2667d211e.0000000000000175/head//19 by client.6526574.0:148614 2014-12-03 20:04:46.118789 2014-12-08 18:13:53.096648 7f291ac78900 20 read_log 70963'39951261 (70963'39951260) modify 56a8be71/rbd_data.6240a2667d211e.0000000000000175/head//19 by client.6526574.0:148619 2014-12-03 20:04:46.164894 -- 2014-12-08 18:13:56.679015 7f291ac78900 10 osd.70 pg_epoch: 71456 pg[19.6e(unlocked)] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:56.679020 7f291ac78900 10 osd.70 pg_epoch: 71439 pg[19.6d(unlocked)] _calc_past_interval_range: already have past intervals back to 71439 2014-12-08 18:13:56.679024 7f291ac78900 10 osd.70 pg_epoch: 71581 pg[218.6c7(unlocked)] _calc_past_interval_range start epoch 71433 >= end epoch 71432, nothing to do 2014-12-08 18:13:56.679027 7f291ac78900 10 osd.70 pg_epoch: 71581 pg[218.6c4(unlocked)] _calc_past_interval_range start epoch 71437 >= end epoch 71432, nothing to do 2014-12-08 18:13:56.679030 7f291ac78900 10 osd.70 pg_epoch: 71581 pg[218.562(unlocked)] _calc_past_interval_range: already have past intervals back to 71437 2014-12-08 18:13:56.679034 7f291ac78900 10 osd.70 pg_epoch: 71435 pg[218.5e(unlocked)] _calc_past_interval_range: already have past intervals back to 71435 2014-12-08 18:13:56.679037 7f291ac78900 10 osd.70 pg_epoch: 71580 pg[218.5f(unlocked)] _calc_past_interval_range start epoch 71437 >= end epoch 71432, nothing to do 2014-12-08 18:13:56.679041 7f291ac78900 10 osd.70 pg_epoch: 71435 pg[217.235(unlocked)] _calc_past_interval_range: already have past intervals back to 71435 2014-12-08 18:13:56.679045 7f291ac78900 10 osd.70 pg_epoch: 71437 pg[218.38e(unlocked)] _calc_past_interval_range start epoch 71437 >= end epoch 71432, nothing to do 2014-12-08 18:13:56.679048 7f291ac78900 10 osd.70 pg_epoch: 71457 pg[218.38f(unlocked)] _calc_past_interval_range start epoch 71457 >= end epoch 71456, nothing to do 2014-12-08 18:13:56.679051 7f291ac78900 10 osd.70 pg_epoch: 71448 pg[218.38c(unlocked)] _calc_past_interval_range: already have past intervals back to 71448 2014-12-08 18:13:56.679055 7f291ac78900 10 osd.70 pg_epoch: 71435 pg[217.51(unlocked)] _calc_past_interval_range: already have past intervals back to 71435 2014-12-08 18:13:56.679071 7f291ac78900 10 osd.70 pg_epoch: 71453 pg[216.380(unlocked)] _calc_past_interval_range: already have past intervals back to 71453 2014-12-08 18:13:56.679074 7f291ac78900 10 osd.70 pg_epoch: 71581 pg[218.51(unlocked)] _calc_past_interval_range start epoch 71434 >= end epoch 71432, nothing to do 2014-12-08 18:13:56.679078 7f291ac78900 10 osd.70 pg_epoch: 71450 pg[218.380(unlocked)] _calc_past_interval_range: already have past intervals back to 71450 2014-12-08 18:13:56.679082 7f291ac78900 10 osd.70 pg_epoch: 71581 pg[218.518(unlocked)] _calc_past_interval_range: already have past intervals back to 71433 2014-12-08 18:13:56.679087 7f291ac78900 10 osd.70 pg_epoch: 71437 pg[217.22d(unlocked)] _calc_past_interval_range: already have past intervals back to 71437 2014-12-08 18:13:56.679097 7f291ac78900 10 osd.70 pg_epoch: 71435 pg[216.384(unlocked)] _calc_past_interval_range: already have past intervals back to 71435 2014-12-08 18:13:56.679102 7f291ac78900 10 osd.70 pg_epoch: 71450 pg[216.22f(unlocked)] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:56.679111 7f291ac78900 10 osd.70 pg_epoch: 71581 pg[218.4b(unlocked)] _calc_past_interval_range: already have past intervals back to 71453 2014-12-08 18:13:56.679115 7f291ac78900 10 osd.70 pg_epoch: 71434 pg[217.220(unlocked)] _calc_past_interval_range: already have past intervals back to 71434 -- 2014-12-08 18:13:56.700970 7f291ac78900 10 osd.70 pg_epoch: 71456 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] null 2014-12-08 18:13:56.700984 7f291ac78900 10 osd.70 pg_epoch: 71439 pg[19.6d( v 71397'33614009 (70963'33611008,71397'33614009] local-les=71439 n=896 ec=21298 les/c 71439/71439 71432/71438/71169) [4,70,69] r=1 lpr=71439 pi=71156-71437/22 crt=71397'33614007 lcod 0'0 inactive NOTIFY] null 2014-12-08 18:13:56.700998 7f291ac78900 10 osd.70 pg_epoch: 71581 pg[218.6c7( v 71581'2 (0'0,71581'2] local-les=71433 n=2 ec=71423 les/c 71433/71433 71432/71432/71432) [70,10,18] r=0 lpr=71581 crt=0'0 lcod 0'0 mlcod 0'0 inactive] null 2014-12-08 18:13:56.701010 7f291ac78900 10 osd.70 pg_epoch: 71581 pg[218.6c4( v 71581'3 (0'0,71581'3] local-les=71437 n=3 ec=71423 les/c 71437/71437 71432/71432/71432) [70,76,13] r=0 lpr=71581 crt=71580'1 lcod 0'0 mlcod 0'0 inactive] null 2014-12-08 18:13:56.701019 7f291ac78900 10 osd.70 pg_epoch: 71581 pg[218.562( v 71581'3 (0'0,71581'3] local-les=71437 n=3 ec=71423 les/c 71437/71437 71432/71432/71425) [9,18,70] r=2 lpr=71581 pi=71423-71431/4 crt=71580'1 lcod 0'0 inactive NOTIFY] null 2014-12-08 18:13:56.701030 7f291ac78900 10 osd.70 pg_epoch: 71435 pg[218.5e( empty local-les=71435 n=0 ec=71423 les/c 71435/71435 71432/71432/71425) [16,22,70] r=2 lpr=71435 pi=71423-71431/4 crt=0'0 inactive NOTIFY] null 2014-12-08 18:13:56.701260 7f291ac78900 10 osd.70 pg_epoch: 71580 pg[218.5f( v 71580'2 (0'0,71580'2] local-les=71437 n=2 ec=71423 les/c 71437/71437 71432/71432/71432) [70,15,24] r=0 lpr=71580 crt=0'0 lcod 0'0 mlcod 0'0 inactive] null 2014-12-08 18:13:56.701271 7f291ac78900 10 osd.70 pg_epoch: 71435 pg[217.235( v 71377'4108 (71377'1107,71377'4108] local-les=71435 n=11 ec=71094 les/c 71435/71435 71432/71434/71434) [78,25,70] r=2 lpr=71435 pi=71094-71433/26 crt=71377'4105 lcod 0'0 inactive NOTIFY] null 2014-12-08 18:13:56.701285 7f291ac78900 10 osd.70 pg_epoch: 71437 pg[218.38e( empty local-les=71437 n=0 ec=71423 les/c 71437/71437 71432/71432/71432) [70,13,3] r=0 lpr=71437 crt=0'0 mlcod 0'0 inactive] null 2014-12-08 18:13:56.701428 7f28fcb46700 10 osd.70 pg_epoch: 71486 pg[216.249( v 71097'244 (0'0,71097'244] local-les=71457 n=119 ec=70987 les/c 71457/71486 71456/71456/71432) [70,68,5] r=0 lpr=71486 crt=71097'244 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,68,5]/[70,68,5] -- 70/70 2014-12-08 18:13:56.701452 7f28fcb46700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:56.701440 7f28fc345700 10 osd.70 pg_epoch: 71435 pg[216.257( v 71097'238 (0'0,71097'238] local-les=71434 n=110 ec=70987 les/c 71434/71434 71432/71432/71168) [8,70,16] r=1 lpr=71434 pi=70987-71431/25 crt=71097'236 lcod 0'0 inactive NOTIFY] handle_advance_map [8,70,16]/[8,70,16] -- 8/8 2014-12-08 18:13:56.701464 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:56.701463 7f28fcb46700 10 osd.70 pg_epoch: 71487 pg[216.249( v 71097'244 (0'0,71097'244] local-les=71457 n=119 ec=70987 les/c 71457/71486 71456/71456/71432) [70,68,5] r=0 lpr=71486 crt=71097'244 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:56.701474 7f28fcb46700 10 osd.70 pg_epoch: 71487 pg[216.249( v 71097'244 (0'0,71097'244] local-les=71457 n=119 ec=70987 les/c 71457/71486 71456/71456/71432) [70,68,5] r=0 lpr=71486 crt=71097'244 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71486 >= end epoch 71456, nothing to do 2014-12-08 18:13:56.701469 7f28fc345700 10 osd.70 pg_epoch: 71436 pg[216.257( v 71097'238 (0'0,71097'238] local-les=71434 n=110 ec=70987 les/c 71434/71434 71432/71432/71168) [8,70,16] r=1 lpr=71434 pi=70987-71431/25 crt=71097'236 lcod 0'0 inactive NOTIFY] state: Reset advmap 2014-12-08 18:13:56.701489 7f28fcb46700 10 osd.70 pg_epoch: 71487 pg[216.249( v 71097'244 (0'0,71097'244] local-les=71457 n=119 ec=70987 les/c 71457/71486 71456/71456/71432) [70,68,5] r=0 lpr=71486 crt=71097'244 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:56.701493 7f28fc345700 10 osd.70 pg_epoch: 71436 pg[216.257( v 71097'238 (0'0,71097'238] local-les=71434 n=110 ec=70987 les/c 71434/71434 71432/71432/71168) [8,70,16] r=1 lpr=71434 pi=70987-71431/25 crt=71097'236 lcod 0'0 inactive NOTIFY] _calc_past_interval_range: already have past intervals back to 71434 2014-12-08 18:13:56.701507 7f28fcb46700 20 osd.70 71581 get_map 71488 - loading and decoding 0x5535e00 2014-12-08 18:13:56.701295 7f291ac78900 10 osd.70 pg_epoch: 71457 pg[218.38f( empty local-les=71457 n=0 ec=71423 les/c 71457/71457 71456/71456/71432) [70,23,25] r=0 lpr=71457 crt=0'0 mlcod 0'0 inactive] null 2014-12-08 18:13:56.701508 7f28fc345700 10 osd.70 pg_epoch: 71436 pg[216.257( v 71097'238 (0'0,71097'238] local-les=71434 n=110 ec=70987 les/c 71434/71434 71432/71432/71168) [8,70,16] r=1 lpr=71434 pi=70987-71431/25 crt=71097'236 lcod 0'0 inactive NOTIFY] check_recovery_sources no source osds () went down -- 2014-12-08 18:13:57.761701 7f28fcb46700 10 osd.70 pg_epoch: 71456 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.761708 7f28fc345700 10 osd.70 pg_epoch: 71581 pg[216.384( v 70990'209 (0'0,70990'209] local-les=71435 n=101 ec=70987 les/c 71435/71435 71432/71432/71432) [77,22,70] r=2 lpr=71435 pi=70987-71431/22 crt=70990'207 lcod 0'0 inactive NOTIFY] state: Reset advmap 2014-12-08 18:13:57.761718 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.761717 7f28fc345700 10 osd.70 pg_epoch: 71581 pg[216.384( v 70990'209 (0'0,70990'209] local-les=71435 n=101 ec=70987 les/c 71435/71435 71432/71432/71432) [77,22,70] r=2 lpr=71435 pi=70987-71431/22 crt=70990'207 lcod 0'0 inactive NOTIFY] _calc_past_interval_range: already have past intervals back to 71435 2014-12-08 18:13:57.761725 7f28fcb46700 10 osd.70 pg_epoch: 71457 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.761733 7f28fcb46700 10 osd.70 pg_epoch: 71457 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.761729 7f28fc345700 10 osd.70 pg_epoch: 71581 pg[216.384( v 70990'209 (0'0,70990'209] local-les=71435 n=101 ec=70987 les/c 71435/71435 71432/71432/71432) [77,22,70] r=2 lpr=71435 pi=70987-71431/22 crt=70990'207 lcod 0'0 inactive NOTIFY] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.761745 7f28fcb46700 10 osd.70 pg_epoch: 71457 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.761760 7f28fcb46700 10 osd.70 pg_epoch: 71457 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.761769 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.761761 7f28fc345700 10 osd.70 pg_epoch: 71581 pg[216.384( v 70990'209 (0'0,70990'209] local-les=71435 n=101 ec=70987 les/c 71435/71435 71432/71432/71432) [77,22,70] r=2 lpr=71435 pi=70987-71431/22 crt=70990'207 lcod 0'0 inactive NOTIFY] handle_activate_map 2014-12-08 18:13:57.761771 7f28fcb46700 10 osd.70 pg_epoch: 71458 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.761779 7f28fcb46700 10 osd.70 pg_epoch: 71458 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.761786 7f28fcb46700 10 osd.70 pg_epoch: 71458 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.761799 7f28fcb46700 10 osd.70 pg_epoch: 71458 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.761808 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.761810 7f28fcb46700 10 osd.70 pg_epoch: 71459 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.761816 7f28fcb46700 10 osd.70 pg_epoch: 71459 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.761813 7f28fc345700 10 osd.70 pg_epoch: 71581 pg[216.384( v 70990'209 (0'0,70990'209] local-les=71435 n=101 ec=70987 les/c 71435/71435 71432/71432/71432) [77,22,70] r=2 lpr=71435 pi=70987-71431/22 crt=70990'207 lcod 0'0 inactive NOTIFY] update_heartbeat_peers unchanged 2014-12-08 18:13:57.761824 7f28fc345700 10 osd.70 pg_epoch: 71581 pg[216.384( v 70990'209 (0'0,70990'209] local-les=71435 n=101 ec=70987 les/c 71435/71435 71432/71432/71432) [77,22,70] r=2 lpr=71435 pi=70987-71431/22 crt=70990'207 lcod 0'0 inactive NOTIFY] take_waiters 2014-12-08 18:13:57.761823 7f28fcb46700 10 osd.70 pg_epoch: 71459 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.761832 7f28fc345700 5 osd.70 pg_epoch: 71581 pg[216.384( v 70990'209 (0'0,70990'209] local-les=71435 n=101 ec=70987 les/c 71435/71435 71432/71432/71432) [77,22,70] r=2 lpr=71435 pi=70987-71431/22 crt=70990'207 lcod 0'0 inactive NOTIFY] exit Reset 3.896814 147 0.007942 2014-12-08 18:13:57.761837 7f28fcb46700 10 osd.70 pg_epoch: 71459 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.761846 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.761841 7f28fc345700 5 osd.70 pg_epoch: 71581 pg[216.384( v 70990'209 (0'0,70990'209] local-les=71435 n=101 ec=70987 les/c 71435/71435 71432/71432/71432) [77,22,70] r=2 lpr=71435 pi=70987-71431/22 crt=70990'207 lcod 0'0 inactive NOTIFY] enter Started 2014-12-08 18:13:57.761849 7f28fc345700 5 osd.70 pg_epoch: 71581 pg[216.384( v 70990'209 (0'0,70990'209] local-les=71435 n=101 ec=70987 les/c 71435/71435 71432/71432/71432) [77,22,70] r=2 lpr=71435 pi=70987-71431/22 crt=70990'207 lcod 0'0 inactive NOTIFY] enter Start 2014-12-08 18:13:57.761848 7f28fcb46700 10 osd.70 pg_epoch: 71460 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.761859 7f28fcb46700 10 osd.70 pg_epoch: 71460 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.761858 7f28fc345700 1 osd.70 pg_epoch: 71581 pg[216.384( v 70990'209 (0'0,70990'209] local-les=71435 n=101 ec=70987 les/c 71435/71435 71432/71432/71432) [77,22,70] r=2 lpr=71435 pi=70987-71431/22 crt=70990'207 lcod 0'0 inactive NOTIFY] state: transitioning to Stray 2014-12-08 18:13:57.761868 7f28fcb46700 10 osd.70 pg_epoch: 71460 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.761871 7f28fc345700 5 osd.70 pg_epoch: 71581 pg[216.384( v 70990'209 (0'0,70990'209] local-les=71435 n=101 ec=70987 les/c 71435/71435 71432/71432/71432) [77,22,70] r=2 lpr=71435 pi=70987-71431/22 crt=70990'207 lcod 0'0 inactive NOTIFY] exit Start 0.000023 0 0.000000 2014-12-08 18:13:57.761880 7f28fc345700 5 osd.70 pg_epoch: 71581 pg[216.384( v 70990'209 (0'0,70990'209] local-les=71435 n=101 ec=70987 les/c 71435/71435 71432/71432/71432) [77,22,70] r=2 lpr=71435 pi=70987-71431/22 crt=70990'207 lcod 0'0 inactive NOTIFY] enter Started/Stray 2014-12-08 18:13:57.761881 7f28fcb46700 10 osd.70 pg_epoch: 71460 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.761890 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.761890 7f28fc345700 20 osd.70 pg_epoch: 71581 pg[216.384( v 70990'209 (0'0,70990'209] local-les=71435 n=101 ec=70987 les/c 71435/71435 71432/71432/71432) [77,22,70] r=2 lpr=71435 pi=70987-71431/22 crt=70990'207 lcod 0'0 inactive NOTIFY] handle_activate_map: Not dirtying info: last_persisted is 71435 while current is 71581 2014-12-08 18:13:57.765738 7f28fcb46700 10 osd.70 pg_epoch: 71461 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.765742 7f28fc345700 10 osd.70 pg_epoch: 71581 pg[216.384( v 70990'209 (0'0,70990'209] local-les=71435 n=101 ec=70987 les/c 71435/71435 71432/71432/71432) [77,22,70] r=2 lpr=71435 pi=70987-71431/22 crt=70990'207 lcod 0'0 inactive NOTIFY] handle_peering_event: epoch_sent: 71581 epoch_requested: 71581 NullEvt 2014-12-08 18:13:57.762133 7f290a83a700 10 osd.70 pg_epoch: 71581 pg[216.392( v 71093'208 (0'0,71093'208] local-les=71438 n=104 ec=70987 les/c 71438/71438 71432/71432/70989) [69,7,70] r=2 lpr=71438 pi=71183-71431/13 crt=71093'208 lcod 0'0 inactive NOTIFY] flushed 2014-12-08 18:13:57.765751 7f28fcb46700 10 osd.70 pg_epoch: 71461 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.765756 7f28fc345700 10 write_log with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, dirty_divergent_priors: 0, writeout_from: 4294967295'18446744073709551615, trimmed: 2014-12-08 18:13:57.765760 7f28fcb46700 10 osd.70 pg_epoch: 71461 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.765776 7f28fcb46700 10 osd.70 pg_epoch: 71461 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.765785 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.765788 7f28fcb46700 10 osd.70 pg_epoch: 71462 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.765795 7f28fcb46700 10 osd.70 pg_epoch: 71462 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.765802 7f28fcb46700 10 osd.70 pg_epoch: 71462 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.765802 7f28fc345700 10 osd.70 pg_epoch: 71450 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.765815 7f28fcb46700 10 osd.70 pg_epoch: 71462 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.765824 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.765824 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.765826 7f28fcb46700 10 osd.70 pg_epoch: 71463 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.765834 7f28fcb46700 10 osd.70 pg_epoch: 71463 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.765829 7f28fc345700 10 osd.70 pg_epoch: 71451 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.765841 7f28fcb46700 10 osd.70 pg_epoch: 71463 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.765846 7f28fc345700 10 osd.70 pg_epoch: 71451 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.765854 7f28fcb46700 10 osd.70 pg_epoch: 71463 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.765862 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.765864 7f28fcb46700 10 osd.70 pg_epoch: 71464 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.765860 7f28fc345700 10 osd.70 pg_epoch: 71451 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.765871 7f28fcb46700 10 osd.70 pg_epoch: 71464 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.765878 7f28fcb46700 10 osd.70 pg_epoch: 71464 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.765884 7f28fc345700 10 osd.70 pg_epoch: 71451 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.765890 7f28fcb46700 10 osd.70 pg_epoch: 71464 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.765900 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.765902 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.765904 7f28fcb46700 10 osd.70 pg_epoch: 71465 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.765904 7f28fc345700 10 osd.70 pg_epoch: 71452 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.765911 7f28fcb46700 10 osd.70 pg_epoch: 71465 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.765919 7f28fcb46700 10 osd.70 pg_epoch: 71465 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.765917 7f28fc345700 10 osd.70 pg_epoch: 71452 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.765929 7f28fc345700 10 osd.70 pg_epoch: 71452 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.765933 7f28fcb46700 10 osd.70 pg_epoch: 71465 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.765942 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.765944 7f28fcb46700 10 osd.70 pg_epoch: 71466 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.765951 7f28fcb46700 10 osd.70 pg_epoch: 71466 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.765957 7f28fcb46700 10 osd.70 pg_epoch: 71466 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.765971 7f28fcb46700 10 osd.70 pg_epoch: 71466 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.765979 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.765981 7f28fcb46700 10 osd.70 pg_epoch: 71467 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.765988 7f28fcb46700 10 osd.70 pg_epoch: 71467 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.765951 7f28fc345700 10 osd.70 pg_epoch: 71452 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.765999 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.765995 7f28fcb46700 10 osd.70 pg_epoch: 71467 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.766003 7f28fc345700 10 osd.70 pg_epoch: 71453 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.766008 7f28fcb46700 10 osd.70 pg_epoch: 71467 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.766018 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.766014 7f28fc345700 10 osd.70 pg_epoch: 71453 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.766020 7f28fcb46700 10 osd.70 pg_epoch: 71468 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.766023 7f28fc345700 10 osd.70 pg_epoch: 71453 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.766027 7f28fcb46700 10 osd.70 pg_epoch: 71468 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.766034 7f28fcb46700 10 osd.70 pg_epoch: 71468 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.766040 7f28fc345700 10 osd.70 pg_epoch: 71453 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.766049 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.766047 7f28fcb46700 10 osd.70 pg_epoch: 71468 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.766055 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.766057 7f28fcb46700 10 osd.70 pg_epoch: 71469 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.766064 7f28fcb46700 10 osd.70 pg_epoch: 71469 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.766051 7f28fc345700 10 osd.70 pg_epoch: 71454 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.766070 7f28fcb46700 10 osd.70 pg_epoch: 71469 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.766074 7f28fc345700 10 osd.70 pg_epoch: 71454 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.766082 7f28fc345700 10 osd.70 pg_epoch: 71454 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.766083 7f28fcb46700 10 osd.70 pg_epoch: 71469 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.766092 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.766094 7f28fcb46700 10 osd.70 pg_epoch: 71470 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.766097 7f28fc345700 10 osd.70 pg_epoch: 71454 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.766100 7f28fcb46700 10 osd.70 pg_epoch: 71470 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.766107 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.766107 7f28fcb46700 10 osd.70 pg_epoch: 71470 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.766109 7f28fc345700 10 osd.70 pg_epoch: 71455 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.766106 7f290a83a700 10 osd.70 pg_epoch: 71581 pg[216.384( v 70990'209 (0'0,70990'209] local-les=71435 n=101 ec=70987 les/c 71435/71435 71432/71432/71432) [77,22,70] r=2 lpr=71435 pi=70987-71431/22 crt=70990'207 lcod 0'0 inactive NOTIFY] flushed 2014-12-08 18:13:57.766116 7f28fc345700 10 osd.70 pg_epoch: 71455 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.766121 7f28fcb46700 10 osd.70 pg_epoch: 71470 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.766125 7f28fc345700 10 osd.70 pg_epoch: 71455 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.766130 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.766132 7f28fcb46700 10 osd.70 pg_epoch: 71471 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.766137 7f28fc345700 10 osd.70 pg_epoch: 71455 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.766139 7f28fcb46700 10 osd.70 pg_epoch: 71471 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.766150 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.766150 7f28fcb46700 10 osd.70 pg_epoch: 71471 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.766152 7f28fc345700 10 osd.70 pg_epoch: 71456 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.766160 7f28fc345700 10 osd.70 pg_epoch: 71456 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.766163 7f28fcb46700 10 osd.70 pg_epoch: 71471 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.766168 7f28fc345700 10 osd.70 pg_epoch: 71456 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.766173 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.766175 7f28fcb46700 10 osd.70 pg_epoch: 71472 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.766181 7f28fc345700 10 osd.70 pg_epoch: 71456 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.766182 7f28fcb46700 10 osd.70 pg_epoch: 71472 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.766190 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.766190 7f28fcb46700 10 osd.70 pg_epoch: 71472 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.766193 7f28fc345700 10 osd.70 pg_epoch: 71457 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.766200 7f28fc345700 10 osd.70 pg_epoch: 71457 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.766203 7f28fcb46700 10 osd.70 pg_epoch: 71472 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.766207 7f28fc345700 10 osd.70 pg_epoch: 71457 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.766213 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.766215 7f28fcb46700 10 osd.70 pg_epoch: 71473 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.766219 7f28fc345700 10 osd.70 pg_epoch: 71457 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.766221 7f28fcb46700 10 osd.70 pg_epoch: 71473 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.766228 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.766229 7f28fcb46700 10 osd.70 pg_epoch: 71473 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.766230 7f28fc345700 10 osd.70 pg_epoch: 71458 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.766237 7f28fc345700 10 osd.70 pg_epoch: 71458 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.766242 7f28fcb46700 10 osd.70 pg_epoch: 71473 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.766244 7f28fc345700 10 osd.70 pg_epoch: 71458 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.766257 7f28fc345700 10 osd.70 pg_epoch: 71458 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.766265 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.766267 7f28fc345700 10 osd.70 pg_epoch: 71459 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.766250 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.766274 7f28fc345700 10 osd.70 pg_epoch: 71459 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.766281 7f28fc345700 10 osd.70 pg_epoch: 71459 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.766281 7f28fcb46700 10 osd.70 pg_epoch: 71474 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.766289 7f28fcb46700 10 osd.70 pg_epoch: 71474 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.766294 7f28fc345700 10 osd.70 pg_epoch: 71459 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.766296 7f28fcb46700 10 osd.70 pg_epoch: 71474 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.766302 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.766304 7f28fc345700 10 osd.70 pg_epoch: 71460 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.766309 7f28fcb46700 10 osd.70 pg_epoch: 71474 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.766310 7f28fc345700 10 osd.70 pg_epoch: 71460 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.766318 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.766317 7f28fc345700 10 osd.70 pg_epoch: 71460 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.766320 7f28fcb46700 10 osd.70 pg_epoch: 71475 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.766327 7f28fcb46700 10 osd.70 pg_epoch: 71475 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.766329 7f28fc345700 10 osd.70 pg_epoch: 71460 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.766338 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.766334 7f28fcb46700 10 osd.70 pg_epoch: 71475 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.766340 7f28fc345700 10 osd.70 pg_epoch: 71461 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.766347 7f28fc345700 10 osd.70 pg_epoch: 71461 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.766347 7f28fcb46700 10 osd.70 pg_epoch: 71475 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.766356 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.766355 7f28fc345700 10 osd.70 pg_epoch: 71461 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.766358 7f28fcb46700 10 osd.70 pg_epoch: 71476 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.766366 7f28fcb46700 10 osd.70 pg_epoch: 71476 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.766367 7f28fc345700 10 osd.70 pg_epoch: 71461 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.766375 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.766373 7f28fcb46700 10 osd.70 pg_epoch: 71476 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.766377 7f28fc345700 10 osd.70 pg_epoch: 71462 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.766385 7f28fc345700 10 osd.70 pg_epoch: 71462 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.766386 7f28fcb46700 10 osd.70 pg_epoch: 71476 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.766395 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.766392 7f28fc345700 10 osd.70 pg_epoch: 71462 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.766397 7f28fcb46700 10 osd.70 pg_epoch: 71477 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.766404 7f28fcb46700 10 osd.70 pg_epoch: 71477 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.766404 7f28fc345700 10 osd.70 pg_epoch: 71462 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.766414 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.766412 7f28fcb46700 10 osd.70 pg_epoch: 71477 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.766416 7f28fc345700 10 osd.70 pg_epoch: 71463 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.766423 7f28fc345700 10 osd.70 pg_epoch: 71463 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.766425 7f28fcb46700 10 osd.70 pg_epoch: 71477 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.766435 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.766430 7f28fc345700 10 osd.70 pg_epoch: 71463 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.766440 7f28fcb46700 10 osd.70 pg_epoch: 71478 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.766447 7f28fcb46700 10 osd.70 pg_epoch: 71478 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.766445 7f28fc345700 10 osd.70 pg_epoch: 71463 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.766457 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.766455 7f28fcb46700 10 osd.70 pg_epoch: 71478 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.766459 7f28fc345700 10 osd.70 pg_epoch: 71464 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.766466 7f28fc345700 10 osd.70 pg_epoch: 71464 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.766469 7f28fcb46700 10 osd.70 pg_epoch: 71478 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.766473 7f28fc345700 10 osd.70 pg_epoch: 71464 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.766478 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.766483 7f28fcb46700 10 osd.70 pg_epoch: 71479 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.766488 7f28fc345700 10 osd.70 pg_epoch: 71464 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.766491 7f28fcb46700 10 osd.70 pg_epoch: 71479 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.766497 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.766498 7f28fcb46700 10 osd.70 pg_epoch: 71479 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.766499 7f28fc345700 10 osd.70 pg_epoch: 71465 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.766507 7f28fc345700 10 osd.70 pg_epoch: 71465 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.766512 7f28fcb46700 10 osd.70 pg_epoch: 71479 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.766514 7f28fc345700 10 osd.70 pg_epoch: 71465 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.766521 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.766523 7f28fcb46700 10 osd.70 pg_epoch: 71480 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.766526 7f28fc345700 10 osd.70 pg_epoch: 71465 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.766530 7f28fcb46700 10 osd.70 pg_epoch: 71480 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.766536 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.766539 7f28fcb46700 10 osd.70 pg_epoch: 71480 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.766540 7f28fc345700 10 osd.70 pg_epoch: 71466 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.766547 7f28fc345700 10 osd.70 pg_epoch: 71466 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.766553 7f28fcb46700 10 osd.70 pg_epoch: 71480 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.766555 7f28fc345700 10 osd.70 pg_epoch: 71466 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.766562 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.766564 7f28fcb46700 10 osd.70 pg_epoch: 71481 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.766567 7f28fc345700 10 osd.70 pg_epoch: 71466 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.766571 7f28fcb46700 10 osd.70 pg_epoch: 71481 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.766577 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.766579 7f28fc345700 10 osd.70 pg_epoch: 71467 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.766579 7f28fcb46700 10 osd.70 pg_epoch: 71481 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.766588 7f28fc345700 10 osd.70 pg_epoch: 71467 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.766596 7f28fc345700 10 osd.70 pg_epoch: 71467 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.766595 7f28fcb46700 10 osd.70 pg_epoch: 71481 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.766605 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.766607 7f28fcb46700 10 osd.70 pg_epoch: 71482 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.766609 7f28fc345700 10 osd.70 pg_epoch: 71467 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.766618 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.766614 7f28fcb46700 10 osd.70 pg_epoch: 71482 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.766620 7f28fc345700 10 osd.70 pg_epoch: 71468 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.766622 7f28fcb46700 10 osd.70 pg_epoch: 71482 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.766627 7f28fc345700 10 osd.70 pg_epoch: 71468 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.766634 7f28fc345700 10 osd.70 pg_epoch: 71468 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.766635 7f28fcb46700 10 osd.70 pg_epoch: 71482 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.766644 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.766646 7f28fcb46700 10 osd.70 pg_epoch: 71483 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.766646 7f28fc345700 10 osd.70 pg_epoch: 71468 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.766656 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.766655 7f28fcb46700 10 osd.70 pg_epoch: 71483 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.766658 7f28fc345700 10 osd.70 pg_epoch: 71469 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.766662 7f28fcb46700 10 osd.70 pg_epoch: 71483 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.766666 7f28fc345700 10 osd.70 pg_epoch: 71469 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.766673 7f28fc345700 10 osd.70 pg_epoch: 71469 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.766676 7f28fcb46700 10 osd.70 pg_epoch: 71483 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.766685 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.766685 7f28fc345700 10 osd.70 pg_epoch: 71469 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.766693 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.766687 7f28fcb46700 10 osd.70 pg_epoch: 71484 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.766695 7f28fc345700 10 osd.70 pg_epoch: 71470 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.766695 7f28fcb46700 10 osd.70 pg_epoch: 71484 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.766702 7f28fcb46700 10 osd.70 pg_epoch: 71484 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.766703 7f28fc345700 10 osd.70 pg_epoch: 71470 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.766712 7f28fc345700 10 osd.70 pg_epoch: 71470 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.766717 7f28fcb46700 10 osd.70 pg_epoch: 71484 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.766726 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.766723 7f28fc345700 10 osd.70 pg_epoch: 71470 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.766733 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.766728 7f28fcb46700 10 osd.70 pg_epoch: 71485 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.766735 7f28fc345700 10 osd.70 pg_epoch: 71471 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.766736 7f28fcb46700 10 osd.70 pg_epoch: 71485 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.766743 7f28fcb46700 10 osd.70 pg_epoch: 71485 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.766742 7f28fc345700 10 osd.70 pg_epoch: 71471 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.766753 7f28fc345700 10 osd.70 pg_epoch: 71471 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.766758 7f28fcb46700 10 osd.70 pg_epoch: 71485 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.766766 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.766765 7f28fc345700 10 osd.70 pg_epoch: 71471 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.766774 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.766769 7f28fcb46700 10 osd.70 pg_epoch: 71486 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.766778 7f28fc345700 10 osd.70 pg_epoch: 71472 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.766778 7f28fcb46700 10 osd.70 pg_epoch: 71486 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.766786 7f28fcb46700 10 osd.70 pg_epoch: 71486 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.766786 7f28fc345700 10 osd.70 pg_epoch: 71472 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.766795 7f28fc345700 10 osd.70 pg_epoch: 71472 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.766801 7f28fcb46700 10 osd.70 pg_epoch: 71486 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.766810 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.766807 7f28fc345700 10 osd.70 pg_epoch: 71472 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.766816 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.766812 7f28fcb46700 10 osd.70 pg_epoch: 71487 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.766818 7f28fc345700 10 osd.70 pg_epoch: 71473 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.766820 7f28fcb46700 10 osd.70 pg_epoch: 71487 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.766825 7f28fc345700 10 osd.70 pg_epoch: 71473 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.766827 7f28fcb46700 10 osd.70 pg_epoch: 71487 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.766833 7f28fc345700 10 osd.70 pg_epoch: 71473 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.766840 7f28fcb46700 10 osd.70 pg_epoch: 71487 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.766848 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.766845 7f28fc345700 10 osd.70 pg_epoch: 71473 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.766854 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.766850 7f28fcb46700 10 osd.70 pg_epoch: 71488 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.766855 7f28fc345700 10 osd.70 pg_epoch: 71474 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.766857 7f28fcb46700 10 osd.70 pg_epoch: 71488 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.766862 7f28fc345700 10 osd.70 pg_epoch: 71474 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.766863 7f28fcb46700 10 osd.70 pg_epoch: 71488 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.766872 7f28fc345700 10 osd.70 pg_epoch: 71474 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.766879 7f28fcb46700 10 osd.70 pg_epoch: 71488 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.766888 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.766884 7f28fc345700 10 osd.70 pg_epoch: 71474 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.766893 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.766890 7f28fcb46700 10 osd.70 pg_epoch: 71489 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.766895 7f28fc345700 10 osd.70 pg_epoch: 71475 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.766897 7f28fcb46700 10 osd.70 pg_epoch: 71489 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.766903 7f28fc345700 10 osd.70 pg_epoch: 71475 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.766904 7f28fcb46700 10 osd.70 pg_epoch: 71489 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.766912 7f28fc345700 10 osd.70 pg_epoch: 71475 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.766919 7f28fcb46700 10 osd.70 pg_epoch: 71489 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.766928 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.766925 7f28fc345700 10 osd.70 pg_epoch: 71475 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.766934 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.766930 7f28fcb46700 10 osd.70 pg_epoch: 71490 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.766936 7f28fc345700 10 osd.70 pg_epoch: 71476 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.766938 7f28fcb46700 10 osd.70 pg_epoch: 71490 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.766943 7f28fc345700 10 osd.70 pg_epoch: 71476 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.766945 7f28fcb46700 10 osd.70 pg_epoch: 71490 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.766951 7f28fc345700 10 osd.70 pg_epoch: 71476 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.766957 7f28fcb46700 10 osd.70 pg_epoch: 71490 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.766966 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.766963 7f28fc345700 10 osd.70 pg_epoch: 71476 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.766971 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.766968 7f28fcb46700 10 osd.70 pg_epoch: 71491 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.766974 7f28fc345700 10 osd.70 pg_epoch: 71477 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.766975 7f28fcb46700 10 osd.70 pg_epoch: 71491 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.766981 7f28fc345700 10 osd.70 pg_epoch: 71477 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.766982 7f28fcb46700 10 osd.70 pg_epoch: 71491 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.766988 7f28fc345700 10 osd.70 pg_epoch: 71477 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.766995 7f28fcb46700 10 osd.70 pg_epoch: 71491 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.767004 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.767000 7f28fc345700 10 osd.70 pg_epoch: 71477 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.767008 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.767006 7f28fcb46700 10 osd.70 pg_epoch: 71492 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.767010 7f28fc345700 10 osd.70 pg_epoch: 71478 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.767013 7f28fcb46700 10 osd.70 pg_epoch: 71492 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.767018 7f28fc345700 10 osd.70 pg_epoch: 71478 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.767020 7f28fcb46700 10 osd.70 pg_epoch: 71492 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.767026 7f28fc345700 10 osd.70 pg_epoch: 71478 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.767033 7f28fcb46700 10 osd.70 pg_epoch: 71492 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.767042 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.767038 7f28fc345700 10 osd.70 pg_epoch: 71478 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.767047 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.767045 7f28fcb46700 10 osd.70 pg_epoch: 71493 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.767049 7f28fc345700 10 osd.70 pg_epoch: 71479 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.767053 7f28fcb46700 10 osd.70 pg_epoch: 71493 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.767057 7f28fc345700 10 osd.70 pg_epoch: 71479 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.767060 7f28fcb46700 10 osd.70 pg_epoch: 71493 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.767065 7f28fc345700 10 osd.70 pg_epoch: 71479 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.767072 7f28fcb46700 10 osd.70 pg_epoch: 71493 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.767081 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.767077 7f28fc345700 10 osd.70 pg_epoch: 71479 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.767087 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.767083 7f28fcb46700 10 osd.70 pg_epoch: 71494 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.767089 7f28fc345700 10 osd.70 pg_epoch: 71480 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.767090 7f28fcb46700 10 osd.70 pg_epoch: 71494 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.767096 7f28fc345700 10 osd.70 pg_epoch: 71480 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.767097 7f28fcb46700 10 osd.70 pg_epoch: 71494 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.767103 7f28fc345700 10 osd.70 pg_epoch: 71480 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.767111 7f28fcb46700 10 osd.70 pg_epoch: 71494 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.767119 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.767115 7f28fc345700 10 osd.70 pg_epoch: 71480 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.767125 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.767121 7f28fcb46700 10 osd.70 pg_epoch: 71495 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.767126 7f28fc345700 10 osd.70 pg_epoch: 71481 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.767128 7f28fcb46700 10 osd.70 pg_epoch: 71495 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.767134 7f28fc345700 10 osd.70 pg_epoch: 71481 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.767134 7f28fcb46700 10 osd.70 pg_epoch: 71495 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.767144 7f28fc345700 10 osd.70 pg_epoch: 71481 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.767151 7f28fcb46700 10 osd.70 pg_epoch: 71495 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.767160 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.767156 7f28fc345700 10 osd.70 pg_epoch: 71481 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.767165 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.767162 7f28fcb46700 10 osd.70 pg_epoch: 71496 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.767167 7f28fc345700 10 osd.70 pg_epoch: 71482 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.767169 7f28fcb46700 10 osd.70 pg_epoch: 71496 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.767176 7f28fcb46700 10 osd.70 pg_epoch: 71496 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.767189 7f28fcb46700 10 osd.70 pg_epoch: 71496 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.767197 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.767174 7f28fc345700 10 osd.70 pg_epoch: 71482 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.767199 7f28fcb46700 10 osd.70 pg_epoch: 71497 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.767200 7f28fc345700 10 osd.70 pg_epoch: 71482 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.767206 7f28fcb46700 10 osd.70 pg_epoch: 71497 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.767213 7f28fcb46700 10 osd.70 pg_epoch: 71497 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.767213 7f28fc345700 10 osd.70 pg_epoch: 71482 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.767223 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.767225 7f28fc345700 10 osd.70 pg_epoch: 71483 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.767227 7f28fcb46700 10 osd.70 pg_epoch: 71497 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.767235 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.767232 7f28fc345700 10 osd.70 pg_epoch: 71483 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.767237 7f28fcb46700 10 osd.70 pg_epoch: 71498 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.767239 7f28fc345700 10 osd.70 pg_epoch: 71483 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.767244 7f28fcb46700 10 osd.70 pg_epoch: 71498 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.767251 7f28fcb46700 10 osd.70 pg_epoch: 71498 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.767251 7f28fc345700 10 osd.70 pg_epoch: 71483 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.767261 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.767262 7f28fc345700 10 osd.70 pg_epoch: 71484 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.767264 7f28fcb46700 10 osd.70 pg_epoch: 71498 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.767272 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.767270 7f28fc345700 10 osd.70 pg_epoch: 71484 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.767274 7f28fcb46700 10 osd.70 pg_epoch: 71499 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.767278 7f28fc345700 10 osd.70 pg_epoch: 71484 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.767281 7f28fcb46700 10 osd.70 pg_epoch: 71499 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.767288 7f28fcb46700 10 osd.70 pg_epoch: 71499 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.767290 7f28fc345700 10 osd.70 pg_epoch: 71484 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.767300 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.767300 7f28fcb46700 10 osd.70 pg_epoch: 71499 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.767301 7f28fc345700 10 osd.70 pg_epoch: 71485 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.767308 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.767309 7f28fc345700 10 osd.70 pg_epoch: 71485 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.767310 7f28fcb46700 10 osd.70 pg_epoch: 71500 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.767316 7f28fcb46700 10 osd.70 pg_epoch: 71500 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.767316 7f28fc345700 10 osd.70 pg_epoch: 71485 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.767324 7f28fcb46700 10 osd.70 pg_epoch: 71500 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.767329 7f28fc345700 10 osd.70 pg_epoch: 71485 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.767337 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.767335 7f28fcb46700 10 osd.70 pg_epoch: 71500 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.767344 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.767339 7f28fc345700 10 osd.70 pg_epoch: 71486 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.767346 7f28fcb46700 10 osd.70 pg_epoch: 71501 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.767350 7f28fc345700 10 osd.70 pg_epoch: 71486 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.767353 7f28fcb46700 10 osd.70 pg_epoch: 71501 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.767362 7f28fcb46700 10 osd.70 pg_epoch: 71501 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.767362 7f28fc345700 10 osd.70 pg_epoch: 71486 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.767398 7f28fcb46700 10 osd.70 pg_epoch: 71501 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.767407 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.767405 7f28fc345700 10 osd.70 pg_epoch: 71486 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.767409 7f28fcb46700 10 osd.70 pg_epoch: 71502 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.767416 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.767416 7f28fcb46700 10 osd.70 pg_epoch: 71502 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.767418 7f28fc345700 10 osd.70 pg_epoch: 71487 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.767423 7f28fcb46700 10 osd.70 pg_epoch: 71502 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.767426 7f28fc345700 10 osd.70 pg_epoch: 71487 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.767433 7f28fc345700 10 osd.70 pg_epoch: 71487 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.767435 7f28fcb46700 10 osd.70 pg_epoch: 71502 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.767444 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.767446 7f28fcb46700 10 osd.70 pg_epoch: 71503 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.767446 7f28fc345700 10 osd.70 pg_epoch: 71487 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.767455 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.767454 7f28fcb46700 10 osd.70 pg_epoch: 71503 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.767457 7f28fc345700 10 osd.70 pg_epoch: 71488 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.767461 7f28fcb46700 10 osd.70 pg_epoch: 71503 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.767465 7f28fc345700 10 osd.70 pg_epoch: 71488 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.767473 7f28fc345700 10 osd.70 pg_epoch: 71488 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.767475 7f28fcb46700 10 osd.70 pg_epoch: 71503 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.767483 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.767485 7f28fcb46700 10 osd.70 pg_epoch: 71504 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.767485 7f28fc345700 10 osd.70 pg_epoch: 71488 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.767495 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.767493 7f28fcb46700 10 osd.70 pg_epoch: 71504 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.767497 7f28fc345700 10 osd.70 pg_epoch: 71489 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.767500 7f28fcb46700 10 osd.70 pg_epoch: 71504 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.767504 7f28fc345700 10 osd.70 pg_epoch: 71489 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.767512 7f28fc345700 10 osd.70 pg_epoch: 71489 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.767512 7f28fcb46700 10 osd.70 pg_epoch: 71504 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.767521 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.767523 7f28fcb46700 10 osd.70 pg_epoch: 71505 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.767529 7f28fcb46700 10 osd.70 pg_epoch: 71505 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.767536 7f28fcb46700 10 osd.70 pg_epoch: 71505 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.767596 7f28fcb46700 10 osd.70 pg_epoch: 71505 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.767596 7f28fc345700 10 osd.70 pg_epoch: 71489 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.767627 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.767628 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.767633 7f28fcb46700 10 osd.70 pg_epoch: 71506 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.767634 7f28fc345700 10 osd.70 pg_epoch: 71490 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.767643 7f28fcb46700 10 osd.70 pg_epoch: 71506 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.767655 7f28fcb46700 10 osd.70 pg_epoch: 71506 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.767651 7f28fc345700 10 osd.70 pg_epoch: 71490 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.767667 7f28fc345700 10 osd.70 pg_epoch: 71490 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.767678 7f28fcb46700 10 osd.70 pg_epoch: 71506 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.767687 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.767690 7f28fcb46700 10 osd.70 pg_epoch: 71507 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.767690 7f28fc345700 10 osd.70 pg_epoch: 71490 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.767698 7f28fcb46700 10 osd.70 pg_epoch: 71507 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.767706 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.767705 7f28fcb46700 10 osd.70 pg_epoch: 71507 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.767710 7f28fc345700 10 osd.70 pg_epoch: 71491 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.767719 7f28fcb46700 10 osd.70 pg_epoch: 71507 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.767728 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.767723 7f28fc345700 10 osd.70 pg_epoch: 71491 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.767731 7f28fcb46700 10 osd.70 pg_epoch: 71508 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.767736 7f28fc345700 10 osd.70 pg_epoch: 71491 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.767738 7f28fcb46700 10 osd.70 pg_epoch: 71508 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.767745 7f28fcb46700 10 osd.70 pg_epoch: 71508 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.767754 7f28fc345700 10 osd.70 pg_epoch: 71491 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.767764 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.767761 7f28fcb46700 10 osd.70 pg_epoch: 71508 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.767770 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.767767 7f28fc345700 10 osd.70 pg_epoch: 71492 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.767772 7f28fcb46700 10 osd.70 pg_epoch: 71509 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.767774 7f28fc345700 10 osd.70 pg_epoch: 71492 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.767780 7f28fcb46700 10 osd.70 pg_epoch: 71509 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.767782 7f28fc345700 10 osd.70 pg_epoch: 71492 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.767787 7f28fcb46700 10 osd.70 pg_epoch: 71509 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.767797 7f28fc345700 10 osd.70 pg_epoch: 71492 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.767806 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.767802 7f28fcb46700 10 osd.70 pg_epoch: 71509 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.767811 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.767808 7f28fc345700 10 osd.70 pg_epoch: 71493 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.767813 7f28fcb46700 10 osd.70 pg_epoch: 71510 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.767815 7f28fc345700 10 osd.70 pg_epoch: 71493 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.767820 7f28fcb46700 10 osd.70 pg_epoch: 71510 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.767826 7f28fc345700 10 osd.70 pg_epoch: 71493 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.767828 7f28fcb46700 10 osd.70 pg_epoch: 71510 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.767847 7f28fc345700 10 osd.70 pg_epoch: 71493 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.767856 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.767851 7f28fcb46700 10 osd.70 pg_epoch: 71510 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.767860 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.767858 7f28fc345700 10 osd.70 pg_epoch: 71494 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.767862 7f28fcb46700 10 osd.70 pg_epoch: 71511 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.767866 7f28fc345700 10 osd.70 pg_epoch: 71494 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.767870 7f28fcb46700 10 osd.70 pg_epoch: 71511 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.767874 7f28fc345700 10 osd.70 pg_epoch: 71494 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.767878 7f28fcb46700 10 osd.70 pg_epoch: 71511 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.767889 7f28fc345700 10 osd.70 pg_epoch: 71494 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.767893 7f28fcb46700 10 osd.70 pg_epoch: 71511 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.767901 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.767902 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.767903 7f28fc345700 10 osd.70 pg_epoch: 71495 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.767904 7f28fcb46700 10 osd.70 pg_epoch: 71512 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.767911 7f28fcb46700 10 osd.70 pg_epoch: 71512 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.767912 7f28fc345700 10 osd.70 pg_epoch: 71495 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.767919 7f28fcb46700 10 osd.70 pg_epoch: 71512 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.767922 7f28fc345700 10 osd.70 pg_epoch: 71495 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.767934 7f28fcb46700 10 osd.70 pg_epoch: 71512 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.767937 7f28fc345700 10 osd.70 pg_epoch: 71495 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.767944 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.767946 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.767946 7f28fcb46700 10 osd.70 pg_epoch: 71513 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.767948 7f28fc345700 10 osd.70 pg_epoch: 71496 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.767954 7f28fcb46700 10 osd.70 pg_epoch: 71513 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.767956 7f28fc345700 10 osd.70 pg_epoch: 71496 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.767961 7f28fcb46700 10 osd.70 pg_epoch: 71513 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.767963 7f28fc345700 10 osd.70 pg_epoch: 71496 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.767976 7f28fcb46700 10 osd.70 pg_epoch: 71513 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.767985 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.767978 7f28fc345700 10 osd.70 pg_epoch: 71496 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.767990 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.767987 7f28fcb46700 10 osd.70 pg_epoch: 71514 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.767993 7f28fc345700 10 osd.70 pg_epoch: 71497 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.767994 7f28fcb46700 10 osd.70 pg_epoch: 71514 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.768002 7f28fcb46700 10 osd.70 pg_epoch: 71514 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.768002 7f28fc345700 10 osd.70 pg_epoch: 71497 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.768017 7f28fcb46700 10 osd.70 pg_epoch: 71514 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.768015 7f28fc345700 10 osd.70 pg_epoch: 71497 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.768026 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.768028 7f28fcb46700 10 osd.70 pg_epoch: 71515 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.768035 7f28fcb46700 10 osd.70 pg_epoch: 71515 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.768042 7f28fcb46700 10 osd.70 pg_epoch: 71515 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.768037 7f28fc345700 10 osd.70 pg_epoch: 71497 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.768054 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.768057 7f28fcb46700 10 osd.70 pg_epoch: 71515 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.768057 7f28fc345700 10 osd.70 pg_epoch: 71498 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.768067 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.768075 7f28fcb46700 10 osd.70 pg_epoch: 71516 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.768076 7f28fc345700 10 osd.70 pg_epoch: 71498 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.768083 7f28fcb46700 10 osd.70 pg_epoch: 71516 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.768086 7f28fc345700 10 osd.70 pg_epoch: 71498 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.768090 7f28fcb46700 10 osd.70 pg_epoch: 71516 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.768111 7f28fc345700 10 osd.70 pg_epoch: 71498 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.768117 7f28fcb46700 10 osd.70 pg_epoch: 71516 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.768126 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.768128 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.768129 7f28fc345700 10 osd.70 pg_epoch: 71499 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.768131 7f28fcb46700 10 osd.70 pg_epoch: 71517 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.768141 7f28fc345700 10 osd.70 pg_epoch: 71499 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.768143 7f28fcb46700 10 osd.70 pg_epoch: 71517 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.768151 7f28fcb46700 10 osd.70 pg_epoch: 71517 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.768150 7f28fc345700 10 osd.70 pg_epoch: 71499 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.768167 7f28fcb46700 10 osd.70 pg_epoch: 71517 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.768177 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.768171 7f28fc345700 10 osd.70 pg_epoch: 71499 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.768197 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.768179 7f28fcb46700 10 osd.70 pg_epoch: 71518 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.768201 7f28fc345700 10 osd.70 pg_epoch: 71500 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.768208 7f28fcb46700 10 osd.70 pg_epoch: 71518 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.768218 7f28fcb46700 10 osd.70 pg_epoch: 71518 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.768214 7f28fc345700 10 osd.70 pg_epoch: 71500 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.768227 7f28fc345700 10 osd.70 pg_epoch: 71500 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.768235 7f28fcb46700 10 osd.70 pg_epoch: 71518 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.768245 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.768247 7f28fcb46700 10 osd.70 pg_epoch: 71519 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.768254 7f28fcb46700 10 osd.70 pg_epoch: 71519 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.768249 7f28fc345700 10 osd.70 pg_epoch: 71500 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.768262 7f28fcb46700 10 osd.70 pg_epoch: 71519 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.768266 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.768270 7f28fc345700 10 osd.70 pg_epoch: 71501 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.768276 7f28fcb46700 10 osd.70 pg_epoch: 71519 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.768286 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.768288 7f28fcb46700 10 osd.70 pg_epoch: 71520 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.768284 7f28fc345700 10 osd.70 pg_epoch: 71501 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.768295 7f28fcb46700 10 osd.70 pg_epoch: 71520 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.768303 7f28fcb46700 10 osd.70 pg_epoch: 71520 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.768299 7f28fc345700 10 osd.70 pg_epoch: 71501 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.768317 7f28fcb46700 10 osd.70 pg_epoch: 71520 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.768326 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.768328 7f28fcb46700 10 osd.70 pg_epoch: 71521 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.768322 7f28fc345700 10 osd.70 pg_epoch: 71501 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.768339 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.768335 7f28fcb46700 10 osd.70 pg_epoch: 71521 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.768342 7f28fcb46700 10 osd.70 pg_epoch: 71521 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.768343 7f28fc345700 10 osd.70 pg_epoch: 71502 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.768356 7f28fcb46700 10 osd.70 pg_epoch: 71521 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.768366 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.768357 7f28fc345700 10 osd.70 pg_epoch: 71502 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.768368 7f28fcb46700 10 osd.70 pg_epoch: 71522 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.768369 7f28fc345700 10 osd.70 pg_epoch: 71502 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.768376 7f28fcb46700 10 osd.70 pg_epoch: 71522 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.768383 7f28fcb46700 10 osd.70 pg_epoch: 71522 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.768386 7f28fc345700 10 osd.70 pg_epoch: 71502 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.768395 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.768397 7f28fc345700 10 osd.70 pg_epoch: 71503 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.768397 7f28fcb46700 10 osd.70 pg_epoch: 71522 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.768407 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.768405 7f28fc345700 10 osd.70 pg_epoch: 71503 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.768409 7f28fcb46700 10 osd.70 pg_epoch: 71523 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.768413 7f28fc345700 10 osd.70 pg_epoch: 71503 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.768417 7f28fcb46700 10 osd.70 pg_epoch: 71523 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.768425 7f28fcb46700 10 osd.70 pg_epoch: 71523 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.768427 7f28fc345700 10 osd.70 pg_epoch: 71503 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.768436 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.768438 7f28fc345700 10 osd.70 pg_epoch: 71504 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.768439 7f28fcb46700 10 osd.70 pg_epoch: 71523 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.768449 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.768446 7f28fc345700 10 osd.70 pg_epoch: 71504 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.768453 7f28fc345700 10 osd.70 pg_epoch: 71504 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.768453 7f28fcb46700 10 osd.70 pg_epoch: 71524 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.768465 7f28fc345700 10 osd.70 pg_epoch: 71504 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.768464 7f28fcb46700 10 osd.70 pg_epoch: 71524 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.768473 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.768475 7f28fcb46700 10 osd.70 pg_epoch: 71524 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.768479 7f28fc345700 10 osd.70 pg_epoch: 71505 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.768486 7f28fc345700 10 osd.70 pg_epoch: 71505 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.768493 7f28fc345700 10 osd.70 pg_epoch: 71505 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.768492 7f28fcb46700 10 osd.70 pg_epoch: 71524 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.768503 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.768505 7f28fcb46700 10 osd.70 pg_epoch: 71525 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.768506 7f28fc345700 10 osd.70 pg_epoch: 71505 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.768515 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.768513 7f28fcb46700 10 osd.70 pg_epoch: 71525 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.768517 7f28fc345700 10 osd.70 pg_epoch: 71506 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.768520 7f28fcb46700 10 osd.70 pg_epoch: 71525 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.768526 7f28fc345700 10 osd.70 pg_epoch: 71506 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.768532 7f28fc345700 10 osd.70 pg_epoch: 71506 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.768537 7f28fcb46700 10 osd.70 pg_epoch: 71525 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.768547 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.768544 7f28fc345700 10 osd.70 pg_epoch: 71506 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.768553 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.768555 7f28fc345700 10 osd.70 pg_epoch: 71507 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.768550 7f28fcb46700 10 osd.70 pg_epoch: 71526 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.768562 7f28fc345700 10 osd.70 pg_epoch: 71507 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.768562 7f28fcb46700 10 osd.70 pg_epoch: 71526 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.768570 7f28fc345700 10 osd.70 pg_epoch: 71507 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.768575 7f28fcb46700 10 osd.70 pg_epoch: 71526 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.768582 7f28fc345700 10 osd.70 pg_epoch: 71507 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.768590 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.768591 7f28fcb46700 10 osd.70 pg_epoch: 71526 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.768591 7f28fc345700 10 osd.70 pg_epoch: 71508 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.768602 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.768603 7f28fc345700 10 osd.70 pg_epoch: 71508 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.768607 7f28fcb46700 10 osd.70 pg_epoch: 71527 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.768610 7f28fc345700 10 osd.70 pg_epoch: 71508 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.768616 7f28fcb46700 10 osd.70 pg_epoch: 71527 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.768623 7f28fc345700 10 osd.70 pg_epoch: 71508 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.768632 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.768626 7f28fcb46700 10 osd.70 pg_epoch: 71527 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.768634 7f28fc345700 10 osd.70 pg_epoch: 71509 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.768641 7f28fc345700 10 osd.70 pg_epoch: 71509 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.768648 7f28fc345700 10 osd.70 pg_epoch: 71509 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.768643 7f28fcb46700 10 osd.70 pg_epoch: 71527 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.768657 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.768659 7f28fcb46700 10 osd.70 pg_epoch: 71528 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.768660 7f28fc345700 10 osd.70 pg_epoch: 71509 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.768669 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.768667 7f28fcb46700 10 osd.70 pg_epoch: 71528 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.768671 7f28fc345700 10 osd.70 pg_epoch: 71510 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.768677 7f28fcb46700 10 osd.70 pg_epoch: 71528 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.768678 7f28fc345700 10 osd.70 pg_epoch: 71510 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.768685 7f28fc345700 10 osd.70 pg_epoch: 71510 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.768692 7f28fcb46700 10 osd.70 pg_epoch: 71528 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.768697 7f28fc345700 10 osd.70 pg_epoch: 71510 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.768705 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.768706 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.768708 7f28fc345700 10 osd.70 pg_epoch: 71511 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.768708 7f28fcb46700 10 osd.70 pg_epoch: 71529 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.768715 7f28fc345700 10 osd.70 pg_epoch: 71511 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.768716 7f28fcb46700 10 osd.70 pg_epoch: 71529 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.768723 7f28fc345700 10 osd.70 pg_epoch: 71511 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.768723 7f28fcb46700 10 osd.70 pg_epoch: 71529 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.768735 7f28fc345700 10 osd.70 pg_epoch: 71511 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.768744 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.768746 7f28fc345700 10 osd.70 pg_epoch: 71512 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.768744 7f28fcb46700 10 osd.70 pg_epoch: 71529 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.768755 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.768753 7f28fc345700 10 osd.70 pg_epoch: 71512 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.768758 7f28fcb46700 10 osd.70 pg_epoch: 71530 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.768761 7f28fc345700 10 osd.70 pg_epoch: 71512 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.768767 7f28fcb46700 10 osd.70 pg_epoch: 71530 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.768773 7f28fc345700 10 osd.70 pg_epoch: 71512 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.768782 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.768777 7f28fcb46700 10 osd.70 pg_epoch: 71530 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.768786 7f28fc345700 10 osd.70 pg_epoch: 71513 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.768794 7f28fc345700 10 osd.70 pg_epoch: 71513 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.768797 7f28fcb46700 10 osd.70 pg_epoch: 71530 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.768801 7f28fc345700 10 osd.70 pg_epoch: 71513 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.768808 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.768813 7f28fc345700 10 osd.70 pg_epoch: 71513 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.768812 7f28fcb46700 10 osd.70 pg_epoch: 71531 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.768824 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.768825 7f28fcb46700 10 osd.70 pg_epoch: 71531 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.768826 7f28fc345700 10 osd.70 pg_epoch: 71514 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.768833 7f28fc345700 10 osd.70 pg_epoch: 71514 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.768833 7f28fcb46700 10 osd.70 pg_epoch: 71531 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.768842 7f28fc345700 10 osd.70 pg_epoch: 71514 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.768851 7f28fcb46700 10 osd.70 pg_epoch: 71531 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.768860 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.768855 7f28fc345700 10 osd.70 pg_epoch: 71514 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.768864 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.768863 7f28fcb46700 10 osd.70 pg_epoch: 71532 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.768866 7f28fc345700 10 osd.70 pg_epoch: 71515 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.768870 7f28fcb46700 10 osd.70 pg_epoch: 71532 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.768874 7f28fc345700 10 osd.70 pg_epoch: 71515 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.768878 7f28fcb46700 10 osd.70 pg_epoch: 71532 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.768882 7f28fc345700 10 osd.70 pg_epoch: 71515 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.768893 7f28fcb46700 10 osd.70 pg_epoch: 71532 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.768894 7f28fc345700 10 osd.70 pg_epoch: 71515 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.768902 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.768903 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.768905 7f28fc345700 10 osd.70 pg_epoch: 71516 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.768904 7f28fcb46700 10 osd.70 pg_epoch: 71533 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.768914 7f28fc345700 10 osd.70 pg_epoch: 71516 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.768914 7f28fcb46700 10 osd.70 pg_epoch: 71533 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.768922 7f28fc345700 10 osd.70 pg_epoch: 71516 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.768922 7f28fcb46700 10 osd.70 pg_epoch: 71533 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.768935 7f28fc345700 10 osd.70 pg_epoch: 71516 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.768936 7f28fcb46700 10 osd.70 pg_epoch: 71533 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.768944 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.768945 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.768946 7f28fc345700 10 osd.70 pg_epoch: 71517 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.768947 7f28fcb46700 10 osd.70 pg_epoch: 71534 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.768953 7f28fc345700 10 osd.70 pg_epoch: 71517 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.768955 7f28fcb46700 10 osd.70 pg_epoch: 71534 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.768960 7f28fc345700 10 osd.70 pg_epoch: 71517 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.768962 7f28fcb46700 10 osd.70 pg_epoch: 71534 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.768972 7f28fc345700 10 osd.70 pg_epoch: 71517 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.768981 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.768976 7f28fcb46700 10 osd.70 pg_epoch: 71534 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.768986 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.768983 7f28fc345700 10 osd.70 pg_epoch: 71518 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.768988 7f28fcb46700 10 osd.70 pg_epoch: 71535 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.768990 7f28fc345700 10 osd.70 pg_epoch: 71518 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.768995 7f28fcb46700 10 osd.70 pg_epoch: 71535 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.768996 7f28fc345700 10 osd.70 pg_epoch: 71518 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.769002 7f28fcb46700 10 osd.70 pg_epoch: 71535 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.769008 7f28fc345700 10 osd.70 pg_epoch: 71518 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.769017 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.769016 7f28fcb46700 10 osd.70 pg_epoch: 71535 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.769019 7f28fc345700 10 osd.70 pg_epoch: 71519 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.769024 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.769026 7f28fc345700 10 osd.70 pg_epoch: 71519 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.769026 7f28fcb46700 10 osd.70 pg_epoch: 71536 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.769034 7f28fc345700 10 osd.70 pg_epoch: 71519 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.769035 7f28fcb46700 10 osd.70 pg_epoch: 71536 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.769042 7f28fcb46700 10 osd.70 pg_epoch: 71536 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.769047 7f28fc345700 10 osd.70 pg_epoch: 71519 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.769055 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.769055 7f28fcb46700 10 osd.70 pg_epoch: 71536 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.769058 7f28fc345700 10 osd.70 pg_epoch: 71520 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.769066 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.769068 7f28fc345700 10 osd.70 pg_epoch: 71520 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.769069 7f28fcb46700 10 osd.70 pg_epoch: 71537 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.769077 7f28fcb46700 10 osd.70 pg_epoch: 71537 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.769076 7f28fc345700 10 osd.70 pg_epoch: 71520 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.769084 7f28fcb46700 10 osd.70 pg_epoch: 71537 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.769098 7f28fcb46700 10 osd.70 pg_epoch: 71537 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.769106 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.769095 7f28fc345700 10 osd.70 pg_epoch: 71520 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.769110 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.769108 7f28fcb46700 10 osd.70 pg_epoch: 71538 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.769115 7f28fcb46700 10 osd.70 pg_epoch: 71538 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.769114 7f28fc345700 10 osd.70 pg_epoch: 71521 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.769122 7f28fcb46700 10 osd.70 pg_epoch: 71538 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.769125 7f28fc345700 10 osd.70 pg_epoch: 71521 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.769136 7f28fcb46700 10 osd.70 pg_epoch: 71538 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.769145 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.769137 7f28fc345700 10 osd.70 pg_epoch: 71521 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.769147 7f28fcb46700 10 osd.70 pg_epoch: 71539 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.769154 7f28fcb46700 10 osd.70 pg_epoch: 71539 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.769161 7f28fcb46700 10 osd.70 pg_epoch: 71539 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.769156 7f28fc345700 10 osd.70 pg_epoch: 71521 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.769171 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.769175 7f28fcb46700 10 osd.70 pg_epoch: 71539 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.769184 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.769174 7f28fc345700 10 osd.70 pg_epoch: 71522 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.769188 7f28fcb46700 10 osd.70 pg_epoch: 71540 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.769196 7f28fcb46700 10 osd.70 pg_epoch: 71540 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.769193 7f28fc345700 10 osd.70 pg_epoch: 71522 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.769203 7f28fcb46700 10 osd.70 pg_epoch: 71540 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.769206 7f28fc345700 10 osd.70 pg_epoch: 71522 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.769219 7f28fcb46700 10 osd.70 pg_epoch: 71540 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.769227 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.769226 7f28fc345700 10 osd.70 pg_epoch: 71522 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.769237 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.769240 7f28fc345700 10 osd.70 pg_epoch: 71523 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.769247 7f28fc345700 10 osd.70 pg_epoch: 71523 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.769255 7f28fc345700 10 osd.70 pg_epoch: 71523 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.769229 7f28fcb46700 10 osd.70 pg_epoch: 71541 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.769269 7f28fc345700 10 osd.70 pg_epoch: 71523 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.769268 7f28fcb46700 10 osd.70 pg_epoch: 71541 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.769288 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.769291 7f28fc345700 10 osd.70 pg_epoch: 71524 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.769300 7f28fc345700 10 osd.70 pg_epoch: 71524 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.769311 7f28fc345700 10 osd.70 pg_epoch: 71524 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.769292 7f28fcb46700 10 osd.70 pg_epoch: 71541 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.769343 7f28fcb46700 10 osd.70 pg_epoch: 71541 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.769361 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.769349 7f28fc345700 10 osd.70 pg_epoch: 71524 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.769370 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.769366 7f28fcb46700 10 osd.70 pg_epoch: 71542 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.769373 7f28fc345700 10 osd.70 pg_epoch: 71525 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.769379 7f28fcb46700 10 osd.70 pg_epoch: 71542 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.769385 7f28fc345700 10 osd.70 pg_epoch: 71525 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.769394 7f28fc345700 10 osd.70 pg_epoch: 71525 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.769392 7f28fcb46700 10 osd.70 pg_epoch: 71542 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.769411 7f28fc345700 10 osd.70 pg_epoch: 71525 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.769420 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.769422 7f28fc345700 10 osd.70 pg_epoch: 71526 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.769429 7f28fc345700 10 osd.70 pg_epoch: 71526 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.769437 7f28fc345700 10 osd.70 pg_epoch: 71526 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.769429 7f28fcb46700 10 osd.70 pg_epoch: 71542 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.769447 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.769451 7f28fc345700 10 osd.70 pg_epoch: 71526 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.769460 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.769451 7f28fcb46700 10 osd.70 pg_epoch: 71543 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.769462 7f28fc345700 10 osd.70 pg_epoch: 71527 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.769469 7f28fc345700 10 osd.70 pg_epoch: 71527 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.769477 7f28fc345700 10 osd.70 pg_epoch: 71527 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.769464 7f28fcb46700 10 osd.70 pg_epoch: 71543 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.769490 7f28fc345700 10 osd.70 pg_epoch: 71527 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.769488 7f28fcb46700 10 osd.70 pg_epoch: 71543 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.769502 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.769504 7f28fc345700 10 osd.70 pg_epoch: 71528 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.769511 7f28fc345700 10 osd.70 pg_epoch: 71528 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.769515 7f28fcb46700 10 osd.70 pg_epoch: 71543 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.769518 7f28fc345700 10 osd.70 pg_epoch: 71528 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.769527 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.769529 7f28fcb46700 10 osd.70 pg_epoch: 71544 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.769534 7f28fc345700 10 osd.70 pg_epoch: 71528 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.769537 7f28fcb46700 10 osd.70 pg_epoch: 71544 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.769544 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.769545 7f28fcb46700 10 osd.70 pg_epoch: 71544 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.769546 7f28fc345700 10 osd.70 pg_epoch: 71529 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.769554 7f28fc345700 10 osd.70 pg_epoch: 71529 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.769561 7f28fc345700 10 osd.70 pg_epoch: 71529 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.769560 7f28fcb46700 10 osd.70 pg_epoch: 71544 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.769570 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.769575 7f28fc345700 10 osd.70 pg_epoch: 71529 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.769572 7f28fcb46700 10 osd.70 pg_epoch: 71545 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.769585 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.769584 7f28fcb46700 10 osd.70 pg_epoch: 71545 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.769587 7f28fc345700 10 osd.70 pg_epoch: 71530 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.769591 7f28fcb46700 10 osd.70 pg_epoch: 71545 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.769595 7f28fc345700 10 osd.70 pg_epoch: 71530 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.769602 7f28fc345700 10 osd.70 pg_epoch: 71530 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.769607 7f28fcb46700 10 osd.70 pg_epoch: 71545 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.769619 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.769615 7f28fc345700 10 osd.70 pg_epoch: 71530 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.769624 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.769621 7f28fcb46700 10 osd.70 pg_epoch: 71546 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.769626 7f28fc345700 10 osd.70 pg_epoch: 71531 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.769629 7f28fcb46700 10 osd.70 pg_epoch: 71546 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.769634 7f28fc345700 10 osd.70 pg_epoch: 71531 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.769636 7f28fcb46700 10 osd.70 pg_epoch: 71546 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.769641 7f28fc345700 10 osd.70 pg_epoch: 71531 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.769651 7f28fcb46700 10 osd.70 pg_epoch: 71546 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.769660 7f28fc345700 10 osd.70 pg_epoch: 71531 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.769666 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.769669 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.769671 7f28fc345700 10 osd.70 pg_epoch: 71532 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.769670 7f28fcb46700 10 osd.70 pg_epoch: 71547 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.769679 7f28fc345700 10 osd.70 pg_epoch: 71532 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.769686 7f28fc345700 10 osd.70 pg_epoch: 71532 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.769682 7f28fcb46700 10 osd.70 pg_epoch: 71547 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.769695 7f28fcb46700 10 osd.70 pg_epoch: 71547 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.769700 7f28fc345700 10 osd.70 pg_epoch: 71532 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.769709 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.769711 7f28fc345700 10 osd.70 pg_epoch: 71533 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.769718 7f28fc345700 10 osd.70 pg_epoch: 71533 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.769725 7f28fc345700 10 osd.70 pg_epoch: 71533 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.769717 7f28fcb46700 10 osd.70 pg_epoch: 71547 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.769737 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.769739 7f28fc345700 10 osd.70 pg_epoch: 71533 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.769747 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.769741 7f28fcb46700 10 osd.70 pg_epoch: 71548 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.769749 7f28fc345700 10 osd.70 pg_epoch: 71534 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.769756 7f28fc345700 10 osd.70 pg_epoch: 71534 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.769754 7f28fcb46700 10 osd.70 pg_epoch: 71548 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.769763 7f28fc345700 10 osd.70 pg_epoch: 71534 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.769767 7f28fcb46700 10 osd.70 pg_epoch: 71548 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.769778 7f28fc345700 10 osd.70 pg_epoch: 71534 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.769786 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.769788 7f28fc345700 10 osd.70 pg_epoch: 71535 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.769790 7f28fcb46700 10 osd.70 pg_epoch: 71548 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.769795 7f28fc345700 10 osd.70 pg_epoch: 71535 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.769803 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.769803 7f28fc345700 10 osd.70 pg_epoch: 71535 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.769807 7f28fcb46700 10 osd.70 pg_epoch: 71549 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.769816 7f28fc345700 10 osd.70 pg_epoch: 71535 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.769824 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.769819 7f28fcb46700 10 osd.70 pg_epoch: 71549 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.769826 7f28fc345700 10 osd.70 pg_epoch: 71536 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.769833 7f28fc345700 10 osd.70 pg_epoch: 71536 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.769833 7f28fcb46700 10 osd.70 pg_epoch: 71549 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.769840 7f28fc345700 10 osd.70 pg_epoch: 71536 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.769856 7f28fc345700 10 osd.70 pg_epoch: 71536 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.769864 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.769866 7f28fc345700 10 osd.70 pg_epoch: 71537 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.769857 7f28fcb46700 10 osd.70 pg_epoch: 71549 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.769874 7f28fc345700 10 osd.70 pg_epoch: 71537 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.769878 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.769881 7f28fc345700 10 osd.70 pg_epoch: 71537 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.769883 7f28fcb46700 10 osd.70 pg_epoch: 71550 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.769892 7f28fcb46700 10 osd.70 pg_epoch: 71550 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.769899 7f28fcb46700 10 osd.70 pg_epoch: 71550 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.769894 7f28fc345700 10 osd.70 pg_epoch: 71537 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.769919 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.769914 7f28fcb46700 10 osd.70 pg_epoch: 71550 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.769924 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.769921 7f28fc345700 10 osd.70 pg_epoch: 71538 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.769926 7f28fcb46700 10 osd.70 pg_epoch: 71551 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.769932 7f28fc345700 10 osd.70 pg_epoch: 71538 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.769934 7f28fcb46700 10 osd.70 pg_epoch: 71551 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.769939 7f28fc345700 10 osd.70 pg_epoch: 71538 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.769941 7f28fcb46700 10 osd.70 pg_epoch: 71551 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.769954 7f28fc345700 10 osd.70 pg_epoch: 71538 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.769955 7f28fcb46700 10 osd.70 pg_epoch: 71551 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.769963 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.769964 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.769965 7f28fc345700 10 osd.70 pg_epoch: 71539 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.769967 7f28fcb46700 10 osd.70 pg_epoch: 71552 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.769974 7f28fcb46700 10 osd.70 pg_epoch: 71552 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.769972 7f28fc345700 10 osd.70 pg_epoch: 71539 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.769981 7f28fcb46700 10 osd.70 pg_epoch: 71552 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.769983 7f28fc345700 10 osd.70 pg_epoch: 71539 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.769996 7f28fc345700 10 osd.70 pg_epoch: 71539 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.769996 7f28fcb46700 10 osd.70 pg_epoch: 71552 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.770008 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.770009 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.770010 7f28fc345700 10 osd.70 pg_epoch: 71540 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.770011 7f28fcb46700 10 osd.70 pg_epoch: 71553 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.770017 7f28fc345700 10 osd.70 pg_epoch: 71540 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.770018 7f28fcb46700 10 osd.70 pg_epoch: 71553 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.770025 7f28fc345700 10 osd.70 pg_epoch: 71540 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.770026 7f28fcb46700 10 osd.70 pg_epoch: 71553 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.770039 7f28fc345700 10 osd.70 pg_epoch: 71540 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.770047 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.770041 7f28fcb46700 10 osd.70 pg_epoch: 71553 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.770051 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.770053 7f28fcb46700 10 osd.70 pg_epoch: 71554 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.770049 7f28fc345700 10 osd.70 pg_epoch: 71541 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.770059 7f28fcb46700 10 osd.70 pg_epoch: 71554 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.770064 7f28fc345700 10 osd.70 pg_epoch: 71541 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.770067 7f28fcb46700 10 osd.70 pg_epoch: 71554 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.770073 7f28fc345700 10 osd.70 pg_epoch: 71541 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.770080 7f28fcb46700 10 osd.70 pg_epoch: 71554 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.770086 7f28fc345700 10 osd.70 pg_epoch: 71541 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.770095 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.770095 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.770097 7f28fc345700 10 osd.70 pg_epoch: 71542 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.770099 7f28fcb46700 10 osd.70 pg_epoch: 71555 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.770105 7f28fc345700 10 osd.70 pg_epoch: 71542 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.770112 7f28fc345700 10 osd.70 pg_epoch: 71542 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.770112 7f28fcb46700 10 osd.70 pg_epoch: 71555 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.770121 7f28fcb46700 10 osd.70 pg_epoch: 71555 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.770126 7f28fc345700 10 osd.70 pg_epoch: 71542 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.770134 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.770136 7f28fc345700 10 osd.70 pg_epoch: 71543 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.770136 7f28fcb46700 10 osd.70 pg_epoch: 71555 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.770143 7f28fc345700 10 osd.70 pg_epoch: 71543 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.770147 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.770150 7f28fc345700 10 osd.70 pg_epoch: 71543 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.770152 7f28fcb46700 10 osd.70 pg_epoch: 71556 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.770162 7f28fc345700 10 osd.70 pg_epoch: 71543 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.770170 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.770164 7f28fcb46700 10 osd.70 pg_epoch: 71556 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.770172 7f28fc345700 10 osd.70 pg_epoch: 71544 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.770179 7f28fc345700 10 osd.70 pg_epoch: 71544 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.770178 7f28fcb46700 10 osd.70 pg_epoch: 71556 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.770187 7f28fc345700 10 osd.70 pg_epoch: 71544 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.770202 7f28fc345700 10 osd.70 pg_epoch: 71544 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.770210 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.770212 7f28fc345700 10 osd.70 pg_epoch: 71545 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.770220 7f28fc345700 10 osd.70 pg_epoch: 71545 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.770210 7f28fcb46700 10 osd.70 pg_epoch: 71556 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.770229 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.770227 7f28fc345700 10 osd.70 pg_epoch: 71545 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.770234 7f28fcb46700 10 osd.70 pg_epoch: 71557 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.770239 7f28fc345700 10 osd.70 pg_epoch: 71545 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.770248 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.770250 7f28fc345700 10 osd.70 pg_epoch: 71546 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.770246 7f28fcb46700 10 osd.70 pg_epoch: 71557 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.770257 7f28fc345700 10 osd.70 pg_epoch: 71546 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.770265 7f28fc345700 10 osd.70 pg_epoch: 71546 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.770259 7f28fcb46700 10 osd.70 pg_epoch: 71557 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.770276 7f28fc345700 10 osd.70 pg_epoch: 71546 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.770284 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.770286 7f28fc345700 10 osd.70 pg_epoch: 71547 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.770282 7f28fcb46700 10 osd.70 pg_epoch: 71557 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.770293 7f28fc345700 10 osd.70 pg_epoch: 71547 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.770299 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.770300 7f28fc345700 10 osd.70 pg_epoch: 71547 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.770303 7f28fcb46700 10 osd.70 pg_epoch: 71558 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.770312 7f28fc345700 10 osd.70 pg_epoch: 71547 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.770321 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.770316 7f28fcb46700 10 osd.70 pg_epoch: 71558 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.770323 7f28fc345700 10 osd.70 pg_epoch: 71548 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.770328 7f28fcb46700 10 osd.70 pg_epoch: 71558 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.770330 7f28fc345700 10 osd.70 pg_epoch: 71548 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.770338 7f28fc345700 10 osd.70 pg_epoch: 71548 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.770344 7f28fcb46700 10 osd.70 pg_epoch: 71558 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.770354 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.770350 7f28fc345700 10 osd.70 pg_epoch: 71548 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.770359 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.770361 7f28fc345700 10 osd.70 pg_epoch: 71549 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.770359 7f28fcb46700 10 osd.70 pg_epoch: 71559 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.770367 7f28fc345700 10 osd.70 pg_epoch: 71549 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.770375 7f28fc345700 10 osd.70 pg_epoch: 71549 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.770371 7f28fcb46700 10 osd.70 pg_epoch: 71559 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.770387 7f28fc345700 10 osd.70 pg_epoch: 71549 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.770385 7f28fcb46700 10 osd.70 pg_epoch: 71559 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.770395 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.770397 7f28fc345700 10 osd.70 pg_epoch: 71550 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.770403 7f28fc345700 10 osd.70 pg_epoch: 71550 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.770410 7f28fc345700 10 osd.70 pg_epoch: 71550 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.770407 7f28fcb46700 10 osd.70 pg_epoch: 71559 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.770423 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.770422 7f28fc345700 10 osd.70 pg_epoch: 71550 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.770426 7f28fcb46700 10 osd.70 pg_epoch: 71560 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.776009 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.776019 7f28fc345700 10 osd.70 pg_epoch: 71551 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.776013 7f28fcb46700 10 osd.70 pg_epoch: 71560 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.776031 7f28fcb46700 10 osd.70 pg_epoch: 71560 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.776032 7f28fc345700 10 osd.70 pg_epoch: 71551 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.776044 7f28fc345700 10 osd.70 pg_epoch: 71551 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.776062 7f28fc345700 10 osd.70 pg_epoch: 71551 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.776054 7f28fcb46700 10 osd.70 pg_epoch: 71560 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.776071 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.776074 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.776073 7f28fc345700 10 osd.70 pg_epoch: 71552 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.776077 7f28fcb46700 10 osd.70 pg_epoch: 71561 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.776081 7f28fc345700 10 osd.70 pg_epoch: 71552 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.776085 7f28fcb46700 10 osd.70 pg_epoch: 71561 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.776089 7f28fc345700 10 osd.70 pg_epoch: 71552 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.776093 7f28fcb46700 10 osd.70 pg_epoch: 71561 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.776104 7f28fc345700 10 osd.70 pg_epoch: 71552 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.776113 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.776107 7f28fcb46700 10 osd.70 pg_epoch: 71561 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.776116 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.776115 7f28fc345700 10 osd.70 pg_epoch: 71553 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.776118 7f28fcb46700 10 osd.70 pg_epoch: 71562 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.776122 7f28fc345700 10 osd.70 pg_epoch: 71553 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.776126 7f28fcb46700 10 osd.70 pg_epoch: 71562 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.776130 7f28fc345700 10 osd.70 pg_epoch: 71553 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.776133 7f28fcb46700 10 osd.70 pg_epoch: 71562 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.776144 7f28fc345700 10 osd.70 pg_epoch: 71553 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.776153 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.776146 7f28fcb46700 10 osd.70 pg_epoch: 71562 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.776156 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.776155 7f28fc345700 10 osd.70 pg_epoch: 71554 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.776158 7f28fcb46700 10 osd.70 pg_epoch: 71563 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.776162 7f28fc345700 10 osd.70 pg_epoch: 71554 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.776165 7f28fcb46700 10 osd.70 pg_epoch: 71563 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.776172 7f28fcb46700 10 osd.70 pg_epoch: 71563 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.776175 7f28fc345700 10 osd.70 pg_epoch: 71554 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.776273 7f28fc345700 10 osd.70 pg_epoch: 71554 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.776286 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.776294 7f28fc345700 10 osd.70 pg_epoch: 71555 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.776301 7f28fc345700 10 osd.70 pg_epoch: 71555 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.776293 7f28fcb46700 10 osd.70 pg_epoch: 71563 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.776312 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.776309 7f28fc345700 10 osd.70 pg_epoch: 71555 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.776315 7f28fcb46700 10 osd.70 pg_epoch: 71564 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.776326 7f28fcb46700 10 osd.70 pg_epoch: 71564 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.776328 7f28fc345700 10 osd.70 pg_epoch: 71555 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.776339 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.776341 7f28fc345700 10 osd.70 pg_epoch: 71556 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.776338 7f28fcb46700 10 osd.70 pg_epoch: 71564 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.776348 7f28fc345700 10 osd.70 pg_epoch: 71556 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.776355 7f28fc345700 10 osd.70 pg_epoch: 71556 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.776361 7f28fcb46700 10 osd.70 pg_epoch: 71564 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.776368 7f28fc345700 10 osd.70 pg_epoch: 71556 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.776378 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.776377 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.776380 7f28fc345700 10 osd.70 pg_epoch: 71557 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.776381 7f28fcb46700 10 osd.70 pg_epoch: 71565 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.776387 7f28fc345700 10 osd.70 pg_epoch: 71557 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.776394 7f28fc345700 10 osd.70 pg_epoch: 71557 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.776393 7f28fcb46700 10 osd.70 pg_epoch: 71565 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.776407 7f28fc345700 10 osd.70 pg_epoch: 71557 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.776405 7f28fcb46700 10 osd.70 pg_epoch: 71565 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.776416 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.776418 7f28fc345700 10 osd.70 pg_epoch: 71558 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.776424 7f28fc345700 10 osd.70 pg_epoch: 71558 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.776431 7f28fc345700 10 osd.70 pg_epoch: 71558 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.776443 7f28fc345700 10 osd.70 pg_epoch: 71558 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.776452 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.776453 7f28fc345700 10 osd.70 pg_epoch: 71559 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.776460 7f28fc345700 10 osd.70 pg_epoch: 71559 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.776467 7f28fc345700 10 osd.70 pg_epoch: 71559 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.776427 7f28fcb46700 10 osd.70 pg_epoch: 71565 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.776479 7f28fc345700 10 osd.70 pg_epoch: 71559 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.776488 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.776488 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.776490 7f28fc345700 10 osd.70 pg_epoch: 71560 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.776491 7f28fcb46700 10 osd.70 pg_epoch: 71566 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.776497 7f28fc345700 10 osd.70 pg_epoch: 71560 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.776499 7f28fcb46700 10 osd.70 pg_epoch: 71566 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.776505 7f28fc345700 10 osd.70 pg_epoch: 71560 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.776507 7f28fcb46700 10 osd.70 pg_epoch: 71566 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.776518 7f28fc345700 10 osd.70 pg_epoch: 71560 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.776526 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.776523 7f28fcb46700 10 osd.70 pg_epoch: 71566 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.776532 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.776528 7f28fc345700 10 osd.70 pg_epoch: 71561 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.776534 7f28fcb46700 10 osd.70 pg_epoch: 71567 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.776535 7f28fc345700 10 osd.70 pg_epoch: 71561 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.776541 7f28fcb46700 10 osd.70 pg_epoch: 71567 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.776543 7f28fc345700 10 osd.70 pg_epoch: 71561 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.776548 7f28fcb46700 10 osd.70 pg_epoch: 71567 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.776556 7f28fc345700 10 osd.70 pg_epoch: 71561 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.776564 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.776561 7f28fcb46700 10 osd.70 pg_epoch: 71567 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.776566 7f28fc345700 10 osd.70 pg_epoch: 71562 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.776571 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.776574 7f28fc345700 10 osd.70 pg_epoch: 71562 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.776576 7f28fcb46700 10 osd.70 pg_epoch: 71568 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.776581 7f28fc345700 10 osd.70 pg_epoch: 71562 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.776584 7f28fcb46700 10 osd.70 pg_epoch: 71568 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.776591 7f28fcb46700 10 osd.70 pg_epoch: 71568 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.776594 7f28fc345700 10 osd.70 pg_epoch: 71562 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.776603 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.776605 7f28fc345700 10 osd.70 pg_epoch: 71563 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.776605 7f28fcb46700 10 osd.70 pg_epoch: 71568 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.776614 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.776612 7f28fc345700 10 osd.70 pg_epoch: 71563 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.776616 7f28fcb46700 10 osd.70 pg_epoch: 71569 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.776619 7f28fc345700 10 osd.70 pg_epoch: 71563 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.776623 7f28fcb46700 10 osd.70 pg_epoch: 71569 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.776630 7f28fcb46700 10 osd.70 pg_epoch: 71569 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.776631 7f28fc345700 10 osd.70 pg_epoch: 71563 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.776640 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.776642 7f28fc345700 10 osd.70 pg_epoch: 71564 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.776643 7f28fcb46700 10 osd.70 pg_epoch: 71569 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.776653 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.776648 7f28fc345700 10 osd.70 pg_epoch: 71564 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.776655 7f28fcb46700 10 osd.70 pg_epoch: 71570 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.776656 7f28fc345700 10 osd.70 pg_epoch: 71564 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.776661 7f28fcb46700 10 osd.70 pg_epoch: 71570 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.776667 7f28fcb46700 10 osd.70 pg_epoch: 71570 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.776669 7f28fc345700 10 osd.70 pg_epoch: 71564 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.776678 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.776680 7f28fc345700 10 osd.70 pg_epoch: 71565 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.776680 7f28fcb46700 10 osd.70 pg_epoch: 71570 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.776689 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.776687 7f28fc345700 10 osd.70 pg_epoch: 71565 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.776691 7f28fcb46700 10 osd.70 pg_epoch: 71571 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.776695 7f28fc345700 10 osd.70 pg_epoch: 71565 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.776699 7f28fcb46700 10 osd.70 pg_epoch: 71571 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.776706 7f28fcb46700 10 osd.70 pg_epoch: 71571 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.776708 7f28fc345700 10 osd.70 pg_epoch: 71565 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.776717 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.776719 7f28fc345700 10 osd.70 pg_epoch: 71566 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.776718 7f28fcb46700 10 osd.70 pg_epoch: 71571 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.776728 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.776726 7f28fc345700 10 osd.70 pg_epoch: 71566 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.776729 7f28fcb46700 10 osd.70 pg_epoch: 71572 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.776734 7f28fc345700 10 osd.70 pg_epoch: 71566 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.776737 7f28fcb46700 10 osd.70 pg_epoch: 71572 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.776745 7f28fcb46700 10 osd.70 pg_epoch: 71572 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.776748 7f28fc345700 10 osd.70 pg_epoch: 71566 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.776757 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.776758 7f28fc345700 10 osd.70 pg_epoch: 71567 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.776758 7f28fcb46700 10 osd.70 pg_epoch: 71572 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.776768 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.776766 7f28fc345700 10 osd.70 pg_epoch: 71567 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.776770 7f28fcb46700 10 osd.70 pg_epoch: 71573 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.776773 7f28fc345700 10 osd.70 pg_epoch: 71567 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.776777 7f28fcb46700 10 osd.70 pg_epoch: 71573 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.776784 7f28fcb46700 10 osd.70 pg_epoch: 71573 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.776786 7f28fc345700 10 osd.70 pg_epoch: 71567 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.776795 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.776797 7f28fc345700 10 osd.70 pg_epoch: 71568 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.776797 7f28fcb46700 10 osd.70 pg_epoch: 71573 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.776806 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.776804 7f28fc345700 10 osd.70 pg_epoch: 71568 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.776808 7f28fcb46700 10 osd.70 pg_epoch: 71574 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.776811 7f28fc345700 10 osd.70 pg_epoch: 71568 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.776816 7f28fcb46700 10 osd.70 pg_epoch: 71574 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.776822 7f28fcb46700 10 osd.70 pg_epoch: 71574 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.776825 7f28fc345700 10 osd.70 pg_epoch: 71568 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.776833 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.776835 7f28fcb46700 10 osd.70 pg_epoch: 71574 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.776844 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.776834 7f28fc345700 10 osd.70 pg_epoch: 71569 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.776847 7f28fcb46700 10 osd.70 pg_epoch: 71575 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.776853 7f28fc345700 10 osd.70 pg_epoch: 71569 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.776858 7f28fcb46700 10 osd.70 pg_epoch: 71575 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.776861 7f28fc345700 10 osd.70 pg_epoch: 71569 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.776866 7f28fcb46700 10 osd.70 pg_epoch: 71575 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.776873 7f28fc345700 10 osd.70 pg_epoch: 71569 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.776881 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.776880 7f28fcb46700 10 osd.70 pg_epoch: 71575 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.776883 7f28fc345700 10 osd.70 pg_epoch: 71570 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.776889 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.776890 7f28fc345700 10 osd.70 pg_epoch: 71570 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.776891 7f28fcb46700 10 osd.70 pg_epoch: 71576 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.776897 7f28fc345700 10 osd.70 pg_epoch: 71570 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.776898 7f28fcb46700 10 osd.70 pg_epoch: 71576 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.776905 7f28fcb46700 10 osd.70 pg_epoch: 71576 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.776909 7f28fc345700 10 osd.70 pg_epoch: 71570 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.776918 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.776920 7f28fc345700 10 osd.70 pg_epoch: 71571 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.776919 7f28fcb46700 10 osd.70 pg_epoch: 71576 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.776931 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.776930 7f28fc345700 10 osd.70 pg_epoch: 71571 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.776934 7f28fcb46700 10 osd.70 pg_epoch: 71577 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.776937 7f28fc345700 10 osd.70 pg_epoch: 71571 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.776941 7f28fcb46700 10 osd.70 pg_epoch: 71577 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.776948 7f28fcb46700 10 osd.70 pg_epoch: 71577 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.776950 7f28fc345700 10 osd.70 pg_epoch: 71571 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.776959 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.776961 7f28fc345700 10 osd.70 pg_epoch: 71572 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.776961 7f28fcb46700 10 osd.70 pg_epoch: 71577 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.776971 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.776968 7f28fc345700 10 osd.70 pg_epoch: 71572 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.776973 7f28fcb46700 10 osd.70 pg_epoch: 71578 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.776976 7f28fc345700 10 osd.70 pg_epoch: 71572 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.776980 7f28fcb46700 10 osd.70 pg_epoch: 71578 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.776987 7f28fcb46700 10 osd.70 pg_epoch: 71578 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.776989 7f28fc345700 10 osd.70 pg_epoch: 71572 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.776997 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.776999 7f28fc345700 10 osd.70 pg_epoch: 71573 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.776999 7f28fcb46700 10 osd.70 pg_epoch: 71578 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.777009 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.777006 7f28fc345700 10 osd.70 pg_epoch: 71573 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.777010 7f28fcb46700 10 osd.70 pg_epoch: 71579 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.777014 7f28fc345700 10 osd.70 pg_epoch: 71573 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.777018 7f28fcb46700 10 osd.70 pg_epoch: 71579 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.777025 7f28fcb46700 10 osd.70 pg_epoch: 71579 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.777027 7f28fc345700 10 osd.70 pg_epoch: 71573 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.777035 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.777037 7f28fc345700 10 osd.70 pg_epoch: 71574 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.777038 7f28fcb46700 10 osd.70 pg_epoch: 71579 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.777046 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.777044 7f28fc345700 10 osd.70 pg_epoch: 71574 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.777048 7f28fcb46700 10 osd.70 pg_epoch: 71580 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.777051 7f28fc345700 10 osd.70 pg_epoch: 71574 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.777055 7f28fcb46700 10 osd.70 pg_epoch: 71580 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.777062 7f28fcb46700 10 osd.70 pg_epoch: 71580 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.777063 7f28fc345700 10 osd.70 pg_epoch: 71574 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.777071 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.777073 7f28fc345700 10 osd.70 pg_epoch: 71575 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.777074 7f28fcb46700 10 osd.70 pg_epoch: 71580 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:13:57.777083 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.777080 7f28fc345700 10 osd.70 pg_epoch: 71575 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.777084 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.777087 7f28fc345700 10 osd.70 pg_epoch: 71575 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.777091 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.777098 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.777099 7f28fc345700 10 osd.70 pg_epoch: 71575 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.777108 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.777110 7f28fc345700 10 osd.70 pg_epoch: 71576 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.777112 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_activate_map 2014-12-08 18:13:57.777117 7f28fc345700 10 osd.70 pg_epoch: 71576 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.777121 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] update_heartbeat_peers -> 13,16,70 2014-12-08 18:13:57.777125 7f28fc345700 10 osd.70 pg_epoch: 71576 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.777130 7f28fcb46700 20 osd.70 71581 need_heartbeat_peer_update 2014-12-08 18:13:57.777132 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] take_waiters 2014-12-08 18:13:57.777137 7f28fc345700 10 osd.70 pg_epoch: 71576 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.777140 7f28fcb46700 5 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] exit Reset 4.682363 126 0.008794 2014-12-08 18:13:57.777146 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.777148 7f28fc345700 10 osd.70 pg_epoch: 71577 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.777150 7f28fcb46700 5 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] enter Started 2014-12-08 18:13:57.777155 7f28fc345700 10 osd.70 pg_epoch: 71577 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.777157 7f28fcb46700 5 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] enter Start 2014-12-08 18:13:57.777162 7f28fc345700 10 osd.70 pg_epoch: 71577 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.777165 7f28fcb46700 1 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: transitioning to Primary 2014-12-08 18:13:57.777174 7f28fc345700 10 osd.70 pg_epoch: 71577 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.777174 7f28fcb46700 5 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] exit Start 0.000017 0 0.000000 2014-12-08 18:13:57.777186 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.777186 7f28fcb46700 5 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] enter Started/Primary 2014-12-08 18:13:57.777187 7f28fc345700 10 osd.70 pg_epoch: 71578 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.777194 7f28fcb46700 5 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] enter Started/Primary/Peering 2014-12-08 18:13:57.777195 7f28fc345700 10 osd.70 pg_epoch: 71578 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.777203 7f28fc345700 10 osd.70 pg_epoch: 71578 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.777203 7f28fcb46700 5 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] enter Started/Primary/Peering/GetInfo 2014-12-08 18:13:57.777211 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:13:57.777214 7f28fc345700 10 osd.70 pg_epoch: 71578 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.777224 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.777226 7f28fc345700 10 osd.70 pg_epoch: 71579 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.777219 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(71444-71455 up [70,16,56](70) acting [70,16,56](70) maybe_went_rw) 2014-12-08 18:13:57.777233 7f28fc345700 10 osd.70 pg_epoch: 71579 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.777236 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(71441-71443 up [70,16,13](70) acting [70,16,13](70) maybe_went_rw) 2014-12-08 18:13:57.777240 7f28fc345700 10 osd.70 pg_epoch: 71579 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.777244 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(71432-71440 up [70,16,56](70) acting [70,16,56](70) maybe_went_rw) 2014-12-08 18:13:57.777252 7f28fc345700 10 osd.70 pg_epoch: 71579 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.777253 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(71429-71431 up [76,16,56](76) acting [76,16,56](76) maybe_went_rw) 2014-12-08 18:13:57.777260 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.777262 7f28fc345700 10 osd.70 pg_epoch: 71580 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.777261 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(71427-71428 up [16,56](16) acting [16,56](16) maybe_went_rw) 2014-12-08 18:13:57.777269 7f28fc345700 10 osd.70 pg_epoch: 71580 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.777270 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(71406-71426 up [70,16,56](70) acting [70,16,56](70) maybe_went_rw) 2014-12-08 18:13:57.777277 7f28fc345700 10 osd.70 pg_epoch: 71580 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.777278 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(71401-71405 up [76,16,56](76) acting [76,16,56](76) maybe_went_rw) 2014-12-08 18:13:57.777286 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(71399-71400 up [16,56](16) acting [16,56](16) maybe_went_rw) 2014-12-08 18:13:57.777288 7f28fc345700 10 osd.70 pg_epoch: 71580 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_advance_map [70,6,21]/[70,6,21] -- 70/70 2014-12-08 18:13:57.777297 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.777294 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(71392-71398 up [70,16,56](70) acting [70,16,56](70) maybe_went_rw) 2014-12-08 18:13:57.777299 7f28fc345700 10 osd.70 pg_epoch: 71581 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:13:57.777302 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(71384-71391 up [16,56](16) acting [16,56](16) maybe_went_rw) 2014-12-08 18:13:57.777306 7f28fc345700 10 osd.70 pg_epoch: 71581 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.777311 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(71380-71383 up [76,16,56](76) acting [76,16,56](76) maybe_went_rw) 2014-12-08 18:13:57.777314 7f28fc345700 10 osd.70 pg_epoch: 71581 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.777319 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(71378-71379 up [16,56](16) acting [16,56](16) maybe_went_rw) 2014-12-08 18:13:57.777328 7f28fc345700 10 osd.70 pg_epoch: 71581 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] handle_activate_map 2014-12-08 18:13:57.777326 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(71280-71377 up [70,16,56](70) acting [70,16,56](70) maybe_went_rw) 2014-12-08 18:13:57.777335 7f28fc345700 10 osd.70 pg_epoch: 71581 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] update_heartbeat_peers -> 6,21,70 2014-12-08 18:13:57.777343 7f28fc345700 20 osd.70 71581 need_heartbeat_peer_update 2014-12-08 18:13:57.777337 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(71183-71279 up [70,16,63](70) acting [70,16,63](70) maybe_went_rw) 2014-12-08 18:13:57.777344 7f28fc345700 10 osd.70 pg_epoch: 71581 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] take_waiters 2014-12-08 18:13:57.777348 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(71181-71182 up [70,16](70) acting [70,16](70) maybe_went_rw) 2014-12-08 18:13:57.777352 7f28fc345700 5 osd.70 pg_epoch: 71581 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] exit Reset 4.171613 132 0.003274 2014-12-08 18:13:57.777357 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(71150-71180 up [70,16,56](70) acting [70,16,56](70) maybe_went_rw) 2014-12-08 18:13:57.777361 7f28fc345700 5 osd.70 pg_epoch: 71581 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] enter Started 2014-12-08 18:13:57.777366 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(71136-71149 up [76,16,56](76) acting [76,16,56](76) maybe_went_rw) 2014-12-08 18:13:57.777369 7f28fc345700 5 osd.70 pg_epoch: 71581 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] enter Start 2014-12-08 18:13:57.777376 7f28fc345700 1 osd.70 pg_epoch: 71581 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] state: transitioning to Primary 2014-12-08 18:13:57.777375 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(71134-71135 up [16,56](16) acting [16,56](16) maybe_went_rw) 2014-12-08 18:13:57.777384 7f28fc345700 5 osd.70 pg_epoch: 71581 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] exit Start 0.000015 0 0.000000 2014-12-08 18:13:57.777385 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(71129-71133 up [70,16,56](70) acting [70,16,56](70) maybe_went_rw) 2014-12-08 18:13:57.777392 7f28fc345700 5 osd.70 pg_epoch: 71581 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] enter Started/Primary 2014-12-08 18:13:57.777393 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(71108-71128 up [70,16,56](70) acting [70,16,56](70) maybe_went_rw) 2014-12-08 18:13:57.777399 7f28fc345700 5 osd.70 pg_epoch: 71581 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 inactive] enter Started/Primary/Peering 2014-12-08 18:13:57.777401 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(71107-71107 up [16,56](16) acting [16,56](16)) 2014-12-08 18:13:57.777407 7f28fc345700 5 osd.70 pg_epoch: 71581 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 peering] enter Started/Primary/Peering/GetInfo 2014-12-08 18:13:57.777409 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(71101-71106 up [76,16,56](76) acting [76,16,56](76) maybe_went_rw) 2014-12-08 18:13:57.777415 7f28fc345700 10 osd.70 pg_epoch: 71581 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 peering] _calc_past_interval_range start epoch 71450 >= end epoch 71444, nothing to do 2014-12-08 18:13:57.777417 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(71099-71100 up [16,56](16) acting [16,56](16) maybe_went_rw) 2014-12-08 18:13:57.777424 7f28fc345700 10 osd.70 pg_epoch: 71581 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior final: probe 6,21,70 down blocked_by {} 2014-12-08 18:13:57.777425 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(71091-71098 up [70,16,56](70) acting [70,16,56](70) maybe_went_rw) 2014-12-08 18:13:57.777432 7f28fc345700 10 osd.70 pg_epoch: 71581 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 peering] up_thru 71491 >= same_since 71444, all is well 2014-12-08 18:13:57.777433 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(71059-71090 up [70,16,56](70) acting [70,16,56](70) maybe_went_rw) 2014-12-08 18:13:57.777441 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(71038-71058 up [70,16](70) acting [70,16](70) maybe_went_rw) 2014-12-08 18:13:57.777442 7f28fc345700 10 osd.70 pg_epoch: 71581 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 peering] state: querying info from osd.6 2014-12-08 18:13:57.777449 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(71021-71037 up [70,16,13](70) acting [70,16,13](70) maybe_went_rw) 2014-12-08 18:13:57.777452 7f28fc345700 10 osd.70 pg_epoch: 71581 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 peering] state: querying info from osd.21 2014-12-08 18:13:57.777457 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(71008-71020 up [70,16](70) acting [70,16](70) maybe_went_rw) 2014-12-08 18:13:57.777462 7f28fc345700 15 osd.70 pg_epoch: 71581 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 peering] publish_stats_to_osd 71581:675 2014-12-08 18:13:57.777465 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(70997-71007 up [70,16,13](70) acting [70,16,13](70) maybe_went_rw) 2014-12-08 18:13:57.777471 7f28fc345700 20 osd.70 pg_epoch: 71581 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 peering] handle_activate_map: Not dirtying info: last_persisted is 71450 while current is 71581 2014-12-08 18:13:57.777473 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(70993-70996 up [70,16](70) acting [70,16](70) maybe_went_rw) 2014-12-08 18:13:57.777479 7f28fc345700 10 osd.70 pg_epoch: 71581 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 peering] handle_peering_event: epoch_sent: 71581 epoch_requested: 71581 NullEvt 2014-12-08 18:13:57.777480 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(70991-70992 up [70,16,11](70) acting [70,16,11](70) maybe_went_rw) 2014-12-08 18:13:57.777490 7f28fc345700 10 write_log with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, dirty_divergent_priors: 0, writeout_from: 4294967295'18446744073709551615, trimmed: 2014-12-08 18:13:57.777508 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(70977-70990 up [70,16,11](70) acting [70,16,11](70) maybe_went_rw) 2014-12-08 18:13:57.777519 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(70970-70976 up [16,11](16) acting [16,11](16) maybe_went_rw) 2014-12-08 18:13:57.777529 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(70967-70969 up [76,16,11](76) acting [76,16,11](76) maybe_went_rw) 2014-12-08 18:13:57.777530 7f28fc345700 10 osd.70 pg_epoch: 71581 pg[218.4b( v 71581'2 (0'0,71581'2] local-les=71448 n=2 ec=71423 les/c 71448/71453 71444/71444/71425) [18,17,70] r=2 lpr=71581 pi=71423-71443/6 crt=0'0 lcod 0'0 inactive NOTIFY] handle_peering_event: epoch_sent: 71581 epoch_requested: 71581 NullEvt 2014-12-08 18:13:57.777537 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(70965-70966 up [16,11](16) acting [16,11](16) maybe_went_rw) 2014-12-08 18:13:57.777543 7f28fc345700 10 write_log with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, dirty_divergent_priors: 0, writeout_from: 4294967295'18446744073709551615, trimmed: 2014-12-08 18:13:57.777554 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(69763-70964 up [70,16,11](70) acting [70,16,11](70) maybe_went_rw) 2014-12-08 18:13:57.777562 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(69680-69762 up [](-1) acting [](-1)) 2014-12-08 18:13:57.777569 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(69677-69679 up [78](78) acting [78](78)) 2014-12-08 18:13:57.777576 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(69674-69676 up [77](77) acting [77](77)) 2014-12-08 18:13:57.777583 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(69651-69673 up [70](70) acting [70](70) maybe_went_rw) 2014-12-08 18:13:57.777578 7f28fc345700 10 osd.70 pg_epoch: 71434 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71432/71433/71433) [68,18,70] r=2 lpr=71434 pi=71183-71432/17 crt=71377'4110 lcod 0'0 inactive NOTIFY] handle_advance_map [68,18,70]/[68,18,70] -- 68/68 2014-12-08 18:13:57.777592 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.777591 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(69645-69650 up [70,11](70) acting [70,11](70) maybe_went_rw) 2014-12-08 18:13:57.777595 7f28fc345700 10 osd.70 pg_epoch: 71435 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71432/71433/71433) [68,18,70] r=2 lpr=71434 pi=71183-71432/17 crt=71377'4110 lcod 0'0 inactive NOTIFY] state: Reset advmap 2014-12-08 18:13:57.777598 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(69395-69644 up [70,16,11](70) acting [70,16,11](70) maybe_went_rw) 2014-12-08 18:13:57.777603 7f28fc345700 10 osd.70 pg_epoch: 71435 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71432/71433/71433) [68,18,70] r=2 lpr=71434 pi=71183-71432/17 crt=71377'4110 lcod 0'0 inactive NOTIFY] _calc_past_interval_range: already have past intervals back to 71434 2014-12-08 18:13:57.777606 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(66323-69394 up [70,20,11](70) acting [70,20,11](70) maybe_went_rw) 2014-12-08 18:13:57.777612 7f28fc345700 10 osd.70 pg_epoch: 71435 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71432/71433/71433) [68,18,70] r=2 lpr=71434 pi=71183-71432/17 crt=71377'4110 lcod 0'0 inactive NOTIFY] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.777614 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(66286-66322 up [70,11](70) acting [70,11](70) maybe_went_rw) 2014-12-08 18:13:57.777621 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(64997-66285 up [70,16,11](70) acting [70,16,11](70) maybe_went_rw) 2014-12-08 18:13:57.777627 7f28fc345700 10 osd.70 pg_epoch: 71435 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71432/71433/71433) [68,18,70] r=2 lpr=71434 pi=71183-71432/17 crt=71377'4110 lcod 0'0 inactive NOTIFY] handle_advance_map [68,18,70]/[68,18,70] -- 68/68 2014-12-08 18:13:57.777629 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(64292-64996 up [](-1) acting [](-1)) 2014-12-08 18:13:57.777636 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.777638 7f28fc345700 10 osd.70 pg_epoch: 71436 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71432/71433/71433) [68,18,70] r=2 lpr=71434 pi=71183-71432/17 crt=71377'4110 lcod 0'0 inactive NOTIFY] state: Reset advmap 2014-12-08 18:13:57.777635 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(64288-64291 up [70,58,63](70) acting [70,58,63](70)) 2014-12-08 18:13:57.777644 7f28fc345700 10 osd.70 pg_epoch: 71436 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71432/71433/71433) [68,18,70] r=2 lpr=71434 pi=71183-71432/17 crt=71377'4110 lcod 0'0 inactive NOTIFY] _calc_past_interval_range: already have past intervals back to 71434 2014-12-08 18:13:57.777650 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(64287-64287 up [70,58](70) acting [70,58](70)) 2014-12-08 18:13:57.777652 7f28fc345700 10 osd.70 pg_epoch: 71436 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71432/71433/71433) [68,18,70] r=2 lpr=71434 pi=71183-71432/17 crt=71377'4110 lcod 0'0 inactive NOTIFY] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.777657 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(64273-64286 up [70](70) acting [70](70) maybe_went_rw) 2014-12-08 18:13:57.777664 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(64268-64272 up [70,11](70) acting [70,11](70) maybe_went_rw) 2014-12-08 18:13:57.777667 7f28fc345700 10 osd.70 pg_epoch: 71436 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71432/71433/71433) [68,18,70] r=2 lpr=71434 pi=71183-71432/17 crt=71377'4110 lcod 0'0 inactive NOTIFY] handle_advance_map [68,18,70]/[68,18,70] -- 68/68 2014-12-08 18:13:57.777677 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.777671 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(58298-64267 up [70,20,11](70) acting [70,20,11](70) maybe_went_rw) 2014-12-08 18:13:57.777678 7f28fc345700 10 osd.70 pg_epoch: 71437 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71432/71433/71433) [68,18,70] r=2 lpr=71434 pi=71183-71432/17 crt=71377'4110 lcod 0'0 inactive NOTIFY] state: Reset advmap 2014-12-08 18:13:57.777679 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(58233-58297 up [70,11](70) acting [70,11](70) maybe_went_rw) 2014-12-08 18:13:57.777685 7f28fc345700 10 osd.70 pg_epoch: 71437 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71432/71433/71433) [68,18,70] r=2 lpr=71434 pi=71183-71432/17 crt=71377'4110 lcod 0'0 inactive NOTIFY] _calc_past_interval_range: already have past intervals back to 71434 2014-12-08 18:13:57.777687 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(57990-58232 up [70,16,11](70) acting [70,16,11](70) maybe_went_rw) 2014-12-08 18:13:57.777693 7f28fc345700 10 osd.70 pg_epoch: 71437 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71432/71433/71433) [68,18,70] r=2 lpr=71434 pi=71183-71432/17 crt=71377'4110 lcod 0'0 inactive NOTIFY] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.777695 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(57835-57989 up [70,16](70) acting [70,16](70) maybe_went_rw) 2014-12-08 18:13:57.777702 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(57650-57834 up [](-1) acting [](-1)) 2014-12-08 18:13:57.777706 7f28fc345700 10 osd.70 pg_epoch: 71437 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71432/71433/71433) [68,18,70] r=2 lpr=71434 pi=71183-71432/17 crt=71377'4110 lcod 0'0 inactive NOTIFY] handle_advance_map [68,18,70]/[68,18,70] -- 68/68 2014-12-08 18:13:57.777715 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.777710 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(57447-57649 up [4,16](4) acting [4,16](4) maybe_went_rw) 2014-12-08 18:13:57.777717 7f28fc345700 10 osd.70 pg_epoch: 71438 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71432/71433/71433) [68,18,70] r=2 lpr=71434 pi=71183-71432/17 crt=71377'4110 lcod 0'0 inactive NOTIFY] state: Reset advmap 2014-12-08 18:13:57.777718 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(57438-57446 up [16](16) acting [16](16) maybe_went_rw) 2014-12-08 18:13:57.777723 7f28fc345700 10 osd.70 pg_epoch: 71438 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71432/71433/71433) [68,18,70] r=2 lpr=71434 pi=71183-71432/17 crt=71377'4110 lcod 0'0 inactive NOTIFY] _calc_past_interval_range: already have past intervals back to 71434 2014-12-08 18:13:57.777731 7f28fc345700 10 osd.70 pg_epoch: 71438 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71432/71433/71433) [68,18,70] r=2 lpr=71434 pi=71183-71432/17 crt=71377'4110 lcod 0'0 inactive NOTIFY] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.777729 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(53820-57437 up [70,16](70) acting [70,16](70) maybe_went_rw) 2014-12-08 18:13:57.777738 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(53718-53819 up [4,58](4) acting [4,58](4) maybe_went_rw) 2014-12-08 18:13:57.777743 7f28fc345700 10 osd.70 pg_epoch: 71438 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71432/71433/71433) [68,18,70] r=2 lpr=71434 pi=71183-71432/17 crt=71377'4110 lcod 0'0 inactive NOTIFY] handle_advance_map [68,18,70]/[68,18,70] -- 68/68 2014-12-08 18:13:57.777746 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(53708-53717 up [58](58) acting [58](58) maybe_went_rw) 2014-12-08 18:13:57.777752 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.777753 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(53692-53707 up [4,58](4) acting [4,58](4) maybe_went_rw) 2014-12-08 18:13:57.777754 7f28fc345700 10 osd.70 pg_epoch: 71439 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71432/71433/71433) [68,18,70] r=2 lpr=71434 pi=71183-71432/17 crt=71377'4110 lcod 0'0 inactive NOTIFY] state: Reset advmap 2014-12-08 18:13:57.777766 7f28fc345700 10 osd.70 pg_epoch: 71439 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71432/71433/71433) [68,18,70] r=2 lpr=71434 pi=71183-71432/17 crt=71377'4110 lcod 0'0 inactive NOTIFY] _calc_past_interval_range: already have past intervals back to 71434 2014-12-08 18:13:57.777764 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(53691-53691 up [58](58) acting [58](58)) 2014-12-08 18:13:57.777772 7f28fc345700 10 osd.70 pg_epoch: 71439 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71432/71433/71433) [68,18,70] r=2 lpr=71434 pi=71183-71432/17 crt=71377'4110 lcod 0'0 inactive NOTIFY] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.777777 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(53689-53690 up [4,58](4) acting [4,58](4)) 2014-12-08 18:13:57.777786 7f28fc345700 10 osd.70 pg_epoch: 71439 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71432/71433/71433) [68,18,70] r=2 lpr=71434 pi=71183-71432/17 crt=71377'4110 lcod 0'0 inactive NOTIFY] handle_advance_map [68,18,70]/[68,18,70] -- 68/68 2014-12-08 18:13:57.777796 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.777789 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(53670-53688 up [4](4) acting [4](4) maybe_went_rw) 2014-12-08 18:13:57.777797 7f28fc345700 10 osd.70 pg_epoch: 71440 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71432/71433/71433) [68,18,70] r=2 lpr=71434 pi=71183-71432/17 crt=71377'4110 lcod 0'0 inactive NOTIFY] state: Reset advmap 2014-12-08 18:13:57.777804 7f28fc345700 10 osd.70 pg_epoch: 71440 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71432/71433/71433) [68,18,70] r=2 lpr=71434 pi=71183-71432/17 crt=71377'4110 lcod 0'0 inactive NOTIFY] _calc_past_interval_range: already have past intervals back to 71434 2014-12-08 18:13:57.777803 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(53096-53669 up [4,58](4) acting [4,58](4) maybe_went_rw) 2014-12-08 18:13:57.777811 7f28fc345700 10 osd.70 pg_epoch: 71440 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71432/71433/71433) [68,18,70] r=2 lpr=71434 pi=71183-71432/17 crt=71377'4110 lcod 0'0 inactive NOTIFY] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.777817 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(52968-53095 up [4,30](4) acting [4,30](4) maybe_went_rw) 2014-12-08 18:13:57.777826 7f28fc345700 10 osd.70 pg_epoch: 71440 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71432/71433/71433) [68,18,70] r=2 lpr=71434 pi=71183-71432/17 crt=71377'4110 lcod 0'0 inactive NOTIFY] handle_advance_map [68,18,70]/[68,18,70] -- 68/68 2014-12-08 18:13:57.777835 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.777830 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(52952-52967 up [30](30) acting [30](30) maybe_went_rw) 2014-12-08 18:13:57.777837 7f28fc345700 10 osd.70 pg_epoch: 71441 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71432/71433/71433) [68,18,70] r=2 lpr=71434 pi=71183-71432/17 crt=71377'4110 lcod 0'0 inactive NOTIFY] state: Reset advmap 2014-12-08 18:13:57.777844 7f28fc345700 10 osd.70 pg_epoch: 71441 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71432/71433/71433) [68,18,70] r=2 lpr=71434 pi=71183-71432/17 crt=71377'4110 lcod 0'0 inactive NOTIFY] _calc_past_interval_range: already have past intervals back to 71434 2014-12-08 18:13:57.777844 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(52944-52951 up [4,30](4) acting [4,30](4) maybe_went_rw) 2014-12-08 18:13:57.777851 7f28fc345700 10 osd.70 pg_epoch: 71441 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71432/71433/71433) [68,18,70] r=2 lpr=71434 pi=71183-71432/17 crt=71377'4110 lcod 0'0 inactive NOTIFY] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.777854 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(52924-52943 up [30](30) acting [30](30) maybe_went_rw) 2014-12-08 18:13:57.777861 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(52826-52923 up [4,30](4) acting [4,30](4) maybe_went_rw) 2014-12-08 18:13:57.777868 7f28fc345700 10 osd.70 pg_epoch: 71441 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71432/71433/71433) [68,18,70] r=2 lpr=71434 pi=71183-71432/17 crt=71377'4110 lcod 0'0 inactive NOTIFY] handle_advance_map [68,18,70]/[68,18,70] -- 68/68 2014-12-08 18:13:57.777869 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(52819-52825 up [30](30) acting [30](30) maybe_went_rw) 2014-12-08 18:13:57.777877 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.777876 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(52749-52818 up [4,30](4) acting [4,30](4) maybe_went_rw) 2014-12-08 18:13:57.777879 7f28fc345700 10 osd.70 pg_epoch: 71442 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71432/71433/71433) [68,18,70] r=2 lpr=71434 pi=71183-71432/17 crt=71377'4110 lcod 0'0 inactive NOTIFY] state: Reset advmap 2014-12-08 18:13:57.777884 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(52685-52748 up [4](4) acting [4](4) maybe_went_rw) 2014-12-08 18:13:57.777886 7f28fc345700 10 osd.70 pg_epoch: 71442 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71432/71433/71433) [68,18,70] r=2 lpr=71434 pi=71183-71432/17 crt=71377'4110 lcod 0'0 inactive NOTIFY] _calc_past_interval_range: already have past intervals back to 71434 2014-12-08 18:13:57.777892 7f28fc345700 10 osd.70 pg_epoch: 71442 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71432/71433/71433) [68,18,70] r=2 lpr=71434 pi=71183-71432/17 crt=71377'4110 lcod 0'0 inactive NOTIFY] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.777892 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(52582-52684 up [4,30](4) acting [4,30](4) maybe_went_rw) 2014-12-08 18:13:57.777900 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(52578-52581 up [30](30) acting [30](30) maybe_went_rw) 2014-12-08 18:13:57.777907 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(52568-52577 up [4,30](4) acting [4,30](4) maybe_went_rw) 2014-12-08 18:13:57.777908 7f28fc345700 10 osd.70 pg_epoch: 71442 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71432/71433/71433) [68,18,70] r=2 lpr=71434 pi=71183-71432/17 crt=71377'4110 lcod 0'0 inactive NOTIFY] handle_advance_map [68,18,70]/[68,18,70] -- 68/68 2014-12-08 18:13:57.777918 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.777914 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(52561-52567 up [30](30) acting [30](30) maybe_went_rw) 2014-12-08 18:13:57.777919 7f28fc345700 10 osd.70 pg_epoch: 71443 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71432/71433/71433) [68,18,70] r=2 lpr=71434 pi=71183-71432/17 crt=71377'4110 lcod 0'0 inactive NOTIFY] state: Reset advmap 2014-12-08 18:13:57.777921 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(52550-52560 up [4,30](4) acting [4,30](4) maybe_went_rw) 2014-12-08 18:13:57.777930 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(52549-52549 up [4](4) acting [4](4)) 2014-12-08 18:13:57.777928 7f28fc345700 10 osd.70 pg_epoch: 71443 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71432/71433/71433) [68,18,70] r=2 lpr=71434 pi=71183-71432/17 crt=71377'4110 lcod 0'0 inactive NOTIFY] _calc_past_interval_range: already have past intervals back to 71434 2014-12-08 18:13:57.777937 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(52546-52548 up [4,30](4) acting [4,30](4) maybe_went_rw) 2014-12-08 18:13:57.777940 7f28fc345700 10 osd.70 pg_epoch: 71443 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71432/71433/71433) [68,18,70] r=2 lpr=71434 pi=71183-71432/17 crt=71377'4110 lcod 0'0 inactive NOTIFY] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.777944 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(52532-52545 up [30](30) acting [30](30)) 2014-12-08 18:13:57.777952 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(52375-52531 up [4,30](4) acting [4,30](4) maybe_went_rw) 2014-12-08 18:13:57.777959 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(52372-52374 up [30,11](30) acting [30,11](30) maybe_went_rw) 2014-12-08 18:13:57.777966 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(52369-52371 up [30,31](30) acting [30,31](30) maybe_went_rw) 2014-12-08 18:13:57.777963 7f28fc345700 10 osd.70 pg_epoch: 71443 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71432/71433/71433) [68,18,70] r=2 lpr=71434 pi=71183-71432/17 crt=71377'4110 lcod 0'0 inactive NOTIFY] handle_advance_map [68,18,70]/[68,18,70] -- 68/68 2014-12-08 18:13:57.777974 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(52340-52368 up [31,59](31) acting [31,59](31) maybe_went_rw) 2014-12-08 18:13:57.777980 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.777981 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(52327-52339 up [11,31](11) acting [11,31](11) maybe_went_rw) 2014-12-08 18:13:57.777984 7f28fc345700 10 osd.70 pg_epoch: 71444 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71432/71433/71433) [68,18,70] r=2 lpr=71434 pi=71183-71432/17 crt=71377'4110 lcod 0'0 inactive NOTIFY] state: Reset advmap 2014-12-08 18:13:57.777989 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(52320-52326 up [30,11](30) acting [30,11](30) maybe_went_rw) 2014-12-08 18:13:57.777997 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(52319-52319 up [30](30) acting [30](30)) 2014-12-08 18:13:57.777996 7f28fc345700 10 osd.70 pg_epoch: 71444 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71432/71433/71433) [68,18,70] r=2 lpr=71434 pi=71183-71432/17 crt=71377'4110 lcod 0'0 inactive NOTIFY] _calc_past_interval_range: already have past intervals back to 71434 2014-12-08 18:13:57.778004 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(52306-52318 up [30,11](30) acting [30,11](30) maybe_went_rw) 2014-12-08 18:13:57.778012 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(52298-52305 up [11](11) acting [11](11) maybe_went_rw) 2014-12-08 18:13:57.778010 7f28fc345700 10 osd.70 pg_epoch: 71444 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71432/71433/71433) [68,18,70] r=2 lpr=71434 pi=71183-71432/17 crt=71377'4110 lcod 0'0 inactive NOTIFY] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.778019 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(52295-52297 up [30,11](30) acting [30,11](30) maybe_went_rw) 2014-12-08 18:13:57.778026 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(52292-52294 up [11](11) acting [11](11) maybe_went_rw) 2014-12-08 18:13:57.778033 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(52279-52291 up [30,11](30) acting [30,11](30) maybe_went_rw) 2014-12-08 18:13:57.778032 7f28fc345700 10 osd.70 pg_epoch: 71444 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71432/71433/71433) [68,18,70] r=2 lpr=71434 pi=71183-71432/17 crt=71377'4110 lcod 0'0 inactive NOTIFY] handle_advance_map [68,18,70]/[68,18,70] -- 68/68 2014-12-08 18:13:57.778041 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(52272-52278 up [11](11) acting [11](11) maybe_went_rw) 2014-12-08 18:13:57.778049 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.778049 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(52265-52271 up [30,11](30) acting [30,11](30) maybe_went_rw) 2014-12-08 18:13:57.778057 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(52157-52264 up [4,30](4) acting [4,30](4) maybe_went_rw) 2014-12-08 18:13:57.778053 7f28fc345700 10 osd.70 pg_epoch: 71445 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71432/71433/71433) [68,18,70] r=2 lpr=71434 pi=71183-71432/17 crt=71377'4110 lcod 0'0 inactive NOTIFY] state: Reset advmap 2014-12-08 18:13:57.778065 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(52143-52156 up [30](30) acting [30](30) maybe_went_rw) 2014-12-08 18:13:57.778065 7f28fc345700 10 osd.70 pg_epoch: 71445 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71432/71433/71433) [68,18,70] r=2 lpr=71434 pi=71183-71432/17 crt=71377'4110 lcod 0'0 inactive NOTIFY] _calc_past_interval_range: already have past intervals back to 71434 2014-12-08 18:13:57.778073 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(52061-52142 up [4,30](4) acting [4,30](4) maybe_went_rw) 2014-12-08 18:13:57.778081 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(52058-52060 up [4,11](4) acting [4,11](4) maybe_went_rw) 2014-12-08 18:13:57.778088 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(52033-52057 up [11,31](11) acting [11,31](11) maybe_went_rw) 2014-12-08 18:13:57.778095 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(52031-52032 up [31](31) acting [31](31) maybe_went_rw) 2014-12-08 18:13:57.778102 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(51949-52030 up [11,31](11) acting [11,31](11) maybe_went_rw) 2014-12-08 18:13:57.778078 7f28fc345700 10 osd.70 pg_epoch: 71445 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71432/71433/71433) [68,18,70] r=2 lpr=71434 pi=71183-71432/17 crt=71377'4110 lcod 0'0 inactive NOTIFY] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.778109 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(51938-51948 up [31](31) acting [31](31) maybe_went_rw) 2014-12-08 18:13:57.778116 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(51772-51937 up [11,31](11) acting [11,31](11) maybe_went_rw) 2014-12-08 18:13:57.778121 7f28fc345700 10 osd.70 pg_epoch: 71445 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71432/71433/71433) [68,18,70] r=2 lpr=71434 pi=71183-71432/17 crt=71377'4110 lcod 0'0 inactive NOTIFY] handle_advance_map [68,18,70]/[68,18,70] -- 68/68 2014-12-08 18:13:57.778123 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(51770-51771 up [4,11](4) acting [4,11](4) maybe_went_rw) 2014-12-08 18:13:57.778132 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.778131 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(51768-51769 up [11](11) acting [11](11) maybe_went_rw) 2014-12-08 18:13:57.778135 7f28fc345700 10 osd.70 pg_epoch: 71446 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71432/71433/71433) [68,18,70] r=2 lpr=71434 pi=71183-71432/17 crt=71377'4110 lcod 0'0 inactive NOTIFY] state: Reset advmap 2014-12-08 18:13:57.778138 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(51765-51767 up [4,11](4) acting [4,11](4) maybe_went_rw) 2014-12-08 18:13:57.778142 7f28fc345700 10 osd.70 pg_epoch: 71446 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71432/71433/71433) [68,18,70] r=2 lpr=71434 pi=71183-71432/17 crt=71377'4110 lcod 0'0 inactive NOTIFY] _calc_past_interval_range: already have past intervals back to 71434 2014-12-08 18:13:57.778147 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(51763-51764 up [11](11) acting [11](11) maybe_went_rw) 2014-12-08 18:13:57.778150 7f28fc345700 10 osd.70 pg_epoch: 71446 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71432/71433/71433) [68,18,70] r=2 lpr=71434 pi=71183-71432/17 crt=71377'4110 lcod 0'0 inactive NOTIFY] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.778154 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(51757-51762 up [4,11](4) acting [4,11](4) maybe_went_rw) 2014-12-08 18:13:57.778162 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(51749-51756 up [11](11) acting [11](11) maybe_went_rw) 2014-12-08 18:13:57.778165 7f28fc345700 10 osd.70 pg_epoch: 71446 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71432/71433/71433) [68,18,70] r=2 lpr=71434 pi=71183-71432/17 crt=71377'4110 lcod 0'0 inactive NOTIFY] handle_advance_map [68,18,70]/[68,18,70] -- 68/68 2014-12-08 18:13:57.778174 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.778169 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(51743-51748 up [4,11](4) acting [4,11](4) maybe_went_rw) 2014-12-08 18:13:57.778177 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(51736-51742 up [11](11) acting [11](11) maybe_went_rw) 2014-12-08 18:13:57.778184 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(51732-51735 up [4,11](4) acting [4,11](4) maybe_went_rw) 2014-12-08 18:13:57.778191 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(51729-51731 up [11](11) acting [11](11) maybe_went_rw) 2014-12-08 18:13:57.778197 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(51723-51728 up [4,11](4) acting [4,11](4) maybe_went_rw) 2014-12-08 18:13:57.778205 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(51721-51722 up [11](11) acting [11](11) maybe_went_rw) 2014-12-08 18:13:57.778214 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(51649-51720 up [4,11](4) acting [4,11](4) maybe_went_rw) 2014-12-08 18:13:57.778176 7f28fc345700 10 osd.70 pg_epoch: 71447 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71432/71433/71433) [68,18,70] r=2 lpr=71434 pi=71183-71432/17 crt=71377'4110 lcod 0'0 inactive NOTIFY] state: Reset advmap 2014-12-08 18:13:57.778222 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(51637-51648 up [11](11) acting [11](11) maybe_went_rw) 2014-12-08 18:13:57.778226 7f28fc345700 10 osd.70 pg_epoch: 71447 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71432/71433/71433) [68,18,70] r=2 lpr=71434 pi=71183-71432/17 crt=71377'4110 lcod 0'0 inactive NOTIFY] _calc_past_interval_range: already have past intervals back to 71434 2014-12-08 18:13:57.778230 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(51483-51636 up [4,11](4) acting [4,11](4) maybe_went_rw) 2014-12-08 18:13:57.778245 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(51467-51482 up [11](11) acting [11](11) maybe_went_rw) 2014-12-08 18:13:57.778248 7f28fc345700 10 osd.70 pg_epoch: 71447 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71432/71433/71433) [68,18,70] r=2 lpr=71434 pi=71183-71432/17 crt=71377'4110 lcod 0'0 inactive NOTIFY] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.778255 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(51363-51466 up [4,11](4) acting [4,11](4) maybe_went_rw) 2014-12-08 18:13:57.778268 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(51338-51362 up [11](11) acting [11](11) maybe_went_rw) 2014-12-08 18:13:57.778280 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(51148-51337 up [4,11](4) acting [4,11](4) maybe_went_rw) 2014-12-08 18:13:57.778283 7f28fc345700 10 osd.70 pg_epoch: 71447 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71432/71433/71433) [68,18,70] r=2 lpr=71434 pi=71183-71432/17 crt=71377'4110 lcod 0'0 inactive NOTIFY] handle_advance_map [68,18,70]/[68,18,70] -- 68/68 2014-12-08 18:13:57.778301 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.778295 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(51147-51147 up [11](11) acting [11](11)) 2014-12-08 18:13:57.778309 7f28fc345700 10 osd.70 pg_epoch: 71448 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71432/71433/71433) [68,18,70] r=2 lpr=71434 pi=71183-71432/17 crt=71377'4110 lcod 0'0 inactive NOTIFY] state: Reset advmap 2014-12-08 18:13:57.778310 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(51140-51146 up [4,11](4) acting [4,11](4) maybe_went_rw) 2014-12-08 18:13:57.778327 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(51119-51139 up [11,31](11) acting [11,31](11) maybe_went_rw) 2014-12-08 18:13:57.778325 7f28fc345700 10 osd.70 pg_epoch: 71448 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71432/71433/71433) [68,18,70] r=2 lpr=71434 pi=71183-71432/17 crt=71377'4110 lcod 0'0 inactive NOTIFY] _calc_past_interval_range: already have past intervals back to 71434 2014-12-08 18:13:57.778338 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(51100-51118 up [11,31](11) acting [0](0) maybe_went_rw) 2014-12-08 18:13:57.778340 7f28fc345700 10 osd.70 pg_epoch: 71448 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71432/71433/71433) [68,18,70] r=2 lpr=71434 pi=71183-71432/17 crt=71377'4110 lcod 0'0 inactive NOTIFY] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.778349 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(51073-51099 up [11,31](11) acting [11,31](11) maybe_went_rw) 2014-12-08 18:13:57.778367 7f28fc345700 10 osd.70 pg_epoch: 71448 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71432/71433/71433) [68,18,70] r=2 lpr=71434 pi=71183-71432/17 crt=71377'4110 lcod 0'0 inactive NOTIFY] handle_advance_map [68,18,70]/[68,18,70] -- 68/68 2014-12-08 18:13:57.778374 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior final: probe 0,4,11,13,16,20,30,31,56,58,59,63,70,76 down blocked_by {} 2014-12-08 18:13:57.778388 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.778387 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] up_thru 71491 >= same_since 71456, all is well 2014-12-08 18:13:57.778397 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] state: querying info from osd.0 2014-12-08 18:13:57.778392 7f28fc345700 10 osd.70 pg_epoch: 71449 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71432/71433/71433) [68,18,70] r=2 lpr=71434 pi=71183-71432/17 crt=71377'4110 lcod 0'0 inactive NOTIFY] state: Reset advmap 2014-12-08 18:13:57.778409 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] state: querying info from osd.4 2014-12-08 18:13:57.778411 7f28fc345700 10 osd.70 pg_epoch: 71449 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71432/71433/71433) [68,18,70] r=2 lpr=71434 pi=71183-71432/17 crt=71377'4110 lcod 0'0 inactive NOTIFY] _calc_past_interval_range: already have past intervals back to 71434 2014-12-08 18:13:57.778419 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] state: querying info from osd.11 2014-12-08 18:13:57.778428 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] state: querying info from osd.13 2014-12-08 18:13:57.778436 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] state: querying info from osd.16 2014-12-08 18:13:57.778445 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] state: querying info from osd.20 2014-12-08 18:13:57.778428 7f28fc345700 10 osd.70 pg_epoch: 71449 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71432/71433/71433) [68,18,70] r=2 lpr=71434 pi=71183-71432/17 crt=71377'4110 lcod 0'0 inactive NOTIFY] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.778456 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] state: querying info from osd.30 2014-12-08 18:13:57.778463 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] state: querying info from osd.31 2014-12-08 18:13:57.778469 7f28fc345700 10 osd.70 pg_epoch: 71449 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71432/71433/71433) [68,18,70] r=2 lpr=71434 pi=71183-71432/17 crt=71377'4110 lcod 0'0 inactive NOTIFY] handle_advance_map [68,18,70]/[68,18,70] -- 68/68 2014-12-08 18:13:57.778473 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] state: querying info from osd.56 2014-12-08 18:13:57.778482 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.778483 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] state: querying info from osd.58 2014-12-08 18:13:57.778485 7f28fc345700 10 osd.70 pg_epoch: 71450 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71432/71433/71433) [68,18,70] r=2 lpr=71434 pi=71183-71432/17 crt=71377'4110 lcod 0'0 inactive NOTIFY] state: Reset advmap 2014-12-08 18:13:57.778493 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] state: querying info from osd.59 2014-12-08 18:13:57.778494 7f28fc345700 10 osd.70 pg_epoch: 71450 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71432/71433/71433) [68,18,70] r=2 lpr=71434 pi=71183-71432/17 crt=71377'4110 lcod 0'0 inactive NOTIFY] _calc_past_interval_range: already have past intervals back to 71434 2014-12-08 18:13:57.778503 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] state: querying info from osd.63 2014-12-08 18:13:57.778504 7f28fc345700 10 osd.70 pg_epoch: 71450 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71432/71433/71433) [68,18,70] r=2 lpr=71434 pi=71183-71432/17 crt=71377'4110 lcod 0'0 inactive NOTIFY] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.778513 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] state: querying info from osd.76 2014-12-08 18:13:57.778520 7f28fc345700 10 osd.70 pg_epoch: 71450 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71432/71433/71433) [68,18,70] r=2 lpr=71434 pi=71183-71432/17 crt=71377'4110 lcod 0'0 inactive NOTIFY] handle_advance_map [68,18,70]/[68,18,70] -- 68/68 2014-12-08 18:13:57.778531 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.778523 7f28fcb46700 15 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] publish_stats_to_osd 71581:63844727 2014-12-08 18:13:57.778533 7f28fc345700 10 osd.70 pg_epoch: 71451 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71432/71433/71433) [68,18,70] r=2 lpr=71434 pi=71183-71432/17 crt=71377'4110 lcod 0'0 inactive NOTIFY] state: Reset advmap 2014-12-08 18:13:57.778536 7f28fcb46700 20 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] handle_activate_map: Not dirtying info: last_persisted is 71456 while current is 71581 2014-12-08 18:13:57.778543 7f28fc345700 10 osd.70 pg_epoch: 71451 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71432/71433/71433) [68,18,70] r=2 lpr=71434 pi=71183-71432/17 crt=71377'4110 lcod 0'0 inactive NOTIFY] _calc_past_interval_range: already have past intervals back to 71434 2014-12-08 18:13:57.778548 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] handle_peering_event: epoch_sent: 71581 epoch_requested: 71581 NullEvt 2014-12-08 18:13:57.778554 7f28fc345700 10 osd.70 pg_epoch: 71451 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71432/71433/71433) [68,18,70] r=2 lpr=71434 pi=71183-71432/17 crt=71377'4110 lcod 0'0 inactive NOTIFY] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.778563 7f28fcb46700 10 write_log with: dirty_to: 0'0, dirty_from: 4294967295'18446744073709551615, dirty_divergent_priors: 0, writeout_from: 4294967295'18446744073709551615, trimmed: 2014-12-08 18:13:57.778570 7f28fc345700 10 osd.70 pg_epoch: 71451 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71432/71433/71433) [68,18,70] r=2 lpr=71434 pi=71183-71432/17 crt=71377'4110 lcod 0'0 inactive NOTIFY] handle_advance_map [68,18,70]/[68,18,70] -- 68/68 2014-12-08 18:13:57.778579 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.778584 7f28fc345700 10 osd.70 pg_epoch: 71452 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71432/71433/71433) [68,18,70] r=2 lpr=71434 pi=71183-71432/17 crt=71377'4110 lcod 0'0 inactive NOTIFY] state: Reset advmap 2014-12-08 18:13:57.778590 7f28fc345700 10 osd.70 pg_epoch: 71452 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71432/71433/71433) [68,18,70] r=2 lpr=71434 pi=71183-71432/17 crt=71377'4110 lcod 0'0 inactive NOTIFY] _calc_past_interval_range: already have past intervals back to 71434 2014-12-08 18:13:57.778599 7f28fc345700 10 osd.70 pg_epoch: 71452 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71432/71433/71433) [68,18,70] r=2 lpr=71434 pi=71183-71432/17 crt=71377'4110 lcod 0'0 inactive NOTIFY] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.778617 7f28fc345700 10 osd.70 pg_epoch: 71452 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71432/71433/71433) [68,18,70] r=2 lpr=71434 pi=71183-71432/17 crt=71377'4110 lcod 0'0 inactive NOTIFY] handle_advance_map [68,18,70]/[68,18,70] -- 68/68 2014-12-08 18:13:57.778629 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.778631 7f28fc345700 10 osd.70 pg_epoch: 71453 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71432/71433/71433) [68,18,70] r=2 lpr=71434 pi=71183-71432/17 crt=71377'4110 lcod 0'0 inactive NOTIFY] state: Reset advmap 2014-12-08 18:13:57.778638 7f28fc345700 10 osd.70 pg_epoch: 71453 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71432/71433/71433) [68,18,70] r=2 lpr=71434 pi=71183-71432/17 crt=71377'4110 lcod 0'0 inactive NOTIFY] _calc_past_interval_range: already have past intervals back to 71434 2014-12-08 18:13:57.778644 7f28fc345700 10 osd.70 pg_epoch: 71453 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71432/71433/71433) [68,18,70] r=2 lpr=71434 pi=71183-71432/17 crt=71377'4110 lcod 0'0 inactive NOTIFY] check_recovery_sources no source osds () went down 2014-12-08 18:13:57.778660 7f28fc345700 10 osd.70 pg_epoch: 71453 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71432/71433/71433) [68,18,70] r=2 lpr=71434 pi=71183-71432/17 crt=71377'4110 lcod 0'0 inactive NOTIFY] handle_advance_map [68,18,70]/[68,18,70] -- 68/68 2014-12-08 18:13:57.778658 7f28fcb46700 10 osd.70 pg_epoch: 71439 pg[19.6d( v 71397'33614009 (70963'33611008,71397'33614009] local-les=71439 n=896 ec=21298 les/c 71439/71439 71432/71438/71169) [4,70,69] r=1 lpr=71439 pi=71156-71437/22 crt=71397'33614007 lcod 0'0 inactive NOTIFY] handle_advance_map [4,70,69]/[4,70,69] -- 4/4 2014-12-08 18:13:57.778671 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:13:57.778678 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:13:57.778678 7f28fc345700 10 osd.70 pg_epoch: 71454 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71432/71433/71433) [68,18,70] r=2 lpr=71434 pi=71183-71432/17 crt=71377'4110 lcod 0'0 inactive NOTIFY] state: Reset advmap 2014-12-08 18:13:57.778683 7f28fcb46700 10 osd.70 pg_epoch: 71440 pg[19.6d( v 71397'33614009 (70963'33611008,71397'33614009] local-les=71439 n=896 ec=21298 les/c 71439/71439 71432/71438/71169) [4,70,69] r=1 lpr=71439 pi=71156-71437/22 crt=71397'33614007 lcod 0'0 inactive NOTIFY] state: Reset advmap 2014-12-08 18:13:57.778688 7f28fc345700 10 osd.70 pg_epoch: 71454 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71432/71433/71433) [68,18,70] r=2 lpr=71434 pi=71183-71432/17 crt=71377'4110 lcod 0'0 inactive NOTIFY] _calc_past_interval_range: already have past intervals back to 71434 2014-12-08 18:13:57.778698 7f28fc345700 10 osd.70 pg_epoch: 71454 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71432/71433/71433) [68,18,70] r=2 lpr=71434 pi=71183-71432/17 crt=71377'4110 lcod 0'0 inactive NOTIFY] check_recovery_sources no source osds () went down -- 2014-12-08 18:13:59.777307 7f2904b56700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] null 2014-12-08 18:13:59.777328 7f2904b56700 10 osd.70 pg_epoch: 71581 pg[19.6d( v 71397'33614009 (70963'33611008,71397'33614009] local-les=71439 n=896 ec=21298 les/c 71439/71439 71432/71438/71169) [4,70,69] r=1 lpr=71439 pi=71156-71437/22 crt=71397'33614007 lcod 0'0 inactive NOTIFY] null 2014-12-08 18:13:59.777350 7f2904b56700 10 osd.70 pg_epoch: 71581 pg[218.6c7( v 71581'2 (0'0,71581'2] local-les=71433 n=2 ec=71423 les/c 71433/71433 71432/71432/71432) [70,10,18] r=0 lpr=71581 crt=0'0 lcod 0'0 mlcod 0'0 inactive] null 2014-12-08 18:13:59.777368 7f2904b56700 10 osd.70 pg_epoch: 71581 pg[218.6c4( v 71581'3 (0'0,71581'3] local-les=71437 n=3 ec=71423 les/c 71437/71437 71432/71432/71432) [70,76,13] r=0 lpr=71581 crt=71580'1 lcod 0'0 mlcod 0'0 inactive] null 2014-12-08 18:13:59.777384 7f2904b56700 10 osd.70 pg_epoch: 71581 pg[218.562( v 71581'3 (0'0,71581'3] local-les=71437 n=3 ec=71423 les/c 71437/71437 71432/71432/71425) [9,18,70] r=2 lpr=71581 pi=71423-71431/4 crt=71580'1 lcod 0'0 inactive NOTIFY] null 2014-12-08 18:13:59.777409 7f2904b56700 10 osd.70 pg_epoch: 71581 pg[218.5e( empty local-les=71435 n=0 ec=71423 les/c 71435/71435 71432/71432/71425) [16,22,70] r=2 lpr=71435 pi=71423-71431/4 crt=0'0 inactive NOTIFY] null 2014-12-08 18:13:59.777425 7f2904b56700 10 osd.70 pg_epoch: 71581 pg[218.5f( v 71580'2 (0'0,71580'2] local-les=71437 n=2 ec=71423 les/c 71437/71437 71432/71432/71432) [70,15,24] r=0 lpr=71580 crt=0'0 lcod 0'0 mlcod 0'0 peering] null 2014-12-08 18:13:59.777444 7f2904b56700 10 osd.70 pg_epoch: 71581 pg[217.235( v 71377'4108 (71377'1107,71377'4108] local-les=71435 n=11 ec=71094 les/c 71435/71435 71432/71434/71434) [78,25,70] r=2 lpr=71435 pi=71094-71433/26 crt=71377'4105 lcod 0'0 inactive NOTIFY] null 2014-12-08 18:13:59.777462 7f2904b56700 10 osd.70 pg_epoch: 71581 pg[218.38e( empty local-les=71437 n=0 ec=71423 les/c 71437/71437 71432/71432/71432) [70,13,3] r=0 lpr=71437 crt=0'0 mlcod 0'0 peering] null 2014-12-08 18:13:59.777478 7f2904b56700 10 osd.70 pg_epoch: 71581 pg[218.38f( empty local-les=71457 n=0 ec=71423 les/c 71457/71457 71456/71456/71432) [70,23,25] r=0 lpr=71457 crt=0'0 mlcod 0'0 peering] null 2014-12-08 18:13:59.777496 7f2904b56700 10 osd.70 pg_epoch: 71581 pg[218.38c( empty local-les=71448 n=0 ec=71423 les/c 71448/71448 71444/71444/71444) [9,18,70] r=2 lpr=71448 pi=71423-71443/6 crt=0'0 inactive NOTIFY] null 2014-12-08 18:13:59.777513 7f2904b56700 10 osd.70 pg_epoch: 71581 pg[217.51( v 71377'4117 (71377'1116,71377'4117] local-les=71435 n=16 ec=71094 les/c 71435/71435 71432/71434/71096) [75,17,70] r=2 lpr=71435 pi=71183-71433/16 crt=71377'4114 lcod 0'0 inactive NOTIFY] null 2014-12-08 18:13:59.777532 7f2904b56700 10 osd.70 pg_epoch: 71581 pg[216.380( v 70990'233 (0'0,70990'233] local-les=71447 n=112 ec=70987 les/c 71447/71453 71444/71444/71444) [17,70,7] r=1 lpr=71453 pi=70987-71443/26 crt=70990'231 lcod 0'0 inactive NOTIFY] null 2014-12-08 18:13:59.777551 7f2904b56700 10 osd.70 pg_epoch: 71581 pg[218.51( v 71581'3 (0'0,71581'3] local-les=71433 n=3 ec=71423 les/c 71433/71434 71432/71432/71432) [70,66,75] r=0 lpr=71581 crt=71580'1 lcod 0'0 mlcod 0'0 inactive] null 2014-12-08 18:13:59.777567 7f2904b56700 10 osd.70 pg_epoch: 71581 pg[218.380( empty local-les=71450 n=0 ec=71423 les/c 71450/71450 71444/71444/71425) [18,21,70] r=2 lpr=71450 pi=71423-71443/6 crt=0'0 inactive NOTIFY] null 2014-12-08 18:13:59.777582 7f2904b56700 10 osd.70 pg_epoch: 71581 pg[218.518( v 71581'1 (0'0,71581'1] local-les=71433 n=1 ec=71423 les/c 71433/71433 71432/71432/71432) [68,6,70] r=2 lpr=71581 pi=71423-71431/4 crt=0'0 lcod 0'0 inactive NOTIFY] null 2014-12-08 18:13:59.777599 7f2904b56700 10 osd.70 pg_epoch: 71581 pg[217.22d( v 71377'4107 (71377'1106,71377'4107] local-les=71437 n=8 ec=71094 les/c 71437/71437 71432/71436/71436) [63,70,1] r=1 lpr=71437 pi=71094-71435/26 crt=71377'4104 lcod 0'0 inactive NOTIFY] null 2014-12-08 18:13:59.777616 7f2904b56700 10 osd.70 pg_epoch: 71581 pg[216.384( v 70990'209 (0'0,70990'209] local-les=71435 n=101 ec=70987 les/c 71435/71435 71432/71432/71432) [77,22,70] r=2 lpr=71435 pi=70987-71431/22 crt=70990'207 lcod 0'0 inactive NOTIFY] null 2014-12-08 18:13:59.777633 7f2904b56700 10 osd.70 pg_epoch: 71581 pg[216.22f( v 71097'207 (0'0,71097'207] local-les=71450 n=99 ec=70987 les/c 71450/71450 71444/71444/71432) [70,6,21] r=0 lpr=71450 crt=71097'205 lcod 0'0 mlcod 0'0 peering] null 2014-12-08 18:13:59.777650 7f2904b56700 10 osd.70 pg_epoch: 71581 pg[218.4b( v 71581'2 (0'0,71581'2] local-les=71448 n=2 ec=71423 les/c 71448/71453 71444/71444/71425) [18,17,70] r=2 lpr=71581 pi=71423-71443/6 crt=0'0 lcod 0'0 inactive NOTIFY] null 2014-12-08 18:13:59.777667 7f2904b56700 10 osd.70 pg_epoch: 71581 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71432/71433/71433) [68,18,70] r=2 lpr=71434 pi=71183-71432/17 crt=71377'4110 lcod 0'0 inactive NOTIFY] null -- 2014-12-08 18:14:00.274128 7f2904b56700 20 osd.70 71591 19.6e heartbeat_peers 13,16,70 2014-12-08 18:14:00.274134 7f28fcb46700 10 log is not dirty 2014-12-08 18:14:00.274130 7f28fc345700 10 osd.70 pg_epoch: 71582 pg[216.ee( empty local-les=71437 n=0 ec=70987 les/c 71437/71437 71432/71432/70989) [9,8,70] r=2 lpr=71582 pi=71401-71431/4 crt=0'0 inactive NOTIFY] state: should restart peering, calling start_peering_interval again 2014-12-08 18:14:00.274137 7f28fc345700 20 osd.70 pg_epoch: 71582 pg[216.ee( empty local-les=71437 n=0 ec=70987 les/c 71437/71437 71432/71432/70989) [9,8,70] r=2 lpr=71582 pi=71401-71431/4 crt=0'0 inactive NOTIFY] set_last_peering_reset 71582 2014-12-08 18:14:00.274154 7f28fc345700 10 osd.70 pg_epoch: 71582 pg[216.ee( empty local-les=71437 n=0 ec=70987 les/c 71437/71437 71432/71432/70989) [9,8] r=-1 lpr=71582 pi=71401-71581/5 crt=0'0 inactive NOTIFY] start_peering_interval: check_new_interval output: generate_past_intervals interval(71432-71581 up [9,8,70](9) acting [9,8,70](9)): not rw, up_thru 71514 up_from 69689 last_epoch_clean 71437 generate_past_intervals interval(71432-71581 up [9,8,70](9) acting [9,8,70](9) maybe_went_rw) : primary up 69689-71514 includes interval 2014-12-08 18:14:00.274159 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[19.5a( v 71397'49665417 (50495'49662416,71397'49665417] local-les=71434 n=896 ec=21298 les/c 71434/71434 71432/71433/71433) [70,20,21] r=0 lpr=71434 crt=71397'49665415 lcod 0'0 mlcod 0'0 peering] handle_advance_map [20,21]/[20,21] -- 20/20 2014-12-08 18:14:00.274170 7f28fc345700 10 osd.70 pg_epoch: 71582 pg[216.ee( empty local-les=71437 n=0 ec=70987 les/c 71437/71437 71432/71432/70989) [9,8] r=-1 lpr=71582 pi=71401-71581/5 crt=0'0 inactive NOTIFY] noting past interval(71432-71581 up [9,8,70](9) acting [9,8,70](9) maybe_went_rw) 2014-12-08 18:14:00.274179 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:14:00.274177 7f28fc345700 10 osd.70 pg_epoch: 71582 pg[216.ee( empty local-les=71437 n=0 ec=70987 les/c 71437/71437 71582/71582/70989) [9,8] r=-1 lpr=71582 pi=71401-71581/5 crt=0'0 inactive NOTIFY] up [9,8,70] -> [9,8], acting [9,8,70] -> [9,8], acting_primary 9 -> 9, up_primary 9 -> 9, role 2 -> -1 2014-12-08 18:14:00.274184 7f28fc345700 10 osd.70 pg_epoch: 71582 pg[216.ee( empty local-les=71437 n=0 ec=70987 les/c 71437/71437 71582/71582/70989) [9,8] r=-1 lpr=71582 pi=71401-71581/5 crt=0'0 inactive NOTIFY] on_change 2014-12-08 18:14:00.274190 7f28fc345700 15 osd.70 pg_epoch: 71582 pg[216.ee( empty local-les=71437 n=0 ec=70987 les/c 71437/71437 71582/71582/70989) [9,8] r=-1 lpr=71582 pi=71401-71581/5 crt=0'0 inactive NOTIFY] publish_stats_to_osd -- not primary 2014-12-08 18:14:00.274189 7f2904b56700 10 osd.70 71591 _add_heartbeat_peer: new peer osd.0 172.16.10.102:6990/4210 172.16.10.102:6991/4210 2014-12-08 18:14:00.274185 7f28fcb46700 10 osd.70 pg_epoch: 71582 pg[19.5a( v 71397'49665417 (50495'49662416,71397'49665417] local-les=71434 n=896 ec=21298 les/c 71434/71434 71432/71433/71433) [70,20,21] r=0 lpr=71434 crt=71397'49665415 lcod 0'0 mlcod 0'0 peering] state: Peering advmap 2014-12-08 18:14:00.274197 7f28fc345700 15 osd.70 pg_epoch: 71582 pg[216.ee( empty local-les=71437 n=0 ec=70987 les/c 71437/71437 71582/71582/70989) [9,8] r=-1 lpr=71582 pi=71401-71581/5 crt=0'0 inactive NOTIFY] requeue_ops 2014-12-08 18:14:00.274204 7f28fc345700 10 osd.70 pg_epoch: 71582 pg[216.ee( empty local-les=71437 n=0 ec=70987 les/c 71437/71437 71582/71582/70989) [9,8] r=-1 lpr=71582 pi=71401-71581/5 crt=0'0 inactive NOTIFY] cancel_copy_ops 2014-12-08 18:14:00.274204 7f28fcb46700 10 osd.70 pg_epoch: 71582 pg[19.5a( v 71397'49665417 (50495'49662416,71397'49665417] local-les=71434 n=896 ec=21298 les/c 71434/71434 71432/71433/71433) [70,20,21] r=0 lpr=71434 crt=71397'49665415 lcod 0'0 mlcod 0'0 peering] PriorSet: affected_by_map osd.70 now down 2014-12-08 18:14:00.274209 7f28fc345700 10 osd.70 pg_epoch: 71582 pg[216.ee( empty local-les=71437 n=0 ec=70987 les/c 71437/71437 71582/71582/70989) [9,8] r=-1 lpr=71582 pi=71401-71581/5 crt=0'0 inactive NOTIFY] cancel_flush_ops 2014-12-08 18:14:00.274215 7f28fc345700 10 osd.70 pg_epoch: 71582 pg[216.ee( empty local-les=71437 n=0 ec=70987 les/c 71437/71437 71582/71582/70989) [9,8] r=-1 lpr=71582 pi=71401-71581/5 crt=0'0 inactive NOTIFY] on_change_cleanup 2014-12-08 18:14:00.274212 7f28fcb46700 1 osd.70 pg_epoch: 71582 pg[19.5a( v 71397'49665417 (50495'49662416,71397'49665417] local-les=71434 n=896 ec=21298 les/c 71434/71434 71432/71433/71433) [70,20,21] r=0 lpr=71434 crt=71397'49665415 lcod 0'0 mlcod 0'0 peering] state: Peering, affected_by_map, going to Reset -- 2014-12-08 18:14:00.295754 7f2904b56700 20 osd.70 71591 _dispatch 0x12ca5860 pg_notify(19.6e(150) epoch 71591) v5 2014-12-08 18:14:00.295751 7f28fcb46700 10 osd.70 pg_epoch: 71587 pg[216.269( v 71097'270 (0'0,71097'270] local-les=71449 n=128 ec=70987 les/c 71449/71449 71587/71587/70989) [75,9] r=-1 lpr=71587 pi=70987-71586/29 crt=71097'268 lcod 0'0 inactive NOTIFY] on_change 2014-12-08 18:14:00.295756 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[217.246( v 71377'15269 (71377'12268,71377'15269] local-les=71451 n=13 ec=71094 les/c 71451/71451 71591/71591/71591) [70,21,3] r=0 lpr=71591 pi=71450-71590/6 crt=71377'15266 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(71587-71589 up [21,3](21) acting [3,21](3) maybe_went_rw) 2014-12-08 18:14:00.295765 7f28fcb46700 15 osd.70 pg_epoch: 71587 pg[216.269( v 71097'270 (0'0,71097'270] local-les=71449 n=128 ec=70987 les/c 71449/71449 71587/71587/70989) [75,9] r=-1 lpr=71587 pi=70987-71586/29 crt=71097'268 lcod 0'0 inactive NOTIFY] publish_stats_to_osd -- not primary 2014-12-08 18:14:00.295780 7f28fcb46700 15 osd.70 pg_epoch: 71587 pg[216.269( v 71097'270 (0'0,71097'270] local-les=71449 n=128 ec=70987 les/c 71449/71449 71587/71587/70989) [75,9] r=-1 lpr=71587 pi=70987-71586/29 crt=71097'268 lcod 0'0 inactive NOTIFY] requeue_ops 2014-12-08 18:14:00.295774 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[217.246( v 71377'15269 (71377'12268,71377'15269] local-les=71451 n=13 ec=71094 les/c 71451/71451 71591/71591/71591) [70,21,3] r=0 lpr=71591 pi=71450-71590/6 crt=71377'15266 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(71585-71586 up [23,21,3](23) acting [3,21](3) maybe_went_rw) 2014-12-08 18:14:00.295793 7f28fcb46700 10 osd.70 pg_epoch: 71587 pg[216.269( v 71097'270 (0'0,71097'270] local-les=71449 n=128 ec=70987 les/c 71449/71449 71587/71587/70989) [75,9] r=-1 lpr=71587 pi=70987-71586/29 crt=71097'268 lcod 0'0 inactive NOTIFY] cancel_copy_ops 2014-12-08 18:14:00.295807 7f2904b56700 7 osd.70 71591 handle_pg_notify from osd.58 2014-12-08 18:14:00.295810 7f2904b56700 15 osd.70 71591 require_same_or_newer_map 71591 (i am 71591) 0x12ca5860 2014-12-08 18:14:00.295803 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[217.246( v 71377'15269 (71377'12268,71377'15269] local-les=71451 n=13 ec=71094 les/c 71451/71451 71591/71591/71591) [70,21,3] r=0 lpr=71591 pi=71450-71590/6 crt=71377'15266 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(71584-71584 up [23,21,3](23) acting [23,21,3](23)) 2014-12-08 18:14:00.295808 7f28fcb46700 10 osd.70 pg_epoch: 71587 pg[216.269( v 71097'270 (0'0,71097'270] local-les=71449 n=128 ec=70987 les/c 71449/71449 71587/71587/70989) [75,9] r=-1 lpr=71587 pi=70987-71586/29 crt=71097'268 lcod 0'0 inactive NOTIFY] cancel_flush_ops 2014-12-08 18:14:00.295819 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[217.246( v 71377'15269 (71377'12268,71377'15269] local-les=71451 n=13 ec=71094 les/c 71451/71451 71591/71591/71591) [70,21,3] r=0 lpr=71591 pi=71450-71590/6 crt=71377'15266 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(71582-71583 up [21,3](21) acting [21,3](21) maybe_went_rw) 2014-12-08 18:14:00.295825 7f28fcb46700 10 osd.70 pg_epoch: 71587 pg[216.269( v 71097'270 (0'0,71097'270] local-les=71449 n=128 ec=70987 les/c 71449/71449 71587/71587/70989) [75,9] r=-1 lpr=71587 pi=70987-71586/29 crt=71097'268 lcod 0'0 inactive NOTIFY] on_change_cleanup 2014-12-08 18:14:00.295845 7f2904b56700 10 osd.70 71591 do_waiters -- start 2014-12-08 18:14:00.295848 7f2904b56700 10 osd.70 71591 do_waiters -- finish 2014-12-08 18:14:00.295836 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[217.246( v 71377'15269 (71377'12268,71377'15269] local-les=71451 n=13 ec=71094 les/c 71451/71451 71591/71591/71591) [70,21,3] r=0 lpr=71591 pi=71450-71590/6 crt=71377'15266 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(71450-71581 up [70,21,3](70) acting [70,21,3](70) maybe_went_rw) 2014-12-08 18:14:00.295843 7f28fcb46700 10 osd.70 pg_epoch: 71587 pg[216.269( v 71097'270 (0'0,71097'270] local-les=71449 n=128 ec=70987 les/c 71449/71449 71587/71587/70989) [75,9] r=-1 lpr=71587 pi=70987-71586/29 crt=71097'268 lcod 0'0 inactive NOTIFY] on_change 2014-12-08 18:14:00.295864 7f2904b56700 10 osd.70 71591 do_waiters -- start 2014-12-08 18:14:00.295853 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[217.246( v 71377'15269 (71377'12268,71377'15269] local-les=71451 n=13 ec=71094 les/c 71451/71451 71591/71591/71591) [70,21,3] r=0 lpr=71591 pi=71450-71590/6 crt=71377'15266 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior final: probe 3,21,70 down blocked_by {} 2014-12-08 18:14:00.295867 7f2904b56700 10 osd.70 71591 do_waiters -- finish 2014-12-08 18:14:00.295873 7f2904b56700 20 osd.70 71591 _dispatch 0x12c70760 pg_notify(19.71(23) epoch 71591) v5 -- 2014-12-08 18:14:00.305095 7f2904b56700 20 osd.70 71591 _dispatch 0x12ca5680 pg_notify(19.6e(150) epoch 71591) v5 2014-12-08 18:14:00.305095 7f28fcb46700 20 osd.70 pg_epoch: 71582 pg[19.5e( v 71422'34705118 (70963'34702116,71422'34705118] local-les=71436 n=884 ec=21298 les/c 71436/71436 71432/71435/69765) [3,16,70] r=2 lpr=71436 pi=69698-71434/36 crt=71397'34705115 lcod 0'0 inactive NOTIFY] set_last_peering_reset 71582 2014-12-08 18:14:00.305110 7f28fcb46700 10 osd.70 pg_epoch: 71582 pg[19.5e( v 71422'34705118 (70963'34702116,71422'34705118] local-les=71436 n=884 ec=21298 les/c 71436/71436 71432/71435/69765) [3,16,70] r=2 lpr=71582 pi=69698-71434/36 crt=71397'34705115 lcod 0'0 inactive NOTIFY] Clearing blocked outgoing recovery messages 2014-12-08 18:14:00.305137 7f2904b56700 7 osd.70 71591 handle_pg_notify from osd.30 2014-12-08 18:14:00.305140 7f2904b56700 15 osd.70 71591 require_same_or_newer_map 71591 (i am 71591) 0x12ca5680 2014-12-08 18:14:00.305129 7f28fcb46700 10 osd.70 pg_epoch: 71582 pg[19.5e( v 71422'34705118 (70963'34702116,71422'34705118] local-les=71436 n=884 ec=21298 les/c 71436/71436 71432/71435/69765) [3,16,70] r=2 lpr=71582 pi=69698-71434/36 crt=71397'34705115 lcod 0'0 inactive NOTIFY] Not blocking outgoing recovery messages 2014-12-08 18:14:00.305144 7f28fcb46700 10 osd.70 pg_epoch: 71582 pg[19.5e( v 71422'34705118 (70963'34702116,71422'34705118] local-les=71436 n=884 ec=21298 les/c 71436/71436 71432/71435/69765) [3,16,70] r=2 lpr=71582 pi=69698-71434/36 crt=71397'34705115 lcod 0'0 inactive NOTIFY] state: Reset advmap 2014-12-08 18:14:00.305159 7f28fcb46700 10 osd.70 pg_epoch: 71582 pg[19.5e( v 71422'34705118 (70963'34702116,71422'34705118] local-les=71436 n=884 ec=21298 les/c 71436/71436 71432/71435/69765) [3,16,70] r=2 lpr=71582 pi=69698-71434/36 crt=71397'34705115 lcod 0'0 inactive NOTIFY] _calc_past_interval_range: already have past intervals back to 71436 2014-12-08 18:14:00.305171 7f28fcb46700 20 osd.70 pg_epoch: 71582 pg[19.5e( v 71422'34705118 (70963'34702116,71422'34705118] local-les=71436 n=884 ec=21298 les/c 71436/71436 71432/71435/69765) [3,16,70] r=2 lpr=71582 pi=69698-71434/36 crt=71397'34705115 lcod 0'0 inactive NOTIFY] new interval newup [3,16] newacting [3,16] 2014-12-08 18:14:00.305181 7f2904b56700 10 osd.70 71591 do_waiters -- start 2014-12-08 18:14:00.305184 7f2904b56700 10 osd.70 71591 do_waiters -- finish 2014-12-08 18:14:00.305183 7f28fcb46700 10 osd.70 pg_epoch: 71582 pg[19.5e( v 71422'34705118 (70963'34702116,71422'34705118] local-les=71436 n=884 ec=21298 les/c 71436/71436 71432/71435/69765) [3,16,70] r=2 lpr=71582 pi=69698-71434/36 crt=71397'34705115 lcod 0'0 inactive NOTIFY] state: should restart peering, calling start_peering_interval again 2014-12-08 18:14:00.305206 7f2904b56700 10 osd.70 71591 do_waiters -- start 2014-12-08 18:14:00.305208 7f2904b56700 10 osd.70 71591 do_waiters -- finish 2014-12-08 18:14:00.305195 7f28fcb46700 20 osd.70 pg_epoch: 71582 pg[19.5e( v 71422'34705118 (70963'34702116,71422'34705118] local-les=71436 n=884 ec=21298 les/c 71436/71436 71432/71435/69765) [3,16,70] r=2 lpr=71582 pi=69698-71434/36 crt=71397'34705115 lcod 0'0 inactive NOTIFY] set_last_peering_reset 71582 2014-12-08 18:14:00.305209 7f2904b56700 20 osd.70 71591 _dispatch 0x12b75840 pg_notify(19.30(68),218.3dd(7) epoch 71591) v5 2014-12-08 18:14:00.305236 7f2904b56700 7 osd.70 71591 handle_pg_notify from osd.8 2014-12-08 18:14:00.305239 7f2904b56700 15 osd.70 71591 require_same_or_newer_map 71591 (i am 71591) 0x12b75840 2014-12-08 18:14:00.305225 7f28fcb46700 10 osd.70 pg_epoch: 71582 pg[19.5e( v 71422'34705118 (70963'34702116,71422'34705118] local-les=71436 n=884 ec=21298 les/c 71436/71436 71432/71435/69765) [3,16] r=-1 lpr=71582 pi=69698-71581/37 crt=71397'34705115 lcod 0'0 inactive NOTIFY] start_peering_interval: check_new_interval output: generate_past_intervals interval(71435-71581 up [3,16,70](3) acting [3,16,70](3)): not rw, up_thru 71503 up_from 69692 last_epoch_clean 71436 generate_past_intervals interval(71435-71581 up [3,16,70](3) acting [3,16,70](3) maybe_went_rw) : primary up 69692-71503 includes interval -- 2014-12-08 18:14:00.306751 7f2904b56700 20 osd.70 71591 _dispatch 0x12c70b20 pg_notify(19.6e(150) epoch 71591) v5 2014-12-08 18:14:00.306752 7f28fcb46700 10 osd.70 pg_epoch: 71591 pg[19.5e( v 71422'34705118 (70963'34702116,71422'34705118] local-les=71436 n=884 ec=21298 les/c 71436/71436 71591/71591/69765) [3,16,70] r=2 lpr=71591 pi=69698-71590/41 crt=71397'34705115 lcod 0'0 inactive NOTIFY] handle_activate_map 2014-12-08 18:14:00.306785 7f2904b56700 7 osd.70 71591 handle_pg_notify from osd.76 2014-12-08 18:14:00.306787 7f2904b56700 15 osd.70 71591 require_same_or_newer_map 71591 (i am 71591) 0x12c70b20 2014-12-08 18:14:00.306813 7f2904b56700 10 osd.70 71591 do_waiters -- start 2014-12-08 18:14:00.306816 7f2904b56700 10 osd.70 71591 do_waiters -- finish 2014-12-08 18:14:00.306841 7f28fcb46700 10 osd.70 pg_epoch: 71591 pg[19.5e( v 71422'34705118 (70963'34702116,71422'34705118] local-les=71436 n=884 ec=21298 les/c 71436/71436 71591/71591/69765) [3,16,70] r=2 lpr=71591 pi=69698-71590/41 crt=71397'34705115 lcod 0'0 inactive NOTIFY] update_heartbeat_peers unchanged 2014-12-08 18:14:00.306849 7f28fcb46700 10 osd.70 pg_epoch: 71591 pg[19.5e( v 71422'34705118 (70963'34702116,71422'34705118] local-les=71436 n=884 ec=21298 les/c 71436/71436 71591/71591/69765) [3,16,70] r=2 lpr=71591 pi=69698-71590/41 crt=71397'34705115 lcod 0'0 inactive NOTIFY] take_waiters 2014-12-08 18:14:00.306857 7f28fcb46700 5 osd.70 pg_epoch: 71591 pg[19.5e( v 71422'34705118 (70963'34702116,71422'34705118] local-les=71436 n=884 ec=21298 les/c 71436/71436 71591/71591/69765) [3,16,70] r=2 lpr=71591 pi=69698-71590/41 crt=71397'34705115 lcod 0'0 inactive NOTIFY] exit Reset 0.001776 10 0.001505 2014-12-08 18:14:00.306868 7f2904b56700 10 osd.70 71591 do_waiters -- start 2014-12-08 18:14:00.306871 7f2904b56700 10 osd.70 71591 do_waiters -- finish 2014-12-08 18:14:00.306873 7f2904b56700 20 osd.70 71591 _dispatch 0x12cd1a40 pg_query(218.4d epoch 71591) v3 2014-12-08 18:14:00.306868 7f28fcb46700 5 osd.70 pg_epoch: 71591 pg[19.5e( v 71422'34705118 (70963'34702116,71422'34705118] local-les=71436 n=884 ec=21298 les/c 71436/71436 71591/71591/69765) [3,16,70] r=2 lpr=71591 pi=69698-71590/41 crt=71397'34705115 lcod 0'0 inactive NOTIFY] enter Started 2014-12-08 18:14:00.306879 7f28fcb46700 5 osd.70 pg_epoch: 71591 pg[19.5e( v 71422'34705118 (70963'34702116,71422'34705118] local-les=71436 n=884 ec=21298 les/c 71436/71436 71591/71591/69765) [3,16,70] r=2 lpr=71591 pi=69698-71590/41 crt=71397'34705115 lcod 0'0 inactive NOTIFY] enter Start 2014-12-08 18:14:00.306888 7f28fcb46700 1 osd.70 pg_epoch: 71591 pg[19.5e( v 71422'34705118 (70963'34702116,71422'34705118] local-les=71436 n=884 ec=21298 les/c 71436/71436 71591/71591/69765) [3,16,70] r=2 lpr=71591 pi=69698-71590/41 crt=71397'34705115 lcod 0'0 inactive NOTIFY] state: transitioning to Stray 2014-12-08 18:14:00.306896 7f28fcb46700 5 osd.70 pg_epoch: 71591 pg[19.5e( v 71422'34705118 (70963'34702116,71422'34705118] local-les=71436 n=884 ec=21298 les/c 71436/71436 71591/71591/69765) [3,16,70] r=2 lpr=71591 pi=69698-71590/41 crt=71397'34705115 lcod 0'0 inactive NOTIFY] exit Start 0.000017 0 0.000000 2014-12-08 18:14:00.306907 7f28fcb46700 5 osd.70 pg_epoch: 71591 pg[19.5e( v 71422'34705118 (70963'34702116,71422'34705118] local-les=71436 n=884 ec=21298 les/c 71436/71436 71591/71591/69765) [3,16,70] r=2 lpr=71591 pi=69698-71590/41 crt=71397'34705115 lcod 0'0 inactive NOTIFY] enter Started/Stray 2014-12-08 18:14:00.306919 7f28fcb46700 20 osd.70 pg_epoch: 71591 pg[19.5e( v 71422'34705118 (70963'34702116,71422'34705118] local-les=71436 n=884 ec=21298 les/c 71436/71436 71591/71591/69765) [3,16,70] r=2 lpr=71591 pi=69698-71590/41 crt=71397'34705115 lcod 0'0 inactive NOTIFY] handle_activate_map: Not dirtying info: last_persisted is 71436 while current is 71591 2014-12-08 18:14:00.306927 7f28fcb46700 10 osd.70 pg_epoch: 71591 pg[19.5e( v 71422'34705118 (70963'34702116,71422'34705118] local-les=71436 n=884 ec=21298 les/c 71436/71436 71591/71591/69765) [3,16,70] r=2 lpr=71591 pi=69698-71590/41 crt=71397'34705115 lcod 0'0 inactive NOTIFY] handle_peering_event: epoch_sent: 71591 epoch_requested: 71591 NullEvt 2014-12-08 18:14:00.306976 7f28fcb46700 10 log is not dirty 2014-12-08 18:14:00.307009 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[216.3c4( v 71097'219 (0'0,71097'219] local-les=71437 n=102 ec=70987 les/c 71437/71438 71432/71432/71432) [70,69,77] r=0 lpr=71438 crt=71097'219 lcod 0'0 mlcod 0'0 peering] handle_advance_map [69,77]/[69,77] -- 69/69 -- 2014-12-08 18:14:00.308550 7f2904b56700 20 osd.70 71591 _dispatch 0x12c6e1e0 pg_notify(19.6e(150) epoch 71591) v5 2014-12-08 18:14:00.308547 7f28fcb46700 10 osd.70 pg_epoch: 71591 pg[216.3c4( v 71097'219 (0'0,71097'219] local-les=71437 n=102 ec=70987 les/c 71437/71438 71591/71591/71591) [70,69,77] r=0 lpr=71591 pi=71432-71590/4 crt=71097'219 lcod 0'0 mlcod 0'0 inactive NOTIFY] on_change_cleanup 2014-12-08 18:14:00.308555 7f28fcb46700 10 osd.70 pg_epoch: 71591 pg[216.3c4( v 71097'219 (0'0,71097'219] local-les=71437 n=102 ec=70987 les/c 71437/71438 71591/71591/71591) [70,69,77] r=0 lpr=71591 pi=71432-71590/4 crt=71097'219 lcod 0'0 mlcod 0'0 inactive NOTIFY] on_change 2014-12-08 18:14:00.308566 7f28fcb46700 20 osd.70 pg_epoch: 71591 pg[216.3c4( v 71097'219 (0'0,71097'219] local-les=71437 n=102 ec=70987 les/c 71437/71438 71591/71591/71591) [70,69,77] r=0 lpr=71591 pi=71432-71590/4 crt=71097'219 lcod 0'0 mlcod 0'0 inactive NOTIFY] exit NotTrimming 2014-12-08 18:14:00.308586 7f2904b56700 7 osd.70 71591 handle_pg_notify from osd.4 2014-12-08 18:14:00.308580 7f28fcb46700 20 osd.70 pg_epoch: 71591 pg[216.3c4( v 71097'219 (0'0,71097'219] local-les=71437 n=102 ec=70987 les/c 71437/71438 71591/71591/71591) [70,69,77] r=0 lpr=71591 pi=71432-71590/4 crt=71097'219 lcod 0'0 mlcod 0'0 inactive NOTIFY] enter NotTrimming 2014-12-08 18:14:00.308589 7f2904b56700 15 osd.70 71591 require_same_or_newer_map 71591 (i am 71591) 0x12c6e1e0 2014-12-08 18:14:00.308590 7f28fcb46700 15 osd.70 pg_epoch: 71591 pg[216.3c4( v 71097'219 (0'0,71097'219] local-les=71437 n=102 ec=70987 les/c 71437/71438 71591/71591/71591) [70,69,77] r=0 lpr=71591 pi=71432-71590/4 crt=71097'219 lcod 0'0 mlcod 0'0 inactive] clear_stats 2014-12-08 18:14:00.308599 7f28fcb46700 15 osd.70 pg_epoch: 71591 pg[216.3c4( v 71097'219 (0'0,71097'219] local-les=71437 n=102 ec=70987 les/c 71437/71438 71591/71591/71591) [70,69,77] r=0 lpr=71591 pi=71432-71590/4 crt=71097'219 lcod 0'0 mlcod 0'0 inactive] requeue_ops 2014-12-08 18:14:00.308607 7f28fcb46700 10 osd.70 pg_epoch: 71591 pg[216.3c4( v 71097'219 (0'0,71097'219] local-les=71437 n=102 ec=70987 les/c 71437/71438 71591/71591/71591) [70,69,77] r=0 lpr=71591 pi=71432-71590/4 crt=71097'219 lcod 0'0 mlcod 0'0 inactive] on_role_change 2014-12-08 18:14:00.308615 7f28fcb46700 15 osd.70 pg_epoch: 71591 pg[216.3c4( v 71097'219 (0'0,71097'219] local-les=71437 n=102 ec=70987 les/c 71437/71438 71591/71591/71591) [70,69,77] r=0 lpr=71591 pi=71432-71590/4 crt=71097'219 lcod 0'0 mlcod 0'0 inactive] requeue_ops 2014-12-08 18:14:00.308627 7f2904b56700 10 osd.70 71591 do_waiters -- start 2014-12-08 18:14:00.308623 7f28fcb46700 10 osd.70 pg_epoch: 71591 pg[216.3c4( v 71097'219 (0'0,71097'219] local-les=71437 n=102 ec=70987 les/c 71437/71438 71591/71591/71591) [70,69,77] r=0 lpr=71591 pi=71432-71590/4 crt=71097'219 lcod 0'0 mlcod 0'0 inactive] cancel_recovery 2014-12-08 18:14:00.308630 7f2904b56700 10 osd.70 71591 do_waiters -- finish 2014-12-08 18:14:00.308631 7f28fcb46700 10 osd.70 pg_epoch: 71591 pg[216.3c4( v 71097'219 (0'0,71097'219] local-les=71437 n=102 ec=70987 les/c 71437/71438 71591/71591/71591) [70,69,77] r=0 lpr=71591 pi=71432-71590/4 crt=71097'219 lcod 0'0 mlcod 0'0 inactive] clear_recovery_state 2014-12-08 18:14:00.308640 7f28fcb46700 10 osd.70 pg_epoch: 71591 pg[216.3c4( v 71097'219 (0'0,71097'219] local-les=71437 n=102 ec=70987 les/c 71437/71438 71591/71591/71591) [70,69,77] r=0 lpr=71591 pi=71432-71590/4 crt=71097'219 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:14:00.308647 7f2904b56700 10 osd.70 71591 do_waiters -- start 2014-12-08 18:14:00.308649 7f2904b56700 10 osd.70 71591 do_waiters -- finish 2014-12-08 18:14:00.308651 7f2904b56700 20 osd.70 71591 _dispatch 0x12cd0960 pg_notify(216.25e(17) epoch 71591) v5 2014-12-08 18:14:00.308653 7f28fcb46700 10 osd.70 pg_epoch: 71591 pg[216.3c4( v 71097'219 (0'0,71097'219] local-les=71437 n=102 ec=70987 les/c 71437/71438 71591/71591/71591) [70,69,77] r=0 lpr=71591 pi=71432-71590/4 crt=71097'219 lcod 0'0 mlcod 0'0 inactive] handle_activate_map 2014-12-08 18:14:00.308663 7f28fcb46700 10 osd.70 pg_epoch: 71591 pg[216.3c4( v 71097'219 (0'0,71097'219] local-les=71437 n=102 ec=70987 les/c 71437/71438 71591/71591/71591) [70,69,77] r=0 lpr=71591 pi=71432-71590/4 crt=71097'219 lcod 0'0 mlcod 0'0 inactive] update_heartbeat_peers 69,70,77 unchanged -- 2014-12-08 18:14:00.328661 7f2904b56700 20 osd.70 71591 _dispatch 0x12ead840 pg_notify(19.6e(150) epoch 71591) v5 2014-12-08 18:14:00.328659 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[218.553( empty local-les=71435 n=0 ec=71423 les/c 71435/71435 71432/71432/71425) [17,70,63] r=1 lpr=71435 pi=71423-71431/4 crt=0'0 inactive NOTIFY] handle_advance_map [17,63]/[17,63] -- 17/17 2014-12-08 18:14:00.328683 7f2904b56700 7 osd.70 71591 handle_pg_notify from osd.16 2014-12-08 18:14:00.328669 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[218.5ad( empty local-les=71448 n=0 ec=71423 les/c 71448/71449 71591/71591/71444) [7,70,13]/[7,13,8] r=-1 lpr=71591 pi=71423-71590/10 crt=0'0 remapped NOTIFY] update_heartbeat_peers unchanged 2014-12-08 18:14:00.328685 7f2904b56700 15 osd.70 71591 require_same_or_newer_map 71591 (i am 71591) 0x12ead840 2014-12-08 18:14:00.328688 7f28fcb46700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:14:00.328686 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[218.5ad( empty local-les=71448 n=0 ec=71423 les/c 71448/71449 71591/71591/71444) [7,70,13]/[7,13,8] r=-1 lpr=71591 pi=71423-71590/10 crt=0'0 remapped NOTIFY] take_waiters 2014-12-08 18:14:00.328696 7f28fcb46700 10 osd.70 pg_epoch: 71582 pg[218.553( empty local-les=71435 n=0 ec=71423 les/c 71435/71435 71432/71432/71425) [17,70,63] r=1 lpr=71435 pi=71423-71431/4 crt=0'0 inactive NOTIFY] state: Started advmap 2014-12-08 18:14:00.328700 7f28fc345700 5 osd.70 pg_epoch: 71591 pg[218.5ad( empty local-les=71448 n=0 ec=71423 les/c 71448/71449 71591/71591/71444) [7,70,13]/[7,13,8] r=-1 lpr=71591 pi=71423-71590/10 crt=0'0 remapped NOTIFY] exit Reset 0.007551 10 0.007242 2014-12-08 18:14:00.328709 7f28fcb46700 20 osd.70 pg_epoch: 71582 pg[218.553( empty local-les=71435 n=0 ec=71423 les/c 71435/71435 71432/71432/71425) [17,70,63] r=1 lpr=71435 pi=71423-71431/4 crt=0'0 inactive NOTIFY] new interval newup [17,63] newacting [17,63] 2014-12-08 18:14:00.328718 7f28fc345700 5 osd.70 pg_epoch: 71591 pg[218.5ad( empty local-les=71448 n=0 ec=71423 les/c 71448/71449 71591/71591/71444) [7,70,13]/[7,13,8] r=-1 lpr=71591 pi=71423-71590/10 crt=0'0 remapped NOTIFY] enter Started 2014-12-08 18:14:00.328727 7f2904b56700 10 osd.70 71591 do_waiters -- start 2014-12-08 18:14:00.328729 7f2904b56700 10 osd.70 71591 do_waiters -- finish 2014-12-08 18:14:00.328722 7f28fcb46700 10 osd.70 pg_epoch: 71582 pg[218.553( empty local-les=71435 n=0 ec=71423 les/c 71435/71435 71432/71432/71425) [17,70,63] r=1 lpr=71435 pi=71423-71431/4 crt=0'0 inactive NOTIFY] state: should_restart_peering, transitioning to Reset 2014-12-08 18:14:00.328727 7f28fc345700 5 osd.70 pg_epoch: 71591 pg[218.5ad( empty local-les=71448 n=0 ec=71423 les/c 71448/71449 71591/71591/71444) [7,70,13]/[7,13,8] r=-1 lpr=71591 pi=71423-71590/10 crt=0'0 remapped NOTIFY] enter Start 2014-12-08 18:14:00.328739 7f28fcb46700 5 osd.70 pg_epoch: 71582 pg[218.553( empty local-les=71435 n=0 ec=71423 les/c 71435/71435 71432/71432/71425) [17,70,63] r=1 lpr=71435 pi=71423-71431/4 crt=0'0 inactive NOTIFY] exit Started/Stray 2.915548 3 0.000091 2014-12-08 18:14:00.328760 7f28fcb46700 5 osd.70 pg_epoch: 71582 pg[218.553( empty local-les=71435 n=0 ec=71423 les/c 71435/71435 71432/71432/71425) [17,70,63] r=1 lpr=71435 pi=71423-71431/4 crt=0'0 inactive NOTIFY] exit Started 2.915599 0 0.000000 2014-12-08 18:14:00.328773 7f28fcb46700 5 osd.70 pg_epoch: 71582 pg[218.553( empty local-les=71435 n=0 ec=71423 les/c 71435/71435 71432/71432/71425) [17,70,63] r=1 lpr=71435 pi=71423-71431/4 crt=0'0 inactive NOTIFY] enter Reset 2014-12-08 18:14:00.328745 7f28fc345700 1 osd.70 pg_epoch: 71591 pg[218.5ad( empty local-les=71448 n=0 ec=71423 les/c 71448/71449 71591/71591/71444) [7,70,13]/[7,13,8] r=-1 lpr=71591 pi=71423-71590/10 crt=0'0 remapped NOTIFY] state: transitioning to Stray 2014-12-08 18:14:00.328794 7f2904b56700 10 osd.70 71591 do_waiters -- start 2014-12-08 18:14:00.328798 7f2904b56700 10 osd.70 71591 do_waiters -- finish -- 2014-12-08 18:14:00.430519 7f28fc345700 10 osd.70 pg_epoch: 71581 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] handle_advance_map [16,13]/[16,13] -- 16/16 2014-12-08 18:14:00.430531 7f28fcb46700 10 osd.70 pg_epoch: 71582 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71582/71582/70961) [75] r=-1 lpr=71582 pi=71183-71581/14 crt=57981'40 lcod 0'0 inactive NOTIFY] cancel_recovery 2014-12-08 18:14:00.430542 7f28fc345700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:14:00.430540 7f28fcb46700 10 osd.70 pg_epoch: 71582 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71582/71582/70961) [75] r=-1 lpr=71582 pi=71183-71581/14 crt=57981'40 lcod 0'0 inactive NOTIFY] clear_recovery_state 2014-12-08 18:14:00.430548 7f28fc345700 10 osd.70 pg_epoch: 71582 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] state: Peering advmap 2014-12-08 18:14:00.430552 7f28fcb46700 10 osd.70 pg_epoch: 71582 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71582/71582/70961) [75] r=-1 lpr=71582 pi=71183-71581/14 crt=57981'40 lcod 0'0 inactive NOTIFY] check_recovery_sources no source osds () went down 2014-12-08 18:14:00.430561 7f28fc345700 10 osd.70 pg_epoch: 71582 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: affected_by_map osd.70 now down 2014-12-08 18:14:00.430568 7f28fc345700 1 osd.70 pg_epoch: 71582 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] state: Peering, affected_by_map, going to Reset 2014-12-08 18:14:00.430571 7f28fcb46700 10 osd.70 pg_epoch: 71582 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71582/71582/70961) [75] r=-1 lpr=71582 pi=71183-71581/14 crt=57981'40 lcod 0'0 inactive NOTIFY] handle_advance_map [75]/[75] -- 75/75 2014-12-08 18:14:00.430577 7f28fc345700 5 osd.70 pg_epoch: 71582 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] exit Started/Primary/Peering/GetInfo 2.653374 2 0.001416 2014-12-08 18:14:00.430586 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1,9~2,d~2] newly_removed_snaps [] snapc f=[] (no change) 2014-12-08 18:14:00.430588 7f28fc345700 10 osd.70 pg_epoch: 71582 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] state: Leaving Peering 2014-12-08 18:14:00.430590 7f28fcb46700 10 osd.70 pg_epoch: 71583 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71582/71582/70961) [75] r=-1 lpr=71582 pi=71183-71581/14 crt=57981'40 lcod 0'0 inactive NOTIFY] state: Reset advmap 2014-12-08 18:14:00.430595 7f28fc345700 5 osd.70 pg_epoch: 71582 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] exit Started/Primary/Peering 2.653401 0 0.000000 2014-12-08 18:14:00.430604 7f28fc345700 5 osd.70 pg_epoch: 71582 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] exit Started/Primary 2.653418 0 0.000000 2014-12-08 18:14:00.430601 7f28fcb46700 10 osd.70 pg_epoch: 71583 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71582/71582/70961) [75] r=-1 lpr=71582 pi=71183-71581/14 crt=57981'40 lcod 0'0 inactive NOTIFY] _calc_past_interval_range: already have past intervals back to 71434 2014-12-08 18:14:00.430612 7f28fc345700 5 osd.70 pg_epoch: 71582 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] exit Started 2.653463 0 0.000000 2014-12-08 18:14:00.430613 7f28fcb46700 10 osd.70 pg_epoch: 71583 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71582/71582/70961) [75] r=-1 lpr=71582 pi=71183-71581/14 crt=57981'40 lcod 0'0 inactive NOTIFY] check_recovery_sources no source osds () went down 2014-12-08 18:14:00.430619 7f28fc345700 5 osd.70 pg_epoch: 71582 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] enter Reset 2014-12-08 18:14:00.430626 7f28fc345700 20 osd.70 pg_epoch: 71582 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71456 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] set_last_peering_reset 71582 2014-12-08 18:14:00.430632 7f28fc345700 10 osd.70 pg_epoch: 71582 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71582 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] Clearing blocked outgoing recovery messages 2014-12-08 18:14:00.430624 7f290a83a700 10 osd.70 pg_epoch: 71591 pg[218.aa( v 71581'3 (0'0,71581'3] local-les=71446 n=3 ec=71423 les/c 71446/71446 71591/71591/71425) [10,13,70]/[10,13,56] r=-1 lpr=71591 pi=71423-71590/11 crt=71581'1 lcod 0'0 remapped NOTIFY] flushed 2014-12-08 18:14:00.430631 7f28fcb46700 10 osd.70 pg_epoch: 71583 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71582/71582/70961) [75] r=-1 lpr=71582 pi=71183-71581/14 crt=57981'40 lcod 0'0 inactive NOTIFY] handle_advance_map [75,25]/[75,25] -- 75/75 2014-12-08 18:14:00.430641 7f28fc345700 10 osd.70 pg_epoch: 71582 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71582 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] Not blocking outgoing recovery messages 2014-12-08 18:14:00.430646 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1,9~2,d~2] newly_removed_snaps [] snapc f=[] (no change) 2014-12-08 18:14:00.430647 7f28fc345700 10 osd.70 pg_epoch: 71582 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71582 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: Reset advmap 2014-12-08 18:14:00.430653 7f28fc345700 10 osd.70 pg_epoch: 71582 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71582 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:14:00.430650 7f28fcb46700 10 osd.70 pg_epoch: 71584 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71582/71582/70961) [75] r=-1 lpr=71582 pi=71183-71581/14 crt=57981'40 lcod 0'0 inactive NOTIFY] state: Reset advmap 2014-12-08 18:14:00.430660 7f28fc345700 20 osd.70 pg_epoch: 71582 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71582 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] new interval newup [16,13] newacting [16,13] 2014-12-08 18:14:00.430666 7f28fc345700 10 osd.70 pg_epoch: 71582 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71582 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: should restart peering, calling start_peering_interval again 2014-12-08 18:14:00.430661 7f28fcb46700 10 osd.70 pg_epoch: 71584 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71582/71582/70961) [75] r=-1 lpr=71582 pi=71183-71581/14 crt=57981'40 lcod 0'0 inactive NOTIFY] _calc_past_interval_range: already have past intervals back to 71434 2014-12-08 18:14:00.430672 7f28fc345700 20 osd.70 pg_epoch: 71582 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [70,16,13] r=0 lpr=71582 pi=50489-71455/146 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] set_last_peering_reset 71582 2014-12-08 18:14:00.430675 7f28fcb46700 20 osd.70 pg_epoch: 71584 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71582/71582/70961) [75] r=-1 lpr=71582 pi=71183-71581/14 crt=57981'40 lcod 0'0 inactive NOTIFY] new interval newup [75,25] newacting [75,25] 2014-12-08 18:14:00.430688 7f28fc345700 10 osd.70 pg_epoch: 71582 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [16,13] r=-1 lpr=71582 pi=50489-71581/147 crt=51071'48910301 lcod 0'0 inactive] start_peering_interval: check_new_interval output: generate_past_intervals interval(71456-71581 up [70,16,13](70) acting [70,16,13](70)): not rw, up_thru 71491 up_from 71432 last_epoch_clean 50495 generate_past_intervals interval(71456-71581 up [70,16,13](70) acting [70,16,13](70) maybe_went_rw) : primary up 71432-71491 includes interval 2014-12-08 18:14:00.430687 7f28fcb46700 10 osd.70 pg_epoch: 71584 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71582/71582/70961) [75] r=-1 lpr=71582 pi=71183-71581/14 crt=57981'40 lcod 0'0 inactive NOTIFY] state: should restart peering, calling start_peering_interval again 2014-12-08 18:14:00.430695 7f28fc345700 10 osd.70 pg_epoch: 71582 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71456/71456/71432) [16,13] r=-1 lpr=71582 pi=50489-71581/147 crt=51071'48910301 lcod 0'0 inactive] noting past interval(71456-71581 up [70,16,13](70) acting [70,16,13](70) maybe_went_rw) 2014-12-08 18:14:00.430702 7f28fc345700 10 osd.70 pg_epoch: 71582 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71582/71582/71582) [16,13] r=-1 lpr=71582 pi=50489-71581/147 crt=51071'48910301 lcod 0'0 inactive] up [70,16,13] -> [16,13], acting [70,16,13] -> [16,13], acting_primary 70 -> 16, up_primary 70 -> 16, role 0 -> -1 2014-12-08 18:14:00.430700 7f28fcb46700 20 osd.70 pg_epoch: 71584 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71582/71582/70961) [75] r=-1 lpr=71582 pi=71183-71581/14 crt=57981'40 lcod 0'0 inactive NOTIFY] set_last_peering_reset 71584 2014-12-08 18:14:00.430709 7f28fc345700 10 osd.70 pg_epoch: 71582 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71582/71582/71582) [16,13] r=-1 lpr=71582 pi=50489-71581/147 crt=51071'48910301 lcod 0'0 inactive] clear_primary_state 2014-12-08 18:14:00.430715 7f28fc345700 20 osd.70 pg_epoch: 71582 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71582/71582/71582) [16,13] r=-1 lpr=71582 pi=50489-71581/147 crt=51071'48910301 lcod 0'0 inactive] agent_stop 2014-12-08 18:14:00.430710 7f28fcb46700 10 osd.70 pg_epoch: 71584 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71582/71582/70961) [75] r=-1 lpr=71584 pi=71183-71581/14 crt=57981'40 lcod 0'0 inactive NOTIFY] Clearing blocked outgoing recovery messages 2014-12-08 18:14:00.430721 7f28fc345700 10 osd.70 pg_epoch: 71582 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71582/71582/71582) [16,13] r=-1 lpr=71582 pi=50489-71581/147 crt=51071'48910301 lcod 0'0 inactive] on_change 2014-12-08 18:14:00.430726 7f28fc345700 15 osd.70 pg_epoch: 71582 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71582/71582/71582) [16,13] r=-1 lpr=71582 pi=50489-71581/147 crt=51071'48910301 lcod 0'0 inactive] publish_stats_to_osd -- not primary 2014-12-08 18:14:00.430723 7f28fcb46700 10 osd.70 pg_epoch: 71584 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71582/71582/70961) [75] r=-1 lpr=71584 pi=71183-71581/14 crt=57981'40 lcod 0'0 inactive NOTIFY] Not blocking outgoing recovery messages 2014-12-08 18:14:00.430732 7f28fc345700 15 osd.70 pg_epoch: 71582 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71582/71582/71582) [16,13] r=-1 lpr=71582 pi=50489-71581/147 crt=51071'48910301 lcod 0'0 inactive] requeue_ops 2014-12-08 18:14:00.430738 7f28fc345700 10 osd.70 pg_epoch: 71582 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71582/71582/71582) [16,13] r=-1 lpr=71582 pi=50489-71581/147 crt=51071'48910301 lcod 0'0 inactive] cancel_copy_ops 2014-12-08 18:14:00.430744 7f28fc345700 10 osd.70 pg_epoch: 71582 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71582/71582/71582) [16,13] r=-1 lpr=71582 pi=50489-71581/147 crt=51071'48910301 lcod 0'0 inactive] cancel_flush_ops 2014-12-08 18:14:00.430750 7f28fc345700 10 osd.70 pg_epoch: 71582 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71582/71582/71582) [16,13] r=-1 lpr=71582 pi=50489-71581/147 crt=51071'48910301 lcod 0'0 inactive] on_change_cleanup 2014-12-08 18:14:00.430743 7f28fcb46700 10 osd.70 pg_epoch: 71584 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71582/71582/70961) [75,25] r=-1 lpr=71584 pi=71183-71583/15 crt=57981'40 lcod 0'0 inactive NOTIFY] start_peering_interval: check_new_interval output: generate_past_intervals interval(71582-71583 up [75](75) acting [75](75)): not rw, up_thru 71582 up_from 69683 last_epoch_clean 71434 generate_past_intervals interval(71582-71583 up [75](75) acting [75](75) maybe_went_rw) : primary up 69683-71582 includes interval 2014-12-08 18:14:00.430756 7f28fc345700 10 osd.70 pg_epoch: 71582 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71582/71582/71582) [16,13] r=-1 lpr=71582 pi=50489-71581/147 crt=51071'48910301 lcod 0'0 inactive] on_change 2014-12-08 18:14:00.430757 7f28fcb46700 10 osd.70 pg_epoch: 71584 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71582/71582/70961) [75,25] r=-1 lpr=71584 pi=71183-71583/15 crt=57981'40 lcod 0'0 inactive NOTIFY] noting past interval(71582-71583 up [75](75) acting [75](75) maybe_went_rw) 2014-12-08 18:14:00.430764 7f28fc345700 20 osd.70 pg_epoch: 71582 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71582/71582/71582) [16,13] r=-1 lpr=71582 pi=50489-71581/147 crt=51071'48910301 lcod 0'0 inactive] exit NotTrimming 2014-12-08 18:14:00.430771 7f28fc345700 20 osd.70 pg_epoch: 71582 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71582/71582/71582) [16,13] r=-1 lpr=71582 pi=50489-71581/147 crt=51071'48910301 lcod 0'0 inactive] enter NotTrimming 2014-12-08 18:14:00.430776 7f28fc345700 15 osd.70 pg_epoch: 71582 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71582/71582/71582) [16,13] r=-1 lpr=71582 pi=50489-71581/147 crt=51071'48910301 lcod 0'0 inactive NOTIFY] clear_stats 2014-12-08 18:14:00.430769 7f28fcb46700 10 osd.70 pg_epoch: 71584 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71584/71584/70961) [75,25] r=-1 lpr=71584 pi=71183-71583/15 crt=57981'40 lcod 0'0 inactive NOTIFY] up [75] -> [75,25], acting [75] -> [75,25], acting_primary 75 -> 75, up_primary 75 -> 75, role -1 -> -1 2014-12-08 18:14:00.430783 7f28fc345700 15 osd.70 pg_epoch: 71582 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71582/71582/71582) [16,13] r=-1 lpr=71582 pi=50489-71581/147 crt=51071'48910301 lcod 0'0 inactive NOTIFY] requeue_ops 2014-12-08 18:14:00.430783 7f28fcb46700 10 osd.70 pg_epoch: 71584 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71584/71584/70961) [75,25] r=-1 lpr=71584 pi=71183-71583/15 crt=57981'40 lcod 0'0 inactive NOTIFY] on_change 2014-12-08 18:14:00.430789 7f28fc345700 10 osd.70 pg_epoch: 71582 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71582/71582/71582) [16,13] r=-1 lpr=71582 pi=50489-71581/147 crt=51071'48910301 lcod 0'0 inactive NOTIFY] on_role_change 2014-12-08 18:14:00.430794 7f28fc345700 15 osd.70 pg_epoch: 71582 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71582/71582/71582) [16,13] r=-1 lpr=71582 pi=50489-71581/147 crt=51071'48910301 lcod 0'0 inactive NOTIFY] requeue_ops 2014-12-08 18:14:00.430794 7f28fcb46700 15 osd.70 pg_epoch: 71584 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71584/71584/70961) [75,25] r=-1 lpr=71584 pi=71183-71583/15 crt=57981'40 lcod 0'0 inactive NOTIFY] publish_stats_to_osd -- not primary 2014-12-08 18:14:00.430800 7f28fc345700 10 osd.70 pg_epoch: 71582 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71582/71582/71582) [16,13] r=-1 lpr=71582 pi=50489-71581/147 crt=51071'48910301 lcod 0'0 inactive NOTIFY] cancel_recovery 2014-12-08 18:14:00.430806 7f28fc345700 10 osd.70 pg_epoch: 71582 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71582/71582/71582) [16,13] r=-1 lpr=71582 pi=50489-71581/147 crt=51071'48910301 lcod 0'0 inactive NOTIFY] clear_recovery_state 2014-12-08 18:14:00.430806 7f28fcb46700 15 osd.70 pg_epoch: 71584 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71584/71584/70961) [75,25] r=-1 lpr=71584 pi=71183-71583/15 crt=57981'40 lcod 0'0 inactive NOTIFY] requeue_ops 2014-12-08 18:14:00.430813 7f28fc345700 10 osd.70 pg_epoch: 71582 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71582/71582/71582) [16,13] r=-1 lpr=71582 pi=50489-71581/147 crt=51071'48910301 lcod 0'0 inactive NOTIFY] check_recovery_sources no source osds () went down 2014-12-08 18:14:00.430818 7f28fcb46700 10 osd.70 pg_epoch: 71584 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71584/71584/70961) [75,25] r=-1 lpr=71584 pi=71183-71583/15 crt=57981'40 lcod 0'0 inactive NOTIFY] cancel_copy_ops 2014-12-08 18:14:00.430826 7f28fc345700 10 osd.70 pg_epoch: 71582 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71582/71582/71582) [16,13] r=-1 lpr=71582 pi=50489-71581/147 crt=51071'48910301 lcod 0'0 inactive NOTIFY] handle_advance_map [16,13]/[16,13] -- 16/16 2014-12-08 18:14:00.430834 7f28fc345700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:14:00.430828 7f28fcb46700 10 osd.70 pg_epoch: 71584 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71584/71584/70961) [75,25] r=-1 lpr=71584 pi=71183-71583/15 crt=57981'40 lcod 0'0 inactive NOTIFY] cancel_flush_ops 2014-12-08 18:14:00.430836 7f28fc345700 10 osd.70 pg_epoch: 71583 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71582/71582/71582) [16,13] r=-1 lpr=71582 pi=50489-71581/147 crt=51071'48910301 lcod 0'0 inactive NOTIFY] state: Reset advmap 2014-12-08 18:14:00.430842 7f28fc345700 10 osd.70 pg_epoch: 71583 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71582/71582/71582) [16,13] r=-1 lpr=71582 pi=50489-71581/147 crt=51071'48910301 lcod 0'0 inactive NOTIFY] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:14:00.430839 7f28fcb46700 10 osd.70 pg_epoch: 71584 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71584/71584/70961) [75,25] r=-1 lpr=71584 pi=71183-71583/15 crt=57981'40 lcod 0'0 inactive NOTIFY] on_change_cleanup 2014-12-08 18:14:00.430849 7f28fc345700 10 osd.70 pg_epoch: 71583 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71582/71582/71582) [16,13] r=-1 lpr=71582 pi=50489-71581/147 crt=51071'48910301 lcod 0'0 inactive NOTIFY] check_recovery_sources no source osds () went down 2014-12-08 18:14:00.430849 7f28fcb46700 10 osd.70 pg_epoch: 71584 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71584/71584/70961) [75,25] r=-1 lpr=71584 pi=71183-71583/15 crt=57981'40 lcod 0'0 inactive NOTIFY] on_change 2014-12-08 18:14:00.430859 7f28fc345700 10 osd.70 pg_epoch: 71583 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71582/71582/71582) [16,13] r=-1 lpr=71582 pi=50489-71581/147 crt=51071'48910301 lcod 0'0 inactive NOTIFY] handle_advance_map [76,16,13]/[76,16,13] -- 76/76 2014-12-08 18:14:00.430867 7f28fc345700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:14:00.430861 7f28fcb46700 20 osd.70 pg_epoch: 71584 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71584/71584/70961) [75,25] r=-1 lpr=71584 pi=71183-71583/15 crt=57981'40 lcod 0'0 inactive NOTIFY] exit NotTrimming 2014-12-08 18:14:00.430869 7f28fc345700 10 osd.70 pg_epoch: 71584 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71582/71582/71582) [16,13] r=-1 lpr=71582 pi=50489-71581/147 crt=51071'48910301 lcod 0'0 inactive NOTIFY] state: Reset advmap 2014-12-08 18:14:00.430875 7f28fc345700 10 osd.70 pg_epoch: 71584 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71582/71582/71582) [16,13] r=-1 lpr=71582 pi=50489-71581/147 crt=51071'48910301 lcod 0'0 inactive NOTIFY] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:14:00.430872 7f28fcb46700 20 osd.70 pg_epoch: 71584 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71584/71584/70961) [75,25] r=-1 lpr=71584 pi=71183-71583/15 crt=57981'40 lcod 0'0 inactive NOTIFY] enter NotTrimming 2014-12-08 18:14:00.430881 7f28fc345700 20 osd.70 pg_epoch: 71584 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71582/71582/71582) [16,13] r=-1 lpr=71582 pi=50489-71581/147 crt=51071'48910301 lcod 0'0 inactive NOTIFY] new interval newup [76,16,13] newacting [76,16,13] 2014-12-08 18:14:00.430887 7f28fc345700 10 osd.70 pg_epoch: 71584 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71582/71582/71582) [16,13] r=-1 lpr=71582 pi=50489-71581/147 crt=51071'48910301 lcod 0'0 inactive NOTIFY] state: should restart peering, calling start_peering_interval again 2014-12-08 18:14:00.430883 7f28fcb46700 10 osd.70 pg_epoch: 71584 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71584/71584/70961) [75,25] r=-1 lpr=71584 pi=71183-71583/15 crt=57981'40 lcod 0'0 inactive NOTIFY] cancel_recovery 2014-12-08 18:14:00.430893 7f28fc345700 20 osd.70 pg_epoch: 71584 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71582/71582/71582) [16,13] r=-1 lpr=71582 pi=50489-71581/147 crt=51071'48910301 lcod 0'0 inactive NOTIFY] set_last_peering_reset 71584 2014-12-08 18:14:00.430894 7f28fcb46700 10 osd.70 pg_epoch: 71584 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71584/71584/70961) [75,25] r=-1 lpr=71584 pi=71183-71583/15 crt=57981'40 lcod 0'0 inactive NOTIFY] clear_recovery_state 2014-12-08 18:14:00.430899 7f28fc345700 10 osd.70 pg_epoch: 71584 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71582/71582/71582) [16,13] r=-1 lpr=71584 pi=50489-71581/147 crt=51071'48910301 lcod 0'0 inactive NOTIFY] Clearing blocked outgoing recovery messages 2014-12-08 18:14:00.430905 7f28fc345700 10 osd.70 pg_epoch: 71584 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71582/71582/71582) [16,13] r=-1 lpr=71584 pi=50489-71581/147 crt=51071'48910301 lcod 0'0 inactive NOTIFY] Not blocking outgoing recovery messages 2014-12-08 18:14:00.430906 7f28fcb46700 10 osd.70 pg_epoch: 71584 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71584/71584/70961) [75,25] r=-1 lpr=71584 pi=71183-71583/15 crt=57981'40 lcod 0'0 inactive NOTIFY] check_recovery_sources no source osds () went down 2014-12-08 18:14:00.430916 7f28fc345700 10 osd.70 pg_epoch: 71584 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71582/71582/71582) [76,16,13] r=-1 lpr=71584 pi=50489-71583/148 crt=51071'48910301 lcod 0'0 inactive NOTIFY] start_peering_interval: check_new_interval output: generate_past_intervals interval(71582-71583 up [16,13](16) acting [16,13](16)): not rw, up_thru 71582 up_from 69690 last_epoch_clean 50495 generate_past_intervals interval(71582-71583 up [16,13](16) acting [16,13](16) maybe_went_rw) : primary up 69690-71582 includes interval 2014-12-08 18:14:00.430922 7f28fc345700 10 osd.70 pg_epoch: 71584 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71582/71582/71582) [76,16,13] r=-1 lpr=71584 pi=50489-71583/148 crt=51071'48910301 lcod 0'0 inactive NOTIFY] noting past interval(71582-71583 up [16,13](16) acting [16,13](16) maybe_went_rw) 2014-12-08 18:14:00.430928 7f28fc345700 10 osd.70 pg_epoch: 71584 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71584/71584/71584) [76,16,13] r=-1 lpr=71584 pi=50489-71583/148 crt=51071'48910301 lcod 0'0 inactive NOTIFY] up [16,13] -> [76,16,13], acting [16,13] -> [76,16,13], acting_primary 16 -> 76, up_primary 16 -> 76, role -1 -> -1 2014-12-08 18:14:00.430926 7f28fcb46700 10 osd.70 pg_epoch: 71584 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71584/71584/70961) [75,25] r=-1 lpr=71584 pi=71183-71583/15 crt=57981'40 lcod 0'0 inactive NOTIFY] handle_advance_map [75,25]/[75,25] -- 75/75 2014-12-08 18:14:00.430935 7f28fc345700 10 osd.70 pg_epoch: 71584 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71584/71584/71584) [76,16,13] r=-1 lpr=71584 pi=50489-71583/148 crt=51071'48910301 lcod 0'0 inactive NOTIFY] on_change 2014-12-08 18:14:00.430941 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1,9~2,d~2] newly_removed_snaps [] snapc f=[] (no change) 2014-12-08 18:14:00.430941 7f28fc345700 15 osd.70 pg_epoch: 71584 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71584/71584/71584) [76,16,13] r=-1 lpr=71584 pi=50489-71583/148 crt=51071'48910301 lcod 0'0 inactive NOTIFY] publish_stats_to_osd -- not primary 2014-12-08 18:14:00.430947 7f28fc345700 15 osd.70 pg_epoch: 71584 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71584/71584/71584) [76,16,13] r=-1 lpr=71584 pi=50489-71583/148 crt=51071'48910301 lcod 0'0 inactive NOTIFY] requeue_ops 2014-12-08 18:14:00.430946 7f28fcb46700 10 osd.70 pg_epoch: 71585 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71584/71584/70961) [75,25] r=-1 lpr=71584 pi=71183-71583/15 crt=57981'40 lcod 0'0 inactive NOTIFY] state: Reset advmap 2014-12-08 18:14:00.430953 7f28fc345700 10 osd.70 pg_epoch: 71584 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71584/71584/71584) [76,16,13] r=-1 lpr=71584 pi=50489-71583/148 crt=51071'48910301 lcod 0'0 inactive NOTIFY] cancel_copy_ops 2014-12-08 18:14:00.430960 7f28fc345700 10 osd.70 pg_epoch: 71584 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71584/71584/71584) [76,16,13] r=-1 lpr=71584 pi=50489-71583/148 crt=51071'48910301 lcod 0'0 inactive NOTIFY] cancel_flush_ops 2014-12-08 18:14:00.430957 7f28fcb46700 10 osd.70 pg_epoch: 71585 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71584/71584/70961) [75,25] r=-1 lpr=71584 pi=71183-71583/15 crt=57981'40 lcod 0'0 inactive NOTIFY] _calc_past_interval_range: already have past intervals back to 71434 2014-12-08 18:14:00.430965 7f28fc345700 10 osd.70 pg_epoch: 71584 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71584/71584/71584) [76,16,13] r=-1 lpr=71584 pi=50489-71583/148 crt=51071'48910301 lcod 0'0 inactive NOTIFY] on_change_cleanup 2014-12-08 18:14:00.430971 7f28fc345700 10 osd.70 pg_epoch: 71584 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71584/71584/71584) [76,16,13] r=-1 lpr=71584 pi=50489-71583/148 crt=51071'48910301 lcod 0'0 inactive NOTIFY] on_change 2014-12-08 18:14:00.430969 7f28fcb46700 10 osd.70 pg_epoch: 71585 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71584/71584/70961) [75,25] r=-1 lpr=71584 pi=71183-71583/15 crt=57981'40 lcod 0'0 inactive NOTIFY] check_recovery_sources no source osds () went down 2014-12-08 18:14:00.430977 7f28fc345700 20 osd.70 pg_epoch: 71584 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71584/71584/71584) [76,16,13] r=-1 lpr=71584 pi=50489-71583/148 crt=51071'48910301 lcod 0'0 inactive NOTIFY] exit NotTrimming 2014-12-08 18:14:00.430983 7f28fc345700 20 osd.70 pg_epoch: 71584 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71584/71584/71584) [76,16,13] r=-1 lpr=71584 pi=50489-71583/148 crt=51071'48910301 lcod 0'0 inactive NOTIFY] enter NotTrimming 2014-12-08 18:14:00.430986 7f28fcb46700 10 osd.70 pg_epoch: 71585 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71584/71584/70961) [75,25] r=-1 lpr=71584 pi=71183-71583/15 crt=57981'40 lcod 0'0 inactive NOTIFY] handle_advance_map [75,25]/[75,25] -- 75/75 2014-12-08 18:14:00.431001 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1,9~2,d~2] newly_removed_snaps [] snapc f=[] (no change) 2014-12-08 18:14:00.430992 7f28fc345700 10 osd.70 pg_epoch: 71584 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71584/71584/71584) [76,16,13] r=-1 lpr=71584 pi=50489-71583/148 crt=51071'48910301 lcod 0'0 inactive NOTIFY] pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71584/71584/71584) [76,16,13] r=-1 lpr=71584 pi=50489-71583/148 crt=51071'48910301 lcod 0'0 inactive NOTIFY] [16,13] -> [76,16,13], acting primary 16 -> 76 2014-12-08 18:14:00.431009 7f28fc345700 10 osd.70 pg_epoch: 71584 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71584/71584/71584) [76,16,13] r=-1 lpr=71584 pi=50489-71583/148 crt=51071'48910301 lcod 0'0 inactive NOTIFY] cancel_recovery 2014-12-08 18:14:00.431005 7f28fcb46700 10 osd.70 pg_epoch: 71586 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71584/71584/70961) [75,25] r=-1 lpr=71584 pi=71183-71583/15 crt=57981'40 lcod 0'0 inactive NOTIFY] state: Reset advmap 2014-12-08 18:14:00.431014 7f28fc345700 10 osd.70 pg_epoch: 71584 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71584/71584/71584) [76,16,13] r=-1 lpr=71584 pi=50489-71583/148 crt=51071'48910301 lcod 0'0 inactive NOTIFY] clear_recovery_state 2014-12-08 18:14:00.431021 7f28fc345700 10 osd.70 pg_epoch: 71584 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71584/71584/71584) [76,16,13] r=-1 lpr=71584 pi=50489-71583/148 crt=51071'48910301 lcod 0'0 inactive NOTIFY] check_recovery_sources no source osds () went down 2014-12-08 18:14:00.431017 7f28fcb46700 10 osd.70 pg_epoch: 71586 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71584/71584/70961) [75,25] r=-1 lpr=71584 pi=71183-71583/15 crt=57981'40 lcod 0'0 inactive NOTIFY] _calc_past_interval_range: already have past intervals back to 71434 2014-12-08 18:14:00.431037 7f28fc345700 10 osd.70 pg_epoch: 71584 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71584/71584/71584) [76,16,13] r=-1 lpr=71584 pi=50489-71583/148 crt=51071'48910301 lcod 0'0 inactive NOTIFY] handle_advance_map [76,16,13]/[76,16,13] -- 76/76 2014-12-08 18:14:00.431033 7f28fcb46700 10 osd.70 pg_epoch: 71586 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71584/71584/70961) [75,25] r=-1 lpr=71584 pi=71183-71583/15 crt=57981'40 lcod 0'0 inactive NOTIFY] check_recovery_sources no source osds () went down 2014-12-08 18:14:00.431045 7f28fc345700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:14:00.431047 7f28fc345700 10 osd.70 pg_epoch: 71585 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71584/71584/71584) [76,16,13] r=-1 lpr=71584 pi=50489-71583/148 crt=51071'48910301 lcod 0'0 inactive NOTIFY] state: Reset advmap 2014-12-08 18:14:00.431053 7f28fc345700 10 osd.70 pg_epoch: 71585 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71584/71584/71584) [76,16,13] r=-1 lpr=71584 pi=50489-71583/148 crt=51071'48910301 lcod 0'0 inactive NOTIFY] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:14:00.431059 7f28fc345700 10 osd.70 pg_epoch: 71585 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71584/71584/71584) [76,16,13] r=-1 lpr=71584 pi=50489-71583/148 crt=51071'48910301 lcod 0'0 inactive NOTIFY] check_recovery_sources no source osds () went down 2014-12-08 18:14:00.431053 7f28fcb46700 10 osd.70 pg_epoch: 71586 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71584/71584/70961) [75,25] r=-1 lpr=71584 pi=71183-71583/15 crt=57981'40 lcod 0'0 inactive NOTIFY] handle_advance_map [75]/[75] -- 75/75 2014-12-08 18:14:00.431067 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1,9~2,d~2] newly_removed_snaps [] snapc f=[] (no change) 2014-12-08 18:14:00.431069 7f28fc345700 10 osd.70 pg_epoch: 71585 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71584/71584/71584) [76,16,13] r=-1 lpr=71584 pi=50489-71583/148 crt=51071'48910301 lcod 0'0 inactive NOTIFY] handle_advance_map [76,16,13]/[76,16,13] -- 76/76 2014-12-08 18:14:00.431077 7f28fc345700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:14:00.431071 7f28fcb46700 10 osd.70 pg_epoch: 71587 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71584/71584/70961) [75,25] r=-1 lpr=71584 pi=71183-71583/15 crt=57981'40 lcod 0'0 inactive NOTIFY] state: Reset advmap 2014-12-08 18:14:00.431078 7f28fc345700 10 osd.70 pg_epoch: 71586 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71584/71584/71584) [76,16,13] r=-1 lpr=71584 pi=50489-71583/148 crt=51071'48910301 lcod 0'0 inactive NOTIFY] state: Reset advmap 2014-12-08 18:14:00.431085 7f28fc345700 10 osd.70 pg_epoch: 71586 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71584/71584/71584) [76,16,13] r=-1 lpr=71584 pi=50489-71583/148 crt=51071'48910301 lcod 0'0 inactive NOTIFY] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:14:00.431082 7f28fcb46700 10 osd.70 pg_epoch: 71587 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71584/71584/70961) [75,25] r=-1 lpr=71584 pi=71183-71583/15 crt=57981'40 lcod 0'0 inactive NOTIFY] _calc_past_interval_range: already have past intervals back to 71434 2014-12-08 18:14:00.431091 7f28fc345700 10 osd.70 pg_epoch: 71586 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71584/71584/71584) [76,16,13] r=-1 lpr=71584 pi=50489-71583/148 crt=51071'48910301 lcod 0'0 inactive NOTIFY] check_recovery_sources no source osds () went down 2014-12-08 18:14:00.431094 7f28fcb46700 20 osd.70 pg_epoch: 71587 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71584/71584/70961) [75,25] r=-1 lpr=71584 pi=71183-71583/15 crt=57981'40 lcod 0'0 inactive NOTIFY] new interval newup [75] newacting [75] 2014-12-08 18:14:00.431103 7f28fc345700 10 osd.70 pg_epoch: 71586 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71584/71584/71584) [76,16,13] r=-1 lpr=71584 pi=50489-71583/148 crt=51071'48910301 lcod 0'0 inactive NOTIFY] handle_advance_map [16,13]/[16,13] -- 16/16 2014-12-08 18:14:00.431111 7f28fc345700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:14:00.431097 7f290a039700 10 osd.70 pg_epoch: 71591 pg[216.392( v 71093'208 (0'0,71093'208] local-les=71438 n=104 ec=70987 les/c 71438/71438 71591/71591/70989) [69,7,70] r=2 lpr=71591 pi=71183-71590/17 crt=71093'208 lcod 0'0 inactive NOTIFY] flushed 2014-12-08 18:14:00.431106 7f28fcb46700 10 osd.70 pg_epoch: 71587 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71584/71584/70961) [75,25] r=-1 lpr=71584 pi=71183-71583/15 crt=57981'40 lcod 0'0 inactive NOTIFY] state: should restart peering, calling start_peering_interval again 2014-12-08 18:14:00.431112 7f28fc345700 10 osd.70 pg_epoch: 71587 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71584/71584/71584) [76,16,13] r=-1 lpr=71584 pi=50489-71583/148 crt=51071'48910301 lcod 0'0 inactive NOTIFY] state: Reset advmap 2014-12-08 18:14:00.431119 7f28fc345700 10 osd.70 pg_epoch: 71587 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71584/71584/71584) [76,16,13] r=-1 lpr=71584 pi=50489-71583/148 crt=51071'48910301 lcod 0'0 inactive NOTIFY] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:14:00.431118 7f28fcb46700 20 osd.70 pg_epoch: 71587 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71584/71584/70961) [75,25] r=-1 lpr=71584 pi=71183-71583/15 crt=57981'40 lcod 0'0 inactive NOTIFY] set_last_peering_reset 71587 2014-12-08 18:14:00.431125 7f28fc345700 20 osd.70 pg_epoch: 71587 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71584/71584/71584) [76,16,13] r=-1 lpr=71584 pi=50489-71583/148 crt=51071'48910301 lcod 0'0 inactive NOTIFY] new interval newup [16,13] newacting [16,13] 2014-12-08 18:14:00.431131 7f28fc345700 10 osd.70 pg_epoch: 71587 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71584/71584/71584) [76,16,13] r=-1 lpr=71584 pi=50489-71583/148 crt=51071'48910301 lcod 0'0 inactive NOTIFY] state: should restart peering, calling start_peering_interval again 2014-12-08 18:14:00.431129 7f28fcb46700 10 osd.70 pg_epoch: 71587 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71584/71584/70961) [75,25] r=-1 lpr=71587 pi=71183-71583/15 crt=57981'40 lcod 0'0 inactive NOTIFY] Clearing blocked outgoing recovery messages 2014-12-08 18:14:00.431137 7f28fc345700 20 osd.70 pg_epoch: 71587 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71584/71584/71584) [76,16,13] r=-1 lpr=71584 pi=50489-71583/148 crt=51071'48910301 lcod 0'0 inactive NOTIFY] set_last_peering_reset 71587 2014-12-08 18:14:00.431141 7f28fcb46700 10 osd.70 pg_epoch: 71587 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71584/71584/70961) [75,25] r=-1 lpr=71587 pi=71183-71583/15 crt=57981'40 lcod 0'0 inactive NOTIFY] Not blocking outgoing recovery messages 2014-12-08 18:14:00.431147 7f28fc345700 10 osd.70 pg_epoch: 71587 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71584/71584/71584) [76,16,13] r=-1 lpr=71587 pi=50489-71583/148 crt=51071'48910301 lcod 0'0 inactive NOTIFY] Clearing blocked outgoing recovery messages 2014-12-08 18:14:00.431158 7f28fc345700 10 osd.70 pg_epoch: 71587 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71584/71584/71584) [76,16,13] r=-1 lpr=71587 pi=50489-71583/148 crt=51071'48910301 lcod 0'0 inactive NOTIFY] Not blocking outgoing recovery messages 2014-12-08 18:14:00.431161 7f28fcb46700 10 osd.70 pg_epoch: 71587 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71584/71584/70961) [75] r=-1 lpr=71587 pi=71183-71586/16 crt=57981'40 lcod 0'0 inactive NOTIFY] start_peering_interval: check_new_interval output: generate_past_intervals interval(71584-71586 up [75,25](75) acting [75,25](75)): not rw, up_thru 71585 up_from 69683 last_epoch_clean 71434 generate_past_intervals interval(71584-71586 up [75,25](75) acting [75,25](75) maybe_went_rw) : primary up 69683-71585 includes interval 2014-12-08 18:14:00.431174 7f28fcb46700 10 osd.70 pg_epoch: 71587 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71584/71584/70961) [75] r=-1 lpr=71587 pi=71183-71586/16 crt=57981'40 lcod 0'0 inactive NOTIFY] noting past interval(71584-71586 up [75,25](75) acting [75,25](75) maybe_went_rw) 2014-12-08 18:14:00.431177 7f28fc345700 10 osd.70 pg_epoch: 71587 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71584/71584/71584) [16,13] r=-1 lpr=71587 pi=50489-71586/149 crt=51071'48910301 lcod 0'0 inactive NOTIFY] start_peering_interval: check_new_interval output: generate_past_intervals interval(71584-71586 up [76,16,13](76) acting [76,16,13](76)): not rw, up_thru 71585 up_from 69673 last_epoch_clean 50495 generate_past_intervals interval(71584-71586 up [76,16,13](76) acting [76,16,13](76) maybe_went_rw) : primary up 69673-71585 includes interval 2014-12-08 18:14:00.431188 7f28fc345700 10 osd.70 pg_epoch: 71587 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71584/71584/71584) [16,13] r=-1 lpr=71587 pi=50489-71586/149 crt=51071'48910301 lcod 0'0 inactive NOTIFY] noting past interval(71584-71586 up [76,16,13](76) acting [76,16,13](76) maybe_went_rw) 2014-12-08 18:14:00.431186 7f28fcb46700 10 osd.70 pg_epoch: 71587 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71587/71587/70961) [75] r=-1 lpr=71587 pi=71183-71586/16 crt=57981'40 lcod 0'0 inactive NOTIFY] up [75,25] -> [75], acting [75,25] -> [75], acting_primary 75 -> 75, up_primary 75 -> 75, role -1 -> -1 2014-12-08 18:14:00.431196 7f28fc345700 10 osd.70 pg_epoch: 71587 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71587/71587/71587) [16,13] r=-1 lpr=71587 pi=50489-71586/149 crt=51071'48910301 lcod 0'0 inactive NOTIFY] up [76,16,13] -> [16,13], acting [76,16,13] -> [16,13], acting_primary 76 -> 16, up_primary 76 -> 16, role -1 -> -1 2014-12-08 18:14:00.431199 7f28fcb46700 10 osd.70 pg_epoch: 71587 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71587/71587/70961) [75] r=-1 lpr=71587 pi=71183-71586/16 crt=57981'40 lcod 0'0 inactive NOTIFY] on_change 2014-12-08 18:14:00.431204 7f28fc345700 10 osd.70 pg_epoch: 71587 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71587/71587/71587) [16,13] r=-1 lpr=71587 pi=50489-71586/149 crt=51071'48910301 lcod 0'0 inactive NOTIFY] on_change 2014-12-08 18:14:00.431210 7f28fc345700 15 osd.70 pg_epoch: 71587 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71587/71587/71587) [16,13] r=-1 lpr=71587 pi=50489-71586/149 crt=51071'48910301 lcod 0'0 inactive NOTIFY] publish_stats_to_osd -- not primary 2014-12-08 18:14:00.431210 7f28fcb46700 15 osd.70 pg_epoch: 71587 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71587/71587/70961) [75] r=-1 lpr=71587 pi=71183-71586/16 crt=57981'40 lcod 0'0 inactive NOTIFY] publish_stats_to_osd -- not primary 2014-12-08 18:14:00.431216 7f28fc345700 15 osd.70 pg_epoch: 71587 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71587/71587/71587) [16,13] r=-1 lpr=71587 pi=50489-71586/149 crt=51071'48910301 lcod 0'0 inactive NOTIFY] requeue_ops 2014-12-08 18:14:00.431221 7f28fcb46700 15 osd.70 pg_epoch: 71587 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71587/71587/70961) [75] r=-1 lpr=71587 pi=71183-71586/16 crt=57981'40 lcod 0'0 inactive NOTIFY] requeue_ops 2014-12-08 18:14:00.431223 7f28fc345700 10 osd.70 pg_epoch: 71587 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71587/71587/71587) [16,13] r=-1 lpr=71587 pi=50489-71586/149 crt=51071'48910301 lcod 0'0 inactive NOTIFY] cancel_copy_ops 2014-12-08 18:14:00.431233 7f28fc345700 10 osd.70 pg_epoch: 71587 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71587/71587/71587) [16,13] r=-1 lpr=71587 pi=50489-71586/149 crt=51071'48910301 lcod 0'0 inactive NOTIFY] cancel_flush_ops 2014-12-08 18:14:00.431232 7f28fcb46700 10 osd.70 pg_epoch: 71587 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71587/71587/70961) [75] r=-1 lpr=71587 pi=71183-71586/16 crt=57981'40 lcod 0'0 inactive NOTIFY] cancel_copy_ops 2014-12-08 18:14:00.431239 7f28fc345700 10 osd.70 pg_epoch: 71587 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71587/71587/71587) [16,13] r=-1 lpr=71587 pi=50489-71586/149 crt=51071'48910301 lcod 0'0 inactive NOTIFY] on_change_cleanup 2014-12-08 18:14:00.431246 7f28fc345700 10 osd.70 pg_epoch: 71587 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71587/71587/71587) [16,13] r=-1 lpr=71587 pi=50489-71586/149 crt=51071'48910301 lcod 0'0 inactive NOTIFY] on_change 2014-12-08 18:14:00.431243 7f28fcb46700 10 osd.70 pg_epoch: 71587 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71587/71587/70961) [75] r=-1 lpr=71587 pi=71183-71586/16 crt=57981'40 lcod 0'0 inactive NOTIFY] cancel_flush_ops 2014-12-08 18:14:00.431252 7f28fc345700 20 osd.70 pg_epoch: 71587 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71587/71587/71587) [16,13] r=-1 lpr=71587 pi=50489-71586/149 crt=51071'48910301 lcod 0'0 inactive NOTIFY] exit NotTrimming 2014-12-08 18:14:00.431254 7f28fcb46700 10 osd.70 pg_epoch: 71587 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71587/71587/70961) [75] r=-1 lpr=71587 pi=71183-71586/16 crt=57981'40 lcod 0'0 inactive NOTIFY] on_change_cleanup 2014-12-08 18:14:00.431258 7f28fc345700 20 osd.70 pg_epoch: 71587 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71587/71587/71587) [16,13] r=-1 lpr=71587 pi=50489-71586/149 crt=51071'48910301 lcod 0'0 inactive NOTIFY] enter NotTrimming 2014-12-08 18:14:00.431267 7f28fc345700 10 osd.70 pg_epoch: 71587 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71587/71587/71587) [16,13] r=-1 lpr=71587 pi=50489-71586/149 crt=51071'48910301 lcod 0'0 inactive NOTIFY] pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71587/71587/71587) [16,13] r=-1 lpr=71587 pi=50489-71586/149 crt=51071'48910301 lcod 0'0 inactive NOTIFY] [76,16,13] -> [16,13], acting primary 76 -> 16 2014-12-08 18:14:00.431266 7f28fcb46700 10 osd.70 pg_epoch: 71587 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71587/71587/70961) [75] r=-1 lpr=71587 pi=71183-71586/16 crt=57981'40 lcod 0'0 inactive NOTIFY] on_change 2014-12-08 18:14:00.431277 7f28fc345700 10 osd.70 pg_epoch: 71587 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71587/71587/71587) [16,13] r=-1 lpr=71587 pi=50489-71586/149 crt=51071'48910301 lcod 0'0 inactive NOTIFY] cancel_recovery 2014-12-08 18:14:00.431283 7f28fc345700 10 osd.70 pg_epoch: 71587 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71587/71587/71587) [16,13] r=-1 lpr=71587 pi=50489-71586/149 crt=51071'48910301 lcod 0'0 inactive NOTIFY] clear_recovery_state 2014-12-08 18:14:00.431279 7f28fcb46700 20 osd.70 pg_epoch: 71587 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71587/71587/70961) [75] r=-1 lpr=71587 pi=71183-71586/16 crt=57981'40 lcod 0'0 inactive NOTIFY] exit NotTrimming 2014-12-08 18:14:00.431289 7f28fc345700 10 osd.70 pg_epoch: 71587 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71587/71587/71587) [16,13] r=-1 lpr=71587 pi=50489-71586/149 crt=51071'48910301 lcod 0'0 inactive NOTIFY] check_recovery_sources no source osds () went down 2014-12-08 18:14:00.431290 7f28fcb46700 20 osd.70 pg_epoch: 71587 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71587/71587/70961) [75] r=-1 lpr=71587 pi=71183-71586/16 crt=57981'40 lcod 0'0 inactive NOTIFY] enter NotTrimming 2014-12-08 18:14:00.431303 7f28fc345700 10 osd.70 pg_epoch: 71587 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71587/71587/71587) [16,13] r=-1 lpr=71587 pi=50489-71586/149 crt=51071'48910301 lcod 0'0 inactive NOTIFY] handle_advance_map [16,13]/[16,13] -- 16/16 2014-12-08 18:14:00.431300 7f28fcb46700 10 osd.70 pg_epoch: 71587 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71587/71587/70961) [75] r=-1 lpr=71587 pi=71183-71586/16 crt=57981'40 lcod 0'0 inactive NOTIFY] cancel_recovery 2014-12-08 18:14:00.431310 7f28fc345700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:14:00.431311 7f28fcb46700 10 osd.70 pg_epoch: 71587 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71587/71587/70961) [75] r=-1 lpr=71587 pi=71183-71586/16 crt=57981'40 lcod 0'0 inactive NOTIFY] clear_recovery_state 2014-12-08 18:14:00.431313 7f28fc345700 10 osd.70 pg_epoch: 71588 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71587/71587/71587) [16,13] r=-1 lpr=71587 pi=50489-71586/149 crt=51071'48910301 lcod 0'0 inactive NOTIFY] state: Reset advmap 2014-12-08 18:14:00.431323 7f28fc345700 10 osd.70 pg_epoch: 71588 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71587/71587/71587) [16,13] r=-1 lpr=71587 pi=50489-71586/149 crt=51071'48910301 lcod 0'0 inactive NOTIFY] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:14:00.431322 7f28fcb46700 10 osd.70 pg_epoch: 71587 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71587/71587/70961) [75] r=-1 lpr=71587 pi=71183-71586/16 crt=57981'40 lcod 0'0 inactive NOTIFY] check_recovery_sources no source osds () went down 2014-12-08 18:14:00.431334 7f28fc345700 10 osd.70 pg_epoch: 71588 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71587/71587/71587) [16,13] r=-1 lpr=71587 pi=50489-71586/149 crt=51071'48910301 lcod 0'0 inactive NOTIFY] check_recovery_sources no source osds () went down 2014-12-08 18:14:00.431342 7f28fcb46700 10 osd.70 pg_epoch: 71587 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71587/71587/70961) [75] r=-1 lpr=71587 pi=71183-71586/16 crt=57981'40 lcod 0'0 inactive NOTIFY] handle_advance_map [75]/[75] -- 75/75 2014-12-08 18:14:00.431349 7f28fc345700 10 osd.70 pg_epoch: 71588 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71587/71587/71587) [16,13] r=-1 lpr=71587 pi=50489-71586/149 crt=51071'48910301 lcod 0'0 inactive NOTIFY] handle_advance_map [16,13]/[16,13] -- 16/16 2014-12-08 18:14:00.431355 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1,9~2,d~2] newly_removed_snaps [] snapc f=[] (no change) 2014-12-08 18:14:00.431358 7f28fc345700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:14:00.431359 7f28fc345700 10 osd.70 pg_epoch: 71589 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71587/71587/71587) [16,13] r=-1 lpr=71587 pi=50489-71586/149 crt=51071'48910301 lcod 0'0 inactive NOTIFY] state: Reset advmap 2014-12-08 18:14:00.431359 7f28fcb46700 10 osd.70 pg_epoch: 71588 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71587/71587/70961) [75] r=-1 lpr=71587 pi=71183-71586/16 crt=57981'40 lcod 0'0 inactive NOTIFY] state: Reset advmap 2014-12-08 18:14:00.431366 7f28fc345700 10 osd.70 pg_epoch: 71589 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71587/71587/71587) [16,13] r=-1 lpr=71587 pi=50489-71586/149 crt=51071'48910301 lcod 0'0 inactive NOTIFY] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:14:00.431372 7f28fc345700 10 osd.70 pg_epoch: 71589 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71587/71587/71587) [16,13] r=-1 lpr=71587 pi=50489-71586/149 crt=51071'48910301 lcod 0'0 inactive NOTIFY] check_recovery_sources no source osds () went down 2014-12-08 18:14:00.431371 7f28fcb46700 10 osd.70 pg_epoch: 71588 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71587/71587/70961) [75] r=-1 lpr=71587 pi=71183-71586/16 crt=57981'40 lcod 0'0 inactive NOTIFY] _calc_past_interval_range: already have past intervals back to 71434 2014-12-08 18:14:00.431383 7f28fc345700 10 osd.70 pg_epoch: 71589 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71587/71587/71587) [16,13] r=-1 lpr=71587 pi=50489-71586/149 crt=51071'48910301 lcod 0'0 inactive NOTIFY] handle_advance_map [16,13]/[16,13] -- 16/16 2014-12-08 18:14:00.431390 7f28fc345700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:14:00.431383 7f28fcb46700 10 osd.70 pg_epoch: 71588 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71587/71587/70961) [75] r=-1 lpr=71587 pi=71183-71586/16 crt=57981'40 lcod 0'0 inactive NOTIFY] check_recovery_sources no source osds () went down 2014-12-08 18:14:00.431392 7f28fc345700 10 osd.70 pg_epoch: 71590 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71587/71587/71587) [16,13] r=-1 lpr=71587 pi=50489-71586/149 crt=51071'48910301 lcod 0'0 inactive NOTIFY] state: Reset advmap 2014-12-08 18:14:00.431398 7f28fc345700 10 osd.70 pg_epoch: 71590 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71587/71587/71587) [16,13] r=-1 lpr=71587 pi=50489-71586/149 crt=51071'48910301 lcod 0'0 inactive NOTIFY] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:14:00.431404 7f28fc345700 10 osd.70 pg_epoch: 71590 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71587/71587/71587) [16,13] r=-1 lpr=71587 pi=50489-71586/149 crt=51071'48910301 lcod 0'0 inactive NOTIFY] check_recovery_sources no source osds () went down 2014-12-08 18:14:00.431401 7f28fcb46700 10 osd.70 pg_epoch: 71588 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71587/71587/70961) [75] r=-1 lpr=71587 pi=71183-71586/16 crt=57981'40 lcod 0'0 inactive NOTIFY] handle_advance_map [75]/[75,25] -- 75/75 2014-12-08 18:14:00.431415 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1,9~2,d~2] newly_removed_snaps [] snapc f=[] (no change) 2014-12-08 18:14:00.431414 7f28fc345700 10 osd.70 pg_epoch: 71590 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71587/71587/71587) [16,13] r=-1 lpr=71587 pi=50489-71586/149 crt=51071'48910301 lcod 0'0 inactive NOTIFY] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:14:00.431422 7f28fc345700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:14:00.431423 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71587/71587/71587) [16,13] r=-1 lpr=71587 pi=50489-71586/149 crt=51071'48910301 lcod 0'0 inactive NOTIFY] state: Reset advmap 2014-12-08 18:14:00.431419 7f28fcb46700 10 osd.70 pg_epoch: 71589 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71587/71587/70961) [75] r=-1 lpr=71587 pi=71183-71586/16 crt=57981'40 lcod 0'0 inactive NOTIFY] state: Reset advmap 2014-12-08 18:14:00.431430 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71587/71587/71587) [16,13] r=-1 lpr=71587 pi=50489-71586/149 crt=51071'48910301 lcod 0'0 inactive NOTIFY] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:14:00.431436 7f28fc345700 20 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71587/71587/71587) [16,13] r=-1 lpr=71587 pi=50489-71586/149 crt=51071'48910301 lcod 0'0 inactive NOTIFY] new interval newup [70,16,13] newacting [70,16,13] 2014-12-08 18:14:00.431430 7f28fcb46700 10 osd.70 pg_epoch: 71589 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71587/71587/70961) [75] r=-1 lpr=71587 pi=71183-71586/16 crt=57981'40 lcod 0'0 inactive NOTIFY] _calc_past_interval_range: already have past intervals back to 71434 2014-12-08 18:14:00.431442 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71587/71587/71587) [16,13] r=-1 lpr=71587 pi=50489-71586/149 crt=51071'48910301 lcod 0'0 inactive NOTIFY] state: should restart peering, calling start_peering_interval again 2014-12-08 18:14:00.431448 7f28fc345700 20 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71587/71587/71587) [16,13] r=-1 lpr=71587 pi=50489-71586/149 crt=51071'48910301 lcod 0'0 inactive NOTIFY] set_last_peering_reset 71591 2014-12-08 18:14:00.431443 7f28fcb46700 20 osd.70 pg_epoch: 71589 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71587/71587/70961) [75] r=-1 lpr=71587 pi=71183-71586/16 crt=57981'40 lcod 0'0 inactive NOTIFY] new interval newup [75] newacting [75,25] 2014-12-08 18:14:00.431453 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71587/71587/71587) [16,13] r=-1 lpr=71591 pi=50489-71586/149 crt=51071'48910301 lcod 0'0 inactive NOTIFY] Clearing blocked outgoing recovery messages 2014-12-08 18:14:00.431454 7f28fcb46700 10 osd.70 pg_epoch: 71589 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71587/71587/70961) [75] r=-1 lpr=71587 pi=71183-71586/16 crt=57981'40 lcod 0'0 inactive NOTIFY] state: should restart peering, calling start_peering_interval again 2014-12-08 18:14:00.431460 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71587/71587/71587) [16,13] r=-1 lpr=71591 pi=50489-71586/149 crt=51071'48910301 lcod 0'0 inactive NOTIFY] Not blocking outgoing recovery messages 2014-12-08 18:14:00.431465 7f28fcb46700 20 osd.70 pg_epoch: 71589 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71587/71587/70961) [75] r=-1 lpr=71587 pi=71183-71586/16 crt=57981'40 lcod 0'0 inactive NOTIFY] set_last_peering_reset 71589 2014-12-08 18:14:00.431474 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71587/71587/71587) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive NOTIFY] start_peering_interval: check_new_interval output: generate_past_intervals interval(71587-71590 up [16,13](16) acting [16,13](16)): not rw, up_thru 71585 up_from 69690 last_epoch_clean 50495 generate_past_intervals interval(71587-71590 up [16,13](16) acting [16,13](16)) : primary up 69690-71585 does not include interval 2014-12-08 18:14:00.431475 7f28fcb46700 10 osd.70 pg_epoch: 71589 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71587/71587/70961) [75] r=-1 lpr=71589 pi=71183-71586/16 crt=57981'40 lcod 0'0 inactive NOTIFY] Clearing blocked outgoing recovery messages 2014-12-08 18:14:00.431481 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71587/71587/71587) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive NOTIFY] noting past interval(71587-71590 up [16,13](16) acting [16,13](16)) 2014-12-08 18:14:00.431487 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive NOTIFY] up [16,13] -> [70,16,13], acting [16,13] -> [70,16,13], acting_primary 16 -> 70, up_primary 16 -> 70, role -1 -> 0 2014-12-08 18:14:00.431487 7f28fcb46700 10 osd.70 pg_epoch: 71589 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71587/71587/70961) [75] r=-1 lpr=71589 pi=71183-71586/16 crt=57981'40 lcod 0'0 inactive NOTIFY] Not blocking outgoing recovery messages 2014-12-08 18:14:00.431495 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive NOTIFY] clear_primary_state 2014-12-08 18:14:00.431501 7f28fc345700 20 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive NOTIFY] agent_stop 2014-12-08 18:14:00.431507 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive NOTIFY] on_change 2014-12-08 18:14:00.431507 7f28fcb46700 10 osd.70 pg_epoch: 71589 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71587/71587/70961) [75]/[75,25] r=-1 lpr=71589 pi=71183-71588/17 crt=57981'40 lcod 0'0 remapped NOTIFY] start_peering_interval: check_new_interval output: generate_past_intervals interval(71587-71588 up [75](75) acting [75](75)): not rw, up_thru 71587 up_from 69683 last_epoch_clean 71434 generate_past_intervals interval(71587-71588 up [75](75) acting [75](75) maybe_went_rw) : primary up 69683-71587 includes interval 2014-12-08 18:14:00.431515 7f28fc345700 15 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive NOTIFY] publish_stats_to_osd 71591:63844728 2014-12-08 18:14:00.431522 7f28fc345700 15 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive NOTIFY] requeue_ops 2014-12-08 18:14:00.431520 7f28fcb46700 10 osd.70 pg_epoch: 71589 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71587/71587/70961) [75]/[75,25] r=-1 lpr=71589 pi=71183-71588/17 crt=57981'40 lcod 0'0 remapped NOTIFY] noting past interval(71587-71588 up [75](75) acting [75](75) maybe_went_rw) 2014-12-08 18:14:00.431528 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive NOTIFY] cancel_copy_ops 2014-12-08 18:14:00.431534 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive NOTIFY] cancel_flush_ops 2014-12-08 18:14:00.431540 7f28fc345700 15 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive NOTIFY] requeue_ops 2014-12-08 18:14:00.431533 7f28fcb46700 10 osd.70 pg_epoch: 71589 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71587/71589/70961) [75]/[75,25] r=-1 lpr=71589 pi=71183-71588/17 crt=57981'40 lcod 0'0 remapped NOTIFY] up [75] -> [75], acting [75] -> [75,25], acting_primary 75 -> 75, up_primary 75 -> 75, role -1 -> -1 2014-12-08 18:14:00.431546 7f28fc345700 15 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive NOTIFY] requeue_ops 2014-12-08 18:14:00.431547 7f28fcb46700 10 osd.70 pg_epoch: 71589 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71587/71589/70961) [75]/[75,25] r=-1 lpr=71589 pi=71183-71588/17 crt=57981'40 lcod 0'0 remapped NOTIFY] on_change 2014-12-08 18:14:00.431553 7f28fc345700 15 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive NOTIFY] requeue_ops 2014-12-08 18:14:00.431558 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive NOTIFY] on_change_cleanup 2014-12-08 18:14:00.431558 7f28fcb46700 15 osd.70 pg_epoch: 71589 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71587/71589/70961) [75]/[75,25] r=-1 lpr=71589 pi=71183-71588/17 crt=57981'40 lcod 0'0 remapped NOTIFY] publish_stats_to_osd -- not primary 2014-12-08 18:14:00.431565 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive NOTIFY] on_change 2014-12-08 18:14:00.431571 7f28fc345700 20 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive NOTIFY] exit NotTrimming 2014-12-08 18:14:00.431570 7f28fcb46700 15 osd.70 pg_epoch: 71589 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71587/71589/70961) [75]/[75,25] r=-1 lpr=71589 pi=71183-71588/17 crt=57981'40 lcod 0'0 remapped NOTIFY] requeue_ops 2014-12-08 18:14:00.431578 7f28fc345700 20 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive NOTIFY] enter NotTrimming 2014-12-08 18:14:00.431584 7f28fc345700 15 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] clear_stats 2014-12-08 18:14:00.431582 7f28fcb46700 10 osd.70 pg_epoch: 71589 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71587/71589/70961) [75]/[75,25] r=-1 lpr=71589 pi=71183-71588/17 crt=57981'40 lcod 0'0 remapped NOTIFY] cancel_copy_ops 2014-12-08 18:14:00.431590 7f28fc345700 15 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] requeue_ops 2014-12-08 18:14:00.431596 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] on_role_change 2014-12-08 18:14:00.431592 7f28fcb46700 10 osd.70 pg_epoch: 71589 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71587/71589/70961) [75]/[75,25] r=-1 lpr=71589 pi=71183-71588/17 crt=57981'40 lcod 0'0 remapped NOTIFY] cancel_flush_ops 2014-12-08 18:14:00.431602 7f28fc345700 15 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] requeue_ops 2014-12-08 18:14:00.431608 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] cancel_recovery 2014-12-08 18:14:00.431604 7f28fcb46700 10 osd.70 pg_epoch: 71589 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71587/71589/70961) [75]/[75,25] r=-1 lpr=71589 pi=71183-71588/17 crt=57981'40 lcod 0'0 remapped NOTIFY] on_change_cleanup 2014-12-08 18:14:00.431614 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] clear_recovery_state 2014-12-08 18:14:00.431616 7f28fcb46700 10 osd.70 pg_epoch: 71589 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71587/71589/70961) [75]/[75,25] r=-1 lpr=71589 pi=71183-71588/17 crt=57981'40 lcod 0'0 remapped NOTIFY] on_change 2014-12-08 18:14:00.431620 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:14:00.431631 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] handle_activate_map 2014-12-08 18:14:00.431627 7f28fcb46700 20 osd.70 pg_epoch: 71589 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71587/71589/70961) [75]/[75,25] r=-1 lpr=71589 pi=71183-71588/17 crt=57981'40 lcod 0'0 remapped NOTIFY] exit NotTrimming 2014-12-08 18:14:00.431638 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] update_heartbeat_peers 13,16,70 unchanged 2014-12-08 18:14:00.431645 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] take_waiters 2014-12-08 18:14:00.431640 7f28fcb46700 20 osd.70 pg_epoch: 71589 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71587/71589/70961) [75]/[75,25] r=-1 lpr=71589 pi=71183-71588/17 crt=57981'40 lcod 0'0 remapped NOTIFY] enter NotTrimming 2014-12-08 18:14:00.431651 7f28fc345700 5 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] exit Reset 0.001032 10 0.000946 2014-12-08 18:14:00.431652 7f28fcb46700 10 osd.70 pg_epoch: 71589 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71587/71589/70961) [75]/[75,25] r=-1 lpr=71589 pi=71183-71588/17 crt=57981'40 lcod 0'0 remapped NOTIFY] cancel_recovery 2014-12-08 18:14:00.431659 7f28fc345700 5 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] enter Started 2014-12-08 18:14:00.431665 7f28fc345700 5 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] enter Start 2014-12-08 18:14:00.431663 7f28fcb46700 10 osd.70 pg_epoch: 71589 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71587/71589/70961) [75]/[75,25] r=-1 lpr=71589 pi=71183-71588/17 crt=57981'40 lcod 0'0 remapped NOTIFY] clear_recovery_state 2014-12-08 18:14:00.431671 7f28fc345700 1 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] state: transitioning to Primary 2014-12-08 18:14:00.431678 7f28fc345700 5 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] exit Start 0.000013 0 0.000000 2014-12-08 18:14:00.431676 7f28fcb46700 10 osd.70 pg_epoch: 71589 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71587/71589/70961) [75]/[75,25] r=-1 lpr=71589 pi=71183-71588/17 crt=57981'40 lcod 0'0 remapped NOTIFY] check_recovery_sources no source osds () went down 2014-12-08 18:14:00.431685 7f28fc345700 5 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] enter Started/Primary 2014-12-08 18:14:00.431691 7f28fc345700 5 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 inactive] enter Started/Primary/Peering 2014-12-08 18:14:00.431697 7f28fc345700 5 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] enter Started/Primary/Peering/GetInfo 2014-12-08 18:14:00.431696 7f28fcb46700 10 osd.70 pg_epoch: 71589 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71587/71589/70961) [75]/[75,25] r=-1 lpr=71589 pi=71183-71588/17 crt=57981'40 lcod 0'0 remapped NOTIFY] handle_advance_map [75]/[75,25] -- 75/75 2014-12-08 18:14:00.431704 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] _calc_past_interval_range: already have past intervals back to 50495 2014-12-08 18:14:00.431712 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1,9~2,d~2] newly_removed_snaps [] snapc f=[] (no change) 2014-12-08 18:14:00.431712 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(71587-71590 up [16,13](16) acting [16,13](16)) 2014-12-08 18:14:00.431719 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(71584-71586 up [76,16,13](76) acting [76,16,13](76) maybe_went_rw) 2014-12-08 18:14:00.431716 7f28fcb46700 10 osd.70 pg_epoch: 71590 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71587/71589/70961) [75]/[75,25] r=-1 lpr=71589 pi=71183-71588/17 crt=57981'40 lcod 0'0 remapped NOTIFY] state: Reset advmap 2014-12-08 18:14:00.431727 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(71582-71583 up [16,13](16) acting [16,13](16) maybe_went_rw) 2014-12-08 18:14:00.431728 7f28fcb46700 10 osd.70 pg_epoch: 71590 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71587/71589/70961) [75]/[75,25] r=-1 lpr=71589 pi=71183-71588/17 crt=57981'40 lcod 0'0 remapped NOTIFY] _calc_past_interval_range: already have past intervals back to 71434 2014-12-08 18:14:00.431734 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(71456-71581 up [70,16,13](70) acting [70,16,13](70) maybe_went_rw) 2014-12-08 18:14:00.431741 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(71444-71455 up [70,16,56](70) acting [70,16,56](70) maybe_went_rw) 2014-12-08 18:14:00.431740 7f28fcb46700 10 osd.70 pg_epoch: 71590 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71587/71589/70961) [75]/[75,25] r=-1 lpr=71589 pi=71183-71588/17 crt=57981'40 lcod 0'0 remapped NOTIFY] check_recovery_sources no source osds () went down 2014-12-08 18:14:00.431749 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(71441-71443 up [70,16,13](70) acting [70,16,13](70) maybe_went_rw) 2014-12-08 18:14:00.431756 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(71432-71440 up [70,16,56](70) acting [70,16,56](70) maybe_went_rw) 2014-12-08 18:14:00.431763 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(71429-71431 up [76,16,56](76) acting [76,16,56](76) maybe_went_rw) 2014-12-08 18:14:00.431759 7f28fcb46700 10 osd.70 pg_epoch: 71590 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71587/71589/70961) [75]/[75,25] r=-1 lpr=71589 pi=71183-71588/17 crt=57981'40 lcod 0'0 remapped NOTIFY] handle_advance_map [75,70]/[75,25] -- 75/75 2014-12-08 18:14:00.431769 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(71427-71428 up [16,56](16) acting [16,56](16) maybe_went_rw) 2014-12-08 18:14:00.431774 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1,9~2,d~2] newly_removed_snaps [] snapc f=[] (no change) 2014-12-08 18:14:00.431776 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(71406-71426 up [70,16,56](70) acting [70,16,56](70) maybe_went_rw) 2014-12-08 18:14:00.431778 7f28fcb46700 10 osd.70 pg_epoch: 71591 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71587/71589/70961) [75]/[75,25] r=-1 lpr=71589 pi=71183-71588/17 crt=57981'40 lcod 0'0 remapped NOTIFY] state: Reset advmap 2014-12-08 18:14:00.431783 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(71401-71405 up [76,16,56](76) acting [76,16,56](76) maybe_went_rw) 2014-12-08 18:14:00.431791 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(71399-71400 up [16,56](16) acting [16,56](16) maybe_went_rw) 2014-12-08 18:14:00.431789 7f28fcb46700 10 osd.70 pg_epoch: 71591 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71587/71589/70961) [75]/[75,25] r=-1 lpr=71589 pi=71183-71588/17 crt=57981'40 lcod 0'0 remapped NOTIFY] _calc_past_interval_range: already have past intervals back to 71434 2014-12-08 18:14:00.431798 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(71392-71398 up [70,16,56](70) acting [70,16,56](70) maybe_went_rw) 2014-12-08 18:14:00.431805 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(71384-71391 up [16,56](16) acting [16,56](16) maybe_went_rw) 2014-12-08 18:14:00.431802 7f28fcb46700 20 osd.70 pg_epoch: 71591 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71587/71589/70961) [75]/[75,25] r=-1 lpr=71589 pi=71183-71588/17 crt=57981'40 lcod 0'0 remapped NOTIFY] new interval newup [75,70] newacting [75,25] 2014-12-08 18:14:00.431812 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(71380-71383 up [76,16,56](76) acting [76,16,56](76) maybe_went_rw) 2014-12-08 18:14:00.431814 7f28fcb46700 10 osd.70 pg_epoch: 71591 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71587/71589/70961) [75]/[75,25] r=-1 lpr=71589 pi=71183-71588/17 crt=57981'40 lcod 0'0 remapped NOTIFY] state: should restart peering, calling start_peering_interval again 2014-12-08 18:14:00.431819 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(71378-71379 up [16,56](16) acting [16,56](16) maybe_went_rw) 2014-12-08 18:14:00.431827 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(71280-71377 up [70,16,56](70) acting [70,16,56](70) maybe_went_rw) 2014-12-08 18:14:00.431826 7f28fcb46700 20 osd.70 pg_epoch: 71591 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71587/71589/70961) [75]/[75,25] r=-1 lpr=71589 pi=71183-71588/17 crt=57981'40 lcod 0'0 remapped NOTIFY] set_last_peering_reset 71591 2014-12-08 18:14:00.431834 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(71183-71279 up [70,16,63](70) acting [70,16,63](70) maybe_went_rw) 2014-12-08 18:14:00.431842 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(71181-71182 up [70,16](70) acting [70,16](70) maybe_went_rw) 2014-12-08 18:14:00.431838 7f28fcb46700 10 osd.70 pg_epoch: 71591 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71587/71589/70961) [75]/[75,25] r=-1 lpr=71591 pi=71183-71588/17 crt=57981'40 lcod 0'0 remapped NOTIFY] Clearing blocked outgoing recovery messages 2014-12-08 18:14:00.431849 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(71150-71180 up [70,16,56](70) acting [70,16,56](70) maybe_went_rw) 2014-12-08 18:14:00.431851 7f28fcb46700 10 osd.70 pg_epoch: 71591 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71587/71589/70961) [75]/[75,25] r=-1 lpr=71591 pi=71183-71588/17 crt=57981'40 lcod 0'0 remapped NOTIFY] Not blocking outgoing recovery messages 2014-12-08 18:14:00.431856 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(71136-71149 up [76,16,56](76) acting [76,16,56](76) maybe_went_rw) 2014-12-08 18:14:00.431864 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(71134-71135 up [16,56](16) acting [16,56](16) maybe_went_rw) 2014-12-08 18:14:00.431871 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(71129-71133 up [70,16,56](70) acting [70,16,56](70) maybe_went_rw) 2014-12-08 18:14:00.431872 7f28fcb46700 10 osd.70 pg_epoch: 71591 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71587/71589/70961) [75,70]/[75,25] r=-1 lpr=71591 pi=71183-71590/18 crt=57981'40 lcod 0'0 remapped NOTIFY] start_peering_interval: check_new_interval output: generate_past_intervals interval(71589-71590 up [75](75) acting [75,25](75)): not rw, up_thru 71589 up_from 69683 last_epoch_clean 71434 generate_past_intervals interval(71589-71590 up [75](75) acting [75,25](75) maybe_went_rw) : primary up 69683-71589 includes interval 2014-12-08 18:14:00.431879 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(71108-71128 up [70,16,56](70) acting [70,16,56](70) maybe_went_rw) 2014-12-08 18:14:00.431886 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(71107-71107 up [16,56](16) acting [16,56](16)) 2014-12-08 18:14:00.431886 7f28fcb46700 10 osd.70 pg_epoch: 71591 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71587/71589/70961) [75,70]/[75,25] r=-1 lpr=71591 pi=71183-71590/18 crt=57981'40 lcod 0'0 remapped NOTIFY] noting past interval(71589-71590 up [75](75) acting [75,25](75) maybe_went_rw) 2014-12-08 18:14:00.431893 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(71101-71106 up [76,16,56](76) acting [76,16,56](76) maybe_went_rw) 2014-12-08 18:14:00.431901 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(71099-71100 up [16,56](16) acting [16,56](16) maybe_went_rw) 2014-12-08 18:14:00.431899 7f28fcb46700 10 osd.70 pg_epoch: 71591 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71591/71591/70961) [75,70]/[75,25] r=-1 lpr=71591 pi=71183-71590/18 crt=57981'40 lcod 0'0 remapped NOTIFY] up [75] -> [75,70], acting [75,25] -> [75,25], acting_primary 75 -> 75, up_primary 75 -> 75, role -1 -> -1 2014-12-08 18:14:00.431908 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(71091-71098 up [70,16,56](70) acting [70,16,56](70) maybe_went_rw) 2014-12-08 18:14:00.431915 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(71059-71090 up [70,16,56](70) acting [70,16,56](70) maybe_went_rw) 2014-12-08 18:14:00.431914 7f28fcb46700 10 osd.70 pg_epoch: 71591 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71591/71591/70961) [75,70]/[75,25] r=-1 lpr=71591 pi=71183-71590/18 crt=57981'40 lcod 0'0 remapped NOTIFY] on_change 2014-12-08 18:14:00.431922 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(71038-71058 up [70,16](70) acting [70,16](70) maybe_went_rw) 2014-12-08 18:14:00.431930 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(71021-71037 up [70,16,13](70) acting [70,16,13](70) maybe_went_rw) 2014-12-08 18:14:00.431926 7f28fcb46700 15 osd.70 pg_epoch: 71591 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71591/71591/70961) [75,70]/[75,25] r=-1 lpr=71591 pi=71183-71590/18 crt=57981'40 lcod 0'0 remapped NOTIFY] publish_stats_to_osd -- not primary 2014-12-08 18:14:00.431937 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(71008-71020 up [70,16](70) acting [70,16](70) maybe_went_rw) 2014-12-08 18:14:00.431938 7f28fcb46700 15 osd.70 pg_epoch: 71591 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71591/71591/70961) [75,70]/[75,25] r=-1 lpr=71591 pi=71183-71590/18 crt=57981'40 lcod 0'0 remapped NOTIFY] requeue_ops 2014-12-08 18:14:00.431944 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(70997-71007 up [70,16,13](70) acting [70,16,13](70) maybe_went_rw) 2014-12-08 18:14:00.431951 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(70993-70996 up [70,16](70) acting [70,16](70) maybe_went_rw) 2014-12-08 18:14:00.431950 7f28fcb46700 10 osd.70 pg_epoch: 71591 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71591/71591/70961) [75,70]/[75,25] r=-1 lpr=71591 pi=71183-71590/18 crt=57981'40 lcod 0'0 remapped NOTIFY] cancel_copy_ops 2014-12-08 18:14:00.431958 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(70991-70992 up [70,16,11](70) acting [70,16,11](70) maybe_went_rw) 2014-12-08 18:14:00.431962 7f28fcb46700 10 osd.70 pg_epoch: 71591 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71591/71591/70961) [75,70]/[75,25] r=-1 lpr=71591 pi=71183-71590/18 crt=57981'40 lcod 0'0 remapped NOTIFY] cancel_flush_ops 2014-12-08 18:14:00.431966 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(70977-70990 up [70,16,11](70) acting [70,16,11](70) maybe_went_rw) 2014-12-08 18:14:00.431977 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(70970-70976 up [16,11](16) acting [16,11](16) maybe_went_rw) 2014-12-08 18:14:00.431976 7f28fcb46700 10 osd.70 pg_epoch: 71591 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71591/71591/70961) [75,70]/[75,25] r=-1 lpr=71591 pi=71183-71590/18 crt=57981'40 lcod 0'0 remapped NOTIFY] on_change_cleanup 2014-12-08 18:14:00.431985 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(70967-70969 up [76,16,11](76) acting [76,16,11](76) maybe_went_rw) 2014-12-08 18:14:00.431992 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(70965-70966 up [16,11](16) acting [16,11](16) maybe_went_rw) 2014-12-08 18:14:00.431988 7f28fcb46700 10 osd.70 pg_epoch: 71591 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71591/71591/70961) [75,70]/[75,25] r=-1 lpr=71591 pi=71183-71590/18 crt=57981'40 lcod 0'0 remapped NOTIFY] on_change 2014-12-08 18:14:00.432000 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(69763-70964 up [70,16,11](70) acting [70,16,11](70) maybe_went_rw) 2014-12-08 18:14:00.432001 7f28fcb46700 20 osd.70 pg_epoch: 71591 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71591/71591/70961) [75,70]/[75,25] r=-1 lpr=71591 pi=71183-71590/18 crt=57981'40 lcod 0'0 remapped NOTIFY] exit NotTrimming 2014-12-08 18:14:00.432007 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(69680-69762 up [](-1) acting [](-1)) 2014-12-08 18:14:00.432014 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(69677-69679 up [78](78) acting [78](78)) 2014-12-08 18:14:00.432012 7f28fcb46700 20 osd.70 pg_epoch: 71591 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71591/71591/70961) [75,70]/[75,25] r=-1 lpr=71591 pi=71183-71590/18 crt=57981'40 lcod 0'0 remapped NOTIFY] enter NotTrimming 2014-12-08 18:14:00.432020 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(69674-69676 up [77](77) acting [77](77)) 2014-12-08 18:14:00.432028 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(69651-69673 up [70](70) acting [70](70) maybe_went_rw) 2014-12-08 18:14:00.432024 7f28fcb46700 10 osd.70 pg_epoch: 71591 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71591/71591/70961) [75,70]/[75,25] r=-1 lpr=71591 pi=71183-71590/18 crt=57981'40 lcod 0'0 remapped NOTIFY] cancel_recovery 2014-12-08 18:14:00.432035 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(69645-69650 up [70,11](70) acting [70,11](70) maybe_went_rw) 2014-12-08 18:14:00.432035 7f28fcb46700 10 osd.70 pg_epoch: 71591 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71591/71591/70961) [75,70]/[75,25] r=-1 lpr=71591 pi=71183-71590/18 crt=57981'40 lcod 0'0 remapped NOTIFY] clear_recovery_state 2014-12-08 18:14:00.432042 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(69395-69644 up [70,16,11](70) acting [70,16,11](70) maybe_went_rw) 2014-12-08 18:14:00.432049 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(66323-69394 up [70,20,11](70) acting [70,20,11](70) maybe_went_rw) 2014-12-08 18:14:00.432048 7f28fcb46700 10 osd.70 pg_epoch: 71591 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71591/71591/70961) [75,70]/[75,25] r=-1 lpr=71591 pi=71183-71590/18 crt=57981'40 lcod 0'0 remapped NOTIFY] check_recovery_sources no source osds () went down 2014-12-08 18:14:00.432056 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(66286-66322 up [70,11](70) acting [70,11](70) maybe_went_rw) 2014-12-08 18:14:00.432063 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(64997-66285 up [70,16,11](70) acting [70,16,11](70) maybe_went_rw) 2014-12-08 18:14:00.432065 7f28fcb46700 10 osd.70 pg_epoch: 71591 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71591/71591/70961) [75,70]/[75,25] r=-1 lpr=71591 pi=71183-71590/18 crt=57981'40 lcod 0'0 remapped NOTIFY] handle_activate_map 2014-12-08 18:14:00.432071 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(64292-64996 up [](-1) acting [](-1)) 2014-12-08 18:14:00.432077 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(64288-64291 up [70,58,63](70) acting [70,58,63](70)) 2014-12-08 18:14:00.432084 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(64287-64287 up [70,58](70) acting [70,58](70)) 2014-12-08 18:14:00.432090 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(64273-64286 up [70](70) acting [70](70) maybe_went_rw) 2014-12-08 18:14:00.432096 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(64268-64272 up [70,11](70) acting [70,11](70) maybe_went_rw) 2014-12-08 18:14:00.432103 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(58298-64267 up [70,20,11](70) acting [70,20,11](70) maybe_went_rw) 2014-12-08 18:14:00.432103 7f28fcb46700 10 osd.70 pg_epoch: 71591 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71591/71591/70961) [75,70]/[75,25] r=-1 lpr=71591 pi=71183-71590/18 crt=57981'40 lcod 0'0 remapped NOTIFY] update_heartbeat_peers unchanged 2014-12-08 18:14:00.432110 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(58233-58297 up [70,11](70) acting [70,11](70) maybe_went_rw) 2014-12-08 18:14:00.432118 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(57990-58232 up [70,16,11](70) acting [70,16,11](70) maybe_went_rw) 2014-12-08 18:14:00.432117 7f28fcb46700 10 osd.70 pg_epoch: 71591 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71591/71591/70961) [75,70]/[75,25] r=-1 lpr=71591 pi=71183-71590/18 crt=57981'40 lcod 0'0 remapped NOTIFY] take_waiters 2014-12-08 18:14:00.432125 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(57835-57989 up [70,16](70) acting [70,16](70) maybe_went_rw) 2014-12-08 18:14:00.432133 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(57650-57834 up [](-1) acting [](-1)) 2014-12-08 18:14:00.432130 7f28fcb46700 5 osd.70 pg_epoch: 71591 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71591/71591/70961) [75,70]/[75,25] r=-1 lpr=71591 pi=71183-71590/18 crt=57981'40 lcod 0'0 remapped NOTIFY] exit Reset 0.001878 10 0.001656 2014-12-08 18:14:00.432139 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(57447-57649 up [4,16](4) acting [4,16](4) maybe_went_rw) 2014-12-08 18:14:00.432146 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(57438-57446 up [16](16) acting [16](16) maybe_went_rw) 2014-12-08 18:14:00.432145 7f28fcb46700 5 osd.70 pg_epoch: 71591 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71591/71591/70961) [75,70]/[75,25] r=-1 lpr=71591 pi=71183-71590/18 crt=57981'40 lcod 0'0 remapped NOTIFY] enter Started 2014-12-08 18:14:00.432153 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(53820-57437 up [70,16](70) acting [70,16](70) maybe_went_rw) 2014-12-08 18:14:00.432160 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(53718-53819 up [4,58](4) acting [4,58](4) maybe_went_rw) 2014-12-08 18:14:00.432157 7f28fcb46700 5 osd.70 pg_epoch: 71591 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71591/71591/70961) [75,70]/[75,25] r=-1 lpr=71591 pi=71183-71590/18 crt=57981'40 lcod 0'0 remapped NOTIFY] enter Start 2014-12-08 18:14:00.432167 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(53708-53717 up [58](58) acting [58](58) maybe_went_rw) 2014-12-08 18:14:00.432174 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(53692-53707 up [4,58](4) acting [4,58](4) maybe_went_rw) 2014-12-08 18:14:00.432169 7f28fcb46700 1 osd.70 pg_epoch: 71591 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71591/71591/70961) [75,70]/[75,25] r=-1 lpr=71591 pi=71183-71590/18 crt=57981'40 lcod 0'0 remapped NOTIFY] state: transitioning to Stray 2014-12-08 18:14:00.432207 7f28fcb46700 5 osd.70 pg_epoch: 71591 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71591/71591/70961) [75,70]/[75,25] r=-1 lpr=71591 pi=71183-71590/18 crt=57981'40 lcod 0'0 remapped NOTIFY] exit Start 0.000050 0 0.000000 2014-12-08 18:14:00.432228 7f28fcb46700 5 osd.70 pg_epoch: 71591 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71591/71591/70961) [75,70]/[75,25] r=-1 lpr=71591 pi=71183-71590/18 crt=57981'40 lcod 0'0 remapped NOTIFY] enter Started/Stray 2014-12-08 18:14:00.432242 7f28fcb46700 20 osd.70 pg_epoch: 71591 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71591/71591/70961) [75,70]/[75,25] r=-1 lpr=71591 pi=71183-71590/18 crt=57981'40 lcod 0'0 remapped NOTIFY] handle_activate_map: Not dirtying info: last_persisted is 71434 while current is 71591 2014-12-08 18:14:00.432255 7f28fcb46700 10 osd.70 pg_epoch: 71591 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71591/71591/70961) [75,70]/[75,25] r=-1 lpr=71591 pi=71183-71590/18 crt=57981'40 lcod 0'0 remapped NOTIFY] handle_peering_event: epoch_sent: 71591 epoch_requested: 71591 NullEvt 2014-12-08 18:14:00.432181 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(53691-53691 up [58](58) acting [58](58)) 2014-12-08 18:14:00.432269 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(53689-53690 up [4,58](4) acting [4,58](4)) 2014-12-08 18:14:00.432289 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(53670-53688 up [4](4) acting [4](4) maybe_went_rw) 2014-12-08 18:14:00.432296 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(53096-53669 up [4,58](4) acting [4,58](4) maybe_went_rw) 2014-12-08 18:14:00.432302 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(52968-53095 up [4,30](4) acting [4,30](4) maybe_went_rw) 2014-12-08 18:14:00.432309 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(52952-52967 up [30](30) acting [30](30) maybe_went_rw) 2014-12-08 18:14:00.432318 7f28fcb46700 10 log is not dirty 2014-12-08 18:14:00.432316 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(52944-52951 up [4,30](4) acting [4,30](4) maybe_went_rw) 2014-12-08 18:14:00.432322 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(52924-52943 up [30](30) acting [30](30) maybe_went_rw) 2014-12-08 18:14:00.432329 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(52826-52923 up [4,30](4) acting [4,30](4) maybe_went_rw) 2014-12-08 18:14:00.432335 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(52819-52825 up [30](30) acting [30](30) maybe_went_rw) 2014-12-08 18:14:00.432354 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(52749-52818 up [4,30](4) acting [4,30](4) maybe_went_rw) 2014-12-08 18:14:00.432361 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(52685-52748 up [4](4) acting [4](4) maybe_went_rw) 2014-12-08 18:14:00.432368 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(52582-52684 up [4,30](4) acting [4,30](4) maybe_went_rw) 2014-12-08 18:14:00.432359 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[218.57c( v 71424'1 (0'0,71424'1] local-les=71434 n=1 ec=71423 les/c 71434/71434 71432/71432/71425) [24,16,70] r=2 lpr=71434 pi=71423-71431/4 crt=71424'1 lcod 0'0 inactive NOTIFY] handle_advance_map [24,16]/[24,16] -- 24/24 2014-12-08 18:14:00.432375 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(52578-52581 up [30](30) acting [30](30) maybe_went_rw) 2014-12-08 18:14:00.432380 7f28fcb46700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:14:00.432382 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(52568-52577 up [4,30](4) acting [4,30](4) maybe_went_rw) 2014-12-08 18:14:00.432389 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(52561-52567 up [30](30) acting [30](30) maybe_went_rw) 2014-12-08 18:14:00.432385 7f28fcb46700 10 osd.70 pg_epoch: 71582 pg[218.57c( v 71424'1 (0'0,71424'1] local-les=71434 n=1 ec=71423 les/c 71434/71434 71432/71432/71425) [24,16,70] r=2 lpr=71434 pi=71423-71431/4 crt=71424'1 lcod 0'0 inactive NOTIFY] state: Started advmap 2014-12-08 18:14:00.432396 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(52550-52560 up [4,30](4) acting [4,30](4) maybe_went_rw) 2014-12-08 18:14:00.432397 7f28fcb46700 20 osd.70 pg_epoch: 71582 pg[218.57c( v 71424'1 (0'0,71424'1] local-les=71434 n=1 ec=71423 les/c 71434/71434 71432/71432/71425) [24,16,70] r=2 lpr=71434 pi=71423-71431/4 crt=71424'1 lcod 0'0 inactive NOTIFY] new interval newup [24,16] newacting [24,16] 2014-12-08 18:14:00.432402 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(52549-52549 up [4](4) acting [4](4)) 2014-12-08 18:14:00.432409 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(52546-52548 up [4,30](4) acting [4,30](4) maybe_went_rw) 2014-12-08 18:14:00.432409 7f28fcb46700 10 osd.70 pg_epoch: 71582 pg[218.57c( v 71424'1 (0'0,71424'1] local-les=71434 n=1 ec=71423 les/c 71434/71434 71432/71432/71425) [24,16,70] r=2 lpr=71434 pi=71423-71431/4 crt=71424'1 lcod 0'0 inactive NOTIFY] state: should_restart_peering, transitioning to Reset 2014-12-08 18:14:00.432416 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(52532-52545 up [30](30) acting [30](30)) 2014-12-08 18:14:00.432423 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(52375-52531 up [4,30](4) acting [4,30](4) maybe_went_rw) 2014-12-08 18:14:00.432423 7f28fcb46700 5 osd.70 pg_epoch: 71582 pg[218.57c( v 71424'1 (0'0,71424'1] local-les=71434 n=1 ec=71423 les/c 71434/71434 71432/71432/71425) [24,16,70] r=2 lpr=71434 pi=71423-71431/4 crt=71424'1 lcod 0'0 inactive NOTIFY] exit Started/Stray 2.826858 3 0.000095 2014-12-08 18:14:00.432431 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(52372-52374 up [30,11](30) acting [30,11](30) maybe_went_rw) 2014-12-08 18:14:00.432451 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(52369-52371 up [30,31](30) acting [30,31](30) maybe_went_rw) 2014-12-08 18:14:00.432449 7f28fcb46700 5 osd.70 pg_epoch: 71582 pg[218.57c( v 71424'1 (0'0,71424'1] local-les=71434 n=1 ec=71423 les/c 71434/71434 71432/71432/71425) [24,16,70] r=2 lpr=71434 pi=71423-71431/4 crt=71424'1 lcod 0'0 inactive NOTIFY] exit Started 2.826918 0 0.000000 2014-12-08 18:14:00.432458 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(52340-52368 up [31,59](31) acting [31,59](31) maybe_went_rw) 2014-12-08 18:14:00.432462 7f28fcb46700 5 osd.70 pg_epoch: 71582 pg[218.57c( v 71424'1 (0'0,71424'1] local-les=71434 n=1 ec=71423 les/c 71434/71434 71432/71432/71425) [24,16,70] r=2 lpr=71434 pi=71423-71431/4 crt=71424'1 lcod 0'0 inactive NOTIFY] enter Reset 2014-12-08 18:14:00.432761 7f290a83a700 10 osd.70 pg_epoch: 71591 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71434/71434 71591/71591/70961) [75,70]/[75,25] r=-1 lpr=71591 pi=71183-71590/18 crt=57981'40 lcod 0'0 remapped NOTIFY] flushed 2014-12-08 18:14:00.437044 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(52327-52339 up [11,31](11) acting [11,31](11) maybe_went_rw) 2014-12-08 18:14:00.437050 7f28fcb46700 20 osd.70 pg_epoch: 71582 pg[218.57c( v 71424'1 (0'0,71424'1] local-les=71434 n=1 ec=71423 les/c 71434/71434 71432/71432/71425) [24,16,70] r=2 lpr=71434 pi=71423-71431/4 crt=71424'1 lcod 0'0 inactive NOTIFY] set_last_peering_reset 71582 2014-12-08 18:14:00.437060 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(52320-52326 up [30,11](30) acting [30,11](30) maybe_went_rw) 2014-12-08 18:14:00.437063 7f28fcb46700 10 osd.70 pg_epoch: 71582 pg[218.57c( v 71424'1 (0'0,71424'1] local-les=71434 n=1 ec=71423 les/c 71434/71434 71432/71432/71425) [24,16,70] r=2 lpr=71582 pi=71423-71431/4 crt=71424'1 lcod 0'0 inactive NOTIFY] Clearing blocked outgoing recovery messages 2014-12-08 18:14:00.437069 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(52319-52319 up [30](30) acting [30](30)) 2014-12-08 18:14:00.437071 7f28fcb46700 10 osd.70 pg_epoch: 71582 pg[218.57c( v 71424'1 (0'0,71424'1] local-les=71434 n=1 ec=71423 les/c 71434/71434 71432/71432/71425) [24,16,70] r=2 lpr=71582 pi=71423-71431/4 crt=71424'1 lcod 0'0 inactive NOTIFY] Not blocking outgoing recovery messages 2014-12-08 18:14:00.437076 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(52306-52318 up [30,11](30) acting [30,11](30) maybe_went_rw) 2014-12-08 18:14:00.437078 7f28fcb46700 10 osd.70 pg_epoch: 71582 pg[218.57c( v 71424'1 (0'0,71424'1] local-les=71434 n=1 ec=71423 les/c 71434/71434 71432/71432/71425) [24,16,70] r=2 lpr=71582 pi=71423-71431/4 crt=71424'1 lcod 0'0 inactive NOTIFY] state: Reset advmap 2014-12-08 18:14:00.437083 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(52298-52305 up [11](11) acting [11](11) maybe_went_rw) 2014-12-08 18:14:00.437084 7f28fcb46700 10 osd.70 pg_epoch: 71582 pg[218.57c( v 71424'1 (0'0,71424'1] local-les=71434 n=1 ec=71423 les/c 71434/71434 71432/71432/71425) [24,16,70] r=2 lpr=71582 pi=71423-71431/4 crt=71424'1 lcod 0'0 inactive NOTIFY] _calc_past_interval_range: already have past intervals back to 71434 2014-12-08 18:14:00.437091 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(52295-52297 up [30,11](30) acting [30,11](30) maybe_went_rw) 2014-12-08 18:14:00.437092 7f28fcb46700 20 osd.70 pg_epoch: 71582 pg[218.57c( v 71424'1 (0'0,71424'1] local-les=71434 n=1 ec=71423 les/c 71434/71434 71432/71432/71425) [24,16,70] r=2 lpr=71582 pi=71423-71431/4 crt=71424'1 lcod 0'0 inactive NOTIFY] new interval newup [24,16] newacting [24,16] 2014-12-08 18:14:00.437098 7f28fcb46700 10 osd.70 pg_epoch: 71582 pg[218.57c( v 71424'1 (0'0,71424'1] local-les=71434 n=1 ec=71423 les/c 71434/71434 71432/71432/71425) [24,16,70] r=2 lpr=71582 pi=71423-71431/4 crt=71424'1 lcod 0'0 inactive NOTIFY] state: should restart peering, calling start_peering_interval again 2014-12-08 18:14:00.437098 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(52292-52294 up [11](11) acting [11](11) maybe_went_rw) 2014-12-08 18:14:00.437105 7f28fcb46700 20 osd.70 pg_epoch: 71582 pg[218.57c( v 71424'1 (0'0,71424'1] local-les=71434 n=1 ec=71423 les/c 71434/71434 71432/71432/71425) [24,16,70] r=2 lpr=71582 pi=71423-71431/4 crt=71424'1 lcod 0'0 inactive NOTIFY] set_last_peering_reset 71582 2014-12-08 18:14:00.437105 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(52279-52291 up [30,11](30) acting [30,11](30) maybe_went_rw) 2014-12-08 18:14:00.437113 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(52272-52278 up [11](11) acting [11](11) maybe_went_rw) 2014-12-08 18:14:00.437120 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(52265-52271 up [30,11](30) acting [30,11](30) maybe_went_rw) 2014-12-08 18:14:00.437119 7f28fcb46700 10 osd.70 pg_epoch: 71582 pg[218.57c( v 71424'1 (0'0,71424'1] local-les=71434 n=1 ec=71423 les/c 71434/71434 71432/71432/71425) [24,16] r=-1 lpr=71582 pi=71423-71581/5 crt=71424'1 lcod 0'0 inactive NOTIFY] start_peering_interval: check_new_interval output: generate_past_intervals interval(71432-71581 up [24,16,70](24) acting [24,16,70](24)): not rw, up_thru 71456 up_from 69676 last_epoch_clean 71434 generate_past_intervals interval(71432-71581 up [24,16,70](24) acting [24,16,70](24) maybe_went_rw) : primary up 69676-71456 includes interval 2014-12-08 18:14:00.437127 7f28fcb46700 10 osd.70 pg_epoch: 71582 pg[218.57c( v 71424'1 (0'0,71424'1] local-les=71434 n=1 ec=71423 les/c 71434/71434 71432/71432/71425) [24,16] r=-1 lpr=71582 pi=71423-71581/5 crt=71424'1 lcod 0'0 inactive NOTIFY] noting past interval(71432-71581 up [24,16,70](24) acting [24,16,70](24) maybe_went_rw) 2014-12-08 18:14:00.437127 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(52157-52264 up [4,30](4) acting [4,30](4) maybe_went_rw) 2014-12-08 18:14:00.437135 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(52143-52156 up [30](30) acting [30](30) maybe_went_rw) 2014-12-08 18:14:00.437135 7f28fcb46700 10 osd.70 pg_epoch: 71582 pg[218.57c( v 71424'1 (0'0,71424'1] local-les=71434 n=1 ec=71423 les/c 71434/71434 71582/71582/71425) [24,16] r=-1 lpr=71582 pi=71423-71581/5 crt=71424'1 lcod 0'0 inactive NOTIFY] up [24,16,70] -> [24,16], acting [24,16,70] -> [24,16], acting_primary 24 -> 24, up_primary 24 -> 24, role 2 -> -1 2014-12-08 18:14:00.437143 7f28fcb46700 10 osd.70 pg_epoch: 71582 pg[218.57c( v 71424'1 (0'0,71424'1] local-les=71434 n=1 ec=71423 les/c 71434/71434 71582/71582/71425) [24,16] r=-1 lpr=71582 pi=71423-71581/5 crt=71424'1 lcod 0'0 inactive NOTIFY] on_change 2014-12-08 18:14:00.437143 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(52061-52142 up [4,30](4) acting [4,30](4) maybe_went_rw) 2014-12-08 18:14:00.437150 7f28fcb46700 15 osd.70 pg_epoch: 71582 pg[218.57c( v 71424'1 (0'0,71424'1] local-les=71434 n=1 ec=71423 les/c 71434/71434 71582/71582/71425) [24,16] r=-1 lpr=71582 pi=71423-71581/5 crt=71424'1 lcod 0'0 inactive NOTIFY] publish_stats_to_osd -- not primary 2014-12-08 18:14:00.437151 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(52058-52060 up [4,11](4) acting [4,11](4) maybe_went_rw) 2014-12-08 18:14:00.437156 7f28fcb46700 15 osd.70 pg_epoch: 71582 pg[218.57c( v 71424'1 (0'0,71424'1] local-les=71434 n=1 ec=71423 les/c 71434/71434 71582/71582/71425) [24,16] r=-1 lpr=71582 pi=71423-71581/5 crt=71424'1 lcod 0'0 inactive NOTIFY] requeue_ops 2014-12-08 18:14:00.437159 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(52033-52057 up [11,31](11) acting [11,31](11) maybe_went_rw) 2014-12-08 18:14:00.437164 7f28fcb46700 10 osd.70 pg_epoch: 71582 pg[218.57c( v 71424'1 (0'0,71424'1] local-les=71434 n=1 ec=71423 les/c 71434/71434 71582/71582/71425) [24,16] r=-1 lpr=71582 pi=71423-71581/5 crt=71424'1 lcod 0'0 inactive NOTIFY] cancel_copy_ops 2014-12-08 18:14:00.437166 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(52031-52032 up [31](31) acting [31](31) maybe_went_rw) 2014-12-08 18:14:00.437169 7f28fcb46700 10 osd.70 pg_epoch: 71582 pg[218.57c( v 71424'1 (0'0,71424'1] local-les=71434 n=1 ec=71423 les/c 71434/71434 71582/71582/71425) [24,16] r=-1 lpr=71582 pi=71423-71581/5 crt=71424'1 lcod 0'0 inactive NOTIFY] cancel_flush_ops 2014-12-08 18:14:00.437174 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(51949-52030 up [11,31](11) acting [11,31](11) maybe_went_rw) 2014-12-08 18:14:00.437175 7f28fcb46700 10 osd.70 pg_epoch: 71582 pg[218.57c( v 71424'1 (0'0,71424'1] local-les=71434 n=1 ec=71423 les/c 71434/71434 71582/71582/71425) [24,16] r=-1 lpr=71582 pi=71423-71581/5 crt=71424'1 lcod 0'0 inactive NOTIFY] on_change_cleanup 2014-12-08 18:14:00.437181 7f28fcb46700 10 osd.70 pg_epoch: 71582 pg[218.57c( v 71424'1 (0'0,71424'1] local-les=71434 n=1 ec=71423 les/c 71434/71434 71582/71582/71425) [24,16] r=-1 lpr=71582 pi=71423-71581/5 crt=71424'1 lcod 0'0 inactive NOTIFY] on_change 2014-12-08 18:14:00.437181 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(51938-51948 up [31](31) acting [31](31) maybe_went_rw) 2014-12-08 18:14:00.437189 7f28fcb46700 20 osd.70 pg_epoch: 71582 pg[218.57c( v 71424'1 (0'0,71424'1] local-les=71434 n=1 ec=71423 les/c 71434/71434 71582/71582/71425) [24,16] r=-1 lpr=71582 pi=71423-71581/5 crt=71424'1 lcod 0'0 inactive NOTIFY] exit NotTrimming 2014-12-08 18:14:00.437188 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(51772-51937 up [11,31](11) acting [11,31](11) maybe_went_rw) 2014-12-08 18:14:00.437195 7f28fcb46700 20 osd.70 pg_epoch: 71582 pg[218.57c( v 71424'1 (0'0,71424'1] local-les=71434 n=1 ec=71423 les/c 71434/71434 71582/71582/71425) [24,16] r=-1 lpr=71582 pi=71423-71581/5 crt=71424'1 lcod 0'0 inactive NOTIFY] enter NotTrimming 2014-12-08 18:14:00.437196 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(51770-51771 up [4,11](4) acting [4,11](4) maybe_went_rw) 2014-12-08 18:14:00.437202 7f28fcb46700 10 osd.70 pg_epoch: 71582 pg[218.57c( v 71424'1 (0'0,71424'1] local-les=71434 n=1 ec=71423 les/c 71434/71434 71582/71582/71425) [24,16] r=-1 lpr=71582 pi=71423-71581/5 crt=71424'1 lcod 0'0 inactive NOTIFY] on_role_change 2014-12-08 18:14:00.437204 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(51768-51769 up [11](11) acting [11](11) maybe_went_rw) 2014-12-08 18:14:00.437208 7f28fcb46700 15 osd.70 pg_epoch: 71582 pg[218.57c( v 71424'1 (0'0,71424'1] local-les=71434 n=1 ec=71423 les/c 71434/71434 71582/71582/71425) [24,16] r=-1 lpr=71582 pi=71423-71581/5 crt=71424'1 lcod 0'0 inactive NOTIFY] requeue_ops 2014-12-08 18:14:00.437214 7f28fcb46700 10 osd.70 pg_epoch: 71582 pg[218.57c( v 71424'1 (0'0,71424'1] local-les=71434 n=1 ec=71423 les/c 71434/71434 71582/71582/71425) [24,16] r=-1 lpr=71582 pi=71423-71581/5 crt=71424'1 lcod 0'0 inactive NOTIFY] cancel_recovery 2014-12-08 18:14:00.437211 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(51765-51767 up [4,11](4) acting [4,11](4) maybe_went_rw) 2014-12-08 18:14:00.437219 7f28fcb46700 10 osd.70 pg_epoch: 71582 pg[218.57c( v 71424'1 (0'0,71424'1] local-les=71434 n=1 ec=71423 les/c 71434/71434 71582/71582/71425) [24,16] r=-1 lpr=71582 pi=71423-71581/5 crt=71424'1 lcod 0'0 inactive NOTIFY] clear_recovery_state 2014-12-08 18:14:00.437220 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(51763-51764 up [11](11) acting [11](11) maybe_went_rw) 2014-12-08 18:14:00.437227 7f28fcb46700 10 osd.70 pg_epoch: 71582 pg[218.57c( v 71424'1 (0'0,71424'1] local-les=71434 n=1 ec=71423 les/c 71434/71434 71582/71582/71425) [24,16] r=-1 lpr=71582 pi=71423-71581/5 crt=71424'1 lcod 0'0 inactive NOTIFY] check_recovery_sources no source osds () went down 2014-12-08 18:14:00.437227 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(51757-51762 up [4,11](4) acting [4,11](4) maybe_went_rw) 2014-12-08 18:14:00.437235 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(51749-51756 up [11](11) acting [11](11) maybe_went_rw) 2014-12-08 18:14:00.437241 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(51743-51748 up [4,11](4) acting [4,11](4) maybe_went_rw) 2014-12-08 18:14:00.437241 7f28fcb46700 10 osd.70 pg_epoch: 71582 pg[218.57c( v 71424'1 (0'0,71424'1] local-les=71434 n=1 ec=71423 les/c 71434/71434 71582/71582/71425) [24,16] r=-1 lpr=71582 pi=71423-71581/5 crt=71424'1 lcod 0'0 inactive NOTIFY] handle_advance_map [24,16]/[24,16] -- 24/24 2014-12-08 18:14:00.437251 7f28fcb46700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:14:00.437249 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(51736-51742 up [11](11) acting [11](11) maybe_went_rw) 2014-12-08 18:14:00.437253 7f28fcb46700 10 osd.70 pg_epoch: 71583 pg[218.57c( v 71424'1 (0'0,71424'1] local-les=71434 n=1 ec=71423 les/c 71434/71434 71582/71582/71425) [24,16] r=-1 lpr=71582 pi=71423-71581/5 crt=71424'1 lcod 0'0 inactive NOTIFY] state: Reset advmap 2014-12-08 18:14:00.437257 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(51732-51735 up [4,11](4) acting [4,11](4) maybe_went_rw) 2014-12-08 18:14:00.437260 7f28fcb46700 10 osd.70 pg_epoch: 71583 pg[218.57c( v 71424'1 (0'0,71424'1] local-les=71434 n=1 ec=71423 les/c 71434/71434 71582/71582/71425) [24,16] r=-1 lpr=71582 pi=71423-71581/5 crt=71424'1 lcod 0'0 inactive NOTIFY] _calc_past_interval_range: already have past intervals back to 71434 2014-12-08 18:14:00.437266 7f28fcb46700 10 osd.70 pg_epoch: 71583 pg[218.57c( v 71424'1 (0'0,71424'1] local-les=71434 n=1 ec=71423 les/c 71434/71434 71582/71582/71425) [24,16] r=-1 lpr=71582 pi=71423-71581/5 crt=71424'1 lcod 0'0 inactive NOTIFY] check_recovery_sources no source osds () went down 2014-12-08 18:14:00.437265 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(51729-51731 up [11](11) acting [11](11) maybe_went_rw) 2014-12-08 18:14:00.437275 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(51723-51728 up [4,11](4) acting [4,11](4) maybe_went_rw) 2014-12-08 18:14:00.437279 7f28fcb46700 10 osd.70 pg_epoch: 71583 pg[218.57c( v 71424'1 (0'0,71424'1] local-les=71434 n=1 ec=71423 les/c 71434/71434 71582/71582/71425) [24,16] r=-1 lpr=71582 pi=71423-71581/5 crt=71424'1 lcod 0'0 inactive NOTIFY] handle_advance_map [24,16,56]/[24,16,56] -- 24/24 2014-12-08 18:14:00.437287 7f28fcb46700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:14:00.437282 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(51721-51722 up [11](11) acting [11](11) maybe_went_rw) 2014-12-08 18:14:00.437289 7f28fcb46700 10 osd.70 pg_epoch: 71584 pg[218.57c( v 71424'1 (0'0,71424'1] local-les=71434 n=1 ec=71423 les/c 71434/71434 71582/71582/71425) [24,16] r=-1 lpr=71582 pi=71423-71581/5 crt=71424'1 lcod 0'0 inactive NOTIFY] state: Reset advmap 2014-12-08 18:14:00.437290 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(51649-51720 up [4,11](4) acting [4,11](4) maybe_went_rw) 2014-12-08 18:14:00.437295 7f28fcb46700 10 osd.70 pg_epoch: 71584 pg[218.57c( v 71424'1 (0'0,71424'1] local-les=71434 n=1 ec=71423 les/c 71434/71434 71582/71582/71425) [24,16] r=-1 lpr=71582 pi=71423-71581/5 crt=71424'1 lcod 0'0 inactive NOTIFY] _calc_past_interval_range: already have past intervals back to 71434 2014-12-08 18:14:00.437297 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(51637-51648 up [11](11) acting [11](11) maybe_went_rw) 2014-12-08 18:14:00.437301 7f28fcb46700 20 osd.70 pg_epoch: 71584 pg[218.57c( v 71424'1 (0'0,71424'1] local-les=71434 n=1 ec=71423 les/c 71434/71434 71582/71582/71425) [24,16] r=-1 lpr=71582 pi=71423-71581/5 crt=71424'1 lcod 0'0 inactive NOTIFY] new interval newup [24,16,56] newacting [24,16,56] 2014-12-08 18:14:00.437305 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(51483-51636 up [4,11](4) acting [4,11](4) maybe_went_rw) 2014-12-08 18:14:00.437308 7f28fcb46700 10 osd.70 pg_epoch: 71584 pg[218.57c( v 71424'1 (0'0,71424'1] local-les=71434 n=1 ec=71423 les/c 71434/71434 71582/71582/71425) [24,16] r=-1 lpr=71582 pi=71423-71581/5 crt=71424'1 lcod 0'0 inactive NOTIFY] state: should restart peering, calling start_peering_interval again 2014-12-08 18:14:00.437314 7f28fcb46700 20 osd.70 pg_epoch: 71584 pg[218.57c( v 71424'1 (0'0,71424'1] local-les=71434 n=1 ec=71423 les/c 71434/71434 71582/71582/71425) [24,16] r=-1 lpr=71582 pi=71423-71581/5 crt=71424'1 lcod 0'0 inactive NOTIFY] set_last_peering_reset 71584 2014-12-08 18:14:00.437313 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(51467-51482 up [11](11) acting [11](11) maybe_went_rw) 2014-12-08 18:14:00.437319 7f28fcb46700 10 osd.70 pg_epoch: 71584 pg[218.57c( v 71424'1 (0'0,71424'1] local-les=71434 n=1 ec=71423 les/c 71434/71434 71582/71582/71425) [24,16] r=-1 lpr=71584 pi=71423-71581/5 crt=71424'1 lcod 0'0 inactive NOTIFY] Clearing blocked outgoing recovery messages 2014-12-08 18:14:00.437320 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(51363-51466 up [4,11](4) acting [4,11](4) maybe_went_rw) 2014-12-08 18:14:00.437326 7f28fcb46700 10 osd.70 pg_epoch: 71584 pg[218.57c( v 71424'1 (0'0,71424'1] local-les=71434 n=1 ec=71423 les/c 71434/71434 71582/71582/71425) [24,16] r=-1 lpr=71584 pi=71423-71581/5 crt=71424'1 lcod 0'0 inactive NOTIFY] Not blocking outgoing recovery messages 2014-12-08 18:14:00.437328 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(51338-51362 up [11](11) acting [11](11) maybe_went_rw) 2014-12-08 18:14:00.437335 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(51148-51337 up [4,11](4) acting [4,11](4) maybe_went_rw) 2014-12-08 18:14:00.437337 7f28fcb46700 10 osd.70 pg_epoch: 71584 pg[218.57c( v 71424'1 (0'0,71424'1] local-les=71434 n=1 ec=71423 les/c 71434/71434 71582/71582/71425) [24,16,56] r=-1 lpr=71584 pi=71423-71583/6 crt=71424'1 lcod 0'0 inactive NOTIFY] start_peering_interval: check_new_interval output: generate_past_intervals interval(71582-71583 up [24,16](24) acting [24,16](24)): not rw, up_thru 71582 up_from 69676 last_epoch_clean 71434 generate_past_intervals interval(71582-71583 up [24,16](24) acting [24,16](24) maybe_went_rw) : primary up 69676-71582 includes interval 2014-12-08 18:14:00.437342 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(51147-51147 up [11](11) acting [11](11)) 2014-12-08 18:14:00.437345 7f28fcb46700 10 osd.70 pg_epoch: 71584 pg[218.57c( v 71424'1 (0'0,71424'1] local-les=71434 n=1 ec=71423 les/c 71434/71434 71582/71582/71425) [24,16,56] r=-1 lpr=71584 pi=71423-71583/6 crt=71424'1 lcod 0'0 inactive NOTIFY] noting past interval(71582-71583 up [24,16](24) acting [24,16](24) maybe_went_rw) 2014-12-08 18:14:00.437349 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(51140-51146 up [4,11](4) acting [4,11](4) maybe_went_rw) 2014-12-08 18:14:00.437352 7f28fcb46700 10 osd.70 pg_epoch: 71584 pg[218.57c( v 71424'1 (0'0,71424'1] local-les=71434 n=1 ec=71423 les/c 71434/71434 71584/71584/71425) [24,16,56] r=-1 lpr=71584 pi=71423-71583/6 crt=71424'1 lcod 0'0 inactive NOTIFY] up [24,16] -> [24,16,56], acting [24,16] -> [24,16,56], acting_primary 24 -> 24, up_primary 24 -> 24, role -1 -> -1 2014-12-08 18:14:00.437357 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(51119-51139 up [11,31](11) acting [11,31](11) maybe_went_rw) 2014-12-08 18:14:00.437360 7f28fcb46700 10 osd.70 pg_epoch: 71584 pg[218.57c( v 71424'1 (0'0,71424'1] local-les=71434 n=1 ec=71423 les/c 71434/71434 71584/71584/71425) [24,16,56] r=-1 lpr=71584 pi=71423-71583/6 crt=71424'1 lcod 0'0 inactive NOTIFY] on_change 2014-12-08 18:14:00.437365 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(51100-51118 up [11,31](11) acting [0](0) maybe_went_rw) 2014-12-08 18:14:00.437368 7f28fcb46700 15 osd.70 pg_epoch: 71584 pg[218.57c( v 71424'1 (0'0,71424'1] local-les=71434 n=1 ec=71423 les/c 71434/71434 71584/71584/71425) [24,16,56] r=-1 lpr=71584 pi=71423-71583/6 crt=71424'1 lcod 0'0 inactive NOTIFY] publish_stats_to_osd -- not primary 2014-12-08 18:14:00.437372 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(51073-51099 up [11,31](11) acting [11,31](11) maybe_went_rw) 2014-12-08 18:14:00.437380 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior final: probe 0,4,11,13,16,20,30,31,56,58,59,63,70,76 down blocked_by {} 2014-12-08 18:14:00.437379 7f28fcb46700 15 osd.70 pg_epoch: 71584 pg[218.57c( v 71424'1 (0'0,71424'1] local-les=71434 n=1 ec=71423 les/c 71434/71434 71584/71584/71425) [24,16,56] r=-1 lpr=71584 pi=71423-71583/6 crt=71424'1 lcod 0'0 inactive NOTIFY] requeue_ops 2014-12-08 18:14:00.437388 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] up_thru 71491 < same_since 71591, must notify monitor 2014-12-08 18:14:00.437389 7f28fcb46700 10 osd.70 pg_epoch: 71584 pg[218.57c( v 71424'1 (0'0,71424'1] local-les=71434 n=1 ec=71423 les/c 71434/71434 71584/71584/71425) [24,16,56] r=-1 lpr=71584 pi=71423-71583/6 crt=71424'1 lcod 0'0 inactive NOTIFY] cancel_copy_ops 2014-12-08 18:14:00.437397 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] state: querying info from osd.0 2014-12-08 18:14:00.437400 7f28fcb46700 10 osd.70 pg_epoch: 71584 pg[218.57c( v 71424'1 (0'0,71424'1] local-les=71434 n=1 ec=71423 les/c 71434/71434 71584/71584/71425) [24,16,56] r=-1 lpr=71584 pi=71423-71583/6 crt=71424'1 lcod 0'0 inactive NOTIFY] cancel_flush_ops 2014-12-08 18:14:00.437409 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] state: querying info from osd.4 2014-12-08 18:14:00.437410 7f28fcb46700 10 osd.70 pg_epoch: 71584 pg[218.57c( v 71424'1 (0'0,71424'1] local-les=71434 n=1 ec=71423 les/c 71434/71434 71584/71584/71425) [24,16,56] r=-1 lpr=71584 pi=71423-71583/6 crt=71424'1 lcod 0'0 inactive NOTIFY] on_change_cleanup 2014-12-08 18:14:00.437416 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] state: querying info from osd.11 2014-12-08 18:14:00.437423 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] state: querying info from osd.13 2014-12-08 18:14:00.437420 7f28fcb46700 10 osd.70 pg_epoch: 71584 pg[218.57c( v 71424'1 (0'0,71424'1] local-les=71434 n=1 ec=71423 les/c 71434/71434 71584/71584/71425) [24,16,56] r=-1 lpr=71584 pi=71423-71583/6 crt=71424'1 lcod 0'0 inactive NOTIFY] on_change 2014-12-08 18:14:00.437430 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] state: querying info from osd.16 2014-12-08 18:14:00.437433 7f28fcb46700 20 osd.70 pg_epoch: 71584 pg[218.57c( v 71424'1 (0'0,71424'1] local-les=71434 n=1 ec=71423 les/c 71434/71434 71584/71584/71425) [24,16,56] r=-1 lpr=71584 pi=71423-71583/6 crt=71424'1 lcod 0'0 inactive NOTIFY] exit NotTrimming 2014-12-08 18:14:00.437437 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] state: querying info from osd.20 2014-12-08 18:14:00.437440 7f28fcb46700 20 osd.70 pg_epoch: 71584 pg[218.57c( v 71424'1 (0'0,71424'1] local-les=71434 n=1 ec=71423 les/c 71434/71434 71584/71584/71425) [24,16,56] r=-1 lpr=71584 pi=71423-71583/6 crt=71424'1 lcod 0'0 inactive NOTIFY] enter NotTrimming 2014-12-08 18:14:00.437444 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] state: querying info from osd.30 2014-12-08 18:14:00.437447 7f28fcb46700 10 osd.70 pg_epoch: 71584 pg[218.57c( v 71424'1 (0'0,71424'1] local-les=71434 n=1 ec=71423 les/c 71434/71434 71584/71584/71425) [24,16,56] r=-1 lpr=71584 pi=71423-71583/6 crt=71424'1 lcod 0'0 inactive NOTIFY] cancel_recovery 2014-12-08 18:14:00.437452 7f28fcb46700 10 osd.70 pg_epoch: 71584 pg[218.57c( v 71424'1 (0'0,71424'1] local-les=71434 n=1 ec=71423 les/c 71434/71434 71584/71584/71425) [24,16,56] r=-1 lpr=71584 pi=71423-71583/6 crt=71424'1 lcod 0'0 inactive NOTIFY] clear_recovery_state 2014-12-08 18:14:00.437452 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] state: querying info from osd.31 2014-12-08 18:14:00.437459 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] state: querying info from osd.56 2014-12-08 18:14:00.437459 7f28fcb46700 10 osd.70 pg_epoch: 71584 pg[218.57c( v 71424'1 (0'0,71424'1] local-les=71434 n=1 ec=71423 les/c 71434/71434 71584/71584/71425) [24,16,56] r=-1 lpr=71584 pi=71423-71583/6 crt=71424'1 lcod 0'0 inactive NOTIFY] check_recovery_sources no source osds () went down 2014-12-08 18:14:00.437466 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] state: querying info from osd.58 2014-12-08 18:14:00.437473 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] state: querying info from osd.59 2014-12-08 18:14:00.437474 7f28fcb46700 10 osd.70 pg_epoch: 71584 pg[218.57c( v 71424'1 (0'0,71424'1] local-les=71434 n=1 ec=71423 les/c 71434/71434 71584/71584/71425) [24,16,56] r=-1 lpr=71584 pi=71423-71583/6 crt=71424'1 lcod 0'0 inactive NOTIFY] handle_advance_map [24,16,56]/[24,16,56] -- 24/24 2014-12-08 18:14:00.437482 7f28fcb46700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:14:00.437480 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] state: querying info from osd.63 2014-12-08 18:14:00.437484 7f28fcb46700 10 osd.70 pg_epoch: 71585 pg[218.57c( v 71424'1 (0'0,71424'1] local-les=71434 n=1 ec=71423 les/c 71434/71434 71584/71584/71425) [24,16,56] r=-1 lpr=71584 pi=71423-71583/6 crt=71424'1 lcod 0'0 inactive NOTIFY] state: Reset advmap 2014-12-08 18:14:00.437488 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] state: querying info from osd.76 2014-12-08 18:14:00.437491 7f28fcb46700 10 osd.70 pg_epoch: 71585 pg[218.57c( v 71424'1 (0'0,71424'1] local-les=71434 n=1 ec=71423 les/c 71434/71434 71584/71584/71425) [24,16,56] r=-1 lpr=71584 pi=71423-71583/6 crt=71424'1 lcod 0'0 inactive NOTIFY] _calc_past_interval_range: already have past intervals back to 71434 2014-12-08 18:14:00.437496 7f28fc345700 15 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] publish_stats_to_osd 71591:63844729 2014-12-08 18:14:00.437498 7f28fcb46700 10 osd.70 pg_epoch: 71585 pg[218.57c( v 71424'1 (0'0,71424'1] local-les=71434 n=1 ec=71423 les/c 71434/71434 71584/71584/71425) [24,16,56] r=-1 lpr=71584 pi=71423-71583/6 crt=71424'1 lcod 0'0 inactive NOTIFY] check_recovery_sources no source osds () went down 2014-12-08 18:14:00.437504 7f28fc345700 20 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] handle_activate_map: Not dirtying info: last_persisted is 71456 while current is 71591 2014-12-08 18:14:00.437508 7f28fcb46700 10 osd.70 pg_epoch: 71585 pg[218.57c( v 71424'1 (0'0,71424'1] local-les=71434 n=1 ec=71423 les/c 71434/71434 71584/71584/71425) [24,16,56] r=-1 lpr=71584 pi=71423-71583/6 crt=71424'1 lcod 0'0 inactive NOTIFY] handle_advance_map [24,16,56]/[24,16,56] -- 24/24 2014-12-08 18:14:00.437516 7f28fcb46700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:14:00.437511 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] handle_peering_event: epoch_sent: 71591 epoch_requested: 71591 NullEvt 2014-12-08 18:14:00.437518 7f28fcb46700 10 osd.70 pg_epoch: 71586 pg[218.57c( v 71424'1 (0'0,71424'1] local-les=71434 n=1 ec=71423 les/c 71434/71434 71584/71584/71425) [24,16,56] r=-1 lpr=71584 pi=71423-71583/6 crt=71424'1 lcod 0'0 inactive NOTIFY] state: Reset advmap 2014-12-08 18:14:00.437523 7f28fcb46700 10 osd.70 pg_epoch: 71586 pg[218.57c( v 71424'1 (0'0,71424'1] local-les=71434 n=1 ec=71423 les/c 71434/71434 71584/71584/71425) [24,16,56] r=-1 lpr=71584 pi=71423-71583/6 crt=71424'1 lcod 0'0 inactive NOTIFY] _calc_past_interval_range: already have past intervals back to 71434 2014-12-08 18:14:00.437529 7f28fcb46700 10 osd.70 pg_epoch: 71586 pg[218.57c( v 71424'1 (0'0,71424'1] local-les=71434 n=1 ec=71423 les/c 71434/71434 71584/71584/71425) [24,16,56] r=-1 lpr=71584 pi=71423-71583/6 crt=71424'1 lcod 0'0 inactive NOTIFY] check_recovery_sources no source osds () went down 2014-12-08 18:14:00.437539 7f28fcb46700 10 osd.70 pg_epoch: 71586 pg[218.57c( v 71424'1 (0'0,71424'1] local-les=71434 n=1 ec=71423 les/c 71434/71434 71584/71584/71425) [24,16,56] r=-1 lpr=71584 pi=71423-71583/6 crt=71424'1 lcod 0'0 inactive NOTIFY] handle_advance_map [24,16]/[24,16] -- 24/24 2014-12-08 18:14:00.437546 7f28fcb46700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:14:00.437548 7f28fcb46700 10 osd.70 pg_epoch: 71587 pg[218.57c( v 71424'1 (0'0,71424'1] local-les=71434 n=1 ec=71423 les/c 71434/71434 71584/71584/71425) [24,16,56] r=-1 lpr=71584 pi=71423-71583/6 crt=71424'1 lcod 0'0 inactive NOTIFY] state: Reset advmap 2014-12-08 18:14:00.437553 7f28fcb46700 10 osd.70 pg_epoch: 71587 pg[218.57c( v 71424'1 (0'0,71424'1] local-les=71434 n=1 ec=71423 les/c 71434/71434 71584/71584/71425) [24,16,56] r=-1 lpr=71584 pi=71423-71583/6 crt=71424'1 lcod 0'0 inactive NOTIFY] _calc_past_interval_range: already have past intervals back to 71434 2014-12-08 18:14:00.437558 7f28fcb46700 20 osd.70 pg_epoch: 71587 pg[218.57c( v 71424'1 (0'0,71424'1] local-les=71434 n=1 ec=71423 les/c 71434/71434 71584/71584/71425) [24,16,56] r=-1 lpr=71584 pi=71423-71583/6 crt=71424'1 lcod 0'0 inactive NOTIFY] new interval newup [24,16] newacting [24,16] 2014-12-08 18:14:00.437564 7f28fcb46700 10 osd.70 pg_epoch: 71587 pg[218.57c( v 71424'1 (0'0,71424'1] local-les=71434 n=1 ec=71423 les/c 71434/71434 71584/71584/71425) [24,16,56] r=-1 lpr=71584 pi=71423-71583/6 crt=71424'1 lcod 0'0 inactive NOTIFY] state: should restart peering, calling start_peering_interval again 2014-12-08 18:14:00.437577 7f28fcb46700 20 osd.70 pg_epoch: 71587 pg[218.57c( v 71424'1 (0'0,71424'1] local-les=71434 n=1 ec=71423 les/c 71434/71434 71584/71584/71425) [24,16,56] r=-1 lpr=71584 pi=71423-71583/6 crt=71424'1 lcod 0'0 inactive NOTIFY] set_last_peering_reset 71587 2014-12-08 18:14:00.437582 7f28fcb46700 10 osd.70 pg_epoch: 71587 pg[218.57c( v 71424'1 (0'0,71424'1] local-les=71434 n=1 ec=71423 les/c 71434/71434 71584/71584/71425) [24,16,56] r=-1 lpr=71587 pi=71423-71583/6 crt=71424'1 lcod 0'0 inactive NOTIFY] Clearing blocked outgoing recovery messages 2014-12-08 18:14:00.437588 7f28fcb46700 10 osd.70 pg_epoch: 71587 pg[218.57c( v 71424'1 (0'0,71424'1] local-les=71434 n=1 ec=71423 les/c 71434/71434 71584/71584/71425) [24,16,56] r=-1 lpr=71587 pi=71423-71583/6 crt=71424'1 lcod 0'0 inactive NOTIFY] Not blocking outgoing recovery messages 2014-12-08 18:14:00.437602 7f28fc345700 10 log is not dirty 2014-12-08 18:14:00.437598 7f28fcb46700 10 osd.70 pg_epoch: 71587 pg[218.57c( v 71424'1 (0'0,71424'1] local-les=71434 n=1 ec=71423 les/c 71434/71434 71584/71584/71425) [24,16] r=-1 lpr=71587 pi=71423-71586/7 crt=71424'1 lcod 0'0 inactive NOTIFY] start_peering_interval: check_new_interval output: generate_past_intervals interval(71584-71586 up [24,16,56](24) acting [24,16,56](24)): not rw, up_thru 71585 up_from 69676 last_epoch_clean 71434 generate_past_intervals interval(71584-71586 up [24,16,56](24) acting [24,16,56](24) maybe_went_rw) : primary up 69676-71585 includes interval 2014-12-08 18:14:00.437605 7f28fcb46700 10 osd.70 pg_epoch: 71587 pg[218.57c( v 71424'1 (0'0,71424'1] local-les=71434 n=1 ec=71423 les/c 71434/71434 71584/71584/71425) [24,16] r=-1 lpr=71587 pi=71423-71586/7 crt=71424'1 lcod 0'0 inactive NOTIFY] noting past interval(71584-71586 up [24,16,56](24) acting [24,16,56](24) maybe_went_rw) 2014-12-08 18:14:00.437611 7f28fcb46700 10 osd.70 pg_epoch: 71587 pg[218.57c( v 71424'1 (0'0,71424'1] local-les=71434 n=1 ec=71423 les/c 71434/71434 71587/71587/71425) [24,16] r=-1 lpr=71587 pi=71423-71586/7 crt=71424'1 lcod 0'0 inactive NOTIFY] up [24,16,56] -> [24,16], acting [24,16,56] -> [24,16], acting_primary 24 -> 24, up_primary 24 -> 24, role -1 -> -1 2014-12-08 18:14:00.437618 7f28fcb46700 10 osd.70 pg_epoch: 71587 pg[218.57c( v 71424'1 (0'0,71424'1] local-les=71434 n=1 ec=71423 les/c 71434/71434 71587/71587/71425) [24,16] r=-1 lpr=71587 pi=71423-71586/7 crt=71424'1 lcod 0'0 inactive NOTIFY] on_change 2014-12-08 18:14:00.437623 7f28fcb46700 15 osd.70 pg_epoch: 71587 pg[218.57c( v 71424'1 (0'0,71424'1] local-les=71434 n=1 ec=71423 les/c 71434/71434 71587/71587/71425) [24,16] r=-1 lpr=71587 pi=71423-71586/7 crt=71424'1 lcod 0'0 inactive NOTIFY] publish_stats_to_osd -- not primary -- 2014-12-08 18:14:00.482727 7f2904b56700 20 osd.70 71591 _dispatch 0x12c181c0 pg_notify(19.6e epoch 71591) v5 2014-12-08 18:14:00.482717 7f28fcb46700 10 osd.70 pg_epoch: 71583 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71582/71582/71433) [68,18] r=-1 lpr=71582 pi=71183-71581/18 crt=71377'4110 lcod 0'0 inactive NOTIFY] handle_advance_map [15,18,17]/[15,18,17] -- 15/15 2014-12-08 18:14:00.482731 7f28fcb46700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:14:00.482729 7f28fc345700 5 osd.70 pg_epoch: 71582 pg[218.7f( empty local-les=71437 n=0 ec=71423 les/c 71437/71437 71432/71432/71425) [77,70,18] r=1 lpr=71437 pi=71423-71431/4 crt=0'0 inactive NOTIFY] exit Started/Stray 2.535925 3 0.000120 2014-12-08 18:14:00.482735 7f28fcb46700 10 osd.70 pg_epoch: 71584 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71582/71582/71433) [68,18] r=-1 lpr=71582 pi=71183-71581/18 crt=71377'4110 lcod 0'0 inactive NOTIFY] state: Reset advmap 2014-12-08 18:14:00.482739 7f28fc345700 5 osd.70 pg_epoch: 71582 pg[218.7f( empty local-les=71437 n=0 ec=71423 les/c 71437/71437 71432/71432/71425) [77,70,18] r=1 lpr=71437 pi=71423-71431/4 crt=0'0 inactive NOTIFY] exit Started 2.535965 0 0.000000 2014-12-08 18:14:00.482750 7f28fc345700 5 osd.70 pg_epoch: 71582 pg[218.7f( empty local-les=71437 n=0 ec=71423 les/c 71437/71437 71432/71432/71425) [77,70,18] r=1 lpr=71437 pi=71423-71431/4 crt=0'0 inactive NOTIFY] enter Reset 2014-12-08 18:14:00.482759 7f2904b56700 7 osd.70 71591 handle_pg_notify from osd.13 2014-12-08 18:14:00.482761 7f2904b56700 15 osd.70 71591 require_same_or_newer_map 71591 (i am 71591) 0x12c181c0 2014-12-08 18:14:00.482748 7f28fcb46700 10 osd.70 pg_epoch: 71584 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71582/71582/71433) [68,18] r=-1 lpr=71582 pi=71183-71581/18 crt=71377'4110 lcod 0'0 inactive NOTIFY] _calc_past_interval_range: already have past intervals back to 71434 2014-12-08 18:14:00.482759 7f28dd8ef700 10 osd.70 71591 OSD::ms_get_authorizer type=osd 2014-12-08 18:14:00.482763 7f28fc345700 20 osd.70 pg_epoch: 71582 pg[218.7f( empty local-les=71437 n=0 ec=71423 les/c 71437/71437 71432/71432/71425) [77,70,18] r=1 lpr=71437 pi=71423-71431/4 crt=0'0 inactive NOTIFY] set_last_peering_reset 71582 2014-12-08 18:14:00.482755 7f28ddbf2700 10 osd.70 71591 new session (outgoing) 0x128dd980 con=0x1264ef20 addr=172.16.10.102:6989/4210 2014-12-08 18:14:00.482769 7f28fc345700 10 osd.70 pg_epoch: 71582 pg[218.7f( empty local-les=71437 n=0 ec=71423 les/c 71437/71437 71432/71432/71425) [77,70,18] r=1 lpr=71582 pi=71423-71431/4 crt=0'0 inactive NOTIFY] Clearing blocked outgoing recovery messages 2014-12-08 18:14:00.482768 7f28fcb46700 20 osd.70 pg_epoch: 71584 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71582/71582/71433) [68,18] r=-1 lpr=71582 pi=71183-71581/18 crt=71377'4110 lcod 0'0 inactive NOTIFY] new interval newup [15,18,17] newacting [15,18,17] 2014-12-08 18:14:00.482779 7f28fc345700 10 osd.70 pg_epoch: 71582 pg[218.7f( empty local-les=71437 n=0 ec=71423 les/c 71437/71437 71432/71432/71425) [77,70,18] r=1 lpr=71582 pi=71423-71431/4 crt=0'0 inactive NOTIFY] Not blocking outgoing recovery messages 2014-12-08 18:14:00.482782 7f28fcb46700 10 osd.70 pg_epoch: 71584 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71582/71582/71433) [68,18] r=-1 lpr=71582 pi=71183-71581/18 crt=71377'4110 lcod 0'0 inactive NOTIFY] state: should restart peering, calling start_peering_interval again 2014-12-08 18:14:00.482795 7f2904b56700 10 osd.70 71591 do_waiters -- start 2014-12-08 18:14:00.482798 7f2904b56700 10 osd.70 71591 do_waiters -- finish 2014-12-08 18:14:00.482789 7f28fc345700 10 osd.70 pg_epoch: 71582 pg[218.7f( empty local-les=71437 n=0 ec=71423 les/c 71437/71437 71432/71432/71425) [77,70,18] r=1 lpr=71582 pi=71423-71431/4 crt=0'0 inactive NOTIFY] state: Reset advmap 2014-12-08 18:14:00.482794 7f28fcb46700 20 osd.70 pg_epoch: 71584 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71582/71582/71433) [68,18] r=-1 lpr=71582 pi=71183-71581/18 crt=71377'4110 lcod 0'0 inactive NOTIFY] set_last_peering_reset 71584 -- 2014-12-08 18:14:00.483542 7f2904b56700 20 osd.70 71591 _dispatch 0x12c19840 pg_notify(19.6e(150) epoch 71591) v5 2014-12-08 18:14:00.483539 7f28fc345700 15 osd.70 pg_epoch: 71587 pg[218.7f( empty local-les=71437 n=0 ec=71423 les/c 71437/71437 71587/71587/71425) [77,18] r=-1 lpr=71587 pi=71423-71586/7 crt=0'0 inactive NOTIFY] publish_stats_to_osd -- not primary 2014-12-08 18:14:00.483545 7f28fc345700 15 osd.70 pg_epoch: 71587 pg[218.7f( empty local-les=71437 n=0 ec=71423 les/c 71437/71437 71587/71587/71425) [77,18] r=-1 lpr=71587 pi=71423-71586/7 crt=0'0 inactive NOTIFY] requeue_ops 2014-12-08 18:14:00.483553 7f28fc345700 10 osd.70 pg_epoch: 71587 pg[218.7f( empty local-les=71437 n=0 ec=71423 les/c 71437/71437 71587/71587/71425) [77,18] r=-1 lpr=71587 pi=71423-71586/7 crt=0'0 inactive NOTIFY] cancel_copy_ops 2014-12-08 18:14:00.483566 7f2904b56700 7 osd.70 71591 handle_pg_notify from osd.16 2014-12-08 18:14:00.483547 7f28fcb46700 10 osd.70 pg_epoch: 71586 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71584/71585/71585) [15,18,17]/[18,68] r=-1 lpr=71585 pi=71183-71584/20 crt=71377'4110 lcod 0'0 remapped NOTIFY] check_recovery_sources no source osds () went down 2014-12-08 18:14:00.483562 7f28fc345700 10 osd.70 pg_epoch: 71587 pg[218.7f( empty local-les=71437 n=0 ec=71423 les/c 71437/71437 71587/71587/71425) [77,18] r=-1 lpr=71587 pi=71423-71586/7 crt=0'0 inactive NOTIFY] cancel_flush_ops 2014-12-08 18:14:00.483568 7f2904b56700 15 osd.70 71591 require_same_or_newer_map 71591 (i am 71591) 0x12c19840 2014-12-08 18:14:00.483568 7f28fc345700 10 osd.70 pg_epoch: 71587 pg[218.7f( empty local-les=71437 n=0 ec=71423 les/c 71437/71437 71587/71587/71425) [77,18] r=-1 lpr=71587 pi=71423-71586/7 crt=0'0 inactive NOTIFY] on_change_cleanup 2014-12-08 18:14:00.483578 7f28fc345700 10 osd.70 pg_epoch: 71587 pg[218.7f( empty local-les=71437 n=0 ec=71423 les/c 71437/71437 71587/71587/71425) [77,18] r=-1 lpr=71587 pi=71423-71586/7 crt=0'0 inactive NOTIFY] on_change 2014-12-08 18:14:00.483588 7f28fc345700 20 osd.70 pg_epoch: 71587 pg[218.7f( empty local-les=71437 n=0 ec=71423 les/c 71437/71437 71587/71587/71425) [77,18] r=-1 lpr=71587 pi=71423-71586/7 crt=0'0 inactive NOTIFY] exit NotTrimming 2014-12-08 18:14:00.483579 7f28fcb46700 10 osd.70 pg_epoch: 71586 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71584/71585/71585) [15,18,17]/[18,68] r=-1 lpr=71585 pi=71183-71584/20 crt=71377'4110 lcod 0'0 remapped NOTIFY] handle_advance_map [68,18]/[18,68] -- 68/18 2014-12-08 18:14:00.483595 7f28fc345700 20 osd.70 pg_epoch: 71587 pg[218.7f( empty local-les=71437 n=0 ec=71423 les/c 71437/71437 71587/71587/71425) [77,18] r=-1 lpr=71587 pi=71423-71586/7 crt=0'0 inactive NOTIFY] enter NotTrimming 2014-12-08 18:14:00.483600 7f28fcb46700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:14:00.483602 7f2904b56700 10 osd.70 71591 do_waiters -- start 2014-12-08 18:14:00.483604 7f2904b56700 10 osd.70 71591 do_waiters -- finish 2014-12-08 18:14:00.483601 7f28fc345700 10 osd.70 pg_epoch: 71587 pg[218.7f( empty local-les=71437 n=0 ec=71423 les/c 71437/71437 71587/71587/71425) [77,18] r=-1 lpr=71587 pi=71423-71586/7 crt=0'0 inactive NOTIFY] cancel_recovery 2014-12-08 18:14:00.483608 7f28fc345700 10 osd.70 pg_epoch: 71587 pg[218.7f( empty local-les=71437 n=0 ec=71423 les/c 71437/71437 71587/71587/71425) [77,18] r=-1 lpr=71587 pi=71423-71586/7 crt=0'0 inactive NOTIFY] clear_recovery_state 2014-12-08 18:14:00.483604 7f28fcb46700 10 osd.70 pg_epoch: 71587 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71584/71585/71585) [15,18,17]/[18,68] r=-1 lpr=71585 pi=71183-71584/20 crt=71377'4110 lcod 0'0 remapped NOTIFY] state: Reset advmap 2014-12-08 18:14:00.483617 7f28fc345700 10 osd.70 pg_epoch: 71587 pg[218.7f( empty local-les=71437 n=0 ec=71423 les/c 71437/71437 71587/71587/71425) [77,18] r=-1 lpr=71587 pi=71423-71586/7 crt=0'0 inactive NOTIFY] check_recovery_sources no source osds () went down 2014-12-08 18:14:00.483616 7f28fcb46700 10 osd.70 pg_epoch: 71587 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71584/71585/71585) [15,18,17]/[18,68] r=-1 lpr=71585 pi=71183-71584/20 crt=71377'4110 lcod 0'0 remapped NOTIFY] _calc_past_interval_range: already have past intervals back to 71434 -- 2014-12-08 18:14:00.483666 7f2904b56700 20 osd.70 71591 _dispatch 0x12b77e00 pg_notify(19.6e(150) epoch 71591) v5 2014-12-08 18:14:00.483656 7f28fcb46700 20 osd.70 pg_epoch: 71587 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71584/71585/71585) [15,18,17]/[18,68] r=-1 lpr=71585 pi=71183-71584/20 crt=71377'4110 lcod 0'0 remapped NOTIFY] set_last_peering_reset 71587 2014-12-08 18:14:00.483671 7f28fcb46700 10 osd.70 pg_epoch: 71587 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71584/71585/71585) [15,18,17]/[18,68] r=-1 lpr=71587 pi=71183-71584/20 crt=71377'4110 lcod 0'0 remapped NOTIFY] Clearing blocked outgoing recovery messages 2014-12-08 18:14:00.483685 7f2904b56700 7 osd.70 71591 handle_pg_notify from osd.58 2014-12-08 18:14:00.483688 7f2904b56700 15 osd.70 71591 require_same_or_newer_map 71591 (i am 71591) 0x12b77e00 2014-12-08 18:14:00.483684 7f28fcb46700 10 osd.70 pg_epoch: 71587 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71584/71585/71585) [15,18,17]/[18,68] r=-1 lpr=71587 pi=71183-71584/20 crt=71377'4110 lcod 0'0 remapped NOTIFY] Not blocking outgoing recovery messages 2014-12-08 18:14:00.483681 7f28fc345700 10 osd.70 pg_epoch: 71588 pg[218.7f( empty local-les=71437 n=0 ec=71423 les/c 71437/71437 71587/71587/71425) [77,18] r=-1 lpr=71587 pi=71423-71586/7 crt=0'0 inactive NOTIFY] handle_advance_map [77,18]/[77,18] -- 77/77 2014-12-08 18:14:00.483700 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:14:00.483702 7f28fcb46700 10 osd.70 pg_epoch: 71587 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71584/71585/71585) [68,18]/[18,68] r=-1 lpr=71587 pi=71183-71586/21 crt=71377'4110 lcod 0'0 remapped NOTIFY] start_peering_interval: check_new_interval output: generate_past_intervals interval(71585-71586 up [15,18,17](15) acting [18,68](18)): not rw, up_thru 71585 up_from 69663 last_epoch_clean 71434 generate_past_intervals interval(71585-71586 up [15,18,17](15) acting [18,68](18) maybe_went_rw) : primary up 69663-71585 includes interval 2014-12-08 18:14:00.483717 7f2904b56700 10 osd.70 71591 do_waiters -- start 2014-12-08 18:14:00.483718 7f2904b56700 10 osd.70 71591 do_waiters -- finish 2014-12-08 18:14:00.483704 7f28fc345700 10 osd.70 pg_epoch: 71589 pg[218.7f( empty local-les=71437 n=0 ec=71423 les/c 71437/71437 71587/71587/71425) [77,18] r=-1 lpr=71587 pi=71423-71586/7 crt=0'0 inactive NOTIFY] state: Reset advmap 2014-12-08 18:14:00.483716 7f28fcb46700 10 osd.70 pg_epoch: 71587 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71584/71585/71585) [68,18]/[18,68] r=-1 lpr=71587 pi=71183-71586/21 crt=71377'4110 lcod 0'0 remapped NOTIFY] noting past interval(71585-71586 up [15,18,17](15) acting [18,68](18) maybe_went_rw) 2014-12-08 18:14:00.483724 7f28fc345700 10 osd.70 pg_epoch: 71589 pg[218.7f( empty local-les=71437 n=0 ec=71423 les/c 71437/71437 71587/71587/71425) [77,18] r=-1 lpr=71587 pi=71423-71586/7 crt=0'0 inactive NOTIFY] _calc_past_interval_range: already have past intervals back to 71437 2014-12-08 18:14:00.483733 7f28fc345700 10 osd.70 pg_epoch: 71589 pg[218.7f( empty local-les=71437 n=0 ec=71423 les/c 71437/71437 71587/71587/71425) [77,18] r=-1 lpr=71587 pi=71423-71586/7 crt=0'0 inactive NOTIFY] check_recovery_sources no source osds () went down 2014-12-08 18:14:00.483730 7f28fcb46700 10 osd.70 pg_epoch: 71587 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71587/71587/71585) [68,18]/[18,68] r=-1 lpr=71587 pi=71183-71586/21 crt=71377'4110 lcod 0'0 remapped NOTIFY] up [15,18,17] -> [68,18], acting [18,68] -> [18,68], acting_primary 18 -> 18, up_primary 15 -> 68, role -1 -> -1 2014-12-08 18:14:00.483747 7f2904b56700 10 osd.70 71591 do_waiters -- start 2014-12-08 18:14:00.483749 7f2904b56700 10 osd.70 71591 do_waiters -- finish 2014-12-08 18:14:00.483742 7f28fcb46700 10 osd.70 pg_epoch: 71587 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71587/71587/71585) [68,18]/[18,68] r=-1 lpr=71587 pi=71183-71586/21 crt=71377'4110 lcod 0'0 remapped NOTIFY] on_change -- 2014-12-08 18:14:00.483812 7f2904b56700 20 osd.70 71591 _dispatch 0x12b79c00 pg_notify(19.6e epoch 71591) v5 2014-12-08 18:14:00.483805 7f290a83a700 10 osd.70 pg_epoch: 71591 pg[217.22d( v 71377'4107 (71377'1106,71377'4107] local-les=71437 n=8 ec=71094 les/c 71437/71437 71591/71591/71590) [63,70,1] r=1 lpr=71591 pi=71094-71590/32 crt=71377'4104 lcod 0'0 inactive NOTIFY] flushed 2014-12-08 18:14:00.483814 7f28fcb46700 10 osd.70 pg_epoch: 71587 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71587/71587/71585) [68,18]/[18,68] r=-1 lpr=71587 pi=71183-71586/21 crt=71377'4110 lcod 0'0 remapped NOTIFY] on_change 2014-12-08 18:14:00.483820 7f28fc345700 10 osd.70 pg_epoch: 71590 pg[218.7f( empty local-les=71437 n=0 ec=71423 les/c 71437/71437 71587/71587/71425) [77,18] r=-1 lpr=71587 pi=71423-71586/7 crt=0'0 inactive NOTIFY] handle_advance_map [77,70,18]/[77,70,18] -- 77/77 2014-12-08 18:14:00.483838 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:14:00.483839 7f2904b56700 7 osd.70 71591 handle_pg_notify from osd.11 2014-12-08 18:14:00.483841 7f2904b56700 15 osd.70 71591 require_same_or_newer_map 71591 (i am 71591) 0x12b79c00 2014-12-08 18:14:00.483835 7f28fcb46700 20 osd.70 pg_epoch: 71587 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71587/71587/71585) [68,18]/[18,68] r=-1 lpr=71587 pi=71183-71586/21 crt=71377'4110 lcod 0'0 remapped NOTIFY] exit NotTrimming 2014-12-08 18:14:00.483841 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[218.7f( empty local-les=71437 n=0 ec=71423 les/c 71437/71437 71587/71587/71425) [77,18] r=-1 lpr=71587 pi=71423-71586/7 crt=0'0 inactive NOTIFY] state: Reset advmap 2014-12-08 18:14:00.483854 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[218.7f( empty local-les=71437 n=0 ec=71423 les/c 71437/71437 71587/71587/71425) [77,18] r=-1 lpr=71587 pi=71423-71586/7 crt=0'0 inactive NOTIFY] _calc_past_interval_range: already have past intervals back to 71437 2014-12-08 18:14:00.483853 7f28fcb46700 20 osd.70 pg_epoch: 71587 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71587/71587/71585) [68,18]/[18,68] r=-1 lpr=71587 pi=71183-71586/21 crt=71377'4110 lcod 0'0 remapped NOTIFY] enter NotTrimming 2014-12-08 18:14:00.483862 7f28fc345700 20 osd.70 pg_epoch: 71591 pg[218.7f( empty local-les=71437 n=0 ec=71423 les/c 71437/71437 71587/71587/71425) [77,18] r=-1 lpr=71587 pi=71423-71586/7 crt=0'0 inactive NOTIFY] new interval newup [77,70,18] newacting [77,70,18] 2014-12-08 18:14:00.483874 7f2904b56700 10 osd.70 71591 do_waiters -- start 2014-12-08 18:14:00.483870 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[218.7f( empty local-les=71437 n=0 ec=71423 les/c 71437/71437 71587/71587/71425) [77,18] r=-1 lpr=71587 pi=71423-71586/7 crt=0'0 inactive NOTIFY] state: should restart peering, calling start_peering_interval again 2014-12-08 18:14:00.483877 7f2904b56700 10 osd.70 71591 do_waiters -- finish 2014-12-08 18:14:00.483868 7f28fcb46700 10 osd.70 pg_epoch: 71587 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71587/71587/71585) [68,18]/[18,68] r=-1 lpr=71587 pi=71183-71586/21 crt=71377'4110 lcod 0'0 remapped NOTIFY] cancel_recovery 2014-12-08 18:14:00.483878 7f28fc345700 20 osd.70 pg_epoch: 71591 pg[218.7f( empty local-les=71437 n=0 ec=71423 les/c 71437/71437 71587/71587/71425) [77,18] r=-1 lpr=71587 pi=71423-71586/7 crt=0'0 inactive NOTIFY] set_last_peering_reset 71591 2014-12-08 18:14:00.483883 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[218.7f( empty local-les=71437 n=0 ec=71423 les/c 71437/71437 71587/71587/71425) [77,18] r=-1 lpr=71591 pi=71423-71586/7 crt=0'0 inactive NOTIFY] Clearing blocked outgoing recovery messages 2014-12-08 18:14:00.483883 7f28fcb46700 10 osd.70 pg_epoch: 71587 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71587/71587/71585) [68,18]/[18,68] r=-1 lpr=71587 pi=71183-71586/21 crt=71377'4110 lcod 0'0 remapped NOTIFY] clear_recovery_state 2014-12-08 18:14:00.483891 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[218.7f( empty local-les=71437 n=0 ec=71423 les/c 71437/71437 71587/71587/71425) [77,18] r=-1 lpr=71591 pi=71423-71586/7 crt=0'0 inactive NOTIFY] Not blocking outgoing recovery messages 2014-12-08 18:14:00.483897 7f2904b56700 10 osd.70 71591 do_waiters -- start -- 2014-12-08 18:14:00.483948 7f2904b56700 20 osd.70 71591 _dispatch 0x12c19a20 pg_notify(19.6e epoch 71591) v5 2014-12-08 18:14:00.483943 7f28fc345700 15 osd.70 pg_epoch: 71591 pg[218.7f( empty local-les=71437 n=0 ec=71423 les/c 71437/71437 71591/71591/71425) [77,70,18] r=1 lpr=71591 pi=71423-71590/8 crt=0'0 inactive NOTIFY] publish_stats_to_osd -- not primary 2014-12-08 18:14:00.483939 7f28fcb46700 10 osd.70 pg_epoch: 71588 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71587/71587/71585) [68,18]/[18,68] r=-1 lpr=71587 pi=71183-71586/21 crt=71377'4110 lcod 0'0 remapped NOTIFY] state: Reset advmap 2014-12-08 18:14:00.483953 7f28fc345700 15 osd.70 pg_epoch: 71591 pg[218.7f( empty local-les=71437 n=0 ec=71423 les/c 71437/71437 71591/71591/71425) [77,70,18] r=1 lpr=71591 pi=71423-71590/8 crt=0'0 inactive NOTIFY] requeue_ops 2014-12-08 18:14:00.483953 7f28fcb46700 10 osd.70 pg_epoch: 71588 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71587/71587/71585) [68,18]/[18,68] r=-1 lpr=71587 pi=71183-71586/21 crt=71377'4110 lcod 0'0 remapped NOTIFY] _calc_past_interval_range: already have past intervals back to 71434 2014-12-08 18:14:00.483961 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[218.7f( empty local-les=71437 n=0 ec=71423 les/c 71437/71437 71591/71591/71425) [77,70,18] r=1 lpr=71591 pi=71423-71590/8 crt=0'0 inactive NOTIFY] cancel_copy_ops 2014-12-08 18:14:00.483970 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[218.7f( empty local-les=71437 n=0 ec=71423 les/c 71437/71437 71591/71591/71425) [77,70,18] r=1 lpr=71591 pi=71423-71590/8 crt=0'0 inactive NOTIFY] cancel_flush_ops 2014-12-08 18:14:00.483967 7f28fcb46700 10 osd.70 pg_epoch: 71588 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71587/71587/71585) [68,18]/[18,68] r=-1 lpr=71587 pi=71183-71586/21 crt=71377'4110 lcod 0'0 remapped NOTIFY] check_recovery_sources no source osds () went down 2014-12-08 18:14:00.483978 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[218.7f( empty local-les=71437 n=0 ec=71423 les/c 71437/71437 71591/71591/71425) [77,70,18] r=1 lpr=71591 pi=71423-71590/8 crt=0'0 inactive NOTIFY] on_change_cleanup 2014-12-08 18:14:00.483982 7f2904b56700 7 osd.70 71591 handle_pg_notify from osd.31 2014-12-08 18:14:00.483986 7f2904b56700 15 osd.70 71591 require_same_or_newer_map 71591 (i am 71591) 0x12c19a20 2014-12-08 18:14:00.483983 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[218.7f( empty local-les=71437 n=0 ec=71423 les/c 71437/71437 71591/71591/71425) [77,70,18] r=1 lpr=71591 pi=71423-71590/8 crt=0'0 inactive NOTIFY] on_change 2014-12-08 18:14:00.483986 7f28fcb46700 10 osd.70 pg_epoch: 71588 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71587/71587/71585) [68,18]/[18,68] r=-1 lpr=71587 pi=71183-71586/21 crt=71377'4110 lcod 0'0 remapped NOTIFY] handle_advance_map [68,18]/[68,18] -- 68/68 2014-12-08 18:14:00.490407 7f28fc345700 20 osd.70 pg_epoch: 71591 pg[218.7f( empty local-les=71437 n=0 ec=71423 les/c 71437/71437 71591/71591/71425) [77,70,18] r=1 lpr=71591 pi=71423-71590/8 crt=0'0 inactive NOTIFY] exit NotTrimming 2014-12-08 18:14:00.484269 7f290a83a700 10 osd.70 pg_epoch: 71591 pg[216.384( v 70990'209 (0'0,70990'209] local-les=71435 n=101 ec=70987 les/c 71435/71435 71591/71591/71587) [77,22,70]/[77,22,25] r=-1 lpr=71591 pi=70987-71590/26 crt=70990'207 lcod 0'0 remapped NOTIFY] flushed 2014-12-08 18:14:00.490435 7f28fc345700 20 osd.70 pg_epoch: 71591 pg[218.7f( empty local-les=71437 n=0 ec=71423 les/c 71437/71437 71591/71591/71425) [77,70,18] r=1 lpr=71591 pi=71423-71590/8 crt=0'0 inactive NOTIFY] enter NotTrimming 2014-12-08 18:14:00.490435 7f28fcb46700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:14:00.490448 7f2904b56700 10 osd.70 71591 do_waiters -- start 2014-12-08 18:14:00.490444 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[218.7f( empty local-les=71437 n=0 ec=71423 les/c 71437/71437 71591/71591/71425) [77,70,18] r=1 lpr=71591 pi=71423-71590/8 crt=0'0 inactive NOTIFY] on_role_change 2014-12-08 18:14:00.490457 7f2904b56700 10 osd.70 71591 do_waiters -- finish 2014-12-08 18:14:00.490458 7f28fc345700 15 osd.70 pg_epoch: 71591 pg[218.7f( empty local-les=71437 n=0 ec=71423 les/c 71437/71437 71591/71591/71425) [77,70,18] r=1 lpr=71591 pi=71423-71590/8 crt=0'0 inactive NOTIFY] requeue_ops -- 2014-12-08 18:14:00.490478 7f2904b56700 20 osd.70 71591 _dispatch 0x12b79a20 pg_notify(19.6e(150) epoch 71591) v5 2014-12-08 18:14:00.490466 7f290a83a700 10 osd.70 pg_epoch: 71591 pg[218.4b( v 71581'2 (0'0,71581'2] local-les=71448 n=2 ec=71423 les/c 71448/71453 71591/71591/71425) [18,17,70]/[18,17,25] r=-1 lpr=71591 pi=71423-71590/11 crt=0'0 lcod 0'0 remapped NOTIFY] flushed 2014-12-08 18:14:00.490480 7f28fcb46700 10 osd.70 pg_epoch: 71589 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71587/71587/71585) [68,18]/[18,68] r=-1 lpr=71587 pi=71183-71586/21 crt=71377'4110 lcod 0'0 remapped NOTIFY] _calc_past_interval_range: already have past intervals back to 71434 2014-12-08 18:14:00.490485 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[218.7f( empty local-les=71437 n=0 ec=71423 les/c 71437/71437 71591/71591/71425) [77,70,18] r=1 lpr=71591 pi=71423-71590/8 crt=0'0 inactive NOTIFY] check_recovery_sources no source osds () went down 2014-12-08 18:14:00.490503 7f2904b56700 7 osd.70 71591 handle_pg_notify from osd.30 2014-12-08 18:14:00.490505 7f2904b56700 15 osd.70 71591 require_same_or_newer_map 71591 (i am 71591) 0x12b79a20 2014-12-08 18:14:00.490498 7f28fcb46700 20 osd.70 pg_epoch: 71589 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71587/71587/71585) [68,18]/[18,68] r=-1 lpr=71587 pi=71183-71586/21 crt=71377'4110 lcod 0'0 remapped NOTIFY] new interval newup [68,18] newacting [68,18] 2014-12-08 18:14:00.490511 7f28fcb46700 10 osd.70 pg_epoch: 71589 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71587/71587/71585) [68,18]/[18,68] r=-1 lpr=71587 pi=71183-71586/21 crt=71377'4110 lcod 0'0 remapped NOTIFY] state: should restart peering, calling start_peering_interval again 2014-12-08 18:14:00.490517 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[218.7f( empty local-les=71437 n=0 ec=71423 les/c 71437/71437 71591/71591/71425) [77,70,18] r=1 lpr=71591 pi=71423-71590/8 crt=0'0 inactive NOTIFY] handle_activate_map 2014-12-08 18:14:00.490524 7f28fcb46700 20 osd.70 pg_epoch: 71589 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71587/71587/71585) [68,18]/[18,68] r=-1 lpr=71587 pi=71183-71586/21 crt=71377'4110 lcod 0'0 remapped NOTIFY] set_last_peering_reset 71589 2014-12-08 18:14:00.490536 7f2904b56700 10 osd.70 71591 do_waiters -- start 2014-12-08 18:14:00.490538 7f2904b56700 10 osd.70 71591 do_waiters -- finish 2014-12-08 18:14:00.490534 7f28fcb46700 10 osd.70 pg_epoch: 71589 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71587/71587/71585) [68,18]/[18,68] r=-1 lpr=71589 pi=71183-71586/21 crt=71377'4110 lcod 0'0 remapped NOTIFY] Clearing blocked outgoing recovery messages 2014-12-08 18:14:00.490546 7f28fcb46700 10 osd.70 pg_epoch: 71589 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71587/71587/71585) [68,18]/[18,68] r=-1 lpr=71589 pi=71183-71586/21 crt=71377'4110 lcod 0'0 remapped NOTIFY] Not blocking outgoing recovery messages 2014-12-08 18:14:00.490553 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[218.7f( empty local-les=71437 n=0 ec=71423 les/c 71437/71437 71591/71591/71425) [77,70,18] r=1 lpr=71591 pi=71423-71590/8 crt=0'0 inactive NOTIFY] update_heartbeat_peers unchanged 2014-12-08 18:14:00.490560 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[218.7f( empty local-les=71437 n=0 ec=71423 les/c 71437/71437 71591/71591/71425) [77,70,18] r=1 lpr=71591 pi=71423-71590/8 crt=0'0 inactive NOTIFY] take_waiters 2014-12-08 18:14:00.490566 7f28fc345700 5 osd.70 pg_epoch: 71591 pg[218.7f( empty local-les=71437 n=0 ec=71423 les/c 71437/71437 71591/71591/71425) [77,70,18] r=1 lpr=71591 pi=71423-71590/8 crt=0'0 inactive NOTIFY] exit Reset 0.007816 10 0.007550 2014-12-08 18:14:00.490577 7f28fc345700 5 osd.70 pg_epoch: 71591 pg[218.7f( empty local-les=71437 n=0 ec=71423 les/c 71437/71437 71591/71591/71425) [77,70,18] r=1 lpr=71591 pi=71423-71590/8 crt=0'0 inactive NOTIFY] enter Started 2014-12-08 18:14:00.490577 7f28fcb46700 10 osd.70 pg_epoch: 71589 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71434 n=14 ec=71094 les/c 71434/71434 71587/71587/71585) [68,18] r=-1 lpr=71589 pi=71183-71588/22 crt=71377'4110 lcod 0'0 inactive NOTIFY] start_peering_interval: check_new_interval output: generate_past_intervals interval(71587-71588 up [68,18](68) acting [18,68](18)): not rw, up_thru 71587 up_from 69663 last_epoch_clean 71434 generate_past_intervals interval(71587-71588 up [68,18](68) acting [18,68](18) maybe_went_rw) : primary up 69663-71587 includes interval -- 2014-12-08 18:14:00.521500 7f2904b56700 20 osd.70 71591 _dispatch 0x12b76b40 pg_notify(19.6e epoch 71591) v5 2014-12-08 18:14:00.521506 7f28fc345700 10 osd.70 pg_epoch: 71582 pg[218.1c0( v 71581'3 (0'0,71581'3] local-les=71434 n=3 ec=71423 les/c 71434/71434 71582/71582/71425) [75,3] r=-1 lpr=71582 pi=71423-71581/5 crt=71580'1 lcod 0'0 inactive NOTIFY] up [75,70,3] -> [75,3], acting [75,70,3] -> [75,3], acting_primary 75 -> 75, up_primary 75 -> 75, role 1 -> -1 2014-12-08 18:14:00.521515 7f28fcb46700 15 osd.70 pg_epoch: 71587 pg[216.42( v 71097'231 (0'0,71097'231] local-les=71437 n=109 ec=70987 les/c 71437/71437 71587/71587/70989) [9,10] r=-1 lpr=71587 pi=70987-71586/25 crt=71097'229 lcod 0'0 inactive NOTIFY] publish_stats_to_osd -- not primary 2014-12-08 18:14:00.521518 7f28fc345700 10 osd.70 pg_epoch: 71582 pg[218.1c0( v 71581'3 (0'0,71581'3] local-les=71434 n=3 ec=71423 les/c 71434/71434 71582/71582/71425) [75,3] r=-1 lpr=71582 pi=71423-71581/5 crt=71580'1 lcod 0'0 inactive NOTIFY] on_change 2014-12-08 18:14:00.521528 7f28fc345700 15 osd.70 pg_epoch: 71582 pg[218.1c0( v 71581'3 (0'0,71581'3] local-les=71434 n=3 ec=71423 les/c 71434/71434 71582/71582/71425) [75,3] r=-1 lpr=71582 pi=71423-71581/5 crt=71580'1 lcod 0'0 inactive NOTIFY] publish_stats_to_osd -- not primary 2014-12-08 18:14:00.521527 7f28fcb46700 15 osd.70 pg_epoch: 71587 pg[216.42( v 71097'231 (0'0,71097'231] local-les=71437 n=109 ec=70987 les/c 71437/71437 71587/71587/70989) [9,10] r=-1 lpr=71587 pi=70987-71586/25 crt=71097'229 lcod 0'0 inactive NOTIFY] requeue_ops 2014-12-08 18:14:00.521536 7f2904b56700 7 osd.70 71591 handle_pg_notify from osd.63 2014-12-08 18:14:00.521538 7f2904b56700 15 osd.70 71591 require_same_or_newer_map 71591 (i am 71591) 0x12b76b40 2014-12-08 18:14:00.521537 7f28fc345700 15 osd.70 pg_epoch: 71582 pg[218.1c0( v 71581'3 (0'0,71581'3] local-les=71434 n=3 ec=71423 les/c 71434/71434 71582/71582/71425) [75,3] r=-1 lpr=71582 pi=71423-71581/5 crt=71580'1 lcod 0'0 inactive NOTIFY] requeue_ops 2014-12-08 18:14:00.521539 7f28fcb46700 10 osd.70 pg_epoch: 71587 pg[216.42( v 71097'231 (0'0,71097'231] local-les=71437 n=109 ec=70987 les/c 71437/71437 71587/71587/70989) [9,10] r=-1 lpr=71587 pi=70987-71586/25 crt=71097'229 lcod 0'0 inactive NOTIFY] cancel_copy_ops 2014-12-08 18:14:00.521550 7f28fc345700 10 osd.70 pg_epoch: 71582 pg[218.1c0( v 71581'3 (0'0,71581'3] local-les=71434 n=3 ec=71423 les/c 71434/71434 71582/71582/71425) [75,3] r=-1 lpr=71582 pi=71423-71581/5 crt=71580'1 lcod 0'0 inactive NOTIFY] cancel_copy_ops 2014-12-08 18:14:00.521549 7f28fcb46700 10 osd.70 pg_epoch: 71587 pg[216.42( v 71097'231 (0'0,71097'231] local-les=71437 n=109 ec=70987 les/c 71437/71437 71587/71587/70989) [9,10] r=-1 lpr=71587 pi=70987-71586/25 crt=71097'229 lcod 0'0 inactive NOTIFY] cancel_flush_ops 2014-12-08 18:14:00.521558 7f28fc345700 10 osd.70 pg_epoch: 71582 pg[218.1c0( v 71581'3 (0'0,71581'3] local-les=71434 n=3 ec=71423 les/c 71434/71434 71582/71582/71425) [75,3] r=-1 lpr=71582 pi=71423-71581/5 crt=71580'1 lcod 0'0 inactive NOTIFY] cancel_flush_ops 2014-12-08 18:14:00.521566 7f2904b56700 10 osd.70 71591 do_waiters -- start 2014-12-08 18:14:00.521559 7f28fcb46700 10 osd.70 pg_epoch: 71587 pg[216.42( v 71097'231 (0'0,71097'231] local-les=71437 n=109 ec=70987 les/c 71437/71437 71587/71587/70989) [9,10] r=-1 lpr=71587 pi=70987-71586/25 crt=71097'229 lcod 0'0 inactive NOTIFY] on_change_cleanup 2014-12-08 18:14:00.521569 7f2904b56700 10 osd.70 71591 do_waiters -- finish 2014-12-08 18:14:00.521566 7f28fc345700 10 osd.70 pg_epoch: 71582 pg[218.1c0( v 71581'3 (0'0,71581'3] local-les=71434 n=3 ec=71423 les/c 71434/71434 71582/71582/71425) [75,3] r=-1 lpr=71582 pi=71423-71581/5 crt=71580'1 lcod 0'0 inactive NOTIFY] on_change_cleanup 2014-12-08 18:14:00.521580 7f2904b56700 10 osd.70 71591 do_waiters -- start 2014-12-08 18:14:00.521570 7f28fcb46700 10 osd.70 pg_epoch: 71587 pg[216.42( v 71097'231 (0'0,71097'231] local-les=71437 n=109 ec=70987 les/c 71437/71437 71587/71587/70989) [9,10] r=-1 lpr=71587 pi=70987-71586/25 crt=71097'229 lcod 0'0 inactive NOTIFY] on_change 2014-12-08 18:14:00.521576 7f28fc345700 10 osd.70 pg_epoch: 71582 pg[218.1c0( v 71581'3 (0'0,71581'3] local-les=71434 n=3 ec=71423 les/c 71434/71434 71582/71582/71425) [75,3] r=-1 lpr=71582 pi=71423-71581/5 crt=71580'1 lcod 0'0 inactive NOTIFY] on_change 2014-12-08 18:14:00.521583 7f2904b56700 10 osd.70 71591 do_waiters -- finish -- 2014-12-08 18:14:00.521588 7f2904b56700 20 osd.70 71591 _dispatch 0x12b763c0 pg_notify(19.6e epoch 71591) v5 2014-12-08 18:14:00.521590 7f28fc345700 20 osd.70 pg_epoch: 71582 pg[218.1c0( v 71581'3 (0'0,71581'3] local-les=71434 n=3 ec=71423 les/c 71434/71434 71582/71582/71425) [75,3] r=-1 lpr=71582 pi=71423-71581/5 crt=71580'1 lcod 0'0 inactive NOTIFY] exit NotTrimming 2014-12-08 18:14:00.521597 7f28fc345700 20 osd.70 pg_epoch: 71582 pg[218.1c0( v 71581'3 (0'0,71581'3] local-les=71434 n=3 ec=71423 les/c 71434/71434 71582/71582/71425) [75,3] r=-1 lpr=71582 pi=71423-71581/5 crt=71580'1 lcod 0'0 inactive NOTIFY] enter NotTrimming 2014-12-08 18:14:00.521605 7f2904b56700 7 osd.70 71591 handle_pg_notify from osd.56 2014-12-08 18:14:00.521606 7f2904b56700 15 osd.70 71591 require_same_or_newer_map 71591 (i am 71591) 0x12b763c0 2014-12-08 18:14:00.521603 7f28fc345700 10 osd.70 pg_epoch: 71582 pg[218.1c0( v 71581'3 (0'0,71581'3] local-les=71434 n=3 ec=71423 les/c 71434/71434 71582/71582/71425) [75,3] r=-1 lpr=71582 pi=71423-71581/5 crt=71580'1 lcod 0'0 inactive NOTIFY] on_role_change 2014-12-08 18:14:00.521608 7f28fc345700 15 osd.70 pg_epoch: 71582 pg[218.1c0( v 71581'3 (0'0,71581'3] local-les=71434 n=3 ec=71423 les/c 71434/71434 71582/71582/71425) [75,3] r=-1 lpr=71582 pi=71423-71581/5 crt=71580'1 lcod 0'0 inactive NOTIFY] requeue_ops 2014-12-08 18:14:00.521615 7f28fc345700 10 osd.70 pg_epoch: 71582 pg[218.1c0( v 71581'3 (0'0,71581'3] local-les=71434 n=3 ec=71423 les/c 71434/71434 71582/71582/71425) [75,3] r=-1 lpr=71582 pi=71423-71581/5 crt=71580'1 lcod 0'0 inactive NOTIFY] cancel_recovery 2014-12-08 18:14:00.521621 7f2904b56700 10 osd.70 71591 do_waiters -- start 2014-12-08 18:14:00.521622 7f2904b56700 10 osd.70 71591 do_waiters -- finish 2014-12-08 18:14:00.521620 7f28fc345700 10 osd.70 pg_epoch: 71582 pg[218.1c0( v 71581'3 (0'0,71581'3] local-les=71434 n=3 ec=71423 les/c 71434/71434 71582/71582/71425) [75,3] r=-1 lpr=71582 pi=71423-71581/5 crt=71580'1 lcod 0'0 inactive NOTIFY] clear_recovery_state 2014-12-08 18:14:00.521627 7f28fc345700 10 osd.70 pg_epoch: 71582 pg[218.1c0( v 71581'3 (0'0,71581'3] local-les=71434 n=3 ec=71423 les/c 71434/71434 71582/71582/71425) [75,3] r=-1 lpr=71582 pi=71423-71581/5 crt=71580'1 lcod 0'0 inactive NOTIFY] check_recovery_sources no source osds () went down 2014-12-08 18:14:00.521640 7f28fc345700 10 osd.70 pg_epoch: 71582 pg[218.1c0( v 71581'3 (0'0,71581'3] local-les=71434 n=3 ec=71423 les/c 71434/71434 71582/71582/71425) [75,3] r=-1 lpr=71582 pi=71423-71581/5 crt=71580'1 lcod 0'0 inactive NOTIFY] handle_advance_map [75,3]/[75,3] -- 75/75 2014-12-08 18:14:00.521647 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:14:00.521649 7f28fc345700 10 osd.70 pg_epoch: 71583 pg[218.1c0( v 71581'3 (0'0,71581'3] local-les=71434 n=3 ec=71423 les/c 71434/71434 71582/71582/71425) [75,3] r=-1 lpr=71582 pi=71423-71581/5 crt=71580'1 lcod 0'0 inactive NOTIFY] state: Reset advmap 2014-12-08 18:14:00.521655 7f28fc345700 10 osd.70 pg_epoch: 71583 pg[218.1c0( v 71581'3 (0'0,71581'3] local-les=71434 n=3 ec=71423 les/c 71434/71434 71582/71582/71425) [75,3] r=-1 lpr=71582 pi=71423-71581/5 crt=71580'1 lcod 0'0 inactive NOTIFY] _calc_past_interval_range: already have past intervals back to 71434 2014-12-08 18:14:00.521661 7f28fc345700 10 osd.70 pg_epoch: 71583 pg[218.1c0( v 71581'3 (0'0,71581'3] local-les=71434 n=3 ec=71423 les/c 71434/71434 71582/71582/71425) [75,3] r=-1 lpr=71582 pi=71423-71581/5 crt=71580'1 lcod 0'0 inactive NOTIFY] check_recovery_sources no source osds () went down 2014-12-08 18:14:00.521672 7f28fc345700 10 osd.70 pg_epoch: 71583 pg[218.1c0( v 71581'3 (0'0,71581'3] local-les=71434 n=3 ec=71423 les/c 71434/71434 71582/71582/71425) [75,3] r=-1 lpr=71582 pi=71423-71581/5 crt=71580'1 lcod 0'0 inactive NOTIFY] handle_advance_map [75,6,7]/[75,6,7] -- 75/75 2014-12-08 18:14:00.521679 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:14:00.521681 7f28fc345700 10 osd.70 pg_epoch: 71584 pg[218.1c0( v 71581'3 (0'0,71581'3] local-les=71434 n=3 ec=71423 les/c 71434/71434 71582/71582/71425) [75,3] r=-1 lpr=71582 pi=71423-71581/5 crt=71580'1 lcod 0'0 inactive NOTIFY] state: Reset advmap 2014-12-08 18:14:00.521686 7f28fc345700 10 osd.70 pg_epoch: 71584 pg[218.1c0( v 71581'3 (0'0,71581'3] local-les=71434 n=3 ec=71423 les/c 71434/71434 71582/71582/71425) [75,3] r=-1 lpr=71582 pi=71423-71581/5 crt=71580'1 lcod 0'0 inactive NOTIFY] _calc_past_interval_range: already have past intervals back to 71434 -- 2014-12-08 18:14:00.526202 7f2904b56700 20 osd.70 71591 _dispatch 0x12c18940 pg_notify(19.6e epoch 71591) v5 2014-12-08 18:14:00.526196 7f28fc345700 10 osd.70 pg_epoch: 71587 pg[218.1c0( v 71581'3 (0'0,71581'3] local-les=71434 n=3 ec=71423 les/c 71434/71434 71587/71587/71425) [75,3] r=-1 lpr=71587 pi=71423-71586/7 crt=71580'1 lcod 0'0 inactive NOTIFY] check_recovery_sources no source osds () went down 2014-12-08 18:14:00.526189 7f28fcb46700 10 osd.70 pg_epoch: 71589 pg[216.42( v 71097'231 (0'0,71097'231] local-les=71437 n=109 ec=70987 les/c 71437/71437 71587/71589/70989) [9,10]/[9,10,76] r=-1 lpr=71589 pi=70987-71588/26 crt=71097'229 lcod 0'0 remapped NOTIFY] clear_recovery_state 2014-12-08 18:14:00.526222 7f2904b56700 7 osd.70 71591 handle_pg_notify from osd.20 2014-12-08 18:14:00.526224 7f2904b56700 15 osd.70 71591 require_same_or_newer_map 71591 (i am 71591) 0x12c18940 2014-12-08 18:14:00.526216 7f28fc345700 10 osd.70 pg_epoch: 71587 pg[218.1c0( v 71581'3 (0'0,71581'3] local-les=71434 n=3 ec=71423 les/c 71434/71434 71587/71587/71425) [75,3] r=-1 lpr=71587 pi=71423-71586/7 crt=71580'1 lcod 0'0 inactive NOTIFY] handle_advance_map [75,3]/[75,3] -- 75/75 2014-12-08 18:14:00.526229 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:14:00.526216 7f28fcb46700 10 osd.70 pg_epoch: 71589 pg[216.42( v 71097'231 (0'0,71097'231] local-les=71437 n=109 ec=70987 les/c 71437/71437 71587/71589/70989) [9,10]/[9,10,76] r=-1 lpr=71589 pi=70987-71588/26 crt=71097'229 lcod 0'0 remapped NOTIFY] check_recovery_sources no source osds () went down 2014-12-08 18:14:00.526234 7f28fc345700 10 osd.70 pg_epoch: 71588 pg[218.1c0( v 71581'3 (0'0,71581'3] local-les=71434 n=3 ec=71423 les/c 71434/71434 71587/71587/71425) [75,3] r=-1 lpr=71587 pi=71423-71586/7 crt=71580'1 lcod 0'0 inactive NOTIFY] state: Reset advmap 2014-12-08 18:14:00.526240 7f28fc345700 10 osd.70 pg_epoch: 71588 pg[218.1c0( v 71581'3 (0'0,71581'3] local-les=71434 n=3 ec=71423 les/c 71434/71434 71587/71587/71425) [75,3] r=-1 lpr=71587 pi=71423-71586/7 crt=71580'1 lcod 0'0 inactive NOTIFY] _calc_past_interval_range: already have past intervals back to 71434 2014-12-08 18:14:00.526249 7f2904b56700 10 osd.70 71591 do_waiters -- start 2014-12-08 18:14:00.526251 7f2904b56700 10 osd.70 71591 do_waiters -- finish 2014-12-08 18:14:00.526248 7f28fc345700 10 osd.70 pg_epoch: 71588 pg[218.1c0( v 71581'3 (0'0,71581'3] local-les=71434 n=3 ec=71423 les/c 71434/71434 71587/71587/71425) [75,3] r=-1 lpr=71587 pi=71423-71586/7 crt=71580'1 lcod 0'0 inactive NOTIFY] check_recovery_sources no source osds () went down 2014-12-08 18:14:00.526260 7f2904b56700 10 osd.70 71591 do_waiters -- start 2014-12-08 18:14:00.526261 7f2904b56700 10 osd.70 71591 do_waiters -- finish 2014-12-08 18:14:00.526262 7f2904b56700 20 osd.70 71591 _dispatch 0x12cedc00 pg_notify(217.214(37) epoch 71591) v5 2014-12-08 18:14:00.526263 7f28fc345700 10 osd.70 pg_epoch: 71588 pg[218.1c0( v 71581'3 (0'0,71581'3] local-les=71434 n=3 ec=71423 les/c 71434/71434 71587/71587/71425) [75,3] r=-1 lpr=71587 pi=71423-71586/7 crt=71580'1 lcod 0'0 inactive NOTIFY] handle_advance_map [75,3]/[75,3] -- 75/75 2014-12-08 18:14:00.526251 7f28fcb46700 10 osd.70 pg_epoch: 71589 pg[216.42( v 71097'231 (0'0,71097'231] local-les=71437 n=109 ec=70987 les/c 71437/71437 71587/71589/70989) [9,10]/[9,10,76] r=-1 lpr=71589 pi=70987-71588/26 crt=71097'229 lcod 0'0 remapped NOTIFY] handle_advance_map [9,10]/[9,10,76] -- 9/9 2014-12-08 18:14:00.526277 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:14:00.526280 7f28fcb46700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:14:00.526284 7f2904b56700 7 osd.70 71591 handle_pg_notify from osd.4 -- 2014-12-08 18:14:00.526390 7f2904b56700 20 osd.70 71591 _dispatch 0x12c18b20 pg_notify(19.6e(150) epoch 71591) v5 2014-12-08 18:14:00.526387 7f28fc345700 10 osd.70 pg_epoch: 71590 pg[218.1c0( v 71581'3 (0'0,71581'3] local-les=71434 n=3 ec=71423 les/c 71434/71434 71587/71587/71425) [75,3] r=-1 lpr=71590 pi=71423-71586/7 crt=71580'1 lcod 0'0 inactive NOTIFY] Not blocking outgoing recovery messages 2014-12-08 18:14:00.526392 7f28fcb46700 20 osd.70 pg_epoch: 71591 pg[216.42( v 71097'231 (0'0,71097'231] local-les=71437 n=109 ec=70987 les/c 71437/71437 71587/71589/70989) [9,10]/[9,10,76] r=-1 lpr=71589 pi=70987-71588/26 crt=71097'229 lcod 0'0 remapped NOTIFY] new interval newup [9,10,70] newacting [9,10,76] 2014-12-08 18:14:00.526409 7f2904b56700 7 osd.70 71591 handle_pg_notify from osd.4 2014-12-08 18:14:00.526404 7f28fc345700 10 osd.70 pg_epoch: 71590 pg[218.1c0( v 71581'3 (0'0,71581'3] local-les=71434 n=3 ec=71423 les/c 71434/71434 71587/71587/71425) [75,3]/[75,3,6] r=-1 lpr=71590 pi=71423-71589/8 crt=71580'1 lcod 0'0 remapped NOTIFY] start_peering_interval: check_new_interval output: generate_past_intervals interval(71587-71589 up [75,3](75) acting [75,3](75)): not rw, up_thru 71588 up_from 69683 last_epoch_clean 71434 generate_past_intervals interval(71587-71589 up [75,3](75) acting [75,3](75) maybe_went_rw) : primary up 69683-71588 includes interval 2014-12-08 18:14:00.526414 7f2904b56700 15 osd.70 71591 require_same_or_newer_map 71591 (i am 71591) 0x12c18b20 2014-12-08 18:14:00.526414 7f28fc345700 10 osd.70 pg_epoch: 71590 pg[218.1c0( v 71581'3 (0'0,71581'3] local-les=71434 n=3 ec=71423 les/c 71434/71434 71587/71587/71425) [75,3]/[75,3,6] r=-1 lpr=71590 pi=71423-71589/8 crt=71580'1 lcod 0'0 remapped NOTIFY] noting past interval(71587-71589 up [75,3](75) acting [75,3](75) maybe_went_rw) 2014-12-08 18:14:00.526414 7f28fcb46700 10 osd.70 pg_epoch: 71591 pg[216.42( v 71097'231 (0'0,71097'231] local-les=71437 n=109 ec=70987 les/c 71437/71437 71587/71589/70989) [9,10]/[9,10,76] r=-1 lpr=71589 pi=70987-71588/26 crt=71097'229 lcod 0'0 remapped NOTIFY] state: should restart peering, calling start_peering_interval again 2014-12-08 18:14:00.526435 7f2904b56700 10 osd.70 71591 do_waiters -- start 2014-12-08 18:14:00.526436 7f2904b56700 10 osd.70 71591 do_waiters -- finish 2014-12-08 18:14:00.526426 7f28fc345700 10 osd.70 pg_epoch: 71590 pg[218.1c0( v 71581'3 (0'0,71581'3] local-les=71434 n=3 ec=71423 les/c 71434/71434 71587/71590/71425) [75,3]/[75,3,6] r=-1 lpr=71590 pi=71423-71589/8 crt=71580'1 lcod 0'0 remapped NOTIFY] up [75,3] -> [75,3], acting [75,3] -> [75,3,6], acting_primary 75 -> 75, up_primary 75 -> 75, role -1 -> -1 2014-12-08 18:14:00.526438 7f28fc345700 10 osd.70 pg_epoch: 71590 pg[218.1c0( v 71581'3 (0'0,71581'3] local-les=71434 n=3 ec=71423 les/c 71434/71434 71587/71590/71425) [75,3]/[75,3,6] r=-1 lpr=71590 pi=71423-71589/8 crt=71580'1 lcod 0'0 remapped NOTIFY] on_change 2014-12-08 18:14:00.526436 7f28fcb46700 20 osd.70 pg_epoch: 71591 pg[216.42( v 71097'231 (0'0,71097'231] local-les=71437 n=109 ec=70987 les/c 71437/71437 71587/71589/70989) [9,10]/[9,10,76] r=-1 lpr=71589 pi=70987-71588/26 crt=71097'229 lcod 0'0 remapped NOTIFY] set_last_peering_reset 71591 2014-12-08 18:14:00.526447 7f28fc345700 15 osd.70 pg_epoch: 71590 pg[218.1c0( v 71581'3 (0'0,71581'3] local-les=71434 n=3 ec=71423 les/c 71434/71434 71587/71590/71425) [75,3]/[75,3,6] r=-1 lpr=71590 pi=71423-71589/8 crt=71580'1 lcod 0'0 remapped NOTIFY] publish_stats_to_osd -- not primary 2014-12-08 18:14:00.526456 7f28fc345700 15 osd.70 pg_epoch: 71590 pg[218.1c0( v 71581'3 (0'0,71581'3] local-les=71434 n=3 ec=71423 les/c 71434/71434 71587/71590/71425) [75,3]/[75,3,6] r=-1 lpr=71590 pi=71423-71589/8 crt=71580'1 lcod 0'0 remapped NOTIFY] requeue_ops 2014-12-08 18:14:00.526455 7f28fcb46700 10 osd.70 pg_epoch: 71591 pg[216.42( v 71097'231 (0'0,71097'231] local-les=71437 n=109 ec=70987 les/c 71437/71437 71587/71589/70989) [9,10]/[9,10,76] r=-1 lpr=71591 pi=70987-71588/26 crt=71097'229 lcod 0'0 remapped NOTIFY] Clearing blocked outgoing recovery messages 2014-12-08 18:14:00.526465 7f28fc345700 10 osd.70 pg_epoch: 71590 pg[218.1c0( v 71581'3 (0'0,71581'3] local-les=71434 n=3 ec=71423 les/c 71434/71434 71587/71590/71425) [75,3]/[75,3,6] r=-1 lpr=71590 pi=71423-71589/8 crt=71580'1 lcod 0'0 remapped NOTIFY] cancel_copy_ops 2014-12-08 18:14:00.526472 7f28fc345700 10 osd.70 pg_epoch: 71590 pg[218.1c0( v 71581'3 (0'0,71581'3] local-les=71434 n=3 ec=71423 les/c 71434/71434 71587/71590/71425) [75,3]/[75,3,6] r=-1 lpr=71590 pi=71423-71589/8 crt=71580'1 lcod 0'0 remapped NOTIFY] cancel_flush_ops 2014-12-08 18:14:00.526471 7f28fcb46700 10 osd.70 pg_epoch: 71591 pg[216.42( v 71097'231 (0'0,71097'231] local-les=71437 n=109 ec=70987 les/c 71437/71437 71587/71589/70989) [9,10]/[9,10,76] r=-1 lpr=71591 pi=70987-71588/26 crt=71097'229 lcod 0'0 remapped NOTIFY] Not blocking outgoing recovery messages -- 2014-12-08 18:14:00.549070 7f2904b56700 20 osd.70 71591 _dispatch 0x12c17e00 pg_notify(19.6e(150) epoch 71591) v5 2014-12-08 18:14:00.549075 7f28fc345700 10 osd.70 pg_epoch: 71582 pg[217.350( v 71377'4121 (71377'1120,71377'4121] local-les=71446 n=17 ec=71094 les/c 71446/71448 71444/71444/71096) [13,70,7] r=1 lpr=71448 pi=71183-71443/19 crt=71377'4119 lcod 0'0 inactive NOTIFY] state: Started advmap 2014-12-08 18:14:00.549088 7f28fc345700 20 osd.70 pg_epoch: 71582 pg[217.350( v 71377'4121 (71377'1120,71377'4121] local-les=71446 n=17 ec=71094 les/c 71446/71448 71444/71444/71096) [13,70,7] r=1 lpr=71448 pi=71183-71443/19 crt=71377'4119 lcod 0'0 inactive NOTIFY] new interval newup [13,7] newacting [13,7] 2014-12-08 18:14:00.549077 7f28fcb46700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:14:00.549100 7f28fc345700 10 osd.70 pg_epoch: 71582 pg[217.350( v 71377'4121 (71377'1120,71377'4121] local-les=71446 n=17 ec=71094 les/c 71446/71448 71444/71444/71096) [13,70,7] r=1 lpr=71448 pi=71183-71443/19 crt=71377'4119 lcod 0'0 inactive NOTIFY] state: should_restart_peering, transitioning to Reset 2014-12-08 18:14:00.549109 7f28fcb46700 10 osd.70 pg_epoch: 71585 pg[216.1d0( v 71097'265 (0'0,71097'265] local-les=71437 n=123 ec=70987 les/c 71437/71437 71584/71584/71584) [22,13,20] r=-1 lpr=71584 pi=71432-71583/2 crt=71097'265 lcod 0'0 inactive NOTIFY] state: Reset advmap 2014-12-08 18:14:00.549124 7f2904b56700 7 osd.70 71591 handle_pg_notify from osd.76 2014-12-08 18:14:00.549116 7f28fc345700 5 osd.70 pg_epoch: 71582 pg[217.350( v 71377'4121 (71377'1120,71377'4121] local-les=71446 n=17 ec=71094 les/c 71446/71448 71444/71444/71096) [13,70,7] r=1 lpr=71448 pi=71183-71443/19 crt=71377'4119 lcod 0'0 inactive NOTIFY] exit Started/Stray 2.590350 3 0.000134 2014-12-08 18:14:00.549128 7f2904b56700 15 osd.70 71591 require_same_or_newer_map 71591 (i am 71591) 0x12c17e00 2014-12-08 18:14:00.549122 7f28fcb46700 10 osd.70 pg_epoch: 71585 pg[216.1d0( v 71097'265 (0'0,71097'265] local-les=71437 n=123 ec=70987 les/c 71437/71437 71584/71584/71584) [22,13,20] r=-1 lpr=71584 pi=71432-71583/2 crt=71097'265 lcod 0'0 inactive NOTIFY] _calc_past_interval_range: already have past intervals back to 71437 2014-12-08 18:14:00.549132 7f28fc345700 5 osd.70 pg_epoch: 71582 pg[217.350( v 71377'4121 (71377'1120,71377'4121] local-les=71446 n=17 ec=71094 les/c 71446/71448 71444/71444/71096) [13,70,7] r=1 lpr=71448 pi=71183-71443/19 crt=71377'4119 lcod 0'0 inactive NOTIFY] exit Started 2.590407 0 0.000000 2014-12-08 18:14:00.549136 7f28fcb46700 10 osd.70 pg_epoch: 71585 pg[216.1d0( v 71097'265 (0'0,71097'265] local-les=71437 n=123 ec=70987 les/c 71437/71437 71584/71584/71584) [22,13,20] r=-1 lpr=71584 pi=71432-71583/2 crt=71097'265 lcod 0'0 inactive NOTIFY] check_recovery_sources no source osds () went down 2014-12-08 18:14:00.549149 7f28fc345700 5 osd.70 pg_epoch: 71582 pg[217.350( v 71377'4121 (71377'1120,71377'4121] local-les=71446 n=17 ec=71094 les/c 71446/71448 71444/71444/71096) [13,70,7] r=1 lpr=71448 pi=71183-71443/19 crt=71377'4119 lcod 0'0 inactive NOTIFY] enter Reset 2014-12-08 18:14:00.549157 7f28fcb46700 10 osd.70 pg_epoch: 71585 pg[216.1d0( v 71097'265 (0'0,71097'265] local-les=71437 n=123 ec=70987 les/c 71437/71437 71584/71584/71584) [22,13,20] r=-1 lpr=71584 pi=71432-71583/2 crt=71097'265 lcod 0'0 inactive NOTIFY] handle_advance_map [22,13,20]/[22,13,20] -- 22/22 2014-12-08 18:14:00.549172 7f28fcb46700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:14:00.549163 7f28fc345700 20 osd.70 pg_epoch: 71582 pg[217.350( v 71377'4121 (71377'1120,71377'4121] local-les=71446 n=17 ec=71094 les/c 71446/71448 71444/71444/71096) [13,70,7] r=1 lpr=71448 pi=71183-71443/19 crt=71377'4119 lcod 0'0 inactive NOTIFY] set_last_peering_reset 71582 2014-12-08 18:14:00.549177 7f2904b56700 10 osd.70 71591 do_waiters -- start 2014-12-08 18:14:00.549182 7f2904b56700 10 osd.70 71591 do_waiters -- finish 2014-12-08 18:14:00.549175 7f28fcb46700 10 osd.70 pg_epoch: 71586 pg[216.1d0( v 71097'265 (0'0,71097'265] local-les=71437 n=123 ec=70987 les/c 71437/71437 71584/71584/71584) [22,13,20] r=-1 lpr=71584 pi=71432-71583/2 crt=71097'265 lcod 0'0 inactive NOTIFY] state: Reset advmap 2014-12-08 18:14:00.549178 7f28fc345700 10 osd.70 pg_epoch: 71582 pg[217.350( v 71377'4121 (71377'1120,71377'4121] local-les=71446 n=17 ec=71094 les/c 71446/71448 71444/71444/71096) [13,70,7] r=1 lpr=71582 pi=71183-71443/19 crt=71377'4119 lcod 0'0 inactive NOTIFY] Clearing blocked outgoing recovery messages 2014-12-08 18:14:00.549189 7f28fcb46700 10 osd.70 pg_epoch: 71586 pg[216.1d0( v 71097'265 (0'0,71097'265] local-les=71437 n=123 ec=70987 les/c 71437/71437 71584/71584/71584) [22,13,20] r=-1 lpr=71584 pi=71432-71583/2 crt=71097'265 lcod 0'0 inactive NOTIFY] _calc_past_interval_range: already have past intervals back to 71437 -- 2014-12-08 18:14:00.551049 7f2904b56700 20 osd.70 71591 _dispatch 0x12cec940 pg_notify(19.6e epoch 71591) v5 2014-12-08 18:14:00.551050 7f28fc345700 1 osd.70 pg_epoch: 71591 pg[217.350( v 71377'4121 (71377'1120,71377'4121] local-les=71446 n=17 ec=71094 les/c 71446/71448 71591/71591/71096) [13,70,7] r=1 lpr=71591 pi=71183-71590/24 crt=71377'4119 lcod 0'0 inactive NOTIFY] state: transitioning to Stray 2014-12-08 18:14:00.551066 7f28fc345700 5 osd.70 pg_epoch: 71591 pg[217.350( v 71377'4121 (71377'1120,71377'4121] local-les=71446 n=17 ec=71094 les/c 71446/71448 71591/71591/71096) [13,70,7] r=1 lpr=71591 pi=71183-71590/24 crt=71377'4119 lcod 0'0 inactive NOTIFY] exit Start 0.000028 0 0.000000 2014-12-08 18:14:00.551085 7f2904b56700 7 osd.70 71591 handle_pg_notify from osd.0 2014-12-08 18:14:00.551080 7f28fc345700 5 osd.70 pg_epoch: 71591 pg[217.350( v 71377'4121 (71377'1120,71377'4121] local-les=71446 n=17 ec=71094 les/c 71446/71448 71591/71591/71096) [13,70,7] r=1 lpr=71591 pi=71183-71590/24 crt=71377'4119 lcod 0'0 inactive NOTIFY] enter Started/Stray 2014-12-08 18:14:00.551088 7f2904b56700 15 osd.70 71591 require_same_or_newer_map 71591 (i am 71591) 0x12cec940 2014-12-08 18:14:00.551095 7f28fc345700 20 osd.70 pg_epoch: 71591 pg[217.350( v 71377'4121 (71377'1120,71377'4121] local-les=71446 n=17 ec=71094 les/c 71446/71448 71591/71591/71096) [13,70,7] r=1 lpr=71591 pi=71183-71590/24 crt=71377'4119 lcod 0'0 inactive NOTIFY] handle_activate_map: Not dirtying info: last_persisted is 71448 while current is 71591 2014-12-08 18:14:00.551106 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[217.350( v 71377'4121 (71377'1120,71377'4121] local-les=71446 n=17 ec=71094 les/c 71446/71448 71591/71591/71096) [13,70,7] r=1 lpr=71591 pi=71183-71590/24 crt=71377'4119 lcod 0'0 inactive NOTIFY] handle_peering_event: epoch_sent: 71591 epoch_requested: 71591 NullEvt 2014-12-08 18:14:00.551127 7f2904b56700 10 osd.70 71591 do_waiters -- start 2014-12-08 18:14:00.551130 7f2904b56700 10 osd.70 71591 do_waiters -- finish 2014-12-08 18:14:00.551159 7f28fc345700 10 log is not dirty 2014-12-08 18:14:00.551191 7f28fc345700 10 osd.70 pg_epoch: 71581 pg[217.1fa( v 71377'4113 (71377'1112,71377'4113] local-les=71440 n=11 ec=71094 les/c 71440/71440 71432/71439/71439) [70,14,69] r=0 lpr=71440 crt=71377'4110 lcod 0'0 mlcod 0'0 peering] handle_advance_map [14,69]/[14,69] -- 14/14 2014-12-08 18:14:00.551208 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:14:00.551214 7f28fc345700 10 osd.70 pg_epoch: 71582 pg[217.1fa( v 71377'4113 (71377'1112,71377'4113] local-les=71440 n=11 ec=71094 les/c 71440/71440 71432/71439/71439) [70,14,69] r=0 lpr=71440 crt=71377'4110 lcod 0'0 mlcod 0'0 peering] state: Peering advmap 2014-12-08 18:14:00.551223 7f28fc345700 10 osd.70 pg_epoch: 71582 pg[217.1fa( v 71377'4113 (71377'1112,71377'4113] local-les=71440 n=11 ec=71094 les/c 71440/71440 71432/71439/71439) [70,14,69] r=0 lpr=71440 crt=71377'4110 lcod 0'0 mlcod 0'0 peering] PriorSet: affected_by_map osd.70 now down 2014-12-08 18:14:00.551232 7f28fc345700 1 osd.70 pg_epoch: 71582 pg[217.1fa( v 71377'4113 (71377'1112,71377'4113] local-les=71440 n=11 ec=71094 les/c 71440/71440 71432/71439/71439) [70,14,69] r=0 lpr=71440 crt=71377'4110 lcod 0'0 mlcod 0'0 peering] state: Peering, affected_by_map, going to Reset 2014-12-08 18:14:00.551242 7f28fc345700 5 osd.70 pg_epoch: 71582 pg[217.1fa( v 71377'4113 (71377'1112,71377'4113] local-les=71440 n=11 ec=71094 les/c 71440/71440 71432/71439/71439) [70,14,69] r=0 lpr=71440 crt=71377'4110 lcod 0'0 mlcod 0'0 peering] exit Started/Primary/Peering/GetInfo 2.576648 2 0.000155 2014-12-08 18:14:00.551254 7f28fc345700 10 osd.70 pg_epoch: 71582 pg[217.1fa( v 71377'4113 (71377'1112,71377'4113] local-les=71440 n=11 ec=71094 les/c 71440/71440 71432/71439/71439) [70,14,69] r=0 lpr=71440 crt=71377'4110 lcod 0'0 mlcod 0'0 peering] state: Leaving Peering 2014-12-08 18:14:00.551263 7f28fc345700 5 osd.70 pg_epoch: 71582 pg[217.1fa( v 71377'4113 (71377'1112,71377'4113] local-les=71440 n=11 ec=71094 les/c 71440/71440 71432/71439/71439) [70,14,69] r=0 lpr=71440 crt=71377'4110 lcod 0'0 mlcod 0'0 peering] exit Started/Primary/Peering 2.576677 0 0.000000 2014-12-08 18:14:00.551276 7f28fc345700 5 osd.70 pg_epoch: 71582 pg[217.1fa( v 71377'4113 (71377'1112,71377'4113] local-les=71440 n=11 ec=71094 les/c 71440/71440 71432/71439/71439) [70,14,69] r=0 lpr=71440 crt=71377'4110 lcod 0'0 mlcod 0'0 inactive] exit Started/Primary 2.576696 0 0.000000 2014-12-08 18:14:00.551287 7f28fc345700 5 osd.70 pg_epoch: 71582 pg[217.1fa( v 71377'4113 (71377'1112,71377'4113] local-les=71440 n=11 ec=71094 les/c 71440/71440 71432/71439/71439) [70,14,69] r=0 lpr=71440 crt=71377'4110 lcod 0'0 mlcod 0'0 inactive] exit Started 2.576746 0 0.000000 -- 2014-12-08 18:14:00.630932 7f2904b56700 20 osd.70 71591 _dispatch 0x12b761e0 pg_notify(19.6e epoch 71591) v5 2014-12-08 18:14:00.630983 7f2904b56700 7 osd.70 71591 handle_pg_notify from osd.59 2014-12-08 18:14:00.630986 7f2904b56700 15 osd.70 71591 require_same_or_newer_map 71591 (i am 71591) 0x12b761e0 2014-12-08 18:14:00.630989 7f2902351700 20 osd.70 71591 share_map_peer 0x12769700 already has epoch 71591 2014-12-08 18:14:00.631020 7f2904b56700 10 osd.70 71591 do_waiters -- start 2014-12-08 18:14:00.631022 7f2904b56700 10 osd.70 71591 do_waiters -- finish 2014-12-08 18:14:00.630935 7f28fcb46700 10 osd.70 pg_epoch: 71589 pg[218.38( v 71581'3 (0'0,71581'3] local-les=71434 n=3 ec=71423 les/c 71434/71434 71587/71589/71425) [56,17]/[56,17,15] r=-1 lpr=71589 pi=71423-71588/8 crt=71580'1 lcod 0'0 remapped NOTIFY] up [56,17] -> [56,17], acting [56,17] -> [56,17,15], acting_primary 56 -> 56, up_primary 56 -> 56, role -1 -> -1 2014-12-08 18:14:00.631075 7f28fcb46700 10 osd.70 pg_epoch: 71589 pg[218.38( v 71581'3 (0'0,71581'3] local-les=71434 n=3 ec=71423 les/c 71434/71434 71587/71589/71425) [56,17]/[56,17,15] r=-1 lpr=71589 pi=71423-71588/8 crt=71580'1 lcod 0'0 remapped NOTIFY] on_change 2014-12-08 18:14:00.631074 7f28fc345700 10 osd.70 pg_epoch: 71586 pg[217.2e( v 71377'4787 (71377'1786,71377'4787] local-les=71436 n=13 ec=71094 les/c 71436/71436 71584/71585/71585) [21,68,16]/[16,68] r=-1 lpr=71585 pi=71435-71584/3 crt=71377'4784 lcod 0'0 remapped NOTIFY] handle_advance_map [68,16]/[16,68] -- 68/16 2014-12-08 18:14:00.631090 7f28fcb46700 15 osd.70 pg_epoch: 71589 pg[218.38( v 71581'3 (0'0,71581'3] local-les=71434 n=3 ec=71423 les/c 71434/71434 71587/71589/71425) [56,17]/[56,17,15] r=-1 lpr=71589 pi=71423-71588/8 crt=71580'1 lcod 0'0 remapped NOTIFY] publish_stats_to_osd -- not primary 2014-12-08 18:14:00.631101 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:14:00.631101 7f28fcb46700 15 osd.70 pg_epoch: 71589 pg[218.38( v 71581'3 (0'0,71581'3] local-les=71434 n=3 ec=71423 les/c 71434/71434 71587/71589/71425) [56,17]/[56,17,15] r=-1 lpr=71589 pi=71423-71588/8 crt=71580'1 lcod 0'0 remapped NOTIFY] requeue_ops 2014-12-08 18:14:00.631110 7f28fc345700 10 osd.70 pg_epoch: 71587 pg[217.2e( v 71377'4787 (71377'1786,71377'4787] local-les=71436 n=13 ec=71094 les/c 71436/71436 71584/71585/71585) [21,68,16]/[16,68] r=-1 lpr=71585 pi=71435-71584/3 crt=71377'4784 lcod 0'0 remapped NOTIFY] state: Reset advmap 2014-12-08 18:14:00.631122 7f28fc345700 10 osd.70 pg_epoch: 71587 pg[217.2e( v 71377'4787 (71377'1786,71377'4787] local-les=71436 n=13 ec=71094 les/c 71436/71436 71584/71585/71585) [21,68,16]/[16,68] r=-1 lpr=71585 pi=71435-71584/3 crt=71377'4784 lcod 0'0 remapped NOTIFY] _calc_past_interval_range: already have past intervals back to 71436 2014-12-08 18:14:00.631126 7f28fcb46700 10 osd.70 pg_epoch: 71589 pg[218.38( v 71581'3 (0'0,71581'3] local-les=71434 n=3 ec=71423 les/c 71434/71434 71587/71589/71425) [56,17]/[56,17,15] r=-1 lpr=71589 pi=71423-71588/8 crt=71580'1 lcod 0'0 remapped NOTIFY] cancel_copy_ops 2014-12-08 18:14:00.631137 7f28fcb46700 10 osd.70 pg_epoch: 71589 pg[218.38( v 71581'3 (0'0,71581'3] local-les=71434 n=3 ec=71423 les/c 71434/71434 71587/71589/71425) [56,17]/[56,17,15] r=-1 lpr=71589 pi=71423-71588/8 crt=71580'1 lcod 0'0 remapped NOTIFY] cancel_flush_ops 2014-12-08 18:14:00.631136 7f28fc345700 20 osd.70 pg_epoch: 71587 pg[217.2e( v 71377'4787 (71377'1786,71377'4787] local-les=71436 n=13 ec=71094 les/c 71436/71436 71584/71585/71585) [21,68,16]/[16,68] r=-1 lpr=71585 pi=71435-71584/3 crt=71377'4784 lcod 0'0 remapped NOTIFY] new interval newup [68,16] newacting [16,68] 2014-12-08 18:14:00.631147 7f28fc345700 10 osd.70 pg_epoch: 71587 pg[217.2e( v 71377'4787 (71377'1786,71377'4787] local-les=71436 n=13 ec=71094 les/c 71436/71436 71584/71585/71585) [21,68,16]/[16,68] r=-1 lpr=71585 pi=71435-71584/3 crt=71377'4784 lcod 0'0 remapped NOTIFY] state: should restart peering, calling start_peering_interval again 2014-12-08 18:14:00.631149 7f28fcb46700 10 osd.70 pg_epoch: 71589 pg[218.38( v 71581'3 (0'0,71581'3] local-les=71434 n=3 ec=71423 les/c 71434/71434 71587/71589/71425) [56,17]/[56,17,15] r=-1 lpr=71589 pi=71423-71588/8 crt=71580'1 lcod 0'0 remapped NOTIFY] on_change_cleanup 2014-12-08 18:14:00.631159 7f28fc345700 20 osd.70 pg_epoch: 71587 pg[217.2e( v 71377'4787 (71377'1786,71377'4787] local-les=71436 n=13 ec=71094 les/c 71436/71436 71584/71585/71585) [21,68,16]/[16,68] r=-1 lpr=71585 pi=71435-71584/3 crt=71377'4784 lcod 0'0 remapped NOTIFY] set_last_peering_reset 71587 2014-12-08 18:14:00.631160 7f28fcb46700 10 osd.70 pg_epoch: 71589 pg[218.38( v 71581'3 (0'0,71581'3] local-les=71434 n=3 ec=71423 les/c 71434/71434 71587/71589/71425) [56,17]/[56,17,15] r=-1 lpr=71589 pi=71423-71588/8 crt=71580'1 lcod 0'0 remapped NOTIFY] on_change -- 2014-12-08 18:14:00.765763 7f2904b56700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] null 2014-12-08 18:14:00.765776 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[218.338( empty local-les=71436 n=0 ec=71423 les/c 71436/71436 71591/71591/71425) [23,63,70]/[23,63,21] r=-1 lpr=71591 pi=71423-71590/8 crt=0'0 remapped NOTIFY] check_recovery_sources no source osds () went down 2014-12-08 18:14:00.765782 7f2904b56700 10 osd.70 pg_epoch: 71591 pg[19.6d( v 71397'33614009 (70963'33611008,71397'33614009] local-les=71439 n=896 ec=21298 les/c 71439/71439 71591/71591/71169) [4,70,69] r=1 lpr=71591 pi=71156-71590/27 crt=71397'33614007 lcod 0'0 inactive NOTIFY] null 2014-12-08 18:14:00.765799 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[218.338( empty local-les=71436 n=0 ec=71423 les/c 71436/71436 71591/71591/71425) [23,63,70]/[23,63,21] r=-1 lpr=71591 pi=71423-71590/8 crt=0'0 remapped NOTIFY] handle_activate_map 2014-12-08 18:14:00.765799 7f2904b56700 10 osd.70 pg_epoch: 71591 pg[218.6c7( v 71581'2 (0'0,71581'2] local-les=71433 n=2 ec=71423 les/c 71433/71433 71591/71591/71587) [70,10,18]/[10,18,76] r=-1 lpr=71591 pi=71432-71590/5 crt=0'0 lcod 0'0 remapped NOTIFY] null 2014-12-08 18:14:00.765816 7f2904b56700 10 osd.70 pg_epoch: 71591 pg[218.6c4( v 71581'3 (0'0,71581'3] local-les=71437 n=3 ec=71423 les/c 71437/71437 71591/71591/71587) [70,76,13]/[76,13,15] r=-1 lpr=71591 pi=71432-71590/5 crt=71580'1 lcod 0'0 remapped NOTIFY] null 2014-12-08 18:14:00.765830 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[218.338( empty local-les=71436 n=0 ec=71423 les/c 71436/71436 71591/71591/71425) [23,63,70]/[23,63,21] r=-1 lpr=71591 pi=71423-71590/8 crt=0'0 remapped NOTIFY] update_heartbeat_peers unchanged 2014-12-08 18:14:00.765834 7f2904b56700 10 osd.70 pg_epoch: 71591 pg[218.562( v 71581'3 (0'0,71581'3] local-les=71437 n=3 ec=71423 les/c 71437/71437 71591/71591/71425) [9,18,70]/[9,18,58] r=-1 lpr=71591 pi=71423-71590/9 crt=71580'1 lcod 0'0 remapped NOTIFY] null 2014-12-08 18:14:00.765842 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[218.338( empty local-les=71436 n=0 ec=71423 les/c 71436/71436 71591/71591/71425) [23,63,70]/[23,63,21] r=-1 lpr=71591 pi=71423-71590/8 crt=0'0 remapped NOTIFY] take_waiters 2014-12-08 18:14:00.765854 7f28fc345700 5 osd.70 pg_epoch: 71592 pg[218.338( empty local-les=71436 n=0 ec=71423 les/c 71436/71436 71591/71591/71425) [23,63,70]/[23,63,21] r=-1 lpr=71591 pi=71423-71590/8 crt=0'0 remapped NOTIFY] exit Reset 0.002027 11 0.001725 2014-12-08 18:14:00.765853 7f2904b56700 10 osd.70 pg_epoch: 71591 pg[218.5e( empty local-les=71435 n=0 ec=71423 les/c 71435/71435 71591/71591/71425) [16,22,70] r=2 lpr=71591 pi=71423-71590/8 crt=0'0 inactive NOTIFY] null 2014-12-08 18:14:00.765868 7f28fc345700 5 osd.70 pg_epoch: 71592 pg[218.338( empty local-les=71436 n=0 ec=71423 les/c 71436/71436 71591/71591/71425) [23,63,70]/[23,63,21] r=-1 lpr=71591 pi=71423-71590/8 crt=0'0 remapped NOTIFY] enter Started 2014-12-08 18:14:00.765872 7f2904b56700 10 osd.70 pg_epoch: 71591 pg[218.5f( v 71580'2 (0'0,71580'2] local-les=71437 n=2 ec=71423 les/c 71437/71437 71591/71591/71587) [70,15,24]/[15,24,9] r=-1 lpr=71591 pi=71432-71590/5 crt=0'0 lcod 0'0 remapped NOTIFY] null 2014-12-08 18:14:00.765879 7f28fc345700 5 osd.70 pg_epoch: 71592 pg[218.338( empty local-les=71436 n=0 ec=71423 les/c 71436/71436 71591/71591/71425) [23,63,70]/[23,63,21] r=-1 lpr=71591 pi=71423-71590/8 crt=0'0 remapped NOTIFY] enter Start 2014-12-08 18:14:00.765889 7f28fc345700 1 osd.70 pg_epoch: 71592 pg[218.338( empty local-les=71436 n=0 ec=71423 les/c 71436/71436 71591/71591/71425) [23,63,70]/[23,63,21] r=-1 lpr=71591 pi=71423-71590/8 crt=0'0 remapped NOTIFY] state: transitioning to Stray 2014-12-08 18:14:00.765890 7f2904b56700 10 osd.70 pg_epoch: 71591 pg[217.235( v 71377'4108 (71377'1107,71377'4108] local-les=71435 n=11 ec=71094 les/c 71435/71435 71591/71591/71589) [78,25,70] r=2 lpr=71591 pi=71094-71590/32 crt=71377'4105 lcod 0'0 inactive NOTIFY] null 2014-12-08 18:14:00.765903 7f28fc345700 5 osd.70 pg_epoch: 71592 pg[218.338( empty local-les=71436 n=0 ec=71423 les/c 71436/71436 71591/71591/71425) [23,63,70]/[23,63,21] r=-1 lpr=71591 pi=71423-71590/8 crt=0'0 remapped NOTIFY] exit Start 0.000024 0 0.000000 2014-12-08 18:14:00.765906 7f2904b56700 10 osd.70 pg_epoch: 71591 pg[218.38e( empty local-les=71437 n=0 ec=71423 les/c 71437/71437 71591/71591/71587) [70,13,3]/[13,3,5] r=-1 lpr=71591 pi=71432-71590/5 crt=0'0 remapped NOTIFY] null 2014-12-08 18:14:00.765917 7f28fc345700 5 osd.70 pg_epoch: 71592 pg[218.338( empty local-les=71436 n=0 ec=71423 les/c 71436/71436 71591/71591/71425) [23,63,70]/[23,63,21] r=-1 lpr=71591 pi=71423-71590/8 crt=0'0 remapped NOTIFY] enter Started/Stray 2014-12-08 18:14:00.765921 7f2904b56700 10 osd.70 pg_epoch: 71591 pg[218.38f( empty local-les=71457 n=0 ec=71423 les/c 71457/71457 71591/71591/71587) [70,23,25]/[23,25,58] r=-1 lpr=71591 pi=71456-71590/4 crt=0'0 remapped NOTIFY] null 2014-12-08 18:14:00.765932 7f28fc345700 20 osd.70 pg_epoch: 71592 pg[218.338( empty local-les=71436 n=0 ec=71423 les/c 71436/71436 71591/71591/71425) [23,63,70]/[23,63,21] r=-1 lpr=71591 pi=71423-71590/8 crt=0'0 remapped NOTIFY] handle_activate_map: Not dirtying info: last_persisted is 71436 while current is 71592 -- 2014-12-08 18:14:00.826376 7f2904b56700 20 osd.70 71592 19.6e heartbeat_peers 13,16,70 2014-12-08 18:14:00.826382 7f2904b56700 20 osd.70 71592 19.6d heartbeat_peers 2014-12-08 18:14:00.826384 7f2904b56700 20 osd.70 71592 218.6c7 heartbeat_peers 2014-12-08 18:14:00.826375 7f28fc345700 10 osd.70 pg_epoch: 71590 pg[218.7e8( v 71581'1 (0'0,71581'1] local-les=71438 n=1 ec=71423 les/c 71438/71438 71587/71587/71425) [69,77] r=-1 lpr=71587 pi=71423-71586/7 crt=0'0 lcod 0'0 inactive NOTIFY] _calc_past_interval_range: already have past intervals back to 71438 2014-12-08 18:14:00.831463 7f2904b56700 20 osd.70 71592 218.6c4 heartbeat_peers 2014-12-08 18:14:00.831477 7f2904b56700 20 osd.70 71592 218.562 heartbeat_peers 2014-12-08 18:14:00.826383 7f28fcb46700 10 osd.70 pg_epoch: 71583 pg[216.16e( empty local-les=71434 n=0 ec=70987 les/c 71434/71434 71582/71582/70989) [78,56] r=-1 lpr=71582 pi=71401-71581/5 crt=0'0 inactive NOTIFY] handle_advance_map [78,56,16]/[78,56,16] -- 78/78 2014-12-08 18:14:00.831479 7f2904b56700 20 osd.70 71592 218.5e heartbeat_peers 2014-12-08 18:14:00.831481 7f2904b56700 20 osd.70 71592 218.5f heartbeat_peers 2014-12-08 18:14:00.831482 7f2904b56700 20 osd.70 71592 217.235 heartbeat_peers 2014-12-08 18:14:00.831484 7f2904b56700 20 osd.70 71592 218.38e heartbeat_peers 2014-12-08 18:14:00.831486 7f2904b56700 20 osd.70 71592 218.38f heartbeat_peers 2014-12-08 18:14:00.831487 7f2904b56700 20 osd.70 71592 218.38c heartbeat_peers 2014-12-08 18:14:00.831489 7f2904b56700 20 osd.70 71592 217.51 heartbeat_peers 2014-12-08 18:14:00.831491 7f2904b56700 20 osd.70 71592 216.380 heartbeat_peers 2014-12-08 18:14:00.831493 7f2904b56700 20 osd.70 71592 218.51 heartbeat_peers 66,70,75 2014-12-08 18:14:00.831490 7f28fcb46700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:14:00.831500 7f2904b56700 20 osd.70 71592 218.380 heartbeat_peers 2014-12-08 18:14:00.831501 7f2904b56700 20 osd.70 71592 218.518 heartbeat_peers 2014-12-08 18:14:00.831503 7f2904b56700 20 osd.70 71592 217.22d heartbeat_peers 2014-12-08 18:14:00.831505 7f2904b56700 20 osd.70 71592 216.384 heartbeat_peers -- 2014-12-08 18:14:00.833186 7f2904b56700 20 osd.70 71592 _dispatch 0x12f894a0 pg_notify(19.6e(150) epoch 71592) v5 2014-12-08 18:14:00.833184 7f28fcb46700 10 osd.70 pg_epoch: 71585 pg[216.2f8( empty local-les=71437 n=0 ec=70987 les/c 71437/71437 71584/71584/70989) [23,75,9] r=-1 lpr=71584 pi=71401-71583/6 crt=0'0 inactive NOTIFY] _calc_past_interval_range: already have past intervals back to 71437 2014-12-08 18:14:00.833191 7f28fcb46700 10 osd.70 pg_epoch: 71585 pg[216.2f8( empty local-les=71437 n=0 ec=70987 les/c 71437/71437 71584/71584/70989) [23,75,9] r=-1 lpr=71584 pi=71401-71583/6 crt=0'0 inactive NOTIFY] check_recovery_sources no source osds () went down 2014-12-08 18:14:00.833202 7f28fcb46700 10 osd.70 pg_epoch: 71585 pg[216.2f8( empty local-les=71437 n=0 ec=70987 les/c 71437/71437 71584/71584/70989) [23,75,9] r=-1 lpr=71584 pi=71401-71583/6 crt=0'0 inactive NOTIFY] handle_advance_map [23,75,9]/[23,75,9] -- 23/23 2014-12-08 18:14:00.833209 7f28fcb46700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:14:00.833184 7f28fc345700 10 osd.70 pg_epoch: 71584 pg[218.611( empty local-les=71434 n=0 ec=71423 les/c 71434/71434 71584/71584/71425) [22,25,76] r=-1 lpr=71584 pi=71423-71583/6 crt=0'0 inactive NOTIFY] up [22,76] -> [22,25,76], acting [22,76] -> [22,25,76], acting_primary 22 -> 22, up_primary 22 -> 22, role -1 -> -1 2014-12-08 18:14:00.833212 7f2904b56700 7 osd.70 71592 handle_pg_notify from osd.30 2014-12-08 18:14:00.833213 7f2904b56700 15 osd.70 71592 require_same_or_newer_map 71592 (i am 71592) 0x12f894a0 2014-12-08 18:14:00.833211 7f28fcb46700 10 osd.70 pg_epoch: 71586 pg[216.2f8( empty local-les=71437 n=0 ec=70987 les/c 71437/71437 71584/71584/70989) [23,75,9] r=-1 lpr=71584 pi=71401-71583/6 crt=0'0 inactive NOTIFY] state: Reset advmap 2014-12-08 18:14:00.833211 7f28fc345700 10 osd.70 pg_epoch: 71584 pg[218.611( empty local-les=71434 n=0 ec=71423 les/c 71434/71434 71584/71584/71425) [22,25,76] r=-1 lpr=71584 pi=71423-71583/6 crt=0'0 inactive NOTIFY] on_change 2014-12-08 18:14:00.833218 7f28fcb46700 10 osd.70 pg_epoch: 71586 pg[216.2f8( empty local-les=71437 n=0 ec=70987 les/c 71437/71437 71584/71584/70989) [23,75,9] r=-1 lpr=71584 pi=71401-71583/6 crt=0'0 inactive NOTIFY] _calc_past_interval_range: already have past intervals back to 71437 2014-12-08 18:14:00.833222 7f28fc345700 15 osd.70 pg_epoch: 71584 pg[218.611( empty local-les=71434 n=0 ec=71423 les/c 71434/71434 71584/71584/71425) [22,25,76] r=-1 lpr=71584 pi=71423-71583/6 crt=0'0 inactive NOTIFY] publish_stats_to_osd -- not primary 2014-12-08 18:14:00.833231 7f2904b56700 10 osd.70 71592 do_waiters -- start 2014-12-08 18:14:00.833225 7f28fcb46700 10 osd.70 pg_epoch: 71586 pg[216.2f8( empty local-les=71437 n=0 ec=70987 les/c 71437/71437 71584/71584/70989) [23,75,9] r=-1 lpr=71584 pi=71401-71583/6 crt=0'0 inactive NOTIFY] check_recovery_sources no source osds () went down 2014-12-08 18:14:00.833233 7f2904b56700 10 osd.70 71592 do_waiters -- finish 2014-12-08 18:14:00.833234 7f28fc345700 15 osd.70 pg_epoch: 71584 pg[218.611( empty local-les=71434 n=0 ec=71423 les/c 71434/71434 71584/71584/71425) [22,25,76] r=-1 lpr=71584 pi=71423-71583/6 crt=0'0 inactive NOTIFY] requeue_ops 2014-12-08 18:14:00.833240 7f28fcb46700 10 osd.70 pg_epoch: 71586 pg[216.2f8( empty local-les=71437 n=0 ec=70987 les/c 71437/71437 71584/71584/70989) [23,75,9] r=-1 lpr=71584 pi=71401-71583/6 crt=0'0 inactive NOTIFY] handle_advance_map [23,9]/[23,9] -- 23/23 2014-12-08 18:14:00.833248 7f28fcb46700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:14:00.833243 7f28fc345700 10 osd.70 pg_epoch: 71584 pg[218.611( empty local-les=71434 n=0 ec=71423 les/c 71434/71434 71584/71584/71425) [22,25,76] r=-1 lpr=71584 pi=71423-71583/6 crt=0'0 inactive NOTIFY] cancel_copy_ops 2014-12-08 18:14:00.833250 7f28fcb46700 10 osd.70 pg_epoch: 71587 pg[216.2f8( empty local-les=71437 n=0 ec=70987 les/c 71437/71437 71584/71584/70989) [23,75,9] r=-1 lpr=71584 pi=71401-71583/6 crt=0'0 inactive NOTIFY] state: Reset advmap 2014-12-08 18:14:00.833253 7f28fc345700 10 osd.70 pg_epoch: 71584 pg[218.611( empty local-les=71434 n=0 ec=71423 les/c 71434/71434 71584/71584/71425) [22,25,76] r=-1 lpr=71584 pi=71423-71583/6 crt=0'0 inactive NOTIFY] cancel_flush_ops -- 2014-12-08 18:14:00.844057 7f2904b56700 20 osd.70 71592 _dispatch 0x12b725a0 pg_notify(19.6e(150) epoch 71592) v5 2014-12-08 18:14:00.844084 7f2904b56700 7 osd.70 71592 handle_pg_notify from osd.4 2014-12-08 18:14:00.844087 7f2904b56700 15 osd.70 71592 require_same_or_newer_map 71592 (i am 71592) 0x12b725a0 2014-12-08 18:14:00.844087 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[218.452( v 71581'3 (0'0,71581'3] local-les=71450 n=3 ec=71423 les/c 71450/71450 71591/71591/71444) [21,63,70]/[21,63,68] r=-1 lpr=71591 pi=71423-71590/11 crt=71580'1 lcod 0'0 remapped NOTIFY] handle_activate_map 2014-12-08 18:14:00.844115 7f2904b56700 10 osd.70 71592 do_waiters -- start 2014-12-08 18:14:00.844110 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[218.452( v 71581'3 (0'0,71581'3] local-les=71450 n=3 ec=71423 les/c 71450/71450 71591/71591/71444) [21,63,70]/[21,63,68] r=-1 lpr=71591 pi=71423-71590/11 crt=71580'1 lcod 0'0 remapped NOTIFY] update_heartbeat_peers unchanged 2014-12-08 18:14:00.844118 7f2904b56700 10 osd.70 71592 do_waiters -- finish 2014-12-08 18:14:00.844105 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[218.611( empty local-les=71434 n=0 ec=71423 les/c 71434/71434 71591/71591/71425) [22,70,76]/[22,76,25] r=-1 lpr=71591 pi=71423-71590/9 crt=0'0 remapped NOTIFY] handle_activate_map 2014-12-08 18:14:00.844119 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[218.452( v 71581'3 (0'0,71581'3] local-les=71450 n=3 ec=71423 les/c 71450/71450 71591/71591/71444) [21,63,70]/[21,63,68] r=-1 lpr=71591 pi=71423-71590/11 crt=71580'1 lcod 0'0 remapped NOTIFY] take_waiters 2014-12-08 18:14:00.844127 7f28fcb46700 5 osd.70 pg_epoch: 71592 pg[218.452( v 71581'3 (0'0,71581'3] local-les=71450 n=3 ec=71423 les/c 71450/71450 71591/71591/71444) [21,63,70]/[21,63,68] r=-1 lpr=71591 pi=71423-71590/11 crt=71580'1 lcod 0'0 remapped NOTIFY] exit Reset 4.191756 13 0.001350 2014-12-08 18:14:00.844138 7f28fcb46700 5 osd.70 pg_epoch: 71592 pg[218.452( v 71581'3 (0'0,71581'3] local-les=71450 n=3 ec=71423 les/c 71450/71450 71591/71591/71444) [21,63,70]/[21,63,68] r=-1 lpr=71591 pi=71423-71590/11 crt=71580'1 lcod 0'0 remapped NOTIFY] enter Started 2014-12-08 18:14:00.844143 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[218.611( empty local-les=71434 n=0 ec=71423 les/c 71434/71434 71591/71591/71425) [22,70,76]/[22,76,25] r=-1 lpr=71591 pi=71423-71590/9 crt=0'0 remapped NOTIFY] update_heartbeat_peers unchanged 2014-12-08 18:14:00.844151 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[218.611( empty local-les=71434 n=0 ec=71423 les/c 71434/71434 71591/71591/71425) [22,70,76]/[22,76,25] r=-1 lpr=71591 pi=71423-71590/9 crt=0'0 remapped NOTIFY] take_waiters 2014-12-08 18:14:00.844149 7f28fcb46700 5 osd.70 pg_epoch: 71592 pg[218.452( v 71581'3 (0'0,71581'3] local-les=71450 n=3 ec=71423 les/c 71450/71450 71591/71591/71444) [21,63,70]/[21,63,68] r=-1 lpr=71591 pi=71423-71590/11 crt=71580'1 lcod 0'0 remapped NOTIFY] enter Start 2014-12-08 18:14:00.844159 7f28fc345700 5 osd.70 pg_epoch: 71592 pg[218.611( empty local-les=71434 n=0 ec=71423 les/c 71434/71434 71591/71591/71425) [22,70,76]/[22,76,25] r=-1 lpr=71591 pi=71423-71590/9 crt=0'0 remapped NOTIFY] exit Reset 0.011528 11 0.011042 2014-12-08 18:14:00.844163 7f28fcb46700 1 osd.70 pg_epoch: 71592 pg[218.452( v 71581'3 (0'0,71581'3] local-les=71450 n=3 ec=71423 les/c 71450/71450 71591/71591/71444) [21,63,70]/[21,63,68] r=-1 lpr=71591 pi=71423-71590/11 crt=71580'1 lcod 0'0 remapped NOTIFY] state: transitioning to Stray 2014-12-08 18:14:00.844174 7f2904b56700 10 osd.70 71592 do_waiters -- start 2014-12-08 18:14:00.844169 7f28fc345700 5 osd.70 pg_epoch: 71592 pg[218.611( empty local-les=71434 n=0 ec=71423 les/c 71434/71434 71591/71591/71425) [22,70,76]/[22,76,25] r=-1 lpr=71591 pi=71423-71590/9 crt=0'0 remapped NOTIFY] enter Started 2014-12-08 18:14:00.844176 7f2904b56700 10 osd.70 71592 do_waiters -- finish 2014-12-08 18:14:00.844178 7f2904b56700 20 osd.70 71592 _dispatch 0x12b79c00 pg_notify(217.bb(5) epoch 71592) v5 2014-12-08 18:14:00.844177 7f28fc345700 5 osd.70 pg_epoch: 71592 pg[218.611( empty local-les=71434 n=0 ec=71423 les/c 71434/71434 71591/71591/71425) [22,70,76]/[22,76,25] r=-1 lpr=71591 pi=71423-71590/9 crt=0'0 remapped NOTIFY] enter Start -- 2014-12-08 18:14:00.850061 7f2904b56700 20 osd.70 71592 _dispatch 0x12f80960 pg_notify(217.399(28),19.6e(150) epoch 71592) v5 2014-12-08 18:14:00.850071 7f28fc345700 10 log is not dirty 2014-12-08 18:14:00.850079 7f2904b56700 7 osd.70 71592 handle_pg_notify from osd.58 2014-12-08 18:14:00.850081 7f2904b56700 15 osd.70 71592 require_same_or_newer_map 71592 (i am 71592) 0x12f80960 2014-12-08 18:14:00.850105 7f2904b56700 10 osd.70 71592 do_waiters -- start 2014-12-08 18:14:00.850107 7f2904b56700 10 osd.70 71592 do_waiters -- finish 2014-12-08 18:14:00.850098 7f28fcb46700 10 osd.70 pg_epoch: 71581 pg[217.2fd( v 71377'4110 (71377'1109,71377'4110] local-les=71446 n=14 ec=71094 les/c 71446/71447 71444/71444/71433) [70,10,19] r=0 lpr=71447 crt=71377'4107 lcod 0'0 mlcod 0'0 peering] handle_advance_map [10,19]/[10,19] -- 10/10 2014-12-08 18:14:00.850099 7f28fc345700 10 osd.70 pg_epoch: 71581 pg[218.7b7( v 71581'1 (0'0,71581'1] local-les=71457 n=1 ec=71423 les/c 71457/71457 71456/71456/71425) [58,17,70] r=2 lpr=71581 pi=71423-71455/7 crt=0'0 lcod 0'0 inactive NOTIFY] handle_advance_map [58,17]/[58,17] -- 58/58 2014-12-08 18:14:00.850113 7f28fcb46700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:14:00.850114 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:14:00.850117 7f28fc345700 10 osd.70 pg_epoch: 71582 pg[218.7b7( v 71581'1 (0'0,71581'1] local-les=71457 n=1 ec=71423 les/c 71457/71457 71456/71456/71425) [58,17,70] r=2 lpr=71581 pi=71423-71455/7 crt=0'0 lcod 0'0 inactive NOTIFY] state: Reset advmap 2014-12-08 18:14:00.850118 7f28fcb46700 10 osd.70 pg_epoch: 71582 pg[217.2fd( v 71377'4110 (71377'1109,71377'4110] local-les=71446 n=14 ec=71094 les/c 71446/71447 71444/71444/71433) [70,10,19] r=0 lpr=71447 crt=71377'4107 lcod 0'0 mlcod 0'0 peering] state: Peering advmap 2014-12-08 18:14:00.850125 7f28fc345700 10 osd.70 pg_epoch: 71582 pg[218.7b7( v 71581'1 (0'0,71581'1] local-les=71457 n=1 ec=71423 les/c 71457/71457 71456/71456/71425) [58,17,70] r=2 lpr=71581 pi=71423-71455/7 crt=0'0 lcod 0'0 inactive NOTIFY] _calc_past_interval_range: already have past intervals back to 71457 2014-12-08 18:14:00.850128 7f28fcb46700 10 osd.70 pg_epoch: 71582 pg[217.2fd( v 71377'4110 (71377'1109,71377'4110] local-les=71446 n=14 ec=71094 les/c 71446/71447 71444/71444/71433) [70,10,19] r=0 lpr=71447 crt=71377'4107 lcod 0'0 mlcod 0'0 peering] PriorSet: affected_by_map osd.70 now down 2014-12-08 18:14:00.850135 7f28fc345700 20 osd.70 pg_epoch: 71582 pg[218.7b7( v 71581'1 (0'0,71581'1] local-les=71457 n=1 ec=71423 les/c 71457/71457 71456/71456/71425) [58,17,70] r=2 lpr=71581 pi=71423-71455/7 crt=0'0 lcod 0'0 inactive NOTIFY] new interval newup [58,17] newacting [58,17] 2014-12-08 18:14:00.850135 7f28fcb46700 1 osd.70 pg_epoch: 71582 pg[217.2fd( v 71377'4110 (71377'1109,71377'4110] local-les=71446 n=14 ec=71094 les/c 71446/71447 71444/71444/71433) [70,10,19] r=0 lpr=71447 crt=71377'4107 lcod 0'0 mlcod 0'0 peering] state: Peering, affected_by_map, going to Reset 2014-12-08 18:14:00.850143 7f28fc345700 10 osd.70 pg_epoch: 71582 pg[218.7b7( v 71581'1 (0'0,71581'1] local-les=71457 n=1 ec=71423 les/c 71457/71457 71456/71456/71425) [58,17,70] r=2 lpr=71581 pi=71423-71455/7 crt=0'0 lcod 0'0 inactive NOTIFY] state: should restart peering, calling start_peering_interval again 2014-12-08 18:14:00.850151 7f28fc345700 20 osd.70 pg_epoch: 71582 pg[218.7b7( v 71581'1 (0'0,71581'1] local-les=71457 n=1 ec=71423 les/c 71457/71457 71456/71456/71425) [58,17,70] r=2 lpr=71581 pi=71423-71455/7 crt=0'0 lcod 0'0 inactive NOTIFY] set_last_peering_reset 71582 2014-12-08 18:14:00.850148 7f28fcb46700 5 osd.70 pg_epoch: 71582 pg[217.2fd( v 71377'4110 (71377'1109,71377'4110] local-les=71446 n=14 ec=71094 les/c 71446/71447 71444/71444/71433) [70,10,19] r=0 lpr=71447 crt=71377'4107 lcod 0'0 mlcod 0'0 peering] exit Started/Primary/Peering/GetInfo 2.337991 2 0.000167 2014-12-08 18:14:00.850158 7f28fc345700 10 osd.70 pg_epoch: 71582 pg[218.7b7( v 71581'1 (0'0,71581'1] local-les=71457 n=1 ec=71423 les/c 71457/71457 71456/71456/71425) [58,17,70] r=2 lpr=71582 pi=71423-71455/7 crt=0'0 lcod 0'0 inactive NOTIFY] Clearing blocked outgoing recovery messages 2014-12-08 18:14:00.850167 7f2904b56700 10 osd.70 71592 do_waiters -- start -- 2014-12-08 18:14:00.850170 7f2904b56700 20 osd.70 71592 _dispatch 0x12b79660 pg_notify(19.6e(150),218.6c7(5),218.6c4(5) epoch 71592) v5 2014-12-08 18:14:00.850165 7f28fcb46700 10 osd.70 pg_epoch: 71582 pg[217.2fd( v 71377'4110 (71377'1109,71377'4110] local-les=71446 n=14 ec=71094 les/c 71446/71447 71444/71444/71433) [70,10,19] r=0 lpr=71447 crt=71377'4107 lcod 0'0 mlcod 0'0 peering] state: Leaving Peering 2014-12-08 18:14:00.850170 7f28fc345700 10 osd.70 pg_epoch: 71582 pg[218.7b7( v 71581'1 (0'0,71581'1] local-les=71457 n=1 ec=71423 les/c 71457/71457 71456/71456/71425) [58,17,70] r=2 lpr=71582 pi=71423-71455/7 crt=0'0 lcod 0'0 inactive NOTIFY] Not blocking outgoing recovery messages 2014-12-08 18:14:00.850178 7f28fcb46700 5 osd.70 pg_epoch: 71582 pg[217.2fd( v 71377'4110 (71377'1109,71377'4110] local-les=71446 n=14 ec=71094 les/c 71446/71447 71444/71444/71433) [70,10,19] r=0 lpr=71447 crt=71377'4107 lcod 0'0 mlcod 0'0 peering] exit Started/Primary/Peering 2.338028 0 0.000000 2014-12-08 18:14:00.850194 7f2904b56700 7 osd.70 71592 handle_pg_notify from osd.76 2014-12-08 18:14:00.850188 7f28fcb46700 5 osd.70 pg_epoch: 71582 pg[217.2fd( v 71377'4110 (71377'1109,71377'4110] local-les=71446 n=14 ec=71094 les/c 71446/71447 71444/71444/71433) [70,10,19] r=0 lpr=71447 crt=71377'4107 lcod 0'0 mlcod 0'0 inactive] exit Started/Primary 2.338046 0 0.000000 2014-12-08 18:14:00.850186 7f28fc345700 10 osd.70 pg_epoch: 71582 pg[218.7b7( v 71581'1 (0'0,71581'1] local-les=71457 n=1 ec=71423 les/c 71457/71457 71456/71456/71425) [58,17] r=-1 lpr=71582 pi=71423-71581/8 crt=0'0 lcod 0'0 inactive NOTIFY] start_peering_interval: check_new_interval output: generate_past_intervals interval(71456-71581 up [58,17,70](58) acting [58,17,70](58)): not rw, up_thru 71456 up_from 69683 last_epoch_clean 71457 generate_past_intervals interval(71456-71581 up [58,17,70](58) acting [58,17,70](58) maybe_went_rw) : primary up 69683-71456 includes interval 2014-12-08 18:14:00.850198 7f2904b56700 15 osd.70 71592 require_same_or_newer_map 71592 (i am 71592) 0x12b79660 2014-12-08 18:14:00.850198 7f28fcb46700 5 osd.70 pg_epoch: 71582 pg[217.2fd( v 71377'4110 (71377'1109,71377'4110] local-les=71446 n=14 ec=71094 les/c 71446/71447 71444/71444/71433) [70,10,19] r=0 lpr=71447 crt=71377'4107 lcod 0'0 mlcod 0'0 inactive] exit Started 2.338089 0 0.000000 2014-12-08 18:14:00.850198 7f28fc345700 10 osd.70 pg_epoch: 71582 pg[218.7b7( v 71581'1 (0'0,71581'1] local-les=71457 n=1 ec=71423 les/c 71457/71457 71456/71456/71425) [58,17] r=-1 lpr=71582 pi=71423-71581/8 crt=0'0 lcod 0'0 inactive NOTIFY] noting past interval(71456-71581 up [58,17,70](58) acting [58,17,70](58) maybe_went_rw) 2014-12-08 18:14:00.850207 7f28fcb46700 5 osd.70 pg_epoch: 71582 pg[217.2fd( v 71377'4110 (71377'1109,71377'4110] local-les=71446 n=14 ec=71094 les/c 71446/71447 71444/71444/71433) [70,10,19] r=0 lpr=71447 crt=71377'4107 lcod 0'0 mlcod 0'0 inactive] enter Reset 2014-12-08 18:14:00.850208 7f28fc345700 10 osd.70 pg_epoch: 71582 pg[218.7b7( v 71581'1 (0'0,71581'1] local-les=71457 n=1 ec=71423 les/c 71457/71457 71582/71582/71425) [58,17] r=-1 lpr=71582 pi=71423-71581/8 crt=0'0 lcod 0'0 inactive NOTIFY] up [58,17,70] -> [58,17], acting [58,17,70] -> [58,17], acting_primary 58 -> 58, up_primary 58 -> 58, role 2 -> -1 2014-12-08 18:14:00.850214 7f28fcb46700 20 osd.70 pg_epoch: 71582 pg[217.2fd( v 71377'4110 (71377'1109,71377'4110] local-les=71446 n=14 ec=71094 les/c 71446/71447 71444/71444/71433) [70,10,19] r=0 lpr=71447 crt=71377'4107 lcod 0'0 mlcod 0'0 inactive] set_last_peering_reset 71582 2014-12-08 18:14:00.850218 7f28fc345700 10 osd.70 pg_epoch: 71582 pg[218.7b7( v 71581'1 (0'0,71581'1] local-les=71457 n=1 ec=71423 les/c 71457/71457 71582/71582/71425) [58,17] r=-1 lpr=71582 pi=71423-71581/8 crt=0'0 lcod 0'0 inactive NOTIFY] on_change 2014-12-08 18:14:00.850222 7f28fcb46700 10 osd.70 pg_epoch: 71582 pg[217.2fd( v 71377'4110 (71377'1109,71377'4110] local-les=71446 n=14 ec=71094 les/c 71446/71447 71444/71444/71433) [70,10,19] r=0 lpr=71582 crt=71377'4107 lcod 0'0 mlcod 0'0 inactive] Clearing blocked outgoing recovery messages 2014-12-08 18:14:00.850226 7f28fc345700 15 osd.70 pg_epoch: 71582 pg[218.7b7( v 71581'1 (0'0,71581'1] local-les=71457 n=1 ec=71423 les/c 71457/71457 71582/71582/71425) [58,17] r=-1 lpr=71582 pi=71423-71581/8 crt=0'0 lcod 0'0 inactive NOTIFY] publish_stats_to_osd -- not primary 2014-12-08 18:14:00.850235 7f2904b56700 10 osd.70 71592 do_waiters -- start 2014-12-08 18:14:00.850237 7f2904b56700 10 osd.70 71592 do_waiters -- finish 2014-12-08 18:14:00.850232 7f28fcb46700 10 osd.70 pg_epoch: 71582 pg[217.2fd( v 71377'4110 (71377'1109,71377'4110] local-les=71446 n=14 ec=71094 les/c 71446/71447 71444/71444/71433) [70,10,19] r=0 lpr=71582 crt=71377'4107 lcod 0'0 mlcod 0'0 inactive] Not blocking outgoing recovery messages -- 2014-12-08 18:14:00.857439 7f2904b56700 20 osd.70 71592 _dispatch 0x12b72780 pg_notify(19.6e(150) epoch 71592) v5 2014-12-08 18:14:00.857442 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:14:00.857445 7f28fc345700 10 osd.70 pg_epoch: 71587 pg[216.2e2( v 71097'191 (0'0,71097'191] local-les=71433 n=91 ec=70987 les/c 71433/71433 71584/71584/71168) [20,78,10] r=-1 lpr=71584 pi=70987-71583/26 crt=71097'189 lcod 0'0 inactive NOTIFY] state: Reset advmap 2014-12-08 18:14:00.857448 7f28fcb46700 15 osd.70 pg_epoch: 71591 pg[218.2fc( empty local-les=71434 n=0 ec=71423 les/c 71434/71434 71591/71591/71591) [70,16,75] r=0 lpr=71591 pi=71432-71590/4 crt=0'0 mlcod 0'0 inactive NOTIFY] requeue_ops 2014-12-08 18:14:00.857452 7f28fc345700 10 osd.70 pg_epoch: 71587 pg[216.2e2( v 71097'191 (0'0,71097'191] local-les=71433 n=91 ec=70987 les/c 71433/71433 71584/71584/71168) [20,78,10] r=-1 lpr=71584 pi=70987-71583/26 crt=71097'189 lcod 0'0 inactive NOTIFY] _calc_past_interval_range: already have past intervals back to 71433 2014-12-08 18:14:00.857461 7f2904b56700 7 osd.70 71592 handle_pg_notify from osd.16 2014-12-08 18:14:00.857456 7f28fcb46700 10 osd.70 pg_epoch: 71591 pg[218.2fc( empty local-les=71434 n=0 ec=71423 les/c 71434/71434 71591/71591/71591) [70,16,75] r=0 lpr=71591 pi=71432-71590/4 crt=0'0 mlcod 0'0 inactive NOTIFY] on_change_cleanup 2014-12-08 18:14:00.857463 7f2904b56700 15 osd.70 71592 require_same_or_newer_map 71592 (i am 71592) 0x12b72780 2014-12-08 18:14:00.857462 7f28fc345700 20 osd.70 pg_epoch: 71587 pg[216.2e2( v 71097'191 (0'0,71097'191] local-les=71433 n=91 ec=70987 les/c 71433/71433 71584/71584/71168) [20,78,10] r=-1 lpr=71584 pi=70987-71583/26 crt=71097'189 lcod 0'0 inactive NOTIFY] new interval newup [20,10] newacting [20,10] 2014-12-08 18:14:00.857465 7f28fcb46700 10 osd.70 pg_epoch: 71591 pg[218.2fc( empty local-les=71434 n=0 ec=71423 les/c 71434/71434 71591/71591/71591) [70,16,75] r=0 lpr=71591 pi=71432-71590/4 crt=0'0 mlcod 0'0 inactive NOTIFY] on_change 2014-12-08 18:14:00.857470 7f28fc345700 10 osd.70 pg_epoch: 71587 pg[216.2e2( v 71097'191 (0'0,71097'191] local-les=71433 n=91 ec=70987 les/c 71433/71433 71584/71584/71168) [20,78,10] r=-1 lpr=71584 pi=70987-71583/26 crt=71097'189 lcod 0'0 inactive NOTIFY] state: should restart peering, calling start_peering_interval again 2014-12-08 18:14:00.857472 7f28fcb46700 20 osd.70 pg_epoch: 71591 pg[218.2fc( empty local-les=71434 n=0 ec=71423 les/c 71434/71434 71591/71591/71591) [70,16,75] r=0 lpr=71591 pi=71432-71590/4 crt=0'0 mlcod 0'0 inactive NOTIFY] exit NotTrimming 2014-12-08 18:14:00.857478 7f28fc345700 20 osd.70 pg_epoch: 71587 pg[216.2e2( v 71097'191 (0'0,71097'191] local-les=71433 n=91 ec=70987 les/c 71433/71433 71584/71584/71168) [20,78,10] r=-1 lpr=71584 pi=70987-71583/26 crt=71097'189 lcod 0'0 inactive NOTIFY] set_last_peering_reset 71587 2014-12-08 18:14:00.857480 7f28fcb46700 20 osd.70 pg_epoch: 71591 pg[218.2fc( empty local-les=71434 n=0 ec=71423 les/c 71434/71434 71591/71591/71591) [70,16,75] r=0 lpr=71591 pi=71432-71590/4 crt=0'0 mlcod 0'0 inactive NOTIFY] enter NotTrimming 2014-12-08 18:14:00.857486 7f2904b56700 10 osd.70 71592 do_waiters -- start 2014-12-08 18:14:00.857489 7f2904b56700 10 osd.70 71592 do_waiters -- finish 2014-12-08 18:14:00.857487 7f28fcb46700 15 osd.70 pg_epoch: 71591 pg[218.2fc( empty local-les=71434 n=0 ec=71423 les/c 71434/71434 71591/71591/71591) [70,16,75] r=0 lpr=71591 pi=71432-71590/4 crt=0'0 mlcod 0'0 inactive] clear_stats 2014-12-08 18:14:00.857486 7f28fc345700 10 osd.70 pg_epoch: 71587 pg[216.2e2( v 71097'191 (0'0,71097'191] local-les=71433 n=91 ec=70987 les/c 71433/71433 71584/71584/71168) [20,78,10] r=-1 lpr=71587 pi=70987-71583/26 crt=71097'189 lcod 0'0 inactive NOTIFY] Clearing blocked outgoing recovery messages 2014-12-08 18:14:00.857494 7f28fcb46700 15 osd.70 pg_epoch: 71591 pg[218.2fc( empty local-les=71434 n=0 ec=71423 les/c 71434/71434 71591/71591/71591) [70,16,75] r=0 lpr=71591 pi=71432-71590/4 crt=0'0 mlcod 0'0 inactive] requeue_ops 2014-12-08 18:14:00.857494 7f28fc345700 10 osd.70 pg_epoch: 71587 pg[216.2e2( v 71097'191 (0'0,71097'191] local-les=71433 n=91 ec=70987 les/c 71433/71433 71584/71584/71168) [20,78,10] r=-1 lpr=71587 pi=70987-71583/26 crt=71097'189 lcod 0'0 inactive NOTIFY] Not blocking outgoing recovery messages 2014-12-08 18:14:00.857501 7f28fcb46700 10 osd.70 pg_epoch: 71591 pg[218.2fc( empty local-les=71434 n=0 ec=71423 les/c 71434/71434 71591/71591/71591) [70,16,75] r=0 lpr=71591 pi=71432-71590/4 crt=0'0 mlcod 0'0 inactive] on_role_change -- 2014-12-08 18:14:00.996680 7f28fcb46700 10 osd.70 pg_epoch: 71591 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:14:00.996697 7f28fc345700 15 osd.70 pg_epoch: 71582 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71433/71433 71582/71582/71582) [78,1] r=-1 lpr=71582 pi=71432-71581/1 crt=0'0 inactive NOTIFY] requeue_ops 2014-12-08 18:14:00.996701 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:14:00.996702 7f28fc345700 10 osd.70 pg_epoch: 71582 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71433/71433 71582/71582/71582) [78,1] r=-1 lpr=71582 pi=71432-71581/1 crt=0'0 inactive NOTIFY] cancel_recovery 2014-12-08 18:14:00.996707 7f28fc345700 10 osd.70 pg_epoch: 71582 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71433/71433 71582/71582/71582) [78,1] r=-1 lpr=71582 pi=71432-71581/1 crt=0'0 inactive NOTIFY] clear_recovery_state 2014-12-08 18:14:00.996706 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] state: Peering advmap 2014-12-08 18:14:00.996715 7f28fc345700 10 osd.70 pg_epoch: 71582 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71433/71433 71582/71582/71582) [78,1] r=-1 lpr=71582 pi=71432-71581/1 crt=0'0 inactive NOTIFY] check_recovery_sources no source osds () went down 2014-12-08 18:14:00.996720 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] adjust_need_up_thru now 71591, need_up_thru now false 2014-12-08 18:14:00.996727 7f28fc345700 10 osd.70 pg_epoch: 71582 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71433/71433 71582/71582/71582) [78,1] r=-1 lpr=71582 pi=71432-71581/1 crt=0'0 inactive NOTIFY] handle_advance_map [78,1]/[78,1] -- 78/78 2014-12-08 18:14:00.996736 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:14:00.996728 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] state: Started advmap 2014-12-08 18:14:00.996739 7f28fc345700 10 osd.70 pg_epoch: 71583 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71433/71433 71582/71582/71582) [78,1] r=-1 lpr=71582 pi=71432-71581/1 crt=0'0 inactive NOTIFY] state: Reset advmap 2014-12-08 18:14:00.996746 7f28fc345700 10 osd.70 pg_epoch: 71583 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71433/71433 71582/71582/71582) [78,1] r=-1 lpr=71582 pi=71432-71581/1 crt=0'0 inactive NOTIFY] _calc_past_interval_range: already have past intervals back to 71433 2014-12-08 18:14:00.996741 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] check_recovery_sources no source osds () went down 2014-12-08 18:14:00.996753 7f28fc345700 10 osd.70 pg_epoch: 71583 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71433/71433 71582/71582/71582) [78,1] r=-1 lpr=71582 pi=71432-71581/1 crt=0'0 inactive NOTIFY] check_recovery_sources no source osds () went down 2014-12-08 18:14:00.996758 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] handle_activate_map 2014-12-08 18:14:00.996763 7f28fc345700 10 osd.70 pg_epoch: 71583 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71433/71433 71582/71582/71582) [78,1] r=-1 lpr=71582 pi=71432-71581/1 crt=0'0 inactive NOTIFY] handle_advance_map [14,20,1]/[14,20,1] -- 14/14 2014-12-08 18:14:00.996772 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:14:00.996769 7f28fcb46700 7 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] state: handle ActMap primary 2014-12-08 18:14:00.996773 7f28fc345700 10 osd.70 pg_epoch: 71584 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71433/71433 71582/71582/71582) [78,1] r=-1 lpr=71582 pi=71432-71581/1 crt=0'0 inactive NOTIFY] state: Reset advmap 2014-12-08 18:14:00.996781 7f28fc345700 10 osd.70 pg_epoch: 71584 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71433/71433 71582/71582/71582) [78,1] r=-1 lpr=71582 pi=71432-71581/1 crt=0'0 inactive NOTIFY] _calc_past_interval_range: already have past intervals back to 71433 2014-12-08 18:14:00.996786 7f28fc345700 20 osd.70 pg_epoch: 71584 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71433/71433 71582/71582/71582) [78,1] r=-1 lpr=71582 pi=71432-71581/1 crt=0'0 inactive NOTIFY] new interval newup [14,20,1] newacting [14,20,1] 2014-12-08 18:14:00.996784 7f28fcb46700 15 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] publish_stats_to_osd 71592:63844730 2014-12-08 18:14:00.996793 7f28fc345700 10 osd.70 pg_epoch: 71584 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71433/71433 71582/71582/71582) [78,1] r=-1 lpr=71582 pi=71432-71581/1 crt=0'0 inactive NOTIFY] state: should restart peering, calling start_peering_interval again 2014-12-08 18:14:00.996800 7f28fc345700 20 osd.70 pg_epoch: 71584 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71433/71433 71582/71582/71582) [78,1] r=-1 lpr=71582 pi=71432-71581/1 crt=0'0 inactive NOTIFY] set_last_peering_reset 71584 2014-12-08 18:14:00.996797 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] take_waiters 2014-12-08 18:14:00.996806 7f28fc345700 10 osd.70 pg_epoch: 71584 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71433/71433 71582/71582/71582) [78,1] r=-1 lpr=71584 pi=71432-71581/1 crt=0'0 inactive NOTIFY] Clearing blocked outgoing recovery messages 2014-12-08 18:14:00.996811 7f28fc345700 10 osd.70 pg_epoch: 71584 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71433/71433 71582/71582/71582) [78,1] r=-1 lpr=71584 pi=71432-71581/1 crt=0'0 inactive NOTIFY] Not blocking outgoing recovery messages 2014-12-08 18:14:00.996820 7f28fc345700 10 osd.70 pg_epoch: 71584 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71433/71433 71582/71582/71582) [14,20,1] r=-1 lpr=71584 pi=71432-71583/2 crt=0'0 inactive NOTIFY] start_peering_interval: check_new_interval output: generate_past_intervals interval(71582-71583 up [78,1](78) acting [78,1](78)): not rw, up_thru 71582 up_from 69689 last_epoch_clean 71433 generate_past_intervals interval(71582-71583 up [78,1](78) acting [78,1](78) maybe_went_rw) : primary up 69689-71582 includes interval 2014-12-08 18:14:00.996826 7f28fc345700 10 osd.70 pg_epoch: 71584 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71433/71433 71582/71582/71582) [14,20,1] r=-1 lpr=71584 pi=71432-71583/2 crt=0'0 inactive NOTIFY] noting past interval(71582-71583 up [78,1](78) acting [78,1](78) maybe_went_rw) 2014-12-08 18:14:00.996832 7f28fc345700 10 osd.70 pg_epoch: 71584 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71433/71433 71584/71584/71584) [14,20,1] r=-1 lpr=71584 pi=71432-71583/2 crt=0'0 inactive NOTIFY] up [78,1] -> [14,20,1], acting [78,1] -> [14,20,1], acting_primary 78 -> 14, up_primary 78 -> 14, role -1 -> -1 2014-12-08 18:14:00.996831 7f28fcb46700 20 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] handle_activate_map: Not dirtying info: last_persisted is 71591 while current is 71592 2014-12-08 18:14:00.996844 7f28fc345700 10 osd.70 pg_epoch: 71584 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71433/71433 71584/71584/71584) [14,20,1] r=-1 lpr=71584 pi=71432-71583/2 crt=0'0 inactive NOTIFY] on_change 2014-12-08 18:14:00.996850 7f28fc345700 15 osd.70 pg_epoch: 71584 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71433/71433 71584/71584/71584) [14,20,1] r=-1 lpr=71584 pi=71432-71583/2 crt=0'0 inactive NOTIFY] publish_stats_to_osd -- not primary 2014-12-08 18:14:00.996846 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] handle_peering_event: epoch_sent: 71591 epoch_requested: 71591 MNotifyRec from 58 notify: (query_epoch:71591, epoch_sent:71591, info:19.6e( empty local-les=0 n=0 ec=21298 les/c 51101/50495 71591/71591/71591)) features: 0x3fffffffffff 2014-12-08 18:14:00.996856 7f28fc345700 15 osd.70 pg_epoch: 71584 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71433/71433 71584/71584/71584) [14,20,1] r=-1 lpr=71584 pi=71432-71583/2 crt=0'0 inactive NOTIFY] requeue_ops 2014-12-08 18:14:00.996862 7f28fc345700 10 osd.70 pg_epoch: 71584 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71433/71433 71584/71584/71584) [14,20,1] r=-1 lpr=71584 pi=71432-71583/2 crt=0'0 inactive NOTIFY] cancel_copy_ops 2014-12-08 18:14:00.996860 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] got osd.58 19.6e( empty local-les=0 n=0 ec=21298 les/c 51101/50495 71591/71591/71591) 2014-12-08 18:14:00.996869 7f28fc345700 10 osd.70 pg_epoch: 71584 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71433/71433 71584/71584/71584) [14,20,1] r=-1 lpr=71584 pi=71432-71583/2 crt=0'0 inactive NOTIFY] cancel_flush_ops 2014-12-08 18:14:00.996874 7f28fc345700 10 osd.70 pg_epoch: 71584 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71433/71433 71584/71584/71584) [14,20,1] r=-1 lpr=71584 pi=71432-71583/2 crt=0'0 inactive NOTIFY] on_change_cleanup 2014-12-08 18:14:00.996879 7f28fc345700 10 osd.70 pg_epoch: 71584 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71433/71433 71584/71584/71584) [14,20,1] r=-1 lpr=71584 pi=71432-71583/2 crt=0'0 inactive NOTIFY] on_change 2014-12-08 18:14:00.996876 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] osd.58 has stray content: 19.6e( empty local-les=0 n=0 ec=21298 les/c 51101/50495 71591/71591/71591) 2014-12-08 18:14:00.996886 7f28fc345700 20 osd.70 pg_epoch: 71584 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71433/71433 71584/71584/71584) [14,20,1] r=-1 lpr=71584 pi=71432-71583/2 crt=0'0 inactive NOTIFY] exit NotTrimming 2014-12-08 18:14:00.996893 7f28fc345700 20 osd.70 pg_epoch: 71584 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71433/71433 71584/71584/71584) [14,20,1] r=-1 lpr=71584 pi=71432-71583/2 crt=0'0 inactive NOTIFY] enter NotTrimming 2014-12-08 18:14:00.996890 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] update_heartbeat_peers 13,16,70 -> 13,16,58,70 2014-12-08 18:14:00.996903 7f28fcb46700 20 osd.70 71592 need_heartbeat_peer_update 2014-12-08 18:14:00.996899 7f28fc345700 10 osd.70 pg_epoch: 71584 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71433/71433 71584/71584/71584) [14,20,1] r=-1 lpr=71584 pi=71432-71583/2 crt=0'0 inactive NOTIFY] pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71433/71433 71584/71584/71584) [14,20,1] r=-1 lpr=71584 pi=71432-71583/2 crt=0'0 inactive NOTIFY] [78,1] -> [14,20,1], acting primary 78 -> 14 2014-12-08 18:14:00.996909 7f28fc345700 10 osd.70 pg_epoch: 71584 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71433/71433 71584/71584/71584) [14,20,1] r=-1 lpr=71584 pi=71432-71583/2 crt=0'0 inactive NOTIFY] cancel_recovery 2014-12-08 18:14:00.996905 7f28fcb46700 20 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] state: Adding osd: 58 features: 3fffffffffff 2014-12-08 18:14:00.996918 7f28fcb46700 10 log is not dirty 2014-12-08 18:14:00.996916 7f28fc345700 10 osd.70 pg_epoch: 71584 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71433/71433 71584/71584/71584) [14,20,1] r=-1 lpr=71584 pi=71432-71583/2 crt=0'0 inactive NOTIFY] clear_recovery_state 2014-12-08 18:14:00.996922 7f28fc345700 10 osd.70 pg_epoch: 71584 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71433/71433 71584/71584/71584) [14,20,1] r=-1 lpr=71584 pi=71432-71583/2 crt=0'0 inactive NOTIFY] check_recovery_sources no source osds () went down 2014-12-08 18:14:00.996931 7f28fcb46700 10 osd.70 71592 queue_want_up_thru want 71592 <= queued 71592, currently 71591 2014-12-08 18:14:00.996935 7f28fcb46700 20 osd.70 71592 share_map_peer 0x12750420 don't know epoch, doing nothing 2014-12-08 18:14:00.996932 7f28fc345700 10 osd.70 pg_epoch: 71584 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71433/71433 71584/71584/71584) [14,20,1] r=-1 lpr=71584 pi=71432-71583/2 crt=0'0 inactive NOTIFY] handle_advance_map [14,20,1]/[14,20,1] -- 14/14 2014-12-08 18:14:00.996938 7f28fcb46700 7 osd.70 71592 do_notifies osd 3 on 1 PGs 2014-12-08 18:14:00.996941 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:14:00.996943 7f28fc345700 10 osd.70 pg_epoch: 71585 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71433/71433 71584/71584/71584) [14,20,1] r=-1 lpr=71584 pi=71432-71583/2 crt=0'0 inactive NOTIFY] state: Reset advmap 2014-12-08 18:14:00.996948 7f28fc345700 10 osd.70 pg_epoch: 71585 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71433/71433 71584/71584/71584) [14,20,1] r=-1 lpr=71584 pi=71432-71583/2 crt=0'0 inactive NOTIFY] _calc_past_interval_range: already have past intervals back to 71433 2014-12-08 18:14:00.996955 7f28fc345700 10 osd.70 pg_epoch: 71585 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71433/71433 71584/71584/71584) [14,20,1] r=-1 lpr=71584 pi=71432-71583/2 crt=0'0 inactive NOTIFY] check_recovery_sources no source osds () went down 2014-12-08 18:14:00.996936 7f28fbb44700 10 osd.70 71592 dequeue_op 0x1285d300 prio 63 cost 4096 latency 0.220713 osd_op(client.3273665.0:2102384 rbd_data.31f57912639066.0000000000000001 [stat,set-alloc-hint object_size 8388608 write_size 8388608,write 3440640~4096] 19.e66bec6e RETRY=100 ack+ondisk+retry+write e71592) v4 pg pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] 2014-12-08 18:14:00.996964 7f28fcb46700 20 osd.70 71592 share_map_peer 0x118b9e40 don't know epoch, doing nothing 2014-12-08 18:14:00.996967 7f28fcb46700 7 osd.70 71592 do_notifies osd 4 on 1 PGs 2014-12-08 18:14:00.996964 7f28fc345700 10 osd.70 pg_epoch: 71585 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71433/71433 71584/71584/71584) [14,20,1] r=-1 lpr=71584 pi=71432-71583/2 crt=0'0 inactive NOTIFY] handle_advance_map [14,20,1]/[14,20,1] -- 14/14 2014-12-08 18:14:00.996972 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:14:00.996974 7f28fc345700 10 osd.70 pg_epoch: 71586 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71433/71433 71584/71584/71584) [14,20,1] r=-1 lpr=71584 pi=71432-71583/2 crt=0'0 inactive NOTIFY] state: Reset advmap 2014-12-08 18:14:00.996983 7f28fcb46700 20 osd.70 71592 share_map_peer 0x1262d9c0 already has epoch 71592 2014-12-08 18:14:00.996986 7f28fcb46700 7 osd.70 71592 do_notifies osd 5 on 1 PGs 2014-12-08 18:14:00.996984 7f28fc345700 10 osd.70 pg_epoch: 71586 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71433/71433 71584/71584/71584) [14,20,1] r=-1 lpr=71584 pi=71432-71583/2 crt=0'0 inactive NOTIFY] _calc_past_interval_range: already have past intervals back to 71433 2014-12-08 18:14:00.996993 7f28fc345700 10 osd.70 pg_epoch: 71586 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71433/71433 71584/71584/71584) [14,20,1] r=-1 lpr=71584 pi=71432-71583/2 crt=0'0 inactive NOTIFY] check_recovery_sources no source osds () went down 2014-12-08 18:14:00.996987 7f28fbb44700 20 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] op_has_sufficient_caps pool=19 (volumes ) owner=0 need_read_cap=1 need_write_cap=1 need_class_read_cap=0 need_class_write_cap=0 -> yes 2014-12-08 18:14:00.997002 7f28fcb46700 20 osd.70 71592 share_map_peer 0x127b7fa0 already has epoch 71592 2014-12-08 18:14:00.997004 7f28fcb46700 7 osd.70 71592 do_notifies osd 7 on 1 PGs 2014-12-08 18:14:00.997003 7f28fc345700 10 osd.70 pg_epoch: 71586 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71433/71433 71584/71584/71584) [14,20,1] r=-1 lpr=71584 pi=71432-71583/2 crt=0'0 inactive NOTIFY] handle_advance_map [78,1]/[78,1] -- 78/78 2014-12-08 18:14:00.997013 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:14:00.997003 7f28fbb44700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] can_handle_while_inactive: 0x1285d300 2014-12-08 18:14:00.997018 7f28fbb44700 10 osd.70 71592 dequeue_op 0x1285d300 finish 2014-12-08 18:14:00.997017 7f28fc345700 10 osd.70 pg_epoch: 71587 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71433/71433 71584/71584/71584) [14,20,1] r=-1 lpr=71584 pi=71432-71583/2 crt=0'0 inactive NOTIFY] state: Reset advmap 2014-12-08 18:14:00.997022 7f28fcb46700 20 osd.70 71592 share_map_peer 0x127f8100 don't know epoch, doing nothing 2014-12-08 18:14:00.997025 7f28fcb46700 7 osd.70 71592 do_notifies osd 9 on 2 PGs 2014-12-08 18:14:00.997022 7f28fc345700 10 osd.70 pg_epoch: 71587 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71433/71433 71584/71584/71584) [14,20,1] r=-1 lpr=71584 pi=71432-71583/2 crt=0'0 inactive NOTIFY] _calc_past_interval_range: already have past intervals back to 71433 2014-12-08 18:14:00.997029 7f28fc345700 20 osd.70 pg_epoch: 71587 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71433/71433 71584/71584/71584) [14,20,1] r=-1 lpr=71584 pi=71432-71583/2 crt=0'0 inactive NOTIFY] new interval newup [78,1] newacting [78,1] 2014-12-08 18:14:00.997035 7f28fc345700 10 osd.70 pg_epoch: 71587 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71433/71433 71584/71584/71584) [14,20,1] r=-1 lpr=71584 pi=71432-71583/2 crt=0'0 inactive NOTIFY] state: should restart peering, calling start_peering_interval again 2014-12-08 18:14:00.997041 7f28fc345700 20 osd.70 pg_epoch: 71587 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71433/71433 71584/71584/71584) [14,20,1] r=-1 lpr=71584 pi=71432-71583/2 crt=0'0 inactive NOTIFY] set_last_peering_reset 71587 2014-12-08 18:14:00.997044 7f28fcb46700 20 osd.70 71592 share_map_peer 0x126b8c00 don't know epoch, doing nothing 2014-12-08 18:14:00.997047 7f28fcb46700 7 osd.70 71592 do_notifies osd 16 on 1 PGs 2014-12-08 18:14:00.997046 7f28fc345700 10 osd.70 pg_epoch: 71587 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71433/71433 71584/71584/71584) [14,20,1] r=-1 lpr=71587 pi=71432-71583/2 crt=0'0 inactive NOTIFY] Clearing blocked outgoing recovery messages 2014-12-08 18:14:00.997030 7f28f933f700 10 osd.70 71592 dequeue_op 0x1285d400 prio 63 cost 4096 latency 0.220541 osd_op(client.3273665.0:2102385 rbd_data.31f57912639066.0000000000000001 [stat,set-alloc-hint object_size 8388608 write_size 8388608,write 3457024~4096] 19.e66bec6e RETRY=100 ack+ondisk+retry+write e71592) v4 pg pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] 2014-12-08 18:14:00.997054 7f28fc345700 10 osd.70 pg_epoch: 71587 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71433/71433 71584/71584/71584) [14,20,1] r=-1 lpr=71587 pi=71432-71583/2 crt=0'0 inactive NOTIFY] Not blocking outgoing recovery messages 2014-12-08 18:14:00.997065 7f28fcb46700 20 osd.70 71592 share_map_peer 0x126b8940 don't know epoch, doing nothing 2014-12-08 18:14:00.997067 7f28fcb46700 7 osd.70 71592 do_notifies osd 19 on 1 PGs 2014-12-08 18:14:00.997065 7f28fc345700 10 osd.70 pg_epoch: 71587 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71433/71433 71584/71584/71584) [78,1] r=-1 lpr=71587 pi=71432-71586/3 crt=0'0 inactive NOTIFY] start_peering_interval: check_new_interval output: generate_past_intervals interval(71584-71586 up [14,20,1](14) acting [14,20,1](14)): not rw, up_thru 71585 up_from 69670 last_epoch_clean 71433 generate_past_intervals interval(71584-71586 up [14,20,1](14) acting [14,20,1](14) maybe_went_rw) : primary up 69670-71585 includes interval 2014-12-08 18:14:00.997075 7f28fc345700 10 osd.70 pg_epoch: 71587 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71433/71433 71584/71584/71584) [78,1] r=-1 lpr=71587 pi=71432-71586/3 crt=0'0 inactive NOTIFY] noting past interval(71584-71586 up [14,20,1](14) acting [14,20,1](14) maybe_went_rw) 2014-12-08 18:14:00.997083 7f28fcb46700 20 osd.70 71592 share_map_peer 0x1268b760 don't know epoch, doing nothing 2014-12-08 18:14:00.997086 7f28fcb46700 7 osd.70 71592 do_notifies osd 23 on 2 PGs 2014-12-08 18:14:00.997081 7f28fc345700 10 osd.70 pg_epoch: 71587 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71433/71433 71587/71587/71587) [78,1] r=-1 lpr=71587 pi=71432-71586/3 crt=0'0 inactive NOTIFY] up [14,20,1] -> [78,1], acting [14,20,1] -> [78,1], acting_primary 14 -> 78, up_primary 14 -> 78, role -1 -> -1 2014-12-08 18:14:00.997076 7f28f933f700 20 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] op_has_sufficient_caps pool=19 (volumes ) owner=0 need_read_cap=1 need_write_cap=1 need_class_read_cap=0 need_class_write_cap=0 -> yes 2014-12-08 18:14:00.997091 7f28fc345700 10 osd.70 pg_epoch: 71587 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71433/71433 71587/71587/71587) [78,1] r=-1 lpr=71587 pi=71432-71586/3 crt=0'0 inactive NOTIFY] on_change 2014-12-08 18:14:00.997096 7f28fc345700 15 osd.70 pg_epoch: 71587 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71433/71433 71587/71587/71587) [78,1] r=-1 lpr=71587 pi=71432-71586/3 crt=0'0 inactive NOTIFY] publish_stats_to_osd -- not primary 2014-12-08 18:14:00.997095 7f28f933f700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] can_handle_while_inactive: 0x1285d400 2014-12-08 18:14:00.997105 7f28f933f700 10 osd.70 71592 dequeue_op 0x1285d400 finish 2014-12-08 18:14:00.997104 7f28fcb46700 20 osd.70 71592 share_map_peer 0x12769700 already has epoch 71592 2014-12-08 18:14:00.997102 7f28fc345700 15 osd.70 pg_epoch: 71587 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71433/71433 71587/71587/71587) [78,1] r=-1 lpr=71587 pi=71432-71586/3 crt=0'0 inactive NOTIFY] requeue_ops 2014-12-08 18:14:00.997108 7f28fcb46700 7 osd.70 71592 do_notifies osd 25 on 1 PGs 2014-12-08 18:14:00.997109 7f28fc345700 10 osd.70 pg_epoch: 71587 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71433/71433 71587/71587/71587) [78,1] r=-1 lpr=71587 pi=71432-71586/3 crt=0'0 inactive NOTIFY] cancel_copy_ops 2014-12-08 18:14:00.997115 7f28fc345700 10 osd.70 pg_epoch: 71587 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71433/71433 71587/71587/71587) [78,1] r=-1 lpr=71587 pi=71432-71586/3 crt=0'0 inactive NOTIFY] cancel_flush_ops 2014-12-08 18:14:00.997120 7f28fc345700 10 osd.70 pg_epoch: 71587 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71433/71433 71587/71587/71587) [78,1] r=-1 lpr=71587 pi=71432-71586/3 crt=0'0 inactive NOTIFY] on_change_cleanup 2014-12-08 18:14:00.997124 7f28fcb46700 20 osd.70 71592 share_map_peer 0x126b8680 already has epoch 71592 2014-12-08 18:14:00.997126 7f28fcb46700 7 osd.70 71592 do_notifies osd 56 on 1 PGs 2014-12-08 18:14:00.997125 7f28fc345700 10 osd.70 pg_epoch: 71587 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71433/71433 71587/71587/71587) [78,1] r=-1 lpr=71587 pi=71432-71586/3 crt=0'0 inactive NOTIFY] on_change 2014-12-08 18:14:00.997133 7f28fc345700 20 osd.70 pg_epoch: 71587 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71433/71433 71587/71587/71587) [78,1] r=-1 lpr=71587 pi=71432-71586/3 crt=0'0 inactive NOTIFY] exit NotTrimming 2014-12-08 18:14:00.997139 7f28fc345700 20 osd.70 pg_epoch: 71587 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71433/71433 71587/71587/71587) [78,1] r=-1 lpr=71587 pi=71432-71586/3 crt=0'0 inactive NOTIFY] enter NotTrimming 2014-12-08 18:14:00.997143 7f28fcb46700 20 osd.70 71592 share_map_peer 0x12889ce0 don't know epoch, doing nothing 2014-12-08 18:14:00.997146 7f28fcb46700 7 osd.70 71592 do_notifies osd 66 on 1 PGs 2014-12-08 18:14:00.997143 7f28fc345700 10 osd.70 pg_epoch: 71587 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71433/71433 71587/71587/71587) [78,1] r=-1 lpr=71587 pi=71432-71586/3 crt=0'0 inactive NOTIFY] pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71433/71433 71587/71587/71587) [78,1] r=-1 lpr=71587 pi=71432-71586/3 crt=0'0 inactive NOTIFY] [14,20,1] -> [78,1], acting primary 14 -> 78 2014-12-08 18:14:00.997155 7f28fc345700 10 osd.70 pg_epoch: 71587 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71433/71433 71587/71587/71587) [78,1] r=-1 lpr=71587 pi=71432-71586/3 crt=0'0 inactive NOTIFY] cancel_recovery 2014-12-08 18:14:00.997159 7f28fc345700 10 osd.70 pg_epoch: 71587 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71433/71433 71587/71587/71587) [78,1] r=-1 lpr=71587 pi=71432-71586/3 crt=0'0 inactive NOTIFY] clear_recovery_state 2014-12-08 18:14:00.997166 7f28fc345700 10 osd.70 pg_epoch: 71587 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71433/71433 71587/71587/71587) [78,1] r=-1 lpr=71587 pi=71432-71586/3 crt=0'0 inactive NOTIFY] check_recovery_sources no source osds () went down 2014-12-08 18:14:00.997179 7f28fc345700 10 osd.70 pg_epoch: 71587 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71433/71433 71587/71587/71587) [78,1] r=-1 lpr=71587 pi=71432-71586/3 crt=0'0 inactive NOTIFY] handle_advance_map [78,1]/[78,1] -- 78/78 2014-12-08 18:14:00.997186 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) -- 2014-12-08 18:14:01.016564 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] handle_peering_event: epoch_sent: 71591 epoch_requested: 71591 MNotifyRec from 30 notify: (query_epoch:71591, epoch_sent:71591, info:19.6e( empty local-les=0 n=0 ec=21298 les/c 51101/50495 71591/71591/71591)) features: 0x3fffffffffff 2014-12-08 18:14:01.016576 7f28fc345700 10 osd.70 pg_epoch: 71586 pg[218.71a( v 71581'3 (0'0,71581'3] local-les=71433 n=3 ec=71423 les/c 71433/71433 71584/71584/71425) [10,68,4] r=-1 lpr=71584 pi=71423-71583/6 crt=71580'1 lcod 0'0 inactive NOTIFY] handle_advance_map [10,4]/[10,4] -- 10/10 2014-12-08 18:14:01.016583 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:14:01.016586 7f28fc345700 10 osd.70 pg_epoch: 71587 pg[218.71a( v 71581'3 (0'0,71581'3] local-les=71433 n=3 ec=71423 les/c 71433/71433 71584/71584/71425) [10,68,4] r=-1 lpr=71584 pi=71423-71583/6 crt=71580'1 lcod 0'0 inactive NOTIFY] state: Reset advmap 2014-12-08 18:14:01.016582 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] got osd.30 19.6e( empty local-les=0 n=0 ec=21298 les/c 51101/50495 71591/71591/71591) 2014-12-08 18:14:01.016593 7f28fc345700 10 osd.70 pg_epoch: 71587 pg[218.71a( v 71581'3 (0'0,71581'3] local-les=71433 n=3 ec=71423 les/c 71433/71433 71584/71584/71425) [10,68,4] r=-1 lpr=71584 pi=71423-71583/6 crt=71580'1 lcod 0'0 inactive NOTIFY] _calc_past_interval_range: already have past intervals back to 71433 2014-12-08 18:14:01.016601 7f28fc345700 20 osd.70 pg_epoch: 71587 pg[218.71a( v 71581'3 (0'0,71581'3] local-les=71433 n=3 ec=71423 les/c 71433/71433 71584/71584/71425) [10,68,4] r=-1 lpr=71584 pi=71423-71583/6 crt=71580'1 lcod 0'0 inactive NOTIFY] new interval newup [10,4] newacting [10,4] 2014-12-08 18:14:01.016609 7f28fc345700 10 osd.70 pg_epoch: 71587 pg[218.71a( v 71581'3 (0'0,71581'3] local-les=71433 n=3 ec=71423 les/c 71433/71433 71584/71584/71425) [10,68,4] r=-1 lpr=71584 pi=71423-71583/6 crt=71580'1 lcod 0'0 inactive NOTIFY] state: should restart peering, calling start_peering_interval again 2014-12-08 18:14:01.016600 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] osd.30 has stray content: 19.6e( empty local-les=0 n=0 ec=21298 les/c 51101/50495 71591/71591/71591) 2014-12-08 18:14:01.016616 7f28fc345700 20 osd.70 pg_epoch: 71587 pg[218.71a( v 71581'3 (0'0,71581'3] local-les=71433 n=3 ec=71423 les/c 71433/71433 71584/71584/71425) [10,68,4] r=-1 lpr=71584 pi=71423-71583/6 crt=71580'1 lcod 0'0 inactive NOTIFY] set_last_peering_reset 71587 2014-12-08 18:14:01.016624 7f28fc345700 10 osd.70 pg_epoch: 71587 pg[218.71a( v 71581'3 (0'0,71581'3] local-les=71433 n=3 ec=71423 les/c 71433/71433 71584/71584/71425) [10,68,4] r=-1 lpr=71587 pi=71423-71583/6 crt=71580'1 lcod 0'0 inactive NOTIFY] Clearing blocked outgoing recovery messages 2014-12-08 18:14:01.016618 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] update_heartbeat_peers 13,16,58,70 -> 13,16,30,58,70 2014-12-08 18:14:01.016633 7f28fcb46700 20 osd.70 71592 need_heartbeat_peer_update 2014-12-08 18:14:01.016632 7f28fc345700 10 osd.70 pg_epoch: 71587 pg[218.71a( v 71581'3 (0'0,71581'3] local-les=71433 n=3 ec=71423 les/c 71433/71433 71584/71584/71425) [10,68,4] r=-1 lpr=71587 pi=71423-71583/6 crt=71580'1 lcod 0'0 inactive NOTIFY] Not blocking outgoing recovery messages 2014-12-08 18:14:01.016636 7f28fcb46700 20 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] state: Adding osd: 30 features: 3fffffffffff 2014-12-08 18:14:01.016649 7f28fcb46700 10 log is not dirty 2014-12-08 18:14:01.016647 7f28fc345700 10 osd.70 pg_epoch: 71587 pg[218.71a( v 71581'3 (0'0,71581'3] local-les=71433 n=3 ec=71423 les/c 71433/71433 71584/71584/71425) [10,4] r=-1 lpr=71587 pi=71423-71586/7 crt=71580'1 lcod 0'0 inactive NOTIFY] start_peering_interval: check_new_interval output: generate_past_intervals interval(71584-71586 up [10,68,4](10) acting [10,68,4](10)): not rw, up_thru 71585 up_from 69670 last_epoch_clean 71433 generate_past_intervals interval(71584-71586 up [10,68,4](10) acting [10,68,4](10) maybe_went_rw) : primary up 69670-71585 includes interval 2014-12-08 18:14:01.016657 7f28fc345700 10 osd.70 pg_epoch: 71587 pg[218.71a( v 71581'3 (0'0,71581'3] local-les=71433 n=3 ec=71423 les/c 71433/71433 71584/71584/71425) [10,4] r=-1 lpr=71587 pi=71423-71586/7 crt=71580'1 lcod 0'0 inactive NOTIFY] noting past interval(71584-71586 up [10,68,4](10) acting [10,68,4](10) maybe_went_rw) 2014-12-08 18:14:01.016656 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[218.3dd( v 71581'2 (0'0,71581'2] local-les=71437 n=2 ec=71423 les/c 71437/71437 71591/71592/71592) [70,66,14]/[66,14,8] r=-1 lpr=71592 pi=71432-71591/5 crt=0'0 lcod 0'0 remapped NOTIFY] handle_peering_event: epoch_sent: 71591 epoch_requested: 71591 MNotifyRec from 8 notify: (query_epoch:71591, epoch_sent:71591, info:218.3dd( v 71581'2 lc 0'0 (0'0,71581'2] local-les=71585 n=2 ec=71423 les/c 71585/71583 71591/71591/71591)) features: 0x3fffffffffff 2014-12-08 18:14:01.016666 7f28fc345700 10 osd.70 pg_epoch: 71587 pg[218.71a( v 71581'3 (0'0,71581'3] local-les=71433 n=3 ec=71423 les/c 71433/71433 71587/71587/71425) [10,4] r=-1 lpr=71587 pi=71423-71586/7 crt=71580'1 lcod 0'0 inactive NOTIFY] up [10,68,4] -> [10,4], acting [10,68,4] -> [10,4], acting_primary 10 -> 10, up_primary 10 -> 10, role -1 -> -1 2014-12-08 18:14:01.016669 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[218.3dd( v 71581'2 (0'0,71581'2] local-les=71437 n=2 ec=71423 les/c 71437/71437 71591/71592/71592) [70,66,14]/[66,14,8] r=-1 lpr=71592 pi=71432-71591/5 crt=0'0 lcod 0'0 remapped NOTIFY] old_peering_msg reply_epoch 71591 query_epoch 71591 last_peering_reset 71592 2014-12-08 18:14:01.016682 7f28fcb46700 10 log is not dirty 2014-12-08 18:14:01.016675 7f28fc345700 10 osd.70 pg_epoch: 71587 pg[218.71a( v 71581'3 (0'0,71581'3] local-les=71433 n=3 ec=71423 les/c 71433/71433 71587/71587/71425) [10,4] r=-1 lpr=71587 pi=71423-71586/7 crt=71580'1 lcod 0'0 inactive NOTIFY] on_change 2014-12-08 18:14:01.016687 7f28fc345700 15 osd.70 pg_epoch: 71587 pg[218.71a( v 71581'3 (0'0,71581'3] local-les=71433 n=3 ec=71423 les/c 71433/71433 71587/71587/71425) [10,4] r=-1 lpr=71587 pi=71423-71586/7 crt=71580'1 lcod 0'0 inactive NOTIFY] publish_stats_to_osd -- not primary 2014-12-08 18:14:01.016693 7f28fc345700 15 osd.70 pg_epoch: 71587 pg[218.71a( v 71581'3 (0'0,71581'3] local-les=71433 n=3 ec=71423 les/c 71433/71433 71587/71587/71425) [10,4] r=-1 lpr=71587 pi=71423-71586/7 crt=71580'1 lcod 0'0 inactive NOTIFY] requeue_ops 2014-12-08 18:14:01.016691 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[217.bb( v 71377'16307 (71377'13306,71377'16307] local-les=71435 n=22 ec=71094 les/c 71586/71583 71591/71591/71591) [70,78,25] r=0 lpr=71591 pi=71434-71590/6 crt=71377'16305 lcod 0'0 mlcod 0'0 peering] handle_peering_event: epoch_sent: 71591 epoch_requested: 71591 MNotifyRec from 5 notify: (query_epoch:71591, epoch_sent:71591, info:217.bb( v 71377'16307 (71377'13307,71377'16307] lb 0//0//-1 local-les=0 n=0 ec=71094 les/c 71586/71583 71591/71591/71591)) features: 0x3fffffffffff 2014-12-08 18:14:01.016701 7f28fc345700 10 osd.70 pg_epoch: 71587 pg[218.71a( v 71581'3 (0'0,71581'3] local-les=71433 n=3 ec=71423 les/c 71433/71433 71587/71587/71425) [10,4] r=-1 lpr=71587 pi=71423-71586/7 crt=71580'1 lcod 0'0 inactive NOTIFY] cancel_copy_ops 2014-12-08 18:14:01.016707 7f28fc345700 10 osd.70 pg_epoch: 71587 pg[218.71a( v 71581'3 (0'0,71581'3] local-les=71433 n=3 ec=71423 les/c 71433/71433 71587/71587/71425) [10,4] r=-1 lpr=71587 pi=71423-71586/7 crt=71580'1 lcod 0'0 inactive NOTIFY] cancel_flush_ops 2014-12-08 18:14:01.016714 7f28fc345700 10 osd.70 pg_epoch: 71587 pg[218.71a( v 71581'3 (0'0,71581'3] local-les=71433 n=3 ec=71423 les/c 71433/71433 71587/71587/71425) [10,4] r=-1 lpr=71587 pi=71423-71586/7 crt=71580'1 lcod 0'0 inactive NOTIFY] on_change_cleanup 2014-12-08 18:14:01.016707 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[217.bb( v 71377'16307 (71377'13306,71377'16307] local-les=71435 n=22 ec=71094 les/c 71586/71583 71591/71591/71591) [70,78,25] r=0 lpr=71591 pi=71434-71590/6 crt=71377'16305 lcod 0'0 mlcod 0'0 peering] got osd.5 217.bb( v 71377'16307 (71377'13307,71377'16307] lb 0//0//-1 local-les=0 n=0 ec=71094 les/c 71586/71583 71591/71591/71591) 2014-12-08 18:14:01.016722 7f28fc345700 10 osd.70 pg_epoch: 71587 pg[218.71a( v 71581'3 (0'0,71581'3] local-les=71433 n=3 ec=71423 les/c 71433/71433 71587/71587/71425) [10,4] r=-1 lpr=71587 pi=71423-71586/7 crt=71580'1 lcod 0'0 inactive NOTIFY] on_change 2014-12-08 18:14:01.016729 7f28fc345700 20 osd.70 pg_epoch: 71587 pg[218.71a( v 71581'3 (0'0,71581'3] local-les=71433 n=3 ec=71423 les/c 71433/71433 71587/71587/71425) [10,4] r=-1 lpr=71587 pi=71423-71586/7 crt=71580'1 lcod 0'0 inactive NOTIFY] exit NotTrimming 2014-12-08 18:14:01.016727 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[217.bb( v 71377'16307 (71377'13306,71377'16307] local-les=71435 n=22 ec=71094 les/c 71586/71583 71591/71591/71591) [70,78,25] r=0 lpr=71591 pi=71434-71590/6 crt=71377'16305 lcod 0'0 mlcod 0'0 peering] osd.5 has stray content: 217.bb( v 71377'16307 (71377'13307,71377'16307] lb 0//0//-1 local-les=0 n=0 ec=71094 les/c 71586/71583 71591/71591/71591) -- 2014-12-08 18:14:01.020602 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] handle_peering_event: epoch_sent: 71591 epoch_requested: 71591 MNotifyRec from 76 notify: (query_epoch:71591, epoch_sent:71591, info:19.6e( empty local-les=0 n=0 ec=21298 les/c 51101/50495 71591/71591/71591)) features: 0x3fffffffffff 2014-12-08 18:14:01.020617 7f28fc345700 10 osd.70 pg_epoch: 71585 pg[217.fc( v 71377'4103 (71377'1102,71377'4103] local-les=71434 n=12 ec=71094 les/c 71434/71434 71584/71585/71168) [18,4,64]/[18,4] r=-1 lpr=71585 pi=71094-71584/31 crt=71377'4100 lcod 0'0 remapped NOTIFY] on_change_cleanup 2014-12-08 18:14:01.020623 7f28fc345700 10 osd.70 pg_epoch: 71585 pg[217.fc( v 71377'4103 (71377'1102,71377'4103] local-les=71434 n=12 ec=71094 les/c 71434/71434 71584/71585/71168) [18,4,64]/[18,4] r=-1 lpr=71585 pi=71094-71584/31 crt=71377'4100 lcod 0'0 remapped NOTIFY] on_change 2014-12-08 18:14:01.020622 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] got osd.76 19.6e( empty local-les=0 n=0 ec=21298 les/c 51101/50495 71591/71591/71591) 2014-12-08 18:14:01.020631 7f28fc345700 20 osd.70 pg_epoch: 71585 pg[217.fc( v 71377'4103 (71377'1102,71377'4103] local-les=71434 n=12 ec=71094 les/c 71434/71434 71584/71585/71168) [18,4,64]/[18,4] r=-1 lpr=71585 pi=71094-71584/31 crt=71377'4100 lcod 0'0 remapped NOTIFY] exit NotTrimming 2014-12-08 18:14:01.020638 7f28fc345700 20 osd.70 pg_epoch: 71585 pg[217.fc( v 71377'4103 (71377'1102,71377'4103] local-les=71434 n=12 ec=71094 les/c 71434/71434 71584/71585/71168) [18,4,64]/[18,4] r=-1 lpr=71585 pi=71094-71584/31 crt=71377'4100 lcod 0'0 remapped NOTIFY] enter NotTrimming 2014-12-08 18:14:01.020638 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] osd.76 has stray content: 19.6e( empty local-les=0 n=0 ec=21298 les/c 51101/50495 71591/71591/71591) 2014-12-08 18:14:01.020647 7f28fc345700 10 osd.70 pg_epoch: 71585 pg[217.fc( v 71377'4103 (71377'1102,71377'4103] local-les=71434 n=12 ec=71094 les/c 71434/71434 71584/71585/71168) [18,4,64]/[18,4] r=-1 lpr=71585 pi=71094-71584/31 crt=71377'4100 lcod 0'0 remapped NOTIFY] cancel_recovery 2014-12-08 18:14:01.020653 7f28fc345700 10 osd.70 pg_epoch: 71585 pg[217.fc( v 71377'4103 (71377'1102,71377'4103] local-les=71434 n=12 ec=71094 les/c 71434/71434 71584/71585/71168) [18,4,64]/[18,4] r=-1 lpr=71585 pi=71094-71584/31 crt=71377'4100 lcod 0'0 remapped NOTIFY] clear_recovery_state 2014-12-08 18:14:01.020651 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] update_heartbeat_peers 13,16,30,58,70 -> 13,16,30,58,70,76 2014-12-08 18:14:01.020664 7f28fcb46700 20 osd.70 71592 need_heartbeat_peer_update 2014-12-08 18:14:01.020662 7f28fc345700 10 osd.70 pg_epoch: 71585 pg[217.fc( v 71377'4103 (71377'1102,71377'4103] local-les=71434 n=12 ec=71094 les/c 71434/71434 71584/71585/71168) [18,4,64]/[18,4] r=-1 lpr=71585 pi=71094-71584/31 crt=71377'4100 lcod 0'0 remapped NOTIFY] check_recovery_sources no source osds () went down 2014-12-08 18:14:01.020666 7f28fcb46700 20 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] state: Adding osd: 76 features: 3fffffffffff 2014-12-08 18:14:01.020677 7f28fcb46700 10 log is not dirty 2014-12-08 18:14:01.020676 7f28fc345700 10 osd.70 pg_epoch: 71585 pg[217.fc( v 71377'4103 (71377'1102,71377'4103] local-les=71434 n=12 ec=71094 les/c 71434/71434 71584/71585/71168) [18,4,64]/[18,4] r=-1 lpr=71585 pi=71094-71584/31 crt=71377'4100 lcod 0'0 remapped NOTIFY] handle_advance_map [18,4,64]/[18,4] -- 18/18 2014-12-08 18:14:01.020684 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:14:01.020686 7f28fc345700 10 osd.70 pg_epoch: 71586 pg[217.fc( v 71377'4103 (71377'1102,71377'4103] local-les=71434 n=12 ec=71094 les/c 71434/71434 71584/71585/71168) [18,4,64]/[18,4] r=-1 lpr=71585 pi=71094-71584/31 crt=71377'4100 lcod 0'0 remapped NOTIFY] state: Reset advmap 2014-12-08 18:14:01.020693 7f28fc345700 10 osd.70 pg_epoch: 71586 pg[217.fc( v 71377'4103 (71377'1102,71377'4103] local-les=71434 n=12 ec=71094 les/c 71434/71434 71584/71585/71168) [18,4,64]/[18,4] r=-1 lpr=71585 pi=71094-71584/31 crt=71377'4100 lcod 0'0 remapped NOTIFY] _calc_past_interval_range: already have past intervals back to 71434 2014-12-08 18:14:01.020690 7f28fcb46700 10 osd.70 pg_epoch: 71591 pg[218.4d( v 71581'1 (0'0,71581'1] local-les=71457 n=1 ec=71423 les/c 71457/71457 71591/71591/71425) [6,19,70] r=2 lpr=71591 pi=71423-71590/9 crt=0'0 lcod 0'0 inactive NOTIFY] handle_advance_map [6,19,70]/[6,19,9] -- 6/6 2014-12-08 18:14:01.020705 7f28fcb46700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:14:01.020701 7f28fc345700 10 osd.70 pg_epoch: 71586 pg[217.fc( v 71377'4103 (71377'1102,71377'4103] local-les=71434 n=12 ec=71094 les/c 71434/71434 71584/71585/71168) [18,4,64]/[18,4] r=-1 lpr=71585 pi=71094-71584/31 crt=71377'4100 lcod 0'0 remapped NOTIFY] check_recovery_sources no source osds () went down 2014-12-08 18:14:01.020710 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[218.4d( v 71581'1 (0'0,71581'1] local-les=71457 n=1 ec=71423 les/c 71457/71457 71591/71591/71425) [6,19,70] r=2 lpr=71591 pi=71423-71590/9 crt=0'0 lcod 0'0 inactive NOTIFY] state: Started advmap 2014-12-08 18:14:01.020715 7f28fc345700 10 osd.70 pg_epoch: 71586 pg[217.fc( v 71377'4103 (71377'1102,71377'4103] local-les=71434 n=12 ec=71094 les/c 71434/71434 71584/71585/71168) [18,4,64]/[18,4] r=-1 lpr=71585 pi=71094-71584/31 crt=71377'4100 lcod 0'0 remapped NOTIFY] handle_advance_map [18,4]/[18,4] -- 18/18 2014-12-08 18:14:01.020725 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:14:01.020719 7f28fcb46700 20 osd.70 pg_epoch: 71592 pg[218.4d( v 71581'1 (0'0,71581'1] local-les=71457 n=1 ec=71423 les/c 71457/71457 71591/71591/71425) [6,19,70] r=2 lpr=71591 pi=71423-71590/9 crt=0'0 lcod 0'0 inactive NOTIFY] new interval newup [6,19,70] newacting [6,19,9] 2014-12-08 18:14:01.020728 7f28fc345700 10 osd.70 pg_epoch: 71587 pg[217.fc( v 71377'4103 (71377'1102,71377'4103] local-les=71434 n=12 ec=71094 les/c 71434/71434 71584/71585/71168) [18,4,64]/[18,4] r=-1 lpr=71585 pi=71094-71584/31 crt=71377'4100 lcod 0'0 remapped NOTIFY] state: Reset advmap 2014-12-08 18:14:01.020730 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[218.4d( v 71581'1 (0'0,71581'1] local-les=71457 n=1 ec=71423 les/c 71457/71457 71591/71591/71425) [6,19,70] r=2 lpr=71591 pi=71423-71590/9 crt=0'0 lcod 0'0 inactive NOTIFY] state: should_restart_peering, transitioning to Reset 2014-12-08 18:14:01.020737 7f28fc345700 10 osd.70 pg_epoch: 71587 pg[217.fc( v 71377'4103 (71377'1102,71377'4103] local-les=71434 n=12 ec=71094 les/c 71434/71434 71584/71585/71168) [18,4,64]/[18,4] r=-1 lpr=71585 pi=71094-71584/31 crt=71377'4100 lcod 0'0 remapped NOTIFY] _calc_past_interval_range: already have past intervals back to 71434 2014-12-08 18:14:01.020745 7f28fcb46700 5 osd.70 pg_epoch: 71592 pg[218.4d( v 71581'1 (0'0,71581'1] local-les=71457 n=1 ec=71423 les/c 71457/71457 71591/71591/71425) [6,19,70] r=2 lpr=71591 pi=71423-71590/9 crt=0'0 lcod 0'0 inactive NOTIFY] exit Started/Stray 0.502858 2 0.000120 2014-12-08 18:14:01.020747 7f28fc345700 20 osd.70 pg_epoch: 71587 pg[217.fc( v 71377'4103 (71377'1102,71377'4103] local-les=71434 n=12 ec=71094 les/c 71434/71434 71584/71585/71168) [18,4,64]/[18,4] r=-1 lpr=71585 pi=71094-71584/31 crt=71377'4100 lcod 0'0 remapped NOTIFY] new interval newup [18,4] newacting [18,4] 2014-12-08 18:14:01.020756 7f28fc345700 10 osd.70 pg_epoch: 71587 pg[217.fc( v 71377'4103 (71377'1102,71377'4103] local-les=71434 n=12 ec=71094 les/c 71434/71434 71584/71585/71168) [18,4,64]/[18,4] r=-1 lpr=71585 pi=71094-71584/31 crt=71377'4100 lcod 0'0 remapped NOTIFY] state: should restart peering, calling start_peering_interval again 2014-12-08 18:14:01.020757 7f28fcb46700 5 osd.70 pg_epoch: 71592 pg[218.4d( v 71581'1 (0'0,71581'1] local-les=71457 n=1 ec=71423 les/c 71457/71457 71591/71591/71425) [6,19,70] r=2 lpr=71591 pi=71423-71590/9 crt=0'0 lcod 0'0 inactive NOTIFY] exit Started 0.502915 0 0.000000 2014-12-08 18:14:01.020764 7f28fc345700 20 osd.70 pg_epoch: 71587 pg[217.fc( v 71377'4103 (71377'1102,71377'4103] local-les=71434 n=12 ec=71094 les/c 71434/71434 71584/71585/71168) [18,4,64]/[18,4] r=-1 lpr=71585 pi=71094-71584/31 crt=71377'4100 lcod 0'0 remapped NOTIFY] set_last_peering_reset 71587 -- 2014-12-08 18:14:01.026014 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] handle_peering_event: epoch_sent: 71591 epoch_requested: 71591 MNotifyRec from 4 notify: (query_epoch:71591, epoch_sent:71591, info:19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591)) features: 0x3fffffffffff 2014-12-08 18:14:01.026032 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] got osd.4 19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) 2014-12-08 18:14:01.026037 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[19.30( v 71397'38291943 (70960'38288942,71397'38291943] local-les=71445 n=911 ec=21298 les/c 71583/71583 71591/71591/71591) [70,66,1] r=0 lpr=71591 pi=71444-71590/6 crt=71397'38291940 lcod 0'0 mlcod 0'0 peering] calc_acting osd.1 19.30( v 71397'38291943 (70960'38288942,71397'38291943] local-les=71586 n=911 ec=21298 les/c 71583/71583 71591/71591/71591) 2014-12-08 18:14:01.026054 7f2904b56700 7 osd.70 71592 handle_pg_info pg_info(1 pgs e71592:20.31(3)) v4 from osd.16 2014-12-08 18:14:01.026047 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[19.30( v 71397'38291943 (70960'38288942,71397'38291943] local-les=71445 n=911 ec=21298 les/c 71583/71583 71591/71591/71591) [70,66,1] r=0 lpr=71591 pi=71444-71590/6 crt=71397'38291940 lcod 0'0 mlcod 0'0 peering] calc_acting osd.8 19.30( v 71397'38291943 (70960'38288943,71397'38291943] lb 0//0//-1 local-les=0 n=0 ec=21298 les/c 71583/71583 71591/71591/71591) 2014-12-08 18:14:01.026058 7f2904b56700 15 osd.70 71592 require_same_or_newer_map 71592 (i am 71592) 0x12b79c00 2014-12-08 18:14:01.026047 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] osd.4 has stray content: 19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) 2014-12-08 18:14:01.026060 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[19.30( v 71397'38291943 (70960'38288942,71397'38291943] local-les=71445 n=911 ec=21298 les/c 71583/71583 71591/71591/71591) [70,66,1] r=0 lpr=71591 pi=71444-71590/6 crt=71397'38291940 lcod 0'0 mlcod 0'0 peering] calc_acting osd.66 19.30( v 71397'38291943 (70960'38288942,71397'38291943] local-les=71583 n=911 ec=21298 les/c 71583/71583 71591/71591/71591) 2014-12-08 18:14:01.026066 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] update_heartbeat_peers 13,16,30,58,70,76 -> 4,13,16,30,58,70,76 2014-12-08 18:14:01.026071 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[19.30( v 71397'38291943 (70960'38288942,71397'38291943] local-les=71445 n=911 ec=21298 les/c 71583/71583 71591/71591/71591) [70,66,1] r=0 lpr=71591 pi=71444-71590/6 crt=71397'38291940 lcod 0'0 mlcod 0'0 peering] calc_acting osd.70 19.30( v 71397'38291943 (70960'38288942,71397'38291943] local-les=71445 n=911 ec=21298 les/c 71583/71583 71591/71591/71591) 2014-12-08 18:14:01.026081 7f28fcb46700 20 osd.70 71592 need_heartbeat_peer_update 2014-12-08 18:14:01.026083 7f2904b56700 10 osd.70 71592 do_waiters -- start 2014-12-08 18:14:01.026086 7f2904b56700 10 osd.70 71592 do_waiters -- finish 2014-12-08 18:14:01.026083 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[19.30( v 71397'38291943 (70960'38288942,71397'38291943] local-les=71445 n=911 ec=21298 les/c 71583/71583 71591/71591/71591) [70,66,1] r=0 lpr=71591 pi=71444-71590/6 crt=71397'38291940 lcod 0'0 mlcod 0'0 peering] calc_acting prefer osd.70 because it is current primary 2014-12-08 18:14:01.026084 7f28fcb46700 20 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] state: Adding osd: 4 features: 3fffffffffff 2014-12-08 18:14:01.026121 7f28fcb46700 10 log is not dirty 2014-12-08 18:14:01.026113 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[19.30( v 71397'38291943 (70960'38288942,71397'38291943] local-les=71445 n=911 ec=21298 les/c 71583/71583 71591/71591/71591) [70,66,1] r=0 lpr=71591 pi=71444-71590/6 crt=71397'38291940 lcod 0'0 mlcod 0'0 peering] calc_acting newest update on osd.70 with 19.30( v 71397'38291943 (70960'38288942,71397'38291943] local-les=71445 n=911 ec=21298 les/c 71583/71583 71591/71591/71591) up_primary: 70) selected as primary calc_acting primary is osd.70 with 19.30( v 71397'38291943 (70960'38288942,71397'38291943] local-les=71445 n=911 ec=21298 les/c 71583/71583 71591/71591/71591) osd.66 (up) accepted 19.30( v 71397'38291943 (70960'38288942,71397'38291943] local-les=71583 n=911 ec=21298 les/c 71583/71583 71591/71591/71591) osd.1 (up) accepted 19.30( v 71397'38291943 (70960'38288942,71397'38291943] local-les=71586 n=911 ec=21298 les/c 71583/71583 71591/71591/71591) 2014-12-08 18:14:01.026127 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[19.0( v 71377'41814799 (71180'41811798,71377'41814799] local-les=71436 n=828 ec=21298 les/c 71436/71436 71591/71591/71589) [63,70,14] r=1 lpr=71591 pi=57611-71590/80 crt=71377'41814797 lcod 0'0 inactive NOTIFY] handle_peering_event: epoch_sent: 71591 epoch_requested: 71591 MQuery from 63 query_epoch 71591 query: query(info 0'0) 2014-12-08 18:14:01.026140 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[19.0( v 71377'41814799 (71180'41811798,71377'41814799] local-les=71436 n=828 ec=21298 les/c 71590/71590 71591/71591/71589) [63,70,14] r=1 lpr=71591 pi=57611-71590/80 crt=71377'41814797 lcod 0'0 inactive NOTIFY] sending info 2014-12-08 18:14:01.026138 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[19.30( v 71397'38291943 (70960'38288942,71397'38291943] local-les=71445 n=911 ec=21298 les/c 71583/71583 71591/71591/71591) [70,66,1] r=0 lpr=71591 pi=71444-71590/6 crt=71397'38291940 lcod 0'0 mlcod 0'0 peering] actingbackfill is 1,66,70 2014-12-08 18:14:01.026152 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[19.30( v 71397'38291943 (70960'38288942,71397'38291943] local-les=71445 n=911 ec=21298 les/c 71583/71583 71591/71591/71591) [70,66,1] r=0 lpr=71591 pi=71444-71590/6 crt=71397'38291940 lcod 0'0 mlcod 0'0 peering] choose_acting want [70,66,1] (== acting) backfill_targets 2014-12-08 18:14:01.026169 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[19.30( v 71397'38291943 (70960'38288942,71397'38291943] local-les=71445 n=911 ec=21298 les/c 71583/71583 71591/71591/71591) [70,66,1] r=0 lpr=71591 pi=71444-71590/6 crt=71397'38291940 lcod 0'0 mlcod 0'0 peering] state: leaving GetLog 2014-12-08 18:14:01.026186 7f28fc345700 5 osd.70 pg_epoch: 71592 pg[19.30( v 71397'38291943 (70960'38288942,71397'38291943] local-les=71445 n=911 ec=21298 les/c 71583/71583 71591/71591/71591) [70,66,1] r=0 lpr=71591 pi=71444-71590/6 crt=71397'38291940 lcod 0'0 mlcod 0'0 peering] exit Started/Primary/Peering/GetLog 0.000174 0 0.000000 2014-12-08 18:14:01.026219 7f28fc345700 15 osd.70 pg_epoch: 71592 pg[19.30( v 71397'38291943 (70960'38288942,71397'38291943] local-les=71445 n=911 ec=21298 les/c 71583/71583 71591/71591/71591) [70,66,1] r=0 lpr=71591 pi=71444-71590/6 crt=71397'38291940 lcod 0'0 mlcod 0'0 peering] publish_stats_to_osd 71592:46072339 2014-12-08 18:14:01.026227 7f28fc345700 5 osd.70 pg_epoch: 71592 pg[19.30( v 71397'38291943 (70960'38288942,71397'38291943] local-les=71445 n=911 ec=21298 les/c 71583/71583 71591/71591/71591) [70,66,1] r=0 lpr=71591 pi=71444-71590/6 crt=71397'38291940 lcod 0'0 mlcod 0'0 peering] enter Started/Primary/Peering/GetMissing 2014-12-08 18:14:01.026234 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[19.30( v 71397'38291943 (70960'38288942,71397'38291943] local-les=71445 n=911 ec=21298 les/c 71583/71583 71591/71591/71591) [70,66,1] r=0 lpr=71591 pi=71444-71590/6 crt=71397'38291940 lcod 0'0 mlcod 0'0 peering] state: osd.1 has no missing, identical log 2014-12-08 18:14:01.026243 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[19.30( v 71397'38291943 (70960'38288942,71397'38291943] local-les=71445 n=911 ec=21298 les/c 71583/71583 71591/71591/71591) [70,66,1] r=0 lpr=71591 pi=71444-71590/6 crt=71397'38291940 lcod 0'0 mlcod 0'0 peering] state: osd.66 has no missing, identical log 2014-12-08 18:14:01.026252 7f28fc345700 5 osd.70 pg_epoch: 71592 pg[19.30( v 71397'38291943 (70960'38288942,71397'38291943] local-les=71445 n=911 ec=21298 les/c 71583/71583 71591/71591/71591) [70,66,1] r=0 lpr=71591 pi=71444-71590/6 crt=71397'38291940 lcod 0'0 mlcod 0'0 peering] exit Started/Primary/Peering/GetMissing 0.000025 0 0.000000 2014-12-08 18:14:01.026260 7f28fc345700 15 osd.70 pg_epoch: 71592 pg[19.30( v 71397'38291943 (70960'38288942,71397'38291943] local-les=71445 n=911 ec=21298 les/c 71583/71583 71591/71591/71591) [70,66,1] r=0 lpr=71591 pi=71444-71590/6 crt=71397'38291940 lcod 0'0 mlcod 0'0 peering] publish_stats_to_osd 71592:46072340 2014-12-08 18:14:01.026267 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[19.30( v 71397'38291943 (70960'38288942,71397'38291943] local-les=71445 n=911 ec=21298 les/c 71583/71583 71591/71591/71591) [70,66,1] r=0 lpr=71591 pi=71444-71590/6 crt=71397'38291940 lcod 0'0 mlcod 0'0 peering] state: Leaving Peering -- 2014-12-08 18:14:01.045753 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] handle_peering_event: epoch_sent: 71591 epoch_requested: 71591 MNotifyRec from 16 notify: (query_epoch:71591, epoch_sent:71591, info:19.6e( empty local-les=0 n=0 ec=21298 les/c 51101/50495 71591/71591/71591)) features: 0x3fffffffffff 2014-12-08 18:14:01.045763 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[218.39f( empty local-les=71446 n=0 ec=71423 les/c 71585/71585 71591/71591/71591) [70,13,16] r=0 lpr=71591 pi=71444-71590/4 crt=0'0 mlcod 0'0 peering] got osd.13 218.39f( empty local-les=71585 n=0 ec=71423 les/c 71585/71585 71591/71591/71591) 2014-12-08 18:14:01.045768 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] got osd.16 19.6e( empty local-les=0 n=0 ec=21298 les/c 51101/50495 71591/71591/71591) 2014-12-08 18:14:01.045790 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[218.39f( empty local-les=71446 n=0 ec=71423 les/c 71585/71585 71591/71591/71591) [70,13,16] r=0 lpr=71591 pi=71444-71590/4 crt=0'0 mlcod 0'0 peering] update_heartbeat_peers 13,16,22,70 unchanged 2014-12-08 18:14:01.045795 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] update_heartbeat_peers 4,13,16,30,58,70,76 unchanged 2014-12-08 18:14:01.045805 7f28fc345700 20 osd.70 pg_epoch: 71592 pg[218.39f( empty local-les=71446 n=0 ec=71423 les/c 71585/71585 71591/71591/71591) [70,13,16] r=0 lpr=71591 pi=71444-71590/4 crt=0'0 mlcod 0'0 peering] state: Adding osd: 13 features: 3fffffffffff 2014-12-08 18:14:01.045809 7f28fcb46700 20 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] state: Adding osd: 16 features: 3fffffffffff 2014-12-08 18:14:01.045821 7f28fcb46700 10 log is not dirty 2014-12-08 18:14:01.045822 7f28fc345700 10 log is not dirty 2014-12-08 18:14:01.045826 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[216.29a( v 71093'209 (0'0,71093'209] local-les=71453 n=104 ec=70987 les/c 71453/71453 71591/71591/71444) [19,70,14]/[19,14,58] r=-1 lpr=71591 pi=70987-71590/31 crt=71093'207 lcod 0'0 remapped NOTIFY] handle_peering_event: epoch_sent: 71591 epoch_requested: 71591 MQuery from 19 query_epoch 71591 query: query(info 0'0) 2014-12-08 18:14:01.045840 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[216.29a( v 71093'209 (0'0,71093'209] local-les=71453 n=104 ec=70987 les/c 71585/71583 71591/71591/71444) [19,70,14]/[19,14,58] r=-1 lpr=71591 pi=70987-71590/31 crt=71093'207 lcod 0'0 remapped NOTIFY] sending info 2014-12-08 18:14:01.045832 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[217.fc( v 71377'4103 (71377'1102,71377'4103] local-les=71434 n=12 ec=71094 les/c 71434/71434 71591/71591/71168) [18,4,70] r=2 lpr=71591 pi=71094-71590/33 crt=71377'4100 lcod 0'0 inactive NOTIFY] handle_peering_event: epoch_sent: 71591 epoch_requested: 71591 MQuery from 18 query_epoch 71591 query: query(info 0'0) 2014-12-08 18:14:01.045852 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[217.fc( v 71377'4103 (71377'1102,71377'4103] local-les=71434 n=12 ec=71094 les/c 71588/71588 71591/71591/71168) [18,4,70] r=2 lpr=71591 pi=71094-71590/33 crt=71377'4100 lcod 0'0 inactive NOTIFY] sending info 2014-12-08 18:14:01.045898 7f28fcb46700 10 log is not dirty 2014-12-08 18:14:01.045905 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[217.18( v 71377'4106 (71377'1105,71377'4106] local-les=71445 n=8 ec=71094 les/c 71586/71583 71591/71591/71591) [70,56,15] r=0 lpr=71591 pi=71444-71590/6 crt=71377'4103 lcod 0'0 mlcod 0'0 peering] handle_peering_event: epoch_sent: 71591 epoch_requested: 71591 MNotifyRec from 56 notify: (query_epoch:71591, epoch_sent:71591, info:217.18( v 71377'4106 (71377'1105,71377'4106] local-les=71586 n=8 ec=71094 les/c 71586/71583 71591/71591/71591)) features: 0x3fffffffffff 2014-12-08 18:14:01.045922 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[217.18( v 71377'4106 (71377'1105,71377'4106] local-les=71445 n=8 ec=71094 les/c 71586/71583 71591/71591/71591) [70,56,15] r=0 lpr=71591 pi=71444-71590/6 crt=71377'4103 lcod 0'0 mlcod 0'0 peering] got osd.56 217.18( v 71377'4106 (71377'1105,71377'4106] local-les=71586 n=8 ec=71094 les/c 71586/71583 71591/71591/71591) 2014-12-08 18:14:01.045944 7f28fc345700 10 log is not dirty 2014-12-08 18:14:01.045945 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[217.18( v 71377'4106 (71377'1105,71377'4106] local-les=71445 n=8 ec=71094 les/c 71586/71583 71591/71591/71591) [70,56,15] r=0 lpr=71591 pi=71444-71590/6 crt=71377'4103 lcod 0'0 mlcod 0'0 peering] update_heartbeat_peers 4,9,15,56,70 unchanged 2014-12-08 18:14:01.045957 7f28fcb46700 20 osd.70 pg_epoch: 71592 pg[217.18( v 71377'4106 (71377'1105,71377'4106] local-les=71445 n=8 ec=71094 les/c 71586/71583 71591/71591/71591) [70,56,15] r=0 lpr=71591 pi=71444-71590/6 crt=71377'4103 lcod 0'0 mlcod 0'0 peering] state: Adding osd: 56 features: 3fffffffffff 2014-12-08 18:14:01.045953 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[218.760( empty local-les=71433 n=0 ec=71423 les/c 71433/71433 71591/71591/71425) [18,70,5]/[18,5,25] r=-1 lpr=71591 pi=71423-71590/9 crt=0'0 remapped NOTIFY] handle_peering_event: epoch_sent: 71591 epoch_requested: 71591 MQuery from 18 query_epoch 71591 query: query(info 0'0) 2014-12-08 18:14:01.045966 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[217.18( v 71377'4106 (71377'1105,71377'4106] local-les=71445 n=8 ec=71094 les/c 71586/71583 71591/71591/71591) [70,56,15] r=0 lpr=71591 pi=71444-71590/6 crt=71377'4103 lcod 0'0 mlcod 0'0 peering] state: last maybe_went_rw interval was interval(71587-71589 up [56,15](56) acting [15,56](15) maybe_went_rw) 2014-12-08 18:14:01.045975 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[218.760( empty local-les=71433 n=0 ec=71423 les/c 71590/71590 71591/71591/71425) [18,70,5]/[18,5,25] r=-1 lpr=71591 pi=71423-71590/9 crt=0'0 remapped NOTIFY] sending info 2014-12-08 18:14:01.047030 7f28fcb46700 20 osd.70 pg_epoch: 71592 pg[217.18( v 71377'4106 (71377'1105,71377'4106] local-les=71445 n=8 ec=71094 les/c 71586/71583 71591/71591/71591) [70,56,15] r=0 lpr=71591 pi=71444-71590/6 crt=71377'4103 lcod 0'0 mlcod 0'0 peering] state: Common features: 3fffffffffff 2014-12-08 18:14:01.047047 7f28fcb46700 5 osd.70 pg_epoch: 71592 pg[217.18( v 71377'4106 (71377'1105,71377'4106] local-les=71445 n=8 ec=71094 les/c 71586/71583 71591/71591/71591) [70,56,15] r=0 lpr=71591 pi=71444-71590/6 crt=71377'4103 lcod 0'0 mlcod 0'0 peering] exit Started/Primary/Peering/GetInfo 0.460160 7 0.000645 2014-12-08 18:14:01.047052 7f2904b56700 10 osd.70 71592 do_waiters -- start 2014-12-08 18:14:01.047057 7f2904b56700 10 osd.70 71592 do_waiters -- finish 2014-12-08 18:14:01.047058 7f2904b56700 20 osd.70 71592 _dispatch 0x12cd0960 pg_notify(218.279(4) epoch 71592) v5 -- 2014-12-08 18:14:01.103973 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] handle_peering_event: epoch_sent: 71591 epoch_requested: 71591 MNotifyRec from 13 notify: (query_epoch:71591, epoch_sent:71591, info:19.6e( DNE empty local-les=0 n=0 ec=0 les/c 0/0 0/0/0)) features: 0x3fffffffffff 2014-12-08 18:14:01.103970 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[217.bb( v 71377'16307 (71377'13306,71377'16307] local-les=71592 n=22 ec=71094 les/c 71586/71583 71591/71591/71591) [70,78,25] r=0 lpr=71591 pi=71434-71590/6 crt=71377'16305 lcod 0'0 mlcod 0'0 inactive] handle_peering_event: epoch_sent: 71591 epoch_requested: 71591 MNotifyRec from 78 notify: (query_epoch:71591, epoch_sent:71591, info:217.bb( v 71377'16307 (71377'13306,71377'16307] local-les=71590 n=22 ec=71094 les/c 71586/71583 71591/71591/71591)) features: 0x3fffffffffff 2014-12-08 18:14:01.103995 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] got osd.13 19.6e( DNE empty local-les=0 n=0 ec=0 les/c 0/0 0/0/0) 2014-12-08 18:14:01.103996 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[217.bb( v 71377'16307 (71377'13306,71377'16307] local-les=71592 n=22 ec=71094 les/c 71586/71583 71591/71591/71591) [70,78,25] r=0 lpr=71591 pi=71434-71590/6 crt=71377'16305 lcod 0'0 mlcod 0'0 inactive] state: Active: got notify from 78, already have info from that osd, ignoring 2014-12-08 18:14:01.104012 7f28fc345700 10 log is not dirty 2014-12-08 18:14:01.104014 7f2904b56700 10 osd.70 71592 do_waiters -- start 2014-12-08 18:14:01.104016 7f2904b56700 10 osd.70 71592 do_waiters -- finish 2014-12-08 18:14:01.104015 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] update_heartbeat_peers 4,13,16,30,58,70,76 unchanged 2014-12-08 18:14:01.104030 7f2904b56700 10 osd.70 71592 do_waiters -- start 2014-12-08 18:14:01.104024 7f28fcb46700 20 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] state: Adding osd: 13 features: 3fffffffffff 2014-12-08 18:14:01.104032 7f2904b56700 10 osd.70 71592 do_waiters -- finish 2014-12-08 18:14:01.104033 7f28fcb46700 10 log is not dirty 2014-12-08 18:14:01.104033 7f2904b56700 20 osd.70 71592 _dispatch 0x12cf7fe0 pg_info(1 pgs e71592:217.15a) v4 2014-12-08 18:14:01.104058 7f2904b56700 7 osd.70 71592 handle_pg_info pg_info(1 pgs e71592:217.15a) v4 from osd.15 2014-12-08 18:14:01.104061 7f2904b56700 15 osd.70 71592 require_same_or_newer_map 71592 (i am 71592) 0x12cf7fe0 2014-12-08 18:14:01.104090 7f2904b56700 10 osd.70 71592 do_waiters -- start 2014-12-08 18:14:01.104093 7f2904b56700 10 osd.70 71592 do_waiters -- finish 2014-12-08 18:14:01.104328 7f2904b56700 10 osd.70 71592 do_waiters -- start 2014-12-08 18:14:01.104330 7f2904b56700 10 osd.70 71592 do_waiters -- finish 2014-12-08 18:14:01.104331 7f2904b56700 20 osd.70 71592 _dispatch 0x12cf74a0 pg_info(1 pgs e71592:218.40e) v4 2014-12-08 18:14:01.104391 7f2904b56700 7 osd.70 71592 handle_pg_info pg_info(1 pgs e71592:218.40e) v4 from osd.64 2014-12-08 18:14:01.104394 7f2904b56700 15 osd.70 71592 require_same_or_newer_map 71592 (i am 71592) 0x12cf74a0 2014-12-08 18:14:01.104424 7f2904b56700 10 osd.70 71592 do_waiters -- start 2014-12-08 18:14:01.104426 7f2904b56700 10 osd.70 71592 do_waiters -- finish 2014-12-08 18:14:01.104803 7f2904b56700 10 osd.70 71592 do_waiters -- start 2014-12-08 18:14:01.104806 7f2904b56700 10 osd.70 71592 do_waiters -- finish 2014-12-08 18:14:01.104807 7f2904b56700 20 osd.70 71592 _dispatch 0x12cf7860 pg_info(1 pgs e71592:216.2a1) v4 2014-12-08 18:14:01.104828 7f2904b56700 7 osd.70 71592 handle_pg_info pg_info(1 pgs e71592:216.2a1) v4 from osd.16 2014-12-08 18:14:01.104830 7f2904b56700 15 osd.70 71592 require_same_or_newer_map 71592 (i am 71592) 0x12cf7860 2014-12-08 18:14:01.104857 7f2904b56700 10 osd.70 71592 do_waiters -- start -- 2014-12-08 18:14:01.105105 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] handle_peering_event: epoch_sent: 71591 epoch_requested: 71591 MNotifyRec from 16 notify: (query_epoch:71591, epoch_sent:71591, info:19.6e( empty local-les=0 n=0 ec=21298 les/c 51101/50495 71591/71591/71591)) features: 0x3fffffffffff 2014-12-08 18:14:01.105118 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[217.3a5( v 71377'8160 (71377'5159,71377'8160] local-les=71439 n=11 ec=71094 les/c 71583/71583 71591/71591/71585) [69,13,70]/[13,69] r=-1 lpr=71591 pi=71183-71590/22 crt=71377'8158 lcod 0'0 remapped NOTIFY] handle_peering_event: epoch_sent: 71591 epoch_requested: 71591 FlushedEvt 2014-12-08 18:14:01.105124 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] got dup osd.16 info 19.6e( empty local-les=0 n=0 ec=21298 les/c 51101/50495 71591/71591/71591), identical to ours 2014-12-08 18:14:01.105139 7f28fcb46700 10 log is not dirty 2014-12-08 18:14:01.105132 7f28fc345700 15 osd.70 pg_epoch: 71592 pg[217.3a5( v 71377'8160 (71377'5159,71377'8160] local-les=71439 n=11 ec=71094 les/c 71583/71583 71591/71591/71585) [69,13,70]/[13,69] r=-1 lpr=71591 pi=71183-71590/22 crt=71377'8158 lcod 0'0 remapped NOTIFY] requeue_ops 2014-12-08 18:14:01.105143 7f28fc345700 10 log is not dirty 2014-12-08 18:14:01.105144 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] handle_peering_event: epoch_sent: 71591 epoch_requested: 71591 MNotifyRec from 58 notify: (query_epoch:71591, epoch_sent:71591, info:19.6e( empty local-les=0 n=0 ec=21298 les/c 51101/50495 71591/71591/71591)) features: 0x3fffffffffff 2014-12-08 18:14:01.105147 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[217.3bd( v 71377'8155 (71377'5154,71377'8155] local-les=71436 n=11 ec=71094 les/c 71588/71589 71591/71591/71096) [3,68,70] r=2 lpr=71591 pi=71183-71590/21 crt=71377'8151 lcod 0'0 inactive NOTIFY] handle_peering_event: epoch_sent: 71591 epoch_requested: 71591 FlushedEvt 2014-12-08 18:14:01.105161 7f28fc345700 15 osd.70 pg_epoch: 71592 pg[217.3bd( v 71377'8155 (71377'5154,71377'8155] local-les=71436 n=11 ec=71094 les/c 71588/71589 71591/71591/71096) [3,68,70] r=2 lpr=71591 pi=71183-71590/21 crt=71377'8151 lcod 0'0 inactive NOTIFY] requeue_ops 2014-12-08 18:14:01.105156 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] got dup osd.58 info 19.6e( empty local-les=0 n=0 ec=21298 les/c 51101/50495 71591/71591/71591), identical to ours 2014-12-08 18:14:01.105174 7f28fc345700 10 log is not dirty 2014-12-08 18:14:01.105177 7f28fcb46700 10 log is not dirty 2014-12-08 18:14:01.105178 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[218.4d( v 71581'1 (0'0,71581'1] local-les=71457 n=1 ec=71423 les/c 71457/71457 71591/71592/71425) [6,19,70]/[6,19,9] r=-1 lpr=71592 pi=71423-71591/10 crt=0'0 lcod 0'0 remapped NOTIFY] handle_peering_event: epoch_sent: 71591 epoch_requested: 71591 FlushedEvt 2014-12-08 18:14:01.105190 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[218.4d( v 71581'1 (0'0,71581'1] local-les=71457 n=1 ec=71423 les/c 71457/71457 71591/71592/71425) [6,19,70]/[6,19,9] r=-1 lpr=71592 pi=71423-71591/10 crt=0'0 lcod 0'0 remapped NOTIFY] old_peering_msg reply_epoch 71591 query_epoch 71591 last_peering_reset 71592 2014-12-08 18:14:01.105201 7f28fc345700 10 log is not dirty 2014-12-08 18:14:01.105184 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[217.22d( v 71377'4107 (71377'1106,71377'4107] local-les=71437 n=8 ec=71094 les/c 71583/71583 71591/71591/71590) [63,70,1] r=1 lpr=71591 pi=71094-71590/32 crt=71377'4104 lcod 0'0 inactive NOTIFY] handle_peering_event: epoch_sent: 71591 epoch_requested: 71591 FlushedEvt 2014-12-08 18:14:01.105206 7f28fc345700 20 osd.70 71592 share_map_peer 0x128fe6e0 don't know epoch, doing nothing 2014-12-08 18:14:01.105209 7f28fc345700 7 osd.70 71592 do_notifies osd 75 on 1 PGs 2014-12-08 18:14:01.105209 7f28fcb46700 15 osd.70 pg_epoch: 71592 pg[217.22d( v 71377'4107 (71377'1106,71377'4107] local-les=71437 n=8 ec=71094 les/c 71583/71583 71591/71591/71590) [63,70,1] r=1 lpr=71591 pi=71094-71590/32 crt=71377'4104 lcod 0'0 inactive NOTIFY] requeue_ops 2014-12-08 18:14:01.105223 7f28fcb46700 10 log is not dirty 2014-12-08 18:14:01.105229 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] handle_peering_event: epoch_sent: 71591 epoch_requested: 71591 MNotifyRec from 11 notify: (query_epoch:71591, epoch_sent:71591, info:19.6e( DNE empty local-les=0 n=0 ec=0 les/c 0/0 0/0/0)) features: 0x3fffffffffff 2014-12-08 18:14:01.105242 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] got osd.11 19.6e( DNE empty local-les=0 n=0 ec=0 les/c 0/0 0/0/0) 2014-12-08 18:14:01.105248 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[218.697( empty local-les=71434 n=0 ec=71423 les/c 71434/71434 71591/71592/71425) [20,70,4]/[20,4,15] r=-1 lpr=71592 pi=71423-71591/9 crt=0'0 remapped NOTIFY] handle_peering_event: epoch_sent: 71591 epoch_requested: 71591 FlushedEvt 2014-12-08 18:14:01.105260 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[218.697( empty local-les=71434 n=0 ec=71423 les/c 71434/71434 71591/71592/71425) [20,70,4]/[20,4,15] r=-1 lpr=71592 pi=71423-71591/9 crt=0'0 remapped NOTIFY] old_peering_msg reply_epoch 71591 query_epoch 71591 last_peering_reset 71592 2014-12-08 18:14:01.105258 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] osd.11 has stray content: 19.6e( DNE empty local-les=0 n=0 ec=0 les/c 0/0 0/0/0) 2014-12-08 18:14:01.105271 7f28fc345700 10 log is not dirty 2014-12-08 18:14:01.105273 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] update_heartbeat_peers 4,13,16,30,58,70,76 -> 4,11,13,16,30,58,70,76 2014-12-08 18:14:01.105276 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[217.1c0( v 71377'4125 (71377'1124,71377'4125] local-les=71437 n=14 ec=71094 les/c 71588/71589 71591/71591/71096) [15,70,75] r=1 lpr=71591 pi=71094-71590/30 crt=71377'4123 lcod 0'0 inactive NOTIFY] handle_peering_event: epoch_sent: 71591 epoch_requested: 71591 FlushedEvt 2014-12-08 18:14:01.105288 7f28fcb46700 20 osd.70 71592 need_heartbeat_peer_update 2014-12-08 18:14:01.105291 7f28fc345700 15 osd.70 pg_epoch: 71592 pg[217.1c0( v 71377'4125 (71377'1124,71377'4125] local-les=71437 n=14 ec=71094 les/c 71588/71589 71591/71591/71096) [15,70,75] r=1 lpr=71591 pi=71094-71590/30 crt=71377'4123 lcod 0'0 inactive NOTIFY] requeue_ops 2014-12-08 18:14:01.105301 7f28fc345700 10 log is not dirty 2014-12-08 18:14:01.105291 7f28fcb46700 20 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] state: Adding osd: 11 features: 3fffffffffff 2014-12-08 18:14:01.105304 7f28fcb46700 10 log is not dirty 2014-12-08 18:14:01.105551 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[216.358( v 71422'212 (0'0,71422'212] local-les=71450 n=100 ec=70987 les/c 71450/71450 71591/71592/71444) [5,77,70] r=2 lpr=71592 pi=71183-71591/18 crt=71097'211 lcod 0'0 inactive NOTIFY] handle_peering_event: epoch_sent: 71591 epoch_requested: 71591 FlushedEvt 2014-12-08 18:14:01.105566 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[216.358( v 71422'212 (0'0,71422'212] local-les=71450 n=100 ec=70987 les/c 71450/71450 71591/71592/71444) [5,77,70] r=2 lpr=71592 pi=71183-71591/18 crt=71097'211 lcod 0'0 inactive NOTIFY] old_peering_msg reply_epoch 71591 query_epoch 71591 last_peering_reset 71592 2014-12-08 18:14:01.105574 7f28fc345700 10 log is not dirty 2014-12-08 18:14:01.105579 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[217.214( v 71377'21369 (71377'18368,71377'21369] local-les=71592 n=14 ec=71094 les/c 71589/71589 71591/71591/71591) [70,9,4] r=0 lpr=71591 pi=71438-71590/6 crt=71377'21367 lcod 0'0 mlcod 0'0 inactive] handle_peering_event: epoch_sent: 71591 epoch_requested: 71591 MNotifyRec from 4 notify: (query_epoch:71591, epoch_sent:71591, info:217.214( v 71377'21369 (71377'18368,71377'21369] local-les=71589 n=14 ec=71094 les/c 71589/71589 71591/71591/71591)) features: 0x3fffffffffff 2014-12-08 18:14:01.105592 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[217.214( v 71377'21369 (71377'18368,71377'21369] local-les=71592 n=14 ec=71094 les/c 71589/71589 71591/71591/71591) [70,9,4] r=0 lpr=71591 pi=71438-71590/6 crt=71377'21367 lcod 0'0 mlcod 0'0 inactive] state: Active: got notify from 4, already have info from that osd, ignoring 2014-12-08 18:14:01.105601 7f28fc345700 10 log is not dirty 2014-12-08 18:14:01.105605 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[216.42( v 71097'231 (0'0,71097'231] local-les=71437 n=109 ec=70987 les/c 71437/71437 71591/71592/70989) [9,10,70] r=2 lpr=71592 pi=70987-71591/28 crt=71097'229 lcod 0'0 inactive NOTIFY] handle_peering_event: epoch_sent: 71591 epoch_requested: 71591 FlushedEvt 2014-12-08 18:14:01.105614 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[216.42( v 71097'231 (0'0,71097'231] local-les=71437 n=109 ec=70987 les/c 71437/71437 71591/71592/70989) [9,10,70] r=2 lpr=71592 pi=70987-71591/28 crt=71097'229 lcod 0'0 inactive NOTIFY] old_peering_msg reply_epoch 71591 query_epoch 71591 last_peering_reset 71592 2014-12-08 18:14:01.105622 7f28fc345700 10 log is not dirty 2014-12-08 18:14:01.105638 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[218.1c0( v 71581'3 (0'0,71581'3] local-les=71434 n=3 ec=71423 les/c 71434/71434 71591/71591/71425) [75,70,3]/[75,3,6] r=-1 lpr=71591 pi=71423-71590/9 crt=71580'1 lcod 0'0 remapped NOTIFY] handle_advance_map [75,70,3]/[75,3,6] -- 75/75 2014-12-08 18:14:01.105652 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:14:01.105656 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[218.1c0( v 71581'3 (0'0,71581'3] local-les=71434 n=3 ec=71423 les/c 71434/71434 71591/71591/71425) [75,70,3]/[75,3,6] r=-1 lpr=71591 pi=71423-71590/9 crt=71580'1 lcod 0'0 remapped NOTIFY] state: Started advmap 2014-12-08 18:14:01.105666 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[218.1c0( v 71581'3 (0'0,71581'3] local-les=71434 n=3 ec=71423 les/c 71434/71434 71591/71591/71425) [75,70,3]/[75,3,6] r=-1 lpr=71591 pi=71423-71590/9 crt=71580'1 lcod 0'0 remapped NOTIFY] check_recovery_sources no source osds () went down 2014-12-08 18:14:01.105684 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[218.1c0( v 71581'3 (0'0,71581'3] local-les=71434 n=3 ec=71423 les/c 71434/71434 71591/71591/71425) [75,70,3]/[75,3,6] r=-1 lpr=71591 pi=71423-71590/9 crt=71580'1 lcod 0'0 remapped NOTIFY] handle_activate_map 2014-12-08 18:14:01.105711 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[218.1c0( v 71581'3 (0'0,71581'3] local-les=71434 n=3 ec=71423 les/c 71434/71434 71591/71591/71425) [75,70,3]/[75,3,6] r=-1 lpr=71591 pi=71423-71590/9 crt=71580'1 lcod 0'0 remapped NOTIFY] take_waiters 2014-12-08 18:14:01.105725 7f28fc345700 20 osd.70 pg_epoch: 71592 pg[218.1c0( v 71581'3 (0'0,71581'3] local-les=71434 n=3 ec=71423 les/c 71434/71434 71591/71591/71425) [75,70,3]/[75,3,6] r=-1 lpr=71591 pi=71423-71590/9 crt=71580'1 lcod 0'0 remapped NOTIFY] handle_activate_map: Not dirtying info: last_persisted is 71591 while current is 71592 2014-12-08 18:14:01.105736 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] handle_peering_event: epoch_sent: 71591 epoch_requested: 71591 MNotifyRec from 31 notify: (query_epoch:71591, epoch_sent:71591, info:19.6e( DNE empty local-les=0 n=0 ec=0 les/c 0/0 0/0/0)) features: 0x3fffffffffff 2014-12-08 18:14:01.105739 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[218.1c0( v 71581'3 (0'0,71581'3] local-les=71434 n=3 ec=71423 les/c 71434/71434 71591/71591/71425) [75,70,3]/[75,3,6] r=-1 lpr=71591 pi=71423-71590/9 crt=71580'1 lcod 0'0 remapped NOTIFY] handle_peering_event: epoch_sent: 71591 epoch_requested: 71591 FlushedEvt 2014-12-08 18:14:01.105759 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] got osd.31 19.6e( DNE empty local-les=0 n=0 ec=0 les/c 0/0 0/0/0) 2014-12-08 18:14:01.105771 7f28fc345700 15 osd.70 pg_epoch: 71592 pg[218.1c0( v 71581'3 (0'0,71581'3] local-les=71434 n=3 ec=71423 les/c 71434/71434 71591/71591/71425) [75,70,3]/[75,3,6] r=-1 lpr=71591 pi=71423-71590/9 crt=71580'1 lcod 0'0 remapped NOTIFY] requeue_ops 2014-12-08 18:14:01.105786 7f28fc345700 10 log is not dirty 2014-12-08 18:14:01.105782 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] osd.31 has stray content: 19.6e( DNE empty local-les=0 n=0 ec=0 les/c 0/0 0/0/0) 2014-12-08 18:14:01.105795 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[217.6a( v 71376'14 (0'0,71376'14] local-les=71447 n=7 ec=71094 les/c 71447/71448 71591/71592/71592) [70,24,19] r=0 lpr=71592 pi=71444-71591/6 crt=71097'12 lcod 0'0 mlcod 0'0 peering] handle_peering_event: epoch_sent: 71591 epoch_requested: 71591 FlushedEvt 2014-12-08 18:14:01.105795 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] update_heartbeat_peers 4,11,13,16,30,58,70,76 -> 4,11,13,16,30,31,58,70,76 2014-12-08 18:14:01.105816 7f28fcb46700 20 osd.70 71592 need_heartbeat_peer_update 2014-12-08 18:14:01.105811 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[217.6a( v 71376'14 (0'0,71376'14] local-les=71447 n=7 ec=71094 les/c 71447/71448 71591/71592/71592) [70,24,19] r=0 lpr=71592 pi=71444-71591/6 crt=71097'12 lcod 0'0 mlcod 0'0 peering] old_peering_msg reply_epoch 71591 query_epoch 71591 last_peering_reset 71592 2014-12-08 18:14:01.105821 7f28fc345700 10 log is not dirty 2014-12-08 18:14:01.105824 7f2904b56700 10 osd.70 71592 do_waiters -- start 2014-12-08 18:14:01.105828 7f2904b56700 10 osd.70 71592 do_waiters -- finish 2014-12-08 18:14:01.105820 7f28fcb46700 20 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] state: Adding osd: 31 features: 3fffffffffff 2014-12-08 18:14:01.105832 7f2904b56700 20 osd.70 71592 _dispatch 0x12cf9de0 pg_info(1 pgs e71592:218.40e) v4 2014-12-08 18:14:01.105836 7f28fcb46700 10 log is not dirty 2014-12-08 18:14:01.105825 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[217.42( v 71377'43 (0'0,71377'43] local-les=71438 n=8 ec=71094 les/c 71438/71438 71591/71592/71096) [13,70,77] r=1 lpr=71592 pi=71183-71591/17 crt=71377'40 lcod 0'0 inactive NOTIFY] handle_peering_event: epoch_sent: 71591 epoch_requested: 71591 FlushedEvt 2014-12-08 18:14:01.105844 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[217.42( v 71377'43 (0'0,71377'43] local-les=71438 n=8 ec=71094 les/c 71438/71438 71591/71592/71096) [13,70,77] r=1 lpr=71592 pi=71183-71591/17 crt=71377'40 lcod 0'0 inactive NOTIFY] old_peering_msg reply_epoch 71591 query_epoch 71591 last_peering_reset 71592 2014-12-08 18:14:01.105859 7f28fc345700 10 log is not dirty 2014-12-08 18:14:01.105845 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[216.384( v 70990'209 (0'0,70990'209] local-les=71435 n=101 ec=70987 les/c 71435/71435 71591/71592/71587) [77,22,70] r=2 lpr=71592 pi=70987-71591/27 crt=70990'207 lcod 0'0 inactive NOTIFY] handle_peering_event: epoch_sent: 71591 epoch_requested: 71591 FlushedEvt 2014-12-08 18:14:01.105873 7f2904b56700 7 osd.70 71592 handle_pg_info pg_info(1 pgs e71592:218.40e) v4 from osd.6 2014-12-08 18:14:01.105876 7f2904b56700 15 osd.70 71592 require_same_or_newer_map 71592 (i am 71592) 0x12cf9de0 2014-12-08 18:14:01.105865 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[218.531( v 71580'1 (0'0,71580'1] local-les=71433 n=1 ec=71423 les/c 71433/71433 71591/71592/71425) [18,14,70] r=2 lpr=71592 pi=71423-71591/10 crt=0'0 lcod 0'0 inactive NOTIFY] handle_peering_event: epoch_sent: 71591 epoch_requested: 71591 FlushedEvt 2014-12-08 18:14:01.105868 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[216.384( v 70990'209 (0'0,70990'209] local-les=71435 n=101 ec=70987 les/c 71435/71435 71591/71592/71587) [77,22,70] r=2 lpr=71592 pi=70987-71591/27 crt=70990'207 lcod 0'0 inactive NOTIFY] old_peering_msg reply_epoch 71591 query_epoch 71591 last_peering_reset 71592 2014-12-08 18:14:01.105889 7f28fcb46700 10 log is not dirty 2014-12-08 18:14:01.105883 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[218.531( v 71580'1 (0'0,71580'1] local-les=71433 n=1 ec=71423 les/c 71433/71433 71591/71592/71425) [18,14,70] r=2 lpr=71592 pi=71423-71591/10 crt=0'0 lcod 0'0 inactive NOTIFY] old_peering_msg reply_epoch 71591 query_epoch 71591 last_peering_reset 71592 2014-12-08 18:14:01.105897 7f28fc345700 10 log is not dirty 2014-12-08 18:14:01.105897 7f28fcb46700 10 osd.70 71592 queue_want_up_thru want 71592 <= queued 71592, currently 71591 2014-12-08 18:14:01.105906 7f28fcb46700 20 osd.70 71592 share_map_peer 0x12839760 don't know epoch, doing nothing 2014-12-08 18:14:01.105911 7f28fcb46700 7 osd.70 71592 do_notifies osd 17 on 1 PGs 2014-12-08 18:14:01.105906 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[217.6f( v 71377'4120 (71377'1119,71377'4120] local-les=71469 n=7 ec=71094 les/c 71469/71470 71591/71592/71592) [70,17,66] r=0 lpr=71592 pi=71463-71591/6 crt=71377'4116 lcod 0'0 mlcod 0'0 peering] handle_peering_event: epoch_sent: 71591 epoch_requested: 71591 FlushedEvt 2014-12-08 18:14:01.105924 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[217.6f( v 71377'4120 (71377'1119,71377'4120] local-les=71469 n=7 ec=71094 les/c 71469/71470 71591/71592/71592) [70,17,66] r=0 lpr=71592 pi=71463-71591/6 crt=71377'4116 lcod 0'0 mlcod 0'0 peering] old_peering_msg reply_epoch 71591 query_epoch 71591 last_peering_reset 71592 2014-12-08 18:14:01.105938 7f28fc345700 10 log is not dirty 2014-12-08 18:14:01.105944 7f2904b56700 10 osd.70 71592 do_waiters -- start -- 2014-12-08 18:14:01.106112 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] handle_peering_event: epoch_sent: 71591 epoch_requested: 71591 MNotifyRec from 30 notify: (query_epoch:71591, epoch_sent:71591, info:19.6e( empty local-les=0 n=0 ec=21298 les/c 51101/50495 71591/71591/71591)) features: 0x3fffffffffff 2014-12-08 18:14:01.106137 7f28fc345700 10 log is not dirty 2014-12-08 18:14:01.106139 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] got dup osd.30 info 19.6e( empty local-les=0 n=0 ec=21298 les/c 51101/50495 71591/71591/71591), identical to ours 2014-12-08 18:14:01.106151 7f28fcb46700 10 log is not dirty 2014-12-08 18:14:01.106142 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[218.3a1( v 71581'1 (0'0,71581'1] local-les=71451 n=1 ec=71423 les/c 71585/71583 71591/71591/71591) [70,23,69] r=0 lpr=71591 pi=71444-71590/4 crt=0'0 lcod 0'0 mlcod 0'0 peering] handle_peering_event: epoch_sent: 71591 epoch_requested: 71591 MNotifyRec from 23 notify: (query_epoch:71591, epoch_sent:71591, info:218.3a1( v 71581'1 (0'0,71581'1] local-les=71585 n=1 ec=71423 les/c 71585/71583 71591/71591/71591)) features: 0x3fffffffffff 2014-12-08 18:14:01.106156 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[218.3a1( v 71581'1 (0'0,71581'1] local-les=71451 n=1 ec=71423 les/c 71585/71583 71591/71591/71591) [70,23,69] r=0 lpr=71591 pi=71444-71590/4 crt=0'0 lcod 0'0 mlcod 0'0 peering] got dup osd.23 info 218.3a1( v 71581'1 (0'0,71581'1] local-les=71585 n=1 ec=71423 les/c 71585/71583 71591/71591/71591), identical to ours 2014-12-08 18:14:01.106168 7f28fc345700 10 log is not dirty 2014-12-08 18:14:01.106157 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[218.51( v 71581'3 (0'0,71581'3] local-les=71433 n=3 ec=71423 les/c 71433/71434 71591/71592/71592) [70,66,75]/[66,75,6] r=-1 lpr=71592 pi=71432-71591/5 crt=71580'1 lcod 0'0 remapped NOTIFY] handle_peering_event: epoch_sent: 71591 epoch_requested: 71591 MNotifyRec from 24 notify: (query_epoch:71591, epoch_sent:71591, info:218.51( v 71581'3 lc 0'0 (0'0,71581'3] local-les=71585 n=3 ec=71423 les/c 71585/71583 71591/71591/71591)) features: 0x3fffffffffff 2014-12-08 18:14:01.106173 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[218.3a4( empty local-les=71435 n=0 ec=71423 les/c 71585/71585 71591/71591/71591) [70,6,66] r=0 lpr=71591 pi=71432-71590/4 crt=0'0 mlcod 0'0 peering] handle_peering_event: epoch_sent: 71591 epoch_requested: 71591 MNotifyRec from 6 notify: (query_epoch:71591, epoch_sent:71591, info:218.3a4( empty local-les=71585 n=0 ec=71423 les/c 71585/71585 71591/71591/71591)) features: 0x3fffffffffff 2014-12-08 18:14:01.106173 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[218.51( v 71581'3 (0'0,71581'3] local-les=71433 n=3 ec=71423 les/c 71433/71434 71591/71592/71592) [70,66,75]/[66,75,6] r=-1 lpr=71592 pi=71432-71591/5 crt=71580'1 lcod 0'0 remapped NOTIFY] old_peering_msg reply_epoch 71591 query_epoch 71591 last_peering_reset 71592 2014-12-08 18:14:01.106185 7f28fcb46700 10 log is not dirty 2014-12-08 18:14:01.106184 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[218.3a4( empty local-les=71435 n=0 ec=71423 les/c 71585/71585 71591/71591/71591) [70,6,66] r=0 lpr=71591 pi=71432-71590/4 crt=0'0 mlcod 0'0 peering] got dup osd.6 info 218.3a4( empty local-les=71585 n=0 ec=71423 les/c 71585/71585 71591/71591/71591), identical to ours 2014-12-08 18:14:01.106200 7f28fc345700 10 log is not dirty 2014-12-08 18:14:01.106192 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] handle_peering_event: epoch_sent: 71591 epoch_requested: 71591 MNotifyRec from 63 notify: (query_epoch:71591, epoch_sent:71591, info:19.6e( DNE empty local-les=0 n=0 ec=0 les/c 0/0 0/0/0)) features: 0x3fffffffffff 2014-12-08 18:14:01.106204 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] got osd.63 19.6e( DNE empty local-les=0 n=0 ec=0 les/c 0/0 0/0/0) 2014-12-08 18:14:01.106206 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[218.3a4( empty local-les=71435 n=0 ec=71423 les/c 71585/71585 71591/71591/71591) [70,6,66] r=0 lpr=71591 pi=71432-71590/4 crt=0'0 mlcod 0'0 peering] handle_peering_event: epoch_sent: 71591 epoch_requested: 71591 MNotifyRec from 64 notify: (query_epoch:71591, epoch_sent:71591, info:218.3a4( empty local-les=71585 n=0 ec=71423 les/c 71585/71585 71591/71591/71591)) features: 0x3fffffffffff 2014-12-08 18:14:01.106218 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] osd.63 has stray content: 19.6e( DNE empty local-les=0 n=0 ec=0 les/c 0/0 0/0/0) 2014-12-08 18:14:01.106223 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[218.3a4( empty local-les=71435 n=0 ec=71423 les/c 71585/71585 71591/71591/71591) [70,6,66] r=0 lpr=71591 pi=71432-71590/4 crt=0'0 mlcod 0'0 peering] got dup osd.64 info 218.3a4( empty local-les=71585 n=0 ec=71423 les/c 71585/71585 71591/71591/71591), identical to ours 2014-12-08 18:14:01.106241 7f28fc345700 10 log is not dirty 2014-12-08 18:14:01.106231 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] update_heartbeat_peers 4,11,13,16,30,31,58,70,76 -> 4,11,13,16,30,31,58,63,70,76 2014-12-08 18:14:01.106244 7f28fcb46700 20 osd.70 71592 need_heartbeat_peer_update 2014-12-08 18:14:01.106246 7f28fcb46700 20 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] state: Adding osd: 63 features: 3fffffffffff 2014-12-08 18:14:01.106257 7f28fcb46700 10 log is not dirty 2014-12-08 18:14:01.106247 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[218.3a4( empty local-les=71435 n=0 ec=71423 les/c 71585/71585 71591/71591/71591) [70,6,66] r=0 lpr=71591 pi=71432-71590/4 crt=0'0 mlcod 0'0 peering] handle_peering_event: epoch_sent: 71591 epoch_requested: 71591 MNotifyRec from 66 notify: (query_epoch:71591, epoch_sent:71591, info:218.3a4( DNE empty local-les=0 n=0 ec=0 les/c 0/0 0/0/0)) features: 0x3fffffffffff 2014-12-08 18:14:01.106261 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] handle_peering_event: epoch_sent: 71591 epoch_requested: 71591 MNotifyRec from 56 notify: (query_epoch:71591, epoch_sent:71591, info:19.6e( DNE empty local-les=0 n=0 ec=0 les/c 0/0 0/0/0)) features: 0x3fffffffffff 2014-12-08 18:14:01.106264 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[218.3a4( empty local-les=71435 n=0 ec=71423 les/c 71585/71585 71591/71591/71591) [70,6,66] r=0 lpr=71591 pi=71432-71590/4 crt=0'0 mlcod 0'0 peering] got osd.66 218.3a4( DNE empty local-les=0 n=0 ec=0 les/c 0/0 0/0/0) 2014-12-08 18:14:01.106274 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] got osd.56 19.6e( DNE empty local-les=0 n=0 ec=0 les/c 0/0 0/0/0) 2014-12-08 18:14:01.106297 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] osd.56 has stray content: 19.6e( DNE empty local-les=0 n=0 ec=0 les/c 0/0 0/0/0) 2014-12-08 18:14:01.106298 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[218.3a4( empty local-les=71435 n=0 ec=71423 les/c 71585/71585 71591/71591/71591) [70,6,66] r=0 lpr=71591 pi=71432-71590/4 crt=0'0 mlcod 0'0 peering] update_heartbeat_peers 3,6,64,66,70 unchanged 2014-12-08 18:14:01.106313 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] update_heartbeat_peers 4,11,13,16,30,31,58,63,70,76 -> 4,11,13,16,30,31,56,58,63,70,76 2014-12-08 18:14:01.106327 7f28fcb46700 20 osd.70 71592 need_heartbeat_peer_update 2014-12-08 18:14:01.106317 7f28fc345700 20 osd.70 pg_epoch: 71592 pg[218.3a4( empty local-les=71435 n=0 ec=71423 les/c 71585/71585 71591/71591/71591) [70,6,66] r=0 lpr=71591 pi=71432-71590/4 crt=0'0 mlcod 0'0 peering] state: Adding osd: 66 features: 3fffffffffff 2014-12-08 18:14:01.106329 7f28fcb46700 20 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] state: Adding osd: 56 features: 3fffffffffff 2014-12-08 18:14:01.106341 7f28fcb46700 10 log is not dirty 2014-12-08 18:14:01.106332 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[218.3a4( empty local-les=71435 n=0 ec=71423 les/c 71585/71585 71591/71591/71591) [70,6,66] r=0 lpr=71591 pi=71432-71590/4 crt=0'0 mlcod 0'0 peering] state: last maybe_went_rw interval was interval(71587-71590 up [6,66](6) acting [6,66](6) maybe_went_rw) 2014-12-08 18:14:01.106346 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[218.a1( v 71581'1 (0'0,71581'1] local-les=71437 n=1 ec=71423 les/c 71585/71585 71591/71591/71591) [70,66,18] r=0 lpr=71591 pi=71432-71590/4 crt=0'0 lcod 0'0 mlcod 0'0 peering] handle_peering_event: epoch_sent: 71591 epoch_requested: 71591 MNotifyRec from 77 notify: (query_epoch:71591, epoch_sent:71591, info:218.a1( v 71581'1 (0'0,71581'1] local-les=71585 n=1 ec=71423 les/c 71585/71585 71591/71591/71591)) features: 0x3fffffffffff 2014-12-08 18:14:01.106352 7f28fc345700 20 osd.70 pg_epoch: 71592 pg[218.3a4( empty local-les=71435 n=0 ec=71423 les/c 71585/71585 71591/71591/71591) [70,6,66] r=0 lpr=71591 pi=71432-71590/4 crt=0'0 mlcod 0'0 peering] state: Common features: 3fffffffffff 2014-12-08 18:14:01.106362 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[218.a1( v 71581'1 (0'0,71581'1] local-les=71437 n=1 ec=71423 les/c 71585/71585 71591/71591/71591) [70,66,18] r=0 lpr=71591 pi=71432-71590/4 crt=0'0 lcod 0'0 mlcod 0'0 peering] got dup osd.77 info 218.a1( v 71581'1 (0'0,71581'1] local-les=71585 n=1 ec=71423 les/c 71585/71585 71591/71591/71591), identical to ours 2014-12-08 18:14:01.106373 7f28fcb46700 10 log is not dirty 2014-12-08 18:14:01.106368 7f28fc345700 5 osd.70 pg_epoch: 71592 pg[218.3a4( empty local-les=71435 n=0 ec=71423 les/c 71585/71585 71591/71591/71591) [70,6,66] r=0 lpr=71591 pi=71432-71590/4 crt=0'0 mlcod 0'0 peering] exit Started/Primary/Peering/GetInfo 0.592821 10 0.000595 2014-12-08 18:14:01.106378 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] handle_peering_event: epoch_sent: 71591 epoch_requested: 71591 MNotifyRec from 20 notify: (query_epoch:71591, epoch_sent:71591, info:19.6e( DNE empty local-les=0 n=0 ec=0 les/c 0/0 0/0/0)) features: 0x3fffffffffff 2014-12-08 18:14:01.106383 7f28fc345700 5 osd.70 pg_epoch: 71592 pg[218.3a4( empty local-les=71435 n=0 ec=71423 les/c 71585/71585 71591/71591/71591) [70,6,66] r=0 lpr=71591 pi=71432-71590/4 crt=0'0 mlcod 0'0 peering] enter Started/Primary/Peering/GetLog 2014-12-08 18:14:01.106392 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] got osd.20 19.6e( DNE empty local-les=0 n=0 ec=0 les/c 0/0 0/0/0) 2014-12-08 18:14:01.106404 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] osd.20 has stray content: 19.6e( DNE empty local-les=0 n=0 ec=0 les/c 0/0 0/0/0) 2014-12-08 18:14:01.106403 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[218.3a4( empty local-les=71435 n=0 ec=71423 les/c 71585/71585 71591/71591/71591) [70,6,66] r=0 lpr=71591 pi=71432-71590/4 crt=0'0 mlcod 0'0 peering] calc_acting osd.3 218.3a4( empty local-les=71585 n=0 ec=71423 les/c 71585/71585 71591/71591/71591) 2014-12-08 18:14:01.106418 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] update_heartbeat_peers 4,11,13,16,30,31,56,58,63,70,76 -> 4,11,13,16,20,30,31,56,58,63,70,76 2014-12-08 18:14:01.106419 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[218.3a4( empty local-les=71435 n=0 ec=71423 les/c 71585/71585 71591/71591/71591) [70,6,66] r=0 lpr=71591 pi=71432-71590/4 crt=0'0 mlcod 0'0 peering] calc_acting osd.6 218.3a4( empty local-les=71585 n=0 ec=71423 les/c 71585/71585 71591/71591/71591) 2014-12-08 18:14:01.106432 7f28fcb46700 20 osd.70 71592 need_heartbeat_peer_update 2014-12-08 18:14:01.106435 7f28fcb46700 20 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] state: Adding osd: 20 features: 3fffffffffff 2014-12-08 18:14:01.106434 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[218.3a4( empty local-les=71435 n=0 ec=71423 les/c 71585/71585 71591/71591/71591) [70,6,66] r=0 lpr=71591 pi=71432-71590/4 crt=0'0 mlcod 0'0 peering] calc_acting osd.64 218.3a4( empty local-les=71585 n=0 ec=71423 les/c 71585/71585 71591/71591/71591) 2014-12-08 18:14:01.106448 7f28fcb46700 10 log is not dirty 2014-12-08 18:14:01.106449 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[218.3a4( empty local-les=71435 n=0 ec=71423 les/c 71585/71585 71591/71591/71591) [70,6,66] r=0 lpr=71591 pi=71432-71590/4 crt=0'0 mlcod 0'0 peering] calc_acting osd.66 218.3a4( DNE empty local-les=0 n=0 ec=0 les/c 0/0 0/0/0) 2014-12-08 18:14:01.106452 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[218.a3( v 71581'1 (0'0,71581'1] local-les=71434 n=1 ec=71423 les/c 71434/71436 71591/71592/71592) [70,20,23]/[20,23,75] r=-1 lpr=71592 pi=71432-71591/5 crt=0'0 lcod 0'0 remapped NOTIFY] handle_peering_event: epoch_sent: 71591 epoch_requested: 71591 MNotifyRec from 20 notify: (query_epoch:71591, epoch_sent:71591, info:218.a3( v 71581'1 (0'0,71581'1] local-les=71585 n=1 ec=71423 les/c 71585/71583 71591/71591/71591)) features: 0x3fffffffffff 2014-12-08 18:14:01.106467 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[218.a3( v 71581'1 (0'0,71581'1] local-les=71434 n=1 ec=71423 les/c 71434/71436 71591/71592/71592) [70,20,23]/[20,23,75] r=-1 lpr=71592 pi=71432-71591/5 crt=0'0 lcod 0'0 remapped NOTIFY] old_peering_msg reply_epoch 71591 query_epoch 71591 last_peering_reset 71592 2014-12-08 18:14:01.106464 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[218.3a4( empty local-les=71435 n=0 ec=71423 les/c 71585/71585 71591/71591/71591) [70,6,66] r=0 lpr=71591 pi=71432-71590/4 crt=0'0 mlcod 0'0 peering] calc_acting osd.70 218.3a4( empty local-les=71435 n=0 ec=71423 les/c 71585/71585 71591/71591/71591) 2014-12-08 18:14:01.106487 7f28fcb46700 10 log is not dirty 2014-12-08 18:14:01.106479 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[218.3a4( empty local-les=71435 n=0 ec=71423 les/c 71585/71585 71591/71591/71591) [70,6,66] r=0 lpr=71591 pi=71432-71590/4 crt=0'0 mlcod 0'0 peering] calc_acting prefer osd.70 because it is current primary 2014-12-08 18:14:01.106493 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] handle_peering_event: epoch_sent: 71591 epoch_requested: 71591 MNotifyRec from 4 notify: (query_epoch:71591, epoch_sent:71591, info:19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591)) features: 0x3fffffffffff 2014-12-08 18:14:01.106505 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[218.3a4( empty local-les=71435 n=0 ec=71423 les/c 71585/71585 71591/71591/71591) [70,6,66] r=0 lpr=71591 pi=71432-71590/4 crt=0'0 mlcod 0'0 peering] calc_acting newest update on osd.70 with 218.3a4( empty local-les=71435 n=0 ec=71423 les/c 71585/71585 71591/71591/71591) up_primary: 70) selected as primary calc_acting primary is osd.70 with 218.3a4( empty local-les=71435 n=0 ec=71423 les/c 71585/71585 71591/71591/71591) osd.6 (up) accepted 218.3a4( empty local-les=71585 n=0 ec=71423 les/c 71585/71585 71591/71591/71591) osd.66 (up) accepted 218.3a4( DNE empty local-les=0 n=0 ec=0 les/c 0/0 0/0/0) 2014-12-08 18:14:01.106507 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] got dup osd.4 info 19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591), identical to ours 2014-12-08 18:14:01.106529 7f28fcb46700 10 log is not dirty 2014-12-08 18:14:01.106529 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[218.3a4( empty local-les=71435 n=0 ec=71423 les/c 71585/71585 71591/71591/71591) [70,6,66] r=0 lpr=71591 pi=71432-71590/4 crt=0'0 mlcod 0'0 peering] actingbackfill is 6,66,70 2014-12-08 18:14:01.106535 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[218.535( empty local-les=71452 n=0 ec=71423 les/c 71585/71585 71591/71591/71444) [3,70,19]/[3,19,18] r=-1 lpr=71591 pi=71423-71590/11 crt=0'0 remapped NOTIFY] handle_peering_event: epoch_sent: 71591 epoch_requested: 71591 FlushedEvt 2014-12-08 18:14:01.106542 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[218.3a4( empty local-les=71435 n=0 ec=71423 les/c 71585/71585 71591/71591/71591) [70,6,66] r=0 lpr=71591 pi=71432-71590/4 crt=0'0 mlcod 0'0 peering] choose_acting want [70,6,66] (== acting) backfill_targets 2014-12-08 18:14:01.106551 7f28fcb46700 15 osd.70 pg_epoch: 71592 pg[218.535( empty local-les=71452 n=0 ec=71423 les/c 71585/71585 71591/71591/71444) [3,70,19]/[3,19,18] r=-1 lpr=71591 pi=71423-71590/11 crt=0'0 remapped NOTIFY] requeue_ops 2014-12-08 18:14:01.106561 7f28fcb46700 10 log is not dirty 2014-12-08 18:14:01.106560 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[218.3a4( empty local-les=71435 n=0 ec=71423 les/c 71585/71585 71591/71591/71591) [70,6,66] r=0 lpr=71591 pi=71432-71590/4 crt=0'0 mlcod 0'0 peering] state: leaving GetLog 2014-12-08 18:14:01.106567 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[218.3b1( v 71581'1 (0'0,71581'1] local-les=71450 n=1 ec=71423 les/c 71585/71583 71591/71591/71444) [5,10,70] r=2 lpr=71591 pi=71423-71590/10 crt=0'0 lcod 0'0 inactive NOTIFY] handle_peering_event: epoch_sent: 71591 epoch_requested: 71591 FlushedEvt 2014-12-08 18:14:01.106575 7f28fc345700 5 osd.70 pg_epoch: 71592 pg[218.3a4( empty local-les=71435 n=0 ec=71423 les/c 71585/71585 71591/71591/71591) [70,6,66] r=0 lpr=71591 pi=71432-71590/4 crt=0'0 mlcod 0'0 peering] exit Started/Primary/Peering/GetLog 0.000192 0 0.000000 2014-12-08 18:14:01.106580 7f28fcb46700 15 osd.70 pg_epoch: 71592 pg[218.3b1( v 71581'1 (0'0,71581'1] local-les=71450 n=1 ec=71423 les/c 71585/71583 71591/71591/71444) [5,10,70] r=2 lpr=71591 pi=71423-71590/10 crt=0'0 lcod 0'0 inactive NOTIFY] requeue_ops 2014-12-08 18:14:01.106591 7f28fcb46700 10 log is not dirty 2014-12-08 18:14:01.106591 7f28fc345700 15 osd.70 pg_epoch: 71592 pg[218.3a4( empty local-les=71435 n=0 ec=71423 les/c 71585/71585 71591/71591/71591) [70,6,66] r=0 lpr=71591 pi=71432-71590/4 crt=0'0 mlcod 0'0 peering] publish_stats_to_osd 71592:24 2014-12-08 18:14:01.106596 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[216.1de( v 71097'224 (0'0,71097'224] local-les=71435 n=109 ec=70987 les/c 71435/71435 71591/71592/70989) [15,78,70] r=2 lpr=71592 pi=71183-71591/19 crt=71097'224 lcod 0'0 inactive NOTIFY] handle_peering_event: epoch_sent: 71591 epoch_requested: 71591 FlushedEvt 2014-12-08 18:14:01.106607 7f28fc345700 5 osd.70 pg_epoch: 71592 pg[218.3a4( empty local-les=71435 n=0 ec=71423 les/c 71585/71585 71591/71591/71591) [70,6,66] r=0 lpr=71591 pi=71432-71590/4 crt=0'0 mlcod 0'0 peering] enter Started/Primary/Peering/GetMissing 2014-12-08 18:14:01.106610 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[216.1de( v 71097'224 (0'0,71097'224] local-les=71435 n=109 ec=70987 les/c 71435/71435 71591/71592/70989) [15,78,70] r=2 lpr=71592 pi=71183-71591/19 crt=71097'224 lcod 0'0 inactive NOTIFY] old_peering_msg reply_epoch 71591 query_epoch 71591 last_peering_reset 71592 2014-12-08 18:14:01.106622 7f28fcb46700 10 log is not dirty 2014-12-08 18:14:01.106624 7f28fc345700 5 osd.70 pg_epoch: 71592 pg[218.3a4( empty local-les=71435 n=0 ec=71423 les/c 71585/71585 71591/71591/71591) [70,6,66] r=0 lpr=71591 pi=71432-71590/4 crt=0'0 mlcod 0'0 peering] exit Started/Primary/Peering/GetMissing 0.000018 0 0.000000 2014-12-08 18:14:01.106626 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] handle_peering_event: epoch_sent: 71591 epoch_requested: 71591 MNotifyRec from 76 notify: (query_epoch:71591, epoch_sent:71591, info:19.6e( empty local-les=0 n=0 ec=21298 les/c 51101/50495 71591/71591/71591)) features: 0x3fffffffffff 2014-12-08 18:14:01.106642 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] got dup osd.76 info 19.6e( empty local-les=0 n=0 ec=21298 les/c 51101/50495 71591/71591/71591), identical to ours 2014-12-08 18:14:01.106643 7f28fc345700 15 osd.70 pg_epoch: 71592 pg[218.3a4( empty local-les=71435 n=0 ec=71423 les/c 71585/71585 71591/71591/71591) [70,6,66] r=0 lpr=71591 pi=71432-71590/4 crt=0'0 mlcod 0'0 peering] publish_stats_to_osd 71592:25 2014-12-08 18:14:01.106656 7f28fcb46700 10 log is not dirty 2014-12-08 18:14:01.106656 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[218.3a4( empty local-les=71435 n=0 ec=71423 les/c 71585/71585 71591/71591/71591) [70,6,66] r=0 lpr=71591 pi=71432-71590/4 crt=0'0 mlcod 0'0 peering] state: Leaving Peering 2014-12-08 18:14:01.106660 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[216.1f9( empty local-les=71437 n=0 ec=70987 les/c 71585/71585 71591/71591/71587) [70,9,77]/[9,77,3] r=-1 lpr=71591 pi=71432-71590/4 crt=0'0 remapped NOTIFY] handle_peering_event: epoch_sent: 71591 epoch_requested: 71591 FlushedEvt 2014-12-08 18:14:01.106668 7f28fc345700 5 osd.70 pg_epoch: 71592 pg[218.3a4( empty local-les=71435 n=0 ec=71423 les/c 71585/71585 71591/71591/71591) [70,6,66] r=0 lpr=71591 pi=71432-71590/4 crt=0'0 mlcod 0'0 peering] exit Started/Primary/Peering 0.593131 0 0.000000 2014-12-08 18:14:01.106677 7f28fcb46700 15 osd.70 pg_epoch: 71592 pg[216.1f9( empty local-les=71437 n=0 ec=70987 les/c 71585/71585 71591/71591/71587) [70,9,77]/[9,77,3] r=-1 lpr=71591 pi=71432-71590/4 crt=0'0 remapped NOTIFY] requeue_ops 2014-12-08 18:14:01.106686 7f28fcb46700 10 log is not dirty 2014-12-08 18:14:01.106684 7f28fc345700 5 osd.70 pg_epoch: 71592 pg[218.3a4( empty local-les=71435 n=0 ec=71423 les/c 71585/71585 71591/71591/71591) [70,6,66] r=0 lpr=71591 pi=71432-71590/4 crt=0'0 mlcod 0'0 inactive] enter Started/Primary/Active 2014-12-08 18:14:01.106691 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[216.1d0( v 71097'265 (0'0,71097'265] local-les=71437 n=123 ec=70987 les/c 71437/71437 71591/71592/71592) [70,13,20] r=0 lpr=71592 pi=71432-71591/5 crt=71097'265 lcod 0'0 mlcod 0'0 peering] handle_peering_event: epoch_sent: 71591 epoch_requested: 71591 FlushedEvt 2014-12-08 18:14:01.106694 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[218.3a4( empty local-les=71435 n=0 ec=71423 les/c 71585/71585 71591/71591/71591) [70,6,66] r=0 lpr=71591 pi=71432-71590/4 crt=0'0 mlcod 0'0 inactive] state: In Active, about to call activate 2014-12-08 18:14:01.106703 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[216.1d0( v 71097'265 (0'0,71097'265] local-les=71437 n=123 ec=70987 les/c 71437/71437 71591/71592/71592) [70,13,20] r=0 lpr=71592 pi=71432-71591/5 crt=71097'265 lcod 0'0 mlcod 0'0 peering] old_peering_msg reply_epoch 71591 query_epoch 71591 last_peering_reset 71592 2014-12-08 18:14:01.106715 7f28fcb46700 10 log is not dirty 2014-12-08 18:14:01.106708 7f28fc345700 20 osd.70 pg_epoch: 71592 pg[218.3a4( empty local-les=71592 n=0 ec=71423 les/c 71585/71585 71591/71591/71591) [70,6,66] r=0 lpr=71591 pi=71432-71590/4 crt=0'0 mlcod 0'0 inactive] activate - purged_snaps [] cached_removed_snaps [] 2014-12-08 18:14:01.106719 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] handle_peering_event: epoch_sent: 71591 epoch_requested: 71591 MNotifyRec from 0 notify: (query_epoch:71591, epoch_sent:71591, info:19.6e( DNE empty local-les=0 n=0 ec=0 les/c 0/0 0/0/0)) features: 0x3fffffffffff 2014-12-08 18:14:01.106722 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[218.3a4( empty local-les=71592 n=0 ec=71423 les/c 71585/71585 71591/71591/71591) [70,6,66] r=0 lpr=71591 pi=71432-71590/4 crt=0'0 mlcod 0'0 inactive] activate - snap_trimq [] 2014-12-08 18:14:01.106732 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[218.3a4( empty local-les=71592 n=0 ec=71423 les/c 71585/71585 71591/71591/71591) [70,6,66] r=0 lpr=71591 pi=71432-71590/4 crt=0'0 mlcod 0'0 inactive] activate - no missing, moving last_complete 0'0 -> 0'0 2014-12-08 18:14:01.106732 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] got osd.0 19.6e( DNE empty local-les=0 n=0 ec=0 les/c 0/0 0/0/0) 2014-12-08 18:14:01.106744 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[218.3a4( empty local-les=71592 n=0 ec=71423 les/c 71585/71585 71591/71591/71591) [70,6,66] r=0 lpr=71591 pi=71432-71590/4 crt=0'0 mlcod 0'0 inactive] activate peer osd.6 218.3a4( empty local-les=71585 n=0 ec=71423 les/c 71585/71585 71591/71591/71591) 2014-12-08 18:14:01.106746 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] osd.0 has stray content: 19.6e( DNE empty local-les=0 n=0 ec=0 les/c 0/0 0/0/0) 2014-12-08 18:14:01.106758 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[218.3a4( empty local-les=71592 n=0 ec=71423 les/c 71585/71585 71591/71591/71591) [70,6,66] r=0 lpr=71591 pi=71432-71590/4 crt=0'0 mlcod 0'0 inactive] activate peer osd.6 is up to date, but sending pg_log anyway 2014-12-08 18:14:01.106762 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] update_heartbeat_peers 4,11,13,16,20,30,31,56,58,63,70,76 -> 0,4,11,13,16,20,30,31,56,58,63,70,76 2014-12-08 18:14:01.106778 7f28fcb46700 20 osd.70 71592 need_heartbeat_peer_update 2014-12-08 18:14:01.106773 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[218.3a4( empty local-les=71592 n=0 ec=71423 les/c 71585/71585 71591/71591/71591) [70,6,66] r=0 lpr=71591 pi=71432-71590/4 crt=0'0 mlcod 0'0 inactive] activate peer osd.6 sending log((0'0,0'0], crt=0'0) 2014-12-08 18:14:01.106781 7f28fcb46700 20 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] state: Adding osd: 0 features: 3fffffffffff 2014-12-08 18:14:01.106790 7f28fcb46700 10 log is not dirty 2014-12-08 18:14:01.106788 7f28fc345700 20 osd.70 71592 share_map_peer 0x128a0b00 already has epoch 71592 2014-12-08 18:14:01.106795 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[217.350( v 71377'4121 (71377'1120,71377'4121] local-les=71446 n=17 ec=71094 les/c 71588/71589 71591/71591/71096) [13,70,7] r=1 lpr=71591 pi=71183-71590/24 crt=71377'4119 lcod 0'0 inactive NOTIFY] handle_peering_event: epoch_sent: 71591 epoch_requested: 71591 FlushedEvt 2014-12-08 18:14:01.106807 7f28fcb46700 15 osd.70 pg_epoch: 71592 pg[217.350( v 71377'4121 (71377'1120,71377'4121] local-les=71446 n=17 ec=71094 les/c 71588/71589 71591/71591/71096) [13,70,7] r=1 lpr=71591 pi=71183-71590/24 crt=71377'4119 lcod 0'0 inactive NOTIFY] requeue_ops 2014-12-08 18:14:01.106814 7f28fcb46700 10 log is not dirty 2014-12-08 18:14:01.106813 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[218.3a4( empty local-les=71592 n=0 ec=71423 les/c 71585/71585 71591/71591/71591) [70,6,66] r=0 lpr=71591 pi=71432-71590/4 crt=0'0 mlcod 0'0 inactive] activate peer osd.6 218.3a4( empty local-les=71585 n=0 ec=71423 les/c 71585/71585 71591/71591/71591) uptodate 2014-12-08 18:14:01.106819 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[218.50b( v 71581'1 (0'0,71581'1] local-les=71592 n=1 ec=71423 les/c 71585/71585 71591/71591/71591) [70,16,23] r=0 lpr=71591 pi=71444-71590/4 crt=0'0 lcod 0'0 mlcod 0'0 inactive] handle_peering_event: epoch_sent: 71591 epoch_requested: 71591 MNotifyRec from 64 notify: (query_epoch:71591, epoch_sent:71591, info:218.50b( v 71581'1 (0'0,71581'1] local-les=71585 n=1 ec=71423 les/c 71585/71585 71591/71591/71591)) features: 0x3fffffffffff 2014-12-08 18:14:01.106828 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[218.3a4( empty local-les=71592 n=0 ec=71423 les/c 71585/71585 71591/71591/71591) [70,6,66] r=0 lpr=71591 pi=71432-71590/4 crt=0'0 mlcod 0'0 inactive] activate peer osd.66 218.3a4( DNE empty local-les=0 n=0 ec=0 les/c 0/0 0/0/0) 2014-12-08 18:14:01.106838 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[218.50b( v 71581'1 (0'0,71581'1] local-les=71592 n=1 ec=71423 les/c 71585/71585 71591/71591/71591) [70,16,23] r=0 lpr=71591 pi=71444-71590/4 crt=0'0 lcod 0'0 mlcod 0'0 inactive] state: Active: got notify from 64, already have info from that osd, ignoring 2014-12-08 18:14:01.106841 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[218.3a4( empty local-les=71592 n=0 ec=71423 les/c 71585/71585 71591/71591/71591) [70,6,66] r=0 lpr=71591 pi=71432-71590/4 crt=0'0 mlcod 0'0 inactive] activate peer osd.66 is up to date, but sending pg_log anyway 2014-12-08 18:14:01.106850 7f28fcb46700 10 log is not dirty 2014-12-08 18:14:01.106854 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[218.50b( v 71581'1 (0'0,71581'1] local-les=71592 n=1 ec=71423 les/c 71585/71585 71591/71591/71591) [70,16,23] r=0 lpr=71591 pi=71444-71590/4 crt=0'0 lcod 0'0 mlcod 0'0 inactive] handle_peering_event: epoch_sent: 71591 epoch_requested: 71591 MNotifyRec from 23 notify: (query_epoch:71591, epoch_sent:71591, info:218.50b( v 71581'1 (0'0,71581'1] local-les=71585 n=1 ec=71423 les/c 71585/71583 71591/71591/71591)) features: 0x3fffffffffff 2014-12-08 18:14:01.106857 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[218.3a4( empty local-les=71592 n=0 ec=71423 les/c 71585/71585 71591/71591/71591) [70,6,66] r=0 lpr=71591 pi=71432-71590/4 crt=0'0 mlcod 0'0 inactive] activate peer osd.66 sending log((0'0,0'0], crt=0'0) 2014-12-08 18:14:01.106866 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[218.50b( v 71581'1 (0'0,71581'1] local-les=71592 n=1 ec=71423 les/c 71585/71585 71591/71591/71591) [70,16,23] r=0 lpr=71591 pi=71444-71590/4 crt=0'0 lcod 0'0 mlcod 0'0 inactive] state: Active: got notify from 23, already have info from that osd, ignoring 2014-12-08 18:14:01.106871 7f28fc345700 20 osd.70 71592 share_map_peer 0x12889ce0 don't know epoch, doing nothing 2014-12-08 18:14:01.106877 7f28fcb46700 10 log is not dirty 2014-12-08 18:14:01.106882 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[217.227( v 71377'12247 (71377'9246,71377'12247] local-les=71592 n=15 ec=71094 les/c 71583/71583 71591/71591/71591) [70,22,1] r=0 lpr=71591 pi=71444-71590/6 crt=71377'12245 lcod 0'0 mlcod 0'0 inactive] handle_peering_event: epoch_sent: 71591 epoch_requested: 71591 MNotifyRec from 22 notify: (query_epoch:71591, epoch_sent:71591, info:217.227( v 71377'12247 (71377'9246,71377'12247] local-les=71586 n=15 ec=71094 les/c 71583/71583 71591/71591/71591)) features: 0x3fffffffffff 2014-12-08 18:14:01.106896 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[217.227( v 71377'12247 (71377'9246,71377'12247] local-les=71592 n=15 ec=71094 les/c 71583/71583 71591/71591/71591) [70,22,1] r=0 lpr=71591 pi=71444-71590/6 crt=71377'12245 lcod 0'0 mlcod 0'0 inactive] state: Active: got notify from 22, already have info from that osd, ignoring 2014-12-08 18:14:01.106906 7f28fcb46700 10 log is not dirty 2014-12-08 18:14:01.106898 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[218.3a4( empty local-les=71592 n=0 ec=71423 les/c 71585/71585 71591/71591/71591) [70,6,66] r=0 lpr=71591 pi=71432-71590/4 crt=0'0 mlcod 0'0 inactive] activate peer osd.66 218.3a4( DNE empty local-les=0 n=0 ec=0 les/c 0/0 0/0/0) uptodate -- 2014-12-08 18:14:01.108051 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] handle_peering_event: epoch_sent: 71591 epoch_requested: 71591 MNotifyRec from 59 notify: (query_epoch:71591, epoch_sent:71591, info:19.6e( DNE empty local-les=0 n=0 ec=0 les/c 0/0 0/0/0)) features: 0x3fffffffffff 2014-12-08 18:14:01.108064 7f28fc345700 10 log is not dirty 2014-12-08 18:14:01.108065 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] got osd.59 19.6e( DNE empty local-les=0 n=0 ec=0 les/c 0/0 0/0/0) 2014-12-08 18:14:01.108071 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[217.370( v 71377'12257 (71377'9256,71377'12257] local-les=71592 n=16 ec=71094 les/c 71588/71588 71591/71591/71591) [70,10,56] r=0 lpr=71591 pi=71434-71590/5 crt=71377'12255 lcod 0'0 mlcod 0'0 inactive] handle_peering_event: epoch_sent: 71591 epoch_requested: 71591 MNotifyRec from 10 notify: (query_epoch:71591, epoch_sent:71591, info:217.370( v 71377'12257 (71377'9256,71377'12257] local-les=71588 n=16 ec=71094 les/c 71588/71588 71591/71591/71591)) features: 0x3fffffffffff 2014-12-08 18:14:01.108079 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] osd.59 has stray content: 19.6e( DNE empty local-les=0 n=0 ec=0 les/c 0/0 0/0/0) 2014-12-08 18:14:01.108089 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[217.370( v 71377'12257 (71377'9256,71377'12257] local-les=71592 n=16 ec=71094 les/c 71588/71588 71591/71591/71591) [70,10,56] r=0 lpr=71591 pi=71434-71590/5 crt=71377'12255 lcod 0'0 mlcod 0'0 inactive] state: Active: got notify from 10, already have info from that osd, ignoring 2014-12-08 18:14:01.108100 7f28fc345700 10 log is not dirty 2014-12-08 18:14:01.108092 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] update_heartbeat_peers 0,4,11,13,16,20,30,31,56,58,63,70,76 -> 0,4,11,13,16,20,30,31,56,58,59,63,70,76 2014-12-08 18:14:01.108108 7f28fcb46700 20 osd.70 71592 need_heartbeat_peer_update 2014-12-08 18:14:01.108105 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[216.3( v 70990'236 (0'0,70990'236] local-les=71592 n=109 ec=70987 les/c 71585/71583 71591/71591/71591) [70,16,56] r=0 lpr=71591 pi=71432-71590/4 crt=70990'236 lcod 0'0 mlcod 0'0 inactive] handle_peering_event: epoch_sent: 71591 epoch_requested: 71591 MNotifyRec from 16 notify: (query_epoch:71591, epoch_sent:71591, info:216.3( v 70990'236 (0'0,70990'236] local-les=71585 n=109 ec=70987 les/c 71585/71583 71591/71591/71591)) features: 0x3fffffffffff 2014-12-08 18:14:01.108111 7f28fcb46700 20 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] state: Adding osd: 59 features: 3fffffffffff 2014-12-08 18:14:01.108121 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[216.3( v 70990'236 (0'0,70990'236] local-les=71592 n=109 ec=70987 les/c 71585/71583 71591/71591/71591) [70,16,56] r=0 lpr=71591 pi=71432-71590/4 crt=70990'236 lcod 0'0 mlcod 0'0 inactive] state: Active: got notify from 16, already have info from that osd, ignoring 2014-12-08 18:14:01.108133 7f28fc345700 10 log is not dirty 2014-12-08 18:14:01.108122 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] state: last maybe_went_rw interval was interval(71584-71586 up [76,16,13](76) acting [76,16,13](76) maybe_went_rw) 2014-12-08 18:14:01.108136 7f28fcb46700 20 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] state: Common features: 3fffffffffff 2014-12-08 18:14:01.108137 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[216.3( v 70990'236 (0'0,70990'236] local-les=71592 n=109 ec=70987 les/c 71585/71583 71591/71591/71591) [70,16,56] r=0 lpr=71591 pi=71432-71590/4 crt=70990'236 lcod 0'0 mlcod 0'0 inactive] handle_peering_event: epoch_sent: 71591 epoch_requested: 71591 MNotifyRec from 20 notify: (query_epoch:71591, epoch_sent:71591, info:216.3( v 70990'236 lc 70990'1 (0'0,70990'236] local-les=71585 n=109 ec=70987 les/c 71585/71583 71591/71591/71591)) features: 0x3fffffffffff 2014-12-08 18:14:01.108149 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[216.3( v 70990'236 (0'0,70990'236] local-les=71592 n=109 ec=70987 les/c 71585/71583 71591/71591/71591) [70,16,56] r=0 lpr=71591 pi=71432-71590/4 crt=70990'236 lcod 0'0 mlcod 0'0 inactive] state: Active: got notify from 20, already have info from that osd, ignoring 2014-12-08 18:14:01.108148 7f28fcb46700 5 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] exit Started/Primary/Peering/GetInfo 0.676451 21 0.006813 2014-12-08 18:14:01.108160 7f28fc345700 10 log is not dirty 2014-12-08 18:14:01.108163 7f28fcb46700 5 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] enter Started/Primary/Peering/GetLog 2014-12-08 18:14:01.108165 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[217.18( v 71377'4106 (71377'1105,71377'4106] local-les=71592 n=8 ec=71094 les/c 71586/71583 71591/71591/71591) [70,56,15] r=0 lpr=71591 pi=71444-71590/6 crt=71377'4103 lcod 0'0 mlcod 0'0 inactive] handle_peering_event: epoch_sent: 71591 epoch_requested: 71591 MNotifyRec from 15 notify: (query_epoch:71591, epoch_sent:71591, info:217.18( v 71377'4106 (71377'1105,71377'4106] local-les=71586 n=8 ec=71094 les/c 71586/71583 71591/71591/71591)) features: 0x3fffffffffff 2014-12-08 18:14:01.108179 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[217.18( v 71377'4106 (71377'1105,71377'4106] local-les=71592 n=8 ec=71094 les/c 71586/71583 71591/71591/71591) [70,56,15] r=0 lpr=71591 pi=71444-71590/6 crt=71377'4103 lcod 0'0 mlcod 0'0 inactive] state: Active: got notify from 15, already have info from that osd, ignoring 2014-12-08 18:14:01.108229 7f28fc345700 10 log is not dirty 2014-12-08 18:14:01.108222 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] calc_acting osd.0 19.6e( DNE empty local-les=0 n=0 ec=0 les/c 0/0 0/0/0) 2014-12-08 18:14:01.108236 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] calc_acting osd.4 19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) 2014-12-08 18:14:01.108238 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[216.19f( v 71097'244 (0'0,71097'244] local-les=71435 n=117 ec=70987 les/c 71585/71583 71591/71591/70989) [24,70,23]/[24,23,63] r=-1 lpr=71591 pi=70987-71590/25 crt=71097'242 lcod 0'0 remapped NOTIFY] handle_peering_event: epoch_sent: 71591 epoch_requested: 71591 FlushedEvt 2014-12-08 18:14:01.108254 7f28fc345700 15 osd.70 pg_epoch: 71592 pg[216.19f( v 71097'244 (0'0,71097'244] local-les=71435 n=117 ec=70987 les/c 71585/71583 71591/71591/70989) [24,70,23]/[24,23,63] r=-1 lpr=71591 pi=70987-71590/25 crt=71097'242 lcod 0'0 remapped NOTIFY] requeue_ops 2014-12-08 18:14:01.108251 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] calc_acting osd.11 19.6e( DNE empty local-les=0 n=0 ec=0 les/c 0/0 0/0/0) 2014-12-08 18:14:01.108265 7f28fc345700 10 log is not dirty 2014-12-08 18:14:01.108271 7f28fc345700 20 osd.70 71592 share_map_peer 0x12839760 don't know epoch, doing nothing 2014-12-08 18:14:01.108274 7f28fc345700 7 osd.70 71592 do_notifies osd 17 on 1 PGs 2014-12-08 18:14:01.108266 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] calc_acting osd.13 19.6e( DNE empty local-les=0 n=0 ec=0 les/c 0/0 0/0/0) 2014-12-08 18:14:01.108276 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] calc_acting osd.16 19.6e( empty local-les=0 n=0 ec=21298 les/c 51101/50495 71591/71591/71591) 2014-12-08 18:14:01.108287 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] calc_acting osd.20 19.6e( DNE empty local-les=0 n=0 ec=0 les/c 0/0 0/0/0) 2014-12-08 18:14:01.108296 7f28fc345700 20 osd.70 71592 share_map_peer 0x11a2eb00 don't know epoch, doing nothing 2014-12-08 18:14:01.108298 7f28fc345700 7 osd.70 71592 do_notifies osd 69 on 1 PGs 2014-12-08 18:14:01.108296 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] calc_acting osd.30 19.6e( empty local-les=0 n=0 ec=21298 les/c 51101/50495 71591/71591/71591) 2014-12-08 18:14:01.108313 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] calc_acting osd.31 19.6e( DNE empty local-les=0 n=0 ec=0 les/c 0/0 0/0/0) 2014-12-08 18:14:01.108325 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] calc_acting osd.56 19.6e( DNE empty local-les=0 n=0 ec=0 les/c 0/0 0/0/0) 2014-12-08 18:14:01.108334 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[216.3( v 70990'236 (0'0,70990'236] local-les=71592 n=109 ec=70987 les/c 71585/71583 71591/71591/71591) [70,16,56] r=0 lpr=71591 pi=71432-71590/4 crt=70990'236 lcod 0'0 mlcod 0'0 inactive] handle_peering_event: epoch_sent: 71591 epoch_requested: 71591 MNotifyRec from 56 notify: (query_epoch:71591, epoch_sent:71591, info:216.3( v 70990'236 (0'0,70990'236] local-les=71585 n=109 ec=70987 les/c 71585/71583 71591/71591/71591)) features: 0x3fffffffffff 2014-12-08 18:14:01.108339 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] calc_acting osd.58 19.6e( empty local-les=0 n=0 ec=21298 les/c 51101/50495 71591/71591/71591) 2014-12-08 18:14:01.108350 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[216.3( v 70990'236 (0'0,70990'236] local-les=71592 n=109 ec=70987 les/c 71585/71583 71591/71591/71591) [70,16,56] r=0 lpr=71591 pi=71432-71590/4 crt=70990'236 lcod 0'0 mlcod 0'0 inactive] state: Active: got notify from 56, already have info from that osd, ignoring 2014-12-08 18:14:01.108359 7f28fc345700 10 log is not dirty 2014-12-08 18:14:01.108356 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] calc_acting osd.59 19.6e( DNE empty local-les=0 n=0 ec=0 les/c 0/0 0/0/0) 2014-12-08 18:14:01.108363 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[217.370( v 71377'12257 (71377'9256,71377'12257] local-les=71592 n=16 ec=71094 les/c 71588/71588 71591/71591/71591) [70,10,56] r=0 lpr=71591 pi=71434-71590/5 crt=71377'12255 lcod 0'0 mlcod 0'0 inactive] handle_peering_event: epoch_sent: 71591 epoch_requested: 71591 MNotifyRec from 56 notify: (query_epoch:71591, epoch_sent:71591, info:217.370( v 71377'12257 (71377'9256,71377'12257] local-les=71588 n=16 ec=71094 les/c 71588/71588 71591/71591/71591)) features: 0x3fffffffffff 2014-12-08 18:14:01.108376 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[217.370( v 71377'12257 (71377'9256,71377'12257] local-les=71592 n=16 ec=71094 les/c 71588/71588 71591/71591/71591) [70,10,56] r=0 lpr=71591 pi=71434-71590/5 crt=71377'12255 lcod 0'0 mlcod 0'0 inactive] state: Active: got notify from 56, already have info from that osd, ignoring 2014-12-08 18:14:01.108385 7f28fc345700 10 log is not dirty 2014-12-08 18:14:01.108374 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] calc_acting osd.63 19.6e( DNE empty local-les=0 n=0 ec=0 les/c 0/0 0/0/0) 2014-12-08 18:14:01.108388 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[217.18( v 71377'4106 (71377'1105,71377'4106] local-les=71592 n=8 ec=71094 les/c 71586/71583 71591/71591/71591) [70,56,15] r=0 lpr=71591 pi=71444-71590/6 crt=71377'4103 lcod 0'0 mlcod 0'0 inactive] handle_peering_event: epoch_sent: 71591 epoch_requested: 71591 MNotifyRec from 56 notify: (query_epoch:71591, epoch_sent:71591, info:217.18( v 71377'4106 (71377'1105,71377'4106] local-les=71586 n=8 ec=71094 les/c 71586/71583 71591/71591/71591)) features: 0x3fffffffffff 2014-12-08 18:14:01.108401 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[217.18( v 71377'4106 (71377'1105,71377'4106] local-les=71592 n=8 ec=71094 les/c 71586/71583 71591/71591/71591) [70,56,15] r=0 lpr=71591 pi=71444-71590/6 crt=71377'4103 lcod 0'0 mlcod 0'0 inactive] state: Active: got notify from 56, already have info from that osd, ignoring 2014-12-08 18:14:01.108411 7f28fc345700 10 log is not dirty 2014-12-08 18:14:01.108391 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] calc_acting osd.70 19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) 2014-12-08 18:14:01.108415 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[217.32( v 71377'8183 (71377'5182,71377'8183] local-les=71438 n=15 ec=71094 les/c 71590/71590 71591/71591/71096) [23,22,70] r=2 lpr=71591 pi=71183-71590/21 crt=71377'8181 lcod 0'0 inactive NOTIFY] handle_peering_event: epoch_sent: 71591 epoch_requested: 71591 FlushedEvt 2014-12-08 18:14:01.108414 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] calc_acting osd.76 19.6e( empty local-les=0 n=0 ec=21298 les/c 51101/50495 71591/71591/71591) 2014-12-08 18:14:01.108430 7f28fc345700 15 osd.70 pg_epoch: 71592 pg[217.32( v 71377'8183 (71377'5182,71377'8183] local-les=71438 n=15 ec=71094 les/c 71590/71590 71591/71591/71096) [23,22,70] r=2 lpr=71591 pi=71183-71590/21 crt=71377'8181 lcod 0'0 inactive NOTIFY] requeue_ops 2014-12-08 18:14:01.108440 7f28fc345700 10 log is not dirty 2014-12-08 18:14:01.108431 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] choose_acting failed 2014-12-08 18:14:01.108444 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[216.1b1( v 71097'260 (0'0,71097'260] local-les=71434 n=123 ec=70987 les/c 71585/71583 71591/71591/70989) [22,70,18]/[22,18,16] r=-1 lpr=71591 pi=71183-71590/16 crt=71097'260 lcod 0'0 remapped NOTIFY] handle_peering_event: epoch_sent: 71591 epoch_requested: 71591 FlushedEvt 2014-12-08 18:14:01.108456 7f28fc345700 15 osd.70 pg_epoch: 71592 pg[216.1b1( v 71097'260 (0'0,71097'260] local-les=71434 n=123 ec=70987 les/c 71585/71583 71591/71591/70989) [22,70,18]/[22,18,16] r=-1 lpr=71591 pi=71183-71590/16 crt=71097'260 lcod 0'0 remapped NOTIFY] requeue_ops 2014-12-08 18:14:01.108466 7f28fc345700 10 log is not dirty 2014-12-08 18:14:01.108452 7f28fcb46700 5 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] exit Started/Primary/Peering/GetLog 0.000289 0 0.000000 2014-12-08 18:14:01.108471 7f28fcb46700 15 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] publish_stats_to_osd 71592:63844731 2014-12-08 18:14:01.108481 7f28fcb46700 5 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 peering] enter Started/Primary/Peering/Incomplete 2014-12-08 18:14:01.108479 7f28fc345700 10 osd.70 pg_epoch: 71591 pg[218.36( empty local-les=71438 n=0 ec=71423 les/c 71438/71438 71591/71591/71587) [69,70,77] r=1 lpr=71591 pi=71423-71590/8 crt=0'0 inactive NOTIFY] handle_advance_map [69,70,77]/[69,70,77] -- 69/69 2014-12-08 18:14:01.108495 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:14:01.108495 7f28fcb46700 15 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] publish_stats_to_osd 71592:63844732 2014-12-08 18:14:01.108500 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[218.36( empty local-les=71438 n=0 ec=71423 les/c 71438/71438 71591/71591/71587) [69,70,77] r=1 lpr=71591 pi=71423-71590/8 crt=0'0 inactive NOTIFY] state: Started advmap 2014-12-08 18:14:01.108508 7f28fcb46700 10 log is not dirty 2014-12-08 18:14:01.108510 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[218.36( empty local-les=71438 n=0 ec=71423 les/c 71438/71438 71591/71591/71587) [69,70,77] r=1 lpr=71591 pi=71423-71590/8 crt=0'0 inactive NOTIFY] check_recovery_sources no source osds () went down 2014-12-08 18:14:01.108516 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[218.38( v 71581'3 (0'0,71581'3] local-les=71434 n=3 ec=71423 les/c 71590/71583 71591/71591/71425) [56,17,70]/[56,17,15] r=-1 lpr=71591 pi=71423-71590/9 crt=71580'1 lcod 0'0 remapped NOTIFY] handle_peering_event: epoch_sent: 71591 epoch_requested: 71591 FlushedEvt 2014-12-08 18:14:01.108522 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[218.36( empty local-les=71438 n=0 ec=71423 les/c 71438/71438 71591/71591/71587) [69,70,77] r=1 lpr=71591 pi=71423-71590/8 crt=0'0 inactive NOTIFY] handle_activate_map 2014-12-08 18:14:01.108530 7f28fcb46700 15 osd.70 pg_epoch: 71592 pg[218.38( v 71581'3 (0'0,71581'3] local-les=71434 n=3 ec=71423 les/c 71590/71583 71591/71591/71425) [56,17,70]/[56,17,15] r=-1 lpr=71591 pi=71423-71590/9 crt=71580'1 lcod 0'0 remapped NOTIFY] requeue_ops 2014-12-08 18:14:01.108539 7f28fcb46700 10 log is not dirty 2014-12-08 18:14:01.108541 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[218.36( empty local-les=71438 n=0 ec=71423 les/c 71438/71438 71591/71591/71587) [69,70,77] r=1 lpr=71591 pi=71423-71590/8 crt=0'0 inactive NOTIFY] take_waiters 2014-12-08 18:14:01.108545 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[217.2e( v 71377'4787 (71377'1786,71377'4787] local-les=71436 n=13 ec=71094 les/c 71583/71583 71591/71591/71585) [70,68,16]/[16,68] r=-1 lpr=71591 pi=71435-71590/5 crt=71377'4784 lcod 0'0 remapped NOTIFY] handle_peering_event: epoch_sent: 71591 epoch_requested: 71591 FlushedEvt 2014-12-08 18:14:01.108550 7f28fc345700 20 osd.70 pg_epoch: 71592 pg[218.36( empty local-les=71438 n=0 ec=71423 les/c 71438/71438 71591/71591/71587) [69,70,77] r=1 lpr=71591 pi=71423-71590/8 crt=0'0 inactive NOTIFY] handle_activate_map: Not dirtying info: last_persisted is 71591 while current is 71592 2014-12-08 18:14:01.108560 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[218.36( empty local-les=71438 n=0 ec=71423 les/c 71438/71438 71591/71591/71587) [69,70,77] r=1 lpr=71591 pi=71423-71590/8 crt=0'0 inactive NOTIFY] handle_peering_event: epoch_sent: 71591 epoch_requested: 71591 FlushedEvt 2014-12-08 18:14:01.108560 7f28fcb46700 15 osd.70 pg_epoch: 71592 pg[217.2e( v 71377'4787 (71377'1786,71377'4787] local-les=71436 n=13 ec=71094 les/c 71583/71583 71591/71591/71585) [70,68,16]/[16,68] r=-1 lpr=71591 pi=71435-71590/5 crt=71377'4784 lcod 0'0 remapped NOTIFY] requeue_ops 2014-12-08 18:14:01.108574 7f28fcb46700 10 log is not dirty 2014-12-08 18:14:01.108570 7f28fc345700 15 osd.70 pg_epoch: 71592 pg[218.36( empty local-les=71438 n=0 ec=71423 les/c 71438/71438 71591/71591/71587) [69,70,77] r=1 lpr=71591 pi=71423-71590/8 crt=0'0 inactive NOTIFY] requeue_ops 2014-12-08 18:14:01.108579 7f28fc345700 10 log is not dirty 2014-12-08 18:14:01.108579 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[217.187( v 71377'12194 (71377'9193,71377'12194] local-les=71435 n=10 ec=71094 les/c 71583/71583 71591/71591/71096) [19,70,66] r=1 lpr=71591 pi=71136-71590/21 crt=71377'12191 lcod 0'0 inactive NOTIFY] handle_peering_event: epoch_sent: 71591 epoch_requested: 71591 FlushedEvt 2014-12-08 18:14:01.108584 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[216.362( v 71097'201 (0'0,71097'201] local-les=71435 n=95 ec=70987 les/c 71589/71583 71591/71591/71587) [78,70,5]/[78,5,17] r=-1 lpr=71591 pi=70987-71590/27 crt=71097'199 lcod 0'0 remapped NOTIFY] handle_peering_event: epoch_sent: 71591 epoch_requested: 71591 FlushedEvt 2014-12-08 18:14:01.108593 7f28fcb46700 15 osd.70 pg_epoch: 71592 pg[217.187( v 71377'12194 (71377'9193,71377'12194] local-les=71435 n=10 ec=71094 les/c 71583/71583 71591/71591/71096) [19,70,66] r=1 lpr=71591 pi=71136-71590/21 crt=71377'12191 lcod 0'0 inactive NOTIFY] requeue_ops 2014-12-08 18:14:01.108604 7f28fcb46700 10 log is not dirty 2014-12-08 18:14:01.108599 7f28fc345700 15 osd.70 pg_epoch: 71592 pg[216.362( v 71097'201 (0'0,71097'201] local-les=71435 n=95 ec=70987 les/c 71589/71583 71591/71591/71587) [78,70,5]/[78,5,17] r=-1 lpr=71591 pi=70987-71590/27 crt=71097'199 lcod 0'0 remapped NOTIFY] requeue_ops -- 2014-12-08 18:14:01.110197 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] handle_peering_event: epoch_sent: 71592 epoch_requested: 71592 NullEvt 2014-12-08 18:14:01.110210 7f28fcb46700 10 log is not dirty 2014-12-08 18:14:01.110201 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[217.206( v 71377'1055 (0'0,71377'1055] local-les=71448 n=11 ec=71094 les/c 71448/71448 71591/71592/71096) [76,70,13] r=1 lpr=71592 pi=71094-71591/26 crt=71377'1053 lcod 0'0 inactive NOTIFY] handle_peering_event: epoch_sent: 71592 epoch_requested: 71592 NullEvt 2014-12-08 18:14:01.110214 7f28fc345700 10 log is not dirty 2014-12-08 18:14:01.110215 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[19.6d( v 71397'33614009 (70963'33611008,71397'33614009] local-les=71439 n=896 ec=21298 les/c 71583/71583 71591/71591/71169) [4,70,69] r=1 lpr=71591 pi=71156-71590/27 crt=71397'33614007 lcod 0'0 inactive NOTIFY] handle_peering_event: epoch_sent: 71592 epoch_requested: 71592 NullEvt 2014-12-08 18:14:01.110218 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[216.39e( v 71422'199 (0'0,71422'199] local-les=71436 n=94 ec=70987 les/c 71583/71583 71591/71591/71587) [64,70,76] r=1 lpr=71591 pi=71156-71590/18 crt=71097'198 lcod 0'0 inactive NOTIFY] handle_peering_event: epoch_sent: 71592 epoch_requested: 71592 NullEvt 2014-12-08 18:14:01.110228 7f28fcb46700 10 log is not dirty 2014-12-08 18:14:01.110234 7f28fc345700 10 log is not dirty 2014-12-08 18:14:01.110246 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[218.6c7( v 71581'2 (0'0,71581'2] local-les=71433 n=2 ec=71423 les/c 71433/71433 71591/71592/71592) [70,10,18] r=0 lpr=71592 pi=71432-71591/6 crt=0'0 lcod 0'0 mlcod 0'0 peering] handle_peering_event: epoch_sent: 71592 epoch_requested: 71592 NullEvt 2014-12-08 18:14:01.110259 7f28fcb46700 10 log is not dirty 2014-12-08 18:14:01.110250 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[218.a3( v 71581'1 (0'0,71581'1] local-les=71434 n=1 ec=71423 les/c 71434/71436 71591/71592/71592) [70,20,23]/[20,23,75] r=-1 lpr=71592 pi=71432-71591/5 crt=0'0 lcod 0'0 remapped NOTIFY] handle_peering_event: epoch_sent: 71592 epoch_requested: 71592 NullEvt 2014-12-08 18:14:01.110264 7f28fc345700 10 log is not dirty 2014-12-08 18:14:01.110264 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[218.6c4( v 71581'3 (0'0,71581'3] local-les=71437 n=3 ec=71423 les/c 71437/71437 71591/71592/71592) [70,76,13] r=0 lpr=71592 pi=71432-71591/6 crt=71580'1 lcod 0'0 mlcod 0'0 peering] handle_peering_event: epoch_sent: 71592 epoch_requested: 71592 NullEvt 2014-12-08 18:14:01.110277 7f28fcb46700 10 log is not dirty 2014-12-08 18:14:01.110268 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[19.6a( v 71397'36030933 (50495'36027932,71397'36030933] local-les=71439 n=942 ec=21298 les/c 71588/71590 71591/71591/69763) [13,58,70] r=2 lpr=71591 pi=57840-71590/51 crt=71397'36030931 lcod 0'0 inactive NOTIFY] handle_peering_event: epoch_sent: 71592 epoch_requested: 71592 NullEvt 2014-12-08 18:14:01.110282 7f28fc345700 10 log is not dirty 2014-12-08 18:14:01.110286 7f28fc345700 10 osd.70 71592 queue_want_up_thru want 71592 <= queued 71592, currently 71591 2014-12-08 18:14:01.110282 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[218.562( v 71581'3 (0'0,71581'3] local-les=71437 n=3 ec=71423 les/c 71437/71437 71591/71592/71425) [9,18,70] r=2 lpr=71592 pi=71423-71591/10 crt=71580'1 lcod 0'0 inactive NOTIFY] handle_peering_event: epoch_sent: 71592 epoch_requested: 71592 NullEvt 2014-12-08 18:14:01.110294 7f28fcb46700 10 log is not dirty 2014-12-08 18:14:01.110298 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[218.5e( empty local-les=71435 n=0 ec=71423 les/c 71585/71585 71591/71591/71425) [16,22,70] r=2 lpr=71591 pi=71423-71590/8 crt=0'0 inactive NOTIFY] handle_peering_event: epoch_sent: 71592 epoch_requested: 71592 NullEvt 2014-12-08 18:14:01.110308 7f28fcb46700 10 log is not dirty -- 2014-12-08 18:14:01.114563 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] handle_peering_event: epoch_sent: 71592 epoch_requested: 71592 MNotifyRec from 30 notify: (query_epoch:71592, epoch_sent:71592, info:19.6e( empty local-les=0 n=0 ec=21298 les/c 51101/50495 71591/71591/71591)) features: 0x3fffffffffff 2014-12-08 18:14:01.114576 7f28fc345700 7 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] state: handle_pg_notify from osd.30 2014-12-08 18:14:01.114585 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] state: pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] got dup osd.30 info 19.6e( empty local-les=0 n=0 ec=21298 les/c 51101/50495 71591/71591/71591), identical to ours 2014-12-08 18:14:01.114599 7f28fc345700 10 log is not dirty 2014-12-08 18:14:01.114603 7f28fcb46700 10 log is not dirty 2014-12-08 18:14:01.114602 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[19.30( v 71397'38291943 (70960'38288942,71397'38291943] local-les=71592 n=911 ec=21298 les/c 71583/71583 71591/71591/71591) [70,66,1] r=0 lpr=71591 pi=71444-71590/6 crt=71397'38291940 lcod 0'0 mlcod 0'0 inactive] handle_peering_event: epoch_sent: 71592 epoch_requested: 71592 MNotifyRec from 8 notify: (query_epoch:71592, epoch_sent:71592, info:19.30( v 71397'38291943 (70960'38288943,71397'38291943] lb 0//0//-1 local-les=0 n=0 ec=21298 les/c 71583/71583 71591/71591/71591)) features: 0x3fffffffffff 2014-12-08 18:14:01.114612 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[19.30( v 71397'38291943 (70960'38288942,71397'38291943] local-les=71592 n=911 ec=21298 les/c 71583/71583 71591/71591/71591) [70,66,1] r=0 lpr=71591 pi=71444-71590/6 crt=71397'38291940 lcod 0'0 mlcod 0'0 inactive] state: Active: got notify from 8, already have info from that osd, ignoring 2014-12-08 18:14:01.114619 7f28fc345700 10 log is not dirty 2014-12-08 18:14:01.114623 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[216.3c5( v 71097'233 (0'0,71097'233] local-les=71449 n=113 ec=70987 les/c 71449/71450 71591/71592/71592) [70,5,20] r=0 lpr=71592 pi=71444-71591/5 crt=71097'233 lcod 0'0 mlcod 0'0 peering] handle_peering_event: epoch_sent: 71592 epoch_requested: 71592 MNotifyRec from 5 notify: (query_epoch:71592, epoch_sent:71592, info:216.3c5( v 71097'233 (0'0,71097'233] local-les=71585 n=113 ec=70987 les/c 71585/71583 71591/71592/71592)) features: 0x3fffffffffff 2014-12-08 18:14:01.114634 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[216.3c5( v 71097'233 (0'0,71097'233] local-les=71449 n=113 ec=70987 les/c 71449/71450 71591/71592/71592) [70,5,20] r=0 lpr=71592 pi=71444-71591/5 crt=71097'233 lcod 0'0 mlcod 0'0 peering] got osd.5 216.3c5( v 71097'233 (0'0,71097'233] local-les=71585 n=113 ec=70987 les/c 71585/71583 71591/71592/71592) 2014-12-08 18:14:01.114628 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[218.705( v 71581'1 (0'0,71581'1] local-les=71437 n=1 ec=71423 les/c 71437/71437 71591/71592/71592) [70,56,22] r=0 lpr=71592 pi=71432-71591/6 crt=0'0 lcod 0'0 mlcod 0'0 peering] handle_peering_event: epoch_sent: 71592 epoch_requested: 71592 MNotifyRec from 56 notify: (query_epoch:71592, epoch_sent:71592, info:218.705( v 71581'1 (0'0,71581'1] local-les=71585 n=1 ec=71423 les/c 71585/71586 71591/71592/71592)) features: 0x3fffffffffff 2014-12-08 18:14:01.114651 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[216.3c5( v 71097'233 (0'0,71097'233] local-les=71449 n=113 ec=70987 les/c 71585/71583 71591/71592/71592) [70,5,20] r=0 lpr=71592 pi=71444-71591/5 crt=71097'233 lcod 0'0 mlcod 0'0 peering] update_heartbeat_peers 5,20,70 unchanged 2014-12-08 18:14:01.114653 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[218.705( v 71581'1 (0'0,71581'1] local-les=71437 n=1 ec=71423 les/c 71437/71437 71591/71592/71592) [70,56,22] r=0 lpr=71592 pi=71432-71591/6 crt=0'0 lcod 0'0 mlcod 0'0 peering] got osd.56 218.705( v 71581'1 (0'0,71581'1] local-les=71585 n=1 ec=71423 les/c 71585/71586 71591/71592/71592) 2014-12-08 18:14:01.114663 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[216.3c5( v 71097'233 (0'0,71097'233] local-les=71449 n=113 ec=70987 les/c 71585/71583 71591/71592/71592) [70,5,20] r=0 lpr=71592 pi=71444-71591/5 crt=71097'233 lcod 0'0 mlcod 0'0 peering] state: last_epoch_started moved forward, rebuilding prior 2014-12-08 18:14:01.114672 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[216.3c5( v 71097'233 (0'0,71097'233] local-les=71449 n=113 ec=70987 les/c 71585/71583 71591/71592/71592) [70,5,20] r=0 lpr=71592 pi=71444-71591/5 crt=71097'233 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(71591-71591 up [70,5,20](70) acting [5,20,1](5)) 2014-12-08 18:14:01.114673 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[218.705( v 71581'1 (0'0,71581'1] local-les=71437 n=1 ec=71423 les/c 71585/71586 71591/71592/71592) [70,56,22] r=0 lpr=71592 pi=71432-71591/6 crt=0'0 lcod 0'0 mlcod 0'0 peering] update_heartbeat_peers 22,56,70 unchanged 2014-12-08 18:14:01.114684 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[216.3c5( v 71097'233 (0'0,71097'233] local-les=71449 n=113 ec=70987 les/c 71585/71583 71591/71592/71592) [70,5,20] r=0 lpr=71592 pi=71444-71591/5 crt=71097'233 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(71587-71590 up [5,20](5) acting [5,20](5) maybe_went_rw) 2014-12-08 18:14:01.114686 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[218.705( v 71581'1 (0'0,71581'1] local-les=71437 n=1 ec=71423 les/c 71585/71586 71591/71592/71592) [70,56,22] r=0 lpr=71592 pi=71432-71591/6 crt=0'0 lcod 0'0 mlcod 0'0 peering] state: last_epoch_started moved forward, rebuilding prior 2014-12-08 18:14:01.114696 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[216.3c5( v 71097'233 (0'0,71097'233] local-les=71449 n=113 ec=70987 les/c 71585/71583 71591/71592/71592) [70,5,20] r=0 lpr=71592 pi=71444-71591/5 crt=71097'233 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(71584-71586 up [1,5,20](1) acting [1,5,20](1) maybe_went_rw) 2014-12-08 18:14:01.114699 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[218.705( v 71581'1 (0'0,71581'1] local-les=71437 n=1 ec=71423 les/c 71585/71586 71591/71592/71592) [70,56,22] r=0 lpr=71592 pi=71432-71591/6 crt=0'0 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(71591-71591 up [70,56,22](70) acting [56,22,9](56)) 2014-12-08 18:14:01.114707 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[216.3c5( v 71097'233 (0'0,71097'233] local-les=71449 n=113 ec=70987 les/c 71585/71583 71591/71592/71592) [70,5,20] r=0 lpr=71592 pi=71444-71591/5 crt=71097'233 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(71582-71583 up [5,20](5) acting [5,20](5) maybe_went_rw) 2014-12-08 18:14:01.114713 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[218.705( v 71581'1 (0'0,71581'1] local-les=71437 n=1 ec=71423 les/c 71585/71586 71591/71592/71592) [70,56,22] r=0 lpr=71592 pi=71432-71591/6 crt=0'0 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior interval(71590-71590 up [56,22](56) acting [56,22,9](56)) 2014-12-08 18:14:01.114718 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[216.3c5( v 71097'233 (0'0,71097'233] local-les=71449 n=113 ec=70987 les/c 71585/71583 71591/71592/71592) [70,5,20] r=0 lpr=71592 pi=71444-71591/5 crt=71097'233 lcod 0'0 mlcod 0'0 peering] PriorSet: build_prior final: probe 1,5,20,70 down blocked_by {} -- 2014-12-08 18:14:01.115027 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] handle_peering_event: epoch_sent: 71592 epoch_requested: 71592 MNotifyRec from 4 notify: (query_epoch:71592, epoch_sent:71592, info:19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591)) features: 0x3fffffffffff 2014-12-08 18:14:01.115036 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[216.3cd( v 71097'258 (0'0,71097'258] local-les=71449 n=120 ec=70987 les/c 71449/71450 71591/71592/71168) [18,5,70] r=2 lpr=71592 pi=70987-71591/34 crt=71097'256 lcod 0'0 inactive NOTIFY] handle_peering_event: epoch_sent: 71592 epoch_requested: 71592 MQuery from 18 query_epoch 71592 query: query(info 0'0) 2014-12-08 18:14:01.115039 7f28fc345700 7 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] state: handle_pg_notify from osd.4 2014-12-08 18:14:01.115049 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[216.3cd( v 71097'258 (0'0,71097'258] local-les=71449 n=120 ec=70987 les/c 71590/71583 71591/71592/71168) [18,5,70] r=2 lpr=71592 pi=70987-71591/34 crt=71097'256 lcod 0'0 inactive NOTIFY] sending info 2014-12-08 18:14:01.115049 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] state: pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] got dup osd.4 info 19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591), identical to ours 2014-12-08 18:14:01.115068 7f28fc345700 10 log is not dirty 2014-12-08 18:14:01.115071 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[217.bb( v 71377'16307 (71377'13306,71377'16307] local-les=71592 n=22 ec=71094 les/c 71586/71583 71591/71591/71591) [70,78,25] r=0 lpr=71591 pi=71434-71590/6 crt=71377'16305 lcod 0'0 mlcod 0'0 inactive] handle_peering_event: epoch_sent: 71592 epoch_requested: 71592 MNotifyRec from 78 notify: (query_epoch:71592, epoch_sent:71592, info:217.bb( v 71377'16307 (71377'13306,71377'16307] local-les=71590 n=22 ec=71094 les/c 71586/71583 71591/71591/71591)) features: 0x3fffffffffff 2014-12-08 18:14:01.115080 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[217.bb( v 71377'16307 (71377'13306,71377'16307] local-les=71592 n=22 ec=71094 les/c 71586/71583 71591/71591/71591) [70,78,25] r=0 lpr=71591 pi=71434-71590/6 crt=71377'16305 lcod 0'0 mlcod 0'0 inactive] state: Active: got notify from 78, already have info from that osd, ignoring 2014-12-08 18:14:01.115087 7f28fc345700 10 log is not dirty 2014-12-08 18:14:01.115091 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[218.38f( empty local-les=71457 n=0 ec=71423 les/c 71457/71457 71591/71592/71592) [70,23,25] r=0 lpr=71592 pi=71456-71591/5 crt=0'0 mlcod 0'0 peering] handle_peering_event: epoch_sent: 71592 epoch_requested: 71592 MNotifyRec from 25 notify: (query_epoch:71592, epoch_sent:71592, info:218.38f( empty local-les=71585 n=0 ec=71423 les/c 71585/71585 71591/71592/71592)) features: 0x3fffffffffff 2014-12-08 18:14:01.115104 7f28fcb46700 10 log is not dirty 2014-12-08 18:14:01.115103 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[218.38f( empty local-les=71457 n=0 ec=71423 les/c 71457/71457 71591/71592/71592) [70,23,25] r=0 lpr=71592 pi=71456-71591/5 crt=0'0 mlcod 0'0 peering] got osd.25 218.38f( empty local-les=71585 n=0 ec=71423 les/c 71585/71585 71591/71592/71592) 2014-12-08 18:14:01.115111 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[218.73e( v 71581'2 (0'0,71581'2] local-les=71449 n=2 ec=71423 les/c 71449/71450 71591/71592/71425) [7,70,9] r=1 lpr=71592 pi=71423-71591/12 crt=71424'1 lcod 0'0 inactive NOTIFY] handle_peering_event: epoch_sent: 71592 epoch_requested: 71592 MQuery from 7 query_epoch 71592 query: query(info 0'0) 2014-12-08 18:14:01.115116 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[218.38f( empty local-les=71457 n=0 ec=71423 les/c 71585/71585 71591/71592/71592) [70,23,25] r=0 lpr=71592 pi=71456-71591/5 crt=0'0 mlcod 0'0 peering] update_heartbeat_peers 23,25,70 unchanged 2014-12-08 18:14:01.115126 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[218.38f( empty local-les=71457 n=0 ec=71423 les/c 71585/71585 71591/71592/71592) [70,23,25] r=0 lpr=71592 pi=71456-71591/5 crt=0'0 mlcod 0'0 peering] state: last_epoch_started moved forward, rebuilding prior 2014-12-08 18:14:01.115124 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[218.73e( v 71581'2 (0'0,71581'2] local-les=71449 n=2 ec=71423 les/c 71590/71590 71591/71592/71425) [7,70,9] r=1 lpr=71592 pi=71423-71591/12 crt=71424'1 lcod 0'0 inactive NOTIFY] sending info 2014-12-08 18:14:01.115135 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[218.38f( empty local-les=71457 n=0 ec=71423 les/c 71585/71585 71591/71592/71592) [70,23,25] r=0 lpr=71592 pi=71456-71591/5 crt=0'0 mlcod 0'0 peering] PriorSet: build_prior interval(71591-71591 up [70,23,25](70) acting [23,25,58](23)) 2014-12-08 18:14:01.115144 7f28fcb46700 10 log is not dirty 2014-12-08 18:14:01.115143 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[218.38f( empty local-les=71457 n=0 ec=71423 les/c 71585/71585 71591/71592/71592) [70,23,25] r=0 lpr=71592 pi=71456-71591/5 crt=0'0 mlcod 0'0 peering] PriorSet: build_prior interval(71587-71590 up [23,25](23) acting [23,25](23) maybe_went_rw) 2014-12-08 18:14:01.115152 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[218.38f( empty local-les=71457 n=0 ec=71423 les/c 71585/71585 71591/71592/71592) [70,23,25] r=0 lpr=71592 pi=71456-71591/5 crt=0'0 mlcod 0'0 peering] PriorSet: build_prior interval(71584-71586 up [58,23,25](58) acting [58,23,25](58) maybe_went_rw) 2014-12-08 18:14:01.115151 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[216.ee( empty local-les=71437 n=0 ec=70987 les/c 71437/71437 71591/71592/70989) [9,8,70] r=2 lpr=71592 pi=71401-71591/10 crt=0'0 inactive NOTIFY] handle_peering_event: epoch_sent: 71592 epoch_requested: 71592 MQuery from 9 query_epoch 71592 query: query(info 0'0) 2014-12-08 18:14:01.115163 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[218.38f( empty local-les=71457 n=0 ec=71423 les/c 71585/71585 71591/71592/71592) [70,23,25] r=0 lpr=71592 pi=71456-71591/5 crt=0'0 mlcod 0'0 peering] PriorSet: build_prior interval(71582-71583 up [23,25](23) acting [23,25](23) maybe_went_rw) 2014-12-08 18:14:01.115165 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[216.ee( empty local-les=71437 n=0 ec=70987 les/c 71590/71590 71591/71592/70989) [9,8,70] r=2 lpr=71592 pi=71401-71591/10 crt=0'0 inactive NOTIFY] sending info 2014-12-08 18:14:01.115174 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[218.38f( empty local-les=71457 n=0 ec=71423 les/c 71585/71585 71591/71592/71592) [70,23,25] r=0 lpr=71592 pi=71456-71591/5 crt=0'0 mlcod 0'0 peering] PriorSet: build_prior final: probe 23,25,58,70 down blocked_by {} 2014-12-08 18:14:01.115184 7f28fcb46700 10 log is not dirty -- 2014-12-08 18:14:01.119719 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] handle_peering_event: epoch_sent: 71592 epoch_requested: 71592 MNotifyRec from 58 notify: (query_epoch:71592, epoch_sent:71592, info:19.6e( empty local-les=0 n=0 ec=21298 les/c 51101/50495 71591/71591/71591)) features: 0x3fffffffffff 2014-12-08 18:14:01.119730 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[217.399( v 71377'57 (0'0,71377'57] local-les=71445 n=12 ec=71094 les/c 71590/71583 71591/71592/71592) [70,25,58] r=0 lpr=71592 pi=71444-71591/6 crt=71377'54 lcod 0'0 mlcod 0'0 peering] state: osd.58 has no missing, identical log 2014-12-08 18:14:01.119734 7f28fc345700 7 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] state: handle_pg_notify from osd.58 2014-12-08 18:14:01.119744 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[217.399( v 71377'57 (0'0,71377'57] local-les=71445 n=12 ec=71094 les/c 71590/71583 71591/71592/71592) [70,25,58] r=0 lpr=71592 pi=71444-71591/6 crt=71377'54 lcod 0'0 mlcod 0'0 peering] state: still need up_thru update before going active 2014-12-08 18:14:01.119746 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] state: pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] got dup osd.58 info 19.6e( empty local-les=0 n=0 ec=21298 les/c 51101/50495 71591/71591/71591), identical to ours 2014-12-08 18:14:01.119765 7f28fc345700 10 log is not dirty 2014-12-08 18:14:01.119760 7f28fcb46700 5 osd.70 pg_epoch: 71592 pg[217.399( v 71377'57 (0'0,71377'57] local-les=71445 n=12 ec=71094 les/c 71590/71583 71591/71592/71592) [70,25,58] r=0 lpr=71592 pi=71444-71591/6 crt=71377'54 lcod 0'0 mlcod 0'0 peering] exit Started/Primary/Peering/GetMissing 0.000059 0 0.000000 2014-12-08 18:14:01.119770 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] handle_peering_event: epoch_sent: 71592 epoch_requested: 71592 MNotifyRec from 76 notify: (query_epoch:71592, epoch_sent:71592, info:19.6e( empty local-les=0 n=0 ec=21298 les/c 51101/50495 71591/71591/71591)) features: 0x3fffffffffff 2014-12-08 18:14:01.119779 7f28fc345700 7 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] state: handle_pg_notify from osd.76 2014-12-08 18:14:01.119777 7f28fcb46700 15 osd.70 pg_epoch: 71592 pg[217.399( v 71377'57 (0'0,71377'57] local-les=71445 n=12 ec=71094 les/c 71590/71583 71591/71592/71592) [70,25,58] r=0 lpr=71592 pi=71444-71591/6 crt=71377'54 lcod 0'0 mlcod 0'0 peering] publish_stats_to_osd 71592:5116 2014-12-08 18:14:01.119789 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] state: pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] got dup osd.76 info 19.6e( empty local-les=0 n=0 ec=21298 les/c 51101/50495 71591/71591/71591), identical to ours 2014-12-08 18:14:01.119804 7f28fc345700 10 log is not dirty 2014-12-08 18:14:01.119794 7f28fcb46700 5 osd.70 pg_epoch: 71592 pg[217.399( v 71377'57 (0'0,71377'57] local-les=71445 n=12 ec=71094 les/c 71590/71583 71591/71592/71592) [70,25,58] r=0 lpr=71592 pi=71444-71591/6 crt=71377'54 lcod 0'0 mlcod 0'0 peering] enter Started/Primary/Peering/WaitUpThru 2014-12-08 18:14:01.119810 7f28fcb46700 10 log is not dirty 2014-12-08 18:14:01.119807 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[218.6c7( v 71581'2 (0'0,71581'2] local-les=71433 n=2 ec=71423 les/c 71590/71583 71591/71592/71592) [70,10,18] r=0 lpr=71592 pi=71432-71591/6 crt=0'0 lcod 0'0 mlcod 0'0 peering] handle_peering_event: epoch_sent: 71592 epoch_requested: 71592 MNotifyRec from 76 notify: (query_epoch:71592, epoch_sent:71592, info:218.6c7( v 71581'2 lc 0'0 (0'0,71581'2] local-les=71590 n=2 ec=71423 les/c 71590/71583 71591/71592/71592)) features: 0x3fffffffffff 2014-12-08 18:14:01.119823 7f2904b56700 10 osd.70 71592 do_waiters -- start 2014-12-08 18:14:01.119827 7f2904b56700 10 osd.70 71592 do_waiters -- finish 2014-12-08 18:14:01.119829 7f2904b56700 20 osd.70 71592 _dispatch 0x12cec940 pg_info(1 pgs e71592:217.86) v4 2014-12-08 18:14:01.119821 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[218.6c7( v 71581'2 (0'0,71581'2] local-les=71433 n=2 ec=71423 les/c 71590/71583 71591/71592/71592) [70,10,18] r=0 lpr=71592 pi=71432-71591/6 crt=0'0 lcod 0'0 mlcod 0'0 peering] got osd.76 218.6c7( v 71581'2 lc 0'0 (0'0,71581'2] local-les=71590 n=2 ec=71423 les/c 71590/71583 71591/71592/71592) 2014-12-08 18:14:01.119816 7f290a83a700 10 osd.70 pg_epoch: 71592 pg[19.57( v 71397'43972637 (51252'43969636,71397'43972637] local-les=71592 n=872 ec=21298 les/c 71590/71590 71591/71591/71507) [23,14,70] r=2 lpr=71591 pi=69651-71590/65 crt=71397'43972635 lcod 0'0 inactive] _activate_committed 71592 telling primary 2014-12-08 18:14:01.119844 7f290a83a700 20 osd.70 71592 share_map_peer 0x1268b760 don't know epoch, doing nothing 2014-12-08 18:14:01.119837 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[218.6c7( v 71581'2 (0'0,71581'2] local-les=71433 n=2 ec=71423 les/c 71590/71583 71591/71592/71592) [70,10,18] r=0 lpr=71592 pi=71432-71591/6 crt=0'0 lcod 0'0 mlcod 0'0 peering] osd.76 has stray content: 218.6c7( v 71581'2 lc 0'0 (0'0,71581'2] local-les=71590 n=2 ec=71423 les/c 71590/71583 71591/71592/71592) 2014-12-08 18:14:01.119839 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[218.39f( empty local-les=71592 n=0 ec=71423 les/c 71585/71585 71591/71591/71591) [70,13,16] r=0 lpr=71591 pi=71444-71590/4 crt=0'0 mlcod 0'0 inactive] handle_peering_event: epoch_sent: 71592 epoch_requested: 71592 MNotifyRec from 13 notify: (query_epoch:71592, epoch_sent:71592, info:218.39f( empty local-les=71585 n=0 ec=71423 les/c 71585/71585 71591/71591/71591)) features: 0x3fffffffffff 2014-12-08 18:14:01.119852 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[218.6c7( v 71581'2 (0'0,71581'2] local-les=71433 n=2 ec=71423 les/c 71590/71583 71591/71592/71592) [70,10,18] r=0 lpr=71592 pi=71432-71591/6 crt=0'0 lcod 0'0 mlcod 0'0 peering] update_heartbeat_peers 10,18,70 -> 10,18,70,76 2014-12-08 18:14:01.119865 7f28fc345700 20 osd.70 71592 need_heartbeat_peer_update 2014-12-08 18:14:01.119871 7f2904b56700 7 osd.70 71592 handle_pg_info pg_info(1 pgs e71592:217.86) v4 from osd.63 2014-12-08 18:14:01.119877 7f2904b56700 15 osd.70 71592 require_same_or_newer_map 71592 (i am 71592) 0x12cec940 2014-12-08 18:14:01.119870 7f290a83a700 10 osd.70 pg_epoch: 71592 pg[19.57( v 71397'43972637 (51252'43969636,71397'43972637] local-les=71592 n=872 ec=21298 les/c 71590/71590 71591/71591/71507) [23,14,70] r=2 lpr=71591 pi=69651-71590/65 luod=0'0 crt=71397'43972635 lcod 0'0 active] flushed 2014-12-08 18:14:01.119869 7f28fc345700 20 osd.70 pg_epoch: 71592 pg[218.6c7( v 71581'2 (0'0,71581'2] local-les=71433 n=2 ec=71423 les/c 71590/71583 71591/71592/71592) [70,10,18] r=0 lpr=71592 pi=71432-71591/6 crt=0'0 lcod 0'0 mlcod 0'0 peering] state: Adding osd: 76 features: 3fffffffffff 2014-12-08 18:14:01.119868 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[218.39f( empty local-les=71592 n=0 ec=71423 les/c 71585/71585 71591/71591/71591) [70,13,16] r=0 lpr=71591 pi=71444-71590/4 crt=0'0 mlcod 0'0 inactive] state: Active: got notify from 13, already have info from that osd, ignoring 2014-12-08 18:14:01.119892 7f28fc345700 10 log is not dirty -- 2014-12-08 18:14:01.123843 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] handle_peering_event: epoch_sent: 71592 epoch_requested: 71592 MNotifyRec from 16 notify: (query_epoch:71592, epoch_sent:71592, info:19.6e( empty local-les=0 n=0 ec=21298 les/c 51101/50495 71591/71591/71591)) features: 0x3fffffffffff 2014-12-08 18:14:01.123858 7f28fc345700 7 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] state: handle_pg_notify from osd.16 2014-12-08 18:14:01.123871 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] state: pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] got dup osd.16 info 19.6e( empty local-les=0 n=0 ec=21298 les/c 51101/50495 71591/71591/71591), identical to ours 2014-12-08 18:14:01.123902 7f28fc345700 10 log is not dirty 2014-12-08 18:14:01.123917 7f2904b56700 7 osd.70 71592 handle_pg_info pg_info(1 pgs e71592:218.5e) v4 from osd.16 2014-12-08 18:14:01.123923 7f2904b56700 15 osd.70 71592 require_same_or_newer_map 71592 (i am 71592) 0x12ca7a20 2014-12-08 18:14:01.123912 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[217.2fd( v 71377'4110 (71377'1109,71377'4110] local-les=71592 n=14 ec=71094 les/c 71589/71589 71591/71591/71591) [70,10,19] r=0 lpr=71591 pi=71444-71590/5 crt=71377'4107 lcod 0'0 mlcod 0'0 inactive] handle_peering_event: epoch_sent: 71592 epoch_requested: 71592 MNotifyRec from 19 notify: (query_epoch:71592, epoch_sent:71592, info:217.2fd( v 71377'4110 (71377'1109,71377'4110] local-les=71589 n=14 ec=71094 les/c 71589/71589 71591/71591/71591)) features: 0x3fffffffffff 2014-12-08 18:14:01.123931 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[217.2fd( v 71377'4110 (71377'1109,71377'4110] local-les=71592 n=14 ec=71094 les/c 71589/71589 71591/71591/71591) [70,10,19] r=0 lpr=71591 pi=71444-71590/5 crt=71377'4107 lcod 0'0 mlcod 0'0 inactive] state: Active: got notify from 19, already have info from that osd, ignoring 2014-12-08 18:14:01.123940 7f28fc345700 10 log is not dirty 2014-12-08 18:14:01.123945 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[216.3( v 70990'236 (0'0,70990'236] local-les=71592 n=109 ec=70987 les/c 71585/71583 71591/71591/71591) [70,16,56] r=0 lpr=71591 pi=71432-71590/4 crt=70990'236 lcod 0'0 mlcod 0'0 inactive] handle_peering_event: epoch_sent: 71592 epoch_requested: 71592 MNotifyRec from 56 notify: (query_epoch:71592, epoch_sent:71592, info:216.3( v 70990'236 (0'0,70990'236] local-les=71585 n=109 ec=70987 les/c 71585/71583 71591/71591/71591)) features: 0x3fffffffffff 2014-12-08 18:14:01.123961 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[216.3( v 70990'236 (0'0,70990'236] local-les=71592 n=109 ec=70987 les/c 71585/71583 71591/71591/71591) [70,16,56] r=0 lpr=71591 pi=71432-71590/4 crt=70990'236 lcod 0'0 mlcod 0'0 inactive] state: Active: got notify from 56, already have info from that osd, ignoring 2014-12-08 18:14:01.123969 7f2904b56700 10 osd.70 71592 do_waiters -- start 2014-12-08 18:14:01.123970 7f28fc345700 10 log is not dirty 2014-12-08 18:14:01.123977 7f2904b56700 10 osd.70 71592 do_waiters -- finish 2014-12-08 18:14:01.123982 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[217.370( v 71377'12257 (71377'9256,71377'12257] local-les=71592 n=16 ec=71094 les/c 71588/71588 71591/71591/71591) [70,10,56] r=0 lpr=71591 pi=71434-71590/5 crt=71377'12255 lcod 0'0 mlcod 0'0 inactive] handle_peering_event: epoch_sent: 71592 epoch_requested: 71592 MNotifyRec from 56 notify: (query_epoch:71592, epoch_sent:71592, info:217.370( v 71377'12257 (71377'9256,71377'12257] local-les=71588 n=16 ec=71094 les/c 71588/71588 71591/71591/71591)) features: 0x3fffffffffff 2014-12-08 18:14:01.123998 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[217.370( v 71377'12257 (71377'9256,71377'12257] local-les=71592 n=16 ec=71094 les/c 71588/71588 71591/71591/71591) [70,10,56] r=0 lpr=71591 pi=71434-71590/5 crt=71377'12255 lcod 0'0 mlcod 0'0 inactive] state: Active: got notify from 56, already have info from that osd, ignoring 2014-12-08 18:14:01.124007 7f28fc345700 10 log is not dirty 2014-12-08 18:14:01.124013 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[218.38f( empty local-les=71457 n=0 ec=71423 les/c 71585/71585 71591/71592/71592) [70,23,25] r=0 lpr=71592 pi=71456-71591/5 crt=0'0 mlcod 0'0 peering] handle_peering_event: epoch_sent: 71592 epoch_requested: 71592 MNotifyRec from 23 notify: (query_epoch:71592, epoch_sent:71592, info:218.38f( empty local-les=71585 n=0 ec=71423 les/c 71585/71585 71591/71592/71592)) features: 0x3fffffffffff 2014-12-08 18:14:01.124028 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[218.38f( empty local-les=71457 n=0 ec=71423 les/c 71585/71585 71591/71592/71592) [70,23,25] r=0 lpr=71592 pi=71456-71591/5 crt=0'0 mlcod 0'0 peering] got osd.23 218.38f( empty local-les=71585 n=0 ec=71423 les/c 71585/71585 71591/71592/71592) 2014-12-08 18:14:01.124042 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[218.38f( empty local-les=71457 n=0 ec=71423 les/c 71585/71585 71591/71592/71592) [70,23,25] r=0 lpr=71592 pi=71456-71591/5 crt=0'0 mlcod 0'0 peering] update_heartbeat_peers 23,25,70 unchanged 2014-12-08 18:14:01.124050 7f28fc345700 20 osd.70 pg_epoch: 71592 pg[218.38f( empty local-les=71457 n=0 ec=71423 les/c 71585/71585 71591/71592/71592) [70,23,25] r=0 lpr=71592 pi=71456-71591/5 crt=0'0 mlcod 0'0 peering] state: Adding osd: 23 features: 3fffffffffff 2014-12-08 18:14:01.124060 7f28fc345700 10 log is not dirty 2014-12-08 18:14:01.124066 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[218.3a1( v 71581'1 (0'0,71581'1] local-les=71451 n=1 ec=71423 les/c 71585/71583 71591/71591/71591) [70,23,69] r=0 lpr=71591 pi=71444-71590/4 crt=0'0 lcod 0'0 mlcod 0'0 peering] handle_peering_event: epoch_sent: 71592 epoch_requested: 71592 MNotifyRec from 23 notify: (query_epoch:71592, epoch_sent:71592, info:218.3a1( v 71581'1 (0'0,71581'1] local-les=71585 n=1 ec=71423 les/c 71585/71583 71591/71591/71591)) features: 0x3fffffffffff -- 2014-12-08 18:14:01.445398 7f28fbb44700 10 osd.70 71592 dequeue_op 0x4481500 prio 63 cost 4096 latency 0.000413 osd_op(client.3273605.0:6509137 rbd_data.31f5466c26f78a.0000000000000a17 [set-alloc-hint object_size 8388608 write_size 8388608,write 0~4096] 19.8b7d6b6e RETRY=104 ack+ondisk+retry+write e71592) v4 pg pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] 2014-12-08 18:14:01.445497 7f28dabc2700 20 osd.70 71592 should_share_map client.3273605 172.16.5.108:0/1015139 71592 2014-12-08 18:14:01.445485 7f28fbb44700 20 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] op_has_sufficient_caps pool=19 (volumes ) owner=0 need_read_cap=0 need_write_cap=1 need_class_read_cap=0 need_class_write_cap=0 -> yes 2014-12-08 18:14:01.445505 7f28fbb44700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] can_handle_while_inactive: 0x4481500 2014-12-08 18:14:01.445519 7f28fbb44700 10 osd.70 71592 dequeue_op 0x4481500 finish 2014-12-08 18:14:01.445519 7f28dabc2700 15 osd.70 71592 enqueue_op 0x447ec00 prio 63 cost 4096 latency 0.000162 osd_op(client.3273605.0:6509138 rbd_data.31f5466c26f78a.0000000000000a17 [set-alloc-hint object_size 8388608 write_size 8388608,write 4096~4096] 19.8b7d6b6e RETRY=104 ack+ondisk+retry+write e71592) v4 2014-12-08 18:14:01.445590 7f28f933f700 10 osd.70 71592 dequeue_op 0x447ec00 prio 63 cost 4096 latency 0.000232 osd_op(client.3273605.0:6509138 rbd_data.31f5466c26f78a.0000000000000a17 [set-alloc-hint object_size 8388608 write_size 8388608,write 4096~4096] 19.8b7d6b6e RETRY=104 ack+ondisk+retry+write e71592) v4 pg pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] 2014-12-08 18:14:01.445643 7f28f933f700 20 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] op_has_sufficient_caps pool=19 (volumes ) owner=0 need_read_cap=0 need_write_cap=1 need_class_read_cap=0 need_class_write_cap=0 -> yes 2014-12-08 18:14:01.445656 7f28dabc2700 20 osd.70 71592 should_share_map client.3273605 172.16.5.108:0/1015139 71592 2014-12-08 18:14:01.445658 7f28f933f700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] can_handle_while_inactive: 0x447ec00 2014-12-08 18:14:01.445670 7f28f933f700 10 osd.70 71592 dequeue_op 0x447ec00 finish 2014-12-08 18:14:01.445677 7f28dabc2700 15 osd.70 71592 enqueue_op 0x4481700 prio 63 cost 24576 latency 0.000130 osd_op(client.3273605.0:6509139 rbd_data.31f5466c26f78a.0000000000000a17 [set-alloc-hint object_size 8388608 write_size 8388608,write 8192~24576] 19.8b7d6b6e RETRY=104 ack+ondisk+retry+write e71592) v4 2014-12-08 18:14:01.445741 7f28fbb44700 10 osd.70 71592 dequeue_op 0x4481700 prio 63 cost 24576 latency 0.000194 osd_op(client.3273605.0:6509139 rbd_data.31f5466c26f78a.0000000000000a17 [set-alloc-hint object_size 8388608 write_size 8388608,write 8192~24576] 19.8b7d6b6e RETRY=104 ack+ondisk+retry+write e71592) v4 pg pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] 2014-12-08 18:14:01.445785 7f28fbb44700 20 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] op_has_sufficient_caps pool=19 (volumes ) owner=0 need_read_cap=0 need_write_cap=1 need_class_read_cap=0 need_class_write_cap=0 -> yes 2014-12-08 18:14:01.445820 7f28dabc2700 20 osd.70 71592 should_share_map client.3273605 172.16.5.108:0/1015139 71592 2014-12-08 18:14:01.445854 7f28dabc2700 15 osd.70 71592 enqueue_op 0x447fb00 prio 63 cost 4096 latency 0.000149 osd_op(client.3273605.0:6509140 rbd_data.31f5466c26f78a.0000000000000a17 [set-alloc-hint object_size 8388608 write_size 8388608,write 32768~4096] 19.8b7d6b6e RETRY=104 ack+ondisk+retry+write e71592) v4 2014-12-08 18:14:01.445818 7f28fbb44700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] can_handle_while_inactive: 0x4481700 2014-12-08 18:14:01.445934 7f28fbb44700 10 osd.70 71592 dequeue_op 0x4481700 finish 2014-12-08 18:14:01.445998 7f28f933f700 10 osd.70 71592 dequeue_op 0x447fb00 prio 63 cost 4096 latency 0.000292 osd_op(client.3273605.0:6509140 rbd_data.31f5466c26f78a.0000000000000a17 [set-alloc-hint object_size 8388608 write_size 8388608,write 32768~4096] 19.8b7d6b6e RETRY=104 ack+ondisk+retry+write e71592) v4 pg pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] 2014-12-08 18:14:01.446035 7f28dabc2700 20 osd.70 71592 should_share_map client.3273605 172.16.5.108:0/1015139 71592 2014-12-08 18:14:01.446045 7f28f933f700 20 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] op_has_sufficient_caps pool=19 (volumes ) owner=0 need_read_cap=0 need_write_cap=1 need_class_read_cap=0 need_class_write_cap=0 -> yes 2014-12-08 18:14:01.446055 7f28dabc2700 15 osd.70 71592 enqueue_op 0x4481800 prio 63 cost 4096 latency 0.000160 osd_op(client.3273605.0:6518766 rbd_data.31f5466c26f78a.0000000000000a17 [set-alloc-hint object_size 8388608 write_size 8388608,write 36864~4096] 19.8b7d6b6e RETRY=45 ack+ondisk+retry+write e71592) v4 2014-12-08 18:14:01.446058 7f28f933f700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] can_handle_while_inactive: 0x447fb00 2014-12-08 18:14:01.446070 7f28f933f700 10 osd.70 71592 dequeue_op 0x447fb00 finish 2014-12-08 18:14:01.446078 7f28f933f700 10 osd.70 71592 dequeue_op 0x4481800 prio 63 cost 4096 latency 0.000182 osd_op(client.3273605.0:6518766 rbd_data.31f5466c26f78a.0000000000000a17 [set-alloc-hint object_size 8388608 write_size 8388608,write 36864~4096] 19.8b7d6b6e RETRY=45 ack+ondisk+retry+write e71592) v4 pg pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] 2014-12-08 18:14:01.446100 7f28f933f700 20 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] op_has_sufficient_caps pool=19 (volumes ) owner=0 need_read_cap=0 need_write_cap=1 need_class_read_cap=0 need_class_write_cap=0 -> yes 2014-12-08 18:14:01.446112 7f28f933f700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] can_handle_while_inactive: 0x4481800 2014-12-08 18:14:01.446120 7f28f933f700 10 osd.70 71592 dequeue_op 0x4481800 finish 2014-12-08 18:14:01.529894 7f2904b56700 10 osd.70 71592 do_waiters -- start 2014-12-08 18:14:01.529899 7f2904b56700 10 osd.70 71592 do_waiters -- finish 2014-12-08 18:14:01.529901 7f2904b56700 20 osd.70 71592 _dispatch 0x12f8a760 pg_info(1 pgs e71592:218.428(3)) v4 2014-12-08 18:14:01.529934 7f2904b56700 7 osd.70 71592 handle_pg_info pg_info(1 pgs e71592:218.428(3)) v4 from osd.75 2014-12-08 18:14:01.529938 7f2904b56700 15 osd.70 71592 require_same_or_newer_map 71592 (i am 71592) 0x12f8a760 2014-12-08 18:14:01.529999 7f2904b56700 10 osd.70 71592 do_waiters -- start 2014-12-08 18:14:01.530002 7f2904b56700 10 osd.70 71592 do_waiters -- finish 2014-12-08 18:14:01.529995 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[218.428( v 71581'2 (0'0,71581'2] local-les=71457 n=2 ec=71423 les/c 71585/71583 71591/71591/71587) [75,70,7] r=1 lpr=71591 pi=71423-71590/11 crt=0'0 lcod 0'0 inactive NOTIFY] handle_peering_event: epoch_sent: 71592 epoch_requested: 71592 MInfoRec from 75 info: 218.428( v 71581'2 (0'0,71581'2] local-les=71592 n=2 ec=71423 les/c 71585/71583 71591/71591/71587) 2014-12-08 18:14:01.530017 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[218.428( v 71581'2 (0'0,71581'2] local-les=71457 n=2 ec=71423 les/c 71585/71583 71591/71591/71587) [75,70,7] r=1 lpr=71591 pi=71423-71590/11 crt=0'0 lcod 0'0 inactive NOTIFY] state: got info from osd.75 218.428( v 71581'2 (0'0,71581'2] local-les=71592 n=2 ec=71423 les/c 71585/71583 71591/71591/71587) 2014-12-08 18:14:01.530031 7f28fc345700 5 osd.70 pg_epoch: 71592 pg[218.428( v 71581'2 (0'0,71581'2] local-les=71457 n=2 ec=71423 les/c 71585/71583 71591/71591/71587) [75,70,7] r=1 lpr=71591 pi=71423-71590/11 crt=0'0 lcod 0'0 inactive NOTIFY] exit Started/Stray 0.511360 5 0.000179 2014-12-08 18:14:01.530043 7f28fc345700 5 osd.70 pg_epoch: 71592 pg[218.428( v 71581'2 (0'0,71581'2] local-les=71457 n=2 ec=71423 les/c 71585/71583 71591/71591/71587) [75,70,7] r=1 lpr=71591 pi=71423-71590/11 crt=0'0 lcod 0'0 inactive NOTIFY] enter Started/ReplicaActive 2014-12-08 18:14:01.530054 7f28fc345700 5 osd.70 pg_epoch: 71592 pg[218.428( v 71581'2 (0'0,71581'2] local-les=71457 n=2 ec=71423 les/c 71585/71583 71591/71591/71587) [75,70,7] r=1 lpr=71591 pi=71423-71590/11 crt=0'0 lcod 0'0 inactive NOTIFY] enter Started/ReplicaActive/RepNotRecovering 2014-12-08 18:14:01.530065 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[218.428( v 71581'2 (0'0,71581'2] local-les=71457 n=2 ec=71423 les/c 71585/71583 71591/71591/71587) [75,70,7] r=1 lpr=71591 pi=71423-71590/11 crt=0'0 lcod 0'0 inactive NOTIFY] state: In ReplicaActive, about to call activate 2014-12-08 18:14:01.530075 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[218.428( v 71581'2 (0'0,71581'2] local-les=71592 n=2 ec=71423 les/c 71585/71583 71591/71591/71587) [75,70,7] r=1 lpr=71591 pi=71423-71590/11 crt=0'0 lcod 0'0 inactive] activate - no missing, moving last_complete 71581'2 -> 71581'2 2014-12-08 18:14:01.530084 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[218.428( v 71581'2 (0'0,71581'2] local-les=71592 n=2 ec=71423 les/c 71585/71583 71591/71591/71587) [75,70,7] r=1 lpr=71591 pi=71423-71590/11 crt=0'0 lcod 0'0 inactive] state: Activate Finished 2014-12-08 18:14:01.530124 7f28fc345700 10 log is not dirty 2014-12-08 18:14:01.530553 7f2904b56700 10 osd.70 71592 do_waiters -- start 2014-12-08 18:14:01.530556 7f2904b56700 10 osd.70 71592 do_waiters -- finish 2014-12-08 18:14:01.530559 7f2904b56700 20 osd.70 71592 _dispatch 0x12c481c0 osd_map(71582..71591 src has 50399..71591) v3 -- 2014-12-08 18:14:01.879281 7f2904b56700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] null 2014-12-08 18:14:01.879292 7f2904b56700 10 osd.70 pg_epoch: 71592 pg[19.6d( v 71397'33614009 (70963'33611008,71397'33614009] local-les=71592 n=896 ec=21298 les/c 71583/71583 71591/71591/71169) [4,70,69] r=1 lpr=71591 pi=71156-71590/27 luod=0'0 crt=71397'33614007 lcod 0'0 active] null 2014-12-08 18:14:01.879293 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[19.5a( v 71397'49665417 (50495'49662416,71397'49665417] local-les=71592 n=896 ec=21298 les/c 71592/71592 71591/71591/71591) [70,20,21] r=0 lpr=71591 crt=71397'49665415 lcod 0'0 mlcod 0'0 active+clean] handle_advance_map [70,20,21]/[70,20,21] -- 70/70 2014-12-08 18:14:01.879309 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:14:01.879299 7f28fc345700 10 osd.70 pg_epoch: 71593 pg[216.243( v 71097'212 (0'0,71097'212] local-les=71593 n=101 ec=70987 les/c 71585/71583 71591/71592/71592) [70,23,6] r=0 lpr=71592 pi=71444-71591/5 crt=71097'210 lcod 0'0 mlcod 0'0 inactive] activate peer osd.6 216.243( v 71097'212 (0'0,71097'212] local-les=71585 n=101 ec=70987 les/c 71585/71583 71591/71592/71592) 2014-12-08 18:14:01.879305 7f2904b56700 10 osd.70 pg_epoch: 71592 pg[218.6c7( v 71581'2 (0'0,71581'2] local-les=71433 n=2 ec=71423 les/c 71590/71583 71591/71592/71592) [70,10,18] r=0 lpr=71592 pi=71432-71591/6 crt=0'0 lcod 0'0 mlcod 0'0 peering] null 2014-12-08 18:14:01.879314 7f28fcb46700 10 osd.70 pg_epoch: 71593 pg[19.5a( v 71397'49665417 (50495'49662416,71397'49665417] local-les=71592 n=896 ec=21298 les/c 71592/71592 71591/71591/71591) [70,20,21] r=0 lpr=71591 crt=71397'49665415 lcod 0'0 mlcod 0'0 active+clean] state: Active advmap 2014-12-08 18:14:01.879317 7f28fc345700 10 osd.70 pg_epoch: 71593 pg[216.243( v 71097'212 (0'0,71097'212] local-les=71593 n=101 ec=70987 les/c 71585/71583 71591/71592/71592) [70,23,6] r=0 lpr=71592 pi=71444-71591/5 crt=71097'210 lcod 0'0 mlcod 0'0 inactive] activate peer osd.6 is up to date, queueing in pending_activators 2014-12-08 18:14:01.879323 7f2904b56700 10 osd.70 pg_epoch: 71592 pg[218.6c4( v 71581'3 (0'0,71581'3] local-les=71437 n=3 ec=71423 les/c 71590/71583 71591/71592/71592) [70,76,13] r=0 lpr=71592 pi=71432-71591/6 crt=71580'1 lcod 0'0 mlcod 0'0 peering] null 2014-12-08 18:14:01.879330 7f28fcb46700 10 osd.70 pg_epoch: 71593 pg[19.5a( v 71397'49665417 (50495'49662416,71397'49665417] local-les=71592 n=896 ec=21298 les/c 71592/71592 71591/71591/71591) [70,20,21] r=0 lpr=71591 crt=71397'49665415 lcod 0'0 mlcod 0'0 active+clean] state: Started advmap 2014-12-08 18:14:01.879337 7f2904b56700 10 osd.70 pg_epoch: 71592 pg[218.562( v 71581'3 (0'0,71581'3] local-les=71437 n=3 ec=71423 les/c 71590/71583 71591/71592/71425) [9,18,70] r=2 lpr=71592 pi=71423-71591/10 crt=71580'1 lcod 0'0 inactive NOTIFY] null 2014-12-08 18:14:01.879340 7f28fcb46700 10 osd.70 pg_epoch: 71593 pg[19.5a( v 71397'49665417 (50495'49662416,71397'49665417] local-les=71592 n=896 ec=21298 les/c 71592/71592 71591/71591/71591) [70,20,21] r=0 lpr=71591 crt=71397'49665415 lcod 0'0 mlcod 0'0 active+clean] check_recovery_sources no source osds () went down 2014-12-08 18:14:01.879337 7f28fc345700 10 osd.70 pg_epoch: 71593 pg[216.243( v 71097'212 (0'0,71097'212] local-les=71593 n=101 ec=70987 les/c 71585/71583 71591/71592/71592) [70,23,6] r=0 lpr=71592 pi=71444-71591/5 crt=71097'210 lcod 0'0 mlcod 0'0 inactive] activate peer osd.6 216.243( v 71097'212 (0'0,71097'212] local-les=71585 n=101 ec=70987 les/c 71585/71583 71591/71592/71592) uptodate 2014-12-08 18:14:01.879355 7f28fcb46700 10 osd.70 pg_epoch: 71593 pg[19.5a( v 71397'49665417 (50495'49662416,71397'49665417] local-les=71592 n=896 ec=21298 les/c 71592/71592 71591/71591/71591) [70,20,21] r=0 lpr=71591 crt=71397'49665415 lcod 0'0 mlcod 0'0 active+clean] handle_activate_map 2014-12-08 18:14:01.879354 7f2904b56700 10 osd.70 pg_epoch: 71592 pg[218.5e( empty local-les=71592 n=0 ec=71423 les/c 71592/71592 71591/71591/71425) [16,22,70] r=2 lpr=71591 pi=71423-71590/8 crt=0'0 active] null 2014-12-08 18:14:01.879353 7f28fc345700 10 osd.70 pg_epoch: 71593 pg[216.243( v 71097'212 (0'0,71097'212] local-les=71593 n=101 ec=70987 les/c 71585/71583 71591/71592/71592) [70,23,6] r=0 lpr=71592 pi=71444-71591/5 crt=71097'210 lcod 0'0 mlcod 0'0 inactive] activate peer osd.23 216.243( v 71097'212 (0'0,71097'212] local-les=71585 n=101 ec=70987 les/c 71585/71583 71591/71592/71592) 2014-12-08 18:14:01.879369 7f28fcb46700 10 osd.70 pg_epoch: 71593 pg[19.5a( v 71397'49665417 (50495'49662416,71397'49665417] local-les=71592 n=896 ec=21298 les/c 71592/71592 71591/71591/71591) [70,20,21] r=0 lpr=71591 crt=71397'49665415 lcod 0'0 mlcod 0'0 active+clean] state: Active: handling ActMap 2014-12-08 18:14:01.879371 7f28fc345700 10 osd.70 pg_epoch: 71593 pg[216.243( v 71097'212 (0'0,71097'212] local-les=71593 n=101 ec=70987 les/c 71585/71583 71591/71592/71592) [70,23,6] r=0 lpr=71592 pi=71444-71591/5 crt=71097'210 lcod 0'0 mlcod 0'0 inactive] activate peer osd.23 is up to date, queueing in pending_activators 2014-12-08 18:14:01.879373 7f2904b56700 10 osd.70 pg_epoch: 71592 pg[218.5f( v 71580'2 (0'0,71580'2] local-les=71437 n=2 ec=71423 les/c 71585/71583 71591/71592/71592) [70,15,24] r=0 lpr=71592 pi=71432-71591/6 crt=0'0 lcod 0'0 mlcod 0'0 peering] null 2014-12-08 18:14:01.879381 7f28fcb46700 7 osd.70 pg_epoch: 71593 pg[19.5a( v 71397'49665417 (50495'49662416,71397'49665417] local-les=71592 n=896 ec=21298 les/c 71592/71592 71591/71591/71591) [70,20,21] r=0 lpr=71591 crt=71397'49665415 lcod 0'0 mlcod 0'0 active+clean] state: handle ActMap primary 2014-12-08 18:14:01.879386 7f28fc345700 10 osd.70 pg_epoch: 71593 pg[216.243( v 71097'212 (0'0,71097'212] local-les=71593 n=101 ec=70987 les/c 71585/71583 71591/71592/71592) [70,23,6] r=0 lpr=71592 pi=71444-71591/5 crt=71097'210 lcod 0'0 mlcod 0'0 inactive] activate peer osd.23 216.243( v 71097'212 (0'0,71097'212] local-les=71585 n=101 ec=70987 les/c 71585/71583 71591/71592/71592) uptodate -- 2014-12-08 18:14:01.883815 7f2904b56700 20 osd.70 71593 19.6e heartbeat_peers 0,4,11,13,16,20,30,31,56,58,59,63,70,76 2014-12-08 18:14:01.883821 7f2904b56700 20 osd.70 71593 19.6d heartbeat_peers 2014-12-08 18:14:01.883822 7f2904b56700 20 osd.70 71593 218.6c7 heartbeat_peers 10,18,70,76 2014-12-08 18:14:01.883825 7f2904b56700 20 osd.70 71593 218.6c4 heartbeat_peers 13,15,70,76 2014-12-08 18:14:01.883828 7f2904b56700 20 osd.70 71593 218.562 heartbeat_peers 2014-12-08 18:14:01.883829 7f2904b56700 20 osd.70 71593 218.5e heartbeat_peers 2014-12-08 18:14:01.883831 7f2904b56700 20 osd.70 71593 218.5f heartbeat_peers 9,15,24,70 2014-12-08 18:14:01.883833 7f2904b56700 20 osd.70 71593 217.235 heartbeat_peers 2014-12-08 18:14:01.883835 7f2904b56700 20 osd.70 71593 218.38e heartbeat_peers 3,5,13,70 2014-12-08 18:14:01.883837 7f2904b56700 20 osd.70 71593 218.38f heartbeat_peers 23,25,58,70 2014-12-08 18:14:01.883840 7f2904b56700 20 osd.70 71593 218.38c heartbeat_peers 2014-12-08 18:14:01.883841 7f2904b56700 20 osd.70 71593 217.51 heartbeat_peers 2014-12-08 18:14:01.883842 7f2904b56700 20 osd.70 71593 216.380 heartbeat_peers 2014-12-08 18:14:01.883844 7f2904b56700 20 osd.70 71593 218.51 heartbeat_peers 2014-12-08 18:14:01.883846 7f2904b56700 20 osd.70 71593 218.380 heartbeat_peers 2014-12-08 18:14:01.883847 7f2904b56700 20 osd.70 71593 218.518 heartbeat_peers 2014-12-08 18:14:01.883848 7f2904b56700 20 osd.70 71593 217.22d heartbeat_peers 2014-12-08 18:14:01.883849 7f2904b56700 20 osd.70 71593 216.384 heartbeat_peers 2014-12-08 18:14:01.883844 7f28fc345700 10 osd.70 71593 send_incremental_map 71592 -> 71593 to 0x12711080 172.16.10.103:6809/23410 2014-12-08 18:14:01.883851 7f2904b56700 20 osd.70 71593 216.22f heartbeat_peers 6,21,70 2014-12-08 18:14:01.883855 7f2904b56700 20 osd.70 71593 218.4b heartbeat_peers -- 2014-12-08 18:14:01.887576 7f2904b56700 20 osd.70 71593 _dispatch 0x12eac940 pg_notify(19.6e(150) epoch 71593) v5 2014-12-08 18:14:01.887572 7f28fcb46700 10 osd.70 pg_epoch: 71593 pg[217.3f7( v 71376'21 (0'0,71376'21] local-les=71434 n=7 ec=71094 les/c 71585/71583 71591/71593/71593) [70,56,68] r=0 lpr=71593 pi=71432-71592/5 crt=71097'19 lcod 0'0 mlcod 0'0 peering] state: querying info from osd.68 2014-12-08 18:14:01.887575 7f28fc345700 10 osd.70 pg_epoch: 71593 pg[217.399( v 71377'57 (0'0,71377'57] local-les=71445 n=12 ec=71094 les/c 71590/71583 71591/71592/71592) [70,25,58] r=0 lpr=71592 pi=71444-71591/6 crt=71377'54 lcod 0'0 mlcod 0'0 peering] state: Started advmap 2014-12-08 18:14:01.887597 7f2903353700 20 osd.70 71593 share_map_peer 0x1264ef20 already has epoch 71593 2014-12-08 18:14:01.887590 7f28fcb46700 15 osd.70 pg_epoch: 71593 pg[217.3f7( v 71376'21 (0'0,71376'21] local-les=71434 n=7 ec=71094 les/c 71585/71583 71591/71593/71593) [70,56,68] r=0 lpr=71593 pi=71432-71592/5 crt=71097'19 lcod 0'0 mlcod 0'0 peering] publish_stats_to_osd 71593:168 2014-12-08 18:14:01.887591 7f28fc345700 10 osd.70 pg_epoch: 71593 pg[217.399( v 71377'57 (0'0,71377'57] local-les=71445 n=12 ec=71094 les/c 71590/71583 71591/71592/71592) [70,25,58] r=0 lpr=71592 pi=71444-71591/6 crt=71377'54 lcod 0'0 mlcod 0'0 peering] check_recovery_sources no source osds () went down 2014-12-08 18:14:01.887609 7f2904b56700 7 osd.70 71593 handle_pg_notify from osd.16 2014-12-08 18:14:01.887612 7f2904b56700 15 osd.70 71593 require_same_or_newer_map 71593 (i am 71593) 0x12eac940 2014-12-08 18:14:01.887605 7f28fcb46700 20 osd.70 pg_epoch: 71593 pg[217.3f7( v 71376'21 (0'0,71376'21] local-les=71434 n=7 ec=71094 les/c 71585/71583 71591/71593/71593) [70,56,68] r=0 lpr=71593 pi=71432-71592/5 crt=71097'19 lcod 0'0 mlcod 0'0 peering] handle_activate_map: Not dirtying info: last_persisted is 71591 while current is 71593 2014-12-08 18:14:01.887628 7f2903353700 10 osd.70 71593 note_peer_epoch osd.11 has 71593 2014-12-08 18:14:01.887619 7f28fcb46700 10 osd.70 pg_epoch: 71593 pg[217.3f7( v 71376'21 (0'0,71376'21] local-les=71434 n=7 ec=71094 les/c 71585/71583 71591/71593/71593) [70,56,68] r=0 lpr=71593 pi=71432-71592/5 crt=71097'19 lcod 0'0 mlcod 0'0 peering] handle_peering_event: epoch_sent: 71593 epoch_requested: 71593 NullEvt 2014-12-08 18:14:01.887615 7f28fc345700 10 osd.70 pg_epoch: 71593 pg[217.399( v 71377'57 (0'0,71377'57] local-les=71445 n=12 ec=71094 les/c 71590/71583 71591/71592/71592) [70,25,58] r=0 lpr=71592 pi=71444-71591/6 crt=71377'54 lcod 0'0 mlcod 0'0 peering] handle_activate_map 2014-12-08 18:14:01.887633 7f2903353700 20 osd.70 71593 share_map_peer 0x12239ce0 already has epoch 71593 2014-12-08 18:14:01.887634 7f2904b56700 10 osd.70 71593 do_waiters -- start 2014-12-08 18:14:01.887637 7f2904b56700 10 osd.70 71593 do_waiters -- finish 2014-12-08 18:14:01.887646 7f2903353700 20 osd.70 71593 share_map_peer 0x12239ce0 already has epoch 71593 2014-12-08 18:14:01.887635 7f28fc345700 7 osd.70 pg_epoch: 71593 pg[217.399( v 71377'57 (0'0,71377'57] local-les=71445 n=12 ec=71094 les/c 71590/71583 71591/71592/71592) [70,25,58] r=0 lpr=71592 pi=71444-71591/6 crt=71377'54 lcod 0'0 mlcod 0'0 peering] state: handle ActMap primary 2014-12-08 18:14:01.887658 7f2903353700 10 osd.70 71593 note_peer_epoch osd.30 has 71593 2014-12-08 18:14:01.887661 7f28fcb46700 10 log is not dirty 2014-12-08 18:14:01.887661 7f2903353700 20 osd.70 71593 share_map_peer 0x127a5020 already has epoch 71593 2014-12-08 18:14:01.887652 7f28fc345700 15 osd.70 pg_epoch: 71593 pg[217.399( v 71377'57 (0'0,71377'57] local-les=71445 n=12 ec=71094 les/c 71590/71583 71591/71592/71592) [70,25,58] r=0 lpr=71592 pi=71444-71591/6 crt=71377'54 lcod 0'0 mlcod 0'0 peering] publish_stats_to_osd 71593:5117 -- 2014-12-08 18:14:01.888283 7f2904b56700 20 osd.70 71593 _dispatch 0x12c6fc20 pg_notify(217.399(28),19.6e(150) epoch 71593) v5 2014-12-08 18:14:01.888271 7f28fc345700 20 osd.70 pg_epoch: 71593 pg[216.ab( v 71097'289 (0'0,71097'289] local-les=71437 n=133 ec=70987 les/c 71585/71583 71591/71592/71168) [20,64,70]/[20,64,13] r=-1 lpr=71592 pi=71183-71591/16 crt=71097'289 lcod 0'0 remapped NOTIFY] set_last_peering_reset 71593 2014-12-08 18:14:01.888278 7f28fcb46700 10 osd.70 pg_epoch: 71593 pg[217.bd( v 71377'4132 (71377'1131,71377'4132] local-les=71592 n=13 ec=71094 les/c 71592/71592 71591/71591/71591) [70,7,25] r=0 lpr=71591 crt=71377'4130 lcod 0'0 mlcod 0'0 active+clean] check_recovery_sources no source osds () went down 2014-12-08 18:14:01.888299 7f2903353700 20 osd.70 71593 share_map_peer 0x12623a20 already has epoch 71593 2014-12-08 18:14:01.888291 7f28fc345700 10 osd.70 pg_epoch: 71593 pg[216.ab( v 71097'289 (0'0,71097'289] local-les=71437 n=133 ec=70987 les/c 71585/71583 71591/71592/71168) [20,64,70]/[20,64,13] r=-1 lpr=71593 pi=71183-71591/16 crt=71097'289 lcod 0'0 remapped NOTIFY] Clearing blocked outgoing recovery messages 2014-12-08 18:14:01.888294 7f28fcb46700 10 osd.70 pg_epoch: 71593 pg[217.bd( v 71377'4132 (71377'1131,71377'4132] local-les=71592 n=13 ec=71094 les/c 71592/71592 71591/71591/71591) [70,7,25] r=0 lpr=71591 crt=71377'4130 lcod 0'0 mlcod 0'0 active+clean] handle_activate_map 2014-12-08 18:14:01.888315 7f2904b56700 7 osd.70 71593 handle_pg_notify from osd.58 2014-12-08 18:14:01.888317 7f2903353700 20 osd.70 71593 share_map_peer 0x1281f600 already has epoch 71593 2014-12-08 18:14:01.888317 7f2904b56700 15 osd.70 71593 require_same_or_newer_map 71593 (i am 71593) 0x12c6fc20 2014-12-08 18:14:01.888308 7f28fcb46700 10 osd.70 pg_epoch: 71593 pg[217.bd( v 71377'4132 (71377'1131,71377'4132] local-les=71592 n=13 ec=71094 les/c 71592/71592 71591/71591/71591) [70,7,25] r=0 lpr=71591 crt=71377'4130 lcod 0'0 mlcod 0'0 active+clean] state: Active: handling ActMap 2014-12-08 18:14:01.888309 7f28fc345700 10 osd.70 pg_epoch: 71593 pg[216.ab( v 71097'289 (0'0,71097'289] local-les=71437 n=133 ec=70987 les/c 71585/71583 71591/71592/71168) [20,64,70]/[20,64,13] r=-1 lpr=71593 pi=71183-71591/16 crt=71097'289 lcod 0'0 remapped NOTIFY] Not blocking outgoing recovery messages 2014-12-08 18:14:01.888323 7f28fcb46700 7 osd.70 pg_epoch: 71593 pg[217.bd( v 71377'4132 (71377'1131,71377'4132] local-les=71592 n=13 ec=71094 les/c 71592/71592 71591/71591/71591) [70,7,25] r=0 lpr=71591 crt=71377'4130 lcod 0'0 mlcod 0'0 active+clean] state: handle ActMap primary 2014-12-08 18:14:01.888325 7f28fc345700 10 osd.70 pg_epoch: 71593 pg[216.ab( v 71097'289 (0'0,71097'289] local-les=71437 n=133 ec=70987 les/c 71585/71583 71591/71592/71168) [20,64,70]/[20,64,13] r=-1 lpr=71593 pi=71183-71591/16 crt=71097'289 lcod 0'0 remapped NOTIFY] state: Reset advmap 2014-12-08 18:14:01.888336 7f2903353700 20 osd.70 71593 share_map_peer 0x1281f600 already has epoch 71593 2014-12-08 18:14:01.888335 7f28fcb46700 15 osd.70 pg_epoch: 71593 pg[217.bd( v 71377'4132 (71377'1131,71377'4132] local-les=71592 n=13 ec=71094 les/c 71592/71592 71591/71591/71591) [70,7,25] r=0 lpr=71591 crt=71377'4130 lcod 0'0 mlcod 0'0 active+clean] publish_stats_to_osd 71593:22936 2014-12-08 18:14:01.888339 7f28fc345700 10 osd.70 pg_epoch: 71593 pg[216.ab( v 71097'289 (0'0,71097'289] local-les=71437 n=133 ec=70987 les/c 71585/71583 71591/71592/71168) [20,64,70]/[20,64,13] r=-1 lpr=71593 pi=71183-71591/16 crt=71097'289 lcod 0'0 remapped NOTIFY] _calc_past_interval_range: already have past intervals back to 71583 2014-12-08 18:14:01.888349 7f28fcb46700 10 osd.70 pg_epoch: 71593 pg[217.bd( v 71377'4132 (71377'1131,71377'4132] local-les=71592 n=13 ec=71094 les/c 71592/71592 71591/71591/71591) [70,7,25] r=0 lpr=71591 crt=71377'4130 lcod 0'0 mlcod 0'0 active+clean] take_waiters 2014-12-08 18:14:01.888359 7f2904b56700 10 osd.70 71593 do_waiters -- start 2014-12-08 18:14:01.888358 7f2903353700 20 osd.70 71593 share_map_peer 0x12889ce0 already has epoch 71593 2014-12-08 18:14:01.888363 7f2904b56700 10 osd.70 71593 do_waiters -- finish 2014-12-08 18:14:01.888356 7f28fc345700 20 osd.70 pg_epoch: 71593 pg[216.ab( v 71097'289 (0'0,71097'289] local-les=71437 n=133 ec=70987 les/c 71585/71583 71591/71592/71168) [20,64,70]/[20,64,13] r=-1 lpr=71593 pi=71183-71591/16 crt=71097'289 lcod 0'0 remapped NOTIFY] new interval newup [20,64,70] newacting [20,64,70] -- 2014-12-08 18:14:01.888442 7f2904b56700 20 osd.70 71593 _dispatch 0x12ceb0e0 pg_notify(19.6e(150),218.6c7(5),218.6c4(5) epoch 71593) v5 2014-12-08 18:14:01.888435 7f28fc345700 10 osd.70 pg_epoch: 71593 pg[216.ab( v 71097'289 (0'0,71097'289] local-les=71437 n=133 ec=70987 les/c 71585/71583 71591/71593/71168) [20,64,70] r=2 lpr=71593 pi=71183-71592/17 crt=71097'289 lcod 0'0 inactive NOTIFY] up [20,64,70] -> [20,64,70], acting [20,64,13] -> [20,64,70], acting_primary 20 -> 20, up_primary 20 -> 20, role -1 -> 2 2014-12-08 18:14:01.888448 7f28fcb46700 10 osd.70 pg_epoch: 71593 pg[216.213( v 71093'217 (0'0,71093'217] local-les=71438 n=101 ec=70987 les/c 71585/71583 71591/71591/70989) [13,17,70]/[13,17,77] r=-1 lpr=71591 pi=70987-71590/23 crt=71093'214 lcod 0'0 remapped NOTIFY] handle_activate_map 2014-12-08 18:14:01.888452 7f28fc345700 10 osd.70 pg_epoch: 71593 pg[216.ab( v 71097'289 (0'0,71097'289] local-les=71437 n=133 ec=70987 les/c 71585/71583 71591/71593/71168) [20,64,70] r=2 lpr=71593 pi=71183-71592/17 crt=71097'289 lcod 0'0 inactive NOTIFY] on_change 2014-12-08 18:14:01.888470 7f2904b56700 7 osd.70 71593 handle_pg_notify from osd.76 2014-12-08 18:14:01.888463 7f28fc345700 15 osd.70 pg_epoch: 71593 pg[216.ab( v 71097'289 (0'0,71097'289] local-les=71437 n=133 ec=70987 les/c 71585/71583 71591/71593/71168) [20,64,70] r=2 lpr=71593 pi=71183-71592/17 crt=71097'289 lcod 0'0 inactive NOTIFY] publish_stats_to_osd -- not primary 2014-12-08 18:14:01.888472 7f2904b56700 15 osd.70 71593 require_same_or_newer_map 71593 (i am 71593) 0x12ceb0e0 2014-12-08 18:14:01.888472 7f28fc345700 15 osd.70 pg_epoch: 71593 pg[216.ab( v 71097'289 (0'0,71097'289] local-les=71437 n=133 ec=70987 les/c 71585/71583 71591/71593/71168) [20,64,70] r=2 lpr=71593 pi=71183-71592/17 crt=71097'289 lcod 0'0 inactive NOTIFY] requeue_ops 2014-12-08 18:14:01.888481 7f28fc345700 10 osd.70 pg_epoch: 71593 pg[216.ab( v 71097'289 (0'0,71097'289] local-les=71437 n=133 ec=70987 les/c 71585/71583 71591/71593/71168) [20,64,70] r=2 lpr=71593 pi=71183-71592/17 crt=71097'289 lcod 0'0 inactive NOTIFY] cancel_copy_ops 2014-12-08 18:14:01.888487 7f2903353700 20 osd.70 71593 share_map_peer 0x11a2eb00 already has epoch 71593 2014-12-08 18:14:01.888489 7f28fc345700 10 osd.70 pg_epoch: 71593 pg[216.ab( v 71097'289 (0'0,71097'289] local-les=71437 n=133 ec=70987 les/c 71585/71583 71591/71593/71168) [20,64,70] r=2 lpr=71593 pi=71183-71592/17 crt=71097'289 lcod 0'0 inactive NOTIFY] cancel_flush_ops 2014-12-08 18:14:01.888487 7f28fcb46700 10 osd.70 pg_epoch: 71593 pg[216.213( v 71093'217 (0'0,71093'217] local-les=71438 n=101 ec=70987 les/c 71585/71583 71591/71591/70989) [13,17,70]/[13,17,77] r=-1 lpr=71591 pi=70987-71590/23 crt=71093'214 lcod 0'0 remapped NOTIFY] take_waiters 2014-12-08 18:14:01.888506 7f2903353700 10 osd.70 71593 note_peer_epoch osd.77 has 71593 2014-12-08 18:14:01.888510 7f2903353700 20 osd.70 71593 share_map_peer 0x12a1fc80 already has epoch 71593 2014-12-08 18:14:01.888502 7f28fc345700 10 osd.70 pg_epoch: 71593 pg[216.ab( v 71097'289 (0'0,71097'289] local-les=71437 n=133 ec=70987 les/c 71585/71583 71591/71593/71168) [20,64,70] r=2 lpr=71593 pi=71183-71592/17 crt=71097'289 lcod 0'0 inactive NOTIFY] on_change_cleanup 2014-12-08 18:14:01.888513 7f2904b56700 10 osd.70 71593 do_waiters -- start 2014-12-08 18:14:01.888516 7f2904b56700 10 osd.70 71593 do_waiters -- finish 2014-12-08 18:14:01.888503 7f28fcb46700 20 osd.70 pg_epoch: 71593 pg[216.213( v 71093'217 (0'0,71093'217] local-les=71438 n=101 ec=70987 les/c 71585/71583 71591/71591/70989) [13,17,70]/[13,17,77] r=-1 lpr=71591 pi=70987-71590/23 crt=71093'214 lcod 0'0 remapped NOTIFY] handle_activate_map: Not dirtying info: last_persisted is 71591 while current is 71593 2014-12-08 18:14:01.888516 7f28fc345700 10 osd.70 pg_epoch: 71593 pg[216.ab( v 71097'289 (0'0,71097'289] local-les=71437 n=133 ec=70987 les/c 71585/71583 71591/71593/71168) [20,64,70] r=2 lpr=71593 pi=71183-71592/17 crt=71097'289 lcod 0'0 inactive NOTIFY] on_change 2014-12-08 18:14:01.888519 7f28fcb46700 10 osd.70 pg_epoch: 71593 pg[216.213( v 71093'217 (0'0,71093'217] local-les=71438 n=101 ec=70987 les/c 71585/71583 71591/71591/70989) [13,17,70]/[13,17,77] r=-1 lpr=71591 pi=70987-71590/23 crt=71093'214 lcod 0'0 remapped NOTIFY] handle_peering_event: epoch_sent: 71593 epoch_requested: 71593 NullEvt 2014-12-08 18:14:01.888531 7f28fcb46700 10 log is not dirty -- 2014-12-08 18:14:01.894447 7f2904b56700 20 osd.70 71593 _dispatch 0x44fb0e0 pg_notify(19.6e(150) epoch 71593) v5 2014-12-08 18:14:01.894432 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[216.39e( v 71422'199 (0'0,71422'199] local-les=71592 n=94 ec=70987 les/c 71592/71592 71591/71591/71587) [64,70,76] r=1 lpr=71591 pi=71156-71590/18 luod=0'0 crt=71097'198 lcod 0'0 active] handle_advance_map [64,70,76]/[64,70,76] -- 64/64 2014-12-08 18:14:01.894453 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:14:01.894453 7f28fcb46700 10 osd.70 pg_epoch: 71593 pg[218.3e2( empty local-les=71438 n=0 ec=71423 les/c 71585/71585 71591/71592/71425) [13,19,70] r=2 lpr=71592 pi=71423-71591/9 crt=0'0 inactive NOTIFY] handle_activate_map 2014-12-08 18:14:01.894458 7f28fc345700 10 osd.70 pg_epoch: 71593 pg[216.39e( v 71422'199 (0'0,71422'199] local-les=71592 n=94 ec=70987 les/c 71592/71592 71591/71591/71587) [64,70,76] r=1 lpr=71591 pi=71156-71590/18 luod=0'0 crt=71097'198 lcod 0'0 active] state: Started advmap 2014-12-08 18:14:01.894476 7f2904b56700 7 osd.70 71593 handle_pg_notify from osd.4 2014-12-08 18:14:01.894478 7f2904b56700 15 osd.70 71593 require_same_or_newer_map 71593 (i am 71593) 0x44fb0e0 2014-12-08 18:14:01.894472 7f28fc345700 10 osd.70 pg_epoch: 71593 pg[216.39e( v 71422'199 (0'0,71422'199] local-les=71592 n=94 ec=70987 les/c 71592/71592 71591/71591/71587) [64,70,76] r=1 lpr=71591 pi=71156-71590/18 luod=0'0 crt=71097'198 lcod 0'0 active] check_recovery_sources no source osds () went down 2014-12-08 18:14:01.894484 7f28fc345700 10 osd.70 pg_epoch: 71593 pg[216.39e( v 71422'199 (0'0,71422'199] local-les=71592 n=94 ec=70987 les/c 71592/71592 71591/71591/71587) [64,70,76] r=1 lpr=71591 pi=71156-71590/18 luod=0'0 crt=71097'198 lcod 0'0 active] handle_activate_map 2014-12-08 18:14:01.894492 7f28fcb46700 10 osd.70 pg_epoch: 71593 pg[218.3e2( empty local-les=71438 n=0 ec=71423 les/c 71585/71585 71591/71592/71425) [13,19,70] r=2 lpr=71592 pi=71423-71591/9 crt=0'0 inactive NOTIFY] take_waiters 2014-12-08 18:14:01.894493 7f28fc345700 10 osd.70 pg_epoch: 71593 pg[216.39e( v 71422'199 (0'0,71422'199] local-les=71592 n=94 ec=70987 les/c 71592/71592 71591/71591/71587) [64,70,76] r=1 lpr=71591 pi=71156-71590/18 luod=0'0 crt=71097'198 lcod 0'0 active] take_waiters 2014-12-08 18:14:01.894503 7f2904b56700 10 osd.70 71593 do_waiters -- start 2014-12-08 18:14:01.894505 7f2904b56700 10 osd.70 71593 do_waiters -- finish 2014-12-08 18:14:01.894503 7f28fcb46700 20 osd.70 pg_epoch: 71593 pg[218.3e2( empty local-les=71438 n=0 ec=71423 les/c 71585/71585 71591/71592/71425) [13,19,70] r=2 lpr=71592 pi=71423-71591/9 crt=0'0 inactive NOTIFY] handle_activate_map: Not dirtying info: last_persisted is 71592 while current is 71593 2014-12-08 18:14:01.894505 7f28fc345700 20 osd.70 pg_epoch: 71593 pg[216.39e( v 71422'199 (0'0,71422'199] local-les=71592 n=94 ec=70987 les/c 71592/71592 71591/71591/71587) [64,70,76] r=1 lpr=71591 pi=71156-71590/18 luod=0'0 crt=71097'198 lcod 0'0 active] handle_activate_map: Not dirtying info: last_persisted is 71592 while current is 71593 2014-12-08 18:14:01.894515 7f28fcb46700 10 osd.70 pg_epoch: 71593 pg[218.3e2( empty local-les=71438 n=0 ec=71423 les/c 71585/71585 71591/71592/71425) [13,19,70] r=2 lpr=71592 pi=71423-71591/9 crt=0'0 inactive NOTIFY] handle_peering_event: epoch_sent: 71593 epoch_requested: 71593 NullEvt 2014-12-08 18:14:01.894526 7f28fcb46700 10 log is not dirty 2014-12-08 18:14:01.894518 7f28fc345700 10 osd.70 pg_epoch: 71593 pg[216.39e( v 71422'199 (0'0,71422'199] local-les=71592 n=94 ec=70987 les/c 71592/71592 71591/71591/71587) [64,70,76] r=1 lpr=71591 pi=71156-71590/18 luod=0'0 crt=71097'198 lcod 0'0 active] handle_peering_event: epoch_sent: 71593 epoch_requested: 71593 NullEvt 2014-12-08 18:14:01.894531 7f28fc345700 10 log is not dirty 2014-12-08 18:14:01.894531 7f28fcb46700 10 osd.70 71593 queue_want_up_thru want 71593 <= queued 71593, currently 71592 2014-12-08 18:14:01.894538 7f28fcb46700 20 osd.70 71593 share_map_peer 0x128a0b00 already has epoch 71593 -- 2014-12-08 18:14:01.894982 7f2904b56700 20 osd.70 71593 _dispatch 0x12f892c0 pg_notify(19.6e(150) epoch 71593) v5 2014-12-08 18:14:01.894973 7f28fc345700 10 osd.70 pg_epoch: 71593 pg[218.a3( v 71581'1 (0'0,71581'1] local-les=71434 n=1 ec=71423 les/c 71585/71583 71591/71593/71593) [70,20,23] r=0 lpr=71593 pi=71432-71592/6 crt=0'0 lcod 0'0 mlcod 0'0 inactive] on_role_change 2014-12-08 18:14:01.894979 7f28fcb46700 10 osd.70 pg_epoch: 71593 pg[19.6a( v 71397'36030933 (50495'36027932,71397'36030933] local-les=71592 n=942 ec=21298 les/c 71592/71592 71591/71591/69763) [13,58,70] r=2 lpr=71591 pi=57840-71590/51 luod=0'0 crt=71397'36030931 lcod 0'0 active] take_waiters 2014-12-08 18:14:01.894987 7f28fc345700 15 osd.70 pg_epoch: 71593 pg[218.a3( v 71581'1 (0'0,71581'1] local-les=71434 n=1 ec=71423 les/c 71585/71583 71591/71593/71593) [70,20,23] r=0 lpr=71593 pi=71432-71592/6 crt=0'0 lcod 0'0 mlcod 0'0 inactive] requeue_ops 2014-12-08 18:14:01.894993 7f28fcb46700 20 osd.70 pg_epoch: 71593 pg[19.6a( v 71397'36030933 (50495'36027932,71397'36030933] local-les=71592 n=942 ec=21298 les/c 71592/71592 71591/71591/69763) [13,58,70] r=2 lpr=71591 pi=57840-71590/51 luod=0'0 crt=71397'36030931 lcod 0'0 active] handle_activate_map: Not dirtying info: last_persisted is 71592 while current is 71593 2014-12-08 18:14:01.894999 7f28fc345700 10 osd.70 pg_epoch: 71593 pg[218.a3( v 71581'1 (0'0,71581'1] local-les=71434 n=1 ec=71423 les/c 71585/71583 71591/71593/71593) [70,20,23] r=0 lpr=71593 pi=71432-71592/6 crt=0'0 lcod 0'0 mlcod 0'0 inactive] cancel_recovery 2014-12-08 18:14:01.895016 7f2904b56700 7 osd.70 71593 handle_pg_notify from osd.30 2014-12-08 18:14:01.895019 7f2904b56700 15 osd.70 71593 require_same_or_newer_map 71593 (i am 71593) 0x12f892c0 2014-12-08 18:14:01.895007 7f28fcb46700 10 osd.70 pg_epoch: 71593 pg[19.6a( v 71397'36030933 (50495'36027932,71397'36030933] local-les=71592 n=942 ec=21298 les/c 71592/71592 71591/71591/69763) [13,58,70] r=2 lpr=71591 pi=57840-71590/51 luod=0'0 crt=71397'36030931 lcod 0'0 active] handle_peering_event: epoch_sent: 71593 epoch_requested: 71593 NullEvt 2014-12-08 18:14:01.895011 7f28fc345700 10 osd.70 pg_epoch: 71593 pg[218.a3( v 71581'1 (0'0,71581'1] local-les=71434 n=1 ec=71423 les/c 71585/71583 71591/71593/71593) [70,20,23] r=0 lpr=71593 pi=71432-71592/6 crt=0'0 lcod 0'0 mlcod 0'0 inactive] clear_recovery_state 2014-12-08 18:14:01.895023 7f28fcb46700 10 log is not dirty 2014-12-08 18:14:01.895025 7f28fc345700 10 osd.70 pg_epoch: 71593 pg[218.a3( v 71581'1 (0'0,71581'1] local-les=71434 n=1 ec=71423 les/c 71585/71583 71591/71593/71593) [70,20,23] r=0 lpr=71593 pi=71432-71592/6 crt=0'0 lcod 0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:14:01.895039 7f2904b56700 10 osd.70 71593 do_waiters -- start 2014-12-08 18:14:01.895042 7f2904b56700 10 osd.70 71593 do_waiters -- finish 2014-12-08 18:14:01.895034 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[20.6e( v 57981'62 (0'0,57981'62] local-les=71433 n=59 ec=21299 les/c 71585/71583 71591/71592/71592) [70,78] r=0 lpr=71592 pi=71432-71591/6 crt=57981'62 lcod 0'0 mlcod 0'0 peering] handle_advance_map [70,78]/[70,78] -- 70/70 2014-12-08 18:14:01.895040 7f28fc345700 10 osd.70 pg_epoch: 71593 pg[218.a3( v 71581'1 (0'0,71581'1] local-les=71434 n=1 ec=71423 les/c 71585/71583 71591/71593/71593) [70,20,23] r=0 lpr=71593 pi=71432-71592/6 crt=0'0 lcod 0'0 mlcod 0'0 inactive] handle_activate_map 2014-12-08 18:14:01.895051 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1,9~2,d~2] newly_removed_snaps [] snapc f=[] (no change) 2014-12-08 18:14:01.895053 7f28fc345700 10 osd.70 pg_epoch: 71593 pg[218.a3( v 71581'1 (0'0,71581'1] local-les=71434 n=1 ec=71423 les/c 71585/71583 71591/71593/71593) [70,20,23] r=0 lpr=71593 pi=71432-71592/6 crt=0'0 lcod 0'0 mlcod 0'0 inactive] update_heartbeat_peers -> 20,23,70 2014-12-08 18:14:01.895063 7f28fc345700 20 osd.70 71593 need_heartbeat_peer_update 2014-12-08 18:14:01.895059 7f28fcb46700 10 osd.70 pg_epoch: 71593 pg[20.6e( v 57981'62 (0'0,57981'62] local-les=71433 n=59 ec=21299 les/c 71585/71583 71591/71592/71592) [70,78] r=0 lpr=71592 pi=71432-71591/6 crt=57981'62 lcod 0'0 mlcod 0'0 peering] state: Peering advmap 2014-12-08 18:14:01.895065 7f28fc345700 10 osd.70 pg_epoch: 71593 pg[218.a3( v 71581'1 (0'0,71581'1] local-les=71434 n=1 ec=71423 les/c 71585/71583 71591/71593/71593) [70,20,23] r=0 lpr=71593 pi=71432-71592/6 crt=0'0 lcod 0'0 mlcod 0'0 inactive] take_waiters -- 2014-12-08 18:14:01.896794 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:14:01.896795 7f28fc345700 10 osd.70 pg_epoch: 71593 pg[218.4b( v 71581'2 (0'0,71581'2] local-les=71448 n=2 ec=71423 les/c 71590/71583 71591/71591/71425) [18,17,70]/[18,17,25] r=-1 lpr=71591 pi=71423-71590/11 crt=0'0 lcod 0'0 remapped NOTIFY] handle_activate_map 2014-12-08 18:14:01.896812 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:14:01.896816 7f2904b56700 10 osd.70 71593 do_waiters -- start 2014-12-08 18:14:01.896818 7f2904b56700 10 osd.70 71593 do_waiters -- finish 2014-12-08 18:14:01.896818 7f2904b56700 20 osd.70 71593 _dispatch 0x135c0d00 pg_query(218.343 epoch 71593) v3 2014-12-08 18:14:01.896818 7f28fcb46700 10 osd.70 pg_epoch: 71593 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] state: Peering advmap 2014-12-08 18:14:01.896842 7f2904b56700 7 osd.70 71593 handle_pg_query from osd.15 epoch 71593 2014-12-08 18:14:01.896832 7f28fcb46700 10 osd.70 pg_epoch: 71593 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] state: Started advmap 2014-12-08 18:14:01.896844 7f2904b56700 15 osd.70 71593 require_same_or_newer_map 71593 (i am 71593) 0x135c0d00 2014-12-08 18:14:01.896834 7f28fc345700 10 osd.70 pg_epoch: 71593 pg[218.4b( v 71581'2 (0'0,71581'2] local-les=71448 n=2 ec=71423 les/c 71590/71583 71591/71591/71425) [18,17,70]/[18,17,25] r=-1 lpr=71591 pi=71423-71590/11 crt=0'0 lcod 0'0 remapped NOTIFY] take_waiters 2014-12-08 18:14:01.896851 7f28fcb46700 10 osd.70 pg_epoch: 71593 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] check_recovery_sources no source osds () went down 2014-12-08 18:14:01.896847 7f290a83a700 10 osd.70 pg_epoch: 71593 pg[216.23b( v 71422'256 (0'0,71422'256] local-les=71593 n=122 ec=70987 les/c 71585/71583 71591/71592/71592) [70,24,63] r=0 lpr=71592 pi=71432-71591/6 crt=71097'255 lcod 0'0 mlcod 0'0 inactive] _activate_committed 71593 peer_activated now 70 last_epoch_started 71585 same_interval_since 71592 2014-12-08 18:14:01.896857 7f2904b56700 10 osd.70 pg_epoch: 71592 pg[218.343( empty local-les=71457 n=0 ec=71423 les/c 71585/71585 71591/71591/71456) [15,70,76]/[15,76,68] r=-1 lpr=71591 pi=71444-71590/6 crt=0'0 remapped NOTIFY] handle_query query(info 0'0) from replica 15 2014-12-08 18:14:01.896855 7f28fc345700 20 osd.70 pg_epoch: 71593 pg[218.4b( v 71581'2 (0'0,71581'2] local-les=71448 n=2 ec=71423 les/c 71590/71583 71591/71591/71425) [18,17,70]/[18,17,25] r=-1 lpr=71591 pi=71423-71590/11 crt=0'0 lcod 0'0 remapped NOTIFY] handle_activate_map: Not dirtying info: last_persisted is 71591 while current is 71593 2014-12-08 18:14:01.896871 7f290a83a700 10 osd.70 pg_epoch: 71593 pg[216.23b( v 71422'256 (0'0,71422'256] local-les=71593 n=122 ec=70987 les/c 71585/71583 71591/71592/71592) [70,24,63] r=0 lpr=71592 pi=71432-71591/6 crt=71097'255 lcod 0'0 mlcod 0'0 inactive] flushed 2014-12-08 18:14:01.896868 7f28fcb46700 10 osd.70 pg_epoch: 71593 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] handle_activate_map 2014-12-08 18:14:01.896886 7f2904b56700 10 osd.70 71593 do_waiters -- start 2014-12-08 18:14:01.896888 7f2904b56700 10 osd.70 71593 do_waiters -- finish 2014-12-08 18:14:01.896883 7f28fc345700 10 osd.70 pg_epoch: 71593 pg[218.4b( v 71581'2 (0'0,71581'2] local-les=71448 n=2 ec=71423 les/c 71590/71583 71591/71591/71425) [18,17,70]/[18,17,25] r=-1 lpr=71591 pi=71423-71590/11 crt=0'0 lcod 0'0 remapped NOTIFY] handle_peering_event: epoch_sent: 71593 epoch_requested: 71593 NullEvt 2014-12-08 18:14:01.896900 7f2904b56700 10 osd.70 71593 do_waiters -- start 2014-12-08 18:14:01.896902 7f2904b56700 10 osd.70 71593 do_waiters -- finish 2014-12-08 18:14:01.896888 7f28fcb46700 7 osd.70 pg_epoch: 71593 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] state: handle ActMap primary 2014-12-08 18:14:01.896904 7f28fc345700 10 log is not dirty 2014-12-08 18:14:01.896903 7f2904b56700 20 osd.70 71593 _dispatch 0x12eaa960 pg_notify(218.3c2(4) epoch 71593) v5 2014-12-08 18:14:01.896912 7f28fcb46700 15 osd.70 pg_epoch: 71593 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] publish_stats_to_osd 71593:63844733 2014-12-08 18:14:01.896923 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71592 n=14 ec=71094 les/c 71592/71592 71591/71591/71589) [68,18,70] r=2 lpr=71591 pi=71183-71590/23 luod=0'0 crt=71377'4110 lcod 0'0 active] handle_advance_map [68,18,70]/[68,18,70] -- 68/68 2014-12-08 18:14:01.896929 7f28fcb46700 10 osd.70 pg_epoch: 71593 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] take_waiters 2014-12-08 18:14:01.896945 7f2904b56700 7 osd.70 71593 handle_pg_notify from osd.24 2014-12-08 18:14:01.896947 7f2904b56700 15 osd.70 71593 require_same_or_newer_map 71593 (i am 71593) 0x12eaa960 2014-12-08 18:14:01.896950 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:14:01.896950 7f28fcb46700 20 osd.70 pg_epoch: 71593 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] handle_activate_map: Not dirtying info: last_persisted is 71591 while current is 71593 2014-12-08 18:14:01.896968 7f2904b56700 10 osd.70 71593 do_waiters -- start 2014-12-08 18:14:01.896970 7f2904b56700 10 osd.70 71593 do_waiters -- finish 2014-12-08 18:14:01.896958 7f28fc345700 10 osd.70 pg_epoch: 71593 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71592 n=14 ec=71094 les/c 71592/71592 71591/71591/71589) [68,18,70] r=2 lpr=71591 pi=71183-71590/23 luod=0'0 crt=71377'4110 lcod 0'0 active] state: Started advmap 2014-12-08 18:14:01.896982 7f2904b56700 10 osd.70 71593 do_waiters -- start 2014-12-08 18:14:01.896984 7f2904b56700 10 osd.70 71593 do_waiters -- finish 2014-12-08 18:14:01.896969 7f28fcb46700 10 osd.70 pg_epoch: 71593 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] handle_peering_event: epoch_sent: 71593 epoch_requested: 71593 NullEvt 2014-12-08 18:14:01.896985 7f2904b56700 20 osd.70 71593 _dispatch 0x12f81e00 pg_notify(216.2e7(4) epoch 71593) v5 2014-12-08 18:14:01.896989 7f28fcb46700 10 log is not dirty 2014-12-08 18:14:01.896980 7f28fc345700 10 osd.70 pg_epoch: 71593 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71592 n=14 ec=71094 les/c 71592/71592 71591/71591/71589) [68,18,70] r=2 lpr=71591 pi=71183-71590/23 luod=0'0 crt=71377'4110 lcod 0'0 active] check_recovery_sources no source osds () went down 2014-12-08 18:14:01.897004 7f2904b56700 7 osd.70 71593 handle_pg_notify from osd.64 2014-12-08 18:14:01.897006 7f2904b56700 15 osd.70 71593 require_same_or_newer_map 71593 (i am 71593) 0x12f81e00 2014-12-08 18:14:01.897011 7f28fc345700 10 osd.70 pg_epoch: 71593 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71592 n=14 ec=71094 les/c 71592/71592 71591/71591/71589) [68,18,70] r=2 lpr=71591 pi=71183-71590/23 luod=0'0 crt=71377'4110 lcod 0'0 active] handle_activate_map 2014-12-08 18:14:01.897025 7f2904b56700 10 osd.70 71593 do_waiters -- start 2014-12-08 18:14:01.897026 7f2904b56700 10 osd.70 71593 do_waiters -- finish 2014-12-08 18:14:01.897022 7f28fc345700 10 osd.70 pg_epoch: 71593 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71592 n=14 ec=71094 les/c 71592/71592 71591/71591/71589) [68,18,70] r=2 lpr=71591 pi=71183-71590/23 luod=0'0 crt=71377'4110 lcod 0'0 active] take_waiters 2014-12-08 18:14:01.897030 7f28fc345700 20 osd.70 pg_epoch: 71593 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71592 n=14 ec=71094 les/c 71592/71592 71591/71591/71589) [68,18,70] r=2 lpr=71591 pi=71183-71590/23 luod=0'0 crt=71377'4110 lcod 0'0 active] handle_activate_map: Not dirtying info: last_persisted is 71592 while current is 71593 2014-12-08 18:14:01.897039 7f2904b56700 10 osd.70 71593 do_waiters -- start 2014-12-08 18:14:01.897042 7f2904b56700 10 osd.70 71593 do_waiters -- finish 2014-12-08 18:14:01.897044 7f2904b56700 20 osd.70 71593 _dispatch 0x12c6e960 pg_notify(216.3c5(33) epoch 71593) v5 2014-12-08 18:14:01.897040 7f28fc345700 10 osd.70 pg_epoch: 71593 pg[217.220( v 71377'4112 (71377'1111,71377'4112] local-les=71592 n=14 ec=71094 les/c 71592/71592 71591/71591/71589) [68,18,70] r=2 lpr=71591 pi=71183-71590/23 luod=0'0 crt=71377'4110 lcod 0'0 active] handle_peering_event: epoch_sent: 71593 epoch_requested: 71593 NullEvt 2014-12-08 18:14:01.897053 7f28fc345700 10 log is not dirty 2014-12-08 18:14:01.897035 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[19.6d( v 71397'33614009 (70963'33611008,71397'33614009] local-les=71592 n=896 ec=21298 les/c 71583/71583 71591/71591/71169) [4,70,69] r=1 lpr=71591 pi=71156-71590/27 luod=0'0 crt=71397'33614007 lcod 0'0 active] handle_advance_map [4,70,69]/[4,70,69] -- 4/4 2014-12-08 18:14:01.897064 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:14:01.897067 7f28fc345700 10 osd.70 pg_epoch: 71592 pg[216.3b8( v 71097'270 (0'0,71097'270] local-les=71437 n=130 ec=70987 les/c 71585/71583 71591/71591/71587) [58,70,18]/[58,18,13] r=-1 lpr=71591 pi=70987-71590/26 crt=71097'270 lcod 0'0 remapped NOTIFY] handle_advance_map [58,70,18]/[58,18,13] -- 58/58 2014-12-08 18:14:01.897070 7f28fcb46700 10 osd.70 pg_epoch: 71593 pg[19.6d( v 71397'33614009 (70963'33611008,71397'33614009] local-les=71592 n=896 ec=21298 les/c 71583/71583 71591/71591/71169) [4,70,69] r=1 lpr=71591 pi=71156-71590/27 luod=0'0 crt=71397'33614007 lcod 0'0 active] state: Started advmap 2014-12-08 18:14:01.897081 7f2904b56700 7 osd.70 71593 handle_pg_notify from osd.1 -- 2014-12-08 18:14:01.931297 7f28fbb44700 10 osd.70 71593 dequeue_op 0x1285ce00 prio 63 cost 917504 latency 0.007363 osd_op(client.1195889.0:533240 rbd_data.124045103eaad3.00000000000007be [set-alloc-hint object_size 8388608 write_size 8388608,write 0~917504] 19.2fb9d06e RETRY=14 ack+ondisk+retry+write e71593) v4 pg pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] 2014-12-08 18:14:01.931327 7f28fc345700 10 osd.70 pg_epoch: 71593 pg[217.1ed( v 71377'36 (0'0,71377'36] local-les=71433 n=14 ec=71094 les/c 71585/71583 71591/71593/71168) [20,70,5] r=1 lpr=71593 pi=71183-71592/17 crt=71377'34 lcod 0'0 inactive NOTIFY] cancel_flush_ops 2014-12-08 18:14:01.931341 7f2904b56700 10 osd.70 71593 do_waiters -- start 2014-12-08 18:14:01.931344 7f2904b56700 10 osd.70 71593 do_waiters -- finish 2014-12-08 18:14:01.931351 7f2904b56700 20 osd.70 71593 _dispatch 0x12eab2c0 pg_notify(216.23b(5) epoch 71593) v5 2014-12-08 18:14:01.931340 7f28fc345700 10 osd.70 pg_epoch: 71593 pg[217.1ed( v 71377'36 (0'0,71377'36] local-les=71433 n=14 ec=71094 les/c 71585/71583 71591/71593/71168) [20,70,5] r=1 lpr=71593 pi=71183-71592/17 crt=71377'34 lcod 0'0 inactive NOTIFY] on_change_cleanup 2014-12-08 18:14:01.931344 7f28fcb46700 10 osd.70 pg_epoch: 71593 pg[218.685( v 71581'2 (0'0,71581'2] local-les=71433 n=2 ec=71423 les/c 71585/71583 71591/71591/71587) [70,10,18] r=0 lpr=71593 pi=71432-71592/6 crt=0'0 lcod 0'0 mlcod 0'0 inactive NOTIFY] start_peering_interval: check_new_interval output: generate_past_intervals interval(71591-71592 up [70,10,18](70) acting [10,18,6](10)): not rw, up_thru 71591 up_from 69670 last_epoch_clean 71583 generate_past_intervals interval(71591-71592 up [70,10,18](70) acting [10,18,6](10) maybe_went_rw) : primary up 69670-71591 includes interval 2014-12-08 18:14:01.931359 7f28fc345700 10 osd.70 pg_epoch: 71593 pg[217.1ed( v 71377'36 (0'0,71377'36] local-les=71433 n=14 ec=71094 les/c 71585/71583 71591/71593/71168) [20,70,5] r=1 lpr=71593 pi=71183-71592/17 crt=71377'34 lcod 0'0 inactive NOTIFY] on_change 2014-12-08 18:14:01.931380 7f2904b56700 7 osd.70 71593 handle_pg_notify from osd.24 2014-12-08 18:14:01.931382 7f2904b56700 15 osd.70 71593 require_same_or_newer_map 71593 (i am 71593) 0x12eab2c0 2014-12-08 18:14:01.931375 7f28fc345700 20 osd.70 pg_epoch: 71593 pg[217.1ed( v 71377'36 (0'0,71377'36] local-les=71433 n=14 ec=71094 les/c 71585/71583 71591/71593/71168) [20,70,5] r=1 lpr=71593 pi=71183-71592/17 crt=71377'34 lcod 0'0 inactive NOTIFY] exit NotTrimming 2014-12-08 18:14:01.931374 7f28fcb46700 10 osd.70 pg_epoch: 71593 pg[218.685( v 71581'2 (0'0,71581'2] local-les=71433 n=2 ec=71423 les/c 71585/71583 71591/71591/71587) [70,10,18] r=0 lpr=71593 pi=71432-71592/6 crt=0'0 lcod 0'0 mlcod 0'0 inactive NOTIFY] noting past interval(71591-71592 up [70,10,18](70) acting [10,18,6](10) maybe_went_rw) 2014-12-08 18:14:01.931390 7f28fc345700 20 osd.70 pg_epoch: 71593 pg[217.1ed( v 71377'36 (0'0,71377'36] local-les=71433 n=14 ec=71094 les/c 71585/71583 71591/71593/71168) [20,70,5] r=1 lpr=71593 pi=71183-71592/17 crt=71377'34 lcod 0'0 inactive NOTIFY] enter NotTrimming 2014-12-08 18:14:01.931411 7f2904b56700 10 osd.70 71593 do_waiters -- start 2014-12-08 18:14:01.931413 7f2904b56700 10 osd.70 71593 do_waiters -- finish 2014-12-08 18:14:01.931405 7f28fc345700 10 osd.70 pg_epoch: 71593 pg[217.1ed( v 71377'36 (0'0,71377'36] local-les=71433 n=14 ec=71094 les/c 71585/71583 71591/71593/71168) [20,70,5] r=1 lpr=71593 pi=71183-71592/17 crt=71377'34 lcod 0'0 inactive NOTIFY] on_role_change 2014-12-08 18:14:01.931417 7f28fc345700 15 osd.70 pg_epoch: 71593 pg[217.1ed( v 71377'36 (0'0,71377'36] local-les=71433 n=14 ec=71094 les/c 71585/71583 71591/71593/71168) [20,70,5] r=1 lpr=71593 pi=71183-71592/17 crt=71377'34 lcod 0'0 inactive NOTIFY] requeue_ops 2014-12-08 18:14:01.931400 7f28fcb46700 10 osd.70 pg_epoch: 71593 pg[218.685( v 71581'2 (0'0,71581'2] local-les=71433 n=2 ec=71423 les/c 71585/71583 71591/71593/71593) [70,10,18] r=0 lpr=71593 pi=71432-71592/6 crt=0'0 lcod 0'0 mlcod 0'0 inactive NOTIFY] up [70,10,18] -> [70,10,18], acting [10,18,6] -> [70,10,18], acting_primary 10 -> 70, up_primary 70 -> 70, role -1 -> 0 2014-12-08 18:14:01.931436 7f2904b56700 10 osd.70 71593 do_waiters -- start -- 2014-12-08 18:14:01.931593 7f28fbb44700 20 osd.70 pg_epoch: 71593 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] op_has_sufficient_caps pool=19 (volumes ) owner=0 need_read_cap=0 need_write_cap=1 need_class_read_cap=0 need_class_write_cap=0 -> yes 2014-12-08 18:14:01.931620 7f28fcb46700 15 osd.70 pg_epoch: 71593 pg[218.685( v 71581'2 (0'0,71581'2] local-les=71433 n=2 ec=71423 les/c 71585/71583 71591/71593/71593) [70,10,18] r=0 lpr=71593 pi=71432-71592/6 crt=0'0 lcod 0'0 mlcod 0'0 inactive NOTIFY] requeue_ops 2014-12-08 18:14:01.931629 7f28fc345700 5 osd.70 pg_epoch: 71593 pg[217.1ed( v 71377'36 (0'0,71377'36] local-les=71433 n=14 ec=71094 les/c 71585/71583 71591/71593/71168) [20,70,5] r=1 lpr=71593 pi=71183-71592/17 crt=71377'34 lcod 0'0 inactive NOTIFY] exit Start 0.000039 0 0.000000 2014-12-08 18:14:01.931633 7f28fbb44700 10 osd.70 pg_epoch: 71593 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] can_handle_while_inactive: 0x1285ce00 2014-12-08 18:14:01.931652 7f28fbb44700 10 osd.70 71593 dequeue_op 0x1285ce00 finish 2014-12-08 18:14:01.931652 7f28fc345700 5 osd.70 pg_epoch: 71593 pg[217.1ed( v 71377'36 (0'0,71377'36] local-les=71433 n=14 ec=71094 les/c 71585/71583 71591/71593/71168) [20,70,5] r=1 lpr=71593 pi=71183-71592/17 crt=71377'34 lcod 0'0 inactive NOTIFY] enter Started/Stray 2014-12-08 18:14:01.931662 7f2904b56700 7 osd.70 71593 handle_pg_info pg_info(1 pgs e71593:218.270(4)) v4 from osd.4 2014-12-08 18:14:01.931670 7f2904b56700 15 osd.70 71593 require_same_or_newer_map 71593 (i am 71593) 0x44fc3a0 2014-12-08 18:14:01.931650 7f28fcb46700 15 osd.70 pg_epoch: 71593 pg[218.685( v 71581'2 (0'0,71581'2] local-les=71433 n=2 ec=71423 les/c 71585/71583 71591/71593/71593) [70,10,18] r=0 lpr=71593 pi=71432-71592/6 crt=0'0 lcod 0'0 mlcod 0'0 inactive NOTIFY] requeue_ops 2014-12-08 18:14:01.931672 7f28fc345700 20 osd.70 pg_epoch: 71593 pg[217.1ed( v 71377'36 (0'0,71377'36] local-les=71433 n=14 ec=71094 les/c 71585/71583 71591/71593/71168) [20,70,5] r=1 lpr=71593 pi=71183-71592/17 crt=71377'34 lcod 0'0 inactive NOTIFY] handle_activate_map: Not dirtying info: last_persisted is 71592 while current is 71593 2014-12-08 18:14:01.931673 7f28fcb46700 10 osd.70 pg_epoch: 71593 pg[218.685( v 71581'2 (0'0,71581'2] local-les=71433 n=2 ec=71423 les/c 71585/71583 71591/71593/71593) [70,10,18] r=0 lpr=71593 pi=71432-71592/6 crt=0'0 lcod 0'0 mlcod 0'0 inactive NOTIFY] on_change_cleanup 2014-12-08 18:14:01.931687 7f28fc345700 10 osd.70 pg_epoch: 71593 pg[217.1ed( v 71377'36 (0'0,71377'36] local-les=71433 n=14 ec=71094 les/c 71585/71583 71591/71593/71168) [20,70,5] r=1 lpr=71593 pi=71183-71592/17 crt=71377'34 lcod 0'0 inactive NOTIFY] handle_peering_event: epoch_sent: 71593 epoch_requested: 71593 NullEvt 2014-12-08 18:14:01.931711 7f2904b56700 10 osd.70 71593 do_waiters -- start 2014-12-08 18:14:01.931714 7f2904b56700 10 osd.70 71593 do_waiters -- finish 2014-12-08 18:14:01.931695 7f28fcb46700 10 osd.70 pg_epoch: 71593 pg[218.685( v 71581'2 (0'0,71581'2] local-les=71433 n=2 ec=71423 les/c 71585/71583 71591/71593/71593) [70,10,18] r=0 lpr=71593 pi=71432-71592/6 crt=0'0 lcod 0'0 mlcod 0'0 inactive NOTIFY] on_change 2014-12-08 18:14:01.931727 7f2904b56700 10 osd.70 71593 do_waiters -- start 2014-12-08 18:14:01.931728 7f2904b56700 10 osd.70 71593 do_waiters -- finish 2014-12-08 18:14:01.931730 7f2904b56700 20 osd.70 71593 _dispatch 0x13ef3c00 pg_query(218.18a epoch 71593) v3 2014-12-08 18:14:01.931718 7f28fcb46700 20 osd.70 pg_epoch: 71593 pg[218.685( v 71581'2 (0'0,71581'2] local-les=71433 n=2 ec=71423 les/c 71585/71583 71591/71593/71593) [70,10,18] r=0 lpr=71593 pi=71432-71592/6 crt=0'0 lcod 0'0 mlcod 0'0 inactive NOTIFY] exit NotTrimming 2014-12-08 18:14:01.931744 7f28fc345700 10 log is not dirty 2014-12-08 18:14:01.931753 7f2904b56700 7 osd.70 71593 handle_pg_query from osd.78 epoch 71593 2014-12-08 18:14:01.931755 7f2904b56700 15 osd.70 71593 require_same_or_newer_map 71593 (i am 71593) 0x13ef3c00 2014-12-08 18:14:01.931737 7f28fcb46700 20 osd.70 pg_epoch: 71593 pg[218.685( v 71581'2 (0'0,71581'2] local-les=71433 n=2 ec=71423 les/c 71585/71583 71591/71593/71593) [70,10,18] r=0 lpr=71593 pi=71432-71592/6 crt=0'0 lcod 0'0 mlcod 0'0 inactive NOTIFY] enter NotTrimming 2014-12-08 18:14:01.931759 7f28fcb46700 15 osd.70 pg_epoch: 71593 pg[218.685( v 71581'2 (0'0,71581'2] local-les=71433 n=2 ec=71423 les/c 71585/71583 71591/71593/71593) [70,10,18] r=0 lpr=71593 pi=71432-71592/6 crt=0'0 lcod 0'0 mlcod 0'0 inactive] clear_stats -- 2014-12-08 18:14:01.933195 7f28f933f700 10 osd.70 71593 dequeue_op 0x1285ca00 prio 63 cost 1048576 latency 0.001915 osd_op(client.1195889.0:533241 rbd_data.124045103eaad3.00000000000007be [set-alloc-hint object_size 8388608 write_size 8388608,write 917504~1048576] 19.2fb9d06e RETRY=14 ack+ondisk+retry+write e71593) v4 pg pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] 2014-12-08 18:14:01.933232 7f2904b56700 20 osd.70 71593 _dispatch 0x13efd4a0 pg_notify(216.1f4(4) epoch 71593) v5 2014-12-08 18:14:01.933220 7f28fcb46700 20 osd.70 pg_epoch: 71593 pg[216.1b( v 71093'263 (0'0,71093'263] local-les=71450 n=121 ec=70987 les/c 71585/71583 71591/71593/71587) [58,23,70] r=2 lpr=71593 pi=70987-71592/31 crt=71093'261 lcod 0'0 inactive NOTIFY] enter NotTrimming 2014-12-08 18:14:01.933234 7f28fc345700 10 osd.70 pg_epoch: 71593 pg[218.1e3( v 71581'1 (0'0,71581'1] local-les=71435 n=1 ec=71423 les/c 71585/71583 71591/71591/71587) [70,76,14]/[76,14,1] r=-1 lpr=71591 pi=71432-71590/5 crt=0'0 lcod 0'0 remapped NOTIFY] state: should_restart_peering, transitioning to Reset 2014-12-08 18:14:01.933243 7f28fcb46700 10 osd.70 pg_epoch: 71593 pg[216.1b( v 71093'263 (0'0,71093'263] local-les=71450 n=121 ec=70987 les/c 71585/71583 71591/71593/71587) [58,23,70] r=2 lpr=71593 pi=70987-71592/31 crt=71093'261 lcod 0'0 inactive NOTIFY] on_role_change 2014-12-08 18:14:01.933251 7f28fc345700 5 osd.70 pg_epoch: 71593 pg[218.1e3( v 71581'1 (0'0,71581'1] local-les=71435 n=1 ec=71423 les/c 71585/71583 71591/71591/71587) [70,76,14]/[76,14,1] r=-1 lpr=71591 pi=71432-71590/5 crt=0'0 lcod 0'0 remapped NOTIFY] exit Started/Stray 1.366116 7 0.000167 2014-12-08 18:14:01.933278 7f2904b56700 7 osd.70 71593 handle_pg_notify from osd.58 2014-12-08 18:14:01.933252 7f28f933f700 20 osd.70 pg_epoch: 71593 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] op_has_sufficient_caps pool=19 (volumes ) owner=0 need_read_cap=0 need_write_cap=1 need_class_read_cap=0 need_class_write_cap=0 -> yes 2014-12-08 18:14:01.933282 7f2904b56700 15 osd.70 71593 require_same_or_newer_map 71593 (i am 71593) 0x13efd4a0 2014-12-08 18:14:01.933274 7f28fc345700 5 osd.70 pg_epoch: 71593 pg[218.1e3( v 71581'1 (0'0,71581'1] local-les=71435 n=1 ec=71423 les/c 71585/71583 71591/71591/71587) [70,76,14]/[76,14,1] r=-1 lpr=71591 pi=71432-71590/5 crt=0'0 lcod 0'0 remapped NOTIFY] exit Started 1.366177 0 0.000000 2014-12-08 18:14:01.933268 7f28fcb46700 15 osd.70 pg_epoch: 71593 pg[216.1b( v 71093'263 (0'0,71093'263] local-les=71450 n=121 ec=70987 les/c 71585/71583 71591/71593/71587) [58,23,70] r=2 lpr=71593 pi=70987-71592/31 crt=71093'261 lcod 0'0 inactive NOTIFY] requeue_ops 2014-12-08 18:14:01.933292 7f28fc345700 5 osd.70 pg_epoch: 71593 pg[218.1e3( v 71581'1 (0'0,71581'1] local-les=71435 n=1 ec=71423 les/c 71585/71583 71591/71591/71587) [70,76,14]/[76,14,1] r=-1 lpr=71591 pi=71432-71590/5 crt=0'0 lcod 0'0 remapped NOTIFY] enter Reset 2014-12-08 18:14:01.933284 7f28f933f700 10 osd.70 pg_epoch: 71593 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] can_handle_while_inactive: 0x1285ca00 2014-12-08 18:14:01.933312 7f28f933f700 10 osd.70 71593 dequeue_op 0x1285ca00 finish 2014-12-08 18:14:01.933294 7f28fcb46700 10 osd.70 pg_epoch: 71593 pg[216.1b( v 71093'263 (0'0,71093'263] local-les=71450 n=121 ec=70987 les/c 71585/71583 71591/71593/71587) [58,23,70] r=2 lpr=71593 pi=70987-71592/31 crt=71093'261 lcod 0'0 inactive NOTIFY] cancel_recovery 2014-12-08 18:14:01.933310 7f28fc345700 20 osd.70 pg_epoch: 71593 pg[218.1e3( v 71581'1 (0'0,71581'1] local-les=71435 n=1 ec=71423 les/c 71585/71583 71591/71591/71587) [70,76,14]/[76,14,1] r=-1 lpr=71591 pi=71432-71590/5 crt=0'0 lcod 0'0 remapped NOTIFY] set_last_peering_reset 71593 2014-12-08 18:14:01.933327 7f2904b56700 10 osd.70 71593 do_waiters -- start 2014-12-08 18:14:01.933333 7f2904b56700 10 osd.70 71593 do_waiters -- finish 2014-12-08 18:14:01.933319 7f28fcb46700 10 osd.70 pg_epoch: 71593 pg[216.1b( v 71093'263 (0'0,71093'263] local-les=71450 n=121 ec=70987 les/c 71585/71583 71591/71593/71587) [58,23,70] r=2 lpr=71593 pi=70987-71592/31 crt=71093'261 lcod 0'0 inactive NOTIFY] clear_recovery_state 2014-12-08 18:14:01.933326 7f28fc345700 10 osd.70 pg_epoch: 71593 pg[218.1e3( v 71581'1 (0'0,71581'1] local-les=71435 n=1 ec=71423 les/c 71585/71583 71591/71591/71587) [70,76,14]/[76,14,1] r=-1 lpr=71593 pi=71432-71590/5 crt=0'0 lcod 0'0 remapped NOTIFY] Clearing blocked outgoing recovery messages 2014-12-08 18:14:01.933343 7f28fc345700 10 osd.70 pg_epoch: 71593 pg[218.1e3( v 71581'1 (0'0,71581'1] local-les=71435 n=1 ec=71423 les/c 71585/71583 71591/71591/71587) [70,76,14]/[76,14,1] r=-1 lpr=71593 pi=71432-71590/5 crt=0'0 lcod 0'0 remapped NOTIFY] Not blocking outgoing recovery messages 2014-12-08 18:14:01.933361 7f2904b56700 10 osd.70 71593 do_waiters -- start 2014-12-08 18:14:01.933344 7f28fcb46700 10 osd.70 pg_epoch: 71593 pg[216.1b( v 71093'263 (0'0,71093'263] local-les=71450 n=121 ec=70987 les/c 71585/71583 71591/71593/71587) [58,23,70] r=2 lpr=71593 pi=70987-71592/31 crt=71093'261 lcod 0'0 inactive NOTIFY] check_recovery_sources no source osds () went down 2014-12-08 18:14:01.933364 7f2904b56700 10 osd.70 71593 do_waiters -- finish 2014-12-08 18:14:01.933355 7f28fc345700 10 osd.70 pg_epoch: 71593 pg[218.1e3( v 71581'1 (0'0,71581'1] local-les=71435 n=1 ec=71423 les/c 71585/71583 71591/71591/71587) [70,76,14]/[76,14,1] r=-1 lpr=71593 pi=71432-71590/5 crt=0'0 lcod 0'0 remapped NOTIFY] state: Reset advmap 2014-12-08 18:14:01.933371 7f2904b56700 20 osd.70 71593 _dispatch 0x13efe3a0 pg_notify(218.131(10) epoch 71593) v5 2014-12-08 18:14:01.933368 7f28fc345700 10 osd.70 pg_epoch: 71593 pg[218.1e3( v 71581'1 (0'0,71581'1] local-les=71435 n=1 ec=71423 les/c 71585/71583 71591/71591/71587) [70,76,14]/[76,14,1] r=-1 lpr=71593 pi=71432-71590/5 crt=0'0 lcod 0'0 remapped NOTIFY] _calc_past_interval_range: already have past intervals back to 71583 2014-12-08 18:14:01.933379 7f28fc345700 20 osd.70 pg_epoch: 71593 pg[218.1e3( v 71581'1 (0'0,71581'1] local-les=71435 n=1 ec=71423 les/c 71585/71583 71591/71591/71587) [70,76,14]/[76,14,1] r=-1 lpr=71593 pi=71432-71590/5 crt=0'0 lcod 0'0 remapped NOTIFY] new interval newup [70,76,14] newacting [70,76,14] 2014-12-08 18:14:01.933395 7f2904b56700 7 osd.70 71593 handle_pg_notify from osd.76 2014-12-08 18:14:01.933398 7f2904b56700 15 osd.70 71593 require_same_or_newer_map 71593 (i am 71593) 0x13efe3a0 2014-12-08 18:14:01.933381 7f28fcb46700 10 osd.70 pg_epoch: 71593 pg[216.1b( v 71093'263 (0'0,71093'263] local-les=71450 n=121 ec=70987 les/c 71585/71583 71591/71593/71587) [58,23,70] r=2 lpr=71593 pi=70987-71592/31 crt=71093'261 lcod 0'0 inactive NOTIFY] handle_activate_map 2014-12-08 18:14:01.933393 7f28fc345700 10 osd.70 pg_epoch: 71593 pg[218.1e3( v 71581'1 (0'0,71581'1] local-les=71435 n=1 ec=71423 les/c 71585/71583 71591/71591/71587) [70,76,14]/[76,14,1] r=-1 lpr=71593 pi=71432-71590/5 crt=0'0 lcod 0'0 remapped NOTIFY] state: should restart peering, calling start_peering_interval again 2014-12-08 18:14:01.933409 7f28fc345700 20 osd.70 pg_epoch: 71593 pg[218.1e3( v 71581'1 (0'0,71581'1] local-les=71435 n=1 ec=71423 les/c 71585/71583 71591/71591/71587) [70,76,14]/[76,14,1] r=-1 lpr=71593 pi=71432-71590/5 crt=0'0 lcod 0'0 remapped NOTIFY] set_last_peering_reset 71593 -- 2014-12-08 18:14:01.999522 7f28fc345700 10 osd.70 pg_epoch: 71593 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] handle_peering_event: epoch_sent: 71593 epoch_requested: 71593 MNotifyRec from 16 notify: (query_epoch:71593, epoch_sent:71593, info:19.6e( empty local-les=0 n=0 ec=21298 les/c 51101/50495 71591/71591/71591)) features: 0x3fffffffffff 2014-12-08 18:14:01.999538 7f2904b56700 10 osd.70 71593 do_waiters -- start 2014-12-08 18:14:01.999541 7f2904b56700 10 osd.70 71593 do_waiters -- finish 2014-12-08 18:14:01.999540 7f28fcb46700 20 osd.70 pg_epoch: 71593 pg[216.c3( v 71097'230 (0'0,71097'230] local-les=71436 n=111 ec=70987 les/c 71585/71583 71591/71593/70989) [23,78,70] r=2 lpr=71593 pi=71183-71592/18 crt=71097'230 lcod 0'0 inactive NOTIFY] exit NotTrimming 2014-12-08 18:14:01.999542 7f28fc345700 7 osd.70 pg_epoch: 71593 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] state: handle_pg_notify from osd.16 2014-12-08 18:14:01.999547 7f28fcb46700 20 osd.70 pg_epoch: 71593 pg[216.c3( v 71097'230 (0'0,71097'230] local-les=71436 n=111 ec=70987 les/c 71585/71583 71591/71593/70989) [23,78,70] r=2 lpr=71593 pi=71183-71592/18 crt=71097'230 lcod 0'0 inactive NOTIFY] enter NotTrimming 2014-12-08 18:14:01.999554 7f28fcb46700 10 osd.70 pg_epoch: 71593 pg[216.c3( v 71097'230 (0'0,71097'230] local-les=71436 n=111 ec=70987 les/c 71585/71583 71591/71593/70989) [23,78,70] r=2 lpr=71593 pi=71183-71592/18 crt=71097'230 lcod 0'0 inactive NOTIFY] on_role_change 2014-12-08 18:14:01.999552 7f28fc345700 10 osd.70 pg_epoch: 71593 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] state: pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] got dup osd.16 info 19.6e( empty local-les=0 n=0 ec=21298 les/c 51101/50495 71591/71591/71591), identical to ours 2014-12-08 18:14:01.999568 7f28fc345700 10 log is not dirty 2014-12-08 18:14:01.999561 7f28fcb46700 15 osd.70 pg_epoch: 71593 pg[216.c3( v 71097'230 (0'0,71097'230] local-les=71436 n=111 ec=70987 les/c 71585/71583 71591/71593/70989) [23,78,70] r=2 lpr=71593 pi=71183-71592/18 crt=71097'230 lcod 0'0 inactive NOTIFY] requeue_ops 2014-12-08 18:14:01.999573 7f28fc345700 10 osd.70 pg_epoch: 71593 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71590/71583 71591/71593/70961) [75,70] r=1 lpr=71593 pi=71183-71592/19 crt=57981'40 lcod 0'0 inactive NOTIFY] handle_peering_event: epoch_sent: 71593 epoch_requested: 71593 FlushedEvt 2014-12-08 18:14:01.999580 7f28fcb46700 10 osd.70 pg_epoch: 71593 pg[216.c3( v 71097'230 (0'0,71097'230] local-les=71436 n=111 ec=70987 les/c 71585/71583 71591/71593/70989) [23,78,70] r=2 lpr=71593 pi=71183-71592/18 crt=71097'230 lcod 0'0 inactive NOTIFY] cancel_recovery 2014-12-08 18:14:01.999587 7f2904b56700 10 osd.70 71593 do_waiters -- start 2014-12-08 18:14:01.999583 7f28fc345700 15 osd.70 pg_epoch: 71593 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71590/71583 71591/71593/70961) [75,70] r=1 lpr=71593 pi=71183-71592/19 crt=57981'40 lcod 0'0 inactive NOTIFY] requeue_ops 2014-12-08 18:14:01.999590 7f2904b56700 10 osd.70 71593 do_waiters -- finish 2014-12-08 18:14:01.999592 7f28fc345700 10 log is not dirty 2014-12-08 18:14:01.999587 7f28fcb46700 10 osd.70 pg_epoch: 71593 pg[216.c3( v 71097'230 (0'0,71097'230] local-les=71436 n=111 ec=70987 les/c 71585/71583 71591/71593/70989) [23,78,70] r=2 lpr=71593 pi=71183-71592/18 crt=71097'230 lcod 0'0 inactive NOTIFY] clear_recovery_state 2014-12-08 18:14:01.999592 7f2904b56700 20 osd.70 71593 _dispatch 0x1335ab20 pg_notify(218.75f(10) epoch 71593) v5 2014-12-08 18:14:01.999595 7f28fc345700 20 osd.70 71593 share_map_peer 0x126955a0 already has epoch 71593 2014-12-08 18:14:01.999597 7f28fc345700 7 osd.70 71593 do_notifies osd 10 on 1 PGs 2014-12-08 18:14:01.999597 7f28fcb46700 10 osd.70 pg_epoch: 71593 pg[216.c3( v 71097'230 (0'0,71097'230] local-les=71436 n=111 ec=70987 les/c 71585/71583 71591/71593/70989) [23,78,70] r=2 lpr=71593 pi=71183-71592/18 crt=71097'230 lcod 0'0 inactive NOTIFY] check_recovery_sources no source osds () went down 2014-12-08 18:14:01.999613 7f28fc345700 20 osd.70 71593 share_map_peer 0x1268bce0 already has epoch 71593 2014-12-08 18:14:01.999615 7f28fc345700 7 osd.70 71593 do_notifies osd 14 on 1 PGs 2014-12-08 18:14:01.999613 7f28fcb46700 10 osd.70 pg_epoch: 71593 pg[216.c3( v 71097'230 (0'0,71097'230] local-les=71436 n=111 ec=70987 les/c 71585/71583 71591/71593/70989) [23,78,70] r=2 lpr=71593 pi=71183-71592/18 crt=71097'230 lcod 0'0 inactive NOTIFY] handle_activate_map 2014-12-08 18:14:01.999624 7f2904b56700 7 osd.70 71593 handle_pg_notify from osd.1 2014-12-08 18:14:01.999626 7f2904b56700 15 osd.70 71593 require_same_or_newer_map 71593 (i am 71593) 0x1335ab20 2014-12-08 18:14:01.999627 7f28fc345700 20 osd.70 71593 share_map_peer 0x1281f600 already has epoch 71593 2014-12-08 18:14:01.999628 7f28fc345700 7 osd.70 71593 do_notifies osd 20 on 1 PGs -- 2014-12-08 18:14:02.000126 7f28fc345700 10 osd.70 pg_epoch: 71593 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] handle_peering_event: epoch_sent: 71593 epoch_requested: 71593 MNotifyRec from 58 notify: (query_epoch:71593, epoch_sent:71593, info:19.6e( empty local-les=0 n=0 ec=21298 les/c 51101/50495 71591/71591/71591)) features: 0x3fffffffffff 2014-12-08 18:14:02.000134 7f28fcb46700 15 osd.70 pg_epoch: 71593 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71585/71585 71591/71593/71593) [70,78,1] r=0 lpr=71593 pi=71432-71592/6 crt=0'0 mlcod 0'0 inactive NOTIFY] requeue_ops 2014-12-08 18:14:02.000136 7f28fc345700 7 osd.70 pg_epoch: 71593 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] state: handle_pg_notify from osd.58 2014-12-08 18:14:02.000139 7f28fcb46700 10 osd.70 pg_epoch: 71593 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71585/71585 71591/71593/71593) [70,78,1] r=0 lpr=71593 pi=71432-71592/6 crt=0'0 mlcod 0'0 inactive NOTIFY] on_change_cleanup 2014-12-08 18:14:02.000146 7f28fcb46700 10 osd.70 pg_epoch: 71593 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71585/71585 71591/71593/71593) [70,78,1] r=0 lpr=71593 pi=71432-71592/6 crt=0'0 mlcod 0'0 inactive NOTIFY] on_change 2014-12-08 18:14:02.000144 7f28fc345700 10 osd.70 pg_epoch: 71593 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] state: pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] got dup osd.58 info 19.6e( empty local-les=0 n=0 ec=21298 les/c 51101/50495 71591/71591/71591), identical to ours 2014-12-08 18:14:02.000153 7f28fcb46700 20 osd.70 pg_epoch: 71593 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71585/71585 71591/71593/71593) [70,78,1] r=0 lpr=71593 pi=71432-71592/6 crt=0'0 mlcod 0'0 inactive NOTIFY] exit NotTrimming 2014-12-08 18:14:02.000159 7f28fc345700 10 log is not dirty 2014-12-08 18:14:02.000159 7f28fcb46700 20 osd.70 pg_epoch: 71593 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71585/71585 71591/71593/71593) [70,78,1] r=0 lpr=71593 pi=71432-71592/6 crt=0'0 mlcod 0'0 inactive NOTIFY] enter NotTrimming 2014-12-08 18:14:02.000162 7f28fc345700 10 osd.70 pg_epoch: 71593 pg[217.399( v 71377'57 (0'0,71377'57] local-les=71593 n=12 ec=71094 les/c 71590/71583 71591/71592/71592) [70,25,58] r=0 lpr=71592 pi=71444-71591/6 crt=71377'54 lcod 0'0 mlcod 0'0 inactive] handle_peering_event: epoch_sent: 71593 epoch_requested: 71593 FlushedEvt 2014-12-08 18:14:02.000166 7f28fcb46700 15 osd.70 pg_epoch: 71593 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71585/71585 71591/71593/71593) [70,78,1] r=0 lpr=71593 pi=71432-71592/6 crt=0'0 mlcod 0'0 inactive] clear_stats 2014-12-08 18:14:02.000171 7f28fc345700 15 osd.70 pg_epoch: 71593 pg[217.399( v 71377'57 (0'0,71377'57] local-les=71593 n=12 ec=71094 les/c 71590/71583 71591/71592/71592) [70,25,58] r=0 lpr=71592 pi=71444-71591/6 crt=71377'54 lcod 0'0 mlcod 0'0 inactive] requeue_ops 2014-12-08 18:14:02.000172 7f28fcb46700 15 osd.70 pg_epoch: 71593 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71585/71585 71591/71593/71593) [70,78,1] r=0 lpr=71593 pi=71432-71592/6 crt=0'0 mlcod 0'0 inactive] requeue_ops 2014-12-08 18:14:02.000179 7f28fc345700 10 log is not dirty 2014-12-08 18:14:02.000178 7f28fcb46700 10 osd.70 pg_epoch: 71593 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71585/71585 71591/71593/71593) [70,78,1] r=0 lpr=71593 pi=71432-71592/6 crt=0'0 mlcod 0'0 inactive] on_role_change 2014-12-08 18:14:02.000201 7f28fcb46700 15 osd.70 pg_epoch: 71593 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71585/71585 71591/71593/71593) [70,78,1] r=0 lpr=71593 pi=71432-71592/6 crt=0'0 mlcod 0'0 inactive] requeue_ops 2014-12-08 18:14:02.000213 7f28fcb46700 10 osd.70 pg_epoch: 71593 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71585/71585 71591/71593/71593) [70,78,1] r=0 lpr=71593 pi=71432-71592/6 crt=0'0 mlcod 0'0 inactive] cancel_recovery 2014-12-08 18:14:02.000222 7f28fcb46700 10 osd.70 pg_epoch: 71593 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71585/71585 71591/71593/71593) [70,78,1] r=0 lpr=71593 pi=71432-71592/6 crt=0'0 mlcod 0'0 inactive] clear_recovery_state 2014-12-08 18:14:02.000233 7f28fcb46700 10 osd.70 pg_epoch: 71593 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71585/71585 71591/71593/71593) [70,78,1] r=0 lpr=71593 pi=71432-71592/6 crt=0'0 mlcod 0'0 inactive] check_recovery_sources no source osds () went down 2014-12-08 18:14:02.000233 7f28fc345700 10 osd.70 pg_epoch: 71593 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] handle_peering_event: epoch_sent: 71593 epoch_requested: 71593 MNotifyRec from 76 notify: (query_epoch:71593, epoch_sent:71593, info:19.6e( empty local-les=0 n=0 ec=21298 les/c 51101/50495 71591/71591/71591)) features: 0x3fffffffffff 2014-12-08 18:14:02.000248 7f28fcb46700 10 osd.70 pg_epoch: 71593 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71585/71585 71591/71593/71593) [70,78,1] r=0 lpr=71593 pi=71432-71592/6 crt=0'0 mlcod 0'0 inactive] handle_activate_map 2014-12-08 18:14:02.000257 7f28fcb46700 10 osd.70 pg_epoch: 71593 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71585/71585 71591/71593/71593) [70,78,1] r=0 lpr=71593 pi=71432-71592/6 crt=0'0 mlcod 0'0 inactive] update_heartbeat_peers -> 1,70,78 2014-12-08 18:14:02.000264 7f28fcb46700 20 osd.70 71593 need_heartbeat_peer_update 2014-12-08 18:14:02.000254 7f28fc345700 7 osd.70 pg_epoch: 71593 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] state: handle_pg_notify from osd.76 2014-12-08 18:14:02.000265 7f28fcb46700 10 osd.70 pg_epoch: 71593 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71585/71585 71591/71593/71593) [70,78,1] r=0 lpr=71593 pi=71432-71592/6 crt=0'0 mlcod 0'0 inactive] take_waiters 2014-12-08 18:14:02.000272 7f28fcb46700 5 osd.70 pg_epoch: 71593 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71585/71585 71591/71593/71593) [70,78,1] r=0 lpr=71593 pi=71432-71592/6 crt=0'0 mlcod 0'0 inactive] exit Reset 0.000293 1 0.000302 2014-12-08 18:14:02.000267 7f28fc345700 10 osd.70 pg_epoch: 71593 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] state: pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] got dup osd.76 info 19.6e( empty local-les=0 n=0 ec=21298 les/c 51101/50495 71591/71591/71591), identical to ours 2014-12-08 18:14:02.000279 7f28fcb46700 5 osd.70 pg_epoch: 71593 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71585/71585 71591/71593/71593) [70,78,1] r=0 lpr=71593 pi=71432-71592/6 crt=0'0 mlcod 0'0 inactive] enter Started 2014-12-08 18:14:02.000285 7f28fc345700 10 log is not dirty 2014-12-08 18:14:02.000286 7f28fcb46700 5 osd.70 pg_epoch: 71593 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71585/71585 71591/71593/71593) [70,78,1] r=0 lpr=71593 pi=71432-71592/6 crt=0'0 mlcod 0'0 inactive] enter Start 2014-12-08 18:14:02.000291 7f28fcb46700 1 osd.70 pg_epoch: 71593 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71585/71585 71591/71593/71593) [70,78,1] r=0 lpr=71593 pi=71432-71592/6 crt=0'0 mlcod 0'0 inactive] state: transitioning to Primary 2014-12-08 18:14:02.000291 7f28fc345700 10 osd.70 pg_epoch: 71593 pg[218.6c7( v 71581'2 (0'0,71581'2] local-les=71593 n=2 ec=71423 les/c 71590/71583 71591/71592/71592) [70,10,18] r=0 lpr=71592 pi=71432-71591/6 crt=0'0 lcod 0'0 mlcod 0'0 inactive] handle_peering_event: epoch_sent: 71593 epoch_requested: 71593 MNotifyRec from 76 notify: (query_epoch:71593, epoch_sent:71593, info:218.6c7( v 71581'2 lc 0'0 (0'0,71581'2] local-les=71590 n=2 ec=71423 les/c 71590/71583 71591/71592/71592)) features: 0x3fffffffffff 2014-12-08 18:14:02.000299 7f28fcb46700 5 osd.70 pg_epoch: 71593 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71585/71585 71591/71593/71593) [70,78,1] r=0 lpr=71593 pi=71432-71592/6 crt=0'0 mlcod 0'0 inactive] exit Start 0.000014 0 0.000000 2014-12-08 18:14:02.000306 7f28fcb46700 5 osd.70 pg_epoch: 71593 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71585/71585 71591/71593/71593) [70,78,1] r=0 lpr=71593 pi=71432-71592/6 crt=0'0 mlcod 0'0 inactive] enter Started/Primary 2014-12-08 18:14:02.000304 7f28fc345700 10 osd.70 pg_epoch: 71593 pg[218.6c7( v 71581'2 (0'0,71581'2] local-les=71593 n=2 ec=71423 les/c 71590/71583 71591/71592/71592) [70,10,18] r=0 lpr=71592 pi=71432-71591/6 crt=0'0 lcod 0'0 mlcod 0'0 inactive] state: Active: got notify from 76, already have info from that osd, ignoring 2014-12-08 18:14:02.000313 7f28fc345700 10 log is not dirty 2014-12-08 18:14:02.000313 7f28fcb46700 5 osd.70 pg_epoch: 71593 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71585/71585 71591/71593/71593) [70,78,1] r=0 lpr=71593 pi=71432-71592/6 crt=0'0 mlcod 0'0 inactive] enter Started/Primary/Peering 2014-12-08 18:14:02.000319 7f28fcb46700 5 osd.70 pg_epoch: 71593 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71585/71585 71591/71593/71593) [70,78,1] r=0 lpr=71593 pi=71432-71592/6 crt=0'0 mlcod 0'0 peering] enter Started/Primary/Peering/GetInfo 2014-12-08 18:14:02.000317 7f28fc345700 10 osd.70 pg_epoch: 71593 pg[218.6c4( v 71581'3 (0'0,71581'3] local-les=71593 n=3 ec=71423 les/c 71590/71583 71591/71592/71592) [70,76,13] r=0 lpr=71592 pi=71432-71591/6 crt=71580'1 lcod 0'0 mlcod 0'0 inactive] handle_peering_event: epoch_sent: 71593 epoch_requested: 71593 MNotifyRec from 76 notify: (query_epoch:71593, epoch_sent:71593, info:218.6c4( v 71581'3 (0'0,71581'3] local-les=71590 n=3 ec=71423 les/c 71590/71583 71591/71592/71592)) features: 0x3fffffffffff 2014-12-08 18:14:02.000325 7f28fcb46700 10 osd.70 pg_epoch: 71593 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71585/71585 71591/71593/71593) [70,78,1] r=0 lpr=71593 pi=71432-71592/6 crt=0'0 mlcod 0'0 peering] _calc_past_interval_range: already have past intervals back to 71585 2014-12-08 18:14:02.000329 7f28fc345700 10 osd.70 pg_epoch: 71593 pg[218.6c4( v 71581'3 (0'0,71581'3] local-les=71593 n=3 ec=71423 les/c 71590/71583 71591/71592/71592) [70,76,13] r=0 lpr=71592 pi=71432-71591/6 crt=71580'1 lcod 0'0 mlcod 0'0 inactive] state: Active: got notify from 76, already have info from that osd, ignoring 2014-12-08 18:14:02.000340 7f28fc345700 10 log is not dirty 2014-12-08 18:14:02.000333 7f28fcb46700 10 osd.70 pg_epoch: 71593 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71585/71585 71591/71593/71593) [70,78,1] r=0 lpr=71593 pi=71432-71592/6 crt=0'0 mlcod 0'0 peering] PriorSet: build_prior interval(71591-71592 up [70,78,1](70) acting [78,1,14](78) maybe_went_rw) 2014-12-08 18:14:02.000330 7f290a83a700 10 osd.70 pg_epoch: 71593 pg[216.3cd( v 71097'258 (0'0,71097'258] local-les=71593 n=120 ec=70987 les/c 71590/71583 71591/71592/71168) [18,5,70] r=2 lpr=71592 pi=70987-71591/34 crt=71097'256 lcod 0'0 inactive] _activate_committed 71593 telling primary 2014-12-08 18:14:02.000346 7f28fcb46700 10 osd.70 pg_epoch: 71593 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71585/71585 71591/71593/71593) [70,78,1] r=0 lpr=71593 pi=71432-71592/6 crt=0'0 mlcod 0'0 peering] PriorSet: build_prior interval(71590-71590 up [78,1](78) acting [78,1,14](78)) 2014-12-08 18:14:02.000345 7f28fc345700 10 osd.70 pg_epoch: 71593 pg[218.51( v 71581'3 (0'0,71581'3] local-les=71433 n=3 ec=71423 les/c 71585/71583 71591/71593/71593) [70,66,75] r=0 lpr=71593 pi=71432-71592/6 crt=71580'1 lcod 0'0 mlcod 0'0 peering] handle_peering_event: epoch_sent: 71593 epoch_requested: 71593 MNotifyRec from 6 notify: (query_epoch:71593, epoch_sent:71593, info:218.51( v 71581'3 lc 0'0 (0'0,71581'3] local-les=71585 n=3 ec=71423 les/c 71585/71583 71591/71593/71593)) features: 0x3fffffffffff 2014-12-08 18:14:02.000356 7f28fcb46700 10 osd.70 pg_epoch: 71593 pg[218.431( empty local-les=71433 n=0 ec=71423 les/c 71585/71585 71591/71593/71593) [70,78,1] r=0 lpr=71593 pi=71432-71592/6 crt=0'0 mlcod 0'0 peering] PriorSet: build_prior interval(71587-71589 up [78,1](78) acting [78,1](78) maybe_went_rw) -- 2014-12-08 18:14:02.000866 7f28fc345700 10 osd.70 pg_epoch: 71593 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] handle_peering_event: epoch_sent: 71593 epoch_requested: 71593 MNotifyRec from 4 notify: (query_epoch:71593, epoch_sent:71593, info:19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591)) features: 0x3fffffffffff 2014-12-08 18:14:02.000875 7f28fc345700 7 osd.70 pg_epoch: 71593 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] state: handle_pg_notify from osd.4 2014-12-08 18:14:02.000883 7f28fcb46700 10 osd.70 pg_epoch: 71593 pg[218.760( empty local-les=71433 n=0 ec=71423 les/c 71590/71590 71591/71593/71425) [18,70,5] r=1 lpr=71593 pi=71423-71592/10 crt=0'0 inactive NOTIFY] update_heartbeat_peers unchanged 2014-12-08 18:14:02.000890 7f28fcb46700 10 osd.70 pg_epoch: 71593 pg[218.760( empty local-les=71433 n=0 ec=71423 les/c 71590/71590 71591/71593/71425) [18,70,5] r=1 lpr=71593 pi=71423-71592/10 crt=0'0 inactive NOTIFY] take_waiters 2014-12-08 18:14:02.000882 7f28fc345700 10 osd.70 pg_epoch: 71593 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] state: pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] got dup osd.4 info 19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591), identical to ours 2014-12-08 18:14:02.000898 7f28fc345700 10 log is not dirty 2014-12-08 18:14:02.000896 7f28fcb46700 5 osd.70 pg_epoch: 71593 pg[218.760( empty local-les=71433 n=0 ec=71423 les/c 71590/71590 71591/71593/71425) [18,70,5] r=1 lpr=71593 pi=71423-71592/10 crt=0'0 inactive NOTIFY] exit Reset 0.000224 1 0.000230 2014-12-08 18:14:02.000903 7f28fcb46700 5 osd.70 pg_epoch: 71593 pg[218.760( empty local-les=71433 n=0 ec=71423 les/c 71590/71590 71591/71593/71425) [18,70,5] r=1 lpr=71593 pi=71423-71592/10 crt=0'0 inactive NOTIFY] enter Started 2014-12-08 18:14:02.000901 7f28fc345700 10 osd.70 pg_epoch: 71593 pg[217.399( v 71377'57 (0'0,71377'57] local-les=71593 n=12 ec=71094 les/c 71590/71583 71591/71592/71592) [70,25,58] r=0 lpr=71592 pi=71444-71591/6 crt=71377'54 lcod 0'0 mlcod 0'0 inactive] handle_peering_event: epoch_sent: 71593 epoch_requested: 71593 MNotifyRec from 25 notify: (query_epoch:71593, epoch_sent:71593, info:217.399( v 71377'57 (0'0,71377'57] local-les=71590 n=12 ec=71094 les/c 71590/71583 71591/71592/71592)) features: 0x3fffffffffff 2014-12-08 18:14:02.000912 7f28fcb46700 5 osd.70 pg_epoch: 71593 pg[218.760( empty local-les=71433 n=0 ec=71423 les/c 71590/71590 71591/71593/71425) [18,70,5] r=1 lpr=71593 pi=71423-71592/10 crt=0'0 inactive NOTIFY] enter Start 2014-12-08 18:14:02.000914 7f28fc345700 10 osd.70 pg_epoch: 71593 pg[217.399( v 71377'57 (0'0,71377'57] local-les=71593 n=12 ec=71094 les/c 71590/71583 71591/71592/71592) [70,25,58] r=0 lpr=71592 pi=71444-71591/6 crt=71377'54 lcod 0'0 mlcod 0'0 inactive] state: Active: got notify from 25, already have info from that osd, ignoring 2014-12-08 18:14:02.000922 7f28fc345700 10 log is not dirty 2014-12-08 18:14:02.000918 7f28fcb46700 1 osd.70 pg_epoch: 71593 pg[218.760( empty local-les=71433 n=0 ec=71423 les/c 71590/71590 71591/71593/71425) [18,70,5] r=1 lpr=71593 pi=71423-71592/10 crt=0'0 inactive NOTIFY] state: transitioning to Stray 2014-12-08 18:14:02.000926 7f28fcb46700 5 osd.70 pg_epoch: 71593 pg[218.760( empty local-les=71433 n=0 ec=71423 les/c 71590/71590 71591/71593/71425) [18,70,5] r=1 lpr=71593 pi=71423-71592/10 crt=0'0 inactive NOTIFY] exit Start 0.000014 0 0.000000 2014-12-08 18:14:02.000926 7f28fc345700 10 osd.70 pg_epoch: 71593 pg[218.71a( v 71581'3 (0'0,71581'3] local-les=71433 n=3 ec=71423 les/c 71589/71589 71591/71593/71425) [10,70,4] r=1 lpr=71593 pi=71423-71592/10 crt=71580'1 lcod 0'0 inactive NOTIFY] handle_peering_event: epoch_sent: 71593 epoch_requested: 71593 MQuery from 10 query_epoch 71593 query: query(info 0'0) 2014-12-08 18:14:02.000933 7f28fcb46700 5 osd.70 pg_epoch: 71593 pg[218.760( empty local-les=71433 n=0 ec=71423 les/c 71590/71590 71591/71593/71425) [18,70,5] r=1 lpr=71593 pi=71423-71592/10 crt=0'0 inactive NOTIFY] enter Started/Stray 2014-12-08 18:14:02.000936 7f28fc345700 10 osd.70 pg_epoch: 71593 pg[218.71a( v 71581'3 (0'0,71581'3] local-les=71433 n=3 ec=71423 les/c 71589/71589 71591/71593/71425) [10,70,4] r=1 lpr=71593 pi=71423-71592/10 crt=71580'1 lcod 0'0 inactive NOTIFY] sending info 2014-12-08 18:14:02.000941 7f28fcb46700 20 osd.70 pg_epoch: 71593 pg[218.760( empty local-les=71433 n=0 ec=71423 les/c 71590/71590 71591/71593/71425) [18,70,5] r=1 lpr=71593 pi=71423-71592/10 crt=0'0 inactive NOTIFY] handle_activate_map: Not dirtying info: last_persisted is 71592 while current is 71593 2014-12-08 18:14:02.000948 7f28fcb46700 10 osd.70 pg_epoch: 71593 pg[218.760( empty local-les=71433 n=0 ec=71423 les/c 71590/71590 71591/71593/71425) [18,70,5] r=1 lpr=71593 pi=71423-71592/10 crt=0'0 inactive NOTIFY] handle_peering_event: epoch_sent: 71593 epoch_requested: 71593 NullEvt 2014-12-08 18:14:02.000959 7f28fc345700 10 log is not dirty 2014-12-08 18:14:02.000962 7f28fc345700 10 osd.70 pg_epoch: 71593 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] handle_peering_event: epoch_sent: 71593 epoch_requested: 71593 MNotifyRec from 30 notify: (query_epoch:71593, epoch_sent:71593, info:19.6e( empty local-les=0 n=0 ec=21298 les/c 51101/50495 71591/71591/71591)) features: 0x3fffffffffff 2014-12-08 18:14:02.000971 7f28fc345700 7 osd.70 pg_epoch: 71593 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] state: handle_pg_notify from osd.30 2014-12-08 18:14:02.000978 7f28fcb46700 10 log is not dirty 2014-12-08 18:14:02.000978 7f28fc345700 10 osd.70 pg_epoch: 71593 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] state: pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] got dup osd.30 info 19.6e( empty local-les=0 n=0 ec=21298 les/c 51101/50495 71591/71591/71591), identical to ours 2014-12-08 18:14:02.000992 7f28fc345700 10 log is not dirty 2014-12-08 18:14:02.000995 7f28fc345700 10 osd.70 pg_epoch: 71593 pg[218.3dd( v 71581'2 (0'0,71581'2] local-les=71437 n=2 ec=71423 les/c 71585/71583 71591/71593/71593) [70,66,14] r=0 lpr=71593 pi=71432-71592/6 crt=0'0 lcod 0'0 mlcod 0'0 peering] handle_peering_event: epoch_sent: 71593 epoch_requested: 71593 MNotifyRec from 8 notify: (query_epoch:71593, epoch_sent:71593, info:218.3dd( v 71581'2 lc 0'0 (0'0,71581'2] local-les=71585 n=2 ec=71423 les/c 71585/71583 71591/71593/71593)) features: 0x3fffffffffff 2014-12-08 18:14:02.001003 7f28fc345700 10 osd.70 pg_epoch: 71593 pg[218.3dd( v 71581'2 (0'0,71581'2] local-les=71437 n=2 ec=71423 les/c 71585/71583 71591/71593/71593) [70,66,14] r=0 lpr=71593 pi=71432-71592/6 crt=0'0 lcod 0'0 mlcod 0'0 peering] got osd.8 218.3dd( v 71581'2 lc 0'0 (0'0,71581'2] local-les=71585 n=2 ec=71423 les/c 71585/71583 71591/71593/71593) 2014-12-08 18:14:02.001002 7f28fcb46700 10 osd.70 pg_epoch: 71592 pg[218.589( empty local-les=71433 n=0 ec=71423 les/c 71585/71585 71591/71591/71425) [18,70,6]/[18,6,21] r=-1 lpr=71591 pi=71423-71590/8 crt=0'0 remapped NOTIFY] handle_advance_map [18,70,6]/[18,70,6] -- 18/18 2014-12-08 18:14:02.001015 7f28fcb46700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:14:02.001015 7f28fc345700 10 osd.70 pg_epoch: 71593 pg[218.3dd( v 71581'2 (0'0,71581'2] local-les=71437 n=2 ec=71423 les/c 71585/71583 71591/71593/71593) [70,66,14] r=0 lpr=71593 pi=71432-71592/6 crt=0'0 lcod 0'0 mlcod 0'0 peering] osd.8 has stray content: 218.3dd( v 71581'2 lc 0'0 (0'0,71581'2] local-les=71585 n=2 ec=71423 les/c 71585/71583 71591/71593/71593) 2014-12-08 18:14:02.001018 7f28fcb46700 10 osd.70 pg_epoch: 71593 pg[218.589( empty local-les=71433 n=0 ec=71423 les/c 71585/71585 71591/71591/71425) [18,70,6]/[18,6,21] r=-1 lpr=71591 pi=71423-71590/8 crt=0'0 remapped NOTIFY] state: Started advmap 2014-12-08 18:14:02.001026 7f28fc345700 10 osd.70 pg_epoch: 71593 pg[218.3dd( v 71581'2 (0'0,71581'2] local-les=71437 n=2 ec=71423 les/c 71585/71583 71591/71593/71593) [70,66,14] r=0 lpr=71593 pi=71432-71592/6 crt=0'0 lcod 0'0 mlcod 0'0 peering] update_heartbeat_peers 13,14,66,70 -> 8,13,14,66,70 2014-12-08 18:14:02.001028 7f28fcb46700 20 osd.70 pg_epoch: 71593 pg[218.589( empty local-les=71433 n=0 ec=71423 les/c 71585/71585 71591/71591/71425) [18,70,6]/[18,6,21] r=-1 lpr=71591 pi=71423-71590/8 crt=0'0 remapped NOTIFY] new interval newup [18,70,6] newacting [18,70,6] 2014-12-08 18:14:02.001034 7f28fc345700 20 osd.70 71593 need_heartbeat_peer_update 2014-12-08 18:14:02.001035 7f28fcb46700 10 osd.70 pg_epoch: 71593 pg[218.589( empty local-les=71433 n=0 ec=71423 les/c 71585/71585 71591/71591/71425) [18,70,6]/[18,6,21] r=-1 lpr=71591 pi=71423-71590/8 crt=0'0 remapped NOTIFY] state: should_restart_peering, transitioning to Reset 2014-12-08 18:14:02.001035 7f28fc345700 20 osd.70 pg_epoch: 71593 pg[218.3dd( v 71581'2 (0'0,71581'2] local-les=71437 n=2 ec=71423 les/c 71585/71583 71591/71593/71593) [70,66,14] r=0 lpr=71593 pi=71432-71592/6 crt=0'0 lcod 0'0 mlcod 0'0 peering] state: Adding osd: 8 features: 3fffffffffff 2014-12-08 18:14:02.001044 7f28fc345700 10 log is not dirty 2014-12-08 18:14:02.001043 7f28fcb46700 5 osd.70 pg_epoch: 71593 pg[218.589( empty local-les=71433 n=0 ec=71423 les/c 71585/71585 71591/71591/71425) [18,70,6]/[18,6,21] r=-1 lpr=71591 pi=71423-71590/8 crt=0'0 remapped NOTIFY] exit Started/Stray 0.997633 5 0.000105 2014-12-08 18:14:02.001051 7f28fcb46700 5 osd.70 pg_epoch: 71593 pg[218.589( empty local-les=71433 n=0 ec=71423 les/c 71585/71585 71591/71591/71425) [18,70,6]/[18,6,21] r=-1 lpr=71591 pi=71423-71590/8 crt=0'0 remapped NOTIFY] exit Started 0.997676 0 0.000000 2014-12-08 18:14:02.001048 7f28fc345700 10 osd.70 pg_epoch: 71593 pg[216.3c5( v 71097'233 (0'0,71097'233] local-les=71593 n=113 ec=70987 les/c 71585/71583 71591/71592/71592) [70,5,20] r=0 lpr=71592 pi=71444-71591/5 crt=71097'233 lcod 0'0 mlcod 0'0 inactive] handle_peering_event: epoch_sent: 71593 epoch_requested: 71593 MNotifyRec from 5 notify: (query_epoch:71593, epoch_sent:71593, info:216.3c5( v 71097'233 (0'0,71097'233] local-les=71585 n=113 ec=70987 les/c 71585/71583 71591/71592/71592)) features: 0x3fffffffffff 2014-12-08 18:14:02.001058 7f28fcb46700 5 osd.70 pg_epoch: 71593 pg[218.589( empty local-les=71433 n=0 ec=71423 les/c 71585/71585 71591/71591/71425) [18,70,6]/[18,6,21] r=-1 lpr=71591 pi=71423-71590/8 crt=0'0 remapped NOTIFY] enter Reset 2014-12-08 18:14:02.001061 7f28fc345700 10 osd.70 pg_epoch: 71593 pg[216.3c5( v 71097'233 (0'0,71097'233] local-les=71593 n=113 ec=70987 les/c 71585/71583 71591/71592/71592) [70,5,20] r=0 lpr=71592 pi=71444-71591/5 crt=71097'233 lcod 0'0 mlcod 0'0 inactive] state: Active: got notify from 5, already have info from that osd, ignoring 2014-12-08 18:14:02.001069 7f28fc345700 10 log is not dirty 2014-12-08 18:14:02.001065 7f28fcb46700 20 osd.70 pg_epoch: 71593 pg[218.589( empty local-les=71433 n=0 ec=71423 les/c 71585/71585 71591/71591/71425) [18,70,6]/[18,6,21] r=-1 lpr=71591 pi=71423-71590/8 crt=0'0 remapped NOTIFY] set_last_peering_reset 71593 -- 2014-12-08 18:14:02.283055 7f28fbb44700 10 osd.70 71593 dequeue_op 0x4481600 prio 63 cost 0 latency 0.000230 osd_op(client.6634234.0:1291 rbd_data.124045103eaad3.0000000000000270 [sparse-read 0~8388608] 19.d0d3a06e RETRY=4 ack+retry+read e71593) v4 pg pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] 2014-12-08 18:14:02.283106 7f28fbb44700 20 osd.70 pg_epoch: 71593 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] op_has_sufficient_caps pool=19 (volumes ) owner=0 need_read_cap=1 need_write_cap=0 need_class_read_cap=0 need_class_write_cap=0 -> yes 2014-12-08 18:14:02.283138 7f28d9fb6700 20 osd.70 71593 should_share_map client.6634234 172.16.5.105:0/1017156 71593 2014-12-08 18:14:02.283133 7f28fbb44700 10 osd.70 pg_epoch: 71593 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] can_handle_while_inactive: 0x4481600 2014-12-08 18:14:02.283147 7f28fbb44700 10 osd.70 71593 dequeue_op 0x4481600 finish 2014-12-08 18:14:02.283156 7f28d9fb6700 15 osd.70 71593 enqueue_op 0x1285f600 prio 63 cost 0 latency 0.000120 osd_op(client.6634234.0:1569 rbd_data.124045103eaad3.00000000000002fd [sparse-read 0~8388608] 19.2b4576ee RETRY=4 ack+retry+read e71593) v4 2014-12-08 18:14:02.283200 7f28f933f700 10 osd.70 71593 dequeue_op 0x1285f600 prio 63 cost 0 latency 0.000163 osd_op(client.6634234.0:1569 rbd_data.124045103eaad3.00000000000002fd [sparse-read 0~8388608] 19.2b4576ee RETRY=4 ack+retry+read e71593) v4 pg pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] 2014-12-08 18:14:02.283257 7f28d9fb6700 20 osd.70 71593 should_share_map client.6634234 172.16.5.105:0/1017156 71593 2014-12-08 18:14:02.283254 7f28f933f700 20 osd.70 pg_epoch: 71593 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] op_has_sufficient_caps pool=19 (volumes ) owner=0 need_read_cap=1 need_write_cap=0 need_class_read_cap=0 need_class_write_cap=0 -> yes 2014-12-08 18:14:02.283277 7f28d9fb6700 15 osd.70 71593 enqueue_op 0x447fe00 prio 63 cost 0 latency 0.000101 osd_op(client.6634234.0:1598 rbd_data.124045103eaad3.0000000000000310 [sparse-read 0~8388608] 19.bd03a26e RETRY=4 ack+retry+read e71593) v4 2014-12-08 18:14:02.283282 7f28f933f700 10 osd.70 pg_epoch: 71593 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] can_handle_while_inactive: 0x1285f600 2014-12-08 18:14:02.283299 7f28f933f700 10 osd.70 71593 dequeue_op 0x1285f600 finish 2014-12-08 18:14:02.283316 7f28fbb44700 10 osd.70 71593 dequeue_op 0x447fe00 prio 63 cost 0 latency 0.000139 osd_op(client.6634234.0:1598 rbd_data.124045103eaad3.0000000000000310 [sparse-read 0~8388608] 19.bd03a26e RETRY=4 ack+retry+read e71593) v4 pg pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] 2014-12-08 18:14:02.283355 7f28fbb44700 20 osd.70 pg_epoch: 71593 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] op_has_sufficient_caps pool=19 (volumes ) owner=0 need_read_cap=1 need_write_cap=0 need_class_read_cap=0 need_class_write_cap=0 -> yes 2014-12-08 18:14:02.283370 7f28fbb44700 10 osd.70 pg_epoch: 71593 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] can_handle_while_inactive: 0x447fe00 2014-12-08 18:14:02.283380 7f28d9fb6700 20 osd.70 71593 should_share_map client.6634234 172.16.5.105:0/1017156 71593 2014-12-08 18:14:02.283382 7f28fbb44700 10 osd.70 71593 dequeue_op 0x447fe00 finish 2014-12-08 18:14:02.283397 7f28d9fb6700 15 osd.70 71593 enqueue_op 0x1285ff00 prio 63 cost 0 latency 0.000098 osd_op(client.6634234.0:1799 rbd_data.124045103eaad3.0000000000000378 [sparse-read 0~8388608] 19.e8b6396e RETRY=4 ack+retry+read e71593) v4 2014-12-08 18:14:02.283429 7f28f933f700 10 osd.70 71593 dequeue_op 0x1285ff00 prio 63 cost 0 latency 0.000129 osd_op(client.6634234.0:1799 rbd_data.124045103eaad3.0000000000000378 [sparse-read 0~8388608] 19.e8b6396e RETRY=4 ack+retry+read e71593) v4 pg pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] 2014-12-08 18:14:02.283467 7f28f933f700 20 osd.70 pg_epoch: 71593 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] op_has_sufficient_caps pool=19 (volumes ) owner=0 need_read_cap=1 need_write_cap=0 need_class_read_cap=0 need_class_write_cap=0 -> yes 2014-12-08 18:14:02.283496 7f28f933f700 10 osd.70 pg_epoch: 71593 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] can_handle_while_inactive: 0x1285ff00 2014-12-08 18:14:02.283509 7f28f933f700 10 osd.70 71593 dequeue_op 0x1285ff00 finish 2014-12-08 18:14:02.283529 7f28d9fb6700 20 osd.70 71593 should_share_map client.6634234 172.16.5.105:0/1017156 71593 2014-12-08 18:14:02.283547 7f28d9fb6700 15 osd.70 71593 enqueue_op 0x1285f100 prio 63 cost 0 latency 0.000128 osd_op(client.6634234.0:2106 rbd_data.124045103eaad3.0000000000000419 [sparse-read 0~8388608] 19.fe2a22ee RETRY=4 ack+retry+read e71593) v4 2014-12-08 18:14:02.283573 7f2904b56700 10 osd.70 71593 do_waiters -- start 2014-12-08 18:14:02.283575 7f2904b56700 10 osd.70 71593 do_waiters -- finish 2014-12-08 18:14:02.283577 7f2904b56700 20 osd.70 71593 _dispatch 0x13f54580 pg_info(1 pgs e71593:217.36e(4)) v4 2014-12-08 18:14:02.283606 7f2904b56700 7 osd.70 71593 handle_pg_info pg_info(1 pgs e71593:217.36e(4)) v4 from osd.69 2014-12-08 18:14:02.283584 7f28fbb44700 10 osd.70 71593 dequeue_op 0x1285f100 prio 63 cost 0 latency 0.000166 osd_op(client.6634234.0:2106 rbd_data.124045103eaad3.0000000000000419 [sparse-read 0~8388608] 19.fe2a22ee RETRY=4 ack+retry+read e71593) v4 pg pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] 2014-12-08 18:14:02.283617 7f2904b56700 15 osd.70 71593 require_same_or_newer_map 71593 (i am 71593) 0x13f54580 2014-12-08 18:14:02.283647 7f28fbb44700 20 osd.70 pg_epoch: 71593 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] op_has_sufficient_caps pool=19 (volumes ) owner=0 need_read_cap=1 need_write_cap=0 need_class_read_cap=0 need_class_write_cap=0 -> yes 2014-12-08 18:14:02.283674 7f28fbb44700 10 osd.70 pg_epoch: 71593 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] can_handle_while_inactive: 0x1285f100 2014-12-08 18:14:02.283683 7f28fbb44700 10 osd.70 71593 dequeue_op 0x1285f100 finish 2014-12-08 18:14:02.283697 7f2904b56700 10 osd.70 71593 do_waiters -- start 2014-12-08 18:14:02.283700 7f2904b56700 10 osd.70 71593 do_waiters -- finish 2014-12-08 18:14:02.283704 7f28d9fb6700 20 osd.70 71593 should_share_map client.6634234 172.16.5.105:0/1017156 71593 2014-12-08 18:14:02.283695 7f28fcb46700 10 osd.70 pg_epoch: 71593 pg[217.36e( v 71377'4109 (71377'1108,71377'4109] local-les=71440 n=11 ec=71094 les/c 71583/71583 71591/71591/71096) [69,18,70] r=2 lpr=71591 pi=71183-71590/21 crt=71377'4106 lcod 0'0 inactive NOTIFY] handle_peering_event: epoch_sent: 71593 epoch_requested: 71593 MInfoRec from 69 info: 217.36e( v 71377'4109 (71377'1108,71377'4109] local-les=71593 n=11 ec=71094 les/c 71583/71583 71591/71591/71096) 2014-12-08 18:14:02.283727 7f28d9fb6700 15 osd.70 71593 enqueue_op 0x1285f200 prio 63 cost 0 latency 0.000152 osd_op(client.6634234.0:2303 rbd_data.124045103eaad3.000000000000047c [sparse-read 0~8388608] 19.16b8f4ee RETRY=4 ack+retry+read e71593) v4 2014-12-08 18:14:02.283724 7f28fcb46700 10 osd.70 pg_epoch: 71593 pg[217.36e( v 71377'4109 (71377'1108,71377'4109] local-les=71440 n=11 ec=71094 les/c 71583/71583 71591/71591/71096) [69,18,70] r=2 lpr=71591 pi=71183-71590/21 crt=71377'4106 lcod 0'0 inactive NOTIFY] state: got info from osd.69 217.36e( v 71377'4109 (71377'1108,71377'4109] local-les=71593 n=11 ec=71094 les/c 71583/71583 71591/71591/71096) 2014-12-08 18:14:02.283738 7f28fcb46700 5 osd.70 pg_epoch: 71593 pg[217.36e( v 71377'4109 (71377'1108,71377'4109] local-les=71440 n=11 ec=71094 les/c 71583/71583 71591/71591/71096) [69,18,70] r=2 lpr=71591 pi=71183-71590/21 crt=71377'4106 lcod 0'0 inactive NOTIFY] exit Started/Stray 1.624856 10 0.000328 2014-12-08 18:14:02.283750 7f28fcb46700 5 osd.70 pg_epoch: 71593 pg[217.36e( v 71377'4109 (71377'1108,71377'4109] local-les=71440 n=11 ec=71094 les/c 71583/71583 71591/71591/71096) [69,18,70] r=2 lpr=71591 pi=71183-71590/21 crt=71377'4106 lcod 0'0 inactive NOTIFY] enter Started/ReplicaActive 2014-12-08 18:14:02.283763 7f28fcb46700 5 osd.70 pg_epoch: 71593 pg[217.36e( v 71377'4109 (71377'1108,71377'4109] local-les=71440 n=11 ec=71094 les/c 71583/71583 71591/71591/71096) [69,18,70] r=2 lpr=71591 pi=71183-71590/21 crt=71377'4106 lcod 0'0 inactive NOTIFY] enter Started/ReplicaActive/RepNotRecovering 2014-12-08 18:14:02.283773 7f28fcb46700 10 osd.70 pg_epoch: 71593 pg[217.36e( v 71377'4109 (71377'1108,71377'4109] local-les=71440 n=11 ec=71094 les/c 71583/71583 71591/71591/71096) [69,18,70] r=2 lpr=71591 pi=71183-71590/21 crt=71377'4106 lcod 0'0 inactive NOTIFY] state: In ReplicaActive, about to call activate 2014-12-08 18:14:02.283784 7f28fcb46700 10 osd.70 pg_epoch: 71593 pg[217.36e( v 71377'4109 (71377'1108,71377'4109] local-les=71593 n=11 ec=71094 les/c 71583/71583 71591/71591/71096) [69,18,70] r=2 lpr=71591 pi=71183-71590/21 crt=71377'4106 lcod 0'0 inactive] activate - no missing, moving last_complete 71377'4109 -> 71377'4109 2014-12-08 18:14:02.283769 7f28f933f700 10 osd.70 71593 dequeue_op 0x1285f200 prio 63 cost 0 latency 0.000194 osd_op(client.6634234.0:2303 rbd_data.124045103eaad3.000000000000047c [sparse-read 0~8388608] 19.16b8f4ee RETRY=4 ack+retry+read e71593) v4 pg pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] 2014-12-08 18:14:02.283813 7f28f933f700 20 osd.70 pg_epoch: 71593 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] op_has_sufficient_caps pool=19 (volumes ) owner=0 need_read_cap=1 need_write_cap=0 need_class_read_cap=0 need_class_write_cap=0 -> yes 2014-12-08 18:14:02.283835 7f28f933f700 10 osd.70 pg_epoch: 71593 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] can_handle_while_inactive: 0x1285f200 2014-12-08 18:14:02.283851 7f28f933f700 10 osd.70 71593 dequeue_op 0x1285f200 finish 2014-12-08 18:14:02.283850 7f28d9fb6700 20 osd.70 71593 should_share_map client.6634234 172.16.5.105:0/1017156 71593 2014-12-08 18:14:02.283872 7f28d9fb6700 15 osd.70 71593 enqueue_op 0x1285fa00 prio 63 cost 0 latency 0.000117 osd_op(client.6634234.0:2335 rbd_data.124045103eaad3.000000000000048c [sparse-read 0~8388608] 19.d02440ee RETRY=4 ack+retry+read e71593) v4 2014-12-08 18:14:02.283907 7f28fbb44700 10 osd.70 71593 dequeue_op 0x1285fa00 prio 63 cost 0 latency 0.000152 osd_op(client.6634234.0:2335 rbd_data.124045103eaad3.000000000000048c [sparse-read 0~8388608] 19.d02440ee RETRY=4 ack+retry+read e71593) v4 pg pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] 2014-12-08 18:14:02.283943 7f28fbb44700 20 osd.70 pg_epoch: 71593 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] op_has_sufficient_caps pool=19 (volumes ) owner=0 need_read_cap=1 need_write_cap=0 need_class_read_cap=0 need_class_write_cap=0 -> yes 2014-12-08 18:14:02.283957 7f28fbb44700 10 osd.70 pg_epoch: 71593 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] can_handle_while_inactive: 0x1285fa00 2014-12-08 18:14:02.283966 7f28fbb44700 10 osd.70 71593 dequeue_op 0x1285fa00 finish 2014-12-08 18:14:02.284081 7f28fcb46700 10 osd.70 pg_epoch: 71593 pg[217.36e( v 71377'4109 (71377'1108,71377'4109] local-les=71593 n=11 ec=71094 les/c 71583/71583 71591/71591/71096) [69,18,70] r=2 lpr=71591 pi=71183-71590/21 crt=71377'4106 lcod 0'0 inactive] state: Activate Finished 2014-12-08 18:14:02.284100 7f28d9fb6700 20 osd.70 71593 should_share_map client.6634234 172.16.5.105:0/1017156 71593 2014-12-08 18:14:02.284119 7f28d9fb6700 15 osd.70 71593 enqueue_op 0x1285f000 prio 63 cost 0 latency 0.000221 osd_op(client.6634234.0:2599 rbd_data.124045103eaad3.0000000000000517 [sparse-read 0~8388608] 19.bc52096e RETRY=4 ack+retry+read e71593) v4 2014-12-08 18:14:02.284149 7f28fcb46700 10 log is not dirty 2014-12-08 18:14:02.284158 7f28f933f700 10 osd.70 71593 dequeue_op 0x1285f000 prio 63 cost 0 latency 0.000260 osd_op(client.6634234.0:2599 rbd_data.124045103eaad3.0000000000000517 [sparse-read 0~8388608] 19.bc52096e RETRY=4 ack+retry+read e71593) v4 pg pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] 2014-12-08 18:14:02.284234 7f28f933f700 20 osd.70 pg_epoch: 71593 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] op_has_sufficient_caps pool=19 (volumes ) owner=0 need_read_cap=1 need_write_cap=0 need_class_read_cap=0 need_class_write_cap=0 -> yes 2014-12-08 18:14:02.284250 7f28f933f700 10 osd.70 pg_epoch: 71593 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] can_handle_while_inactive: 0x1285f000 2014-12-08 18:14:02.284263 7f28d9fb6700 20 osd.70 71593 should_share_map client.6634234 172.16.5.105:0/1017156 71593 2014-12-08 18:14:02.284266 7f28f933f700 10 osd.70 71593 dequeue_op 0x1285f000 finish 2014-12-08 18:14:02.284284 7f28d9fb6700 15 osd.70 71593 enqueue_op 0x1285ef00 prio 63 cost 0 latency 0.000140 osd_op(client.6634234.0:3022 rbd_data.124045103eaad3.00000000000005eb [sparse-read 0~8388608] 19.d827eee RETRY=4 ack+retry+read e71593) v4 2014-12-08 18:14:02.284312 7f2904b56700 10 osd.70 71593 do_waiters -- start 2014-12-08 18:14:02.284316 7f2904b56700 10 osd.70 71593 do_waiters -- finish 2014-12-08 18:14:02.284318 7f2904b56700 20 osd.70 71593 _dispatch 0x134caa00 pg_log(218.36 epoch 71593 log log((0'0,0'0], crt=0'0) query_epoch 71593) v4 2014-12-08 18:14:02.284317 7f28fbb44700 10 osd.70 71593 dequeue_op 0x1285ef00 prio 63 cost 0 latency 0.000172 osd_op(client.6634234.0:3022 rbd_data.124045103eaad3.00000000000005eb [sparse-read 0~8388608] 19.d827eee RETRY=4 ack+retry+read e71593) v4 pg pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] 2014-12-08 18:14:02.284365 7f2904b56700 7 osd.70 71593 handle_pg_log pg_log(218.36 epoch 71593 log log((0'0,0'0], crt=0'0) query_epoch 71593) v4 from osd.69 2014-12-08 18:14:02.284370 7f2904b56700 15 osd.70 71593 require_same_or_newer_map 71593 (i am 71593) 0x134caa00 2014-12-08 18:14:02.284356 7f28fbb44700 20 osd.70 pg_epoch: 71593 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] op_has_sufficient_caps pool=19 (volumes ) owner=0 need_read_cap=1 need_write_cap=0 need_class_read_cap=0 need_class_write_cap=0 -> yes 2014-12-08 18:14:02.284380 7f28fbb44700 10 osd.70 pg_epoch: 71593 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] can_handle_while_inactive: 0x1285ef00 2014-12-08 18:14:02.284395 7f28fbb44700 10 osd.70 71593 dequeue_op 0x1285ef00 finish 2014-12-08 18:14:02.284407 7f28d9fb6700 20 osd.70 71593 should_share_map client.6634234 172.16.5.105:0/1017156 71593 2014-12-08 18:14:02.284435 7f28d9fb6700 15 osd.70 71593 enqueue_op 0x1285fe00 prio 63 cost 0 latency 0.000127 osd_op(client.6634234.0:3146 rbd_data.124045103eaad3.0000000000000630 [sparse-read 0~8388608] 19.d00e23ee RETRY=4 ack+retry+read e71593) v4 2014-12-08 18:14:02.284429 7f28fc345700 10 osd.70 pg_epoch: 71593 pg[218.36( empty local-les=71438 n=0 ec=71423 les/c 71438/71438 71591/71591/71587) [69,70,77] r=1 lpr=71591 pi=71423-71590/8 crt=0'0 inactive NOTIFY] handle_peering_event: epoch_sent: 71593 epoch_requested: 71593 MLogRec from 69 2014-12-08 18:14:02.284457 7f28fc345700 10 osd.70 pg_epoch: 71593 pg[218.36( empty local-les=71438 n=0 ec=71423 les/c 71438/71438 71591/71591/71587) [69,70,77] r=1 lpr=71591 pi=71423-71590/8 crt=0'0 inactive NOTIFY] state: got info+log from osd.69 218.36( empty local-les=71593 n=0 ec=71423 les/c 71585/71585 71591/71591/71587) log((0'0,0'0], crt=0'0) 2014-12-08 18:14:02.284473 7f28fc345700 10 merge_log log((0'0,0'0], crt=0'0) from osd.69 into log((0'0,0'0], crt=0'0) 2014-12-08 18:14:02.284475 7f2904b56700 10 osd.70 71593 do_waiters -- start 2014-12-08 18:14:02.284480 7f2904b56700 10 osd.70 71593 do_waiters -- finish 2014-12-08 18:14:02.284479 7f28fc345700 10 merge_log result log((0'0,0'0], crt=0'0) missing(0) changed=0 2014-12-08 18:14:02.284485 7f28fc345700 5 osd.70 pg_epoch: 71593 pg[218.36( empty local-les=71438 n=0 ec=71423 les/c 71438/71438 71591/71591/71587) [69,70,77] r=1 lpr=71591 pi=71423-71590/8 crt=0'0 inactive NOTIFY] exit Started/Stray 1.612859 10 0.000390 2014-12-08 18:14:02.284501 7f28fc345700 5 osd.70 pg_epoch: 71593 pg[218.36( empty local-les=71438 n=0 ec=71423 les/c 71438/71438 71591/71591/71587) [69,70,77] r=1 lpr=71591 pi=71423-71590/8 crt=0'0 inactive NOTIFY] enter Started/ReplicaActive 2014-12-08 18:14:02.284532 7f28fc345700 5 osd.70 pg_epoch: 71593 pg[218.36( empty local-les=71438 n=0 ec=71423 les/c 71438/71438 71591/71591/71587) [69,70,77] r=1 lpr=71591 pi=71423-71590/8 crt=0'0 inactive NOTIFY] enter Started/ReplicaActive/RepNotRecovering 2014-12-08 18:14:02.284546 7f28fc345700 10 osd.70 pg_epoch: 71593 pg[218.36( empty local-les=71438 n=0 ec=71423 les/c 71438/71438 71591/71591/71587) [69,70,77] r=1 lpr=71591 pi=71423-71590/8 crt=0'0 inactive NOTIFY] state: In ReplicaActive, about to call activate 2014-12-08 18:14:02.284540 7f28f933f700 10 osd.70 71593 dequeue_op 0x1285fe00 prio 63 cost 0 latency 0.000232 osd_op(client.6634234.0:3146 rbd_data.124045103eaad3.0000000000000630 [sparse-read 0~8388608] 19.d00e23ee RETRY=4 ack+retry+read e71593) v4 pg pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] 2014-12-08 18:14:02.284561 7f28fc345700 10 osd.70 pg_epoch: 71593 pg[218.36( empty local-les=71593 n=0 ec=71423 les/c 71438/71438 71591/71591/71587) [69,70,77] r=1 lpr=71591 pi=71423-71590/8 crt=0'0 inactive] activate - no missing, moving last_complete 0'0 -> 0'0 2014-12-08 18:14:02.284575 7f28fc345700 10 osd.70 pg_epoch: 71593 pg[218.36( empty local-les=71593 n=0 ec=71423 les/c 71438/71438 71591/71591/71587) [69,70,77] r=1 lpr=71591 pi=71423-71590/8 crt=0'0 inactive] state: Activate Finished 2014-12-08 18:14:02.284591 7f28f933f700 20 osd.70 pg_epoch: 71593 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] op_has_sufficient_caps pool=19 (volumes ) owner=0 need_read_cap=1 need_write_cap=0 need_class_read_cap=0 need_class_write_cap=0 -> yes 2014-12-08 18:14:02.284606 7f290a83a700 10 osd.70 pg_epoch: 71593 pg[218.1f6( empty local-les=71593 n=0 ec=71423 les/c 71585/71585 71591/71591/71425) [69,70,20] r=1 lpr=71591 pi=71423-71590/8 crt=0'0 inactive] _activate_committed 71593 telling primary 2014-12-08 18:14:02.284610 7f28f933f700 10 osd.70 pg_epoch: 71593 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] can_handle_while_inactive: 0x1285fe00 2014-12-08 18:14:02.284626 7f28f933f700 10 osd.70 71593 dequeue_op 0x1285fe00 finish 2014-12-08 18:14:02.284628 7f290a83a700 20 osd.70 71593 share_map_peer 0x11a2eb00 already has epoch 71593 2014-12-08 18:14:02.284642 7f28fc345700 10 log is not dirty 2014-12-08 18:14:02.284649 7f290a83a700 10 osd.70 pg_epoch: 71593 pg[218.1f6( empty local-les=71593 n=0 ec=71423 les/c 71585/71585 71591/71591/71425) [69,70,20] r=1 lpr=71591 pi=71423-71590/8 crt=0'0 active] flushed 2014-12-08 18:14:02.284699 7f28fcb46700 10 osd.70 pg_epoch: 71593 pg[218.1f6( empty local-les=71593 n=0 ec=71423 les/c 71585/71585 71591/71591/71425) [69,70,20] r=1 lpr=71591 pi=71423-71590/8 crt=0'0 active] handle_peering_event: epoch_sent: 71593 epoch_requested: 71593 FlushedEvt 2014-12-08 18:14:02.284730 7f28fcb46700 15 osd.70 pg_epoch: 71593 pg[218.1f6( empty local-les=71593 n=0 ec=71423 les/c 71585/71585 71591/71591/71425) [69,70,20] r=1 lpr=71591 pi=71423-71590/8 crt=0'0 active] requeue_ops 2014-12-08 18:14:02.284743 7f28fcb46700 10 log is not dirty 2014-12-08 18:14:02.284841 7f290a83a700 10 osd.70 pg_epoch: 71593 pg[217.36e( v 71377'4109 (71377'1108,71377'4109] local-les=71593 n=11 ec=71094 les/c 71583/71583 71591/71591/71096) [69,18,70] r=2 lpr=71591 pi=71183-71590/21 crt=71377'4106 lcod 0'0 inactive] _activate_committed 71593 telling primary 2014-12-08 18:14:02.284874 7f290a83a700 20 osd.70 71593 share_map_peer 0x11a2eb00 already has epoch 71593 2014-12-08 18:14:02.284898 7f290a83a700 10 osd.70 pg_epoch: 71593 pg[217.36e( v 71377'4109 (71377'1108,71377'4109] local-les=71593 n=11 ec=71094 les/c 71583/71583 71591/71591/71096) [69,18,70] r=2 lpr=71591 pi=71183-71590/21 luod=0'0 crt=71377'4106 lcod 0'0 active] flushed 2014-12-08 18:14:02.284934 7f28fc345700 10 osd.70 pg_epoch: 71593 pg[217.36e( v 71377'4109 (71377'1108,71377'4109] local-les=71593 n=11 ec=71094 les/c 71583/71583 71591/71591/71096) [69,18,70] r=2 lpr=71591 pi=71183-71590/21 luod=0'0 crt=71377'4106 lcod 0'0 active] handle_peering_event: epoch_sent: 71593 epoch_requested: 71593 FlushedEvt 2014-12-08 18:14:02.284970 7f28fc345700 15 osd.70 pg_epoch: 71593 pg[217.36e( v 71377'4109 (71377'1108,71377'4109] local-les=71593 n=11 ec=71094 les/c 71583/71583 71591/71591/71096) [69,18,70] r=2 lpr=71591 pi=71183-71590/21 luod=0'0 crt=71377'4106 lcod 0'0 active] requeue_ops 2014-12-08 18:14:02.284990 7f28fc345700 10 log is not dirty 2014-12-08 18:14:02.285151 7f290a83a700 10 osd.70 pg_epoch: 71593 pg[218.36( empty local-les=71593 n=0 ec=71423 les/c 71438/71438 71591/71591/71587) [69,70,77] r=1 lpr=71591 pi=71423-71590/8 crt=0'0 inactive] _activate_committed 71593 telling primary 2014-12-08 18:14:02.285184 7f2904b56700 10 osd.70 71593 do_waiters -- start 2014-12-08 18:14:02.285187 7f2904b56700 10 osd.70 71593 do_waiters -- finish 2014-12-08 18:14:02.285188 7f2904b56700 20 osd.70 71593 _dispatch 0x12b79660 pg_info(1 pgs e71593:218.61a) v4 2014-12-08 18:14:02.285188 7f290a83a700 20 osd.70 71593 share_map_peer 0x11a2eb00 already has epoch 71593 2014-12-08 18:14:02.285217 7f290a83a700 10 osd.70 pg_epoch: 71593 pg[218.36( empty local-les=71593 n=0 ec=71423 les/c 71438/71438 71591/71591/71587) [69,70,77] r=1 lpr=71591 pi=71423-71590/8 crt=0'0 active] flushed 2014-12-08 18:14:02.285224 7f2904b56700 7 osd.70 71593 handle_pg_info pg_info(1 pgs e71593:218.61a) v4 from osd.6 -- 2014-12-08 18:14:02.530268 7f2906359700 10 osd.70 pg_epoch: 71593 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] null 2014-12-08 18:14:02.530288 7f28fc345700 10 osd.70 pg_epoch: 71594 pg[218.279( empty local-les=71593 n=0 ec=71423 les/c 71593/71593 71591/71592/71592) [70,18,64] r=0 lpr=71592 crt=0'0 mlcod 0'0 active+clean] handle_peering_event: epoch_sent: 71594 epoch_requested: 71594 NullEvt 2014-12-08 18:14:02.530295 7f28fc345700 10 log is not dirty 2014-12-08 18:14:02.530294 7f28fcb46700 5 osd.70 pg_epoch: 71594 pg[218.553( empty local-les=71435 n=0 ec=71423 les/c 71589/71589 71591/71591/71425) [17,70,63]/[17,63,75] r=-1 lpr=71591 pi=71423-71590/9 crt=0'0 remapped NOTIFY] exit Started 2.194882 0 0.000000 2014-12-08 18:14:02.530291 7f2906359700 10 osd.70 pg_epoch: 71593 pg[19.6d( v 71397'33614009 (70963'33611008,71397'33614009] local-les=71592 n=896 ec=21298 les/c 71592/71593 71591/71591/71169) [4,70,69] r=1 lpr=71591 pi=71156-71590/27 luod=0'0 crt=71397'33614007 lcod 0'0 active] null 2014-12-08 18:14:02.530302 7f28fcb46700 5 osd.70 pg_epoch: 71594 pg[218.553( empty local-les=71435 n=0 ec=71423 les/c 71589/71589 71591/71591/71425) [17,70,63]/[17,63,75] r=-1 lpr=71591 pi=71423-71590/9 crt=0'0 remapped NOTIFY] enter Reset 2014-12-08 18:14:02.530306 7f28fc345700 10 osd.70 pg_epoch: 71593 pg[217.27a( v 71377'12225 (71377'9224,71377'12225] local-les=71592 n=9 ec=71094 les/c 71592/71592 71591/71591/71591) [70,8,1] r=0 lpr=71591 crt=71377'12222 lcod 0'0 mlcod 0'0 active+clean] handle_advance_map [70,8,1]/[70,8,1] -- 70/70 2014-12-08 18:14:02.530311 7f28fcb46700 20 osd.70 pg_epoch: 71594 pg[218.553( empty local-les=71435 n=0 ec=71423 les/c 71589/71589 71591/71591/71425) [17,70,63]/[17,63,75] r=-1 lpr=71591 pi=71423-71590/9 crt=0'0 remapped NOTIFY] set_last_peering_reset 71594 2014-12-08 18:14:02.530317 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:14:02.530318 7f28fcb46700 10 osd.70 pg_epoch: 71594 pg[218.553( empty local-les=71435 n=0 ec=71423 les/c 71589/71589 71591/71591/71425) [17,70,63]/[17,63,75] r=-1 lpr=71594 pi=71423-71590/9 crt=0'0 remapped NOTIFY] Clearing blocked outgoing recovery messages 2014-12-08 18:14:02.530312 7f2906359700 10 osd.70 pg_epoch: 71593 pg[218.6c7( v 71581'2 (0'0,71581'2] local-les=71593 n=2 ec=71423 les/c 71593/71593 71591/71592/71592) [70,10,18] r=0 lpr=71592 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] null 2014-12-08 18:14:02.530321 7f28fc345700 10 osd.70 pg_epoch: 71594 pg[217.27a( v 71377'12225 (71377'9224,71377'12225] local-les=71592 n=9 ec=71094 les/c 71592/71592 71591/71591/71591) [70,8,1] r=0 lpr=71591 crt=71377'12222 lcod 0'0 mlcod 0'0 active+clean] state: Active advmap 2014-12-08 18:14:02.530327 7f28fcb46700 10 osd.70 pg_epoch: 71594 pg[218.553( empty local-les=71435 n=0 ec=71423 les/c 71589/71589 71591/71591/71425) [17,70,63]/[17,63,75] r=-1 lpr=71594 pi=71423-71590/9 crt=0'0 remapped NOTIFY] Not blocking outgoing recovery messages 2014-12-08 18:14:02.530330 7f28fc345700 10 osd.70 pg_epoch: 71594 pg[217.27a( v 71377'12225 (71377'9224,71377'12225] local-les=71592 n=9 ec=71094 les/c 71592/71592 71591/71591/71591) [70,8,1] r=0 lpr=71591 crt=71377'12222 lcod 0'0 mlcod 0'0 active+clean] state: Started advmap 2014-12-08 18:14:02.530336 7f28fcb46700 10 osd.70 pg_epoch: 71594 pg[218.553( empty local-les=71435 n=0 ec=71423 les/c 71589/71589 71591/71591/71425) [17,70,63]/[17,63,75] r=-1 lpr=71594 pi=71423-71590/9 crt=0'0 remapped NOTIFY] state: Reset advmap 2014-12-08 18:14:02.530331 7f2906359700 10 osd.70 pg_epoch: 71593 pg[218.6c4( v 71581'3 (0'0,71581'3] local-les=71593 n=3 ec=71423 les/c 71593/71593 71591/71592/71592) [70,76,13] r=0 lpr=71592 crt=71580'1 lcod 0'0 mlcod 0'0 active+clean] null 2014-12-08 18:14:02.530339 7f28fc345700 10 osd.70 pg_epoch: 71594 pg[217.27a( v 71377'12225 (71377'9224,71377'12225] local-les=71592 n=9 ec=71094 les/c 71592/71592 71591/71591/71591) [70,8,1] r=0 lpr=71591 crt=71377'12222 lcod 0'0 mlcod 0'0 active+clean] check_recovery_sources no source osds () went down 2014-12-08 18:14:02.530343 7f28fcb46700 10 osd.70 pg_epoch: 71594 pg[218.553( empty local-les=71435 n=0 ec=71423 les/c 71589/71589 71591/71591/71425) [17,70,63]/[17,63,75] r=-1 lpr=71594 pi=71423-71590/9 crt=0'0 remapped NOTIFY] _calc_past_interval_range: already have past intervals back to 71589 2014-12-08 18:14:02.530349 7f28fc345700 10 osd.70 pg_epoch: 71594 pg[217.27a( v 71377'12225 (71377'9224,71377'12225] local-les=71592 n=9 ec=71094 les/c 71592/71592 71591/71591/71591) [70,8,1] r=0 lpr=71591 crt=71377'12222 lcod 0'0 mlcod 0'0 active+clean] handle_activate_map 2014-12-08 18:14:02.530351 7f28fcb46700 20 osd.70 pg_epoch: 71594 pg[218.553( empty local-les=71435 n=0 ec=71423 les/c 71589/71589 71591/71591/71425) [17,70,63]/[17,63,75] r=-1 lpr=71594 pi=71423-71590/9 crt=0'0 remapped NOTIFY] new interval newup [17,70,63] newacting [17,70,63] 2014-12-08 18:14:02.530356 7f28fc345700 10 osd.70 pg_epoch: 71594 pg[217.27a( v 71377'12225 (71377'9224,71377'12225] local-les=71592 n=9 ec=71094 les/c 71592/71592 71591/71591/71591) [70,8,1] r=0 lpr=71591 crt=71377'12222 lcod 0'0 mlcod 0'0 active+clean] state: Active: handling ActMap -- 2014-12-08 18:14:02.536639 7f28fc345700 10 osd.70 pg_epoch: 71593 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:14:02.536659 7f28fc345700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:14:02.536645 7f28fcb46700 10 osd.70 pg_epoch: 71594 pg[217.3f7( v 71376'21 (0'0,71376'21] local-les=71594 n=7 ec=71094 les/c 71585/71583 71591/71593/71593) [70,56,68] r=0 lpr=71593 pi=71432-71592/5 crt=71097'19 lcod 0'0 mlcod 0'0 inactive] activate peer osd.68 217.3f7( v 71376'21 (0'0,71376'21] local-les=71585 n=7 ec=71094 les/c 71585/71583 71591/71593/71593) uptodate 2014-12-08 18:14:02.535843 7f2906359700 10 osd.70 pg_epoch: 71593 pg[216.175( empty local-les=71435 n=0 ec=70987 les/c 71585/71585 71591/71593/71593) [70,78,5] r=0 lpr=71593 pi=71432-71592/6 crt=0'0 mlcod 0'0 peering] null 2014-12-08 18:14:02.536665 7f28fcb46700 10 osd.70 pg_epoch: 71594 pg[217.3f7( v 71376'21 (0'0,71376'21] local-les=71594 n=7 ec=71094 les/c 71585/71583 71591/71593/71593) [70,56,68] r=0 lpr=71593 pi=71432-71592/5 crt=71097'19 lcod 0'0 mlcod 0'0 inactive] needs_recovery is recovered 2014-12-08 18:14:02.536666 7f28fc345700 10 osd.70 pg_epoch: 71594 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] state: Peering advmap 2014-12-08 18:14:02.536677 7f2906359700 10 osd.70 pg_epoch: 71593 pg[218.63e( empty local-les=71437 n=0 ec=71423 les/c 71585/71585 71591/71593/71587) [68,76,70] r=2 lpr=71593 pi=71423-71592/9 crt=0'0 inactive NOTIFY] null 2014-12-08 18:14:02.536682 7f28fcb46700 10 osd.70 pg_epoch: 71594 pg[217.3f7( v 71376'21 (0'0,71376'21] local-les=71594 n=7 ec=71094 les/c 71585/71583 71591/71593/71593) [70,56,68] r=0 lpr=71593 pi=71432-71592/5 crt=71097'19 lcod 0'0 mlcod 0'0 inactive] state: Activate Finished 2014-12-08 18:14:02.536687 7f28fc345700 10 osd.70 pg_epoch: 71594 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] state: Started advmap 2014-12-08 18:14:02.536700 7f28fcb46700 5 osd.70 pg_epoch: 71594 pg[217.3f7( v 71376'21 (0'0,71376'21] local-les=71594 n=7 ec=71094 les/c 71585/71583 71591/71593/71593) [70,56,68] r=0 lpr=71593 pi=71432-71592/5 crt=71097'19 lcod 0'0 mlcod 0'0 inactive] enter Started/Primary/Active/Activating 2014-12-08 18:14:02.536705 7f28fc345700 10 osd.70 pg_epoch: 71594 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] check_recovery_sources no source osds () went down 2014-12-08 18:14:02.536719 7f28fcb46700 20 osd.70 pg_epoch: 71594 pg[217.3f7( v 71376'21 (0'0,71376'21] local-les=71594 n=7 ec=71094 les/c 71585/71583 71591/71593/71593) [70,56,68] r=0 lpr=71593 pi=71432-71592/5 crt=71097'19 lcod 0'0 mlcod 0'0 inactive] handle_activate_map: Not dirtying info: last_persisted is 71593 while current is 71594 2014-12-08 18:14:02.536720 7f2906359700 10 osd.70 pg_epoch: 71593 pg[218.4a7( empty local-les=71592 n=0 ec=71423 les/c 71592/71593 71591/71591/71591) [70,56,21] r=0 lpr=71591 crt=0'0 mlcod 0'0 active+clean] null 2014-12-08 18:14:02.536727 7f28fc345700 10 osd.70 pg_epoch: 71594 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] handle_activate_map 2014-12-08 18:14:02.536735 7f28fcb46700 10 osd.70 pg_epoch: 71594 pg[217.3f7( v 71376'21 (0'0,71376'21] local-les=71594 n=7 ec=71094 les/c 71585/71583 71591/71593/71593) [70,56,68] r=0 lpr=71593 pi=71432-71592/5 crt=71097'19 lcod 0'0 mlcod 0'0 inactive] handle_peering_event: epoch_sent: 71594 epoch_requested: 71594 NullEvt 2014-12-08 18:14:02.536745 7f28fc345700 7 osd.70 pg_epoch: 71594 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] state: handle ActMap primary 2014-12-08 18:14:02.536744 7f2906359700 10 osd.70 pg_epoch: 71593 pg[218.16e( v 71581'2 (0'0,71581'2] local-les=71450 n=2 ec=71423 les/c 71585/71585 71591/71593/71593) [70,23,63] r=0 lpr=71593 pi=71444-71592/5 crt=0'0 lcod 0'0 mlcod 0'0 peering] null 2014-12-08 18:14:02.536764 7f28fc345700 15 osd.70 pg_epoch: 71594 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] publish_stats_to_osd 71594:63844734 2014-12-08 18:14:02.536778 7f28fc345700 10 osd.70 pg_epoch: 71594 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] take_waiters 2014-12-08 18:14:02.536776 7f2906359700 10 osd.70 pg_epoch: 71593 pg[218.16c( v 71581'1 (0'0,71581'1] local-les=71433 n=1 ec=71423 les/c 71585/71583 71591/71593/71425) [6,56,70] r=2 lpr=71593 pi=71423-71592/10 crt=0'0 lcod 0'0 inactive NOTIFY] null 2014-12-08 18:14:02.536793 7f28fcb46700 10 log is not dirty 2014-12-08 18:14:02.536791 7f28fc345700 20 osd.70 pg_epoch: 71594 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] handle_activate_map: Not dirtying info: last_persisted is 71591 while current is 71594 2014-12-08 18:14:02.536797 7f2906359700 10 osd.70 pg_epoch: 71593 pg[216.16e( empty local-les=71592 n=0 ec=70987 les/c 71592/71592 71591/71591/70989) [78,56,70] r=2 lpr=71591 pi=71401-71590/8 crt=0'0 active] null 2014-12-08 18:14:02.536807 7f28fc345700 10 osd.70 pg_epoch: 71594 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] handle_peering_event: epoch_sent: 71594 epoch_requested: 71594 NullEvt 2014-12-08 18:14:02.536814 7f2906359700 10 osd.70 pg_epoch: 71593 pg[216.2f8( empty local-les=71437 n=0 ec=70987 les/c 71585/71585 71591/71593/70989) [23,70,9] r=1 lpr=71593 pi=71401-71592/9 crt=0'0 inactive NOTIFY] null 2014-12-08 18:14:02.536825 7f28fc345700 10 log is not dirty 2014-12-08 18:14:02.536834 7f2906359700 10 osd.70 pg_epoch: 71593 pg[218.452( v 71581'3 (0'0,71581'3] local-les=71450 n=3 ec=71423 les/c 71589/71589 71591/71593/71444) [21,63,70] r=2 lpr=71593 pi=71423-71592/12 crt=71580'1 lcod 0'0 inactive NOTIFY] null 2014-12-08 18:14:02.536829 7f28fcb46700 10 osd.70 pg_epoch: 71593 pg[19.71( v 71379'39954249 (70963'39951247,71379'39954249] local-les=71592 n=908 ec=21298 les/c 71592/71592 71591/71591/71591) [70,10,76] r=0 lpr=71591 crt=71379'39954249 lcod 0'0 mlcod 0'0 active+clean] handle_advance_map [70,10,76]/[70,10,76] -- 70/70 2014-12-08 18:14:02.536853 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:14:02.536853 7f2906359700 10 osd.70 pg_epoch: 71593 pg[217.165( v 71377'4090 (71377'1089,71377'4090] local-les=71592 n=8 ec=71094 les/c 71592/71592 71591/71591/71096) [64,5,70] r=2 lpr=71591 pi=71094-71590/32 luod=0'0 crt=71377'4087 lcod 0'0 active] null 2014-12-08 18:14:02.536860 7f28fcb46700 10 osd.70 pg_epoch: 71594 pg[19.71( v 71379'39954249 (70963'39951247,71379'39954249] local-les=71592 n=908 ec=21298 les/c 71592/71592 71591/71591/71591) [70,10,76] r=0 lpr=71591 crt=71379'39954249 lcod 0'0 mlcod 0'0 active+clean] state: Active advmap 2014-12-08 18:14:02.536869 7f2906359700 10 osd.70 pg_epoch: 71593 pg[217.2fd( v 71377'4110 (71377'1109,71377'4110] local-les=71592 n=14 ec=71094 les/c 71592/71592 71591/71591/71591) [70,10,19] r=0 lpr=71591 crt=71377'4107 lcod 0'0 mlcod 0'0 active+clean] null 2014-12-08 18:14:02.536875 7f28fcb46700 10 osd.70 pg_epoch: 71594 pg[19.71( v 71379'39954249 (70963'39951247,71379'39954249] local-les=71592 n=908 ec=21298 les/c 71592/71592 71591/71591/71591) [70,10,76] r=0 lpr=71591 crt=71379'39954249 lcod 0'0 mlcod 0'0 active+clean] state: Started advmap 2014-12-08 18:14:02.536884 7f2906359700 10 osd.70 pg_epoch: 71593 pg[218.2fc( empty local-les=71592 n=0 ec=71423 les/c 71592/71592 71591/71591/71591) [70,16,75] r=0 lpr=71591 crt=0'0 mlcod 0'0 active+clean] null 2014-12-08 18:14:02.536891 7f28fcb46700 10 osd.70 pg_epoch: 71594 pg[19.71( v 71379'39954249 (70963'39951247,71379'39954249] local-les=71592 n=908 ec=21298 les/c 71592/71592 71591/71591/71591) [70,10,76] r=0 lpr=71591 crt=71379'39954249 lcod 0'0 mlcod 0'0 active+clean] check_recovery_sources no source osds () went down 2014-12-08 18:14:02.536897 7f2906359700 10 osd.70 pg_epoch: 71593 pg[216.167( v 71422'250 (0'0,71422'250] local-les=71446 n=117 ec=70987 les/c 71590/71583 71591/71593/70989) [25,9,70] r=2 lpr=71593 pi=71183-71592/21 crt=71097'248 lcod 0'0 inactive NOTIFY] null 2014-12-08 18:14:02.536910 7f2906359700 10 osd.70 pg_epoch: 71593 pg[218.118( v 71581'3 (0'0,71581'3] local-les=71445 n=3 ec=71423 les/c 71585/71583 71591/71593/71425) [6,23,70] r=2 lpr=71593 pi=71423-71592/12 crt=71424'2 lcod 0'0 inactive NOTIFY] null 2014-12-08 18:14:02.536912 7f28fcb46700 10 osd.70 pg_epoch: 71594 pg[19.71( v 71379'39954249 (70963'39951247,71379'39954249] local-les=71592 n=908 ec=21298 les/c 71592/71592 71591/71591/71591) [70,10,76] r=0 lpr=71591 crt=71379'39954249 lcod 0'0 mlcod 0'0 active+clean] handle_activate_map 2014-12-08 18:14:02.536925 7f28fcb46700 10 osd.70 pg_epoch: 71594 pg[19.71( v 71379'39954249 (70963'39951247,71379'39954249] local-les=71592 n=908 ec=21298 les/c 71592/71592 71591/71591/71591) [70,10,76] r=0 lpr=71591 crt=71379'39954249 lcod 0'0 mlcod 0'0 active+clean] state: Active: handling ActMap 2014-12-08 18:14:02.536926 7f2906359700 10 osd.70 pg_epoch: 71593 pg[218.785( v 71581'2 (0'0,71581'2] local-les=71435 n=2 ec=71423 les/c 71585/71586 71591/71593/71593) [70,18,78] r=0 lpr=71593 pi=71432-71592/6 crt=0'0 lcod 0'0 mlcod 0'0 peering] null 2014-12-08 18:14:02.536937 7f28fcb46700 7 osd.70 pg_epoch: 71594 pg[19.71( v 71379'39954249 (70963'39951247,71379'39954249] local-les=71592 n=908 ec=21298 les/c 71592/71592 71591/71591/71591) [70,10,76] r=0 lpr=71591 crt=71379'39954249 lcod 0'0 mlcod 0'0 active+clean] state: handle ActMap primary 2014-12-08 18:14:02.536944 7f2906359700 10 osd.70 pg_epoch: 71593 pg[216.11b( v 71093'250 (0'0,71093'250] local-les=71435 n=119 ec=70987 les/c 71585/71583 71591/71593/71587) [63,70,16] r=1 lpr=71593 pi=70987-71592/29 crt=71093'248 lcod 0'0 inactive NOTIFY] null 2014-12-08 18:14:02.536952 7f28fcb46700 15 osd.70 pg_epoch: 71594 pg[19.71( v 71379'39954249 (70963'39951247,71379'39954249] local-les=71592 n=908 ec=21298 les/c 71592/71592 71591/71591/71591) [70,10,76] r=0 lpr=71591 crt=71379'39954249 lcod 0'0 mlcod 0'0 active+clean] publish_stats_to_osd 71594:45391760 2014-12-08 18:14:02.536963 7f2906359700 10 osd.70 pg_epoch: 71593 pg[218.11d( v 71580'1 (0'0,71580'1] local-les=71593 n=1 ec=71423 les/c 71593/71593 71591/71591/71591) [70,25,17] r=0 lpr=71591 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] null -- 2014-12-08 18:14:02.540221 7f2906359700 20 osd.70 71594 19.6e heartbeat_peers 0,4,11,13,16,20,30,31,56,58,59,63,70,76 2014-12-08 18:14:02.540241 7f2906359700 20 osd.70 71594 19.6d heartbeat_peers 2014-12-08 18:14:02.540244 7f2906359700 20 osd.70 71594 218.6c7 heartbeat_peers 10,18,70 2014-12-08 18:14:02.540253 7f2906359700 20 osd.70 71594 218.6c4 heartbeat_peers 13,70,76 2014-12-08 18:14:02.540256 7f2906359700 20 osd.70 71594 218.562 heartbeat_peers 2014-12-08 18:14:02.540259 7f2906359700 20 osd.70 71594 218.5e heartbeat_peers 2014-12-08 18:14:02.540261 7f2906359700 20 osd.70 71594 218.5f heartbeat_peers 15,24,70 2014-12-08 18:14:02.540264 7f2906359700 20 osd.70 71594 217.235 heartbeat_peers 2014-12-08 18:14:02.540267 7f2906359700 20 osd.70 71594 218.38e heartbeat_peers 3,13,70 2014-12-08 18:14:02.540270 7f2906359700 20 osd.70 71594 218.38f heartbeat_peers 23,25,70 2014-12-08 18:14:02.540274 7f2906359700 20 osd.70 71594 218.38c heartbeat_peers 2014-12-08 18:14:02.540277 7f2906359700 20 osd.70 71594 217.51 heartbeat_peers 2014-12-08 18:14:02.540280 7f2906359700 20 osd.70 71594 216.380 heartbeat_peers 2014-12-08 18:14:02.540283 7f2906359700 20 osd.70 71594 218.51 heartbeat_peers 6,24,66,70,75 2014-12-08 18:14:02.540287 7f2906359700 20 osd.70 71594 218.380 heartbeat_peers 2014-12-08 18:14:02.540290 7f2906359700 20 osd.70 71594 218.518 heartbeat_peers 2014-12-08 18:14:02.540271 7f28fc345700 5 osd.70 pg_epoch: 71594 pg[216.380( v 70990'233 (0'0,70990'233] local-les=71447 n=112 ec=70987 les/c 71589/71583 71591/71594/71444) [17,70,7] r=1 lpr=71594 pi=70987-71593/32 crt=70990'231 lcod 0'0 inactive NOTIFY] enter Started 2014-12-08 18:14:02.540293 7f2906359700 20 osd.70 71594 217.22d heartbeat_peers 2014-12-08 18:14:02.540296 7f2906359700 20 osd.70 71594 216.384 heartbeat_peers 2014-12-08 18:14:02.540300 7f2906359700 20 osd.70 71594 216.22f heartbeat_peers 6,21,70 2014-12-08 18:14:02.540304 7f2906359700 20 osd.70 71594 218.4b heartbeat_peers -- 2014-12-08 18:14:02.561181 7f2904b56700 20 osd.70 71594 _dispatch 0x12cf74a0 pg_notify(19.6e(150) epoch 71594) v5 2014-12-08 18:14:02.561174 7f28fcb46700 5 osd.70 pg_epoch: 71594 pg[218.685( v 71581'2 (0'0,71581'2] local-les=71433 n=2 ec=71423 les/c 71585/71583 71591/71593/71593) [70,10,18] r=0 lpr=71593 pi=71432-71592/6 crt=0'0 lcod 0'0 mlcod 0'0 inactive] enter Started/Primary/Active 2014-12-08 18:14:02.561173 7f28fc345700 10 osd.70 pg_epoch: 71593 pg[217.37c( v 71377'8150 (71377'5149,71377'8150] local-les=71592 n=10 ec=71094 les/c 71592/71593 71591/71591/71591) [70,58,69] r=0 lpr=71591 crt=71377'8146 lcod 0'0 mlcod 0'0 active+clean] handle_advance_map [70,58,69]/[70,58,69] -- 70/70 2014-12-08 18:14:02.561184 7f290a83a700 10 osd.70 pg_epoch: 71594 pg[218.1e1( v 71581'1 (0'0,71581'1] local-les=71594 n=1 ec=71423 les/c 71590/71590 71591/71593/71593) [70,21,4] r=0 lpr=71593 pi=71444-71592/6 crt=0'0 lcod 0'0 mlcod 0'0 inactive] flushed 2014-12-08 18:14:02.561195 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:14:02.561192 7f28fcb46700 10 osd.70 pg_epoch: 71594 pg[218.685( v 71581'2 (0'0,71581'2] local-les=71433 n=2 ec=71423 les/c 71585/71583 71591/71593/71593) [70,10,18] r=0 lpr=71593 pi=71432-71592/6 crt=0'0 lcod 0'0 mlcod 0'0 inactive] state: In Active, about to call activate 2014-12-08 18:14:02.561201 7f28fc345700 10 osd.70 pg_epoch: 71594 pg[217.37c( v 71377'8150 (71377'5149,71377'8150] local-les=71592 n=10 ec=71094 les/c 71592/71593 71591/71591/71591) [70,58,69] r=0 lpr=71591 crt=71377'8146 lcod 0'0 mlcod 0'0 active+clean] state: Active advmap 2014-12-08 18:14:02.561210 7f28fcb46700 20 osd.70 pg_epoch: 71594 pg[218.685( v 71581'2 (0'0,71581'2] local-les=71594 n=2 ec=71423 les/c 71585/71583 71591/71593/71593) [70,10,18] r=0 lpr=71593 pi=71432-71592/6 crt=0'0 lcod 0'0 mlcod 0'0 inactive] activate - purged_snaps [] cached_removed_snaps [] 2014-12-08 18:14:02.561214 7f28fc345700 10 osd.70 pg_epoch: 71594 pg[217.37c( v 71377'8150 (71377'5149,71377'8150] local-les=71592 n=10 ec=71094 les/c 71592/71593 71591/71591/71591) [70,58,69] r=0 lpr=71591 crt=71377'8146 lcod 0'0 mlcod 0'0 active+clean] state: Started advmap 2014-12-08 18:14:02.561224 7f2904b56700 7 osd.70 71594 handle_pg_notify from osd.16 2014-12-08 18:14:02.561228 7f2904b56700 15 osd.70 71594 require_same_or_newer_map 71594 (i am 71594) 0x12cf74a0 2014-12-08 18:14:02.561222 7f28fcb46700 10 osd.70 pg_epoch: 71594 pg[218.685( v 71581'2 (0'0,71581'2] local-les=71594 n=2 ec=71423 les/c 71585/71583 71591/71593/71593) [70,10,18] r=0 lpr=71593 pi=71432-71592/6 crt=0'0 lcod 0'0 mlcod 0'0 inactive] activate - snap_trimq [] 2014-12-08 18:14:02.561226 7f28fc345700 10 osd.70 pg_epoch: 71594 pg[217.37c( v 71377'8150 (71377'5149,71377'8150] local-les=71592 n=10 ec=71094 les/c 71592/71593 71591/71591/71591) [70,58,69] r=0 lpr=71591 crt=71377'8146 lcod 0'0 mlcod 0'0 active+clean] check_recovery_sources no source osds () went down 2014-12-08 18:14:02.561233 7f28fcb46700 10 osd.70 pg_epoch: 71594 pg[218.685( v 71581'2 (0'0,71581'2] local-les=71594 n=2 ec=71423 les/c 71585/71583 71591/71593/71593) [70,10,18] r=0 lpr=71593 pi=71432-71592/6 crt=0'0 lcod 0'0 mlcod 0'0 inactive] activate - no missing, moving last_complete 71581'2 -> 71581'2 2014-12-08 18:14:02.561240 7f28fc345700 10 osd.70 pg_epoch: 71594 pg[217.37c( v 71377'8150 (71377'5149,71377'8150] local-les=71592 n=10 ec=71094 les/c 71592/71593 71591/71591/71591) [70,58,69] r=0 lpr=71591 crt=71377'8146 lcod 0'0 mlcod 0'0 active+clean] handle_activate_map 2014-12-08 18:14:02.561244 7f28fcb46700 10 osd.70 pg_epoch: 71594 pg[218.685( v 71581'2 (0'0,71581'2] local-les=71594 n=2 ec=71423 les/c 71585/71583 71591/71593/71593) [70,10,18] r=0 lpr=71593 pi=71432-71592/6 crt=0'0 lcod 0'0 mlcod 0'0 inactive] activate peer osd.10 218.685( v 71581'2 (0'0,71581'2] local-les=71585 n=2 ec=71423 les/c 71585/71583 71591/71593/71593) 2014-12-08 18:14:02.561251 7f28fc345700 10 osd.70 pg_epoch: 71594 pg[217.37c( v 71377'8150 (71377'5149,71377'8150] local-les=71592 n=10 ec=71094 les/c 71592/71593 71591/71591/71591) [70,58,69] r=0 lpr=71591 crt=71377'8146 lcod 0'0 mlcod 0'0 active+clean] state: Active: handling ActMap 2014-12-08 18:14:02.561265 7f2904b56700 10 osd.70 71594 do_waiters -- start 2014-12-08 18:14:02.561260 7f28fcb46700 10 osd.70 pg_epoch: 71594 pg[218.685( v 71581'2 (0'0,71581'2] local-les=71594 n=2 ec=71423 les/c 71585/71583 71591/71593/71593) [70,10,18] r=0 lpr=71593 pi=71432-71592/6 crt=0'0 lcod 0'0 mlcod 0'0 inactive] activate peer osd.10 is up to date, queueing in pending_activators 2014-12-08 18:14:02.561268 7f2904b56700 10 osd.70 71594 do_waiters -- finish 2014-12-08 18:14:02.561261 7f28fc345700 7 osd.70 pg_epoch: 71594 pg[217.37c( v 71377'8150 (71377'5149,71377'8150] local-les=71592 n=10 ec=71094 les/c 71592/71593 71591/71591/71591) [70,58,69] r=0 lpr=71591 crt=71377'8146 lcod 0'0 mlcod 0'0 active+clean] state: handle ActMap primary -- 2014-12-08 18:14:02.562170 7f2904b56700 20 osd.70 71594 _dispatch 0x12f8d4a0 pg_notify(19.6e(150) epoch 71594) v5 2014-12-08 18:14:02.562197 7f2904b56700 7 osd.70 71594 handle_pg_notify from osd.58 2014-12-08 18:14:02.562197 7f28fc345700 20 osd.70 71594 share_map_peer 0x1281f600 already has epoch 71594 2014-12-08 18:14:02.562199 7f2904b56700 15 osd.70 71594 require_same_or_newer_map 71594 (i am 71594) 0x12f8d4a0 2014-12-08 18:14:02.562201 7f28fc345700 7 osd.70 71594 do_notifies osd 20 on 1 PGs 2014-12-08 18:14:02.562225 7f28fc345700 10 osd.70 71594 send_incremental_map 71593 -> 71594 to 0x118ba3c0 172.16.10.101:6953/19844 2014-12-08 18:14:02.562235 7f2904b56700 10 osd.70 71594 do_waiters -- start 2014-12-08 18:14:02.562237 7f2904b56700 10 osd.70 71594 do_waiters -- finish 2014-12-08 18:14:02.562224 7f28fcb46700 10 osd.70 pg_epoch: 71593 pg[218.372( v 71581'2 (0'0,71581'2] local-les=71435 n=2 ec=71423 les/c 71590/71590 71591/71593/71425) [63,70,68] r=1 lpr=71593 pi=71423-71592/10 crt=0'0 lcod 0'0 inactive NOTIFY] handle_advance_map [63,70,68]/[63,70,68] -- 63/63 2014-12-08 18:14:02.562244 7f28fcb46700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:14:02.562253 7f28fc345700 10 osd.70 71594 note_peer_epoch osd.22 has 71594 2014-12-08 18:14:02.562250 7f28fcb46700 10 osd.70 pg_epoch: 71594 pg[218.372( v 71581'2 (0'0,71581'2] local-les=71435 n=2 ec=71423 les/c 71590/71590 71591/71593/71425) [63,70,68] r=1 lpr=71593 pi=71423-71592/10 crt=0'0 lcod 0'0 inactive NOTIFY] state: Started advmap 2014-12-08 18:14:02.562256 7f28fc345700 7 osd.70 71594 do_notifies osd 22 on 1 PGs 2014-12-08 18:14:02.562259 7f28fcb46700 10 osd.70 pg_epoch: 71594 pg[218.372( v 71581'2 (0'0,71581'2] local-les=71435 n=2 ec=71423 les/c 71590/71590 71591/71593/71425) [63,70,68] r=1 lpr=71593 pi=71423-71592/10 crt=0'0 lcod 0'0 inactive NOTIFY] check_recovery_sources no source osds () went down 2014-12-08 18:14:02.562275 7f28fcb46700 10 osd.70 pg_epoch: 71594 pg[218.372( v 71581'2 (0'0,71581'2] local-les=71435 n=2 ec=71423 les/c 71590/71590 71591/71593/71425) [63,70,68] r=1 lpr=71593 pi=71423-71592/10 crt=0'0 lcod 0'0 inactive NOTIFY] handle_activate_map 2014-12-08 18:14:02.562291 7f28fcb46700 10 osd.70 pg_epoch: 71594 pg[218.372( v 71581'2 (0'0,71581'2] local-les=71435 n=2 ec=71423 les/c 71590/71590 71591/71593/71425) [63,70,68] r=1 lpr=71593 pi=71423-71592/10 crt=0'0 lcod 0'0 inactive NOTIFY] take_waiters 2014-12-08 18:14:02.562298 7f28fcb46700 20 osd.70 pg_epoch: 71594 pg[218.372( v 71581'2 (0'0,71581'2] local-les=71435 n=2 ec=71423 les/c 71590/71590 71591/71593/71425) [63,70,68] r=1 lpr=71593 pi=71423-71592/10 crt=0'0 lcod 0'0 inactive NOTIFY] handle_activate_map: Not dirtying info: last_persisted is 71593 while current is 71594 2014-12-08 18:14:02.562306 7f28fcb46700 10 osd.70 pg_epoch: 71594 pg[218.372( v 71581'2 (0'0,71581'2] local-les=71435 n=2 ec=71423 les/c 71590/71590 71591/71593/71425) [63,70,68] r=1 lpr=71593 pi=71423-71592/10 crt=0'0 lcod 0'0 inactive NOTIFY] handle_peering_event: epoch_sent: 71594 epoch_requested: 71594 NullEvt 2014-12-08 18:14:02.562314 7f28fcb46700 10 log is not dirty 2014-12-08 18:14:02.562332 7f2904b56700 10 osd.70 71594 do_waiters -- start 2014-12-08 18:14:02.562334 7f2904b56700 10 osd.70 71594 do_waiters -- finish -- 2014-12-08 18:14:02.562336 7f2904b56700 20 osd.70 71594 _dispatch 0x12c19a20 pg_notify(19.6e(150) epoch 71594) v5 2014-12-08 18:14:02.562339 7f28fcb46700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:14:02.562344 7f28fc345700 20 osd.70 71594 share_map_peer 0x12a1fc80 already has epoch 71594 2014-12-08 18:14:02.562343 7f28fcb46700 10 osd.70 pg_epoch: 71594 pg[217.198( v 71377'26 (0'0,71377'26] local-les=71593 n=6 ec=71094 les/c 71593/71593 71591/71591/71096) [25,75,70] r=2 lpr=71591 pi=71094-71590/26 luod=0'0 crt=71377'24 lcod 0'0 active] state: Started advmap 2014-12-08 18:14:02.562348 7f28fc345700 7 osd.70 71594 do_notifies osd 77 on 1 PGs 2014-12-08 18:14:02.562351 7f28fcb46700 10 osd.70 pg_epoch: 71594 pg[217.198( v 71377'26 (0'0,71377'26] local-les=71593 n=6 ec=71094 les/c 71593/71593 71591/71591/71096) [25,75,70] r=2 lpr=71591 pi=71094-71590/26 luod=0'0 crt=71377'24 lcod 0'0 active] check_recovery_sources no source osds () went down 2014-12-08 18:14:02.562366 7f2904b56700 7 osd.70 71594 handle_pg_notify from osd.76 2014-12-08 18:14:02.562363 7f28fcb46700 10 osd.70 pg_epoch: 71594 pg[217.198( v 71377'26 (0'0,71377'26] local-les=71593 n=6 ec=71094 les/c 71593/71593 71591/71591/71096) [25,75,70] r=2 lpr=71591 pi=71094-71590/26 luod=0'0 crt=71377'24 lcod 0'0 active] handle_activate_map 2014-12-08 18:14:02.562368 7f2904b56700 15 osd.70 71594 require_same_or_newer_map 71594 (i am 71594) 0x12c19a20 2014-12-08 18:14:02.562369 7f28fc345700 20 osd.70 71594 do_infos sending info 218.1e1( v 71581'1 (0'0,71581'1] local-les=71594 n=1 ec=71423 les/c 71590/71590 71591/71593/71593) to shard 4 2014-12-08 18:14:02.562376 7f28fc345700 20 osd.70 71594 share_map_peer 0x118b9e40 already has epoch 71594 2014-12-08 18:14:02.562374 7f28fcb46700 10 osd.70 pg_epoch: 71594 pg[217.198( v 71377'26 (0'0,71377'26] local-les=71593 n=6 ec=71094 les/c 71593/71593 71591/71591/71096) [25,75,70] r=2 lpr=71591 pi=71094-71590/26 luod=0'0 crt=71377'24 lcod 0'0 active] take_waiters 2014-12-08 18:14:02.562382 7f28fcb46700 20 osd.70 pg_epoch: 71594 pg[217.198( v 71377'26 (0'0,71377'26] local-les=71593 n=6 ec=71094 les/c 71593/71593 71591/71591/71096) [25,75,70] r=2 lpr=71591 pi=71094-71590/26 luod=0'0 crt=71377'24 lcod 0'0 active] handle_activate_map: Not dirtying info: last_persisted is 71593 while current is 71594 2014-12-08 18:14:02.562394 7f28fc345700 20 osd.70 71594 do_infos sending info 218.1e3( v 71581'1 (0'0,71581'1] local-les=71594 n=1 ec=71423 les/c 71585/71583 71591/71593/71593) to shard 14 2014-12-08 18:14:02.562390 7f28fcb46700 10 osd.70 pg_epoch: 71594 pg[217.198( v 71377'26 (0'0,71377'26] local-les=71593 n=6 ec=71094 les/c 71593/71593 71591/71591/71096) [25,75,70] r=2 lpr=71591 pi=71094-71590/26 luod=0'0 crt=71377'24 lcod 0'0 active] handle_peering_event: epoch_sent: 71594 epoch_requested: 71594 NullEvt 2014-12-08 18:14:02.562402 7f28fc345700 20 osd.70 71594 share_map_peer 0x1268bce0 already has epoch 71594 2014-12-08 18:14:02.562403 7f28fcb46700 10 log is not dirty 2014-12-08 18:14:02.562404 7f2904b56700 10 osd.70 71594 do_waiters -- start 2014-12-08 18:14:02.562407 7f2904b56700 10 osd.70 71594 do_waiters -- finish 2014-12-08 18:14:02.562426 7f28fc345700 20 osd.70 71594 do_infos sending info 218.1e1( v 71581'1 (0'0,71581'1] local-les=71594 n=1 ec=71423 les/c 71590/71590 71591/71593/71593) to shard 21 2014-12-08 18:14:02.562416 7f28fcb46700 10 osd.70 pg_epoch: 71593 pg[217.370( v 71377'12257 (71377'9256,71377'12257] local-les=71592 n=16 ec=71094 les/c 71592/71592 71591/71591/71591) [70,10,56] r=0 lpr=71591 crt=71377'12255 lcod 0'0 mlcod 0'0 active+clean] handle_advance_map [70,10,56]/[70,10,56] -- 70/70 -- 2014-12-08 18:14:02.568448 7f2904b56700 20 osd.70 71594 _dispatch 0x12f8ad00 pg_notify(19.6e(150) epoch 71594) v5 2014-12-08 18:14:02.568452 7f28fcb46700 10 log is not dirty 2014-12-08 18:14:02.568445 7f28fc345700 10 osd.70 pg_epoch: 71594 pg[218.4f3( v 71581'3 (0'0,71581'3] local-les=71594 n=3 ec=71423 les/c 71590/71583 71591/71593/71593) [70,76,3] r=0 lpr=71593 pi=71432-71592/6 crt=71580'1 lcod 0'0 mlcod 0'0 inactive] activate peer osd.3 218.4f3( v 71581'3 (0'0,71581'3] local-les=71590 n=3 ec=71423 les/c 71590/71583 71591/71593/71593) 2014-12-08 18:14:02.568455 7f28fc345700 10 osd.70 pg_epoch: 71594 pg[218.4f3( v 71581'3 (0'0,71581'3] local-les=71594 n=3 ec=71423 les/c 71590/71583 71591/71593/71593) [70,76,3] r=0 lpr=71593 pi=71432-71592/6 crt=71580'1 lcod 0'0 mlcod 0'0 inactive] activate peer osd.3 is up to date, queueing in pending_activators 2014-12-08 18:14:02.568456 7f290a83a700 10 osd.70 pg_epoch: 71594 pg[218.38( v 71581'3 (0'0,71581'3] local-les=71434 n=3 ec=71423 les/c 71590/71583 71591/71594/71425) [56,17,70] r=2 lpr=71594 pi=71423-71593/10 crt=71580'1 lcod 0'0 inactive NOTIFY] flushed 2014-12-08 18:14:02.568461 7f28fcb46700 10 osd.70 pg_epoch: 71593 pg[216.196( v 71093'258 (0'0,71093'258] local-les=71449 n=122 ec=70987 les/c 71590/71583 71591/71591/71444) [21,70,68]/[21,68,17] r=-1 lpr=71591 pi=70987-71590/33 crt=71093'256 lcod 0'0 remapped NOTIFY] handle_advance_map [21,70,68]/[21,70,68] -- 21/21 2014-12-08 18:14:02.568473 7f28fcb46700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:14:02.568468 7f28fc345700 10 osd.70 pg_epoch: 71594 pg[218.4f3( v 71581'3 (0'0,71581'3] local-les=71594 n=3 ec=71423 les/c 71590/71583 71591/71593/71593) [70,76,3] r=0 lpr=71593 pi=71432-71592/6 crt=71580'1 lcod 0'0 mlcod 0'0 inactive] activate peer osd.3 218.4f3( v 71581'3 (0'0,71581'3] local-les=71590 n=3 ec=71423 les/c 71590/71583 71591/71593/71593) uptodate 2014-12-08 18:14:02.568478 7f2904b56700 7 osd.70 71594 handle_pg_notify from osd.4 2014-12-08 18:14:02.568481 7f2904b56700 15 osd.70 71594 require_same_or_newer_map 71594 (i am 71594) 0x12f8ad00 2014-12-08 18:14:02.568476 7f28fcb46700 10 osd.70 pg_epoch: 71594 pg[216.196( v 71093'258 (0'0,71093'258] local-les=71449 n=122 ec=70987 les/c 71590/71583 71591/71591/71444) [21,70,68]/[21,68,17] r=-1 lpr=71591 pi=70987-71590/33 crt=71093'256 lcod 0'0 remapped NOTIFY] state: Started advmap 2014-12-08 18:14:02.568478 7f28fc345700 10 osd.70 pg_epoch: 71594 pg[218.4f3( v 71581'3 (0'0,71581'3] local-les=71594 n=3 ec=71423 les/c 71590/71583 71591/71593/71593) [70,76,3] r=0 lpr=71593 pi=71432-71592/6 crt=71580'1 lcod 0'0 mlcod 0'0 inactive] activate peer osd.76 218.4f3( v 71581'3 (0'0,71581'3] local-les=71590 n=3 ec=71423 les/c 71590/71583 71591/71593/71593) 2014-12-08 18:14:02.568485 7f28fcb46700 20 osd.70 pg_epoch: 71594 pg[216.196( v 71093'258 (0'0,71093'258] local-les=71449 n=122 ec=70987 les/c 71590/71583 71591/71591/71444) [21,70,68]/[21,68,17] r=-1 lpr=71591 pi=70987-71590/33 crt=71093'256 lcod 0'0 remapped NOTIFY] new interval newup [21,70,68] newacting [21,70,68] 2014-12-08 18:14:02.568488 7f28fc345700 10 osd.70 pg_epoch: 71594 pg[218.4f3( v 71581'3 (0'0,71581'3] local-les=71594 n=3 ec=71423 les/c 71590/71583 71591/71593/71593) [70,76,3] r=0 lpr=71593 pi=71432-71592/6 crt=71580'1 lcod 0'0 mlcod 0'0 inactive] activate peer osd.76 is up to date, queueing in pending_activators 2014-12-08 18:14:02.568493 7f28fcb46700 10 osd.70 pg_epoch: 71594 pg[216.196( v 71093'258 (0'0,71093'258] local-les=71449 n=122 ec=70987 les/c 71590/71583 71591/71591/71444) [21,70,68]/[21,68,17] r=-1 lpr=71591 pi=70987-71590/33 crt=71093'256 lcod 0'0 remapped NOTIFY] state: should_restart_peering, transitioning to Reset 2014-12-08 18:14:02.568502 7f28fcb46700 5 osd.70 pg_epoch: 71594 pg[216.196( v 71093'258 (0'0,71093'258] local-les=71449 n=122 ec=70987 les/c 71590/71583 71591/71591/71444) [21,70,68]/[21,68,17] r=-1 lpr=71591 pi=70987-71590/33 crt=71093'256 lcod 0'0 remapped NOTIFY] exit Started/Stray 1.921920 10 0.000562 2014-12-08 18:14:02.568517 7f2904b56700 10 osd.70 71594 do_waiters -- start 2014-12-08 18:14:02.568511 7f28fc345700 10 osd.70 pg_epoch: 71594 pg[218.4f3( v 71581'3 (0'0,71581'3] local-les=71594 n=3 ec=71423 les/c 71590/71583 71591/71593/71593) [70,76,3] r=0 lpr=71593 pi=71432-71592/6 crt=71580'1 lcod 0'0 mlcod 0'0 inactive] activate peer osd.76 218.4f3( v 71581'3 (0'0,71581'3] local-les=71590 n=3 ec=71423 les/c 71590/71583 71591/71593/71593) uptodate 2014-12-08 18:14:02.568519 7f2904b56700 10 osd.70 71594 do_waiters -- finish 2014-12-08 18:14:02.568515 7f28fcb46700 5 osd.70 pg_epoch: 71594 pg[216.196( v 71093'258 (0'0,71093'258] local-les=71449 n=122 ec=70987 les/c 71590/71583 71591/71591/71444) [21,70,68]/[21,68,17] r=-1 lpr=71591 pi=70987-71590/33 crt=71093'256 lcod 0'0 remapped NOTIFY] exit Started 1.921996 0 0.000000 2014-12-08 18:14:02.568522 7f28fc345700 10 osd.70 pg_epoch: 71594 pg[218.4f3( v 71581'3 (0'0,71581'3] local-les=71594 n=3 ec=71423 les/c 71590/71583 71591/71593/71593) [70,76,3] r=0 lpr=71593 pi=71432-71592/6 crt=71580'1 lcod 0'0 mlcod 0'0 inactive] needs_recovery is recovered -- 2014-12-08 18:14:02.617316 7f28fcb46700 10 osd.70 pg_epoch: 71594 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] handle_peering_event: epoch_sent: 71594 epoch_requested: 71594 MNotifyRec from 16 notify: (query_epoch:71594, epoch_sent:71594, info:19.6e( empty local-les=0 n=0 ec=21298 les/c 51101/50495 71591/71591/71591)) features: 0x3fffffffffff 2014-12-08 18:14:02.617333 7f28fc345700 10 log is not dirty 2014-12-08 18:14:02.617337 7f2904b56700 7 osd.70 71594 handle_pg_info pg_info(1 pgs e71594:218.2a2) v4 from osd.76 2014-12-08 18:14:02.617341 7f2904b56700 15 osd.70 71594 require_same_or_newer_map 71594 (i am 71594) 0x13ef10e0 2014-12-08 18:14:02.617341 7f28fc345700 20 osd.70 71594 share_map_peer 0x12839760 already has epoch 71594 2014-12-08 18:14:02.617335 7f28fcb46700 7 osd.70 pg_epoch: 71594 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] state: handle_pg_notify from osd.16 2014-12-08 18:14:02.617346 7f28fc345700 7 osd.70 71594 do_notifies osd 17 on 1 PGs 2014-12-08 18:14:02.617351 7f28fcb46700 10 osd.70 pg_epoch: 71594 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] state: pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] got dup osd.16 info 19.6e( empty local-les=0 n=0 ec=21298 les/c 51101/50495 71591/71591/71591), identical to ours 2014-12-08 18:14:02.617374 7f28fcb46700 10 log is not dirty 2014-12-08 18:14:02.617378 7f2904b56700 10 osd.70 71594 do_waiters -- start 2014-12-08 18:14:02.617381 7f2904b56700 10 osd.70 71594 do_waiters -- finish 2014-12-08 18:14:02.617380 7f28fcb46700 10 osd.70 pg_epoch: 71594 pg[216.29a( v 71093'209 (0'0,71093'209] local-les=71453 n=104 ec=70987 les/c 71585/71583 71591/71593/71444) [19,70,14] r=1 lpr=71593 pi=70987-71592/32 crt=71093'207 lcod 0'0 inactive NOTIFY] handle_peering_event: epoch_sent: 71594 epoch_requested: 71594 MInfoRec from 19 info: 216.29a( v 71093'209 (0'0,71093'209] local-les=71594 n=104 ec=70987 les/c 71585/71583 71591/71593/71444) 2014-12-08 18:14:02.617398 7f2904b56700 10 osd.70 71594 do_waiters -- start 2014-12-08 18:14:02.617401 7f2904b56700 10 osd.70 71594 do_waiters -- finish 2014-12-08 18:14:02.617393 7f28fcb46700 10 osd.70 pg_epoch: 71594 pg[216.29a( v 71093'209 (0'0,71093'209] local-les=71453 n=104 ec=70987 les/c 71585/71583 71591/71593/71444) [19,70,14] r=1 lpr=71593 pi=70987-71592/32 crt=71093'207 lcod 0'0 inactive NOTIFY] state: got info from osd.19 216.29a( v 71093'209 (0'0,71093'209] local-les=71594 n=104 ec=70987 les/c 71585/71583 71591/71593/71444) 2014-12-08 18:14:02.617403 7f2904b56700 20 osd.70 71594 _dispatch 0x12eaa000 pg_info(1 pgs e71594:218.464) v4 2014-12-08 18:14:02.617408 7f28fcb46700 5 osd.70 pg_epoch: 71594 pg[216.29a( v 71093'209 (0'0,71093'209] local-les=71453 n=104 ec=70987 les/c 71585/71583 71591/71593/71444) [19,70,14] r=1 lpr=71593 pi=70987-71592/32 crt=71093'207 lcod 0'0 inactive NOTIFY] exit Started/Stray 0.639907 7 0.000399 2014-12-08 18:14:02.617402 7f28fc345700 10 osd.70 pg_epoch: 71594 pg[216.355( v 71097'225 (0'0,71097'225] local-les=71594 n=109 ec=70987 les/c 71585/71583 71591/71593/71593) [70,8,78] r=0 lpr=71593 pi=71432-71592/5 crt=71097'225 lcod 0'0 mlcod 0'0 inactive] handle_peering_event: epoch_sent: 71594 epoch_requested: 71594 MNotifyRec from 8 notify: (query_epoch:71594, epoch_sent:71594, info:216.355( v 71097'225 (0'0,71097'225] local-les=71585 n=109 ec=70987 les/c 71585/71583 71591/71593/71593)) features: 0x3fffffffffff 2014-12-08 18:14:02.617423 7f28fcb46700 5 osd.70 pg_epoch: 71594 pg[216.29a( v 71093'209 (0'0,71093'209] local-les=71453 n=104 ec=70987 les/c 71585/71583 71591/71593/71444) [19,70,14] r=1 lpr=71593 pi=70987-71592/32 crt=71093'207 lcod 0'0 inactive NOTIFY] enter Started/ReplicaActive 2014-12-08 18:14:02.617437 7f2904b56700 7 osd.70 71594 handle_pg_info pg_info(1 pgs e71594:218.464) v4 from osd.66 2014-12-08 18:14:02.617427 7f28fc345700 10 osd.70 pg_epoch: 71594 pg[216.355( v 71097'225 (0'0,71097'225] local-les=71594 n=109 ec=70987 les/c 71585/71583 71591/71593/71593) [70,8,78] r=0 lpr=71593 pi=71432-71592/5 crt=71097'225 lcod 0'0 mlcod 0'0 inactive] state: Active: got notify from 8, already have info from that osd, ignoring 2014-12-08 18:14:02.617441 7f2904b56700 15 osd.70 71594 require_same_or_newer_map 71594 (i am 71594) 0x12eaa000 2014-12-08 18:14:02.617444 7f28fc345700 10 log is not dirty 2014-12-08 18:14:02.617440 7f28fcb46700 5 osd.70 pg_epoch: 71594 pg[216.29a( v 71093'209 (0'0,71093'209] local-les=71453 n=104 ec=70987 les/c 71585/71583 71591/71593/71444) [19,70,14] r=1 lpr=71593 pi=70987-71592/32 crt=71093'207 lcod 0'0 inactive NOTIFY] enter Started/ReplicaActive/RepNotRecovering 2014-12-08 18:14:02.617454 7f28fcb46700 10 osd.70 pg_epoch: 71594 pg[216.29a( v 71093'209 (0'0,71093'209] local-les=71453 n=104 ec=70987 les/c 71585/71583 71591/71593/71444) [19,70,14] r=1 lpr=71593 pi=70987-71592/32 crt=71093'207 lcod 0'0 inactive NOTIFY] state: In ReplicaActive, about to call activate 2014-12-08 18:14:02.617472 7f2904b56700 10 osd.70 71594 do_waiters -- start 2014-12-08 18:14:02.617466 7f28fcb46700 10 osd.70 pg_epoch: 71594 pg[216.29a( v 71093'209 (0'0,71093'209] local-les=71594 n=104 ec=70987 les/c 71585/71583 71591/71593/71444) [19,70,14] r=1 lpr=71593 pi=70987-71592/32 crt=71093'207 lcod 0'0 inactive] activate - no missing, moving last_complete 71093'209 -> 71093'209 2014-12-08 18:14:02.617476 7f2904b56700 10 osd.70 71594 do_waiters -- finish -- 2014-12-08 18:14:02.618068 7f28fcb46700 10 osd.70 pg_epoch: 71594 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] handle_peering_event: epoch_sent: 71594 epoch_requested: 71594 MNotifyRec from 58 notify: (query_epoch:71594, epoch_sent:71594, info:19.6e( empty local-les=0 n=0 ec=21298 les/c 51101/50495 71591/71591/71591)) features: 0x3fffffffffff 2014-12-08 18:14:02.618082 7f28fc345700 10 osd.70 pg_epoch: 71594 pg[218.4d( v 71581'1 (0'0,71581'1] local-les=71594 n=1 ec=71423 les/c 71585/71583 71591/71593/71425) [6,19,70] r=2 lpr=71593 pi=71423-71592/11 crt=0'0 lcod 0'0 inactive] activate - no missing, moving last_complete 71581'1 -> 71581'1 2014-12-08 18:14:02.618060 7f2904b56700 10 osd.70 71594 do_waiters -- finish 2014-12-08 18:14:02.618102 7f2904b56700 20 osd.70 71594 _dispatch 0x135bed20 pg_info(1 pgs e71594:218.75f) v4 2014-12-08 18:14:02.618090 7f28fcb46700 7 osd.70 pg_epoch: 71594 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] state: handle_pg_notify from osd.58 2014-12-08 18:14:02.618096 7f28fc345700 10 osd.70 pg_epoch: 71594 pg[218.4d( v 71581'1 (0'0,71581'1] local-les=71594 n=1 ec=71423 les/c 71585/71583 71591/71593/71425) [6,19,70] r=2 lpr=71593 pi=71423-71592/11 crt=0'0 lcod 0'0 inactive] state: Activate Finished 2014-12-08 18:14:02.618112 7f28fcb46700 10 osd.70 pg_epoch: 71594 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] state: pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] got dup osd.58 info 19.6e( empty local-les=0 n=0 ec=21298 les/c 51101/50495 71591/71591/71591), identical to ours 2014-12-08 18:14:02.618141 7f28fcb46700 10 log is not dirty 2014-12-08 18:14:02.618147 7f2904b56700 7 osd.70 71594 handle_pg_info pg_info(1 pgs e71594:218.75f) v4 from osd.10 2014-12-08 18:14:02.618150 7f2904b56700 15 osd.70 71594 require_same_or_newer_map 71594 (i am 71594) 0x135bed20 2014-12-08 18:14:02.618156 7f28fc345700 10 log is not dirty 2014-12-08 18:14:02.618150 7f28fcb46700 10 osd.70 pg_epoch: 71594 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] handle_peering_event: epoch_sent: 71594 epoch_requested: 71594 MNotifyRec from 76 notify: (query_epoch:71594, epoch_sent:71594, info:19.6e( empty local-les=0 n=0 ec=21298 les/c 51101/50495 71591/71591/71591)) features: 0x3fffffffffff 2014-12-08 18:14:02.618162 7f28fcb46700 7 osd.70 pg_epoch: 71594 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] state: handle_pg_notify from osd.76 2014-12-08 18:14:02.618186 7f2904b56700 10 osd.70 71594 do_waiters -- start 2014-12-08 18:14:02.618173 7f28fcb46700 10 osd.70 pg_epoch: 71594 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] state: pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] got dup osd.76 info 19.6e( empty local-les=0 n=0 ec=21298 les/c 51101/50495 71591/71591/71591), identical to ours 2014-12-08 18:14:02.618189 7f2904b56700 10 osd.70 71594 do_waiters -- finish 2014-12-08 18:14:02.618192 7f28fcb46700 10 log is not dirty 2014-12-08 18:14:02.618174 7f28fc345700 10 osd.70 pg_epoch: 71594 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71590/71583 71591/71593/70961) [75,70] r=1 lpr=71593 pi=71183-71592/19 crt=57981'40 lcod 0'0 inactive NOTIFY] handle_peering_event: epoch_sent: 71594 epoch_requested: 71594 MInfoRec from 75 info: 20.64( v 57981'40 (0'0,57981'40] local-les=71594 n=40 ec=21299 les/c 71590/71583 71591/71593/70961) 2014-12-08 18:14:02.618197 7f28fcb46700 10 osd.70 pg_epoch: 71594 pg[218.51( v 71581'3 (0'0,71581'3] local-les=71594 n=3 ec=71423 les/c 71585/71583 71591/71593/71593) [70,66,75] r=0 lpr=71593 pi=71432-71592/6 crt=71580'1 lcod 0'0 mlcod 0'0 inactive] handle_peering_event: epoch_sent: 71594 epoch_requested: 71594 MNotifyRec from 66 notify: (query_epoch:71594, epoch_sent:71594, info:218.51( v 71581'3 (0'0,71581'3] local-les=71585 n=3 ec=71423 les/c 71585/71583 71591/71593/71593)) features: 0x3fffffffffff 2014-12-08 18:14:02.618208 7f2904b56700 10 osd.70 71594 do_waiters -- start 2014-12-08 18:14:02.618211 7f2904b56700 10 osd.70 71594 do_waiters -- finish 2014-12-08 18:14:02.618196 7f28fc345700 10 osd.70 pg_epoch: 71594 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71590/71583 71591/71593/70961) [75,70] r=1 lpr=71593 pi=71183-71592/19 crt=57981'40 lcod 0'0 inactive NOTIFY] state: got info from osd.75 20.64( v 57981'40 (0'0,57981'40] local-les=71594 n=40 ec=21299 les/c 71590/71583 71591/71593/70961) 2014-12-08 18:14:02.618214 7f2904b56700 20 osd.70 71594 _dispatch 0x13f943a0 pg_info(1 pgs e71594:216.12) v4 2014-12-08 18:14:02.618210 7f28fcb46700 10 osd.70 pg_epoch: 71594 pg[218.51( v 71581'3 (0'0,71581'3] local-les=71594 n=3 ec=71423 les/c 71585/71583 71591/71593/71593) [70,66,75] r=0 lpr=71593 pi=71432-71592/6 crt=71580'1 lcod 0'0 mlcod 0'0 inactive] state: Active: got notify from 66, already have info from that osd, ignoring 2014-12-08 18:14:02.618224 7f28fcb46700 10 log is not dirty 2014-12-08 18:14:02.618218 7f28fc345700 5 osd.70 pg_epoch: 71594 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71590/71583 71591/71593/70961) [75,70] r=1 lpr=71593 pi=71183-71592/19 crt=57981'40 lcod 0'0 inactive NOTIFY] exit Started/Stray 0.730945 7 0.000276 2014-12-08 18:14:02.618228 7f28fcb46700 10 osd.70 pg_epoch: 71594 pg[216.2e7( v 71097'253 (0'0,71097'253] local-les=71594 n=117 ec=70987 les/c 71585/71583 71591/71593/71593) [70,22,76] r=0 lpr=71593 pi=71432-71592/5 crt=71097'251 lcod 0'0 mlcod 0'0 inactive] handle_peering_event: epoch_sent: 71594 epoch_requested: 71594 MNotifyRec from 64 notify: (query_epoch:71594, epoch_sent:71594, info:216.2e7( v 71097'253 lc 71097'13 (0'0,71097'253] local-les=71585 n=117 ec=70987 les/c 71585/71583 71591/71593/71593)) features: 0x3fffffffffff 2014-12-08 18:14:02.618241 7f28fcb46700 10 osd.70 pg_epoch: 71594 pg[216.2e7( v 71097'253 (0'0,71097'253] local-les=71594 n=117 ec=70987 les/c 71585/71583 71591/71593/71593) [70,22,76] r=0 lpr=71593 pi=71432-71592/5 crt=71097'251 lcod 0'0 mlcod 0'0 inactive] state: Active: got notify from 64, already have info from that osd, ignoring 2014-12-08 18:14:02.618255 7f28fcb46700 10 log is not dirty 2014-12-08 18:14:02.618255 7f2904b56700 7 osd.70 71594 handle_pg_info pg_info(1 pgs e71594:216.12) v4 from osd.77 2014-12-08 18:14:02.618242 7f28fc345700 5 osd.70 pg_epoch: 71594 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71590/71583 71591/71593/70961) [75,70] r=1 lpr=71593 pi=71183-71592/19 crt=57981'40 lcod 0'0 inactive NOTIFY] enter Started/ReplicaActive 2014-12-08 18:14:02.618262 7f2904b56700 15 osd.70 71594 require_same_or_newer_map 71594 (i am 71594) 0x13f943a0 2014-12-08 18:14:02.618260 7f28fcb46700 10 osd.70 pg_epoch: 71594 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] handle_peering_event: epoch_sent: 71594 epoch_requested: 71594 MNotifyRec from 4 notify: (query_epoch:71594, epoch_sent:71594, info:19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591)) features: 0x3fffffffffff 2014-12-08 18:14:02.618268 7f28fc345700 5 osd.70 pg_epoch: 71594 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71590/71583 71591/71593/70961) [75,70] r=1 lpr=71593 pi=71183-71592/19 crt=57981'40 lcod 0'0 inactive NOTIFY] enter Started/ReplicaActive/RepNotRecovering 2014-12-08 18:14:02.618279 7f28fcb46700 7 osd.70 pg_epoch: 71594 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] state: handle_pg_notify from osd.4 2014-12-08 18:14:02.618292 7f2904b56700 10 osd.70 71594 do_waiters -- start 2014-12-08 18:14:02.618294 7f2904b56700 10 osd.70 71594 do_waiters -- finish 2014-12-08 18:14:02.618287 7f28fc345700 10 osd.70 pg_epoch: 71594 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71434 n=40 ec=21299 les/c 71590/71583 71591/71593/70961) [75,70] r=1 lpr=71593 pi=71183-71592/19 crt=57981'40 lcod 0'0 inactive NOTIFY] state: In ReplicaActive, about to call activate 2014-12-08 18:14:02.618311 7f2904b56700 10 osd.70 71594 do_waiters -- start 2014-12-08 18:14:02.618313 7f2904b56700 10 osd.70 71594 do_waiters -- finish 2014-12-08 18:14:02.618293 7f28fcb46700 10 osd.70 pg_epoch: 71594 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] state: pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] got dup osd.4 info 19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591), identical to ours 2014-12-08 18:14:02.618315 7f2904b56700 20 osd.70 71594 _dispatch 0x12eabe00 pg_info(1 pgs e71594:218.47e) v4 2014-12-08 18:14:02.618319 7f28fcb46700 10 log is not dirty 2014-12-08 18:14:02.618307 7f28fc345700 10 osd.70 pg_epoch: 71594 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71594 n=40 ec=21299 les/c 71590/71583 71591/71593/70961) [75,70] r=1 lpr=71593 pi=71183-71592/19 crt=57981'40 lcod 0'0 inactive] activate - no missing, moving last_complete 57981'40 -> 57981'40 2014-12-08 18:14:02.618325 7f28fcb46700 10 osd.70 pg_epoch: 71594 pg[218.1e3( v 71581'1 (0'0,71581'1] local-les=71594 n=1 ec=71423 les/c 71585/71583 71591/71593/71593) [70,76,14] r=0 lpr=71593 pi=71432-71592/6 crt=0'0 lcod 0'0 mlcod 0'0 inactive] handle_peering_event: epoch_sent: 71594 epoch_requested: 71594 MNotifyRec from 63 notify: (query_epoch:71594, epoch_sent:71594, info:218.1e3( v 71581'1 lc 0'0 (0'0,71581'1] local-les=71585 n=1 ec=71423 les/c 71585/71583 71591/71593/71593)) features: 0x3fffffffffff 2014-12-08 18:14:02.618331 7f28fc345700 10 osd.70 pg_epoch: 71594 pg[20.64( v 57981'40 (0'0,57981'40] local-les=71594 n=40 ec=21299 les/c 71590/71583 71591/71593/70961) [75,70] r=1 lpr=71593 pi=71183-71592/19 crt=57981'40 lcod 0'0 inactive] state: Activate Finished 2014-12-08 18:14:02.618341 7f28fcb46700 10 osd.70 pg_epoch: 71594 pg[218.1e3( v 71581'1 (0'0,71581'1] local-les=71594 n=1 ec=71423 les/c 71585/71583 71591/71593/71593) [70,76,14] r=0 lpr=71593 pi=71432-71592/6 crt=0'0 lcod 0'0 mlcod 0'0 inactive] state: Active: got notify from 63, already have info from that osd, ignoring 2014-12-08 18:14:02.618352 7f2904b56700 7 osd.70 71594 handle_pg_info pg_info(1 pgs e71594:218.47e) v4 from osd.14 2014-12-08 18:14:02.618356 7f2904b56700 15 osd.70 71594 require_same_or_newer_map 71594 (i am 71594) 0x12eabe00 2014-12-08 18:14:02.618357 7f28fcb46700 10 log is not dirty 2014-12-08 18:14:02.618362 7f28fcb46700 10 osd.70 pg_epoch: 71594 pg[218.7aa( empty local-les=71457 n=0 ec=71423 les/c 71585/71585 71591/71593/71456) [25,70,58] r=1 lpr=71593 pi=71444-71592/6 crt=0'0 inactive NOTIFY] handle_peering_event: epoch_sent: 71594 epoch_requested: 71594 MLogRec from 25 2014-12-08 18:14:02.618374 7f28fcb46700 10 osd.70 pg_epoch: 71594 pg[218.7aa( empty local-les=71457 n=0 ec=71423 les/c 71585/71585 71591/71593/71456) [25,70,58] r=1 lpr=71593 pi=71444-71592/6 crt=0'0 inactive NOTIFY] state: got info+log from osd.25 218.7aa( empty local-les=71594 n=0 ec=71423 les/c 71585/71585 71591/71593/71456) log((0'0,0'0], crt=0'0) 2014-12-08 18:14:02.618386 7f2904b56700 10 osd.70 71594 do_waiters -- start 2014-12-08 18:14:02.618389 7f2904b56700 10 osd.70 71594 do_waiters -- finish 2014-12-08 18:14:02.618387 7f28fcb46700 10 merge_log log((0'0,0'0], crt=0'0) from osd.25 into log((0'0,0'0], crt=0'0) 2014-12-08 18:14:02.618396 7f28fcb46700 10 merge_log result log((0'0,0'0], crt=0'0) missing(0) changed=0 2014-12-08 18:14:02.618400 7f28fc345700 10 log is not dirty 2014-12-08 18:14:02.618405 7f2904b56700 10 osd.70 71594 do_waiters -- start 2014-12-08 18:14:02.618408 7f2904b56700 10 osd.70 71594 do_waiters -- finish 2014-12-08 18:14:02.618400 7f28fcb46700 5 osd.70 pg_epoch: 71594 pg[218.7aa( empty local-les=71457 n=0 ec=71423 les/c 71585/71585 71591/71593/71456) [25,70,58] r=1 lpr=71593 pi=71444-71592/6 crt=0'0 inactive NOTIFY] exit Started/Stray 0.643274 7 0.004031 2014-12-08 18:14:02.618410 7f2904b56700 20 osd.70 71594 _dispatch 0x12eadde0 pg_info(1 pgs e71594:218.6ae) v4 -- 2014-12-08 18:14:02.635546 7f2904b56700 20 osd.70 71594 _dispatch 0x135ba960 pg_notify(19.6e(150) epoch 71594) v5 2014-12-08 18:14:02.635540 7f28fc345700 10 osd.70 pg_epoch: 71594 pg[218.18a( v 71581'3 (0'0,71581'3] local-les=71434 n=3 ec=71423 les/c 71585/71583 71591/71593/71587) [78,24,70] r=2 lpr=71593 pi=71423-71592/10 crt=71580'1 lcod 0'0 inactive NOTIFY] state: In ReplicaActive, about to call activate 2014-12-08 18:14:02.635540 7f28fcb46700 10 osd.70 pg_epoch: 71594 pg[218.1e3( v 71581'1 (0'0,71581'1] local-les=71594 n=1 ec=71423 les/c 71585/71583 71591/71593/71593) [70,76,14] r=0 lpr=71593 pi=71432-71592/6 crt=0'0 lcod 0'0 mlcod 0'0 inactive] state: Active: got notify from 76, already have info from that osd, ignoring 2014-12-08 18:14:02.635559 7f28fcb46700 10 log is not dirty 2014-12-08 18:14:02.635556 7f28fc345700 10 osd.70 pg_epoch: 71594 pg[218.18a( v 71581'3 (0'0,71581'3] local-les=71594 n=3 ec=71423 les/c 71585/71583 71591/71593/71587) [78,24,70] r=2 lpr=71593 pi=71423-71592/10 crt=71580'1 lcod 0'0 inactive] activate - no missing, moving last_complete 71581'3 -> 71581'3 2014-12-08 18:14:02.635570 7f2904b56700 7 osd.70 71594 handle_pg_notify from osd.30 2014-12-08 18:14:02.635573 7f2904b56700 15 osd.70 71594 require_same_or_newer_map 71594 (i am 71594) 0x135ba960 2014-12-08 18:14:02.635565 7f28fc345700 10 osd.70 pg_epoch: 71594 pg[218.18a( v 71581'3 (0'0,71581'3] local-les=71594 n=3 ec=71423 les/c 71585/71583 71591/71593/71587) [78,24,70] r=2 lpr=71593 pi=71423-71592/10 crt=71580'1 lcod 0'0 inactive] state: Activate Finished 2014-12-08 18:14:02.635567 7f28fcb46700 10 osd.70 pg_epoch: 71594 pg[218.4f3( v 71581'3 (0'0,71581'3] local-les=71594 n=3 ec=71423 les/c 71590/71583 71591/71593/71593) [70,76,3] r=0 lpr=71593 pi=71432-71592/6 crt=71580'1 lcod 0'0 mlcod 0'0 inactive] handle_peering_event: epoch_sent: 71594 epoch_requested: 71594 MNotifyRec from 76 notify: (query_epoch:71594, epoch_sent:71594, info:218.4f3( v 71581'3 (0'0,71581'3] local-les=71590 n=3 ec=71423 les/c 71590/71583 71591/71593/71593)) features: 0x3fffffffffff 2014-12-08 18:14:02.635582 7f28fcb46700 10 osd.70 pg_epoch: 71594 pg[218.4f3( v 71581'3 (0'0,71581'3] local-les=71594 n=3 ec=71423 les/c 71590/71583 71591/71593/71593) [70,76,3] r=0 lpr=71593 pi=71432-71592/6 crt=71580'1 lcod 0'0 mlcod 0'0 inactive] state: Active: got notify from 76, already have info from that osd, ignoring 2014-12-08 18:14:02.635591 7f28fcb46700 10 log is not dirty 2014-12-08 18:14:02.635595 7f2904b56700 10 osd.70 71594 do_waiters -- start 2014-12-08 18:14:02.635597 7f2904b56700 10 osd.70 71594 do_waiters -- finish 2014-12-08 18:14:02.635595 7f28fcb46700 10 osd.70 pg_epoch: 71594 pg[218.16c( v 71581'1 (0'0,71581'1] local-les=71433 n=1 ec=71423 les/c 71585/71583 71591/71593/71425) [6,56,70] r=2 lpr=71593 pi=71423-71592/10 crt=0'0 lcod 0'0 inactive NOTIFY] handle_peering_event: epoch_sent: 71594 epoch_requested: 71594 MInfoRec from 6 info: 218.16c( v 71581'1 (0'0,71581'1] local-les=71594 n=1 ec=71423 les/c 71585/71583 71591/71593/71425) 2014-12-08 18:14:02.635608 7f28fc345700 10 log is not dirty 2014-12-08 18:14:02.635606 7f28fcb46700 10 osd.70 pg_epoch: 71594 pg[218.16c( v 71581'1 (0'0,71581'1] local-les=71433 n=1 ec=71423 les/c 71585/71583 71591/71593/71425) [6,56,70] r=2 lpr=71593 pi=71423-71592/10 crt=0'0 lcod 0'0 inactive NOTIFY] state: got info from osd.6 218.16c( v 71581'1 (0'0,71581'1] local-les=71594 n=1 ec=71423 les/c 71585/71583 71591/71593/71425) 2014-12-08 18:14:02.635617 7f28fcb46700 5 osd.70 pg_epoch: 71594 pg[218.16c( v 71581'1 (0'0,71581'1] local-les=71433 n=1 ec=71423 les/c 71585/71583 71591/71593/71425) [6,56,70] r=2 lpr=71593 pi=71423-71592/10 crt=0'0 lcod 0'0 inactive NOTIFY] exit Started/Stray 0.668477 7 0.000198 2014-12-08 18:14:02.635621 7f28fc345700 10 osd.70 pg_epoch: 71594 pg[218.785( v 71581'2 (0'0,71581'2] local-les=71594 n=2 ec=71423 les/c 71585/71586 71591/71593/71593) [70,18,78] r=0 lpr=71593 pi=71432-71592/6 crt=0'0 lcod 0'0 mlcod 0'0 degraded] handle_peering_event: epoch_sent: 71594 epoch_requested: 71594 MNotifyRec from 63 notify: (query_epoch:71594, epoch_sent:71594, info:218.785( v 71581'2 (0'0,71581'2] local-les=71585 n=2 ec=71423 les/c 71585/71586 71591/71593/71593)) features: 0x3fffffffffff 2014-12-08 18:14:02.635630 7f28fcb46700 5 osd.70 pg_epoch: 71594 pg[218.16c( v 71581'1 (0'0,71581'1] local-les=71433 n=1 ec=71423 les/c 71585/71583 71591/71593/71425) [6,56,70] r=2 lpr=71593 pi=71423-71592/10 crt=0'0 lcod 0'0 inactive NOTIFY] enter Started/ReplicaActive 2014-12-08 18:14:02.635635 7f28fc345700 10 osd.70 pg_epoch: 71594 pg[218.785( v 71581'2 (0'0,71581'2] local-les=71594 n=2 ec=71423 les/c 71585/71586 71591/71593/71593) [70,18,78] r=0 lpr=71593 pi=71432-71592/6 crt=0'0 lcod 0'0 mlcod 0'0 degraded] state: Active: got notify from 63, already have info from that osd, ignoring 2014-12-08 18:14:02.635646 7f28fc345700 10 log is not dirty -- 2014-12-08 18:14:02.657517 7f28fc345700 10 osd.70 pg_epoch: 71594 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] handle_peering_event: epoch_sent: 71594 epoch_requested: 71594 MNotifyRec from 30 notify: (query_epoch:71594, epoch_sent:71594, info:19.6e( empty local-les=0 n=0 ec=21298 les/c 51101/50495 71591/71591/71591)) features: 0x3fffffffffff 2014-12-08 18:14:02.657533 7f28fc345700 7 osd.70 pg_epoch: 71594 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] state: handle_pg_notify from osd.30 2014-12-08 18:14:02.657542 7f28fc345700 10 osd.70 pg_epoch: 71594 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] state: pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] got dup osd.30 info 19.6e( empty local-les=0 n=0 ec=21298 les/c 51101/50495 71591/71591/71591), identical to ours 2014-12-08 18:14:02.657569 7f28fc345700 10 log is not dirty 2014-12-08 18:14:02.657574 7f28fc345700 10 osd.70 pg_epoch: 71594 pg[216.ab( v 71097'289 (0'0,71097'289] local-les=71594 n=133 ec=70987 les/c 71594/71583 71591/71593/71168) [20,64,70] r=2 lpr=71593 pi=71183-71592/17 luod=0'0 crt=71097'289 lcod 0'0 active] handle_peering_event: epoch_sent: 71594 epoch_requested: 71594 MInfoRec from 20 info: 216.ab( v 71097'289 (0'0,71097'289] local-les=71594 n=133 ec=70987 les/c 71594/71594 71591/71593/71168) 2014-12-08 18:14:02.657597 7f28fc345700 10 log is not dirty 2014-12-08 18:14:02.657592 7f28fcb46700 10 osd.70 pg_epoch: 71594 pg[218.2ef( v 71581'2 (0'0,71581'2] local-les=71594 n=2 ec=71423 les/c 71594/71583 71591/71593/71587) [68,70,4] r=1 lpr=71593 pi=71423-71592/10 luod=0'0 crt=0'0 lcod 0'0 active] handle_peering_event: epoch_sent: 71594 epoch_requested: 71594 MInfoRec from 68 info: 218.2ef( v 71581'2 (0'0,71581'2] local-les=71594 n=2 ec=71423 les/c 71594/71594 71591/71593/71587) 2014-12-08 18:14:02.657608 7f28fc345700 10 osd.70 pg_epoch: 71594 pg[218.697( empty local-les=71594 n=0 ec=71423 les/c 71585/71585 71591/71593/71425) [20,70,4] r=1 lpr=71593 pi=71423-71592/10 crt=0'0 active] handle_peering_event: epoch_sent: 71594 epoch_requested: 71594 MInfoRec from 20 info: 218.697( empty local-les=71594 n=0 ec=71423 les/c 71594/71585 71591/71593/71425) 2014-12-08 18:14:02.657632 7f28fc345700 10 log is not dirty 2014-12-08 18:14:02.657631 7f28fcb46700 10 log is not dirty 2014-12-08 18:14:02.657661 7f28fcb46700 10 osd.70 pg_epoch: 71594 pg[217.3f7( v 71376'21 (0'0,71376'21] local-les=71594 n=7 ec=71094 les/c 71594/71583 71591/71593/71593) [70,56,68] r=0 lpr=71593 pi=71432-71592/5 crt=71097'19 lcod 0'0 mlcod 0'0 inactive] handle_peering_event: epoch_sent: 71594 epoch_requested: 71594 AllReplicasActivated 2014-12-08 18:14:02.657682 7f28fcb46700 10 osd.70 pg_epoch: 71594 pg[217.3f7( v 71376'21 (0'0,71376'21] local-les=71594 n=7 ec=71094 les/c 71594/71583 71591/71593/71593) [70,56,68] r=0 lpr=71593 pi=71432-71592/5 crt=71097'19 lcod 0'0 mlcod 0'0 active] check_local 2014-12-08 18:14:02.657692 7f28fcb46700 15 osd.70 pg_epoch: 71594 pg[217.3f7( v 71376'21 (0'0,71376'21] local-les=71594 n=7 ec=71094 les/c 71594/71583 71591/71593/71593) [70,56,68] r=0 lpr=71593 pi=71432-71592/5 crt=71097'19 lcod 0'0 mlcod 0'0 active] requeue_ops 2014-12-08 18:14:02.657704 7f28fcb46700 10 osd.70 pg_epoch: 71594 pg[217.3f7( v 71376'21 (0'0,71376'21] local-les=71594 n=7 ec=71094 les/c 71594/71583 71591/71593/71593) [70,56,68] r=0 lpr=71593 pi=71432-71592/5 crt=71097'19 lcod 0'0 mlcod 0'0 active] needs_recovery is recovered 2014-12-08 18:14:02.657715 7f28fcb46700 10 osd.70 pg_epoch: 71594 pg[217.3f7( v 71376'21 (0'0,71376'21] local-les=71594 n=7 ec=71094 les/c 71594/71583 71591/71593/71593) [70,56,68] r=0 lpr=71593 pi=71432-71592/5 crt=71097'19 lcod 0'0 mlcod 0'0 active] needs_backfill does not need backfill 2014-12-08 18:14:02.657725 7f28fcb46700 10 osd.70 pg_epoch: 71594 pg[217.3f7( v 71376'21 (0'0,71376'21] local-les=71594 n=7 ec=71094 les/c 71594/71583 71591/71593/71593) [70,56,68] r=0 lpr=71593 pi=71432-71592/5 crt=71097'19 lcod 0'0 mlcod 0'0 active] activate all replicas clean, no recovery 2014-12-08 18:14:02.657749 7f28fcb46700 15 osd.70 pg_epoch: 71594 pg[217.3f7( v 71376'21 (0'0,71376'21] local-les=71594 n=7 ec=71094 les/c 71594/71583 71591/71593/71593) [70,56,68] r=0 lpr=71593 pi=71432-71592/5 crt=71097'19 lcod 0'0 mlcod 0'0 active] publish_stats_to_osd 71594:173 2014-12-08 18:14:02.657759 7f28fc345700 10 osd.70 pg_epoch: 71594 pg[218.697( empty local-les=71594 n=0 ec=71423 les/c 71594/71585 71591/71593/71425) [20,70,4] r=1 lpr=71593 pi=71423-71592/10 crt=0'0 active] handle_peering_event: epoch_sent: 71594 epoch_requested: 71594 MInfoRec from 20 info: 218.697( empty local-les=71594 n=0 ec=71423 les/c 71594/71594 71591/71593/71425) 2014-12-08 18:14:02.657763 7f28fcb46700 20 osd.70 pg_epoch: 71594 pg[217.3f7( v 71376'21 (0'0,71376'21] local-les=71594 n=7 ec=71094 les/c 71594/71583 71591/71593/71593) [70,56,68] r=0 lpr=71593 pi=71432-71592/5 crt=71097'19 lcod 0'0 mlcod 0'0 active] hit_set_clear 2014-12-08 18:14:02.657787 7f28fc345700 10 log is not dirty 2014-12-08 18:14:02.657780 7f28fcb46700 20 osd.70 pg_epoch: 71594 pg[217.3f7( v 71376'21 (0'0,71376'21] local-les=71594 n=7 ec=71094 les/c 71594/71583 71591/71593/71593) [70,56,68] r=0 lpr=71593 pi=71432-71592/5 crt=71097'19 lcod 0'0 mlcod 0'0 active] agent_stop 2014-12-08 18:14:02.657794 7f28fcb46700 10 log is not dirty 2014-12-08 18:14:02.657801 7f28fcb46700 10 osd.70 pg_epoch: 71594 pg[218.79a( v 71581'5 (0'0,71581'5] local-les=71594 n=5 ec=71423 les/c 71590/71583 71591/71593/71425) [18,75,70] r=2 lpr=71593 pi=71423-71592/10 luod=0'0 crt=71581'3 lcod 0'0 active] handle_peering_event: epoch_sent: 71594 epoch_requested: 71594 FlushedEvt -- 2014-12-08 18:14:02.682447 7f2911047700 20 osd.70 71594 19.6e heartbeat_peers 0,4,11,13,16,20,30,31,56,58,59,63,70,76 2014-12-08 18:14:02.682442 7f28f6b3a700 10 osd.70 pg_epoch: 71594 pg[218.785( v 71581'2 (0'0,71581'2] local-les=71594 n=2 ec=71423 les/c 71594/71586 71591/71593/71593) [70,18,78] r=0 lpr=71593 pi=71432-71592/6 rops=1 crt=0'0 lcod 0'0 mlcod 0'0 active+recovering+degraded] populate_obc_watchers 23550785/rbd_data.653afa79e2a9e3.0000000000000489/head//218 2014-12-08 18:14:02.682455 7f2911047700 20 osd.70 71594 19.6d heartbeat_peers 2014-12-08 18:14:02.682458 7f2911047700 20 osd.70 71594 218.6c7 heartbeat_peers 10,18,70 2014-12-08 18:14:02.682461 7f2911047700 20 osd.70 71594 218.6c4 heartbeat_peers 13,70,76 2014-12-08 18:14:02.682457 7f28f6b3a700 20 osd.70 pg_epoch: 71594 pg[218.785( v 71581'2 (0'0,71581'2] local-les=71594 n=2 ec=71423 les/c 71594/71586 71591/71593/71593) [70,18,78] r=0 lpr=71593 pi=71432-71592/6 rops=1 crt=0'0 lcod 0'0 mlcod 0'0 active+recovering+degraded] ReplicatedPG::check_blacklisted_obc_watchers for obc 23550785/rbd_data.653afa79e2a9e3.0000000000000489/head//218 2014-12-08 18:14:02.682464 7f2911047700 20 osd.70 71594 218.562 heartbeat_peers 2014-12-08 18:14:02.682466 7f2911047700 20 osd.70 71594 218.5e heartbeat_peers 2014-12-08 18:14:02.682468 7f2911047700 20 osd.70 71594 218.5f heartbeat_peers 15,24,70 2014-12-08 18:14:02.682470 7f2911047700 20 osd.70 71594 217.235 heartbeat_peers 2014-12-08 18:14:02.682465 7f28f6b3a700 10 osd.70 pg_epoch: 71594 pg[218.785( v 71581'2 (0'0,71581'2] local-les=71594 n=2 ec=71423 les/c 71594/71586 71591/71593/71593) [70,18,78] r=0 lpr=71593 pi=71432-71592/6 rops=1 crt=0'0 lcod 0'0 mlcod 0'0 active+recovering+degraded] get_object_context: creating obc from disk: 0x13171600 2014-12-08 18:14:02.682472 7f2911047700 20 osd.70 71594 218.38e heartbeat_peers 3,13,70 2014-12-08 18:14:02.682475 7f2911047700 20 osd.70 71594 218.38f heartbeat_peers 23,25,70 2014-12-08 18:14:02.682478 7f2911047700 20 osd.70 71594 218.38c heartbeat_peers 2014-12-08 18:14:02.682479 7f2911047700 20 osd.70 71594 217.51 heartbeat_peers 2014-12-08 18:14:02.682481 7f2911047700 20 osd.70 71594 216.380 heartbeat_peers 2014-12-08 18:14:02.682483 7f2911047700 20 osd.70 71594 218.51 heartbeat_peers 6,24,66,70,75 2014-12-08 18:14:02.682486 7f2911047700 20 osd.70 71594 218.380 heartbeat_peers 2014-12-08 18:14:02.682474 7f28f6b3a700 10 osd.70 pg_epoch: 71594 pg[218.785( v 71581'2 (0'0,71581'2] local-les=71594 n=2 ec=71423 les/c 71594/71586 71591/71593/71593) [70,18,78] r=0 lpr=71593 pi=71432-71592/6 rops=1 crt=0'0 lcod 0'0 mlcod 0'0 active+recovering+degraded] get_object_context: 0x13171600 23550785/rbd_data.653afa79e2a9e3.0000000000000489/head//218 rwstate(none n=0 w=0) oi: 23550785/rbd_data.653afa79e2a9e3.0000000000000489/head//218(71581'2 client.6634234.0:2333 wrlock_by=unknown.0.0:0 dirty s 8388608 uv2) ssc: 0x45fbdc0 snapset: 0=[]:[]+head 2014-12-08 18:14:02.682488 7f2911047700 20 osd.70 71594 218.518 heartbeat_peers 2014-12-08 18:14:02.682493 7f2911047700 20 osd.70 71594 217.22d heartbeat_peers -- 2014-12-08 18:14:03.264994 7f28fbb44700 10 osd.70 71594 dequeue_op 0x4481900 prio 63 cost 0 latency 0.000534 osd_op(client.6443001.0:180 rbd_data.5d886f14455c93.000000000000009b [delete] 19.3458066e RETRY=39 ack+ondisk+retry+write e71594) v4 pg pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] 2014-12-08 18:14:03.265028 7f28fbb44700 20 osd.70 pg_epoch: 71594 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] op_has_sufficient_caps pool=19 (volumes ) owner=0 need_read_cap=0 need_write_cap=1 need_class_read_cap=0 need_class_write_cap=0 -> yes 2014-12-08 18:14:03.265039 7f28d9bb2700 20 osd.70 71594 should_share_map client.6443001 172.16.5.116:0/109013063 71594 2014-12-08 18:14:03.265042 7f28fbb44700 10 osd.70 pg_epoch: 71594 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] can_handle_while_inactive: 0x4481900 2014-12-08 18:14:03.265052 7f28fbb44700 10 osd.70 71594 dequeue_op 0x4481900 finish 2014-12-08 18:14:03.265055 7f28d9bb2700 15 osd.70 71594 enqueue_op 0x4481a00 prio 63 cost 0 latency 0.000131 osd_op(client.6443001.0:402 rbd_data.5d886f14455c93.0000000000000179 [delete] 19.93050fee RETRY=39 ack+ondisk+retry+write e71594) v4 2014-12-08 18:14:03.265084 7f28f933f700 10 osd.70 71594 dequeue_op 0x4481a00 prio 63 cost 0 latency 0.000160 osd_op(client.6443001.0:402 rbd_data.5d886f14455c93.0000000000000179 [delete] 19.93050fee RETRY=39 ack+ondisk+retry+write e71594) v4 pg pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] 2014-12-08 18:14:03.265152 7f28f933f700 20 osd.70 pg_epoch: 71594 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] op_has_sufficient_caps pool=19 (volumes ) owner=0 need_read_cap=0 need_write_cap=1 need_class_read_cap=0 need_class_write_cap=0 -> yes 2014-12-08 18:14:03.265203 7f28f933f700 10 osd.70 pg_epoch: 71594 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] can_handle_while_inactive: 0x4481a00 2014-12-08 18:14:03.265213 7f28f933f700 10 osd.70 71594 dequeue_op 0x4481a00 finish 2014-12-08 18:14:03.265219 7f28d9bb2700 20 osd.70 71594 should_share_map client.6443001 172.16.5.116:0/109013063 71594 2014-12-08 18:14:03.265249 7f28d9bb2700 15 osd.70 71594 enqueue_op 0x1285de00 prio 63 cost 0 latency 0.000173 osd_op(client.6443001.0:513 rbd_data.5d886f14455c93.00000000000001e8 [delete] 19.557e676e RETRY=39 ack+ondisk+retry+write e71594) v4 2014-12-08 18:14:03.265353 7f28fbb44700 10 osd.70 71594 dequeue_op 0x1285de00 prio 63 cost 0 latency 0.000278 osd_op(client.6443001.0:513 rbd_data.5d886f14455c93.00000000000001e8 [delete] 19.557e676e RETRY=39 ack+ondisk+retry+write e71594) v4 pg pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] 2014-12-08 18:14:03.265386 7f28fbb44700 20 osd.70 pg_epoch: 71594 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] op_has_sufficient_caps pool=19 (volumes ) owner=0 need_read_cap=0 need_write_cap=1 need_class_read_cap=0 need_class_write_cap=0 -> yes 2014-12-08 18:14:03.265402 7f28d9bb2700 20 osd.70 71594 should_share_map client.6443001 172.16.5.116:0/109013063 71594 2014-12-08 18:14:03.265402 7f28fbb44700 10 osd.70 pg_epoch: 71594 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] can_handle_while_inactive: 0x1285de00 2014-12-08 18:14:03.265410 7f28fbb44700 10 osd.70 71594 dequeue_op 0x1285de00 finish 2014-12-08 18:14:03.265418 7f28d9bb2700 15 osd.70 71594 enqueue_op 0x1285dc00 prio 63 cost 0 latency 0.000131 osd_op(client.6443001.0:547 rbd_data.5d886f14455c93.000000000000020a [delete] 19.5052c9ee RETRY=39 ack+ondisk+retry+write e71594) v4 2014-12-08 18:14:03.265440 7f28f933f700 10 osd.70 71594 dequeue_op 0x1285dc00 prio 63 cost 0 latency 0.000153 osd_op(client.6443001.0:547 rbd_data.5d886f14455c93.000000000000020a [delete] 19.5052c9ee RETRY=39 ack+ondisk+retry+write e71594) v4 pg pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] 2014-12-08 18:14:03.265508 7f28f933f700 20 osd.70 pg_epoch: 71594 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] op_has_sufficient_caps pool=19 (volumes ) owner=0 need_read_cap=0 need_write_cap=1 need_class_read_cap=0 need_class_write_cap=0 -> yes 2014-12-08 18:14:03.265559 7f28f933f700 10 osd.70 pg_epoch: 71594 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] can_handle_while_inactive: 0x1285dc00 2014-12-08 18:14:03.265573 7f28f933f700 10 osd.70 71594 dequeue_op 0x1285dc00 finish 2014-12-08 18:14:03.265574 7f28d9bb2700 20 osd.70 71594 should_share_map client.6443001 172.16.5.116:0/109013063 71594 2014-12-08 18:14:03.265590 7f28d9bb2700 15 osd.70 71594 enqueue_op 0x1285c800 prio 63 cost 0 latency 0.000155 osd_op(client.6443001.0:641 rbd_data.5d886f14455c93.0000000000000268 [delete] 19.4826cf6e RETRY=39 ack+ondisk+retry+write e71594) v4 2014-12-08 18:14:03.265668 7f28fbb44700 10 osd.70 71594 dequeue_op 0x1285c800 prio 63 cost 0 latency 0.000232 osd_op(client.6443001.0:641 rbd_data.5d886f14455c93.0000000000000268 [delete] 19.4826cf6e RETRY=39 ack+ondisk+retry+write e71594) v4 pg pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] 2014-12-08 18:14:03.265715 7f28fbb44700 20 osd.70 pg_epoch: 71594 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] op_has_sufficient_caps pool=19 (volumes ) owner=0 need_read_cap=0 need_write_cap=1 need_class_read_cap=0 need_class_write_cap=0 -> yes 2014-12-08 18:14:03.265729 7f28fbb44700 10 osd.70 pg_epoch: 71594 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] can_handle_while_inactive: 0x1285c800 2014-12-08 18:14:03.265736 7f28fbb44700 10 osd.70 71594 dequeue_op 0x1285c800 finish 2014-12-08 18:14:03.265740 7f28d9bb2700 20 osd.70 71594 should_share_map client.6443001 172.16.5.116:0/109013063 71594 2014-12-08 18:14:03.265756 7f28d9bb2700 15 osd.70 71594 enqueue_op 0x1285e400 prio 63 cost 0 latency 0.000116 osd_op(client.6443001.0:642 rbd_data.5d886f14455c93.0000000000000269 [delete] 19.8b6d1e6e RETRY=39 ack+ondisk+retry+write e71594) v4 2014-12-08 18:14:03.265811 7f28f933f700 10 osd.70 71594 dequeue_op 0x1285e400 prio 63 cost 0 latency 0.000171 osd_op(client.6443001.0:642 rbd_data.5d886f14455c93.0000000000000269 [delete] 19.8b6d1e6e RETRY=39 ack+ondisk+retry+write e71594) v4 pg pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] 2014-12-08 18:14:03.265868 7f28f933f700 20 osd.70 pg_epoch: 71594 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] op_has_sufficient_caps pool=19 (volumes ) owner=0 need_read_cap=0 need_write_cap=1 need_class_read_cap=0 need_class_write_cap=0 -> yes 2014-12-08 18:14:03.265907 7f28f933f700 10 osd.70 pg_epoch: 71594 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] can_handle_while_inactive: 0x1285e400 2014-12-08 18:14:03.265916 7f28f933f700 10 osd.70 71594 dequeue_op 0x1285e400 finish 2014-12-08 18:14:03.265923 7f28d9bb2700 20 osd.70 71594 should_share_map client.6443001 172.16.5.116:0/109013063 71594 2014-12-08 18:14:03.265939 7f28d9bb2700 15 osd.70 71594 enqueue_op 0x1285c900 prio 63 cost 0 latency 0.000146 osd_op(client.6443001.0:785 rbd_data.5d886f14455c93.00000000000002f8 [delete] 19.5522f0ee RETRY=39 ack+ondisk+retry+write e71594) v4 2014-12-08 18:14:03.265996 7f28fbb44700 10 osd.70 71594 dequeue_op 0x1285c900 prio 63 cost 0 latency 0.000203 osd_op(client.6443001.0:785 rbd_data.5d886f14455c93.00000000000002f8 [delete] 19.5522f0ee RETRY=39 ack+ondisk+retry+write e71594) v4 pg pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] 2014-12-08 18:14:03.266047 7f28fbb44700 20 osd.70 pg_epoch: 71594 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] op_has_sufficient_caps pool=19 (volumes ) owner=0 need_read_cap=0 need_write_cap=1 need_class_read_cap=0 need_class_write_cap=0 -> yes 2014-12-08 18:14:03.266082 7f28fbb44700 10 osd.70 pg_epoch: 71594 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] can_handle_while_inactive: 0x1285c900 2014-12-08 18:14:03.266091 7f28fbb44700 10 osd.70 71594 dequeue_op 0x1285c900 finish 2014-12-08 18:14:03.266100 7f28d9bb2700 20 osd.70 71594 should_share_map client.6443001 172.16.5.116:0/109013063 71594 2014-12-08 18:14:03.266117 7f28d9bb2700 15 osd.70 71594 enqueue_op 0x1285e100 prio 63 cost 0 latency 0.000144 osd_op(client.6443001.0:956 rbd_data.5d886f14455c93.00000000000003a3 [delete] 19.f90d486e RETRY=39 ack+ondisk+retry+write e71594) v4 2014-12-08 18:14:03.266176 7f28f933f700 10 osd.70 71594 dequeue_op 0x1285e100 prio 63 cost 0 latency 0.000202 osd_op(client.6443001.0:956 rbd_data.5d886f14455c93.00000000000003a3 [delete] 19.f90d486e RETRY=39 ack+ondisk+retry+write e71594) v4 pg pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] 2014-12-08 18:14:03.266225 7f28f933f700 20 osd.70 pg_epoch: 71594 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] op_has_sufficient_caps pool=19 (volumes ) owner=0 need_read_cap=0 need_write_cap=1 need_class_read_cap=0 need_class_write_cap=0 -> yes 2014-12-08 18:14:03.266244 7f28f933f700 10 osd.70 pg_epoch: 71594 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] can_handle_while_inactive: 0x1285e100 2014-12-08 18:14:03.266259 7f28f933f700 10 osd.70 71594 dequeue_op 0x1285e100 finish 2014-12-08 18:14:03.266304 7f28d9bb2700 20 osd.70 71594 should_share_map client.6443001 172.16.5.116:0/109013063 71594 2014-12-08 18:14:03.266323 7f28d9bb2700 15 osd.70 71594 enqueue_op 0x1285e000 prio 63 cost 0 latency 0.000152 osd_op(client.6443001.0:1014 rbd_data.5d886f14455c93.00000000000003dd [delete] 19.f28d3b6e RETRY=39 ack+ondisk+retry+write e71594) v4 2014-12-08 18:14:03.266343 7f28fbb44700 10 osd.70 71594 dequeue_op 0x1285e000 prio 63 cost 0 latency 0.000172 osd_op(client.6443001.0:1014 rbd_data.5d886f14455c93.00000000000003dd [delete] 19.f28d3b6e RETRY=39 ack+ondisk+retry+write e71594) v4 pg pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] 2014-12-08 18:14:03.266366 7f28fbb44700 20 osd.70 pg_epoch: 71594 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] op_has_sufficient_caps pool=19 (volumes ) owner=0 need_read_cap=0 need_write_cap=1 need_class_read_cap=0 need_class_write_cap=0 -> yes 2014-12-08 18:14:03.266379 7f28fbb44700 10 osd.70 pg_epoch: 71594 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] can_handle_while_inactive: 0x1285e000 2014-12-08 18:14:03.266386 7f28fbb44700 10 osd.70 71594 dequeue_op 0x1285e000 finish 2014-12-08 18:14:03.266413 7f28d9bb2700 20 osd.70 71594 should_share_map client.6443001 172.16.5.116:0/109013063 71594 2014-12-08 18:14:03.266431 7f28d9bb2700 15 osd.70 71594 enqueue_op 0x1285f500 prio 63 cost 0 latency 0.000089 osd_op(client.6443001.0:1320 rbd_data.5d886f14455c93.000000000000050f [delete] 19.6bd2726e RETRY=39 ack+ondisk+retry+write e71594) v4 2014-12-08 18:14:03.266480 7f28f933f700 10 osd.70 71594 dequeue_op 0x1285f500 prio 63 cost 0 latency 0.000137 osd_op(client.6443001.0:1320 rbd_data.5d886f14455c93.000000000000050f [delete] 19.6bd2726e RETRY=39 ack+ondisk+retry+write e71594) v4 pg pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] 2014-12-08 18:14:03.266519 7f28f933f700 20 osd.70 pg_epoch: 71594 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] op_has_sufficient_caps pool=19 (volumes ) owner=0 need_read_cap=0 need_write_cap=1 need_class_read_cap=0 need_class_write_cap=0 -> yes 2014-12-08 18:14:03.266532 7f28f933f700 10 osd.70 pg_epoch: 71594 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] can_handle_while_inactive: 0x1285f500 2014-12-08 18:14:03.266540 7f28f933f700 10 osd.70 71594 dequeue_op 0x1285f500 finish 2014-12-08 18:14:03.317364 7f2900b4e700 20 osd.70 71594 share_map_peer 0x12711080 already has epoch 71594 2014-12-08 18:14:03.317525 7f2902351700 20 osd.70 71594 share_map_peer 0x12711080 already has epoch 71594 2014-12-08 18:14:03.372991 7f2902351700 20 osd.70 71594 share_map_peer 0x129c5760 already has epoch 71594 2014-12-08 18:14:03.373072 7f2900b4e700 20 osd.70 71594 share_map_peer 0x129c5760 already has epoch 71594 2014-12-08 18:14:03.509920 7f2900b4e700 20 osd.70 71594 share_map_peer 0x1281f600 already has epoch 71594 2014-12-08 18:14:03.510093 7f2902351700 20 osd.70 71594 share_map_peer 0x1281f600 already has epoch 71594 2014-12-08 18:14:03.529651 7f2900b4e700 20 osd.70 71594 share_map_peer 0x12769700 already has epoch 71594 2014-12-08 18:14:03.529769 7f2902351700 20 osd.70 71594 share_map_peer 0x12769700 already has epoch 71594 2014-12-08 18:14:03.550646 7f2906359700 10 osd.70 71594 do_waiters -- start 2014-12-08 18:14:03.550651 7f2906359700 10 osd.70 71594 do_waiters -- finish 2014-12-08 18:14:03.550654 7f2906359700 20 osd.70 71594 _dispatch 0x12c15840 osd_map(71595..71595 src has 50399..71595) v3 2014-12-08 18:14:03.550712 7f2906359700 3 osd.70 71594 handle_osd_map epochs [71595,71595], i have 71594, src has [50399,71595] 2014-12-08 18:14:03.550720 7f2906359700 10 osd.70 71594 handle_osd_map got inc map for epoch 71595 2014-12-08 18:14:03.552251 7f2906359700 10 osd.70 71594 advance to epoch 71595 (<= newest 71595) 2014-12-08 18:14:03.552287 7f2906359700 7 osd.70 71595 advance_map epoch 71595 2014-12-08 18:14:03.552301 7f2906359700 10 osd.70 71595 write_superblock sb(5509ebdb-0310-4919-ae72-ee26c6358258 osd.70 603f74ce-40a0-43de-8388-a8468497416a e71595 [50399,71595] lci=[71591,71595]) 2014-12-08 18:14:03.552393 7f2906359700 7 osd.70 71595 consume_map version 71595 2014-12-08 18:14:03.552752 7f2906359700 10 osd.70 pg_epoch: 71594 pg[216.257( v 71097'238 (0'0,71097'238] local-les=71593 n=110 ec=70987 les/c 71593/71593 71591/71591/71168) [8,70,16] r=1 lpr=71591 pi=70987-71590/29 luod=0'0 crt=71097'236 lcod 0'0 active] null 2014-12-08 18:14:03.552814 7f2906359700 10 osd.70 pg_epoch: 71594 pg[216.249( v 71097'244 (0'0,71097'244] local-les=71593 n=119 ec=70987 les/c 71593/71593 71591/71592/71592) [70,68,5] r=0 lpr=71592 crt=71097'244 lcod 0'0 mlcod 0'0 active+clean] null -- 2014-12-08 18:14:03.555113 7f2906359700 10 osd.70 pg_epoch: 71594 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] null 2014-12-08 18:14:03.555130 7f28fcb46700 20 osd.70 pg_epoch: 71595 pg[218.553( empty local-les=71435 n=0 ec=71423 les/c 71589/71589 71591/71594/71425) [17,70,63] r=1 lpr=71594 pi=71423-71593/10 crt=0'0 inactive NOTIFY] handle_activate_map: Not dirtying info: last_persisted is 71594 while current is 71595 2014-12-08 18:14:03.555133 7f28fc345700 10 osd.70 pg_epoch: 71595 pg[217.27a( v 71377'12225 (71377'9224,71377'12225] local-les=71592 n=9 ec=71094 les/c 71592/71592 71591/71591/71591) [70,8,1] r=0 lpr=71591 crt=71377'12222 lcod 0'0 mlcod 0'0 active+clean] handle_peering_event: epoch_sent: 71595 epoch_requested: 71595 NullEvt 2014-12-08 18:14:03.555152 7f28fc345700 10 log is not dirty 2014-12-08 18:14:03.555147 7f28fcb46700 10 osd.70 pg_epoch: 71595 pg[218.553( empty local-les=71435 n=0 ec=71423 les/c 71589/71589 71591/71594/71425) [17,70,63] r=1 lpr=71594 pi=71423-71593/10 crt=0'0 inactive NOTIFY] handle_peering_event: epoch_sent: 71595 epoch_requested: 71595 NullEvt 2014-12-08 18:14:03.555164 7f28fcb46700 10 log is not dirty 2014-12-08 18:14:03.555147 7f2906359700 10 osd.70 pg_epoch: 71594 pg[19.6d( v 71397'33614009 (70963'33611008,71397'33614009] local-les=71592 n=896 ec=21298 les/c 71592/71593 71591/71591/71169) [4,70,69] r=1 lpr=71591 pi=71156-71590/27 luod=0'0 crt=71397'33614007 lcod 0'0 active] null 2014-12-08 18:14:03.555172 7f28fc345700 10 osd.70 pg_epoch: 71594 pg[218.5ad( empty local-les=71593 n=0 ec=71423 les/c 71593/71593 71591/71592/71444) [7,70,13] r=1 lpr=71592 pi=71423-71591/11 crt=0'0 active] handle_advance_map [7,70,13]/[7,70,13] -- 7/7 2014-12-08 18:14:03.555178 7f2906359700 10 osd.70 pg_epoch: 71594 pg[218.6c7( v 71581'2 (0'0,71581'2] local-les=71593 n=2 ec=71423 les/c 71593/71593 71591/71592/71592) [70,10,18] r=0 lpr=71592 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] null 2014-12-08 18:14:03.555192 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:14:03.555189 7f28fcb46700 10 osd.70 pg_epoch: 71594 pg[216.3fb( empty local-les=71592 n=0 ec=70987 les/c 71592/71592 71591/71591/70989) [14,64,70] r=2 lpr=71591 pi=71136-71590/17 crt=0'0 active] handle_advance_map [14,64,70]/[14,64,70] -- 14/14 2014-12-08 18:14:03.555211 7f28fcb46700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:14:03.555199 7f28fc345700 10 osd.70 pg_epoch: 71595 pg[218.5ad( empty local-les=71593 n=0 ec=71423 les/c 71593/71593 71591/71592/71444) [7,70,13] r=1 lpr=71592 pi=71423-71591/11 crt=0'0 active] state: Started advmap 2014-12-08 18:14:03.555199 7f2906359700 10 osd.70 pg_epoch: 71594 pg[218.6c4( v 71581'3 (0'0,71581'3] local-les=71593 n=3 ec=71423 les/c 71593/71593 71591/71592/71592) [70,76,13] r=0 lpr=71592 crt=71580'1 lcod 0'0 mlcod 0'0 active+clean] null 2014-12-08 18:14:03.555218 7f28fc345700 10 osd.70 pg_epoch: 71595 pg[218.5ad( empty local-les=71593 n=0 ec=71423 les/c 71593/71593 71591/71592/71444) [7,70,13] r=1 lpr=71592 pi=71423-71591/11 crt=0'0 active] check_recovery_sources no source osds () went down 2014-12-08 18:14:03.555218 7f28fcb46700 10 osd.70 pg_epoch: 71595 pg[216.3fb( empty local-les=71592 n=0 ec=70987 les/c 71592/71592 71591/71591/70989) [14,64,70] r=2 lpr=71591 pi=71136-71590/17 crt=0'0 active] state: Started advmap 2014-12-08 18:14:03.555225 7f2906359700 10 osd.70 pg_epoch: 71594 pg[218.562( v 71581'3 (0'0,71581'3] local-les=71593 n=3 ec=71423 les/c 71593/71593 71591/71592/71425) [9,18,70] r=2 lpr=71592 pi=71423-71591/10 luod=0'0 crt=71580'1 lcod 0'0 active] null 2014-12-08 18:14:03.555244 7f28fcb46700 10 osd.70 pg_epoch: 71595 pg[216.3fb( empty local-les=71592 n=0 ec=70987 les/c 71592/71592 71591/71591/70989) [14,64,70] r=2 lpr=71591 pi=71136-71590/17 crt=0'0 active] check_recovery_sources no source osds () went down 2014-12-08 18:14:03.555246 7f28fc345700 10 osd.70 pg_epoch: 71595 pg[218.5ad( empty local-les=71593 n=0 ec=71423 les/c 71593/71593 71591/71592/71444) [7,70,13] r=1 lpr=71592 pi=71423-71591/11 crt=0'0 active] handle_activate_map 2014-12-08 18:14:03.555248 7f2906359700 10 osd.70 pg_epoch: 71594 pg[218.5e( empty local-les=71592 n=0 ec=71423 les/c 71592/71592 71591/71591/71425) [16,22,70] r=2 lpr=71591 pi=71423-71590/8 crt=0'0 active] null 2014-12-08 18:14:03.555263 7f28fc345700 10 osd.70 pg_epoch: 71595 pg[218.5ad( empty local-les=71593 n=0 ec=71423 les/c 71593/71593 71591/71592/71444) [7,70,13] r=1 lpr=71592 pi=71423-71591/11 crt=0'0 active] take_waiters -- 2014-12-08 18:14:03.562006 7f2906359700 20 osd.70 71595 19.6e heartbeat_peers 0,4,11,13,16,20,30,31,56,58,59,63,70,76 2014-12-08 18:14:03.562011 7f28fcb46700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:14:03.562001 7f28fc345700 10 osd.70 pg_epoch: 71595 pg[218.57c( v 71424'1 (0'0,71424'1] local-les=71592 n=1 ec=71423 les/c 71592/71592 71591/71591/71425) [24,16,70] r=2 lpr=71591 pi=71423-71590/8 luod=0'0 crt=71424'1 lcod 0'0 active] handle_activate_map 2014-12-08 18:14:03.562018 7f2906359700 20 osd.70 71595 19.6d heartbeat_peers 2014-12-08 18:14:03.562023 7f2906359700 20 osd.70 71595 218.6c7 heartbeat_peers 10,18,70 2014-12-08 18:14:03.562027 7f2906359700 20 osd.70 71595 218.6c4 heartbeat_peers 13,70,76 2014-12-08 18:14:03.562031 7f2906359700 20 osd.70 71595 218.562 heartbeat_peers 2014-12-08 18:14:03.562018 7f28fc345700 10 osd.70 pg_epoch: 71595 pg[218.57c( v 71424'1 (0'0,71424'1] local-les=71592 n=1 ec=71423 les/c 71592/71592 71591/71591/71425) [24,16,70] r=2 lpr=71591 pi=71423-71590/8 luod=0'0 crt=71424'1 lcod 0'0 active] take_waiters 2014-12-08 18:14:03.562035 7f2906359700 20 osd.70 71595 218.5e heartbeat_peers 2014-12-08 18:14:03.562017 7f28fcb46700 10 osd.70 pg_epoch: 71595 pg[216.392( v 71093'208 (0'0,71093'208] local-les=71593 n=104 ec=70987 les/c 71593/71593 71591/71591/70989) [69,7,70] r=2 lpr=71591 pi=71183-71590/17 luod=0'0 crt=71093'208 lcod 0'0 active] state: Started advmap 2014-12-08 18:14:03.562038 7f2906359700 20 osd.70 71595 218.5f heartbeat_peers 15,24,70 2014-12-08 18:14:03.562042 7f2906359700 20 osd.70 71595 217.235 heartbeat_peers 2014-12-08 18:14:03.562046 7f2906359700 20 osd.70 71595 218.38e heartbeat_peers 3,13,70 2014-12-08 18:14:03.562049 7f2906359700 20 osd.70 71595 218.38f heartbeat_peers 23,25,70 2014-12-08 18:14:03.562037 7f28fc345700 20 osd.70 pg_epoch: 71595 pg[218.57c( v 71424'1 (0'0,71424'1] local-les=71592 n=1 ec=71423 les/c 71592/71592 71591/71591/71425) [24,16,70] r=2 lpr=71591 pi=71423-71590/8 luod=0'0 crt=71424'1 lcod 0'0 active] handle_activate_map: Not dirtying info: last_persisted is 71592 while current is 71595 2014-12-08 18:14:03.562054 7f2906359700 20 osd.70 71595 218.38c heartbeat_peers 2014-12-08 18:14:03.562040 7f28fcb46700 10 osd.70 pg_epoch: 71595 pg[216.392( v 71093'208 (0'0,71093'208] local-les=71593 n=104 ec=70987 les/c 71593/71593 71591/71591/70989) [69,7,70] r=2 lpr=71591 pi=71183-71590/17 luod=0'0 crt=71093'208 lcod 0'0 active] check_recovery_sources no source osds () went down 2014-12-08 18:14:03.562057 7f2906359700 20 osd.70 71595 217.51 heartbeat_peers 2014-12-08 18:14:03.562060 7f2906359700 20 osd.70 71595 216.380 heartbeat_peers 2014-12-08 18:14:03.562063 7f2906359700 20 osd.70 71595 218.51 heartbeat_peers 66,70,75 2014-12-08 18:14:03.562068 7f2906359700 20 osd.70 71595 218.380 heartbeat_peers -- 2014-12-08 18:14:03.562152 7f28fcb46700 10 osd.70 pg_epoch: 71594 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] handle_advance_map [70,16,13]/[70,16,13] -- 70/70 2014-12-08 18:14:03.562174 7f2906359700 20 osd.70 71595 218.692 heartbeat_peers 2014-12-08 18:14:03.562162 7f28fc345700 10 osd.70 pg_epoch: 71595 pg[217.399( v 71377'57 (0'0,71377'57] local-les=71593 n=12 ec=71094 les/c 71593/71593 71591/71592/71592) [70,25,58] r=0 lpr=71592 crt=71377'54 lcod 0'0 mlcod 0'0 active+clean] check_recovery_sources no source osds () went down 2014-12-08 18:14:03.562177 7f2906359700 20 osd.70 71595 218.3a1 heartbeat_peers 23,69,70 2014-12-08 18:14:03.562178 7f28fcb46700 20 PGPool::update cached_removed_snaps [1~1] newly_removed_snaps [] snapc 2=[] (no change) 2014-12-08 18:14:03.562181 7f2906359700 20 osd.70 71595 216.1cd heartbeat_peers 2014-12-08 18:14:03.562184 7f2906359700 20 osd.70 71595 217.3a5 heartbeat_peers 2014-12-08 18:14:03.562186 7f2906359700 20 osd.70 71595 218.3a4 heartbeat_peers 6,66,70 2014-12-08 18:14:03.562191 7f2906359700 20 osd.70 71595 218.697 heartbeat_peers 2014-12-08 18:14:03.562194 7f2906359700 20 osd.70 71595 217.1c0 heartbeat_peers 2014-12-08 18:14:03.562196 7f2906359700 20 osd.70 71595 216.358 heartbeat_peers 2014-12-08 18:14:03.562185 7f28fc345700 10 osd.70 pg_epoch: 71595 pg[217.399( v 71377'57 (0'0,71377'57] local-les=71593 n=12 ec=71094 les/c 71593/71593 71591/71592/71592) [70,25,58] r=0 lpr=71592 crt=71377'54 lcod 0'0 mlcod 0'0 active+clean] handle_activate_map 2014-12-08 18:14:03.562200 7f2906359700 20 osd.70 71595 218.1c0 heartbeat_peers 2014-12-08 18:14:03.562187 7f28fcb46700 10 osd.70 pg_epoch: 71595 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] state: Peering advmap 2014-12-08 18:14:03.562203 7f2906359700 20 osd.70 71595 217.6a heartbeat_peers 19,24,70 2014-12-08 18:14:03.562208 7f2906359700 20 osd.70 71595 218.531 heartbeat_peers 2014-12-08 18:14:03.562210 7f2906359700 20 osd.70 71595 217.6f heartbeat_peers 17,66,70 2014-12-08 18:14:03.562215 7f2906359700 20 osd.70 71595 218.536 heartbeat_peers 9,70,75 2014-12-08 18:14:03.562203 7f28fc345700 10 osd.70 pg_epoch: 71595 pg[217.399( v 71377'57 (0'0,71377'57] local-les=71593 n=12 ec=71094 les/c 71593/71593 71591/71592/71592) [70,25,58] r=0 lpr=71592 crt=71377'54 lcod 0'0 mlcod 0'0 active+clean] state: Active: handling ActMap 2014-12-08 18:14:03.562219 7f2906359700 20 osd.70 71595 218.534 heartbeat_peers 2014-12-08 18:14:03.562222 7f2906359700 20 osd.70 71595 218.535 heartbeat_peers 2014-12-08 18:14:03.562209 7f28fcb46700 10 osd.70 pg_epoch: 71595 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] state: Started advmap 2014-12-08 18:14:03.562226 7f2906359700 20 osd.70 71595 216.1f9 heartbeat_peers 9,70,77 2014-12-08 18:14:03.562220 7f28fc345700 7 osd.70 pg_epoch: 71595 pg[217.399( v 71377'57 (0'0,71377'57] local-les=71593 n=12 ec=71094 les/c 71593/71593 71591/71592/71592) [70,25,58] r=0 lpr=71592 crt=71377'54 lcod 0'0 mlcod 0'0 active+clean] state: handle ActMap primary 2014-12-08 18:14:03.562236 7f2906359700 20 osd.70 71595 217.350 heartbeat_peers 2014-12-08 18:14:03.562239 7f2906359700 20 osd.70 71595 217.1fa heartbeat_peers 14,69,70 2014-12-08 18:14:03.562244 7f2906359700 20 osd.70 71595 218.351 heartbeat_peers 2014-12-08 18:14:03.562246 7f2906359700 20 osd.70 71595 218.1fe heartbeat_peers 6,15,70 2014-12-08 18:14:03.562250 7f2906359700 20 osd.70 71595 217.1fd heartbeat_peers 2014-12-08 18:14:03.562239 7f28fc345700 15 osd.70 pg_epoch: 71595 pg[217.399( v 71377'57 (0'0,71377'57] local-les=71593 n=12 ec=71094 les/c 71593/71593 71591/71592/71592) [70,25,58] r=0 lpr=71592 crt=71377'54 lcod 0'0 mlcod 0'0 active+clean] publish_stats_to_osd 71595:5123 2014-12-08 18:14:03.562253 7f2906359700 20 osd.70 71595 218.1fc heartbeat_peers 2014-12-08 18:14:03.562238 7f28fcb46700 10 osd.70 pg_epoch: 71595 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] check_recovery_sources no source osds () went down 2014-12-08 18:14:03.562257 7f2906359700 20 osd.70 71595 216.67 heartbeat_peers 2014-12-08 18:14:03.562259 7f2906359700 20 osd.70 71595 216.355 heartbeat_peers 8,70,78 2014-12-08 18:14:03.562263 7f2906359700 20 osd.70 71595 217.18 heartbeat_peers 15,56,70 2014-12-08 18:14:03.562267 7f2906359700 20 osd.70 71595 218.684 heartbeat_peers 2014-12-08 18:14:03.562256 7f28fc345700 10 osd.70 pg_epoch: 71595 pg[217.399( v 71377'57 (0'0,71377'57] local-les=71593 n=12 ec=71094 les/c 71593/71593 71591/71592/71592) [70,25,58] r=0 lpr=71592 crt=71377'54 lcod 0'0 mlcod 0'0 active+clean] take_waiters 2014-12-08 18:14:03.562270 7f2906359700 20 osd.70 71595 218.685 heartbeat_peers 10,18,70 2014-12-08 18:14:03.562274 7f2906359700 20 osd.70 71595 216.1b heartbeat_peers 2014-12-08 18:14:03.562262 7f28fcb46700 10 osd.70 pg_epoch: 71595 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] handle_activate_map 2014-12-08 18:14:03.562278 7f2906359700 20 osd.70 71595 218.1f6 heartbeat_peers 2014-12-08 18:14:03.562281 7f2906359700 20 osd.70 71595 216.1f4 heartbeat_peers 8,58,70 2014-12-08 18:14:03.562285 7f2906359700 20 osd.70 71595 216.1f6 heartbeat_peers 2014-12-08 18:14:03.562271 7f28fc345700 20 osd.70 pg_epoch: 71595 pg[217.399( v 71377'57 (0'0,71377'57] local-les=71593 n=12 ec=71094 les/c 71593/71593 71591/71592/71592) [70,25,58] r=0 lpr=71592 crt=71377'54 lcod 0'0 mlcod 0'0 active+clean] handle_activate_map: Not dirtying info: last_persisted is 71593 while current is 71595 2014-12-08 18:14:03.562288 7f2906359700 20 osd.70 71595 216.10 heartbeat_peers 2014-12-08 18:14:03.562281 7f28fcb46700 7 osd.70 pg_epoch: 71595 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] state: handle ActMap primary 2014-12-08 18:14:03.562291 7f2906359700 20 osd.70 71595 216.11 heartbeat_peers 2014-12-08 18:14:03.562290 7f28fc345700 10 osd.70 pg_epoch: 71595 pg[217.399( v 71377'57 (0'0,71377'57] local-les=71593 n=12 ec=71094 les/c 71593/71593 71591/71592/71592) [70,25,58] r=0 lpr=71592 crt=71377'54 lcod 0'0 mlcod 0'0 active+clean] handle_peering_event: epoch_sent: 71595 epoch_requested: 71595 NullEvt 2014-12-08 18:14:03.562304 7f2906359700 20 osd.70 71595 216.1e8 heartbeat_peers 21,69,70 2014-12-08 18:14:03.562308 7f28fc345700 10 log is not dirty 2014-12-08 18:14:03.562309 7f2906359700 20 osd.70 71595 216.12 heartbeat_peers 2014-12-08 18:14:03.562311 7f2906359700 20 osd.70 71595 218.343 heartbeat_peers 2014-12-08 18:14:03.562314 7f2906359700 20 osd.70 71595 217.1ed heartbeat_peers 2014-12-08 18:14:03.562316 7f2906359700 20 osd.70 71595 218.4df heartbeat_peers 2014-12-08 18:14:03.562319 7f2906359700 20 osd.70 71595 218.345 heartbeat_peers 2014-12-08 18:14:03.562305 7f28fcb46700 15 osd.70 pg_epoch: 71595 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] publish_stats_to_osd 71595:63844735 2014-12-08 18:14:03.562321 7f2906359700 20 osd.70 71595 218.1e3 heartbeat_peers 14,70,76 2014-12-08 18:14:03.562325 7f2906359700 20 osd.70 71595 218.1e0 heartbeat_peers 2014-12-08 18:14:03.562328 7f2906359700 20 osd.70 71595 218.1e1 heartbeat_peers 4,21,70 2014-12-08 18:14:03.562332 7f2906359700 20 osd.70 71595 218.e heartbeat_peers 2014-12-08 18:14:03.562335 7f2906359700 20 osd.70 71595 218.1e4 heartbeat_peers 2014-12-08 18:14:03.562322 7f28fcb46700 10 osd.70 pg_epoch: 71595 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] take_waiters 2014-12-08 18:14:03.562338 7f2906359700 20 osd.70 71595 217.37c heartbeat_peers 58,69,70 2014-12-08 18:14:03.562342 7f2906359700 20 osd.70 71595 217.e heartbeat_peers 2014-12-08 18:14:03.562326 7f28fc345700 10 osd.70 pg_epoch: 71594 pg[216.ab( v 71097'289 (0'0,71097'289] local-les=71594 n=133 ec=70987 les/c 71594/71594 71591/71593/71168) [20,64,70] r=2 lpr=71593 pi=71183-71592/17 luod=0'0 crt=71097'289 lcod 0'0 active] handle_advance_map [20,64,70]/[20,64,70] -- 20/20 2014-12-08 18:14:03.562345 7f2906359700 20 osd.70 71595 218.6ae heartbeat_peers 2014-12-08 18:14:03.562348 7f2906359700 20 osd.70 71595 218.6af heartbeat_peers 2014-12-08 18:14:03.562348 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:14:03.562351 7f2906359700 20 osd.70 71595 218.372 heartbeat_peers 2014-12-08 18:14:03.562354 7f2906359700 20 osd.70 71595 217.198 heartbeat_peers 2014-12-08 18:14:03.562340 7f28fcb46700 20 osd.70 pg_epoch: 71595 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] handle_activate_map: Not dirtying info: last_persisted is 71591 while current is 71595 2014-12-08 18:14:03.562357 7f2906359700 20 osd.70 71595 217.370 heartbeat_peers 10,56,70 2014-12-08 18:14:03.562361 7f2906359700 20 osd.70 71595 216.3 heartbeat_peers 16,56,70 2014-12-08 18:14:03.562365 7f2906359700 20 osd.70 71595 218.6a1 heartbeat_peers 2014-12-08 18:14:03.562368 7f2906359700 20 osd.70 71595 218.374 heartbeat_peers 2014-12-08 18:14:03.562354 7f28fc345700 10 osd.70 pg_epoch: 71595 pg[216.ab( v 71097'289 (0'0,71097'289] local-les=71594 n=133 ec=70987 les/c 71594/71594 71591/71593/71168) [20,64,70] r=2 lpr=71593 pi=71183-71592/17 luod=0'0 crt=71097'289 lcod 0'0 active] state: Started advmap 2014-12-08 18:14:03.562371 7f2906359700 20 osd.70 71595 216.19f heartbeat_peers 2014-12-08 18:14:03.562357 7f28fcb46700 10 osd.70 pg_epoch: 71595 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] handle_peering_event: epoch_sent: 71595 epoch_requested: 71595 NullEvt 2014-12-08 18:14:03.562374 7f2906359700 20 osd.70 71595 218.192 heartbeat_peers 2014-12-08 18:14:03.562377 7f28fcb46700 10 log is not dirty 2014-12-08 18:14:03.562378 7f2906359700 20 osd.70 71595 218.38 heartbeat_peers 2014-12-08 18:14:03.562381 7f2906359700 20 osd.70 71595 217.368 heartbeat_peers 10,18,70 2014-12-08 18:14:03.562384 7f2906359700 20 osd.70 71595 217.36b heartbeat_peers 2014-12-08 18:14:03.562387 7f2906359700 20 osd.70 71595 218.3d heartbeat_peers 2014-12-08 18:14:03.562374 7f28fc345700 10 osd.70 pg_epoch: 71595 pg[216.ab( v 71097'289 (0'0,71097'289] local-les=71594 n=133 ec=70987 les/c 71594/71594 71591/71593/71168) [20,64,70] r=2 lpr=71593 pi=71183-71592/17 luod=0'0 crt=71097'289 lcod 0'0 active] check_recovery_sources no source osds () went down 2014-12-08 18:14:03.562390 7f2906359700 20 osd.70 71595 216.196 heartbeat_peers 2014-12-08 18:14:03.562392 7f2906359700 20 osd.70 71595 218.4c4 heartbeat_peers 2014-12-08 18:14:03.562395 7f2906359700 20 osd.70 71595 218.18a heartbeat_peers 2014-12-08 18:14:03.562397 7f2906359700 20 osd.70 71595 217.36e heartbeat_peers 2014-12-08 18:14:03.562401 7f2906359700 20 osd.70 71595 217.32 heartbeat_peers 2014-12-08 18:14:03.562403 7f2906359700 20 osd.70 71595 218.36 heartbeat_peers 2014-12-08 18:14:03.562406 7f2906359700 20 osd.70 71595 216.362 heartbeat_peers 2014-12-08 18:14:03.562409 7f2906359700 20 osd.70 71595 217.34 heartbeat_peers 4,17,70 2014-12-08 18:14:03.562396 7f28fc345700 10 osd.70 pg_epoch: 71595 pg[216.ab( v 71097'289 (0'0,71097'289] local-les=71594 n=133 ec=70987 les/c 71594/71594 71591/71593/71168) [20,64,70] r=2 lpr=71593 pi=71183-71592/17 luod=0'0 crt=71097'289 lcod 0'0 active] handle_activate_map 2014-12-08 18:14:03.562413 7f2906359700 20 osd.70 71595 218.34 heartbeat_peers 2014-12-08 18:14:03.562395 7f28fcb46700 10 osd.70 pg_epoch: 71594 pg[19.6d( v 71397'33614009 (70963'33611008,71397'33614009] local-les=71592 n=896 ec=21298 les/c 71592/71593 71591/71591/71169) [4,70,69] r=1 lpr=71591 pi=71156-71590/27 luod=0'0 crt=71397'33614007 lcod 0'0 active] handle_advance_map [4,70,69]/[4,70,69] -- 4/4 2014-12-08 18:14:03.562416 7f2906359700 20 osd.70 71595 217.36 heartbeat_peers 2014-12-08 18:14:03.562419 7f2906359700 20 osd.70 71595 218.4ff heartbeat_peers -- 2014-12-08 18:14:03.572712 7f2904b56700 20 osd.70 71595 _dispatch 0x13efde00 pg_notify(19.6e(150) epoch 71595) v5 2014-12-08 18:14:03.572706 7f28fcb46700 10 osd.70 pg_epoch: 71595 pg[218.3a4( empty local-les=71592 n=0 ec=71423 les/c 71592/71592 71591/71591/71591) [70,6,66] r=0 lpr=71591 crt=0'0 mlcod 0'0 active+clean] state: Active advmap 2014-12-08 18:14:03.572717 7f28fcb46700 10 osd.70 pg_epoch: 71595 pg[218.3a4( empty local-les=71592 n=0 ec=71423 les/c 71592/71592 71591/71591/71591) [70,6,66] r=0 lpr=71591 crt=0'0 mlcod 0'0 active+clean] state: Started advmap 2014-12-08 18:14:03.572710 7f28fc345700 10 osd.70 pg_epoch: 71595 pg[218.1c0( v 71581'3 (0'0,71581'3] local-les=71594 n=3 ec=71423 les/c 71594/71594 71591/71593/71425) [75,70,3] r=1 lpr=71593 pi=71423-71592/10 luod=0'0 crt=71580'1 lcod 0'0 active] handle_peering_event: epoch_sent: 71595 epoch_requested: 71595 NullEvt 2014-12-08 18:14:03.572741 7f28fc345700 10 log is not dirty 2014-12-08 18:14:03.572745 7f2904b56700 7 osd.70 71595 handle_pg_notify from osd.58 2014-12-08 18:14:03.572738 7f28fcb46700 10 osd.70 pg_epoch: 71595 pg[218.3a4( empty local-les=71592 n=0 ec=71423 les/c 71592/71592 71591/71591/71591) [70,6,66] r=0 lpr=71591 crt=0'0 mlcod 0'0 active+clean] check_recovery_sources no source osds () went down 2014-12-08 18:14:03.572748 7f2904b56700 15 osd.70 71595 require_same_or_newer_map 71595 (i am 71595) 0x13efde00 2014-12-08 18:14:03.572758 7f28fcb46700 10 osd.70 pg_epoch: 71595 pg[218.3a4( empty local-les=71592 n=0 ec=71423 les/c 71592/71592 71591/71591/71591) [70,6,66] r=0 lpr=71591 crt=0'0 mlcod 0'0 active+clean] handle_activate_map 2014-12-08 18:14:03.572769 7f28fcb46700 10 osd.70 pg_epoch: 71595 pg[218.3a4( empty local-les=71592 n=0 ec=71423 les/c 71592/71592 71591/71591/71591) [70,6,66] r=0 lpr=71591 crt=0'0 mlcod 0'0 active+clean] state: Active: handling ActMap 2014-12-08 18:14:03.572776 7f2904b56700 10 osd.70 71595 do_waiters -- start 2014-12-08 18:14:03.572778 7f2904b56700 10 osd.70 71595 do_waiters -- finish 2014-12-08 18:14:03.572759 7f28fc345700 10 osd.70 pg_epoch: 71594 pg[217.6a( v 71376'14 (0'0,71376'14] local-les=71593 n=7 ec=71094 les/c 71593/71593 71591/71592/71592) [70,24,19] r=0 lpr=71592 crt=71097'12 lcod 0'0 mlcod 0'0 active+clean] handle_advance_map [70,24,19]/[70,24,19] -- 70/70 2014-12-08 18:14:03.572777 7f28fcb46700 7 osd.70 pg_epoch: 71595 pg[218.3a4( empty local-les=71592 n=0 ec=71423 les/c 71592/71592 71591/71591/71591) [70,6,66] r=0 lpr=71591 crt=0'0 mlcod 0'0 active+clean] state: handle ActMap primary 2014-12-08 18:14:03.572786 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:14:03.572788 7f28fcb46700 15 osd.70 pg_epoch: 71595 pg[218.3a4( empty local-les=71592 n=0 ec=71423 les/c 71592/71592 71591/71591/71591) [70,6,66] r=0 lpr=71591 crt=0'0 mlcod 0'0 active+clean] publish_stats_to_osd 71595:32 2014-12-08 18:14:03.572791 7f28fc345700 10 osd.70 pg_epoch: 71595 pg[217.6a( v 71376'14 (0'0,71376'14] local-les=71593 n=7 ec=71094 les/c 71593/71593 71591/71592/71592) [70,24,19] r=0 lpr=71592 crt=71097'12 lcod 0'0 mlcod 0'0 active+clean] state: Active advmap 2014-12-08 18:14:03.572797 7f28fcb46700 10 osd.70 pg_epoch: 71595 pg[218.3a4( empty local-les=71592 n=0 ec=71423 les/c 71592/71592 71591/71591/71591) [70,6,66] r=0 lpr=71591 crt=0'0 mlcod 0'0 active+clean] take_waiters 2014-12-08 18:14:03.572806 7f28fc345700 10 osd.70 pg_epoch: 71595 pg[217.6a( v 71376'14 (0'0,71376'14] local-les=71593 n=7 ec=71094 les/c 71593/71593 71591/71592/71592) [70,24,19] r=0 lpr=71592 crt=71097'12 lcod 0'0 mlcod 0'0 active+clean] state: Started advmap 2014-12-08 18:14:03.572806 7f28fcb46700 20 osd.70 pg_epoch: 71595 pg[218.3a4( empty local-les=71592 n=0 ec=71423 les/c 71592/71592 71591/71591/71591) [70,6,66] r=0 lpr=71591 crt=0'0 mlcod 0'0 active+clean] handle_activate_map: Not dirtying info: last_persisted is 71592 while current is 71595 2014-12-08 18:14:03.572816 7f28fcb46700 10 osd.70 pg_epoch: 71595 pg[218.3a4( empty local-les=71592 n=0 ec=71423 les/c 71592/71592 71591/71591/71591) [70,6,66] r=0 lpr=71591 crt=0'0 mlcod 0'0 active+clean] handle_peering_event: epoch_sent: 71595 epoch_requested: 71595 NullEvt -- 2014-12-08 18:14:03.572892 7f2904b56700 20 osd.70 71595 _dispatch 0x12c70ee0 pg_notify(19.6e(150) epoch 71595) v5 2014-12-08 18:14:03.572889 7f28fc345700 10 osd.70 pg_epoch: 71595 pg[217.6a( v 71376'14 (0'0,71376'14] local-les=71593 n=7 ec=71094 les/c 71593/71593 71591/71592/71592) [70,24,19] r=0 lpr=71592 crt=71097'12 lcod 0'0 mlcod 0'0 active+clean] take_waiters 2014-12-08 18:14:03.572897 7f28fc345700 20 osd.70 pg_epoch: 71595 pg[217.6a( v 71376'14 (0'0,71376'14] local-les=71593 n=7 ec=71094 les/c 71593/71593 71591/71592/71592) [70,24,19] r=0 lpr=71592 crt=71097'12 lcod 0'0 mlcod 0'0 active+clean] handle_activate_map: Not dirtying info: last_persisted is 71593 while current is 71595 2014-12-08 18:14:03.572899 7f28fcb46700 10 osd.70 pg_epoch: 71595 pg[218.697( empty local-les=71594 n=0 ec=71423 les/c 71594/71594 71591/71593/71425) [20,70,4] r=1 lpr=71593 pi=71423-71592/10 crt=0'0 active] handle_activate_map 2014-12-08 18:14:03.572910 7f28fcb46700 10 osd.70 pg_epoch: 71595 pg[218.697( empty local-les=71594 n=0 ec=71423 les/c 71594/71594 71591/71593/71425) [20,70,4] r=1 lpr=71593 pi=71423-71592/10 crt=0'0 active] take_waiters 2014-12-08 18:14:03.572919 7f2904b56700 7 osd.70 71595 handle_pg_notify from osd.76 2014-12-08 18:14:03.572909 7f28fc345700 10 osd.70 pg_epoch: 71595 pg[217.6a( v 71376'14 (0'0,71376'14] local-les=71593 n=7 ec=71094 les/c 71593/71593 71591/71592/71592) [70,24,19] r=0 lpr=71592 crt=71097'12 lcod 0'0 mlcod 0'0 active+clean] handle_peering_event: epoch_sent: 71595 epoch_requested: 71595 NullEvt 2014-12-08 18:14:03.572921 7f2904b56700 15 osd.70 71595 require_same_or_newer_map 71595 (i am 71595) 0x12c70ee0 2014-12-08 18:14:03.572927 7f28fc345700 10 log is not dirty 2014-12-08 18:14:03.572920 7f28fcb46700 20 osd.70 pg_epoch: 71595 pg[218.697( empty local-les=71594 n=0 ec=71423 les/c 71594/71594 71591/71593/71425) [20,70,4] r=1 lpr=71593 pi=71423-71592/10 crt=0'0 active] handle_activate_map: Not dirtying info: last_persisted is 71594 while current is 71595 2014-12-08 18:14:03.572931 7f28fcb46700 10 osd.70 pg_epoch: 71595 pg[218.697( empty local-les=71594 n=0 ec=71423 les/c 71594/71594 71591/71593/71425) [20,70,4] r=1 lpr=71593 pi=71423-71592/10 crt=0'0 active] handle_peering_event: epoch_sent: 71595 epoch_requested: 71595 NullEvt 2014-12-08 18:14:03.572940 7f28fcb46700 10 log is not dirty 2014-12-08 18:14:03.572946 7f2904b56700 10 osd.70 71595 do_waiters -- start 2014-12-08 18:14:03.572947 7f2904b56700 10 osd.70 71595 do_waiters -- finish 2014-12-08 18:14:03.572941 7f28fc345700 10 osd.70 pg_epoch: 71594 pg[218.531( v 71580'1 (0'0,71580'1] local-les=71593 n=1 ec=71423 les/c 71593/71593 71591/71592/71425) [18,14,70] r=2 lpr=71592 pi=71423-71591/10 luod=0'0 crt=0'0 lcod 0'0 active] handle_advance_map [18,14,70]/[18,14,70] -- 18/18 2014-12-08 18:14:03.572954 7f28fc345700 20 PGPool::update cached_removed_snaps [] newly_removed_snaps [] snapc 0=[] (no change) 2014-12-08 18:14:03.572958 7f28fc345700 10 osd.70 pg_epoch: 71595 pg[218.531( v 71580'1 (0'0,71580'1] local-les=71593 n=1 ec=71423 les/c 71593/71593 71591/71592/71425) [18,14,70] r=2 lpr=71592 pi=71423-71591/10 luod=0'0 crt=0'0 lcod 0'0 active] state: Started advmap 2014-12-08 18:14:03.572966 7f28fc345700 10 osd.70 pg_epoch: 71595 pg[218.531( v 71580'1 (0'0,71580'1] local-les=71593 n=1 ec=71423 les/c 71593/71593 71591/71592/71425) [18,14,70] r=2 lpr=71592 pi=71423-71591/10 luod=0'0 crt=0'0 lcod 0'0 active] check_recovery_sources no source osds () went down 2014-12-08 18:14:03.572985 7f28fc345700 10 osd.70 pg_epoch: 71595 pg[218.531( v 71580'1 (0'0,71580'1] local-les=71593 n=1 ec=71423 les/c 71593/71593 71591/71592/71425) [18,14,70] r=2 lpr=71592 pi=71423-71591/10 luod=0'0 crt=0'0 lcod 0'0 active] handle_activate_map 2014-12-08 18:14:03.572987 7f28fcb46700 10 osd.70 pg_epoch: 71594 pg[218.685( v 71581'2 (0'0,71581'2] local-les=71594 n=2 ec=71423 les/c 71594/71594 71591/71593/71593) [70,10,18] r=0 lpr=71593 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] handle_advance_map [70,10,18]/[70,10,18] -- 70/70 2014-12-08 18:14:03.572996 7f28fc345700 10 osd.70 pg_epoch: 71595 pg[218.531( v 71580'1 (0'0,71580'1] local-les=71593 n=1 ec=71423 les/c 71593/71593 71591/71592/71425) [18,14,70] r=2 lpr=71592 pi=71423-71591/10 luod=0'0 crt=0'0 lcod 0'0 active] take_waiters -- 2014-12-08 18:14:03.605159 7f28fc345700 10 osd.70 pg_epoch: 71595 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] handle_peering_event: epoch_sent: 71595 epoch_requested: 71595 MNotifyRec from 58 notify: (query_epoch:71595, epoch_sent:71595, info:19.6e( empty local-les=0 n=0 ec=21298 les/c 51101/50495 71591/71591/71591)) features: 0x3fffffffffff 2014-12-08 18:14:03.605180 7f28fc345700 7 osd.70 pg_epoch: 71595 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] state: handle_pg_notify from osd.58 2014-12-08 18:14:03.605172 7f290a83a700 10 osd.70 pg_epoch: 71595 pg[216.3b8( v 71097'270 (0'0,71097'270] local-les=71595 n=130 ec=70987 les/c 71585/71583 71591/71594/71587) [58,70,18] r=1 lpr=71594 pi=70987-71593/27 crt=71097'270 lcod 0'0 inactive] _activate_committed 71595 telling primary 2014-12-08 18:14:03.605203 7f290a83a700 20 osd.70 71595 share_map_peer 0x12838f20 already has epoch 71595 2014-12-08 18:14:03.605186 7f28fcb46700 10 osd.70 pg_epoch: 71595 pg[217.36( v 71377'48 (0'0,71377'48] local-les=71450 n=12 ec=71094 les/c 71585/71583 71591/71594/71444) [23,70,14] r=1 lpr=71594 pi=71183-71593/20 crt=71377'46 lcod 0'0 inactive NOTIFY] handle_peering_event: epoch_sent: 71595 epoch_requested: 71595 MInfoRec from 23 info: 217.36( v 71377'48 (0'0,71377'48] local-les=71595 n=12 ec=71094 les/c 71585/71583 71591/71594/71444) 2014-12-08 18:14:03.605194 7f28fc345700 10 osd.70 pg_epoch: 71595 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] state: pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] got dup osd.58 info 19.6e( empty local-les=0 n=0 ec=21298 les/c 51101/50495 71591/71591/71591), identical to ours 2014-12-08 18:14:03.605214 7f28fc345700 10 log is not dirty 2014-12-08 18:14:03.605218 7f28fc345700 10 osd.70 pg_epoch: 71595 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] handle_peering_event: epoch_sent: 71595 epoch_requested: 71595 MNotifyRec from 76 notify: (query_epoch:71595, epoch_sent:71595, info:19.6e( empty local-les=0 n=0 ec=21298 les/c 51101/50495 71591/71591/71591)) features: 0x3fffffffffff 2014-12-08 18:14:03.605211 7f28fcb46700 10 osd.70 pg_epoch: 71595 pg[217.36( v 71377'48 (0'0,71377'48] local-les=71450 n=12 ec=71094 les/c 71585/71583 71591/71594/71444) [23,70,14] r=1 lpr=71594 pi=71183-71593/20 crt=71377'46 lcod 0'0 inactive NOTIFY] state: got info from osd.23 217.36( v 71377'48 (0'0,71377'48] local-les=71595 n=12 ec=71094 les/c 71585/71583 71591/71594/71444) 2014-12-08 18:14:03.605230 7f28fc345700 7 osd.70 pg_epoch: 71595 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] state: handle_pg_notify from osd.76 2014-12-08 18:14:03.605231 7f290a83a700 10 osd.70 pg_epoch: 71595 pg[216.3b8( v 71097'270 (0'0,71097'270] local-les=71595 n=130 ec=70987 les/c 71585/71583 71591/71594/71587) [58,70,18] r=1 lpr=71594 pi=70987-71593/27 luod=0'0 crt=71097'270 lcod 0'0 active] flushed 2014-12-08 18:14:03.605233 7f28fcb46700 5 osd.70 pg_epoch: 71595 pg[217.36( v 71377'48 (0'0,71377'48] local-les=71450 n=12 ec=71094 les/c 71585/71583 71591/71594/71444) [23,70,14] r=1 lpr=71594 pi=71183-71593/20 crt=71377'46 lcod 0'0 inactive NOTIFY] exit Started/Stray 1.041987 7 0.000315 2014-12-08 18:14:03.605243 7f28fc345700 10 osd.70 pg_epoch: 71595 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] state: pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] got dup osd.76 info 19.6e( empty local-les=0 n=0 ec=21298 les/c 51101/50495 71591/71591/71591), identical to ours 2014-12-08 18:14:03.605270 7f28fc345700 10 log is not dirty 2014-12-08 18:14:03.605259 7f28fcb46700 5 osd.70 pg_epoch: 71595 pg[217.36( v 71377'48 (0'0,71377'48] local-les=71450 n=12 ec=71094 les/c 71585/71583 71591/71594/71444) [23,70,14] r=1 lpr=71594 pi=71183-71593/20 crt=71377'46 lcod 0'0 inactive NOTIFY] enter Started/ReplicaActive 2014-12-08 18:14:03.605260 7f290a83a700 10 osd.70 pg_epoch: 71595 pg[218.38( v 71581'3 (0'0,71581'3] local-les=71595 n=3 ec=71423 les/c 71590/71583 71591/71594/71425) [56,17,70] r=2 lpr=71594 pi=71423-71593/10 crt=71580'1 lcod 0'0 inactive] _activate_committed 71595 telling primary 2014-12-08 18:14:03.605274 7f28fc345700 10 osd.70 pg_epoch: 71595 pg[217.21d( v 71097'9 (0'0,71097'9] local-les=71595 n=6 ec=71094 les/c 71585/71583 71591/71594/71594) [70,6,14] r=0 lpr=71594 pi=71432-71593/6 crt=71097'9 lcod 0'0 mlcod 0'0 inactive] handle_peering_event: epoch_sent: 71595 epoch_requested: 71595 MNotifyRec from 6 notify: (query_epoch:71595, epoch_sent:71595, info:217.21d( v 71097'9 (0'0,71097'9] local-les=71585 n=6 ec=71094 les/c 71585/71583 71591/71594/71594)) features: 0x3fffffffffff 2014-12-08 18:14:03.605289 7f290a83a700 20 osd.70 71595 share_map_peer 0x126b8680 already has epoch 71595 2014-12-08 18:14:03.605280 7f28fcb46700 5 osd.70 pg_epoch: 71595 pg[217.36( v 71377'48 (0'0,71377'48] local-les=71450 n=12 ec=71094 les/c 71585/71583 71591/71594/71444) [23,70,14] r=1 lpr=71594 pi=71183-71593/20 crt=71377'46 lcod 0'0 inactive NOTIFY] enter Started/ReplicaActive/RepNotRecovering 2014-12-08 18:14:03.605287 7f28fc345700 10 osd.70 pg_epoch: 71595 pg[217.21d( v 71097'9 (0'0,71097'9] local-les=71595 n=6 ec=71094 les/c 71585/71583 71591/71594/71594) [70,6,14] r=0 lpr=71594 pi=71432-71593/6 crt=71097'9 lcod 0'0 mlcod 0'0 inactive] state: Active: got notify from 6, already have info from that osd, ignoring 2014-12-08 18:14:03.605305 7f28fc345700 10 log is not dirty 2014-12-08 18:14:03.605306 7f28fcb46700 10 osd.70 pg_epoch: 71595 pg[217.36( v 71377'48 (0'0,71377'48] local-les=71450 n=12 ec=71094 les/c 71585/71583 71591/71594/71444) [23,70,14] r=1 lpr=71594 pi=71183-71593/20 crt=71377'46 lcod 0'0 inactive NOTIFY] state: In ReplicaActive, about to call activate 2014-12-08 18:14:03.605309 7f28fc345700 10 osd.70 pg_epoch: 71595 pg[218.553( empty local-les=71435 n=0 ec=71423 les/c 71589/71589 71591/71594/71425) [17,70,63] r=1 lpr=71594 pi=71423-71593/10 crt=0'0 inactive NOTIFY] handle_peering_event: epoch_sent: 71595 epoch_requested: 71595 MLogRec from 17 2014-12-08 18:14:03.605313 7f290a83a700 10 osd.70 pg_epoch: 71595 pg[218.38( v 71581'3 (0'0,71581'3] local-les=71595 n=3 ec=71423 les/c 71590/71583 71591/71594/71425) [56,17,70] r=2 lpr=71594 pi=71423-71593/10 luod=0'0 crt=71580'1 lcod 0'0 active] flushed 2014-12-08 18:14:03.605324 7f28fc345700 10 osd.70 pg_epoch: 71595 pg[218.553( empty local-les=71435 n=0 ec=71423 les/c 71589/71589 71591/71594/71425) [17,70,63] r=1 lpr=71594 pi=71423-71593/10 crt=0'0 inactive NOTIFY] state: got info+log from osd.17 218.553( empty local-les=71595 n=0 ec=71423 les/c 71589/71589 71591/71594/71425) log((0'0,0'0], crt=0'0) 2014-12-08 18:14:03.605323 7f28fcb46700 10 osd.70 pg_epoch: 71595 pg[217.36( v 71377'48 (0'0,71377'48] local-les=71595 n=12 ec=71094 les/c 71585/71583 71591/71594/71444) [23,70,14] r=1 lpr=71594 pi=71183-71593/20 crt=71377'46 lcod 0'0 inactive] activate - no missing, moving last_complete 71377'48 -> 71377'48 2014-12-08 18:14:03.605339 7f28fc345700 10 merge_log log((0'0,0'0], crt=0'0) from osd.17 into log((0'0,0'0], crt=0'0) 2014-12-08 18:14:03.605344 7f28fc345700 10 merge_log result log((0'0,0'0], crt=0'0) missing(0) changed=0 2014-12-08 18:14:03.605347 7f28fc345700 5 osd.70 pg_epoch: 71595 pg[218.553( empty local-les=71435 n=0 ec=71423 les/c 71589/71589 71591/71594/71425) [17,70,63] r=1 lpr=71594 pi=71423-71593/10 crt=0'0 inactive NOTIFY] exit Started/Stray 1.074710 7 0.000230 2014-12-08 18:14:03.605351 7f28fcb46700 10 osd.70 pg_epoch: 71595 pg[217.36( v 71377'48 (0'0,71377'48] local-les=71595 n=12 ec=71094 les/c 71585/71583 71591/71594/71444) [23,70,14] r=1 lpr=71594 pi=71183-71593/20 crt=71377'46 lcod 0'0 inactive] state: Activate Finished 2014-12-08 18:14:03.605360 7f28fc345700 5 osd.70 pg_epoch: 71595 pg[218.553( empty local-les=71435 n=0 ec=71423 les/c 71589/71589 71591/71594/71425) [17,70,63] r=1 lpr=71594 pi=71423-71593/10 crt=0'0 inactive NOTIFY] enter Started/ReplicaActive 2014-12-08 18:14:03.605372 7f28fc345700 5 osd.70 pg_epoch: 71595 pg[218.553( empty local-les=71435 n=0 ec=71423 les/c 71589/71589 71591/71594/71425) [17,70,63] r=1 lpr=71594 pi=71423-71593/10 crt=0'0 inactive NOTIFY] enter Started/ReplicaActive/RepNotRecovering 2014-12-08 18:14:03.605381 7f28fc345700 10 osd.70 pg_epoch: 71595 pg[218.553( empty local-les=71435 n=0 ec=71423 les/c 71589/71589 71591/71594/71425) [17,70,63] r=1 lpr=71594 pi=71423-71593/10 crt=0'0 inactive NOTIFY] state: In ReplicaActive, about to call activate -- 2014-12-08 18:14:03.640372 7f2904b56700 20 osd.70 71595 _dispatch 0x12eac760 pg_notify(19.6e(150) epoch 71595) v5 2014-12-08 18:14:03.640397 7f2904b56700 7 osd.70 71595 handle_pg_notify from osd.4 2014-12-08 18:14:03.640399 7f2904b56700 15 osd.70 71595 require_same_or_newer_map 71595 (i am 71595) 0x12eac760 2014-12-08 18:14:03.640442 7f2904b56700 10 osd.70 71595 do_waiters -- start 2014-12-08 18:14:03.640448 7f2904b56700 10 osd.70 71595 do_waiters -- finish 2014-12-08 18:14:03.640484 7f28fc345700 10 osd.70 pg_epoch: 71595 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] handle_peering_event: epoch_sent: 71595 epoch_requested: 71595 MNotifyRec from 4 notify: (query_epoch:71595, epoch_sent:71595, info:19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591)) features: 0x3fffffffffff 2014-12-08 18:14:03.640525 7f28fc345700 7 osd.70 pg_epoch: 71595 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] state: handle_pg_notify from osd.4 2014-12-08 18:14:03.640540 7f28fc345700 10 osd.70 pg_epoch: 71595 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] state: pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] got dup osd.4 info 19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591), identical to ours 2014-12-08 18:14:03.640565 7f28fc345700 10 log is not dirty 2014-12-08 18:14:03.643417 7f2906359700 10 osd.70 71595 do_waiters -- start 2014-12-08 18:14:03.643421 7f2906359700 10 osd.70 71595 do_waiters -- finish 2014-12-08 18:14:03.643423 7f2906359700 20 osd.70 71595 _dispatch 0x1335b2a0 pg_stats_ack(120 pgs tid 1) v1 2014-12-08 18:14:03.643431 7f2906359700 10 osd.70 71595 handle_pg_stats_ack 2014-12-08 18:14:03.643544 7f2906359700 10 osd.70 71595 do_waiters -- start 2014-12-08 18:14:03.643551 7f2906359700 10 osd.70 71595 do_waiters -- finish 2014-12-08 18:14:03.647542 7f2904b56700 10 osd.70 71595 do_waiters -- start 2014-12-08 18:14:03.647546 7f2904b56700 10 osd.70 71595 do_waiters -- finish 2014-12-08 18:14:03.647549 7f2904b56700 20 osd.70 71595 _dispatch 0x134b3840 osd_map(71595..71595 src has 50399..71595) v3 2014-12-08 18:14:03.647574 7f2904b56700 3 osd.70 71595 handle_osd_map epochs [71595,71595], i have 71595, src has [50399,71595] 2014-12-08 18:14:03.647617 7f2904b56700 10 osd.70 71595 no new maps here, dropping 2014-12-08 18:14:03.647620 7f2904b56700 10 osd.70 71595 do_waiters -- start 2014-12-08 18:14:03.647621 7f2904b56700 10 osd.70 71595 do_waiters -- finish 2014-12-08 18:14:03.647907 7f2904b56700 10 osd.70 71595 do_waiters -- start 2014-12-08 18:14:03.647912 7f2904b56700 10 osd.70 71595 do_waiters -- finish 2014-12-08 18:14:03.647914 7f2904b56700 20 osd.70 71595 _dispatch 0x135bb0e0 pg_notify(19.6e(150) epoch 71595) v5 2014-12-08 18:14:03.647955 7f2904b56700 7 osd.70 71595 handle_pg_notify from osd.30 2014-12-08 18:14:03.647958 7f2904b56700 15 osd.70 71595 require_same_or_newer_map 71595 (i am 71595) 0x135bb0e0 2014-12-08 18:14:03.648011 7f2904b56700 10 osd.70 71595 do_waiters -- start 2014-12-08 18:14:03.648013 7f2904b56700 10 osd.70 71595 do_waiters -- finish 2014-12-08 18:14:03.648018 7f28fcb46700 10 osd.70 pg_epoch: 71595 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] handle_peering_event: epoch_sent: 71595 epoch_requested: 71595 MNotifyRec from 30 notify: (query_epoch:71595, epoch_sent:71595, info:19.6e( empty local-les=0 n=0 ec=21298 les/c 51101/50495 71591/71591/71591)) features: 0x3fffffffffff 2014-12-08 18:14:03.648058 7f28fcb46700 7 osd.70 pg_epoch: 71595 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] state: handle_pg_notify from osd.30 2014-12-08 18:14:03.648103 7f28fcb46700 10 osd.70 pg_epoch: 71595 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] state: pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] got dup osd.30 info 19.6e( empty local-les=0 n=0 ec=21298 les/c 51101/50495 71591/71591/71591), identical to ours 2014-12-08 18:14:03.648146 7f28fcb46700 10 log is not dirty 2014-12-08 18:14:03.683419 7f2911047700 5 osd.70 71595 tick 2014-12-08 18:14:03.683443 7f2911047700 10 osd.70 71595 maybe_update_heartbeat_peers updating 2014-12-08 18:14:03.683447 7f2911047700 20 osd.70 71595 216.257 heartbeat_peers 2014-12-08 18:14:03.683450 7f2911047700 20 osd.70 71595 216.249 heartbeat_peers 5,68,70 2014-12-08 18:14:03.683455 7f2911047700 20 osd.70 71595 216.24a heartbeat_peers 2014-12-08 18:14:03.683458 7f2911047700 20 osd.70 71595 218.5bb heartbeat_peers 2014-12-08 18:14:03.683459 7f2911047700 20 osd.70 71595 218.5bc heartbeat_peers 2014-12-08 18:14:03.683461 7f2911047700 20 osd.70 71595 216.e9 heartbeat_peers 2014-12-08 18:14:03.683463 7f2911047700 20 osd.70 71595 216.240 heartbeat_peers 2014-12-08 18:14:03.683465 7f2911047700 20 osd.70 71595 218.240 heartbeat_peers 2014-12-08 18:14:03.683466 7f2911047700 20 osd.70 71595 218.3d8 heartbeat_peers 2014-12-08 18:14:03.683468 7f2911047700 20 osd.70 71595 216.243 heartbeat_peers 6,23,70 2014-12-08 18:14:03.683471 7f2911047700 20 osd.70 71595 216.ee heartbeat_peers 2014-12-08 18:14:03.683473 7f2911047700 20 osd.70 71595 216.3dc heartbeat_peers 2014-12-08 18:14:03.683481 7f2911047700 20 osd.70 71595 19.24 heartbeat_peers 14,16,70 2014-12-08 18:14:03.683485 7f2911047700 20 osd.70 71595 217.246 heartbeat_peers 3,21,70 2014-12-08 18:14:03.683488 7f2911047700 20 osd.70 71595 218.3dd heartbeat_peers 14,66,70 2014-12-08 18:14:03.683491 7f2911047700 20 osd.70 71595 216.278 heartbeat_peers 2014-12-08 18:14:03.683493 7f2911047700 20 osd.70 71595 216.e3 heartbeat_peers -- 2014-12-08 18:14:03.683646 7f2911047700 20 osd.70 71595 19.6e heartbeat_peers 0,4,11,13,16,20,30,31,56,58,59,63,70,76 2014-12-08 18:14:03.683654 7f2911047700 20 osd.70 71595 19.6d heartbeat_peers 2014-12-08 18:14:03.683656 7f2911047700 20 osd.70 71595 218.6c7 heartbeat_peers 10,18,70 2014-12-08 18:14:03.683659 7f2911047700 20 osd.70 71595 218.6c4 heartbeat_peers 13,70,76 2014-12-08 18:14:03.683661 7f2911047700 20 osd.70 71595 218.562 heartbeat_peers 2014-12-08 18:14:03.683663 7f2911047700 20 osd.70 71595 218.5e heartbeat_peers 2014-12-08 18:14:03.683664 7f2911047700 20 osd.70 71595 218.5f heartbeat_peers 15,24,70 2014-12-08 18:14:03.683667 7f2911047700 20 osd.70 71595 217.235 heartbeat_peers 2014-12-08 18:14:03.683669 7f2911047700 20 osd.70 71595 218.38e heartbeat_peers 3,13,70 2014-12-08 18:14:03.683671 7f2911047700 20 osd.70 71595 218.38f heartbeat_peers 23,25,70 2014-12-08 18:14:03.683673 7f2911047700 20 osd.70 71595 218.38c heartbeat_peers 2014-12-08 18:14:03.683675 7f2911047700 20 osd.70 71595 217.51 heartbeat_peers 2014-12-08 18:14:03.683676 7f2911047700 20 osd.70 71595 216.380 heartbeat_peers 2014-12-08 18:14:03.683678 7f2911047700 20 osd.70 71595 218.51 heartbeat_peers 66,70,75 2014-12-08 18:14:03.683680 7f2911047700 20 osd.70 71595 218.380 heartbeat_peers 2014-12-08 18:14:03.683681 7f2911047700 20 osd.70 71595 218.518 heartbeat_peers 2014-12-08 18:14:03.683683 7f2911047700 20 osd.70 71595 217.22d heartbeat_peers 2014-12-08 18:14:03.683684 7f2911047700 20 osd.70 71595 216.384 heartbeat_peers 2014-12-08 18:14:03.683686 7f2911047700 20 osd.70 71595 216.22f heartbeat_peers 6,21,70 2014-12-08 18:14:03.683688 7f2911047700 20 osd.70 71595 218.4b heartbeat_peers 2014-12-08 18:14:03.683689 7f2911047700 20 osd.70 71595 217.220 heartbeat_peers -- 2014-12-08 18:14:03.949255 7f2904b56700 20 osd.70 71595 _dispatch 0x1335bde0 pg_notify(19.6e(150) epoch 71595) v5 2014-12-08 18:14:03.949284 7f2904b56700 7 osd.70 71595 handle_pg_notify from osd.16 2014-12-08 18:14:03.949288 7f2904b56700 15 osd.70 71595 require_same_or_newer_map 71595 (i am 71595) 0x1335bde0 2014-12-08 18:14:03.949352 7f2904b56700 10 osd.70 71595 do_waiters -- start 2014-12-08 18:14:03.949355 7f2904b56700 10 osd.70 71595 do_waiters -- finish 2014-12-08 18:14:03.949381 7f28fc345700 10 osd.70 pg_epoch: 71595 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] handle_peering_event: epoch_sent: 71595 epoch_requested: 71595 MNotifyRec from 16 notify: (query_epoch:71595, epoch_sent:71595, info:19.6e( empty local-les=0 n=0 ec=21298 les/c 51101/50495 71591/71591/71591)) features: 0x3fffffffffff 2014-12-08 18:14:03.949435 7f28fc345700 7 osd.70 pg_epoch: 71595 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] state: handle_pg_notify from osd.16 2014-12-08 18:14:03.949450 7f28fc345700 10 osd.70 pg_epoch: 71595 pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] state: pg[19.6e( v 51072'48910307 (50495'48906592,51072'48910307] local-les=51066 n=855 ec=21298 les/c 51101/50495 71591/71591/71591) [70,16,13] r=0 lpr=71591 pi=50489-71590/150 crt=51071'48910301 lcod 0'0 mlcod 0'0 incomplete] got dup osd.16 info 19.6e( empty local-les=0 n=0 ec=21298 les/c 51101/50495 71591/71591/71591), identical to ours 2014-12-08 18:14:03.949472 7f28fc345700 10 log is not dirty 2014-12-08 18:14:03.979588 7f2902351700 10 osd.70 71595 note_peer_epoch osd.77 has 71595 2014-12-08 18:14:03.979598 7f2902351700 20 osd.70 71595 share_map_peer 0x12a1fc80 already has epoch 71595 2014-12-08 18:14:03.979650 7f2900b4e700 20 osd.70 71595 share_map_peer 0x12a1fc80 already has epoch 71595 2014-12-08 18:14:04.060210 7f2900b4e700 10 osd.70 71595 note_peer_epoch osd.8 has 71595 2014-12-08 18:14:04.060222 7f2900b4e700 20 osd.70 71595 share_map_peer 0x12623a20 already has epoch 71595 2014-12-08 18:14:04.060504 7f2902351700 20 osd.70 71595 share_map_peer 0x12623a20 already has epoch 71595 2014-12-08 18:14:04.290864 7f2902351700 10 osd.70 71595 note_peer_epoch osd.76 has 71595 2014-12-08 18:14:04.290877 7f2902351700 20 osd.70 71595 share_map_peer 0x12889fa0 already has epoch 71595 2014-12-08 18:14:04.290976 7f2900b4e700 20 osd.70 71595 share_map_peer 0x12889fa0 already has epoch 71595 2014-12-08 18:14:04.432678 7f2900b4e700 10 osd.70 71595 note_peer_epoch osd.7 has 71595 2014-12-08 18:14:04.432688 7f2900b4e700 20 osd.70 71595 share_map_peer 0x127b7fa0 already has epoch 71595 2014-12-08 18:14:04.432727 7f2902351700 20 osd.70 71595 share_map_peer 0x127b7fa0 already has epoch 71595 2014-12-08 18:14:04.477331 7f2900b4e700 10 osd.70 71595 note_peer_epoch osd.71 has 71595 2014-12-08 18:14:04.477358 7f2900b4e700 20 osd.70 71595 share_map_peer 0x129274a0 already has epoch 71595 2014-12-08 18:14:04.477410 7f2902351700 20 osd.70 71595 share_map_peer 0x129274a0 already has epoch 71595 2014-12-08 18:14:04.577059 7f2900b4e700 20 osd.70 71595 share_map_peer 0x127b8260 already has epoch 71595 2014-12-08 18:14:04.577178 7f2902351700 20 osd.70 71595 share_map_peer 0x127b8260 already has epoch 71595 2014-12-08 18:14:04.606669 7f2900b4e700 10 osd.70 71595 note_peer_epoch osd.68 has 71595 2014-12-08 18:14:04.606689 7f2900b4e700 20 osd.70 71595 share_map_peer 0x11a2e840 already has epoch 71595