Ceph

root@peon5754:~# ceph pg dump | grep peering|sort
11.291f 48      0       0       0       687588  179425  179425  remapped+peering        2012-07-31 22:06:06.544533      110900'2380     112510'40079    [367,398,426]   [367,398,426,218]       110900'2380     2012-07-31 20:44:13.204916
11.945a 60      0       0       0       706366  180272  180272  remapped+peering        2012-07-31 22:06:02.526352      109184'2444     113487'6        [590,232,154]   [590,232,160,457]       109184'2444     2012-07-31 20:33:43.820623
11.bce6 52      0       0       0       706078  179965  179965  remapped+peering        2012-07-31 22:06:06.721910      111140'2444     113488'2        [652,367,443]   [652,367,443,557]       87916'2392      2012-07-27 06:23:55.325072
5.124a  43      0       0       0       134051137       160450  160450  peering 2012-07-31 22:06:06.625033      113438'1293     113179'62742    [1005,461,778]  [1005,461,778]  111191'1292     2012-07-31 20:27:39.284759
5.339b  51      0       0       0       1384556703      184682  184682  remapped+peering        2012-07-31 22:06:06.588708      111691'3590     113488'3        [567,729,1016]  [567,729,209,346]       87916'3520      2012-07-27 06:46:54.144366
5.3ef9  60      0       0       0       519475076       172842  172842  peering 2012-07-31 22:06:06.614138      108818'1958     112642'974      [557,193,1010]  [557,193,1010]  86527'1892      2012-07-26 04:52:38.136108
5.44c2  41      0       0       0       760422179       176971  176971  peering 2012-07-31 22:06:06.634252      112529'2404     112519'13019    [631,694,102]   [631,694,102]   112529'2404     2012-07-31 20:33:40.998462
5.56f   46      0       0       0       329484213       169239  169239  remapped+peering        2012-07-31 22:06:06.596506      107485'1682     112516'14325    [566,614,338]   [566,614,338,312]       107485'1682     2012-07-31 20:31:09.055944
5.57a6  40      0       0       0       689377258       183668  183668  remapped+peering        2012-07-31 22:05:59.843162      109512'2083     113486'3        [668,582,280]   [668,582,189,323]       102173'2040     2012-07-27 23:37:55.789355
5.727c  45      0       0       0       347239616       174530  174530  peering 2012-07-31 22:06:06.544465      113230'1502     112508'7180     [26,526,157]    [26,526,157]    87916'1450      2012-07-27 06:43:05.001200
5.81d5  34      0       0       0       536475076       179208  179208  peering 2012-07-31 22:06:06.651725      106465'1838     112496'3077     [217,605,615]   [217,605,615]   87916'1805      2012-07-27 06:54:01.394043
5.9240  54      0       0       0       538779844       179240  179240  remapped+peering        2012-07-31 22:06:06.595153      108233'2031     112515'55441    [229,220,639]   [229,220,639,179]       86260'1917      2012-07-26 01:35:43.837070
5.92ed  45      0       0       0       55646163        149715  149715  peering 2012-07-31 22:06:06.586791      113019'1075     113186'6637     [136,530,599]   [136,530,599]   66451'1016      2012-07-27 15:18:07.220745
5.aa8c  45      0       0       0       2618889134      176719  176719  remapped+peering        2012-07-31 22:06:06.618009      112376'6087     113488'1        [205,667,419]   [205,667,419,24]        112376'6087     2012-07-31 20:29:29.331063
5.bb50  42      0       0       0       906570606       178106  178106  peering 2012-07-31 22:06:06.621592      111489'2674     112653'885      [169,711,651]   [169,711,651]   87916'2573      2012-07-27 07:08:35.936853
5.c096  48      0       0       0       451100676       233837  233837  remapped+peering        2012-07-31 22:06:06.638969      113170'2075     113221'53894    [810,413,605]   [810,413,605,104]       101062'2017     2012-07-27 15:22:09.427545
5.ce3c  38      0       0       0       141618117       154912  154912  peering 2012-07-31 22:06:06.612812      112688'1269     103185'10486    [264,33,682]    [264,33,682]    101092'1202     2012-07-27 15:20:48.387908
5.eee3  45      0       0       0       326613008       135472  135472  remapped+peering        2012-07-31 22:06:06.622799      113124'5583     112501'57739    [102,682,551]   [102,682,551,268]       70008'5484      2012-07-26 01:32:48.964843
5.f0da  36      0       0       0       326716396       167339  167339  peering 2012-07-31 22:06:06.565803      110554'1672     112508'8768     [336,737,506]   [336,737,506]   87916'1605      2012-07-27 06:26:14.075392
5.f318  33      0       0       0       295672159       171602  171602  peering 2012-07-31 22:06:06.572761      111963'1541     112519'7806     [591,65,29]     [591,65,29]     111963'1541     2012-07-31 20:27:30.667415
5.f633  48      0       0       0       541973168       176239  176239  remapped+peering        2012-07-31 22:06:06.555449      109521'1914     113473'68       [755,643,358]   [755,643,358,276]       109521'1914     2012-07-31 20:39:14.478257
root@peon5754:~# 
root@peon5754:~# sleep 60 ; ceph pg dump | grep peering | sort
^C
root@peon5754:~# ceph pg dump | grep peering|sort
11.4a53 54      0       0       0       718667  180472  180472  peering 2012-07-31 22:06:29.027662      107592'2479     112510'50062    [87,12,410]     [87,12,410]     87916'2451      2012-07-27 23:37:23.367178
11.b89c 61      0       0       0       715691  179120  179120  peering 2012-07-31 22:06:33.513569      109881'2466     112508'9420     [575,737,164]   [575,737,164]   87327'2409      2012-07-27 16:08:57.401425
5.148c  51      0       0       0       547739238       181283  181283  remapped+peering        2012-07-31 22:06:23.566567      111291'2035     112510'64015    [998,696,449]   [998,696,449,54]        87175'1948      2012-07-27 06:23:11.752541
5.1ae0  43      0       0       0       357005659       134951  134951  remapped+peering        2012-07-31 22:06:33.600706      112824'5676     112497'20463    [691,786,426]   [691,786,426,164]       108211'5675     2012-07-31 20:43:41.037158
5.1c0c  31      0       0       0       73780965        150949  150949  peering 2012-07-31 22:06:29.081523      111618'1175     112504'11290    [103,198,672]   [103,198,672]   87916'1116      2012-07-27 06:23:08.221143
5.2472  48      0       0       0       511338927       172717  172717  peering 2012-07-31 22:06:33.586402      111139'2060     113483'63       [789,702,586]   [789,702,586]   111139'2060     2012-07-31 20:33:38.480391
5.27ea  39      0       0       0       485984250       179060  179060  peering 2012-07-31 22:06:33.513696      109124'1856     113164'5793     [575,696,54]    [575,696,54]    109124'1856     2012-07-31 20:35:02.514350
5.376a  40      0       0       0       338672815       173562  173562  remapped+peering        2012-07-31 22:06:33.556185      106465'5560     113484'55       [592,125,185]   [592,125,185,332]       87916'5507      2012-07-27 16:08:49.446644
5.3f18  36      0       0       0       263560429       170010  170010  peering 2012-07-31 22:06:33.650976      111175'1522     113493'5        [825,273,41]    [825,273,41]    87916'1445      2012-07-27 06:51:05.401231
5.507   48      0       0       0       264844574       165220  165220  peering 2012-07-31 22:06:29.099673      112232'1376     112496'62139    [173,602,619]   [173,602,619]   65060'1260      2012-07-26 05:01:49.831265
5.532c  46      0       0       0       562721368       176473  176473  peering 2012-07-31 22:06:33.514408      112529'2187     112508'14922    [541,592,379]   [541,592,379]   87916'2105      2012-07-27 06:45:23.954225
5.5be0  36      0       0       0       268682631       165809  165809  peering 2012-07-31 22:06:33.517270      112565'1646     112510'59395    [129,606,759]   [129,606,759]   88649'1564      2012-07-27 06:50:11.806803
5.5d40  45      0       0       0       317816060       169466  169466  peering 2012-07-31 22:06:33.645752      110897'1582     112501'53756    [120,238,519]   [120,238,519]   87916'1523      2012-07-27 06:45:02.102176
5.6592  50      0       0       0       303179581       169682  169682  peering 2012-07-31 22:06:29.066659      110824'1509     112507'7984     [132,1006,205]  [132,1006,205]  87916'1460      2012-07-27 06:48:20.420691
5.79a1  43      0       0       0       494348661       180475  180475  peering 2012-07-31 22:06:33.621785      110849'2013     112508'63505    [1071,495,687]  [1071,495,687]  87916'1943      2012-07-27 06:44:39.131806
5.8e67  38      0       0       0       316014478       172660  172660  peering 2012-07-31 20:58:53.198595      108681'1576     113492'3        [519,358,56]    [519,358,56]    87916'1523      2012-07-27 07:10:47.609322
5.b0e7  40      0       0       0       106346423       151758  151758  remapped+peering        2012-07-31 22:06:29.061290      110800'1276     100803'59916    [264,776,562]   [264,776,562,184]       87916'1171      2012-07-27 06:20:16.035395
5.beb2  46      0       0       0       535615909       179726  179726  peering 2012-07-31 22:06:29.079118      112776'1994     112506'58590    [6,185,772]     [6,185,772]     101656'1893     2012-07-27 23:32:31.381404
5.c21c  48      0       0       0       1803995432      170149  170149  peering 2012-07-31 22:06:29.085457      111401'4484     113464'79       [418,706,133]   [418,706,133]   87916'4406      2012-07-27 06:42:20.506818
5.d8bc  33      0       0       0       359765605       164278  164278  remapped+peering        2012-07-31 22:06:29.109812      111902'5631     113492'2        [621,6,175]     [621,6,42,157]  65060'5580      2012-07-26 06:06:02.548055
root@peon5754:~# ceph health detaul | grep stuck
^C
root@peon5754:~# ceph health detaul | grep peering
HEALTH_WARN 9808 pgs backfill; 600 pgs degraded; 15 pgs peering; 13224 pgs recovering; 2 pgs stuck inactive; 13427 pgs stuck unclean; recovery 860057/23861832 degraded (3.604%); 1 mons down, quorum 0,2 peon5752,peon5754
pg 5.7342 is stuck remapped+peering, last acting [618,136,559,270]
pg 5.391e is stuck remapped+peering, last acting [527,678,249,119]
pg 5.3d11 is stuck peering, last acting [584,252,134]
pg 5.391e is stuck remapped+peering, last acting [527,678,249,119]
pg 5.f5bc is stuck peering, last acting [249,200,660]
pg 5.f414 is stuck remapped+peering, last acting [584,374,815,184]
pg 11.f40e is stuck remapped+peering, last acting [584,374,815,184]
pg 11.f2ac is stuck peering, last acting [194,458,412]
pg 11.8672 is stuck peering, last acting [56,268,696]
pg 5.8430 is stuck peering, last acting [757,494,691]
pg 5.842a is stuck peering, last acting [1011,486,775]
pg 5.e015 is stuck peering, last acting [539,1069,116]
pg 11.7d4e is stuck remapped+peering, last acting [536,397,254,347]
pg 5.7342 is stuck remapped+peering, last acting [618,136,559,270]
pg 5.7280 is stuck remapped+peering, last acting [491,96,246,66]
pg 5.cc83 is stuck remapped+peering, last acting [430,832,198,172]
pg 5.6685 is stuck peering, last acting [315,598,106]
pg 5.f5bc is peering, acting [249,200,660]
pg 5.f414 is remapped+peering, acting [584,374,815,184]
pg 11.f40e is remapped+peering, acting [584,374,815,184]
pg 11.f2ac is peering, acting [194,458,412]
pg 5.e015 is peering, acting [539,1069,116]
pg 5.cc83 is remapped+peering, acting [430,832,198,172]
pg 11.8672 is peering, acting [56,268,696]
pg 5.8430 is peering, acting [757,494,691]
pg 5.842a is peering, acting [1011,486,775]
pg 11.7d4e is remapped+peering, acting [536,397,254,347]
pg 5.7342 is remapped+peering, acting [618,136,559,270]
pg 5.7280 is remapped+peering, acting [491,96,246,66]
pg 5.6685 is peering, acting [315,598,106]
pg 5.3d11 is peering, acting [584,252,134]
pg 5.391e is remapped+peering, acting [527,678,249,119]