-189> 2013-09-27 15:39:16.855663 7f1da97a2700 1 -- [2001:620:0:6::102]:0/4426 <== osd.18 [2001:620:0:6::110]:6806/15588 29130 ==== osd_ping(ping_reply e71023 stamp 2013-09-27 15:39:16.854136) v2 ==== 47+0+0 (1027812123 0 0) 0x1eb928c0 con 0x1dc478c0 -188> 2013-09-27 15:39:16.855719 7f1da97a2700 1 -- [2001:620:0:6::102]:0/4426 <== osd.49 [2001:620:0:6::10e]:6808/5732 29130 ==== osd_ping(ping_reply e71023 stamp 2013-09-27 15:39:16.854136) v2 ==== 47+0+0 (1027812123 0 0) 0x1dd2ea80 con 0x1dc47080 -187> 2013-09-27 15:39:16.855766 7f1da97a2700 1 -- [2001:620:0:6::102]:0/4426 <== osd.40 [2001:620:0:6::10a]:6811/1004803 29130 ==== osd_ping(ping_reply e71023 stamp 2013-09-27 15:39:16.854136) v2 ==== 47+0+0 (1027812123 0 0) 0x21f78c40 con 0x1dc46dc0 -186> 2013-09-27 15:39:16.855801 7f1da97a2700 1 -- [2001:620:0:6::102]:0/4426 <== osd.56 [2001:620:0:6::112]:6811/4580 29130 ==== osd_ping(ping_reply e71023 stamp 2013-09-27 15:39:16.854136) v2 ==== 47+0+0 (1027812123 0 0) 0x20ed9340 con 0x1dc46b00 -185> 2013-09-27 15:39:16.855822 7f1da97a2700 1 -- [2001:620:0:6::102]:0/4426 <== osd.14 [2001:620:0:6::10c]:6835/4934 29130 ==== osd_ping(ping_reply e71023 stamp 2013-09-27 15:39:16.854136) v2 ==== 47+0+0 (1027812123 0 0) 0x1f829a40 con 0x1dc43340 -184> 2013-09-27 15:39:16.855841 7f1da97a2700 1 -- [2001:620:0:6::102]:0/4426 <== osd.45 [2001:620:0:6::10a]:6820/5276 29130 ==== osd_ping(ping_reply e71023 stamp 2013-09-27 15:39:16.854136) v2 ==== 47+0+0 (1027812123 0 0) 0x20bc8a80 con 0x1dc43080 -183> 2013-09-27 15:39:16.855869 7f1da97a2700 1 -- [2001:620:0:6::102]:0/4426 <== osd.8 [2001:620:0:6::108]:6811/5452 29127 ==== osd_ping(ping_reply e71023 stamp 2013-09-27 15:39:16.854136) v2 ==== 47+0+0 (1027812123 0 0) 0x1ff1a700 con 0x1dc429a0 -182> 2013-09-27 15:39:16.855890 7f1da97a2700 1 -- [2001:620:0:6::102]:0/4426 <== osd.9 [2001:620:0:6::108]:6814/5575 29127 ==== osd_ping(ping_reply e71023 stamp 2013-09-27 15:39:16.854136) v2 ==== 47+0+0 (1027812123 0 0) 0x23672a80 con 0x1fa0b1e0 -181> 2013-09-27 15:39:16.855912 7f1da97a2700 1 -- [2001:620:0:6::102]:0/4426 <== osd.10 [2001:620:0:6::108]:6802/4423 29127 ==== osd_ping(ping_reply e71023 stamp 2013-09-27 15:39:16.854136) v2 ==== 47+0+0 (1027812123 0 0) 0x1c61f500 con 0x1fa0af20 -180> 2013-09-27 15:39:16.855932 7f1da97a2700 1 -- [2001:620:0:6::102]:0/4426 <== osd.12 [2001:620:0:6::10c]:6812/4468 29127 ==== osd_ping(ping_reply e71023 stamp 2013-09-27 15:39:16.854136) v2 ==== 47+0+0 (1027812123 0 0) 0x25d048c0 con 0x1fa0a9a0 -179> 2013-09-27 15:39:16.855951 7f1da97a2700 1 -- [2001:620:0:6::102]:0/4426 <== osd.11 [2001:620:0:6::108]:6805/4506 29127 ==== osd_ping(ping_reply e71023 stamp 2013-09-27 15:39:16.854136) v2 ==== 47+0+0 (1027812123 0 0) 0x20945880 con 0x20e17340 -178> 2013-09-27 15:39:16.855969 7f1da97a2700 1 -- [2001:620:0:6::102]:0/4426 <== osd.57 [2001:620:0:6::112]:6805/4664 1984 ==== osd_ping(ping_reply e71023 stamp 2013-09-27 15:39:16.854136) v2 ==== 47+0+0 (1027812123 0 0) 0x250c96c0 con 0x1dc46840 -177> 2013-09-27 15:39:16.855987 7f1da97a2700 1 -- [2001:620:0:6::102]:0/4426 <== osd.1 [2001:620:0:6::100]:6816/423 1181 ==== osd_ping(ping_reply e71023 stamp 2013-09-27 15:39:16.854136) v2 ==== 47+0+0 (1027812123 0 0) 0x1e479500 con 0x1dac0000 -176> 2013-09-27 15:39:16.856006 7f1da97a2700 1 -- [2001:620:0:6::102]:0/4426 <== osd.31 [2001:620:0:6::106]:6808/5158 132 ==== osd_ping(ping_reply e71023 stamp 2013-09-27 15:39:16.854136) v2 ==== 47+0+0 (1027812123 0 0) 0x1c61f340 con 0x241bc9a0 -175> 2013-09-27 15:39:16.856024 7f1da97a2700 1 -- [2001:620:0:6::102]:0/4426 <== osd.29 [2001:620:0:6::106]:6802/4977 59 ==== osd_ping(ping_reply e71023 stamp 2013-09-27 15:39:16.854136) v2 ==== 47+0+0 (1027812123 0 0) 0x202d1dc0 con 0x21acc9a0 -174> 2013-09-27 15:39:16.856042 7f1da97a2700 1 -- [2001:620:0:6::102]:0/4426 <== osd.30 [2001:620:0:6::106]:6805/5076 48 ==== osd_ping(ping_reply e71023 stamp 2013-09-27 15:39:16.854136) v2 ==== 47+0+0 (1027812123 0 0) 0xf712540 con 0x1fa15600 -173> 2013-09-27 15:39:16.856061 7f1da97a2700 1 -- [2001:620:0:6::102]:0/4426 <== osd.33 [2001:620:0:6::106]:6814/5323 28 ==== osd_ping(ping_reply e71023 stamp 2013-09-27 15:39:16.854136) v2 ==== 47+0+0 (1027812123 0 0) 0x23c9fa40 con 0x241bda20 -172> 2013-09-27 15:39:16.856080 7f1da97a2700 1 -- [2001:620:0:6::102]:0/4426 <== osd.37 [2001:620:0:6::106]:6820/2005575 22 ==== osd_ping(ping_reply e71023 stamp 2013-09-27 15:39:16.854136) v2 ==== 47+0+0 (1027812123 0 0) 0x24098380 con 0x1f99fe40 -171> 2013-09-27 15:39:16.856098 7f1da97a2700 1 -- [2001:620:0:6::102]:0/4426 <== osd.34 [2001:620:0:6::106]:6817/5408 15 ==== osd_ping(ping_reply e71023 stamp 2013-09-27 15:39:16.854136) v2 ==== 47+0+0 (1027812123 0 0) 0x22a48540 con 0x241bde40 -170> 2013-09-27 15:39:16.856116 7f1da97a2700 1 -- [2001:620:0:6::102]:0/4426 <== osd.35 [2001:620:0:6::106]:6811/2005491 7 ==== osd_ping(ping_reply e71023 stamp 2013-09-27 15:39:16.854136) v2 ==== 47+0+0 (1027812123 0 0) 0x1f199c00 con 0x1c61b340 -169> 2013-09-27 15:39:16.856134 7f1da97a2700 1 -- [2001:620:0:6::102]:0/4426 <== osd.19 [2001:620:0:6::110]:6809/25573 29130 ==== osd_ping(ping_reply e71023 stamp 2013-09-27 15:39:16.854136) v2 ==== 47+0+0 (1027812123 0 0) 0x25b9a8c0 con 0x1cd9da20 -168> 2013-09-27 15:39:16.856153 7f1da97a2700 1 -- [2001:620:0:6::102]:0/4426 <== osd.13 [2001:620:0:6::10c]:6820/4804 29130 ==== osd_ping(ping_reply e71023 stamp 2013-09-27 15:39:16.854136) v2 ==== 47+0+0 (1027812123 0 0) 0x21603500 con 0x1cd9d760 -167> 2013-09-27 15:39:16.856171 7f1da97a2700 1 -- [2001:620:0:6::102]:0/4426 <== osd.58 [2001:620:0:6::112]:6808/4755 29130 ==== osd_ping(ping_reply e71023 stamp 2013-09-27 15:39:16.854136) v2 ==== 47+0+0 (1027812123 0 0) 0x216708c0 con 0x1cd9dce0 -166> 2013-09-27 15:39:16.856189 7f1da97a2700 1 -- [2001:620:0:6::102]:0/4426 <== osd.46 [2001:620:0:6::10a]:6814/1005359 29130 ==== osd_ping(ping_reply e71023 stamp 2013-09-27 15:39:16.854136) v2 ==== 47+0+0 (1027812123 0 0) 0x255a8fc0 con 0x1dac0580 -165> 2013-09-27 15:39:16.856207 7f1da97a2700 1 -- [2001:620:0:6::102]:0/4426 <== osd.51 [2001:620:0:6::10e]:6832/1958 29130 ==== osd_ping(ping_reply e71023 stamp 2013-09-27 15:39:16.854136) v2 ==== 47+0+0 (1027812123 0 0) 0x21f64540 con 0x1dac02c0 -164> 2013-09-27 15:39:16.856226 7f1da97a2700 1 -- [2001:620:0:6::102]:0/4426 <== osd.41 [2001:620:0:6::10a]:6808/4904 29130 ==== osd_ping(ping_reply e71023 stamp 2013-09-27 15:39:16.854136) v2 ==== 47+0+0 (1027812123 0 0) 0x200ace00 con 0x1dac0c60 -163> 2013-09-27 15:39:16.856244 7f1da97a2700 1 -- [2001:620:0:6::102]:0/4426 <== osd.59 [2001:620:0:6::112]:6816/4840 29130 ==== osd_ping(ping_reply e71023 stamp 2013-09-27 15:39:16.854136) v2 ==== 47+0+0 (1027812123 0 0) 0x20087180 con 0x1dac09a0 -162> 2013-09-27 15:39:16.856261 7f1da97a2700 1 -- [2001:620:0:6::102]:0/4426 <== osd.47 [2001:620:0:6::10e]:6831/1005568 29130 ==== osd_ping(ping_reply e71023 stamp 2013-09-27 15:39:16.854136) v2 ==== 47+0+0 (1027812123 0 0) 0x1f4428c0 con 0x1dac1340 -161> 2013-09-27 15:39:16.856279 7f1da97a2700 1 -- [2001:620:0:6::102]:0/4426 <== osd.63 [2001:620:0:6::10c]:6806/6179 29130 ==== osd_ping(ping_reply e71023 stamp 2013-09-27 15:39:16.854136) v2 ==== 47+0+0 (1027812123 0 0) 0x1ef3a8c0 con 0x1dad8840 -160> 2013-09-27 15:39:16.856297 7f1da97a2700 1 -- [2001:620:0:6::102]:0/4426 <== osd.52 [2001:620:0:6::10e]:6817/5995 29130 ==== osd_ping(ping_reply e71023 stamp 2013-09-27 15:39:16.854136) v2 ==== 47+0+0 (1027812123 0 0) 0x250dac40 con 0x1dad9a20 -159> 2013-09-27 15:39:16.856314 7f1da97a2700 1 -- [2001:620:0:6::102]:0/4426 <== osd.62 [2001:620:0:6::10c]:6818/6071 29130 ==== osd_ping(ping_reply e71023 stamp 2013-09-27 15:39:16.854136) v2 ==== 47+0+0 (1027812123 0 0) 0x1fcdcc40 con 0x1dc2d1e0 -158> 2013-09-27 15:39:16.856332 7f1da97a2700 1 -- [2001:620:0:6::102]:0/4426 <== osd.54 [2001:620:0:6::10e]:6823/6159 29130 ==== osd_ping(ping_reply e71023 stamp 2013-09-27 15:39:16.854136) v2 ==== 47+0+0 (1027812123 0 0) 0x2541ca80 con 0x1dc2a6e0 -157> 2013-09-27 15:39:16.856350 7f1da97a2700 1 -- [2001:620:0:6::102]:0/4426 <== osd.64 [2001:620:0:6::10c]:6824/6343 29130 ==== osd_ping(ping_reply e71023 stamp 2013-09-27 15:39:16.854136) v2 ==== 47+0+0 (1027812123 0 0) 0x217a4e00 con 0x1dc2da20 -156> 2013-09-27 15:39:16.856368 7f1da97a2700 1 -- [2001:620:0:6::102]:0/4426 <== osd.55 [2001:620:0:6::10e]:6802/1006241 29130 ==== osd_ping(ping_reply e71023 stamp 2013-09-27 15:39:16.854136) v2 ==== 47+0+0 (1027812123 0 0) 0x20ed9dc0 con 0x1dc2be40 -155> 2013-09-27 15:39:16.856386 7f1da97a2700 1 -- [2001:620:0:6::102]:0/4426 <== osd.53 [2001:620:0:6::10e]:6820/6077 29130 ==== osd_ping(ping_reply e71023 stamp 2013-09-27 15:39:16.854136) v2 ==== 47+0+0 (1027812123 0 0) 0x2175b500 con 0x1dc2b340 -154> 2013-09-27 15:39:16.910618 7f1df54b9700 5 osd.21 71023 tick -153> 2013-09-27 15:39:16.919777 7f1da8fa1700 1 -- [2001:620:0:6::102]:6805/4426 <== osd.18 [2001:620:0:6::110]:0/15588 28917 ==== osd_ping(ping e71023 stamp 2013-09-27 15:39:16.920267) v2 ==== 47+0+0 (80686135 0 0) 0x1f1981c0 con 0x1d569ce0 -152> 2013-09-27 15:39:16.919814 7f1da8fa1700 1 -- [2001:620:0:6::102]:6805/4426 --> [2001:620:0:6::110]:0/15588 -- osd_ping(ping_reply e71023 stamp 2013-09-27 15:39:16.920267) v2 -- ?+0 0x212496c0 con 0x1d569ce0 -151> 2013-09-27 15:39:16.989209 7f1da8fa1700 1 -- [2001:620:0:6::102]:6805/4426 <== osd.52 [2001:620:0:6::10e]:0/5995 28993 ==== osd_ping(ping e71023 stamp 2013-09-27 15:39:16.988227) v2 ==== 47+0+0 (1637260337 0 0) 0x1ee64540 con 0x1c92b8c0 -150> 2013-09-27 15:39:16.989258 7f1da8fa1700 1 -- [2001:620:0:6::102]:6805/4426 --> [2001:620:0:6::10e]:0/5995 -- osd_ping(ping_reply e71023 stamp 2013-09-27 15:39:16.988227) v2 -- ?+0 0x1f1981c0 con 0x1c92b8c0 -149> 2013-09-27 15:39:16.996509 7f1da8fa1700 1 -- [2001:620:0:6::102]:6805/4426 <== osd.48 [2001:620:0:6::10e]:0/5650 29163 ==== osd_ping(ping e71023 stamp 2013-09-27 15:39:16.995615) v2 ==== 47+0+0 (2137618812 0 0) 0x1cb761c0 con 0x1c7c7340 -148> 2013-09-27 15:39:16.996537 7f1da8fa1700 1 -- [2001:620:0:6::102]:6805/4426 --> [2001:620:0:6::10e]:0/5650 -- osd_ping(ping_reply e71023 stamp 2013-09-27 15:39:16.995615) v2 -- ?+0 0x1ee64540 con 0x1c7c7340 -147> 2013-09-27 15:39:17.050822 7f1daa7a4700 1 -- [2001:620:0:6::102]:6804/4426 <== osd.16 [2001:620:0:6::110]:6802/27723 4113 ==== osd_map(71024..71024 src has 64713..71024) v3 ==== 3517+0+0 (793712060 0 0) 0x24c2afc0 con 0x1c75c840 -146> 2013-09-27 15:39:17.050872 7f1daa7a4700 3 osd.21 71023 handle_osd_map epochs [71024,71024], i have 71023, src has [64713,71024] -145> 2013-09-27 15:39:17.053836 7f1da679c700 10 monclient: _send_mon_message to mon.s2 at [2001:620:0:6::108]:6789/0 -144> 2013-09-27 15:39:17.053851 7f1da679c700 1 -- [2001:620:0:6::102]:6803/4426 --> [2001:620:0:6::108]:6789/0 -- osd_pgtemp(e71024 {0.a30=[21,34,1,57]} v71024) v1 -- ?+0 0x1e4c12c0 con 0x20e16160 -143> 2013-09-27 15:39:17.054093 7f1da6f9d700 10 monclient: _send_mon_message to mon.s2 at [2001:620:0:6::108]:6789/0 -142> 2013-09-27 15:39:17.054117 7f1da6f9d700 1 -- [2001:620:0:6::102]:6803/4426 --> [2001:620:0:6::108]:6789/0 -- osd_pgtemp(e71024 {0.a30=[21,34,1,57]} v71024) v1 -- ?+0 0x1f60f680 con 0x20e16160 -141> 2013-09-27 15:39:17.054256 7f1da6f9d700 10 monclient: _send_mon_message to mon.s2 at [2001:620:0:6::108]:6789/0 -140> 2013-09-27 15:39:17.054270 7f1da6f9d700 1 -- [2001:620:0:6::102]:6803/4426 --> [2001:620:0:6::108]:6789/0 -- osd_pgtemp(e71024 {0.a30=[21,34,1,57]} v71024) v1 -- ?+0 0x1f60f4a0 con 0x20e16160 -139> 2013-09-27 15:39:17.054405 7f1da679c700 10 monclient: _send_mon_message to mon.s2 at [2001:620:0:6::108]:6789/0 -138> 2013-09-27 15:39:17.054418 7f1da679c700 1 -- [2001:620:0:6::102]:6803/4426 --> [2001:620:0:6::108]:6789/0 -- osd_pgtemp(e71024 {0.a30=[21,34,1,57]} v71024) v1 -- ?+0 0x25928b40 con 0x20e16160 -137> 2013-09-27 15:39:17.054461 7f1da6f9d700 10 monclient: _send_mon_message to mon.s2 at [2001:620:0:6::108]:6789/0 -136> 2013-09-27 15:39:17.054477 7f1da6f9d700 1 -- [2001:620:0:6::102]:6803/4426 --> [2001:620:0:6::108]:6789/0 -- osd_pgtemp(e71024 {0.a30=[21,34,1,57]} v71024) v1 -- ?+0 0x1f60ef00 con 0x20e16160 -135> 2013-09-27 15:39:17.054702 7f1da6f9d700 10 monclient: _send_mon_message to mon.s2 at [2001:620:0:6::108]:6789/0 -134> 2013-09-27 15:39:17.054716 7f1da6f9d700 1 -- [2001:620:0:6::102]:6803/4426 --> [2001:620:0:6::108]:6789/0 -- osd_pgtemp(e71024 {0.a30=[21,34,1,57]} v71024) v1 -- ?+0 0x1f60f2c0 con 0x20e16160 -133> 2013-09-27 15:39:17.054954 7f1da679c700 10 monclient: _send_mon_message to mon.s2 at [2001:620:0:6::108]:6789/0 -132> 2013-09-27 15:39:17.054968 7f1da679c700 1 -- [2001:620:0:6::102]:6803/4426 --> [2001:620:0:6::108]:6789/0 -- osd_pgtemp(e71024 {0.a30=[21,34,1,57]} v71024) v1 -- ?+0 0x24e34f00 con 0x20e16160 -131> 2013-09-27 15:39:17.055431 7f1da6f9d700 10 monclient: _send_mon_message to mon.s2 at [2001:620:0:6::108]:6789/0 -130> 2013-09-27 15:39:17.055448 7f1da6f9d700 1 -- [2001:620:0:6::102]:6803/4426 --> [2001:620:0:6::108]:6789/0 -- osd_pgtemp(e71024 {0.a30=[21,34,1,57]} v71024) v1 -- ?+0 0x1f60eb40 con 0x20e16160 -129> 2013-09-27 15:39:17.055718 7f1da679c700 10 monclient: _send_mon_message to mon.s2 at [2001:620:0:6::108]:6789/0 -128> 2013-09-27 15:39:17.055733 7f1da679c700 1 -- [2001:620:0:6::102]:6803/4426 --> [2001:620:0:6::108]:6789/0 -- osd_pgtemp(e71024 {0.a30=[21,34,1,57]} v71024) v1 -- ?+0 0x24e341e0 con 0x20e16160 -127> 2013-09-27 15:39:17.056027 7f1da6f9d700 10 monclient: _send_mon_message to mon.s2 at [2001:620:0:6::108]:6789/0 -126> 2013-09-27 15:39:17.056042 7f1da6f9d700 1 -- [2001:620:0:6::102]:6803/4426 --> [2001:620:0:6::108]:6789/0 -- osd_pgtemp(e71024 {0.a30=[21,34,1,57]} v71024) v1 -- ?+0 0x1f60f0e0 con 0x20e16160 -125> 2013-09-27 15:39:17.056380 7f1da679c700 10 monclient: _send_mon_message to mon.s2 at [2001:620:0:6::108]:6789/0 -124> 2013-09-27 15:39:17.056394 7f1da679c700 1 -- [2001:620:0:6::102]:6803/4426 --> [2001:620:0:6::108]:6789/0 -- osd_pgtemp(e71024 {0.a30=[21,34,1,57]} v71024) v1 -- ?+0 0x24f61e00 con 0x20e16160 -123> 2013-09-27 15:39:17.056469 7f1da6f9d700 10 monclient: _send_mon_message to mon.s2 at [2001:620:0:6::108]:6789/0 -122> 2013-09-27 15:39:17.056482 7f1da6f9d700 1 -- [2001:620:0:6::102]:6803/4426 --> [2001:620:0:6::108]:6789/0 -- osd_pgtemp(e71024 {0.a30=[21,34,1,57]} v71024) v1 -- ?+0 0x1f60e960 con 0x20e16160 -121> 2013-09-27 15:39:17.056546 7f1da679c700 1 osd.21 pg_epoch: 71024 pg[0.d56( v 28197'7944 (14521'4944,28197'7944] local-les=71011 n=2313 ec=1 les/c 71011/71011 71023/71023/69955) [21,16] r=0 lpr=71023 pi=71010-71022/1 lcod 0'0 mlcod 0'0 peering] state: Peering, affected_by_map, going to Reset -120> 2013-09-27 15:39:17.056620 7f1da6f9d700 1 osd.21 pg_epoch: 71024 pg[11.d4b( v 69118'1702 (0'0,69118'1702] local-les=71011 n=151 ec=106 les/c 71011/71011 71023/71023/69905) [21] r=0 lpr=71023 pi=71010-71022/1 lcod 0'0 mlcod 0'0 peering] state: Peering, affected_by_map, going to Reset -119> 2013-09-27 15:39:17.056632 7f1da679c700 1 osd.21 pg_epoch: 71024 pg[0.d56( v 28197'7944 (14521'4944,28197'7944] local-les=71011 n=2313 ec=1 les/c 71011/71011 71024/71024/69955) [21,32,16] r=0 lpr=71024 pi=71010-71023/2 lcod 0'0 mlcod 0'0 inactive] state: transitioning to Primary -118> 2013-09-27 15:39:17.056691 7f1da6f9d700 1 osd.21 pg_epoch: 71024 pg[11.d4b( v 69118'1702 (0'0,69118'1702] local-les=71011 n=151 ec=106 les/c 71011/71011 71024/71024/69905) [21,32] r=0 lpr=71024 pi=71010-71023/2 lcod 0'0 mlcod 0'0 inactive] state: transitioning to Primary -117> 2013-09-27 15:39:17.056778 7f1da679c700 1 -- [2001:620:0:6::102]:6804/4426 --> [2001:620:0:6::110]:6802/27723 -- osd_map(71024..71024 src has 64713..71024) v3 -- ?+0 0x25fe4b40 con 0x1c75c840 -116> 2013-09-27 15:39:17.056813 7f1da679c700 1 -- [2001:620:0:6::102]:6804/4426 --> [2001:620:0:6::110]:6802/27723 -- pg_query(0.d56 epoch 71024) v2 -- ?+0 0x24f614a0 con 0x1c75c840 -115> 2013-09-27 15:39:17.056828 7f1da6f9d700 1 -- [2001:620:0:6::102]:6804/4426 --> [2001:620:0:6::106]:6828/1005241 -- osd_map(71023..71024 src has 64713..71024) v3 -- ?+0 0x254da6c0 con 0x2220b760 -114> 2013-09-27 15:39:17.056906 7f1da679c700 1 -- [2001:620:0:6::102]:6804/4426 --> [2001:620:0:6::106]:6828/1005241 -- osd_map(71023..71024 src has 64713..71024) v3 -- ?+0 0x25bc6480 con 0x2220b760 -113> 2013-09-27 15:39:17.056909 7f1da6f9d700 1 -- [2001:620:0:6::102]:6804/4426 --> [2001:620:0:6::106]:6828/1005241 -- pg_query(11.d4b epoch 71024) v2 -- ?+0 0x1f60ed20 con 0x2220b760 -112> 2013-09-27 15:39:17.056935 7f1da679c700 1 -- [2001:620:0:6::102]:6804/4426 --> [2001:620:0:6::106]:6828/1005241 -- pg_query(0.d56 epoch 71024) v2 -- ?+0 0x25928d20 con 0x2220b760 -111> 2013-09-27 15:39:17.057079 7f1da679c700 1 osd.21 pg_epoch: 71024 pg[2.d54( v 20483'258 (0'0,20483'258] local-les=71011 n=4 ec=1 les/c 71011/71011 71023/71023/69905) [21] r=0 lpr=71023 pi=71010-71022/1 lcod 0'0 mlcod 0'0 peering] state: Peering, affected_by_map, going to Reset -110> 2013-09-27 15:39:17.057098 7f1da6f9d700 1 osd.21 pg_epoch: 71024 pg[1.736( v 20605'1335 (0'0,20605'1335] local-les=71011 n=0 ec=1 les/c 71011/71011 71024/71024/71024) [32,53,21] r=2 lpr=71024 pi=66422-71023/34 lcod 0'0 inactive NOTIFY] state: transitioning to Stray -109> 2013-09-27 15:39:17.057158 7f1da679c700 1 osd.21 pg_epoch: 71024 pg[2.d54( v 20483'258 (0'0,20483'258] local-les=71011 n=4 ec=1 les/c 71011/71011 71024/71024/69905) [21,32] r=0 lpr=71024 pi=71010-71023/2 lcod 0'0 mlcod 0'0 inactive] state: transitioning to Primary -108> 2013-09-27 15:39:17.057178 7f1da6f9d700 1 -- [2001:620:0:6::102]:6804/4426 --> [2001:620:0:6::106]:6828/1005241 -- pg_notify(1.736(34) epoch 71024) v4 -- ?+0 0x207dcc40 con 0x2220b760 -107> 2013-09-27 15:39:17.057219 7f1da679c700 1 -- [2001:620:0:6::102]:6804/4426 --> [2001:620:0:6::106]:6828/1005241 -- pg_query(2.d54 epoch 71024) v2 -- ?+0 0x24f61680 con 0x2220b760 -106> 2013-09-27 15:39:17.057328 7f1da679c700 1 osd.21 pg_epoch: 71024 pg[1.d55( v 14626'30 (0'0,14626'30] local-les=71011 n=0 ec=1 les/c 71011/71011 71023/71023/69905) [21,16] r=0 lpr=71023 pi=71010-71022/1 lcod 0'0 mlcod 0'0 peering] state: Peering, affected_by_map, going to Reset -105> 2013-09-27 15:39:17.057388 7f1da679c700 1 osd.21 pg_epoch: 71024 pg[1.d55( v 14626'30 (0'0,14626'30] local-les=71011 n=0 ec=1 les/c 71011/71011 71024/71024/69905) [21,32,16] r=0 lpr=71024 pi=71010-71023/2 lcod 0'0 mlcod 0'0 inactive] state: transitioning to Primary -104> 2013-09-27 15:39:17.057467 7f1da679c700 1 -- [2001:620:0:6::102]:6804/4426 --> [2001:620:0:6::110]:6802/27723 -- pg_query(1.d55 epoch 71024) v2 -- ?+0 0x24f61860 con 0x1c75c840 -103> 2013-09-27 15:39:17.057490 7f1da679c700 1 -- [2001:620:0:6::102]:6804/4426 --> [2001:620:0:6::106]:6828/1005241 -- pg_query(1.d55 epoch 71024) v2 -- ?+0 0x24e343c0 con 0x2220b760 -102> 2013-09-27 15:39:17.057506 7f1da6f9d700 1 osd.21 pg_epoch: 71024 pg[0.737( v 28199'7760 (14513'4760,28199'7760] local-les=71011 n=2282 ec=1 les/c 71011/71011 71024/71024/71024) [32,53,21] r=2 lpr=71024 pi=35657-71023/170 lcod 0'0 inactive NOTIFY] state: transitioning to Stray -101> 2013-09-27 15:39:17.057660 7f1da6f9d700 1 -- [2001:620:0:6::102]:6804/4426 --> [2001:620:0:6::106]:6828/1005241 -- pg_notify(0.737(170) epoch 71024) v4 -- ?+0 0x23f9afc0 con 0x2220b760 -100> 2013-09-27 15:39:17.057711 7f1da679c700 1 osd.21 pg_epoch: 71024 pg[14.d48( v 69931'22 (0'0,69931'22] local-les=71011 n=16 ec=2815 les/c 71011/71011 71023/71023/69905) [21] r=0 lpr=71023 pi=71010-71022/1 mlcod 0'0 peering] state: Peering, affected_by_map, going to Reset -99> 2013-09-27 15:39:17.057765 7f1da679c700 1 osd.21 pg_epoch: 71024 pg[14.d48( v 69931'22 (0'0,69931'22] local-les=71011 n=16 ec=2815 les/c 71011/71011 71024/71024/69905) [21,32] r=0 lpr=71024 pi=71010-71023/2 mlcod 0'0 inactive] state: transitioning to Primary -98> 2013-09-27 15:39:17.057828 7f1da679c700 1 -- [2001:620:0:6::102]:6804/4426 --> [2001:620:0:6::106]:6828/1005241 -- pg_query(14.d48 epoch 71024) v2 -- ?+0 0x25928780 con 0x2220b760 -97> 2013-09-27 15:39:17.057878 7f1da6f9d700 10 monclient: _send_mon_message to mon.s2 at [2001:620:0:6::108]:6789/0 -96> 2013-09-27 15:39:17.057889 7f1da6f9d700 1 -- [2001:620:0:6::102]:6803/4426 --> [2001:620:0:6::108]:6789/0 -- osd_alive(want up_thru 71024 have 71024) v1 -- ?+0 0x21f9a8c0 con 0x20e16160 -95> 2013-09-27 15:39:17.057908 7f1da6f9d700 10 monclient: _send_mon_message to mon.s2 at [2001:620:0:6::108]:6789/0 -94> 2013-09-27 15:39:17.057911 7f1da6f9d700 1 -- [2001:620:0:6::102]:6803/4426 --> [2001:620:0:6::108]:6789/0 -- osd_pgtemp(e71024 {0.a30=[21,34,1,57]} v71024) v1 -- ?+0 0x1f60e5a0 con 0x20e16160 -93> 2013-09-27 15:39:17.057931 7f1da679c700 10 monclient: _send_mon_message to mon.s2 at [2001:620:0:6::108]:6789/0 -92> 2013-09-27 15:39:17.057941 7f1da679c700 1 -- [2001:620:0:6::102]:6803/4426 --> [2001:620:0:6::108]:6789/0 -- osd_pgtemp(e71024 {0.a30=[21,34,1,57]} v71024) v1 -- ?+0 0x24f61a40 con 0x20e16160 -91> 2013-09-27 15:39:17.058423 7f1daa7a4700 1 -- [2001:620:0:6::102]:6804/4426 <== osd.16 [2001:620:0:6::110]:6802/27723 4114 ==== pg_notify(0.d56(126) epoch 71024) v4 ==== 6714+0+0 (3519142453 0 0) 0x1f014380 con 0x1c75c840 -90> 2013-09-27 15:39:17.058447 7f1daa7a4700 5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 479849, time: 2013-09-27 15:39:17.053448, event: header_read, request: pg_notify(0.d56(126) epoch 71024) v4 -89> 2013-09-27 15:39:17.058462 7f1daa7a4700 5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 479849, time: 2013-09-27 15:39:17.053450, event: throttled, request: pg_notify(0.d56(126) epoch 71024) v4 -88> 2013-09-27 15:39:17.058475 7f1daa7a4700 5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 479849, time: 2013-09-27 15:39:17.053681, event: all_read, request: pg_notify(0.d56(126) epoch 71024) v4 -87> 2013-09-27 15:39:17.058486 7f1daa7a4700 5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 479849, time: 2013-09-27 15:39:17.058443, event: dispatched, request: pg_notify(0.d56(126) epoch 71024) v4 -86> 2013-09-27 15:39:17.058499 7f1daa7a4700 5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 479849, time: 2013-09-27 15:39:17.058499, event: waiting_for_osdmap, request: pg_notify(0.d56(126) epoch 71024) v4 -85> 2013-09-27 15:39:17.058504 7f1da6f9d700 10 monclient: _send_mon_message to mon.s2 at [2001:620:0:6::108]:6789/0 -84> 2013-09-27 15:39:17.058513 7f1daa7a4700 5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 479849, time: 2013-09-27 15:39:17.058513, event: started, request: pg_notify(0.d56(126) epoch 71024) v4 -83> 2013-09-27 15:39:17.058516 7f1da6f9d700 1 -- [2001:620:0:6::102]:6803/4426 --> [2001:620:0:6::108]:6789/0 -- osd_pgtemp(e71024 {0.a30=[21,34,1,57]} v71024) v1 -- ?+0 0x1f60e1e0 con 0x20e16160 -82> 2013-09-27 15:39:17.058545 7f1daa7a4700 5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 479849, time: 2013-09-27 15:39:17.058544, event: done, request: pg_notify(0.d56(126) epoch 71024) v4 -81> 2013-09-27 15:39:17.058551 7f1da679c700 10 monclient: _send_mon_message to mon.s2 at [2001:620:0:6::108]:6789/0 -80> 2013-09-27 15:39:17.058561 7f1da679c700 1 -- [2001:620:0:6::102]:6803/4426 --> [2001:620:0:6::108]:6789/0 -- osd_pgtemp(e71024 {0.a30=[21,34,1,57]} v71024) v1 -- ?+0 0x1f60eb40 con 0x20e16160 -79> 2013-09-27 15:39:17.058624 7f1daa7a4700 1 -- [2001:620:0:6::102]:6804/4426 <== osd.16 [2001:620:0:6::110]:6802/27723 4115 ==== pg_notify(1.d55(10) epoch 71024) v4 ==== 1078+0+0 (3557912638 0 0) 0x226e6c40 con 0x1c75c840 -78> 2013-09-27 15:39:17.058645 7f1daa7a4700 5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 479850, time: 2013-09-27 15:39:17.053705, event: header_read, request: pg_notify(1.d55(10) epoch 71024) v4 -77> 2013-09-27 15:39:17.058657 7f1daa7a4700 5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 479850, time: 2013-09-27 15:39:17.053706, event: throttled, request: pg_notify(1.d55(10) epoch 71024) v4 -76> 2013-09-27 15:39:17.058669 7f1daa7a4700 5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 479850, time: 2013-09-27 15:39:17.053777, event: all_read, request: pg_notify(1.d55(10) epoch 71024) v4 -75> 2013-09-27 15:39:17.058681 7f1daa7a4700 5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 479850, time: 2013-09-27 15:39:17.058642, event: dispatched, request: pg_notify(1.d55(10) epoch 71024) v4 -74> 2013-09-27 15:39:17.058693 7f1daa7a4700 5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 479850, time: 2013-09-27 15:39:17.058693, event: waiting_for_osdmap, request: pg_notify(1.d55(10) epoch 71024) v4 -73> 2013-09-27 15:39:17.058707 7f1daa7a4700 5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 479850, time: 2013-09-27 15:39:17.058707, event: started, request: pg_notify(1.d55(10) epoch 71024) v4 -72> 2013-09-27 15:39:17.058734 7f1daa7a4700 5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 479850, time: 2013-09-27 15:39:17.058734, event: done, request: pg_notify(1.d55(10) epoch 71024) v4 -71> 2013-09-27 15:39:17.059093 7f1da679c700 10 monclient: _send_mon_message to mon.s2 at [2001:620:0:6::108]:6789/0 -70> 2013-09-27 15:39:17.059110 7f1da679c700 1 -- [2001:620:0:6::102]:6803/4426 --> [2001:620:0:6::108]:6789/0 -- osd_pgtemp(e71024 {0.a30=[21,34,1,57]} v71024) v1 -- ?+0 0x24e34f00 con 0x20e16160 -69> 2013-09-27 15:39:17.059132 7f1da6f9d700 10 monclient: _send_mon_message to mon.s2 at [2001:620:0:6::108]:6789/0 -68> 2013-09-27 15:39:17.059150 7f1da6f9d700 1 -- [2001:620:0:6::102]:6803/4426 --> [2001:620:0:6::108]:6789/0 -- osd_pgtemp(e71024 {0.a30=[21,34,1,57]} v71024) v1 -- ?+0 0x1f60e000 con 0x20e16160 -67> 2013-09-27 15:39:17.059222 7f1da6f9d700 1 osd.21 pg_epoch: 71024 pg[11.98( v 69155'5271 (20495'2269,69155'5271] local-les=71011 n=149 ec=106 les/c 71011/71015 71023/71023/71023) [21] r=0 lpr=71023 pi=70989-71022/2 lcod 0'0 mlcod 0'0 peering] state: Peering, affected_by_map, going to Reset -66> 2013-09-27 15:39:17.059247 7f1da679c700 1 osd.21 pg_epoch: 71024 pg[2.a1( v 20483'157 (0'0,20483'157] local-les=71011 n=3 ec=1 les/c 71011/71012 71023/71023/71023) [21] r=0 lpr=71023 pi=70989-71022/2 lcod 0'0 mlcod 0'0 peering] state: Peering, affected_by_map, going to Reset -65> 2013-09-27 15:39:17.059307 7f1da679c700 1 osd.21 pg_epoch: 71024 pg[2.a1( v 20483'157 (0'0,20483'157] local-les=71011 n=3 ec=1 les/c 71011/71012 71024/71024/71024) [32,21] r=1 lpr=71024 pi=70989-71023/3 lcod 0'0 inactive NOTIFY] state: transitioning to Stray -64> 2013-09-27 15:39:17.059365 7f1da679c700 1 -- [2001:620:0:6::102]:6804/4426 --> [2001:620:0:6::106]:6828/1005241 -- pg_notify(2.a1(3) epoch 71024) v4 -- ?+0 0x2032c000 con 0x2220b760 -63> 2013-09-27 15:39:17.059305 7f1da6f9d700 1 osd.21 pg_epoch: 71024 pg[11.98( v 69155'5271 (20495'2269,69155'5271] local-les=71011 n=149 ec=106 les/c 71011/71015 71024/71024/71024) [32,21] r=1 lpr=71024 pi=70989-71023/3 lcod 0'0 inactive NOTIFY] state: transitioning to Stray -62> 2013-09-27 15:39:17.059422 7f1da679c700 1 osd.21 pg_epoch: 71024 pg[1.a2( v 69226'1769 (0'0,69226'1769] local-les=71011 n=0 ec=1 les/c 71011/71014 71023/71023/71023) [21,8] r=0 lpr=71023 pi=70989-71022/2 lcod 0'0 mlcod 0'0 peering] state: Peering, affected_by_map, going to Reset -61> 2013-09-27 15:39:17.059441 7f1da6f9d700 1 -- [2001:620:0:6::102]:6804/4426 --> [2001:620:0:6::106]:6828/1005241 -- pg_notify(11.98(3) epoch 71024) v4 -- ?+0 0x255a8c40 con 0x2220b760 -60> 2013-09-27 15:39:17.059481 7f1da679c700 1 osd.21 pg_epoch: 71024 pg[1.a2( v 69226'1769 (0'0,69226'1769] local-les=71011 n=0 ec=1 les/c 71011/71014 71024/71024/71024) [32,21,8] r=1 lpr=71024 pi=70989-71023/3 lcod 0'0 inactive NOTIFY] state: transitioning to Stray -59> 2013-09-27 15:39:17.059531 7f1da679c700 1 -- [2001:620:0:6::102]:6804/4426 --> [2001:620:0:6::106]:6828/1005241 -- pg_notify(1.a2(3) epoch 71024) v4 -- ?+0 0x2297a540 con 0x2220b760 -58> 2013-09-27 15:39:17.059572 7f1da679c700 1 osd.21 pg_epoch: 71024 pg[0.a3( v 28199'7867 (14513'4867,28199'7867] local-les=71011 n=2287 ec=1 les/c 71011/71014 71023/71023/71023) [21,8] r=0 lpr=71023 pi=70989-71022/2 lcod 0'0 mlcod 0'0 peering] state: Peering, affected_by_map, going to Reset -57> 2013-09-27 15:39:17.059661 7f1da679c700 1 osd.21 pg_epoch: 71024 pg[0.a3( v 28199'7867 (14513'4867,28199'7867] local-les=71011 n=2287 ec=1 les/c 71011/71014 71024/71024/71024) [32,21,8] r=1 lpr=71024 pi=70989-71023/3 lcod 0'0 inactive NOTIFY] state: transitioning to Stray -56> 2013-09-27 15:39:17.059708 7f1da679c700 1 -- [2001:620:0:6::102]:6804/4426 --> [2001:620:0:6::106]:6828/1005241 -- pg_notify(0.a3(3) epoch 71024) v4 -- ?+0 0x1e683340 con 0x2220b760 -55> 2013-09-27 15:39:17.059864 7f1dab7a6700 1 -- [2001:620:0:6::102]:6803/4426 <== mon.1 [2001:620:0:6::108]:6789/0 230 ==== osd_map(71018..71024 src has 70160..71024) v3 ==== 8466+0+0 (903183566 0 0) 0x1e98c000 con 0x20e16160 -54> 2013-09-27 15:39:17.059878 7f1da679c700 1 osd.21 pg_epoch: 71024 pg[14.95( v 53228'22 (0'0,53228'22] local-les=71011 n=16 ec=2815 les/c 71011/71012 71023/71023/71023) [21] r=0 lpr=71023 pi=70989-71022/2 lcod 0'0 mlcod 0'0 peering] state: Peering, affected_by_map, going to Reset -53> 2013-09-27 15:39:17.059896 7f1dab7a6700 3 osd.21 71024 handle_osd_map epochs [71018,71024], i have 71024, src has [70160,71024] -52> 2013-09-27 15:39:17.059960 7f1da679c700 1 osd.21 pg_epoch: 71024 pg[14.95( v 53228'22 (0'0,53228'22] local-les=71011 n=16 ec=2815 les/c 71011/71012 71024/71024/71024) [32,21] r=1 lpr=71024 pi=70989-71023/3 lcod 0'0 inactive NOTIFY] state: transitioning to Stray -51> 2013-09-27 15:39:17.059995 7f1da6f9d700 10 monclient: _send_mon_message to mon.s2 at [2001:620:0:6::108]:6789/0 -50> 2013-09-27 15:39:17.060012 7f1da6f9d700 1 -- [2001:620:0:6::102]:6803/4426 --> [2001:620:0:6::108]:6789/0 -- osd_pgtemp(e71024 {0.a30=[21,34,1,57]} v71024) v1 -- ?+0 0x1f60e3c0 con 0x20e16160 -49> 2013-09-27 15:39:17.060026 7f1da679c700 1 -- [2001:620:0:6::102]:6804/4426 --> [2001:620:0:6::106]:6828/1005241 -- pg_notify(14.95(3) epoch 71024) v4 -- ?+0 0x22a49340 con 0x2220b760 -48> 2013-09-27 15:39:17.060327 7f1da679c700 10 monclient: _send_mon_message to mon.s2 at [2001:620:0:6::108]:6789/0 -47> 2013-09-27 15:39:17.060340 7f1da679c700 1 -- [2001:620:0:6::102]:6803/4426 --> [2001:620:0:6::108]:6789/0 -- osd_pgtemp(e71024 {0.a30=[21,34,1,57]} v71024) v1 -- ?+0 0x1f60f2c0 con 0x20e16160 -46> 2013-09-27 15:39:17.060355 7f1da6f9d700 1 osd.21 pg_epoch: 71024 pg[17.92( empty local-les=71011 n=0 ec=48929 les/c 71011/71014 71023/71023/71023) [21] r=0 lpr=71023 pi=70989-71022/2 mlcod 0'0 peering] state: Peering, affected_by_map, going to Reset -45> 2013-09-27 15:39:17.060409 7f1da6f9d700 1 osd.21 pg_epoch: 71024 pg[17.92( empty local-les=71011 n=0 ec=48929 les/c 71011/71014 71024/71024/71024) [32,21] r=1 lpr=71024 pi=70989-71023/3 inactive NOTIFY] state: transitioning to Stray -44> 2013-09-27 15:39:17.060456 7f1da6f9d700 1 -- [2001:620:0:6::102]:6804/4426 --> [2001:620:0:6::106]:6828/1005241 -- pg_notify(17.92(3) epoch 71024) v4 -- ?+0 0x21671340 con 0x2220b760 -43> 2013-09-27 15:39:17.060762 7f1da679c700 10 monclient: _send_mon_message to mon.s2 at [2001:620:0:6::108]:6789/0 -42> 2013-09-27 15:39:17.060776 7f1da679c700 1 -- [2001:620:0:6::102]:6803/4426 --> [2001:620:0:6::108]:6789/0 -- osd_pgtemp(e71024 {0.a30=[21,34,1,57]} v71024) v1 -- ?+0 0x1f60ef00 con 0x20e16160 -41> 2013-09-27 15:39:17.060813 7f1da6f9d700 10 monclient: _send_mon_message to mon.s2 at [2001:620:0:6::108]:6789/0 -40> 2013-09-27 15:39:17.060828 7f1da6f9d700 1 -- [2001:620:0:6::102]:6803/4426 --> [2001:620:0:6::108]:6789/0 -- osd_pgtemp(e71024 {0.a30=[21,34,1,57]} v71024) v1 -- ?+0 0x1e633680 con 0x20e16160 -39> 2013-09-27 15:39:17.061202 7f1d980e7700 2 -- [2001:620:0:6::102]:6804/4426 >> [2001:620:0:6::106]:6828/1005241 pipe(0x1f901900 sd=77 :40803 s=1 pgs=0 cs=0 l=0). got newly_acked_seq 61 vs out_seq 0 -38> 2013-09-27 15:39:17.061234 7f1d980e7700 2 -- [2001:620:0:6::102]:6804/4426 >> [2001:620:0:6::106]:6828/1005241 pipe(0x1f901900 sd=77 :40803 s=1 pgs=0 cs=0 l=0). discarding previously sent 0 osd_map(71023..71024 src has 64713..71024) v3 -37> 2013-09-27 15:39:17.061259 7f1da679c700 10 monclient: _send_mon_message to mon.s2 at [2001:620:0:6::108]:6789/0 -36> 2013-09-27 15:39:17.061259 7f1d980e7700 2 -- [2001:620:0:6::102]:6804/4426 >> [2001:620:0:6::106]:6828/1005241 pipe(0x1f901900 sd=77 :40803 s=1 pgs=0 cs=0 l=0). discarding previously sent 0 osd_map(71023..71024 src has 64713..71024) v3 -35> 2013-09-27 15:39:17.061273 7f1da679c700 1 -- [2001:620:0:6::102]:6803/4426 --> [2001:620:0:6::108]:6789/0 -- osd_pgtemp(e71024 {0.a30=[21,34,1,57]} v71024) v1 -- ?+0 0x25928b40 con 0x20e16160 -34> 2013-09-27 15:39:17.061276 7f1d980e7700 2 -- [2001:620:0:6::102]:6804/4426 >> [2001:620:0:6::106]:6828/1005241 pipe(0x1f901900 sd=77 :40803 s=1 pgs=0 cs=0 l=0). discarding previously sent 0 pg_query(11.d4b epoch 71024) v2 -33> 2013-09-27 15:39:17.061336 7f1d980e7700 2 -- [2001:620:0:6::102]:6804/4426 >> [2001:620:0:6::106]:6828/1005241 pipe(0x1f901900 sd=77 :40803 s=1 pgs=0 cs=0 l=0). discarding previously sent 0 pg_query(0.d56 epoch 71024) v2 -32> 2013-09-27 15:39:17.061389 7f1d980e7700 2 -- [2001:620:0:6::102]:6804/4426 >> [2001:620:0:6::106]:6828/1005241 pipe(0x1f901900 sd=77 :40803 s=1 pgs=0 cs=0 l=0). discarding previously sent 0 pg_notify(1.736(34) epoch 71024) v4 -31> 2013-09-27 15:39:17.061489 7f1d980e7700 2 -- [2001:620:0:6::102]:6804/4426 >> [2001:620:0:6::106]:6828/1005241 pipe(0x1f901900 sd=77 :40803 s=1 pgs=0 cs=0 l=0). discarding previously sent 0 pg_query(2.d54 epoch 71024) v2 -30> 2013-09-27 15:39:17.061513 7f1da6f9d700 10 monclient: _send_mon_message to mon.s2 at [2001:620:0:6::108]:6789/0 -29> 2013-09-27 15:39:17.061511 7f1d980e7700 2 -- [2001:620:0:6::102]:6804/4426 >> [2001:620:0:6::106]:6828/1005241 pipe(0x1f901900 sd=77 :40803 s=1 pgs=0 cs=0 l=0). discarding previously sent 0 pg_query(1.d55 epoch 71024) v2 -28> 2013-09-27 15:39:17.061546 7f1da6f9d700 1 -- [2001:620:0:6::102]:6803/4426 --> [2001:620:0:6::108]:6789/0 -- osd_pgtemp(e71024 {0.a30=[21,34,1,57]} v71024) v1 -- ?+0 0x1e633e00 con 0x20e16160 -27> 2013-09-27 15:39:17.061562 7f1d980e7700 2 -- [2001:620:0:6::102]:6804/4426 >> [2001:620:0:6::106]:6828/1005241 pipe(0x1f901900 sd=77 :40803 s=1 pgs=0 cs=0 l=0). discarding previously sent 0 pg_notify(0.737(170) epoch 71024) v4 -26> 2013-09-27 15:39:17.061575 7f1daa7a4700 1 -- [2001:620:0:6::102]:6804/4426 <== osd.35 [2001:620:0:6::106]:6810/2005491 39 ==== osd_map(71024..71024 src has 62296..71024) v3 ==== 3517+0+0 (2244786833 0 0) 0x20c0c000 con 0x1c855600 -25> 2013-09-27 15:39:17.061618 7f1daa7a4700 3 osd.21 71024 handle_osd_map epochs [71024,71024], i have 71024, src has [62296,71024] -24> 2013-09-27 15:39:17.061665 7f1daa7a4700 1 -- [2001:620:0:6::102]:6804/4426 <== osd.35 [2001:620:0:6::106]:6810/2005491 40 ==== pg_notify(0.a30(10) epoch 71024) v4 ==== 1106+0+0 (2059992953 0 0) 0x206628c0 con 0x1c855600 -23> 2013-09-27 15:39:17.061713 7f1d980e7700 2 -- [2001:620:0:6::102]:6804/4426 >> [2001:620:0:6::106]:6828/1005241 pipe(0x1f901900 sd=77 :40803 s=1 pgs=0 cs=0 l=0). discarding previously sent 0 pg_query(14.d48 epoch 71024) v2 -22> 2013-09-27 15:39:17.061736 7f1d980e7700 2 -- [2001:620:0:6::102]:6804/4426 >> [2001:620:0:6::106]:6828/1005241 pipe(0x1f901900 sd=77 :40803 s=1 pgs=0 cs=0 l=0). discarding previously sent 0 pg_notify(2.a1(3) epoch 71024) v4 -21> 2013-09-27 15:39:17.061717 7f1daa7a4700 5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 479851, time: 2013-09-27 15:39:17.061570, event: header_read, request: pg_notify(0.a30(10) epoch 71024) v4 -20> 2013-09-27 15:39:17.061750 7f1d980e7700 2 -- [2001:620:0:6::102]:6804/4426 >> [2001:620:0:6::106]:6828/1005241 pipe(0x1f901900 sd=77 :40803 s=1 pgs=0 cs=0 l=0). discarding previously sent 0 pg_notify(11.98(3) epoch 71024) v4 -19> 2013-09-27 15:39:17.061755 7f1daa7a4700 5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 479851, time: 2013-09-27 15:39:17.061571, event: throttled, request: pg_notify(0.a30(10) epoch 71024) v4 -18> 2013-09-27 15:39:17.061765 7f1d980e7700 2 -- [2001:620:0:6::102]:6804/4426 >> [2001:620:0:6::106]:6828/1005241 pipe(0x1f901900 sd=77 :40803 s=1 pgs=0 cs=0 l=0). discarding previously sent 0 pg_notify(1.a2(3) epoch 71024) v4 -17> 2013-09-27 15:39:17.061766 7f1daa7a4700 5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 479851, time: 2013-09-27 15:39:17.061645, event: all_read, request: pg_notify(0.a30(10) epoch 71024) v4 -16> 2013-09-27 15:39:17.061804 7f1daa7a4700 5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 479851, time: 2013-09-27 15:39:17.061687, event: dispatched, request: pg_notify(0.a30(10) epoch 71024) v4 -15> 2013-09-27 15:39:17.061807 7f1da679c700 10 monclient: _send_mon_message to mon.s2 at [2001:620:0:6::108]:6789/0 -14> 2013-09-27 15:39:17.061819 7f1daa7a4700 5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 479851, time: 2013-09-27 15:39:17.061819, event: waiting_for_osdmap, request: pg_notify(0.a30(10) epoch 71024) v4 -13> 2013-09-27 15:39:17.061819 7f1da679c700 1 -- [2001:620:0:6::102]:6803/4426 --> [2001:620:0:6::108]:6789/0 -- osd_pgtemp(e71024 {0.a30=[21,34,1,57]} v71024) v1 -- ?+0 0x1f60f4a0 con 0x20e16160 -12> 2013-09-27 15:39:17.061781 7f1d980e7700 2 -- [2001:620:0:6::102]:6804/4426 >> [2001:620:0:6::106]:6828/1005241 pipe(0x1f901900 sd=77 :40803 s=1 pgs=0 cs=0 l=0). discarding previously sent 0 pg_notify(0.a3(3) epoch 71024) v4 -11> 2013-09-27 15:39:17.061925 7f1d980e7700 2 -- [2001:620:0:6::102]:6804/4426 >> [2001:620:0:6::106]:6828/1005241 pipe(0x1f901900 sd=77 :40803 s=1 pgs=0 cs=0 l=0). discarding previously sent 0 pg_notify(14.95(3) epoch 71024) v4 -10> 2013-09-27 15:39:17.061966 7f1d980e7700 2 -- [2001:620:0:6::102]:6804/4426 >> [2001:620:0:6::106]:6828/1005241 pipe(0x1f901900 sd=77 :40803 s=1 pgs=0 cs=0 l=0). discarding previously sent 0 pg_notify(17.92(3) epoch 71024) v4 -9> 2013-09-27 15:39:17.062172 7f1da6f9d700 10 monclient: _send_mon_message to mon.s2 at [2001:620:0:6::108]:6789/0 -8> 2013-09-27 15:39:17.062190 7f1da6f9d700 1 -- [2001:620:0:6::102]:6803/4426 --> [2001:620:0:6::108]:6789/0 -- osd_pgtemp(e71024 {0.a30=[21,34,1,57]} v71024) v1 -- ?+0 0x1e633a40 con 0x20e16160 -7> 2013-09-27 15:39:17.062245 7f1daa7a4700 5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 479851, time: 2013-09-27 15:39:17.062244, event: started, request: pg_notify(0.a30(10) epoch 71024) v4 -6> 2013-09-27 15:39:17.062270 7f1da6f9d700 1 osd.21 pg_epoch: 71024 pg[14.b43( v 53228'13 (0'0,53228'13] local-les=71011 n=11 ec=2815 les/c 71011/71016 71023/71023/71023) [21] r=0 lpr=71023 pi=70989-71022/2 lcod 0'0 mlcod 0'0 peering] state: Peering, affected_by_map, going to Reset -5> 2013-09-27 15:39:17.062279 7f1da679c700 1 osd.21 pg_epoch: 71024 pg[1.b50( v 25179'931 (0'0,25179'931] local-les=71011 n=0 ec=1 les/c 71011/71016 71023/71023/71023) [21,14] r=0 lpr=71023 pi=70989-71022/2 lcod 0'0 mlcod 0'0 peering] state: Peering, affected_by_map, going to Reset -4> 2013-09-27 15:39:17.062339 7f1daa7a4700 5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 479851, time: 2013-09-27 15:39:17.062338, event: done, request: pg_notify(0.a30(10) epoch 71024) v4 -3> 2013-09-27 15:39:17.062354 7f1da679c700 1 osd.21 pg_epoch: 71024 pg[1.b50( v 25179'931 (0'0,25179'931] local-les=71011 n=0 ec=1 les/c 71011/71016 71024/71024/71024) [32,21,14] r=1 lpr=71024 pi=70989-71023/3 lcod 0'0 inactive NOTIFY] state: transitioning to Stray -2> 2013-09-27 15:39:17.062374 7f1daa7a4700 1 -- [2001:620:0:6::102]:6804/4426 <== osd.34 [2001:620:0:6::106]:6816/5408 48 ==== osd_map(71024..71024 src has 61209..71024) v3 ==== 3517+0+0 (2707869344 0 0) 0x26aba240 con 0x20c59080 -1> 2013-09-27 15:39:17.062382 7f1da6f9d700 1 osd.21 pg_epoch: 71024 pg[14.b43( v 53228'13 (0'0,53228'13] local-les=71011 n=11 ec=2815 les/c 71011/71016 71024/71024/71024) [32,21] r=1 lpr=71024 pi=70989-71023/3 lcod 0'0 inactive NOTIFY] state: transitioning to Stray 0> 2013-09-27 15:39:17.063258 7f1d980e7700 -1 msg/Pipe.cc: In function 'int Pipe::connect()' thread 7f1d980e7700 time 2013-09-27 15:39:17.061999 msg/Pipe.cc: 1029: FAILED assert(m) ceph version 0.61.8 (a6fdcca3bddbc9f177e4e2bf0d9cdd85006b028b) 1: (Pipe::connect()+0x39d1) [0x9a6821] 2: (Pipe::writer()+0xb15) [0x9a76c5] 3: (Pipe::Writer::entry()+0xd) [0x9b177d] 4: (()+0x7f8e) [0x7f1dfa4bdf8e] 5: (clone()+0x6d) [0x7f1df876fe1d] NOTE: a copy of the executable, or `objdump -rdS ` is needed to interpret this. --- logging levels --- 0/ 5 none 0/ 1 lockdep 0/ 1 context 1/ 1 crush 1/ 5 mds 1/ 5 mds_balancer 1/ 5 mds_locker 1/ 5 mds_log 1/ 5 mds_log_expire 1/ 5 mds_migrator 0/ 1 buffer 0/ 1 timer 0/ 1 filer 0/ 1 striper 0/ 1 objecter 0/ 5 rados 0/ 5 rbd 0/ 5 journaler 0/ 5 objectcacher 0/ 5 client 0/ 5 osd 0/ 5 optracker 0/ 5 objclass 1/ 3 filestore 1/ 3 journal 0/ 5 ms 1/ 5 mon 0/10 monc 0/ 5 paxos 0/ 5 tp 1/ 5 auth 1/ 5 crypto 1/ 1 finisher 1/ 5 heartbeatmap 1/ 5 perfcounter 1/ 5 rgw 1/ 5 hadoop 1/ 5 javaclient 1/ 5 asok 1/ 1 throttle -2/-2 (syslog threshold) -1/-1 (stderr threshold) max_recent 10000 max_new 1000 log_file /var/log/ceph/ceph-osd.21.log --- end dump of recent events --- 2013-09-27 15:39:17.356484 7f1d980e7700 -1 *** Caught signal (Aborted) ** in thread 7f1d980e7700 ceph version 0.61.8 (a6fdcca3bddbc9f177e4e2bf0d9cdd85006b028b) 1: /usr/bin/ceph-osd() [0x8452f0] 2: (()+0xfbd0) [0x7f1dfa4c5bd0] 3: (gsignal()+0x37) [0x7f1df86ad037] 4: (abort()+0x148) [0x7f1df86b0698] 5: (__gnu_cxx::__verbose_terminate_handler()+0x11d) [0x7f1df8fb9e8d] 6: (()+0x5ef76) [0x7f1df8fb7f76] 7: (()+0x5efa3) [0x7f1df8fb7fa3] 8: (()+0x5f1de) [0x7f1df8fb81de] 9: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x43d) [0x8f1ead] 10: (Pipe::connect()+0x39d1) [0x9a6821] 11: (Pipe::writer()+0xb15) [0x9a76c5] 12: (Pipe::Writer::entry()+0xd) [0x9b177d] 13: (()+0x7f8e) [0x7f1dfa4bdf8e] 14: (clone()+0x6d) [0x7f1df876fe1d] NOTE: a copy of the executable, or `objdump -rdS ` is needed to interpret this. --- begin dump of recent events --- -65> 2013-09-27 15:39:17.062400 7f1daa7a4700 3 osd.21 71024 handle_osd_map epochs [71024,71024], i have 71024, src has [61209,71024] -64> 2013-09-27 15:39:17.065490 7f1daa7a4700 1 -- [2001:620:0:6::102]:6804/4426 <== osd.34 [2001:620:0:6::106]:6816/5408 49 ==== pg_notify(0.a30(56) epoch 71024) v4 ==== 3388+0+0 (2760852020 0 0) 0x1ef3ac40 con 0x20c59080 -63> 2013-09-27 15:39:17.065514 7f1daa7a4700 5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 479852, time: 2013-09-27 15:39:17.062269, event: header_read, request: pg_notify(0.a30(56) epoch 71024) v4 -62> 2013-09-27 15:39:17.065524 7f1daa7a4700 5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 479852, time: 2013-09-27 15:39:17.062270, event: throttled, request: pg_notify(0.a30(56) epoch 71024) v4 -61> 2013-09-27 15:39:17.065531 7f1daa7a4700 5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 479852, time: 2013-09-27 15:39:17.062385, event: all_read, request: pg_notify(0.a30(56) epoch 71024) v4 -60> 2013-09-27 15:39:17.065540 7f1daa7a4700 5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 479852, time: 2013-09-27 15:39:17.065511, event: dispatched, request: pg_notify(0.a30(56) epoch 71024) v4 -59> 2013-09-27 15:39:17.065547 7f1daa7a4700 5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 479852, time: 2013-09-27 15:39:17.065547, event: waiting_for_osdmap, request: pg_notify(0.a30(56) epoch 71024) v4 -58> 2013-09-27 15:39:17.065556 7f1daa7a4700 5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 479852, time: 2013-09-27 15:39:17.065556, event: started, request: pg_notify(0.a30(56) epoch 71024) v4 -57> 2013-09-27 15:39:17.065586 7f1daa7a4700 5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 479852, time: 2013-09-27 15:39:17.065585, event: done, request: pg_notify(0.a30(56) epoch 71024) v4 -56> 2013-09-27 15:39:17.068078 7f1daa7a4700 1 -- [2001:620:0:6::102]:6804/4426 <== osd.57 [2001:620:0:6::112]:6804/4664 6554 ==== osd_map(71024..71024 src has 64743..71024) v3 ==== 3517+0+0 (3541109980 0 0) 0x24545d40 con 0x1c75cf20 -55> 2013-09-27 15:39:17.068105 7f1daa7a4700 3 osd.21 71024 handle_osd_map epochs [71024,71024], i have 71024, src has [64743,71024] -54> 2013-09-27 15:39:17.068117 7f1daa7a4700 1 -- [2001:620:0:6::102]:6804/4426 <== osd.57 [2001:620:0:6::112]:6804/4664 6555 ==== pg_notify(0.a30(50) epoch 71024) v4 ==== 3090+0+0 (1711887934 0 0) 0x201c9340 con 0x1c75cf20 -53> 2013-09-27 15:39:17.068129 7f1daa7a4700 5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 479853, time: 2013-09-27 15:39:17.067962, event: header_read, request: pg_notify(0.a30(50) epoch 71024) v4 -52> 2013-09-27 15:39:17.068136 7f1daa7a4700 5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 479853, time: 2013-09-27 15:39:17.067963, event: throttled, request: pg_notify(0.a30(50) epoch 71024) v4 -51> 2013-09-27 15:39:17.068141 7f1daa7a4700 5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 479853, time: 2013-09-27 15:39:17.068062, event: all_read, request: pg_notify(0.a30(50) epoch 71024) v4 -50> 2013-09-27 15:39:17.068145 7f1daa7a4700 5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 479853, time: 2013-09-27 15:39:17.068127, event: dispatched, request: pg_notify(0.a30(50) epoch 71024) v4 -49> 2013-09-27 15:39:17.068150 7f1daa7a4700 5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 479853, time: 2013-09-27 15:39:17.068150, event: waiting_for_osdmap, request: pg_notify(0.a30(50) epoch 71024) v4 -48> 2013-09-27 15:39:17.068157 7f1daa7a4700 5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 479853, time: 2013-09-27 15:39:17.068157, event: started, request: pg_notify(0.a30(50) epoch 71024) v4 -47> 2013-09-27 15:39:17.068177 7f1daa7a4700 5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 479853, time: 2013-09-27 15:39:17.068177, event: done, request: pg_notify(0.a30(50) epoch 71024) v4 -46> 2013-09-27 15:39:17.068499 7f1daa7a4700 1 -- [2001:620:0:6::102]:6804/4426 <== osd.10 [2001:620:0:6::108]:6801/4423 1225 ==== osd_map(71024..71024 src has 64713..71024) v3 ==== 3517+0+0 (793712060 0 0) 0x2297cd80 con 0x1c6eec60 -45> 2013-09-27 15:39:17.068514 7f1daa7a4700 3 osd.21 71024 handle_osd_map epochs [71024,71024], i have 71024, src has [64713,71024] -44> 2013-09-27 15:39:17.068563 7f1daa7a4700 1 -- [2001:620:0:6::102]:6804/4426 <== osd.10 [2001:620:0:6::108]:6801/4423 1226 ==== pg_query(0.2ce,1.2cd epoch 71024) v2 ==== 262+0+0 (2924756561 0 0) 0x1e4c0960 con 0x1c6eec60 -43> 2013-09-27 15:39:17.068574 7f1daa7a4700 5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 479854, time: 2013-09-27 15:39:17.068514, event: header_read, request: pg_query(0.2ce,1.2cd epoch 71024) v2 -42> 2013-09-27 15:39:17.068580 7f1daa7a4700 5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 479854, time: 2013-09-27 15:39:17.068515, event: throttled, request: pg_query(0.2ce,1.2cd epoch 71024) v2 -41> 2013-09-27 15:39:17.068584 7f1daa7a4700 5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 479854, time: 2013-09-27 15:39:17.068548, event: all_read, request: pg_query(0.2ce,1.2cd epoch 71024) v2 -40> 2013-09-27 15:39:17.068588 7f1daa7a4700 5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 479854, time: 2013-09-27 15:39:17.068573, event: dispatched, request: pg_query(0.2ce,1.2cd epoch 71024) v2 -39> 2013-09-27 15:39:17.068592 7f1daa7a4700 5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 479854, time: 2013-09-27 15:39:17.068592, event: waiting_for_osdmap, request: pg_query(0.2ce,1.2cd epoch 71024) v2 -38> 2013-09-27 15:39:17.068598 7f1daa7a4700 5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 479854, time: 2013-09-27 15:39:17.068598, event: started, request: pg_query(0.2ce,1.2cd epoch 71024) v2 -37> 2013-09-27 15:39:17.068613 7f1daa7a4700 5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 479854, time: 2013-09-27 15:39:17.068613, event: done, request: pg_query(0.2ce,1.2cd epoch 71024) v2 -36> 2013-09-27 15:39:17.074709 7f1daa7a4700 1 -- [2001:620:0:6::102]:6804/4426 <== osd.16 [2001:620:0:6::110]:6802/27723 4116 ==== pg_notify(0.d56(126),1.d55(10) epoch 71024) v4 ==== 7780+0+0 (3203177835 0 0) 0x24e96540 con 0x1c75c840 -35> 2013-09-27 15:39:17.074731 7f1daa7a4700 5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 479855, time: 2013-09-27 15:39:17.074499, event: header_read, request: pg_notify(0.d56(126),1.d55(10) epoch 71024) v4 -34> 2013-09-27 15:39:17.074739 7f1daa7a4700 5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 479855, time: 2013-09-27 15:39:17.074500, event: throttled, request: pg_notify(0.d56(126),1.d55(10) epoch 71024) v4 -33> 2013-09-27 15:39:17.074744 7f1daa7a4700 5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 479855, time: 2013-09-27 15:39:17.074682, event: all_read, request: pg_notify(0.d56(126),1.d55(10) epoch 71024) v4 -32> 2013-09-27 15:39:17.074749 7f1daa7a4700 5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 479855, time: 2013-09-27 15:39:17.074728, event: dispatched, request: pg_notify(0.d56(126),1.d55(10) epoch 71024) v4 -31> 2013-09-27 15:39:17.074755 7f1daa7a4700 5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 479855, time: 2013-09-27 15:39:17.074754, event: waiting_for_osdmap, request: pg_notify(0.d56(126),1.d55(10) epoch 71024) v4 -30> 2013-09-27 15:39:17.074762 7f1daa7a4700 5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 479855, time: 2013-09-27 15:39:17.074762, event: started, request: pg_notify(0.d56(126),1.d55(10) epoch 71024) v4 -29> 2013-09-27 15:39:17.074787 7f1daa7a4700 5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 479855, time: 2013-09-27 15:39:17.074787, event: done, request: pg_notify(0.d56(126),1.d55(10) epoch 71024) v4 -28> 2013-09-27 15:39:17.076220 7f1da8fa1700 1 -- [2001:620:0:6::102]:6805/4426 <== osd.13 [2001:620:0:6::10c]:0/4804 28962 ==== osd_ping(ping e71024 stamp 2013-09-27 15:39:17.076181) v2 ==== 47+0+0 (3138295901 0 0) 0x25dac8c0 con 0x1cd9cf20 -27> 2013-09-27 15:39:17.076256 7f1da8fa1700 1 -- [2001:620:0:6::102]:6805/4426 --> [2001:620:0:6::10c]:0/4804 -- osd_ping(ping_reply e71024 stamp 2013-09-27 15:39:17.076181) v2 -- ?+0 0x1cb761c0 con 0x1cd9cf20 -26> 2013-09-27 15:39:17.077876 7f1daa7a4700 1 -- [2001:620:0:6::102]:6804/4426 <== osd.31 [2001:620:0:6::106]:6807/5158 138 ==== pg_notify(11.51d(25) epoch 71023) v4 ==== 1715+0+0 (784477437 0 0) 0x22d0c380 con 0x22eb6420 -25> 2013-09-27 15:39:17.077896 7f1daa7a4700 5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 479856, time: 2013-09-27 15:39:17.077754, event: header_read, request: pg_notify(11.51d(25) epoch 71023) v4 -24> 2013-09-27 15:39:17.077903 7f1daa7a4700 5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 479856, time: 2013-09-27 15:39:17.077756, event: throttled, request: pg_notify(11.51d(25) epoch 71023) v4 -23> 2013-09-27 15:39:17.077907 7f1daa7a4700 5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 479856, time: 2013-09-27 15:39:17.077845, event: all_read, request: pg_notify(11.51d(25) epoch 71023) v4 -22> 2013-09-27 15:39:17.077911 7f1daa7a4700 5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 479856, time: 2013-09-27 15:39:17.077893, event: dispatched, request: pg_notify(11.51d(25) epoch 71023) v4 -21> 2013-09-27 15:39:17.077916 7f1daa7a4700 5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 479856, time: 2013-09-27 15:39:17.077916, event: waiting_for_osdmap, request: pg_notify(11.51d(25) epoch 71023) v4 -20> 2013-09-27 15:39:17.077922 7f1daa7a4700 5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 479856, time: 2013-09-27 15:39:17.077922, event: started, request: pg_notify(11.51d(25) epoch 71023) v4 -19> 2013-09-27 15:39:17.077940 7f1daa7a4700 5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 479856, time: 2013-09-27 15:39:17.077939, event: done, request: pg_notify(11.51d(25) epoch 71023) v4 -18> 2013-09-27 15:39:17.078001 7f1daa7a4700 1 -- [2001:620:0:6::102]:6804/4426 <== osd.31 [2001:620:0:6::106]:6807/5158 139 ==== pg_notify(11.51d(25) epoch 71023) v4 ==== 1715+0+0 (784477437 0 0) 0x20ed8380 con 0x22eb6420 -17> 2013-09-27 15:39:17.078017 7f1daa7a4700 5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 479857, time: 2013-09-27 15:39:17.077896, event: header_read, request: pg_notify(11.51d(25) epoch 71023) v4 -16> 2013-09-27 15:39:17.078022 7f1daa7a4700 5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 479857, time: 2013-09-27 15:39:17.077897, event: throttled, request: pg_notify(11.51d(25) epoch 71023) v4 -15> 2013-09-27 15:39:17.078027 7f1daa7a4700 5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 479857, time: 2013-09-27 15:39:17.077959, event: all_read, request: pg_notify(11.51d(25) epoch 71023) v4 -14> 2013-09-27 15:39:17.078031 7f1daa7a4700 5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 479857, time: 2013-09-27 15:39:17.078015, event: dispatched, request: pg_notify(11.51d(25) epoch 71023) v4 -13> 2013-09-27 15:39:17.078036 7f1daa7a4700 5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 479857, time: 2013-09-27 15:39:17.078036, event: waiting_for_osdmap, request: pg_notify(11.51d(25) epoch 71023) v4 -12> 2013-09-27 15:39:17.078042 7f1daa7a4700 5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 479857, time: 2013-09-27 15:39:17.078041, event: started, request: pg_notify(11.51d(25) epoch 71023) v4 -11> 2013-09-27 15:39:17.078056 7f1daa7a4700 5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 479857, time: 2013-09-27 15:39:17.078056, event: done, request: pg_notify(11.51d(25) epoch 71023) v4 -10> 2013-09-27 15:39:17.103020 7f1daa7a4700 1 -- [2001:620:0:6::102]:6804/4426 <== osd.1 [2001:620:0:6::100]:6808/423 12822 ==== osd_map(71024..71024 src has 64803..71024) v3 ==== 3517+0+0 (3020881711 0 0) 0x24c438c0 con 0x1dc49760 -9> 2013-09-27 15:39:17.103045 7f1daa7a4700 3 osd.21 71024 handle_osd_map epochs [71024,71024], i have 71024, src has [64803,71024] -8> 2013-09-27 15:39:17.103297 7f1daa7a4700 1 -- [2001:620:0:6::102]:6804/4426 <== osd.1 [2001:620:0:6::100]:6808/423 12823 ==== pg_notify(0.a30(10) epoch 71024) v4 ==== 1106+0+0 (843226241 0 0) 0x21f6ac40 con 0x1dc49760 -7> 2013-09-27 15:39:17.103329 7f1daa7a4700 5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 479858, time: 2013-09-27 15:39:17.103208, event: header_read, request: pg_notify(0.a30(10) epoch 71024) v4 -6> 2013-09-27 15:39:17.103340 7f1daa7a4700 5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 479858, time: 2013-09-27 15:39:17.103210, event: throttled, request: pg_notify(0.a30(10) epoch 71024) v4 -5> 2013-09-27 15:39:17.103348 7f1daa7a4700 5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 479858, time: 2013-09-27 15:39:17.103271, event: all_read, request: pg_notify(0.a30(10) epoch 71024) v4 -4> 2013-09-27 15:39:17.103355 7f1daa7a4700 5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 479858, time: 2013-09-27 15:39:17.103326, event: dispatched, request: pg_notify(0.a30(10) epoch 71024) v4 -3> 2013-09-27 15:39:17.103362 7f1daa7a4700 5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 479858, time: 2013-09-27 15:39:17.103362, event: waiting_for_osdmap, request: pg_notify(0.a30(10) epoch 71024) v4 -2> 2013-09-27 15:39:17.103372 7f1daa7a4700 5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 479858, time: 2013-09-27 15:39:17.103372, event: started, request: pg_notify(0.a30(10) epoch 71024) v4 -1> 2013-09-27 15:39:17.103401 7f1daa7a4700 5 --OSD::tracker-- reqid: unknown.0.0:0, seq: 479858, time: 2013-09-27 15:39:17.103401, event: done, request: pg_notify(0.a30(10) epoch 71024) v4 0> 2013-09-27 15:39:17.356484 7f1d980e7700 -1 *** Caught signal (Aborted) ** in thread 7f1d980e7700 ceph version 0.61.8 (a6fdcca3bddbc9f177e4e2bf0d9cdd85006b028b) 1: /usr/bin/ceph-osd() [0x8452f0] 2: (()+0xfbd0) [0x7f1dfa4c5bd0] 3: (gsignal()+0x37) [0x7f1df86ad037] 4: (abort()+0x148) [0x7f1df86b0698] 5: (__gnu_cxx::__verbose_terminate_handler()+0x11d) [0x7f1df8fb9e8d] 6: (()+0x5ef76) [0x7f1df8fb7f76] 7: (()+0x5efa3) [0x7f1df8fb7fa3] 8: (()+0x5f1de) [0x7f1df8fb81de] 9: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x43d) [0x8f1ead] 10: (Pipe::connect()+0x39d1) [0x9a6821] 11: (Pipe::writer()+0xb15) [0x9a76c5] 12: (Pipe::Writer::entry()+0xd) [0x9b177d] 13: (()+0x7f8e) [0x7f1dfa4bdf8e] 14: (clone()+0x6d) [0x7f1df876fe1d] NOTE: a copy of the executable, or `objdump -rdS ` is needed to interpret this. --- logging levels --- 0/ 5 none 0/ 1 lockdep 0/ 1 context 1/ 1 crush 1/ 5 mds 1/ 5 mds_balancer 1/ 5 mds_locker 1/ 5 mds_log 1/ 5 mds_log_expire 1/ 5 mds_migrator 0/ 1 buffer 0/ 1 timer 0/ 1 filer 0/ 1 striper 0/ 1 objecter 0/ 5 rados 0/ 5 rbd 0/ 5 journaler 0/ 5 objectcacher 0/ 5 client 0/ 5 osd 0/ 5 optracker 0/ 5 objclass 1/ 3 filestore 1/ 3 journal 0/ 5 ms 1/ 5 mon 0/10 monc 0/ 5 paxos 0/ 5 tp 1/ 5 auth 1/ 5 crypto 1/ 1 finisher 1/ 5 heartbeatmap 1/ 5 perfcounter 1/ 5 rgw 1/ 5 hadoop 1/ 5 javaclient 1/ 5 asok 1/ 1 throttle -2/-2 (syslog threshold) -1/-1 (stderr threshold) max_recent 10000 max_new 1000 log_file /var/log/ceph/ceph-osd.21.log --- end dump of recent events ---