10.06.01_22:25:06.461103 --- opened log /var/log/ceph/r1-9.6204 --- ceph version 0.21~rc () 10.06.01_22:25:06.461354 7f018d385720 -- 0.0.0.0:6800/6204 accepter.bind ms_addr is 0.0.0.0:6800/6204 need_addr=1 10.06.01_22:25:06.461400 7f018d385720 -- 0.0.0.0:6800/6204 messenger.start 10.06.01_22:25:06.461409 7f018d385720 -- 0.0.0.0:6800/6204 messenger.start daemonizing 10.06.01_22:25:06.461711 7f018d385720 ---- renamed log /var/log/ceph/r1-9.6204 -> /var/log/ceph/r1-9.6205 ---- 10.06.01_22:25:06.461820 7f018d385720 -- 0.0.0.0:6800/6204 accepter.start 10.06.01_22:25:06.461988 7f018d385720 mds-1.0 120 MDSCacheObject 10.06.01_22:25:06.461999 7f018d385720 mds-1.0 1568 CInode 10.06.01_22:25:06.462005 7f018d385720 mds-1.0 16 elist<>::item *7=112 10.06.01_22:25:06.462011 7f018d385720 mds-1.0 352 inode_t 10.06.01_22:25:06.462017 7f018d385720 mds-1.0 56 nest_info_t 10.06.01_22:25:06.462022 7f018d385720 mds-1.0 32 frag_info_t 10.06.01_22:25:06.462028 7f018d385720 mds-1.0 40 SimpleLock *5=200 10.06.01_22:25:06.462034 7f018d385720 mds-1.0 48 ScatterLock *3=144 10.06.01_22:25:06.462040 7f018d385720 mds-1.0 416 CDentry 10.06.01_22:25:06.462045 7f018d385720 mds-1.0 16 elist<>::item 10.06.01_22:25:06.462053 7f018d385720 mds-1.0 40 SimpleLock 10.06.01_22:25:06.462058 7f018d385720 mds-1.0 1480 CDir 10.06.01_22:25:06.462064 7f018d385720 mds-1.0 16 elist<>::item *2=32 10.06.01_22:25:06.462070 7f018d385720 mds-1.0 192 fnode_t 10.06.01_22:25:06.462076 7f018d385720 mds-1.0 56 nest_info_t *2 10.06.01_22:25:06.462081 7f018d385720 mds-1.0 32 frag_info_t *2 10.06.01_22:25:06.462087 7f018d385720 mds-1.0 168 Capability 10.06.01_22:25:06.462092 7f018d385720 mds-1.0 32 xlist<>::item *2=64 10.06.01_22:25:06.462163 7f018d385720 ---- renamed symlink /var/log/ceph/mds.r1-9.0 -> /var/log/ceph/mds.r1-9.1 ---- 10.06.01_22:25:06.462192 7f018d385720 ---- renamed symlink /var/log/ceph/mds.r1-9 -> /var/log/ceph/mds.r1-9.0 ---- 10.06.01_22:25:06.462226 7f018d385720 ---- created symlink /var/log/ceph/mds.r1-9 -> r1-9.6205 ---- 10.06.01_22:25:06.462343 7f018d385720 -- 0.0.0.0:6800/6204 --> mon1 172.16.2.10:6789/0 -- auth(proto 0 29 bytes) v1 -- ?+0 0x106fb50 10.06.01_22:25:06.462753 7f018a475710 -- 172.16.2.9:6800/6204 learned my addr 172.16.2.9:6800/6204 10.06.01_22:25:06.462786 7f018a475710 mds-1.0 MDS::ms_get_authorizer type=mon 10.06.01_22:25:06.462938 7f018b477710 mds-1.0 ms_handle_connect on 172.16.2.10:6789/0 10.06.01_22:25:06.463229 7f018b477710 -- 172.16.2.9:6800/6204 <== mon1 172.16.2.10:6789/0 1 ==== auth_reply(proto 1 0 Success) v1 ==== 24+0+0 (2348874389 0 0) 0x106fb50 10.06.01_22:25:06.463280 7f018b477710 monclient(hunting): found mon1 10.06.01_22:25:06.463289 7f018b477710 -- 172.16.2.9:6800/6204 --> mon1 172.16.2.10:6789/0 -- mon_subscribe({monmap=0+}) v1 -- ?+0 0x10768d0 10.06.01_22:25:06.463338 7f018d385720 mds-1.0 beacon_send up:boot seq 1 (currently up:boot) 10.06.01_22:25:06.463356 7f018d385720 -- 172.16.2.9:6800/6204 --> mon1 172.16.2.10:6789/0 -- mdsbeacon(11227/r1-9 up:boot seq 1 v0) v1 -- ?+0 0x106fb50 10.06.01_22:25:06.463441 7f018d385720 -- 172.16.2.9:6800/6204 --> mon1 172.16.2.10:6789/0 -- mon_subscribe({monmap=0+,osdmap=0}) v1 -- ?+0 0x10770d0 10.06.01_22:25:06.463466 7f018d385720 -- 172.16.2.9:6800/6204 --> mon1 172.16.2.10:6789/0 -- mon_subscribe({mdsmap=0+,monmap=0+,osdmap=0}) v1 -- ?+0 0x10773a0 10.06.01_22:25:06.463606 7f018b477710 -- 172.16.2.9:6800/6204 <== mon1 172.16.2.10:6789/0 2 ==== mon_map v1 ==== 481+0+0 (3368855601 0 0) 0x1077100 10.06.01_22:25:06.463631 7f018b477710 -- 172.16.2.9:6800/6204 <== mon1 172.16.2.10:6789/0 3 ==== mon_subscribe_ack(300s) v1 ==== 20+0+0 (432179016 0 0) 0x10773b0 10.06.01_22:25:06.463700 7f018b477710 -- 172.16.2.9:6800/6204 <== mon1 172.16.2.10:6789/0 4 ==== mon_map v1 ==== 481+0+0 (3368855601 0 0) 0x1077100 10.06.01_22:25:06.463787 7f018b477710 -- 172.16.2.9:6800/6204 <== mon1 172.16.2.10:6789/0 5 ==== osd_map(178,178) v1 ==== 2428+0+0 (2723171310 0 0) 0x106fb50 10.06.01_22:25:06.463886 7f018b477710 -- 172.16.2.9:6800/6204 <== mon1 172.16.2.10:6789/0 6 ==== mon_subscribe_ack(300s) v1 ==== 20+0+0 (432179016 0 0) 0x1077130 10.06.01_22:25:06.463922 7f018b477710 -- 172.16.2.9:6800/6204 <== mon1 172.16.2.10:6789/0 7 ==== mdsmap(e 1335) v1 ==== 2070+0+0 (2278891905 0 0) 0x107ef10 10.06.01_22:25:06.463938 7f018b477710 mds-1.0 handle_mds_map epoch 1335 from mon1 10.06.01_22:25:06.463970 7f018b477710 mds-1.0 my compat compat={},rocompat={},incompat={1=base v0.20} 10.06.01_22:25:06.463984 7f018b477710 mds-1.0 mdsmap compat compat={},rocompat={},incompat={1=base v0.20} 10.06.01_22:25:06.463992 7f018b477710 mds-1.0 map says i am 172.16.2.9:6800/6204 mds-1 state down:dne 10.06.01_22:25:06.464003 7f018b477710 mds-1.0 not in map yet 10.06.01_22:25:06.464012 7f018b477710 -- 172.16.2.9:6800/6204 <== mon1 172.16.2.10:6789/0 8 ==== mon_map v1 ==== 481+0+0 (3368855601 0 0) 0x107d000 10.06.01_22:25:06.464028 7f018b477710 -- 172.16.2.9:6800/6204 <== mon1 172.16.2.10:6789/0 9 ==== osd_map(178,178) v1 ==== 2428+0+0 (2723171310 0 0) 0x107fb10 10.06.01_22:25:06.464043 7f018b477710 -- 172.16.2.9:6800/6204 <== mon1 172.16.2.10:6789/0 10 ==== mon_subscribe_ack(300s) v1 ==== 20+0+0 (432179016 0 0) 0x1077130 10.06.01_22:25:10.463440 7f0189473710 mds-1.0 beacon_send up:boot seq 2 (currently down:dne) 10.06.01_22:25:10.463463 7f0189473710 -- 172.16.2.9:6800/6204 --> mon1 172.16.2.10:6789/0 -- mdsbeacon(11227/r1-9 up:boot seq 2 v1335) v1 -- ?+0 0x106fb50 10.06.01_22:25:10.671287 7f018b477710 -- 172.16.2.9:6800/6204 <== mon1 172.16.2.10:6789/0 11 ==== mdsmap(e 1336) v1 ==== 2271+0+0 (3068583984 0 0) 0x106fb50 10.06.01_22:25:10.671321 7f018b477710 mds-1.0 handle_mds_map epoch 1336 from mon1 10.06.01_22:25:10.671355 7f018b477710 mds-1.0 my compat compat={},rocompat={},incompat={1=base v0.20} 10.06.01_22:25:10.671365 7f018b477710 mds-1.0 mdsmap compat compat={},rocompat={},incompat={1=base v0.20} 10.06.01_22:25:10.671373 7f018b477710 mds-1.0 map says i am 172.16.2.9:6800/6204 mds-1 state up:standby 10.06.01_22:25:10.671382 7f018b477710 mds-1.0 handle_mds_map standby 10.06.01_22:25:10.976411 7f018b477710 -- 172.16.2.9:6800/6204 <== mon1 172.16.2.10:6789/0 12 ==== mdsmap(e 1337) v1 ==== 2069+0+0 (2433051849 0 0) 0x106fb50 10.06.01_22:25:10.976445 7f018b477710 mds-1.0 handle_mds_map epoch 1337 from mon1 10.06.01_22:25:10.976474 7f018b477710 mds-1.0 my compat compat={},rocompat={},incompat={1=base v0.20} 10.06.01_22:25:10.976483 7f018b477710 mds-1.0 mdsmap compat compat={},rocompat={},incompat={1=base v0.20} 10.06.01_22:25:10.976490 7f018b477710 mds2.0 map says i am 172.16.2.9:6800/6204 mds2 state up:replay 10.06.01_22:25:10.976513 7f018b477710 ---- renamed symlink /var/log/ceph/mds2 -> /var/log/ceph/mds2.0 ---- 10.06.01_22:25:11.059044 7f018b477710 ---- created symlink /var/log/ceph/mds2 -> r1-9.6205 ---- 10.06.01_22:25:11.059251 7f018b477710 mds2.12 handle_mds_map i am now mds2.12 10.06.01_22:25:11.059261 7f018b477710 mds2.12 handle_mds_map state change up:standby --> up:replay 10.06.01_22:25:11.059269 7f018b477710 mds2.12 replay_start 10.06.01_22:25:11.059277 7f018b477710 mds2.12 now replay. my recovery peers are 0,1 10.06.01_22:25:11.059289 7f018b477710 mds2.cache set_recovery_set 0,1 10.06.01_22:25:11.059297 7f018b477710 mds2.12 boot_start 1: opening inotable 10.06.01_22:25:11.059306 7f018b477710 mds2.inotable: load 10.06.01_22:25:11.059337 7f018b477710 -- 172.16.2.9:6800/6204 --> osd0 172.16.2.9:6801/2678 -- osd_op(mds2.12:1 mds2_inotable [read 0~0] 1.6843) v1 -- ?+0 0x7f018400dda0 10.06.01_22:25:11.059382 7f018b477710 mds2.12 boot_start 1: opening sessionmap 10.06.01_22:25:11.059394 7f018b477710 mds2.sessionmap load 10.06.01_22:25:11.059411 7f018b477710 -- 172.16.2.9:6800/6204 --> osd2 172.16.2.11:6801/3015 -- osd_op(mds2.12:2 mds2_sessionmap [read 0~0] 1.3c91) v1 -- ?+0 0x7f018400ea10 10.06.01_22:25:11.059457 7f018b477710 mds2.12 boot_start 1: opening mds log 10.06.01_22:25:11.059469 7f018b477710 mds2.log open discovering log bounds 10.06.01_22:25:11.059499 7f018b477710 -- 172.16.2.9:6800/6204 --> osd0 172.16.2.9:6801/2678 -- osd_op(mds2.12:3 202.00000000 [read 0~0] 1.dda1) v1 -- ?+0 0x7f01840102a0 10.06.01_22:25:11.059527 7f01837fe710 mds2.12 MDS::ms_get_authorizer type=osd 10.06.01_22:25:11.059551 7f018b477710 mds2.cache handle_mds_failure mds2 10.06.01_22:25:11.059565 7f018b477710 mds2.cache handle_mds_failure mds2 : recovery peers are 0,1 10.06.01_22:25:11.059573 7f018b477710 mds2.cache wants_resolve 10.06.01_22:25:11.059578 7f018b477710 mds2.cache got_resolve 10.06.01_22:25:11.059584 7f018b477710 mds2.cache rejoin_sent 10.06.01_22:25:11.059589 7f018b477710 mds2.cache rejoin_gather 10.06.01_22:25:11.059596 7f018b477710 mds2.cache rejoin_ack_gather 10.06.01_22:25:11.059605 7f018b477710 mds2.migrator handle_mds_failure_or_stop mds2 10.06.01_22:25:11.059645 7f018b477710 mds2.cache show_subtrees - no subtrees 10.06.01_22:25:11.059660 7f018b477710 mds2.bal check_targets have need want 10.06.01_22:25:11.059672 7f018b477710 mds2.bal rebalance done 10.06.01_22:25:11.059678 7f018b477710 mds2.cache show_subtrees - no subtrees 10.06.01_22:25:11.059720 7f018b477710 mds2.12 ms_handle_connect on 172.16.2.9:6801/2678 10.06.01_22:25:11.059773 7f0182ffd710 mds2.12 MDS::ms_get_authorizer type=osd 10.06.01_22:25:11.060071 7f018b477710 mds2.12 ms_handle_connect on 172.16.2.11:6801/3015 10.06.01_22:25:11.060710 7f018b477710 -- 172.16.2.9:6800/6204 <== osd2 172.16.2.11:6801/3015 1 ==== osd_op_reply(2 mds2_sessionmap [read 0~17] = 0) v1 ==== 101+0+17 (3920954706 0 2334645356) 0x10770d0 10.06.01_22:25:11.060748 7f018b477710 mds2.sessionmap dump 10.06.01_22:25:11.060758 7f018b477710 mds2.sessionmap _load_finish v 0, 0 sessions, 17 bytes 10.06.01_22:25:11.060766 7f018b477710 mds2.sessionmap dump 10.06.01_22:25:11.060783 7f018b477710 -- 172.16.2.9:6800/6204 <== osd0 172.16.2.9:6801/2678 1 ==== osd_op_reply(1 mds2_inotable [read 0~29] = 0) v1 ==== 99+0+29 (625385979 0 936214950) 0x107d000 10.06.01_22:25:11.060804 7f018b477710 mds2.inotable: load_2 got 29 bytes 10.06.01_22:25:11.060811 7f018b477710 mds2.inotable: load_2 loaded v0 10.06.01_22:25:11.060885 7f018b477710 -- 172.16.2.9:6800/6204 <== osd0 172.16.2.9:6801/2678 2 ==== osd_op_reply(3 202.00000000 [read 0~84] = 0) v1 ==== 98+0+84 (1743268816 0 2965425223) 0x10770d0 10.06.01_22:25:11.060932 7f018b477710 -- 172.16.2.9:6800/6204 --> osd2 172.16.2.11:6801/3015 -- osd_op(mds2.12:4 202.00000001 [stat 0~0] 1.ae91) v1 -- ?+0 0x7f018400dda0 10.06.01_22:25:11.060959 7f018b477710 -- 172.16.2.9:6800/6204 --> osd0 172.16.2.9:6801/2678 -- osd_op(mds2.12:5 202.00000002 [stat 0~0] 1.49be) v1 -- ?+0 0x7f0184002100 10.06.01_22:25:11.061245 7f018b477710 -- 172.16.2.9:6800/6204 <== osd0 172.16.2.9:6801/2678 3 ==== osd_op_reply(5 202.00000002 [stat 0~0] = -2 (No such file or directory)) v1 ==== 98+0+0 (1580813580 0 0) 0x10770d0 10.06.01_22:25:11.103090 7f018b477710 -- 172.16.2.9:6800/6204 <== osd2 172.16.2.11:6801/3015 2 ==== osd_op_reply(4 202.00000001 [stat 0~0] = 0) v1 ==== 98+0+16 (1391434735 0 1675546433) 0x7f018400dda0 10.06.01_22:25:11.103165 7f018b477710 mds2.12 boot_start 3: replaying mds log 10.06.01_22:25:11.103176 7f018b477710 mds2.log replay start, from 4194304 to 4198093 10.06.01_22:25:11.103227 7f01817fa710 mds2.log _replay_thread start 10.06.01_22:25:11.103269 7f01817fa710 -- 172.16.2.9:6800/6204 --> osd2 172.16.2.11:6801/3015 -- osd_op(mds2.12:6 202.00000001 [read 0~3789] 1.ae91) v1 -- ?+0 0x10770d0 10.06.01_22:25:11.103897 7f018b477710 -- 172.16.2.9:6800/6204 <== osd2 172.16.2.11:6801/3015 3 ==== osd_op_reply(6 202.00000001 [read 0~3789] = 0) v1 ==== 98+0+3789 (2228701598 0 3597426558) 0x7f018400dda0 10.06.01_22:25:11.103948 7f01817fa710 mds2.log _replay 4194304~95 / 4198093 : subtree_map 0 subtrees [metablob] 10.06.01_22:25:11.103963 7f01817fa710 mds2.journal ESubtreeMap.replay -- reconstructing (auth) subtree spanning tree 10.06.01_22:25:11.103974 7f01817fa710 mds2.journal EMetaBlob.replay 0 dirlumps 10.06.01_22:25:11.103986 7f01817fa710 mds2.cache show_subtrees - no subtrees 10.06.01_22:25:11.104007 7f01817fa710 mds2.log _replay 4194403~365 / 4198093 : subtree_map 1 subtrees [metablob 102, 1 dirs] 10.06.01_22:25:11.104023 7f01817fa710 mds2.journal ESubtreeMap.replay -- reconstructing (auth) subtree spanning tree 10.06.01_22:25:11.104029 7f01817fa710 mds2.journal EMetaBlob.replay 1 dirlumps 10.06.01_22:25:11.104035 7f01817fa710 mds2.journal EMetaBlob.replay dir 102 10.06.01_22:25:11.104050 7f01817fa710 mds2.journal EMetaBlob.replay created base [inode 102 [...2,head] ~mds2/ auth v1 snaprealm=0x1082090 f(v0 m0.000000 0=0+0) ds=0=0+0 rb=0 rf=0 rd=0 (iauth sync) (ilink sync) (idft sync) (isnap sync) (inest sync) (ifile sync) (ixattr sync) (iversion lock) 0x7f0184010850] 10.06.01_22:25:11.104118 7f01817fa710 mds2.cache adjust_subtree_auth -1,-2 -> -2,-2 on [dir 102 ~mds2/ [2,head] auth v=0 cv=0/0 state=1073741824 f(v0 m0.000000 0=0+0) s=0=0+0 rb=0/0 rf=0/0 rd=0/0 hs=0+0,ss=0+0 0x1086010] 10.06.01_22:25:11.104138 7f01817fa710 mds2.cache show_subtrees - no subtrees 10.06.01_22:25:11.104149 7f01817fa710 mds2.cache current root is [dir 102 ~mds2/ [2,head] auth v=0 cv=0/0 state=1073741824 f(v0 m0.000000 0=0+0) s=0=0+0 rb=0/0 rf=0/0 rd=0/0 hs=0+0,ss=0+0 | nref=1 0x1086010] 10.06.01_22:25:11.104162 7f01817fa710 mds2.cache.dir(102) setting dir_auth=-2,-2 from -1,-2 on [dir 102 ~mds2/ [2,head] auth v=0 cv=0/0 state=1073741824 f(v0 m0.000000 0=0+0) s=0=0+0 rb=0/0 rf=0/0 rd=0/0 hs=0+0,ss=0+0 | nref=1 0x1086010] 10.06.01_22:25:11.104176 7f01817fa710 mds2.cache.dir(102) new subtree root, adjusting auth_pins 10.06.01_22:25:11.104185 7f01817fa710 mds2.cache show_subtrees 10.06.01_22:25:11.104203 7f01817fa710 mds2.cache |__-2 auth [dir 102 ~mds2/ [2,head] auth v=0 cv=0/0 dir_auth=-2 state=1073741824 f(v0 m0.000000 0=0+0) s=0=0+0 rb=0/0 rf=0/0 rd=0/0 hs=0+0,ss=0+0 | nref=1 0x1086010] 10.06.01_22:25:11.104218 7f01817fa710 mds2.journal EMetaBlob.replay added dir [dir 102 ~mds2/ [2,head] auth v=0 cv=0/0 dir_auth=-2 state=1073741824 f(v0 m0.000000 0=0+0) s=0=0+0 rb=0/0 rf=0/0 rd=0/0 hs=0+0,ss=0+0 | nref=1 0x1086010] 10.06.01_22:25:11.104236 7f01817fa710 mds2.cache adjust_bounded_subtree_auth -2,-2 -> 2,-2 on [dir 102 ~mds2/ [2,head] auth v=1 cv=0/0 dir_auth=-2 state=1073741824 f(v0 m0.000000 2=1+1) s=2=1+1 rb=0/0 rf=1/1 rd=1/1 hs=0+0,ss=0+0 | nref=1 0x1086010] bound_dfs [] 10.06.01_22:25:11.104251 7f01817fa710 mds2.cache adjust_bounded_subtree_auth -2,-2 -> 2,-2 on [dir 102 ~mds2/ [2,head] auth v=1 cv=0/0 dir_auth=-2 state=1073741824 f(v0 m0.000000 2=1+1) s=2=1+1 rb=0/0 rf=1/1 rd=1/1 hs=0+0,ss=0+0 | nref=1 0x1086010] bounds 10.06.01_22:25:11.104267 7f01817fa710 mds2.cache show_subtrees 10.06.01_22:25:11.104275 7f01817fa710 mds2.cache |__-2 auth [dir 102 ~mds2/ [2,head] auth v=1 cv=0/0 dir_auth=-2 state=1073741824 f(v0 m0.000000 2=1+1) s=2=1+1 rb=0/0 rf=1/1 rd=1/1 hs=0+0,ss=0+0 | nref=1 0x1086010] 10.06.01_22:25:11.104288 7f01817fa710 mds2.cache current root is [dir 102 ~mds2/ [2,head] auth v=1 cv=0/0 dir_auth=-2 state=1073741824 f(v0 m0.000000 2=1+1) s=2=1+1 rb=0/0 rf=1/1 rd=1/1 hs=0+0,ss=0+0 | nref=1 0x1086010] 10.06.01_22:25:11.104300 7f01817fa710 mds2.cache.dir(102) setting dir_auth=2,-2 from -2,-2 on [dir 102 ~mds2/ [2,head] auth v=1 cv=0/0 dir_auth=-2 state=1073741824 f(v0 m0.000000 2=1+1) s=2=1+1 rb=0/0 rf=1/1 rd=1/1 hs=0+0,ss=0+0 | nref=1 0x1086010] 10.06.01_22:25:11.104316 7f01817fa710 mds2.cache show_subtrees 10.06.01_22:25:11.104325 7f01817fa710 mds2.cache |__ 2 auth [dir 102 ~mds2/ [2,head] auth v=1 cv=0/0 dir_auth=2 state=1073741824 f(v0 m0.000000 2=1+1) s=2=1+1 rb=0/0 rf=1/1 rd=1/1 hs=0+0,ss=0+0 | nref=1 0x1086010] 10.06.01_22:25:11.104338 7f01817fa710 mds2.cache show_subtrees 10.06.01_22:25:11.104346 7f01817fa710 mds2.cache |__ 2 auth [dir 102 ~mds2/ [2,head] auth v=1 cv=0/0 dir_auth=2 state=1073741824 f(v0 m0.000000 2=1+1) s=2=1+1 rb=0/0 rf=1/1 rd=1/1 hs=0+0,ss=0+0 | nref=1 0x1086010] 10.06.01_22:25:11.104365 7f01817fa710 mds2.log _replay 4194772~365 / 4198093 : subtree_map 1 subtrees [metablob 102, 1 dirs] 10.06.01_22:25:11.104374 7f01817fa710 mds2.journal ESubtreeMap.replay -- ignoring, already have import map 10.06.01_22:25:11.104385 7f01817fa710 mds2.log _replay 4195141~365 / 4198093 : subtree_map 1 subtrees [metablob 102, 1 dirs] 10.06.01_22:25:11.104400 7f01817fa710 mds2.journal ESubtreeMap.replay -- ignoring, already have import map 10.06.01_22:25:11.104412 7f01817fa710 mds2.log _replay 4195510~365 / 4198093 : subtree_map 1 subtrees [metablob 102, 1 dirs] 10.06.01_22:25:11.104420 7f01817fa710 mds2.journal ESubtreeMap.replay -- ignoring, already have import map 10.06.01_22:25:11.104431 7f01817fa710 mds2.log _replay 4195879~365 / 4198093 : subtree_map 1 subtrees [metablob 102, 1 dirs] 10.06.01_22:25:11.104440 7f01817fa710 mds2.journal ESubtreeMap.replay -- ignoring, already have import map 10.06.01_22:25:11.104450 7f01817fa710 mds2.log _replay 4196248~365 / 4198093 : subtree_map 1 subtrees [metablob 102, 1 dirs] 10.06.01_22:25:11.104459 7f01817fa710 mds2.journal ESubtreeMap.replay -- ignoring, already have import map 10.06.01_22:25:11.104469 7f01817fa710 mds2.log _replay 4196617~365 / 4198093 : subtree_map 1 subtrees [metablob 102, 1 dirs] 10.06.01_22:25:11.104477 7f01817fa710 mds2.journal ESubtreeMap.replay -- ignoring, already have import map 10.06.01_22:25:11.104488 7f01817fa710 mds2.log _replay 4196986~365 / 4198093 : subtree_map 1 subtrees [metablob 102, 1 dirs] 10.06.01_22:25:11.104497 7f01817fa710 mds2.journal ESubtreeMap.replay -- ignoring, already have import map 10.06.01_22:25:11.104508 7f01817fa710 mds2.log _replay 4197355~365 / 4198093 : subtree_map 1 subtrees [metablob 102, 1 dirs] 10.06.01_22:25:11.104516 7f01817fa710 mds2.journal ESubtreeMap.replay -- ignoring, already have import map 10.06.01_22:25:11.104527 7f01817fa710 mds2.log _replay 4197724~365 / 4198093 : subtree_map 1 subtrees [metablob 102, 1 dirs] 10.06.01_22:25:11.104536 7f01817fa710 mds2.journal ESubtreeMap.replay -- ignoring, already have import map 10.06.01_22:25:11.104543 7f01817fa710 mds2.log _replay - complete, 11 events, new read/expire pos is 4194304 10.06.01_22:25:11.104550 7f01817fa710 mds2.log _replay - truncating at 4198093 10.06.01_22:25:11.104558 7f01817fa710 mds2.log _replay_thread nothing to truncate, kicking waiters 10.06.01_22:25:11.104565 7f01817fa710 mds2.12 replay_done in=3 failed=0 10.06.01_22:25:11.104571 7f01817fa710 mds2.12 i am not alone, moving to state resolve 10.06.01_22:25:11.104577 7f01817fa710 mds2.12 request_state up:resolve 10.06.01_22:25:11.104583 7f01817fa710 mds2.12 beacon_send up:resolve seq 3 (currently up:replay) 10.06.01_22:25:11.104591 7f01817fa710 -- 172.16.2.9:6800/6204 --> mon1 172.16.2.10:6789/0 -- mdsbeacon(11227/r1-9 up:resolve seq 3 v1337) v1 -- ?+0 0x1084730 10.06.01_22:25:11.104656 7f01817fa710 mds2.log start_new_segment at 4198093 10.06.01_22:25:11.104668 7f01817fa710 mds2.cache create_subtree_map 1 subtrees, 1 fullauth 10.06.01_22:25:11.104677 7f01817fa710 mds2.cache subtree [dir 102 ~mds2/ [2,head] auth v=1 cv=0/0 dir_auth=2 state=1073741824 f(v0 m0.000000 2=1+1) s=2=1+1 rb=0/0 rf=1/1 rd=1/1 hs=0+0,ss=0+0 | nref=1 0x1086010] 10.06.01_22:25:11.104709 7f01817fa710 mds2.log submit_entry 4198093~365 : subtree_map 1 subtrees [metablob 102, 1 dirs] 10.06.01_22:25:11.104733 7f01817fa710 mds2.log _replay_thread finish 10.06.01_22:25:11.463572 7f0189473710 -- 172.16.2.9:6800/6204 --> osd2 172.16.2.11:6801/3015 -- osd_op(mds2.12:7 202.00000001 [write 3789~369] 1.ae91) v1 -- ?+0 0x1084990 10.06.01_22:25:11.463631 7f0189473710 -- 172.16.2.9:6800/6204 --> osd0 172.16.2.9:6801/2678 -- osd_op(mds2.12:8 202.00000000 [writefull 0~84] 1.dda1) v1 -- ?+0 0x7f018400ff40 10.06.01_22:25:11.463741 7f0189473710 mds2.bal get_load mdsload<[0,0 0]/[0,0 0], req 0, hr 0, qlen 0, cpu 0.42> 10.06.01_22:25:11.552407 7f018b477710 -- 172.16.2.9:6800/6204 <== osd2 172.16.2.11:6801/3015 4 ==== osd_op_reply(7 202.00000001 [write 3789~369] = 0) v1 ==== 98+0+0 (1381891074 0 0) 0x7f018400dda0 10.06.01_22:25:11.552443 7f018b477710 mds2.log _logged_subtree_map at 4198093 10.06.01_22:25:11.567599 7f018b477710 -- 172.16.2.9:6800/6204 <== osd0 172.16.2.9:6801/2678 4 ==== osd_op_reply(8 202.00000000 [writefull 0~84] = 0) v1 ==== 98+0+0 (940119683 0 0) 0x1084d80 10.06.01_22:25:11.646331 7f018b477710 -- 172.16.2.9:6800/6204 <== mon1 172.16.2.10:6789/0 13 ==== mdsmap(e 1338) v1 ==== 2271+0+0 (2851125949 0 0) 0x1084990 10.06.01_22:25:11.736973 7f018b477710 mds2.12 handle_mds_map epoch 1338 from mon1 10.06.01_22:25:11.737004 7f018b477710 mds2.12 my compat compat={},rocompat={},incompat={1=base v0.20} 10.06.01_22:25:11.737013 7f018b477710 mds2.12 mdsmap compat compat={},rocompat={},incompat={1=base v0.20} 10.06.01_22:25:11.737021 7f018b477710 mds2.12 map says i am 172.16.2.9:6800/6204 mds2 state up:replay 10.06.01_22:25:11.737037 7f018b477710 mds2.bal check_targets have need want 10.06.01_22:25:11.737075 7f018b477710 mds2.bal map: i imported [dir 102 ~mds2/ [2,head] auth v=1 cv=0/0 dir_auth=2 state=1073741824 f(v0 m0.000000 2=1+1) s=2=1+1 rb=0/0 rf=1/1 rd=1/1 hs=0+0,ss=0+0 | nref=1 0x1086010] from 2 10.06.01_22:25:11.737092 7f018b477710 mds2.bal rebalance done 10.06.01_22:25:11.737100 7f018b477710 mds2.cache show_subtrees 10.06.01_22:25:11.737109 7f018b477710 mds2.cache |__ 2 auth [dir 102 ~mds2/ [2,head] auth v=1 cv=0/0 dir_auth=2 state=1073741824 f(v0 m0.000000 2=1+1) s=2=1+1 rb=0/0 rf=1/1 rd=1/1 hs=0+0,ss=0+0 | nref=1 0x1086010] 10.06.01_22:25:11.863785 7f018b477710 -- 172.16.2.9:6800/6204 <== mon1 172.16.2.10:6789/0 14 ==== mdsbeacon(11227/r1-9 up:resolve seq 3 v1338) v1 ==== 70+0+0 (3960253285 0 0) 0x1084990 10.06.01_22:25:11.863811 7f018b477710 mds2.12 handle_mds_beacon up:resolve seq 3 rtt 0.759221 10.06.01_22:25:11.948326 7f018b477710 -- 172.16.2.9:6800/6204 <== mon1 172.16.2.10:6789/0 15 ==== mdsmap(e 1339) v1 ==== 2473+0+0 (2526007582 0 0) 0x1084990 10.06.01_22:25:11.948346 7f018b477710 mds2.12 handle_mds_map epoch 1339 from mon1 10.06.01_22:25:11.948370 7f018b477710 mds2.12 my compat compat={},rocompat={},incompat={1=base v0.20} 10.06.01_22:25:11.948378 7f018b477710 mds2.12 mdsmap compat compat={},rocompat={},incompat={1=base v0.20} 10.06.01_22:25:11.948389 7f018b477710 mds2.12 map says i am 172.16.2.9:6800/6204 mds2 state up:replay 10.06.01_22:25:11.948404 7f018b477710 mds2.bal check_targets have need want 10.06.01_22:25:11.948412 7f018b477710 mds2.bal map: i imported [dir 102 ~mds2/ [2,head] auth v=1 cv=0/0 dir_auth=2 state=1073741824 f(v0 m0.000000 2=1+1) s=2=1+1 rb=0/0 rf=1/1 rd=1/1 hs=0+0,ss=0+0 | nref=1 0x1086010] from 2 10.06.01_22:25:11.948427 7f018b477710 mds2.bal rebalance done 10.06.01_22:25:11.948433 7f018b477710 mds2.cache show_subtrees 10.06.01_22:25:11.948441 7f018b477710 mds2.cache |__ 2 auth [dir 102 ~mds2/ [2,head] auth v=1 cv=0/0 dir_auth=2 state=1073741824 f(v0 m0.000000 2=1+1) s=2=1+1 rb=0/0 rf=1/1 rd=1/1 hs=0+0,ss=0+0 | nref=1 0x1086010] 10.06.01_22:25:15.104706 7f0189473710 mds2.12 beacon_send up:resolve seq 4 (currently up:replay) 10.06.01_22:25:15.104731 7f0189473710 -- 172.16.2.9:6800/6204 --> mon1 172.16.2.10:6789/0 -- mdsbeacon(11227/r1-9 up:resolve seq 4 v1339) v1 -- ?+0 0x7f018400ff40 10.06.01_22:25:15.288611 7f018b477710 -- 172.16.2.9:6800/6204 <== mon1 172.16.2.10:6789/0 16 ==== mdsmap(e 1340) v1 ==== 2473+0+0 (1592609091 0 0) 0x1084990 10.06.01_22:25:15.288647 7f018b477710 mds2.12 handle_mds_map epoch 1340 from mon1 10.06.01_22:25:15.288680 7f018b477710 mds2.12 my compat compat={},rocompat={},incompat={1=base v0.20} 10.06.01_22:25:15.288689 7f018b477710 mds2.12 mdsmap compat compat={},rocompat={},incompat={1=base v0.20} 10.06.01_22:25:15.288696 7f018b477710 mds2.12 map says i am 172.16.2.9:6800/6204 mds2 state up:resolve 10.06.01_22:25:15.288706 7f018b477710 mds2.12 handle_mds_map state change up:replay --> up:resolve 10.06.01_22:25:15.288712 7f018b477710 mds2.12 resolve_start 10.06.01_22:25:15.288724 7f018b477710 mds2.cache resolve_start 10.06.01_22:25:15.288731 7f018b477710 mds2.cache send_resolve_now to mds0 10.06.01_22:25:15.288739 7f018b477710 mds2.cache show_subtrees 10.06.01_22:25:15.288749 7f018b477710 mds2.cache |__ 2 auth [dir 102 ~mds2/ [2,head] auth v=1 cv=0/0 dir_auth=2 state=1073741824 f(v0 m0.000000 2=1+1) s=2=1+1 rb=0/0 rf=1/1 rd=1/1 hs=0+0,ss=0+0 | nref=1 0x1086010] 10.06.01_22:25:15.288788 7f018b477710 -- 172.16.2.9:6800/6204 --> mds0 172.16.2.10:6800/5796 -- mdsmap(e 1340) v1 -- ?+0 0x7f018400ead0 10.06.01_22:25:15.288848 7f018b477710 -- 172.16.2.9:6800/6204 --> mds0 172.16.2.10:6800/5796 -- mds_resolve(1+0 subtrees +0 slave requests) v1 -- ?+0 0x7f018401c650 10.06.01_22:25:15.288865 7f018b477710 mds2.cache send_resolve_now to mds1 10.06.01_22:25:15.288873 7f018b477710 mds2.cache show_subtrees 10.06.01_22:25:15.288881 7f018b477710 mds2.cache |__ 2 auth [dir 102 ~mds2/ [2,head] auth v=1 cv=0/0 dir_auth=2 state=1073741824 f(v0 m0.000000 2=1+1) s=2=1+1 rb=0/0 rf=1/1 rd=1/1 hs=0+0,ss=0+0 | nref=1 0x1086010] 10.06.01_22:25:15.288909 7f018b477710 -- 172.16.2.9:6800/6204 --> mds1 172.16.2.9:6800/6024 -- mdsmap(e 1340) v1 -- ?+0 0x7f018401e290 10.06.01_22:25:15.288945 7f018b477710 -- 172.16.2.9:6800/6204 --> mds1 172.16.2.9:6800/6024 -- mds_resolve(1+0 subtrees +0 slave requests) v1 -- ?+0 0x7f018401e010 10.06.01_22:25:15.288987 7f018b477710 mds2.12 resolve set is 2, was 10.06.01_22:25:15.289002 7f018b477710 mds2.bal check_targets have need want 10.06.01_22:25:15.289011 7f018b477710 mds2.bal map: i imported [dir 102 ~mds2/ [2,head] auth v=1 cv=0/0 dir_auth=2 state=1073741824 f(v0 m0.000000 2=1+1) s=2=1+1 rb=0/0 rf=1/1 rd=1/1 hs=0+0,ss=0+0 | nref=1 0x1086010] from 2 10.06.01_22:25:15.289028 7f018b477710 mds2.bal rebalance done 10.06.01_22:25:15.289035 7f018b477710 mds2.cache show_subtrees 10.06.01_22:25:15.289043 7f018b477710 mds2.cache |__ 2 auth [dir 102 ~mds2/ [2,head] auth v=1 cv=0/0 dir_auth=2 state=1073741824 f(v0 m0.000000 2=1+1) s=2=1+1 rb=0/0 rf=1/1 rd=1/1 hs=0+0,ss=0+0 | nref=1 0x1086010] 10.06.01_22:25:15.289085 7f01807f8710 -- 172.16.2.9:6800/6204 >> 172.16.2.9:6800/6024 pipe(0x7f018401e500 sd=15 pgs=0 cs=0 l=0).connect claims to be 172.16.2.9:6800/6204 not 172.16.2.9:6800/6024 - wrong node! 10.06.01_22:25:15.289132 7f01807f8710 -- 172.16.2.9:6800/6204 >> 172.16.2.9:6800/6024 pipe(0x7f018401e500 sd=-1 pgs=0 cs=0 l=0).fault first fault 10.06.01_22:25:15.289187 7f0180ff9710 -- 172.16.2.9:6800/6204 >> 172.16.2.10:6800/5796 pipe(0x7f018401c8d0 sd=14 pgs=0 cs=0 l=0).connect claims to be 172.16.2.10:6800/5990 not 172.16.2.10:6800/5796 - wrong node! 10.06.01_22:25:15.289230 7f0180ff9710 -- 172.16.2.9:6800/6204 >> 172.16.2.10:6800/5796 pipe(0x7f018401c8d0 sd=-1 pgs=0 cs=0 l=0).fault first fault 10.06.01_22:25:15.289273 7f01807f8710 -- 172.16.2.9:6800/6204 >> 172.16.2.9:6800/6024 pipe(0x7f018401e500 sd=15 pgs=0 cs=0 l=0).connect claims to be 172.16.2.9:6800/6204 not 172.16.2.9:6800/6024 - wrong node! 10.06.01_22:25:15.289501 7f0180ff9710 -- 172.16.2.9:6800/6204 >> 172.16.2.10:6800/5796 pipe(0x7f018401c8d0 sd=14 pgs=0 cs=0 l=0).connect claims to be 172.16.2.10:6800/5990 not 172.16.2.10:6800/5796 - wrong node! 10.06.01_22:25:15.489448 7f01807f8710 -- 172.16.2.9:6800/6204 >> 172.16.2.9:6800/6024 pipe(0x7f018401e500 sd=14 pgs=0 cs=0 l=0).connect claims to be 172.16.2.9:6800/6204 not 172.16.2.9:6800/6024 - wrong node! 10.06.01_22:25:15.489827 7f0180ff9710 -- 172.16.2.9:6800/6204 >> 172.16.2.10:6800/5796 pipe(0x7f018401c8d0 sd=14 pgs=0 cs=0 l=0).connect claims to be 172.16.2.10:6800/5990 not 172.16.2.10:6800/5796 - wrong node! 10.06.01_22:25:15.889629 7f01807f8710 -- 172.16.2.9:6800/6204 >> 172.16.2.9:6800/6024 pipe(0x7f018401e500 sd=14 pgs=0 cs=0 l=0).connect claims to be 172.16.2.9:6800/6204 not 172.16.2.9:6800/6024 - wrong node! 10.06.01_22:25:15.892212 7f0180ff9710 -- 172.16.2.9:6800/6204 >> 172.16.2.10:6800/5796 pipe(0x7f018401c8d0 sd=16 pgs=0 cs=0 l=0).connect claims to be 172.16.2.10:6800/5990 not 172.16.2.10:6800/5796 - wrong node! 10.06.01_22:25:16.463645 7f0189473710 mds2.bal get_load mdsload<[0,0 0]/[0,0 0], req 0, hr 0, qlen 0, cpu 0.63> 10.06.01_22:25:16.692388 7f01807f8710 -- 172.16.2.9:6800/6204 >> 172.16.2.9:6800/6024 pipe(0x7f018401e500 sd=14 pgs=0 cs=0 l=0).connect claims to be 172.16.2.9:6800/6204 not 172.16.2.9:6800/6024 - wrong node! 10.06.01_22:25:16.692574 7f0180ff9710 -- 172.16.2.9:6800/6204 >> 172.16.2.10:6800/5796 pipe(0x7f018401c8d0 sd=15 pgs=0 cs=0 l=0).connect claims to be 172.16.2.10:6800/5990 not 172.16.2.10:6800/5796 - wrong node! 10.06.01_22:25:18.292585 7f01807f8710 -- 172.16.2.9:6800/6204 >> 172.16.2.9:6800/6024 pipe(0x7f018401e500 sd=14 pgs=0 cs=0 l=0).connect claims to be 172.16.2.9:6800/6204 not 172.16.2.9:6800/6024 - wrong node! 10.06.01_22:25:18.292938 7f0180ff9710 -- 172.16.2.9:6800/6204 >> 172.16.2.10:6800/5796 pipe(0x7f018401c8d0 sd=14 pgs=0 cs=0 l=0).connect claims to be 172.16.2.10:6800/5990 not 172.16.2.10:6800/5796 - wrong node! 10.06.01_22:25:19.104813 7f0189473710 mds2.12 beacon_send up:resolve seq 5 (currently up:resolve) 10.06.01_22:25:19.104831 7f0189473710 -- 172.16.2.9:6800/6204 --> mon1 172.16.2.10:6789/0 -- mdsbeacon(11227/r1-9 up:resolve seq 5 v1340) v1 -- ?+0 0x7f0184002b80 10.06.01_22:25:19.492978 7f018b477710 -- 172.16.2.9:6800/6204 <== mon1 172.16.2.10:6789/0 17 ==== mdsmap(e 1341) v1 ==== 2271+0+0 (3167990692 0 0) 0x1084990 10.06.01_22:25:19.493008 7f018b477710 mds2.12 handle_mds_map epoch 1341 from mon1 10.06.01_22:25:19.493043 7f018b477710 mds2.12 my compat compat={},rocompat={},incompat={1=base v0.20} 10.06.01_22:25:19.493052 7f018b477710 mds2.12 mdsmap compat compat={},rocompat={},incompat={1=base v0.20} 10.06.01_22:25:19.493059 7f018b477710 mds2.12 map says i am 172.16.2.9:6800/6204 mds2 state up:resolve 10.06.01_22:25:19.493075 7f018b477710 mds2.cache handle_mds_failure mds0 10.06.01_22:25:19.493083 7f018b477710 mds2.cache handle_mds_failure mds0 : recovery peers are 0,1 10.06.01_22:25:19.493090 7f018b477710 mds2.cache wants_resolve 10.06.01_22:25:19.493096 7f018b477710 mds2.cache got_resolve 10.06.01_22:25:19.493101 7f018b477710 mds2.cache rejoin_sent 10.06.01_22:25:19.493106 7f018b477710 mds2.cache rejoin_gather 10.06.01_22:25:19.493111 7f018b477710 mds2.cache rejoin_ack_gather 10.06.01_22:25:19.493117 7f018b477710 mds2.migrator handle_mds_failure_or_stop mds0 10.06.01_22:25:19.493129 7f018b477710 mds2.cache show_subtrees 10.06.01_22:25:19.493139 7f018b477710 mds2.cache |__ 2 auth [dir 102 ~mds2/ [2,head] auth v=1 cv=0/0 dir_auth=2 state=1073741824 f(v0 m0.000000 2=1+1) s=2=1+1 rb=0/0 rf=1/1 rd=1/1 hs=0+0,ss=0+0 | nref=1 0x1086010] 10.06.01_22:25:19.493159 7f018b477710 mds2.bal check_targets have need want 10.06.01_22:25:19.493168 7f018b477710 mds2.bal map: i imported [dir 102 ~mds2/ [2,head] auth v=1 cv=0/0 dir_auth=2 state=1073741824 f(v0 m0.000000 2=1+1) s=2=1+1 rb=0/0 rf=1/1 rd=1/1 hs=0+0,ss=0+0 | nref=1 0x1086010] from 2 10.06.01_22:25:19.493182 7f018b477710 mds2.bal rebalance done 10.06.01_22:25:19.493188 7f018b477710 mds2.cache show_subtrees 10.06.01_22:25:19.493195 7f018b477710 mds2.cache |__ 2 auth [dir 102 ~mds2/ [2,head] auth v=1 cv=0/0 dir_auth=2 state=1073741824 f(v0 m0.000000 2=1+1) s=2=1+1 rb=0/0 rf=1/1 rd=1/1 hs=0+0,ss=0+0 | nref=1 0x1086010] 10.06.01_22:25:19.961554 7f018b477710 -- 172.16.2.9:6800/6204 <== mon1 172.16.2.10:6789/0 18 ==== mdsbeacon(11227/r1-9 up:resolve seq 5 v1341) v1 ==== 70+0+0 (1449118260 0 0) 0x1084990 10.06.01_22:25:19.961582 7f018b477710 mds2.12 handle_mds_beacon up:resolve seq 5 rtt 0.856753 10.06.01_22:25:20.114540 7f017f7f6710 mds2.12 new session 0x1084990 for mds11226 172.16.2.11:6800/5997 10.06.01_22:25:20.114795 7f018b477710 -- 172.16.2.9:6800/6204 <== mds0 172.16.2.11:6800/5997 1 ==== mdsmap(e 1342) v1 ==== 2271+0+0 (2120242157 0 0) 0x1091ac0 10.06.01_22:25:20.114823 7f018b477710 mds2.12 handle_mds_map epoch 1342 from mds0 10.06.01_22:25:20.114832 7f018b477710 mds2.12 peer mds0 has mdsmap epoch >= 1342 10.06.01_22:25:20.114859 7f018b477710 mds2.12 my compat compat={},rocompat={},incompat={1=base v0.20} 10.06.01_22:25:20.114867 7f018b477710 mds2.12 mdsmap compat compat={},rocompat={},incompat={1=base v0.20} 10.06.01_22:25:20.114875 7f018b477710 mds2.12 map says i am 172.16.2.9:6800/6204 mds2 state up:resolve 10.06.01_22:25:20.114886 7f018b477710 mds2.12 resolve set is 0,2, was 2 10.06.01_22:25:20.114893 7f018b477710 mds2.cache send_resolve_now to mds0 10.06.01_22:25:20.114907 7f018b477710 mds2.cache show_subtrees 10.06.01_22:25:20.114917 7f018b477710 mds2.cache |__ 2 auth [dir 102 ~mds2/ [2,head] auth v=1 cv=0/0 dir_auth=2 state=1073741824 f(v0 m0.000000 2=1+1) s=2=1+1 rb=0/0 rf=1/1 rd=1/1 hs=0+0,ss=0+0 | nref=1 0x1086010] 10.06.01_22:25:20.114936 7f018b477710 -- 172.16.2.9:6800/6204 --> mds0 172.16.2.11:6800/5997 -- mds_resolve(1+0 subtrees +0 slave requests) v1 -- ?+0 0x7f01840017f0 10.06.01_22:25:20.114960 7f018b477710 mds2.bal check_targets have need want 10.06.01_22:25:20.114970 7f018b477710 mds2.bal map: i imported [dir 102 ~mds2/ [2,head] auth v=1 cv=0/0 dir_auth=2 state=1073741824 f(v0 m0.000000 2=1+1) s=2=1+1 rb=0/0 rf=1/1 rd=1/1 hs=0+0,ss=0+0 | nref=1 0x1086010] from 2 10.06.01_22:25:20.114984 7f018b477710 mds2.bal rebalance done 10.06.01_22:25:20.114991 7f018b477710 mds2.cache show_subtrees 10.06.01_22:25:20.114998 7f018b477710 mds2.cache |__ 2 auth [dir 102 ~mds2/ [2,head] auth v=1 cv=0/0 dir_auth=2 state=1073741824 f(v0 m0.000000 2=1+1) s=2=1+1 rb=0/0 rf=1/1 rd=1/1 hs=0+0,ss=0+0 | nref=1 0x1086010] 10.06.01_22:25:20.115022 7f018b477710 -- 172.16.2.9:6800/6204 <== mds0 172.16.2.11:6800/5997 2 ==== mds_resolve(2+0 subtrees +0 slave requests) v1 ==== 56+0+0 (2291435366 0 0) 0x107f430 10.06.01_22:25:20.115041 7f018b477710 mds2.cache handle_resolve from mds0 10.06.01_22:25:20.115054 7f018b477710 mds2.cache show_subtrees 10.06.01_22:25:20.115063 7f018b477710 mds2.cache |__ 2 auth [dir 102 ~mds2/ [2,head] auth v=1 cv=0/0 dir_auth=2 state=1073741824 f(v0 m0.000000 2=1+1) s=2=1+1 rb=0/0 rf=1/1 rd=1/1 hs=0+0,ss=0+0 | nref=1 0x1086010] 10.06.01_22:25:20.115093 7f018b477710 mds2.cache maybe_resolve_finish still waiting for more resolves, got (0), need (0,1) 10.06.01_22:25:20.122349 7f018b477710 -- 172.16.2.9:6800/6204 <== mon1 172.16.2.10:6789/0 19 ==== mdsmap(e 1342) v1 ==== 2271+0+0 (2120242157 0 0) 0x1091a90 10.06.01_22:25:20.131752 7f018b477710 mds2.12 handle_mds_map epoch 1342 from mon1 10.06.01_22:25:20.131762 7f018b477710 mds2.12 old map epoch 1342 <= 1342, discarding 10.06.01_22:25:20.459610 7f018b477710 -- 172.16.2.9:6800/6204 <== mon1 172.16.2.10:6789/0 20 ==== mdsmap(e 1343) v1 ==== 2070+0+0 (583325346 0 0) 0x1091a90 10.06.01_22:25:20.459640 7f018b477710 mds2.12 handle_mds_map epoch 1343 from mon1 10.06.01_22:25:20.459671 7f018b477710 mds2.12 my compat compat={},rocompat={},incompat={1=base v0.20} 10.06.01_22:25:20.459681 7f018b477710 mds2.12 mdsmap compat compat={},rocompat={},incompat={1=base v0.20} 10.06.01_22:25:20.459688 7f018b477710 mds2.12 map says i am 172.16.2.9:6800/6204 mds2 state up:resolve 10.06.01_22:25:20.459704 7f018b477710 mds2.cache handle_mds_failure mds1 10.06.01_22:25:20.459713 7f018b477710 mds2.cache handle_mds_failure mds1 : recovery peers are 0,1 10.06.01_22:25:20.459720 7f018b477710 mds2.cache wants_resolve 10.06.01_22:25:20.459726 7f018b477710 mds2.cache got_resolve 0 10.06.01_22:25:20.459731 7f018b477710 mds2.cache rejoin_sent 10.06.01_22:25:20.459737 7f018b477710 mds2.cache rejoin_gather 10.06.01_22:25:20.459742 7f018b477710 mds2.cache rejoin_ack_gather 10.06.01_22:25:20.459748 7f018b477710 mds2.migrator handle_mds_failure_or_stop mds1 10.06.01_22:25:20.459760 7f018b477710 mds2.cache show_subtrees 10.06.01_22:25:20.459770 7f018b477710 mds2.cache |__ 2 auth [dir 102 ~mds2/ [2,head] auth v=1 cv=0/0 dir_auth=2 state=1073741824 f(v0 m0.000000 2=1+1) s=2=1+1 rb=0/0 rf=1/1 rd=1/1 hs=0+0,ss=0+0 | nref=1 0x1086010] 10.06.01_22:25:20.459789 7f018b477710 mds2.bal check_targets have need want 10.06.01_22:25:20.459798 7f018b477710 mds2.bal map: i imported [dir 102 ~mds2/ [2,head] auth v=1 cv=0/0 dir_auth=2 state=1073741824 f(v0 m0.000000 2=1+1) s=2=1+1 rb=0/0 rf=1/1 rd=1/1 hs=0+0,ss=0+0 | nref=1 0x1086010] from 2 10.06.01_22:25:20.459812 7f018b477710 mds2.bal rebalance done 10.06.01_22:25:20.459818 7f018b477710 mds2.cache show_subtrees 10.06.01_22:25:20.459825 7f018b477710 mds2.cache |__ 2 auth [dir 102 ~mds2/ [2,head] auth v=1 cv=0/0 dir_auth=2 state=1073741824 f(v0 m0.000000 2=1+1) s=2=1+1 rb=0/0 rf=1/1 rd=1/1 hs=0+0,ss=0+0 | nref=1 0x1086010] 10.06.01_22:25:21.433225 7f017eff5710 mds2.12 new session 0x107e7d0 for mds11237 172.16.2.10:6800/5990 10.06.01_22:25:21.433534 7f018b477710 -- 172.16.2.9:6800/6204 <== mds1 172.16.2.10:6800/5990 1 ==== mdsmap(e 1344) v1 ==== 2070+0+0 (711823579 0 0) 0x107f360 10.06.01_22:25:21.433557 7f018b477710 mds2.12 handle_mds_map epoch 1344 from mds1 10.06.01_22:25:21.433565 7f018b477710 mds2.12 peer mds1 has mdsmap epoch >= 1344 10.06.01_22:25:21.433594 7f018b477710 mds2.12 my compat compat={},rocompat={},incompat={1=base v0.20} 10.06.01_22:25:21.433604 7f018b477710 mds2.12 mdsmap compat compat={},rocompat={},incompat={1=base v0.20} 10.06.01_22:25:21.433611 7f018b477710 mds2.12 map says i am 172.16.2.9:6800/6204 mds2 state up:resolve 10.06.01_22:25:21.433622 7f018b477710 mds2.12 resolve set is 0,1,2, was 0,2 10.06.01_22:25:21.433631 7f018b477710 mds2.cache send_resolve_now to mds1 10.06.01_22:25:21.433638 7f018b477710 mds2.cache show_subtrees 10.06.01_22:25:21.433649 7f018b477710 mds2.cache |__ 2 auth [dir 102 ~mds2/ [2,head] auth v=1 cv=0/0 dir_auth=2 state=1073741824 f(v0 m0.000000 2=1+1) s=2=1+1 rb=0/0 rf=1/1 rd=1/1 hs=0+0,ss=0+0 | nref=1 0x1086010] 10.06.01_22:25:21.433668 7f018b477710 -- 172.16.2.9:6800/6204 --> mds1 172.16.2.10:6800/5990 -- mds_resolve(1+0 subtrees +0 slave requests) v1 -- ?+0 0x7f01840017f0 10.06.01_22:25:21.433693 7f018b477710 mds2.bal check_targets have need want 10.06.01_22:25:21.433702 7f018b477710 mds2.bal map: i imported [dir 102 ~mds2/ [2,head] auth v=1 cv=0/0 dir_auth=2 state=1073741824 f(v0 m0.000000 2=1+1) s=2=1+1 rb=0/0 rf=1/1 rd=1/1 hs=0+0,ss=0+0 | nref=1 0x1086010] from 2 10.06.01_22:25:21.433716 7f018b477710 mds2.bal rebalance done 10.06.01_22:25:21.433722 7f018b477710 mds2.cache show_subtrees 10.06.01_22:25:21.433730 7f018b477710 mds2.cache |__ 2 auth [dir 102 ~mds2/ [2,head] auth v=1 cv=0/0 dir_auth=2 state=1073741824 f(v0 m0.000000 2=1+1) s=2=1+1 rb=0/0 rf=1/1 rd=1/1 hs=0+0,ss=0+0 | nref=1 0x1086010] 10.06.01_22:25:21.433754 7f018b477710 -- 172.16.2.9:6800/6204 <== mds1 172.16.2.10:6800/5990 2 ==== mds_resolve(2+0 subtrees +0 slave requests) v1 ==== 44+0+0 (284640460 0 0) 0x107f6b0 10.06.01_22:25:21.433768 7f018b477710 mds2.cache handle_resolve from mds1 10.06.01_22:25:21.433776 7f018b477710 mds2.cache show_subtrees 10.06.01_22:25:21.433784 7f018b477710 mds2.cache |__ 2 auth [dir 102 ~mds2/ [2,head] auth v=1 cv=0/0 dir_auth=2 state=1073741824 f(v0 m0.000000 2=1+1) s=2=1+1 rb=0/0 rf=1/1 rd=1/1 hs=0+0,ss=0+0 | nref=1 0x1086010] 10.06.01_22:25:21.433802 7f018b477710 mds2.cache maybe_resolve_finish got all resolves+resolve_acks, done. 10.06.01_22:25:21.433812 7f018b477710 mds2.cache disambiguate_imports 10.06.01_22:25:21.433819 7f018b477710 mds2.cache show_subtrees 10.06.01_22:25:21.433827 7f018b477710 mds2.cache |__ 2 auth [dir 102 ~mds2/ [2,head] auth v=1 cv=0/0 dir_auth=2 state=1073741824 f(v0 m0.000000 2=1+1) s=2=1+1 rb=0/0 rf=1/1 rd=1/1 hs=0+0,ss=0+0 | nref=1 0x1086010] 10.06.01_22:25:21.433842 7f018b477710 mds2.cache trim_unlinked_inodes 10.06.01_22:25:21.433851 7f018b477710 mds2.cache recalc_auth_bits 10.06.01_22:25:21.433858 7f018b477710 mds2.cache subtree auth=1 for [dir 102 ~mds2/ [2,head] auth v=1 cv=0/0 dir_auth=2 state=1073741824 f(v0 m0.000000 2=1+1) s=2=1+1 rb=0/0 rf=1/1 rd=1/1 hs=0+0,ss=0+0 | nref=1 0x1086010] 10.06.01_22:25:21.433872 7f018b477710 mds2.cache show_subtrees 10.06.01_22:25:21.433879 7f018b477710 mds2.cache |__ 2 auth [dir 102 ~mds2/ [2,head] auth v=1 cv=0/0 dir_auth=2 state=1073741824 f(v0 m0.000000 2=1+1) s=2=1+1 rb=0/0 rf=1/1 rd=1/1 hs=0+0,ss=0+0 | nref=1 0x1086010] 10.06.01_22:25:21.433893 7f018b477710 mds2.cache show_cache 10.06.01_22:25:21.433899 7f018b477710 mds2.cache unlinked [inode 102 [...2,head] ~mds2/ auth v1 snaprealm=0x1082090 f(v0 m0.000000 0=0+0) ds=0=0+0 rb=0 rf=0 rd=0 (iauth sync) (ilink sync) (idft sync) (isnap sync) (inest sync) (ifile sync) (ixattr sync) (iversion lock) | nref=1 0x7f0184010850] 10.06.01_22:25:21.433925 7f018b477710 mds2.cache dirfrag [dir 102 ~mds2/ [2,head] auth v=1 cv=0/0 dir_auth=2 state=1073741824 f(v0 m0.000000 2=1+1) s=2=1+1 rb=0/0 rf=1/1 rd=1/1 hs=0+0,ss=0+0 | nref=1 0x1086010] 10.06.01_22:25:21.433939 7f018b477710 mds2.cache unlinked [inode 1 [...2,head] / rep@0.0 v1 snaprealm=0x7f0184001fe0 f(v0 m0.000000 0=0+0) ds=0=0+0 rb=0 rf=0 rd=0 (iauth sync) (ilink sync) (idft sync) (isnap sync) (inest sync) (ifile sync) (ixattr sync) (iversion lock) 0x7f0184010230] 10.06.01_22:25:21.433963 7f018b477710 mds2.cache trim_non_auth 10.06.01_22:25:21.433970 7f018b477710 mds2.cache ... [inode 1 [...2,head] / rep@0.0 v1 snaprealm=0x7f0184001fe0 f(v0 m0.000000 0=0+0) ds=0=0+0 rb=0 rf=0 rd=0 (iauth sync) (ilink sync) (idft sync) (isnap sync) (inest sync) (ifile sync) (ixattr sync) (iversion lock) 0x7f0184010230] 10.06.01_22:25:21.433986 7f018b477710 mds2.cache remove_inode [inode 1 [...2,head] / rep@0.0 v1 snaprealm=0x7f0184001fe0 f(v0 m0.000000 0=0+0) ds=0=0+0 rb=0 rf=0 rd=0 (iauth sync) (ilink sync) (idft sync) (isnap sync) (inest sync) (ifile sync) (ixattr sync) (iversion lock) 0x7f0184010230] 10.06.01_22:25:21.434004 7f018b477710 mds2.cache.ino(1) close_snaprealm snaprealm(1 seq 1 lc 0 snaps={} 0x7f0184001fe0) 10.06.01_22:25:21.434019 7f018b477710 mds2.cache show_subtrees 10.06.01_22:25:21.434028 7f018b477710 mds2.cache |__ 2 auth [dir 102 ~mds2/ [2,head] auth v=1 cv=0/0 dir_auth=2 state=1073741824 f(v0 m0.000000 2=1+1) s=2=1+1 rb=0/0 rf=1/1 rd=1/1 hs=0+0,ss=0+0 | nref=1 0x1086010] 10.06.01_22:25:21.434042 7f018b477710 mds2.12 resolve_done 10.06.01_22:25:21.434048 7f018b477710 mds2.12 request_state up:reconnect 10.06.01_22:25:21.434054 7f018b477710 mds2.12 beacon_send up:reconnect seq 6 (currently up:resolve) 10.06.01_22:25:21.434062 7f018b477710 -- 172.16.2.9:6800/6204 --> mon1 172.16.2.10:6789/0 -- mdsbeacon(11227/r1-9 up:reconnect seq 6 v1344) v1 -- ?+0 0x7f0184001ac0 10.06.01_22:25:21.463669 7f0189473710 mds2.bal get_load no root, no load 10.06.01_22:25:21.473238 7f0189473710 mds2.bal get_load mdsload<[0,0 0]/[0,0 0], req 0, hr 0, qlen 0, cpu 0.58> 10.06.01_22:25:21.492839 7f01807f8710 -- 172.16.2.9:6800/6204 >> 172.16.2.9:6800/6024 pipe(0x7f018401e500 sd=16 pgs=0 cs=0 l=0).connect claims to be 172.16.2.9:6800/6204 not 172.16.2.9:6800/6024 - wrong node! 10.06.01_22:25:21.493318 7f0180ff9710 -- 172.16.2.9:6800/6204 >> 172.16.2.10:6800/5796 pipe(0x7f018401c8d0 sd=16 pgs=0 cs=0 l=0).connect claims to be 172.16.2.10:6800/5990 not 172.16.2.10:6800/5796 - wrong node! 10.06.01_22:25:21.625560 7f018b477710 -- 172.16.2.9:6800/6204 <== mon1 172.16.2.10:6789/0 21 ==== mdsmap(e 1344) v1 ==== 2070+0+0 (711823579 0 0) 0x107f360 10.06.01_22:25:21.625581 7f018b477710 mds2.12 handle_mds_map epoch 1344 from mon1 10.06.01_22:25:21.625588 7f018b477710 mds2.12 old map epoch 1344 <= 1344, discarding 10.06.01_22:25:21.768856 7f018b477710 -- 172.16.2.9:6800/6204 <== mon1 172.16.2.10:6789/0 22 ==== mdsmap(e 1345) v1 ==== 2070+0+0 (1232947994 0 0) 0x107f360 10.06.01_22:25:21.768878 7f018b477710 mds2.12 handle_mds_map epoch 1345 from mon1 10.06.01_22:25:21.768921 7f018b477710 mds2.12 my compat compat={},rocompat={},incompat={1=base v0.20} 10.06.01_22:25:21.768930 7f018b477710 mds2.12 mdsmap compat compat={},rocompat={},incompat={1=base v0.20} 10.06.01_22:25:21.768938 7f018b477710 mds2.12 map says i am 172.16.2.9:6800/6204 mds2 state up:resolve 10.06.01_22:25:21.768950 7f018b477710 mds2.12 resolve set is 1,2, was 0,1,2 10.06.01_22:25:21.768964 7f018b477710 mds2.bal check_targets have need want 10.06.01_22:25:21.768973 7f018b477710 mds2.bal map: i imported [dir 102 ~mds2/ [2,head] auth v=1 cv=0/0 dir_auth=2 state=1073741824 f(v0 m0.000000 2=1+1) s=2=1+1 rb=0/0 rf=1/1 rd=1/1 hs=0+0,ss=0+0 | nref=1 0x1086010] from 2 10.06.01_22:25:21.768989 7f018b477710 mds2.bal rebalance done 10.06.01_22:25:21.769006 7f018b477710 mds2.cache show_subtrees 10.06.01_22:25:21.769015 7f018b477710 mds2.cache |__ 2 auth [dir 102 ~mds2/ [2,head] auth v=1 cv=0/0 dir_auth=2 state=1073741824 f(v0 m0.000000 2=1+1) s=2=1+1 rb=0/0 rf=1/1 rd=1/1 hs=0+0,ss=0+0 | nref=1 0x1086010] 10.06.01_22:25:21.769130 7f018b477710 -- 172.16.2.9:6800/6204 <== mon1 172.16.2.10:6789/0 23 ==== mdsbeacon(11227/r1-9 up:reconnect seq 6 v1345) v1 ==== 70+0+0 (2812590400 0 0) 0x107f6b0 10.06.01_22:25:21.769146 7f018b477710 mds2.12 handle_mds_beacon up:reconnect seq 6 rtt 0.335085 10.06.01_22:25:25.434150 7f0189473710 mds2.12 beacon_send up:reconnect seq 7 (currently up:resolve) 10.06.01_22:25:25.434175 7f0189473710 -- 172.16.2.9:6800/6204 --> mon1 172.16.2.10:6789/0 -- mdsbeacon(11227/r1-9 up:reconnect seq 7 v1345) v1 -- ?+0 0x7f018400e870 10.06.01_22:25:25.664469 7f018b477710 -- 172.16.2.9:6800/6204 <== mon1 172.16.2.10:6789/0 24 ==== mdsmap(e 1346) v1 ==== 2070+0+0 (1836856898 0 0) 0x107f360 10.06.01_22:25:25.664495 7f018b477710 mds2.12 handle_mds_map epoch 1346 from mon1 10.06.01_22:25:25.664528 7f018b477710 mds2.12 my compat compat={},rocompat={},incompat={1=base v0.20} 10.06.01_22:25:25.664538 7f018b477710 mds2.12 mdsmap compat compat={},rocompat={},incompat={1=base v0.20} 10.06.01_22:25:25.664545 7f018b477710 mds2.12 map says i am 172.16.2.9:6800/6204 mds2 state up:reconnect 10.06.01_22:25:25.664554 7f018b477710 mds2.12 handle_mds_map state change up:resolve --> up:reconnect 10.06.01_22:25:25.664561 7f018b477710 mds2.12 reconnect_start 10.06.01_22:25:25.664572 7f018b477710 mds2.server reconnect_clients -- no sessions, doing nothing. 10.06.01_22:25:25.664580 7f018b477710 mds2.server reconnect_gather_finish. failed on 0 clients 10.06.01_22:25:25.664587 7f018b477710 mds2.12 reconnect_done 10.06.01_22:25:25.664593 7f018b477710 mds2.12 request_state up:rejoin 10.06.01_22:25:25.664598 7f018b477710 mds2.12 beacon_send up:rejoin seq 8 (currently up:reconnect) 10.06.01_22:25:25.664606 7f018b477710 -- 172.16.2.9:6800/6204 --> mon1 172.16.2.10:6789/0 -- mdsbeacon(11227/r1-9 up:rejoin seq 8 v1346) v1 -- ?+0 0x7f01840013d0 10.06.01_22:25:25.664645 7f018b477710 mds2.bal check_targets have need want 10.06.01_22:25:25.664657 7f018b477710 mds2.bal map: i imported [dir 102 ~mds2/ [2,head] auth v=1 cv=0/0 dir_auth=2 state=1073741824 f(v0 m0.000000 2=1+1) s=2=1+1 rb=0/0 rf=1/1 rd=1/1 hs=0+0,ss=0+0 | nref=1 0x1086010] from 2 10.06.01_22:25:25.664674 7f018b477710 mds2.bal rebalance done 10.06.01_22:25:25.664681 7f018b477710 mds2.cache show_subtrees 10.06.01_22:25:25.664691 7f018b477710 mds2.cache |__ 2 auth [dir 102 ~mds2/ [2,head] auth v=1 cv=0/0 dir_auth=2 state=1073741824 f(v0 m0.000000 2=1+1) s=2=1+1 rb=0/0 rf=1/1 rd=1/1 hs=0+0,ss=0+0 | nref=1 0x1086010] 10.06.01_22:25:25.834362 7f018b477710 -- 172.16.2.9:6800/6204 <== mon1 172.16.2.10:6789/0 25 ==== mdsmap(e 1347) v1 ==== 2070+0+0 (2164141388 0 0) 0x107f360 10.06.01_22:25:25.834385 7f018b477710 mds2.12 handle_mds_map epoch 1347 from mon1 10.06.01_22:25:25.834409 7f018b477710 mds2.12 my compat compat={},rocompat={},incompat={1=base v0.20} 10.06.01_22:25:25.834421 7f018b477710 mds2.12 mdsmap compat compat={},rocompat={},incompat={1=base v0.20} 10.06.01_22:25:25.834428 7f018b477710 mds2.12 map says i am 172.16.2.9:6800/6204 mds2 state up:rejoin 10.06.01_22:25:25.834437 7f018b477710 mds2.12 handle_mds_map state change up:reconnect --> up:rejoin 10.06.01_22:25:25.834451 7f018b477710 mds2.bal check_targets have need want 10.06.01_22:25:25.834460 7f018b477710 mds2.bal map: i imported [dir 102 ~mds2/ [2,head] auth v=1 cv=0/0 dir_auth=2 state=1073741824 f(v0 m0.000000 2=1+1) s=2=1+1 rb=0/0 rf=1/1 rd=1/1 hs=0+0,ss=0+0 | nref=1 0x1086010] from 2 10.06.01_22:25:25.834474 7f018b477710 mds2.bal rebalance done 10.06.01_22:25:25.834480 7f018b477710 mds2.cache show_subtrees 10.06.01_22:25:25.834489 7f018b477710 mds2.cache |__ 2 auth [dir 102 ~mds2/ [2,head] auth v=1 cv=0/0 dir_auth=2 state=1073741824 f(v0 m0.000000 2=1+1) s=2=1+1 rb=0/0 rf=1/1 rd=1/1 hs=0+0,ss=0+0 | nref=1 0x1086010] 10.06.01_22:25:26.463729 7f0189473710 mds2.bal get_load no root, no load 10.06.01_22:25:26.463790 7f0189473710 mds2.bal get_load mdsload<[0,0 0]/[0,0 0], req 0, hr 0, qlen 0, cpu 0.53> 10.06.01_22:25:27.893051 7f01807f8710 -- 172.16.2.9:6800/6204 >> 172.16.2.9:6800/6024 pipe(0x7f018401e500 sd=16 pgs=0 cs=0 l=0).connect claims to be 172.16.2.9:6800/6204 not 172.16.2.9:6800/6024 - wrong node! 10.06.01_22:25:27.893679 7f0180ff9710 -- 172.16.2.9:6800/6204 >> 172.16.2.10:6800/5796 pipe(0x7f018401c8d0 sd=16 pgs=0 cs=0 l=0).connect claims to be 172.16.2.10:6800/5990 not 172.16.2.10:6800/5796 - wrong node! 10.06.01_22:25:29.664690 7f0189473710 mds2.12 beacon_send up:rejoin seq 9 (currently up:rejoin) 10.06.01_22:25:29.664709 7f0189473710 -- 172.16.2.9:6800/6204 --> mon1 172.16.2.10:6789/0 -- mdsbeacon(11227/r1-9 up:rejoin seq 9 v1347) v1 -- ?+0 0x7f018400e870 10.06.01_22:25:29.680427 7f018b477710 -- 172.16.2.9:6800/6204 <== mon1 172.16.2.10:6789/0 26 ==== mdsmap(e 1348) v1 ==== 2070+0+0 (1900786125 0 0) 0x107f360 10.06.01_22:25:29.680452 7f018b477710 mds2.12 handle_mds_map epoch 1348 from mon1 10.06.01_22:25:29.680484 7f018b477710 mds2.12 my compat compat={},rocompat={},incompat={1=base v0.20} 10.06.01_22:25:29.680493 7f018b477710 mds2.12 mdsmap compat compat={},rocompat={},incompat={1=base v0.20} 10.06.01_22:25:29.680501 7f018b477710 mds2.12 map says i am 172.16.2.9:6800/6204 mds2 state up:rejoin 10.06.01_22:25:29.680512 7f018b477710 mds2.12 resolve set is , was 1 10.06.01_22:25:29.680525 7f018b477710 mds2.bal check_targets have need want 10.06.01_22:25:29.680535 7f018b477710 mds2.bal map: i imported [dir 102 ~mds2/ [2,head] auth v=1 cv=0/0 dir_auth=2 state=1073741824 f(v0 m0.000000 2=1+1) s=2=1+1 rb=0/0 rf=1/1 rd=1/1 hs=0+0,ss=0+0 | nref=1 0x1086010] from 2 10.06.01_22:25:29.680551 7f018b477710 mds2.bal rebalance done 10.06.01_22:25:29.680558 7f018b477710 mds2.cache show_subtrees 10.06.01_22:25:29.680570 7f018b477710 mds2.cache |__ 2 auth [dir 102 ~mds2/ [2,head] auth v=1 cv=0/0 dir_auth=2 state=1073741824 f(v0 m0.000000 2=1+1) s=2=1+1 rb=0/0 rf=1/1 rd=1/1 hs=0+0,ss=0+0 | nref=1 0x1086010] 10.06.01_22:25:29.680683 7f018b477710 -- 172.16.2.9:6800/6204 <== mon1 172.16.2.10:6789/0 27 ==== mdsbeacon(11227/r1-9 up:rejoin seq 9 v1348) v1 ==== 70+0+0 (3636121626 0 0) 0x1081df0 10.06.01_22:25:29.680699 7f018b477710 mds2.12 handle_mds_beacon up:rejoin seq 9 rtt 0.015992 10.06.01_22:25:31.463786 7f0189473710 mds2.bal get_load no root, no load 10.06.01_22:25:31.463828 7f0189473710 mds2.bal get_load mdsload<[0,0 0]/[0,0 0], req 0, hr 0, qlen 0, cpu 0.49> 10.06.01_22:25:33.664788 7f0189473710 mds2.12 beacon_send up:rejoin seq 10 (currently up:rejoin) 10.06.01_22:25:33.664804 7f0189473710 -- 172.16.2.9:6800/6204 --> mon1 172.16.2.10:6789/0 -- mdsbeacon(11227/r1-9 up:rejoin seq 10 v1348) v1 -- ?+0 0x7f0184002b80 10.06.01_22:25:33.665337 7f018b477710 -- 172.16.2.9:6800/6204 <== mon1 172.16.2.10:6789/0 28 ==== mdsbeacon(11227/r1-9 up:rejoin seq 10 v1348) v1 ==== 70+0+0 (327606649 0 0) 0x107f610 10.06.01_22:25:33.665363 7f018b477710 mds2.12 handle_mds_beacon up:rejoin seq 10 rtt 0.000559 10.06.01_22:25:36.463844 7f0189473710 mds2.bal get_load no root, no load 10.06.01_22:25:36.463885 7f0189473710 mds2.bal get_load mdsload<[0,0 0]/[0,0 0], req 0, hr 0, qlen 0, cpu 0.45> 10.06.01_22:25:37.664881 7f0189473710 mds2.12 beacon_send up:rejoin seq 11 (currently up:rejoin) 10.06.01_22:25:37.664896 7f0189473710 -- 172.16.2.9:6800/6204 --> mon1 172.16.2.10:6789/0 -- mdsbeacon(11227/r1-9 up:rejoin seq 11 v1348) v1 -- ?+0 0x7f018400e870 10.06.01_22:25:37.707093 7f018b477710 -- 172.16.2.9:6800/6204 <== mon1 172.16.2.10:6789/0 29 ==== mdsbeacon(11227/r1-9 up:rejoin seq 11 v1348) v1 ==== 70+0+0 (2848531447 0 0) 0x107f610 10.06.01_22:25:37.707120 7f018b477710 mds2.12 handle_mds_beacon up:rejoin seq 11 rtt 0.042224 10.06.01_22:25:40.693279 7f01807f8710 -- 172.16.2.9:6800/6204 >> 172.16.2.9:6800/6024 pipe(0x7f018401e500 sd=16 pgs=0 cs=0 l=0).connect claims to be 172.16.2.9:6800/6204 not 172.16.2.9:6800/6024 - wrong node! 10.06.01_22:25:40.694055 7f0180ff9710 -- 172.16.2.9:6800/6204 >> 172.16.2.10:6800/5796 pipe(0x7f018401c8d0 sd=16 pgs=0 cs=0 l=0).connect claims to be 172.16.2.10:6800/5990 not 172.16.2.10:6800/5796 - wrong node! 10.06.01_22:25:41.463904 7f0189473710 mds2.bal get_load no root, no load 10.06.01_22:25:41.463944 7f0189473710 mds2.bal get_load mdsload<[0,0 0]/[0,0 0], req 0, hr 0, qlen 0, cpu 0.41> 10.06.01_22:25:41.664974 7f0189473710 mds2.12 beacon_send up:rejoin seq 12 (currently up:rejoin) 10.06.01_22:25:41.664986 7f0189473710 -- 172.16.2.9:6800/6204 --> mon1 172.16.2.10:6789/0 -- mdsbeacon(11227/r1-9 up:rejoin seq 12 v1348) v1 -- ?+0 0x7f0184002b80 10.06.01_22:25:41.665534 7f018b477710 -- 172.16.2.9:6800/6204 <== mon1 172.16.2.10:6789/0 30 ==== mdsbeacon(11227/r1-9 up:rejoin seq 12 v1348) v1 ==== 70+0+0 (2148726094 0 0) 0x107f610 10.06.01_22:25:41.665558 7f018b477710 mds2.12 handle_mds_beacon up:rejoin seq 12 rtt 0.000572 10.06.01_22:25:45.665065 7f0189473710 mds2.12 beacon_send up:rejoin seq 13 (currently up:rejoin) 10.06.01_22:25:45.665086 7f0189473710 -- 172.16.2.9:6800/6204 --> mon1 172.16.2.10:6789/0 -- mdsbeacon(11227/r1-9 up:rejoin seq 13 v1348) v1 -- ?+0 0x7f018400e870 10.06.01_22:25:45.665631 7f018b477710 -- 172.16.2.9:6800/6204 <== mon1 172.16.2.10:6789/0 31 ==== mdsbeacon(11227/r1-9 up:rejoin seq 13 v1348) v1 ==== 70+0+0 (979187648 0 0) 0x107f610 10.06.01_22:25:45.665658 7f018b477710 mds2.12 handle_mds_beacon up:rejoin seq 13 rtt 0.000574 10.06.01_22:25:46.463961 7f0189473710 mds2.bal get_load no root, no load 10.06.01_22:25:46.464002 7f0189473710 mds2.bal get_load mdsload<[0,0 0]/[0,0 0], req 0, hr 0, qlen 0, cpu 0.38> 10.06.01_22:25:49.665166 7f0189473710 mds2.12 beacon_send up:rejoin seq 14 (currently up:rejoin) 10.06.01_22:25:49.665182 7f0189473710 -- 172.16.2.9:6800/6204 --> mon1 172.16.2.10:6789/0 -- mdsbeacon(11227/r1-9 up:rejoin seq 14 v1348) v1 -- ?+0 0x7f0184002b80 10.06.01_22:25:49.830056 7f018b477710 -- 172.16.2.9:6800/6204 <== mon1 172.16.2.10:6789/0 32 ==== mdsbeacon(11227/r1-9 up:rejoin seq 14 v1348) v1 ==== 70+0+0 (4049667747 0 0) 0x107f610 10.06.01_22:25:49.830084 7f018b477710 mds2.12 handle_mds_beacon up:rejoin seq 14 rtt 0.164904 10.06.01_22:25:51.464019 7f0189473710 mds2.bal get_load no root, no load 10.06.01_22:25:51.464057 7f0189473710 mds2.bal get_load mdsload<[0,0 0]/[0,0 0], req 0, hr 0, qlen 0, cpu 0.51> 10.06.01_22:25:53.665259 7f0189473710 mds2.12 beacon_send up:rejoin seq 15 (currently up:rejoin) 10.06.01_22:25:53.665274 7f0189473710 -- 172.16.2.9:6800/6204 --> mon1 172.16.2.10:6789/0 -- mdsbeacon(11227/r1-9 up:rejoin seq 15 v1348) v1 -- ?+0 0x7f018400e870 10.06.01_22:25:53.665809 7f018b477710 -- 172.16.2.9:6800/6204 <== mon1 172.16.2.10:6789/0 33 ==== mdsbeacon(11227/r1-9 up:rejoin seq 15 v1348) v1 ==== 70+0+0 (1261356077 0 0) 0x107f610 10.06.01_22:25:53.665834 7f018b477710 mds2.12 handle_mds_beacon up:rejoin seq 15 rtt 0.000562 10.06.01_22:25:55.693505 7f01807f8710 -- 172.16.2.9:6800/6204 >> 172.16.2.9:6800/6024 pipe(0x7f018401e500 sd=16 pgs=0 cs=0 l=0).connect claims to be 172.16.2.9:6800/6204 not 172.16.2.9:6800/6024 - wrong node! 10.06.01_22:25:55.694427 7f0180ff9710 -- 172.16.2.9:6800/6204 >> 172.16.2.10:6800/5796 pipe(0x7f018401c8d0 sd=16 pgs=0 cs=0 l=0).connect claims to be 172.16.2.10:6800/5990 not 172.16.2.10:6800/5796 - wrong node! 10.06.01_22:25:56.464077 7f0189473710 mds2.bal get_load no root, no load 10.06.01_22:25:56.464115 7f0189473710 mds2.bal get_load mdsload<[0,0 0]/[0,0 0], req 0, hr 0, qlen 0, cpu 0.47> 10.06.01_22:25:57.665350 7f0189473710 mds2.12 beacon_send up:rejoin seq 16 (currently up:rejoin) 10.06.01_22:25:57.665364 7f0189473710 -- 172.16.2.9:6800/6204 --> mon1 172.16.2.10:6789/0 -- mdsbeacon(11227/r1-9 up:rejoin seq 16 v1348) v1 -- ?+0 0x7f0184002b80 10.06.01_22:25:57.665893 7f018b477710 -- 172.16.2.9:6800/6204 <== mon1 172.16.2.10:6789/0 34 ==== mdsbeacon(11227/r1-9 up:rejoin seq 16 v1348) v1 ==== 70+0+0 (662548394 0 0) 0x107f610 10.06.01_22:25:57.665918 7f018b477710 mds2.12 handle_mds_beacon up:rejoin seq 16 rtt 0.000555 10.06.01_22:26:01.464136 7f0189473710 mds2.bal get_load no root, no load 10.06.01_22:26:01.464173 7f0189473710 mds2.bal get_load mdsload<[0,0 0]/[0,0 0], req 0, hr 0, qlen 0, cpu 0.43> 10.06.01_22:26:01.665440 7f0189473710 mds2.12 beacon_send up:rejoin seq 17 (currently up:rejoin) 10.06.01_22:26:01.665455 7f0189473710 -- 172.16.2.9:6800/6204 --> mon1 172.16.2.10:6789/0 -- mdsbeacon(11227/r1-9 up:rejoin seq 17 v1348) v1 -- ?+0 0x7f018400e870 10.06.01_22:26:01.862642 7f018b477710 -- 172.16.2.9:6800/6204 <== mon1 172.16.2.10:6789/0 35 ==== mdsbeacon(11227/r1-9 up:rejoin seq 17 v1348) v1 ==== 70+0+0 (2637327652 0 0) 0x107f610 10.06.01_22:26:01.862671 7f018b477710 mds2.12 handle_mds_beacon up:rejoin seq 17 rtt 0.197217 10.06.01_22:26:05.665534 7f0189473710 mds2.12 beacon_send up:rejoin seq 18 (currently up:rejoin) 10.06.01_22:26:05.665549 7f0189473710 -- 172.16.2.9:6800/6204 --> mon1 172.16.2.10:6789/0 -- mdsbeacon(11227/r1-9 up:rejoin seq 18 v1348) v1 -- ?+0 0x7f0184002b80 10.06.01_22:26:05.666100 7f018b477710 -- 172.16.2.9:6800/6204 <== mon1 172.16.2.10:6789/0 36 ==== mdsbeacon(11227/r1-9 up:rejoin seq 18 v1348) v1 ==== 70+0+0 (1443781703 0 0) 0x107f610 10.06.01_22:26:05.666125 7f018b477710 mds2.12 handle_mds_beacon up:rejoin seq 18 rtt 0.000576 10.06.01_22:26:06.464193 7f0189473710 mds2.bal get_load no root, no load 10.06.01_22:26:06.464232 7f0189473710 mds2.bal get_load mdsload<[0,0 0]/[0,0 0], req 0, hr 0, qlen 0, cpu 0.4> 10.06.01_22:26:09.665627 7f0189473710 mds2.12 beacon_send up:rejoin seq 19 (currently up:rejoin) 10.06.01_22:26:09.665642 7f0189473710 -- 172.16.2.9:6800/6204 --> mon1 172.16.2.10:6789/0 -- mdsbeacon(11227/r1-9 up:rejoin seq 19 v1348) v1 -- ?+0 0x7f018400e870 10.06.01_22:26:09.666181 7f018b477710 -- 172.16.2.9:6800/6204 <== mon1 172.16.2.10:6789/0 37 ==== mdsbeacon(11227/r1-9 up:rejoin seq 19 v1348) v1 ==== 70+0+0 (3963723465 0 0) 0x107f610 10.06.01_22:26:09.666207 7f018b477710 mds2.12 handle_mds_beacon up:rejoin seq 19 rtt 0.000567 10.06.01_22:26:10.693731 7f01807f8710 -- 172.16.2.9:6800/6204 >> 172.16.2.9:6800/6024 pipe(0x7f018401e500 sd=16 pgs=0 cs=0 l=0).connect claims to be 172.16.2.9:6800/6204 not 172.16.2.9:6800/6024 - wrong node! 10.06.01_22:26:10.694804 7f0180ff9710 -- 172.16.2.9:6800/6204 >> 172.16.2.10:6800/5796 pipe(0x7f018401c8d0 sd=16 pgs=0 cs=0 l=0).connect claims to be 172.16.2.10:6800/5990 not 172.16.2.10:6800/5796 - wrong node! 10.06.01_22:26:11.464252 7f0189473710 mds2.bal get_load no root, no load 10.06.01_22:26:11.464290 7f0189473710 mds2.bal get_load mdsload<[0,0 0]/[0,0 0], req 0, hr 0, qlen 0, cpu 0.36> 10.06.01_22:26:11.890388 7f018b477710 -- 172.16.2.9:6800/6204 <== mon1 172.16.2.10:6789/0 38 ==== mdsmap(e 1349) v1 ==== 2070+0+0 (1287710205 0 0) 0x107f760 10.06.01_22:26:11.890416 7f018b477710 mds2.12 handle_mds_map epoch 1349 from mon1 10.06.01_22:26:11.890447 7f018b477710 mds2.12 my compat compat={},rocompat={},incompat={1=base v0.20} 10.06.01_22:26:11.890457 7f018b477710 mds2.12 mdsmap compat compat={},rocompat={},incompat={1=base v0.20} 10.06.01_22:26:11.890465 7f018b477710 mds2.12 map says i am 172.16.2.9:6800/6204 mds2 state up:rejoin 10.06.01_22:26:11.890484 7f018b477710 mds2.bal check_targets have need want 10.06.01_22:26:11.890495 7f018b477710 mds2.bal map: i imported [dir 102 ~mds2/ [2,head] auth v=1 cv=0/0 dir_auth=2 state=1073741824 f(v0 m0.000000 2=1+1) s=2=1+1 rb=0/0 rf=1/1 rd=1/1 hs=0+0,ss=0+0 | nref=1 0x1086010] from 2 10.06.01_22:26:11.890513 7f018b477710 mds2.bal rebalance done 10.06.01_22:26:11.890520 7f018b477710 mds2.cache show_subtrees 10.06.01_22:26:11.890530 7f018b477710 mds2.cache |__ 2 auth [dir 102 ~mds2/ [2,head] auth v=1 cv=0/0 dir_auth=2 state=1073741824 f(v0 m0.000000 2=1+1) s=2=1+1 rb=0/0 rf=1/1 rd=1/1 hs=0+0,ss=0+0 | nref=1 0x1086010] 10.06.01_22:26:13.665720 7f0189473710 mds2.12 beacon_send up:rejoin seq 20 (currently up:rejoin) 10.06.01_22:26:13.665738 7f0189473710 -- 172.16.2.9:6800/6204 --> mon1 172.16.2.10:6789/0 -- mdsbeacon(11227/r1-9 up:rejoin seq 20 v1349) v1 -- ?+0 0x7f0184002b80 10.06.01_22:26:13.845817 7f018b477710 -- 172.16.2.9:6800/6204 <== mon1 172.16.2.10:6789/0 39 ==== mdsbeacon(11227/r1-9 up:rejoin seq 20 v1349) v1 ==== 70+0+0 (3405788170 0 0) 0x107f760 10.06.01_22:26:13.845853 7f018b477710 mds2.12 handle_mds_beacon up:rejoin seq 20 rtt 0.180116 10.06.01_22:26:16.464310 7f0189473710 mds2.bal get_load no root, no load 10.06.01_22:26:16.464350 7f0189473710 mds2.bal get_load mdsload<[0,0 0]/[0,0 0], req 0, hr 0, qlen 0, cpu 0.33> 10.06.01_22:26:17.165245 7f018b477710 -- 172.16.2.9:6800/6204 <== mds0 172.16.2.11:6800/5997 3 ==== mdsmap(e 1350) v1 ==== 2070+0+0 (3201128017 0 0) 0x107f760 10.06.01_22:26:17.165276 7f018b477710 mds2.12 handle_mds_map epoch 1350 from mds0 10.06.01_22:26:17.165289 7f018b477710 mds2.12 peer mds0 has mdsmap epoch >= 1350 10.06.01_22:26:17.165316 7f018b477710 mds2.12 my compat compat={},rocompat={},incompat={1=base v0.20} 10.06.01_22:26:17.165325 7f018b477710 mds2.12 mdsmap compat compat={},rocompat={},incompat={1=base v0.20} 10.06.01_22:26:17.165332 7f018b477710 mds2.12 map says i am 172.16.2.9:6800/6204 mds2 state up:rejoin 10.06.01_22:26:17.165345 7f018b477710 mds2.12 rejoin_joint_start 10.06.01_22:26:17.165355 7f018b477710 mds2.cache rejoin_send_rejoins with recovery_set 0,1 10.06.01_22:26:17.165376 7f018b477710 -- 172.16.2.9:6800/6204 --> mds0 172.16.2.11:6800/5997 -- cache_rejoin weak v1 -- ?+0 0x1092320 10.06.01_22:26:17.165404 7f018b477710 -- 172.16.2.9:6800/6204 --> mds1 172.16.2.10:6800/5990 -- mdsmap(e 1350) v1 -- ?+0 0x1094010 10.06.01_22:26:17.165421 7f018b477710 -- 172.16.2.9:6800/6204 --> mds1 172.16.2.10:6800/5990 -- cache_rejoin weak v1 -- ?+0 0x1092860 10.06.01_22:26:17.165444 7f018b477710 mds2.bal check_targets have need want 10.06.01_22:26:17.165454 7f018b477710 mds2.bal map: i imported [dir 102 ~mds2/ [2,head] auth v=1 cv=0/0 dir_auth=2 state=1073741824 f(v0 m0.000000 2=1+1) s=2=1+1 rb=0/0 rf=1/1 rd=1/1 hs=0+0,ss=0+0 | nref=1 0x1086010] from 2 10.06.01_22:26:17.165470 7f018b477710 mds2.bal rebalance done 10.06.01_22:26:17.165477 7f018b477710 mds2.cache show_subtrees 10.06.01_22:26:17.165486 7f018b477710 mds2.cache |__ 2 auth [dir 102 ~mds2/ [2,head] auth v=1 cv=0/0 dir_auth=2 state=1073741824 f(v0 m0.000000 2=1+1) s=2=1+1 rb=0/0 rf=1/1 rd=1/1 hs=0+0,ss=0+0 | nref=1 0x1086010] 10.06.01_22:26:17.165529 7f018b477710 -- 172.16.2.9:6800/6204 <== mds0 172.16.2.11:6800/5997 4 ==== cache_rejoin weak v1 ==== 64+0+0 (4018619411 0 0) 0x1080ae0 10.06.01_22:26:17.165547 7f018b477710 mds2.cache handle_cache_rejoin cache_rejoin weak v1 from mds0 (64 bytes) 10.06.01_22:26:17.165559 7f018b477710 mds2.cache still need rejoin from (1) 10.06.01_22:26:17.165693 7f018b477710 -- 172.16.2.9:6800/6204 <== mds1 172.16.2.10:6800/5990 3 ==== mdsmap(e 1350) v1 ==== 2070+0+0 (3201128017 0 0) 0x107fb30 10.06.01_22:26:17.165709 7f018b477710 mds2.12 handle_mds_map epoch 1350 from mds1 10.06.01_22:26:17.165716 7f018b477710 mds2.12 old map epoch 1350 <= 1350, discarding 10.06.01_22:26:17.165726 7f018b477710 -- 172.16.2.9:6800/6204 <== mds1 172.16.2.10:6800/5990 4 ==== cache_rejoin weak v1 ==== 64+0+0 (4018619411 0 0) 0x1080ae0 10.06.01_22:26:17.165751 7f018b477710 mds2.cache handle_cache_rejoin cache_rejoin weak v1 from mds1 (64 bytes) 10.06.01_22:26:17.165761 7f018b477710 mds2.cache rejoin_gather_finish 10.06.01_22:26:17.165771 7f018b477710 mds2.cache rejoin_trim_undef_inodes 10.06.01_22:26:17.165779 7f018b477710 mds2.cache process_imported_caps 10.06.01_22:26:17.165787 7f018b477710 mds2.cache process_reconnected_caps 10.06.01_22:26:17.165795 7f018b477710 mds2.cache send_snaps 10.06.01_22:26:17.165801 7f018b477710 mds2.cache identify_files_to_recover 10.06.01_22:26:17.165811 7f018b477710 mds2.cache rejoin_send_acks 10.06.01_22:26:17.165820 7f018b477710 mds2.cache subtree [dir 102 ~mds2/ [2,head] auth v=1 cv=0/0 dir_auth=2 state=1073741824 f(v0 m0.000000 2=1+1) s=2=1+1 rb=0/0 rf=1/1 rd=1/1 hs=0+0,ss=0+0 | nref=1 0x1086010] 10.06.01_22:26:17.165839 7f018b477710 -- 172.16.2.9:6800/6204 --> mds0 172.16.2.11:6800/5997 -- cache_rejoin ack v1 -- ?+0 0x1092860 10.06.01_22:26:17.165855 7f018b477710 -- 172.16.2.9:6800/6204 --> mds1 172.16.2.10:6800/5990 -- cache_rejoin ack v1 -- ?+0 0x107eb40 10.06.01_22:26:17.166045 7f018b477710 -- 172.16.2.9:6800/6204 <== mds1 172.16.2.10:6800/5990 5 ==== cache_rejoin ack v1 ==== 56+0+0 (2805141753 0 0) 0x1080ae0 10.06.01_22:26:17.166060 7f018b477710 mds2.cache handle_cache_rejoin cache_rejoin ack v1 from mds1 (56 bytes) 10.06.01_22:26:17.166069 7f018b477710 mds2.cache handle_cache_rejoin_ack from mds1 10.06.01_22:26:17.166077 7f018b477710 mds2.cache inode_locks len 0 is buffer::list(len=0, ) 10.06.01_22:26:17.166089 7f018b477710 mds2.cache still need rejoin from (), rejoin_ack from (0) 10.06.01_22:26:17.166815 7f018b477710 -- 172.16.2.9:6800/6204 <== mds0 172.16.2.11:6800/5997 5 ==== cache_rejoin ack v1 ==== 56+0+0 (2805141753 0 0) 0x1080ae0 10.06.01_22:26:17.166829 7f018b477710 mds2.cache handle_cache_rejoin cache_rejoin ack v1 from mds0 (56 bytes) 10.06.01_22:26:17.166837 7f018b477710 mds2.cache handle_cache_rejoin_ack from mds0 10.06.01_22:26:17.166843 7f018b477710 mds2.cache inode_locks len 0 is buffer::list(len=0, ) 10.06.01_22:26:17.166854 7f018b477710 mds2.12 rejoin_done 10.06.01_22:26:17.166860 7f018b477710 mds2.cache show_subtrees 10.06.01_22:26:17.166869 7f018b477710 mds2.cache |__ 2 auth [dir 102 ~mds2/ [2,head] auth v=1 cv=0/0 dir_auth=2 state=1073741824 f(v0 m0.000000 2=1+1) s=2=1+1 rb=0/0 rf=1/1 rd=1/1 hs=0+0,ss=0+0 | nref=1 0x1086010] 10.06.01_22:26:17.166883 7f018b477710 mds2.cache show_cache 10.06.01_22:26:17.166889 7f018b477710 mds2.cache unlinked [inode 102 [...2,head] ~mds2/ auth v1 snaprealm=0x1082090 f(v0 m0.000000 0=0+0) ds=0=0+0 rb=0 rf=0 rd=0 (iauth sync) (ilink sync) (idft sync) (isnap sync) (inest sync) (ifile lock) (ixattr sync) (iversion lock) | nref=1 0x7f0184010850] 10.06.01_22:26:17.166908 7f018b477710 mds2.cache dirfrag [dir 102 ~mds2/ [2,head] auth v=1 cv=0/0 dir_auth=2 state=1073741824 f(v0 m0.000000 2=1+1) s=2=1+1 rb=0/0 rf=1/1 rd=1/1 hs=0+0,ss=0+0 | nref=1 0x1086010] 10.06.01_22:26:17.166922 7f018b477710 mds2.12 request_state up:active 10.06.01_22:26:17.166928 7f018b477710 mds2.12 beacon_send up:active seq 21 (currently up:rejoin) 10.06.01_22:26:17.166936 7f018b477710 -- 172.16.2.9:6800/6204 --> mon1 172.16.2.10:6789/0 -- mdsbeacon(11227/r1-9 up:active seq 21 v1350) v1 -- ?+0 0x107fb30 10.06.01_22:26:17.245199 7f018b477710 -- 172.16.2.9:6800/6204 <== mon1 172.16.2.10:6789/0 40 ==== mdsmap(e 1350) v1 ==== 2070+0+0 (3201128017 0 0) 0x107fb30 10.06.01_22:26:17.245220 7f018b477710 mds2.12 handle_mds_map epoch 1350 from mon1 10.06.01_22:26:17.245228 7f018b477710 mds2.12 old map epoch 1350 <= 1350, discarding 10.06.01_22:26:17.396568 7f017eff5710 -- 172.16.2.9:6800/6204 >> 172.16.2.10:6800/5990 pipe(0x1091a90 sd=-1 pgs=6 cs=1 l=0).fault with nothing to send, going to standby 10.06.01_22:26:17.472247 7f018b477710 -- 172.16.2.9:6800/6204 <== mon1 172.16.2.10:6789/0 41 ==== mdsmap(e 1351) v1 ==== 2070+0+0 (297154568 0 0) 0x107fb30 10.06.01_22:26:17.472269 7f018b477710 mds2.12 handle_mds_map epoch 1351 from mon1 10.06.01_22:26:17.472296 7f018b477710 mds2.12 my compat compat={},rocompat={},incompat={1=base v0.20} 10.06.01_22:26:17.472305 7f018b477710 mds2.12 mdsmap compat compat={},rocompat={},incompat={1=base v0.20} 10.06.01_22:26:17.472313 7f018b477710 mds2.12 map says i am 172.16.2.9:6800/6204 mds2 state up:rejoin 10.06.01_22:26:17.472334 7f018b477710 mds2.bal check_targets have need want 10.06.01_22:26:17.472344 7f018b477710 mds2.bal map: i imported [dir 102 ~mds2/ [2,head] auth v=1 cv=0/0 dir_auth=2 state=1073741824 f(v0 m0.000000 2=1+1) s=2=1+1 rb=0/0 rf=1/1 rd=1/1 hs=0+0,ss=0+0 | nref=1 0x1086010] from 2 10.06.01_22:26:17.472361 7f018b477710 mds2.bal rebalance done 10.06.01_22:26:17.472368 7f018b477710 mds2.cache show_subtrees 10.06.01_22:26:17.472377 7f018b477710 mds2.cache |__ 2 auth [dir 102 ~mds2/ [2,head] auth v=1 cv=0/0 dir_auth=2 state=1073741824 f(v0 m0.000000 2=1+1) s=2=1+1 rb=0/0 rf=1/1 rd=1/1 hs=0+0,ss=0+0 | nref=1 0x1086010] 10.06.01_22:26:17.472549 7f018b477710 -- 172.16.2.9:6800/6204 <== mon1 172.16.2.10:6789/0 42 ==== mdsbeacon(11227/r1-9 up:active seq 21 v1351) v1 ==== 70+0+0 (2922557403 0 0) 0x1080830 10.06.01_22:26:17.472572 7f018b477710 mds2.12 handle_mds_beacon up:active seq 21 rtt 0.305637 10.06.01_22:26:21.167024 7f0189473710 mds2.12 beacon_send up:active seq 22 (currently up:rejoin) 10.06.01_22:26:21.167048 7f0189473710 -- 172.16.2.9:6800/6204 --> mon1 172.16.2.10:6789/0 -- mdsbeacon(11227/r1-9 up:active seq 22 v1351) v1 -- ?+0 0x7f018400e870 10.06.01_22:26:21.464371 7f0189473710 mds2.bal get_load no root, no load 10.06.01_22:26:21.464416 7f0189473710 mds2.bal get_load mdsload<[0,0 0]/[0,0 0], req 0, hr 0, qlen 0, cpu 0.31> 10.06.01_22:26:21.761022 7f018b477710 -- 172.16.2.9:6800/6204 <== mon1 172.16.2.10:6789/0 43 ==== mdsmap(e 1352) v1 ==== 2070+0+0 (2702478598 0 0) 0x107fb30 10.06.01_22:26:21.761048 7f018b477710 mds2.12 handle_mds_map epoch 1352 from mon1 10.06.01_22:26:21.761078 7f018b477710 mds2.12 my compat compat={},rocompat={},incompat={1=base v0.20} 10.06.01_22:26:21.761087 7f018b477710 mds2.12 mdsmap compat compat={},rocompat={},incompat={1=base v0.20} 10.06.01_22:26:21.761094 7f018b477710 mds2.12 map says i am 172.16.2.9:6800/6204 mds2 state up:active 10.06.01_22:26:21.761103 7f018b477710 mds2.12 handle_mds_map state change up:rejoin --> up:active 10.06.01_22:26:21.761110 7f018b477710 mds2.12 recovery_done -- successful recovery! 10.06.01_22:26:21.761120 7f018b477710 mds2.tableclient(anchortable) finish_recovery 10.06.01_22:26:21.761127 7f018b477710 mds2.tableclient(snaptable) finish_recovery 10.06.01_22:26:21.761135 7f018b477710 mds2.cache start_recovered_truncates 10.06.01_22:26:21.761143 7f018b477710 mds2.cache do_file_recover 0 queued, 0 recovering 10.06.01_22:26:21.761149 7f018b477710 mds2.cache reissue_all_caps 10.06.01_22:26:21.761159 7f018b477710 mds2.cache populate_mydir [dir 102 ~mds2/ [2,head] auth v=1 cv=0/0 dir_auth=2 state=1073741824 f(v0 m0.000000 2=1+1) s=2=1+1 rb=0/0 rf=1/1 rd=1/1 hs=0+0,ss=0+0 | nref=1 0x1086010] 10.06.01_22:26:21.761177 7f018b477710 mds2.cache.dir(102) fetch on [dir 102 ~mds2/ [2,head] auth v=1 cv=0/0 dir_auth=2 state=1073741824 f(v0 m0.000000 2=1+1) s=2=1+1 rb=0/0 rf=1/1 rd=1/1 hs=0+0,ss=0+0 | nref=1 0x1086010] 10.06.01_22:26:21.761192 7f018b477710 10.06.01_22:26:21.761197 mds2.cache.dir(102) add_waiter 2 0x1077790 on [dir 102 ~mds2/ [2,head] auth v=1 cv=0/0 dir_auth=2 state=1073741824 f(v0 m0.000000 2=1+1) s=2=1+1 rb=0/0 rf=1/1 rd=1/1 hs=0+0,ss=0+0 | nref=2 0x1086010] 10.06.01_22:26:21.761211 7f018b477710 mds2.cache.dir(102) auth_pin by 0x1086010 on [dir 102 ~mds2/ [2,head] auth v=1 cv=0/0 dir_auth=2 ap=1+0+0 state=1073741824 f(v0 m0.000000 2=1+1) s=2=1+1 rb=0/0 rf=1/1 rd=1/1 hs=0+0,ss=0+0 | nref=3 0x1086010] count now 1 + 0 10.06.01_22:26:21.761243 7f018b477710 -- 172.16.2.9:6800/6204 --> osd1 172.16.2.10:6801/2787 -- osd_op(mds2.12:9 102.00000000 [tmapget 0~0] 1.1bfd) v1 -- ?+0 0x1091f40 10.06.01_22:26:21.761288 7f018b477710 mds2.12 active_start 10.06.01_22:26:21.761296 7f018b477710 mds2.cache clean_open_file_lists 10.06.01_22:26:21.761315 7f018b477710 mds2.bal check_targets have need want 10.06.01_22:26:21.761324 7f018b477710 mds2.bal map: i imported [dir 102 ~mds2/ [2,head] auth v=1 cv=0/0 dir_auth=2 ap=1+0+0 state=1073741952 f(v0 m0.000000 2=1+1) s=2=1+1 rb=0/0 rf=1/1 rd=1/1 hs=0+0,ss=0+0 | nref=3 0x1086010] from 2 10.06.01_22:26:21.761339 7f018b477710 mds2.bal rebalance done 10.06.01_22:26:21.761345 7f018b477710 mds2.cache show_subtrees 10.06.01_22:26:21.761354 7f018b477710 mds2.cache |__ 2 auth [dir 102 ~mds2/ [2,head] auth v=1 cv=0/0 dir_auth=2 ap=1+0+0 state=1073741952 f(v0 m0.000000 2=1+1) s=2=1+1 rb=0/0 rf=1/1 rd=1/1 hs=0+0,ss=0+0 | nref=3 0x1086010] 10.06.01_22:26:21.761593 7f017dff3710 mds2.12 MDS::ms_get_authorizer type=osd 10.06.01_22:26:21.761756 7f018b477710 mds2.12 ms_handle_connect on 172.16.2.10:6801/2787 10.06.01_22:26:21.762414 7f018b477710 -- 172.16.2.9:6800/6204 <== osd1 172.16.2.10:6801/2787 1 ==== osd_op_reply(9 102.00000000 [tmapget 0~0] = 0) v1 ==== 98+0+887 (1692991859 0 2985821500) 0x107fb30 10.06.01_22:26:21.762438 7f018b477710 mds2.cache.dir(102) _fetched 887 bytes for [dir 102 ~mds2/ [2,head] auth v=1 cv=0/0 dir_auth=2 ap=1+0+0 state=1073741952 f(v0 m0.000000 2=1+1) s=2=1+1 rb=0/0 rf=1/1 rd=1/1 hs=0+0,ss=0+0 | nref=3 0x1086010] 10.06.01_22:26:21.762465 7f018b477710 mds2.cache.dir(102) _fetched version 1, 887 bytes, 2 keys 10.06.01_22:26:21.762473 7f018b477710 mds2.cache.snaprealm(102 seq 1 0x1082090) have_past_parents_open [1,head] 10.06.01_22:26:21.762485 7f018b477710 mds2.cache.snaprealm(102 seq 1 0x1082090) build_snap_set [0,head] on snaprealm(102 seq 1 lc 0 snaps={} 0x1082090) 10.06.01_22:26:21.762494 7f018b477710 mds2.cache.snaprealm(102 seq 1 0x1082090) build_snap_trace my_snaps [] 10.06.01_22:26:21.762502 7f018b477710 mds2.cache.snaprealm(102 seq 1 0x1082090) check_cache rebuilt seq 1 cached_seq 1 cached_last_created 0 cached_last_destroyed 0) 10.06.01_22:26:21.762524 7f018b477710 mds2.cache.dir(102) lookup (head, 'journal') 10.06.01_22:26:21.762548 7f018b477710 mds2.cache.dir(102) add_primary_dentry [dentry #102/journal [2,head] auth (dn sync) (dversion lock) pv=0 v=1 inode=0x7f0184010230 0x1097010] 10.06.01_22:26:21.762566 7f018b477710 mds2.cache.dir(102) _fetched got [dentry #102/journal [2,head] auth (dn sync) (dversion lock) pv=0 v=1 inode=0x7f0184010230 0x1097010] [inode 202 [2,head] ~mds2/journal auth v1 s=0 nl=1 rb=0 rf=0 rd=0 (iauth sync) (ilink sync) (ifile sync) (ixattr sync) (iversion lock) 0x7f0184010230] 10.06.01_22:26:21.762589 7f018b477710 mds2.cache.dir(102) lookup (head, 'stray') 10.06.01_22:26:21.762604 7f018b477710 mds2.cache.dir(102) add_primary_dentry [dentry #102/stray [2,head] auth (dn sync) (dversion lock) pv=0 v=1 inode=0x7f0184010e70 0x10971b0] 10.06.01_22:26:21.762615 7f018b477710 mds2.cache.dir(102) _fetched got [dentry #102/stray [2,head] auth (dn sync) (dversion lock) pv=0 v=1 inode=0x7f0184010e70 0x10971b0] [inode 602 [...2,head] ~mds2/stray/ auth v1 f(v0 m0.000000 0=0+0) ds=0=0+0 rb=0 rf=0 rd=0 (iauth sync) (ilink sync) (idft sync) (isnap sync) (inest sync) (ifile sync) (ixattr sync) (iversion lock) 0x7f0184010e70] 10.06.01_22:26:21.762636 7f018b477710 mds2.cache.dir(102) auth_unpin by 0x1086010 on [dir 102 ~mds2/ [2,head] auth v=1 cv=0/0 dir_auth=2 state=1073741826|complete f(v0 m0.000000 2=1+1) s=2=1+1 rb=0/0 rf=1/1 rd=1/1 hs=2+0,ss=0+0 | nref=3 0x1086010] count now 0 + 0 10.06.01_22:26:21.762651 7f018b477710 mds2.cache.dir(102) finish_waiting mask 2 result 0 on [dir 102 ~mds2/ [2,head] auth v=1 cv=0/0 dir_auth=2 state=1073741826|complete f(v0 m0.000000 2=1+1) s=2=1+1 rb=0/0 rf=1/1 rd=1/1 hs=2+0,ss=0+0 | nref=3 0x1086010] 10.06.01_22:26:21.762664 7f018b477710 10.06.01_22:26:21.762666 mds2.cache.dir(102) take_waiting mask 2 took 0x1077790 tag 2 on [dir 102 ~mds2/ [2,head] auth v=1 cv=0/0 dir_auth=2 state=1073741826|complete f(v0 m0.000000 2=1+1) s=2=1+1 rb=0/0 rf=1/1 rd=1/1 hs=2+0,ss=0+0 | nref=3 0x1086010] 10.06.01_22:26:21.762682 7f018b477710 mds2.12 mds has 1 queued contexts 10.06.01_22:26:21.762688 7f018b477710 mds2.12 0x1077790 10.06.01_22:26:21.762695 7f018b477710 mds2.cache open_root 10.06.01_22:26:21.762703 7f018b477710 mds2.cache discover_base_ino 1 from mds0 10.06.01_22:26:21.762724 7f018b477710 -- 172.16.2.9:6800/6204 --> mds0 172.16.2.11:6800/5997 -- mdsmap(e 1352) v1 -- ?+0 0x107f760 10.06.01_22:26:21.762741 7f018b477710 -- 172.16.2.9:6800/6204 --> mds0 172.16.2.11:6800/5997 -- discover(1.0/0 ) v1 -- ?+0 0x107fb30 10.06.01_22:26:21.763178 7f018b477710 -- 172.16.2.9:6800/6204 <== mds0 172.16.2.11:6800/5997 6 ==== discover_reply(1) v1 ==== 676+0+0 (4233918993 0 0) 0x1080ae0 10.06.01_22:26:21.763198 7f018b477710 mds2.cache discover_reply discover_reply(1) v1 10.06.01_22:26:21.763220 7f018b477710 mds2.cache.ino(1) decode_snap_blob snaprealm(1 seq 1 lc 0 snaps={} 0x107fb30) 10.06.01_22:26:21.763235 7f018b477710 mds2.cache add_replica_inode added [inode 1 [...2,head] / rep@0.1 v454 snaprealm=0x107fb30 f(v3 m10.06.01_22:09:03.004840 3=1+2) ds=3=1+2 rb=428/0 rf=10/0 rd=24/1 (iauth sync) (ilink sync) (idft sync) (isnap sync) (inest mix) (ifile sync) (ixattr sync) (iversion lock) 0x7f0184011490] 10.06.01_22:26:21.763259 7f018b477710 mds2.cache discover_reply got base inode [inode 1 [...2,head] / rep@0.1 v454 snaprealm=0x107fb30 f(v3 m10.06.01_22:09:03.004840 3=1+2) ds=3=1+2 rb=428/0 rf=10/0 rd=24/1 (iauth sync) (ilink sync) (idft sync) (isnap sync) (inest mix) (ifile sync) (ixattr sync) (iversion lock) 0x7f0184011490] 10.06.01_22:26:21.763285 7f018b477710 mds2.cache adjust_subtree_auth -1,-2 -> 0,-2 on [dir 1 / [2,head] rep@0.1 state=0 f(v3 m10.06.01_22:09:03.004840 2=0+2) s=2=0+2 rb=428/428 rf=10/10 rd=24/24 hs=0+0,ss=0+0 0x10865d8] 10.06.01_22:26:21.763300 7f018b477710 mds2.cache show_subtrees 10.06.01_22:26:21.763309 7f018b477710 mds2.cache |__ 2 auth [dir 102 ~mds2/ [2,head] auth v=1 cv=0/0 dir_auth=2 state=1073741826|complete f(v0 m0.000000 2=1+1) s=2=1+1 rb=0/0 rf=1/1 rd=1/1 hs=2+0,ss=0+0 | nref=2 0x1086010] 10.06.01_22:26:21.763325 7f018b477710 mds2.cache current root is [dir 1 / [2,head] rep@0.1 state=0 f(v3 m10.06.01_22:09:03.004840 2=0+2) s=2=0+2 rb=428/428 rf=10/10 rd=24/24 hs=0+0,ss=0+0 | nref=1 0x10865d8] 10.06.01_22:26:21.763338 7f018b477710 mds2.cache.dir(1) setting dir_auth=0,-2 from -1,-2 on [dir 1 / [2,head] rep@0.1 state=0 f(v3 m10.06.01_22:09:03.004840 2=0+2) s=2=0+2 rb=428/428 rf=10/10 rd=24/24 hs=0+0,ss=0+0 | nref=1 0x10865d8] 10.06.01_22:26:21.763351 7f018b477710 mds2.cache.dir(1) new subtree root, adjusting auth_pins 10.06.01_22:26:21.763357 7f018b477710 mds2.cache show_subtrees 10.06.01_22:26:21.763366 7f018b477710 mds2.cache |__ 2 auth [dir 102 ~mds2/ [2,head] auth v=1 cv=0/0 dir_auth=2 state=1073741826|complete f(v0 m0.000000 2=1+1) s=2=1+1 rb=0/0 rf=1/1 rd=1/1 hs=2+0,ss=0+0 | nref=2 0x1086010] 10.06.01_22:26:21.763380 7f018b477710 mds2.cache |__ 0 rep [dir 1 / [2,head] rep@0.1 dir_auth=0 state=0 f(v3 m10.06.01_22:09:03.004840 2=0+2) s=2=0+2 rb=428/428 rf=10/10 rd=24/24 hs=0+0,ss=0+0 | nref=1 0x10865d8] 10.06.01_22:26:21.763395 7f018b477710 mds2.cache add_replica_dir added [dir 1 / [2,head] rep@0.1 dir_auth=0 state=0 f(v3 m10.06.01_22:09:03.004840 2=0+2) s=2=0+2 rb=428/428 rf=10/10 rd=24/24 hs=0+0,ss=0+0 | nref=1 0x10865d8] nonce 1 10.06.01_22:26:21.763412 7f018b477710 mds2.12 mds has 1 queued contexts 10.06.01_22:26:21.763418 7f018b477710 mds2.12 0x106ff90 10.06.01_22:26:21.763424 7f018b477710 mds2.cache open_root 10.06.01_22:26:21.763431 7f018b477710 mds2.cache populate_mydir [dir 102 ~mds2/ [2,head] auth v=1 cv=0/0 dir_auth=2 state=1073741826|complete f(v0 m0.000000 2=1+1) s=2=1+1 rb=0/0 rf=1/1 rd=1/1 hs=2+0,ss=0+0 | nref=2 0x1086010] 10.06.01_22:26:21.763445 7f018b477710 mds2.cache.dir(102) lookup (head, 'stray') 10.06.01_22:26:21.763451 7f018b477710 mds2.cache.dir(102) hit -> (stray,head) 10.06.01_22:26:21.763458 7f018b477710 mds2.cache.dir(102) lookup (head, 'journal') 10.06.01_22:26:21.763464 7f018b477710 mds2.cache.dir(102) hit -> (journal,head) 10.06.01_22:26:21.763470 7f018b477710 mds2.cache populate_mydir done 10.06.01_22:26:21.763476 7f018b477710 mds2.cache stray is [inode 602 [...2,head] ~mds2/stray/ auth v1 f(v0 m0.000000 0=0+0) ds=0=0+0 rb=0 rf=0 rd=0 (iauth sync) (ilink sync) (idft sync) (isnap sync) (inest sync) (ifile sync) (ixattr sync) (iversion lock) | nref=1 0x7f0184010e70] 10.06.01_22:26:25.167136 7f0189473710 mds2.12 beacon_send up:active seq 23 (currently up:active) 10.06.01_22:26:25.167160 7f0189473710 -- 172.16.2.9:6800/6204 --> mon1 172.16.2.10:6789/0 -- mdsbeacon(11227/r1-9 up:active seq 23 v1352) v1 -- ?+0 0x7f018400e870 10.06.01_22:26:25.167708 7f018b477710 -- 172.16.2.9:6800/6204 <== mon1 172.16.2.10:6789/0 44 ==== mdsbeacon(11227/r1-9 up:active seq 23 v1352) v1 ==== 70+0+0 (2186660480 0 0) 0x107f760 10.06.01_22:26:25.167727 7f018b477710 mds2.12 handle_mds_beacon up:active seq 23 rtt 0.000570 10.06.01_22:26:25.693980 7f01807f8710 -- 172.16.2.9:6800/6204 >> 172.16.2.9:6800/6024 pipe(0x7f018401e500 sd=16 pgs=0 cs=0 l=0).connect claims to be 172.16.2.9:6800/6204 not 172.16.2.9:6800/6024 - wrong node! 10.06.01_22:26:26.464434 7f0189473710 mds2.cache trim max=100000 cur=2 10.06.01_22:26:26.464451 7f0189473710 mds2.cache trim_client_leases 10.06.01_22:26:26.464817 7f0189473710 mds2.cache check_memory_usage total 271400, rss 2756, heap 180672, malloc 178 mmap 0, baseline 180672, buffers 24, max 1048576, 0 / 4 inodes have caps, 0 caps, 0 caps per inode 10.06.01_22:26:26.464836 7f0189473710 mds2.log trim 12 / 30 segments, 12 / -1 events, 0 (0) expiring, 0 (0) expired 10.06.01_22:26:26.464873 7f0189473710 mds2.bal get_load mdsload<[0,0 0]/[0,0 0], req 0, hr 0, qlen 0, cpu 0.28> 10.06.01_22:26:26.464902 7f0189473710 mds2.locker scatter_tick 10.06.01_22:26:26.464915 7f0189473710 mds2.server find_idle_sessions. laggy until 0.000000 10.06.01_22:26:26.464927 7f0189473710 mds2.bal tick last_sample now 10.06.01_22:26:26.464927 10.06.01_22:26:26.464937 7f0189473710 mds2.snap check_osd_map - version unchanged 10.06.01_22:26:29.167248 7f0189473710 mds2.12 beacon_send up:active seq 24 (currently up:active) 10.06.01_22:26:29.167268 7f0189473710 -- 172.16.2.9:6800/6204 --> mon1 172.16.2.10:6789/0 -- mdsbeacon(11227/r1-9 up:active seq 24 v1352) v1 -- ?+0 0x7f018400e870 10.06.01_22:26:29.167809 7f018b477710 -- 172.16.2.9:6800/6204 <== mon1 172.16.2.10:6789/0 45 ==== mdsbeacon(11227/r1-9 up:active seq 24 v1352) v1 ==== 70+0+0 (1806542204 0 0) 0x1091f40 10.06.01_22:26:29.167828 7f018b477710 mds2.12 handle_mds_beacon up:active seq 24 rtt 0.000561 10.06.01_22:26:31.464493 7f0189473710 mds2.cache trim max=100000 cur=2 10.06.01_22:26:31.464509 7f0189473710 mds2.cache trim_client_leases 10.06.01_22:26:31.464830 7f0189473710 mds2.cache check_memory_usage total 271400, rss 2932, heap 180672, malloc 178 mmap 0, baseline 180672, buffers 24, max 1048576, 0 / 4 inodes have caps, 0 caps, 0 caps per inode 10.06.01_22:26:31.464845 7f0189473710 mds2.log trim 12 / 30 segments, 12 / -1 events, 0 (0) expiring, 0 (0) expired 10.06.01_22:26:31.464878 7f0189473710 mds2.bal get_load mdsload<[0,0 0]/[0,0 0], req 0, hr 0, qlen 0, cpu 0.26> 10.06.01_22:26:31.464902 7f0189473710 mds2.locker scatter_tick 10.06.01_22:26:31.464911 7f0189473710 mds2.server find_idle_sessions. laggy until 0.000000 10.06.01_22:26:31.464919 7f0189473710 mds2.bal tick last_sample now 10.06.01_22:26:31.464918 10.06.01_22:26:31.464926 7f0189473710 mds2.snap check_osd_map - version unchanged 10.06.01_22:26:31.670495 7f018b477710 -- 172.16.2.9:6800/6204 <== mds0 172.16.2.11:6800/5997 7 ==== HB v1 ==== 293+0+0 (1616350805 0 0) 0x1080050 10.06.01_22:26:31.713498 7f018b477710 mds2.bal from mds0, new epoch 10.06.01_22:26:31.713526 7f018b477710 mds2.bal get_load mdsload<[0,0 0]/[0,0 0], req 0, hr 0, qlen 0, cpu 0.26> 10.06.01_22:26:31.713556 7f018b477710 mds2.bal mds2 epoch 1 load mdsload<[0,0 0]/[0,0 0], req 0, hr 0, qlen 0, cpu 0.26> 10.06.01_22:26:31.713577 7f018b477710 -- 172.16.2.9:6800/6204 --> mds0 172.16.2.11:6800/5997 -- HB v1 -- ?+0 0x1081b30 10.06.01_22:26:31.713598 7f018b477710 -- 172.16.2.9:6800/6204 --> mds1 172.16.2.10:6800/5990 -- HB v1 -- ?+0 0x107eb40 10.06.01_22:26:31.713616 7f018b477710 mds2.cache show_subtrees 10.06.01_22:26:31.713628 7f018b477710 mds2.cache |__ 2 auth [dir 102 ~mds2/ [2,head] auth v=1 cv=0/0 dir_auth=2 state=1073741826|complete f(v0 m0.000000 2=1+1) s=2=1+1 rb=0/0 rf=1/1 rd=1/1 hs=2+0,ss=0+0 | nref=2 0x1086010] 10.06.01_22:26:31.713646 7f018b477710 mds2.cache |__ 0 rep [dir 1 / [2,head] rep@0.1 dir_auth=0 state=0 f(v3 m10.06.01_22:09:03.004840 2=0+2) s=2=0+2 rb=428/428 rf=10/10 rd=24/24 hs=0+0,ss=0+0 | nref=1 0x10865d8] 10.06.01_22:26:31.713765 7f017e7f4710 -- 172.16.2.9:6800/6204 >> 172.16.2.10:6800/5990 pipe(0x1091a90 sd=-1 pgs=6 cs=2 l=0).fault first fault 10.06.01_22:26:33.167350 7f0189473710 mds2.12 beacon_send up:active seq 25 (currently up:active) 10.06.01_22:26:33.167368 7f0189473710 -- 172.16.2.9:6800/6204 --> mon1 172.16.2.10:6789/0 -- mdsbeacon(11227/r1-9 up:active seq 25 v1352) v1 -- ?+0 0x7f018400e870 10.06.01_22:26:33.279671 7f018b477710 -- 172.16.2.9:6800/6204 <== mon1 172.16.2.10:6789/0 46 ==== mdsbeacon(11227/r1-9 up:active seq 25 v1352) v1 ==== 70+0+0 (3521275890 0 0) 0x107f100 10.06.01_22:26:33.279692 7f018b477710 mds2.12 handle_mds_beacon up:active seq 25 rtt 0.112326 10.06.01_22:26:34.164406 7f018b477710 -- 172.16.2.9:6800/6204 <== mon1 172.16.2.10:6789/0 47 ==== mdsmap(e 1353) v1 ==== 2070+0+0 (1523680486 0 0) 0x107f730 10.06.01_22:26:34.164429 7f018b477710 mds2.12 handle_mds_map epoch 1353 from mon1 10.06.01_22:26:34.164460 7f018b477710 mds2.12 my compat compat={},rocompat={},incompat={1=base v0.20} 10.06.01_22:26:34.164469 7f018b477710 mds2.12 mdsmap compat compat={},rocompat={},incompat={1=base v0.20} 10.06.01_22:26:34.164477 7f018b477710 mds2.12 map says i am 172.16.2.9:6800/6204 mds2 state up:active 10.06.01_22:26:34.164496 7f018b477710 mds2.bal check_targets have need want 10.06.01_22:26:34.164506 7f018b477710 mds2.bal map: i imported [dir 102 ~mds2/ [2,head] auth v=1 cv=0/0 dir_auth=2 state=1073741826|complete f(v0 m0.000000 2=1+1) s=2=1+1 rb=0/0 rf=1/1 rd=1/1 hs=2+0,ss=0+0 | nref=2 0x1086010] from 2 10.06.01_22:26:34.164522 7f018b477710 mds2.bal rebalance done 10.06.01_22:26:34.164530 7f018b477710 mds2.cache show_subtrees 10.06.01_22:26:34.164539 7f018b477710 mds2.cache |__ 2 auth [dir 102 ~mds2/ [2,head] auth v=1 cv=0/0 dir_auth=2 state=1073741826|complete f(v0 m0.000000 2=1+1) s=2=1+1 rb=0/0 rf=1/1 rd=1/1 hs=2+0,ss=0+0 | nref=2 0x1086010] 10.06.01_22:26:34.164554 7f018b477710 mds2.cache |__ 0 rep [dir 1 / [2,head] rep@0.1 dir_auth=0 state=0 f(v3 m10.06.01_22:09:03.004840 2=0+2) s=2=0+2 rb=428/428 rf=10/10 rd=24/24 hs=0+0,ss=0+0 | nref=1 0x10865d8] 10.06.01_22:26:36.464557 7f0189473710 mds2.cache trim max=100000 cur=2 10.06.01_22:26:36.464575 7f0189473710 mds2.cache trim_client_leases 10.06.01_22:26:36.464900 7f0189473710 mds2.cache check_memory_usage total 271400, rss 2996, heap 180672, malloc 180 mmap 0, baseline 180672, buffers 24, max 1048576, 0 / 4 inodes have caps, 0 caps, 0 caps per inode 10.06.01_22:26:36.464916 7f0189473710 mds2.log trim 12 / 30 segments, 12 / -1 events, 0 (0) expiring, 0 (0) expired 10.06.01_22:26:36.464946 7f0189473710 mds2.bal get_load mdsload<[0,0 0]/[0,0 0], req 0, hr 0, qlen 0, cpu 0.24> 10.06.01_22:26:36.464971 7f0189473710 mds2.locker scatter_tick 10.06.01_22:26:36.464979 7f0189473710 mds2.server find_idle_sessions. laggy until 0.000000 10.06.01_22:26:36.464987 7f0189473710 mds2.bal tick last_sample now 10.06.01_22:26:36.464986 10.06.01_22:26:36.464994 7f0189473710 mds2.snap check_osd_map - version unchanged 10.06.01_22:26:37.167447 7f0189473710 mds2.12 beacon_send up:active seq 26 (currently up:active) 10.06.01_22:26:37.167460 7f0189473710 -- 172.16.2.9:6800/6204 --> mon1 172.16.2.10:6789/0 -- mdsbeacon(11227/r1-9 up:active seq 26 v1353) v1 -- ?+0 0x7f018400e870 10.06.01_22:26:37.167996 7f018b477710 -- 172.16.2.9:6800/6204 <== mon1 172.16.2.10:6789/0 48 ==== mdsbeacon(11227/r1-9 up:active seq 26 v1353) v1 ==== 70+0+0 (340011755 0 0) 0x107f730 10.06.01_22:26:37.168015 7f018b477710 mds2.12 handle_mds_beacon up:active seq 26 rtt 0.000556 10.06.01_22:26:40.714499 7f01807f8710 -- 172.16.2.9:6800/6204 >> 172.16.2.9:6800/6024 pipe(0x7f018401e500 sd=16 pgs=0 cs=0 l=0).connect claims to be 172.16.2.9:6800/6204 not 172.16.2.9:6800/6024 - wrong node! 10.06.01_22:26:41.167542 7f0189473710 mds2.12 beacon_send up:active seq 27 (currently up:active) 10.06.01_22:26:41.167560 7f0189473710 -- 172.16.2.9:6800/6204 --> mon1 172.16.2.10:6789/0 -- mdsbeacon(11227/r1-9 up:active seq 27 v1353) v1 -- ?+0 0x7f0184002b80 10.06.01_22:26:41.403254 7f018b477710 -- 172.16.2.9:6800/6204 <== mon1 172.16.2.10:6789/0 49 ==== mdsbeacon(11227/r1-9 up:active seq 27 v1353) v1 ==== 70+0+0 (2920015973 0 0) 0x10806a0 10.06.01_22:26:41.403279 7f018b477710 mds2.12 handle_mds_beacon up:active seq 27 rtt 0.235720 10.06.01_22:26:41.464615 7f0189473710 mds2.cache trim max=100000 cur=2 10.06.01_22:26:41.464628 7f0189473710 mds2.cache trim_client_leases 10.06.01_22:26:41.464947 7f0189473710 mds2.cache check_memory_usage total 271400, rss 3000, heap 180672, malloc 180 mmap 0, baseline 180672, buffers 24, max 1048576, 0 / 4 inodes have caps, 0 caps, 0 caps per inode 10.06.01_22:26:41.464970 7f0189473710 mds2.log trim 12 / 30 segments, 12 / -1 events, 0 (0) expiring, 0 (0) expired 10.06.01_22:26:41.465001 7f0189473710 mds2.bal get_load mdsload<[0,0 0]/[0,0 0], req 0, hr 0, qlen 0, cpu 0.22> 10.06.01_22:26:41.465027 7f0189473710 mds2.locker scatter_tick 10.06.01_22:26:41.465035 7f0189473710 mds2.server find_idle_sessions. laggy until 0.000000 10.06.01_22:26:41.465043 7f0189473710 mds2.bal tick last_sample now 10.06.01_22:26:41.465042 10.06.01_22:26:41.465050 7f0189473710 mds2.snap check_osd_map - version unchanged 10.06.01_22:26:41.670588 7f018b477710 -- 172.16.2.9:6800/6204 <== mds0 172.16.2.11:6800/5997 8 ==== HB v1 ==== 293+0+0 (3212462232 0 0) 0x107f610 10.06.01_22:26:41.670608 7f018b477710 mds2.bal from mds0, new epoch 10.06.01_22:26:41.670634 7f018b477710 mds2.bal get_load mdsload<[0,0 0]/[0,0 0], req 0, hr 0, qlen 0, cpu 0.22> 10.06.01_22:26:41.670664 7f018b477710 mds2.bal mds2 epoch 2 load mdsload<[0,0 0]/[0,0 0], req 0, hr 0, qlen 0, cpu 0.22> 10.06.01_22:26:41.670686 7f018b477710 -- 172.16.2.9:6800/6204 --> mds0 172.16.2.11:6800/5997 -- HB v1 -- ?+0 0x10806a0 10.06.01_22:26:41.670705 7f018b477710 -- 172.16.2.9:6800/6204 --> mds1 172.16.2.10:6800/5990 -- HB v1 -- ?+0 0x1092690 10.06.01_22:26:41.670722 7f018b477710 mds2.cache show_subtrees 10.06.01_22:26:41.670734 7f018b477710 mds2.cache |__ 2 auth [dir 102 ~mds2/ [2,head] auth v=1 cv=0/0 dir_auth=2 state=1073741826|complete f(v0 m0.000000 2=1+1) s=2=1+1 rb=0/0 rf=1/1 rd=1/1 hs=2+0,ss=0+0 | nref=2 0x1086010] 10.06.01_22:26:41.670751 7f018b477710 mds2.cache |__ 0 rep [dir 1 / [2,head] rep@0.1 dir_auth=0 state=0 f(v3 m10.06.01_22:09:03.004840 2=0+2) s=2=0+2 rb=428/428 rf=10/10 rd=24/24 hs=0+0,ss=0+0 | nref=1 0x10865d8] 10.06.01_22:26:45.167645 7f0189473710 mds2.12 beacon_send up:active seq 28 (currently up:active) 10.06.01_22:26:45.167668 7f0189473710 -- 172.16.2.9:6800/6204 --> mon1 172.16.2.10:6789/0 -- mdsbeacon(11227/r1-9 up:active seq 28 v1353) v1 -- ?+0 0x7f0184002b80 10.06.01_22:26:45.168239 7f018b477710 -- 172.16.2.9:6800/6204 <== mon1 172.16.2.10:6789/0 50 ==== mdsbeacon(11227/r1-9 up:active seq 28 v1353) v1 ==== 70+0+0 (2278571740 0 0) 0x10806a0 10.06.01_22:26:45.168259 7f018b477710 mds2.12 handle_mds_beacon up:active seq 28 rtt 0.000593 10.06.01_22:26:46.464678 7f0189473710 mds2.cache trim max=100000 cur=2 10.06.01_22:26:46.464695 7f0189473710 mds2.cache trim_client_leases 10.06.01_22:26:46.465017 7f0189473710 mds2.cache check_memory_usage total 271400, rss 3000, heap 180672, malloc 181 mmap 0, baseline 180672, buffers 24, max 1048576, 0 / 4 inodes have caps, 0 caps, 0 caps per inode 10.06.01_22:26:46.465032 7f0189473710 mds2.log trim 12 / 30 segments, 12 / -1 events, 0 (0) expiring, 0 (0) expired 10.06.01_22:26:46.465062 7f0189473710 mds2.bal get_load mdsload<[0,0 0]/[0,0 0], req 0, hr 0, qlen 0, cpu 0.2> 10.06.01_22:26:46.465087 7f0189473710 mds2.locker scatter_tick 10.06.01_22:26:46.465095 7f0189473710 mds2.server find_idle_sessions. laggy until 0.000000 10.06.01_22:26:46.465103 7f0189473710 mds2.bal tick last_sample now 10.06.01_22:26:46.465103 10.06.01_22:26:46.465111 7f0189473710 mds2.snap check_osd_map - version unchanged 10.06.01_22:26:49.167752 7f0189473710 mds2.12 beacon_send up:active seq 29 (currently up:active) 10.06.01_22:26:49.167770 7f0189473710 -- 172.16.2.9:6800/6204 --> mon1 172.16.2.10:6789/0 -- mdsbeacon(11227/r1-9 up:active seq 29 v1353) v1 -- ?+0 0x7f0184002b80 10.06.01_22:26:49.168303 7f018b477710 -- 172.16.2.9:6800/6204 <== mon1 172.16.2.10:6789/0 51 ==== mdsbeacon(11227/r1-9 up:active seq 29 v1353) v1 ==== 70+0+0 (1033894994 0 0) 0x10806a0 10.06.01_22:26:49.168323 7f018b477710 mds2.12 handle_mds_beacon up:active seq 29 rtt 0.000554 10.06.01_22:26:51.464738 7f0189473710 mds2.cache trim max=100000 cur=2 10.06.01_22:26:51.464755 7f0189473710 mds2.cache trim_client_leases 10.06.01_22:26:51.465072 7f0189473710 mds2.cache check_memory_usage total 271400, rss 3000, heap 180672, malloc 181 mmap 0, baseline 180672, buffers 24, max 1048576, 0 / 4 inodes have caps, 0 caps, 0 caps per inode 10.06.01_22:26:51.465093 7f0189473710 mds2.log trim 12 / 30 segments, 12 / -1 events, 0 (0) expiring, 0 (0) expired 10.06.01_22:26:51.465123 7f0189473710 mds2.bal get_load mdsload<[0,0 0]/[0,0 0], req 0, hr 0, qlen 0, cpu 0.18> 10.06.01_22:26:51.465149 7f0189473710 mds2.locker scatter_tick 10.06.01_22:26:51.465157 7f0189473710 mds2.server find_idle_sessions. laggy until 0.000000 10.06.01_22:26:51.465165 7f0189473710 mds2.bal tick last_sample now 10.06.01_22:26:51.465165 10.06.01_22:26:51.465173 7f0189473710 mds2.snap check_osd_map - version unchanged 10.06.01_22:26:51.670658 7f018b477710 -- 172.16.2.9:6800/6204 <== mds0 172.16.2.11:6800/5997 9 ==== HB v1 ==== 293+0+0 (2534825716 0 0) 0x107f610 10.06.01_22:26:51.670683 7f018b477710 mds2.bal from mds0, new epoch 10.06.01_22:26:51.670716 7f018b477710 mds2.bal get_load mdsload<[0,0 0]/[0,0 0], req 0, hr 0, qlen 0, cpu 0.18> 10.06.01_22:26:51.670743 7f018b477710 mds2.bal mds2 epoch 3 load mdsload<[0,0 0]/[0,0 0], req 0, hr 0, qlen 0, cpu 0.18> 10.06.01_22:26:51.670764 7f018b477710 -- 172.16.2.9:6800/6204 --> mds0 172.16.2.11:6800/5997 -- HB v1 -- ?+0 0x10806a0 10.06.01_22:26:51.670784 7f018b477710 -- 172.16.2.9:6800/6204 --> mds1 172.16.2.10:6800/5990 -- HB v1 -- ?+0 0x1092a30 10.06.01_22:26:51.670801 7f018b477710 mds2.cache show_subtrees 10.06.01_22:26:51.670813 7f018b477710 mds2.cache |__ 2 auth [dir 102 ~mds2/ [2,head] auth v=1 cv=0/0 dir_auth=2 state=1073741826|complete f(v0 m0.000000 2=1+1) s=2=1+1 rb=0/0 rf=1/1 rd=1/1 hs=2+0,ss=0+0 | nref=2 0x1086010] 10.06.01_22:26:51.670831 7f018b477710 mds2.cache |__ 0 rep [dir 1 / [2,head] rep@0.1 dir_auth=0 state=0 f(v3 m10.06.01_22:09:03.004840 2=0+2) s=2=0+2 rb=428/428 rf=10/10 rd=24/24 hs=0+0,ss=0+0 | nref=1 0x10865d8] 10.06.01_22:26:53.167853 7f0189473710 mds2.12 beacon_send up:active seq 30 (currently up:active) 10.06.01_22:26:53.167869 7f0189473710 -- 172.16.2.9:6800/6204 --> mon1 172.16.2.10:6789/0 -- mdsbeacon(11227/r1-9 up:active seq 30 v1353) v1 -- ?+0 0x7f0184002b80 10.06.01_22:26:53.393677 7f018b477710 -- 172.16.2.9:6800/6204 <== mon1 172.16.2.10:6789/0 52 ==== mdsbeacon(11227/r1-9 up:active seq 30 v1353) v1 ==== 70+0+0 (4137930033 0 0) 0x107f610 10.06.01_22:26:53.393710 7f018b477710 mds2.12 handle_mds_beacon up:active seq 30 rtt 0.225843 10.06.01_22:26:55.714713 7f01807f8710 -- 172.16.2.9:6800/6204 >> 172.16.2.9:6800/6024 pipe(0x7f018401e500 sd=16 pgs=0 cs=0 l=0).connect claims to be 172.16.2.9:6800/6204 not 172.16.2.9:6800/6024 - wrong node! 10.06.01_22:26:56.464801 7f0189473710 mds2.cache trim max=100000 cur=2 10.06.01_22:26:56.464819 7f0189473710 mds2.cache trim_client_leases 10.06.01_22:26:56.465142 7f0189473710 mds2.cache check_memory_usage total 271400, rss 3000, heap 180672, malloc 182 mmap 0, baseline 180672, buffers 24, max 1048576, 0 / 4 inodes have caps, 0 caps, 0 caps per inode 10.06.01_22:26:56.465158 7f0189473710 mds2.log trim 12 / 30 segments, 12 / -1 events, 0 (0) expiring, 0 (0) expired 10.06.01_22:26:56.465189 7f0189473710 mds2.bal get_load mdsload<[0,0 0]/[0,0 0], req 0, hr 0, qlen 0, cpu 0.17> 10.06.01_22:26:56.465214 7f0189473710 mds2.locker scatter_tick 10.06.01_22:26:56.465222 7f0189473710 mds2.server find_idle_sessions. laggy until 0.000000 10.06.01_22:26:56.465230 7f0189473710 mds2.bal tick last_sample now 10.06.01_22:26:56.465229 10.06.01_22:26:56.465238 7f0189473710 mds2.snap check_osd_map - version unchanged 10.06.01_22:26:57.167947 7f0189473710 mds2.12 beacon_send up:active seq 31 (currently up:active) 10.06.01_22:26:57.167960 7f0189473710 -- 172.16.2.9:6800/6204 --> mon1 172.16.2.10:6789/0 -- mdsbeacon(11227/r1-9 up:active seq 31 v1353) v1 -- ?+0 0x7f0184002b80 10.06.01_22:26:57.168498 7f018b477710 -- 172.16.2.9:6800/6204 <== mon1 172.16.2.10:6789/0 53 ==== mdsbeacon(11227/r1-9 up:active seq 31 v1353) v1 ==== 70+0+0 (1290538943 0 0) 0x10806a0 10.06.01_22:26:57.168517 7f018b477710 mds2.12 handle_mds_beacon up:active seq 31 rtt 0.000559 10.06.01_22:27:01.168042 7f0189473710 mds2.12 beacon_send up:active seq 32 (currently up:active) 10.06.01_22:27:01.168067 7f0189473710 -- 172.16.2.9:6800/6204 --> mon1 172.16.2.10:6789/0 -- mdsbeacon(11227/r1-9 up:active seq 32 v1353) v1 -- ?+0 0x7f018400e870 10.06.01_22:27:01.168608 7f018b477710 -- 172.16.2.9:6800/6204 <== mon1 172.16.2.10:6789/0 54 ==== mdsbeacon(11227/r1-9 up:active seq 32 v1353) v1 ==== 70+0+0 (772086335 0 0) 0x10806a0 10.06.01_22:27:01.168626 7f018b477710 mds2.12 handle_mds_beacon up:active seq 32 rtt 0.000560 10.06.01_22:27:01.464860 7f0189473710 mds2.cache trim max=100000 cur=2 10.06.01_22:27:01.464871 7f0189473710 mds2.cache trim_client_leases 10.06.01_22:27:01.465198 7f0189473710 mds2.cache check_memory_usage total 271400, rss 3000, heap 180672, malloc 182 mmap 0, baseline 180672, buffers 24, max 1048576, 0 / 4 inodes have caps, 0 caps, 0 caps per inode 10.06.01_22:27:01.465214 7f0189473710 mds2.log trim 12 / 30 segments, 12 / -1 events, 0 (0) expiring, 0 (0) expired 10.06.01_22:27:01.465245 7f0189473710 mds2.bal get_load mdsload<[0,0 0]/[0,0 0], req 0, hr 0, qlen 0, cpu 0.16> 10.06.01_22:27:01.465270 7f0189473710 mds2.locker scatter_tick 10.06.01_22:27:01.465278 7f0189473710 mds2.server find_idle_sessions. laggy until 0.000000 10.06.01_22:27:01.465286 7f0189473710 mds2.bal tick last_sample now 10.06.01_22:27:01.465286 10.06.01_22:27:01.465293 7f0189473710 mds2.snap check_osd_map - version unchanged 10.06.01_22:27:01.670761 7f018b477710 -- 172.16.2.9:6800/6204 <== mds0 172.16.2.11:6800/5997 10 ==== HB v1 ==== 293+0+0 (2028828324 0 0) 0x107f610 10.06.01_22:27:01.670782 7f018b477710 mds2.bal from mds0, new epoch 10.06.01_22:27:01.670809 7f018b477710 mds2.bal get_load mdsload<[0,0 0]/[0,0 0], req 0, hr 0, qlen 0, cpu 0.16> 10.06.01_22:27:01.670839 7f018b477710 mds2.bal mds2 epoch 4 load mdsload<[0,0 0]/[0,0 0], req 0, hr 0, qlen 0, cpu 0.16> 10.06.01_22:27:01.670860 7f018b477710 -- 172.16.2.9:6800/6204 --> mds0 172.16.2.11:6800/5997 -- HB v1 -- ?+0 0x10806a0 10.06.01_22:27:01.670880 7f018b477710 -- 172.16.2.9:6800/6204 --> mds1 172.16.2.10:6800/5990 -- HB v1 -- ?+0 0x1080f00 10.06.01_22:27:01.670897 7f018b477710 mds2.cache show_subtrees 10.06.01_22:27:01.670909 7f018b477710 mds2.cache |__ 2 auth [dir 102 ~mds2/ [2,head] auth v=1 cv=0/0 dir_auth=2 state=1073741826|complete f(v0 m0.000000 2=1+1) s=2=1+1 rb=0/0 rf=1/1 rd=1/1 hs=2+0,ss=0+0 | nref=2 0x1086010] 10.06.01_22:27:01.670926 7f018b477710 mds2.cache |__ 0 rep [dir 1 / [2,head] rep@0.1 dir_auth=0 state=0 f(v3 m10.06.01_22:09:03.004840 2=0+2) s=2=0+2 rb=428/428 rf=10/10 rd=24/24 hs=0+0,ss=0+0 | nref=1 0x10865d8] 10.06.01_22:27:05.168152 7f0189473710 mds2.12 beacon_send up:active seq 33 (currently up:active) 10.06.01_22:27:05.168171 7f0189473710 -- 172.16.2.9:6800/6204 --> mon1 172.16.2.10:6789/0 -- mdsbeacon(11227/r1-9 up:active seq 33 v1353) v1 -- ?+0 0x7f018400e870 10.06.01_22:27:05.247840 7f018b477710 -- 172.16.2.9:6800/6204 <== mon1 172.16.2.10:6789/0 55 ==== mdsbeacon(11227/r1-9 up:active seq 33 v1353) v1 ==== 70+0+0 (2487933105 0 0) 0x107f610 10.06.01_22:27:05.247864 7f018b477710 mds2.12 handle_mds_beacon up:active seq 33 rtt 0.079695 10.06.01_22:27:06.464920 7f0189473710 mds2.cache trim max=100000 cur=2 10.06.01_22:27:06.464935 7f0189473710 mds2.cache trim_client_leases 10.06.01_22:27:06.465255 7f0189473710 mds2.cache check_memory_usage total 271400, rss 3004, heap 180680, malloc 183 mmap 0, baseline 180672, buffers 24, max 1048576, 0 / 4 inodes have caps, 0 caps, 0 caps per inode 10.06.01_22:27:06.465270 7f0189473710 mds2.log trim 12 / 30 segments, 12 / -1 events, 0 (0) expiring, 0 (0) expired 10.06.01_22:27:06.465299 7f0189473710 mds2.bal get_load mdsload<[0,0 0]/[0,0 0], req 0, hr 0, qlen 0, cpu 0.3> 10.06.01_22:27:06.465324 7f0189473710 mds2.locker scatter_tick 10.06.01_22:27:06.465332 7f0189473710 mds2.server find_idle_sessions. laggy until 0.000000 10.06.01_22:27:06.465340 7f0189473710 mds2.bal tick last_sample now 10.06.01_22:27:06.465340 10.06.01_22:27:06.465348 7f0189473710 mds2.snap check_osd_map - version unchanged 10.06.01_22:27:09.168252 7f0189473710 mds2.12 beacon_send up:active seq 34 (currently up:active) 10.06.01_22:27:09.168276 7f0189473710 -- 172.16.2.9:6800/6204 --> mon1 172.16.2.10:6789/0 -- mdsbeacon(11227/r1-9 up:active seq 34 v1353) v1 -- ?+0 0x7f018400e870 10.06.01_22:27:09.168807 7f018b477710 -- 172.16.2.9:6800/6204 <== mon1 172.16.2.10:6789/0 56 ==== mdsbeacon(11227/r1-9 up:active seq 34 v1353) v1 ==== 70+0+0 (1601628626 0 0) 0x107f610 10.06.01_22:27:09.168825 7f018b477710 mds2.12 handle_mds_beacon up:active seq 34 rtt 0.000550 10.06.01_22:27:10.714941 7f01807f8710 -- 172.16.2.9:6800/6204 >> 172.16.2.9:6800/6024 pipe(0x7f018401e500 sd=16 pgs=0 cs=0 l=0).connect claims to be 172.16.2.9:6800/6204 not 172.16.2.9:6800/6024 - wrong node! 10.06.01_22:27:11.464983 7f0189473710 mds2.cache trim max=100000 cur=2 10.06.01_22:27:11.465001 7f0189473710 mds2.cache trim_client_leases 10.06.01_22:27:11.465317 7f0189473710 mds2.cache check_memory_usage total 271400, rss 3004, heap 180680, malloc 183 mmap 0, baseline 180672, buffers 24, max 1048576, 0 / 4 inodes have caps, 0 caps, 0 caps per inode 10.06.01_22:27:11.465333 7f0189473710 mds2.log trim 12 / 30 segments, 12 / -1 events, 0 (0) expiring, 0 (0) expired 10.06.01_22:27:11.465364 7f0189473710 mds2.bal get_load mdsload<[0,0 0]/[0,0 0], req 0, hr 0, qlen 0, cpu 0.28> 10.06.01_22:27:11.465389 7f0189473710 mds2.locker scatter_tick 10.06.01_22:27:11.465398 7f0189473710 mds2.server find_idle_sessions. laggy until 0.000000 10.06.01_22:27:11.465407 7f0189473710 mds2.bal tick last_sample now 10.06.01_22:27:11.465407 10.06.01_22:27:11.465415 7f0189473710 mds2.snap check_osd_map - version unchanged 10.06.01_22:27:11.670860 7f018b477710 -- 172.16.2.9:6800/6204 <== mds0 172.16.2.11:6800/5997 11 ==== HB v1 ==== 293+0+0 (1095371832 0 0) 0x107f610 10.06.01_22:27:11.670884 7f018b477710 mds2.bal from mds0, new epoch 10.06.01_22:27:11.670914 7f018b477710 mds2.bal get_load mdsload<[0,0 0]/[0,0 0], req 0, hr 0, qlen 0, cpu 0.28> 10.06.01_22:27:11.670942 7f018b477710 mds2.bal mds2 epoch 5 load mdsload<[0,0 0]/[0,0 0], req 0, hr 0, qlen 0, cpu 0.28> 10.06.01_22:27:11.670963 7f018b477710 -- 172.16.2.9:6800/6204 --> mds0 172.16.2.11:6800/5997 -- HB v1 -- ?+0 0x10806a0 10.06.01_22:27:11.670984 7f018b477710 -- 172.16.2.9:6800/6204 --> mds1 172.16.2.10:6800/5990 -- HB v1 -- ?+0 0x10812a0 10.06.01_22:27:11.671001 7f018b477710 mds2.cache show_subtrees 10.06.01_22:27:11.671014 7f018b477710 mds2.cache |__ 2 auth [dir 102 ~mds2/ [2,head] auth v=1 cv=0/0 dir_auth=2 state=1073741826|complete f(v0 m0.000000 2=1+1) s=2=1+1 rb=0/0 rf=1/1 rd=1/1 hs=2+0,ss=0+0 | nref=2 0x1086010] 10.06.01_22:27:11.671032 7f018b477710 mds2.cache |__ 0 rep [dir 1 / [2,head] rep@0.1 dir_auth=0 state=0 f(v3 m10.06.01_22:09:03.004840 2=0+2) s=2=0+2 rb=428/428 rf=10/10 rd=24/24 hs=0+0,ss=0+0 | nref=1 0x10865d8] 10.06.01_22:27:13.168357 7f0189473710 mds2.12 beacon_send up:active seq 35 (currently up:active) 10.06.01_22:27:13.168375 7f0189473710 -- 172.16.2.9:6800/6204 --> mon1 172.16.2.10:6789/0 -- mdsbeacon(11227/r1-9 up:active seq 35 v1353) v1 -- ?+0 0x7f018400e870 10.06.01_22:27:13.168906 7f018b477710 -- 172.16.2.9:6800/6204 <== mon1 172.16.2.10:6789/0 57 ==== mdsbeacon(11227/r1-9 up:active seq 35 v1353) v1 ==== 70+0+0 (3845729116 0 0) 0x107f610 10.06.01_22:27:13.168926 7f018b477710 mds2.12 handle_mds_beacon up:active seq 35 rtt 0.000552 10.06.01_22:27:16.465046 7f0189473710 mds2.cache trim max=100000 cur=2 10.06.01_22:27:16.465064 7f0189473710 mds2.cache trim_client_leases 10.06.01_22:27:16.465381 7f0189473710 mds2.cache check_memory_usage total 271400, rss 3004, heap 180680, malloc 184 mmap 0, baseline 180672, buffers 24, max 1048576, 0 / 4 inodes have caps, 0 caps, 0 caps per inode 10.06.01_22:27:16.465397 7f0189473710 mds2.log trim 12 / 30 segments, 12 / -1 events, 0 (0) expiring, 0 (0) expired 10.06.01_22:27:16.465426 7f0189473710 mds2.bal get_load mdsload<[0,0 0]/[0,0 0], req 0, hr 0, qlen 0, cpu 0.26> 10.06.01_22:27:16.465451 7f0189473710 mds2.locker scatter_tick 10.06.01_22:27:16.465460 7f0189473710 mds2.server find_idle_sessions. laggy until 0.000000 10.06.01_22:27:16.465477 7f0189473710 mds2.bal tick last_sample now 10.06.01_22:27:16.465477 10.06.01_22:27:16.465485 7f0189473710 mds2.snap check_osd_map - version unchanged 10.06.01_22:27:17.168455 7f0189473710 mds2.12 beacon_send up:active seq 36 (currently up:active) 10.06.01_22:27:17.168471 7f0189473710 -- 172.16.2.9:6800/6204 --> mon1 172.16.2.10:6789/0 -- mdsbeacon(11227/r1-9 up:active seq 36 v1353) v1 -- ?+0 0x7f018400e870 10.06.01_22:27:17.397487 7f018b477710 -- 172.16.2.9:6800/6204 <== mon1 172.16.2.10:6789/0 58 ==== mdsbeacon(11227/r1-9 up:active seq 36 v1353) v1 ==== 70+0+0 (3437428197 0 0) 0x107f610 10.06.01_22:27:17.397511 7f018b477710 mds2.12 handle_mds_beacon up:active seq 36 rtt 0.229041 10.06.01_22:27:21.168551 7f0189473710 mds2.12 beacon_send up:active seq 37 (currently up:active) 10.06.01_22:27:21.168571 7f0189473710 -- 172.16.2.9:6800/6204 --> mon1 172.16.2.10:6789/0 -- mdsbeacon(11227/r1-9 up:active seq 37 v1353) v1 -- ?+0 0x7f0184002b80 10.06.01_22:27:21.169105 7f018b477710 -- 172.16.2.9:6800/6204 <== mon1 172.16.2.10:6789/0 59 ==== mdsbeacon(11227/r1-9 up:active seq 37 v1353) v1 ==== 70+0+0 (1991065451 0 0) 0x107f610 10.06.01_22:27:21.169123 7f018b477710 mds2.12 handle_mds_beacon up:active seq 37 rtt 0.000554 10.06.01_22:27:21.465105 7f0189473710 mds2.cache trim max=100000 cur=2 10.06.01_22:27:21.465119 7f0189473710 mds2.cache trim_client_leases 10.06.01_22:27:21.465437 7f0189473710 mds2.cache check_memory_usage total 271400, rss 3004, heap 180680, malloc 184 mmap 0, baseline 180672, buffers 24, max 1048576, 0 / 4 inodes have caps, 0 caps, 0 caps per inode 10.06.01_22:27:21.465452 7f0189473710 mds2.log trim 12 / 30 segments, 12 / -1 events, 0 (0) expiring, 0 (0) expired 10.06.01_22:27:21.465482 7f0189473710 mds2.bal get_load mdsload<[0,0 0]/[0,0 0], req 0, hr 0, qlen 0, cpu 0.24> 10.06.01_22:27:21.465514 7f0189473710 mds2.locker scatter_tick 10.06.01_22:27:21.465522 7f0189473710 mds2.server find_idle_sessions. laggy until 0.000000 10.06.01_22:27:21.465530 7f0189473710 mds2.bal tick last_sample now 10.06.01_22:27:21.465529 10.06.01_22:27:21.465537 7f0189473710 mds2.snap check_osd_map - version unchanged 10.06.01_22:27:21.670916 7f018b477710 -- 172.16.2.9:6800/6204 <== mds0 172.16.2.11:6800/5997 12 ==== HB v1 ==== 293+0+0 (2657602293 0 0) 0x10806a0 10.06.01_22:27:21.670939 7f018b477710 mds2.bal from mds0, new epoch 10.06.01_22:27:21.670970 7f018b477710 mds2.bal get_load mdsload<[0,0 0]/[0,0 0], req 0, hr 0, qlen 0, cpu 0.24> 10.06.01_22:27:21.670998 7f018b477710 mds2.bal mds2 epoch 6 load mdsload<[0,0 0]/[0,0 0], req 0, hr 0, qlen 0, cpu 0.24> 10.06.01_22:27:21.671018 7f018b477710 -- 172.16.2.9:6800/6204 --> mds0 172.16.2.11:6800/5997 -- HB v1 -- ?+0 0x107f610 10.06.01_22:27:21.671038 7f018b477710 -- 172.16.2.9:6800/6204 --> mds1 172.16.2.10:6800/5990 -- HB v1 -- ?+0 0x1081640 10.06.01_22:27:21.671055 7f018b477710 mds2.cache show_subtrees 10.06.01_22:27:21.671067 7f018b477710 mds2.cache |__ 2 auth [dir 102 ~mds2/ [2,head] auth v=1 cv=0/0 dir_auth=2 state=1073741826|complete f(v0 m0.000000 2=1+1) s=2=1+1 rb=0/0 rf=1/1 rd=1/1 hs=2+0,ss=0+0 | nref=2 0x1086010] 10.06.01_22:27:21.671084 7f018b477710 mds2.cache |__ 0 rep [dir 1 / [2,head] rep@0.1 dir_auth=0 state=0 f(v3 m10.06.01_22:09:03.004840 2=0+2) s=2=0+2 rb=428/428 rf=10/10 rd=24/24 hs=0+0,ss=0+0 | nref=1 0x10865d8] 10.06.01_22:27:25.168654 7f0189473710 mds2.12 beacon_send up:active seq 38 (currently up:active) 10.06.01_22:27:25.168672 7f0189473710 -- 172.16.2.9:6800/6204 --> mon1 172.16.2.10:6789/0 -- mdsbeacon(11227/r1-9 up:active seq 38 v1353) v1 -- ?+0 0x7f0184002b80 10.06.01_22:27:25.169205 7f018b477710 -- 172.16.2.9:6800/6204 <== mon1 172.16.2.10:6789/0 60 ==== mdsbeacon(11227/r1-9 up:active seq 38 v1353) v1 ==== 70+0+0 (3180400136 0 0) 0x107f610 10.06.01_22:27:25.169224 7f018b477710 mds2.12 handle_mds_beacon up:active seq 38 rtt 0.000553 10.06.01_22:27:25.715153 7f01807f8710 -- 172.16.2.9:6800/6204 >> 172.16.2.9:6800/6024 pipe(0x7f018401e500 sd=16 pgs=0 cs=0 l=0).connect claims to be 172.16.2.9:6800/6204 not 172.16.2.9:6800/6024 - wrong node!