2013-04-26 02:38:58.919753 7f27e022e800 0 ceph version 0.60-665-g3ce35a6 (3ce35a6743e050bf0de5abd5ad32f522c5664f3d), process ceph-mon, pid 16922 2013-04-26 02:38:58.919768 7f27e022e800 10 needs_conversion 2013-04-26 02:38:59.281341 7f27e022e800 10 obtain_monmap 2013-04-26 02:38:59.281486 7f27e022e800 10 obtain_monmap read last committed monmap ver 1 2013-04-26 02:38:59.281902 7f27e022e800 1 mon.1@-1(probing) e1 preinit fsid 5eacb654-3204-4ca1-9b16-83bb0140e497 2013-04-26 02:38:59.282040 7f27e022e800 10 mon.1@-1(probing) e1 check_fsid cluster_uuid contains '5eacb654-3204-4ca1-9b16-83bb0140e497' 2013-04-26 02:38:59.282087 7f27e022e800 10 mon.1@-1(probing) e1 features compat={},rocompat={},incompat={1=initial feature set (~v.18),3=single paxos with k/v store (v0.?)} 2013-04-26 02:38:59.282125 7f27e022e800 10 mon.1@-1(probing) e1 has_ever_joined = 1 2013-04-26 02:38:59.282184 7f27e022e800 10 mon.1@-1(probing) e1 init_paxos 2013-04-26 02:38:59.283213 7f27e022e800 7 mon.1@-1(probing).pg v0 update_from_paxos loading latest full pgmap v5938118 2013-04-26 02:38:59.291714 7f27e022e800 7 mon.1@-1(probing).pg v5938118 update_from_paxos applying incremental 5938119 2013-04-26 02:38:59.292071 7f27e022e800 10 mon.1@-1(probing).pg v5938119 v5938119: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 100275B/s wr, 6op/s 2013-04-26 02:38:59.292152 7f27e022e800 7 mon.1@-1(probing).pg v5938119 update_from_paxos applying incremental 5938120 2013-04-26 02:38:59.292217 7f27e022e800 10 mon.1@-1(probing).pg v5938120 v5938120: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 54528B/s wr, 4op/s 2013-04-26 02:38:59.292281 7f27e022e800 7 mon.1@-1(probing).pg v5938120 update_from_paxos applying incremental 5938121 2013-04-26 02:38:59.292705 7f27e022e800 10 mon.1@-1(probing).pg v5938121 v5938121: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 221KB/s wr, 18op/s 2013-04-26 02:38:59.292781 7f27e022e800 7 mon.1@-1(probing).pg v5938121 update_from_paxos applying incremental 5938122 2013-04-26 02:38:59.293049 7f27e022e800 10 mon.1@-1(probing).pg v5938122 v5938122: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 517KB/s wr, 46op/s 2013-04-26 02:38:59.293107 7f27e022e800 7 mon.1@-1(probing).pg v5938122 update_from_paxos applying incremental 5938123 2013-04-26 02:38:59.293198 7f27e022e800 10 mon.1@-1(probing).pg v5938123 v5938123: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 190KB/s wr, 19op/s 2013-04-26 02:38:59.293240 7f27e022e800 7 mon.1@-1(probing).pg v5938123 update_from_paxos applying incremental 5938124 2013-04-26 02:38:59.293321 7f27e022e800 10 mon.1@-1(probing).pg v5938124 v5938124: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 146KB/s wr, 7op/s 2013-04-26 02:38:59.293362 7f27e022e800 10 mon.1@-1(probing).pg v5938124 send_pg_creates to 0 pgs 2013-04-26 02:38:59.293367 7f27e022e800 10 mon.1@-1(probing).pg v5938124 update_logger 2013-04-26 02:38:59.293398 7f27e022e800 10 mon.1@-1(probing).mds e0 update_from_paxos version 3724, my e 0 2013-04-26 02:38:59.293434 7f27e022e800 10 mon.1@-1(probing).mds e0 update_from_paxos got 3724 2013-04-26 02:38:59.293463 7f27e022e800 4 mon.1@-1(probing).mds e3724 new map 2013-04-26 02:38:59.293466 7f27e022e800 7 mon.1@-1(probing).mds e3724 print_map epoch 3724 flags 0 created 2013-01-28 15:10:54.412329 modified 2013-04-26 02:34:33.291729 tableserver 0 root 0 session_timeout 60 session_autoclose 300 last_failure 3712 last_failure_osd_epoch 10554 compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding} max_mds 1 in 0 up {0=217231} failed stopped data_pools 0,6 metadata_pool 1 217231: 10.227.227.104:6801/10842 '4' mds.0.262 up:active seq 49 217349: 10.227.227.101:6800/29852 '1' mds.-1.0 up:standby seq 1 217445: 10.227.227.103:6800/8011 '3' mds.-1.0 up:standby seq 1 2013-04-26 02:38:59.293525 7f27e022e800 10 mon.1@-1(probing).mds e3724 update_logger 2013-04-26 02:38:59.293613 7f27e022e800 7 mon.1@-1(probing).osd e0 update_from_paxos loading latest full map e10543 2013-04-26 02:38:59.293768 7f27e022e800 7 mon.1@-1(probing).osd e10543 update_from_paxos applying incremental 10544 2013-04-26 02:38:59.293884 7f27e022e800 1 mon.1@-1(probing).osd e10544 e10544: 12 osds: 10 up, 12 in 2013-04-26 02:38:59.293914 7f27e022e800 7 mon.1@-1(probing).osd e10544 update_from_paxos applying incremental 10545 2013-04-26 02:38:59.293960 7f27e022e800 1 mon.1@-1(probing).osd e10545 e10545: 12 osds: 9 up, 12 in 2013-04-26 02:38:59.293984 7f27e022e800 7 mon.1@-1(probing).osd e10545 update_from_paxos applying incremental 10546 2013-04-26 02:38:59.294027 7f27e022e800 1 mon.1@-1(probing).osd e10546 e10546: 12 osds: 9 up, 12 in 2013-04-26 02:38:59.294053 7f27e022e800 7 mon.1@-1(probing).osd e10546 update_from_paxos applying incremental 10547 2013-04-26 02:38:59.294094 7f27e022e800 1 mon.1@-1(probing).osd e10547 e10547: 12 osds: 9 up, 12 in 2013-04-26 02:38:59.294117 7f27e022e800 7 mon.1@-1(probing).osd e10547 update_from_paxos applying incremental 10548 2013-04-26 02:38:59.294158 7f27e022e800 1 mon.1@-1(probing).osd e10548 e10548: 12 osds: 9 up, 12 in 2013-04-26 02:38:59.294180 7f27e022e800 7 mon.1@-1(probing).osd e10548 update_from_paxos applying incremental 10549 2013-04-26 02:38:59.294227 7f27e022e800 1 mon.1@-1(probing).osd e10549 e10549: 12 osds: 10 up, 12 in 2013-04-26 02:38:59.294255 7f27e022e800 7 mon.1@-1(probing).osd e10549 update_from_paxos applying incremental 10550 2013-04-26 02:38:59.294303 7f27e022e800 1 mon.1@-1(probing).osd e10550 e10550: 12 osds: 12 up, 12 in 2013-04-26 02:38:59.294330 7f27e022e800 7 mon.1@-1(probing).osd e10550 update_from_paxos applying incremental 10551 2013-04-26 02:38:59.294374 7f27e022e800 1 mon.1@-1(probing).osd e10551 e10551: 12 osds: 12 up, 12 in 2013-04-26 02:38:59.294400 7f27e022e800 7 mon.1@-1(probing).osd e10551 update_from_paxos applying incremental 10552 2013-04-26 02:38:59.294444 7f27e022e800 1 mon.1@-1(probing).osd e10552 e10552: 12 osds: 6 up, 12 in 2013-04-26 02:38:59.294472 7f27e022e800 7 mon.1@-1(probing).osd e10552 update_from_paxos applying incremental 10553 2013-04-26 02:38:59.294524 7f27e022e800 1 mon.1@-1(probing).osd e10553 e10553: 12 osds: 11 up, 12 in 2013-04-26 02:38:59.294554 7f27e022e800 7 mon.1@-1(probing).osd e10553 update_from_paxos applying incremental 10554 2013-04-26 02:38:59.294600 7f27e022e800 1 mon.1@-1(probing).osd e10554 e10554: 12 osds: 12 up, 12 in 2013-04-26 02:38:59.294628 7f27e022e800 7 mon.1@-1(probing).osd e10554 update_from_paxos applying incremental 10555 2013-04-26 02:38:59.294670 7f27e022e800 1 mon.1@-1(probing).osd e10555 e10555: 12 osds: 12 up, 12 in 2013-04-26 02:38:59.294696 7f27e022e800 7 mon.1@-1(probing).osd e10555 update_from_paxos applying incremental 10556 2013-04-26 02:38:59.294736 7f27e022e800 1 mon.1@-1(probing).osd e10556 e10556: 12 osds: 12 up, 12 in 2013-04-26 02:38:59.381653 7f27e022e800 10 mon.1@-1(probing).osd e10556 send_to_waiting 10556 2013-04-26 02:38:59.381690 7f27e022e800 10 mon.1@-1(probing).osd e10556 share_map_with_random_osd no up osd on our session map 2013-04-26 02:38:59.381697 7f27e022e800 10 mon.1@-1(probing).osd e10556 update_logger 2013-04-26 02:38:59.381868 7f27e022e800 10 mon.1@-1(probing).log v5604010 update_from_paxos 2013-04-26 02:38:59.382027 7f27e022e800 10 mon.1@-1(probing).log v5604010 update_from_paxos version 5604010 summary v 0 2013-04-26 02:38:59.382128 7f27e022e800 10 mon.1@-1(probing).log v5604010 update_from_paxos latest full 5604009 2013-04-26 02:38:59.382217 7f27e022e800 7 mon.1@-1(probing).log v5604010 update_from_paxos loading summary e5604009 2013-04-26 02:38:59.382322 7f27e022e800 7 mon.1@-1(probing).log v5604010 update_from_paxos loaded summary e5604009 2013-04-26 02:38:59.382410 7f27e022e800 7 mon.1@-1(probing).log v5604010 update_from_paxos applying incremental log 5604010 2013-04-26 02:38:28.312537 mon.0 10.227.227.101:6789/0 96 : [INF] pgmap v5938123: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 190KB/s wr, 19op/s 2013-04-26 02:38:59.382588 7f27e022e800 10 mon.1@-1(probing).log v5604010 check_subs 2013-04-26 02:38:59.382695 7f27e022e800 10 mon.1@-1(probing).auth v4369 update_from_paxos 2013-04-26 02:38:59.382787 7f27e022e800 10 mon.1@-1(probing).auth v4369 update_from_paxos version 4369 keys ver 0 latest 4367 2013-04-26 02:38:59.382854 7f27e022e800 7 mon.1@-1(probing).auth v4369 update_from_paxos loading summary e 4367 2013-04-26 02:38:59.382888 7f27e022e800 7 mon.1@-1(probing).auth v4369 update_from_paxos latest length 2392 2013-04-26 02:38:59.383048 7f27e022e800 10 mon.1@-1(probing).auth v4369 update_from_paxos key server version 4367 2013-04-26 02:38:59.383172 7f27e022e800 10 mon.1@-1(probing).auth v4369 update_from_paxos() last_allocated_id=218796 max_global_id=218796 2013-04-26 02:38:59.383210 7f27e022e800 10 mon.1@-1(probing).health(0) init 2013-04-26 02:38:59.385633 7f27e022e800 2 mon.1@-1(probing) e1 init 2013-04-26 02:38:59.385761 7f27e022e800 10 mon.1@-1(probing) e1 bootstrap 2013-04-26 02:38:59.385769 7f27e022e800 10 mon.1@-1(probing) e1 unregister_cluster_logger - not registered 2013-04-26 02:38:59.385791 7f27e022e800 10 mon.1@-1(probing) e1 cancel_probe_timeout (none scheduled) 2013-04-26 02:38:59.385794 7f27e022e800 0 mon.1@-1(probing) e1 my rank is now 0 (was -1) 2013-04-26 02:38:59.385808 7f27e022e800 10 mon.1@0(probing) e1 reset_sync 2013-04-26 02:38:59.385815 7f27e022e800 10 mon.1@0(probing) e1 reset 2013-04-26 02:38:59.385819 7f27e022e800 10 mon.1@0(probing) e1 timecheck_finish 2013-04-26 02:38:59.385863 7f27e022e800 10 mon.1@0(probing) e1 cancel_probe_timeout (none scheduled) 2013-04-26 02:38:59.385867 7f27e022e800 10 mon.1@0(probing) e1 reset_probe_timeout 0x17524b0 after 2 seconds 2013-04-26 02:38:59.385939 7f27e022e800 10 mon.1@0(probing) e1 probing other monitors 2013-04-26 02:38:59.386801 7f27d500c700 10 mon.1@0(probing) e1 ms_get_authorizer for mon 2013-04-26 02:38:59.386808 7f27d4f0b700 10 mon.1@0(probing) e1 ms_get_authorizer for mon 2013-04-26 02:38:59.388033 7f27d4f0b700 10 mon.1@0(probing) e1 ms_get_authorizer for mon 2013-04-26 02:38:59.388085 7f27d500c700 10 mon.1@0(probing) e1 ms_get_authorizer for mon 2013-04-26 02:38:59.389818 7f27d600e700 10 mon.1@0(probing) e1 do not have session, making new one 2013-04-26 02:38:59.389834 7f27d600e700 10 mon.1@0(probing) e1 ms_dispatch new session MonSession: mon.2 10.227.227.104:6789/0 is open for mon.2 10.227.227.104:6789/0 2013-04-26 02:38:59.389847 7f27d600e700 5 mon.1@0(probing) e1 setting monitor caps on this connection 2013-04-26 02:38:59.389858 7f27d600e700 10 mon.1@0(probing) e1 handle_probe mon_probe(reply 5eacb654-3204-4ca1-9b16-83bb0140e497 name 4 quorum 1,2 paxos( fc 11669173 lc 11669190 )) v4 2013-04-26 02:38:59.389870 7f27d600e700 10 mon.1@0(probing) e1 handle_probe_reply mon.2 10.227.227.104:6789/0mon_probe(reply 5eacb654-3204-4ca1-9b16-83bb0140e497 name 4 quorum 1,2 paxos( fc 11669173 lc 11669190 )) v4 2013-04-26 02:38:59.389884 7f27d600e700 10 mon.1@0(probing) e1 monmap is e1: 3 mons at {1=10.227.227.101:6789/0,3=10.227.227.103:6789/0,4=10.227.227.104:6789/0} 2013-04-26 02:38:59.389910 7f27d600e700 10 mon.1@0(probing) e1 peer name is 4 2013-04-26 02:38:59.389917 7f27d600e700 10 mon.1@0(probing) e1 existing quorum 1,2 2013-04-26 02:38:59.389920 7f27d600e700 10 mon.1@0(probing) e1 peer paxos version 11669190 vs my version 11669168 (too far ahead) 2013-04-26 02:38:59.389926 7f27d600e700 10 mon.1@0(probing) e1 cancel_probe_timeout 0x17524b0 2013-04-26 02:38:59.389933 7f27d600e700 10 mon.1@0(probing) e1 sync_start entity( mon.2 10.227.227.104:6789/0 ) 2013-04-26 02:38:59.501497 7f27d600e700 10 mon.1@0(synchronizing sync( requester state start )).monmap v1 get_monmap ver 0 2013-04-26 02:38:59.503787 7f27d600e700 10 mon.1@0(synchronizing sync( requester state start )) e1 sync_store_init backup current monmap 2013-04-26 02:38:59.516162 7f27d600e700 10 mon.1@0(synchronizing sync( requester state start )) e1 do not have session, making new one 2013-04-26 02:38:59.516176 7f27d600e700 10 mon.1@0(synchronizing sync( requester state start )) e1 ms_dispatch new session MonSession: mon.1 10.227.227.103:6789/0 is open for mon.1 10.227.227.103:6789/0 2013-04-26 02:38:59.516187 7f27d600e700 5 mon.1@0(synchronizing sync( requester state start )) e1 setting monitor caps on this connection 2013-04-26 02:38:59.516194 7f27d600e700 10 mon.1@0(synchronizing sync( requester state start )) e1 handle_probe mon_probe(reply 5eacb654-3204-4ca1-9b16-83bb0140e497 name 3 quorum 1,2 paxos( fc 11669173 lc 11669190 )) v4 2013-04-26 02:38:59.516206 7f27d600e700 10 mon.1@0(synchronizing sync( requester state start )) e1 handle_probe_reply mon.1 10.227.227.103:6789/0mon_probe(reply 5eacb654-3204-4ca1-9b16-83bb0140e497 name 3 quorum 1,2 paxos( fc 11669173 lc 11669190 )) v4 2013-04-26 02:38:59.516219 7f27d600e700 10 mon.1@0(synchronizing sync( requester state start )) e1 monmap is e1: 3 mons at {1=10.227.227.101:6789/0,3=10.227.227.103:6789/0,4=10.227.227.104:6789/0} 2013-04-26 02:38:59.517738 7f27d600e700 10 mon.1@0(synchronizing sync( requester state start )) e1 handle_sync mon_sync( start_reply flags( reply-to(mon.2 10.227.227.104:6789/0) ) ) v1 2013-04-26 02:38:59.517753 7f27d600e700 10 mon.1@0(synchronizing sync( requester state start )) e1 handle_sync_start_reply mon_sync( start_reply flags( reply-to(mon.2 10.227.227.104:6789/0) ) ) v1 2013-04-26 02:38:59.517763 7f27d600e700 10 mon.1@0(synchronizing sync( requester state start )) e1 handle_sync_start_reply leader told us to use mon.2 10.227.227.104:6789/0 as sync provider 2013-04-26 02:38:59.517776 7f27d600e700 10 mon.1@0(synchronizing sync( requester state start )) e1 sync_send_heartbeat mon.1 10.227.227.103:6789/0 reply(0) 2013-04-26 02:38:59.517800 7f27d600e700 10 mon.1@0(synchronizing sync( requester state start )) e1 sync_start_chunks provider(mon.2 10.227.227.104:6789/0) 2013-04-26 02:38:59.518501 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync mon_sync( heartbeat_reply ) v1 2013-04-26 02:38:59.518512 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync_heartbeat_reply mon_sync( heartbeat_reply ) v1 2013-04-26 02:38:59.624428 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync mon_sync( chunk bl 1042969 bytes last_key ( logm,full_5603552 ) ) v1 2013-04-26 02:38:59.624446 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync_chunk mon_sync( chunk bl 1042969 bytes last_key ( logm,full_5603552 ) ) v1 2013-04-26 02:38:59.680011 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync mon_sync( chunk bl 1044836 bytes last_key ( logm,full_5603642 ) ) v1 2013-04-26 02:38:59.680025 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync_chunk mon_sync( chunk bl 1044836 bytes last_key ( logm,full_5603642 ) ) v1 2013-04-26 02:38:59.741618 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync mon_sync( chunk bl 1033792 bytes last_key ( logm,full_5603709 ) ) v1 2013-04-26 02:38:59.741631 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync_chunk mon_sync( chunk bl 1033792 bytes last_key ( logm,full_5603709 ) ) v1 2013-04-26 02:38:59.808419 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync mon_sync( chunk bl 1048220 bytes last_key ( logm,full_5603769 ) ) v1 2013-04-26 02:38:59.808432 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync_chunk mon_sync( chunk bl 1048220 bytes last_key ( logm,full_5603769 ) ) v1 2013-04-26 02:38:59.913253 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync mon_sync( chunk bl 1037517 bytes last_key ( logm,full_5603835 ) ) v1 2013-04-26 02:38:59.913268 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync_chunk mon_sync( chunk bl 1037517 bytes last_key ( logm,full_5603835 ) ) v1 2013-04-26 02:38:59.975658 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync mon_sync( chunk bl 1044599 bytes last_key ( logm,full_5603908 ) ) v1 2013-04-26 02:38:59.975671 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync_chunk mon_sync( chunk bl 1044599 bytes last_key ( logm,full_5603908 ) ) v1 2013-04-26 02:39:00.039075 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync mon_sync( chunk bl 1046384 bytes last_key ( logm,full_5603981 ) ) v1 2013-04-26 02:39:00.039090 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync_chunk mon_sync( chunk bl 1046384 bytes last_key ( logm,full_5603981 ) ) v1 2013-04-26 02:39:00.175686 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync mon_sync( chunk bl 1048206 bytes last_key ( mdsmap,1587 ) ) v1 2013-04-26 02:39:00.175699 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync_chunk mon_sync( chunk bl 1048206 bytes last_key ( mdsmap,1587 ) ) v1 2013-04-26 02:39:00.833082 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync mon_sync( chunk bl 1047917 bytes last_key ( mdsmap,31 ) ) v1 2013-04-26 02:39:00.833097 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync_chunk mon_sync( chunk bl 1047917 bytes last_key ( mdsmap,31 ) ) v1 2013-04-26 02:39:01.326846 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync mon_sync( chunk bl 1047982 bytes last_key ( mdsmap,986 ) ) v1 2013-04-26 02:39:01.326864 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync_chunk mon_sync( chunk bl 1047982 bytes last_key ( mdsmap,986 ) ) v1 2013-04-26 02:39:01.462648 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync mon_sync( chunk bl 1045008 bytes last_key ( osdmap,full_10128 ) ) v1 2013-04-26 02:39:01.462665 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync_chunk mon_sync( chunk bl 1045008 bytes last_key ( osdmap,full_10128 ) ) v1 2013-04-26 02:39:01.534890 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync mon_sync( chunk bl 1047381 bytes last_key ( osdmap,full_10263 ) ) v1 2013-04-26 02:39:01.534903 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync_chunk mon_sync( chunk bl 1047381 bytes last_key ( osdmap,full_10263 ) ) v1 2013-04-26 02:39:01.636642 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync mon_sync( chunk bl 1044964 bytes last_key ( osdmap,full_10393 ) ) v1 2013-04-26 02:39:01.636654 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync_chunk mon_sync( chunk bl 1044964 bytes last_key ( osdmap,full_10393 ) ) v1 2013-04-26 02:39:01.707321 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync mon_sync( chunk bl 1044628 bytes last_key ( osdmap,full_10523 ) ) v1 2013-04-26 02:39:01.707332 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync_chunk mon_sync( chunk bl 1044628 bytes last_key ( osdmap,full_10523 ) ) v1 2013-04-26 02:39:01.771290 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync mon_sync( chunk bl 1048177 bytes last_key ( pgmap,5937695 ) ) v1 2013-04-26 02:39:01.771304 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync_chunk mon_sync( chunk bl 1048177 bytes last_key ( pgmap,5937695 ) ) v1 2013-04-26 02:39:01.841606 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync mon_sync( chunk bl 942926 bytes last_key ( pgmap,5937713 ) ) v1 2013-04-26 02:39:01.841618 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync_chunk mon_sync( chunk bl 942926 bytes last_key ( pgmap,5937713 ) ) v1 2013-04-26 02:39:01.904414 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync mon_sync( chunk bl 949287 bytes last_key ( pgmap,5937727 ) ) v1 2013-04-26 02:39:01.904425 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync_chunk mon_sync( chunk bl 949287 bytes last_key ( pgmap,5937727 ) ) v1 2013-04-26 02:39:01.962952 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync mon_sync( chunk bl 898336 bytes last_key ( pgmap,5937733 ) ) v1 2013-04-26 02:39:01.962964 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync_chunk mon_sync( chunk bl 898336 bytes last_key ( pgmap,5937733 ) ) v1 2013-04-26 02:39:02.029397 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync mon_sync( chunk bl 1007417 bytes last_key ( pgmap,5937743 ) ) v1 2013-04-26 02:39:02.029410 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync_chunk mon_sync( chunk bl 1007417 bytes last_key ( pgmap,5937743 ) ) v1 2013-04-26 02:39:02.138800 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync mon_sync( chunk bl 1023720 bytes last_key ( pgmap,5937755 ) ) v1 2013-04-26 02:39:02.138814 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync_chunk mon_sync( chunk bl 1023720 bytes last_key ( pgmap,5937755 ) ) v1 2013-04-26 02:39:02.222554 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync mon_sync( chunk bl 972239 bytes last_key ( pgmap,5937784 ) ) v1 2013-04-26 02:39:02.222570 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync_chunk mon_sync( chunk bl 972239 bytes last_key ( pgmap,5937784 ) ) v1 2013-04-26 02:39:02.339341 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync mon_sync( chunk bl 761601 bytes last_key ( pgmap,5937792 ) ) v1 2013-04-26 02:39:02.339354 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync_chunk mon_sync( chunk bl 761601 bytes last_key ( pgmap,5937792 ) ) v1 2013-04-26 02:39:02.392789 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync mon_sync( chunk bl 1013156 bytes last_key ( pgmap,5937800 ) ) v1 2013-04-26 02:39:02.392800 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync_chunk mon_sync( chunk bl 1013156 bytes last_key ( pgmap,5937800 ) ) v1 2013-04-26 02:39:02.451396 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync mon_sync( chunk bl 1026893 bytes last_key ( pgmap,5937806 ) ) v1 2013-04-26 02:39:02.451407 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync_chunk mon_sync( chunk bl 1026893 bytes last_key ( pgmap,5937806 ) ) v1 2013-04-26 02:39:02.507000 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync mon_sync( chunk bl 995136 bytes last_key ( pgmap,5937811 ) ) v1 2013-04-26 02:39:02.507012 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync_chunk mon_sync( chunk bl 995136 bytes last_key ( pgmap,5937811 ) ) v1 2013-04-26 02:39:02.569098 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync mon_sync( chunk bl 1006642 bytes last_key ( pgmap,5937814 ) ) v1 2013-04-26 02:39:02.569110 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync_chunk mon_sync( chunk bl 1006642 bytes last_key ( pgmap,5937814 ) ) v1 2013-04-26 02:39:02.640408 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync mon_sync( chunk bl 982416 bytes last_key ( pgmap,5937824 ) ) v1 2013-04-26 02:39:02.640417 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync_chunk mon_sync( chunk bl 982416 bytes last_key ( pgmap,5937824 ) ) v1 2013-04-26 02:39:02.699139 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync mon_sync( chunk bl 1046581 bytes last_key ( pgmap,5937862 ) ) v1 2013-04-26 02:39:02.699151 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync_chunk mon_sync( chunk bl 1046581 bytes last_key ( pgmap,5937862 ) ) v1 2013-04-26 02:39:02.766572 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync mon_sync( chunk bl 1045323 bytes last_key ( pgmap,5937888 ) ) v1 2013-04-26 02:39:02.766583 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync_chunk mon_sync( chunk bl 1045323 bytes last_key ( pgmap,5937888 ) ) v1 2013-04-26 02:39:02.833088 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync mon_sync( chunk bl 993517 bytes last_key ( pgmap,5937925 ) ) v1 2013-04-26 02:39:02.833100 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync_chunk mon_sync( chunk bl 993517 bytes last_key ( pgmap,5937925 ) ) v1 2013-04-26 02:39:02.905572 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync mon_sync( chunk bl 1014000 bytes last_key ( pgmap,5937943 ) ) v1 2013-04-26 02:39:02.905585 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync_chunk mon_sync( chunk bl 1014000 bytes last_key ( pgmap,5937943 ) ) v1 2013-04-26 02:39:02.964127 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync mon_sync( chunk bl 760098 bytes last_key ( pgmap,5937950 ) ) v1 2013-04-26 02:39:02.964139 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync_chunk mon_sync( chunk bl 760098 bytes last_key ( pgmap,5937950 ) ) v1 2013-04-26 02:39:03.030153 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync mon_sync( chunk bl 1022640 bytes last_key ( pgmap,5937954 ) ) v1 2013-04-26 02:39:03.030161 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync_chunk mon_sync( chunk bl 1022640 bytes last_key ( pgmap,5937954 ) ) v1 2013-04-26 02:39:03.205826 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync mon_sync( chunk bl 1047130 bytes last_key ( pgmap,5937961 ) ) v1 2013-04-26 02:39:03.205837 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync_chunk mon_sync( chunk bl 1047130 bytes last_key ( pgmap,5937961 ) ) v1 2013-04-26 02:39:03.306406 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync mon_sync( chunk bl 1027606 bytes last_key ( pgmap,5938021 ) ) v1 2013-04-26 02:39:03.306489 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync_chunk mon_sync( chunk bl 1027606 bytes last_key ( pgmap,5938021 ) ) v1 2013-04-26 02:39:03.373470 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync mon_sync( chunk bl 1048339 bytes last_key ( pgmap,5938054 ) ) v1 2013-04-26 02:39:03.373490 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync_chunk mon_sync( chunk bl 1048339 bytes last_key ( pgmap,5938054 ) ) v1 2013-04-26 02:39:03.439859 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync mon_sync( chunk bl 953405 bytes last_key ( pgmap,5938071 ) ) v1 2013-04-26 02:39:03.439872 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync_chunk mon_sync( chunk bl 953405 bytes last_key ( pgmap,5938071 ) ) v1 2013-04-26 02:39:03.506474 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync mon_sync( chunk bl 974217 bytes last_key ( pgmap,5938099 ) ) v1 2013-04-26 02:39:03.506487 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync_chunk mon_sync( chunk bl 974217 bytes last_key ( pgmap,5938099 ) ) v1 2013-04-26 02:39:03.566968 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync mon_sync( chunk bl 1035783 bytes last_key ( pgmap,5938129 ) ) v1 2013-04-26 02:39:03.566979 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync_chunk mon_sync( chunk bl 1035783 bytes last_key ( pgmap,5938129 ) ) v1 2013-04-26 02:39:03.657124 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync mon_sync( chunk bl 209335 bytes last_key ( pgmap,first_committed ) ) v1 2013-04-26 02:39:03.657133 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync_chunk mon_sync( chunk bl 209335 bytes last_key ( pgmap,first_committed ) ) v1 2013-04-26 02:39:03.708822 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync mon_sync( chunk bl 964168 bytes last_key ( pgmap,full_5937650 ) ) v1 2013-04-26 02:39:03.708832 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync_chunk mon_sync( chunk bl 964168 bytes last_key ( pgmap,full_5937650 ) ) v1 2013-04-26 02:39:03.777597 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync mon_sync( chunk bl 964168 bytes last_key ( pgmap,full_5937676 ) ) v1 2013-04-26 02:39:03.777656 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync_chunk mon_sync( chunk bl 964168 bytes last_key ( pgmap,full_5937676 ) ) v1 2013-04-26 02:39:03.849417 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync mon_sync( chunk bl 964168 bytes last_key ( pgmap,full_5937702 ) ) v1 2013-04-26 02:39:03.849436 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync_chunk mon_sync( chunk bl 964168 bytes last_key ( pgmap,full_5937702 ) ) v1 2013-04-26 02:39:03.917533 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync mon_sync( chunk bl 962312 bytes last_key ( pgmap,full_5937728 ) ) v1 2013-04-26 02:39:03.917616 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync_chunk mon_sync( chunk bl 962312 bytes last_key ( pgmap,full_5937728 ) ) v1 2013-04-26 02:39:03.977098 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync mon_sync( chunk bl 963324 bytes last_key ( pgmap,full_5937754 ) ) v1 2013-04-26 02:39:03.977108 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync_chunk mon_sync( chunk bl 963324 bytes last_key ( pgmap,full_5937754 ) ) v1 2013-04-26 02:39:04.060801 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync mon_sync( chunk bl 964168 bytes last_key ( pgmap,full_5937780 ) ) v1 2013-04-26 02:39:04.060813 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync_chunk mon_sync( chunk bl 964168 bytes last_key ( pgmap,full_5937780 ) ) v1 2013-04-26 02:39:04.119233 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync mon_sync( chunk bl 952324 bytes last_key ( pgmap,full_5937806 ) ) v1 2013-04-26 02:39:04.119245 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync_chunk mon_sync( chunk bl 952324 bytes last_key ( pgmap,full_5937806 ) ) v1 2013-04-26 02:39:04.177876 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync mon_sync( chunk bl 964168 bytes last_key ( pgmap,full_5937832 ) ) v1 2013-04-26 02:39:04.177886 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync_chunk mon_sync( chunk bl 964168 bytes last_key ( pgmap,full_5937832 ) ) v1 2013-04-26 02:39:04.252570 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync mon_sync( chunk bl 964168 bytes last_key ( pgmap,full_5937858 ) ) v1 2013-04-26 02:39:04.252580 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync_chunk mon_sync( chunk bl 964168 bytes last_key ( pgmap,full_5937858 ) ) v1 2013-04-26 02:39:04.328325 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync mon_sync( chunk bl 964168 bytes last_key ( pgmap,full_5937884 ) ) v1 2013-04-26 02:39:04.328337 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync_chunk mon_sync( chunk bl 964168 bytes last_key ( pgmap,full_5937884 ) ) v1 2013-04-26 02:39:04.395701 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync mon_sync( chunk bl 964168 bytes last_key ( pgmap,full_5937910 ) ) v1 2013-04-26 02:39:04.395727 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync_chunk mon_sync( chunk bl 964168 bytes last_key ( pgmap,full_5937910 ) ) v1 2013-04-26 02:39:04.462394 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync mon_sync( chunk bl 964168 bytes last_key ( pgmap,full_5937936 ) ) v1 2013-04-26 02:39:04.462406 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync_chunk mon_sync( chunk bl 964168 bytes last_key ( pgmap,full_5937936 ) ) v1 2013-04-26 02:39:04.529242 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync mon_sync( chunk bl 964168 bytes last_key ( pgmap,full_5937962 ) ) v1 2013-04-26 02:39:04.529256 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync_chunk mon_sync( chunk bl 964168 bytes last_key ( pgmap,full_5937962 ) ) v1 2013-04-26 02:39:04.613015 7f27d680f700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 sync_send_heartbeat mon.1 10.227.227.103:6789/0 reply(0) 2013-04-26 02:39:04.613090 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync mon_sync( chunk bl 964168 bytes last_key ( pgmap,full_5937988 ) ) v1 2013-04-26 02:39:04.613109 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync_chunk mon_sync( chunk bl 964168 bytes last_key ( pgmap,full_5937988 ) ) v1 2013-04-26 02:39:04.679006 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync mon_sync( heartbeat_reply ) v1 2013-04-26 02:39:04.679016 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync_heartbeat_reply mon_sync( heartbeat_reply ) v1 2013-04-26 02:39:04.679042 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync mon_sync( chunk bl 964168 bytes last_key ( pgmap,full_5938014 ) ) v1 2013-04-26 02:39:04.679046 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync_chunk mon_sync( chunk bl 964168 bytes last_key ( pgmap,full_5938014 ) ) v1 2013-04-26 02:39:04.737977 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync mon_sync( chunk bl 964168 bytes last_key ( pgmap,full_5938040 ) ) v1 2013-04-26 02:39:04.737986 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync_chunk mon_sync( chunk bl 964168 bytes last_key ( pgmap,full_5938040 ) ) v1 2013-04-26 02:39:04.796802 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync mon_sync( chunk bl 964168 bytes last_key ( pgmap,full_5938066 ) ) v1 2013-04-26 02:39:04.796812 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync_chunk mon_sync( chunk bl 964168 bytes last_key ( pgmap,full_5938066 ) ) v1 2013-04-26 02:39:04.863350 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync mon_sync( chunk bl 964168 bytes last_key ( pgmap,full_5938092 ) ) v1 2013-04-26 02:39:04.863359 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync_chunk mon_sync( chunk bl 964168 bytes last_key ( pgmap,full_5938092 ) ) v1 2013-04-26 02:39:04.939183 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync mon_sync( chunk bl 964257 bytes last_key ( pgmap,last_committed ) ) v1 2013-04-26 02:39:04.939195 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync_chunk mon_sync( chunk bl 964257 bytes last_key ( pgmap,last_committed ) ) v1 2013-04-26 02:39:05.014115 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync mon_sync( chunk bl 869389 bytes last_key ( pgmap,latest ) ) v1 2013-04-26 02:39:05.014128 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync_chunk mon_sync( chunk bl 869389 bytes last_key ( pgmap,latest ) ) v1 2013-04-26 02:39:05.097250 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync mon_sync( chunk v 11669190 flags( last ) bl 342635 bytes last_key ( paxos,last_pn ) ) v1 2013-04-26 02:39:05.097262 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 handle_sync_chunk mon_sync( chunk v 11669190 flags( last ) bl 342635 bytes last_key ( paxos,last_pn ) ) v1 2013-04-26 02:39:05.155402 7f27d600e700 10 mon.1@0(synchronizing sync( requester state chunks )) e1 sync_stop 2013-04-26 02:39:05.156369 7f27d600e700 10 mon.1@0(synchronizing sync( requester state stop )) e1 handle_sync mon_sync( finish_reply ) v1 2013-04-26 02:39:05.156381 7f27d600e700 10 mon.1@0(synchronizing sync( requester state stop )) e1 handle_sync_finish_reply mon_sync( finish_reply ) v1 2013-04-26 02:39:05.223697 7f27d600e700 10 mon.1@0(synchronizing) e1 init_paxos 2013-04-26 02:39:05.224079 7f27d600e700 7 mon.1@0(synchronizing).pg v5938124 update_from_paxos applying incremental 5938125 2013-04-26 02:39:05.227739 7f27d600e700 10 mon.1@0(synchronizing).pg v5938125 v5938125: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 1250KB/s wr, 49op/s 2013-04-26 02:39:05.227955 7f27d600e700 7 mon.1@0(synchronizing).pg v5938125 update_from_paxos applying incremental 5938126 2013-04-26 02:39:05.229249 7f27d600e700 10 mon.1@0(synchronizing).pg v5938126 v5938126: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 1633KB/s wr, 53op/s 2013-04-26 02:39:05.229357 7f27d600e700 7 mon.1@0(synchronizing).pg v5938126 update_from_paxos applying incremental 5938127 2013-04-26 02:39:05.229475 7f27d600e700 10 mon.1@0(synchronizing).pg v5938127 v5938127: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 4240KB/s wr, 63op/s 2013-04-26 02:39:05.229546 7f27d600e700 7 mon.1@0(synchronizing).pg v5938127 update_from_paxos applying incremental 5938128 2013-04-26 02:39:05.229749 7f27d600e700 10 mon.1@0(synchronizing).pg v5938128 v5938128: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 1449KB/s wr, 39op/s 2013-04-26 02:39:05.229825 7f27d600e700 7 mon.1@0(synchronizing).pg v5938128 update_from_paxos applying incremental 5938129 2013-04-26 02:39:05.229924 7f27d600e700 10 mon.1@0(synchronizing).pg v5938129 v5938129: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 179KB/s wr, 9op/s 2013-04-26 02:39:05.230003 7f27d600e700 7 mon.1@0(synchronizing).pg v5938129 update_from_paxos applying incremental 5938130 2013-04-26 02:39:05.230686 7f27d600e700 10 mon.1@0(synchronizing).pg v5938130 v5938130: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 292KB/s wr, 24op/s 2013-04-26 02:39:05.230787 7f27d600e700 7 mon.1@0(synchronizing).pg v5938130 update_from_paxos applying incremental 5938131 2013-04-26 02:39:05.231226 7f27d600e700 10 mon.1@0(synchronizing).pg v5938131 v5938131: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 413KB/s wr, 34op/s 2013-04-26 02:39:05.231307 7f27d600e700 7 mon.1@0(synchronizing).pg v5938131 update_from_paxos applying incremental 5938132 2013-04-26 02:39:05.231439 7f27d600e700 10 mon.1@0(synchronizing).pg v5938132 v5938132: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 183KB/s wr, 19op/s 2013-04-26 02:39:05.231502 7f27d600e700 7 mon.1@0(synchronizing).pg v5938132 update_from_paxos applying incremental 5938133 2013-04-26 02:39:05.231574 7f27d600e700 10 mon.1@0(synchronizing).pg v5938133 v5938133: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 79494B/s wr, 9op/s 2013-04-26 02:39:05.231651 7f27d600e700 7 mon.1@0(synchronizing).pg v5938133 update_from_paxos applying incremental 5938134 2013-04-26 02:39:05.232382 7f27d600e700 10 mon.1@0(synchronizing).pg v5938134 v5938134: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 211KB/s wr, 18op/s 2013-04-26 02:39:05.232552 7f27d600e700 7 mon.1@0(synchronizing).pg v5938134 update_from_paxos applying incremental 5938135 2013-04-26 02:39:05.232964 7f27d600e700 10 mon.1@0(synchronizing).pg v5938135 v5938135: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 292KB/s wr, 32op/s 2013-04-26 02:39:05.233050 7f27d600e700 7 mon.1@0(synchronizing).pg v5938135 update_from_paxos applying incremental 5938136 2013-04-26 02:39:05.233225 7f27d600e700 10 mon.1@0(synchronizing).pg v5938136 v5938136: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 115KB/s wr, 14op/s 2013-04-26 02:39:05.233294 7f27d600e700 10 mon.1@0(synchronizing).pg v5938136 send_pg_creates to 0 pgs 2013-04-26 02:39:05.233300 7f27d600e700 10 mon.1@0(synchronizing).pg v5938136 update_logger 2013-04-26 02:39:05.233403 7f27d600e700 10 mon.1@0(synchronizing).log v5604019 update_from_paxos 2013-04-26 02:39:05.233492 7f27d600e700 10 mon.1@0(synchronizing).log v5604019 update_from_paxos version 5604019 summary v 5604010 2013-04-26 02:39:05.233571 7f27d600e700 10 mon.1@0(synchronizing).log v5604019 update_from_paxos latest full 5604018 2013-04-26 02:39:05.233655 7f27d600e700 7 mon.1@0(synchronizing).log v5604019 update_from_paxos loading summary e5604018 2013-04-26 02:39:05.233770 7f27d600e700 7 mon.1@0(synchronizing).log v5604019 update_from_paxos loaded summary e5604018 2013-04-26 02:39:05.234013 7f27d600e700 7 mon.1@0(synchronizing).log v5604019 update_from_paxos applying incremental log 5604019 2013-04-26 02:38:56.584362 mon.1 10.227.227.103:6789/0 170 : [INF] pgmap v5938135: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 292KB/s wr, 32op/s 2013-04-26 02:39:05.234175 7f27d600e700 10 mon.1@0(synchronizing).log v5604019 check_subs 2013-04-26 02:39:05.234287 7f27d600e700 10 mon.1@0(synchronizing).auth v4369 update_from_paxos 2013-04-26 02:39:05.234361 7f27d600e700 10 mon.1@0(synchronizing) e1 bootstrap 2013-04-26 02:39:05.234368 7f27d600e700 10 mon.1@0(synchronizing) e1 unregister_cluster_logger - not registered 2013-04-26 02:39:05.234373 7f27d600e700 10 mon.1@0(synchronizing) e1 cancel_probe_timeout (none scheduled) 2013-04-26 02:39:05.234376 7f27d600e700 10 mon.1@0(synchronizing) e1 reset_sync 2013-04-26 02:39:05.234380 7f27d600e700 10 mon.1@0(probing) e1 reset 2013-04-26 02:39:05.234382 7f27d600e700 10 mon.1@0(probing) e1 timecheck_finish 2013-04-26 02:39:05.234392 7f27d600e700 10 mon.1@0(probing) e1 cancel_probe_timeout (none scheduled) 2013-04-26 02:39:05.234402 7f27d600e700 10 mon.1@0(probing) e1 reset_probe_timeout 0x17524c0 after 2 seconds 2013-04-26 02:39:05.234419 7f27d600e700 10 mon.1@0(probing) e1 probing other monitors 2013-04-26 02:39:05.235535 7f27d600e700 10 mon.1@0(probing) e1 handle_probe mon_probe(reply 5eacb654-3204-4ca1-9b16-83bb0140e497 name 4 quorum 1,2 paxos( fc 11669173 lc 11669200 )) v4 2013-04-26 02:39:05.235553 7f27d600e700 10 mon.1@0(probing) e1 handle_probe_reply mon.2 10.227.227.104:6789/0mon_probe(reply 5eacb654-3204-4ca1-9b16-83bb0140e497 name 4 quorum 1,2 paxos( fc 11669173 lc 11669200 )) v4 2013-04-26 02:39:05.235567 7f27d600e700 10 mon.1@0(probing) e1 monmap is e1: 3 mons at {1=10.227.227.101:6789/0,3=10.227.227.103:6789/0,4=10.227.227.104:6789/0} 2013-04-26 02:39:05.235587 7f27d600e700 10 mon.1@0(probing) e1 peer name is 4 2013-04-26 02:39:05.235593 7f27d600e700 10 mon.1@0(probing) e1 existing quorum 1,2 2013-04-26 02:39:05.235595 7f27d600e700 10 mon.1@0(probing) e1 peer paxos version 11669200 vs my version 11669190 (ok) 2013-04-26 02:39:05.235599 7f27d600e700 10 mon.1@0(probing) e1 start_election 2013-04-26 02:39:05.235604 7f27d600e700 10 mon.1@0(probing) e1 cancel_probe_timeout 0x17524c0 2013-04-26 02:39:05.235655 7f27d600e700 0 log [INF] : mon.1 calling new monitor election 2013-04-26 02:39:05.235720 7f27d600e700 5 mon.1@0(electing).elector(0) start -- can i be leader? 2013-04-26 02:39:05.235822 7f27d600e700 1 mon.1@0(electing).elector(2810) init, last seen epoch 2810 2013-04-26 02:39:05.235833 7f27d600e700 10 mon.1@0(electing).elector(2810) bump_epoch 2810 to 2811 2013-04-26 02:39:05.245322 7f27d600e700 10 mon.1@0(electing) e1 handle_probe mon_probe(reply 5eacb654-3204-4ca1-9b16-83bb0140e497 name 3 quorum 1,2 paxos( fc 11669173 lc 11669200 )) v4 2013-04-26 02:39:05.245419 7f27d600e700 10 mon.1@0(electing) e1 handle_probe_reply mon.1 10.227.227.103:6789/0mon_probe(reply 5eacb654-3204-4ca1-9b16-83bb0140e497 name 3 quorum 1,2 paxos( fc 11669173 lc 11669200 )) v4 2013-04-26 02:39:05.245442 7f27d600e700 10 mon.1@0(electing) e1 monmap is e1: 3 mons at {1=10.227.227.101:6789/0,3=10.227.227.103:6789/0,4=10.227.227.104:6789/0} 2013-04-26 02:39:05.245473 7f27d600e700 10 mon.1@0(electing) e1 do not have session, making new one 2013-04-26 02:39:05.245478 7f27d600e700 10 mon.1@0(electing) e1 ms_dispatch new session MonSession: mon.0 10.227.227.101:6789/0 is open for mon.0 10.227.227.101:6789/0 2013-04-26 02:39:05.245490 7f27d600e700 5 mon.1@0(electing) e1 setting monitor caps on this connection 2013-04-26 02:39:05.248225 7f27d600e700 5 mon.1@0(electing).elector(2811) handle_propose from mon.1 2013-04-26 02:39:05.248240 7f27d600e700 10 mon.1@0(electing).elector(2811) bump_epoch 2811 to 2813 2013-04-26 02:39:05.257035 7f27d600e700 10 mon.1@0(electing) e1 reset 2013-04-26 02:39:05.257051 7f27d600e700 10 mon.1@0(electing) e1 timecheck_finish 2013-04-26 02:39:05.257058 7f27d600e700 10 mon.1@0(electing) e1 start_election 2013-04-26 02:39:05.257061 7f27d600e700 10 mon.1@0(electing) e1 cancel_probe_timeout (none scheduled) 2013-04-26 02:39:05.257071 7f27d600e700 0 log [INF] : mon.1 calling new monitor election 2013-04-26 02:39:05.257165 7f27d600e700 5 mon.1@0(electing).elector(2813) start -- can i be leader? 2013-04-26 02:39:05.257232 7f27d600e700 1 mon.1@0(electing).elector(2813) init, last seen epoch 2813 2013-04-26 02:39:05.257385 7f27d600e700 5 mon.1@0(electing).elector(2813) handle_ack from mon.1 2013-04-26 02:39:05.257394 7f27d600e700 5 mon.1@0(electing).elector(2813) so far i have {0=4294967295,1=4294967295} 2013-04-26 02:39:05.257427 7f27d600e700 5 mon.1@0(electing).elector(2813) handle_propose from mon.2 2013-04-26 02:39:05.257609 7f27d600e700 5 mon.1@0(electing).elector(2813) handle_ack from mon.2 2013-04-26 02:39:05.257616 7f27d600e700 5 mon.1@0(electing).elector(2813) so far i have {0=4294967295,1=4294967295,2=4294967295} 2013-04-26 02:39:05.257623 7f27d600e700 10 mon.1@0(electing).elector(2813) bump_epoch 2813 to 2814 2013-04-26 02:39:05.265447 7f27d600e700 10 mon.1@0(leader) e1 win_election, epoch 2814 quorum is 0,1,2 features are 4294967295 2013-04-26 02:39:05.265465 7f27d600e700 0 log [INF] : mon.1@0 won leader election with quorum 0,1,2 2013-04-26 02:39:05.273750 7f27d600e700 10 mon.1@0(leader).data_health(2814) start_epoch epoch 2814 2013-04-26 02:39:05.273764 7f27d600e700 10 mon.1@0(leader) e1 timecheck_finish 2013-04-26 02:39:05.273769 7f27d600e700 10 mon.1@0(leader) e1 resend_routed_requests 2013-04-26 02:39:05.273776 7f27d600e700 10 mon.1@0(leader) e1 register_cluster_logger 2013-04-26 02:39:05.273789 7f27d600e700 10 mon.1@0(leader) e1 timecheck_start 2013-04-26 02:39:05.273792 7f27d600e700 10 mon.1@0(leader) e1 timecheck_start_round curr 0 2013-04-26 02:39:05.273795 7f27d600e700 10 mon.1@0(leader) e1 timecheck_start_round new 1 2013-04-26 02:39:05.273799 7f27d600e700 10 mon.1@0(leader) e1 timecheck 2013-04-26 02:39:05.273802 7f27d600e700 10 mon.1@0(leader) e1 timecheck start timecheck epoch 2814 round 1 2013-04-26 02:39:05.273811 7f27d600e700 10 mon.1@0(leader) e1 timecheck send time_check( ping e 2814 r 1 ) v1 to mon.1 10.227.227.103:6789/0 2013-04-26 02:39:05.273837 7f27d600e700 10 mon.1@0(leader) e1 timecheck send time_check( ping e 2814 r 1 ) v1 to mon.2 10.227.227.104:6789/0 2013-04-26 02:39:05.273857 7f27d600e700 10 mon.1@0(leader) e1 timecheck_start_round setting up next event 2013-04-26 02:39:05.273899 7f27d600e700 5 mon.1@0(leader).elector(2814) old epoch, dropping 2013-04-26 02:39:05.274090 7f27d600e700 5 mon.1@0(leader).elector(2814) old epoch, dropping 2013-04-26 02:39:05.274115 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from mon.2 10.227.227.104:6789/0 via mon.2 10.227.227.104:6789/0 2013-04-26 02:39:05.274127 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x17c3680 from 10.227.227.104:6789/0 2013-04-26 02:39:05.274281 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.6 10.227.227.103:6807/8402 via mon.2 10.227.227.104:6789/0 2013-04-26 02:39:05.274295 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x2eba500 from 10.227.227.104:6789/0 2013-04-26 02:39:05.364588 7f27d600e700 10 mon.1@0(leader) e1 handle_timecheck time_check( pong e 2814 r 1 ts 2013-04-26 02:39:05.361559 ) v1 2013-04-26 02:39:05.364602 7f27d600e700 10 mon.1@0(leader) e1 handle_timecheck_leader time_check( pong e 2814 r 1 ts 2013-04-26 02:39:05.361559 ) v1 2013-04-26 02:39:05.364620 7f27d600e700 10 mon.1@0(leader) e1 handle_timecheck_leader from mon.1 10.227.227.103:6789/0 ts 2013-04-26 02:39:05.361559 delta -0.00305319 skew_bound 0 latency 0.090805 2013-04-26 02:39:05.364912 7f27d600e700 7 mon.1@0(leader).pg v5938136 update_from_paxos applying incremental 5938137 2013-04-26 02:39:05.365527 7f27d600e700 10 mon.1@0(leader).pg v5938137 v5938137: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 311KB/s wr, 24op/s 2013-04-26 02:39:05.365638 7f27d600e700 7 mon.1@0(leader).pg v5938137 update_from_paxos applying incremental 5938138 2013-04-26 02:39:05.366537 7f27d600e700 10 mon.1@0(leader).pg v5938138 v5938138: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 1909KB/s wr, 114op/s 2013-04-26 02:39:05.366630 7f27d600e700 7 mon.1@0(leader).pg v5938138 update_from_paxos applying incremental 5938139 2013-04-26 02:39:05.366688 7f27d600e700 10 mon.1@0(leader).pg v5938139 v5938139: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 1191KB/s wr, 66op/s 2013-04-26 02:39:05.366752 7f27d600e700 7 mon.1@0(leader).pg v5938139 update_from_paxos applying incremental 5938140 2013-04-26 02:39:05.366860 7f27d600e700 10 mon.1@0(leader).pg v5938140 v5938140: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 49957B/s wr, 8op/s 2013-04-26 02:39:05.366936 7f27d600e700 7 mon.1@0(leader).pg v5938140 update_from_paxos applying incremental 5938141 2013-04-26 02:39:05.367332 7f27d600e700 10 mon.1@0(leader).pg v5938141 v5938141: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 2718KB/s wr, 43op/s 2013-04-26 02:39:05.367471 7f27d600e700 10 mon.1@0(leader).pg v5938141 send_pg_creates to 0 pgs 2013-04-26 02:39:05.367484 7f27d600e700 10 mon.1@0(leader).pg v5938141 update_logger 2013-04-26 02:39:05.367524 7f27d600e700 10 mon.1@0(leader).pg v5938141 create_pending v 5938142 2013-04-26 02:39:05.367563 7f27d600e700 10 mon.1@0(leader).pg v5938141 check_osd_map already seen 10556 >= 10556 2013-04-26 02:39:05.367568 7f27d600e700 10 mon.1@0(leader).pg v5938141 update_logger 2013-04-26 02:39:05.367590 7f27d600e700 0 log [INF] : pgmap v5938141: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 2718KB/s wr, 43op/s 2013-04-26 02:39:05.367755 7f27d600e700 10 mon.1@0(leader).mds e3724 create_pending e3725 2013-04-26 02:39:05.367825 7f27d600e700 10 mon.1@0(leader).mds e3724 e3724: 1/1/1 up {0=4=up:active}, 2 up:standby 2013-04-26 02:39:05.367853 7f27d600e700 10 mon.1@0(leader).mds e3724 adding 10.227.227.104:6801/10842 mds.0.262 up:active to last_beacon 2013-04-26 02:39:05.367865 7f27d600e700 10 mon.1@0(leader).mds e3724 adding 10.227.227.101:6800/29852 mds.-1.0 up:standby to last_beacon 2013-04-26 02:39:05.367871 7f27d600e700 10 mon.1@0(leader).mds e3724 adding 10.227.227.103:6800/8011 mds.-1.0 up:standby to last_beacon 2013-04-26 02:39:05.367883 7f27d600e700 10 mon.1@0(leader).mds e3724 update_logger 2013-04-26 02:39:05.367896 7f27d600e700 0 log [INF] : mdsmap e3724: 1/1/1 up {0=4=up:active}, 2 up:standby 2013-04-26 02:39:05.368050 7f27d600e700 10 mon.1@0(leader).osd e10556 create_pending e 10557 2013-04-26 02:39:05.368062 7f27d600e700 10 mon.1@0(leader).osd e10556 remove_redundant_pg_temp 2013-04-26 02:39:05.368066 7f27d600e700 10 mon.1@0(leader).osd e10556 remove_down_pg_temp 2013-04-26 02:39:05.368155 7f27d600e700 10 mon.1@0(leader).osd e10556 update_logger 2013-04-26 02:39:05.368170 7f27d600e700 0 log [INF] : osdmap e10556: 12 osds: 12 up, 12 in 2013-04-26 02:39:05.368226 7f27d600e700 10 mon.1@0(leader).log v5604024 update_from_paxos 2013-04-26 02:39:05.368323 7f27d600e700 10 mon.1@0(leader).log v5604024 update_from_paxos version 5604024 summary v 5604019 2013-04-26 02:39:05.368459 7f27d600e700 10 mon.1@0(leader).log v5604024 update_from_paxos latest full 5604023 2013-04-26 02:39:05.368546 7f27d600e700 7 mon.1@0(leader).log v5604024 update_from_paxos loading summary e5604023 2013-04-26 02:39:05.368651 7f27d600e700 7 mon.1@0(leader).log v5604024 update_from_paxos loaded summary e5604023 2013-04-26 02:39:05.368735 7f27d600e700 7 mon.1@0(leader).log v5604024 update_from_paxos applying incremental log 5604024 2013-04-26 02:39:03.934026 mon.1 10.227.227.103:6789/0 175 : [INF] pgmap v5938140: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 49957B/s wr, 8op/s 2013-04-26 02:39:05.368861 7f27d600e700 10 mon.1@0(leader).log v5604024 check_subs 2013-04-26 02:39:05.368959 7f27d600e700 10 mon.1@0(leader).log v5604024 create_pending v 5604025 2013-04-26 02:39:05.369133 7f27d600e700 10 mon.1@0(leader).monmap v1 create_pending monmap epoch 2 2013-04-26 02:39:05.369225 7f27d600e700 0 log [INF] : monmap e1: 3 mons at {1=10.227.227.101:6789/0,3=10.227.227.103:6789/0,4=10.227.227.104:6789/0} 2013-04-26 02:39:05.369285 7f27d600e700 10 mon.1@0(leader).auth v4369 update_from_paxos 2013-04-26 02:39:05.369398 7f27d600e700 10 mon.1@0(leader).auth v4369 create_pending v 4370 2013-04-26 02:39:05.369480 7f27d600e700 10 mon.1@0(leader).auth v4369 AuthMonitor::on_active() 2013-04-26 02:39:05.369587 7f27d600e700 10 mon.1@0(leader).log v5604024 update_from_paxos 2013-04-26 02:39:05.369655 7f27d600e700 10 mon.1@0(leader).log v5604024 update_from_paxos version 5604024 summary v 5604024 2013-04-26 02:39:05.369690 7f27d600e700 10 mon.1@0(leader).log v5604024 preprocess_query log(1 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:05.369734 7f27d600e700 10 mon.1@0(leader).log v5604024 preprocess_log log(1 entries) v1 from mon.0 2013-04-26 02:39:05.369783 7f27d600e700 10 mon.1@0(leader).log v5604024 prepare_update log(1 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:05.369831 7f27d600e700 10 mon.1@0(leader).log v5604024 prepare_log log(1 entries) v1 from mon.0 2013-04-26 02:39:05.369871 7f27d600e700 10 mon.1@0(leader).log v5604024 logging 2013-04-26 02:39:05.235658 mon.0 10.227.227.101:6789/0 1 : [INF] mon.1 calling new monitor election 2013-04-26 02:39:05.370023 7f27d600e700 10 mon.1@0(leader).log v5604024 update_from_paxos 2013-04-26 02:39:05.370133 7f27d600e700 10 mon.1@0(leader).log v5604024 update_from_paxos version 5604024 summary v 5604024 2013-04-26 02:39:05.370174 7f27d600e700 10 mon.1@0(leader).log v5604024 preprocess_query log(1 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:05.370220 7f27d600e700 10 mon.1@0(leader).log v5604024 preprocess_log log(1 entries) v1 from mon.0 2013-04-26 02:39:05.370269 7f27d600e700 10 mon.1@0(leader).log v5604024 nothing new 2013-04-26 02:39:05.370428 7f27d600e700 10 mon.1@0(leader).log v5604024 update_from_paxos 2013-04-26 02:39:05.370497 7f27d600e700 10 mon.1@0(leader).log v5604024 update_from_paxos version 5604024 summary v 5604024 2013-04-26 02:39:05.370534 7f27d600e700 10 mon.1@0(leader).log v5604024 preprocess_query log(2 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:05.370579 7f27d600e700 10 mon.1@0(leader).log v5604024 preprocess_log log(2 entries) v1 from mon.0 2013-04-26 02:39:05.370628 7f27d600e700 10 mon.1@0(leader).log v5604024 prepare_update log(2 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:05.370672 7f27d600e700 10 mon.1@0(leader).log v5604024 prepare_log log(2 entries) v1 from mon.0 2013-04-26 02:39:05.370713 7f27d600e700 10 mon.1@0(leader).log v5604024 logging 2013-04-26 02:39:05.235658 mon.0 10.227.227.101:6789/0 1 : [INF] mon.1 calling new monitor election 2013-04-26 02:39:05.370768 7f27d600e700 10 mon.1@0(leader).log v5604024 logging 2013-04-26 02:39:05.257073 mon.0 10.227.227.101:6789/0 2 : [INF] mon.1 calling new monitor election 2013-04-26 02:39:05.370922 7f27d600e700 10 mon.1@0(leader).log v5604024 update_from_paxos 2013-04-26 02:39:05.370990 7f27d600e700 10 mon.1@0(leader).log v5604024 update_from_paxos version 5604024 summary v 5604024 2013-04-26 02:39:05.371027 7f27d600e700 10 mon.1@0(leader).log v5604024 preprocess_query log(1 entries) v1 from mon.2 10.227.227.104:6789/0 2013-04-26 02:39:05.371071 7f27d600e700 10 mon.1@0(leader).log v5604024 preprocess_log log(1 entries) v1 from mon.2 2013-04-26 02:39:05.371114 7f27d600e700 10 mon.1@0(leader).log v5604024 nothing new 2013-04-26 02:39:05.371249 7f27d600e700 10 mon.1@0(leader).pg v5938141 preprocess_query pg_stats(11 pgs tid 55 v 10556) v1 from osd.6 10.227.227.103:6807/8402 2013-04-26 02:39:05.371337 7f27d600e700 10 mon.1@0(leader).pg v5938141 prepare_update pg_stats(11 pgs tid 55 v 10556) v1 from osd.6 10.227.227.103:6807/8402 2013-04-26 02:39:05.371352 7f27d600e700 10 mon.1@0(leader).pg v5938141 prepare_pg_stats pg_stats(11 pgs tid 55 v 10556) v1 from osd.6 2013-04-26 02:39:05.371370 7f27d600e700 10 mon.1@0(leader).pg v5938141 got osd.6 osd_stat(184 GB used, 745 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[]) (was osd_stat(184 GB used, 745 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[])) 2013-04-26 02:39:05.371536 7f27d600e700 10 mon.1@0(leader).log v5604024 update_from_paxos 2013-04-26 02:39:05.371608 7f27d600e700 10 mon.1@0(leader).log v5604024 update_from_paxos version 5604024 summary v 5604024 2013-04-26 02:39:05.371679 7f27d600e700 10 mon.1@0(leader).auth v4369 update_from_paxos 2013-04-26 02:39:05.371941 7f27d600e700 10 mon.1@0(leader) e1 handle_timecheck time_check( pong e 2814 r 1 ts 2013-04-26 02:39:05.281539 ) v1 2013-04-26 02:39:05.371954 7f27d600e700 10 mon.1@0(leader) e1 handle_timecheck_leader time_check( pong e 2814 r 1 ts 2013-04-26 02:39:05.281539 ) v1 2013-04-26 02:39:05.371987 7f27d600e700 10 mon.1@0(leader) e1 handle_timecheck_leader from mon.2 10.227.227.104:6789/0 ts 2013-04-26 02:39:05.281539 delta -0.0904431 skew_bound 0 latency 0.098146 2013-04-26 02:39:05.372010 7f27d600e700 10 mon.1@0(leader) e1 handle_timecheck_leader got pongs from everybody (3 total) 2013-04-26 02:39:05.372018 7f27d600e700 10 mon.1@0(leader) e1 timecheck_finish_round curr 1 2013-04-26 02:39:05.372023 7f27d600e700 10 mon.1@0(leader) e1 timecheck_report 2013-04-26 02:39:05.372030 7f27d600e700 10 mon.1@0(leader) e1 timecheck_report send report to mon.1 10.227.227.103:6789/0 2013-04-26 02:39:05.372055 7f27d600e700 10 mon.1@0(leader) e1 timecheck_report send report to mon.2 10.227.227.104:6789/0 2013-04-26 02:39:05.372175 7f27d600e700 10 mon.1@0(leader).log v5604024 update_from_paxos 2013-04-26 02:39:05.372243 7f27d600e700 10 mon.1@0(leader).log v5604024 update_from_paxos version 5604024 summary v 5604024 2013-04-26 02:39:05.372280 7f27d600e700 10 mon.1@0(leader).log v5604024 preprocess_query log(2 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:05.372324 7f27d600e700 10 mon.1@0(leader).log v5604024 preprocess_log log(2 entries) v1 from mon.0 2013-04-26 02:39:05.372372 7f27d600e700 10 mon.1@0(leader).log v5604024 prepare_update log(2 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:05.372415 7f27d600e700 10 mon.1@0(leader).log v5604024 prepare_log log(2 entries) v1 from mon.0 2013-04-26 02:39:05.372456 7f27d600e700 10 mon.1@0(leader).log v5604024 logging 2013-04-26 02:39:05.257073 mon.0 10.227.227.101:6789/0 2 : [INF] mon.1 calling new monitor election 2013-04-26 02:39:05.372511 7f27d600e700 10 mon.1@0(leader).log v5604024 logging 2013-04-26 02:39:05.265466 mon.0 10.227.227.101:6789/0 3 : [INF] mon.1@0 won leader election with quorum 0,1,2 2013-04-26 02:39:05.372666 7f27d600e700 0 log [WRN] : message from mon.1 was stamped 0.078044s in the future, clocks not synchronized 2013-04-26 02:39:05.372825 7f27d600e700 10 mon.1@0(leader).log v5604024 update_from_paxos 2013-04-26 02:39:05.372897 7f27d600e700 10 mon.1@0(leader).log v5604024 update_from_paxos version 5604024 summary v 5604024 2013-04-26 02:39:05.372967 7f27d600e700 10 mon.1@0(leader).auth v4369 update_from_paxos 2013-04-26 02:39:05.373148 7f27d600e700 10 mon.1@0(leader).log v5604024 update_from_paxos 2013-04-26 02:39:05.373217 7f27d600e700 10 mon.1@0(leader).log v5604024 update_from_paxos version 5604024 summary v 5604024 2013-04-26 02:39:05.373287 7f27d600e700 10 mon.1@0(leader).auth v4369 update_from_paxos 2013-04-26 02:39:05.373434 7f27d600e700 10 mon.1@0(leader).log v5604024 update_from_paxos 2013-04-26 02:39:05.373500 7f27d600e700 10 mon.1@0(leader).log v5604024 update_from_paxos version 5604024 summary v 5604024 2013-04-26 02:39:05.373536 7f27d600e700 10 mon.1@0(leader).log v5604024 preprocess_query log(2 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:05.373579 7f27d600e700 10 mon.1@0(leader).log v5604024 preprocess_log log(2 entries) v1 from mon.0 2013-04-26 02:39:05.373627 7f27d600e700 10 mon.1@0(leader).log v5604024 prepare_update log(2 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:05.373673 7f27d600e700 10 mon.1@0(leader).log v5604024 prepare_log log(2 entries) v1 from mon.0 2013-04-26 02:39:05.373714 7f27d600e700 10 mon.1@0(leader).log v5604024 logging 2013-04-26 02:39:05.265466 mon.0 10.227.227.101:6789/0 3 : [INF] mon.1@0 won leader election with quorum 0,1,2 2013-04-26 02:39:05.373769 7f27d600e700 10 mon.1@0(leader).log v5604024 logging 2013-04-26 02:39:05.367593 mon.0 10.227.227.101:6789/0 4 : [INF] pgmap v5938141: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 2718KB/s wr, 43op/s 2013-04-26 02:39:05.373891 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from mon.2 10.227.227.104:6789/0 via mon.2 10.227.227.104:6789/0 2013-04-26 02:39:05.373907 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x1768900 from 10.227.227.104:6789/0 2013-04-26 02:39:05.373980 7f27d600e700 10 mon.1@0(leader).log v5604024 update_from_paxos 2013-04-26 02:39:05.374047 7f27d600e700 10 mon.1@0(leader).log v5604024 update_from_paxos version 5604024 summary v 5604024 2013-04-26 02:39:05.374084 7f27d600e700 10 mon.1@0(leader).log v5604024 preprocess_query log(2 entries) v1 from mon.2 10.227.227.104:6789/0 2013-04-26 02:39:05.374127 7f27d600e700 10 mon.1@0(leader).log v5604024 preprocess_log log(2 entries) v1 from mon.2 2013-04-26 02:39:05.374174 7f27d600e700 10 mon.1@0(leader).log v5604024 prepare_update log(2 entries) v1 from mon.2 10.227.227.104:6789/0 2013-04-26 02:39:05.374220 7f27d600e700 10 mon.1@0(leader).log v5604024 prepare_log log(2 entries) v1 from mon.2 2013-04-26 02:39:05.374261 7f27d600e700 10 mon.1@0(leader).log v5604024 logging 2013-04-26 02:38:44.536320 mon.2 10.227.227.104:6789/0 10 : [WRN] message from mon.1 was stamped 0.082823s in the future, clocks not synchronized 2013-04-26 02:39:05.374317 7f27d600e700 10 mon.1@0(leader).log v5604024 logging 2013-04-26 02:39:05.248213 mon.2 10.227.227.104:6789/0 11 : [INF] mon.4 calling new monitor election 2013-04-26 02:39:05.374498 7f27d600e700 10 mon.1@0(leader).log v5604024 update_from_paxos 2013-04-26 02:39:05.374565 7f27d600e700 10 mon.1@0(leader).log v5604024 update_from_paxos version 5604024 summary v 5604024 2013-04-26 02:39:05.374606 7f27d600e700 10 mon.1@0(leader).log v5604024 preprocess_query log(2 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:05.374650 7f27d600e700 10 mon.1@0(leader).log v5604024 preprocess_log log(2 entries) v1 from mon.0 2013-04-26 02:39:05.374698 7f27d600e700 10 mon.1@0(leader).log v5604024 prepare_update log(2 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:05.374744 7f27d600e700 10 mon.1@0(leader).log v5604024 prepare_log log(2 entries) v1 from mon.0 2013-04-26 02:39:05.374784 7f27d600e700 10 mon.1@0(leader).log v5604024 logging 2013-04-26 02:39:05.367593 mon.0 10.227.227.101:6789/0 4 : [INF] pgmap v5938141: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 2718KB/s wr, 43op/s 2013-04-26 02:39:05.374839 7f27d600e700 10 mon.1@0(leader).log v5604024 logging 2013-04-26 02:39:05.367898 mon.0 10.227.227.101:6789/0 5 : [INF] mdsmap e3724: 1/1/1 up {0=4=up:active}, 2 up:standby 2013-04-26 02:39:05.375072 7f27d600e700 10 mon.1@0(leader).log v5604024 update_from_paxos 2013-04-26 02:39:05.375145 7f27d600e700 10 mon.1@0(leader).log v5604024 update_from_paxos version 5604024 summary v 5604024 2013-04-26 02:39:05.375168 7f27d600e700 10 mon.1@0(leader).log v5604024 preprocess_query log(2 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:05.375194 7f27d600e700 10 mon.1@0(leader).log v5604024 preprocess_log log(2 entries) v1 from mon.0 2013-04-26 02:39:05.375222 7f27d600e700 10 mon.1@0(leader).log v5604024 prepare_update log(2 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:05.375249 7f27d600e700 10 mon.1@0(leader).log v5604024 prepare_log log(2 entries) v1 from mon.0 2013-04-26 02:39:05.375277 7f27d600e700 10 mon.1@0(leader).log v5604024 logging 2013-04-26 02:39:05.367898 mon.0 10.227.227.101:6789/0 5 : [INF] mdsmap e3724: 1/1/1 up {0=4=up:active}, 2 up:standby 2013-04-26 02:39:05.375311 7f27d600e700 10 mon.1@0(leader).log v5604024 logging 2013-04-26 02:39:05.368173 mon.0 10.227.227.101:6789/0 6 : [INF] osdmap e10556: 12 osds: 12 up, 12 in 2013-04-26 02:39:05.375429 7f27d600e700 10 mon.1@0(leader).log v5604024 update_from_paxos 2013-04-26 02:39:05.375469 7f27d600e700 10 mon.1@0(leader).log v5604024 update_from_paxos version 5604024 summary v 5604024 2013-04-26 02:39:05.375491 7f27d600e700 10 mon.1@0(leader).log v5604024 preprocess_query log(2 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:05.375516 7f27d600e700 10 mon.1@0(leader).log v5604024 preprocess_log log(2 entries) v1 from mon.0 2013-04-26 02:39:05.375544 7f27d600e700 10 mon.1@0(leader).log v5604024 prepare_update log(2 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:05.375571 7f27d600e700 10 mon.1@0(leader).log v5604024 prepare_log log(2 entries) v1 from mon.0 2013-04-26 02:39:05.375595 7f27d600e700 10 mon.1@0(leader).log v5604024 logging 2013-04-26 02:39:05.368173 mon.0 10.227.227.101:6789/0 6 : [INF] osdmap e10556: 12 osds: 12 up, 12 in 2013-04-26 02:39:05.375627 7f27d600e700 10 mon.1@0(leader).log v5604024 logging 2013-04-26 02:39:05.369230 mon.0 10.227.227.101:6789/0 7 : [INF] monmap e1: 3 mons at {1=10.227.227.101:6789/0,3=10.227.227.103:6789/0,4=10.227.227.104:6789/0} 2013-04-26 02:39:05.375698 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.4 10.227.227.104:6804/14471 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:05.375708 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x2eba280 from 10.227.227.103:6789/0 2013-04-26 02:39:05.375760 7f27d600e700 10 mon.1@0(leader).pg v5938141 preprocess_query pg_stats(33 pgs tid 744 v 10556) v1 from osd.4 10.227.227.104:6804/14471 2013-04-26 02:39:05.375808 7f27d600e700 10 mon.1@0(leader).pg v5938141 prepare_update pg_stats(33 pgs tid 744 v 10556) v1 from osd.4 10.227.227.104:6804/14471 2013-04-26 02:39:05.375816 7f27d600e700 10 mon.1@0(leader).pg v5938141 prepare_pg_stats pg_stats(33 pgs tid 744 v 10556) v1 from osd.4 2013-04-26 02:39:05.375822 7f27d600e700 10 mon.1@0(leader).pg v5938141 got osd.4 osd_stat(423 GB used, 1437 GB avail, 1860 GB total, peers [0,1,2,6,7,8,9,10,11]/[]) (was osd_stat(423 GB used, 1437 GB avail, 1860 GB total, peers [0,1,2,6,7,8,9,10,11]/[])) 2013-04-26 02:39:05.420162 7f27d680f700 10 mon.1@0(leader).log v5604024 encode_full log v 5604024 2013-04-26 02:39:05.420434 7f27d680f700 10 mon.1@0(leader).log v5604024 encode_pending v5604025 2013-04-26 02:39:05.472761 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.3 10.227.227.104:6800/14463 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:05.472807 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x2eb8f00 from 10.227.227.103:6789/0 2013-04-26 02:39:05.472941 7f27d600e700 10 mon.1@0(leader).pg v5938141 preprocess_query pg_stats(21 pgs tid 749 v 10556) v1 from osd.3 10.227.227.104:6800/14463 2013-04-26 02:39:05.473044 7f27d600e700 10 mon.1@0(leader).pg v5938141 prepare_update pg_stats(21 pgs tid 749 v 10556) v1 from osd.3 10.227.227.104:6800/14463 2013-04-26 02:39:05.473058 7f27d600e700 10 mon.1@0(leader).pg v5938141 prepare_pg_stats pg_stats(21 pgs tid 749 v 10556) v1 from osd.3 2013-04-26 02:39:05.473070 7f27d600e700 10 mon.1@0(leader).pg v5938141 got osd.3 osd_stat(383 GB used, 1477 GB avail, 1860 GB total, peers [0,1,2,6,7,8,9,10,11]/[]) (was osd_stat(383 GB used, 1477 GB avail, 1860 GB total, peers [0,1,2,6,7,8,9,10,11]/[])) 2013-04-26 02:39:05.473526 7f27d680f700 10 mon.1@0(leader).pg v5938141 encode_pending v 5938142 2013-04-26 02:39:05.554143 7f27d600e700 10 mon.1@0(leader).log v5604025 update_from_paxos 2013-04-26 02:39:05.554274 7f27d600e700 10 mon.1@0(leader).log v5604025 update_from_paxos version 5604025 summary v 5604024 2013-04-26 02:39:05.554317 7f27d600e700 10 mon.1@0(leader).log v5604025 update_from_paxos latest full 5604024 2013-04-26 02:39:05.554364 7f27d600e700 7 mon.1@0(leader).log v5604025 update_from_paxos applying incremental log 5604025 2013-04-26 02:38:44.536320 mon.2 10.227.227.104:6789/0 10 : [WRN] message from mon.1 was stamped 0.082823s in the future, clocks not synchronized 2013-04-26 02:39:05.554415 7f27d600e700 7 mon.1@0(leader).log v5604025 update_from_paxos applying incremental log 5604025 2013-04-26 02:39:05.235658 mon.0 10.227.227.101:6789/0 1 : [INF] mon.1 calling new monitor election 2013-04-26 02:39:05.554496 7f27d600e700 7 mon.1@0(leader).log v5604025 update_from_paxos applying incremental log 5604025 2013-04-26 02:39:05.248213 mon.2 10.227.227.104:6789/0 11 : [INF] mon.4 calling new monitor election 2013-04-26 02:39:05.554553 7f27d600e700 7 mon.1@0(leader).log v5604025 update_from_paxos applying incremental log 5604025 2013-04-26 02:39:05.257073 mon.0 10.227.227.101:6789/0 2 : [INF] mon.1 calling new monitor election 2013-04-26 02:39:05.554597 7f27d600e700 7 mon.1@0(leader).log v5604025 update_from_paxos applying incremental log 5604025 2013-04-26 02:39:05.265466 mon.0 10.227.227.101:6789/0 3 : [INF] mon.1@0 won leader election with quorum 0,1,2 2013-04-26 02:39:05.554640 7f27d600e700 7 mon.1@0(leader).log v5604025 update_from_paxos applying incremental log 5604025 2013-04-26 02:39:05.367593 mon.0 10.227.227.101:6789/0 4 : [INF] pgmap v5938141: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 2718KB/s wr, 43op/s 2013-04-26 02:39:05.554679 7f27d600e700 7 mon.1@0(leader).log v5604025 update_from_paxos applying incremental log 5604025 2013-04-26 02:39:05.367898 mon.0 10.227.227.101:6789/0 5 : [INF] mdsmap e3724: 1/1/1 up {0=4=up:active}, 2 up:standby 2013-04-26 02:39:05.554717 7f27d600e700 7 mon.1@0(leader).log v5604025 update_from_paxos applying incremental log 5604025 2013-04-26 02:39:05.368173 mon.0 10.227.227.101:6789/0 6 : [INF] osdmap e10556: 12 osds: 12 up, 12 in 2013-04-26 02:39:05.554758 7f27d600e700 7 mon.1@0(leader).log v5604025 update_from_paxos applying incremental log 5604025 2013-04-26 02:39:05.369230 mon.0 10.227.227.101:6789/0 7 : [INF] monmap e1: 3 mons at {1=10.227.227.101:6789/0,3=10.227.227.103:6789/0,4=10.227.227.104:6789/0} 2013-04-26 02:39:05.554824 7f27d600e700 10 mon.1@0(leader).log v5604025 check_subs 2013-04-26 02:39:05.554884 7f27d600e700 10 mon.1@0(leader).log v5604025 create_pending v 5604026 2013-04-26 02:39:05.554941 7f27d600e700 7 mon.1@0(leader).log v5604025 _updated_log for mon.0 10.227.227.101:6789/0 2013-04-26 02:39:05.554981 7f27d600e700 7 mon.1@0(leader).log v5604025 _updated_log for mon.0 10.227.227.101:6789/0 2013-04-26 02:39:05.555059 7f27d600e700 7 mon.1@0(leader).log v5604025 _updated_log for mon.0 10.227.227.101:6789/0 2013-04-26 02:39:05.555123 7f27d600e700 7 mon.1@0(leader).log v5604025 _updated_log for mon.0 10.227.227.101:6789/0 2013-04-26 02:39:05.555185 7f27d600e700 7 mon.1@0(leader).log v5604025 _updated_log for mon.2 10.227.227.104:6789/0 2013-04-26 02:39:05.555256 7f27d600e700 7 mon.1@0(leader).log v5604025 _updated_log for mon.0 10.227.227.101:6789/0 2013-04-26 02:39:05.555374 7f27d600e700 7 mon.1@0(leader).log v5604025 _updated_log for mon.0 10.227.227.101:6789/0 2013-04-26 02:39:05.555449 7f27d600e700 7 mon.1@0(leader).log v5604025 _updated_log for mon.0 10.227.227.101:6789/0 2013-04-26 02:39:05.671287 7f27d600e700 7 mon.1@0(leader).pg v5938141 update_from_paxos applying incremental 5938142 2013-04-26 02:39:05.671590 7f27d600e700 10 mon.1@0(leader).pg v5938142 v5938142: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 3809KB/s wr, 77op/s 2013-04-26 02:39:05.671646 7f27d600e700 10 mon.1@0(leader).pg v5938142 send_pg_creates to 0 pgs 2013-04-26 02:39:05.671650 7f27d600e700 10 mon.1@0(leader).pg v5938142 update_logger 2013-04-26 02:39:05.671697 7f27d600e700 10 mon.1@0(leader).pg v5938142 create_pending v 5938143 2013-04-26 02:39:05.671722 7f27d600e700 7 mon.1@0(leader).pg v5938142 _updated_stats for osd.6 10.227.227.103:6807/8402 2013-04-26 02:39:05.671788 7f27d600e700 7 mon.1@0(leader).pg v5938142 _updated_stats for osd.4 10.227.227.104:6804/14471 2013-04-26 02:39:05.671817 7f27d600e700 7 mon.1@0(leader).pg v5938142 _updated_stats for osd.3 10.227.227.104:6800/14463 2013-04-26 02:39:05.671839 7f27d600e700 10 mon.1@0(leader).pg v5938142 check_osd_map already seen 10556 >= 10556 2013-04-26 02:39:05.671843 7f27d600e700 10 mon.1@0(leader).pg v5938142 update_logger 2013-04-26 02:39:05.671862 7f27d600e700 0 log [INF] : pgmap v5938142: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 3809KB/s wr, 77op/s 2013-04-26 02:39:05.672099 7f27d600e700 10 mon.1@0(leader).log v5604025 update_from_paxos 2013-04-26 02:39:05.672158 7f27d600e700 10 mon.1@0(leader).log v5604025 update_from_paxos version 5604025 summary v 5604025 2013-04-26 02:39:05.672206 7f27d600e700 10 mon.1@0(leader).auth v4369 update_from_paxos 2013-04-26 02:39:05.672316 7f27d600e700 10 mon.1@0(leader).log v5604025 update_from_paxos 2013-04-26 02:39:05.672358 7f27d600e700 10 mon.1@0(leader).log v5604025 update_from_paxos version 5604025 summary v 5604025 2013-04-26 02:39:05.672382 7f27d600e700 10 mon.1@0(leader).log v5604025 preprocess_query log(2 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:05.672410 7f27d600e700 10 mon.1@0(leader).log v5604025 preprocess_log log(2 entries) v1 from mon.0 2013-04-26 02:39:05.672448 7f27d600e700 10 mon.1@0(leader).log v5604025 prepare_update log(2 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:05.672477 7f27d600e700 10 mon.1@0(leader).log v5604025 prepare_log log(2 entries) v1 from mon.0 2013-04-26 02:39:05.672505 7f27d600e700 10 mon.1@0(leader).log v5604025 logging 2013-04-26 02:39:05.372670 mon.0 10.227.227.101:6789/0 8 : [WRN] message from mon.1 was stamped 0.078044s in the future, clocks not synchronized 2013-04-26 02:39:05.672544 7f27d600e700 10 mon.1@0(leader).log v5604025 logging 2013-04-26 02:39:05.671864 mon.0 10.227.227.101:6789/0 9 : [INF] pgmap v5938142: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 3809KB/s wr, 77op/s 2013-04-26 02:39:05.678566 7f27d600e700 10 mon.1@0(leader).log v5604025 update_from_paxos 2013-04-26 02:39:05.678642 7f27d600e700 10 mon.1@0(leader).log v5604025 update_from_paxos version 5604025 summary v 5604025 2013-04-26 02:39:05.678709 7f27d600e700 10 mon.1@0(leader).auth v4369 update_from_paxos 2013-04-26 02:39:05.721513 7f27d600e700 10 mon.1@0(leader).log v5604025 update_from_paxos 2013-04-26 02:39:05.721653 7f27d600e700 10 mon.1@0(leader).log v5604025 update_from_paxos version 5604025 summary v 5604025 2013-04-26 02:39:05.721748 7f27d600e700 10 mon.1@0(leader).auth v4369 update_from_paxos 2013-04-26 02:39:05.913623 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.8 10.227.227.103:6813/8628 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:05.913648 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x2eb8500 from 10.227.227.103:6789/0 2013-04-26 02:39:05.913794 7f27d600e700 10 mon.1@0(leader).pg v5938142 preprocess_query pg_stats(5 pgs tid 53 v 10556) v1 from osd.8 10.227.227.103:6813/8628 2013-04-26 02:39:05.913892 7f27d600e700 10 mon.1@0(leader).pg v5938142 prepare_update pg_stats(5 pgs tid 53 v 10556) v1 from osd.8 10.227.227.103:6813/8628 2013-04-26 02:39:05.913906 7f27d600e700 10 mon.1@0(leader).pg v5938142 prepare_pg_stats pg_stats(5 pgs tid 53 v 10556) v1 from osd.8 2013-04-26 02:39:05.913925 7f27d600e700 10 mon.1@0(leader).pg v5938142 got osd.8 osd_stat(185 GB used, 744 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[]) (was osd_stat(185 GB used, 744 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[])) 2013-04-26 02:39:06.628528 7f27d680f700 10 mon.1@0(leader).log v5604025 encode_full log v 5604025 2013-04-26 02:39:06.628817 7f27d680f700 10 mon.1@0(leader).log v5604025 encode_pending v5604026 2013-04-26 02:39:06.699110 7f27d680f700 10 mon.1@0(leader).pg v5938142 encode_pending v 5938143 2013-04-26 02:39:06.757432 7f27d600e700 10 mon.1@0(leader).log v5604026 update_from_paxos 2013-04-26 02:39:06.757581 7f27d600e700 10 mon.1@0(leader).log v5604026 update_from_paxos version 5604026 summary v 5604025 2013-04-26 02:39:06.757655 7f27d600e700 10 mon.1@0(leader).log v5604026 update_from_paxos latest full 5604025 2013-04-26 02:39:06.757736 7f27d600e700 7 mon.1@0(leader).log v5604026 update_from_paxos applying incremental log 5604026 2013-04-26 02:39:05.372670 mon.0 10.227.227.101:6789/0 8 : [WRN] message from mon.1 was stamped 0.078044s in the future, clocks not synchronized 2013-04-26 02:39:06.757828 7f27d600e700 7 mon.1@0(leader).log v5604026 update_from_paxos applying incremental log 5604026 2013-04-26 02:39:05.671864 mon.0 10.227.227.101:6789/0 9 : [INF] pgmap v5938142: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 3809KB/s wr, 77op/s 2013-04-26 02:39:06.757977 7f27d600e700 10 mon.1@0(leader).log v5604026 check_subs 2013-04-26 02:39:06.758070 7f27d600e700 10 mon.1@0(leader).log v5604026 create_pending v 5604027 2013-04-26 02:39:06.758169 7f27d600e700 7 mon.1@0(leader).log v5604026 _updated_log for mon.0 10.227.227.101:6789/0 2013-04-26 02:39:06.862816 7f27d600e700 7 mon.1@0(leader).pg v5938142 update_from_paxos applying incremental 5938143 2013-04-26 02:39:06.862882 7f27d600e700 10 mon.1@0(leader).pg v5938143 v5938143: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 193KB/s wr, 30op/s 2013-04-26 02:39:06.862952 7f27d600e700 10 mon.1@0(leader).pg v5938143 send_pg_creates to 0 pgs 2013-04-26 02:39:06.862957 7f27d600e700 10 mon.1@0(leader).pg v5938143 update_logger 2013-04-26 02:39:06.862997 7f27d600e700 10 mon.1@0(leader).pg v5938143 create_pending v 5938144 2013-04-26 02:39:06.863029 7f27d600e700 7 mon.1@0(leader).pg v5938143 _updated_stats for osd.8 10.227.227.103:6813/8628 2013-04-26 02:39:06.863069 7f27d600e700 10 mon.1@0(leader).pg v5938143 check_osd_map already seen 10556 >= 10556 2013-04-26 02:39:06.863077 7f27d600e700 10 mon.1@0(leader).pg v5938143 update_logger 2013-04-26 02:39:06.863098 7f27d600e700 0 log [INF] : pgmap v5938143: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 193KB/s wr, 30op/s 2013-04-26 02:39:06.863523 7f27d600e700 10 mon.1@0(leader).log v5604026 update_from_paxos 2013-04-26 02:39:06.863605 7f27d600e700 10 mon.1@0(leader).log v5604026 update_from_paxos version 5604026 summary v 5604026 2013-04-26 02:39:06.863678 7f27d600e700 10 mon.1@0(leader).auth v4369 update_from_paxos 2013-04-26 02:39:06.863832 7f27d600e700 10 mon.1@0(leader).log v5604026 update_from_paxos 2013-04-26 02:39:06.863899 7f27d600e700 10 mon.1@0(leader).log v5604026 update_from_paxos version 5604026 summary v 5604026 2013-04-26 02:39:06.863933 7f27d600e700 10 mon.1@0(leader).log v5604026 preprocess_query log(1 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:06.863973 7f27d600e700 10 mon.1@0(leader).log v5604026 preprocess_log log(1 entries) v1 from mon.0 2013-04-26 02:39:06.864021 7f27d600e700 10 mon.1@0(leader).log v5604026 prepare_update log(1 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:06.864065 7f27d600e700 10 mon.1@0(leader).log v5604026 prepare_log log(1 entries) v1 from mon.0 2013-04-26 02:39:06.864104 7f27d600e700 10 mon.1@0(leader).log v5604026 logging 2013-04-26 02:39:06.863101 mon.0 10.227.227.101:6789/0 10 : [INF] pgmap v5938143: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 193KB/s wr, 30op/s 2013-04-26 02:39:06.864326 7f27d600e700 10 mon.1@0(leader).log v5604026 update_from_paxos 2013-04-26 02:39:06.864406 7f27d600e700 10 mon.1@0(leader).log v5604026 update_from_paxos version 5604026 summary v 5604026 2013-04-26 02:39:06.864471 7f27d600e700 10 mon.1@0(leader).auth v4369 update_from_paxos 2013-04-26 02:39:06.904583 7f27d600e700 10 mon.1@0(leader).log v5604026 update_from_paxos 2013-04-26 02:39:06.904659 7f27d600e700 10 mon.1@0(leader).log v5604026 update_from_paxos version 5604026 summary v 5604026 2013-04-26 02:39:06.904725 7f27d600e700 10 mon.1@0(leader).auth v4369 update_from_paxos 2013-04-26 02:39:07.753849 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.5 10.227.227.104:6807/14479 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:07.753872 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x2eb8a00 from 10.227.227.103:6789/0 2013-04-26 02:39:07.754032 7f27d600e700 10 mon.1@0(leader).pg v5938143 preprocess_query pg_stats(27 pgs tid 742 v 10556) v1 from osd.5 10.227.227.104:6807/14479 2013-04-26 02:39:07.754138 7f27d600e700 10 mon.1@0(leader).pg v5938143 prepare_update pg_stats(27 pgs tid 742 v 10556) v1 from osd.5 10.227.227.104:6807/14479 2013-04-26 02:39:07.754153 7f27d600e700 10 mon.1@0(leader).pg v5938143 prepare_pg_stats pg_stats(27 pgs tid 742 v 10556) v1 from osd.5 2013-04-26 02:39:07.754168 7f27d600e700 10 mon.1@0(leader).pg v5938143 got osd.5 osd_stat(355 GB used, 1505 GB avail, 1860 GB total, peers [0,1,2,6,7,8,9,10,11]/[]) (was osd_stat(355 GB used, 1505 GB avail, 1860 GB total, peers [0,1,2,6,7,8,9,10,11]/[])) 2013-04-26 02:39:07.810920 7f27d680f700 10 mon.1@0(leader).log v5604026 encode_full log v 5604026 2013-04-26 02:39:07.811099 7f27d680f700 10 mon.1@0(leader).log v5604026 encode_pending v5604027 2013-04-26 02:39:07.867464 7f27d680f700 10 mon.1@0(leader).pg v5938143 encode_pending v 5938144 2013-04-26 02:39:07.932890 7f27d600e700 10 mon.1@0(leader).log v5604027 update_from_paxos 2013-04-26 02:39:07.933050 7f27d600e700 10 mon.1@0(leader).log v5604027 update_from_paxos version 5604027 summary v 5604026 2013-04-26 02:39:07.933126 7f27d600e700 10 mon.1@0(leader).log v5604027 update_from_paxos latest full 5604026 2013-04-26 02:39:07.933208 7f27d600e700 7 mon.1@0(leader).log v5604027 update_from_paxos applying incremental log 5604027 2013-04-26 02:39:06.863101 mon.0 10.227.227.101:6789/0 10 : [INF] pgmap v5938143: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 193KB/s wr, 30op/s 2013-04-26 02:39:07.933337 7f27d600e700 10 mon.1@0(leader).log v5604027 check_subs 2013-04-26 02:39:07.933430 7f27d600e700 10 mon.1@0(leader).log v5604027 create_pending v 5604028 2013-04-26 02:39:07.933538 7f27d600e700 7 mon.1@0(leader).log v5604027 _updated_log for mon.0 10.227.227.101:6789/0 2013-04-26 02:39:08.038467 7f27d600e700 7 mon.1@0(leader).pg v5938143 update_from_paxos applying incremental 5938144 2013-04-26 02:39:08.038671 7f27d600e700 10 mon.1@0(leader).pg v5938144 v5938144: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 108KB/s wr, 13op/s 2013-04-26 02:39:08.038741 7f27d600e700 10 mon.1@0(leader).pg v5938144 send_pg_creates to 0 pgs 2013-04-26 02:39:08.038747 7f27d600e700 10 mon.1@0(leader).pg v5938144 update_logger 2013-04-26 02:39:08.038796 7f27d600e700 10 mon.1@0(leader).pg v5938144 create_pending v 5938145 2013-04-26 02:39:08.038829 7f27d600e700 7 mon.1@0(leader).pg v5938144 _updated_stats for osd.5 10.227.227.104:6807/14479 2013-04-26 02:39:08.038877 7f27d600e700 10 mon.1@0(leader).pg v5938144 check_osd_map already seen 10556 >= 10556 2013-04-26 02:39:08.038881 7f27d600e700 10 mon.1@0(leader).pg v5938144 update_logger 2013-04-26 02:39:08.038903 7f27d600e700 0 log [INF] : pgmap v5938144: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 108KB/s wr, 13op/s 2013-04-26 02:39:08.039163 7f27d600e700 10 mon.1@0(leader).log v5604027 update_from_paxos 2013-04-26 02:39:08.039242 7f27d600e700 10 mon.1@0(leader).log v5604027 update_from_paxos version 5604027 summary v 5604027 2013-04-26 02:39:08.039299 7f27d600e700 10 mon.1@0(leader).auth v4369 update_from_paxos 2013-04-26 02:39:08.039423 7f27d600e700 10 mon.1@0(leader).log v5604027 update_from_paxos 2013-04-26 02:39:08.039478 7f27d600e700 10 mon.1@0(leader).log v5604027 update_from_paxos version 5604027 summary v 5604027 2013-04-26 02:39:08.039507 7f27d600e700 10 mon.1@0(leader).log v5604027 preprocess_query log(1 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:08.039544 7f27d600e700 10 mon.1@0(leader).log v5604027 preprocess_log log(1 entries) v1 from mon.0 2013-04-26 02:39:08.039593 7f27d600e700 10 mon.1@0(leader).log v5604027 prepare_update log(1 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:08.039639 7f27d600e700 10 mon.1@0(leader).log v5604027 prepare_log log(1 entries) v1 from mon.0 2013-04-26 02:39:08.039678 7f27d600e700 10 mon.1@0(leader).log v5604027 logging 2013-04-26 02:39:08.038906 mon.0 10.227.227.101:6789/0 11 : [INF] pgmap v5938144: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 108KB/s wr, 13op/s 2013-04-26 02:39:08.039925 7f27d600e700 10 mon.1@0(leader).log v5604027 update_from_paxos 2013-04-26 02:39:08.039997 7f27d600e700 10 mon.1@0(leader).log v5604027 update_from_paxos version 5604027 summary v 5604027 2013-04-26 02:39:08.040063 7f27d600e700 10 mon.1@0(leader).auth v4369 update_from_paxos 2013-04-26 02:39:08.040153 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from mds.? 10.227.227.101:6800/29852 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:08.040169 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x1789600 from 10.227.227.103:6789/0 2013-04-26 02:39:08.040245 7f27d600e700 10 mon.1@0(leader).mds e3724 preprocess_query mdsbeacon(217349/1 up:standby seq 457 v3724) v2 from mds.? 10.227.227.101:6800/29852 2013-04-26 02:39:08.088122 7f27d600e700 10 mon.1@0(leader).log v5604027 update_from_paxos 2013-04-26 02:39:08.088171 7f27d600e700 10 mon.1@0(leader).log v5604027 update_from_paxos version 5604027 summary v 5604027 2013-04-26 02:39:08.088218 7f27d600e700 10 mon.1@0(leader).auth v4369 update_from_paxos 2013-04-26 02:39:08.502302 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from mds.0 10.227.227.104:6801/10842 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:08.502325 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x1789080 from 10.227.227.103:6789/0 2013-04-26 02:39:08.502484 7f27d600e700 10 mon.1@0(leader).mds e3724 preprocess_query mdsbeacon(217231/4 up:active seq 488 v3724) v2 from mds.0 10.227.227.104:6801/10842 2013-04-26 02:39:08.890690 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.11 10.227.227.103:6804/8253 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:08.890713 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x2ebac80 from 10.227.227.103:6789/0 2013-04-26 02:39:08.890872 7f27d600e700 10 mon.1@0(leader).pg v5938144 preprocess_query pg_stats(19 pgs tid 56 v 10556) v1 from osd.11 10.227.227.103:6804/8253 2013-04-26 02:39:08.891011 7f27d600e700 10 mon.1@0(leader).pg v5938144 prepare_update pg_stats(19 pgs tid 56 v 10556) v1 from osd.11 10.227.227.103:6804/8253 2013-04-26 02:39:08.891030 7f27d600e700 10 mon.1@0(leader).pg v5938144 prepare_pg_stats pg_stats(19 pgs tid 56 v 10556) v1 from osd.11 2013-04-26 02:39:08.891047 7f27d600e700 10 mon.1@0(leader).pg v5938144 got osd.11 osd_stat(161 GB used, 768 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[]) (was osd_stat(161 GB used, 768 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[])) 2013-04-26 02:39:08.988468 7f27d680f700 10 mon.1@0(leader).log v5604027 encode_full log v 5604027 2013-04-26 02:39:08.988744 7f27d680f700 10 mon.1@0(leader).log v5604027 encode_pending v5604028 2013-04-26 02:39:09.076563 7f27d680f700 10 mon.1@0(leader).pg v5938144 encode_full pgmap v 5938144 2013-04-26 02:39:09.087740 7f27d680f700 10 mon.1@0(leader).pg v5938144 encode_pending v 5938145 2013-04-26 02:39:09.134818 7f27d600e700 10 mon.1@0(leader).log v5604028 update_from_paxos 2013-04-26 02:39:09.134963 7f27d600e700 10 mon.1@0(leader).log v5604028 update_from_paxos version 5604028 summary v 5604027 2013-04-26 02:39:09.135089 7f27d600e700 10 mon.1@0(leader).log v5604028 update_from_paxos latest full 5604027 2013-04-26 02:39:09.135146 7f27d600e700 7 mon.1@0(leader).log v5604028 update_from_paxos applying incremental log 5604028 2013-04-26 02:39:08.038906 mon.0 10.227.227.101:6789/0 11 : [INF] pgmap v5938144: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 108KB/s wr, 13op/s 2013-04-26 02:39:09.135257 7f27d600e700 10 mon.1@0(leader).log v5604028 check_subs 2013-04-26 02:39:09.135340 7f27d600e700 10 mon.1@0(leader).log v5604028 create_pending v 5604029 2013-04-26 02:39:09.135400 7f27d600e700 7 mon.1@0(leader).log v5604028 _updated_log for mon.0 10.227.227.101:6789/0 2013-04-26 02:39:09.215453 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from mds.? 10.227.227.103:6800/8011 via mon.2 10.227.227.104:6789/0 2013-04-26 02:39:09.215474 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x1788dc0 from 10.227.227.104:6789/0 2013-04-26 02:39:09.215636 7f27d600e700 10 mon.1@0(leader).mds e3724 preprocess_query mdsbeacon(217445/3 up:standby seq 70 v3724) v2 from mds.? 10.227.227.103:6800/8011 2013-04-26 02:39:09.322742 7f27d600e700 7 mon.1@0(leader).pg v5938144 update_from_paxos applying incremental 5938145 2013-04-26 02:39:09.322893 7f27d600e700 10 mon.1@0(leader).pg v5938145 v5938145: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 368KB/s wr, 15op/s 2013-04-26 02:39:09.322957 7f27d600e700 10 mon.1@0(leader).pg v5938145 send_pg_creates to 0 pgs 2013-04-26 02:39:09.322966 7f27d600e700 10 mon.1@0(leader).pg v5938145 update_logger 2013-04-26 02:39:09.323012 7f27d600e700 10 mon.1@0(leader).pg v5938145 create_pending v 5938146 2013-04-26 02:39:09.323048 7f27d600e700 7 mon.1@0(leader).pg v5938145 _updated_stats for osd.11 10.227.227.103:6804/8253 2013-04-26 02:39:09.323095 7f27d600e700 10 mon.1@0(leader).pg v5938145 check_osd_map already seen 10556 >= 10556 2013-04-26 02:39:09.323100 7f27d600e700 10 mon.1@0(leader).pg v5938145 update_logger 2013-04-26 02:39:09.323187 7f27d600e700 0 log [INF] : pgmap v5938145: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 368KB/s wr, 15op/s 2013-04-26 02:39:09.323456 7f27d600e700 10 mon.1@0(leader).log v5604028 update_from_paxos 2013-04-26 02:39:09.323539 7f27d600e700 10 mon.1@0(leader).log v5604028 update_from_paxos version 5604028 summary v 5604028 2013-04-26 02:39:09.323611 7f27d600e700 10 mon.1@0(leader).auth v4369 update_from_paxos 2013-04-26 02:39:09.323770 7f27d600e700 10 mon.1@0(leader).log v5604028 update_from_paxos 2013-04-26 02:39:09.323838 7f27d600e700 10 mon.1@0(leader).log v5604028 update_from_paxos version 5604028 summary v 5604028 2013-04-26 02:39:09.323874 7f27d600e700 10 mon.1@0(leader).log v5604028 preprocess_query log(1 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:09.323918 7f27d600e700 10 mon.1@0(leader).log v5604028 preprocess_log log(1 entries) v1 from mon.0 2013-04-26 02:39:09.323970 7f27d600e700 10 mon.1@0(leader).log v5604028 prepare_update log(1 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:09.324014 7f27d600e700 10 mon.1@0(leader).log v5604028 prepare_log log(1 entries) v1 from mon.0 2013-04-26 02:39:09.324056 7f27d600e700 10 mon.1@0(leader).log v5604028 logging 2013-04-26 02:39:09.323197 mon.0 10.227.227.101:6789/0 12 : [INF] pgmap v5938145: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 368KB/s wr, 15op/s 2013-04-26 02:39:09.339587 7f27d600e700 10 mon.1@0(leader).log v5604028 update_from_paxos 2013-04-26 02:39:09.339663 7f27d600e700 10 mon.1@0(leader).log v5604028 update_from_paxos version 5604028 summary v 5604028 2013-04-26 02:39:09.339729 7f27d600e700 10 mon.1@0(leader).auth v4369 update_from_paxos 2013-04-26 02:39:09.340160 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.7 10.227.227.103:6810/8502 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:09.340175 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x2eba780 from 10.227.227.103:6789/0 2013-04-26 02:39:09.340331 7f27d600e700 10 mon.1@0(leader).pg v5938145 preprocess_query pg_stats(15 pgs tid 53 v 10556) v1 from osd.7 10.227.227.103:6810/8502 2013-04-26 02:39:09.340423 7f27d600e700 10 mon.1@0(leader).pg v5938145 prepare_update pg_stats(15 pgs tid 53 v 10556) v1 from osd.7 10.227.227.103:6810/8502 2013-04-26 02:39:09.340491 7f27d600e700 10 mon.1@0(leader).pg v5938145 prepare_pg_stats pg_stats(15 pgs tid 53 v 10556) v1 from osd.7 2013-04-26 02:39:09.340511 7f27d600e700 10 mon.1@0(leader).pg v5938145 got osd.7 osd_stat(156 GB used, 772 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[]) (was osd_stat(156 GB used, 772 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[])) 2013-04-26 02:39:09.340633 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.9 10.227.227.103:6816/8752 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:09.340646 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x2eb8000 from 10.227.227.103:6789/0 2013-04-26 02:39:09.340736 7f27d600e700 10 mon.1@0(leader).pg v5938145 preprocess_query pg_stats(17 pgs tid 56 v 10556) v1 from osd.9 10.227.227.103:6816/8752 2013-04-26 02:39:09.340819 7f27d600e700 10 mon.1@0(leader).pg v5938145 prepare_update pg_stats(17 pgs tid 56 v 10556) v1 from osd.9 10.227.227.103:6816/8752 2013-04-26 02:39:09.340833 7f27d600e700 10 mon.1@0(leader).pg v5938145 prepare_pg_stats pg_stats(17 pgs tid 56 v 10556) v1 from osd.9 2013-04-26 02:39:09.340843 7f27d600e700 10 mon.1@0(leader).pg v5938145 got osd.9 osd_stat(161 GB used, 767 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[]) (was osd_stat(161 GB used, 767 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[])) 2013-04-26 02:39:09.395829 7f27d680f700 10 mon.1@0(leader).pg v5938145 check_down_pgs 2013-04-26 02:39:09.396082 7f27d680f700 10 mon.1@0(leader).pg v5938145 v5938145: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 368KB/s wr, 15op/s 2013-04-26 02:39:09.396221 7f27d680f700 10 mon.1@0(leader).mds e3724 e3724: 1/1/1 up {0=4=up:active}, 2 up:standby 2013-04-26 02:39:09.396305 7f27d680f700 10 mon.1@0(leader).osd e10556 e10556: 12 osds: 12 up, 12 in 2013-04-26 02:39:09.396577 7f27d680f700 10 mon.1@0(leader).osd e10556 min_last_epoch_clean 10531 2013-04-26 02:39:09.396717 7f27d680f700 10 mon.1@0(leader).log v5604028 update_from_paxos 2013-04-26 02:39:09.396788 7f27d680f700 10 mon.1@0(leader).log v5604028 update_from_paxos version 5604028 summary v 5604028 2013-04-26 02:39:09.396822 7f27d680f700 10 mon.1@0(leader).log v5604028 log 2013-04-26 02:39:09.396889 7f27d680f700 10 mon.1@0(leader).auth v4369 update_from_paxos 2013-04-26 02:39:09.396951 7f27d680f700 10 mon.1@0(leader).auth v4369 auth 2013-04-26 02:39:09.397020 7f27d680f700 10 mon.1@0(leader).auth v4369 AuthMonitor::tick() updated rotating, now calling propose_pending 2013-04-26 02:39:09.397219 7f27d680f700 10 mon.1@0(leader).auth v4369 encode_pending v 4370 2013-04-26 02:39:09.444019 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:09.444138 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos version 4370 keys ver 4369 latest 4367 2013-04-26 02:39:09.444173 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos key server version 4369 2013-04-26 02:39:09.444264 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos() last_allocated_id=218796 max_global_id=218796 2013-04-26 02:39:09.444365 7f27d600e700 10 mon.1@0(leader).auth v4370 create_pending v 4371 2013-04-26 02:39:09.444484 7f27d600e700 10 mon.1@0(leader).auth v4370 AuthMonitor::on_active() 2013-04-26 02:39:09.444678 7f27d600e700 10 mon.1@0(leader).log v5604028 update_from_paxos 2013-04-26 02:39:09.444749 7f27d600e700 10 mon.1@0(leader).log v5604028 update_from_paxos version 5604028 summary v 5604028 2013-04-26 02:39:09.444818 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:09.445057 7f27d600e700 10 mon.1@0(leader).log v5604028 update_from_paxos 2013-04-26 02:39:09.445132 7f27d600e700 10 mon.1@0(leader).log v5604028 update_from_paxos version 5604028 summary v 5604028 2013-04-26 02:39:09.445198 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:09.452638 7f27d600e700 10 mon.1@0(leader).log v5604028 update_from_paxos 2013-04-26 02:39:09.452718 7f27d600e700 10 mon.1@0(leader).log v5604028 update_from_paxos version 5604028 summary v 5604028 2013-04-26 02:39:09.452783 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:09.531549 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.10 10.227.227.103:6801/8155 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:09.531567 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x2eb8a00 from 10.227.227.103:6789/0 2013-04-26 02:39:09.531679 7f27d600e700 10 mon.1@0(leader).pg v5938145 preprocess_query pg_stats(16 pgs tid 51 v 10556) v1 from osd.10 10.227.227.103:6801/8155 2013-04-26 02:39:09.531764 7f27d600e700 10 mon.1@0(leader).pg v5938145 prepare_update pg_stats(16 pgs tid 51 v 10556) v1 from osd.10 10.227.227.103:6801/8155 2013-04-26 02:39:09.531778 7f27d600e700 10 mon.1@0(leader).pg v5938145 prepare_pg_stats pg_stats(16 pgs tid 51 v 10556) v1 from osd.10 2013-04-26 02:39:09.531791 7f27d600e700 10 mon.1@0(leader).pg v5938145 got osd.10 osd_stat(174 GB used, 755 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[]) (was osd_stat(174 GB used, 755 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[])) 2013-04-26 02:39:09.544332 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.2 10.227.227.101:6807/30207 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:09.544349 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x2eb8500 from 10.227.227.103:6789/0 2013-04-26 02:39:09.544454 7f27d600e700 10 mon.1@0(leader).pg v5938145 preprocess_query pg_stats(19 pgs tid 735 v 10556) v1 from osd.2 10.227.227.101:6807/30207 2013-04-26 02:39:09.544533 7f27d600e700 10 mon.1@0(leader).pg v5938145 prepare_update pg_stats(19 pgs tid 735 v 10556) v1 from osd.2 10.227.227.101:6807/30207 2013-04-26 02:39:09.544547 7f27d600e700 10 mon.1@0(leader).pg v5938145 prepare_pg_stats pg_stats(19 pgs tid 735 v 10556) v1 from osd.2 2013-04-26 02:39:09.544557 7f27d600e700 10 mon.1@0(leader).pg v5938145 got osd.2 osd_stat(311 GB used, 1549 GB avail, 1860 GB total, peers [3,4,5,6,7,8,9,10,11]/[]) (was osd_stat(311 GB used, 1549 GB avail, 1860 GB total, peers [3,4,5,6,7,8,9,10,11]/[])) 2013-04-26 02:39:09.641801 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.1 10.227.227.101:6804/30088 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:09.641824 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x2eb8f00 from 10.227.227.103:6789/0 2013-04-26 02:39:09.641939 7f27d600e700 10 mon.1@0(leader).pg v5938145 preprocess_query pg_stats(26 pgs tid 729 v 10556) v1 from osd.1 10.227.227.101:6804/30088 2013-04-26 02:39:09.642026 7f27d600e700 10 mon.1@0(leader).pg v5938145 prepare_update pg_stats(26 pgs tid 729 v 10556) v1 from osd.1 10.227.227.101:6804/30088 2013-04-26 02:39:09.642040 7f27d600e700 10 mon.1@0(leader).pg v5938145 prepare_pg_stats pg_stats(26 pgs tid 729 v 10556) v1 from osd.1 2013-04-26 02:39:09.642055 7f27d600e700 10 mon.1@0(leader).pg v5938145 got osd.1 osd_stat(342 GB used, 1517 GB avail, 1860 GB total, peers [3,4,5,6,7,8,9,10,11]/[]) (was osd_stat(342 GB used, 1517 GB avail, 1860 GB total, peers [3,4,5,6,7,8,9,10,11]/[])) 2013-04-26 02:39:09.685182 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.0 10.227.227.101:6801/29982 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:09.685200 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x2eb9400 from 10.227.227.103:6789/0 2013-04-26 02:39:09.685323 7f27d600e700 10 mon.1@0(leader).pg v5938145 preprocess_query pg_stats(41 pgs tid 740 v 10556) v1 from osd.0 10.227.227.101:6801/29982 2013-04-26 02:39:09.685408 7f27d600e700 10 mon.1@0(leader).pg v5938145 prepare_update pg_stats(41 pgs tid 740 v 10556) v1 from osd.0 10.227.227.101:6801/29982 2013-04-26 02:39:09.685422 7f27d600e700 10 mon.1@0(leader).pg v5938145 prepare_pg_stats pg_stats(41 pgs tid 740 v 10556) v1 from osd.0 2013-04-26 02:39:09.685433 7f27d600e700 10 mon.1@0(leader).pg v5938145 got osd.0 osd_stat(367 GB used, 1493 GB avail, 1860 GB total, peers [3,4,5,6,7,8,9,10,11]/[]) (was osd_stat(367 GB used, 1493 GB avail, 1860 GB total, peers [3,4,5,6,7,8,9,10,11]/[])) 2013-04-26 02:39:09.890189 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.6 10.227.227.103:6807/8402 via mon.2 10.227.227.104:6789/0 2013-04-26 02:39:09.890211 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x2ebb680 from 10.227.227.104:6789/0 2013-04-26 02:39:09.890350 7f27d600e700 10 mon.1@0(leader).pg v5938145 preprocess_query pg_stats(15 pgs tid 56 v 10556) v1 from osd.6 10.227.227.103:6807/8402 2013-04-26 02:39:09.890453 7f27d600e700 10 mon.1@0(leader).pg v5938145 prepare_update pg_stats(15 pgs tid 56 v 10556) v1 from osd.6 10.227.227.103:6807/8402 2013-04-26 02:39:09.890468 7f27d600e700 10 mon.1@0(leader).pg v5938145 prepare_pg_stats pg_stats(15 pgs tid 56 v 10556) v1 from osd.6 2013-04-26 02:39:09.890482 7f27d600e700 10 mon.1@0(leader).pg v5938145 got osd.6 osd_stat(184 GB used, 745 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[]) (was osd_stat(184 GB used, 745 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[])) 2013-04-26 02:39:10.238868 7f27d680f700 10 mon.1@0(leader).pg v5938145 encode_pending v 5938146 2013-04-26 02:39:10.294156 7f27d680f700 10 mon.1@0(leader).log v5604028 encode_full log v 5604028 2013-04-26 02:39:10.294425 7f27d680f700 10 mon.1@0(leader).log v5604028 encode_pending v5604029 2013-04-26 02:39:10.344639 7f27d600e700 7 mon.1@0(leader).pg v5938145 update_from_paxos applying incremental 5938146 2013-04-26 02:39:10.345388 7f27d600e700 10 mon.1@0(leader).pg v5938146 v5938146: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 1022KB/s wr, 49op/s 2013-04-26 02:39:10.345467 7f27d600e700 10 mon.1@0(leader).pg v5938146 send_pg_creates to 0 pgs 2013-04-26 02:39:10.345472 7f27d600e700 10 mon.1@0(leader).pg v5938146 update_logger 2013-04-26 02:39:10.345554 7f27d600e700 10 mon.1@0(leader).pg v5938146 create_pending v 5938147 2013-04-26 02:39:10.345592 7f27d600e700 7 mon.1@0(leader).pg v5938146 _updated_stats for osd.7 10.227.227.103:6810/8502 2013-04-26 02:39:10.345633 7f27d600e700 7 mon.1@0(leader).pg v5938146 _updated_stats for osd.9 10.227.227.103:6816/8752 2013-04-26 02:39:10.345689 7f27d600e700 7 mon.1@0(leader).pg v5938146 _updated_stats for osd.10 10.227.227.103:6801/8155 2013-04-26 02:39:10.345737 7f27d600e700 7 mon.1@0(leader).pg v5938146 _updated_stats for osd.2 10.227.227.101:6807/30207 2013-04-26 02:39:10.345765 7f27d600e700 7 mon.1@0(leader).pg v5938146 _updated_stats for osd.1 10.227.227.101:6804/30088 2013-04-26 02:39:10.345839 7f27d600e700 7 mon.1@0(leader).pg v5938146 _updated_stats for osd.0 10.227.227.101:6801/29982 2013-04-26 02:39:10.345896 7f27d600e700 7 mon.1@0(leader).pg v5938146 _updated_stats for osd.6 10.227.227.103:6807/8402 2013-04-26 02:39:10.345933 7f27d600e700 10 mon.1@0(leader).pg v5938146 check_osd_map already seen 10556 >= 10556 2013-04-26 02:39:10.345937 7f27d600e700 10 mon.1@0(leader).pg v5938146 update_logger 2013-04-26 02:39:10.345956 7f27d600e700 0 log [INF] : pgmap v5938146: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 1022KB/s wr, 49op/s 2013-04-26 02:39:10.394180 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.4 10.227.227.104:6804/14471 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:10.394193 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x2ebb900 from 10.227.227.103:6789/0 2013-04-26 02:39:10.394261 7f27d600e700 10 mon.1@0(leader).pg v5938146 preprocess_query pg_stats(30 pgs tid 745 v 10556) v1 from osd.4 10.227.227.104:6804/14471 2013-04-26 02:39:10.394333 7f27d600e700 10 mon.1@0(leader).pg v5938146 prepare_update pg_stats(30 pgs tid 745 v 10556) v1 from osd.4 10.227.227.104:6804/14471 2013-04-26 02:39:10.394343 7f27d600e700 10 mon.1@0(leader).pg v5938146 prepare_pg_stats pg_stats(30 pgs tid 745 v 10556) v1 from osd.4 2013-04-26 02:39:10.394354 7f27d600e700 10 mon.1@0(leader).pg v5938146 got osd.4 osd_stat(423 GB used, 1437 GB avail, 1860 GB total, peers [0,1,2,6,7,8,9,10,11]/[]) (was osd_stat(423 GB used, 1437 GB avail, 1860 GB total, peers [0,1,2,6,7,8,9,10,11]/[])) 2013-04-26 02:39:10.455593 7f27d600e700 10 mon.1@0(leader).log v5604029 update_from_paxos 2013-04-26 02:39:10.455726 7f27d600e700 10 mon.1@0(leader).log v5604029 update_from_paxos version 5604029 summary v 5604028 2013-04-26 02:39:10.455802 7f27d600e700 10 mon.1@0(leader).log v5604029 update_from_paxos latest full 5604028 2013-04-26 02:39:10.455883 7f27d600e700 7 mon.1@0(leader).log v5604029 update_from_paxos applying incremental log 5604029 2013-04-26 02:39:09.323197 mon.0 10.227.227.101:6789/0 12 : [INF] pgmap v5938145: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 368KB/s wr, 15op/s 2013-04-26 02:39:10.456005 7f27d600e700 10 mon.1@0(leader).log v5604029 check_subs 2013-04-26 02:39:10.456116 7f27d600e700 10 mon.1@0(leader).log v5604029 create_pending v 5604030 2013-04-26 02:39:10.456215 7f27d600e700 7 mon.1@0(leader).log v5604029 _updated_log for mon.0 10.227.227.101:6789/0 2013-04-26 02:39:10.456344 7f27d600e700 10 mon.1@0(leader).log v5604029 update_from_paxos 2013-04-26 02:39:10.456409 7f27d600e700 10 mon.1@0(leader).log v5604029 update_from_paxos version 5604029 summary v 5604029 2013-04-26 02:39:10.456498 7f27d600e700 10 mon.1@0(leader).log v5604029 preprocess_query log(1 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:10.456599 7f27d600e700 10 mon.1@0(leader).log v5604029 preprocess_log log(1 entries) v1 from mon.0 2013-04-26 02:39:10.456664 7f27d600e700 10 mon.1@0(leader).log v5604029 nothing new 2013-04-26 02:39:10.456931 7f27d600e700 10 mon.1@0(leader).log v5604029 update_from_paxos 2013-04-26 02:39:10.457003 7f27d600e700 10 mon.1@0(leader).log v5604029 update_from_paxos version 5604029 summary v 5604029 2013-04-26 02:39:10.457076 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:10.457186 7f27d600e700 0 log [WRN] : message from mon.1 was stamped 0.083603s in the future, clocks not synchronized 2013-04-26 02:39:10.457359 7f27d600e700 10 mon.1@0(leader).log v5604029 update_from_paxos 2013-04-26 02:39:10.457429 7f27d600e700 10 mon.1@0(leader).log v5604029 update_from_paxos version 5604029 summary v 5604029 2013-04-26 02:39:10.457499 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:10.457687 7f27d600e700 10 mon.1@0(leader).log v5604029 update_from_paxos 2013-04-26 02:39:10.457755 7f27d600e700 10 mon.1@0(leader).log v5604029 update_from_paxos version 5604029 summary v 5604029 2013-04-26 02:39:10.457791 7f27d600e700 10 mon.1@0(leader).log v5604029 preprocess_query log(2 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:10.457835 7f27d600e700 10 mon.1@0(leader).log v5604029 preprocess_log log(2 entries) v1 from mon.0 2013-04-26 02:39:10.457884 7f27d600e700 10 mon.1@0(leader).log v5604029 prepare_update log(2 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:10.457929 7f27d600e700 10 mon.1@0(leader).log v5604029 prepare_log log(2 entries) v1 from mon.0 2013-04-26 02:39:10.457970 7f27d600e700 10 mon.1@0(leader).log v5604029 logging 2013-04-26 02:39:09.323197 mon.0 10.227.227.101:6789/0 12 : [INF] pgmap v5938145: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 368KB/s wr, 15op/s 2013-04-26 02:39:10.458024 7f27d600e700 10 mon.1@0(leader).log v5604029 logging 2013-04-26 02:39:10.345958 mon.0 10.227.227.101:6789/0 13 : [INF] pgmap v5938146: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 1022KB/s wr, 49op/s 2013-04-26 02:39:10.467606 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.3 10.227.227.104:6800/14463 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:10.467624 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x1798c80 from 10.227.227.103:6789/0 2013-04-26 02:39:10.467735 7f27d600e700 10 mon.1@0(leader).pg v5938146 preprocess_query pg_stats(31 pgs tid 750 v 10556) v1 from osd.3 10.227.227.104:6800/14463 2013-04-26 02:39:10.467823 7f27d600e700 10 mon.1@0(leader).pg v5938146 prepare_update pg_stats(31 pgs tid 750 v 10556) v1 from osd.3 10.227.227.104:6800/14463 2013-04-26 02:39:10.467837 7f27d600e700 10 mon.1@0(leader).pg v5938146 prepare_pg_stats pg_stats(31 pgs tid 750 v 10556) v1 from osd.3 2013-04-26 02:39:10.467851 7f27d600e700 10 mon.1@0(leader).pg v5938146 got osd.3 osd_stat(383 GB used, 1477 GB avail, 1860 GB total, peers [0,1,2,6,7,8,9,10,11]/[]) (was osd_stat(383 GB used, 1477 GB avail, 1860 GB total, peers [0,1,2,6,7,8,9,10,11]/[])) 2013-04-26 02:39:10.497394 7f27d600e700 10 mon.1@0(leader).log v5604029 update_from_paxos 2013-04-26 02:39:10.497541 7f27d600e700 10 mon.1@0(leader).log v5604029 update_from_paxos version 5604029 summary v 5604029 2013-04-26 02:39:10.497618 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:10.914405 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.8 10.227.227.103:6813/8628 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:10.914431 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x2eb9180 from 10.227.227.103:6789/0 2013-04-26 02:39:10.914577 7f27d600e700 10 mon.1@0(leader).pg v5938146 preprocess_query pg_stats(11 pgs tid 54 v 10556) v1 from osd.8 10.227.227.103:6813/8628 2013-04-26 02:39:10.914692 7f27d600e700 10 mon.1@0(leader).pg v5938146 prepare_update pg_stats(11 pgs tid 54 v 10556) v1 from osd.8 10.227.227.103:6813/8628 2013-04-26 02:39:10.914708 7f27d600e700 10 mon.1@0(leader).pg v5938146 prepare_pg_stats pg_stats(11 pgs tid 54 v 10556) v1 from osd.8 2013-04-26 02:39:10.914722 7f27d600e700 10 mon.1@0(leader).pg v5938146 got osd.8 osd_stat(185 GB used, 744 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[]) (was osd_stat(185 GB used, 744 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[])) 2013-04-26 02:39:11.298856 7f27d680f700 10 mon.1@0(leader).pg v5938146 encode_pending v 5938147 2013-04-26 02:39:11.436347 7f27d680f700 10 mon.1@0(leader).log v5604029 encode_full log v 5604029 2013-04-26 02:39:11.436547 7f27d680f700 10 mon.1@0(leader).log v5604029 encode_pending v5604030 2013-04-26 02:39:11.438877 7f27d600e700 7 mon.1@0(leader).pg v5938146 update_from_paxos applying incremental 5938147 2013-04-26 02:39:11.439178 7f27d600e700 10 mon.1@0(leader).pg v5938147 v5938147: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 1112KB/s wr, 65op/s 2013-04-26 02:39:11.439227 7f27d600e700 10 mon.1@0(leader).pg v5938147 send_pg_creates to 0 pgs 2013-04-26 02:39:11.439230 7f27d600e700 10 mon.1@0(leader).pg v5938147 update_logger 2013-04-26 02:39:11.439275 7f27d600e700 10 mon.1@0(leader).pg v5938147 create_pending v 5938148 2013-04-26 02:39:11.439295 7f27d600e700 7 mon.1@0(leader).pg v5938147 _updated_stats for osd.4 10.227.227.104:6804/14471 2013-04-26 02:39:11.439329 7f27d600e700 7 mon.1@0(leader).pg v5938147 _updated_stats for osd.3 10.227.227.104:6800/14463 2013-04-26 02:39:11.439347 7f27d600e700 7 mon.1@0(leader).pg v5938147 _updated_stats for osd.8 10.227.227.103:6813/8628 2013-04-26 02:39:11.439363 7f27d600e700 10 mon.1@0(leader).pg v5938147 check_osd_map already seen 10556 >= 10556 2013-04-26 02:39:11.439365 7f27d600e700 10 mon.1@0(leader).pg v5938147 update_logger 2013-04-26 02:39:11.439380 7f27d600e700 0 log [INF] : pgmap v5938147: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 1112KB/s wr, 65op/s 2013-04-26 02:39:11.538944 7f27d600e700 10 mon.1@0(leader).log v5604030 update_from_paxos 2013-04-26 02:39:11.539012 7f27d600e700 10 mon.1@0(leader).log v5604030 update_from_paxos version 5604030 summary v 5604029 2013-04-26 02:39:11.539058 7f27d600e700 10 mon.1@0(leader).log v5604030 update_from_paxos latest full 5604029 2013-04-26 02:39:11.539106 7f27d600e700 7 mon.1@0(leader).log v5604030 update_from_paxos applying incremental log 5604030 2013-04-26 02:39:10.345958 mon.0 10.227.227.101:6789/0 13 : [INF] pgmap v5938146: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 1022KB/s wr, 49op/s 2013-04-26 02:39:11.539191 7f27d600e700 10 mon.1@0(leader).log v5604030 check_subs 2013-04-26 02:39:11.539252 7f27d600e700 10 mon.1@0(leader).log v5604030 create_pending v 5604031 2013-04-26 02:39:11.539310 7f27d600e700 7 mon.1@0(leader).log v5604030 _updated_log for mon.0 10.227.227.101:6789/0 2013-04-26 02:39:11.539388 7f27d600e700 10 mon.1@0(leader).log v5604030 update_from_paxos 2013-04-26 02:39:11.539426 7f27d600e700 10 mon.1@0(leader).log v5604030 update_from_paxos version 5604030 summary v 5604030 2013-04-26 02:39:11.539446 7f27d600e700 10 mon.1@0(leader).log v5604030 preprocess_query log(2 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:11.539473 7f27d600e700 10 mon.1@0(leader).log v5604030 preprocess_log log(2 entries) v1 from mon.0 2013-04-26 02:39:11.539502 7f27d600e700 10 mon.1@0(leader).log v5604030 prepare_update log(2 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:11.539528 7f27d600e700 10 mon.1@0(leader).log v5604030 prepare_log log(2 entries) v1 from mon.0 2013-04-26 02:39:11.539552 7f27d600e700 10 mon.1@0(leader).log v5604030 logging 2013-04-26 02:39:10.345958 mon.0 10.227.227.101:6789/0 13 : [INF] pgmap v5938146: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 1022KB/s wr, 49op/s 2013-04-26 02:39:11.539585 7f27d600e700 10 mon.1@0(leader).log v5604030 logging 2013-04-26 02:39:10.457190 mon.0 10.227.227.101:6789/0 14 : [WRN] message from mon.1 was stamped 0.083603s in the future, clocks not synchronized 2013-04-26 02:39:11.539766 7f27d600e700 10 mon.1@0(leader).log v5604030 update_from_paxos 2013-04-26 02:39:11.539807 7f27d600e700 10 mon.1@0(leader).log v5604030 update_from_paxos version 5604030 summary v 5604030 2013-04-26 02:39:11.539849 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:11.548801 7f27d600e700 10 mon.1@0(leader).log v5604030 update_from_paxos 2013-04-26 02:39:11.548874 7f27d600e700 10 mon.1@0(leader).log v5604030 update_from_paxos version 5604030 summary v 5604030 2013-04-26 02:39:11.548951 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:11.589321 7f27d600e700 10 mon.1@0(leader).log v5604030 update_from_paxos 2013-04-26 02:39:11.589398 7f27d600e700 10 mon.1@0(leader).log v5604030 update_from_paxos version 5604030 summary v 5604030 2013-04-26 02:39:11.589469 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:11.954447 7f27d600e700 10 mon.1@0(leader).data_health(2814) service_dispatch mon_health( service 1 op tell e 0 r 0 flags none ) v1 2013-04-26 02:39:11.954459 7f27d600e700 10 mon.1@0(leader).data_health(2814) handle_tell mon_health( service 1 op tell e 0 r 0 flags none ) v1 2013-04-26 02:39:12.029978 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from mds.? 10.227.227.101:6800/29852 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:12.030001 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x1789600 from 10.227.227.103:6789/0 2013-04-26 02:39:12.030169 7f27d600e700 10 mon.1@0(leader).mds e3724 preprocess_query mdsbeacon(217349/1 up:standby seq 458 v3724) v2 from mds.? 10.227.227.101:6800/29852 2013-04-26 02:39:12.498682 7f27d680f700 10 mon.1@0(leader).log v5604030 encode_full log v 5604030 2013-04-26 02:39:12.498960 7f27d680f700 10 mon.1@0(leader).log v5604030 encode_pending v5604031 2013-04-26 02:39:12.586908 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from mds.0 10.227.227.104:6801/10842 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:12.586924 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x1788dc0 from 10.227.227.103:6789/0 2013-04-26 02:39:12.587021 7f27d600e700 10 mon.1@0(leader).mds e3724 preprocess_query mdsbeacon(217231/4 up:active seq 489 v3724) v2 from mds.0 10.227.227.104:6801/10842 2013-04-26 02:39:12.655677 7f27d600e700 10 mon.1@0(leader).log v5604031 update_from_paxos 2013-04-26 02:39:12.655793 7f27d600e700 10 mon.1@0(leader).log v5604031 update_from_paxos version 5604031 summary v 5604030 2013-04-26 02:39:12.655849 7f27d600e700 10 mon.1@0(leader).log v5604031 update_from_paxos latest full 5604030 2013-04-26 02:39:12.655908 7f27d600e700 7 mon.1@0(leader).log v5604031 update_from_paxos applying incremental log 5604031 2013-04-26 02:39:10.457190 mon.0 10.227.227.101:6789/0 14 : [WRN] message from mon.1 was stamped 0.083603s in the future, clocks not synchronized 2013-04-26 02:39:12.656006 7f27d600e700 10 mon.1@0(leader).log v5604031 check_subs 2013-04-26 02:39:12.656079 7f27d600e700 10 mon.1@0(leader).log v5604031 create_pending v 5604032 2013-04-26 02:39:12.656164 7f27d600e700 7 mon.1@0(leader).log v5604031 _updated_log for mon.0 10.227.227.101:6789/0 2013-04-26 02:39:12.734055 7f27d600e700 10 mon.1@0(leader).log v5604031 update_from_paxos 2013-04-26 02:39:12.734135 7f27d600e700 10 mon.1@0(leader).log v5604031 update_from_paxos version 5604031 summary v 5604031 2013-04-26 02:39:12.734204 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:12.736342 7f27d600e700 10 mon.1@0(leader).log v5604031 update_from_paxos 2013-04-26 02:39:12.736423 7f27d600e700 10 mon.1@0(leader).log v5604031 update_from_paxos version 5604031 summary v 5604031 2013-04-26 02:39:12.736490 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:12.754594 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.5 10.227.227.104:6807/14479 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:12.754614 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x2eba000 from 10.227.227.103:6789/0 2013-04-26 02:39:12.754727 7f27d600e700 10 mon.1@0(leader).pg v5938147 preprocess_query pg_stats(26 pgs tid 743 v 10556) v1 from osd.5 10.227.227.104:6807/14479 2013-04-26 02:39:12.754812 7f27d600e700 10 mon.1@0(leader).pg v5938147 prepare_update pg_stats(26 pgs tid 743 v 10556) v1 from osd.5 10.227.227.104:6807/14479 2013-04-26 02:39:12.754827 7f27d600e700 10 mon.1@0(leader).pg v5938147 prepare_pg_stats pg_stats(26 pgs tid 743 v 10556) v1 from osd.5 2013-04-26 02:39:12.754841 7f27d600e700 10 mon.1@0(leader).pg v5938147 got osd.5 osd_stat(355 GB used, 1505 GB avail, 1860 GB total, peers [0,1,2,6,7,8,9,10,11]/[]) (was osd_stat(355 GB used, 1505 GB avail, 1860 GB total, peers [0,1,2,6,7,8,9,10,11]/[])) 2013-04-26 02:39:12.781012 7f27d600e700 10 mon.1@0(leader).log v5604031 update_from_paxos 2013-04-26 02:39:12.781099 7f27d600e700 10 mon.1@0(leader).log v5604031 update_from_paxos version 5604031 summary v 5604031 2013-04-26 02:39:12.781168 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:13.215703 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from mds.? 10.227.227.103:6800/8011 via mon.2 10.227.227.104:6789/0 2013-04-26 02:39:13.215725 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x1789080 from 10.227.227.104:6789/0 2013-04-26 02:39:13.215887 7f27d600e700 10 mon.1@0(leader).mds e3724 preprocess_query mdsbeacon(217445/3 up:standby seq 71 v3724) v2 from mds.? 10.227.227.103:6800/8011 2013-04-26 02:39:13.718778 7f27d680f700 10 mon.1@0(leader).pg v5938147 encode_pending v 5938148 2013-04-26 02:39:13.855939 7f27d600e700 7 mon.1@0(leader).pg v5938147 update_from_paxos applying incremental 5938148 2013-04-26 02:39:13.856085 7f27d600e700 10 mon.1@0(leader).pg v5938148 v5938148: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 249KB/s wr, 16op/s 2013-04-26 02:39:13.856139 7f27d600e700 10 mon.1@0(leader).pg v5938148 send_pg_creates to 0 pgs 2013-04-26 02:39:13.856145 7f27d600e700 10 mon.1@0(leader).pg v5938148 update_logger 2013-04-26 02:39:13.856176 7f27d600e700 10 mon.1@0(leader).pg v5938148 create_pending v 5938149 2013-04-26 02:39:13.856199 7f27d600e700 7 mon.1@0(leader).pg v5938148 _updated_stats for osd.5 10.227.227.104:6807/14479 2013-04-26 02:39:13.856237 7f27d600e700 10 mon.1@0(leader).pg v5938148 check_osd_map already seen 10556 >= 10556 2013-04-26 02:39:13.856241 7f27d600e700 10 mon.1@0(leader).pg v5938148 update_logger 2013-04-26 02:39:13.856257 7f27d600e700 0 log [INF] : pgmap v5938148: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 249KB/s wr, 16op/s 2013-04-26 02:39:13.856435 7f27d600e700 10 mon.1@0(leader).log v5604031 update_from_paxos 2013-04-26 02:39:13.856489 7f27d600e700 10 mon.1@0(leader).log v5604031 update_from_paxos version 5604031 summary v 5604031 2013-04-26 02:39:13.856534 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:13.856632 7f27d600e700 10 mon.1@0(leader).log v5604031 update_from_paxos 2013-04-26 02:39:13.856669 7f27d600e700 10 mon.1@0(leader).log v5604031 update_from_paxos version 5604031 summary v 5604031 2013-04-26 02:39:13.856774 7f27d600e700 10 mon.1@0(leader).log v5604031 preprocess_query log(2 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:13.856805 7f27d600e700 10 mon.1@0(leader).log v5604031 preprocess_log log(2 entries) v1 from mon.0 2013-04-26 02:39:13.856840 7f27d600e700 10 mon.1@0(leader).log v5604031 prepare_update log(2 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:13.856867 7f27d600e700 10 mon.1@0(leader).log v5604031 prepare_log log(2 entries) v1 from mon.0 2013-04-26 02:39:13.856890 7f27d600e700 10 mon.1@0(leader).log v5604031 logging 2013-04-26 02:39:11.439382 mon.0 10.227.227.101:6789/0 15 : [INF] pgmap v5938147: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 1112KB/s wr, 65op/s 2013-04-26 02:39:13.856927 7f27d600e700 10 mon.1@0(leader).log v5604031 logging 2013-04-26 02:39:13.856259 mon.0 10.227.227.101:6789/0 16 : [INF] pgmap v5938148: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 249KB/s wr, 16op/s 2013-04-26 02:39:13.871547 7f27d600e700 10 mon.1@0(leader).log v5604031 update_from_paxos 2013-04-26 02:39:13.871602 7f27d600e700 10 mon.1@0(leader).log v5604031 update_from_paxos version 5604031 summary v 5604031 2013-04-26 02:39:13.871643 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:13.891448 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.11 10.227.227.103:6804/8253 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:13.891465 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x2eb9180 from 10.227.227.103:6789/0 2013-04-26 02:39:13.891579 7f27d600e700 10 mon.1@0(leader).pg v5938148 preprocess_query pg_stats(11 pgs tid 57 v 10556) v1 from osd.11 10.227.227.103:6804/8253 2013-04-26 02:39:13.891649 7f27d600e700 10 mon.1@0(leader).pg v5938148 prepare_update pg_stats(11 pgs tid 57 v 10556) v1 from osd.11 10.227.227.103:6804/8253 2013-04-26 02:39:13.891658 7f27d600e700 10 mon.1@0(leader).pg v5938148 prepare_pg_stats pg_stats(11 pgs tid 57 v 10556) v1 from osd.11 2013-04-26 02:39:13.891669 7f27d600e700 10 mon.1@0(leader).pg v5938148 got osd.11 osd_stat(161 GB used, 768 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[]) (was osd_stat(161 GB used, 768 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[])) 2013-04-26 02:39:13.906317 7f27d600e700 10 mon.1@0(leader).log v5604031 update_from_paxos 2013-04-26 02:39:13.906372 7f27d600e700 10 mon.1@0(leader).log v5604031 update_from_paxos version 5604031 summary v 5604031 2013-04-26 02:39:13.906413 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:14.274686 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.7 10.227.227.103:6810/8502 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:14.274706 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x2eb9400 from 10.227.227.103:6789/0 2013-04-26 02:39:14.274857 7f27d600e700 10 mon.1@0(leader).pg v5938148 preprocess_query pg_stats(13 pgs tid 54 v 10556) v1 from osd.7 10.227.227.103:6810/8502 2013-04-26 02:39:14.274961 7f27d600e700 10 mon.1@0(leader).pg v5938148 prepare_update pg_stats(13 pgs tid 54 v 10556) v1 from osd.7 10.227.227.103:6810/8502 2013-04-26 02:39:14.274975 7f27d600e700 10 mon.1@0(leader).pg v5938148 prepare_pg_stats pg_stats(13 pgs tid 54 v 10556) v1 from osd.7 2013-04-26 02:39:14.275006 7f27d600e700 10 mon.1@0(leader).pg v5938148 got osd.7 osd_stat(156 GB used, 772 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[]) (was osd_stat(156 GB used, 772 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[])) 2013-04-26 02:39:14.300326 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.9 10.227.227.103:6816/8752 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:14.300340 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x1798c80 from 10.227.227.103:6789/0 2013-04-26 02:39:14.300459 7f27d600e700 10 mon.1@0(leader).pg v5938148 preprocess_query pg_stats(13 pgs tid 57 v 10556) v1 from osd.9 10.227.227.103:6816/8752 2013-04-26 02:39:14.300541 7f27d600e700 10 mon.1@0(leader).pg v5938148 prepare_update pg_stats(13 pgs tid 57 v 10556) v1 from osd.9 10.227.227.103:6816/8752 2013-04-26 02:39:14.300555 7f27d600e700 10 mon.1@0(leader).pg v5938148 prepare_pg_stats pg_stats(13 pgs tid 57 v 10556) v1 from osd.9 2013-04-26 02:39:14.300566 7f27d600e700 10 mon.1@0(leader).pg v5938148 got osd.9 osd_stat(161 GB used, 767 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[]) (was osd_stat(161 GB used, 767 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[])) 2013-04-26 02:39:14.411469 7f27d680f700 10 mon.1@0(leader).pg v5938148 check_down_pgs 2013-04-26 02:39:14.411728 7f27d680f700 10 mon.1@0(leader).pg v5938148 v5938148: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 249KB/s wr, 16op/s 2013-04-26 02:39:14.411841 7f27d680f700 10 mon.1@0(leader).mds e3724 e3724: 1/1/1 up {0=4=up:active}, 2 up:standby 2013-04-26 02:39:14.411908 7f27d680f700 10 mon.1@0(leader).osd e10556 e10556: 12 osds: 12 up, 12 in 2013-04-26 02:39:14.412169 7f27d680f700 10 mon.1@0(leader).osd e10556 min_last_epoch_clean 10531 2013-04-26 02:39:14.412310 7f27d680f700 10 mon.1@0(leader).log v5604031 update_from_paxos 2013-04-26 02:39:14.412390 7f27d680f700 10 mon.1@0(leader).log v5604031 update_from_paxos version 5604031 summary v 5604031 2013-04-26 02:39:14.412424 7f27d680f700 10 mon.1@0(leader).log v5604031 log 2013-04-26 02:39:14.412498 7f27d680f700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:14.412582 7f27d680f700 10 mon.1@0(leader).auth v4370 auth 2013-04-26 02:39:14.532713 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.10 10.227.227.103:6801/8155 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:14.532732 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x2ebb680 from 10.227.227.103:6789/0 2013-04-26 02:39:14.532891 7f27d600e700 10 mon.1@0(leader).pg v5938148 preprocess_query pg_stats(12 pgs tid 52 v 10556) v1 from osd.10 10.227.227.103:6801/8155 2013-04-26 02:39:14.532993 7f27d600e700 10 mon.1@0(leader).pg v5938148 prepare_update pg_stats(12 pgs tid 52 v 10556) v1 from osd.10 10.227.227.103:6801/8155 2013-04-26 02:39:14.533007 7f27d600e700 10 mon.1@0(leader).pg v5938148 prepare_pg_stats pg_stats(12 pgs tid 52 v 10556) v1 from osd.10 2013-04-26 02:39:14.533022 7f27d600e700 10 mon.1@0(leader).pg v5938148 got osd.10 osd_stat(174 GB used, 755 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[]) (was osd_stat(174 GB used, 755 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[])) 2013-04-26 02:39:14.546497 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.2 10.227.227.101:6807/30207 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:14.546515 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x6ce4f00 from 10.227.227.103:6789/0 2013-04-26 02:39:14.546628 7f27d600e700 10 mon.1@0(leader).pg v5938148 preprocess_query pg_stats(28 pgs tid 736 v 10556) v1 from osd.2 10.227.227.101:6807/30207 2013-04-26 02:39:14.546708 7f27d600e700 10 mon.1@0(leader).pg v5938148 prepare_update pg_stats(28 pgs tid 736 v 10556) v1 from osd.2 10.227.227.101:6807/30207 2013-04-26 02:39:14.546723 7f27d600e700 10 mon.1@0(leader).pg v5938148 prepare_pg_stats pg_stats(28 pgs tid 736 v 10556) v1 from osd.2 2013-04-26 02:39:14.546734 7f27d600e700 10 mon.1@0(leader).pg v5938148 got osd.2 osd_stat(311 GB used, 1549 GB avail, 1860 GB total, peers [3,4,5,6,7,8,9,10,11]/[]) (was osd_stat(311 GB used, 1549 GB avail, 1860 GB total, peers [3,4,5,6,7,8,9,10,11]/[])) 2013-04-26 02:39:14.642666 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.1 10.227.227.101:6804/30088 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:14.642685 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x6ce4c80 from 10.227.227.103:6789/0 2013-04-26 02:39:14.642812 7f27d600e700 10 mon.1@0(leader).pg v5938148 preprocess_query pg_stats(14 pgs tid 730 v 10556) v1 from osd.1 10.227.227.101:6804/30088 2013-04-26 02:39:14.642904 7f27d600e700 10 mon.1@0(leader).pg v5938148 prepare_update pg_stats(14 pgs tid 730 v 10556) v1 from osd.1 10.227.227.101:6804/30088 2013-04-26 02:39:14.642918 7f27d600e700 10 mon.1@0(leader).pg v5938148 prepare_pg_stats pg_stats(14 pgs tid 730 v 10556) v1 from osd.1 2013-04-26 02:39:14.642932 7f27d600e700 10 mon.1@0(leader).pg v5938148 got osd.1 osd_stat(342 GB used, 1517 GB avail, 1860 GB total, peers [3,4,5,6,7,8,9,10,11]/[]) (was osd_stat(342 GB used, 1517 GB avail, 1860 GB total, peers [3,4,5,6,7,8,9,10,11]/[])) 2013-04-26 02:39:14.685875 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.0 10.227.227.101:6801/29982 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:14.685894 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x6ce4a00 from 10.227.227.103:6789/0 2013-04-26 02:39:14.686009 7f27d600e700 10 mon.1@0(leader).pg v5938148 preprocess_query pg_stats(26 pgs tid 741 v 10556) v1 from osd.0 10.227.227.101:6801/29982 2013-04-26 02:39:14.686096 7f27d600e700 10 mon.1@0(leader).pg v5938148 prepare_update pg_stats(26 pgs tid 741 v 10556) v1 from osd.0 10.227.227.101:6801/29982 2013-04-26 02:39:14.686109 7f27d600e700 10 mon.1@0(leader).pg v5938148 prepare_pg_stats pg_stats(26 pgs tid 741 v 10556) v1 from osd.0 2013-04-26 02:39:14.686120 7f27d600e700 10 mon.1@0(leader).pg v5938148 got osd.0 osd_stat(367 GB used, 1493 GB avail, 1860 GB total, peers [3,4,5,6,7,8,9,10,11]/[]) (was osd_stat(367 GB used, 1493 GB avail, 1860 GB total, peers [3,4,5,6,7,8,9,10,11]/[])) 2013-04-26 02:39:14.818727 7f27d680f700 10 mon.1@0(leader).log v5604031 encode_full log v 5604031 2013-04-26 02:39:14.818998 7f27d680f700 10 mon.1@0(leader).log v5604031 encode_pending v5604032 2013-04-26 02:39:14.888421 7f27d680f700 10 mon.1@0(leader).pg v5938148 encode_pending v 5938149 2013-04-26 02:39:14.946859 7f27d600e700 10 mon.1@0(leader).log v5604032 update_from_paxos 2013-04-26 02:39:14.946989 7f27d600e700 10 mon.1@0(leader).log v5604032 update_from_paxos version 5604032 summary v 5604031 2013-04-26 02:39:14.947047 7f27d600e700 10 mon.1@0(leader).log v5604032 update_from_paxos latest full 5604031 2013-04-26 02:39:14.947104 7f27d600e700 7 mon.1@0(leader).log v5604032 update_from_paxos applying incremental log 5604032 2013-04-26 02:39:11.439382 mon.0 10.227.227.101:6789/0 15 : [INF] pgmap v5938147: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 1112KB/s wr, 65op/s 2013-04-26 02:39:14.947168 7f27d600e700 7 mon.1@0(leader).log v5604032 update_from_paxos applying incremental log 5604032 2013-04-26 02:39:13.856259 mon.0 10.227.227.101:6789/0 16 : [INF] pgmap v5938148: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 249KB/s wr, 16op/s 2013-04-26 02:39:14.947267 7f27d600e700 10 mon.1@0(leader).log v5604032 check_subs 2013-04-26 02:39:14.947348 7f27d600e700 10 mon.1@0(leader).log v5604032 create_pending v 5604033 2013-04-26 02:39:14.947417 7f27d600e700 7 mon.1@0(leader).log v5604032 _updated_log for mon.0 10.227.227.101:6789/0 2013-04-26 02:39:15.005332 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.6 10.227.227.103:6807/8402 via mon.2 10.227.227.104:6789/0 2013-04-26 02:39:15.005353 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x1798f00 from 10.227.227.104:6789/0 2013-04-26 02:39:15.072543 7f27d600e700 7 mon.1@0(leader).pg v5938148 update_from_paxos applying incremental 5938149 2013-04-26 02:39:15.073179 7f27d600e700 10 mon.1@0(leader).pg v5938149 v5938149: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 2588KB/s wr, 48op/s 2013-04-26 02:39:15.073270 7f27d600e700 10 mon.1@0(leader).pg v5938149 send_pg_creates to 0 pgs 2013-04-26 02:39:15.073276 7f27d600e700 10 mon.1@0(leader).pg v5938149 update_logger 2013-04-26 02:39:15.073353 7f27d600e700 10 mon.1@0(leader).pg v5938149 create_pending v 5938150 2013-04-26 02:39:15.073387 7f27d600e700 7 mon.1@0(leader).pg v5938149 _updated_stats for osd.11 10.227.227.103:6804/8253 2013-04-26 02:39:15.073517 7f27d600e700 7 mon.1@0(leader).pg v5938149 _updated_stats for osd.7 10.227.227.103:6810/8502 2013-04-26 02:39:15.073597 7f27d600e700 7 mon.1@0(leader).pg v5938149 _updated_stats for osd.9 10.227.227.103:6816/8752 2013-04-26 02:39:15.073650 7f27d600e700 7 mon.1@0(leader).pg v5938149 _updated_stats for osd.10 10.227.227.103:6801/8155 2013-04-26 02:39:15.073745 7f27d600e700 7 mon.1@0(leader).pg v5938149 _updated_stats for osd.2 10.227.227.101:6807/30207 2013-04-26 02:39:15.073790 7f27d600e700 7 mon.1@0(leader).pg v5938149 _updated_stats for osd.1 10.227.227.101:6804/30088 2013-04-26 02:39:15.073844 7f27d600e700 7 mon.1@0(leader).pg v5938149 _updated_stats for osd.0 10.227.227.101:6801/29982 2013-04-26 02:39:15.074015 7f27d600e700 10 mon.1@0(leader).pg v5938149 preprocess_query pg_stats(14 pgs tid 57 v 10556) v1 from osd.6 10.227.227.103:6807/8402 2013-04-26 02:39:15.074119 7f27d600e700 10 mon.1@0(leader).pg v5938149 prepare_update pg_stats(14 pgs tid 57 v 10556) v1 from osd.6 10.227.227.103:6807/8402 2013-04-26 02:39:15.074133 7f27d600e700 10 mon.1@0(leader).pg v5938149 prepare_pg_stats pg_stats(14 pgs tid 57 v 10556) v1 from osd.6 2013-04-26 02:39:15.074146 7f27d600e700 10 mon.1@0(leader).pg v5938149 got osd.6 osd_stat(184 GB used, 745 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[]) (was osd_stat(184 GB used, 745 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[])) 2013-04-26 02:39:15.074232 7f27d600e700 10 mon.1@0(leader).pg v5938149 check_osd_map already seen 10556 >= 10556 2013-04-26 02:39:15.074238 7f27d600e700 10 mon.1@0(leader).pg v5938149 update_logger 2013-04-26 02:39:15.074264 7f27d600e700 0 log [INF] : pgmap v5938149: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 2588KB/s wr, 48op/s 2013-04-26 02:39:15.074444 7f27d600e700 10 mon.1@0(leader).log v5604032 update_from_paxos 2013-04-26 02:39:15.074506 7f27d600e700 10 mon.1@0(leader).log v5604032 update_from_paxos version 5604032 summary v 5604032 2013-04-26 02:39:15.074561 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:15.074684 7f27d600e700 10 mon.1@0(leader).log v5604032 update_from_paxos 2013-04-26 02:39:15.074734 7f27d600e700 10 mon.1@0(leader).log v5604032 update_from_paxos version 5604032 summary v 5604032 2013-04-26 02:39:15.074760 7f27d600e700 10 mon.1@0(leader).log v5604032 preprocess_query log(1 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:15.074790 7f27d600e700 10 mon.1@0(leader).log v5604032 preprocess_log log(1 entries) v1 from mon.0 2013-04-26 02:39:15.074827 7f27d600e700 10 mon.1@0(leader).log v5604032 prepare_update log(1 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:15.074859 7f27d600e700 10 mon.1@0(leader).log v5604032 prepare_log log(1 entries) v1 from mon.0 2013-04-26 02:39:15.074889 7f27d600e700 10 mon.1@0(leader).log v5604032 logging 2013-04-26 02:39:15.074267 mon.0 10.227.227.101:6789/0 17 : [INF] pgmap v5938149: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 2588KB/s wr, 48op/s 2013-04-26 02:39:15.089237 7f27d600e700 10 mon.1@0(leader).log v5604032 update_from_paxos 2013-04-26 02:39:15.089314 7f27d600e700 10 mon.1@0(leader).log v5604032 update_from_paxos version 5604032 summary v 5604032 2013-04-26 02:39:15.089380 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:15.122731 7f27d600e700 10 mon.1@0(leader).log v5604032 update_from_paxos 2013-04-26 02:39:15.122808 7f27d600e700 10 mon.1@0(leader).log v5604032 update_from_paxos version 5604032 summary v 5604032 2013-04-26 02:39:15.122876 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:15.344731 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.4 10.227.227.104:6804/14471 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:15.344769 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x1799900 from 10.227.227.103:6789/0 2013-04-26 02:39:15.344923 7f27d600e700 10 mon.1@0(leader).pg v5938149 preprocess_query pg_stats(37 pgs tid 746 v 10556) v1 from osd.4 10.227.227.104:6804/14471 2013-04-26 02:39:15.345088 7f27d600e700 10 mon.1@0(leader).pg v5938149 prepare_update pg_stats(37 pgs tid 746 v 10556) v1 from osd.4 10.227.227.104:6804/14471 2013-04-26 02:39:15.345104 7f27d600e700 10 mon.1@0(leader).pg v5938149 prepare_pg_stats pg_stats(37 pgs tid 746 v 10556) v1 from osd.4 2013-04-26 02:39:15.345119 7f27d600e700 10 mon.1@0(leader).pg v5938149 got osd.4 osd_stat(423 GB used, 1437 GB avail, 1860 GB total, peers [0,1,2,6,7,8,9,10,11]/[]) (was osd_stat(423 GB used, 1437 GB avail, 1860 GB total, peers [0,1,2,6,7,8,9,10,11]/[])) 2013-04-26 02:39:15.469082 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.3 10.227.227.104:6800/14463 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:15.469102 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x2eb8f00 from 10.227.227.103:6789/0 2013-04-26 02:39:15.469247 7f27d600e700 10 mon.1@0(leader).pg v5938149 preprocess_query pg_stats(31 pgs tid 751 v 10556) v1 from osd.3 10.227.227.104:6800/14463 2013-04-26 02:39:15.469341 7f27d600e700 10 mon.1@0(leader).pg v5938149 prepare_update pg_stats(31 pgs tid 751 v 10556) v1 from osd.3 10.227.227.104:6800/14463 2013-04-26 02:39:15.469355 7f27d600e700 10 mon.1@0(leader).pg v5938149 prepare_pg_stats pg_stats(31 pgs tid 751 v 10556) v1 from osd.3 2013-04-26 02:39:15.469370 7f27d600e700 10 mon.1@0(leader).pg v5938149 got osd.3 osd_stat(383 GB used, 1477 GB avail, 1860 GB total, peers [0,1,2,6,7,8,9,10,11]/[]) (was osd_stat(383 GB used, 1477 GB avail, 1860 GB total, peers [0,1,2,6,7,8,9,10,11]/[])) 2013-04-26 02:39:15.915219 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.8 10.227.227.103:6813/8628 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:15.915240 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x6ce4780 from 10.227.227.103:6789/0 2013-04-26 02:39:15.915393 7f27d600e700 10 mon.1@0(leader).pg v5938149 preprocess_query pg_stats(12 pgs tid 55 v 10556) v1 from osd.8 10.227.227.103:6813/8628 2013-04-26 02:39:15.915618 7f27d600e700 10 mon.1@0(leader).pg v5938149 prepare_update pg_stats(12 pgs tid 55 v 10556) v1 from osd.8 10.227.227.103:6813/8628 2013-04-26 02:39:15.915638 7f27d600e700 10 mon.1@0(leader).pg v5938149 prepare_pg_stats pg_stats(12 pgs tid 55 v 10556) v1 from osd.8 2013-04-26 02:39:15.915653 7f27d600e700 10 mon.1@0(leader).pg v5938149 got osd.8 osd_stat(185 GB used, 744 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[]) (was osd_stat(185 GB used, 744 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[])) 2013-04-26 02:39:16.018841 7f27d680f700 10 mon.1@0(leader).pg v5938149 encode_pending v 5938150 2013-04-26 02:39:16.114198 7f27d680f700 10 mon.1@0(leader).log v5604032 encode_full log v 5604032 2013-04-26 02:39:16.114429 7f27d680f700 10 mon.1@0(leader).log v5604032 encode_pending v5604033 2013-04-26 02:39:16.114546 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from mds.? 10.227.227.101:6800/29852 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:16.114564 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x1789340 from 10.227.227.103:6789/0 2013-04-26 02:39:16.114666 7f27d600e700 10 mon.1@0(leader).mds e3724 preprocess_query mdsbeacon(217349/1 up:standby seq 459 v3724) v2 from mds.? 10.227.227.101:6800/29852 2013-04-26 02:39:16.172599 7f27d600e700 7 mon.1@0(leader).pg v5938149 update_from_paxos applying incremental 5938150 2013-04-26 02:39:16.173081 7f27d600e700 10 mon.1@0(leader).pg v5938150 v5938150: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 6612KB/s wr, 119op/s 2013-04-26 02:39:16.173162 7f27d600e700 10 mon.1@0(leader).pg v5938150 send_pg_creates to 0 pgs 2013-04-26 02:39:16.173167 7f27d600e700 10 mon.1@0(leader).pg v5938150 update_logger 2013-04-26 02:39:16.173220 7f27d600e700 10 mon.1@0(leader).pg v5938150 create_pending v 5938151 2013-04-26 02:39:16.173247 7f27d600e700 7 mon.1@0(leader).pg v5938150 _updated_stats for osd.6 10.227.227.103:6807/8402 2013-04-26 02:39:16.173282 7f27d600e700 7 mon.1@0(leader).pg v5938150 _updated_stats for osd.4 10.227.227.104:6804/14471 2013-04-26 02:39:16.173311 7f27d600e700 7 mon.1@0(leader).pg v5938150 _updated_stats for osd.3 10.227.227.104:6800/14463 2013-04-26 02:39:16.173338 7f27d600e700 7 mon.1@0(leader).pg v5938150 _updated_stats for osd.8 10.227.227.103:6813/8628 2013-04-26 02:39:16.173561 7f27d600e700 10 mon.1@0(leader).pg v5938150 check_osd_map already seen 10556 >= 10556 2013-04-26 02:39:16.173579 7f27d600e700 10 mon.1@0(leader).pg v5938150 update_logger 2013-04-26 02:39:16.173606 7f27d600e700 0 log [INF] : pgmap v5938150: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 6612KB/s wr, 119op/s 2013-04-26 02:39:16.289526 7f27d600e700 10 mon.1@0(leader).log v5604033 update_from_paxos 2013-04-26 02:39:16.289656 7f27d600e700 10 mon.1@0(leader).log v5604033 update_from_paxos version 5604033 summary v 5604032 2013-04-26 02:39:16.289748 7f27d600e700 10 mon.1@0(leader).log v5604033 update_from_paxos latest full 5604032 2013-04-26 02:39:16.289849 7f27d600e700 7 mon.1@0(leader).log v5604033 update_from_paxos applying incremental log 5604033 2013-04-26 02:39:15.074267 mon.0 10.227.227.101:6789/0 17 : [INF] pgmap v5938149: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 2588KB/s wr, 48op/s 2013-04-26 02:39:16.289979 7f27d600e700 10 mon.1@0(leader).log v5604033 check_subs 2013-04-26 02:39:16.290094 7f27d600e700 10 mon.1@0(leader).log v5604033 create_pending v 5604034 2013-04-26 02:39:16.290214 7f27d600e700 7 mon.1@0(leader).log v5604033 _updated_log for mon.0 10.227.227.101:6789/0 2013-04-26 02:39:16.290337 7f27d600e700 10 mon.1@0(leader).log v5604033 update_from_paxos 2013-04-26 02:39:16.290413 7f27d600e700 10 mon.1@0(leader).log v5604033 update_from_paxos version 5604033 summary v 5604033 2013-04-26 02:39:16.290446 7f27d600e700 10 mon.1@0(leader).log v5604033 preprocess_query log(1 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:16.290487 7f27d600e700 10 mon.1@0(leader).log v5604033 preprocess_log log(1 entries) v1 from mon.0 2013-04-26 02:39:16.290529 7f27d600e700 10 mon.1@0(leader).log v5604033 nothing new 2013-04-26 02:39:16.290733 7f27d600e700 10 mon.1@0(leader).log v5604033 update_from_paxos 2013-04-26 02:39:16.290797 7f27d600e700 10 mon.1@0(leader).log v5604033 update_from_paxos version 5604033 summary v 5604033 2013-04-26 02:39:16.290864 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:16.298603 7f27d600e700 10 mon.1@0(leader).log v5604033 update_from_paxos 2013-04-26 02:39:16.298679 7f27d600e700 10 mon.1@0(leader).log v5604033 update_from_paxos version 5604033 summary v 5604033 2013-04-26 02:39:16.298746 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:16.347821 7f27d600e700 10 mon.1@0(leader).log v5604033 update_from_paxos 2013-04-26 02:39:16.347895 7f27d600e700 10 mon.1@0(leader).log v5604033 update_from_paxos version 5604033 summary v 5604033 2013-04-26 02:39:16.347967 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:16.502424 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from mds.0 10.227.227.104:6801/10842 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:16.502447 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x1789080 from 10.227.227.103:6789/0 2013-04-26 02:39:16.502614 7f27d600e700 10 mon.1@0(leader).mds e3724 preprocess_query mdsbeacon(217231/4 up:active seq 490 v3724) v2 from mds.0 10.227.227.104:6801/10842 2013-04-26 02:39:17.215916 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from mds.? 10.227.227.103:6800/8011 via mon.2 10.227.227.104:6789/0 2013-04-26 02:39:17.215940 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x1788dc0 from 10.227.227.104:6789/0 2013-04-26 02:39:17.216247 7f27d600e700 10 mon.1@0(leader).mds e3724 preprocess_query mdsbeacon(217445/3 up:standby seq 72 v3724) v2 from mds.? 10.227.227.103:6800/8011 2013-04-26 02:39:17.756029 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.5 10.227.227.104:6807/14479 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:17.756052 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x1799680 from 10.227.227.103:6789/0 2013-04-26 02:39:17.756201 7f27d600e700 10 mon.1@0(leader).pg v5938150 preprocess_query pg_stats(27 pgs tid 744 v 10556) v1 from osd.5 10.227.227.104:6807/14479 2013-04-26 02:39:17.756307 7f27d600e700 10 mon.1@0(leader).pg v5938150 prepare_update pg_stats(27 pgs tid 744 v 10556) v1 from osd.5 10.227.227.104:6807/14479 2013-04-26 02:39:17.756319 7f27d600e700 10 mon.1@0(leader).pg v5938150 prepare_pg_stats pg_stats(27 pgs tid 744 v 10556) v1 from osd.5 2013-04-26 02:39:17.756334 7f27d600e700 10 mon.1@0(leader).pg v5938150 got osd.5 osd_stat(355 GB used, 1505 GB avail, 1860 GB total, peers [0,1,2,6,7,8,9,10,11]/[]) (was osd_stat(355 GB used, 1505 GB avail, 1860 GB total, peers [0,1,2,6,7,8,9,10,11]/[])) 2013-04-26 02:39:17.806804 7f27d680f700 10 mon.1@0(leader).pg v5938150 encode_pending v 5938151 2013-04-26 02:39:17.964080 7f27d600e700 7 mon.1@0(leader).pg v5938150 update_from_paxos applying incremental 5938151 2013-04-26 02:39:17.964253 7f27d600e700 10 mon.1@0(leader).pg v5938151 v5938151: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 2162KB/s wr, 52op/s 2013-04-26 02:39:17.964312 7f27d600e700 10 mon.1@0(leader).pg v5938151 send_pg_creates to 0 pgs 2013-04-26 02:39:17.964317 7f27d600e700 10 mon.1@0(leader).pg v5938151 update_logger 2013-04-26 02:39:17.964357 7f27d600e700 10 mon.1@0(leader).pg v5938151 create_pending v 5938152 2013-04-26 02:39:17.964385 7f27d600e700 7 mon.1@0(leader).pg v5938151 _updated_stats for osd.5 10.227.227.104:6807/14479 2013-04-26 02:39:17.964422 7f27d600e700 10 mon.1@0(leader).pg v5938151 check_osd_map already seen 10556 >= 10556 2013-04-26 02:39:17.964426 7f27d600e700 10 mon.1@0(leader).pg v5938151 update_logger 2013-04-26 02:39:17.964445 7f27d600e700 0 log [INF] : pgmap v5938151: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 2162KB/s wr, 52op/s 2013-04-26 02:39:17.964690 7f27d600e700 10 mon.1@0(leader).log v5604033 update_from_paxos 2013-04-26 02:39:17.964775 7f27d600e700 10 mon.1@0(leader).log v5604033 update_from_paxos version 5604033 summary v 5604033 2013-04-26 02:39:17.964857 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:17.965038 7f27d600e700 10 mon.1@0(leader).log v5604033 update_from_paxos 2013-04-26 02:39:17.965110 7f27d600e700 10 mon.1@0(leader).log v5604033 update_from_paxos version 5604033 summary v 5604033 2013-04-26 02:39:17.965146 7f27d600e700 10 mon.1@0(leader).log v5604033 preprocess_query log(2 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:17.965189 7f27d600e700 10 mon.1@0(leader).log v5604033 preprocess_log log(2 entries) v1 from mon.0 2013-04-26 02:39:17.965240 7f27d600e700 10 mon.1@0(leader).log v5604033 prepare_update log(2 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:17.965284 7f27d600e700 10 mon.1@0(leader).log v5604033 prepare_log log(2 entries) v1 from mon.0 2013-04-26 02:39:17.965325 7f27d600e700 10 mon.1@0(leader).log v5604033 logging 2013-04-26 02:39:16.173610 mon.0 10.227.227.101:6789/0 18 : [INF] pgmap v5938150: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 6612KB/s wr, 119op/s 2013-04-26 02:39:17.965385 7f27d600e700 10 mon.1@0(leader).log v5604033 logging 2013-04-26 02:39:17.964447 mon.0 10.227.227.101:6789/0 19 : [INF] pgmap v5938151: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 2162KB/s wr, 52op/s 2013-04-26 02:39:17.968837 7f27d600e700 10 mon.1@0(leader).log v5604033 update_from_paxos 2013-04-26 02:39:17.968909 7f27d600e700 10 mon.1@0(leader).log v5604033 update_from_paxos version 5604033 summary v 5604033 2013-04-26 02:39:17.968977 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:18.014230 7f27d600e700 10 mon.1@0(leader).log v5604033 update_from_paxos 2013-04-26 02:39:18.014307 7f27d600e700 10 mon.1@0(leader).log v5604033 update_from_paxos version 5604033 summary v 5604033 2013-04-26 02:39:18.014374 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:18.892328 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.11 10.227.227.103:6804/8253 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:18.892352 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x6ce5b80 from 10.227.227.103:6789/0 2013-04-26 02:39:18.892518 7f27d600e700 10 mon.1@0(leader).pg v5938151 preprocess_query pg_stats(9 pgs tid 58 v 10556) v1 from osd.11 10.227.227.103:6804/8253 2013-04-26 02:39:18.892623 7f27d600e700 10 mon.1@0(leader).pg v5938151 prepare_update pg_stats(9 pgs tid 58 v 10556) v1 from osd.11 10.227.227.103:6804/8253 2013-04-26 02:39:18.892637 7f27d600e700 10 mon.1@0(leader).pg v5938151 prepare_pg_stats pg_stats(9 pgs tid 58 v 10556) v1 from osd.11 2013-04-26 02:39:18.892652 7f27d600e700 10 mon.1@0(leader).pg v5938151 got osd.11 osd_stat(161 GB used, 768 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[]) (was osd_stat(161 GB used, 768 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[])) 2013-04-26 02:39:18.918934 7f27d680f700 10 mon.1@0(leader).log v5604033 encode_full log v 5604033 2013-04-26 02:39:18.919216 7f27d680f700 10 mon.1@0(leader).log v5604033 encode_pending v5604034 2013-04-26 02:39:19.040590 7f27d680f700 10 mon.1@0(leader).pg v5938151 encode_pending v 5938152 2013-04-26 02:39:19.099137 7f27d600e700 10 mon.1@0(leader).log v5604034 update_from_paxos 2013-04-26 02:39:19.099282 7f27d600e700 10 mon.1@0(leader).log v5604034 update_from_paxos version 5604034 summary v 5604033 2013-04-26 02:39:19.099330 7f27d600e700 10 mon.1@0(leader).log v5604034 update_from_paxos latest full 5604033 2013-04-26 02:39:19.099464 7f27d600e700 7 mon.1@0(leader).log v5604034 update_from_paxos applying incremental log 5604034 2013-04-26 02:39:16.173610 mon.0 10.227.227.101:6789/0 18 : [INF] pgmap v5938150: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 6612KB/s wr, 119op/s 2013-04-26 02:39:19.099524 7f27d600e700 7 mon.1@0(leader).log v5604034 update_from_paxos applying incremental log 5604034 2013-04-26 02:39:17.964447 mon.0 10.227.227.101:6789/0 19 : [INF] pgmap v5938151: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 2162KB/s wr, 52op/s 2013-04-26 02:39:19.099618 7f27d600e700 10 mon.1@0(leader).log v5604034 check_subs 2013-04-26 02:39:19.099684 7f27d600e700 10 mon.1@0(leader).log v5604034 create_pending v 5604035 2013-04-26 02:39:19.099741 7f27d600e700 7 mon.1@0(leader).log v5604034 _updated_log for mon.0 10.227.227.101:6789/0 2013-04-26 02:39:19.214029 7f27d600e700 7 mon.1@0(leader).pg v5938151 update_from_paxos applying incremental 5938152 2013-04-26 02:39:19.214121 7f27d600e700 10 mon.1@0(leader).pg v5938152 v5938152: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 85397B/s wr, 11op/s 2013-04-26 02:39:19.214183 7f27d600e700 10 mon.1@0(leader).pg v5938152 send_pg_creates to 0 pgs 2013-04-26 02:39:19.214188 7f27d600e700 10 mon.1@0(leader).pg v5938152 update_logger 2013-04-26 02:39:19.214230 7f27d600e700 10 mon.1@0(leader).pg v5938152 create_pending v 5938153 2013-04-26 02:39:19.214262 7f27d600e700 7 mon.1@0(leader).pg v5938152 _updated_stats for osd.11 10.227.227.103:6804/8253 2013-04-26 02:39:19.214306 7f27d600e700 10 mon.1@0(leader).pg v5938152 check_osd_map already seen 10556 >= 10556 2013-04-26 02:39:19.214310 7f27d600e700 10 mon.1@0(leader).pg v5938152 update_logger 2013-04-26 02:39:19.214332 7f27d600e700 0 log [INF] : pgmap v5938152: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 85397B/s wr, 11op/s 2013-04-26 02:39:19.214560 7f27d600e700 10 mon.1@0(leader).log v5604034 update_from_paxos 2013-04-26 02:39:19.214639 7f27d600e700 10 mon.1@0(leader).log v5604034 update_from_paxos version 5604034 summary v 5604034 2013-04-26 02:39:19.214707 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:19.214861 7f27d600e700 10 mon.1@0(leader).log v5604034 update_from_paxos 2013-04-26 02:39:19.214927 7f27d600e700 10 mon.1@0(leader).log v5604034 update_from_paxos version 5604034 summary v 5604034 2013-04-26 02:39:19.214966 7f27d600e700 10 mon.1@0(leader).log v5604034 preprocess_query log(1 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:19.215042 7f27d600e700 10 mon.1@0(leader).log v5604034 preprocess_log log(1 entries) v1 from mon.0 2013-04-26 02:39:19.215093 7f27d600e700 10 mon.1@0(leader).log v5604034 prepare_update log(1 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:19.215135 7f27d600e700 10 mon.1@0(leader).log v5604034 prepare_log log(1 entries) v1 from mon.0 2013-04-26 02:39:19.215176 7f27d600e700 10 mon.1@0(leader).log v5604034 logging 2013-04-26 02:39:19.214334 mon.0 10.227.227.101:6789/0 20 : [INF] pgmap v5938152: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 85397B/s wr, 11op/s 2013-04-26 02:39:19.218818 7f27d600e700 10 mon.1@0(leader).log v5604034 update_from_paxos 2013-04-26 02:39:19.218891 7f27d600e700 10 mon.1@0(leader).log v5604034 update_from_paxos version 5604034 summary v 5604034 2013-04-26 02:39:19.218956 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:19.264449 7f27d600e700 10 mon.1@0(leader).log v5604034 update_from_paxos 2013-04-26 02:39:19.264524 7f27d600e700 10 mon.1@0(leader).log v5604034 update_from_paxos version 5604034 summary v 5604034 2013-04-26 02:39:19.264591 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:19.275332 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.7 10.227.227.103:6810/8502 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:19.275352 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x6ce4780 from 10.227.227.103:6789/0 2013-04-26 02:39:19.275462 7f27d600e700 10 mon.1@0(leader).pg v5938152 preprocess_query pg_stats(10 pgs tid 55 v 10556) v1 from osd.7 10.227.227.103:6810/8502 2013-04-26 02:39:19.275559 7f27d600e700 10 mon.1@0(leader).pg v5938152 prepare_update pg_stats(10 pgs tid 55 v 10556) v1 from osd.7 10.227.227.103:6810/8502 2013-04-26 02:39:19.275575 7f27d600e700 10 mon.1@0(leader).pg v5938152 prepare_pg_stats pg_stats(10 pgs tid 55 v 10556) v1 from osd.7 2013-04-26 02:39:19.275587 7f27d600e700 10 mon.1@0(leader).pg v5938152 got osd.7 osd_stat(156 GB used, 772 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[]) (was osd_stat(156 GB used, 772 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[])) 2013-04-26 02:39:19.301028 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.9 10.227.227.103:6816/8752 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:19.301046 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x2eb8f00 from 10.227.227.103:6789/0 2013-04-26 02:39:19.301172 7f27d600e700 10 mon.1@0(leader).pg v5938152 preprocess_query pg_stats(13 pgs tid 58 v 10556) v1 from osd.9 10.227.227.103:6816/8752 2013-04-26 02:39:19.301258 7f27d600e700 10 mon.1@0(leader).pg v5938152 prepare_update pg_stats(13 pgs tid 58 v 10556) v1 from osd.9 10.227.227.103:6816/8752 2013-04-26 02:39:19.301277 7f27d600e700 10 mon.1@0(leader).pg v5938152 prepare_pg_stats pg_stats(13 pgs tid 58 v 10556) v1 from osd.9 2013-04-26 02:39:19.301289 7f27d600e700 10 mon.1@0(leader).pg v5938152 got osd.9 osd_stat(161 GB used, 767 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[]) (was osd_stat(161 GB used, 767 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[])) 2013-04-26 02:39:19.412813 7f27d680f700 10 mon.1@0(leader).pg v5938152 check_down_pgs 2013-04-26 02:39:19.413073 7f27d680f700 10 mon.1@0(leader).pg v5938152 v5938152: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 85397B/s wr, 11op/s 2013-04-26 02:39:19.413151 7f27d680f700 10 mon.1@0(leader).mds e3724 e3724: 1/1/1 up {0=4=up:active}, 2 up:standby 2013-04-26 02:39:19.413214 7f27d680f700 10 mon.1@0(leader).osd e10556 e10556: 12 osds: 12 up, 12 in 2013-04-26 02:39:19.413480 7f27d680f700 10 mon.1@0(leader).osd e10556 min_last_epoch_clean 10531 2013-04-26 02:39:19.413620 7f27d680f700 10 mon.1@0(leader).log v5604034 update_from_paxos 2013-04-26 02:39:19.413693 7f27d680f700 10 mon.1@0(leader).log v5604034 update_from_paxos version 5604034 summary v 5604034 2013-04-26 02:39:19.413727 7f27d680f700 10 mon.1@0(leader).log v5604034 log 2013-04-26 02:39:19.413816 7f27d680f700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:19.413881 7f27d680f700 10 mon.1@0(leader).auth v4370 auth 2013-04-26 02:39:19.533461 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.10 10.227.227.103:6801/8155 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:19.533482 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x1799400 from 10.227.227.103:6789/0 2013-04-26 02:39:19.533599 7f27d600e700 10 mon.1@0(leader).pg v5938152 preprocess_query pg_stats(13 pgs tid 53 v 10556) v1 from osd.10 10.227.227.103:6801/8155 2013-04-26 02:39:19.533686 7f27d600e700 10 mon.1@0(leader).pg v5938152 prepare_update pg_stats(13 pgs tid 53 v 10556) v1 from osd.10 10.227.227.103:6801/8155 2013-04-26 02:39:19.533705 7f27d600e700 10 mon.1@0(leader).pg v5938152 prepare_pg_stats pg_stats(13 pgs tid 53 v 10556) v1 from osd.10 2013-04-26 02:39:19.533719 7f27d600e700 10 mon.1@0(leader).pg v5938152 got osd.10 osd_stat(174 GB used, 755 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[]) (was osd_stat(174 GB used, 755 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[])) 2013-04-26 02:39:19.547172 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.2 10.227.227.101:6807/30207 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:19.547190 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x6ce4500 from 10.227.227.103:6789/0 2013-04-26 02:39:19.547300 7f27d600e700 10 mon.1@0(leader).pg v5938152 preprocess_query pg_stats(19 pgs tid 737 v 10556) v1 from osd.2 10.227.227.101:6807/30207 2013-04-26 02:39:19.547573 7f27d600e700 10 mon.1@0(leader).pg v5938152 prepare_update pg_stats(19 pgs tid 737 v 10556) v1 from osd.2 10.227.227.101:6807/30207 2013-04-26 02:39:19.547596 7f27d600e700 10 mon.1@0(leader).pg v5938152 prepare_pg_stats pg_stats(19 pgs tid 737 v 10556) v1 from osd.2 2013-04-26 02:39:19.547609 7f27d600e700 10 mon.1@0(leader).pg v5938152 got osd.2 osd_stat(311 GB used, 1549 GB avail, 1860 GB total, peers [3,4,5,6,7,8,9,10,11]/[]) (was osd_stat(311 GB used, 1549 GB avail, 1860 GB total, peers [3,4,5,6,7,8,9,10,11]/[])) 2013-04-26 02:39:19.643715 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.1 10.227.227.101:6804/30088 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:19.643737 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x6ce4280 from 10.227.227.103:6789/0 2013-04-26 02:39:19.643858 7f27d600e700 10 mon.1@0(leader).pg v5938152 preprocess_query pg_stats(19 pgs tid 731 v 10556) v1 from osd.1 10.227.227.101:6804/30088 2013-04-26 02:39:19.643945 7f27d600e700 10 mon.1@0(leader).pg v5938152 prepare_update pg_stats(19 pgs tid 731 v 10556) v1 from osd.1 10.227.227.101:6804/30088 2013-04-26 02:39:19.643959 7f27d600e700 10 mon.1@0(leader).pg v5938152 prepare_pg_stats pg_stats(19 pgs tid 731 v 10556) v1 from osd.1 2013-04-26 02:39:19.643971 7f27d600e700 10 mon.1@0(leader).pg v5938152 got osd.1 osd_stat(342 GB used, 1517 GB avail, 1860 GB total, peers [3,4,5,6,7,8,9,10,11]/[]) (was osd_stat(342 GB used, 1517 GB avail, 1860 GB total, peers [3,4,5,6,7,8,9,10,11]/[])) 2013-04-26 02:39:19.686984 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.0 10.227.227.101:6801/29982 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:19.687019 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x6ce4000 from 10.227.227.103:6789/0 2013-04-26 02:39:19.687132 7f27d600e700 10 mon.1@0(leader).pg v5938152 preprocess_query pg_stats(27 pgs tid 742 v 10556) v1 from osd.0 10.227.227.101:6801/29982 2013-04-26 02:39:19.687217 7f27d600e700 10 mon.1@0(leader).pg v5938152 prepare_update pg_stats(27 pgs tid 742 v 10556) v1 from osd.0 10.227.227.101:6801/29982 2013-04-26 02:39:19.687228 7f27d600e700 10 mon.1@0(leader).pg v5938152 prepare_pg_stats pg_stats(27 pgs tid 742 v 10556) v1 from osd.0 2013-04-26 02:39:19.687239 7f27d600e700 10 mon.1@0(leader).pg v5938152 got osd.0 osd_stat(367 GB used, 1493 GB avail, 1860 GB total, peers [3,4,5,6,7,8,9,10,11]/[]) (was osd_stat(367 GB used, 1493 GB avail, 1860 GB total, peers [3,4,5,6,7,8,9,10,11]/[])) 2013-04-26 02:39:19.903842 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.6 10.227.227.103:6807/8402 via mon.2 10.227.227.104:6789/0 2013-04-26 02:39:19.903865 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x1799900 from 10.227.227.104:6789/0 2013-04-26 02:39:19.904012 7f27d600e700 10 mon.1@0(leader).pg v5938152 preprocess_query pg_stats(9 pgs tid 58 v 10556) v1 from osd.6 10.227.227.103:6807/8402 2013-04-26 02:39:19.904115 7f27d600e700 10 mon.1@0(leader).pg v5938152 prepare_update pg_stats(9 pgs tid 58 v 10556) v1 from osd.6 10.227.227.103:6807/8402 2013-04-26 02:39:19.904129 7f27d600e700 10 mon.1@0(leader).pg v5938152 prepare_pg_stats pg_stats(9 pgs tid 58 v 10556) v1 from osd.6 2013-04-26 02:39:19.904144 7f27d600e700 10 mon.1@0(leader).pg v5938152 got osd.6 osd_stat(184 GB used, 745 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[]) (was osd_stat(184 GB used, 745 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[])) 2013-04-26 02:39:20.030113 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from mds.? 10.227.227.101:6800/29852 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:20.030132 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x1789340 from 10.227.227.103:6789/0 2013-04-26 02:39:20.030267 7f27d600e700 10 mon.1@0(leader).mds e3724 preprocess_query mdsbeacon(217349/1 up:standby seq 460 v3724) v2 from mds.? 10.227.227.101:6800/29852 2013-04-26 02:39:20.158721 7f27d680f700 10 mon.1@0(leader).log v5604034 encode_full log v 5604034 2013-04-26 02:39:20.159017 7f27d680f700 10 mon.1@0(leader).log v5604034 encode_pending v5604035 2013-04-26 02:39:20.241415 7f27d680f700 10 mon.1@0(leader).pg v5938152 encode_pending v 5938153 2013-04-26 02:39:20.299550 7f27d600e700 10 mon.1@0(leader).log v5604035 update_from_paxos 2013-04-26 02:39:20.299625 7f27d600e700 10 mon.1@0(leader).log v5604035 update_from_paxos version 5604035 summary v 5604034 2013-04-26 02:39:20.299679 7f27d600e700 10 mon.1@0(leader).log v5604035 update_from_paxos latest full 5604034 2013-04-26 02:39:20.299733 7f27d600e700 7 mon.1@0(leader).log v5604035 update_from_paxos applying incremental log 5604035 2013-04-26 02:39:19.214334 mon.0 10.227.227.101:6789/0 20 : [INF] pgmap v5938152: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 85397B/s wr, 11op/s 2013-04-26 02:39:20.299823 7f27d600e700 10 mon.1@0(leader).log v5604035 check_subs 2013-04-26 02:39:20.299895 7f27d600e700 10 mon.1@0(leader).log v5604035 create_pending v 5604036 2013-04-26 02:39:20.299964 7f27d600e700 7 mon.1@0(leader).log v5604035 _updated_log for mon.0 10.227.227.101:6789/0 2013-04-26 02:39:20.351253 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.4 10.227.227.104:6804/14471 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:20.351267 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x6ce5180 from 10.227.227.103:6789/0 2013-04-26 02:39:20.422603 7f27d600e700 7 mon.1@0(leader).pg v5938152 update_from_paxos applying incremental 5938153 2013-04-26 02:39:20.423191 7f27d600e700 10 mon.1@0(leader).pg v5938153 v5938153: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 202KB/s wr, 19op/s 2013-04-26 02:39:20.423265 7f27d600e700 10 mon.1@0(leader).pg v5938153 send_pg_creates to 0 pgs 2013-04-26 02:39:20.423270 7f27d600e700 10 mon.1@0(leader).pg v5938153 update_logger 2013-04-26 02:39:20.423341 7f27d600e700 10 mon.1@0(leader).pg v5938153 create_pending v 5938154 2013-04-26 02:39:20.423369 7f27d600e700 7 mon.1@0(leader).pg v5938153 _updated_stats for osd.7 10.227.227.103:6810/8502 2013-04-26 02:39:20.423406 7f27d600e700 7 mon.1@0(leader).pg v5938153 _updated_stats for osd.9 10.227.227.103:6816/8752 2013-04-26 02:39:20.423605 7f27d600e700 7 mon.1@0(leader).pg v5938153 _updated_stats for osd.10 10.227.227.103:6801/8155 2013-04-26 02:39:20.423673 7f27d600e700 7 mon.1@0(leader).pg v5938153 _updated_stats for osd.2 10.227.227.101:6807/30207 2013-04-26 02:39:20.423743 7f27d600e700 7 mon.1@0(leader).pg v5938153 _updated_stats for osd.1 10.227.227.101:6804/30088 2013-04-26 02:39:20.423796 7f27d600e700 7 mon.1@0(leader).pg v5938153 _updated_stats for osd.0 10.227.227.101:6801/29982 2013-04-26 02:39:20.423893 7f27d600e700 7 mon.1@0(leader).pg v5938153 _updated_stats for osd.6 10.227.227.103:6807/8402 2013-04-26 02:39:20.424038 7f27d600e700 10 mon.1@0(leader).pg v5938153 preprocess_query pg_stats(22 pgs tid 747 v 10556) v1 from osd.4 10.227.227.104:6804/14471 2013-04-26 02:39:20.424130 7f27d600e700 10 mon.1@0(leader).pg v5938153 prepare_update pg_stats(22 pgs tid 747 v 10556) v1 from osd.4 10.227.227.104:6804/14471 2013-04-26 02:39:20.424140 7f27d600e700 10 mon.1@0(leader).pg v5938153 prepare_pg_stats pg_stats(22 pgs tid 747 v 10556) v1 from osd.4 2013-04-26 02:39:20.424152 7f27d600e700 10 mon.1@0(leader).pg v5938153 got osd.4 osd_stat(423 GB used, 1437 GB avail, 1860 GB total, peers [0,1,2,6,7,8,9,10,11]/[]) (was osd_stat(423 GB used, 1437 GB avail, 1860 GB total, peers [0,1,2,6,7,8,9,10,11]/[])) 2013-04-26 02:39:20.424240 7f27d600e700 10 mon.1@0(leader).pg v5938153 check_osd_map already seen 10556 >= 10556 2013-04-26 02:39:20.424249 7f27d600e700 10 mon.1@0(leader).pg v5938153 update_logger 2013-04-26 02:39:20.424269 7f27d600e700 0 log [INF] : pgmap v5938153: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 202KB/s wr, 19op/s 2013-04-26 02:39:20.424466 7f27d600e700 10 mon.1@0(leader).log v5604035 update_from_paxos 2013-04-26 02:39:20.424542 7f27d600e700 10 mon.1@0(leader).log v5604035 update_from_paxos version 5604035 summary v 5604035 2013-04-26 02:39:20.424600 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:20.424756 7f27d600e700 10 mon.1@0(leader).log v5604035 update_from_paxos 2013-04-26 02:39:20.424813 7f27d600e700 10 mon.1@0(leader).log v5604035 update_from_paxos version 5604035 summary v 5604035 2013-04-26 02:39:20.424869 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:20.425051 7f27d600e700 10 mon.1@0(leader).log v5604035 update_from_paxos 2013-04-26 02:39:20.425126 7f27d600e700 10 mon.1@0(leader).log v5604035 update_from_paxos version 5604035 summary v 5604035 2013-04-26 02:39:20.425163 7f27d600e700 10 mon.1@0(leader).log v5604035 preprocess_query log(1 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:20.425208 7f27d600e700 10 mon.1@0(leader).log v5604035 preprocess_log log(1 entries) v1 from mon.0 2013-04-26 02:39:20.425260 7f27d600e700 10 mon.1@0(leader).log v5604035 prepare_update log(1 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:20.425306 7f27d600e700 10 mon.1@0(leader).log v5604035 prepare_log log(1 entries) v1 from mon.0 2013-04-26 02:39:20.425347 7f27d600e700 10 mon.1@0(leader).log v5604035 logging 2013-04-26 02:39:20.424272 mon.0 10.227.227.101:6789/0 21 : [INF] pgmap v5938153: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 202KB/s wr, 19op/s 2013-04-26 02:39:20.469697 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.3 10.227.227.104:6800/14463 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:20.469719 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x1798f00 from 10.227.227.103:6789/0 2013-04-26 02:39:20.469843 7f27d600e700 10 mon.1@0(leader).pg v5938153 preprocess_query pg_stats(23 pgs tid 752 v 10556) v1 from osd.3 10.227.227.104:6800/14463 2013-04-26 02:39:20.469940 7f27d600e700 10 mon.1@0(leader).pg v5938153 prepare_update pg_stats(23 pgs tid 752 v 10556) v1 from osd.3 10.227.227.104:6800/14463 2013-04-26 02:39:20.469955 7f27d600e700 10 mon.1@0(leader).pg v5938153 prepare_pg_stats pg_stats(23 pgs tid 752 v 10556) v1 from osd.3 2013-04-26 02:39:20.469966 7f27d600e700 10 mon.1@0(leader).pg v5938153 got osd.3 osd_stat(383 GB used, 1477 GB avail, 1860 GB total, peers [0,1,2,6,7,8,9,10,11]/[]) (was osd_stat(383 GB used, 1477 GB avail, 1860 GB total, peers [0,1,2,6,7,8,9,10,11]/[])) 2013-04-26 02:39:20.472784 7f27d600e700 10 mon.1@0(leader).log v5604035 update_from_paxos 2013-04-26 02:39:20.472856 7f27d600e700 10 mon.1@0(leader).log v5604035 update_from_paxos version 5604035 summary v 5604035 2013-04-26 02:39:20.472924 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:20.502584 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from mds.0 10.227.227.104:6801/10842 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:20.502605 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x1788dc0 from 10.227.227.103:6789/0 2013-04-26 02:39:20.502721 7f27d600e700 10 mon.1@0(leader).mds e3724 preprocess_query mdsbeacon(217231/4 up:active seq 491 v3724) v2 from mds.0 10.227.227.104:6801/10842 2013-04-26 02:39:20.916051 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.8 10.227.227.103:6813/8628 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:20.916072 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x5177b80 from 10.227.227.103:6789/0 2013-04-26 02:39:20.916226 7f27d600e700 10 mon.1@0(leader).pg v5938153 preprocess_query pg_stats(9 pgs tid 56 v 10556) v1 from osd.8 10.227.227.103:6813/8628 2013-04-26 02:39:20.916330 7f27d600e700 10 mon.1@0(leader).pg v5938153 prepare_update pg_stats(9 pgs tid 56 v 10556) v1 from osd.8 10.227.227.103:6813/8628 2013-04-26 02:39:20.916344 7f27d600e700 10 mon.1@0(leader).pg v5938153 prepare_pg_stats pg_stats(9 pgs tid 56 v 10556) v1 from osd.8 2013-04-26 02:39:20.916360 7f27d600e700 10 mon.1@0(leader).pg v5938153 got osd.8 osd_stat(185 GB used, 744 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[]) (was osd_stat(185 GB used, 744 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[])) 2013-04-26 02:39:21.216149 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from mds.? 10.227.227.103:6800/8011 via mon.2 10.227.227.104:6789/0 2013-04-26 02:39:21.216173 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x1789080 from 10.227.227.104:6789/0 2013-04-26 02:39:21.216336 7f27d600e700 10 mon.1@0(leader).mds e3724 preprocess_query mdsbeacon(217445/3 up:standby seq 73 v3724) v2 from mds.? 10.227.227.103:6800/8011 2013-04-26 02:39:21.359014 7f27d680f700 10 mon.1@0(leader).pg v5938153 encode_pending v 5938154 2013-04-26 02:39:21.468208 7f27d680f700 10 mon.1@0(leader).log v5604035 encode_full log v 5604035 2013-04-26 02:39:21.468397 7f27d680f700 10 mon.1@0(leader).log v5604035 encode_pending v5604036 2013-04-26 02:39:21.530970 7f27d600e700 7 mon.1@0(leader).pg v5938153 update_from_paxos applying incremental 5938154 2013-04-26 02:39:21.531323 7f27d600e700 10 mon.1@0(leader).pg v5938154 v5938154: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 241KB/s wr, 30op/s 2013-04-26 02:39:21.531396 7f27d600e700 10 mon.1@0(leader).pg v5938154 send_pg_creates to 0 pgs 2013-04-26 02:39:21.531401 7f27d600e700 10 mon.1@0(leader).pg v5938154 update_logger 2013-04-26 02:39:21.531455 7f27d600e700 10 mon.1@0(leader).pg v5938154 create_pending v 5938155 2013-04-26 02:39:21.531488 7f27d600e700 7 mon.1@0(leader).pg v5938154 _updated_stats for osd.4 10.227.227.104:6804/14471 2013-04-26 02:39:21.531528 7f27d600e700 7 mon.1@0(leader).pg v5938154 _updated_stats for osd.3 10.227.227.104:6800/14463 2013-04-26 02:39:21.531555 7f27d600e700 7 mon.1@0(leader).pg v5938154 _updated_stats for osd.8 10.227.227.103:6813/8628 2013-04-26 02:39:21.531590 7f27d600e700 10 mon.1@0(leader).pg v5938154 check_osd_map already seen 10556 >= 10556 2013-04-26 02:39:21.531594 7f27d600e700 10 mon.1@0(leader).pg v5938154 update_logger 2013-04-26 02:39:21.531616 7f27d600e700 0 log [INF] : pgmap v5938154: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 241KB/s wr, 30op/s 2013-04-26 02:39:21.639245 7f27d600e700 10 mon.1@0(leader).log v5604036 update_from_paxos 2013-04-26 02:39:21.639370 7f27d600e700 10 mon.1@0(leader).log v5604036 update_from_paxos version 5604036 summary v 5604035 2013-04-26 02:39:21.639445 7f27d600e700 10 mon.1@0(leader).log v5604036 update_from_paxos latest full 5604035 2013-04-26 02:39:21.639528 7f27d600e700 7 mon.1@0(leader).log v5604036 update_from_paxos applying incremental log 5604036 2013-04-26 02:39:20.424272 mon.0 10.227.227.101:6789/0 21 : [INF] pgmap v5938153: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 202KB/s wr, 19op/s 2013-04-26 02:39:21.639655 7f27d600e700 10 mon.1@0(leader).log v5604036 check_subs 2013-04-26 02:39:21.639760 7f27d600e700 10 mon.1@0(leader).log v5604036 create_pending v 5604037 2013-04-26 02:39:21.639859 7f27d600e700 7 mon.1@0(leader).log v5604036 _updated_log for mon.0 10.227.227.101:6789/0 2013-04-26 02:39:21.640015 7f27d600e700 10 mon.1@0(leader).log v5604036 update_from_paxos 2013-04-26 02:39:21.640083 7f27d600e700 10 mon.1@0(leader).log v5604036 update_from_paxos version 5604036 summary v 5604036 2013-04-26 02:39:21.640117 7f27d600e700 10 mon.1@0(leader).log v5604036 preprocess_query log(1 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:21.640159 7f27d600e700 10 mon.1@0(leader).log v5604036 preprocess_log log(1 entries) v1 from mon.0 2013-04-26 02:39:21.640206 7f27d600e700 10 mon.1@0(leader).log v5604036 nothing new 2013-04-26 02:39:21.640422 7f27d600e700 10 mon.1@0(leader).log v5604036 update_from_paxos 2013-04-26 02:39:21.640491 7f27d600e700 10 mon.1@0(leader).log v5604036 update_from_paxos version 5604036 summary v 5604036 2013-04-26 02:39:21.640563 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:21.640753 7f27d600e700 10 mon.1@0(leader).log v5604036 update_from_paxos 2013-04-26 02:39:21.640822 7f27d600e700 10 mon.1@0(leader).log v5604036 update_from_paxos version 5604036 summary v 5604036 2013-04-26 02:39:21.640905 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:21.706711 7f27d600e700 10 mon.1@0(leader).log v5604036 update_from_paxos 2013-04-26 02:39:21.706789 7f27d600e700 10 mon.1@0(leader).log v5604036 update_from_paxos version 5604036 summary v 5604036 2013-04-26 02:39:21.706855 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:22.756550 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.5 10.227.227.104:6807/14479 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:22.756569 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x5176c80 from 10.227.227.103:6789/0 2013-04-26 02:39:22.756720 7f27d600e700 10 mon.1@0(leader).pg v5938154 preprocess_query pg_stats(14 pgs tid 745 v 10556) v1 from osd.5 10.227.227.104:6807/14479 2013-04-26 02:39:22.756818 7f27d600e700 10 mon.1@0(leader).pg v5938154 prepare_update pg_stats(14 pgs tid 745 v 10556) v1 from osd.5 10.227.227.104:6807/14479 2013-04-26 02:39:22.756833 7f27d600e700 10 mon.1@0(leader).pg v5938154 prepare_pg_stats pg_stats(14 pgs tid 745 v 10556) v1 from osd.5 2013-04-26 02:39:22.756849 7f27d600e700 10 mon.1@0(leader).pg v5938154 got osd.5 osd_stat(355 GB used, 1505 GB avail, 1860 GB total, peers [0,1,2,6,7,8,9,10,11]/[]) (was osd_stat(355 GB used, 1505 GB avail, 1860 GB total, peers [0,1,2,6,7,8,9,10,11]/[])) 2013-04-26 02:39:22.807271 7f27d680f700 10 mon.1@0(leader).pg v5938154 encode_pending v 5938155 2013-04-26 02:39:22.998240 7f27d600e700 7 mon.1@0(leader).pg v5938154 update_from_paxos applying incremental 5938155 2013-04-26 02:39:22.998356 7f27d600e700 10 mon.1@0(leader).pg v5938155 v5938155: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 191KB/s wr, 15op/s 2013-04-26 02:39:22.998423 7f27d600e700 10 mon.1@0(leader).pg v5938155 send_pg_creates to 0 pgs 2013-04-26 02:39:22.998429 7f27d600e700 10 mon.1@0(leader).pg v5938155 update_logger 2013-04-26 02:39:22.998475 7f27d600e700 10 mon.1@0(leader).pg v5938155 create_pending v 5938156 2013-04-26 02:39:22.998509 7f27d600e700 7 mon.1@0(leader).pg v5938155 _updated_stats for osd.5 10.227.227.104:6807/14479 2013-04-26 02:39:22.998640 7f27d600e700 10 mon.1@0(leader).pg v5938155 check_osd_map already seen 10556 >= 10556 2013-04-26 02:39:22.998652 7f27d600e700 10 mon.1@0(leader).pg v5938155 update_logger 2013-04-26 02:39:22.998676 7f27d600e700 0 log [INF] : pgmap v5938155: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 191KB/s wr, 15op/s 2013-04-26 02:39:22.998940 7f27d600e700 10 mon.1@0(leader).log v5604036 update_from_paxos 2013-04-26 02:39:22.999022 7f27d600e700 10 mon.1@0(leader).log v5604036 update_from_paxos version 5604036 summary v 5604036 2013-04-26 02:39:22.999093 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:22.999282 7f27d600e700 10 mon.1@0(leader).log v5604036 update_from_paxos 2013-04-26 02:39:22.999351 7f27d600e700 10 mon.1@0(leader).log v5604036 update_from_paxos version 5604036 summary v 5604036 2013-04-26 02:39:22.999420 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:22.999565 7f27d600e700 10 mon.1@0(leader).log v5604036 update_from_paxos 2013-04-26 02:39:22.999700 7f27d600e700 10 mon.1@0(leader).log v5604036 update_from_paxos version 5604036 summary v 5604036 2013-04-26 02:39:22.999742 7f27d600e700 10 mon.1@0(leader).log v5604036 preprocess_query log(2 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:22.999788 7f27d600e700 10 mon.1@0(leader).log v5604036 preprocess_log log(2 entries) v1 from mon.0 2013-04-26 02:39:22.999845 7f27d600e700 10 mon.1@0(leader).log v5604036 prepare_update log(2 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:22.999891 7f27d600e700 10 mon.1@0(leader).log v5604036 prepare_log log(2 entries) v1 from mon.0 2013-04-26 02:39:22.999933 7f27d600e700 10 mon.1@0(leader).log v5604036 logging 2013-04-26 02:39:21.531618 mon.0 10.227.227.101:6789/0 22 : [INF] pgmap v5938154: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 241KB/s wr, 30op/s 2013-04-26 02:39:22.999991 7f27d600e700 10 mon.1@0(leader).log v5604036 logging 2013-04-26 02:39:22.998679 mon.0 10.227.227.101:6789/0 23 : [INF] pgmap v5938155: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 191KB/s wr, 15op/s 2013-04-26 02:39:23.048067 7f27d600e700 10 mon.1@0(leader).log v5604036 update_from_paxos 2013-04-26 02:39:23.048149 7f27d600e700 10 mon.1@0(leader).log v5604036 update_from_paxos version 5604036 summary v 5604036 2013-04-26 02:39:23.048217 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:23.893765 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.11 10.227.227.103:6804/8253 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:23.893791 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x5177b80 from 10.227.227.103:6789/0 2013-04-26 02:39:23.893954 7f27d600e700 10 mon.1@0(leader).pg v5938155 preprocess_query pg_stats(9 pgs tid 59 v 10556) v1 from osd.11 10.227.227.103:6804/8253 2013-04-26 02:39:23.894057 7f27d600e700 10 mon.1@0(leader).pg v5938155 prepare_update pg_stats(9 pgs tid 59 v 10556) v1 from osd.11 10.227.227.103:6804/8253 2013-04-26 02:39:23.894074 7f27d600e700 10 mon.1@0(leader).pg v5938155 prepare_pg_stats pg_stats(9 pgs tid 59 v 10556) v1 from osd.11 2013-04-26 02:39:23.894089 7f27d600e700 10 mon.1@0(leader).pg v5938155 got osd.11 osd_stat(161 GB used, 768 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[]) (was osd_stat(161 GB used, 768 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[])) 2013-04-26 02:39:23.939066 7f27d680f700 10 mon.1@0(leader).log v5604036 encode_full log v 5604036 2013-04-26 02:39:23.939357 7f27d680f700 10 mon.1@0(leader).log v5604036 encode_pending v5604037 2013-04-26 02:39:24.011344 7f27d680f700 10 mon.1@0(leader).pg v5938155 encode_pending v 5938156 2013-04-26 02:39:24.061381 7f27d600e700 10 mon.1@0(leader).log v5604037 update_from_paxos 2013-04-26 02:39:24.061578 7f27d600e700 10 mon.1@0(leader).log v5604037 update_from_paxos version 5604037 summary v 5604036 2013-04-26 02:39:24.061656 7f27d600e700 10 mon.1@0(leader).log v5604037 update_from_paxos latest full 5604036 2013-04-26 02:39:24.061733 7f27d600e700 7 mon.1@0(leader).log v5604037 update_from_paxos applying incremental log 5604037 2013-04-26 02:39:21.531618 mon.0 10.227.227.101:6789/0 22 : [INF] pgmap v5938154: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 241KB/s wr, 30op/s 2013-04-26 02:39:24.061831 7f27d600e700 7 mon.1@0(leader).log v5604037 update_from_paxos applying incremental log 5604037 2013-04-26 02:39:22.998679 mon.0 10.227.227.101:6789/0 23 : [INF] pgmap v5938155: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 191KB/s wr, 15op/s 2013-04-26 02:39:24.061960 7f27d600e700 10 mon.1@0(leader).log v5604037 check_subs 2013-04-26 02:39:24.062058 7f27d600e700 10 mon.1@0(leader).log v5604037 create_pending v 5604038 2013-04-26 02:39:24.062157 7f27d600e700 7 mon.1@0(leader).log v5604037 _updated_log for mon.0 10.227.227.101:6789/0 2013-04-26 02:39:24.119910 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from mds.? 10.227.227.101:6800/29852 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:24.119928 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x1789340 from 10.227.227.103:6789/0 2013-04-26 02:39:24.120049 7f27d600e700 10 mon.1@0(leader).mds e3724 preprocess_query mdsbeacon(217349/1 up:standby seq 461 v3724) v2 from mds.? 10.227.227.101:6800/29852 2013-04-26 02:39:24.198347 7f27d600e700 7 mon.1@0(leader).pg v5938155 update_from_paxos applying incremental 5938156 2013-04-26 02:39:24.198572 7f27d600e700 10 mon.1@0(leader).pg v5938156 v5938156: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 149KB/s wr, 6op/s 2013-04-26 02:39:24.198653 7f27d600e700 10 mon.1@0(leader).pg v5938156 send_pg_creates to 0 pgs 2013-04-26 02:39:24.198659 7f27d600e700 10 mon.1@0(leader).pg v5938156 update_logger 2013-04-26 02:39:24.198702 7f27d600e700 10 mon.1@0(leader).pg v5938156 create_pending v 5938157 2013-04-26 02:39:24.198734 7f27d600e700 7 mon.1@0(leader).pg v5938156 _updated_stats for osd.11 10.227.227.103:6804/8253 2013-04-26 02:39:24.198803 7f27d600e700 10 mon.1@0(leader).pg v5938156 check_osd_map already seen 10556 >= 10556 2013-04-26 02:39:24.198812 7f27d600e700 10 mon.1@0(leader).pg v5938156 update_logger 2013-04-26 02:39:24.198835 7f27d600e700 0 log [INF] : pgmap v5938156: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 149KB/s wr, 6op/s 2013-04-26 02:39:24.199069 7f27d600e700 10 mon.1@0(leader).log v5604037 update_from_paxos 2013-04-26 02:39:24.199146 7f27d600e700 10 mon.1@0(leader).log v5604037 update_from_paxos version 5604037 summary v 5604037 2013-04-26 02:39:24.199215 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:24.199409 7f27d600e700 10 mon.1@0(leader).log v5604037 update_from_paxos 2013-04-26 02:39:24.199543 7f27d600e700 10 mon.1@0(leader).log v5604037 update_from_paxos version 5604037 summary v 5604037 2013-04-26 02:39:24.199619 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:24.199763 7f27d600e700 10 mon.1@0(leader).log v5604037 update_from_paxos 2013-04-26 02:39:24.199830 7f27d600e700 10 mon.1@0(leader).log v5604037 update_from_paxos version 5604037 summary v 5604037 2013-04-26 02:39:24.199865 7f27d600e700 10 mon.1@0(leader).log v5604037 preprocess_query log(1 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:24.199905 7f27d600e700 10 mon.1@0(leader).log v5604037 preprocess_log log(1 entries) v1 from mon.0 2013-04-26 02:39:24.199955 7f27d600e700 10 mon.1@0(leader).log v5604037 prepare_update log(1 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:24.200000 7f27d600e700 10 mon.1@0(leader).log v5604037 prepare_log log(1 entries) v1 from mon.0 2013-04-26 02:39:24.200039 7f27d600e700 10 mon.1@0(leader).log v5604037 logging 2013-04-26 02:39:24.198838 mon.0 10.227.227.101:6789/0 24 : [INF] pgmap v5938156: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 149KB/s wr, 6op/s 2013-04-26 02:39:24.256545 7f27d600e700 10 mon.1@0(leader).log v5604037 update_from_paxos 2013-04-26 02:39:24.256684 7f27d600e700 10 mon.1@0(leader).log v5604037 update_from_paxos version 5604037 summary v 5604037 2013-04-26 02:39:24.256762 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:24.276094 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.7 10.227.227.103:6810/8502 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:24.276116 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x5176280 from 10.227.227.103:6789/0 2013-04-26 02:39:24.276263 7f27d600e700 10 mon.1@0(leader).pg v5938156 preprocess_query pg_stats(5 pgs tid 56 v 10556) v1 from osd.7 10.227.227.103:6810/8502 2013-04-26 02:39:24.276361 7f27d600e700 10 mon.1@0(leader).pg v5938156 prepare_update pg_stats(5 pgs tid 56 v 10556) v1 from osd.7 10.227.227.103:6810/8502 2013-04-26 02:39:24.276375 7f27d600e700 10 mon.1@0(leader).pg v5938156 prepare_pg_stats pg_stats(5 pgs tid 56 v 10556) v1 from osd.7 2013-04-26 02:39:24.276392 7f27d600e700 10 mon.1@0(leader).pg v5938156 got osd.7 osd_stat(156 GB used, 772 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[]) (was osd_stat(156 GB used, 772 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[])) 2013-04-26 02:39:24.301752 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.9 10.227.227.103:6816/8752 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:24.301770 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x1799680 from 10.227.227.103:6789/0 2013-04-26 02:39:24.301881 7f27d600e700 10 mon.1@0(leader).pg v5938156 preprocess_query pg_stats(5 pgs tid 59 v 10556) v1 from osd.9 10.227.227.103:6816/8752 2013-04-26 02:39:24.301967 7f27d600e700 10 mon.1@0(leader).pg v5938156 prepare_update pg_stats(5 pgs tid 59 v 10556) v1 from osd.9 10.227.227.103:6816/8752 2013-04-26 02:39:24.301980 7f27d600e700 10 mon.1@0(leader).pg v5938156 prepare_pg_stats pg_stats(5 pgs tid 59 v 10556) v1 from osd.9 2013-04-26 02:39:24.302014 7f27d600e700 10 mon.1@0(leader).pg v5938156 got osd.9 osd_stat(161 GB used, 767 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[]) (was osd_stat(161 GB used, 767 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[])) 2013-04-26 02:39:24.414112 7f27d680f700 10 mon.1@0(leader).pg v5938156 check_down_pgs 2013-04-26 02:39:24.414375 7f27d680f700 10 mon.1@0(leader).pg v5938156 v5938156: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 149KB/s wr, 6op/s 2013-04-26 02:39:24.414458 7f27d680f700 10 mon.1@0(leader).mds e3724 e3724: 1/1/1 up {0=4=up:active}, 2 up:standby 2013-04-26 02:39:24.414520 7f27d680f700 10 mon.1@0(leader).osd e10556 e10556: 12 osds: 12 up, 12 in 2013-04-26 02:39:24.414784 7f27d680f700 10 mon.1@0(leader).osd e10556 min_last_epoch_clean 10531 2013-04-26 02:39:24.414924 7f27d680f700 10 mon.1@0(leader).log v5604037 update_from_paxos 2013-04-26 02:39:24.415017 7f27d680f700 10 mon.1@0(leader).log v5604037 update_from_paxos version 5604037 summary v 5604037 2013-04-26 02:39:24.415054 7f27d680f700 10 mon.1@0(leader).log v5604037 log 2013-04-26 02:39:24.415130 7f27d680f700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:24.415194 7f27d680f700 10 mon.1@0(leader).auth v4370 auth 2013-04-26 02:39:24.502783 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from mds.0 10.227.227.104:6801/10842 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:24.502801 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x1789080 from 10.227.227.103:6789/0 2013-04-26 02:39:24.502954 7f27d600e700 10 mon.1@0(leader).mds e3724 preprocess_query mdsbeacon(217231/4 up:active seq 492 v3724) v2 from mds.0 10.227.227.104:6801/10842 2013-04-26 02:39:24.534308 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.10 10.227.227.103:6801/8155 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:24.534327 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x2511900 from 10.227.227.103:6789/0 2013-04-26 02:39:24.534455 7f27d600e700 10 mon.1@0(leader).pg v5938156 preprocess_query pg_stats(6 pgs tid 54 v 10556) v1 from osd.10 10.227.227.103:6801/8155 2013-04-26 02:39:24.534553 7f27d600e700 10 mon.1@0(leader).pg v5938156 prepare_update pg_stats(6 pgs tid 54 v 10556) v1 from osd.10 10.227.227.103:6801/8155 2013-04-26 02:39:24.534567 7f27d600e700 10 mon.1@0(leader).pg v5938156 prepare_pg_stats pg_stats(6 pgs tid 54 v 10556) v1 from osd.10 2013-04-26 02:39:24.534581 7f27d600e700 10 mon.1@0(leader).pg v5938156 got osd.10 osd_stat(174 GB used, 755 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[]) (was osd_stat(174 GB used, 755 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[])) 2013-04-26 02:39:24.547201 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.2 10.227.227.101:6807/30207 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:24.547222 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x5177900 from 10.227.227.103:6789/0 2013-04-26 02:39:24.547334 7f27d600e700 10 mon.1@0(leader).pg v5938156 preprocess_query pg_stats(7 pgs tid 738 v 10556) v1 from osd.2 10.227.227.101:6807/30207 2013-04-26 02:39:24.547414 7f27d600e700 10 mon.1@0(leader).pg v5938156 prepare_update pg_stats(7 pgs tid 738 v 10556) v1 from osd.2 10.227.227.101:6807/30207 2013-04-26 02:39:24.547436 7f27d600e700 10 mon.1@0(leader).pg v5938156 prepare_pg_stats pg_stats(7 pgs tid 738 v 10556) v1 from osd.2 2013-04-26 02:39:24.547447 7f27d600e700 10 mon.1@0(leader).pg v5938156 got osd.2 osd_stat(311 GB used, 1549 GB avail, 1860 GB total, peers [3,4,5,6,7,8,9,10,11]/[]) (was osd_stat(311 GB used, 1549 GB avail, 1860 GB total, peers [3,4,5,6,7,8,9,10,11]/[])) 2013-04-26 02:39:24.644104 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.1 10.227.227.101:6804/30088 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:24.644124 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x5177680 from 10.227.227.103:6789/0 2013-04-26 02:39:24.644247 7f27d600e700 10 mon.1@0(leader).pg v5938156 preprocess_query pg_stats(10 pgs tid 732 v 10556) v1 from osd.1 10.227.227.101:6804/30088 2013-04-26 02:39:24.644344 7f27d600e700 10 mon.1@0(leader).pg v5938156 prepare_update pg_stats(10 pgs tid 732 v 10556) v1 from osd.1 10.227.227.101:6804/30088 2013-04-26 02:39:24.644356 7f27d600e700 10 mon.1@0(leader).pg v5938156 prepare_pg_stats pg_stats(10 pgs tid 732 v 10556) v1 from osd.1 2013-04-26 02:39:24.644368 7f27d600e700 10 mon.1@0(leader).pg v5938156 got osd.1 osd_stat(342 GB used, 1517 GB avail, 1860 GB total, peers [3,4,5,6,7,8,9,10,11]/[]) (was osd_stat(342 GB used, 1517 GB avail, 1860 GB total, peers [3,4,5,6,7,8,9,10,11]/[])) 2013-04-26 02:39:24.687099 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.0 10.227.227.101:6801/29982 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:24.687116 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x5177400 from 10.227.227.103:6789/0 2013-04-26 02:39:24.687241 7f27d600e700 10 mon.1@0(leader).pg v5938156 preprocess_query pg_stats(12 pgs tid 743 v 10556) v1 from osd.0 10.227.227.101:6801/29982 2013-04-26 02:39:24.687348 7f27d600e700 10 mon.1@0(leader).pg v5938156 prepare_update pg_stats(12 pgs tid 743 v 10556) v1 from osd.0 10.227.227.101:6801/29982 2013-04-26 02:39:24.687364 7f27d600e700 10 mon.1@0(leader).pg v5938156 prepare_pg_stats pg_stats(12 pgs tid 743 v 10556) v1 from osd.0 2013-04-26 02:39:24.687376 7f27d600e700 10 mon.1@0(leader).pg v5938156 got osd.0 osd_stat(367 GB used, 1493 GB avail, 1860 GB total, peers [3,4,5,6,7,8,9,10,11]/[]) (was osd_stat(367 GB used, 1493 GB avail, 1860 GB total, peers [3,4,5,6,7,8,9,10,11]/[])) 2013-04-26 02:39:24.892041 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.6 10.227.227.103:6807/8402 via mon.2 10.227.227.104:6789/0 2013-04-26 02:39:24.892062 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x1798f00 from 10.227.227.104:6789/0 2013-04-26 02:39:24.892182 7f27d600e700 10 mon.1@0(leader).pg v5938156 preprocess_query pg_stats(9 pgs tid 59 v 10556) v1 from osd.6 10.227.227.103:6807/8402 2013-04-26 02:39:24.892269 7f27d600e700 10 mon.1@0(leader).pg v5938156 prepare_update pg_stats(9 pgs tid 59 v 10556) v1 from osd.6 10.227.227.103:6807/8402 2013-04-26 02:39:24.892284 7f27d600e700 10 mon.1@0(leader).pg v5938156 prepare_pg_stats pg_stats(9 pgs tid 59 v 10556) v1 from osd.6 2013-04-26 02:39:24.892296 7f27d600e700 10 mon.1@0(leader).pg v5938156 got osd.6 osd_stat(184 GB used, 745 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[]) (was osd_stat(184 GB used, 745 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[])) 2013-04-26 02:39:25.129565 7f27d680f700 10 mon.1@0(leader).pg v5938156 encode_pending v 5938157 2013-04-26 02:39:25.220329 7f27d680f700 10 mon.1@0(leader).log v5604037 encode_full log v 5604037 2013-04-26 02:39:25.220508 7f27d680f700 10 mon.1@0(leader).log v5604037 encode_pending v5604038 2013-04-26 02:39:25.220633 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from mds.? 10.227.227.103:6800/8011 via mon.2 10.227.227.104:6789/0 2013-04-26 02:39:25.220656 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x1788dc0 from 10.227.227.104:6789/0 2013-04-26 02:39:25.220769 7f27d600e700 10 mon.1@0(leader).mds e3724 preprocess_query mdsbeacon(217445/3 up:standby seq 74 v3724) v2 from mds.? 10.227.227.103:6800/8011 2013-04-26 02:39:25.281218 7f27d600e700 7 mon.1@0(leader).pg v5938156 update_from_paxos applying incremental 5938157 2013-04-26 02:39:25.281568 7f27d600e700 10 mon.1@0(leader).pg v5938157 v5938157: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 217KB/s wr, 12op/s 2013-04-26 02:39:25.281643 7f27d600e700 10 mon.1@0(leader).pg v5938157 send_pg_creates to 0 pgs 2013-04-26 02:39:25.281652 7f27d600e700 10 mon.1@0(leader).pg v5938157 update_logger 2013-04-26 02:39:25.281724 7f27d600e700 10 mon.1@0(leader).pg v5938157 create_pending v 5938158 2013-04-26 02:39:25.281833 7f27d600e700 7 mon.1@0(leader).pg v5938157 _updated_stats for osd.7 10.227.227.103:6810/8502 2013-04-26 02:39:25.281881 7f27d600e700 7 mon.1@0(leader).pg v5938157 _updated_stats for osd.9 10.227.227.103:6816/8752 2013-04-26 02:39:25.281959 7f27d600e700 7 mon.1@0(leader).pg v5938157 _updated_stats for osd.10 10.227.227.103:6801/8155 2013-04-26 02:39:25.281995 7f27d600e700 7 mon.1@0(leader).pg v5938157 _updated_stats for osd.2 10.227.227.101:6807/30207 2013-04-26 02:39:25.282059 7f27d600e700 7 mon.1@0(leader).pg v5938157 _updated_stats for osd.1 10.227.227.101:6804/30088 2013-04-26 02:39:25.282099 7f27d600e700 7 mon.1@0(leader).pg v5938157 _updated_stats for osd.0 10.227.227.101:6801/29982 2013-04-26 02:39:25.282168 7f27d600e700 7 mon.1@0(leader).pg v5938157 _updated_stats for osd.6 10.227.227.103:6807/8402 2013-04-26 02:39:25.282207 7f27d600e700 10 mon.1@0(leader).pg v5938157 check_osd_map already seen 10556 >= 10556 2013-04-26 02:39:25.282213 7f27d600e700 10 mon.1@0(leader).pg v5938157 update_logger 2013-04-26 02:39:25.282243 7f27d600e700 0 log [INF] : pgmap v5938157: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 217KB/s wr, 12op/s 2013-04-26 02:39:25.398030 7f27d600e700 10 mon.1@0(leader).log v5604038 update_from_paxos 2013-04-26 02:39:25.398188 7f27d600e700 10 mon.1@0(leader).log v5604038 update_from_paxos version 5604038 summary v 5604037 2013-04-26 02:39:25.398263 7f27d600e700 10 mon.1@0(leader).log v5604038 update_from_paxos latest full 5604037 2013-04-26 02:39:25.398347 7f27d600e700 7 mon.1@0(leader).log v5604038 update_from_paxos applying incremental log 5604038 2013-04-26 02:39:24.198838 mon.0 10.227.227.101:6789/0 24 : [INF] pgmap v5938156: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 149KB/s wr, 6op/s 2013-04-26 02:39:25.398534 7f27d600e700 10 mon.1@0(leader).log v5604038 check_subs 2013-04-26 02:39:25.398635 7f27d600e700 10 mon.1@0(leader).log v5604038 create_pending v 5604039 2013-04-26 02:39:25.398738 7f27d600e700 7 mon.1@0(leader).log v5604038 _updated_log for mon.0 10.227.227.101:6789/0 2013-04-26 02:39:25.398868 7f27d600e700 10 mon.1@0(leader).log v5604038 update_from_paxos 2013-04-26 02:39:25.398934 7f27d600e700 10 mon.1@0(leader).log v5604038 update_from_paxos version 5604038 summary v 5604038 2013-04-26 02:39:25.398967 7f27d600e700 10 mon.1@0(leader).log v5604038 preprocess_query log(1 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:25.399023 7f27d600e700 10 mon.1@0(leader).log v5604038 preprocess_log log(1 entries) v1 from mon.0 2013-04-26 02:39:25.399067 7f27d600e700 10 mon.1@0(leader).log v5604038 nothing new 2013-04-26 02:39:25.399329 7f27d600e700 10 mon.1@0(leader).log v5604038 update_from_paxos 2013-04-26 02:39:25.399401 7f27d600e700 10 mon.1@0(leader).log v5604038 update_from_paxos version 5604038 summary v 5604038 2013-04-26 02:39:25.399474 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:25.399799 7f27d600e700 10 mon.1@0(leader).log v5604038 update_from_paxos 2013-04-26 02:39:25.399874 7f27d600e700 10 mon.1@0(leader).log v5604038 update_from_paxos version 5604038 summary v 5604038 2013-04-26 02:39:25.399953 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:25.400082 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.4 10.227.227.104:6804/14471 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:25.400099 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x4bcb400 from 10.227.227.103:6789/0 2013-04-26 02:39:25.400194 7f27d600e700 10 mon.1@0(leader).pg v5938157 preprocess_query pg_stats(13 pgs tid 748 v 10556) v1 from osd.4 10.227.227.104:6804/14471 2013-04-26 02:39:25.400274 7f27d600e700 10 mon.1@0(leader).pg v5938157 prepare_update pg_stats(13 pgs tid 748 v 10556) v1 from osd.4 10.227.227.104:6804/14471 2013-04-26 02:39:25.400288 7f27d600e700 10 mon.1@0(leader).pg v5938157 prepare_pg_stats pg_stats(13 pgs tid 748 v 10556) v1 from osd.4 2013-04-26 02:39:25.400303 7f27d600e700 10 mon.1@0(leader).pg v5938157 got osd.4 osd_stat(423 GB used, 1437 GB avail, 1860 GB total, peers [0,1,2,6,7,8,9,10,11]/[]) (was osd_stat(423 GB used, 1437 GB avail, 1860 GB total, peers [0,1,2,6,7,8,9,10,11]/[])) 2013-04-26 02:39:25.457128 7f27d600e700 10 mon.1@0(leader).log v5604038 update_from_paxos 2013-04-26 02:39:25.457211 7f27d600e700 10 mon.1@0(leader).log v5604038 update_from_paxos version 5604038 summary v 5604038 2013-04-26 02:39:25.457280 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:25.470213 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.3 10.227.227.104:6800/14463 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:25.470232 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x5176c80 from 10.227.227.103:6789/0 2013-04-26 02:39:25.470392 7f27d600e700 10 mon.1@0(leader).pg v5938157 preprocess_query pg_stats(9 pgs tid 753 v 10556) v1 from osd.3 10.227.227.104:6800/14463 2013-04-26 02:39:25.470496 7f27d600e700 10 mon.1@0(leader).pg v5938157 prepare_update pg_stats(9 pgs tid 753 v 10556) v1 from osd.3 10.227.227.104:6800/14463 2013-04-26 02:39:25.470511 7f27d600e700 10 mon.1@0(leader).pg v5938157 prepare_pg_stats pg_stats(9 pgs tid 753 v 10556) v1 from osd.3 2013-04-26 02:39:25.470528 7f27d600e700 10 mon.1@0(leader).pg v5938157 got osd.3 osd_stat(383 GB used, 1477 GB avail, 1860 GB total, peers [0,1,2,6,7,8,9,10,11]/[]) (was osd_stat(383 GB used, 1477 GB avail, 1860 GB total, peers [0,1,2,6,7,8,9,10,11]/[])) 2013-04-26 02:39:25.916658 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.8 10.227.227.103:6813/8628 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:25.916680 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x4bcbb80 from 10.227.227.103:6789/0 2013-04-26 02:39:25.916841 7f27d600e700 10 mon.1@0(leader).pg v5938157 preprocess_query pg_stats(3 pgs tid 57 v 10556) v1 from osd.8 10.227.227.103:6813/8628 2013-04-26 02:39:25.916943 7f27d600e700 10 mon.1@0(leader).pg v5938157 prepare_update pg_stats(3 pgs tid 57 v 10556) v1 from osd.8 10.227.227.103:6813/8628 2013-04-26 02:39:25.916958 7f27d600e700 10 mon.1@0(leader).pg v5938157 prepare_pg_stats pg_stats(3 pgs tid 57 v 10556) v1 from osd.8 2013-04-26 02:39:25.916975 7f27d600e700 10 mon.1@0(leader).pg v5938157 got osd.8 osd_stat(185 GB used, 744 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[]) (was osd_stat(185 GB used, 744 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[])) 2013-04-26 02:39:26.330176 7f27d680f700 10 mon.1@0(leader).pg v5938157 encode_pending v 5938158 2013-04-26 02:39:26.465327 7f27d600e700 7 mon.1@0(leader).pg v5938157 update_from_paxos applying incremental 5938158 2013-04-26 02:39:26.465521 7f27d600e700 10 mon.1@0(leader).pg v5938158 v5938158: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 223KB/s wr, 13op/s 2013-04-26 02:39:26.465593 7f27d600e700 10 mon.1@0(leader).pg v5938158 send_pg_creates to 0 pgs 2013-04-26 02:39:26.465599 7f27d600e700 10 mon.1@0(leader).pg v5938158 update_logger 2013-04-26 02:39:26.465647 7f27d600e700 10 mon.1@0(leader).pg v5938158 create_pending v 5938159 2013-04-26 02:39:26.465680 7f27d600e700 7 mon.1@0(leader).pg v5938158 _updated_stats for osd.4 10.227.227.104:6804/14471 2013-04-26 02:39:26.465724 7f27d600e700 7 mon.1@0(leader).pg v5938158 _updated_stats for osd.3 10.227.227.104:6800/14463 2013-04-26 02:39:26.465784 7f27d600e700 7 mon.1@0(leader).pg v5938158 _updated_stats for osd.8 10.227.227.103:6813/8628 2013-04-26 02:39:26.465856 7f27d600e700 10 mon.1@0(leader).pg v5938158 check_osd_map already seen 10556 >= 10556 2013-04-26 02:39:26.465898 7f27d600e700 10 mon.1@0(leader).pg v5938158 update_logger 2013-04-26 02:39:26.465935 7f27d600e700 0 log [INF] : pgmap v5938158: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 223KB/s wr, 13op/s 2013-04-26 02:39:26.466207 7f27d600e700 10 mon.1@0(leader).log v5604038 update_from_paxos 2013-04-26 02:39:26.466290 7f27d600e700 10 mon.1@0(leader).log v5604038 update_from_paxos version 5604038 summary v 5604038 2013-04-26 02:39:26.466478 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:26.466680 7f27d600e700 10 mon.1@0(leader).log v5604038 update_from_paxos 2013-04-26 02:39:26.466749 7f27d600e700 10 mon.1@0(leader).log v5604038 update_from_paxos version 5604038 summary v 5604038 2013-04-26 02:39:26.466817 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:26.466969 7f27d600e700 10 mon.1@0(leader).log v5604038 update_from_paxos 2013-04-26 02:39:26.467034 7f27d600e700 10 mon.1@0(leader).log v5604038 update_from_paxos version 5604038 summary v 5604038 2013-04-26 02:39:26.467070 7f27d600e700 10 mon.1@0(leader).log v5604038 preprocess_query log(2 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:26.467113 7f27d600e700 10 mon.1@0(leader).log v5604038 preprocess_log log(2 entries) v1 from mon.0 2013-04-26 02:39:26.467169 7f27d600e700 10 mon.1@0(leader).log v5604038 prepare_update log(2 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:26.467214 7f27d600e700 10 mon.1@0(leader).log v5604038 prepare_log log(2 entries) v1 from mon.0 2013-04-26 02:39:26.467255 7f27d600e700 10 mon.1@0(leader).log v5604038 logging 2013-04-26 02:39:25.282245 mon.0 10.227.227.101:6789/0 25 : [INF] pgmap v5938157: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 217KB/s wr, 12op/s 2013-04-26 02:39:26.467315 7f27d600e700 10 mon.1@0(leader).log v5604038 logging 2013-04-26 02:39:26.465938 mon.0 10.227.227.101:6789/0 26 : [INF] pgmap v5938158: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 223KB/s wr, 13op/s 2013-04-26 02:39:26.523562 7f27d600e700 10 mon.1@0(leader).log v5604038 update_from_paxos 2013-04-26 02:39:26.523644 7f27d600e700 10 mon.1@0(leader).log v5604038 update_from_paxos version 5604038 summary v 5604038 2013-04-26 02:39:26.523712 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:27.419024 7f27d680f700 10 mon.1@0(leader).log v5604038 encode_full log v 5604038 2013-04-26 02:39:27.419308 7f27d680f700 10 mon.1@0(leader).log v5604038 encode_pending v5604039 2013-04-26 02:39:27.556580 7f27d600e700 10 mon.1@0(leader).log v5604039 update_from_paxos 2013-04-26 02:39:27.556719 7f27d600e700 10 mon.1@0(leader).log v5604039 update_from_paxos version 5604039 summary v 5604038 2013-04-26 02:39:27.556792 7f27d600e700 10 mon.1@0(leader).log v5604039 update_from_paxos latest full 5604038 2013-04-26 02:39:27.556878 7f27d600e700 7 mon.1@0(leader).log v5604039 update_from_paxos applying incremental log 5604039 2013-04-26 02:39:25.282245 mon.0 10.227.227.101:6789/0 25 : [INF] pgmap v5938157: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 217KB/s wr, 12op/s 2013-04-26 02:39:27.556971 7f27d600e700 7 mon.1@0(leader).log v5604039 update_from_paxos applying incremental log 5604039 2013-04-26 02:39:26.465938 mon.0 10.227.227.101:6789/0 26 : [INF] pgmap v5938158: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 223KB/s wr, 13op/s 2013-04-26 02:39:27.557096 7f27d600e700 10 mon.1@0(leader).log v5604039 check_subs 2013-04-26 02:39:27.557205 7f27d600e700 10 mon.1@0(leader).log v5604039 create_pending v 5604040 2013-04-26 02:39:27.557304 7f27d600e700 7 mon.1@0(leader).log v5604039 _updated_log for mon.0 10.227.227.101:6789/0 2013-04-26 02:39:27.557660 7f27d600e700 10 mon.1@0(leader).log v5604039 update_from_paxos 2013-04-26 02:39:27.557735 7f27d600e700 10 mon.1@0(leader).log v5604039 update_from_paxos version 5604039 summary v 5604039 2013-04-26 02:39:27.557804 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:27.559006 7f27d600e700 10 mon.1@0(leader).log v5604039 update_from_paxos 2013-04-26 02:39:27.559083 7f27d600e700 10 mon.1@0(leader).log v5604039 update_from_paxos version 5604039 summary v 5604039 2013-04-26 02:39:27.559150 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:27.606754 7f27d600e700 10 mon.1@0(leader).log v5604039 update_from_paxos 2013-04-26 02:39:27.606832 7f27d600e700 10 mon.1@0(leader).log v5604039 update_from_paxos version 5604039 summary v 5604039 2013-04-26 02:39:27.606898 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:27.757204 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.5 10.227.227.104:6807/14479 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:27.757227 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x6ce5180 from 10.227.227.103:6789/0 2013-04-26 02:39:27.757386 7f27d600e700 10 mon.1@0(leader).pg v5938158 preprocess_query pg_stats(16 pgs tid 746 v 10556) v1 from osd.5 10.227.227.104:6807/14479 2013-04-26 02:39:27.757484 7f27d600e700 10 mon.1@0(leader).pg v5938158 prepare_update pg_stats(16 pgs tid 746 v 10556) v1 from osd.5 10.227.227.104:6807/14479 2013-04-26 02:39:27.757506 7f27d600e700 10 mon.1@0(leader).pg v5938158 prepare_pg_stats pg_stats(16 pgs tid 746 v 10556) v1 from osd.5 2013-04-26 02:39:27.757525 7f27d600e700 10 mon.1@0(leader).pg v5938158 got osd.5 osd_stat(355 GB used, 1505 GB avail, 1860 GB total, peers [0,1,2,6,7,8,9,10,11]/[]) (was osd_stat(355 GB used, 1505 GB avail, 1860 GB total, peers [0,1,2,6,7,8,9,10,11]/[])) 2013-04-26 02:39:28.030700 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from mds.? 10.227.227.101:6800/29852 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:28.030725 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x1789340 from 10.227.227.103:6789/0 2013-04-26 02:39:28.030899 7f27d600e700 10 mon.1@0(leader).mds e3724 preprocess_query mdsbeacon(217349/1 up:standby seq 462 v3724) v2 from mds.? 10.227.227.101:6800/29852 2013-04-26 02:39:28.503077 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from mds.0 10.227.227.104:6801/10842 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:28.503101 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x1788dc0 from 10.227.227.103:6789/0 2013-04-26 02:39:28.503277 7f27d600e700 10 mon.1@0(leader).mds e3724 preprocess_query mdsbeacon(217231/4 up:active seq 493 v3724) v2 from mds.0 10.227.227.104:6801/10842 2013-04-26 02:39:28.509167 7f27d680f700 10 mon.1@0(leader).pg v5938158 encode_pending v 5938159 2013-04-26 02:39:28.623157 7f27d600e700 7 mon.1@0(leader).pg v5938158 update_from_paxos applying incremental 5938159 2013-04-26 02:39:28.623290 7f27d600e700 10 mon.1@0(leader).pg v5938159 v5938159: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 32723B/s wr, 4op/s 2013-04-26 02:39:28.623355 7f27d600e700 10 mon.1@0(leader).pg v5938159 send_pg_creates to 0 pgs 2013-04-26 02:39:28.623363 7f27d600e700 10 mon.1@0(leader).pg v5938159 update_logger 2013-04-26 02:39:28.623411 7f27d600e700 10 mon.1@0(leader).pg v5938159 create_pending v 5938160 2013-04-26 02:39:28.623448 7f27d600e700 7 mon.1@0(leader).pg v5938159 _updated_stats for osd.5 10.227.227.104:6807/14479 2013-04-26 02:39:28.623583 7f27d600e700 10 mon.1@0(leader).pg v5938159 check_osd_map already seen 10556 >= 10556 2013-04-26 02:39:28.623595 7f27d600e700 10 mon.1@0(leader).pg v5938159 update_logger 2013-04-26 02:39:28.623620 7f27d600e700 0 log [INF] : pgmap v5938159: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 32723B/s wr, 4op/s 2013-04-26 02:39:28.623899 7f27d600e700 10 mon.1@0(leader).log v5604039 update_from_paxos 2013-04-26 02:39:28.623982 7f27d600e700 10 mon.1@0(leader).log v5604039 update_from_paxos version 5604039 summary v 5604039 2013-04-26 02:39:28.624054 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:28.624210 7f27d600e700 10 mon.1@0(leader).log v5604039 update_from_paxos 2013-04-26 02:39:28.624275 7f27d600e700 10 mon.1@0(leader).log v5604039 update_from_paxos version 5604039 summary v 5604039 2013-04-26 02:39:28.624311 7f27d600e700 10 mon.1@0(leader).log v5604039 preprocess_query log(1 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:28.624356 7f27d600e700 10 mon.1@0(leader).log v5604039 preprocess_log log(1 entries) v1 from mon.0 2013-04-26 02:39:28.624409 7f27d600e700 10 mon.1@0(leader).log v5604039 prepare_update log(1 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:28.624454 7f27d600e700 10 mon.1@0(leader).log v5604039 prepare_log log(1 entries) v1 from mon.0 2013-04-26 02:39:28.624495 7f27d600e700 10 mon.1@0(leader).log v5604039 logging 2013-04-26 02:39:28.623623 mon.0 10.227.227.101:6789/0 27 : [INF] pgmap v5938159: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 32723B/s wr, 4op/s 2013-04-26 02:39:28.624710 7f27d600e700 10 mon.1@0(leader).log v5604039 update_from_paxos 2013-04-26 02:39:28.624778 7f27d600e700 10 mon.1@0(leader).log v5604039 update_from_paxos version 5604039 summary v 5604039 2013-04-26 02:39:28.624846 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:28.682013 7f27d600e700 10 mon.1@0(leader).log v5604039 update_from_paxos 2013-04-26 02:39:28.682279 7f27d600e700 10 mon.1@0(leader).log v5604039 update_from_paxos version 5604039 summary v 5604039 2013-04-26 02:39:28.682374 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:28.894299 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.11 10.227.227.103:6804/8253 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:28.894322 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x4bcb180 from 10.227.227.103:6789/0 2013-04-26 02:39:28.894623 7f27d600e700 10 mon.1@0(leader).pg v5938159 preprocess_query pg_stats(8 pgs tid 60 v 10556) v1 from osd.11 10.227.227.103:6804/8253 2013-04-26 02:39:28.894745 7f27d600e700 10 mon.1@0(leader).pg v5938159 prepare_update pg_stats(8 pgs tid 60 v 10556) v1 from osd.11 10.227.227.103:6804/8253 2013-04-26 02:39:28.894761 7f27d600e700 10 mon.1@0(leader).pg v5938159 prepare_pg_stats pg_stats(8 pgs tid 60 v 10556) v1 from osd.11 2013-04-26 02:39:28.894775 7f27d600e700 10 mon.1@0(leader).pg v5938159 got osd.11 osd_stat(161 GB used, 768 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[]) (was osd_stat(161 GB used, 768 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[])) 2013-04-26 02:39:29.216438 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from mds.? 10.227.227.103:6800/8011 via mon.2 10.227.227.104:6789/0 2013-04-26 02:39:29.216459 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x1789080 from 10.227.227.104:6789/0 2013-04-26 02:39:29.216627 7f27d600e700 10 mon.1@0(leader).mds e3724 preprocess_query mdsbeacon(217445/3 up:standby seq 75 v3724) v2 from mds.? 10.227.227.103:6800/8011 2013-04-26 02:39:29.277152 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.7 10.227.227.103:6810/8502 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:29.277175 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x1798f00 from 10.227.227.103:6789/0 2013-04-26 02:39:29.277418 7f27d600e700 10 mon.1@0(leader).pg v5938159 preprocess_query pg_stats(6 pgs tid 57 v 10556) v1 from osd.7 10.227.227.103:6810/8502 2013-04-26 02:39:29.277567 7f27d600e700 10 mon.1@0(leader).pg v5938159 prepare_update pg_stats(6 pgs tid 57 v 10556) v1 from osd.7 10.227.227.103:6810/8502 2013-04-26 02:39:29.277586 7f27d600e700 10 mon.1@0(leader).pg v5938159 prepare_pg_stats pg_stats(6 pgs tid 57 v 10556) v1 from osd.7 2013-04-26 02:39:29.277602 7f27d600e700 10 mon.1@0(leader).pg v5938159 got osd.7 osd_stat(156 GB used, 772 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[]) (was osd_stat(156 GB used, 772 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[])) 2013-04-26 02:39:29.302381 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.9 10.227.227.103:6816/8752 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:29.302461 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x4bcb900 from 10.227.227.103:6789/0 2013-04-26 02:39:29.302613 7f27d600e700 10 mon.1@0(leader).pg v5938159 preprocess_query pg_stats(6 pgs tid 60 v 10556) v1 from osd.9 10.227.227.103:6816/8752 2013-04-26 02:39:29.302713 7f27d600e700 10 mon.1@0(leader).pg v5938159 prepare_update pg_stats(6 pgs tid 60 v 10556) v1 from osd.9 10.227.227.103:6816/8752 2013-04-26 02:39:29.302727 7f27d600e700 10 mon.1@0(leader).pg v5938159 prepare_pg_stats pg_stats(6 pgs tid 60 v 10556) v1 from osd.9 2013-04-26 02:39:29.302742 7f27d600e700 10 mon.1@0(leader).pg v5938159 got osd.9 osd_stat(161 GB used, 767 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[]) (was osd_stat(161 GB used, 767 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[])) 2013-04-26 02:39:29.415450 7f27d680f700 10 mon.1@0(leader).pg v5938159 check_down_pgs 2013-04-26 02:39:29.415720 7f27d680f700 10 mon.1@0(leader).pg v5938159 v5938159: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 32723B/s wr, 4op/s 2013-04-26 02:39:29.415812 7f27d680f700 10 mon.1@0(leader).mds e3724 e3724: 1/1/1 up {0=4=up:active}, 2 up:standby 2013-04-26 02:39:29.415883 7f27d680f700 10 mon.1@0(leader).osd e10556 e10556: 12 osds: 12 up, 12 in 2013-04-26 02:39:29.416152 7f27d680f700 10 mon.1@0(leader).osd e10556 min_last_epoch_clean 10531 2013-04-26 02:39:29.416293 7f27d680f700 10 mon.1@0(leader).log v5604039 update_from_paxos 2013-04-26 02:39:29.416374 7f27d680f700 10 mon.1@0(leader).log v5604039 update_from_paxos version 5604039 summary v 5604039 2013-04-26 02:39:29.416416 7f27d680f700 10 mon.1@0(leader).log v5604039 log 2013-04-26 02:39:29.416486 7f27d680f700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:29.416557 7f27d680f700 10 mon.1@0(leader).auth v4370 auth 2013-04-26 02:39:29.535371 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.10 10.227.227.103:6801/8155 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:29.535393 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x4bcb680 from 10.227.227.103:6789/0 2013-04-26 02:39:29.535549 7f27d600e700 10 mon.1@0(leader).pg v5938159 preprocess_query pg_stats(13 pgs tid 55 v 10556) v1 from osd.10 10.227.227.103:6801/8155 2013-04-26 02:39:29.535663 7f27d600e700 10 mon.1@0(leader).pg v5938159 prepare_update pg_stats(13 pgs tid 55 v 10556) v1 from osd.10 10.227.227.103:6801/8155 2013-04-26 02:39:29.535679 7f27d600e700 10 mon.1@0(leader).pg v5938159 prepare_pg_stats pg_stats(13 pgs tid 55 v 10556) v1 from osd.10 2013-04-26 02:39:29.535695 7f27d600e700 10 mon.1@0(leader).pg v5938159 got osd.10 osd_stat(174 GB used, 755 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[]) (was osd_stat(174 GB used, 755 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[])) 2013-04-26 02:39:29.548183 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.2 10.227.227.101:6807/30207 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:29.548203 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x4c15680 from 10.227.227.103:6789/0 2013-04-26 02:39:29.548348 7f27d600e700 10 mon.1@0(leader).pg v5938159 preprocess_query pg_stats(9 pgs tid 739 v 10556) v1 from osd.2 10.227.227.101:6807/30207 2013-04-26 02:39:29.548442 7f27d600e700 10 mon.1@0(leader).pg v5938159 prepare_update pg_stats(9 pgs tid 739 v 10556) v1 from osd.2 10.227.227.101:6807/30207 2013-04-26 02:39:29.548454 7f27d600e700 10 mon.1@0(leader).pg v5938159 prepare_pg_stats pg_stats(9 pgs tid 739 v 10556) v1 from osd.2 2013-04-26 02:39:29.548465 7f27d600e700 10 mon.1@0(leader).pg v5938159 got osd.2 osd_stat(311 GB used, 1549 GB avail, 1860 GB total, peers [3,4,5,6,7,8,9,10,11]/[]) (was osd_stat(311 GB used, 1549 GB avail, 1860 GB total, peers [3,4,5,6,7,8,9,10,11]/[])) 2013-04-26 02:39:29.568915 7f27d680f700 10 mon.1@0(leader).log v5604039 encode_full log v 5604039 2013-04-26 02:39:29.569223 7f27d680f700 10 mon.1@0(leader).log v5604039 encode_pending v5604040 2013-04-26 02:39:29.639003 7f27d680f700 10 mon.1@0(leader).pg v5938159 encode_pending v 5938160 2013-04-26 02:39:29.697879 7f27d600e700 10 mon.1@0(leader).log v5604040 update_from_paxos 2013-04-26 02:39:29.697990 7f27d600e700 10 mon.1@0(leader).log v5604040 update_from_paxos version 5604040 summary v 5604039 2013-04-26 02:39:29.698042 7f27d600e700 10 mon.1@0(leader).log v5604040 update_from_paxos latest full 5604039 2013-04-26 02:39:29.698127 7f27d600e700 7 mon.1@0(leader).log v5604040 update_from_paxos applying incremental log 5604040 2013-04-26 02:39:28.623623 mon.0 10.227.227.101:6789/0 27 : [INF] pgmap v5938159: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 32723B/s wr, 4op/s 2013-04-26 02:39:29.698245 7f27d600e700 10 mon.1@0(leader).log v5604040 check_subs 2013-04-26 02:39:29.698318 7f27d600e700 10 mon.1@0(leader).log v5604040 create_pending v 5604041 2013-04-26 02:39:29.698385 7f27d600e700 7 mon.1@0(leader).log v5604040 _updated_log for mon.0 10.227.227.101:6789/0 2013-04-26 02:39:29.756072 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.1 10.227.227.101:6804/30088 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:29.756120 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x4bcaf00 from 10.227.227.103:6789/0 2013-04-26 02:39:29.756229 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.0 10.227.227.101:6801/29982 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:29.756241 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x4c15400 from 10.227.227.103:6789/0 2013-04-26 02:39:29.831730 7f27d600e700 7 mon.1@0(leader).pg v5938159 update_from_paxos applying incremental 5938160 2013-04-26 02:39:29.831940 7f27d600e700 10 mon.1@0(leader).pg v5938160 v5938160: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 231KB/s wr, 13op/s 2013-04-26 02:39:29.831991 7f27d600e700 10 mon.1@0(leader).pg v5938160 send_pg_creates to 0 pgs 2013-04-26 02:39:29.831996 7f27d600e700 10 mon.1@0(leader).pg v5938160 update_logger 2013-04-26 02:39:29.832038 7f27d600e700 10 mon.1@0(leader).pg v5938160 create_pending v 5938161 2013-04-26 02:39:29.832082 7f27d600e700 7 mon.1@0(leader).pg v5938160 _updated_stats for osd.11 10.227.227.103:6804/8253 2013-04-26 02:39:29.832117 7f27d600e700 7 mon.1@0(leader).pg v5938160 _updated_stats for osd.7 10.227.227.103:6810/8502 2013-04-26 02:39:29.832149 7f27d600e700 7 mon.1@0(leader).pg v5938160 _updated_stats for osd.9 10.227.227.103:6816/8752 2013-04-26 02:39:29.832214 7f27d600e700 7 mon.1@0(leader).pg v5938160 _updated_stats for osd.10 10.227.227.103:6801/8155 2013-04-26 02:39:29.832331 7f27d600e700 7 mon.1@0(leader).pg v5938160 _updated_stats for osd.2 10.227.227.101:6807/30207 2013-04-26 02:39:29.832434 7f27d600e700 10 mon.1@0(leader).pg v5938160 preprocess_query pg_stats(12 pgs tid 733 v 10556) v1 from osd.1 10.227.227.101:6804/30088 2013-04-26 02:39:29.832512 7f27d600e700 10 mon.1@0(leader).pg v5938160 prepare_update pg_stats(12 pgs tid 733 v 10556) v1 from osd.1 10.227.227.101:6804/30088 2013-04-26 02:39:29.832523 7f27d600e700 10 mon.1@0(leader).pg v5938160 prepare_pg_stats pg_stats(12 pgs tid 733 v 10556) v1 from osd.1 2013-04-26 02:39:29.832533 7f27d600e700 10 mon.1@0(leader).pg v5938160 got osd.1 osd_stat(342 GB used, 1517 GB avail, 1860 GB total, peers [3,4,5,6,7,8,9,10,11]/[]) (was osd_stat(342 GB used, 1517 GB avail, 1860 GB total, peers [3,4,5,6,7,8,9,10,11]/[])) 2013-04-26 02:39:29.832700 7f27d600e700 10 mon.1@0(leader).pg v5938160 preprocess_query pg_stats(17 pgs tid 744 v 10556) v1 from osd.0 10.227.227.101:6801/29982 2013-04-26 02:39:29.832760 7f27d600e700 10 mon.1@0(leader).pg v5938160 prepare_update pg_stats(17 pgs tid 744 v 10556) v1 from osd.0 10.227.227.101:6801/29982 2013-04-26 02:39:29.832770 7f27d600e700 10 mon.1@0(leader).pg v5938160 prepare_pg_stats pg_stats(17 pgs tid 744 v 10556) v1 from osd.0 2013-04-26 02:39:29.832776 7f27d600e700 10 mon.1@0(leader).pg v5938160 got osd.0 osd_stat(367 GB used, 1493 GB avail, 1860 GB total, peers [3,4,5,6,7,8,9,10,11]/[]) (was osd_stat(367 GB used, 1493 GB avail, 1860 GB total, peers [3,4,5,6,7,8,9,10,11]/[])) 2013-04-26 02:39:29.832852 7f27d600e700 10 mon.1@0(leader).pg v5938160 check_osd_map already seen 10556 >= 10556 2013-04-26 02:39:29.832858 7f27d600e700 10 mon.1@0(leader).pg v5938160 update_logger 2013-04-26 02:39:29.832875 7f27d600e700 0 log [INF] : pgmap v5938160: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 231KB/s wr, 13op/s 2013-04-26 02:39:29.833033 7f27d600e700 10 mon.1@0(leader).log v5604040 update_from_paxos 2013-04-26 02:39:29.833091 7f27d600e700 10 mon.1@0(leader).log v5604040 update_from_paxos version 5604040 summary v 5604040 2013-04-26 02:39:29.833138 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:29.833269 7f27d600e700 10 mon.1@0(leader).log v5604040 update_from_paxos 2013-04-26 02:39:29.833367 7f27d600e700 10 mon.1@0(leader).log v5604040 update_from_paxos version 5604040 summary v 5604040 2013-04-26 02:39:29.833421 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:29.833522 7f27d600e700 10 mon.1@0(leader).log v5604040 update_from_paxos 2013-04-26 02:39:29.833565 7f27d600e700 10 mon.1@0(leader).log v5604040 update_from_paxos version 5604040 summary v 5604040 2013-04-26 02:39:29.833588 7f27d600e700 10 mon.1@0(leader).log v5604040 preprocess_query log(1 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:29.833616 7f27d600e700 10 mon.1@0(leader).log v5604040 preprocess_log log(1 entries) v1 from mon.0 2013-04-26 02:39:29.833654 7f27d600e700 10 mon.1@0(leader).log v5604040 prepare_update log(1 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:29.833684 7f27d600e700 10 mon.1@0(leader).log v5604040 prepare_log log(1 entries) v1 from mon.0 2013-04-26 02:39:29.833713 7f27d600e700 10 mon.1@0(leader).log v5604040 logging 2013-04-26 02:39:29.832877 mon.0 10.227.227.101:6789/0 28 : [INF] pgmap v5938160: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 231KB/s wr, 13op/s 2013-04-26 02:39:29.882121 7f27d600e700 10 mon.1@0(leader).log v5604040 update_from_paxos 2013-04-26 02:39:29.882205 7f27d600e700 10 mon.1@0(leader).log v5604040 update_from_paxos version 5604040 summary v 5604040 2013-04-26 02:39:29.882285 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:29.892903 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.6 10.227.227.103:6807/8402 via mon.2 10.227.227.104:6789/0 2013-04-26 02:39:29.892922 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x4bcbb80 from 10.227.227.104:6789/0 2013-04-26 02:39:29.893039 7f27d600e700 10 mon.1@0(leader).pg v5938160 preprocess_query pg_stats(8 pgs tid 60 v 10556) v1 from osd.6 10.227.227.103:6807/8402 2013-04-26 02:39:29.893125 7f27d600e700 10 mon.1@0(leader).pg v5938160 prepare_update pg_stats(8 pgs tid 60 v 10556) v1 from osd.6 10.227.227.103:6807/8402 2013-04-26 02:39:29.893139 7f27d600e700 10 mon.1@0(leader).pg v5938160 prepare_pg_stats pg_stats(8 pgs tid 60 v 10556) v1 from osd.6 2013-04-26 02:39:29.893155 7f27d600e700 10 mon.1@0(leader).pg v5938160 got osd.6 osd_stat(184 GB used, 745 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[]) (was osd_stat(184 GB used, 745 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[])) 2013-04-26 02:39:30.346775 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.4 10.227.227.104:6804/14471 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:30.346798 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x2511b80 from 10.227.227.103:6789/0 2013-04-26 02:39:30.346956 7f27d600e700 10 mon.1@0(leader).pg v5938160 preprocess_query pg_stats(19 pgs tid 749 v 10556) v1 from osd.4 10.227.227.104:6804/14471 2013-04-26 02:39:30.347061 7f27d600e700 10 mon.1@0(leader).pg v5938160 prepare_update pg_stats(19 pgs tid 749 v 10556) v1 from osd.4 10.227.227.104:6804/14471 2013-04-26 02:39:30.347075 7f27d600e700 10 mon.1@0(leader).pg v5938160 prepare_pg_stats pg_stats(19 pgs tid 749 v 10556) v1 from osd.4 2013-04-26 02:39:30.347102 7f27d600e700 10 mon.1@0(leader).pg v5938160 got osd.4 osd_stat(423 GB used, 1437 GB avail, 1860 GB total, peers [0,1,2,6,7,8,9,10,11]/[]) (was osd_stat(423 GB used, 1437 GB avail, 1860 GB total, peers [0,1,2,6,7,8,9,10,11]/[])) 2013-04-26 02:39:30.471101 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.3 10.227.227.104:6800/14463 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:30.471124 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x4c15180 from 10.227.227.103:6789/0 2013-04-26 02:39:30.471285 7f27d600e700 10 mon.1@0(leader).pg v5938160 preprocess_query pg_stats(20 pgs tid 754 v 10556) v1 from osd.3 10.227.227.104:6800/14463 2013-04-26 02:39:30.471389 7f27d600e700 10 mon.1@0(leader).pg v5938160 prepare_update pg_stats(20 pgs tid 754 v 10556) v1 from osd.3 10.227.227.104:6800/14463 2013-04-26 02:39:30.471403 7f27d600e700 10 mon.1@0(leader).pg v5938160 prepare_pg_stats pg_stats(20 pgs tid 754 v 10556) v1 from osd.3 2013-04-26 02:39:30.471418 7f27d600e700 10 mon.1@0(leader).pg v5938160 got osd.3 osd_stat(383 GB used, 1477 GB avail, 1860 GB total, peers [0,1,2,6,7,8,9,10,11]/[]) (was osd_stat(383 GB used, 1477 GB avail, 1860 GB total, peers [0,1,2,6,7,8,9,10,11]/[])) 2013-04-26 02:39:30.769170 7f27d680f700 10 mon.1@0(leader).log v5604040 encode_full log v 5604040 2013-04-26 02:39:30.769474 7f27d680f700 10 mon.1@0(leader).log v5604040 encode_pending v5604041 2013-04-26 02:39:30.814708 7f27d680f700 10 mon.1@0(leader).pg v5938160 encode_pending v 5938161 2013-04-26 02:39:30.864779 7f27d600e700 10 mon.1@0(leader).log v5604041 update_from_paxos 2013-04-26 02:39:30.864886 7f27d600e700 10 mon.1@0(leader).log v5604041 update_from_paxos version 5604041 summary v 5604040 2013-04-26 02:39:30.864973 7f27d600e700 10 mon.1@0(leader).log v5604041 update_from_paxos latest full 5604040 2013-04-26 02:39:30.865099 7f27d600e700 7 mon.1@0(leader).log v5604041 update_from_paxos applying incremental log 5604041 2013-04-26 02:39:29.832877 mon.0 10.227.227.101:6789/0 28 : [INF] pgmap v5938160: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 231KB/s wr, 13op/s 2013-04-26 02:39:30.865203 7f27d600e700 10 mon.1@0(leader).log v5604041 check_subs 2013-04-26 02:39:30.865275 7f27d600e700 10 mon.1@0(leader).log v5604041 create_pending v 5604042 2013-04-26 02:39:30.865341 7f27d600e700 7 mon.1@0(leader).log v5604041 _updated_log for mon.0 10.227.227.101:6789/0 2013-04-26 02:39:30.923588 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.8 10.227.227.103:6813/8628 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:30.923604 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x4c15b80 from 10.227.227.103:6789/0 2013-04-26 02:39:30.973732 7f27d600e700 7 mon.1@0(leader).pg v5938160 update_from_paxos applying incremental 5938161 2013-04-26 02:39:30.974221 7f27d600e700 10 mon.1@0(leader).pg v5938161 v5938161: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 661KB/s wr, 35op/s 2013-04-26 02:39:30.974322 7f27d600e700 10 mon.1@0(leader).pg v5938161 send_pg_creates to 0 pgs 2013-04-26 02:39:30.974332 7f27d600e700 10 mon.1@0(leader).pg v5938161 update_logger 2013-04-26 02:39:30.974470 7f27d600e700 10 mon.1@0(leader).pg v5938161 create_pending v 5938162 2013-04-26 02:39:30.974529 7f27d600e700 7 mon.1@0(leader).pg v5938161 _updated_stats for osd.1 10.227.227.101:6804/30088 2013-04-26 02:39:30.974572 7f27d600e700 7 mon.1@0(leader).pg v5938161 _updated_stats for osd.0 10.227.227.101:6801/29982 2013-04-26 02:39:30.974597 7f27d600e700 7 mon.1@0(leader).pg v5938161 _updated_stats for osd.6 10.227.227.103:6807/8402 2013-04-26 02:39:30.974624 7f27d600e700 7 mon.1@0(leader).pg v5938161 _updated_stats for osd.4 10.227.227.104:6804/14471 2013-04-26 02:39:30.974650 7f27d600e700 7 mon.1@0(leader).pg v5938161 _updated_stats for osd.3 10.227.227.104:6800/14463 2013-04-26 02:39:30.974878 7f27d600e700 10 mon.1@0(leader).pg v5938161 preprocess_query pg_stats(5 pgs tid 58 v 10556) v1 from osd.8 10.227.227.103:6813/8628 2013-04-26 02:39:30.974981 7f27d600e700 10 mon.1@0(leader).pg v5938161 prepare_update pg_stats(5 pgs tid 58 v 10556) v1 from osd.8 10.227.227.103:6813/8628 2013-04-26 02:39:30.975050 7f27d600e700 10 mon.1@0(leader).pg v5938161 prepare_pg_stats pg_stats(5 pgs tid 58 v 10556) v1 from osd.8 2013-04-26 02:39:30.975064 7f27d600e700 10 mon.1@0(leader).pg v5938161 got osd.8 osd_stat(185 GB used, 744 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[]) (was osd_stat(185 GB used, 744 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[])) 2013-04-26 02:39:30.975128 7f27d600e700 10 mon.1@0(leader).pg v5938161 check_osd_map already seen 10556 >= 10556 2013-04-26 02:39:30.975133 7f27d600e700 10 mon.1@0(leader).pg v5938161 update_logger 2013-04-26 02:39:30.975155 7f27d600e700 0 log [INF] : pgmap v5938161: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 661KB/s wr, 35op/s 2013-04-26 02:39:30.975354 7f27d600e700 10 mon.1@0(leader).log v5604041 update_from_paxos 2013-04-26 02:39:30.975431 7f27d600e700 10 mon.1@0(leader).log v5604041 update_from_paxos version 5604041 summary v 5604041 2013-04-26 02:39:30.975500 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:30.975649 7f27d600e700 10 mon.1@0(leader).log v5604041 update_from_paxos 2013-04-26 02:39:30.975709 7f27d600e700 10 mon.1@0(leader).log v5604041 update_from_paxos version 5604041 summary v 5604041 2013-04-26 02:39:30.975742 7f27d600e700 10 mon.1@0(leader).log v5604041 preprocess_query log(1 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:30.975779 7f27d600e700 10 mon.1@0(leader).log v5604041 preprocess_log log(1 entries) v1 from mon.0 2013-04-26 02:39:30.975829 7f27d600e700 10 mon.1@0(leader).log v5604041 prepare_update log(1 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:30.975870 7f27d600e700 10 mon.1@0(leader).log v5604041 prepare_log log(1 entries) v1 from mon.0 2013-04-26 02:39:30.975907 7f27d600e700 10 mon.1@0(leader).log v5604041 logging 2013-04-26 02:39:30.975158 mon.0 10.227.227.101:6789/0 29 : [INF] pgmap v5938161: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 661KB/s wr, 35op/s 2013-04-26 02:39:30.979178 7f27d600e700 10 mon.1@0(leader).log v5604041 update_from_paxos 2013-04-26 02:39:30.979249 7f27d600e700 10 mon.1@0(leader).log v5604041 update_from_paxos version 5604041 summary v 5604041 2013-04-26 02:39:30.979313 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:31.015467 7f27d600e700 10 mon.1@0(leader).log v5604041 update_from_paxos 2013-04-26 02:39:31.015624 7f27d600e700 10 mon.1@0(leader).log v5604041 update_from_paxos version 5604041 summary v 5604041 2013-04-26 02:39:31.015706 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:31.929015 7f27d680f700 10 mon.1@0(leader).log v5604041 encode_full log v 5604041 2013-04-26 02:39:31.929314 7f27d680f700 10 mon.1@0(leader).log v5604041 encode_pending v5604042 2013-04-26 02:39:31.969849 7f27d4c08700 10 mon.1@0(leader) e1 ms_verify_authorizer 10.227.227.101:6801/29982 osd protocol 0 2013-04-26 02:39:31.970111 7f27d4b07700 10 mon.1@0(leader) e1 ms_verify_authorizer 10.227.227.103:6813/8628 osd protocol 0 2013-04-26 02:39:31.998644 7f27d680f700 10 mon.1@0(leader).pg v5938161 encode_pending v 5938162 2013-04-26 02:39:31.998786 7f27d600e700 10 mon.1@0(leader) e1 do not have session, making new one 2013-04-26 02:39:31.998799 7f27d600e700 10 mon.1@0(leader) e1 ms_dispatch new session MonSession: osd.0 10.227.227.101:6801/29982 is open for osd.0 10.227.227.101:6801/29982 2013-04-26 02:39:31.998811 7f27d600e700 10 mon.1@0(leader) e1 setting timeout on session 2013-04-26 02:39:31.998899 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:31.998943 7f27d600e700 10 mon.1@0(leader).auth v4370 preprocess_query auth(proto 0 26 bytes epoch 1) v1 from osd.0 10.227.227.101:6801/29982 2013-04-26 02:39:31.998969 7f27d600e700 10 mon.1@0(leader).auth v4370 prep_auth() blob_size=26 2013-04-26 02:39:31.999138 7f27d600e700 10 mon.1@0(leader) e1 do not have session, making new one 2013-04-26 02:39:31.999150 7f27d600e700 10 mon.1@0(leader) e1 ms_dispatch new session MonSession: osd.8 10.227.227.103:6813/8628 is open for osd.8 10.227.227.103:6813/8628 2013-04-26 02:39:31.999155 7f27d600e700 10 mon.1@0(leader) e1 setting timeout on session 2013-04-26 02:39:31.999219 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:31.999268 7f27d600e700 10 mon.1@0(leader).auth v4370 preprocess_query auth(proto 0 26 bytes epoch 1) v1 from osd.8 10.227.227.103:6813/8628 2013-04-26 02:39:31.999299 7f27d600e700 10 mon.1@0(leader).auth v4370 prep_auth() blob_size=26 2013-04-26 02:39:31.999385 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.11 10.227.227.103:6804/8253 via mon.2 10.227.227.104:6789/0 2013-04-26 02:39:31.999396 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x4c15900 from 10.227.227.104:6789/0 2013-04-26 02:39:31.999444 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.10 10.227.227.103:6801/8155 via mon.2 10.227.227.104:6789/0 2013-04-26 02:39:31.999451 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x6ce5180 from 10.227.227.104:6789/0 2013-04-26 02:39:31.999847 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:31.999885 7f27d600e700 10 mon.1@0(leader).auth v4370 preprocess_query auth(proto 2 128 bytes epoch 0) v1 from osd.0 10.227.227.101:6801/29982 2013-04-26 02:39:31.999908 7f27d600e700 10 mon.1@0(leader).auth v4370 prep_auth() blob_size=128 2013-04-26 02:39:32.000212 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:32.000249 7f27d600e700 10 mon.1@0(leader).auth v4370 preprocess_query auth(proto 2 128 bytes epoch 0) v1 from osd.8 10.227.227.103:6813/8628 2013-04-26 02:39:32.000271 7f27d600e700 10 mon.1@0(leader).auth v4370 prep_auth() blob_size=128 2013-04-26 02:39:32.000689 7f27d600e700 10 mon.1@0(leader) e1 handle_subscribe mon_subscribe({monmap=2+,osd_pg_creates=0,osdmap=10554}) v2 2013-04-26 02:39:32.000701 7f27d600e700 10 mon.1@0(leader) e1 check_sub monmap next 2 have 1 2013-04-26 02:39:32.000709 7f27d600e700 5 mon.1@0(leader).osd e10556 send_incremental [10554..10556] to osd.0 10.227.227.101:6801/29982 2013-04-26 02:39:32.000761 7f27d600e700 10 mon.1@0(leader).osd e10556 build_incremental [10554..10556] 2013-04-26 02:39:32.000990 7f27d600e700 10 mon.1@0(leader) e1 handle_subscribe mon_subscribe({monmap=2+,osd_pg_creates=0,osdmap=10554}) v2 2013-04-26 02:39:32.000995 7f27d600e700 10 mon.1@0(leader) e1 check_sub monmap next 2 have 1 2013-04-26 02:39:32.000999 7f27d600e700 5 mon.1@0(leader).osd e10556 send_incremental [10554..10556] to osd.0 10.227.227.101:6801/29982 2013-04-26 02:39:32.001029 7f27d600e700 10 mon.1@0(leader).osd e10556 build_incremental [10554..10556] 2013-04-26 02:39:32.001431 7f27d600e700 10 mon.1@0(leader) e1 handle_subscribe mon_subscribe({monmap=2+,osd_pg_creates=0}) v2 2013-04-26 02:39:32.001440 7f27d600e700 10 mon.1@0(leader) e1 check_sub monmap next 2 have 1 2013-04-26 02:39:32.001518 7f27d600e700 10 mon.1@0(leader) e1 handle_subscribe mon_subscribe({monmap=2+,osd_pg_creates=0}) v2 2013-04-26 02:39:32.001522 7f27d600e700 10 mon.1@0(leader) e1 check_sub monmap next 2 have 1 2013-04-26 02:39:32.082514 7f27d600e700 10 mon.1@0(leader).log v5604042 update_from_paxos 2013-04-26 02:39:32.082604 7f27d600e700 10 mon.1@0(leader).log v5604042 update_from_paxos version 5604042 summary v 5604041 2013-04-26 02:39:32.082649 7f27d600e700 10 mon.1@0(leader).log v5604042 update_from_paxos latest full 5604041 2013-04-26 02:39:32.082714 7f27d600e700 7 mon.1@0(leader).log v5604042 update_from_paxos applying incremental log 5604042 2013-04-26 02:39:30.975158 mon.0 10.227.227.101:6789/0 29 : [INF] pgmap v5938161: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 661KB/s wr, 35op/s 2013-04-26 02:39:32.082809 7f27d600e700 10 mon.1@0(leader).log v5604042 check_subs 2013-04-26 02:39:32.082876 7f27d600e700 10 mon.1@0(leader).log v5604042 create_pending v 5604043 2013-04-26 02:39:32.082954 7f27d600e700 7 mon.1@0(leader).log v5604042 _updated_log for mon.0 10.227.227.101:6789/0 2013-04-26 02:39:32.140771 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from mds.? 10.227.227.101:6800/29852 via mon.2 10.227.227.104:6789/0 2013-04-26 02:39:32.140786 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x1788dc0 from 10.227.227.104:6789/0 2013-04-26 02:39:32.140927 7f27d600e700 10 mon.1@0(leader).mds e3724 preprocess_query mdsbeacon(217349/1 up:standby seq 463 v3724) v2 from mds.? 10.227.227.101:6800/29852 2013-04-26 02:39:32.207112 7f27d600e700 7 mon.1@0(leader).pg v5938161 update_from_paxos applying incremental 5938162 2013-04-26 02:39:32.207183 7f27d600e700 10 mon.1@0(leader).pg v5938162 v5938162: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 380KB/s wr, 22op/s 2013-04-26 02:39:32.207244 7f27d600e700 10 mon.1@0(leader).pg v5938162 send_pg_creates to 0 pgs 2013-04-26 02:39:32.207250 7f27d600e700 10 mon.1@0(leader).pg v5938162 update_logger 2013-04-26 02:39:32.207291 7f27d600e700 10 mon.1@0(leader).pg v5938162 create_pending v 5938163 2013-04-26 02:39:32.207324 7f27d600e700 7 mon.1@0(leader).pg v5938162 _updated_stats for osd.8 10.227.227.103:6813/8628 2013-04-26 02:39:32.207459 7f27d600e700 10 mon.1@0(leader).pg v5938162 preprocess_query pg_stats(9 pgs tid 61 v 10556) v1 from osd.11 10.227.227.103:6804/8253 2013-04-26 02:39:32.207771 7f27d600e700 10 mon.1@0(leader).pg v5938162 prepare_update pg_stats(9 pgs tid 61 v 10556) v1 from osd.11 10.227.227.103:6804/8253 2013-04-26 02:39:32.207794 7f27d600e700 10 mon.1@0(leader).pg v5938162 prepare_pg_stats pg_stats(9 pgs tid 61 v 10556) v1 from osd.11 2013-04-26 02:39:32.207808 7f27d600e700 10 mon.1@0(leader).pg v5938162 got osd.11 osd_stat(161 GB used, 768 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[]) (was osd_stat(161 GB used, 768 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[])) 2013-04-26 02:39:32.207970 7f27d600e700 10 mon.1@0(leader).pg v5938162 preprocess_query pg_stats(5 pgs tid 56 v 10556) v1 from osd.10 10.227.227.103:6801/8155 2013-04-26 02:39:32.208055 7f27d600e700 10 mon.1@0(leader).pg v5938162 prepare_update pg_stats(5 pgs tid 56 v 10556) v1 from osd.10 10.227.227.103:6801/8155 2013-04-26 02:39:32.208070 7f27d600e700 10 mon.1@0(leader).pg v5938162 prepare_pg_stats pg_stats(5 pgs tid 56 v 10556) v1 from osd.10 2013-04-26 02:39:32.208080 7f27d600e700 10 mon.1@0(leader).pg v5938162 got osd.10 osd_stat(174 GB used, 755 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[]) (was osd_stat(174 GB used, 755 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[])) 2013-04-26 02:39:32.208222 7f27d600e700 10 mon.1@0(leader).pg v5938162 preprocess_query pg_stats(7 pgs tid 745 v 10556) v1 from osd.0 10.227.227.101:6801/29982 2013-04-26 02:39:32.208376 7f27d600e700 10 mon.1@0(leader).pg v5938162 prepare_update pg_stats(7 pgs tid 745 v 10556) v1 from osd.0 10.227.227.101:6801/29982 2013-04-26 02:39:32.208395 7f27d600e700 10 mon.1@0(leader).pg v5938162 prepare_pg_stats pg_stats(7 pgs tid 745 v 10556) v1 from osd.0 2013-04-26 02:39:32.208420 7f27d600e700 10 mon.1@0(leader).pg v5938162 got osd.0 osd_stat(367 GB used, 1493 GB avail, 1860 GB total, peers [3,4,5,6,7,8,9,10,11]/[]) (was osd_stat(367 GB used, 1493 GB avail, 1860 GB total, peers [3,4,5,6,7,8,9,10,11]/[])) 2013-04-26 02:39:32.208567 7f27d600e700 10 mon.1@0(leader).pg v5938162 preprocess_query pg_stats(7 pgs tid 59 v 10556) v1 from osd.8 10.227.227.103:6813/8628 2013-04-26 02:39:32.208650 7f27d600e700 10 mon.1@0(leader).pg v5938162 prepare_update pg_stats(7 pgs tid 59 v 10556) v1 from osd.8 10.227.227.103:6813/8628 2013-04-26 02:39:32.208664 7f27d600e700 10 mon.1@0(leader).pg v5938162 prepare_pg_stats pg_stats(7 pgs tid 59 v 10556) v1 from osd.8 2013-04-26 02:39:32.208674 7f27d600e700 10 mon.1@0(leader).pg v5938162 got osd.8 osd_stat(185 GB used, 744 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[]) (was osd_stat(185 GB used, 744 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[])) 2013-04-26 02:39:32.208738 7f27d600e700 10 mon.1@0(leader).pg v5938162 check_osd_map already seen 10556 >= 10556 2013-04-26 02:39:32.208744 7f27d600e700 10 mon.1@0(leader).pg v5938162 update_logger 2013-04-26 02:39:32.208767 7f27d600e700 0 log [INF] : pgmap v5938162: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 380KB/s wr, 22op/s 2013-04-26 02:39:32.208998 7f27d600e700 10 mon.1@0(leader).log v5604042 update_from_paxos 2013-04-26 02:39:32.209081 7f27d600e700 10 mon.1@0(leader).log v5604042 update_from_paxos version 5604042 summary v 5604042 2013-04-26 02:39:32.209149 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:32.209344 7f27d600e700 10 mon.1@0(leader).log v5604042 update_from_paxos 2013-04-26 02:39:32.209414 7f27d600e700 10 mon.1@0(leader).log v5604042 update_from_paxos version 5604042 summary v 5604042 2013-04-26 02:39:32.209480 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:32.209631 7f27d600e700 10 mon.1@0(leader).log v5604042 update_from_paxos 2013-04-26 02:39:32.209696 7f27d600e700 10 mon.1@0(leader).log v5604042 update_from_paxos version 5604042 summary v 5604042 2013-04-26 02:39:32.209732 7f27d600e700 10 mon.1@0(leader).log v5604042 preprocess_query log(1 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:32.209773 7f27d600e700 10 mon.1@0(leader).log v5604042 preprocess_log log(1 entries) v1 from mon.0 2013-04-26 02:39:32.209826 7f27d600e700 10 mon.1@0(leader).log v5604042 prepare_update log(1 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:32.209872 7f27d600e700 10 mon.1@0(leader).log v5604042 prepare_log log(1 entries) v1 from mon.0 2013-04-26 02:39:32.209911 7f27d600e700 10 mon.1@0(leader).log v5604042 logging 2013-04-26 02:39:32.208770 mon.0 10.227.227.101:6789/0 30 : [INF] pgmap v5938162: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 380KB/s wr, 22op/s 2013-04-26 02:39:32.248916 7f27d600e700 10 mon.1@0(leader).log v5604042 update_from_paxos 2013-04-26 02:39:32.248995 7f27d600e700 10 mon.1@0(leader).log v5604042 update_from_paxos version 5604042 summary v 5604042 2013-04-26 02:39:32.249063 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:32.503352 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from mds.0 10.227.227.104:6801/10842 via mon.2 10.227.227.104:6789/0 2013-04-26 02:39:32.503372 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x1789080 from 10.227.227.104:6789/0 2013-04-26 02:39:32.503549 7f27d600e700 10 mon.1@0(leader).mds e3724 preprocess_query mdsbeacon(217231/4 up:active seq 494 v3724) v2 from mds.0 10.227.227.104:6801/10842 2013-04-26 02:39:32.758216 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.5 10.227.227.104:6807/14479 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:32.758243 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x4bcb680 from 10.227.227.103:6789/0 2013-04-26 02:39:32.758601 7f27d600e700 10 mon.1@0(leader).pg v5938162 preprocess_query pg_stats(12 pgs tid 747 v 10556) v1 from osd.5 10.227.227.104:6807/14479 2013-04-26 02:39:32.758714 7f27d600e700 10 mon.1@0(leader).pg v5938162 prepare_update pg_stats(12 pgs tid 747 v 10556) v1 from osd.5 10.227.227.104:6807/14479 2013-04-26 02:39:32.758728 7f27d600e700 10 mon.1@0(leader).pg v5938162 prepare_pg_stats pg_stats(12 pgs tid 747 v 10556) v1 from osd.5 2013-04-26 02:39:32.758744 7f27d600e700 10 mon.1@0(leader).pg v5938162 got osd.5 osd_stat(355 GB used, 1505 GB avail, 1860 GB total, peers [0,1,2,6,7,8,9,10,11]/[]) (was osd_stat(355 GB used, 1505 GB avail, 1860 GB total, peers [0,1,2,6,7,8,9,10,11]/[])) 2013-04-26 02:39:33.143180 7f27d680f700 10 mon.1@0(leader).log v5604042 encode_full log v 5604042 2013-04-26 02:39:33.143474 7f27d680f700 10 mon.1@0(leader).log v5604042 encode_pending v5604043 2013-04-26 02:39:33.191145 7f27d680f700 10 mon.1@0(leader).pg v5938162 encode_pending v 5938163 2013-04-26 02:39:33.250065 7f27d600e700 10 mon.1@0(leader).log v5604043 update_from_paxos 2013-04-26 02:39:33.250170 7f27d600e700 10 mon.1@0(leader).log v5604043 update_from_paxos version 5604043 summary v 5604042 2013-04-26 02:39:33.250216 7f27d600e700 10 mon.1@0(leader).log v5604043 update_from_paxos latest full 5604042 2013-04-26 02:39:33.250282 7f27d600e700 7 mon.1@0(leader).log v5604043 update_from_paxos applying incremental log 5604043 2013-04-26 02:39:32.208770 mon.0 10.227.227.101:6789/0 30 : [INF] pgmap v5938162: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 380KB/s wr, 22op/s 2013-04-26 02:39:33.250435 7f27d600e700 10 mon.1@0(leader).log v5604043 check_subs 2013-04-26 02:39:33.250516 7f27d600e700 10 mon.1@0(leader).log v5604043 create_pending v 5604044 2013-04-26 02:39:33.250579 7f27d600e700 7 mon.1@0(leader).log v5604043 _updated_log for mon.0 10.227.227.101:6789/0 2013-04-26 02:39:33.308162 7f27d600e700 10 mon.1@0(leader).data_health(2814) service_dispatch mon_health( service 1 op tell e 0 r 0 flags none ) v1 2013-04-26 02:39:33.308169 7f27d600e700 10 mon.1@0(leader).data_health(2814) handle_tell mon_health( service 1 op tell e 0 r 0 flags none ) v1 2013-04-26 02:39:33.308194 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from mds.? 10.227.227.103:6800/8011 via mon.2 10.227.227.104:6789/0 2013-04-26 02:39:33.308205 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x1788dc0 from 10.227.227.104:6789/0 2013-04-26 02:39:33.308322 7f27d600e700 10 mon.1@0(leader).mds e3724 preprocess_query mdsbeacon(217445/3 up:standby seq 76 v3724) v2 from mds.? 10.227.227.103:6800/8011 2013-04-26 02:39:33.391964 7f27d600e700 7 mon.1@0(leader).pg v5938162 update_from_paxos applying incremental 5938163 2013-04-26 02:39:33.392171 7f27d600e700 10 mon.1@0(leader).pg v5938163 v5938163: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 225KB/s wr, 12op/s 2013-04-26 02:39:33.392222 7f27d600e700 10 mon.1@0(leader).pg v5938163 send_pg_creates to 0 pgs 2013-04-26 02:39:33.392227 7f27d600e700 10 mon.1@0(leader).pg v5938163 update_logger 2013-04-26 02:39:33.392267 7f27d600e700 10 mon.1@0(leader).pg v5938163 create_pending v 5938164 2013-04-26 02:39:33.392288 7f27d600e700 7 mon.1@0(leader).pg v5938163 _updated_stats for osd.11 10.227.227.103:6804/8253 2013-04-26 02:39:33.392318 7f27d600e700 7 mon.1@0(leader).pg v5938163 _updated_stats for osd.10 10.227.227.103:6801/8155 2013-04-26 02:39:33.392370 7f27d600e700 7 mon.1@0(leader).pg v5938163 _updated_stats for osd.0 10.227.227.101:6801/29982 2013-04-26 02:39:33.392498 7f27d600e700 7 mon.1@0(leader).pg v5938163 _updated_stats for osd.8 10.227.227.103:6813/8628 2013-04-26 02:39:33.392531 7f27d600e700 7 mon.1@0(leader).pg v5938163 _updated_stats for osd.5 10.227.227.104:6807/14479 2013-04-26 02:39:33.392552 7f27d600e700 10 mon.1@0(leader).pg v5938163 check_osd_map already seen 10556 >= 10556 2013-04-26 02:39:33.392555 7f27d600e700 10 mon.1@0(leader).pg v5938163 update_logger 2013-04-26 02:39:33.392571 7f27d600e700 0 log [INF] : pgmap v5938163: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 225KB/s wr, 12op/s 2013-04-26 02:39:33.392751 7f27d600e700 10 mon.1@0(leader).log v5604043 update_from_paxos 2013-04-26 02:39:33.392809 7f27d600e700 10 mon.1@0(leader).log v5604043 update_from_paxos version 5604043 summary v 5604043 2013-04-26 02:39:33.392856 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:33.392958 7f27d600e700 10 mon.1@0(leader).log v5604043 update_from_paxos 2013-04-26 02:39:33.392998 7f27d600e700 10 mon.1@0(leader).log v5604043 update_from_paxos version 5604043 summary v 5604043 2013-04-26 02:39:33.393020 7f27d600e700 10 mon.1@0(leader).log v5604043 preprocess_query log(1 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:33.393047 7f27d600e700 10 mon.1@0(leader).log v5604043 preprocess_log log(1 entries) v1 from mon.0 2013-04-26 02:39:33.393084 7f27d600e700 10 mon.1@0(leader).log v5604043 prepare_update log(1 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:33.393126 7f27d600e700 10 mon.1@0(leader).log v5604043 prepare_log log(1 entries) v1 from mon.0 2013-04-26 02:39:33.393153 7f27d600e700 10 mon.1@0(leader).log v5604043 logging 2013-04-26 02:39:33.392574 mon.0 10.227.227.101:6789/0 31 : [INF] pgmap v5938163: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 225KB/s wr, 12op/s 2013-04-26 02:39:33.398942 7f27d600e700 10 mon.1@0(leader).log v5604043 update_from_paxos 2013-04-26 02:39:33.398999 7f27d600e700 10 mon.1@0(leader).log v5604043 update_from_paxos version 5604043 summary v 5604043 2013-04-26 02:39:33.399050 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:33.441010 7f27d600e700 10 mon.1@0(leader).log v5604043 update_from_paxos 2013-04-26 02:39:33.441092 7f27d600e700 10 mon.1@0(leader).log v5604043 update_from_paxos version 5604043 summary v 5604043 2013-04-26 02:39:33.441161 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:33.894945 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.11 10.227.227.103:6804/8253 via mon.2 10.227.227.104:6789/0 2013-04-26 02:39:33.894973 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x4f37180 from 10.227.227.104:6789/0 2013-04-26 02:39:33.895160 7f27d600e700 10 mon.1@0(leader).pg v5938163 preprocess_query pg_stats(6 pgs tid 62 v 10556) v1 from osd.11 10.227.227.103:6804/8253 2013-04-26 02:39:33.895267 7f27d600e700 10 mon.1@0(leader).pg v5938163 prepare_update pg_stats(6 pgs tid 62 v 10556) v1 from osd.11 10.227.227.103:6804/8253 2013-04-26 02:39:33.895282 7f27d600e700 10 mon.1@0(leader).pg v5938163 prepare_pg_stats pg_stats(6 pgs tid 62 v 10556) v1 from osd.11 2013-04-26 02:39:33.895298 7f27d600e700 10 mon.1@0(leader).pg v5938163 got osd.11 osd_stat(161 GB used, 768 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[]) (was osd_stat(161 GB used, 768 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[])) 2013-04-26 02:39:34.277940 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.7 10.227.227.103:6810/8502 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:34.277965 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x4c14f00 from 10.227.227.103:6789/0 2013-04-26 02:39:34.278120 7f27d600e700 10 mon.1@0(leader).pg v5938163 preprocess_query pg_stats(5 pgs tid 58 v 10556) v1 from osd.7 10.227.227.103:6810/8502 2013-04-26 02:39:34.278224 7f27d600e700 10 mon.1@0(leader).pg v5938163 prepare_update pg_stats(5 pgs tid 58 v 10556) v1 from osd.7 10.227.227.103:6810/8502 2013-04-26 02:39:34.278238 7f27d600e700 10 mon.1@0(leader).pg v5938163 prepare_pg_stats pg_stats(5 pgs tid 58 v 10556) v1 from osd.7 2013-04-26 02:39:34.278253 7f27d600e700 10 mon.1@0(leader).pg v5938163 got osd.7 osd_stat(156 GB used, 772 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[]) (was osd_stat(156 GB used, 772 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[])) 2013-04-26 02:39:34.303087 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.9 10.227.227.103:6816/8752 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:34.303106 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x4c14c80 from 10.227.227.103:6789/0 2013-04-26 02:39:34.303229 7f27d600e700 10 mon.1@0(leader).pg v5938163 preprocess_query pg_stats(6 pgs tid 61 v 10556) v1 from osd.9 10.227.227.103:6816/8752 2013-04-26 02:39:34.303334 7f27d600e700 10 mon.1@0(leader).pg v5938163 prepare_update pg_stats(6 pgs tid 61 v 10556) v1 from osd.9 10.227.227.103:6816/8752 2013-04-26 02:39:34.303347 7f27d600e700 10 mon.1@0(leader).pg v5938163 prepare_pg_stats pg_stats(6 pgs tid 61 v 10556) v1 from osd.9 2013-04-26 02:39:34.303359 7f27d600e700 10 mon.1@0(leader).pg v5938163 got osd.9 osd_stat(161 GB used, 767 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[]) (was osd_stat(161 GB used, 767 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[])) 2013-04-26 02:39:34.329133 7f27d680f700 10 mon.1@0(leader).log v5604043 encode_full log v 5604043 2013-04-26 02:39:34.329430 7f27d680f700 10 mon.1@0(leader).log v5604043 encode_pending v5604044 2013-04-26 02:39:34.417041 7f27d680f700 10 mon.1@0(leader).pg v5938163 encode_pending v 5938164 2013-04-26 02:39:34.417320 7f27d680f700 10 mon.1@0(leader).mds e3724 e3724: 1/1/1 up {0=4=up:active}, 2 up:standby 2013-04-26 02:39:34.417398 7f27d680f700 10 mon.1@0(leader).osd e10556 e10556: 12 osds: 12 up, 12 in 2013-04-26 02:39:34.417499 7f27d680f700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:34.417548 7f27d680f700 10 mon.1@0(leader).auth v4370 auth 2013-04-26 02:39:34.475584 7f27d600e700 10 mon.1@0(leader).log v5604044 update_from_paxos 2013-04-26 02:39:34.475698 7f27d600e700 10 mon.1@0(leader).log v5604044 update_from_paxos version 5604044 summary v 5604043 2013-04-26 02:39:34.475748 7f27d600e700 10 mon.1@0(leader).log v5604044 update_from_paxos latest full 5604043 2013-04-26 02:39:34.475803 7f27d600e700 7 mon.1@0(leader).log v5604044 update_from_paxos applying incremental log 5604044 2013-04-26 02:39:33.392574 mon.0 10.227.227.101:6789/0 31 : [INF] pgmap v5938163: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 225KB/s wr, 12op/s 2013-04-26 02:39:34.475900 7f27d600e700 10 mon.1@0(leader).log v5604044 check_subs 2013-04-26 02:39:34.475981 7f27d600e700 10 mon.1@0(leader).log v5604044 create_pending v 5604045 2013-04-26 02:39:34.476052 7f27d600e700 7 mon.1@0(leader).log v5604044 _updated_log for mon.0 10.227.227.101:6789/0 2013-04-26 02:39:34.590864 7f27d600e700 7 mon.1@0(leader).pg v5938163 update_from_paxos applying incremental 5938164 2013-04-26 02:39:34.591008 7f27d600e700 10 mon.1@0(leader).pg v5938164 v5938164: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 267KB/s wr, 11op/s 2013-04-26 02:39:34.591075 7f27d600e700 10 mon.1@0(leader).pg v5938164 send_pg_creates to 0 pgs 2013-04-26 02:39:34.591084 7f27d600e700 10 mon.1@0(leader).pg v5938164 update_logger 2013-04-26 02:39:34.591132 7f27d600e700 10 mon.1@0(leader).pg v5938164 create_pending v 5938165 2013-04-26 02:39:34.591166 7f27d600e700 7 mon.1@0(leader).pg v5938164 _updated_stats for osd.11 10.227.227.103:6804/8253 2013-04-26 02:39:34.591291 7f27d600e700 7 mon.1@0(leader).pg v5938164 _updated_stats for osd.7 10.227.227.103:6810/8502 2013-04-26 02:39:34.591395 7f27d600e700 7 mon.1@0(leader).pg v5938164 _updated_stats for osd.9 10.227.227.103:6816/8752 2013-04-26 02:39:34.591466 7f27d600e700 10 mon.1@0(leader).pg v5938164 check_osd_map already seen 10556 >= 10556 2013-04-26 02:39:34.591478 7f27d600e700 10 mon.1@0(leader).pg v5938164 update_logger 2013-04-26 02:39:34.591504 7f27d600e700 0 log [INF] : pgmap v5938164: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 267KB/s wr, 11op/s 2013-04-26 02:39:34.607797 7f27d600e700 10 mon.1@0(leader).log v5604044 update_from_paxos 2013-04-26 02:39:34.607870 7f27d600e700 10 mon.1@0(leader).log v5604044 update_from_paxos version 5604044 summary v 5604044 2013-04-26 02:39:34.607905 7f27d600e700 10 mon.1@0(leader).log v5604044 preprocess_query log(1 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:34.607947 7f27d600e700 10 mon.1@0(leader).log v5604044 preprocess_log log(1 entries) v1 from mon.0 2013-04-26 02:39:34.607999 7f27d600e700 10 mon.1@0(leader).log v5604044 prepare_update log(1 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:34.608049 7f27d600e700 10 mon.1@0(leader).log v5604044 prepare_log log(1 entries) v1 from mon.0 2013-04-26 02:39:34.608092 7f27d600e700 10 mon.1@0(leader).log v5604044 logging 2013-04-26 02:39:34.591508 mon.0 10.227.227.101:6789/0 32 : [INF] pgmap v5938164: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 267KB/s wr, 11op/s 2013-04-26 02:39:34.608232 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.2 10.227.227.101:6807/30207 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:34.608247 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x4bcbb80 from 10.227.227.103:6789/0 2013-04-26 02:39:34.608345 7f27d600e700 10 mon.1@0(leader).pg v5938164 preprocess_query pg_stats(7 pgs tid 740 v 10556) v1 from osd.2 10.227.227.101:6807/30207 2013-04-26 02:39:34.608440 7f27d600e700 10 mon.1@0(leader).pg v5938164 prepare_update pg_stats(7 pgs tid 740 v 10556) v1 from osd.2 10.227.227.101:6807/30207 2013-04-26 02:39:34.608452 7f27d600e700 10 mon.1@0(leader).pg v5938164 prepare_pg_stats pg_stats(7 pgs tid 740 v 10556) v1 from osd.2 2013-04-26 02:39:34.608466 7f27d600e700 10 mon.1@0(leader).pg v5938164 got osd.2 osd_stat(311 GB used, 1549 GB avail, 1860 GB total, peers [3,4,5,6,7,8,9,10,11]/[]) (was osd_stat(311 GB used, 1549 GB avail, 1860 GB total, peers [3,4,5,6,7,8,9,10,11]/[])) 2013-04-26 02:39:34.699503 7f27d600e700 10 mon.1@0(leader).pg v5938164 preprocess_query pg_stats(4 pgs tid 746 v 10556) v1 from osd.0 10.227.227.101:6801/29982 2013-04-26 02:39:34.699607 7f27d600e700 10 mon.1@0(leader).pg v5938164 prepare_update pg_stats(4 pgs tid 746 v 10556) v1 from osd.0 10.227.227.101:6801/29982 2013-04-26 02:39:34.699621 7f27d600e700 10 mon.1@0(leader).pg v5938164 prepare_pg_stats pg_stats(4 pgs tid 746 v 10556) v1 from osd.0 2013-04-26 02:39:34.699638 7f27d600e700 10 mon.1@0(leader).pg v5938164 got osd.0 osd_stat(367 GB used, 1493 GB avail, 1860 GB total, peers [3,4,5,6,7,8,9,10,11]/[]) (was osd_stat(367 GB used, 1493 GB avail, 1860 GB total, peers [3,4,5,6,7,8,9,10,11]/[])) 2013-04-26 02:39:34.699802 7f27d600e700 10 mon.1@0(leader).log v5604044 update_from_paxos 2013-04-26 02:39:34.699883 7f27d600e700 10 mon.1@0(leader).log v5604044 update_from_paxos version 5604044 summary v 5604044 2013-04-26 02:39:34.699955 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:34.700147 7f27d600e700 10 mon.1@0(leader).log v5604044 update_from_paxos 2013-04-26 02:39:34.700224 7f27d600e700 10 mon.1@0(leader).log v5604044 update_from_paxos version 5604044 summary v 5604044 2013-04-26 02:39:34.700291 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:34.700673 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.1 10.227.227.101:6804/30088 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:34.700689 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x4bcb680 from 10.227.227.103:6789/0 2013-04-26 02:39:34.700801 7f27d600e700 10 mon.1@0(leader).pg v5938164 preprocess_query pg_stats(9 pgs tid 734 v 10556) v1 from osd.1 10.227.227.101:6804/30088 2013-04-26 02:39:34.700898 7f27d600e700 10 mon.1@0(leader).pg v5938164 prepare_update pg_stats(9 pgs tid 734 v 10556) v1 from osd.1 10.227.227.101:6804/30088 2013-04-26 02:39:34.700912 7f27d600e700 10 mon.1@0(leader).pg v5938164 prepare_pg_stats pg_stats(9 pgs tid 734 v 10556) v1 from osd.1 2013-04-26 02:39:34.700934 7f27d600e700 10 mon.1@0(leader).pg v5938164 got osd.1 osd_stat(342 GB used, 1517 GB avail, 1860 GB total, peers [3,4,5,6,7,8,9,10,11]/[]) (was osd_stat(342 GB used, 1517 GB avail, 1860 GB total, peers [3,4,5,6,7,8,9,10,11]/[])) 2013-04-26 02:39:34.721356 7f27d600e700 10 mon.1@0(leader).log v5604044 update_from_paxos 2013-04-26 02:39:34.721437 7f27d600e700 10 mon.1@0(leader).log v5604044 update_from_paxos version 5604044 summary v 5604044 2013-04-26 02:39:34.721505 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:34.721896 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.10 10.227.227.103:6801/8155 via mon.2 10.227.227.104:6789/0 2013-04-26 02:39:34.721915 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x4f36a00 from 10.227.227.104:6789/0 2013-04-26 02:39:34.722020 7f27d600e700 10 mon.1@0(leader).pg v5938164 preprocess_query pg_stats(2 pgs tid 57 v 10556) v1 from osd.10 10.227.227.103:6801/8155 2013-04-26 02:39:34.722105 7f27d600e700 10 mon.1@0(leader).pg v5938164 prepare_update pg_stats(2 pgs tid 57 v 10556) v1 from osd.10 10.227.227.103:6801/8155 2013-04-26 02:39:34.722118 7f27d600e700 10 mon.1@0(leader).pg v5938164 prepare_pg_stats pg_stats(2 pgs tid 57 v 10556) v1 from osd.10 2013-04-26 02:39:34.722131 7f27d600e700 10 mon.1@0(leader).pg v5938164 got osd.10 osd_stat(174 GB used, 755 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[]) (was osd_stat(174 GB used, 755 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[])) 2013-04-26 02:39:34.893713 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.6 10.227.227.103:6807/8402 via mon.2 10.227.227.104:6789/0 2013-04-26 02:39:34.893737 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x459e780 from 10.227.227.104:6789/0 2013-04-26 02:39:34.893894 7f27d600e700 10 mon.1@0(leader).pg v5938164 preprocess_query pg_stats(6 pgs tid 61 v 10556) v1 from osd.6 10.227.227.103:6807/8402 2013-04-26 02:39:34.893999 7f27d600e700 10 mon.1@0(leader).pg v5938164 prepare_update pg_stats(6 pgs tid 61 v 10556) v1 from osd.6 10.227.227.103:6807/8402 2013-04-26 02:39:34.894014 7f27d600e700 10 mon.1@0(leader).pg v5938164 prepare_pg_stats pg_stats(6 pgs tid 61 v 10556) v1 from osd.6 2013-04-26 02:39:34.894031 7f27d600e700 10 mon.1@0(leader).pg v5938164 got osd.6 osd_stat(184 GB used, 745 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[]) (was osd_stat(184 GB used, 745 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[])) 2013-04-26 02:39:35.075482 7f27d4804700 10 mon.1@0(leader) e1 ms_verify_authorizer 10.227.227.103:6800/8011 mds protocol 0 2013-04-26 02:39:35.075974 7f27d600e700 10 mon.1@0(leader) e1 do not have session, making new one 2013-04-26 02:39:35.075987 7f27d600e700 10 mon.1@0(leader) e1 ms_dispatch new session MonSession: mds.? 10.227.227.103:6800/8011 is open for mds.? 10.227.227.103:6800/8011 2013-04-26 02:39:35.075999 7f27d600e700 10 mon.1@0(leader) e1 setting timeout on session 2013-04-26 02:39:35.076139 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:35.076202 7f27d600e700 10 mon.1@0(leader).auth v4370 preprocess_query auth(proto 0 26 bytes epoch 1) v1 from mds.? 10.227.227.103:6800/8011 2013-04-26 02:39:35.076241 7f27d600e700 10 mon.1@0(leader).auth v4370 prep_auth() blob_size=26 2013-04-26 02:39:35.077205 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:35.077250 7f27d600e700 10 mon.1@0(leader).auth v4370 preprocess_query auth(proto 2 128 bytes epoch 0) v1 from mds.? 10.227.227.103:6800/8011 2013-04-26 02:39:35.077273 7f27d600e700 10 mon.1@0(leader).auth v4370 prep_auth() blob_size=128 2013-04-26 02:39:35.078497 7f27d600e700 10 mon.1@0(leader) e1 handle_subscribe mon_subscribe({mdsmap=3725+,monmap=2+}) v2 2013-04-26 02:39:35.078512 7f27d600e700 10 mon.1@0(leader) e1 check_sub monmap next 2 have 1 2013-04-26 02:39:35.347111 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.4 10.227.227.104:6804/14471 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:35.347133 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x4f36f00 from 10.227.227.103:6789/0 2013-04-26 02:39:35.347295 7f27d600e700 10 mon.1@0(leader).pg v5938164 preprocess_query pg_stats(5 pgs tid 750 v 10556) v1 from osd.4 10.227.227.104:6804/14471 2013-04-26 02:39:35.347402 7f27d600e700 10 mon.1@0(leader).pg v5938164 prepare_update pg_stats(5 pgs tid 750 v 10556) v1 from osd.4 10.227.227.104:6804/14471 2013-04-26 02:39:35.347417 7f27d600e700 10 mon.1@0(leader).pg v5938164 prepare_pg_stats pg_stats(5 pgs tid 750 v 10556) v1 from osd.4 2013-04-26 02:39:35.347439 7f27d600e700 10 mon.1@0(leader).pg v5938164 got osd.4 osd_stat(423 GB used, 1437 GB avail, 1860 GB total, peers [0,1,2,6,7,8,9,10,11]/[]) (was osd_stat(423 GB used, 1437 GB avail, 1860 GB total, peers [0,1,2,6,7,8,9,10,11]/[])) 2013-04-26 02:39:35.471596 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.3 10.227.227.104:6800/14463 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:35.471620 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x4c14a00 from 10.227.227.103:6789/0 2013-04-26 02:39:35.471776 7f27d600e700 10 mon.1@0(leader).pg v5938164 preprocess_query pg_stats(4 pgs tid 755 v 10556) v1 from osd.3 10.227.227.104:6800/14463 2013-04-26 02:39:35.471881 7f27d600e700 10 mon.1@0(leader).pg v5938164 prepare_update pg_stats(4 pgs tid 755 v 10556) v1 from osd.3 10.227.227.104:6800/14463 2013-04-26 02:39:35.471895 7f27d600e700 10 mon.1@0(leader).pg v5938164 prepare_pg_stats pg_stats(4 pgs tid 755 v 10556) v1 from osd.3 2013-04-26 02:39:35.471913 7f27d600e700 10 mon.1@0(leader).pg v5938164 got osd.3 osd_stat(383 GB used, 1477 GB avail, 1860 GB total, peers [0,1,2,6,7,8,9,10,11]/[]) (was osd_stat(383 GB used, 1477 GB avail, 1860 GB total, peers [0,1,2,6,7,8,9,10,11]/[])) 2013-04-26 02:39:35.538718 7f27d680f700 10 mon.1@0(leader).log v5604044 encode_full log v 5604044 2013-04-26 02:39:35.539014 7f27d680f700 10 mon.1@0(leader).log v5604044 encode_pending v5604045 2013-04-26 02:39:35.627487 7f27d680f700 10 mon.1@0(leader).pg v5938164 encode_pending v 5938165 2013-04-26 02:39:35.686123 7f27d600e700 10 mon.1@0(leader).log v5604045 update_from_paxos 2013-04-26 02:39:35.686263 7f27d600e700 10 mon.1@0(leader).log v5604045 update_from_paxos version 5604045 summary v 5604044 2013-04-26 02:39:35.686331 7f27d600e700 10 mon.1@0(leader).log v5604045 update_from_paxos latest full 5604044 2013-04-26 02:39:35.686441 7f27d600e700 7 mon.1@0(leader).log v5604045 update_from_paxos applying incremental log 5604045 2013-04-26 02:39:34.591508 mon.0 10.227.227.101:6789/0 32 : [INF] pgmap v5938164: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 267KB/s wr, 11op/s 2013-04-26 02:39:35.686654 7f27d600e700 10 mon.1@0(leader).log v5604045 check_subs 2013-04-26 02:39:35.686758 7f27d600e700 10 mon.1@0(leader).log v5604045 create_pending v 5604046 2013-04-26 02:39:35.686850 7f27d600e700 7 mon.1@0(leader).log v5604045 _updated_log for mon.0 10.227.227.101:6789/0 2013-04-26 02:39:35.999267 7f27d600e700 7 mon.1@0(leader).pg v5938164 update_from_paxos applying incremental 5938165 2013-04-26 02:39:35.999441 7f27d600e700 10 mon.1@0(leader).pg v5938165 v5938165: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 124KB/s wr, 7op/s 2013-04-26 02:39:35.999488 7f27d600e700 10 mon.1@0(leader).pg v5938165 send_pg_creates to 0 pgs 2013-04-26 02:39:35.999492 7f27d600e700 10 mon.1@0(leader).pg v5938165 update_logger 2013-04-26 02:39:35.999531 7f27d600e700 10 mon.1@0(leader).pg v5938165 create_pending v 5938166 2013-04-26 02:39:35.999551 7f27d600e700 7 mon.1@0(leader).pg v5938165 _updated_stats for osd.2 10.227.227.101:6807/30207 2013-04-26 02:39:35.999578 7f27d600e700 7 mon.1@0(leader).pg v5938165 _updated_stats for osd.0 10.227.227.101:6801/29982 2013-04-26 02:39:35.999599 7f27d600e700 7 mon.1@0(leader).pg v5938165 _updated_stats for osd.1 10.227.227.101:6804/30088 2013-04-26 02:39:35.999614 7f27d600e700 7 mon.1@0(leader).pg v5938165 _updated_stats for osd.10 10.227.227.103:6801/8155 2013-04-26 02:39:35.999627 7f27d600e700 7 mon.1@0(leader).pg v5938165 _updated_stats for osd.6 10.227.227.103:6807/8402 2013-04-26 02:39:35.999641 7f27d600e700 7 mon.1@0(leader).pg v5938165 _updated_stats for osd.4 10.227.227.104:6804/14471 2013-04-26 02:39:35.999677 7f27d600e700 7 mon.1@0(leader).pg v5938165 _updated_stats for osd.3 10.227.227.104:6800/14463 2013-04-26 02:39:35.999866 7f27d600e700 10 mon.1@0(leader).pg v5938165 preprocess_query pg_stats(3 pgs tid 60 v 10556) v1 from osd.8 10.227.227.103:6813/8628 2013-04-26 02:39:35.999944 7f27d600e700 10 mon.1@0(leader).pg v5938165 prepare_update pg_stats(3 pgs tid 60 v 10556) v1 from osd.8 10.227.227.103:6813/8628 2013-04-26 02:39:35.999953 7f27d600e700 10 mon.1@0(leader).pg v5938165 prepare_pg_stats pg_stats(3 pgs tid 60 v 10556) v1 from osd.8 2013-04-26 02:39:35.999963 7f27d600e700 10 mon.1@0(leader).pg v5938165 got osd.8 osd_stat(185 GB used, 744 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[]) (was osd_stat(185 GB used, 744 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[])) 2013-04-26 02:39:36.000002 7f27d600e700 10 mon.1@0(leader).pg v5938165 check_osd_map already seen 10556 >= 10556 2013-04-26 02:39:36.000006 7f27d600e700 10 mon.1@0(leader).pg v5938165 update_logger 2013-04-26 02:39:36.000021 7f27d600e700 0 log [INF] : pgmap v5938165: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 124KB/s wr, 7op/s 2013-04-26 02:39:36.000171 7f27d600e700 10 mon.1@0(leader).log v5604045 update_from_paxos 2013-04-26 02:39:36.000224 7f27d600e700 10 mon.1@0(leader).log v5604045 update_from_paxos version 5604045 summary v 5604045 2013-04-26 02:39:36.000265 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:36.000331 7f27d600e700 0 log [WRN] : message from mon.1 was stamped 0.084234s in the future, clocks not synchronized 2013-04-26 02:39:36.000444 7f27d600e700 10 mon.1@0(leader).log v5604045 update_from_paxos 2013-04-26 02:39:36.000488 7f27d600e700 10 mon.1@0(leader).log v5604045 update_from_paxos version 5604045 summary v 5604045 2013-04-26 02:39:36.000525 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:36.000608 7f27d600e700 10 mon.1@0(leader).log v5604045 update_from_paxos 2013-04-26 02:39:36.000646 7f27d600e700 10 mon.1@0(leader).log v5604045 update_from_paxos version 5604045 summary v 5604045 2013-04-26 02:39:36.000667 7f27d600e700 10 mon.1@0(leader).log v5604045 preprocess_query log(1 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:36.000692 7f27d600e700 10 mon.1@0(leader).log v5604045 preprocess_log log(1 entries) v1 from mon.0 2013-04-26 02:39:36.000727 7f27d600e700 10 mon.1@0(leader).log v5604045 prepare_update log(1 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:36.000754 7f27d600e700 10 mon.1@0(leader).log v5604045 prepare_log log(1 entries) v1 from mon.0 2013-04-26 02:39:36.000778 7f27d600e700 10 mon.1@0(leader).log v5604045 logging 2013-04-26 02:39:36.000024 mon.0 10.227.227.101:6789/0 33 : [INF] pgmap v5938165: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 124KB/s wr, 7op/s 2013-04-26 02:39:36.000889 7f27d600e700 10 mon.1@0(leader).log v5604045 update_from_paxos 2013-04-26 02:39:36.000927 7f27d600e700 10 mon.1@0(leader).log v5604045 update_from_paxos version 5604045 summary v 5604045 2013-04-26 02:39:36.000947 7f27d600e700 10 mon.1@0(leader).log v5604045 preprocess_query log(1 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:36.000973 7f27d600e700 10 mon.1@0(leader).log v5604045 preprocess_log log(1 entries) v1 from mon.0 2013-04-26 02:39:36.000999 7f27d600e700 10 mon.1@0(leader).log v5604045 nothing new 2013-04-26 02:39:36.057775 7f27d600e700 10 mon.1@0(leader).log v5604045 update_from_paxos 2013-04-26 02:39:36.057865 7f27d600e700 10 mon.1@0(leader).log v5604045 update_from_paxos version 5604045 summary v 5604045 2013-04-26 02:39:36.057935 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:36.058263 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from mds.? 10.227.227.101:6800/29852 via mon.2 10.227.227.104:6789/0 2013-04-26 02:39:36.058283 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x1789080 from 10.227.227.104:6789/0 2013-04-26 02:39:36.058401 7f27d600e700 10 mon.1@0(leader).mds e3724 preprocess_query mdsbeacon(217349/1 up:standby seq 464 v3724) v2 from mds.? 10.227.227.101:6800/29852 2013-04-26 02:39:36.503133 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from mds.0 10.227.227.104:6801/10842 via mon.2 10.227.227.104:6789/0 2013-04-26 02:39:36.503155 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x1788dc0 from 10.227.227.104:6789/0 2013-04-26 02:39:36.503339 7f27d600e700 10 mon.1@0(leader).mds e3724 preprocess_query mdsbeacon(217231/4 up:active seq 495 v3724) v2 from mds.0 10.227.227.104:6801/10842 2013-04-26 02:39:36.958770 7f27d680f700 10 mon.1@0(leader).log v5604045 encode_full log v 5604045 2013-04-26 02:39:36.959066 7f27d680f700 10 mon.1@0(leader).log v5604045 encode_pending v5604046 2013-04-26 02:39:37.061658 7f27d680f700 10 mon.1@0(leader).pg v5938165 encode_pending v 5938166 2013-04-26 02:39:37.128622 7f27d600e700 10 mon.1@0(leader).log v5604046 update_from_paxos 2013-04-26 02:39:37.128720 7f27d600e700 10 mon.1@0(leader).log v5604046 update_from_paxos version 5604046 summary v 5604045 2013-04-26 02:39:37.128764 7f27d600e700 10 mon.1@0(leader).log v5604046 update_from_paxos latest full 5604045 2013-04-26 02:39:37.128816 7f27d600e700 7 mon.1@0(leader).log v5604046 update_from_paxos applying incremental log 5604046 2013-04-26 02:39:36.000024 mon.0 10.227.227.101:6789/0 33 : [INF] pgmap v5938165: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 124KB/s wr, 7op/s 2013-04-26 02:39:37.128907 7f27d600e700 10 mon.1@0(leader).log v5604046 check_subs 2013-04-26 02:39:37.128981 7f27d600e700 10 mon.1@0(leader).log v5604046 create_pending v 5604047 2013-04-26 02:39:37.129039 7f27d600e700 7 mon.1@0(leader).log v5604046 _updated_log for mon.0 10.227.227.101:6789/0 2013-04-26 02:39:37.196553 7f27d600e700 7 mon.1@0(leader).pg v5938165 update_from_paxos applying incremental 5938166 2013-04-26 02:39:37.196611 7f27d600e700 10 mon.1@0(leader).pg v5938166 v5938166: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 52659B/s wr, 6op/s 2013-04-26 02:39:37.196672 7f27d600e700 10 mon.1@0(leader).pg v5938166 send_pg_creates to 0 pgs 2013-04-26 02:39:37.196678 7f27d600e700 10 mon.1@0(leader).pg v5938166 update_logger 2013-04-26 02:39:37.196718 7f27d600e700 10 mon.1@0(leader).pg v5938166 create_pending v 5938167 2013-04-26 02:39:37.196751 7f27d600e700 7 mon.1@0(leader).pg v5938166 _updated_stats for osd.8 10.227.227.103:6813/8628 2013-04-26 02:39:37.196792 7f27d600e700 10 mon.1@0(leader).pg v5938166 check_osd_map already seen 10556 >= 10556 2013-04-26 02:39:37.196798 7f27d600e700 10 mon.1@0(leader).pg v5938166 update_logger 2013-04-26 02:39:37.196820 7f27d600e700 0 log [INF] : pgmap v5938166: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 52659B/s wr, 6op/s 2013-04-26 02:39:37.197067 7f27d600e700 10 mon.1@0(leader).log v5604046 update_from_paxos 2013-04-26 02:39:37.197160 7f27d600e700 10 mon.1@0(leader).log v5604046 update_from_paxos version 5604046 summary v 5604046 2013-04-26 02:39:37.197233 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:37.197456 7f27d600e700 10 mon.1@0(leader).log v5604046 update_from_paxos 2013-04-26 02:39:37.197536 7f27d600e700 10 mon.1@0(leader).log v5604046 update_from_paxos version 5604046 summary v 5604046 2013-04-26 02:39:37.197630 7f27d600e700 10 mon.1@0(leader).log v5604046 preprocess_query log(2 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:37.197692 7f27d600e700 10 mon.1@0(leader).log v5604046 preprocess_log log(2 entries) v1 from mon.0 2013-04-26 02:39:37.197748 7f27d600e700 10 mon.1@0(leader).log v5604046 prepare_update log(2 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:37.197791 7f27d600e700 10 mon.1@0(leader).log v5604046 prepare_log log(2 entries) v1 from mon.0 2013-04-26 02:39:37.197833 7f27d600e700 10 mon.1@0(leader).log v5604046 logging 2013-04-26 02:39:36.000333 mon.0 10.227.227.101:6789/0 34 : [WRN] message from mon.1 was stamped 0.084234s in the future, clocks not synchronized 2013-04-26 02:39:37.197890 7f27d600e700 10 mon.1@0(leader).log v5604046 logging 2013-04-26 02:39:37.196825 mon.0 10.227.227.101:6789/0 35 : [INF] pgmap v5938166: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 52659B/s wr, 6op/s 2013-04-26 02:39:37.198119 7f27d600e700 10 mon.1@0(leader).log v5604046 update_from_paxos 2013-04-26 02:39:37.198189 7f27d600e700 10 mon.1@0(leader).log v5604046 update_from_paxos version 5604046 summary v 5604046 2013-04-26 02:39:37.198260 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:37.216078 7f27d600e700 10 mon.1@0(leader).mds e3724 preprocess_query mdsbeacon(217445/3 up:standby seq 77 v3724) v2 from mds.? 10.227.227.103:6800/8011 2013-04-26 02:39:37.257858 7f27d600e700 10 mon.1@0(leader).log v5604046 update_from_paxos 2013-04-26 02:39:37.257939 7f27d600e700 10 mon.1@0(leader).log v5604046 update_from_paxos version 5604046 summary v 5604046 2013-04-26 02:39:37.258008 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:37.759057 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.5 10.227.227.104:6807/14479 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:37.759083 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x4f36280 from 10.227.227.103:6789/0 2013-04-26 02:39:37.759236 7f27d600e700 10 mon.1@0(leader).pg v5938166 preprocess_query pg_stats(5 pgs tid 748 v 10556) v1 from osd.5 10.227.227.104:6807/14479 2013-04-26 02:39:37.759340 7f27d600e700 10 mon.1@0(leader).pg v5938166 prepare_update pg_stats(5 pgs tid 748 v 10556) v1 from osd.5 10.227.227.104:6807/14479 2013-04-26 02:39:37.759355 7f27d600e700 10 mon.1@0(leader).pg v5938166 prepare_pg_stats pg_stats(5 pgs tid 748 v 10556) v1 from osd.5 2013-04-26 02:39:37.759369 7f27d600e700 10 mon.1@0(leader).pg v5938166 got osd.5 osd_stat(355 GB used, 1505 GB avail, 1860 GB total, peers [0,1,2,6,7,8,9,10,11]/[]) (was osd_stat(355 GB used, 1505 GB avail, 1860 GB total, peers [0,1,2,6,7,8,9,10,11]/[])) 2013-04-26 02:39:38.148683 7f27d680f700 10 mon.1@0(leader).log v5604046 encode_full log v 5604046 2013-04-26 02:39:38.150015 7f27d680f700 10 mon.1@0(leader).log v5604046 encode_pending v5604047 2013-04-26 02:39:38.237243 7f27d680f700 10 mon.1@0(leader).pg v5938166 encode_pending v 5938167 2013-04-26 02:39:38.312387 7f27d600e700 10 mon.1@0(leader).log v5604047 update_from_paxos 2013-04-26 02:39:38.312566 7f27d600e700 10 mon.1@0(leader).log v5604047 update_from_paxos version 5604047 summary v 5604046 2013-04-26 02:39:38.312709 7f27d600e700 10 mon.1@0(leader).log v5604047 update_from_paxos latest full 5604046 2013-04-26 02:39:38.312795 7f27d600e700 7 mon.1@0(leader).log v5604047 update_from_paxos applying incremental log 5604047 2013-04-26 02:39:36.000333 mon.0 10.227.227.101:6789/0 34 : [WRN] message from mon.1 was stamped 0.084234s in the future, clocks not synchronized 2013-04-26 02:39:38.312937 7f27d600e700 7 mon.1@0(leader).log v5604047 update_from_paxos applying incremental log 5604047 2013-04-26 02:39:37.196825 mon.0 10.227.227.101:6789/0 35 : [INF] pgmap v5938166: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 52659B/s wr, 6op/s 2013-04-26 02:39:38.313076 7f27d600e700 10 mon.1@0(leader).log v5604047 check_subs 2013-04-26 02:39:38.313164 7f27d600e700 10 mon.1@0(leader).log v5604047 create_pending v 5604048 2013-04-26 02:39:38.313240 7f27d600e700 7 mon.1@0(leader).log v5604047 _updated_log for mon.0 10.227.227.101:6789/0 2013-04-26 02:39:38.429191 7f27d600e700 7 mon.1@0(leader).pg v5938166 update_from_paxos applying incremental 5938167 2013-04-26 02:39:38.429242 7f27d600e700 10 mon.1@0(leader).pg v5938167 v5938167: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 36098B/s wr, 3op/s 2013-04-26 02:39:38.429280 7f27d600e700 10 mon.1@0(leader).pg v5938167 send_pg_creates to 0 pgs 2013-04-26 02:39:38.429285 7f27d600e700 10 mon.1@0(leader).pg v5938167 update_logger 2013-04-26 02:39:38.429311 7f27d600e700 10 mon.1@0(leader).pg v5938167 create_pending v 5938168 2013-04-26 02:39:38.429333 7f27d600e700 7 mon.1@0(leader).pg v5938167 _updated_stats for osd.5 10.227.227.104:6807/14479 2013-04-26 02:39:38.429364 7f27d600e700 10 mon.1@0(leader).pg v5938167 check_osd_map already seen 10556 >= 10556 2013-04-26 02:39:38.429366 7f27d600e700 10 mon.1@0(leader).pg v5938167 update_logger 2013-04-26 02:39:38.429380 7f27d600e700 0 log [INF] : pgmap v5938167: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 36098B/s wr, 3op/s 2013-04-26 02:39:38.429541 7f27d600e700 10 mon.1@0(leader).log v5604047 update_from_paxos 2013-04-26 02:39:38.429595 7f27d600e700 10 mon.1@0(leader).log v5604047 update_from_paxos version 5604047 summary v 5604047 2013-04-26 02:39:38.429636 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:38.429728 7f27d600e700 10 mon.1@0(leader).log v5604047 update_from_paxos 2013-04-26 02:39:38.429767 7f27d600e700 10 mon.1@0(leader).log v5604047 update_from_paxos version 5604047 summary v 5604047 2013-04-26 02:39:38.429799 7f27d600e700 10 mon.1@0(leader).log v5604047 preprocess_query log(1 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:38.429823 7f27d600e700 10 mon.1@0(leader).log v5604047 preprocess_log log(1 entries) v1 from mon.0 2013-04-26 02:39:38.429861 7f27d600e700 10 mon.1@0(leader).log v5604047 prepare_update log(1 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:38.429889 7f27d600e700 10 mon.1@0(leader).log v5604047 prepare_log log(1 entries) v1 from mon.0 2013-04-26 02:39:38.429912 7f27d600e700 10 mon.1@0(leader).log v5604047 logging 2013-04-26 02:39:38.429383 mon.0 10.227.227.101:6789/0 36 : [INF] pgmap v5938167: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 36098B/s wr, 3op/s 2013-04-26 02:39:38.439066 7f27d600e700 10 mon.1@0(leader).log v5604047 update_from_paxos 2013-04-26 02:39:38.439149 7f27d600e700 10 mon.1@0(leader).log v5604047 update_from_paxos version 5604047 summary v 5604047 2013-04-26 02:39:38.439219 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:38.483637 7f27d600e700 10 mon.1@0(leader).log v5604047 update_from_paxos 2013-04-26 02:39:38.483795 7f27d600e700 10 mon.1@0(leader).log v5604047 update_from_paxos version 5604047 summary v 5604047 2013-04-26 02:39:38.483876 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:38.895979 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.11 10.227.227.103:6804/8253 via mon.2 10.227.227.104:6789/0 2013-04-26 02:39:38.896005 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x459fb80 from 10.227.227.104:6789/0 2013-04-26 02:39:38.896165 7f27d600e700 10 mon.1@0(leader).pg v5938167 preprocess_query pg_stats(10 pgs tid 63 v 10556) v1 from osd.11 10.227.227.103:6804/8253 2013-04-26 02:39:38.896269 7f27d600e700 10 mon.1@0(leader).pg v5938167 prepare_update pg_stats(10 pgs tid 63 v 10556) v1 from osd.11 10.227.227.103:6804/8253 2013-04-26 02:39:38.896284 7f27d600e700 10 mon.1@0(leader).pg v5938167 prepare_pg_stats pg_stats(10 pgs tid 63 v 10556) v1 from osd.11 2013-04-26 02:39:38.896299 7f27d600e700 10 mon.1@0(leader).pg v5938167 got osd.11 osd_stat(161 GB used, 768 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[]) (was osd_stat(161 GB used, 768 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[])) 2013-04-26 02:39:39.278659 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.7 10.227.227.103:6810/8502 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:39.278683 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x4f36f00 from 10.227.227.103:6789/0 2013-04-26 02:39:39.278851 7f27d600e700 10 mon.1@0(leader).pg v5938167 preprocess_query pg_stats(1 pgs tid 59 v 10556) v1 from osd.7 10.227.227.103:6810/8502 2013-04-26 02:39:39.278954 7f27d600e700 10 mon.1@0(leader).pg v5938167 prepare_update pg_stats(1 pgs tid 59 v 10556) v1 from osd.7 10.227.227.103:6810/8502 2013-04-26 02:39:39.278968 7f27d600e700 10 mon.1@0(leader).pg v5938167 prepare_pg_stats pg_stats(1 pgs tid 59 v 10556) v1 from osd.7 2013-04-26 02:39:39.278984 7f27d600e700 10 mon.1@0(leader).pg v5938167 got osd.7 osd_stat(156 GB used, 772 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[]) (was osd_stat(156 GB used, 772 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[])) 2013-04-26 02:39:39.304041 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.9 10.227.227.103:6816/8752 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:39.304060 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x4f36500 from 10.227.227.103:6789/0 2013-04-26 02:39:39.304175 7f27d600e700 10 mon.1@0(leader).pg v5938167 preprocess_query pg_stats(7 pgs tid 62 v 10556) v1 from osd.9 10.227.227.103:6816/8752 2013-04-26 02:39:39.304261 7f27d600e700 10 mon.1@0(leader).pg v5938167 prepare_update pg_stats(7 pgs tid 62 v 10556) v1 from osd.9 10.227.227.103:6816/8752 2013-04-26 02:39:39.304275 7f27d600e700 10 mon.1@0(leader).pg v5938167 prepare_pg_stats pg_stats(7 pgs tid 62 v 10556) v1 from osd.9 2013-04-26 02:39:39.304290 7f27d600e700 10 mon.1@0(leader).pg v5938167 got osd.9 osd_stat(161 GB used, 767 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[]) (was osd_stat(161 GB used, 767 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[])) 2013-04-26 02:39:39.388282 7f27d680f700 10 mon.1@0(leader).log v5604047 encode_full log v 5604047 2013-04-26 02:39:39.388609 7f27d680f700 10 mon.1@0(leader).log v5604047 encode_pending v5604048 2013-04-26 02:39:39.471167 7f27d680f700 10 mon.1@0(leader).pg v5938167 encode_pending v 5938168 2013-04-26 02:39:39.471558 7f27d680f700 10 mon.1@0(leader).mds e3724 e3724: 1/1/1 up {0=4=up:active}, 2 up:standby 2013-04-26 02:39:39.471646 7f27d680f700 10 mon.1@0(leader).osd e10556 e10556: 12 osds: 12 up, 12 in 2013-04-26 02:39:39.471793 7f27d680f700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:39.471890 7f27d680f700 10 mon.1@0(leader).auth v4370 auth 2013-04-26 02:39:39.524316 7f27d600e700 10 mon.1@0(leader).log v5604048 update_from_paxos 2013-04-26 02:39:39.524429 7f27d600e700 10 mon.1@0(leader).log v5604048 update_from_paxos version 5604048 summary v 5604047 2013-04-26 02:39:39.524497 7f27d600e700 10 mon.1@0(leader).log v5604048 update_from_paxos latest full 5604047 2013-04-26 02:39:39.524548 7f27d600e700 7 mon.1@0(leader).log v5604048 update_from_paxos applying incremental log 5604048 2013-04-26 02:39:38.429383 mon.0 10.227.227.101:6789/0 36 : [INF] pgmap v5938167: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 36098B/s wr, 3op/s 2013-04-26 02:39:39.524655 7f27d600e700 10 mon.1@0(leader).log v5604048 check_subs 2013-04-26 02:39:39.524759 7f27d600e700 10 mon.1@0(leader).log v5604048 create_pending v 5604049 2013-04-26 02:39:39.524843 7f27d600e700 7 mon.1@0(leader).log v5604048 _updated_log for mon.0 10.227.227.101:6789/0 2013-04-26 02:39:39.580010 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.2 10.227.227.101:6807/30207 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:39.580075 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x459f900 from 10.227.227.103:6789/0 2013-04-26 02:39:39.580199 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.10 10.227.227.103:6801/8155 via mon.2 10.227.227.104:6789/0 2013-04-26 02:39:39.580214 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x459f400 from 10.227.227.104:6789/0 2013-04-26 02:39:39.649645 7f27d600e700 7 mon.1@0(leader).pg v5938167 update_from_paxos applying incremental 5938168 2013-04-26 02:39:39.649792 7f27d600e700 10 mon.1@0(leader).pg v5938168 v5938168: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 849KB/s wr, 12op/s 2013-04-26 02:39:39.649857 7f27d600e700 10 mon.1@0(leader).pg v5938168 send_pg_creates to 0 pgs 2013-04-26 02:39:39.649863 7f27d600e700 10 mon.1@0(leader).pg v5938168 update_logger 2013-04-26 02:39:39.649910 7f27d600e700 10 mon.1@0(leader).pg v5938168 create_pending v 5938169 2013-04-26 02:39:39.649943 7f27d600e700 7 mon.1@0(leader).pg v5938168 _updated_stats for osd.11 10.227.227.103:6804/8253 2013-04-26 02:39:39.649984 7f27d600e700 7 mon.1@0(leader).pg v5938168 _updated_stats for osd.7 10.227.227.103:6810/8502 2013-04-26 02:39:39.650008 7f27d600e700 7 mon.1@0(leader).pg v5938168 _updated_stats for osd.9 10.227.227.103:6816/8752 2013-04-26 02:39:39.650164 7f27d600e700 10 mon.1@0(leader).pg v5938168 preprocess_query pg_stats(6 pgs tid 741 v 10556) v1 from osd.2 10.227.227.101:6807/30207 2013-04-26 02:39:39.650334 7f27d600e700 10 mon.1@0(leader).pg v5938168 prepare_update pg_stats(6 pgs tid 741 v 10556) v1 from osd.2 10.227.227.101:6807/30207 2013-04-26 02:39:39.650354 7f27d600e700 10 mon.1@0(leader).pg v5938168 prepare_pg_stats pg_stats(6 pgs tid 741 v 10556) v1 from osd.2 2013-04-26 02:39:39.650369 7f27d600e700 10 mon.1@0(leader).pg v5938168 got osd.2 osd_stat(311 GB used, 1549 GB avail, 1860 GB total, peers [3,4,5,6,7,8,9,10,11]/[]) (was osd_stat(311 GB used, 1549 GB avail, 1860 GB total, peers [3,4,5,6,7,8,9,10,11]/[])) 2013-04-26 02:39:39.650515 7f27d600e700 10 mon.1@0(leader).pg v5938168 preprocess_query pg_stats(6 pgs tid 58 v 10556) v1 from osd.10 10.227.227.103:6801/8155 2013-04-26 02:39:39.650652 7f27d600e700 10 mon.1@0(leader).pg v5938168 prepare_update pg_stats(6 pgs tid 58 v 10556) v1 from osd.10 10.227.227.103:6801/8155 2013-04-26 02:39:39.650671 7f27d600e700 10 mon.1@0(leader).pg v5938168 prepare_pg_stats pg_stats(6 pgs tid 58 v 10556) v1 from osd.10 2013-04-26 02:39:39.650681 7f27d600e700 10 mon.1@0(leader).pg v5938168 got osd.10 osd_stat(174 GB used, 755 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[]) (was osd_stat(174 GB used, 755 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[])) 2013-04-26 02:39:39.650747 7f27d600e700 10 mon.1@0(leader).pg v5938168 check_osd_map already seen 10556 >= 10556 2013-04-26 02:39:39.650753 7f27d600e700 10 mon.1@0(leader).pg v5938168 update_logger 2013-04-26 02:39:39.650786 7f27d600e700 0 log [INF] : pgmap v5938168: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 849KB/s wr, 12op/s 2013-04-26 02:39:39.651017 7f27d600e700 10 mon.1@0(leader).log v5604048 update_from_paxos 2013-04-26 02:39:39.651098 7f27d600e700 10 mon.1@0(leader).log v5604048 update_from_paxos version 5604048 summary v 5604048 2013-04-26 02:39:39.651166 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:39.651374 7f27d600e700 10 mon.1@0(leader).log v5604048 update_from_paxos 2013-04-26 02:39:39.651445 7f27d600e700 10 mon.1@0(leader).log v5604048 update_from_paxos version 5604048 summary v 5604048 2013-04-26 02:39:39.651514 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:39.651664 7f27d600e700 10 mon.1@0(leader).log v5604048 update_from_paxos 2013-04-26 02:39:39.651730 7f27d600e700 10 mon.1@0(leader).log v5604048 update_from_paxos version 5604048 summary v 5604048 2013-04-26 02:39:39.651766 7f27d600e700 10 mon.1@0(leader).log v5604048 preprocess_query log(1 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:39.651810 7f27d600e700 10 mon.1@0(leader).log v5604048 preprocess_log log(1 entries) v1 from mon.0 2013-04-26 02:39:39.651863 7f27d600e700 10 mon.1@0(leader).log v5604048 prepare_update log(1 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:39.651909 7f27d600e700 10 mon.1@0(leader).log v5604048 prepare_log log(1 entries) v1 from mon.0 2013-04-26 02:39:39.651950 7f27d600e700 10 mon.1@0(leader).log v5604048 logging 2013-04-26 02:39:39.650789 mon.0 10.227.227.101:6789/0 37 : [INF] pgmap v5938168: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 849KB/s wr, 12op/s 2013-04-26 02:39:39.652071 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.1 10.227.227.101:6804/30088 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:39.652086 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x459f180 from 10.227.227.103:6789/0 2013-04-26 02:39:39.652177 7f27d600e700 10 mon.1@0(leader).pg v5938168 preprocess_query pg_stats(7 pgs tid 735 v 10556) v1 from osd.1 10.227.227.101:6804/30088 2013-04-26 02:39:39.652314 7f27d600e700 10 mon.1@0(leader).pg v5938168 prepare_update pg_stats(7 pgs tid 735 v 10556) v1 from osd.1 10.227.227.101:6804/30088 2013-04-26 02:39:39.652333 7f27d600e700 10 mon.1@0(leader).pg v5938168 prepare_pg_stats pg_stats(7 pgs tid 735 v 10556) v1 from osd.1 2013-04-26 02:39:39.652343 7f27d600e700 10 mon.1@0(leader).pg v5938168 got osd.1 osd_stat(342 GB used, 1517 GB avail, 1860 GB total, peers [3,4,5,6,7,8,9,10,11]/[]) (was osd_stat(342 GB used, 1517 GB avail, 1860 GB total, peers [3,4,5,6,7,8,9,10,11]/[])) 2013-04-26 02:39:39.688695 7f27d600e700 10 mon.1@0(leader).pg v5938168 preprocess_query pg_stats(14 pgs tid 747 v 10556) v1 from osd.0 10.227.227.101:6801/29982 2013-04-26 02:39:39.688798 7f27d600e700 10 mon.1@0(leader).pg v5938168 prepare_update pg_stats(14 pgs tid 747 v 10556) v1 from osd.0 10.227.227.101:6801/29982 2013-04-26 02:39:39.688811 7f27d600e700 10 mon.1@0(leader).pg v5938168 prepare_pg_stats pg_stats(14 pgs tid 747 v 10556) v1 from osd.0 2013-04-26 02:39:39.688824 7f27d600e700 10 mon.1@0(leader).pg v5938168 got osd.0 osd_stat(367 GB used, 1493 GB avail, 1860 GB total, peers [3,4,5,6,7,8,9,10,11]/[]) (was osd_stat(367 GB used, 1493 GB avail, 1860 GB total, peers [3,4,5,6,7,8,9,10,11]/[])) 2013-04-26 02:39:39.708150 7f27d600e700 10 mon.1@0(leader).log v5604048 update_from_paxos 2013-04-26 02:39:39.708232 7f27d600e700 10 mon.1@0(leader).log v5604048 update_from_paxos version 5604048 summary v 5604048 2013-04-26 02:39:39.708301 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:39.895061 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.6 10.227.227.103:6807/8402 via mon.2 10.227.227.104:6789/0 2013-04-26 02:39:39.895079 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x4f36a00 from 10.227.227.104:6789/0 2013-04-26 02:39:39.895222 7f27d600e700 10 mon.1@0(leader).pg v5938168 preprocess_query pg_stats(11 pgs tid 62 v 10556) v1 from osd.6 10.227.227.103:6807/8402 2013-04-26 02:39:39.895313 7f27d600e700 10 mon.1@0(leader).pg v5938168 prepare_update pg_stats(11 pgs tid 62 v 10556) v1 from osd.6 10.227.227.103:6807/8402 2013-04-26 02:39:39.895327 7f27d600e700 10 mon.1@0(leader).pg v5938168 prepare_pg_stats pg_stats(11 pgs tid 62 v 10556) v1 from osd.6 2013-04-26 02:39:39.895340 7f27d600e700 10 mon.1@0(leader).pg v5938168 got osd.6 osd_stat(184 GB used, 745 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[]) (was osd_stat(184 GB used, 745 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[])) 2013-04-26 02:39:40.030972 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from mds.? 10.227.227.101:6800/29852 via mon.2 10.227.227.104:6789/0 2013-04-26 02:39:40.031000 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x1789080 from 10.227.227.104:6789/0 2013-04-26 02:39:40.031170 7f27d600e700 10 mon.1@0(leader).mds e3724 preprocess_query mdsbeacon(217349/1 up:standby seq 465 v3724) v2 from mds.? 10.227.227.101:6800/29852 2013-04-26 02:39:40.348358 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.4 10.227.227.104:6804/14471 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:40.348380 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x4f37680 from 10.227.227.103:6789/0 2013-04-26 02:39:40.348546 7f27d600e700 10 mon.1@0(leader).pg v5938168 preprocess_query pg_stats(13 pgs tid 751 v 10556) v1 from osd.4 10.227.227.104:6804/14471 2013-04-26 02:39:40.348712 7f27d600e700 10 mon.1@0(leader).pg v5938168 prepare_update pg_stats(13 pgs tid 751 v 10556) v1 from osd.4 10.227.227.104:6804/14471 2013-04-26 02:39:40.348732 7f27d600e700 10 mon.1@0(leader).pg v5938168 prepare_pg_stats pg_stats(13 pgs tid 751 v 10556) v1 from osd.4 2013-04-26 02:39:40.348748 7f27d600e700 10 mon.1@0(leader).pg v5938168 got osd.4 osd_stat(423 GB used, 1437 GB avail, 1860 GB total, peers [0,1,2,6,7,8,9,10,11]/[]) (was osd_stat(423 GB used, 1437 GB avail, 1860 GB total, peers [0,1,2,6,7,8,9,10,11]/[])) 2013-04-26 02:39:40.472850 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.3 10.227.227.104:6800/14463 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:40.472872 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x4f37400 from 10.227.227.103:6789/0 2013-04-26 02:39:40.473033 7f27d600e700 10 mon.1@0(leader).pg v5938168 preprocess_query pg_stats(12 pgs tid 756 v 10556) v1 from osd.3 10.227.227.104:6800/14463 2013-04-26 02:39:40.473140 7f27d600e700 10 mon.1@0(leader).pg v5938168 prepare_update pg_stats(12 pgs tid 756 v 10556) v1 from osd.3 10.227.227.104:6800/14463 2013-04-26 02:39:40.473155 7f27d600e700 10 mon.1@0(leader).pg v5938168 prepare_pg_stats pg_stats(12 pgs tid 756 v 10556) v1 from osd.3 2013-04-26 02:39:40.473171 7f27d600e700 10 mon.1@0(leader).pg v5938168 got osd.3 osd_stat(383 GB used, 1477 GB avail, 1860 GB total, peers [0,1,2,6,7,8,9,10,11]/[]) (was osd_stat(383 GB used, 1477 GB avail, 1860 GB total, peers [0,1,2,6,7,8,9,10,11]/[])) 2013-04-26 02:39:40.503272 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from mds.0 10.227.227.104:6801/10842 via mon.2 10.227.227.104:6789/0 2013-04-26 02:39:40.503294 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x1789600 from 10.227.227.104:6789/0 2013-04-26 02:39:40.503454 7f27d600e700 10 mon.1@0(leader).mds e3724 preprocess_query mdsbeacon(217231/4 up:active seq 496 v3724) v2 from mds.0 10.227.227.104:6801/10842 2013-04-26 02:39:40.598991 7f27d680f700 10 mon.1@0(leader).pg v5938168 encode_pending v 5938169 2013-04-26 02:39:40.663547 7f27d680f700 10 mon.1@0(leader).log v5604048 encode_full log v 5604048 2013-04-26 02:39:40.663841 7f27d680f700 10 mon.1@0(leader).log v5604048 encode_pending v5604049 2013-04-26 02:39:40.724182 7f27d600e700 7 mon.1@0(leader).pg v5938168 update_from_paxos applying incremental 5938169 2013-04-26 02:39:40.724577 7f27d600e700 10 mon.1@0(leader).pg v5938169 v5938169: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 2185KB/s wr, 38op/s 2013-04-26 02:39:40.724648 7f27d600e700 10 mon.1@0(leader).pg v5938169 send_pg_creates to 0 pgs 2013-04-26 02:39:40.724653 7f27d600e700 10 mon.1@0(leader).pg v5938169 update_logger 2013-04-26 02:39:40.724717 7f27d600e700 10 mon.1@0(leader).pg v5938169 create_pending v 5938170 2013-04-26 02:39:40.724751 7f27d600e700 7 mon.1@0(leader).pg v5938169 _updated_stats for osd.2 10.227.227.101:6807/30207 2013-04-26 02:39:40.724786 7f27d600e700 7 mon.1@0(leader).pg v5938169 _updated_stats for osd.10 10.227.227.103:6801/8155 2013-04-26 02:39:40.724808 7f27d600e700 7 mon.1@0(leader).pg v5938169 _updated_stats for osd.1 10.227.227.101:6804/30088 2013-04-26 02:39:40.724826 7f27d600e700 7 mon.1@0(leader).pg v5938169 _updated_stats for osd.0 10.227.227.101:6801/29982 2013-04-26 02:39:40.724913 7f27d600e700 7 mon.1@0(leader).pg v5938169 _updated_stats for osd.6 10.227.227.103:6807/8402 2013-04-26 02:39:40.724946 7f27d600e700 7 mon.1@0(leader).pg v5938169 _updated_stats for osd.4 10.227.227.104:6804/14471 2013-04-26 02:39:40.725134 7f27d600e700 7 mon.1@0(leader).pg v5938169 _updated_stats for osd.3 10.227.227.104:6800/14463 2013-04-26 02:39:40.725184 7f27d600e700 10 mon.1@0(leader).pg v5938169 check_osd_map already seen 10556 >= 10556 2013-04-26 02:39:40.725189 7f27d600e700 10 mon.1@0(leader).pg v5938169 update_logger 2013-04-26 02:39:40.725212 7f27d600e700 0 log [INF] : pgmap v5938169: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 2185KB/s wr, 38op/s 2013-04-26 02:39:40.816048 7f27d600e700 10 mon.1@0(leader).log v5604049 update_from_paxos 2013-04-26 02:39:40.816169 7f27d600e700 10 mon.1@0(leader).log v5604049 update_from_paxos version 5604049 summary v 5604048 2013-04-26 02:39:40.816243 7f27d600e700 10 mon.1@0(leader).log v5604049 update_from_paxos latest full 5604048 2013-04-26 02:39:40.816324 7f27d600e700 7 mon.1@0(leader).log v5604049 update_from_paxos applying incremental log 5604049 2013-04-26 02:39:39.650789 mon.0 10.227.227.101:6789/0 37 : [INF] pgmap v5938168: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 849KB/s wr, 12op/s 2013-04-26 02:39:40.816503 7f27d600e700 10 mon.1@0(leader).log v5604049 check_subs 2013-04-26 02:39:40.816610 7f27d600e700 10 mon.1@0(leader).log v5604049 create_pending v 5604050 2013-04-26 02:39:40.816726 7f27d600e700 7 mon.1@0(leader).log v5604049 _updated_log for mon.0 10.227.227.101:6789/0 2013-04-26 02:39:40.816856 7f27d600e700 10 mon.1@0(leader).log v5604049 update_from_paxos 2013-04-26 02:39:40.816922 7f27d600e700 10 mon.1@0(leader).log v5604049 update_from_paxos version 5604049 summary v 5604049 2013-04-26 02:39:40.816960 7f27d600e700 10 mon.1@0(leader).log v5604049 preprocess_query log(1 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:40.817003 7f27d600e700 10 mon.1@0(leader).log v5604049 preprocess_log log(1 entries) v1 from mon.0 2013-04-26 02:39:40.817047 7f27d600e700 10 mon.1@0(leader).log v5604049 nothing new 2013-04-26 02:39:40.817268 7f27d600e700 10 mon.1@0(leader).log v5604049 update_from_paxos 2013-04-26 02:39:40.817337 7f27d600e700 10 mon.1@0(leader).log v5604049 update_from_paxos version 5604049 summary v 5604049 2013-04-26 02:39:40.817408 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:40.819005 7f27d600e700 10 mon.1@0(leader).log v5604049 update_from_paxos 2013-04-26 02:39:40.819081 7f27d600e700 10 mon.1@0(leader).log v5604049 update_from_paxos version 5604049 summary v 5604049 2013-04-26 02:39:40.819157 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:40.858150 7f27d600e700 10 mon.1@0(leader).log v5604049 update_from_paxos 2013-04-26 02:39:40.858228 7f27d600e700 10 mon.1@0(leader).log v5604049 update_from_paxos version 5604049 summary v 5604049 2013-04-26 02:39:40.858306 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:40.918176 7f27d600e700 10 mon.1@0(leader).pg v5938169 preprocess_query pg_stats(10 pgs tid 61 v 10556) v1 from osd.8 10.227.227.103:6813/8628 2013-04-26 02:39:40.918288 7f27d600e700 10 mon.1@0(leader).pg v5938169 prepare_update pg_stats(10 pgs tid 61 v 10556) v1 from osd.8 10.227.227.103:6813/8628 2013-04-26 02:39:40.918304 7f27d600e700 10 mon.1@0(leader).pg v5938169 prepare_pg_stats pg_stats(10 pgs tid 61 v 10556) v1 from osd.8 2013-04-26 02:39:40.918320 7f27d600e700 10 mon.1@0(leader).pg v5938169 got osd.8 osd_stat(185 GB used, 744 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[]) (was osd_stat(185 GB used, 744 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[])) 2013-04-26 02:39:41.216315 7f27d600e700 10 mon.1@0(leader).mds e3724 preprocess_query mdsbeacon(217445/3 up:standby seq 78 v3724) v2 from mds.? 10.227.227.103:6800/8011 2013-04-26 02:39:41.779036 7f27d680f700 10 mon.1@0(leader).pg v5938169 encode_pending v 5938170 2013-04-26 02:39:41.933011 7f27d600e700 7 mon.1@0(leader).pg v5938169 update_from_paxos applying incremental 5938170 2013-04-26 02:39:41.933106 7f27d600e700 10 mon.1@0(leader).pg v5938170 v5938170: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 1672KB/s wr, 32op/s 2013-04-26 02:39:41.933272 7f27d600e700 10 mon.1@0(leader).pg v5938170 send_pg_creates to 0 pgs 2013-04-26 02:39:41.933286 7f27d600e700 10 mon.1@0(leader).pg v5938170 update_logger 2013-04-26 02:39:41.933403 7f27d600e700 10 mon.1@0(leader).pg v5938170 create_pending v 5938171 2013-04-26 02:39:41.933536 7f27d600e700 7 mon.1@0(leader).pg v5938170 _updated_stats for osd.8 10.227.227.103:6813/8628 2013-04-26 02:39:41.933583 7f27d600e700 10 mon.1@0(leader).pg v5938170 check_osd_map already seen 10556 >= 10556 2013-04-26 02:39:41.933588 7f27d600e700 10 mon.1@0(leader).pg v5938170 update_logger 2013-04-26 02:39:41.933611 7f27d600e700 0 log [INF] : pgmap v5938170: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 1672KB/s wr, 32op/s 2013-04-26 02:39:41.933846 7f27d600e700 10 mon.1@0(leader).log v5604049 update_from_paxos 2013-04-26 02:39:41.933960 7f27d600e700 10 mon.1@0(leader).log v5604049 update_from_paxos version 5604049 summary v 5604049 2013-04-26 02:39:41.934033 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:41.934237 7f27d600e700 10 mon.1@0(leader).log v5604049 update_from_paxos 2013-04-26 02:39:41.934308 7f27d600e700 10 mon.1@0(leader).log v5604049 update_from_paxos version 5604049 summary v 5604049 2013-04-26 02:39:41.934378 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:41.934600 7f27d600e700 10 mon.1@0(leader).log v5604049 update_from_paxos 2013-04-26 02:39:41.934667 7f27d600e700 10 mon.1@0(leader).log v5604049 update_from_paxos version 5604049 summary v 5604049 2013-04-26 02:39:41.934704 7f27d600e700 10 mon.1@0(leader).log v5604049 preprocess_query log(2 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:41.934748 7f27d600e700 10 mon.1@0(leader).log v5604049 preprocess_log log(2 entries) v1 from mon.0 2013-04-26 02:39:41.934809 7f27d600e700 10 mon.1@0(leader).log v5604049 prepare_update log(2 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:41.934876 7f27d600e700 10 mon.1@0(leader).log v5604049 prepare_log log(2 entries) v1 from mon.0 2013-04-26 02:39:41.934927 7f27d600e700 10 mon.1@0(leader).log v5604049 logging 2013-04-26 02:39:40.725215 mon.0 10.227.227.101:6789/0 38 : [INF] pgmap v5938169: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 2185KB/s wr, 38op/s 2013-04-26 02:39:41.935015 7f27d600e700 10 mon.1@0(leader).log v5604049 logging 2013-04-26 02:39:41.933613 mon.0 10.227.227.101:6789/0 39 : [INF] pgmap v5938170: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 1672KB/s wr, 32op/s 2013-04-26 02:39:41.975961 7f27d600e700 10 mon.1@0(leader).log v5604049 update_from_paxos 2013-04-26 02:39:41.976083 7f27d600e700 10 mon.1@0(leader).log v5604049 update_from_paxos version 5604049 summary v 5604049 2013-04-26 02:39:41.976232 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:42.760488 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.5 10.227.227.104:6807/14479 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:42.760509 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x4f36280 from 10.227.227.103:6789/0 2013-04-26 02:39:42.760668 7f27d600e700 10 mon.1@0(leader).pg v5938170 preprocess_query pg_stats(29 pgs tid 749 v 10556) v1 from osd.5 10.227.227.104:6807/14479 2013-04-26 02:39:42.760763 7f27d600e700 10 mon.1@0(leader).pg v5938170 prepare_update pg_stats(29 pgs tid 749 v 10556) v1 from osd.5 10.227.227.104:6807/14479 2013-04-26 02:39:42.760774 7f27d600e700 10 mon.1@0(leader).pg v5938170 prepare_pg_stats pg_stats(29 pgs tid 749 v 10556) v1 from osd.5 2013-04-26 02:39:42.760789 7f27d600e700 10 mon.1@0(leader).pg v5938170 got osd.5 osd_stat(355 GB used, 1505 GB avail, 1860 GB total, peers [0,1,2,6,7,8,9,10,11]/[]) (was osd_stat(355 GB used, 1505 GB avail, 1860 GB total, peers [0,1,2,6,7,8,9,10,11]/[])) 2013-04-26 02:39:42.869157 7f27d680f700 10 mon.1@0(leader).log v5604049 encode_full log v 5604049 2013-04-26 02:39:42.869499 7f27d680f700 10 mon.1@0(leader).log v5604049 encode_pending v5604050 2013-04-26 02:39:42.956803 7f27d680f700 10 mon.1@0(leader).pg v5938170 encode_full pgmap v 5938170 2013-04-26 02:39:42.966888 7f27d680f700 10 mon.1@0(leader).pg v5938170 encode_pending v 5938171 2013-04-26 02:39:43.041821 7f27d600e700 10 mon.1@0(leader).log v5604050 update_from_paxos 2013-04-26 02:39:43.041957 7f27d600e700 10 mon.1@0(leader).log v5604050 update_from_paxos version 5604050 summary v 5604049 2013-04-26 02:39:43.042032 7f27d600e700 10 mon.1@0(leader).log v5604050 update_from_paxos latest full 5604049 2013-04-26 02:39:43.042117 7f27d600e700 7 mon.1@0(leader).log v5604050 update_from_paxos applying incremental log 5604050 2013-04-26 02:39:40.725215 mon.0 10.227.227.101:6789/0 38 : [INF] pgmap v5938169: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 2185KB/s wr, 38op/s 2013-04-26 02:39:43.042203 7f27d600e700 7 mon.1@0(leader).log v5604050 update_from_paxos applying incremental log 5604050 2013-04-26 02:39:41.933613 mon.0 10.227.227.101:6789/0 39 : [INF] pgmap v5938170: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 1672KB/s wr, 32op/s 2013-04-26 02:39:43.042372 7f27d600e700 10 mon.1@0(leader).log v5604050 check_subs 2013-04-26 02:39:43.042484 7f27d600e700 10 mon.1@0(leader).log v5604050 create_pending v 5604051 2013-04-26 02:39:43.042597 7f27d600e700 7 mon.1@0(leader).log v5604050 _updated_log for mon.0 10.227.227.101:6789/0 2013-04-26 02:39:43.251094 7f27d600e700 7 mon.1@0(leader).pg v5938170 update_from_paxos applying incremental 5938171 2013-04-26 02:39:43.251307 7f27d600e700 10 mon.1@0(leader).pg v5938171 v5938171: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 336KB/s wr, 10op/s 2013-04-26 02:39:43.251378 7f27d600e700 10 mon.1@0(leader).pg v5938171 send_pg_creates to 0 pgs 2013-04-26 02:39:43.251384 7f27d600e700 10 mon.1@0(leader).pg v5938171 update_logger 2013-04-26 02:39:43.251436 7f27d600e700 10 mon.1@0(leader).pg v5938171 create_pending v 5938172 2013-04-26 02:39:43.251469 7f27d600e700 7 mon.1@0(leader).pg v5938171 _updated_stats for osd.5 10.227.227.104:6807/14479 2013-04-26 02:39:43.251519 7f27d600e700 10 mon.1@0(leader).pg v5938171 check_osd_map already seen 10556 >= 10556 2013-04-26 02:39:43.251524 7f27d600e700 10 mon.1@0(leader).pg v5938171 update_logger 2013-04-26 02:39:43.251546 7f27d600e700 0 log [INF] : pgmap v5938171: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 336KB/s wr, 10op/s 2013-04-26 02:39:43.251800 7f27d600e700 10 mon.1@0(leader).log v5604050 update_from_paxos 2013-04-26 02:39:43.251884 7f27d600e700 10 mon.1@0(leader).log v5604050 update_from_paxos version 5604050 summary v 5604050 2013-04-26 02:39:43.251960 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:43.252123 7f27d600e700 10 mon.1@0(leader).log v5604050 update_from_paxos 2013-04-26 02:39:43.252191 7f27d600e700 10 mon.1@0(leader).log v5604050 update_from_paxos version 5604050 summary v 5604050 2013-04-26 02:39:43.252226 7f27d600e700 10 mon.1@0(leader).log v5604050 preprocess_query log(1 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:43.252266 7f27d600e700 10 mon.1@0(leader).log v5604050 preprocess_log log(1 entries) v1 from mon.0 2013-04-26 02:39:43.252319 7f27d600e700 10 mon.1@0(leader).log v5604050 prepare_update log(1 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:43.252362 7f27d600e700 10 mon.1@0(leader).log v5604050 prepare_log log(1 entries) v1 from mon.0 2013-04-26 02:39:43.252400 7f27d600e700 10 mon.1@0(leader).log v5604050 logging 2013-04-26 02:39:43.251549 mon.0 10.227.227.101:6789/0 40 : [INF] pgmap v5938171: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 336KB/s wr, 10op/s 2013-04-26 02:39:43.269894 7f27d600e700 10 mon.1@0(leader).log v5604050 update_from_paxos 2013-04-26 02:39:43.269977 7f27d600e700 10 mon.1@0(leader).log v5604050 update_from_paxos version 5604050 summary v 5604050 2013-04-26 02:39:43.270051 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:43.352947 7f27d600e700 10 mon.1@0(leader).log v5604050 update_from_paxos 2013-04-26 02:39:43.353033 7f27d600e700 10 mon.1@0(leader).log v5604050 update_from_paxos version 5604050 summary v 5604050 2013-04-26 02:39:43.353103 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:43.897053 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.11 10.227.227.103:6804/8253 via mon.2 10.227.227.104:6789/0 2013-04-26 02:39:43.897150 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x4f36a00 from 10.227.227.104:6789/0 2013-04-26 02:39:43.897332 7f27d600e700 10 mon.1@0(leader).pg v5938171 preprocess_query pg_stats(13 pgs tid 64 v 10556) v1 from osd.11 10.227.227.103:6804/8253 2013-04-26 02:39:43.897507 7f27d600e700 10 mon.1@0(leader).pg v5938171 prepare_update pg_stats(13 pgs tid 64 v 10556) v1 from osd.11 10.227.227.103:6804/8253 2013-04-26 02:39:43.897527 7f27d600e700 10 mon.1@0(leader).pg v5938171 prepare_pg_stats pg_stats(13 pgs tid 64 v 10556) v1 from osd.11 2013-04-26 02:39:43.897542 7f27d600e700 10 mon.1@0(leader).pg v5938171 got osd.11 osd_stat(161 GB used, 768 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[]) (was osd_stat(161 GB used, 768 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[])) 2013-04-26 02:39:44.030951 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from mds.? 10.227.227.101:6800/29852 via mon.2 10.227.227.104:6789/0 2013-04-26 02:39:44.030976 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x1789080 from 10.227.227.104:6789/0 2013-04-26 02:39:44.031141 7f27d600e700 10 mon.1@0(leader).mds e3724 preprocess_query mdsbeacon(217349/1 up:standby seq 466 v3724) v2 from mds.? 10.227.227.101:6800/29852 2013-04-26 02:39:44.170341 7f27d680f700 10 mon.1@0(leader).pg v5938171 encode_pending v 5938172 2013-04-26 02:39:44.249247 7f27d680f700 10 mon.1@0(leader).log v5604050 encode_full log v 5604050 2013-04-26 02:39:44.249482 7f27d680f700 10 mon.1@0(leader).log v5604050 encode_pending v5604051 2013-04-26 02:39:44.308063 7f27d600e700 7 mon.1@0(leader).pg v5938171 update_from_paxos applying incremental 5938172 2013-04-26 02:39:44.308246 7f27d600e700 10 mon.1@0(leader).pg v5938172 v5938172: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 61663B/s wr, 9op/s 2013-04-26 02:39:44.308347 7f27d600e700 10 mon.1@0(leader).pg v5938172 send_pg_creates to 0 pgs 2013-04-26 02:39:44.308355 7f27d600e700 10 mon.1@0(leader).pg v5938172 update_logger 2013-04-26 02:39:44.308387 7f27d600e700 10 mon.1@0(leader).pg v5938172 create_pending v 5938173 2013-04-26 02:39:44.308407 7f27d600e700 7 mon.1@0(leader).pg v5938172 _updated_stats for osd.11 10.227.227.103:6804/8253 2013-04-26 02:39:44.308521 7f27d600e700 10 mon.1@0(leader).pg v5938172 check_osd_map already seen 10556 >= 10556 2013-04-26 02:39:44.308544 7f27d600e700 10 mon.1@0(leader).pg v5938172 update_logger 2013-04-26 02:39:44.308569 7f27d600e700 0 log [INF] : pgmap v5938172: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 61663B/s wr, 9op/s 2013-04-26 02:39:44.358032 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.7 10.227.227.103:6810/8502 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:44.358092 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x5023680 from 10.227.227.103:6789/0 2013-04-26 02:39:44.358173 7f27d600e700 10 mon.1@0(leader).pg v5938172 preprocess_query pg_stats(16 pgs tid 60 v 10556) v1 from osd.7 10.227.227.103:6810/8502 2013-04-26 02:39:44.358239 7f27d600e700 10 mon.1@0(leader).pg v5938172 prepare_update pg_stats(16 pgs tid 60 v 10556) v1 from osd.7 10.227.227.103:6810/8502 2013-04-26 02:39:44.358248 7f27d600e700 10 mon.1@0(leader).pg v5938172 prepare_pg_stats pg_stats(16 pgs tid 60 v 10556) v1 from osd.7 2013-04-26 02:39:44.358258 7f27d600e700 10 mon.1@0(leader).pg v5938172 got osd.7 osd_stat(156 GB used, 772 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[]) (was osd_stat(156 GB used, 772 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[])) 2013-04-26 02:39:44.358362 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.9 10.227.227.103:6816/8752 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:44.358375 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x4f37400 from 10.227.227.103:6789/0 2013-04-26 02:39:44.358430 7f27d600e700 10 mon.1@0(leader).pg v5938172 preprocess_query pg_stats(12 pgs tid 63 v 10556) v1 from osd.9 10.227.227.103:6816/8752 2013-04-26 02:39:44.358478 7f27d600e700 10 mon.1@0(leader).pg v5938172 prepare_update pg_stats(12 pgs tid 63 v 10556) v1 from osd.9 10.227.227.103:6816/8752 2013-04-26 02:39:44.358486 7f27d600e700 10 mon.1@0(leader).pg v5938172 prepare_pg_stats pg_stats(12 pgs tid 63 v 10556) v1 from osd.9 2013-04-26 02:39:44.358493 7f27d600e700 10 mon.1@0(leader).pg v5938172 got osd.9 osd_stat(161 GB used, 767 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[]) (was osd_stat(161 GB used, 767 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[])) 2013-04-26 02:39:44.416510 7f27d600e700 10 mon.1@0(leader).log v5604051 update_from_paxos 2013-04-26 02:39:44.416638 7f27d600e700 10 mon.1@0(leader).log v5604051 update_from_paxos version 5604051 summary v 5604050 2013-04-26 02:39:44.416713 7f27d600e700 10 mon.1@0(leader).log v5604051 update_from_paxos latest full 5604050 2013-04-26 02:39:44.416793 7f27d600e700 7 mon.1@0(leader).log v5604051 update_from_paxos applying incremental log 5604051 2013-04-26 02:39:43.251549 mon.0 10.227.227.101:6789/0 40 : [INF] pgmap v5938171: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 336KB/s wr, 10op/s 2013-04-26 02:39:44.416925 7f27d600e700 10 mon.1@0(leader).log v5604051 check_subs 2013-04-26 02:39:44.416999 7f27d600e700 10 mon.1@0(leader).log v5604051 create_pending v 5604052 2013-04-26 02:39:44.417057 7f27d600e700 7 mon.1@0(leader).log v5604051 _updated_log for mon.0 10.227.227.101:6789/0 2013-04-26 02:39:44.417189 7f27d600e700 10 mon.1@0(leader).log v5604051 update_from_paxos 2013-04-26 02:39:44.417256 7f27d600e700 10 mon.1@0(leader).log v5604051 update_from_paxos version 5604051 summary v 5604051 2013-04-26 02:39:44.417278 7f27d600e700 10 mon.1@0(leader).log v5604051 preprocess_query log(1 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:44.417306 7f27d600e700 10 mon.1@0(leader).log v5604051 preprocess_log log(1 entries) v1 from mon.0 2013-04-26 02:39:44.417335 7f27d600e700 10 mon.1@0(leader).log v5604051 nothing new 2013-04-26 02:39:44.417486 7f27d600e700 10 mon.1@0(leader).log v5604051 update_from_paxos 2013-04-26 02:39:44.417529 7f27d600e700 10 mon.1@0(leader).log v5604051 update_from_paxos version 5604051 summary v 5604051 2013-04-26 02:39:44.417587 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:44.428968 7f27d600e700 10 mon.1@0(leader).log v5604051 update_from_paxos 2013-04-26 02:39:44.429043 7f27d600e700 10 mon.1@0(leader).log v5604051 update_from_paxos version 5604051 summary v 5604051 2013-04-26 02:39:44.429111 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:44.467714 7f27d600e700 10 mon.1@0(leader).log v5604051 update_from_paxos 2013-04-26 02:39:44.467794 7f27d600e700 10 mon.1@0(leader).log v5604051 update_from_paxos version 5604051 summary v 5604051 2013-04-26 02:39:44.467862 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:44.472130 7f27d680f700 10 mon.1@0(leader).pg v5938172 check_down_pgs 2013-04-26 02:39:44.472397 7f27d680f700 10 mon.1@0(leader).pg v5938172 v5938172: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 61663B/s wr, 9op/s 2013-04-26 02:39:44.472476 7f27d680f700 10 mon.1@0(leader).mds e3724 e3724: 1/1/1 up {0=4=up:active}, 2 up:standby 2013-04-26 02:39:44.472544 7f27d680f700 10 mon.1@0(leader).osd e10556 e10556: 12 osds: 12 up, 12 in 2013-04-26 02:39:44.472813 7f27d680f700 10 mon.1@0(leader).osd e10556 min_last_epoch_clean 10531 2013-04-26 02:39:44.472954 7f27d680f700 10 mon.1@0(leader).log v5604051 update_from_paxos 2013-04-26 02:39:44.473027 7f27d680f700 10 mon.1@0(leader).log v5604051 update_from_paxos version 5604051 summary v 5604051 2013-04-26 02:39:44.473062 7f27d680f700 10 mon.1@0(leader).log v5604051 log 2013-04-26 02:39:44.473130 7f27d680f700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:44.473194 7f27d680f700 10 mon.1@0(leader).auth v4370 auth 2013-04-26 02:39:44.503475 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from mds.0 10.227.227.104:6801/10842 via mon.2 10.227.227.104:6789/0 2013-04-26 02:39:44.503499 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x1789b80 from 10.227.227.104:6789/0 2013-04-26 02:39:44.503640 7f27d600e700 10 mon.1@0(leader).mds e3724 preprocess_query mdsbeacon(217231/4 up:active seq 497 v3724) v2 from mds.0 10.227.227.104:6801/10842 2013-04-26 02:39:44.537788 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.10 10.227.227.103:6801/8155 via mon.2 10.227.227.104:6789/0 2013-04-26 02:39:44.537812 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x5022c80 from 10.227.227.104:6789/0 2013-04-26 02:39:44.537950 7f27d600e700 10 mon.1@0(leader).pg v5938172 preprocess_query pg_stats(14 pgs tid 59 v 10556) v1 from osd.10 10.227.227.103:6801/8155 2013-04-26 02:39:44.538049 7f27d600e700 10 mon.1@0(leader).pg v5938172 prepare_update pg_stats(14 pgs tid 59 v 10556) v1 from osd.10 10.227.227.103:6801/8155 2013-04-26 02:39:44.538062 7f27d600e700 10 mon.1@0(leader).pg v5938172 prepare_pg_stats pg_stats(14 pgs tid 59 v 10556) v1 from osd.10 2013-04-26 02:39:44.538077 7f27d600e700 10 mon.1@0(leader).pg v5938172 got osd.10 osd_stat(174 GB used, 755 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[]) (was osd_stat(174 GB used, 755 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[])) 2013-04-26 02:39:44.551971 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.2 10.227.227.101:6807/30207 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:44.551987 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x4f36280 from 10.227.227.103:6789/0 2013-04-26 02:39:44.552114 7f27d600e700 10 mon.1@0(leader).pg v5938172 preprocess_query pg_stats(27 pgs tid 742 v 10556) v1 from osd.2 10.227.227.101:6807/30207 2013-04-26 02:39:44.552200 7f27d600e700 10 mon.1@0(leader).pg v5938172 prepare_update pg_stats(27 pgs tid 742 v 10556) v1 from osd.2 10.227.227.101:6807/30207 2013-04-26 02:39:44.552276 7f27d600e700 10 mon.1@0(leader).pg v5938172 prepare_pg_stats pg_stats(27 pgs tid 742 v 10556) v1 from osd.2 2013-04-26 02:39:44.552297 7f27d600e700 10 mon.1@0(leader).pg v5938172 got osd.2 osd_stat(311 GB used, 1549 GB avail, 1860 GB total, peers [3,4,5,6,7,8,9,10,11]/[]) (was osd_stat(311 GB used, 1549 GB avail, 1860 GB total, peers [3,4,5,6,7,8,9,10,11]/[])) 2013-04-26 02:39:44.648065 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.1 10.227.227.101:6804/30088 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:44.648082 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x5023b80 from 10.227.227.103:6789/0 2013-04-26 02:39:44.648213 7f27d600e700 10 mon.1@0(leader).pg v5938172 preprocess_query pg_stats(24 pgs tid 736 v 10556) v1 from osd.1 10.227.227.101:6804/30088 2013-04-26 02:39:44.648306 7f27d600e700 10 mon.1@0(leader).pg v5938172 prepare_update pg_stats(24 pgs tid 736 v 10556) v1 from osd.1 10.227.227.101:6804/30088 2013-04-26 02:39:44.648321 7f27d600e700 10 mon.1@0(leader).pg v5938172 prepare_pg_stats pg_stats(24 pgs tid 736 v 10556) v1 from osd.1 2013-04-26 02:39:44.648335 7f27d600e700 10 mon.1@0(leader).pg v5938172 got osd.1 osd_stat(342 GB used, 1517 GB avail, 1860 GB total, peers [3,4,5,6,7,8,9,10,11]/[]) (was osd_stat(342 GB used, 1517 GB avail, 1860 GB total, peers [3,4,5,6,7,8,9,10,11]/[])) 2013-04-26 02:39:44.690029 7f27d600e700 10 mon.1@0(leader).pg v5938172 preprocess_query pg_stats(37 pgs tid 748 v 10556) v1 from osd.0 10.227.227.101:6801/29982 2013-04-26 02:39:44.690125 7f27d600e700 10 mon.1@0(leader).pg v5938172 prepare_update pg_stats(37 pgs tid 748 v 10556) v1 from osd.0 10.227.227.101:6801/29982 2013-04-26 02:39:44.690137 7f27d600e700 10 mon.1@0(leader).pg v5938172 prepare_pg_stats pg_stats(37 pgs tid 748 v 10556) v1 from osd.0 2013-04-26 02:39:44.690148 7f27d600e700 10 mon.1@0(leader).pg v5938172 got osd.0 osd_stat(367 GB used, 1493 GB avail, 1860 GB total, peers [3,4,5,6,7,8,9,10,11]/[]) (was osd_stat(367 GB used, 1493 GB avail, 1860 GB total, peers [3,4,5,6,7,8,9,10,11]/[])) 2013-04-26 02:39:44.895998 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.6 10.227.227.103:6807/8402 via mon.2 10.227.227.104:6789/0 2013-04-26 02:39:44.896021 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x459ef00 from 10.227.227.104:6789/0 2013-04-26 02:39:44.896171 7f27d600e700 10 mon.1@0(leader).pg v5938172 preprocess_query pg_stats(22 pgs tid 63 v 10556) v1 from osd.6 10.227.227.103:6807/8402 2013-04-26 02:39:44.896276 7f27d600e700 10 mon.1@0(leader).pg v5938172 prepare_update pg_stats(22 pgs tid 63 v 10556) v1 from osd.6 10.227.227.103:6807/8402 2013-04-26 02:39:44.896288 7f27d600e700 10 mon.1@0(leader).pg v5938172 prepare_pg_stats pg_stats(22 pgs tid 63 v 10556) v1 from osd.6 2013-04-26 02:39:44.896302 7f27d600e700 10 mon.1@0(leader).pg v5938172 got osd.6 osd_stat(184 GB used, 745 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[]) (was osd_stat(184 GB used, 745 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[])) 2013-04-26 02:39:45.216541 7f27d600e700 10 mon.1@0(leader).mds e3724 preprocess_query mdsbeacon(217445/3 up:standby seq 79 v3724) v2 from mds.? 10.227.227.103:6800/8011 2013-04-26 02:39:45.258949 7f27d680f700 10 mon.1@0(leader).pg v5938172 encode_pending v 5938173 2013-04-26 02:39:45.358815 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.4 10.227.227.104:6804/14471 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:45.358955 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x459f680 from 10.227.227.103:6789/0 2013-04-26 02:39:45.423973 7f27d600e700 7 mon.1@0(leader).pg v5938172 update_from_paxos applying incremental 5938173 2013-04-26 02:39:45.424886 7f27d600e700 10 mon.1@0(leader).pg v5938173 v5938173: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 480KB/s wr, 36op/s 2013-04-26 02:39:45.424996 7f27d600e700 10 mon.1@0(leader).pg v5938173 send_pg_creates to 0 pgs 2013-04-26 02:39:45.425003 7f27d600e700 10 mon.1@0(leader).pg v5938173 update_logger 2013-04-26 02:39:45.425097 7f27d600e700 10 mon.1@0(leader).pg v5938173 create_pending v 5938174 2013-04-26 02:39:45.425144 7f27d600e700 7 mon.1@0(leader).pg v5938173 _updated_stats for osd.7 10.227.227.103:6810/8502 2013-04-26 02:39:45.425190 7f27d600e700 7 mon.1@0(leader).pg v5938173 _updated_stats for osd.9 10.227.227.103:6816/8752 2013-04-26 02:39:45.425341 7f27d600e700 7 mon.1@0(leader).pg v5938173 _updated_stats for osd.10 10.227.227.103:6801/8155 2013-04-26 02:39:45.425385 7f27d600e700 7 mon.1@0(leader).pg v5938173 _updated_stats for osd.2 10.227.227.101:6807/30207 2013-04-26 02:39:45.425512 7f27d600e700 7 mon.1@0(leader).pg v5938173 _updated_stats for osd.1 10.227.227.101:6804/30088 2013-04-26 02:39:45.425556 7f27d600e700 7 mon.1@0(leader).pg v5938173 _updated_stats for osd.0 10.227.227.101:6801/29982 2013-04-26 02:39:45.425589 7f27d600e700 7 mon.1@0(leader).pg v5938173 _updated_stats for osd.6 10.227.227.103:6807/8402 2013-04-26 02:39:45.425720 7f27d600e700 10 mon.1@0(leader).pg v5938173 preprocess_query pg_stats(47 pgs tid 752 v 10556) v1 from osd.4 10.227.227.104:6804/14471 2013-04-26 02:39:45.425901 7f27d600e700 10 mon.1@0(leader).pg v5938173 prepare_update pg_stats(47 pgs tid 752 v 10556) v1 from osd.4 10.227.227.104:6804/14471 2013-04-26 02:39:45.425924 7f27d600e700 10 mon.1@0(leader).pg v5938173 prepare_pg_stats pg_stats(47 pgs tid 752 v 10556) v1 from osd.4 2013-04-26 02:39:45.425938 7f27d600e700 10 mon.1@0(leader).pg v5938173 got osd.4 osd_stat(423 GB used, 1437 GB avail, 1860 GB total, peers [0,1,2,6,7,8,9,10,11]/[]) (was osd_stat(423 GB used, 1437 GB avail, 1860 GB total, peers [0,1,2,6,7,8,9,10,11]/[])) 2013-04-26 02:39:45.426102 7f27d600e700 10 mon.1@0(leader).pg v5938173 check_osd_map already seen 10556 >= 10556 2013-04-26 02:39:45.426112 7f27d600e700 10 mon.1@0(leader).pg v5938173 update_logger 2013-04-26 02:39:45.426136 7f27d600e700 0 log [INF] : pgmap v5938173: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 480KB/s wr, 36op/s 2013-04-26 02:39:45.426372 7f27d600e700 10 mon.1@0(leader).log v5604051 update_from_paxos 2013-04-26 02:39:45.426455 7f27d600e700 10 mon.1@0(leader).log v5604051 update_from_paxos version 5604051 summary v 5604051 2013-04-26 02:39:45.426524 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:45.426699 7f27d600e700 10 mon.1@0(leader).log v5604051 update_from_paxos 2013-04-26 02:39:45.426767 7f27d600e700 10 mon.1@0(leader).log v5604051 update_from_paxos version 5604051 summary v 5604051 2013-04-26 02:39:45.426865 7f27d600e700 10 mon.1@0(leader).log v5604051 preprocess_query log(2 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:45.426927 7f27d600e700 10 mon.1@0(leader).log v5604051 preprocess_log log(2 entries) v1 from mon.0 2013-04-26 02:39:45.426985 7f27d600e700 10 mon.1@0(leader).log v5604051 prepare_update log(2 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:45.427030 7f27d600e700 10 mon.1@0(leader).log v5604051 prepare_log log(2 entries) v1 from mon.0 2013-04-26 02:39:45.427073 7f27d600e700 10 mon.1@0(leader).log v5604051 logging 2013-04-26 02:39:44.308572 mon.0 10.227.227.101:6789/0 41 : [INF] pgmap v5938172: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 61663B/s wr, 9op/s 2013-04-26 02:39:45.427133 7f27d600e700 10 mon.1@0(leader).log v5604051 logging 2013-04-26 02:39:45.426139 mon.0 10.227.227.101:6789/0 42 : [INF] pgmap v5938173: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 480KB/s wr, 36op/s 2013-04-26 02:39:45.429310 7f27d600e700 10 mon.1@0(leader).log v5604051 update_from_paxos 2013-04-26 02:39:45.429392 7f27d600e700 10 mon.1@0(leader).log v5604051 update_from_paxos version 5604051 summary v 5604051 2013-04-26 02:39:45.429458 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:45.467794 7f27d600e700 10 mon.1@0(leader).log v5604051 update_from_paxos 2013-04-26 02:39:45.467876 7f27d600e700 10 mon.1@0(leader).log v5604051 update_from_paxos version 5604051 summary v 5604051 2013-04-26 02:39:45.467950 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:45.474378 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.3 10.227.227.104:6800/14463 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:45.474404 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x5022a00 from 10.227.227.103:6789/0 2013-04-26 02:39:45.474610 7f27d600e700 10 mon.1@0(leader).pg v5938173 preprocess_query pg_stats(42 pgs tid 757 v 10556) v1 from osd.3 10.227.227.104:6800/14463 2013-04-26 02:39:45.474703 7f27d600e700 10 mon.1@0(leader).pg v5938173 prepare_update pg_stats(42 pgs tid 757 v 10556) v1 from osd.3 10.227.227.104:6800/14463 2013-04-26 02:39:45.474718 7f27d600e700 10 mon.1@0(leader).pg v5938173 prepare_pg_stats pg_stats(42 pgs tid 757 v 10556) v1 from osd.3 2013-04-26 02:39:45.474733 7f27d600e700 10 mon.1@0(leader).pg v5938173 got osd.3 osd_stat(383 GB used, 1477 GB avail, 1860 GB total, peers [0,1,2,6,7,8,9,10,11]/[]) (was osd_stat(383 GB used, 1477 GB avail, 1860 GB total, peers [0,1,2,6,7,8,9,10,11]/[])) 2013-04-26 02:39:45.919076 7f27d600e700 10 mon.1@0(leader).pg v5938173 preprocess_query pg_stats(13 pgs tid 62 v 10556) v1 from osd.8 10.227.227.103:6813/8628 2013-04-26 02:39:45.919180 7f27d600e700 10 mon.1@0(leader).pg v5938173 prepare_update pg_stats(13 pgs tid 62 v 10556) v1 from osd.8 10.227.227.103:6813/8628 2013-04-26 02:39:45.919192 7f27d600e700 10 mon.1@0(leader).pg v5938173 prepare_pg_stats pg_stats(13 pgs tid 62 v 10556) v1 from osd.8 2013-04-26 02:39:45.919208 7f27d600e700 10 mon.1@0(leader).pg v5938173 got osd.8 osd_stat(185 GB used, 744 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[]) (was osd_stat(185 GB used, 744 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[])) 2013-04-26 02:39:46.369754 7f27d680f700 10 mon.1@0(leader).pg v5938173 encode_pending v 5938174 2013-04-26 02:39:46.459473 7f27d680f700 10 mon.1@0(leader).log v5604051 encode_full log v 5604051 2013-04-26 02:39:46.459722 7f27d680f700 10 mon.1@0(leader).log v5604051 encode_pending v5604052 2013-04-26 02:39:46.518649 7f27d600e700 7 mon.1@0(leader).pg v5938173 update_from_paxos applying incremental 5938174 2013-04-26 02:39:46.519274 7f27d600e700 10 mon.1@0(leader).pg v5938174 v5938174: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 1099KB/s wr, 64op/s 2013-04-26 02:39:46.519460 7f27d600e700 10 mon.1@0(leader).pg v5938174 send_pg_creates to 0 pgs 2013-04-26 02:39:46.519474 7f27d600e700 10 mon.1@0(leader).pg v5938174 update_logger 2013-04-26 02:39:46.519558 7f27d600e700 10 mon.1@0(leader).pg v5938174 create_pending v 5938175 2013-04-26 02:39:46.519599 7f27d600e700 7 mon.1@0(leader).pg v5938174 _updated_stats for osd.4 10.227.227.104:6804/14471 2013-04-26 02:39:46.519660 7f27d600e700 7 mon.1@0(leader).pg v5938174 _updated_stats for osd.3 10.227.227.104:6800/14463 2013-04-26 02:39:46.519768 7f27d600e700 7 mon.1@0(leader).pg v5938174 _updated_stats for osd.8 10.227.227.103:6813/8628 2013-04-26 02:39:46.519817 7f27d600e700 10 mon.1@0(leader).pg v5938174 check_osd_map already seen 10556 >= 10556 2013-04-26 02:39:46.519825 7f27d600e700 10 mon.1@0(leader).pg v5938174 update_logger 2013-04-26 02:39:46.519849 7f27d600e700 0 log [INF] : pgmap v5938174: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 1099KB/s wr, 64op/s 2013-04-26 02:39:46.634595 7f27d600e700 10 mon.1@0(leader).log v5604052 update_from_paxos 2013-04-26 02:39:46.634752 7f27d600e700 10 mon.1@0(leader).log v5604052 update_from_paxos version 5604052 summary v 5604051 2013-04-26 02:39:46.634829 7f27d600e700 10 mon.1@0(leader).log v5604052 update_from_paxos latest full 5604051 2013-04-26 02:39:46.634920 7f27d600e700 7 mon.1@0(leader).log v5604052 update_from_paxos applying incremental log 5604052 2013-04-26 02:39:44.308572 mon.0 10.227.227.101:6789/0 41 : [INF] pgmap v5938172: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 61663B/s wr, 9op/s 2013-04-26 02:39:46.635125 7f27d600e700 7 mon.1@0(leader).log v5604052 update_from_paxos applying incremental log 5604052 2013-04-26 02:39:45.426139 mon.0 10.227.227.101:6789/0 42 : [INF] pgmap v5938173: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 480KB/s wr, 36op/s 2013-04-26 02:39:46.635282 7f27d600e700 10 mon.1@0(leader).log v5604052 check_subs 2013-04-26 02:39:46.635390 7f27d600e700 10 mon.1@0(leader).log v5604052 create_pending v 5604053 2013-04-26 02:39:46.635490 7f27d600e700 7 mon.1@0(leader).log v5604052 _updated_log for mon.0 10.227.227.101:6789/0 2013-04-26 02:39:46.635719 7f27d600e700 10 mon.1@0(leader).log v5604052 update_from_paxos 2013-04-26 02:39:46.635815 7f27d600e700 10 mon.1@0(leader).log v5604052 update_from_paxos version 5604052 summary v 5604052 2013-04-26 02:39:46.635852 7f27d600e700 10 mon.1@0(leader).log v5604052 preprocess_query log(1 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:46.635895 7f27d600e700 10 mon.1@0(leader).log v5604052 preprocess_log log(1 entries) v1 from mon.0 2013-04-26 02:39:46.635940 7f27d600e700 10 mon.1@0(leader).log v5604052 nothing new 2013-04-26 02:39:46.636171 7f27d600e700 10 mon.1@0(leader).log v5604052 update_from_paxos 2013-04-26 02:39:46.636242 7f27d600e700 10 mon.1@0(leader).log v5604052 update_from_paxos version 5604052 summary v 5604052 2013-04-26 02:39:46.636314 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:46.636506 7f27d600e700 10 mon.1@0(leader).log v5604052 update_from_paxos 2013-04-26 02:39:46.636604 7f27d600e700 10 mon.1@0(leader).log v5604052 update_from_paxos version 5604052 summary v 5604052 2013-04-26 02:39:46.636808 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:46.698455 7f27d600e700 10 mon.1@0(leader).log v5604052 update_from_paxos 2013-04-26 02:39:46.698557 7f27d600e700 10 mon.1@0(leader).log v5604052 update_from_paxos version 5604052 summary v 5604052 2013-04-26 02:39:46.698629 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:47.761889 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.5 10.227.227.104:6807/14479 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:47.761914 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x5022a00 from 10.227.227.103:6789/0 2013-04-26 02:39:47.762069 7f27d600e700 10 mon.1@0(leader).pg v5938174 preprocess_query pg_stats(41 pgs tid 750 v 10556) v1 from osd.5 10.227.227.104:6807/14479 2013-04-26 02:39:47.762176 7f27d600e700 10 mon.1@0(leader).pg v5938174 prepare_update pg_stats(41 pgs tid 750 v 10556) v1 from osd.5 10.227.227.104:6807/14479 2013-04-26 02:39:47.762191 7f27d600e700 10 mon.1@0(leader).pg v5938174 prepare_pg_stats pg_stats(41 pgs tid 750 v 10556) v1 from osd.5 2013-04-26 02:39:47.762206 7f27d600e700 10 mon.1@0(leader).pg v5938174 got osd.5 osd_stat(355 GB used, 1505 GB avail, 1860 GB total, peers [0,1,2,6,7,8,9,10,11]/[]) (was osd_stat(355 GB used, 1505 GB avail, 1860 GB total, peers [0,1,2,6,7,8,9,10,11]/[])) 2013-04-26 02:39:47.812708 7f27d680f700 10 mon.1@0(leader).pg v5938174 encode_pending v 5938175 2013-04-26 02:39:48.162033 7f27d600e700 7 mon.1@0(leader).pg v5938174 update_from_paxos applying incremental 5938175 2013-04-26 02:39:48.162280 7f27d600e700 10 mon.1@0(leader).pg v5938175 v5938175: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 610KB/s wr, 37op/s 2013-04-26 02:39:48.162336 7f27d600e700 10 mon.1@0(leader).pg v5938175 send_pg_creates to 0 pgs 2013-04-26 02:39:48.162342 7f27d600e700 10 mon.1@0(leader).pg v5938175 update_logger 2013-04-26 02:39:48.162379 7f27d600e700 10 mon.1@0(leader).pg v5938175 create_pending v 5938176 2013-04-26 02:39:48.162403 7f27d600e700 7 mon.1@0(leader).pg v5938175 _updated_stats for osd.5 10.227.227.104:6807/14479 2013-04-26 02:39:48.162443 7f27d600e700 10 mon.1@0(leader).pg v5938175 check_osd_map already seen 10556 >= 10556 2013-04-26 02:39:48.162448 7f27d600e700 10 mon.1@0(leader).pg v5938175 update_logger 2013-04-26 02:39:48.162463 7f27d600e700 0 log [INF] : pgmap v5938175: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 610KB/s wr, 37op/s 2013-04-26 02:39:48.162643 7f27d600e700 10 mon.1@0(leader).log v5604052 update_from_paxos 2013-04-26 02:39:48.162697 7f27d600e700 10 mon.1@0(leader).log v5604052 update_from_paxos version 5604052 summary v 5604052 2013-04-26 02:39:48.162755 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:48.162851 7f27d600e700 10 mon.1@0(leader).log v5604052 update_from_paxos 2013-04-26 02:39:48.162889 7f27d600e700 10 mon.1@0(leader).log v5604052 update_from_paxos version 5604052 summary v 5604052 2013-04-26 02:39:48.162909 7f27d600e700 10 mon.1@0(leader).log v5604052 preprocess_query log(2 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:48.162932 7f27d600e700 10 mon.1@0(leader).log v5604052 preprocess_log log(2 entries) v1 from mon.0 2013-04-26 02:39:48.162979 7f27d600e700 10 mon.1@0(leader).log v5604052 prepare_update log(2 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:48.163012 7f27d600e700 10 mon.1@0(leader).log v5604052 prepare_log log(2 entries) v1 from mon.0 2013-04-26 02:39:48.163042 7f27d600e700 10 mon.1@0(leader).log v5604052 logging 2013-04-26 02:39:46.519851 mon.0 10.227.227.101:6789/0 43 : [INF] pgmap v5938174: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 1099KB/s wr, 64op/s 2013-04-26 02:39:48.163084 7f27d600e700 10 mon.1@0(leader).log v5604052 logging 2013-04-26 02:39:48.162465 mon.0 10.227.227.101:6789/0 44 : [INF] pgmap v5938175: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 610KB/s wr, 37op/s 2013-04-26 02:39:48.168995 7f27d600e700 10 mon.1@0(leader).log v5604052 update_from_paxos 2013-04-26 02:39:48.169042 7f27d600e700 10 mon.1@0(leader).log v5604052 update_from_paxos version 5604052 summary v 5604052 2013-04-26 02:39:48.169083 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:48.209458 7f27d600e700 10 mon.1@0(leader).log v5604052 update_from_paxos 2013-04-26 02:39:48.209507 7f27d600e700 10 mon.1@0(leader).log v5604052 update_from_paxos version 5604052 summary v 5604052 2013-04-26 02:39:48.209548 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:48.209606 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from mds.? 10.227.227.101:6800/29852 via mon.2 10.227.227.104:6789/0 2013-04-26 02:39:48.209617 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x1789080 from 10.227.227.104:6789/0 2013-04-26 02:39:48.209671 7f27d600e700 10 mon.1@0(leader).mds e3724 preprocess_query mdsbeacon(217349/1 up:standby seq 467 v3724) v2 from mds.? 10.227.227.101:6800/29852 2013-04-26 02:39:48.503527 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from mds.0 10.227.227.104:6801/10842 via mon.2 10.227.227.104:6789/0 2013-04-26 02:39:48.503543 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x17898c0 from 10.227.227.104:6789/0 2013-04-26 02:39:48.503668 7f27d600e700 10 mon.1@0(leader).mds e3724 preprocess_query mdsbeacon(217231/4 up:active seq 498 v3724) v2 from mds.0 10.227.227.104:6801/10842 2013-04-26 02:39:48.899742 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.11 10.227.227.103:6804/8253 via mon.2 10.227.227.104:6789/0 2013-04-26 02:39:48.899765 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x709a780 from 10.227.227.104:6789/0 2013-04-26 02:39:48.899950 7f27d600e700 10 mon.1@0(leader).pg v5938175 preprocess_query pg_stats(21 pgs tid 65 v 10556) v1 from osd.11 10.227.227.103:6804/8253 2013-04-26 02:39:48.900061 7f27d600e700 10 mon.1@0(leader).pg v5938175 prepare_update pg_stats(21 pgs tid 65 v 10556) v1 from osd.11 10.227.227.103:6804/8253 2013-04-26 02:39:48.900076 7f27d600e700 10 mon.1@0(leader).pg v5938175 prepare_pg_stats pg_stats(21 pgs tid 65 v 10556) v1 from osd.11 2013-04-26 02:39:48.900091 7f27d600e700 10 mon.1@0(leader).pg v5938175 got osd.11 osd_stat(161 GB used, 768 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[]) (was osd_stat(161 GB used, 768 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[])) 2013-04-26 02:39:48.959398 7f27d680f700 10 mon.1@0(leader).log v5604052 encode_full log v 5604052 2013-04-26 02:39:48.959809 7f27d680f700 10 mon.1@0(leader).log v5604052 encode_pending v5604053 2013-04-26 02:39:49.297500 7f27d680f700 10 mon.1@0(leader).pg v5938175 encode_pending v 5938176 2013-04-26 02:39:49.298444 7f27d600e700 10 mon.1@0(leader).mds e3724 preprocess_query mdsbeacon(217445/3 up:standby seq 80 v3724) v2 from mds.? 10.227.227.103:6800/8011 2013-04-26 02:39:49.298528 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.7 10.227.227.103:6810/8502 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:49.298544 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x709b900 from 10.227.227.103:6789/0 2013-04-26 02:39:49.372569 7f27d600e700 10 mon.1@0(leader).log v5604053 update_from_paxos 2013-04-26 02:39:49.372700 7f27d600e700 10 mon.1@0(leader).log v5604053 update_from_paxos version 5604053 summary v 5604052 2013-04-26 02:39:49.372748 7f27d600e700 10 mon.1@0(leader).log v5604053 update_from_paxos latest full 5604052 2013-04-26 02:39:49.372800 7f27d600e700 7 mon.1@0(leader).log v5604053 update_from_paxos applying incremental log 5604053 2013-04-26 02:39:46.519851 mon.0 10.227.227.101:6789/0 43 : [INF] pgmap v5938174: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 1099KB/s wr, 64op/s 2013-04-26 02:39:49.372879 7f27d600e700 7 mon.1@0(leader).log v5604053 update_from_paxos applying incremental log 5604053 2013-04-26 02:39:48.162465 mon.0 10.227.227.101:6789/0 44 : [INF] pgmap v5938175: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 610KB/s wr, 37op/s 2013-04-26 02:39:49.372966 7f27d600e700 10 mon.1@0(leader).log v5604053 check_subs 2013-04-26 02:39:49.373061 7f27d600e700 10 mon.1@0(leader).log v5604053 create_pending v 5604054 2013-04-26 02:39:49.373122 7f27d600e700 7 mon.1@0(leader).log v5604053 _updated_log for mon.0 10.227.227.101:6789/0 2013-04-26 02:39:49.422691 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.9 10.227.227.103:6816/8752 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:49.422703 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x709bb80 from 10.227.227.103:6789/0 2013-04-26 02:39:49.473371 7f27d680f700 10 mon.1@0(leader).mds e3724 e3724: 1/1/1 up {0=4=up:active}, 2 up:standby 2013-04-26 02:39:49.473435 7f27d680f700 10 mon.1@0(leader).osd e10556 e10556: 12 osds: 12 up, 12 in 2013-04-26 02:39:49.473519 7f27d680f700 10 mon.1@0(leader).log v5604053 update_from_paxos 2013-04-26 02:39:49.473575 7f27d680f700 10 mon.1@0(leader).log v5604053 update_from_paxos version 5604053 summary v 5604053 2013-04-26 02:39:49.473597 7f27d680f700 10 mon.1@0(leader).log v5604053 log 2013-04-26 02:39:49.473643 7f27d680f700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:49.473689 7f27d680f700 10 mon.1@0(leader).auth v4370 auth 2013-04-26 02:39:49.526643 7f27d600e700 7 mon.1@0(leader).pg v5938175 update_from_paxos applying incremental 5938176 2013-04-26 02:39:49.526812 7f27d600e700 10 mon.1@0(leader).pg v5938176 v5938176: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 93035B/s wr, 13op/s 2013-04-26 02:39:49.526880 7f27d600e700 10 mon.1@0(leader).pg v5938176 send_pg_creates to 0 pgs 2013-04-26 02:39:49.526886 7f27d600e700 10 mon.1@0(leader).pg v5938176 update_logger 2013-04-26 02:39:49.526942 7f27d600e700 10 mon.1@0(leader).pg v5938176 create_pending v 5938177 2013-04-26 02:39:49.526979 7f27d600e700 7 mon.1@0(leader).pg v5938176 _updated_stats for osd.11 10.227.227.103:6804/8253 2013-04-26 02:39:49.527115 7f27d600e700 10 mon.1@0(leader).pg v5938176 preprocess_query pg_stats(21 pgs tid 61 v 10556) v1 from osd.7 10.227.227.103:6810/8502 2013-04-26 02:39:49.527210 7f27d600e700 10 mon.1@0(leader).pg v5938176 prepare_update pg_stats(21 pgs tid 61 v 10556) v1 from osd.7 10.227.227.103:6810/8502 2013-04-26 02:39:49.527221 7f27d600e700 10 mon.1@0(leader).pg v5938176 prepare_pg_stats pg_stats(21 pgs tid 61 v 10556) v1 from osd.7 2013-04-26 02:39:49.527234 7f27d600e700 10 mon.1@0(leader).pg v5938176 got osd.7 osd_stat(156 GB used, 772 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[]) (was osd_stat(156 GB used, 772 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[])) 2013-04-26 02:39:49.527425 7f27d600e700 10 mon.1@0(leader).pg v5938176 preprocess_query pg_stats(17 pgs tid 64 v 10556) v1 from osd.9 10.227.227.103:6816/8752 2013-04-26 02:39:49.527510 7f27d600e700 10 mon.1@0(leader).pg v5938176 prepare_update pg_stats(17 pgs tid 64 v 10556) v1 from osd.9 10.227.227.103:6816/8752 2013-04-26 02:39:49.527521 7f27d600e700 10 mon.1@0(leader).pg v5938176 prepare_pg_stats pg_stats(17 pgs tid 64 v 10556) v1 from osd.9 2013-04-26 02:39:49.527530 7f27d600e700 10 mon.1@0(leader).pg v5938176 got osd.9 osd_stat(161 GB used, 767 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[]) (was osd_stat(161 GB used, 767 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[])) 2013-04-26 02:39:49.527630 7f27d600e700 10 mon.1@0(leader).pg v5938176 check_osd_map already seen 10556 >= 10556 2013-04-26 02:39:49.527636 7f27d600e700 10 mon.1@0(leader).pg v5938176 update_logger 2013-04-26 02:39:49.527659 7f27d600e700 0 log [INF] : pgmap v5938176: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 93035B/s wr, 13op/s 2013-04-26 02:39:49.527886 7f27d600e700 10 mon.1@0(leader).log v5604053 update_from_paxos 2013-04-26 02:39:49.527963 7f27d600e700 10 mon.1@0(leader).log v5604053 update_from_paxos version 5604053 summary v 5604053 2013-04-26 02:39:49.528031 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:49.528202 7f27d600e700 10 mon.1@0(leader).log v5604053 update_from_paxos 2013-04-26 02:39:49.528265 7f27d600e700 10 mon.1@0(leader).log v5604053 update_from_paxos version 5604053 summary v 5604053 2013-04-26 02:39:49.528299 7f27d600e700 10 mon.1@0(leader).log v5604053 preprocess_query log(1 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:49.528339 7f27d600e700 10 mon.1@0(leader).log v5604053 preprocess_log log(1 entries) v1 from mon.0 2013-04-26 02:39:49.528391 7f27d600e700 10 mon.1@0(leader).log v5604053 prepare_update log(1 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:49.528433 7f27d600e700 10 mon.1@0(leader).log v5604053 prepare_log log(1 entries) v1 from mon.0 2013-04-26 02:39:49.528470 7f27d600e700 10 mon.1@0(leader).log v5604053 logging 2013-04-26 02:39:49.527662 mon.0 10.227.227.101:6789/0 45 : [INF] pgmap v5938176: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 93035B/s wr, 13op/s 2013-04-26 02:39:49.530207 7f27d600e700 10 mon.1@0(leader).log v5604053 update_from_paxos 2013-04-26 02:39:49.530282 7f27d600e700 10 mon.1@0(leader).log v5604053 update_from_paxos version 5604053 summary v 5604053 2013-04-26 02:39:49.530349 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:49.552422 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.2 10.227.227.101:6807/30207 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:49.552441 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x709b180 from 10.227.227.103:6789/0 2013-04-26 02:39:49.552590 7f27d600e700 10 mon.1@0(leader).pg v5938176 preprocess_query pg_stats(38 pgs tid 743 v 10556) v1 from osd.2 10.227.227.101:6807/30207 2013-04-26 02:39:49.552687 7f27d600e700 10 mon.1@0(leader).pg v5938176 prepare_update pg_stats(38 pgs tid 743 v 10556) v1 from osd.2 10.227.227.101:6807/30207 2013-04-26 02:39:49.552701 7f27d600e700 10 mon.1@0(leader).pg v5938176 prepare_pg_stats pg_stats(38 pgs tid 743 v 10556) v1 from osd.2 2013-04-26 02:39:49.552716 7f27d600e700 10 mon.1@0(leader).pg v5938176 got osd.2 osd_stat(311 GB used, 1549 GB avail, 1860 GB total, peers [3,4,5,6,7,8,9,10,11]/[]) (was osd_stat(311 GB used, 1549 GB avail, 1860 GB total, peers [3,4,5,6,7,8,9,10,11]/[])) 2013-04-26 02:39:49.576657 7f27d600e700 10 mon.1@0(leader).log v5604053 update_from_paxos 2013-04-26 02:39:49.576739 7f27d600e700 10 mon.1@0(leader).log v5604053 update_from_paxos version 5604053 summary v 5604053 2013-04-26 02:39:49.576809 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:49.577685 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.10 10.227.227.103:6801/8155 via mon.2 10.227.227.104:6789/0 2013-04-26 02:39:49.577709 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x709b400 from 10.227.227.104:6789/0 2013-04-26 02:39:49.577843 7f27d600e700 10 mon.1@0(leader).pg v5938176 preprocess_query pg_stats(20 pgs tid 60 v 10556) v1 from osd.10 10.227.227.103:6801/8155 2013-04-26 02:39:49.577935 7f27d600e700 10 mon.1@0(leader).pg v5938176 prepare_update pg_stats(20 pgs tid 60 v 10556) v1 from osd.10 10.227.227.103:6801/8155 2013-04-26 02:39:49.577949 7f27d600e700 10 mon.1@0(leader).pg v5938176 prepare_pg_stats pg_stats(20 pgs tid 60 v 10556) v1 from osd.10 2013-04-26 02:39:49.577963 7f27d600e700 10 mon.1@0(leader).pg v5938176 got osd.10 osd_stat(174 GB used, 755 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[]) (was osd_stat(174 GB used, 755 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[])) 2013-04-26 02:39:49.649174 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.1 10.227.227.101:6804/30088 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:49.649199 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x1799680 from 10.227.227.103:6789/0 2013-04-26 02:39:49.649350 7f27d600e700 10 mon.1@0(leader).pg v5938176 preprocess_query pg_stats(27 pgs tid 737 v 10556) v1 from osd.1 10.227.227.101:6804/30088 2013-04-26 02:39:49.649450 7f27d600e700 10 mon.1@0(leader).pg v5938176 prepare_update pg_stats(27 pgs tid 737 v 10556) v1 from osd.1 10.227.227.101:6804/30088 2013-04-26 02:39:49.649462 7f27d600e700 10 mon.1@0(leader).pg v5938176 prepare_pg_stats pg_stats(27 pgs tid 737 v 10556) v1 from osd.1 2013-04-26 02:39:49.649476 7f27d600e700 10 mon.1@0(leader).pg v5938176 got osd.1 osd_stat(343 GB used, 1517 GB avail, 1860 GB total, peers [3,4,5,6,7,8,9,10,11]/[]) (was osd_stat(342 GB used, 1517 GB avail, 1860 GB total, peers [3,4,5,6,7,8,9,10,11]/[])) 2013-04-26 02:39:49.691056 7f27d600e700 10 mon.1@0(leader).pg v5938176 preprocess_query pg_stats(55 pgs tid 749 v 10556) v1 from osd.0 10.227.227.101:6801/29982 2013-04-26 02:39:49.691156 7f27d600e700 10 mon.1@0(leader).pg v5938176 prepare_update pg_stats(55 pgs tid 749 v 10556) v1 from osd.0 10.227.227.101:6801/29982 2013-04-26 02:39:49.691168 7f27d600e700 10 mon.1@0(leader).pg v5938176 prepare_pg_stats pg_stats(55 pgs tid 749 v 10556) v1 from osd.0 2013-04-26 02:39:49.691181 7f27d600e700 10 mon.1@0(leader).pg v5938176 got osd.0 osd_stat(367 GB used, 1493 GB avail, 1860 GB total, peers [3,4,5,6,7,8,9,10,11]/[]) (was osd_stat(367 GB used, 1493 GB avail, 1860 GB total, peers [3,4,5,6,7,8,9,10,11]/[])) 2013-04-26 02:39:49.896659 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.6 10.227.227.103:6807/8402 via mon.2 10.227.227.104:6789/0 2013-04-26 02:39:49.896682 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x709b680 from 10.227.227.104:6789/0 2013-04-26 02:39:49.896839 7f27d600e700 10 mon.1@0(leader).pg v5938176 preprocess_query pg_stats(25 pgs tid 64 v 10556) v1 from osd.6 10.227.227.103:6807/8402 2013-04-26 02:39:49.896943 7f27d600e700 10 mon.1@0(leader).pg v5938176 prepare_update pg_stats(25 pgs tid 64 v 10556) v1 from osd.6 10.227.227.103:6807/8402 2013-04-26 02:39:49.896955 7f27d600e700 10 mon.1@0(leader).pg v5938176 prepare_pg_stats pg_stats(25 pgs tid 64 v 10556) v1 from osd.6 2013-04-26 02:39:49.896970 7f27d600e700 10 mon.1@0(leader).pg v5938176 got osd.6 osd_stat(184 GB used, 745 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[]) (was osd_stat(184 GB used, 745 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[])) 2013-04-26 02:39:50.350708 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.4 10.227.227.104:6804/14471 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:50.350735 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x459e780 from 10.227.227.103:6789/0 2013-04-26 02:39:50.350895 7f27d600e700 10 mon.1@0(leader).pg v5938176 preprocess_query pg_stats(49 pgs tid 753 v 10556) v1 from osd.4 10.227.227.104:6804/14471 2013-04-26 02:39:50.351006 7f27d600e700 10 mon.1@0(leader).pg v5938176 prepare_update pg_stats(49 pgs tid 753 v 10556) v1 from osd.4 10.227.227.104:6804/14471 2013-04-26 02:39:50.351021 7f27d600e700 10 mon.1@0(leader).pg v5938176 prepare_pg_stats pg_stats(49 pgs tid 753 v 10556) v1 from osd.4 2013-04-26 02:39:50.351037 7f27d600e700 10 mon.1@0(leader).pg v5938176 got osd.4 osd_stat(423 GB used, 1437 GB avail, 1860 GB total, peers [0,1,2,6,7,8,9,10,11]/[]) (was osd_stat(423 GB used, 1437 GB avail, 1860 GB total, peers [0,1,2,6,7,8,9,10,11]/[])) 2013-04-26 02:39:50.428877 7f27d680f700 10 mon.1@0(leader).log v5604053 encode_full log v 5604053 2013-04-26 02:39:50.429165 7f27d680f700 10 mon.1@0(leader).log v5604053 encode_pending v5604054 2013-04-26 02:39:50.515126 7f27d680f700 10 mon.1@0(leader).pg v5938176 encode_pending v 5938177 2013-04-26 02:39:50.516232 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.3 10.227.227.104:6800/14463 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:50.516257 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x4f37900 from 10.227.227.103:6789/0 2013-04-26 02:39:50.565256 7f27d600e700 10 mon.1@0(leader).log v5604054 update_from_paxos 2013-04-26 02:39:50.565374 7f27d600e700 10 mon.1@0(leader).log v5604054 update_from_paxos version 5604054 summary v 5604053 2013-04-26 02:39:50.565424 7f27d600e700 10 mon.1@0(leader).log v5604054 update_from_paxos latest full 5604053 2013-04-26 02:39:50.565482 7f27d600e700 7 mon.1@0(leader).log v5604054 update_from_paxos applying incremental log 5604054 2013-04-26 02:39:49.527662 mon.0 10.227.227.101:6789/0 45 : [INF] pgmap v5938176: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 93035B/s wr, 13op/s 2013-04-26 02:39:50.565570 7f27d600e700 10 mon.1@0(leader).log v5604054 check_subs 2013-04-26 02:39:50.565641 7f27d600e700 10 mon.1@0(leader).log v5604054 create_pending v 5604055 2013-04-26 02:39:50.565703 7f27d600e700 7 mon.1@0(leader).log v5604054 _updated_log for mon.0 10.227.227.101:6789/0 2013-04-26 02:39:50.692977 7f27d600e700 7 mon.1@0(leader).pg v5938176 update_from_paxos applying incremental 5938177 2013-04-26 02:39:50.694534 7f27d600e700 10 mon.1@0(leader).pg v5938177 v5938177: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 635KB/s wr, 77op/s 2013-04-26 02:39:50.694649 7f27d600e700 10 mon.1@0(leader).pg v5938177 send_pg_creates to 0 pgs 2013-04-26 02:39:50.694658 7f27d600e700 10 mon.1@0(leader).pg v5938177 update_logger 2013-04-26 02:39:50.694782 7f27d600e700 10 mon.1@0(leader).pg v5938177 create_pending v 5938178 2013-04-26 02:39:50.694823 7f27d600e700 7 mon.1@0(leader).pg v5938177 _updated_stats for osd.7 10.227.227.103:6810/8502 2013-04-26 02:39:50.694880 7f27d600e700 7 mon.1@0(leader).pg v5938177 _updated_stats for osd.9 10.227.227.103:6816/8752 2013-04-26 02:39:50.695009 7f27d600e700 7 mon.1@0(leader).pg v5938177 _updated_stats for osd.2 10.227.227.101:6807/30207 2013-04-26 02:39:50.695244 7f27d600e700 7 mon.1@0(leader).pg v5938177 _updated_stats for osd.10 10.227.227.103:6801/8155 2013-04-26 02:39:50.695291 7f27d600e700 7 mon.1@0(leader).pg v5938177 _updated_stats for osd.1 10.227.227.101:6804/30088 2013-04-26 02:39:50.695321 7f27d600e700 7 mon.1@0(leader).pg v5938177 _updated_stats for osd.0 10.227.227.101:6801/29982 2013-04-26 02:39:50.695362 7f27d600e700 7 mon.1@0(leader).pg v5938177 _updated_stats for osd.6 10.227.227.103:6807/8402 2013-04-26 02:39:50.695399 7f27d600e700 7 mon.1@0(leader).pg v5938177 _updated_stats for osd.4 10.227.227.104:6804/14471 2013-04-26 02:39:50.695580 7f27d600e700 10 mon.1@0(leader).pg v5938177 preprocess_query pg_stats(53 pgs tid 758 v 10556) v1 from osd.3 10.227.227.104:6800/14463 2013-04-26 02:39:50.695813 7f27d600e700 10 mon.1@0(leader).pg v5938177 prepare_update pg_stats(53 pgs tid 758 v 10556) v1 from osd.3 10.227.227.104:6800/14463 2013-04-26 02:39:50.695835 7f27d600e700 10 mon.1@0(leader).pg v5938177 prepare_pg_stats pg_stats(53 pgs tid 758 v 10556) v1 from osd.3 2013-04-26 02:39:50.695850 7f27d600e700 10 mon.1@0(leader).pg v5938177 got osd.3 osd_stat(383 GB used, 1477 GB avail, 1860 GB total, peers [0,1,2,6,7,8,9,10,11]/[]) (was osd_stat(383 GB used, 1477 GB avail, 1860 GB total, peers [0,1,2,6,7,8,9,10,11]/[])) 2013-04-26 02:39:50.696034 7f27d600e700 10 mon.1@0(leader).pg v5938177 check_osd_map already seen 10556 >= 10556 2013-04-26 02:39:50.696046 7f27d600e700 10 mon.1@0(leader).pg v5938177 update_logger 2013-04-26 02:39:50.696075 7f27d600e700 0 log [INF] : pgmap v5938177: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 635KB/s wr, 77op/s 2013-04-26 02:39:50.696299 7f27d600e700 10 mon.1@0(leader).log v5604054 update_from_paxos 2013-04-26 02:39:50.696379 7f27d600e700 10 mon.1@0(leader).log v5604054 update_from_paxos version 5604054 summary v 5604054 2013-04-26 02:39:50.696452 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:50.696652 7f27d600e700 10 mon.1@0(leader).log v5604054 update_from_paxos 2013-04-26 02:39:50.696722 7f27d600e700 10 mon.1@0(leader).log v5604054 update_from_paxos version 5604054 summary v 5604054 2013-04-26 02:39:50.696790 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:50.696939 7f27d600e700 10 mon.1@0(leader).log v5604054 update_from_paxos 2013-04-26 02:39:50.697005 7f27d600e700 10 mon.1@0(leader).log v5604054 update_from_paxos version 5604054 summary v 5604054 2013-04-26 02:39:50.697041 7f27d600e700 10 mon.1@0(leader).log v5604054 preprocess_query log(1 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:50.697083 7f27d600e700 10 mon.1@0(leader).log v5604054 preprocess_log log(1 entries) v1 from mon.0 2013-04-26 02:39:50.697137 7f27d600e700 10 mon.1@0(leader).log v5604054 prepare_update log(1 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:50.697182 7f27d600e700 10 mon.1@0(leader).log v5604054 prepare_log log(1 entries) v1 from mon.0 2013-04-26 02:39:50.697223 7f27d600e700 10 mon.1@0(leader).log v5604054 logging 2013-04-26 02:39:50.696077 mon.0 10.227.227.101:6789/0 46 : [INF] pgmap v5938177: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 635KB/s wr, 77op/s 2013-04-26 02:39:50.743708 7f27d600e700 10 mon.1@0(leader).log v5604054 update_from_paxos 2013-04-26 02:39:50.743787 7f27d600e700 10 mon.1@0(leader).log v5604054 update_from_paxos version 5604054 summary v 5604054 2013-04-26 02:39:50.743858 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:50.919598 7f27d600e700 10 mon.1@0(leader).pg v5938177 preprocess_query pg_stats(21 pgs tid 63 v 10556) v1 from osd.8 10.227.227.103:6813/8628 2013-04-26 02:39:50.919667 7f27d600e700 10 mon.1@0(leader).pg v5938177 prepare_update pg_stats(21 pgs tid 63 v 10556) v1 from osd.8 10.227.227.103:6813/8628 2013-04-26 02:39:50.919676 7f27d600e700 10 mon.1@0(leader).pg v5938177 prepare_pg_stats pg_stats(21 pgs tid 63 v 10556) v1 from osd.8 2013-04-26 02:39:50.919689 7f27d600e700 10 mon.1@0(leader).pg v5938177 got osd.8 osd_stat(185 GB used, 744 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[]) (was osd_stat(185 GB used, 744 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[])) 2013-04-26 02:39:51.638719 7f27d680f700 10 mon.1@0(leader).pg v5938177 encode_pending v 5938178 2013-04-26 02:39:51.707659 7f27d680f700 10 mon.1@0(leader).log v5604054 encode_full log v 5604054 2013-04-26 02:39:51.707942 7f27d680f700 10 mon.1@0(leader).log v5604054 encode_pending v5604055 2013-04-26 02:39:51.757953 7f27d600e700 7 mon.1@0(leader).pg v5938177 update_from_paxos applying incremental 5938178 2013-04-26 02:39:51.758352 7f27d600e700 10 mon.1@0(leader).pg v5938178 v5938178: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 1313KB/s wr, 157op/s 2013-04-26 02:39:51.758419 7f27d600e700 10 mon.1@0(leader).pg v5938178 send_pg_creates to 0 pgs 2013-04-26 02:39:51.758424 7f27d600e700 10 mon.1@0(leader).pg v5938178 update_logger 2013-04-26 02:39:51.758475 7f27d600e700 10 mon.1@0(leader).pg v5938178 create_pending v 5938179 2013-04-26 02:39:51.758503 7f27d600e700 7 mon.1@0(leader).pg v5938178 _updated_stats for osd.3 10.227.227.104:6800/14463 2013-04-26 02:39:51.758561 7f27d600e700 7 mon.1@0(leader).pg v5938178 _updated_stats for osd.8 10.227.227.103:6813/8628 2013-04-26 02:39:51.758597 7f27d600e700 10 mon.1@0(leader).pg v5938178 check_osd_map already seen 10556 >= 10556 2013-04-26 02:39:51.758623 7f27d600e700 10 mon.1@0(leader).pg v5938178 update_logger 2013-04-26 02:39:51.758648 7f27d600e700 0 log [INF] : pgmap v5938178: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 1313KB/s wr, 157op/s 2013-04-26 02:39:51.859697 7f27d600e700 10 mon.1@0(leader).log v5604055 update_from_paxos 2013-04-26 02:39:51.859791 7f27d600e700 10 mon.1@0(leader).log v5604055 update_from_paxos version 5604055 summary v 5604054 2013-04-26 02:39:51.859857 7f27d600e700 10 mon.1@0(leader).log v5604055 update_from_paxos latest full 5604054 2013-04-26 02:39:51.859922 7f27d600e700 7 mon.1@0(leader).log v5604055 update_from_paxos applying incremental log 5604055 2013-04-26 02:39:50.696077 mon.0 10.227.227.101:6789/0 46 : [INF] pgmap v5938177: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 635KB/s wr, 77op/s 2013-04-26 02:39:51.860030 7f27d600e700 10 mon.1@0(leader).log v5604055 check_subs 2013-04-26 02:39:51.860121 7f27d600e700 10 mon.1@0(leader).log v5604055 create_pending v 5604056 2013-04-26 02:39:51.860214 7f27d600e700 7 mon.1@0(leader).log v5604055 _updated_log for mon.0 10.227.227.101:6789/0 2013-04-26 02:39:51.860343 7f27d600e700 10 mon.1@0(leader).log v5604055 update_from_paxos 2013-04-26 02:39:51.860413 7f27d600e700 10 mon.1@0(leader).log v5604055 update_from_paxos version 5604055 summary v 5604055 2013-04-26 02:39:51.860446 7f27d600e700 10 mon.1@0(leader).log v5604055 preprocess_query log(1 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:51.860485 7f27d600e700 10 mon.1@0(leader).log v5604055 preprocess_log log(1 entries) v1 from mon.0 2013-04-26 02:39:51.860528 7f27d600e700 10 mon.1@0(leader).log v5604055 nothing new 2013-04-26 02:39:51.860743 7f27d600e700 10 mon.1@0(leader).log v5604055 update_from_paxos 2013-04-26 02:39:51.860809 7f27d600e700 10 mon.1@0(leader).log v5604055 update_from_paxos version 5604055 summary v 5604055 2013-04-26 02:39:51.860873 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:51.861052 7f27d600e700 10 mon.1@0(leader).log v5604055 update_from_paxos 2013-04-26 02:39:51.861115 7f27d600e700 10 mon.1@0(leader).log v5604055 update_from_paxos version 5604055 summary v 5604055 2013-04-26 02:39:51.861178 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:51.910184 7f27d600e700 10 mon.1@0(leader).log v5604055 update_from_paxos 2013-04-26 02:39:51.910241 7f27d600e700 10 mon.1@0(leader).log v5604055 update_from_paxos version 5604055 summary v 5604055 2013-04-26 02:39:51.910282 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:52.031563 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from mds.? 10.227.227.101:6800/29852 via mon.2 10.227.227.104:6789/0 2013-04-26 02:39:52.031586 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x1789080 from 10.227.227.104:6789/0 2013-04-26 02:39:52.031730 7f27d600e700 10 mon.1@0(leader).mds e3724 preprocess_query mdsbeacon(217349/1 up:standby seq 468 v3724) v2 from mds.? 10.227.227.101:6800/29852 2013-04-26 02:39:52.503860 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from mds.0 10.227.227.104:6801/10842 via mon.2 10.227.227.104:6789/0 2013-04-26 02:39:52.503886 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x1788840 from 10.227.227.104:6789/0 2013-04-26 02:39:52.504055 7f27d600e700 10 mon.1@0(leader).mds e3724 preprocess_query mdsbeacon(217231/4 up:active seq 499 v3724) v2 from mds.0 10.227.227.104:6801/10842 2013-04-26 02:39:52.762253 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.5 10.227.227.104:6807/14479 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:52.762277 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x709bb80 from 10.227.227.103:6789/0 2013-04-26 02:39:52.762492 7f27d600e700 10 mon.1@0(leader).pg v5938178 preprocess_query pg_stats(35 pgs tid 751 v 10556) v1 from osd.5 10.227.227.104:6807/14479 2013-04-26 02:39:52.762604 7f27d600e700 10 mon.1@0(leader).pg v5938178 prepare_update pg_stats(35 pgs tid 751 v 10556) v1 from osd.5 10.227.227.104:6807/14479 2013-04-26 02:39:52.762619 7f27d600e700 10 mon.1@0(leader).pg v5938178 prepare_pg_stats pg_stats(35 pgs tid 751 v 10556) v1 from osd.5 2013-04-26 02:39:52.762635 7f27d600e700 10 mon.1@0(leader).pg v5938178 got osd.5 osd_stat(355 GB used, 1505 GB avail, 1860 GB total, peers [0,1,2,6,7,8,9,10,11]/[]) (was osd_stat(355 GB used, 1505 GB avail, 1860 GB total, peers [0,1,2,6,7,8,9,10,11]/[])) 2013-04-26 02:39:52.808660 7f27d680f700 10 mon.1@0(leader).pg v5938178 encode_pending v 5938179 2013-04-26 02:39:52.993216 7f27d600e700 7 mon.1@0(leader).pg v5938178 update_from_paxos applying incremental 5938179 2013-04-26 02:39:52.993465 7f27d600e700 10 mon.1@0(leader).pg v5938179 v5938179: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 1041KB/s wr, 95op/s 2013-04-26 02:39:52.993537 7f27d600e700 10 mon.1@0(leader).pg v5938179 send_pg_creates to 0 pgs 2013-04-26 02:39:52.993543 7f27d600e700 10 mon.1@0(leader).pg v5938179 update_logger 2013-04-26 02:39:52.993593 7f27d600e700 10 mon.1@0(leader).pg v5938179 create_pending v 5938180 2013-04-26 02:39:52.993634 7f27d600e700 7 mon.1@0(leader).pg v5938179 _updated_stats for osd.5 10.227.227.104:6807/14479 2013-04-26 02:39:52.993690 7f27d600e700 10 mon.1@0(leader).pg v5938179 check_osd_map already seen 10556 >= 10556 2013-04-26 02:39:52.993695 7f27d600e700 10 mon.1@0(leader).pg v5938179 update_logger 2013-04-26 02:39:52.993717 7f27d600e700 0 log [INF] : pgmap v5938179: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 1041KB/s wr, 95op/s 2013-04-26 02:39:52.993990 7f27d600e700 10 mon.1@0(leader).log v5604055 update_from_paxos 2013-04-26 02:39:52.994074 7f27d600e700 10 mon.1@0(leader).log v5604055 update_from_paxos version 5604055 summary v 5604055 2013-04-26 02:39:52.994147 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:52.994342 7f27d600e700 10 mon.1@0(leader).log v5604055 update_from_paxos 2013-04-26 02:39:52.994411 7f27d600e700 10 mon.1@0(leader).log v5604055 update_from_paxos version 5604055 summary v 5604055 2013-04-26 02:39:52.994480 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:52.994626 7f27d600e700 10 mon.1@0(leader).log v5604055 update_from_paxos 2013-04-26 02:39:52.994692 7f27d600e700 10 mon.1@0(leader).log v5604055 update_from_paxos version 5604055 summary v 5604055 2013-04-26 02:39:52.994728 7f27d600e700 10 mon.1@0(leader).log v5604055 preprocess_query log(2 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:52.994772 7f27d600e700 10 mon.1@0(leader).log v5604055 preprocess_log log(2 entries) v1 from mon.0 2013-04-26 02:39:52.994835 7f27d600e700 10 mon.1@0(leader).log v5604055 prepare_update log(2 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:52.994884 7f27d600e700 10 mon.1@0(leader).log v5604055 prepare_log log(2 entries) v1 from mon.0 2013-04-26 02:39:52.994925 7f27d600e700 10 mon.1@0(leader).log v5604055 logging 2013-04-26 02:39:51.758650 mon.0 10.227.227.101:6789/0 47 : [INF] pgmap v5938178: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 1313KB/s wr, 157op/s 2013-04-26 02:39:52.994998 7f27d600e700 10 mon.1@0(leader).log v5604055 logging 2013-04-26 02:39:52.993720 mon.0 10.227.227.101:6789/0 48 : [INF] pgmap v5938179: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 1041KB/s wr, 95op/s 2013-04-26 02:39:53.035004 7f27d600e700 10 mon.1@0(leader).log v5604055 update_from_paxos 2013-04-26 02:39:53.035081 7f27d600e700 10 mon.1@0(leader).log v5604055 update_from_paxos version 5604055 summary v 5604055 2013-04-26 02:39:53.035148 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:53.216757 7f27d600e700 10 mon.1@0(leader).mds e3724 preprocess_query mdsbeacon(217445/3 up:standby seq 81 v3724) v2 from mds.? 10.227.227.103:6800/8011 2013-04-26 02:39:53.899032 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.11 10.227.227.103:6804/8253 via mon.2 10.227.227.104:6789/0 2013-04-26 02:39:53.899055 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x1799400 from 10.227.227.104:6789/0 2013-04-26 02:39:53.899195 7f27d600e700 10 mon.1@0(leader).pg v5938179 preprocess_query pg_stats(21 pgs tid 66 v 10556) v1 from osd.11 10.227.227.103:6804/8253 2013-04-26 02:39:53.899293 7f27d600e700 10 mon.1@0(leader).pg v5938179 prepare_update pg_stats(21 pgs tid 66 v 10556) v1 from osd.11 10.227.227.103:6804/8253 2013-04-26 02:39:53.899303 7f27d600e700 10 mon.1@0(leader).pg v5938179 prepare_pg_stats pg_stats(21 pgs tid 66 v 10556) v1 from osd.11 2013-04-26 02:39:53.899315 7f27d600e700 10 mon.1@0(leader).pg v5938179 got osd.11 osd_stat(161 GB used, 768 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[]) (was osd_stat(161 GB used, 768 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[])) 2013-04-26 02:39:53.908864 7f27d680f700 10 mon.1@0(leader).log v5604055 encode_full log v 5604055 2013-04-26 02:39:53.909197 7f27d680f700 10 mon.1@0(leader).log v5604055 encode_pending v5604056 2013-04-26 02:39:53.992026 7f27d680f700 10 mon.1@0(leader).pg v5938179 encode_pending v 5938180 2013-04-26 02:39:54.042093 7f27d600e700 10 mon.1@0(leader).log v5604056 update_from_paxos 2013-04-26 02:39:54.042188 7f27d600e700 10 mon.1@0(leader).log v5604056 update_from_paxos version 5604056 summary v 5604055 2013-04-26 02:39:54.042233 7f27d600e700 10 mon.1@0(leader).log v5604056 update_from_paxos latest full 5604055 2013-04-26 02:39:54.042282 7f27d600e700 7 mon.1@0(leader).log v5604056 update_from_paxos applying incremental log 5604056 2013-04-26 02:39:51.758650 mon.0 10.227.227.101:6789/0 47 : [INF] pgmap v5938178: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 1313KB/s wr, 157op/s 2013-04-26 02:39:54.042338 7f27d600e700 7 mon.1@0(leader).log v5604056 update_from_paxos applying incremental log 5604056 2013-04-26 02:39:52.993720 mon.0 10.227.227.101:6789/0 48 : [INF] pgmap v5938179: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 1041KB/s wr, 95op/s 2013-04-26 02:39:54.042408 7f27d600e700 10 mon.1@0(leader).log v5604056 check_subs 2013-04-26 02:39:54.042473 7f27d600e700 10 mon.1@0(leader).log v5604056 create_pending v 5604057 2013-04-26 02:39:54.042527 7f27d600e700 7 mon.1@0(leader).log v5604056 _updated_log for mon.0 10.227.227.101:6789/0 2013-04-26 02:39:54.151653 7f27d600e700 7 mon.1@0(leader).pg v5938179 update_from_paxos applying incremental 5938180 2013-04-26 02:39:54.151771 7f27d600e700 10 mon.1@0(leader).pg v5938180 v5938180: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 1131KB/s wr, 43op/s 2013-04-26 02:39:54.151814 7f27d600e700 10 mon.1@0(leader).pg v5938180 send_pg_creates to 0 pgs 2013-04-26 02:39:54.151817 7f27d600e700 10 mon.1@0(leader).pg v5938180 update_logger 2013-04-26 02:39:54.151848 7f27d600e700 10 mon.1@0(leader).pg v5938180 create_pending v 5938181 2013-04-26 02:39:54.151868 7f27d600e700 7 mon.1@0(leader).pg v5938180 _updated_stats for osd.11 10.227.227.103:6804/8253 2013-04-26 02:39:54.151897 7f27d600e700 10 mon.1@0(leader).pg v5938180 check_osd_map already seen 10556 >= 10556 2013-04-26 02:39:54.151900 7f27d600e700 10 mon.1@0(leader).pg v5938180 update_logger 2013-04-26 02:39:54.151915 7f27d600e700 0 log [INF] : pgmap v5938180: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 1131KB/s wr, 43op/s 2013-04-26 02:39:54.152075 7f27d600e700 10 mon.1@0(leader).log v5604056 update_from_paxos 2013-04-26 02:39:54.152128 7f27d600e700 10 mon.1@0(leader).log v5604056 update_from_paxos version 5604056 summary v 5604056 2013-04-26 02:39:54.152169 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:54.152266 7f27d600e700 10 mon.1@0(leader).log v5604056 update_from_paxos 2013-04-26 02:39:54.152304 7f27d600e700 10 mon.1@0(leader).log v5604056 update_from_paxos version 5604056 summary v 5604056 2013-04-26 02:39:54.152324 7f27d600e700 10 mon.1@0(leader).log v5604056 preprocess_query log(1 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:54.152347 7f27d600e700 10 mon.1@0(leader).log v5604056 preprocess_log log(1 entries) v1 from mon.0 2013-04-26 02:39:54.152382 7f27d600e700 10 mon.1@0(leader).log v5604056 prepare_update log(1 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:54.152409 7f27d600e700 10 mon.1@0(leader).log v5604056 prepare_log log(1 entries) v1 from mon.0 2013-04-26 02:39:54.152432 7f27d600e700 10 mon.1@0(leader).log v5604056 logging 2013-04-26 02:39:54.151917 mon.0 10.227.227.101:6789/0 49 : [INF] pgmap v5938180: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 1131KB/s wr, 43op/s 2013-04-26 02:39:54.152565 7f27d600e700 10 mon.1@0(leader).log v5604056 update_from_paxos 2013-04-26 02:39:54.152606 7f27d600e700 10 mon.1@0(leader).log v5604056 update_from_paxos version 5604056 summary v 5604056 2013-04-26 02:39:54.152645 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:54.193480 7f27d600e700 10 mon.1@0(leader).log v5604056 update_from_paxos 2013-04-26 02:39:54.193553 7f27d600e700 10 mon.1@0(leader).log v5604056 update_from_paxos version 5604056 summary v 5604056 2013-04-26 02:39:54.193625 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:54.280948 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.7 10.227.227.103:6810/8502 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:54.280969 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x4f37900 from 10.227.227.103:6789/0 2013-04-26 02:39:54.281091 7f27d600e700 10 mon.1@0(leader).pg v5938180 preprocess_query pg_stats(13 pgs tid 62 v 10556) v1 from osd.7 10.227.227.103:6810/8502 2013-04-26 02:39:54.281180 7f27d600e700 10 mon.1@0(leader).pg v5938180 prepare_update pg_stats(13 pgs tid 62 v 10556) v1 from osd.7 10.227.227.103:6810/8502 2013-04-26 02:39:54.281194 7f27d600e700 10 mon.1@0(leader).pg v5938180 prepare_pg_stats pg_stats(13 pgs tid 62 v 10556) v1 from osd.7 2013-04-26 02:39:54.281209 7f27d600e700 10 mon.1@0(leader).pg v5938180 got osd.7 osd_stat(156 GB used, 772 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[]) (was osd_stat(156 GB used, 772 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[])) 2013-04-26 02:39:54.306317 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.9 10.227.227.103:6816/8752 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:54.306336 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x709b900 from 10.227.227.103:6789/0 2013-04-26 02:39:54.306536 7f27d600e700 10 mon.1@0(leader).pg v5938180 preprocess_query pg_stats(16 pgs tid 65 v 10556) v1 from osd.9 10.227.227.103:6816/8752 2013-04-26 02:39:54.306629 7f27d600e700 10 mon.1@0(leader).pg v5938180 prepare_update pg_stats(16 pgs tid 65 v 10556) v1 from osd.9 10.227.227.103:6816/8752 2013-04-26 02:39:54.306643 7f27d600e700 10 mon.1@0(leader).pg v5938180 prepare_pg_stats pg_stats(16 pgs tid 65 v 10556) v1 from osd.9 2013-04-26 02:39:54.306654 7f27d600e700 10 mon.1@0(leader).pg v5938180 got osd.9 osd_stat(161 GB used, 767 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[]) (was osd_stat(161 GB used, 767 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[])) 2013-04-26 02:39:54.473930 7f27d680f700 10 mon.1@0(leader).pg v5938180 check_down_pgs 2013-04-26 02:39:54.474204 7f27d680f700 10 mon.1@0(leader).pg v5938180 v5938180: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 1131KB/s wr, 43op/s 2013-04-26 02:39:54.474301 7f27d680f700 10 mon.1@0(leader).mds e3724 e3724: 1/1/1 up {0=4=up:active}, 2 up:standby 2013-04-26 02:39:54.474373 7f27d680f700 10 mon.1@0(leader).osd e10556 e10556: 12 osds: 12 up, 12 in 2013-04-26 02:39:54.474644 7f27d680f700 10 mon.1@0(leader).osd e10556 min_last_epoch_clean 10531 2013-04-26 02:39:54.474783 7f27d680f700 10 mon.1@0(leader).log v5604056 update_from_paxos 2013-04-26 02:39:54.474863 7f27d680f700 10 mon.1@0(leader).log v5604056 update_from_paxos version 5604056 summary v 5604056 2013-04-26 02:39:54.474903 7f27d680f700 10 mon.1@0(leader).log v5604056 log 2013-04-26 02:39:54.474978 7f27d680f700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:54.475064 7f27d680f700 10 mon.1@0(leader).auth v4370 auth 2013-04-26 02:39:54.538982 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.10 10.227.227.103:6801/8155 via mon.2 10.227.227.104:6789/0 2013-04-26 02:39:54.539006 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x459e780 from 10.227.227.104:6789/0 2013-04-26 02:39:54.539129 7f27d600e700 10 mon.1@0(leader).pg v5938180 preprocess_query pg_stats(21 pgs tid 61 v 10556) v1 from osd.10 10.227.227.103:6801/8155 2013-04-26 02:39:54.539219 7f27d600e700 10 mon.1@0(leader).pg v5938180 prepare_update pg_stats(21 pgs tid 61 v 10556) v1 from osd.10 10.227.227.103:6801/8155 2013-04-26 02:39:54.539230 7f27d600e700 10 mon.1@0(leader).pg v5938180 prepare_pg_stats pg_stats(21 pgs tid 61 v 10556) v1 from osd.10 2013-04-26 02:39:54.539245 7f27d600e700 10 mon.1@0(leader).pg v5938180 got osd.10 osd_stat(174 GB used, 755 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[]) (was osd_stat(174 GB used, 755 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[])) 2013-04-26 02:39:54.557928 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.2 10.227.227.101:6807/30207 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:54.557947 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x5022780 from 10.227.227.103:6789/0 2013-04-26 02:39:54.558061 7f27d600e700 10 mon.1@0(leader).pg v5938180 preprocess_query pg_stats(31 pgs tid 744 v 10556) v1 from osd.2 10.227.227.101:6807/30207 2013-04-26 02:39:54.558154 7f27d600e700 10 mon.1@0(leader).pg v5938180 prepare_update pg_stats(31 pgs tid 744 v 10556) v1 from osd.2 10.227.227.101:6807/30207 2013-04-26 02:39:54.558169 7f27d600e700 10 mon.1@0(leader).pg v5938180 prepare_pg_stats pg_stats(31 pgs tid 744 v 10556) v1 from osd.2 2013-04-26 02:39:54.558181 7f27d600e700 10 mon.1@0(leader).pg v5938180 got osd.2 osd_stat(311 GB used, 1549 GB avail, 1860 GB total, peers [3,4,5,6,7,8,9,10,11]/[]) (was osd_stat(311 GB used, 1549 GB avail, 1860 GB total, peers [3,4,5,6,7,8,9,10,11]/[])) 2013-04-26 02:39:54.649524 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.1 10.227.227.101:6804/30088 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:54.649553 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x709aa00 from 10.227.227.103:6789/0 2013-04-26 02:39:54.649695 7f27d600e700 10 mon.1@0(leader).pg v5938180 preprocess_query pg_stats(24 pgs tid 738 v 10556) v1 from osd.1 10.227.227.101:6804/30088 2013-04-26 02:39:54.649856 7f27d600e700 10 mon.1@0(leader).pg v5938180 prepare_update pg_stats(24 pgs tid 738 v 10556) v1 from osd.1 10.227.227.101:6804/30088 2013-04-26 02:39:54.649876 7f27d600e700 10 mon.1@0(leader).pg v5938180 prepare_pg_stats pg_stats(24 pgs tid 738 v 10556) v1 from osd.1 2013-04-26 02:39:54.649890 7f27d600e700 10 mon.1@0(leader).pg v5938180 got osd.1 osd_stat(343 GB used, 1517 GB avail, 1860 GB total, peers [3,4,5,6,7,8,9,10,11]/[]) (was osd_stat(343 GB used, 1517 GB avail, 1860 GB total, peers [3,4,5,6,7,8,9,10,11]/[])) 2013-04-26 02:39:54.691759 7f27d600e700 10 mon.1@0(leader).pg v5938180 preprocess_query pg_stats(44 pgs tid 750 v 10556) v1 from osd.0 10.227.227.101:6801/29982 2013-04-26 02:39:54.691850 7f27d600e700 10 mon.1@0(leader).pg v5938180 prepare_update pg_stats(44 pgs tid 750 v 10556) v1 from osd.0 10.227.227.101:6801/29982 2013-04-26 02:39:54.691862 7f27d600e700 10 mon.1@0(leader).pg v5938180 prepare_pg_stats pg_stats(44 pgs tid 750 v 10556) v1 from osd.0 2013-04-26 02:39:54.691884 7f27d600e700 10 mon.1@0(leader).pg v5938180 got osd.0 osd_stat(367 GB used, 1493 GB avail, 1860 GB total, peers [3,4,5,6,7,8,9,10,11]/[]) (was osd_stat(367 GB used, 1493 GB avail, 1860 GB total, peers [3,4,5,6,7,8,9,10,11]/[])) 2013-04-26 02:39:54.897100 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.6 10.227.227.103:6807/8402 via mon.2 10.227.227.104:6789/0 2013-04-26 02:39:54.897122 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x1799900 from 10.227.227.104:6789/0 2013-04-26 02:39:54.897267 7f27d600e700 10 mon.1@0(leader).pg v5938180 preprocess_query pg_stats(18 pgs tid 65 v 10556) v1 from osd.6 10.227.227.103:6807/8402 2013-04-26 02:39:54.897364 7f27d600e700 10 mon.1@0(leader).pg v5938180 prepare_update pg_stats(18 pgs tid 65 v 10556) v1 from osd.6 10.227.227.103:6807/8402 2013-04-26 02:39:54.897376 7f27d600e700 10 mon.1@0(leader).pg v5938180 prepare_pg_stats pg_stats(18 pgs tid 65 v 10556) v1 from osd.6 2013-04-26 02:39:54.897389 7f27d600e700 10 mon.1@0(leader).pg v5938180 got osd.6 osd_stat(184 GB used, 745 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[]) (was osd_stat(184 GB used, 745 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[])) 2013-04-26 02:39:55.098621 7f27d680f700 10 mon.1@0(leader).log v5604056 encode_full log v 5604056 2013-04-26 02:39:55.098910 7f27d680f700 10 mon.1@0(leader).log v5604056 encode_pending v5604057 2013-04-26 02:39:55.201646 7f27d680f700 10 mon.1@0(leader).pg v5938180 encode_pending v 5938181 2013-04-26 02:39:55.251628 7f27d600e700 10 mon.1@0(leader).log v5604057 update_from_paxos 2013-04-26 02:39:55.251741 7f27d600e700 10 mon.1@0(leader).log v5604057 update_from_paxos version 5604057 summary v 5604056 2013-04-26 02:39:55.251803 7f27d600e700 10 mon.1@0(leader).log v5604057 update_from_paxos latest full 5604056 2013-04-26 02:39:55.251871 7f27d600e700 7 mon.1@0(leader).log v5604057 update_from_paxos applying incremental log 5604057 2013-04-26 02:39:54.151917 mon.0 10.227.227.101:6789/0 49 : [INF] pgmap v5938180: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 1131KB/s wr, 43op/s 2013-04-26 02:39:55.252032 7f27d600e700 10 mon.1@0(leader).log v5604057 check_subs 2013-04-26 02:39:55.252143 7f27d600e700 10 mon.1@0(leader).log v5604057 create_pending v 5604058 2013-04-26 02:39:55.252235 7f27d600e700 7 mon.1@0(leader).log v5604057 _updated_log for mon.0 10.227.227.101:6789/0 2013-04-26 02:39:55.376882 7f27d600e700 7 mon.1@0(leader).pg v5938180 update_from_paxos applying incremental 5938181 2013-04-26 02:39:55.377843 7f27d600e700 10 mon.1@0(leader).pg v5938181 v5938181: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 5381KB/s wr, 193op/s 2013-04-26 02:39:55.377941 7f27d600e700 10 mon.1@0(leader).pg v5938181 send_pg_creates to 0 pgs 2013-04-26 02:39:55.377948 7f27d600e700 10 mon.1@0(leader).pg v5938181 update_logger 2013-04-26 02:39:55.378054 7f27d600e700 10 mon.1@0(leader).pg v5938181 create_pending v 5938182 2013-04-26 02:39:55.378096 7f27d600e700 7 mon.1@0(leader).pg v5938181 _updated_stats for osd.7 10.227.227.103:6810/8502 2013-04-26 02:39:55.378140 7f27d600e700 7 mon.1@0(leader).pg v5938181 _updated_stats for osd.9 10.227.227.103:6816/8752 2013-04-26 02:39:55.378184 7f27d600e700 7 mon.1@0(leader).pg v5938181 _updated_stats for osd.10 10.227.227.103:6801/8155 2013-04-26 02:39:55.378213 7f27d600e700 7 mon.1@0(leader).pg v5938181 _updated_stats for osd.2 10.227.227.101:6807/30207 2013-04-26 02:39:55.378295 7f27d600e700 7 mon.1@0(leader).pg v5938181 _updated_stats for osd.1 10.227.227.101:6804/30088 2013-04-26 02:39:55.378427 7f27d600e700 7 mon.1@0(leader).pg v5938181 _updated_stats for osd.0 10.227.227.101:6801/29982 2013-04-26 02:39:55.378470 7f27d600e700 7 mon.1@0(leader).pg v5938181 _updated_stats for osd.6 10.227.227.103:6807/8402 2013-04-26 02:39:55.378529 7f27d600e700 10 mon.1@0(leader).pg v5938181 check_osd_map already seen 10556 >= 10556 2013-04-26 02:39:55.378534 7f27d600e700 10 mon.1@0(leader).pg v5938181 update_logger 2013-04-26 02:39:55.378558 7f27d600e700 0 log [INF] : pgmap v5938181: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 5381KB/s wr, 193op/s 2013-04-26 02:39:55.419050 7f27d600e700 10 mon.1@0(leader).log v5604057 update_from_paxos 2013-04-26 02:39:55.419125 7f27d600e700 10 mon.1@0(leader).log v5604057 update_from_paxos version 5604057 summary v 5604057 2013-04-26 02:39:55.419162 7f27d600e700 10 mon.1@0(leader).log v5604057 preprocess_query log(1 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:55.419209 7f27d600e700 10 mon.1@0(leader).log v5604057 preprocess_log log(1 entries) v1 from mon.0 2013-04-26 02:39:55.419269 7f27d600e700 10 mon.1@0(leader).log v5604057 prepare_update log(1 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:55.419316 7f27d600e700 10 mon.1@0(leader).log v5604057 prepare_log log(1 entries) v1 from mon.0 2013-04-26 02:39:55.419358 7f27d600e700 10 mon.1@0(leader).log v5604057 logging 2013-04-26 02:39:55.378561 mon.0 10.227.227.101:6789/0 50 : [INF] pgmap v5938181: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 5381KB/s wr, 193op/s 2013-04-26 02:39:55.419503 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.4 10.227.227.104:6804/14471 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:55.419522 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x1798c80 from 10.227.227.103:6789/0 2013-04-26 02:39:55.419624 7f27d600e700 10 mon.1@0(leader).pg v5938181 preprocess_query pg_stats(36 pgs tid 754 v 10556) v1 from osd.4 10.227.227.104:6804/14471 2013-04-26 02:39:55.419731 7f27d600e700 10 mon.1@0(leader).pg v5938181 prepare_update pg_stats(36 pgs tid 754 v 10556) v1 from osd.4 10.227.227.104:6804/14471 2013-04-26 02:39:55.419748 7f27d600e700 10 mon.1@0(leader).pg v5938181 prepare_pg_stats pg_stats(36 pgs tid 754 v 10556) v1 from osd.4 2013-04-26 02:39:55.419760 7f27d600e700 10 mon.1@0(leader).pg v5938181 got osd.4 osd_stat(423 GB used, 1437 GB avail, 1860 GB total, peers [0,1,2,6,7,8,9,10,11]/[]) (was osd_stat(423 GB used, 1437 GB avail, 1860 GB total, peers [0,1,2,6,7,8,9,10,11]/[])) 2013-04-26 02:39:55.460455 7f27d600e700 10 mon.1@0(leader).log v5604057 update_from_paxos 2013-04-26 02:39:55.460634 7f27d600e700 10 mon.1@0(leader).log v5604057 update_from_paxos version 5604057 summary v 5604057 2013-04-26 02:39:55.460715 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:55.460984 7f27d600e700 10 mon.1@0(leader).log v5604057 update_from_paxos 2013-04-26 02:39:55.461057 7f27d600e700 10 mon.1@0(leader).log v5604057 update_from_paxos version 5604057 summary v 5604057 2013-04-26 02:39:55.461124 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:55.476148 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.3 10.227.227.104:6800/14463 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:55.476169 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x1799400 from 10.227.227.103:6789/0 2013-04-26 02:39:55.476315 7f27d600e700 10 mon.1@0(leader).pg v5938181 preprocess_query pg_stats(43 pgs tid 759 v 10556) v1 from osd.3 10.227.227.104:6800/14463 2013-04-26 02:39:55.476413 7f27d600e700 10 mon.1@0(leader).pg v5938181 prepare_update pg_stats(43 pgs tid 759 v 10556) v1 from osd.3 10.227.227.104:6800/14463 2013-04-26 02:39:55.476425 7f27d600e700 10 mon.1@0(leader).pg v5938181 prepare_pg_stats pg_stats(43 pgs tid 759 v 10556) v1 from osd.3 2013-04-26 02:39:55.476440 7f27d600e700 10 mon.1@0(leader).pg v5938181 got osd.3 osd_stat(383 GB used, 1477 GB avail, 1860 GB total, peers [0,1,2,6,7,8,9,10,11]/[]) (was osd_stat(383 GB used, 1477 GB avail, 1860 GB total, peers [0,1,2,6,7,8,9,10,11]/[])) 2013-04-26 02:39:55.502105 7f27d600e700 10 mon.1@0(leader).log v5604057 update_from_paxos 2013-04-26 02:39:55.502182 7f27d600e700 10 mon.1@0(leader).log v5604057 update_from_paxos version 5604057 summary v 5604057 2013-04-26 02:39:55.502250 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:55.920411 7f27d600e700 10 mon.1@0(leader).pg v5938181 preprocess_query pg_stats(20 pgs tid 64 v 10556) v1 from osd.8 10.227.227.103:6813/8628 2013-04-26 02:39:55.920534 7f27d600e700 10 mon.1@0(leader).pg v5938181 prepare_update pg_stats(20 pgs tid 64 v 10556) v1 from osd.8 10.227.227.103:6813/8628 2013-04-26 02:39:55.920547 7f27d600e700 10 mon.1@0(leader).pg v5938181 prepare_pg_stats pg_stats(20 pgs tid 64 v 10556) v1 from osd.8 2013-04-26 02:39:55.920563 7f27d600e700 10 mon.1@0(leader).pg v5938181 got osd.8 osd_stat(185 GB used, 744 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[]) (was osd_stat(185 GB used, 744 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[])) 2013-04-26 02:39:56.031574 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from mds.? 10.227.227.101:6800/29852 via mon.2 10.227.227.104:6789/0 2013-04-26 02:39:56.031593 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x1789080 from 10.227.227.104:6789/0 2013-04-26 02:39:56.031734 7f27d600e700 10 mon.1@0(leader).mds e3724 preprocess_query mdsbeacon(217349/1 up:standby seq 469 v3724) v2 from mds.? 10.227.227.101:6800/29852 2013-04-26 02:39:56.318899 7f27d680f700 10 mon.1@0(leader).log v5604057 encode_full log v 5604057 2013-04-26 02:39:56.319194 7f27d680f700 10 mon.1@0(leader).log v5604057 encode_pending v5604058 2013-04-26 02:39:56.394315 7f27d680f700 10 mon.1@0(leader).pg v5938181 encode_pending v 5938182 2013-04-26 02:39:56.444352 7f27d600e700 10 mon.1@0(leader).log v5604058 update_from_paxos 2013-04-26 02:39:56.444509 7f27d600e700 10 mon.1@0(leader).log v5604058 update_from_paxos version 5604058 summary v 5604057 2013-04-26 02:39:56.444617 7f27d600e700 10 mon.1@0(leader).log v5604058 update_from_paxos latest full 5604057 2013-04-26 02:39:56.444700 7f27d600e700 7 mon.1@0(leader).log v5604058 update_from_paxos applying incremental log 5604058 2013-04-26 02:39:55.378561 mon.0 10.227.227.101:6789/0 50 : [INF] pgmap v5938181: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 5381KB/s wr, 193op/s 2013-04-26 02:39:56.444860 7f27d600e700 10 mon.1@0(leader).log v5604058 check_subs 2013-04-26 02:39:56.445272 7f27d600e700 10 mon.1@0(leader).log v5604058 create_pending v 5604059 2013-04-26 02:39:56.445378 7f27d600e700 7 mon.1@0(leader).log v5604058 _updated_log for mon.0 10.227.227.101:6789/0 2013-04-26 02:39:56.503825 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from mds.0 10.227.227.104:6801/10842 via mon.2 10.227.227.104:6789/0 2013-04-26 02:39:56.503837 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x1789600 from 10.227.227.104:6789/0 2013-04-26 02:39:56.503932 7f27d600e700 10 mon.1@0(leader).mds e3724 preprocess_query mdsbeacon(217231/4 up:active seq 500 v3724) v2 from mds.0 10.227.227.104:6801/10842 2013-04-26 02:39:56.562926 7f27d600e700 7 mon.1@0(leader).pg v5938181 update_from_paxos applying incremental 5938182 2013-04-26 02:39:56.563401 7f27d600e700 10 mon.1@0(leader).pg v5938182 v5938182: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 5364KB/s wr, 237op/s 2013-04-26 02:39:56.563465 7f27d600e700 10 mon.1@0(leader).pg v5938182 send_pg_creates to 0 pgs 2013-04-26 02:39:56.563470 7f27d600e700 10 mon.1@0(leader).pg v5938182 update_logger 2013-04-26 02:39:56.563517 7f27d600e700 10 mon.1@0(leader).pg v5938182 create_pending v 5938183 2013-04-26 02:39:56.563537 7f27d600e700 7 mon.1@0(leader).pg v5938182 _updated_stats for osd.4 10.227.227.104:6804/14471 2013-04-26 02:39:56.563571 7f27d600e700 7 mon.1@0(leader).pg v5938182 _updated_stats for osd.3 10.227.227.104:6800/14463 2013-04-26 02:39:56.563780 7f27d600e700 7 mon.1@0(leader).pg v5938182 _updated_stats for osd.8 10.227.227.103:6813/8628 2013-04-26 02:39:56.563829 7f27d600e700 10 mon.1@0(leader).pg v5938182 check_osd_map already seen 10556 >= 10556 2013-04-26 02:39:56.563835 7f27d600e700 10 mon.1@0(leader).pg v5938182 update_logger 2013-04-26 02:39:56.563865 7f27d600e700 0 log [INF] : pgmap v5938182: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 5364KB/s wr, 237op/s 2013-04-26 02:39:56.564037 7f27d600e700 10 mon.1@0(leader).log v5604058 update_from_paxos 2013-04-26 02:39:56.564086 7f27d600e700 10 mon.1@0(leader).log v5604058 update_from_paxos version 5604058 summary v 5604058 2013-04-26 02:39:56.564127 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:56.564218 7f27d600e700 10 mon.1@0(leader).log v5604058 update_from_paxos 2013-04-26 02:39:56.564256 7f27d600e700 10 mon.1@0(leader).log v5604058 update_from_paxos version 5604058 summary v 5604058 2013-04-26 02:39:56.564276 7f27d600e700 10 mon.1@0(leader).log v5604058 preprocess_query log(1 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:56.564301 7f27d600e700 10 mon.1@0(leader).log v5604058 preprocess_log log(1 entries) v1 from mon.0 2013-04-26 02:39:56.564336 7f27d600e700 10 mon.1@0(leader).log v5604058 prepare_update log(1 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:56.564363 7f27d600e700 10 mon.1@0(leader).log v5604058 prepare_log log(1 entries) v1 from mon.0 2013-04-26 02:39:56.564386 7f27d600e700 10 mon.1@0(leader).log v5604058 logging 2013-04-26 02:39:56.563868 mon.0 10.227.227.101:6789/0 51 : [INF] pgmap v5938182: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 5364KB/s wr, 237op/s 2013-04-26 02:39:56.568790 7f27d600e700 10 mon.1@0(leader).log v5604058 update_from_paxos 2013-04-26 02:39:56.568842 7f27d600e700 10 mon.1@0(leader).log v5604058 update_from_paxos version 5604058 summary v 5604058 2013-04-26 02:39:56.568883 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:56.619254 7f27d600e700 10 mon.1@0(leader).log v5604058 update_from_paxos 2013-04-26 02:39:56.619328 7f27d600e700 10 mon.1@0(leader).log v5604058 update_from_paxos version 5604058 summary v 5604058 2013-04-26 02:39:56.619395 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:57.216922 7f27d600e700 10 mon.1@0(leader).mds e3724 preprocess_query mdsbeacon(217445/3 up:standby seq 82 v3724) v2 from mds.? 10.227.227.103:6800/8011 2013-04-26 02:39:57.508953 7f27d680f700 10 mon.1@0(leader).log v5604058 encode_full log v 5604058 2013-04-26 02:39:57.509266 7f27d680f700 10 mon.1@0(leader).log v5604058 encode_pending v5604059 2013-04-26 02:39:57.696315 7f27d600e700 10 mon.1@0(leader).log v5604059 update_from_paxos 2013-04-26 02:39:57.696414 7f27d600e700 10 mon.1@0(leader).log v5604059 update_from_paxos version 5604059 summary v 5604058 2013-04-26 02:39:57.696468 7f27d600e700 10 mon.1@0(leader).log v5604059 update_from_paxos latest full 5604058 2013-04-26 02:39:57.696518 7f27d600e700 7 mon.1@0(leader).log v5604059 update_from_paxos applying incremental log 5604059 2013-04-26 02:39:56.563868 mon.0 10.227.227.101:6789/0 51 : [INF] pgmap v5938182: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 5364KB/s wr, 237op/s 2013-04-26 02:39:57.696598 7f27d600e700 10 mon.1@0(leader).log v5604059 check_subs 2013-04-26 02:39:57.696667 7f27d600e700 10 mon.1@0(leader).log v5604059 create_pending v 5604060 2013-04-26 02:39:57.696751 7f27d600e700 7 mon.1@0(leader).log v5604059 _updated_log for mon.0 10.227.227.101:6789/0 2013-04-26 02:39:57.696936 7f27d600e700 10 mon.1@0(leader).log v5604059 update_from_paxos 2013-04-26 02:39:57.696998 7f27d600e700 10 mon.1@0(leader).log v5604059 update_from_paxos version 5604059 summary v 5604059 2013-04-26 02:39:57.697039 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:57.709183 7f27d600e700 10 mon.1@0(leader).log v5604059 update_from_paxos 2013-04-26 02:39:57.709258 7f27d600e700 10 mon.1@0(leader).log v5604059 update_from_paxos version 5604059 summary v 5604059 2013-04-26 02:39:57.709325 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:57.752757 7f27d600e700 10 mon.1@0(leader).log v5604059 update_from_paxos 2013-04-26 02:39:57.752830 7f27d600e700 10 mon.1@0(leader).log v5604059 update_from_paxos version 5604059 summary v 5604059 2013-04-26 02:39:57.752897 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:57.762484 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.5 10.227.227.104:6807/14479 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:57.762502 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x6ce4a00 from 10.227.227.103:6789/0 2013-04-26 02:39:57.762647 7f27d600e700 10 mon.1@0(leader).pg v5938182 preprocess_query pg_stats(33 pgs tid 752 v 10556) v1 from osd.5 10.227.227.104:6807/14479 2013-04-26 02:39:57.762739 7f27d600e700 10 mon.1@0(leader).pg v5938182 prepare_update pg_stats(33 pgs tid 752 v 10556) v1 from osd.5 10.227.227.104:6807/14479 2013-04-26 02:39:57.762751 7f27d600e700 10 mon.1@0(leader).pg v5938182 prepare_pg_stats pg_stats(33 pgs tid 752 v 10556) v1 from osd.5 2013-04-26 02:39:57.762766 7f27d600e700 10 mon.1@0(leader).pg v5938182 got osd.5 osd_stat(355 GB used, 1505 GB avail, 1860 GB total, peers [0,1,2,6,7,8,9,10,11]/[]) (was osd_stat(355 GB used, 1505 GB avail, 1860 GB total, peers [0,1,2,6,7,8,9,10,11]/[])) 2013-04-26 02:39:58.658635 7f27d680f700 10 mon.1@0(leader).pg v5938182 encode_pending v 5938183 2013-04-26 02:39:58.811344 7f27d600e700 7 mon.1@0(leader).pg v5938182 update_from_paxos applying incremental 5938183 2013-04-26 02:39:58.811574 7f27d600e700 10 mon.1@0(leader).pg v5938183 v5938183: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 557KB/s wr, 50op/s 2013-04-26 02:39:58.811645 7f27d600e700 10 mon.1@0(leader).pg v5938183 send_pg_creates to 0 pgs 2013-04-26 02:39:58.811653 7f27d600e700 10 mon.1@0(leader).pg v5938183 update_logger 2013-04-26 02:39:58.811703 7f27d600e700 10 mon.1@0(leader).pg v5938183 create_pending v 5938184 2013-04-26 02:39:58.811755 7f27d600e700 7 mon.1@0(leader).pg v5938183 _updated_stats for osd.5 10.227.227.104:6807/14479 2013-04-26 02:39:58.811812 7f27d600e700 10 mon.1@0(leader).pg v5938183 check_osd_map already seen 10556 >= 10556 2013-04-26 02:39:58.811818 7f27d600e700 10 mon.1@0(leader).pg v5938183 update_logger 2013-04-26 02:39:58.811841 7f27d600e700 0 log [INF] : pgmap v5938183: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 557KB/s wr, 50op/s 2013-04-26 02:39:58.812080 7f27d600e700 10 mon.1@0(leader).log v5604059 update_from_paxos 2013-04-26 02:39:58.812161 7f27d600e700 10 mon.1@0(leader).log v5604059 update_from_paxos version 5604059 summary v 5604059 2013-04-26 02:39:58.812233 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:58.812426 7f27d600e700 10 mon.1@0(leader).log v5604059 update_from_paxos 2013-04-26 02:39:58.812497 7f27d600e700 10 mon.1@0(leader).log v5604059 update_from_paxos version 5604059 summary v 5604059 2013-04-26 02:39:58.812563 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:58.812704 7f27d600e700 10 mon.1@0(leader).log v5604059 update_from_paxos 2013-04-26 02:39:58.812767 7f27d600e700 10 mon.1@0(leader).log v5604059 update_from_paxos version 5604059 summary v 5604059 2013-04-26 02:39:58.812801 7f27d600e700 10 mon.1@0(leader).log v5604059 preprocess_query log(1 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:58.812841 7f27d600e700 10 mon.1@0(leader).log v5604059 preprocess_log log(1 entries) v1 from mon.0 2013-04-26 02:39:58.812897 7f27d600e700 10 mon.1@0(leader).log v5604059 prepare_update log(1 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:58.812943 7f27d600e700 10 mon.1@0(leader).log v5604059 prepare_log log(1 entries) v1 from mon.0 2013-04-26 02:39:58.812982 7f27d600e700 10 mon.1@0(leader).log v5604059 logging 2013-04-26 02:39:58.811843 mon.0 10.227.227.101:6789/0 52 : [INF] pgmap v5938183: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 557KB/s wr, 50op/s 2013-04-26 02:39:58.860961 7f27d600e700 10 mon.1@0(leader).log v5604059 update_from_paxos 2013-04-26 02:39:58.861054 7f27d600e700 10 mon.1@0(leader).log v5604059 update_from_paxos version 5604059 summary v 5604059 2013-04-26 02:39:58.861127 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:58.899563 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.11 10.227.227.103:6804/8253 via mon.2 10.227.227.104:6789/0 2013-04-26 02:39:58.899582 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x2ebb680 from 10.227.227.104:6789/0 2013-04-26 02:39:58.899718 7f27d600e700 10 mon.1@0(leader).pg v5938183 preprocess_query pg_stats(13 pgs tid 67 v 10556) v1 from osd.11 10.227.227.103:6804/8253 2013-04-26 02:39:58.899816 7f27d600e700 10 mon.1@0(leader).pg v5938183 prepare_update pg_stats(13 pgs tid 67 v 10556) v1 from osd.11 10.227.227.103:6804/8253 2013-04-26 02:39:58.899828 7f27d600e700 10 mon.1@0(leader).pg v5938183 prepare_pg_stats pg_stats(13 pgs tid 67 v 10556) v1 from osd.11 2013-04-26 02:39:58.899843 7f27d600e700 10 mon.1@0(leader).pg v5938183 got osd.11 osd_stat(161 GB used, 768 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[]) (was osd_stat(161 GB used, 768 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[])) 2013-04-26 02:39:59.281447 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.7 10.227.227.103:6810/8502 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:59.281468 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x5022780 from 10.227.227.103:6789/0 2013-04-26 02:39:59.281626 7f27d600e700 10 mon.1@0(leader).pg v5938183 preprocess_query pg_stats(10 pgs tid 63 v 10556) v1 from osd.7 10.227.227.103:6810/8502 2013-04-26 02:39:59.281742 7f27d600e700 10 mon.1@0(leader).pg v5938183 prepare_update pg_stats(10 pgs tid 63 v 10556) v1 from osd.7 10.227.227.103:6810/8502 2013-04-26 02:39:59.281758 7f27d600e700 10 mon.1@0(leader).pg v5938183 prepare_pg_stats pg_stats(10 pgs tid 63 v 10556) v1 from osd.7 2013-04-26 02:39:59.281774 7f27d600e700 10 mon.1@0(leader).pg v5938183 got osd.7 osd_stat(156 GB used, 772 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[]) (was osd_stat(156 GB used, 772 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[])) 2013-04-26 02:39:59.306812 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.9 10.227.227.103:6816/8752 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:59.306834 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x6ce4280 from 10.227.227.103:6789/0 2013-04-26 02:39:59.306950 7f27d600e700 10 mon.1@0(leader).pg v5938183 preprocess_query pg_stats(12 pgs tid 66 v 10556) v1 from osd.9 10.227.227.103:6816/8752 2013-04-26 02:39:59.307036 7f27d600e700 10 mon.1@0(leader).pg v5938183 prepare_update pg_stats(12 pgs tid 66 v 10556) v1 from osd.9 10.227.227.103:6816/8752 2013-04-26 02:39:59.307050 7f27d600e700 10 mon.1@0(leader).pg v5938183 prepare_pg_stats pg_stats(12 pgs tid 66 v 10556) v1 from osd.9 2013-04-26 02:39:59.307060 7f27d600e700 10 mon.1@0(leader).pg v5938183 got osd.9 osd_stat(161 GB used, 767 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[]) (was osd_stat(161 GB used, 767 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[])) 2013-04-26 02:39:59.383301 7f27d680f700 10 mon.1@0(leader).data_health(2814) service_tick 2013-04-26 02:39:59.383360 7f27d680f700 0 mon.1@0(leader).data_health(2814) update_stats avail 44% total 683567804 used 376056116 avail 307511688 2013-04-26 02:39:59.383371 7f27d680f700 10 mon.1@0(leader).data_health(2814) share_stats 2013-04-26 02:39:59.475306 7f27d680f700 10 mon.1@0(leader).pg v5938183 check_down_pgs 2013-04-26 02:39:59.475579 7f27d680f700 10 mon.1@0(leader).pg v5938183 v5938183: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 557KB/s wr, 50op/s 2013-04-26 02:39:59.475680 7f27d680f700 10 mon.1@0(leader).mds e3724 e3724: 1/1/1 up {0=4=up:active}, 2 up:standby 2013-04-26 02:39:59.475753 7f27d680f700 10 mon.1@0(leader).osd e10556 e10556: 12 osds: 12 up, 12 in 2013-04-26 02:39:59.476020 7f27d680f700 10 mon.1@0(leader).osd e10556 min_last_epoch_clean 10531 2013-04-26 02:39:59.476171 7f27d680f700 10 mon.1@0(leader).log v5604059 update_from_paxos 2013-04-26 02:39:59.476270 7f27d680f700 10 mon.1@0(leader).log v5604059 update_from_paxos version 5604059 summary v 5604059 2013-04-26 02:39:59.476305 7f27d680f700 10 mon.1@0(leader).log v5604059 log 2013-04-26 02:39:59.476375 7f27d680f700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:59.476448 7f27d680f700 10 mon.1@0(leader).auth v4370 auth 2013-04-26 02:39:59.540160 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.10 10.227.227.103:6801/8155 via mon.2 10.227.227.104:6789/0 2013-04-26 02:39:59.540182 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x709aa00 from 10.227.227.104:6789/0 2013-04-26 02:39:59.540340 7f27d600e700 10 mon.1@0(leader).pg v5938183 preprocess_query pg_stats(13 pgs tid 62 v 10556) v1 from osd.10 10.227.227.103:6801/8155 2013-04-26 02:39:59.540436 7f27d600e700 10 mon.1@0(leader).pg v5938183 prepare_update pg_stats(13 pgs tid 62 v 10556) v1 from osd.10 10.227.227.103:6801/8155 2013-04-26 02:39:59.540447 7f27d600e700 10 mon.1@0(leader).pg v5938183 prepare_pg_stats pg_stats(13 pgs tid 62 v 10556) v1 from osd.10 2013-04-26 02:39:59.540462 7f27d600e700 10 mon.1@0(leader).pg v5938183 got osd.10 osd_stat(174 GB used, 755 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[]) (was osd_stat(174 GB used, 755 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[])) 2013-04-26 02:39:59.553855 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.2 10.227.227.101:6807/30207 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:59.553870 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x1799900 from 10.227.227.103:6789/0 2013-04-26 02:39:59.553982 7f27d600e700 10 mon.1@0(leader).pg v5938183 preprocess_query pg_stats(23 pgs tid 745 v 10556) v1 from osd.2 10.227.227.101:6807/30207 2013-04-26 02:39:59.554081 7f27d600e700 10 mon.1@0(leader).pg v5938183 prepare_update pg_stats(23 pgs tid 745 v 10556) v1 from osd.2 10.227.227.101:6807/30207 2013-04-26 02:39:59.554096 7f27d600e700 10 mon.1@0(leader).pg v5938183 prepare_pg_stats pg_stats(23 pgs tid 745 v 10556) v1 from osd.2 2013-04-26 02:39:59.554107 7f27d600e700 10 mon.1@0(leader).pg v5938183 got osd.2 osd_stat(311 GB used, 1549 GB avail, 1860 GB total, peers [3,4,5,6,7,8,9,10,11]/[]) (was osd_stat(311 GB used, 1549 GB avail, 1860 GB total, peers [3,4,5,6,7,8,9,10,11]/[])) 2013-04-26 02:39:59.650580 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.1 10.227.227.101:6804/30088 via mon.1 10.227.227.103:6789/0 2013-04-26 02:39:59.650599 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x6ce4000 from 10.227.227.103:6789/0 2013-04-26 02:39:59.650749 7f27d600e700 10 mon.1@0(leader).pg v5938183 preprocess_query pg_stats(19 pgs tid 739 v 10556) v1 from osd.1 10.227.227.101:6804/30088 2013-04-26 02:39:59.650846 7f27d600e700 10 mon.1@0(leader).pg v5938183 prepare_update pg_stats(19 pgs tid 739 v 10556) v1 from osd.1 10.227.227.101:6804/30088 2013-04-26 02:39:59.650858 7f27d600e700 10 mon.1@0(leader).pg v5938183 prepare_pg_stats pg_stats(19 pgs tid 739 v 10556) v1 from osd.1 2013-04-26 02:39:59.650873 7f27d600e700 10 mon.1@0(leader).pg v5938183 got osd.1 osd_stat(343 GB used, 1517 GB avail, 1860 GB total, peers [3,4,5,6,7,8,9,10,11]/[]) (was osd_stat(343 GB used, 1517 GB avail, 1860 GB total, peers [3,4,5,6,7,8,9,10,11]/[])) 2013-04-26 02:39:59.692550 7f27d600e700 10 mon.1@0(leader).pg v5938183 preprocess_query pg_stats(28 pgs tid 751 v 10556) v1 from osd.0 10.227.227.101:6801/29982 2013-04-26 02:39:59.692657 7f27d600e700 10 mon.1@0(leader).pg v5938183 prepare_update pg_stats(28 pgs tid 751 v 10556) v1 from osd.0 10.227.227.101:6801/29982 2013-04-26 02:39:59.692672 7f27d600e700 10 mon.1@0(leader).pg v5938183 prepare_pg_stats pg_stats(28 pgs tid 751 v 10556) v1 from osd.0 2013-04-26 02:39:59.692687 7f27d600e700 10 mon.1@0(leader).pg v5938183 got osd.0 osd_stat(367 GB used, 1493 GB avail, 1860 GB total, peers [3,4,5,6,7,8,9,10,11]/[]) (was osd_stat(367 GB used, 1493 GB avail, 1860 GB total, peers [3,4,5,6,7,8,9,10,11]/[])) 2013-04-26 02:39:59.729136 7f27d680f700 10 mon.1@0(leader).pg v5938183 encode_pending v 5938184 2013-04-26 02:39:59.813589 7f27d680f700 10 mon.1@0(leader).log v5604059 encode_full log v 5604059 2013-04-26 02:39:59.813763 7f27d680f700 10 mon.1@0(leader).log v5604059 encode_pending v5604060 2013-04-26 02:39:59.872395 7f27d600e700 7 mon.1@0(leader).pg v5938183 update_from_paxos applying incremental 5938184 2013-04-26 02:39:59.873126 7f27d600e700 10 mon.1@0(leader).pg v5938184 v5938184: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 277KB/s wr, 17op/s 2013-04-26 02:39:59.873218 7f27d600e700 10 mon.1@0(leader).pg v5938184 send_pg_creates to 0 pgs 2013-04-26 02:39:59.873225 7f27d600e700 10 mon.1@0(leader).pg v5938184 update_logger 2013-04-26 02:39:59.873288 7f27d600e700 10 mon.1@0(leader).pg v5938184 create_pending v 5938185 2013-04-26 02:39:59.873328 7f27d600e700 7 mon.1@0(leader).pg v5938184 _updated_stats for osd.11 10.227.227.103:6804/8253 2013-04-26 02:39:59.873376 7f27d600e700 7 mon.1@0(leader).pg v5938184 _updated_stats for osd.7 10.227.227.103:6810/8502 2013-04-26 02:39:59.873403 7f27d600e700 7 mon.1@0(leader).pg v5938184 _updated_stats for osd.9 10.227.227.103:6816/8752 2013-04-26 02:39:59.873429 7f27d600e700 7 mon.1@0(leader).pg v5938184 _updated_stats for osd.10 10.227.227.103:6801/8155 2013-04-26 02:39:59.873451 7f27d600e700 7 mon.1@0(leader).pg v5938184 _updated_stats for osd.2 10.227.227.101:6807/30207 2013-04-26 02:39:59.873486 7f27d600e700 7 mon.1@0(leader).pg v5938184 _updated_stats for osd.1 10.227.227.101:6804/30088 2013-04-26 02:39:59.873562 7f27d600e700 7 mon.1@0(leader).pg v5938184 _updated_stats for osd.0 10.227.227.101:6801/29982 2013-04-26 02:39:59.873607 7f27d600e700 10 mon.1@0(leader).pg v5938184 check_osd_map already seen 10556 >= 10556 2013-04-26 02:39:59.873612 7f27d600e700 10 mon.1@0(leader).pg v5938184 update_logger 2013-04-26 02:39:59.873633 7f27d600e700 0 log [INF] : pgmap v5938184: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 277KB/s wr, 17op/s 2013-04-26 02:39:59.922892 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.6 10.227.227.103:6807/8402 via mon.2 10.227.227.104:6789/0 2013-04-26 02:39:59.922913 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x2eb8f00 from 10.227.227.104:6789/0 2013-04-26 02:39:59.923012 7f27d600e700 10 mon.1@0(leader).pg v5938184 preprocess_query pg_stats(11 pgs tid 66 v 10556) v1 from osd.6 10.227.227.103:6807/8402 2013-04-26 02:39:59.923102 7f27d600e700 10 mon.1@0(leader).pg v5938184 prepare_update pg_stats(11 pgs tid 66 v 10556) v1 from osd.6 10.227.227.103:6807/8402 2013-04-26 02:39:59.923117 7f27d600e700 10 mon.1@0(leader).pg v5938184 prepare_pg_stats pg_stats(11 pgs tid 66 v 10556) v1 from osd.6 2013-04-26 02:39:59.923142 7f27d600e700 10 mon.1@0(leader).pg v5938184 got osd.6 osd_stat(184 GB used, 745 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[]) (was osd_stat(184 GB used, 745 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[])) 2013-04-26 02:39:59.980804 7f27d600e700 10 mon.1@0(leader).log v5604060 update_from_paxos 2013-04-26 02:39:59.980913 7f27d600e700 10 mon.1@0(leader).log v5604060 update_from_paxos version 5604060 summary v 5604059 2013-04-26 02:39:59.980991 7f27d600e700 10 mon.1@0(leader).log v5604060 update_from_paxos latest full 5604059 2013-04-26 02:39:59.981065 7f27d600e700 7 mon.1@0(leader).log v5604060 update_from_paxos applying incremental log 5604060 2013-04-26 02:39:58.811843 mon.0 10.227.227.101:6789/0 52 : [INF] pgmap v5938183: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 557KB/s wr, 50op/s 2013-04-26 02:39:59.981196 7f27d600e700 10 mon.1@0(leader).log v5604060 check_subs 2013-04-26 02:39:59.981301 7f27d600e700 10 mon.1@0(leader).log v5604060 create_pending v 5604061 2013-04-26 02:39:59.981400 7f27d600e700 7 mon.1@0(leader).log v5604060 _updated_log for mon.0 10.227.227.101:6789/0 2013-04-26 02:39:59.981528 7f27d600e700 10 mon.1@0(leader).log v5604060 update_from_paxos 2013-04-26 02:39:59.981597 7f27d600e700 10 mon.1@0(leader).log v5604060 update_from_paxos version 5604060 summary v 5604060 2013-04-26 02:39:59.981634 7f27d600e700 10 mon.1@0(leader).log v5604060 preprocess_query log(1 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:39:59.981674 7f27d600e700 10 mon.1@0(leader).log v5604060 preprocess_log log(1 entries) v1 from mon.0 2013-04-26 02:39:59.981727 7f27d600e700 10 mon.1@0(leader).log v5604060 nothing new 2013-04-26 02:39:59.981925 7f27d600e700 10 mon.1@0(leader).log v5604060 update_from_paxos 2013-04-26 02:39:59.981993 7f27d600e700 10 mon.1@0(leader).log v5604060 update_from_paxos version 5604060 summary v 5604060 2013-04-26 02:39:59.982061 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:39:59.988859 7f27d600e700 10 mon.1@0(leader).log v5604060 update_from_paxos 2013-04-26 02:39:59.988934 7f27d600e700 10 mon.1@0(leader).log v5604060 update_from_paxos version 5604060 summary v 5604060 2013-04-26 02:39:59.989001 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:40:00.027853 7f27d600e700 10 mon.1@0(leader).log v5604060 update_from_paxos 2013-04-26 02:40:00.027929 7f27d600e700 10 mon.1@0(leader).log v5604060 update_from_paxos version 5604060 summary v 5604060 2013-04-26 02:40:00.027996 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:40:00.031674 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from mds.? 10.227.227.101:6800/29852 via mon.2 10.227.227.104:6789/0 2013-04-26 02:40:00.031692 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x1789080 from 10.227.227.104:6789/0 2013-04-26 02:40:00.031808 7f27d600e700 10 mon.1@0(leader).mds e3724 preprocess_query mdsbeacon(217349/1 up:standby seq 470 v3724) v2 from mds.? 10.227.227.101:6800/29852 2013-04-26 02:40:00.351243 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.4 10.227.227.104:6804/14471 via mon.1 10.227.227.103:6789/0 2013-04-26 02:40:00.351290 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x6ce4c80 from 10.227.227.103:6789/0 2013-04-26 02:40:00.351491 7f27d600e700 10 mon.1@0(leader).pg v5938184 preprocess_query pg_stats(23 pgs tid 755 v 10556) v1 from osd.4 10.227.227.104:6804/14471 2013-04-26 02:40:00.351624 7f27d600e700 10 mon.1@0(leader).pg v5938184 prepare_update pg_stats(23 pgs tid 755 v 10556) v1 from osd.4 10.227.227.104:6804/14471 2013-04-26 02:40:00.351641 7f27d600e700 10 mon.1@0(leader).pg v5938184 prepare_pg_stats pg_stats(23 pgs tid 755 v 10556) v1 from osd.4 2013-04-26 02:40:00.351657 7f27d600e700 10 mon.1@0(leader).pg v5938184 got osd.4 osd_stat(423 GB used, 1437 GB avail, 1860 GB total, peers [0,1,2,6,7,8,9,10,11]/[]) (was osd_stat(423 GB used, 1437 GB avail, 1860 GB total, peers [0,1,2,6,7,8,9,10,11]/[])) 2013-04-26 02:40:00.476540 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from osd.3 10.227.227.104:6800/14463 via mon.1 10.227.227.103:6789/0 2013-04-26 02:40:00.476565 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x6ce4f00 from 10.227.227.103:6789/0 2013-04-26 02:40:00.476802 7f27d600e700 10 mon.1@0(leader).pg v5938184 preprocess_query pg_stats(19 pgs tid 760 v 10556) v1 from osd.3 10.227.227.104:6800/14463 2013-04-26 02:40:00.476924 7f27d600e700 10 mon.1@0(leader).pg v5938184 prepare_update pg_stats(19 pgs tid 760 v 10556) v1 from osd.3 10.227.227.104:6800/14463 2013-04-26 02:40:00.476944 7f27d600e700 10 mon.1@0(leader).pg v5938184 prepare_pg_stats pg_stats(19 pgs tid 760 v 10556) v1 from osd.3 2013-04-26 02:40:00.476961 7f27d600e700 10 mon.1@0(leader).pg v5938184 got osd.3 osd_stat(383 GB used, 1477 GB avail, 1860 GB total, peers [0,1,2,6,7,8,9,10,11]/[]) (was osd_stat(383 GB used, 1477 GB avail, 1860 GB total, peers [0,1,2,6,7,8,9,10,11]/[])) 2013-04-26 02:40:00.504235 7f27d600e700 10 mon.1@0(leader) e1 received forwarded message from mds.0 10.227.227.104:6801/10842 via mon.2 10.227.227.104:6789/0 2013-04-26 02:40:00.504253 7f27d600e700 10 mon.1@0(leader) e1 mesg 0x1788dc0 from 10.227.227.104:6789/0 2013-04-26 02:40:00.504382 7f27d600e700 10 mon.1@0(leader).mds e3724 preprocess_query mdsbeacon(217231/4 up:active seq 501 v3724) v2 from mds.0 10.227.227.104:6801/10842 2013-04-26 02:40:00.819277 7f27d680f700 10 mon.1@0(leader).pg v5938184 encode_pending v 5938185 2013-04-26 02:40:01.253658 7f27d4602700 10 mon.1@0(leader) e1 ms_verify_authorizer 10.227.227.103:0/11179 client protocol 0 2013-04-26 02:40:01.509053 7f27d600e700 10 mon.1@0(leader).mds e3724 preprocess_query mdsbeacon(217445/3 up:standby seq 83 v3724) v2 from mds.? 10.227.227.103:6800/8011 2013-04-26 02:40:01.509124 7f27d600e700 10 mon.1@0(leader) e1 do not have session, making new one 2013-04-26 02:40:01.509128 7f27d600e700 10 mon.1@0(leader) e1 ms_dispatch new session MonSession: client.? 10.227.227.103:0/11179 is open for client.? 10.227.227.103:0/11179 2013-04-26 02:40:01.509136 7f27d600e700 10 mon.1@0(leader) e1 setting timeout on session 2013-04-26 02:40:01.509213 7f27d600e700 10 mon.1@0(leader).auth v4370 update_from_paxos 2013-04-26 02:40:01.509278 7f27d600e700 10 mon.1@0(leader).auth v4370 preprocess_query auth(proto 0 30 bytes epoch 0) v1 from client.? 10.227.227.103:0/11179 2013-04-26 02:40:01.509319 7f27d600e700 10 mon.1@0(leader).auth v4370 prep_auth() blob_size=30 2013-04-26 02:40:01.509367 7f27d600e700 10 mon.1@0(leader).auth v4370 AuthMonitor::assign_global_id m=auth(proto 0 30 bytes epoch 0) v1 mon=0/3 last_allocated=218796 max_global_id=218796 2013-04-26 02:40:01.509408 7f27d600e700 10 mon.1@0(leader).auth v4370 next_global_id should be 218799 2013-04-26 02:40:01.509441 7f27d600e700 10 mon.1@0(leader).auth v4370 prepare_update auth(proto 0 30 bytes epoch 0) v1 from client.? 10.227.227.103:0/11179 2013-04-26 02:40:01.509487 7f27d600e700 10 mon.1@0(leader).auth v4370 prep_auth() blob_size=30 2013-04-26 02:40:01.509529 7f27d600e700 10 mon.1@0(leader).auth v4370 AuthMonitor::assign_global_id m=auth(proto 0 30 bytes epoch 0) v1 mon=0/3 last_allocated=218796 max_global_id=218796 2013-04-26 02:40:01.509582 7f27d600e700 10 mon.1@0(leader).auth v4370 next_global_id should be 218799 2013-04-26 02:40:01.509616 7f27d600e700 10 mon.1@0(leader).auth v4370 increasing max_global_id to 218896 2013-04-26 02:40:01.509648 7f27d600e700 10 mon.1@0(leader).auth v4370 increasing global id, waitlisting message 2013-04-26 02:40:01.509809 7f27d600e700 10 mon.1@0(leader).auth v4370 encode_pending v 4371 2013-04-26 02:40:01.594574 7f27d600e700 7 mon.1@0(leader).pg v5938184 update_from_paxos applying incremental 5938185 2013-04-26 02:40:01.594958 7f27d600e700 10 mon.1@0(leader).pg v5938185 v5938185: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 700KB/s wr, 69op/s 2013-04-26 02:40:01.595097 7f27d600e700 10 mon.1@0(leader).pg v5938185 send_pg_creates to 0 pgs 2013-04-26 02:40:01.595104 7f27d600e700 10 mon.1@0(leader).pg v5938185 update_logger 2013-04-26 02:40:01.595357 7f27d600e700 10 mon.1@0(leader).pg v5938185 create_pending v 5938186 2013-04-26 02:40:01.595406 7f27d600e700 7 mon.1@0(leader).pg v5938185 _updated_stats for osd.6 10.227.227.103:6807/8402 2013-04-26 02:40:01.595488 7f27d600e700 7 mon.1@0(leader).pg v5938185 _updated_stats for osd.4 10.227.227.104:6804/14471 2013-04-26 02:40:01.595522 7f27d600e700 7 mon.1@0(leader).pg v5938185 _updated_stats for osd.3 10.227.227.104:6800/14463 2013-04-26 02:40:01.595704 7f27d600e700 10 mon.1@0(leader).pg v5938185 preprocess_query pg_stats(9 pgs tid 65 v 10556) v1 from osd.8 10.227.227.103:6813/8628 2013-04-26 02:40:01.595806 7f27d600e700 10 mon.1@0(leader).pg v5938185 prepare_update pg_stats(9 pgs tid 65 v 10556) v1 from osd.8 10.227.227.103:6813/8628 2013-04-26 02:40:01.595820 7f27d600e700 10 mon.1@0(leader).pg v5938185 prepare_pg_stats pg_stats(9 pgs tid 65 v 10556) v1 from osd.8 2013-04-26 02:40:01.595833 7f27d600e700 10 mon.1@0(leader).pg v5938185 got osd.8 osd_stat(185 GB used, 744 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[]) (was osd_stat(185 GB used, 744 GB avail, 929 GB total, peers [0,1,2,3,4,5]/[])) 2013-04-26 02:40:01.595915 7f27d600e700 10 mon.1@0(leader).pg v5938185 check_osd_map already seen 10556 >= 10556 2013-04-26 02:40:01.595923 7f27d600e700 10 mon.1@0(leader).pg v5938185 update_logger 2013-04-26 02:40:01.595947 7f27d600e700 0 log [INF] : pgmap v5938185: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 700KB/s wr, 69op/s 2013-04-26 02:40:01.610255 7f27d600e700 10 mon.1@0(leader).log v5604060 update_from_paxos 2013-04-26 02:40:01.610321 7f27d600e700 10 mon.1@0(leader).log v5604060 update_from_paxos version 5604060 summary v 5604060 2013-04-26 02:40:01.610356 7f27d600e700 10 mon.1@0(leader).log v5604060 preprocess_query log(2 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:40:01.610396 7f27d600e700 10 mon.1@0(leader).log v5604060 preprocess_log log(2 entries) v1 from mon.0 2013-04-26 02:40:01.610455 7f27d600e700 10 mon.1@0(leader).log v5604060 prepare_update log(2 entries) v1 from mon.0 10.227.227.101:6789/0 2013-04-26 02:40:01.610498 7f27d600e700 10 mon.1@0(leader).log v5604060 prepare_log log(2 entries) v1 from mon.0 2013-04-26 02:40:01.610545 7f27d600e700 10 mon.1@0(leader).log v5604060 logging 2013-04-26 02:39:59.873636 mon.0 10.227.227.101:6789/0 53 : [INF] pgmap v5938184: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 277KB/s wr, 17op/s 2013-04-26 02:40:01.610605 7f27d600e700 10 mon.1@0(leader).log v5604060 logging 2013-04-26 02:40:01.595950 mon.0 10.227.227.101:6789/0 54 : [INF] pgmap v5938185: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 700KB/s wr, 69op/s 2013-04-26 02:40:01.654028 7f27d600e700 10 mon.1@0(leader).auth v4371 update_from_paxos 2013-04-26 02:40:01.654159 7f27d600e700 10 mon.1@0(leader).auth v4371 update_from_paxos version 4371 keys ver 4370 latest 4367 2013-04-26 02:40:01.654193 7f27d600e700 10 mon.1@0(leader).auth v4371 update_from_paxos key server version 4370 2013-04-26 02:40:01.654258 7f27d600e700 10 mon.1@0(leader).auth v4371 update_from_paxos() last_allocated_id=218796 max_global_id=218896 2013-04-26 02:40:01.654315 7f27d600e700 10 mon.1@0(leader).auth v4371 create_pending v 4372 2013-04-26 02:40:01.654513 7f27d600e700 10 mon.1@0(leader).auth v4371 update_from_paxos 2013-04-26 02:40:01.654570 7f27d600e700 10 mon.1@0(leader).auth v4371 preprocess_query auth(proto 0 30 bytes epoch 0) v1 from client.? 10.227.227.103:0/11179 2013-04-26 02:40:01.654611 7f27d600e700 10 mon.1@0(leader).auth v4371 prep_auth() blob_size=30 2013-04-26 02:40:01.654654 7f27d600e700 10 mon.1@0(leader).auth v4371 AuthMonitor::assign_global_id m=auth(proto 0 30 bytes epoch 0) v1 mon=0/3 last_allocated=218796 max_global_id=218896 2013-04-26 02:40:01.654688 7f27d600e700 10 mon.1@0(leader).auth v4371 next_global_id should be 218799 2013-04-26 02:40:01.654839 7f27d600e700 10 mon.1@0(leader).auth v4371 AuthMonitor::on_active() 2013-04-26 02:40:01.655122 7f27d600e700 10 mon.1@0(leader).log v5604060 update_from_paxos 2013-04-26 02:40:01.655182 7f27d600e700 10 mon.1@0(leader).log v5604060 update_from_paxos version 5604060 summary v 5604060 2013-04-26 02:40:01.655224 7f27d600e700 10 mon.1@0(leader).auth v4371 update_from_paxos 2013-04-26 02:40:01.655334 7f27d600e700 10 mon.1@0(leader).log v5604060 update_from_paxos 2013-04-26 02:40:01.655375 7f27d600e700 10 mon.1@0(leader).log v5604060 update_from_paxos version 5604060 summary v 5604060 2013-04-26 02:40:01.655468 7f27d600e700 10 mon.1@0(leader).auth v4371 update_from_paxos 2013-04-26 02:40:01.656188 7f27d600e700 10 mon.1@0(leader).auth v4371 update_from_paxos 2013-04-26 02:40:01.656233 7f27d600e700 10 mon.1@0(leader).auth v4371 preprocess_query auth(proto 2 32 bytes epoch 0) v1 from client.? 10.227.227.103:0/11179 2013-04-26 02:40:01.656257 7f27d600e700 10 mon.1@0(leader).auth v4371 prep_auth() blob_size=32 2013-04-26 02:40:01.657178 7f27d600e700 10 mon.1@0(leader).auth v4371 update_from_paxos 2013-04-26 02:40:01.657218 7f27d600e700 10 mon.1@0(leader).auth v4371 preprocess_query auth(proto 2 165 bytes epoch 0) v1 from client.? 10.227.227.103:0/11179 2013-04-26 02:40:01.657243 7f27d600e700 10 mon.1@0(leader).auth v4371 prep_auth() blob_size=165 2013-04-26 02:40:01.658131 7f27d600e700 10 mon.1@0(leader) e1 handle_subscribe mon_subscribe({monmap=0+}) v2 2013-04-26 02:40:01.658145 7f27d600e700 10 mon.1@0(leader) e1 check_sub monmap next 0 have 1 2013-04-26 02:40:01.658191 7f27d600e700 0 mon.1@0(leader) e1 handle_command mon_command(health detail v 0) v1 2013-04-26 02:40:01.658532 7f27d600e700 10 mon.1@0(leader).data_health(2814) get_health 2013-04-26 02:40:01.659292 7f27d600e700 10 mon.1@0(leader) e1 ms_handle_reset 0x45a8580 10.227.227.103:0/11179 2013-04-26 02:40:01.659308 7f27d600e700 10 mon.1@0(leader) e1 reset/close on session client.? 10.227.227.103:0/11179 2013-04-26 02:40:01.659315 7f27d600e700 10 mon.1@0(leader) e1 remove_session MonSession: client.? 10.227.227.103:0/11179 is openallow * client.? 10.227.227.103:0/11179 2013-04-26 02:40:01.662607 7f27d600e700 10 mon.1@0(leader).log v5604060 update_from_paxos 2013-04-26 02:40:01.662660 7f27d600e700 10 mon.1@0(leader).log v5604060 update_from_paxos version 5604060 summary v 5604060 2013-04-26 02:40:01.662701 7f27d600e700 10 mon.1@0(leader).auth v4371 update_from_paxos 2013-04-26 02:40:02.518966 7f27d680f700 10 mon.1@0(leader).log v5604060 encode_full log v 5604060 2013-04-26 02:40:02.519248 7f27d680f700 10 mon.1@0(leader).log v5604060 encode_pending v5604061 2013-04-26 02:40:02.582293 7f27d680f700 10 mon.1@0(leader).pg v5938185 encode_pending v 5938186 2013-04-26 02:40:02.644116 7f27d600e700 10 mon.1@0(leader).log v5604061 update_from_paxos 2013-04-26 02:40:02.644227 7f27d600e700 10 mon.1@0(leader).log v5604061 update_from_paxos version 5604061 summary v 5604060 2013-04-26 02:40:02.644294 7f27d600e700 10 mon.1@0(leader).log v5604061 update_from_paxos latest full 5604060 2013-04-26 02:40:02.644370 7f27d600e700 7 mon.1@0(leader).log v5604061 update_from_paxos applying incremental log 5604061 2013-04-26 02:39:59.873636 mon.0 10.227.227.101:6789/0 53 : [INF] pgmap v5938184: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 277KB/s wr, 17op/s 2013-04-26 02:40:02.644450 7f27d600e700 7 mon.1@0(leader).log v5604061 update_from_paxos applying incremental log 5604061 2013-04-26 02:40:01.595950 mon.0 10.227.227.101:6789/0 54 : [INF] pgmap v5938185: 2428 pgs: 2428 active+clean; 1836 GB data, 3207 GB used, 13536 GB / 16744 GB avail; 700KB/s wr, 69op/s 2013-04-26 02:40:02.644554 7f27d600e700 10 mon.1@0(leader).log v5604061 check_subs 2013-04-26 02:40:02.644646 7f27d600e700 10 mon.1@0(leader).log v5604061 create_pending v 5604062 2013-04-26 02:40:02.644732 7f27d600e700 7 mon.1@0(leader).log v5604061 _updated_log for mon.0 10.227.227.101:6789/0 2013-04-26 02:40:27.183282 7f27d4b07700 10 mon.1@0(leader) e1 ms_verify_authorizer 10.227.227.104:0/26847 client protocol 0 2013-04-26 02:40:43.820201 7f27d4c08700 10 mon.1@0(leader) e1 ms_verify_authorizer 10.227.227.101:0/17880 client protocol 0