Project

General

Profile

Bug #6223 » ceph.txt

jordi arcas, 09/05/2013 01:25 AM

 
ceph.conf

[global]
fsid = f0ed8a65-acc4-4f60-b849-345d413daa0b
mon_initial_members = ceph-node1
mon_host = X.X.X.X
auth_supported = cephx
osd_journal_size = 1024
filestore_xattr_use_omap = true

-------
ceph-mon -f
IO error: /var/lib/ceph/mon/ceph-admin/store.db/LOCK: No such file or directory
IO error: /var/lib/ceph/mon/ceph-admin/store.db/LOCK: No such file or directory
2013-09-05 09:54:05.733324 7f889d0fd780 -1 failed to create new leveldb store

-------
ceph daemon mon.XXXX mon_status
{ "name": "XXXX",
"rank": 0,
"state": "leader",
"election_epoch": 1,
"quorum": [
0],
"outside_quorum": [],
"extra_probe_peers": [],
"sync_provider": [],
"monmap": { "epoch": 1,
"fsid": "a318feba-9e9c-43e2-9cfc-41a1e91eae8d",
"modified": "0.000000",
"created": "0.000000",
"mons": [
{ "rank": 0,
"name": "XXXX",
"addr": "X.X.X.X:6789\/0"}]}}

-------
/etc/hosts
127.0.0.1 localhost
X.X.X.X XXXX.local XXXX
80.31.93.204 ubuntu

# The following lines are desirable for IPv6 capable hosts
::1 ip6-localhost ip6-loopback
fe00::0 ip6-localnet
ff00::0 ip6-mcastprefix
ff02::1 ip6-allnodes
ff02::2 ip6-allrouters
-------

cat /var/log/ceph/ceph-mon.admin.log
2013-09-05 09:50:44.484498 7f68ff4b3780 0 ceph version 0.67.2 (eb4380dd036a0b644c6283869911d615ed729ac8), process ceph-mon, pid 27371
2013-09-05 09:50:44.487313 7f68ff4b3780 -1 failed to create new leveldb store
2013-09-05 09:54:05.729951 7f889d0fd780 0 ceph version 0.67.2 (eb4380dd036a0b644c6283869911d615ed729ac8), process ceph-mon, pid 27396
2013-09-05 09:54:05.733324 7f889d0fd780 -1 failed to create new leveldb store





--------
cat /var/log/ceph/ceph-mon.XXXX.log

ive c 1..57) is_readable now=2013-09-05 10:04:52.650119 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:04:52.650144 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:04:52.650146 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:04:52.650151 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:04:52.650152 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:04:57.650314 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:04:57.650317 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:04:57.650343 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:04:57.650344 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:04:57.650348 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:04:57.650349 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:05:02.650524 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:05:02.650536 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:05:02.650580 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:05:02.650599 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:05:02.650607 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:05:02.650607 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:05:07.650882 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:05:07.650893 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:05:07.650935 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:05:07.650936 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:05:07.650943 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:05:07.650944 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:05:12.651144 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:05:12.651148 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:05:12.651174 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:05:12.651175 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:05:12.651180 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:05:12.651181 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:05:17.651358 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:05:17.651369 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:05:17.651407 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:05:17.651409 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:05:17.651416 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:05:17.651417 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:05:22.651632 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:05:22.651644 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:05:22.654706 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:05:22.654706 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:05:22.654721 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:05:22.654721 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:05:27.654852 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:05:27.654855 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:05:27.654881 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:05:27.654883 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:05:27.654888 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:05:27.654889 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:05:32.655049 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:05:32.655053 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:05:32.655079 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:05:32.655080 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:05:32.655085 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:05:32.655085 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:05:33.202932 7f6dfa910700 0 mon.XXXX@0(leader).data_health(1) update_stats avail 92% total 161367312 used 3430448 avail 149857916
2013-09-05 10:05:37.655175 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:05:37.655179 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:05:37.655203 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:05:37.655204 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:05:37.655209 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:05:37.655210 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:05:42.655409 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:05:42.655416 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:05:42.655449 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:05:42.655450 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:05:42.655456 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:05:42.655457 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:05:47.655617 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:05:47.655620 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:05:47.655647 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:05:47.655648 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:05:47.655653 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:05:47.655653 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:05:52.655846 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:05:52.655850 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:05:52.655876 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:05:52.655878 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:05:52.655883 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:05:52.655884 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:05:57.657096 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:05:57.657096 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:05:57.657096 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:05:57.657096 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:05:57.657096 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:05:57.657096 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:06:02.657261 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:06:02.657272 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:06:02.657314 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:06:02.657316 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:06:02.657323 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:06:02.657324 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:06:07.657528 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:06:07.657539 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:06:07.657581 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:06:07.657582 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:06:07.657589 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:06:07.657589 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:06:12.657800 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:06:12.657803 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:06:12.657827 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:06:12.657828 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:06:12.657833 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:06:12.657833 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:06:17.657980 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:06:17.657983 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:06:17.658007 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:06:17.658008 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:06:17.658013 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:06:17.658013 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:06:22.658241 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:06:22.658252 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:06:22.658292 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:06:22.658293 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:06:22.658300 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:06:22.658301 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:06:27.658534 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:06:27.658537 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:06:27.658561 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:06:27.658562 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:06:27.658566 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:06:27.658567 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:06:32.658783 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:06:32.658794 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:06:32.658834 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:06:32.658836 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:06:32.658843 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:06:32.658844 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:06:33.203122 7f6dfa910700 0 mon.XXXX@0(leader).data_health(1) update_stats avail 92% total 161367312 used 3430448 avail 149857916
2013-09-05 10:06:37.659010 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:06:37.659013 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:06:37.659037 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:06:37.659038 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:06:37.659043 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:06:37.659043 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:06:42.659250 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:06:42.659254 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:06:42.659278 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:06:42.659279 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:06:42.659283 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:06:42.659285 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:06:47.659471 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:06:47.659474 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:06:47.659498 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:06:47.659500 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:06:47.659505 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:06:47.659505 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:06:52.659685 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:06:52.659689 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:06:52.659713 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:06:52.659714 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:06:52.659719 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:06:52.659719 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:06:57.659900 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:06:57.659903 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:06:57.659929 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:06:57.659931 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:06:57.659935 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:06:57.659936 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:07:02.660127 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:07:02.660138 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:07:02.660179 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:07:02.660181 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:07:02.660188 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:07:02.660188 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:07:07.660461 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:07:07.660472 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:07:07.660514 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:07:07.660515 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:07:07.660522 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:07:07.660522 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:07:12.660685 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:07:12.660689 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:07:12.660714 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:07:12.660715 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:07:12.660719 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:07:12.660720 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:07:17.660879 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:07:17.660883 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:07:17.660908 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:07:17.660909 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:07:17.660914 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:07:17.660914 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:07:22.661164 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:07:22.661176 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:07:22.661218 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:07:22.661219 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:07:22.661227 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:07:22.661227 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:07:27.661459 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:07:27.661463 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:07:27.661490 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:07:27.661491 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:07:27.661496 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:07:27.661497 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:07:32.661682 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:07:32.661685 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:07:32.661710 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:07:32.661712 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:07:32.661717 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:07:32.661718 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:07:33.203319 7f6dfa910700 0 mon.XXXX@0(leader).data_health(1) update_stats avail 92% total 161367312 used 3430456 avail 149857908
2013-09-05 10:07:37.661886 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:07:37.661891 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:07:37.661919 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:07:37.661920 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:07:37.661925 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:07:37.661926 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:07:42.662127 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:07:42.662130 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:07:42.662158 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:07:42.662159 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:07:42.662164 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:07:42.662164 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:07:47.662362 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:07:47.662374 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:07:47.662416 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:07:47.662418 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:07:47.662425 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:07:47.662425 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:07:52.662571 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:07:52.662574 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:07:52.662626 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:07:52.662628 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:07:52.662633 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:07:52.662634 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:07:57.662795 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:07:57.662799 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:07:57.662824 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:07:57.662825 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:07:57.662830 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:07:57.662831 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:08:02.662969 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:08:02.662980 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:08:02.663024 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:08:02.663025 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:08:02.663032 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:08:02.663033 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:08:07.663258 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:08:07.663269 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:08:07.663314 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:08:07.663315 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:08:07.663322 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:08:07.663323 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:08:12.663494 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:08:12.663497 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:08:12.663523 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:08:12.663524 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:08:12.663528 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:08:12.663529 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:08:17.663664 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:08:17.663668 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:08:17.663695 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:08:17.663697 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:08:17.663701 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:08:17.663702 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:08:22.663881 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:08:22.663892 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:08:22.663933 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:08:22.663934 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:08:22.663941 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:08:22.663941 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:08:27.664086 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:08:27.664089 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:08:27.664115 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:08:27.664116 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:08:27.664121 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:08:27.664123 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:08:32.664311 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:08:32.664316 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:08:32.664347 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:08:32.664348 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:08:32.664353 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:08:32.664354 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:08:33.203522 7f6dfa910700 0 mon.XXXX@0(leader).data_health(1) update_stats avail 92% total 161367312 used 3430460 avail 149857904
2013-09-05 10:08:37.664494 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:08:37.664498 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:08:37.664523 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:08:37.664525 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:08:37.664530 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:08:37.664531 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:08:42.664689 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:08:42.664694 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:08:42.664722 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:08:42.664724 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:08:42.664729 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:08:42.664729 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:08:47.664882 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:08:47.664885 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:08:47.664912 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:08:47.664914 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:08:47.664919 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:08:47.664920 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:08:52.665080 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:08:52.665083 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:08:52.665110 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:08:52.665111 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:08:52.665116 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:08:52.665116 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:08:57.665254 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:08:57.665257 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:08:57.665284 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:08:57.665285 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:08:57.665290 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:08:57.665291 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:09:02.665488 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:09:02.665499 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:09:02.665542 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:09:02.665543 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:09:02.665550 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:09:02.665551 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:09:07.665713 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:09:07.665724 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:09:07.665764 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:09:07.665765 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:09:07.665772 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:09:07.665773 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:09:12.665822 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:09:12.665825 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:09:12.665848 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:09:12.665849 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:09:12.665854 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:09:12.665854 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:09:17.665994 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:09:17.665997 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:09:17.666023 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:09:17.666024 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:09:17.666029 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:09:17.666030 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:09:22.666209 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:09:22.666221 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:09:22.666262 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:09:22.666263 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:09:22.666270 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:09:22.666271 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:09:27.666423 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:09:27.666426 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:09:27.666453 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:09:27.666454 lease_expire=0.000000 has v0 lc 57
2013-09-05 10:09:27.666459 7f6dfa910700 1 mon.XXXX@0(leader).paxos(paxos active c 1..57) is_readable now=2013-09-05 10:09:27.666460 lease_expire=0.000000 has v0 lc 57






(1-1/2)