|
2011-11-10 14:50:47.087423 7f2f73271760 ceph version 0.37-364-ga3dd5bd (commit:a3dd5bd67ba19aae51a51318138ef10213a91449), process ceph-mon, pid 12888
|
|
2011-11-10 14:50:47.089633 7f2f73271760 store(/data/mon0) created monfs at /data/mon0 for 0
|
|
2011-11-10 15:28:24.866249 7f3facd8f760 ceph version 0.37-364-ga3dd5bd (commit:a3dd5bd67ba19aae51a51318138ef10213a91449), process ceph-mon, pid 13863
|
|
2011-11-10 15:28:24.868399 7f3facd8f760 store(/data/mon0) created monfs at /data/mon0 for 0
|
|
2011-11-10 15:28:34.997639 7ffc711be760 ceph version 0.37-364-ga3dd5bd (commit:a3dd5bd67ba19aae51a51318138ef10213a91449), process ceph-mon, pid 13927
|
|
2011-11-10 15:28:34.997711 7ffc711be760 store(/data/mon0) mount
|
|
2011-11-10 15:28:35.000915 7ffc711be760 mon.0@0(probing) e1 init fsid fc6e103e-e856-40d4-59e7-aab2a4255b3b
|
|
2011-11-10 15:28:35.101474 7ffc6c4ce700 -- 192.168.122.95:6789/0 >> 192.168.122.74:6789/0 pipe(0x2458280 sd=9 pgs=0 cs=0 l=0).fault first fault
|
|
2011-11-10 15:28:35.101602 7ffc711ba700 -- 192.168.122.95:6789/0 >> 192.168.122.37:6789/0 pipe(0x2458500 sd=10 pgs=0 cs=0 l=0).fault first fault
|
|
2011-11-10 15:28:49.292594 7ffc6c3cd700 -- 192.168.122.95:6789/0 >> 192.168.122.37:6789/0 pipe(0x2458a00 sd=11 pgs=0 cs=0 l=0).accept connect_seq 0 vs existing 0 state 1
|
|
2011-11-10 15:28:49.294154 7ffc6d4d0700 log [INF] : mon.0 calling new monitor election
|
|
2011-11-10 15:28:54.486321 7ffc6cccf700 log [INF] : mon.0@0 won leader election with quorum 0,1
|
|
2011-11-10 15:28:55.505728 7ffc6d4d0700 mon.0@0(leader).osd e1 e1: 0 osds: 0 up, 0 in
|
|
2011-11-10 15:28:57.430655 7ffc6bec8700 -- 192.168.122.95:6789/0 >> 192.168.122.74:6789/0 pipe(0x2458c80 sd=14 pgs=0 cs=0 l=0).accept connect_seq 0 vs existing 0 state 1
|
|
2011-11-10 15:28:57.441423 7ffc6d4d0700 mon.0@0(leader).mds e1 preprocess_beacon on fsid 00000000-0000-0000-0000-000000000000 != fc6e103e-e856-40d4-59e7-aab2a4255b3b
|
|
2011-11-10 15:28:57.442396 7ffc6d4d0700 mon.0@0(leader).mds e1 preprocess_beacon on fsid 00000000-0000-0000-0000-000000000000 != fc6e103e-e856-40d4-59e7-aab2a4255b3b
|
|
2011-11-10 15:28:57.603229 7ffc6d4d0700 log [INF] : mon.0 calling new monitor election
|
|
2011-11-10 15:28:58.049942 7ffc6d4d0700 log [INF] : mon.0@0 won leader election with quorum 0,1,2
|
|
2011-11-10 15:29:02.904967 7ffc6d4d0700 mon.0@0(leader).osd e2 e2: 1 osds: 1 up, 1 in
|
|
2011-11-10 15:29:03.319655 7ffc6d4d0700 log [INF] : osd.1 192.168.122.37:6801/9027 boot
|
|
2011-11-10 15:29:04.026644 7ffc6d4d0700 mon.0@0(leader).mds e2 adding standby 192.168.122.95:6800/14002 as mds.0
|
|
2011-11-10 15:29:04.197576 7ffc6d4d0700 log [INF] : mds.? 192.168.122.95:6800/14002 up:boot
|
|
2011-11-10 15:29:04.197622 7ffc6d4d0700 mon.0@0(leader).mds e2 preprocess_beacon on fsid 00000000-0000-0000-0000-000000000000 != fc6e103e-e856-40d4-59e7-aab2a4255b3b
|
|
2011-11-10 15:29:05.064093 7ffc6d4d0700 mon.0@0(leader).pg v3 handle_statfs on fsid 00000000-0000-0000-0000-000000000000 != fc6e103e-e856-40d4-59e7-aab2a4255b3b
|
|
2011-11-10 15:29:06.913011 7ffc6d4d0700 mon.0@0(leader).osd e3 e3: 3 osds: 3 up, 3 in
|
|
2011-11-10 15:29:07.416290 7ffc6d4d0700 log [INF] : osd.0 192.168.122.95:6801/14089 boot
|
|
2011-11-10 15:29:07.416498 7ffc6d4d0700 log [INF] : osd.2 192.168.122.74:6801/8722 boot
|
|
2011-11-10 15:29:11.371892 7ffc6d4d0700 log [INF] : mds.? 192.168.122.74:6800/8637 up:boot
|
|
2011-11-10 15:29:15.601058 7ffc6d4d0700 log [INF] : mds.? 192.168.122.37:6800/8943 up:boot
|
|
2011-11-10 15:29:17.982445 7ffc6d4d0700 mon.0@0(leader).pg v4 handle_statfs on fsid 00000000-0000-0000-0000-000000000000 != fc6e103e-e856-40d4-59e7-aab2a4255b3b
|
|
2011-11-10 15:29:17.982899 7ffc6d4d0700 mon.0@0(leader).pg v4 handle_statfs on fsid 00000000-0000-0000-0000-000000000000 != fc6e103e-e856-40d4-59e7-aab2a4255b3b
|
|
2011-11-10 15:29:22.673408 7ffc6d4d0700 mon.0@0(leader).osd e4 e4: 3 osds: 3 up, 3 in
|
|
2011-11-10 15:29:27.925018 7ffc6d4d0700 log [INF] : mon.0@0 won leader election with quorum 0,1,2
|
|
2011-11-10 15:30:02.341924 7ffc6d4d0700 log [INF] : mds.0 192.168.122.95:6800/14002 up:active
|
|
2011-11-10 15:31:21.659015 7ffc6d4d0700 mon.0@0(leader) e1 handle_command mon_command(osd v 0) v1
|
|
2011-11-10 15:31:24.298351 7ffc6d4d0700 mon.0@0(leader) e1 handle_command mon_command(osd help v 0) v1
|
|
2011-11-10 15:31:26.654101 7ffc6d4d0700 mon.0@0(leader) e1 handle_command mon_command(osd --help v 0) v1
|
|
2011-11-10 15:31:33.719861 7ffc6d4d0700 mon.0@0(leader) e1 handle_command mon_command(osd h v 0) v1
|
|
2011-11-10 15:31:35.499394 7ffc6d4d0700 mon.0@0(leader) e1 handle_command mon_command(osd ? v 0) v1
|
|
2011-11-10 15:31:37.499701 7ffc6d4d0700 mon.0@0(leader) e1 handle_command mon_command(osd help v 0) v1
|
|
2011-11-10 15:32:08.263259 7ffc6d4d0700 mon.0@0(leader) e1 handle_command mon_command(health v 0) v1
|
|
2011-11-10 15:32:13.758677 7ffc6d4d0700 mon.0@0(leader) e1 handle_command mon_command(health v 0) v1
|
|
2011-11-10 16:28:31.002185 7ffc6c3cd700 -- 192.168.122.95:6789/0 >> 192.168.122.37:6789/0 pipe(0x2458a00 sd=11 pgs=2 cs=1 l=0).fault with nothing to send, going to standby
|
|
2011-11-10 16:28:31.178703 7ffc711ba700 -- 192.168.122.95:6789/0 >> 192.168.122.37:6789/0 pipe(0x2458a00 sd=11 pgs=2 cs=2 l=0).fault first fault
|
|
2011-11-10 16:28:37.539386 7ffc6d4d0700 mon.0@0(leader) e1 handle_command mon_command(health v 0) v1
|
|
2011-11-10 16:28:41.178673 7ffc6d4d0700 log [INF] : mon.0 calling new monitor election
|
|
2011-11-10 16:28:46.227213 7ffc6cccf700 log [INF] : mon.0@0 won leader election with quorum 0,2
|
|
2011-11-10 16:28:48.715589 7ffc6d4d0700 mon.0@0(leader) e1 handle_command mon_command(health v 0) v1
|
|
2011-11-10 16:28:54.535906 7ffc6d4d0700 mon.0@0(leader) e1 handle_command mon_command(health v 0) v1
|
|
2011-11-10 16:28:54.855821 7ffc6d4d0700 mon.0@0(leader).osd e4 prepare_failure osd.1 192.168.122.37:6801/9027 from osd.2 192.168.122.74:6801/8722 is reporting failure:1
|
|
2011-11-10 16:28:54.855863 7ffc6d4d0700 log [INF] : osd.1 192.168.122.37:6801/9027 failed (by osd.2 192.168.122.74:6801/8722)
|
|
2011-11-10 16:28:57.856423 7ffc6d4d0700 mon.0@0(leader).osd e4 prepare_failure osd.1 192.168.122.37:6801/9027 from osd.0 192.168.122.95:6801/14089 is reporting failure:1
|
|
2011-11-10 16:28:57.856460 7ffc6d4d0700 log [INF] : osd.1 192.168.122.37:6801/9027 failed (by osd.0 192.168.122.95:6801/14089)
|
|
2011-11-10 16:28:59.857471 7ffc6d4d0700 mon.0@0(leader).osd e4 prepare_failure osd.1 192.168.122.37:6801/9027 from osd.2 192.168.122.74:6801/8722 is reporting failure:1
|
|
2011-11-10 16:28:59.857520 7ffc6d4d0700 log [INF] : osd.1 192.168.122.37:6801/9027 failed (by osd.2 192.168.122.74:6801/8722)
|
|
2011-11-10 16:28:59.857542 7ffc6d4d0700 mon.0@0(leader).osd e4 have enough reports/reporters to mark osd.1 as down
|
|
2011-11-10 16:29:00.259223 7ffc6d4d0700 mon.0@0(leader).osd e5 e5: 3 osds: 2 up, 3 in
|
|
2011-11-10 16:29:02.385482 7ffc6b7c1700 -- 192.168.122.95:6789/0 >> 192.168.122.37:6789/0 pipe(0x24c0000 sd=11 pgs=0 cs=0 l=0).accept connect_seq 0 vs existing 2 state 1
|
|
2011-11-10 16:29:02.385509 7ffc6b7c1700 -- 192.168.122.95:6789/0 >> 192.168.122.37:6789/0 pipe(0x24c0000 sd=11 pgs=0 cs=0 l=0).accept peer reset, then tried to connect to us, replacing
|
|
2011-11-10 16:29:02.550347 7ffc6d4d0700 log [INF] : mon.0 calling new monitor election
|
|
2011-11-10 16:29:07.380005 7ffc6d4d0700 log [INF] : mon.0@0 won leader election with quorum 0,1,2
|
|
2011-11-10 16:29:10.762104 7ffc6d4d0700 mon.0@0(leader).osd e6 e6: 3 osds: 2 up, 3 in
|
|
2011-11-10 16:29:15.709077 7ffc6d4d0700 mon.0@0(leader).osd e7 e7: 3 osds: 3 up, 3 in
|
|
2011-11-10 16:29:16.579720 7ffc6d4d0700 log [INF] : osd.1 192.168.122.37:6803/9701 boot
|
|
2011-11-10 16:29:31.284162 7ffc6d4d0700 log [INF] : mon.0 calling new monitor election
|
|
2011-11-10 16:29:32.116275 7ffc6d4d0700 log [INF] : mon.0@0 won leader election with quorum 0,1,2
|
|
2011-11-10 16:29:39.674265 7ffc6d4d0700 log [INF] : mds.? 192.168.122.37:6800/9616 up:boot
|
|
2011-11-10 16:29:40.257802 7ffc6d4d0700 mon.0@0(leader).osd e8 e8: 3 osds: 3 up, 3 in
|
|
2011-11-10 16:29:50.225042 7ffc6d4d0700 log [INF] : mon.0 calling new monitor election
|
|
2011-11-10 16:29:52.790820 7ffc6d4d0700 log [INF] : mon.0@0 won leader election with quorum 0,1,2
|
|
2011-11-10 16:30:24.056237 7ffc6d4d0700 log [INF] : mds.? 192.168.122.37:6800/9616 up:boot
|
|
2011-11-10 16:30:32.414858 7ffc6bec8700 -- 192.168.122.95:6789/0 >> 192.168.122.74:6789/0 pipe(0x2458c80 sd=14 pgs=2 cs=1 l=0).fault with nothing to send, going to standby
|
|
2011-11-10 16:30:32.415774 7ffc6b6c0700 -- 192.168.122.95:6789/0 >> 192.168.122.1:0/2894128163 pipe(0x24bca00 sd=16 pgs=0 cs=0 l=0).accept peer addr is really 192.168.122.1:0/2894128163 (socket is 192.168.122.1:37566/0)
|
|
2011-11-10 16:30:32.540154 7ffc6c4ce700 -- 192.168.122.95:6789/0 >> 192.168.122.74:6789/0 pipe(0x2458c80 sd=10 pgs=2 cs=2 l=0).fault first fault
|
|
2011-11-10 16:30:37.401926 7ffc6d4d0700 mon.0@0(leader) e1 handle_command mon_command(health v 0) v1
|
|
2011-11-10 16:30:42.540376 7ffc6d4d0700 log [INF] : mon.0 calling new monitor election
|
|
2011-11-10 16:30:47.668314 7ffc6cccf700 log [INF] : mon.0@0 won leader election with quorum 0,1
|
|
2011-11-10 16:30:48.793787 7ffc6d4d0700 mon.0@0(leader) e1 handle_command mon_command(health v 0) v1
|
|
2011-11-10 16:30:56.110905 7ffc6d4d0700 mon.0@0(leader).osd e8 prepare_failure osd.2 192.168.122.74:6801/8722 from osd.0 192.168.122.95:6801/14089 is reporting failure:1
|
|
2011-11-10 16:30:56.110945 7ffc6d4d0700 log [INF] : osd.2 192.168.122.74:6801/8722 failed (by osd.0 192.168.122.95:6801/14089)
|
|
2011-11-10 16:30:57.303567 7ffc6d4d0700 mon.0@0(leader).osd e8 prepare_failure osd.2 192.168.122.74:6801/8722 from osd.1 192.168.122.37:6803/9701 is reporting failure:1
|
|
2011-11-10 16:30:57.303609 7ffc6d4d0700 log [INF] : osd.2 192.168.122.74:6801/8722 failed (by osd.1 192.168.122.37:6803/9701)
|
|
2011-11-10 16:31:01.112276 7ffc6d4d0700 mon.0@0(leader).osd e8 prepare_failure osd.2 192.168.122.74:6801/8722 from osd.0 192.168.122.95:6801/14089 is reporting failure:1
|
|
2011-11-10 16:31:01.112315 7ffc6d4d0700 log [INF] : osd.2 192.168.122.74:6801/8722 failed (by osd.0 192.168.122.95:6801/14089)
|
|
2011-11-10 16:31:01.112335 7ffc6d4d0700 mon.0@0(leader).osd e8 have enough reports/reporters to mark osd.2 as down
|
|
2011-11-10 16:31:01.503510 7ffc6d4d0700 mon.0@0(leader).osd e9 e9: 3 osds: 2 up, 3 in
|
|
2011-11-10 16:31:08.173541 7ffc6d4d0700 mon.0@0(leader).osd e10 e10: 3 osds: 2 up, 3 in
|
|
2011-11-10 16:31:10.767215 7ffc6d4d0700 mon.0@0(leader) e1 handle_command mon_command(health v 0) v1
|
|
2011-11-10 16:31:13.021156 7ffc6d4d0700 mon.0@0(leader) e1 handle_command mon_command(health v 0) v1
|
|
2011-11-10 16:32:49.619734 7ffc6d4d0700 mon.0@0(leader) e1 handle_command mon_command(health v 0) v1
|
|
2011-11-10 16:33:10.322364 7ffc6d4d0700 mon.0@0(leader) e1 handle_command mon_command(health v 0) v1
|
|
2011-11-10 16:34:33.098137 7ffc6b9c3700 -- 192.168.122.95:6789/0 >> 192.168.122.74:6789/0 pipe(0x2564000 sd=10 pgs=0 cs=0 l=0).accept connect_seq 0 vs existing 2 state 1
|
|
2011-11-10 16:34:33.098170 7ffc6b9c3700 -- 192.168.122.95:6789/0 >> 192.168.122.74:6789/0 pipe(0x2564000 sd=10 pgs=0 cs=0 l=0).accept peer reset, then tried to connect to us, replacing
|
|
2011-11-10 16:34:35.682500 7ffc6d4d0700 mon.0@0(leader).osd e11 e11: 3 osds: 3 up, 3 in
|
|
2011-11-10 16:34:36.342888 7ffc6d4d0700 log [INF] : osd.2 192.168.122.74:6801/9388 boot
|
|
2011-11-10 16:34:36.344352 7ffc6d4d0700 mon.0@0(leader).mds e11 preprocess_beacon on fsid 00000000-0000-0000-0000-000000000000 != fc6e103e-e856-40d4-59e7-aab2a4255b3b
|
|
2011-11-10 16:34:45.474258 7ffc6d4d0700 mon.0@0(leader).osd e12 e12: 3 osds: 3 up, 3 in
|
|
2011-11-10 16:34:46.211114 7ffc6d4d0700 mon.0@0(leader) e1 handle_command mon_command(health v 0) v1
|
|
2011-11-10 16:34:52.792849 7ffc6cccf700 log [INF] : mon.0@0 won leader election with quorum 0,1
|
|
2011-11-10 16:34:58.979151 7ffc6d4d0700 log [INF] : mds.? 192.168.122.74:6800/9301 up:boot
|
|
2011-11-10 16:34:58.983151 7ffc6d4d0700 mon.0@0(leader) e1 handle_command mon_command(health v 0) v1
|
|
mon/PGMonitor.cc: In function 'virtual void PGMonitor::encode_pending(ceph::bufferlist&)', in thread '7ffc6d4d0700'
|
|
mon/PGMonitor.cc: 218: FAILED assert(paxos->get_version() + 1 == pending_inc.version)
|
|
ceph version 0.37-364-ga3dd5bd (commit:a3dd5bd67ba19aae51a51318138ef10213a91449)
|
|
1: (PGMonitor::encode_pending(ceph::buffer::list&)+0x108) [0x4c9fd8]
|
|
2: (PaxosService::propose_pending()+0xd2) [0x48e532]
|
|
3: (PGMonitor::check_osd_map(unsigned int)+0xca0) [0x4d0520]
|
|
4: (Context::complete(int)+0xa) [0x478ffa]
|
|
5: (finish_contexts(CephContext*, std::list<Context*, std::allocator<Context*> >&, int)+0xca) [0x47a52a]
|
|
6: (Paxos::handle_accept(MMonPaxos*)+0x5d8) [0x488948]
|
|
7: (Paxos::dispatch(PaxosServiceMessage*)+0x23b) [0x48b66b]
|
|
8: (Monitor::_ms_dispatch(Message*)+0xb99) [0x478409]
|
|
9: (Monitor::ms_dispatch(Message*)+0x35) [0x4832b5]
|
|
10: (SimpleMessenger::dispatch_entry()+0x84b) [0x57612b]
|
|
11: (SimpleMessenger::DispatchThread::entry()+0x1c) [0x462e9c]
|
|
12: (()+0x7efc) [0x7ffc70d95efc]
|
|
13: (clone()+0x6d) [0x7ffc6f7cf89d]
|
|
ceph version 0.37-364-ga3dd5bd (commit:a3dd5bd67ba19aae51a51318138ef10213a91449)
|
|
1: (PGMonitor::encode_pending(ceph::buffer::list&)+0x108) [0x4c9fd8]
|
|
2: (PaxosService::propose_pending()+0xd2) [0x48e532]
|
|
3: (PGMonitor::check_osd_map(unsigned int)+0xca0) [0x4d0520]
|
|
4: (Context::complete(int)+0xa) [0x478ffa]
|
|
5: (finish_contexts(CephContext*, std::list<Context*, std::allocator<Context*> >&, int)+0xca) [0x47a52a]
|
|
6: (Paxos::handle_accept(MMonPaxos*)+0x5d8) [0x488948]
|
|
7: (Paxos::dispatch(PaxosServiceMessage*)+0x23b) [0x48b66b]
|
|
8: (Monitor::_ms_dispatch(Message*)+0xb99) [0x478409]
|
|
9: (Monitor::ms_dispatch(Message*)+0x35) [0x4832b5]
|
|
10: (SimpleMessenger::dispatch_entry()+0x84b) [0x57612b]
|
|
11: (SimpleMessenger::DispatchThread::entry()+0x1c) [0x462e9c]
|
|
12: (()+0x7efc) [0x7ffc70d95efc]
|
|
13: (clone()+0x6d) [0x7ffc6f7cf89d]
|
|
*** Caught signal (Aborted) **
|
|
in thread 7ffc6d4d0700
|
|
ceph version 0.37-364-ga3dd5bd (commit:a3dd5bd67ba19aae51a51318138ef10213a91449)
|
|
1: /usr/bin/ceph-mon() [0x5b4df6]
|
|
2: (()+0x10060) [0x7ffc70d9e060]
|
|
3: (gsignal()+0x35) [0x7ffc6f7243a5]
|
|
4: (abort()+0x17b) [0x7ffc6f727b0b]
|
|
5: (__gnu_cxx::__verbose_terminate_handler()+0x11d) [0x7ffc6ffe2d7d]
|
|
6: (()+0xb9f26) [0x7ffc6ffe0f26]
|
|
7: (()+0xb9f53) [0x7ffc6ffe0f53]
|
|
8: (()+0xba04e) [0x7ffc6ffe104e]
|
|
9: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x193) [0x586fb3]
|
|
10: (PGMonitor::encode_pending(ceph::buffer::list&)+0x108) [0x4c9fd8]
|
|
11: (PaxosService::propose_pending()+0xd2) [0x48e532]
|
|
12: (PGMonitor::check_osd_map(unsigned int)+0xca0) [0x4d0520]
|
|
13: (Context::complete(int)+0xa) [0x478ffa]
|
|
14: (finish_contexts(CephContext*, std::list<Context*, std::allocator<Context*> >&, int)+0xca) [0x47a52a]
|
|
15: (Paxos::handle_accept(MMonPaxos*)+0x5d8) [0x488948]
|
|
16: (Paxos::dispatch(PaxosServiceMessage*)+0x23b) [0x48b66b]
|
|
17: (Monitor::_ms_dispatch(Message*)+0xb99) [0x478409]
|
|
18: (Monitor::ms_dispatch(Message*)+0x35) [0x4832b5]
|
|
19: (SimpleMessenger::dispatch_entry()+0x84b) [0x57612b]
|
|
20: (SimpleMessenger::DispatchThread::entry()+0x1c) [0x462e9c]
|
|
21: (()+0x7efc) [0x7ffc70d95efc]
|
|
22: (clone()+0x6d) [0x7ffc6f7cf89d]
|