Bug #1473
Updated by Sage Weil over 12 years ago
2011-08-31 10:04:29.898531 7fa020ddd700 -- 192.168.101.11:6801/7607 >> 192.168.101.14:6806/2173 pipe(0x78a7000 sd=34 pgs=521 cs=29 l=0).fault initiating reconnect 2011-08-31 10:04:29.898962 7fa01ca9a700 -- 192.168.101.11:6802/7607 >> 192.168.101.14:6808/2174 pipe(0x7154a00 sd=51 pgs=2 cs=2 l=0).fault first fault 2011-08-31 10:04:29.899049 7fa01fdcd700 -- 192.168.101.11:6801/7607 >> 192.168.101.14:6806/2173 pipe(0x78a7000 sd=34 pgs=521 cs=30 l=0).fault first fault 2011-08-31 10:04:50.199412 7fa02600c700 log [ERR] : 0.211 10000000158.00000000/head oi.size 18425 but stat got 0 size 17785 ../../src/osd/ReplicatedPG.cc: In function 'int ReplicatedPG::do_osd_ops(ReplicatedPG::OpContext*, std::vector<OSDOp>&, ceph::bufferlist&)', in thread '0x7fa02600c700' ../../src/osd/ReplicatedPG.cc: 1332: FAILED assert(0 == "oi disagrees with stat, or error code on stat") ceph version (commit:) 1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x89) [0x91ba61] 2: (ReplicatedPG::do_osd_ops(ReplicatedPG::OpContext*, std::vector<OSDOp, std::allocator<OSDOp> >&, ceph::buffer::list&)+0x2051) [0x78ce93] 3: (ReplicatedPG::prepare_transaction(ReplicatedPG::OpContext*)+0xca) [0x7968d0] 4: (ReplicatedPG::do_op(MOSDOp*)+0x20fe) [0x7864c0] 5: (OSD::dequeue_op(PG*)+0x2ef) [0x846f5d] 6: (OSD::OpWQ::_process(PG*)+0x27) [0x8526cd] 7: (ThreadPool::WorkQueue<PG>::_void_process(void*)+0x2e) [0x899e46] 8: (ThreadPool::worker()+0x422) [0x8b4d02] 9: (ThreadPool::WorkThread::entry()+0x1c) [0x851168] 10: (Thread::_entry_func(void*)+0x23) [0x8a0e21] 11: (()+0x6d8c) [0x7fa034be5d8c] 12: (clone()+0x6d) [0x7fa03342704d] ceph version (commit:) 1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x89) [0x91ba61] 2: (ReplicatedPG::do_osd_ops(ReplicatedPG::OpContext*, std::vector<OSDOp, std::allocator<OSDOp> >&, ceph::buffer::list&)+0x2051) [0x78ce93] 3: (ReplicatedPG::prepare_transaction(ReplicatedPG::OpContext*)+0xca) [0x7968d0] 4: (ReplicatedPG::do_op(MOSDOp*)+0x20fe) [0x7864c0] 5: (OSD::dequeue_op(PG*)+0x2ef) [0x846f5d] 6: (OSD::OpWQ::_process(PG*)+0x27) [0x8526cd] 7: (ThreadPool::WorkQueue<PG>::_void_process(void*)+0x2e) [0x899e46] 8: (ThreadPool::worker()+0x422) [0x8b4d02] 9: (ThreadPool::WorkThread::entry()+0x1c) [0x851168] 10: (Thread::_entry_func(void*)+0x23) [0x8a0e21] 11: (()+0x6d8c) [0x7fa034be5d8c] 12: (clone()+0x6d) [0x7fa03342704d] *** Caught signal (Aborted) ** in thread 0x7fa02600c700 ceph version (commit:) 1: (ceph::BackTrace::BackTrace(int)+0x2d) [0x8f3031] 2: /usr/ceph/bin/cosd() [0x91c0d3] 3: (()+0xfc60) [0x7fa034beec60] 4: (gsignal()+0x35) [0x7fa033374d05] 5: (abort()+0x186) [0x7fa033378ab6] 6: (__gnu_cxx::__verbose_terminate_handler()+0x11d) [0x7fa033c2b6dd] 7: (()+0xb9926) [0x7fa033c29926] 8: (()+0xb9953) [0x7fa033c29953] 9: (()+0xb9a5e) [0x7fa033c29a5e] 10: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x1f3) [0x91bbcb] 11: (ReplicatedPG::do_osd_ops(ReplicatedPG::OpContext*, std::vector<OSDOp, std::allocator<OSDOp> >&, ceph::buffer::list&)+0x2051) [0x78ce93] 12: (ReplicatedPG::prepare_transaction(ReplicatedPG::OpContext*)+0xca) [0x7968d0] 13: (ReplicatedPG::do_op(MOSDOp*)+0x20fe) [0x7864c0] 14: (OSD::dequeue_op(PG*)+0x2ef) [0x846f5d] 15: (OSD::OpWQ::_process(PG*)+0x27) [0x8526cd] 16: (ThreadPool::WorkQueue<PG>::_void_process(void*)+0x2e) [0x899e46] 17: (ThreadPool::worker()+0x422) [0x8b4d02] 18: (ThreadPool::WorkThread::entry()+0x1c) [0x851168] 19: (Thread::_entry_func(void*)+0x23) [0x8a0e21] 20: (()+0x6d8c) [0x7fa034be5d8c] 21: (clone()+0x6d) [0x7fa03342704d] (gdb) bt #0 0x00007fa034beeb3b in raise () from /lib/x86_64-linux-gnu/libpthread.so.0 #1 0x000000000091bf26 in reraise_fatal (signum=6) at ../../src/global/signal_handler.cc:59 #2 0x000000000091c144 in handle_fatal_signal (signum=6) at ../../src/global/signal_handler.cc:106 #3 <signal handler called> #4 0x00007fa033374d05 in raise () from /lib/x86_64-linux-gnu/libc.so.6 #5 0x00007fa033378ab6 in abort () from /lib/x86_64-linux-gnu/libc.so.6 #6 0x00007fa033c2b6dd in __gnu_cxx::__verbose_terminate_handler() () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 #7 0x00007fa033c29926 in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 #8 0x00007fa033c29953 in std::terminate() () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 #9 0x00007fa033c29a5e in __cxa_throw () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6 #10 0x000000000091bbcb in ceph::__ceph_assert_fail (assertion=0xa2d288 "0 == \"oi disagrees with stat, or error code on stat\"", file=0xa2c981 "../../src/osd/ReplicatedPG.cc", line=1332, func=0xa36a20 "int ReplicatedPG::do_osd_ops(ReplicatedPG::OpContext*, std::vector<OSDOp>&, ceph::bufferlist&)") at ../../src/common/assert.cc:70 #11 0x000000000078ce93 in ReplicatedPG::do_osd_ops (this=0x7890000, ctx=0x35aab00, ops=..., odata=...) at ../../src/osd/ReplicatedPG.cc:1332 #12 0x00000000007968d0 in ReplicatedPG::prepare_transaction (this=0x7890000, ctx=0x35aab00) at ../../src/osd/ReplicatedPG.cc:2472 #13 0x00000000007864c0 in ReplicatedPG::do_op (this=0x7890000, op=0x3245000) at ../../src/osd/ReplicatedPG.cc:595 #14 0x0000000000846f5d in OSD::dequeue_op (this=0x2169000, pg=0x7890000) at ../../src/osd/OSD.cc:5146 #15 0x00000000008526cd in OSD::OpWQ::_process (this=0x2169a48, pg=0x7890000) at ../../src/osd/OSD.h:342 #16 0x0000000000899e46 in WorkQueue<PG>::_void_process (this=0x2169a48, p=0x7890000) at ../../src/common/WorkQueue.h:62 #17 0x00000000008b4d02 in ThreadPool::worker (this=0x2169400) at ../../src/common/WorkQueue.cc:54 #18 0x0000000000851168 in ThreadPool::WorkThread::entry (this=0x2131ca0) at ../../src/common/WorkQueue.h:119 #19 0x00000000008a0e21 in Thread::_entry_func (arg=0x2131ca0) at ../../src/common/Thread.cc:45 #20 0x00007fa034be5d8c in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0 #21 0x00007fa03342704d in clone () from /lib/x86_64-linux-gnu/libc.so.6 #22 0x0000000000000000 in ?? ()