1. 没有卡住，但一直在运行此scrub线程 PG::chunky_scrub case PG::Scrubber::BUILD_MAP: (1)wait_for_unreadable_object 2015-08-26 10:12:13.218057 7fc8f682a700 0 log_channel(default) log [WRN] : slow request 960.623533 seconds old, received at 2015-08-26 09:56:12.594450: osd_op(client.1869732.0:27515622 rbd_data.1c87906bf7b7ad.000000000000054f [stat,set-alloc-hint object_size 8388608 write_size 8388608,write 3215360~4096] 42.d7686978 RETRY=2 ack+ondisk+retry+write+known_if_redirected e180041) currently waiting for missing object 2015-08-26 10:12:13.218063 7fc8f682a700 0 log_channel(default) log [WRN] : slow request 960.627636 seconds old, received at 2015-08-26 09:56:12.590347: osd_op(client.1869723.0:27638086 rbd_data.1c87751402b3e0.00000000000003e2 [stat,set-alloc-hint object_size 8388608 write_size 8388608,write 1048576~516096] 42.ac247601 RETRY=4 ack+ondisk+retry+write+known_if_redirected e180041) currently no flag points reached (2)void ReplicatedBackend::be_deep_scrub ReplicatedBackend.cc:734 int DBObjectMap::DBObjectMapIteratorImpl::seek_to_first() DBObjectMap.cc:336 { init(); r = 0; if (parent_iter) { r = parent_iter->seek_to_first(); if (r < 0) return r; } r = key_iter->seek_to_first(); if (r < 0) return r; return adjust(); } #0 0x00007fc8fc5a84cf in std::string::reserve(unsigned long) () from /lib64/libstdc++.so.6 #1 0x00007fc8fc5a873f in std::string::append(char const*, unsigned long) () from /lib64/libstdc++.so.6 #2 0x00007fc8fde6d6db in leveldb::Block::Iter::Seek(leveldb::Slice const&) () from /lib64/libleveldb.so.1 #3 0x00007fc8fde73083 in leveldb::(anonymous namespace)::TwoLevelIterator::Seek(leveldb::Slice const&) () from /lib64/libleveldb.so.1 #4 0x00007fc8fde73083 in leveldb::(anonymous namespace)::TwoLevelIterator::Seek(leveldb::Slice const&) () from /lib64/libleveldb.so.1 #5 0x00007fc8fde70028 in leveldb::(anonymous namespace)::MergingIterator::Seek(leveldb::Slice const&) () from /lib64/libleveldb.so.1 #6 0x00007fc8fde59a45 in leveldb::(anonymous namespace)::DBIter::Seek(leveldb::Slice const&) () from /lib64/libleveldb.so.1 #7 0x000000000094f75d in LevelDBStore::LevelDBWholeSpaceIteratorImpl::seek_to_first (this=0x19e56170, prefix=...) at os/LevelDBStore.h:222 #8 0x0000000000a1e347 in DBObjectMap::DBObjectMapIteratorImpl::seek_to_first (this=0xa9819c0) at os/DBObjectMap.cc:340 #9 0x0000000000a1e331 in DBObjectMap::DBObjectMapIteratorImpl::seek_to_first (this=0xa97e840) at os/DBObjectMap.cc:336 #10 0x0000000000a1e331 in DBObjectMap::DBObjectMapIteratorImpl::seek_to_first (this=0xa97ff80) at os/DBObjectMap.cc:336 #11 0x0000000000a1e331 in DBObjectMap::DBObjectMapIteratorImpl::seek_to_first (this=0x124fe640) at os/DBObjectMap.cc:336 #2333 0x0000000000a1e331 in DBObjectMap::DBObjectMapIteratorImpl::seek_to_first (this=0x1a237780) at os/DBObjectMap.cc:336 #2334 0x0000000000a1e331 in DBObjectMap::DBObjectMapIteratorImpl::seek_to_first (this=0x1a236b80) at os/DBObjectMap.cc:336 #2335 0x0000000000a1e331 in DBObjectMap::DBObjectMapIteratorImpl::seek_to_first (this=0xce6f380) at os/DBObjectMap.cc:336 #2336 0x0000000000a1e331 in DBObjectMap::DBObjectMapIteratorImpl::seek_to_first (this=0xce71f00) at os/DBObjectMap.cc:336 #2337 0x0000000000a1e331 in DBObjectMap::DBObjectMapIteratorImpl::seek_to_first (this=0xa972f40) at os/DBObjectMap.cc:336 #31 0x00000000009e0d1f in ReplicatedBackend::be_deep_scrub (this=0x52ede80, poid=..., o=..., handle=...) at osd/ReplicatedBackend.cc:734 #32 0x00000000008b3004 in PGBackend::be_scan_list (this=0x52ede80, map=..., ls=std::vector of length 13118, capacity 13118 = {...}, deep=deep@entry=true, handle=...) at osd/PGBackend.cc:348 #33 0x00000000007b51a7 in PG::build_scrub_map_chunk (this=this@entry=0x64b2800, map=..., start=..., end=..., deep=deep@entry=true, handle=...) at osd/PG.cc:3494 #34 0x00000000007bba6b in PG::chunky_scrub (this=this@entry=0x64b2800, handle=...) at osd/PG.cc:3954 #35 0x00000000007bd076 in PG::scrub (this=this@entry=0x64b2800, handle=...) at osd/PG.cc:3727 #36 0x000000000069fe29 in OSD::ScrubWQ::_process (this=, pg=0x64b2800, handle=...) at osd/OSD.h:2127 #37 0x0000000000b6c226 in ThreadPool::worker (this=0x3f50908, wt=0x3a3c6d0) at common/WorkQueue.cc:128 #38 0x0000000000b6d2b0 in ThreadPool::WorkThread::entry (this=) at common/WorkQueue.h:318 #39 0x00007fc8fd224df3 in start_thread () from /lib64/libpthread.so.0 #40 0x00007fc8fbd0754d in clone () from /lib64/libc.so.6 (3) DBObjectMap::lookup_parent (gdb) where #0 0x00007f85ef3fbb63 in __strlen_sse2_pminub () from /lib64/libc.so.6 #1 0x00007f85efc314c5 in std::basic_string, std::allocator >::basic_string(char const*, std::allocator const&) () from /lib64/libstdc++.so.6 #2 0x0000000000a1d2bf in coll_t (this=0x17930938) at osd/osd_types.h:472 #3 DBObjectMap::lookup_parent (this=this@entry=0x4c82780, input=std::tr1::shared_ptr (count 3, weak 0) 0x1de36310) at os/DBObjectMap.h:319 #4 0x0000000000a1e551 in DBObjectMap::_get_header (this=this@entry=0x4c82780, header=std::tr1::shared_ptr (count 3, weak 0) 0x1de36310, bl=bl@entry=0x7f85cf22be50) at os/DBObjectMap.cc:575 #5 0x0000000000a23bd2 in DBObjectMap::get_header (this=0x4c82780, oid=..., bl=0x7f85cf22be50) at os/DBObjectMap.cc:556 #6 0x00000000008d7606 in FileStore::omap_get_header (this=0x4bae000, c=..., hoid=..., bl=bl@entry=0x7f85cf22be50, allow_eio=allow_eio@entry=true) at os/FileStore.cc:4601 #7 0x00000000009e0c2a in ReplicatedBackend::be_deep_scrub (this=0x60b9200, poid=..., o=..., handle=...) at osd/ReplicatedBackend.cc:715 #8 0x00000000008b3004 in PGBackend::be_scan_list (this=0x60b9200, map=..., ls=std::vector of length 13118, capacity 13118 = {...}, deep=deep@entry=true, handle=...) at osd/PGBackend.cc:348 #9 0x00000000007b51a7 in PG::build_scrub_map_chunk (this=this@entry=0x6b20800, map=..., start=..., end=..., deep=deep@entry=true, handle=...) at osd/PG.cc:3494 #10 0x00000000007b5a92 in PG::replica_scrub (this=this@entry=0x6b20800, msg=msg@entry=0x17122f80, handle=...) at osd/PG.cc:3643 #11 0x00000000006b976a in OSD::RepScrubWQ::_process (this=0x4bff610, msg=0x17122f80, handle=...) at osd/OSD.h:2221 #12 0x0000000000b6c226 in ThreadPool::worker (this=0x4bfe908, wt=0x4dddae0) at common/WorkQueue.cc:128 #13 0x0000000000b6d2b0 in ThreadPool::WorkThread::entry (this=) at common/WorkQueue.h:318 #14 0x00007f85f08acdf3 in start_thread () from /lib64/libpthread.so.0 #15 0x00007f85ef38f54d in clone () froNIndex::list_objects osd/PGBackend.cc:134 2.PG::chunky_scrub case PG::Scrubber::NEW_CHUNK: chunky_scrub函数 while (!boundary_found) { (gdb) bt #0 0x00007f572efaab0c in std::string::_M_mutate(unsigned long, unsigned long, unsigned long) () from /lib64/libstdc++.so.6 #1 0x00007f572efaaf9e in std::string::_M_replace_safe(unsigned long, unsigned long, char const*, unsigned long) () from /lib64/libstdc++.so.6 #2 0x00007f572efac1df in std::string::replace(unsigned long, unsigned long, char const*, unsigned long) () from /lib64/libstdc++.so.6 #3 0x0000000000942ea4 in replace (__k2=48 '0', __k1=53 '5', __i2=..., __i1=..., this=0x7f570d78c6a0) at /usr/include/c++/4.8.2/bits/basic_string.h:1674 #4 append<__gnu_cxx::__normal_iterator > > (__last=..., __first=..., this=0x7f570d78c6a0) at /usr/include/c++/4.8.2/bits/basic_string.h:1045 #5 append_escaped (begin=..., begin@entry=114 'r', end=0 '\000', out=out@entry=0x7f570d78c6a0) at os/LFNIndex.cc:626 #6 0x00000000009437a9 in LFNIndex::lfn_generate_object_name (this=this@entry=0x3b33e40, oid=...) at os/LFNIndex.cc:647 #7 0x0000000000948955 in LFNIndex::list_objects (this=this@entry=0x3b33e40, to_list=std::vector of length 8, capacity 14 = {...}, max_objs=max_objs@entry=0, handle=handle@entry=0x0, out=out@entry=0x7f570d78fc20) at os/LFNIndex.cc:434 #8 0x0000000000a545c4 in HashIndex::get_path_contents_by_hash (this=this@entry=0x3b33e40, path=std::vector of length 8, capacity 14 = {...}, lower_bound=lower_bound@entry=0x0, next_object=next_object@entry=0x7f570d790310, seq=seq@entry=0x7f570d78fd70, hash_prefixes=hash_prefixes@entry=0x7f570d78fe10, objects=objects@entry=0x7f570d78fe40) at os/HashIndex.cc:783 #9 0x0000000000a5722f in HashIndex::list_by_hash (this=this@entry=0x3b33e40, path=std::vector of length 8, capacity 14 = {...}, min_count=min_count@entry=5, max_count=max_count@entry=25, seq=..., next=next@entry=0x7f570d790310, out=out@entry=0x7f570d7924a0) at os/HashIndex.cc:833 #10 0x0000000000a57779 in HashIndex::list_by_hash (this=this@entry=0x3b33e40, path=std::vector of length 7, capacity 12 = {...}, min_count=min_count@entry=5, max_count=max_count@entry=25, seq=..., next=next@entry=0x7f570d790700, out=out@entry=0x7f570d7924a0) at os/HashIndex.cc:857 #11 0x0000000000a57779 in HashIndex::list_by_hash (this=this@entry=0x3b33e40, path=std::vector of length 6, capacity 10 = {...}, min_count=min_count@entry=5, max_count=max_count@entry=25, seq=..., next=next@entry=0x7f570d790af0, out=out@entry=0x7f570d7924a0) at os/HashIndex.cc:857 #12 0x0000000000a57779 in HashIndex::list_by_hash (this=this@entry=0x3b33e40, path=std::vector of length 5, capacity 8 = {...}, min_count=min_count@entry=5, max_count=max_count@entry=25, seq=..., next=next@entry=0x7f570d790ee0, out=out@entry=0x7f570d7924a0) at os/HashIndex.cc:857 #13 0x0000000000a57779 in HashIndex::list_by_hash (this=this@entry=0x3b33e40, path=std::vector of length 4, capacity 6 = {...}, min_count=min_count@entry=5, max_count=max_count@entry=25, seq=..., next=next@entry=0x7f570d7912d0, out=out@entry=0x7f570d7924a0) at os/HashIndex.cc:857 #14 0x0000000000a57779 in HashIndex::list_by_hash (this=this@entry=0x3b33e40, path=std::vector of length 3, capacity 4 = {...}, min_count=min_count@entry=5, max_count=max_count@entry=25, seq=..., next=next@entry=0x7f570d7916c0, out=out@entry=0x7f570d7924a0) at os/HashIndex.cc:857 #15 0x0000000000a57779 in HashIndex::list_by_hash (this=this@entry=0x3b33e40, ---Type to continue, or q to quit--- path=std::vector of length 2, capacity 2 = {...}, min_count=min_count@entry=5, max_count=max_count@entry=25, seq=..., next=next@entry=0x7f570d791ab0, out=out@entry=0x7f570d7924a0) at os/HashIndex.cc:857 #16 0x0000000000a57779 in HashIndex::list_by_hash (this=this@entry=0x3b33e40, path=std::vector of length 1, capacity 1 = {...}, min_count=min_count@entry=5, max_count=max_count@entry=25, seq=..., next=next@entry=0x7f570d791ea0, out=out@entry=0x7f570d7924a0) at os/HashIndex.cc:8 #17 0x0000000000a57779 in HashIndex::list_by_hash (this=this@entry=0x3b33e40, path=std::vector of length 0, capacity 0, min_count=min_count@entry=5, max_count=max_count@entry=25, seq=..., next=next@entry=0x7f570d7924c0, out=out@entry=0x7f570d7924a0) at os/HashIndex.cc:857 #18 0x0000000000a5811d in HashIndex::_collection_list_partial (this=0x3b33e40, start=..., min_count=5, max_count=25, seq=..., ls=0x7f570d7924a0, next=0x7f570d7924c0) at os/HashIndex.cc:340 #19 0x00000000008c1621 in FileStore::collection_list_partial (this=0x3af0000, c=..., start=..., min=min@entry=5, max=max@entry=25, seq=..., seq@entry=..., ls=ls@entry=0x7f570d7924a0, next=next@entry=0x7f570d7924c0) at os/FileStore.cc:4528 #20 0x00000000008b2ae9 in PGBackend::objects_list_partial (this=0x6d5d480, begin=..., min=min@entry=5, max=max@entry=25, seq=..., seq@entry=..., ls=ls@entry=0x7f570d7925f0, next=next@entry=0x7f570d792610) at osd/PGBackend.cc:134 #21 0x00000000007bc000 in PG::chunky_scrub (this=this@entry=0x7146800, handle=...) at osd/PG.cc:3864 #22 0x00000000007bd076 in PG::scrub (this=this@entry=0x7146800, handle=...) at osd/PG.cc:3727 #23 0x000000000069fe29 in OSD::ScrubWQ::_process (this=, pg=0x7146800, handle=...) at osd/OSD.h:2127 #24 0x0000000000b6c226 in ThreadPool::worker (this=0x3cb2908, wt=0x43e9570) at common/WorkQueue.cc:128 #25 0x0000000000b6d2b0 in ThreadPool::WorkThread::entry (this=) at common/WorkQueue.h:318 #26 0x00007f572fc27df3 in start_thread () from /lib64/libpthread.so.0 #27 0x00007f572e70a54d in clone () from /lib64/libc.so.6