|
1. û?п?ס????һֱ?????д?scrub?߳?
|
|
PG::chunky_scrub
|
|
case PG::Scrubber::BUILD_MAP:
|
|
|
|
|
|
|
|
(1)wait_for_unreadable_object
|
|
|
|
2015-08-26 10:12:13.218057 7fc8f682a700 0 log_channel(default) log [WRN] : slow request 960.623533 seconds old, received at 2015-08-26 09:56:12.594450: osd_op(client.1869732.0:27515622 rbd_data.1c87906bf7b7ad.000000000000054f [stat,set-alloc-hint object_size 8388608 write_size 8388608,write 3215360~4096] 42.d7686978 RETRY=2 ack+ondisk+retry+write+known_if_redirected e180041) currently waiting for missing object
|
|
2015-08-26 10:12:13.218063 7fc8f682a700 0 log_channel(default) log [WRN] : slow request 960.627636 seconds old, received at 2015-08-26 09:56:12.590347: osd_op(client.1869723.0:27638086 rbd_data.1c87751402b3e0.00000000000003e2 [stat,set-alloc-hint object_size 8388608 write_size 8388608,write 1048576~516096] 42.ac247601 RETRY=4 ack+ondisk+retry+write+known_if_redirected e180041) currently no flag points reached
|
|
|
|
|
|
(2)void ReplicatedBackend::be_deep_scrub ReplicatedBackend.cc:734
|
|
|
|
|
|
int DBObjectMap::DBObjectMapIteratorImpl::seek_to_first() DBObjectMap.cc:336
|
|
{
|
|
init();
|
|
r = 0;
|
|
if (parent_iter) {
|
|
r = parent_iter->seek_to_first();
|
|
if (r < 0)
|
|
return r;
|
|
}
|
|
r = key_iter->seek_to_first();
|
|
if (r < 0)
|
|
return r;
|
|
return adjust();
|
|
}
|
|
|
|
|
|
|
|
#0 0x00007fc8fc5a84cf in std::string::reserve(unsigned long) () from /lib64/libstdc++.so.6
|
|
#1 0x00007fc8fc5a873f in std::string::append(char const*, unsigned long) () from /lib64/libstdc++.so.6
|
|
#2 0x00007fc8fde6d6db in leveldb::Block::Iter::Seek(leveldb::Slice const&) () from /lib64/libleveldb.so.1
|
|
#3 0x00007fc8fde73083 in leveldb::(anonymous namespace)::TwoLevelIterator::Seek(leveldb::Slice const&) () from /lib64/libleveldb.so.1
|
|
#4 0x00007fc8fde73083 in leveldb::(anonymous namespace)::TwoLevelIterator::Seek(leveldb::Slice const&) () from /lib64/libleveldb.so.1
|
|
#5 0x00007fc8fde70028 in leveldb::(anonymous namespace)::MergingIterator::Seek(leveldb::Slice const&) () from /lib64/libleveldb.so.1
|
|
#6 0x00007fc8fde59a45 in leveldb::(anonymous namespace)::DBIter::Seek(leveldb::Slice const&) () from /lib64/libleveldb.so.1
|
|
#7 0x000000000094f75d in LevelDBStore::LevelDBWholeSpaceIteratorImpl::seek_to_first (this=0x19e56170, prefix=...) at os/LevelDBStore.h:222
|
|
#8 0x0000000000a1e347 in DBObjectMap::DBObjectMapIteratorImpl::seek_to_first (this=0xa9819c0) at os/DBObjectMap.cc:340
|
|
#9 0x0000000000a1e331 in DBObjectMap::DBObjectMapIteratorImpl::seek_to_first (this=0xa97e840) at os/DBObjectMap.cc:336
|
|
#10 0x0000000000a1e331 in DBObjectMap::DBObjectMapIteratorImpl::seek_to_first (this=0xa97ff80) at os/DBObjectMap.cc:336
|
|
#11 0x0000000000a1e331 in DBObjectMap::DBObjectMapIteratorImpl::seek_to_first (this=0x124fe640) at os/DBObjectMap.cc:336
|
|
#2333 0x0000000000a1e331 in DBObjectMap::DBObjectMapIteratorImpl::seek_to_first (this=0x1a237780) at os/DBObjectMap.cc:336
|
|
#2334 0x0000000000a1e331 in DBObjectMap::DBObjectMapIteratorImpl::seek_to_first (this=0x1a236b80) at os/DBObjectMap.cc:336
|
|
#2335 0x0000000000a1e331 in DBObjectMap::DBObjectMapIteratorImpl::seek_to_first (this=0xce6f380) at os/DBObjectMap.cc:336
|
|
#2336 0x0000000000a1e331 in DBObjectMap::DBObjectMapIteratorImpl::seek_to_first (this=0xce71f00) at os/DBObjectMap.cc:336
|
|
#2337 0x0000000000a1e331 in DBObjectMap::DBObjectMapIteratorImpl::seek_to_first (this=0xa972f40) at os/DBObjectMap.cc:336
|
|
#31 0x00000000009e0d1f in ReplicatedBackend::be_deep_scrub (this=0x52ede80, poid=..., o=..., handle=...) at osd/ReplicatedBackend.cc:734
|
|
#32 0x00000000008b3004 in PGBackend::be_scan_list (this=0x52ede80, map=..., ls=std::vector of length 13118, capacity 13118 = {...}, deep=deep@entry=true, handle=...)
|
|
at osd/PGBackend.cc:348
|
|
#33 0x00000000007b51a7 in PG::build_scrub_map_chunk (this=this@entry=0x64b2800, map=..., start=..., end=..., deep=deep@entry=true, handle=...) at osd/PG.cc:3494
|
|
#34 0x00000000007bba6b in PG::chunky_scrub (this=this@entry=0x64b2800, handle=...) at osd/PG.cc:3954
|
|
#35 0x00000000007bd076 in PG::scrub (this=this@entry=0x64b2800, handle=...) at osd/PG.cc:3727
|
|
#36 0x000000000069fe29 in OSD::ScrubWQ::_process (this=<optimized out>, pg=0x64b2800, handle=...) at osd/OSD.h:2127
|
|
#37 0x0000000000b6c226 in ThreadPool::worker (this=0x3f50908, wt=0x3a3c6d0) at common/WorkQueue.cc:128
|
|
#38 0x0000000000b6d2b0 in ThreadPool::WorkThread::entry (this=<optimized out>) at common/WorkQueue.h:318
|
|
#39 0x00007fc8fd224df3 in start_thread () from /lib64/libpthread.so.0
|
|
#40 0x00007fc8fbd0754d in clone () from /lib64/libc.so.6
|
|
|
|
|
|
|
|
(3) DBObjectMap::lookup_parent
|
|
|
|
(gdb) where
|
|
#0 0x00007f85ef3fbb63 in __strlen_sse2_pminub () from /lib64/libc.so.6
|
|
#1 0x00007f85efc314c5 in std::basic_string<char, std::char_traits<char>, std::allocator<char> >::basic_string(char const*, std::allocator<char> const&) () from /lib64/libstdc++.so.6
|
|
#2 0x0000000000a1d2bf in coll_t (this=0x17930938) at osd/osd_types.h:472
|
|
#3 DBObjectMap::lookup_parent (this=this@entry=0x4c82780, input=std::tr1::shared_ptr (count 3, weak 0) 0x1de36310)
|
|
at os/DBObjectMap.h:319
|
|
#4 0x0000000000a1e551 in DBObjectMap::_get_header (this=this@entry=0x4c82780,
|
|
header=std::tr1::shared_ptr (count 3, weak 0) 0x1de36310, bl=bl@entry=0x7f85cf22be50) at os/DBObjectMap.cc:575
|
|
#5 0x0000000000a23bd2 in DBObjectMap::get_header (this=0x4c82780, oid=..., bl=0x7f85cf22be50) at os/DBObjectMap.cc:556
|
|
#6 0x00000000008d7606 in FileStore::omap_get_header (this=0x4bae000, c=..., hoid=..., bl=bl@entry=0x7f85cf22be50,
|
|
allow_eio=allow_eio@entry=true) at os/FileStore.cc:4601
|
|
#7 0x00000000009e0c2a in ReplicatedBackend::be_deep_scrub (this=0x60b9200, poid=..., o=..., handle=...)
|
|
at osd/ReplicatedBackend.cc:715
|
|
#8 0x00000000008b3004 in PGBackend::be_scan_list (this=0x60b9200, map=...,
|
|
ls=std::vector of length 13118, capacity 13118 = {...}, deep=deep@entry=true, handle=...) at osd/PGBackend.cc:348
|
|
#9 0x00000000007b51a7 in PG::build_scrub_map_chunk (this=this@entry=0x6b20800, map=..., start=..., end=...,
|
|
deep=deep@entry=true, handle=...) at osd/PG.cc:3494
|
|
#10 0x00000000007b5a92 in PG::replica_scrub (this=this@entry=0x6b20800, msg=msg@entry=0x17122f80, handle=...)
|
|
at osd/PG.cc:3643
|
|
#11 0x00000000006b976a in OSD::RepScrubWQ::_process (this=0x4bff610, msg=0x17122f80, handle=...) at osd/OSD.h:2221
|
|
#12 0x0000000000b6c226 in ThreadPool::worker (this=0x4bfe908, wt=0x4dddae0) at common/WorkQueue.cc:128
|
|
#13 0x0000000000b6d2b0 in ThreadPool::WorkThread::entry (this=<optimized out>) at common/WorkQueue.h:318
|
|
#14 0x00007f85f08acdf3 in start_thread () from /lib64/libpthread.so.0
|
|
#15 0x00007f85ef38f54d in clone () froNIndex::list_objects osd/PGBackend.cc:134
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2.PG::chunky_scrub
|
|
case PG::Scrubber::NEW_CHUNK:
|
|
|
|
|
|
chunky_scrub????
|
|
while (!boundary_found) {
|
|
|
|
(gdb) bt
|
|
#0 0x00007f572efaab0c in std::string::_M_mutate(unsigned long, unsigned long, unsigned long) () from /lib64/libstdc++.so.6
|
|
#1 0x00007f572efaaf9e in std::string::_M_replace_safe(unsigned long, unsigned long, char const*, unsigned long) ()
|
|
from /lib64/libstdc++.so.6
|
|
#2 0x00007f572efac1df in std::string::replace(unsigned long, unsigned long, char const*, unsigned long) ()
|
|
from /lib64/libstdc++.so.6
|
|
#3 0x0000000000942ea4 in replace (__k2=48 '0', __k1=53 '5', __i2=..., __i1=..., this=0x7f570d78c6a0)
|
|
at /usr/include/c++/4.8.2/bits/basic_string.h:1674
|
|
#4 append<__gnu_cxx::__normal_iterator<char const*, std::basic_string<char> > > (__last=..., __first=...,
|
|
this=0x7f570d78c6a0) at /usr/include/c++/4.8.2/bits/basic_string.h:1045
|
|
#5 append_escaped (begin=..., begin@entry=114 'r', end=0 '\000', out=out@entry=0x7f570d78c6a0) at os/LFNIndex.cc:626
|
|
#6 0x00000000009437a9 in LFNIndex::lfn_generate_object_name (this=this@entry=0x3b33e40, oid=...) at os/LFNIndex.cc:647
|
|
#7 0x0000000000948955 in LFNIndex::list_objects (this=this@entry=0x3b33e40,
|
|
to_list=std::vector of length 8, capacity 14 = {...}, max_objs=max_objs@entry=0, handle=handle@entry=0x0,
|
|
out=out@entry=0x7f570d78fc20) at os/LFNIndex.cc:434
|
|
#8 0x0000000000a545c4 in HashIndex::get_path_contents_by_hash (this=this@entry=0x3b33e40,
|
|
path=std::vector of length 8, capacity 14 = {...}, lower_bound=lower_bound@entry=0x0,
|
|
next_object=next_object@entry=0x7f570d790310, seq=seq@entry=0x7f570d78fd70,
|
|
hash_prefixes=hash_prefixes@entry=0x7f570d78fe10, objects=objects@entry=0x7f570d78fe40) at os/HashIndex.cc:783
|
|
#9 0x0000000000a5722f in HashIndex::list_by_hash (this=this@entry=0x3b33e40,
|
|
path=std::vector of length 8, capacity 14 = {...}, min_count=min_count@entry=5, max_count=max_count@entry=25, seq=...,
|
|
next=next@entry=0x7f570d790310, out=out@entry=0x7f570d7924a0) at os/HashIndex.cc:833
|
|
#10 0x0000000000a57779 in HashIndex::list_by_hash (this=this@entry=0x3b33e40,
|
|
path=std::vector of length 7, capacity 12 = {...}, min_count=min_count@entry=5, max_count=max_count@entry=25, seq=...,
|
|
next=next@entry=0x7f570d790700, out=out@entry=0x7f570d7924a0) at os/HashIndex.cc:857
|
|
#11 0x0000000000a57779 in HashIndex::list_by_hash (this=this@entry=0x3b33e40,
|
|
path=std::vector of length 6, capacity 10 = {...}, min_count=min_count@entry=5, max_count=max_count@entry=25, seq=...,
|
|
next=next@entry=0x7f570d790af0, out=out@entry=0x7f570d7924a0) at os/HashIndex.cc:857
|
|
#12 0x0000000000a57779 in HashIndex::list_by_hash (this=this@entry=0x3b33e40,
|
|
path=std::vector of length 5, capacity 8 = {...}, min_count=min_count@entry=5, max_count=max_count@entry=25, seq=...,
|
|
next=next@entry=0x7f570d790ee0, out=out@entry=0x7f570d7924a0) at os/HashIndex.cc:857
|
|
#13 0x0000000000a57779 in HashIndex::list_by_hash (this=this@entry=0x3b33e40,
|
|
path=std::vector of length 4, capacity 6 = {...}, min_count=min_count@entry=5, max_count=max_count@entry=25, seq=...,
|
|
next=next@entry=0x7f570d7912d0, out=out@entry=0x7f570d7924a0) at os/HashIndex.cc:857
|
|
#14 0x0000000000a57779 in HashIndex::list_by_hash (this=this@entry=0x3b33e40,
|
|
path=std::vector of length 3, capacity 4 = {...}, min_count=min_count@entry=5, max_count=max_count@entry=25, seq=...,
|
|
next=next@entry=0x7f570d7916c0, out=out@entry=0x7f570d7924a0) at os/HashIndex.cc:857
|
|
#15 0x0000000000a57779 in HashIndex::list_by_hash (this=this@entry=0x3b33e40,
|
|
---Type <return> to continue, or q <return> to quit---
|
|
path=std::vector of length 2, capacity 2 = {...}, min_count=min_count@entry=5, max_count=max_count@entry=25, seq=...,
|
|
next=next@entry=0x7f570d791ab0, out=out@entry=0x7f570d7924a0) at os/HashIndex.cc:857
|
|
#16 0x0000000000a57779 in HashIndex::list_by_hash (this=this@entry=0x3b33e40,
|
|
path=std::vector of length 1, capacity 1 = {...}, min_count=min_count@entry=5, max_count=max_count@entry=25, seq=...,
|
|
next=next@entry=0x7f570d791ea0, out=out@entry=0x7f570d7924a0) at os/HashIndex.cc:8
|
|
|
|
#17 0x0000000000a57779 in HashIndex::list_by_hash (this=this@entry=0x3b33e40, path=std::vector of length 0, capacity 0,
|
|
min_count=min_count@entry=5, max_count=max_count@entry=25, seq=..., next=next@entry=0x7f570d7924c0,
|
|
out=out@entry=0x7f570d7924a0) at os/HashIndex.cc:857
|
|
#18 0x0000000000a5811d in HashIndex::_collection_list_partial (this=0x3b33e40, start=..., min_count=5, max_count=25,
|
|
seq=..., ls=0x7f570d7924a0, next=0x7f570d7924c0) at os/HashIndex.cc:340
|
|
#19 0x00000000008c1621 in FileStore::collection_list_partial (this=0x3af0000, c=..., start=..., min=min@entry=5,
|
|
max=max@entry=25, seq=..., seq@entry=..., ls=ls@entry=0x7f570d7924a0, next=next@entry=0x7f570d7924c0)
|
|
at os/FileStore.cc:4528
|
|
#20 0x00000000008b2ae9 in PGBackend::objects_list_partial (this=0x6d5d480, begin=..., min=min@entry=5, max=max@entry=25,
|
|
seq=..., seq@entry=..., ls=ls@entry=0x7f570d7925f0, next=next@entry=0x7f570d792610) at osd/PGBackend.cc:134
|
|
#21 0x00000000007bc000 in PG::chunky_scrub (this=this@entry=0x7146800, handle=...) at osd/PG.cc:3864
|
|
#22 0x00000000007bd076 in PG::scrub (this=this@entry=0x7146800, handle=...) at osd/PG.cc:3727
|
|
#23 0x000000000069fe29 in OSD::ScrubWQ::_process (this=<optimized out>, pg=0x7146800, handle=...) at osd/OSD.h:2127
|
|
#24 0x0000000000b6c226 in ThreadPool::worker (this=0x3cb2908, wt=0x43e9570) at common/WorkQueue.cc:128
|
|
#25 0x0000000000b6d2b0 in ThreadPool::WorkThread::entry (this=<optimized out>) at common/WorkQueue.h:318
|
|
#26 0x00007f572fc27df3 in start_thread () from /lib64/libpthread.so.0
|
|
#27 0x00007f572e70a54d in clone () from /lib64/libc.so.6
|
|
|
|
|
|
|
|
|
|
|