-------------------------------------------------------------------------------- Command: /usr/bin/radosgw -f --cluster ceph --name client.rgw.ceph-1 --setuser ceph --setgroup ceph Massif arguments: --massif-out-file=/tmp/out.3.massif ms_print arguments: /tmp/out.3.massif -------------------------------------------------------------------------------- MB 195.1^ # | ::@# | @:: @# | ::@@:@:: @# | @@@: @ :@:: @# | @@@@@ : @ :@:: @# | @::@@@ @@ : @ :@:: @# | ::@@@: @@@ @@ : @ :@:: @# | @:::: @ @: @@@ @@ : @ :@:: @# | @::::::@: :: @ @: @@@ @@ : @ :@:: @# | @@: ::: @: :: @ @: @@@ @@ : @ :@:: @# | @@@@@: ::: @: :: @ @: @@@ @@ : @ :@:: @# | ::@@ @@@: ::: @: :: @ @: @@@ @@ : @ :@:: @# | @@@@: @@ @@@: ::: @: :: @ @: @@@ @@ : @ :@:: @# | @@@ @: @@ @@@: ::: @: :: @ @: @@@ @@ : @ :@:: @# | ::@@@@@ @: @@ @@@: ::: @: :: @ @: @@@ @@ : @ :@:: @# | :::::::: @ @@@ @: @@ @@@: ::: @: :: @ @: @@@ @@ : @ :@:: @# | :::::: : :: : @ @@@ @: @@ @@@: ::: @: :: @ @: @@@ @@ : @ :@:: @# | @@@: :::: : :: : @ @@@ @: @@ @@@: ::: @: :: @ @: @@@ @@ : @ :@:: @# | :::@@ @: :::: : :: : @ @@@ @: @@ @@@: ::: @: :: @ @: @@@ @@ : @ :@:: @# 0 +----------------------------------------------------------------------->Gi 0 87.08 Number of snapshots: 54 Detailed snapshots: [3, 4, 5, 15, 16, 17, 18, 19, 20, 22, 23, 24, 25, 26, 31, 35, 36, 38, 39, 40, 41, 42, 44, 46, 49, 51, 52, 53 (peak)] -------------------------------------------------------------------------------- n time(i) total(B) useful-heap(B) extra-heap(B) stacks(B) -------------------------------------------------------------------------------- 0 0 0 0 0 0 1 2,248,980,426 12,404,640 11,801,501 603,139 0 2 3,859,584,948 15,841,088 15,087,880 753,208 0 3 5,464,683,000 19,264,064 18,357,800 906,264 0 95.30% (18,357,800B) (heap allocation functions) malloc/new/new[], --alloc-fns, etc. ->26.99% (5,200,272B) 0x51E7C89: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | ->26.99% (5,200,272B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | ->26.99% (5,200,272B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | ->26.99% (5,200,272B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | ->26.99% (5,200,272B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | ->26.99% (5,200,272B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | ->26.99% (5,200,272B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | ->26.99% (5,200,272B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | ->26.99% (5,200,272B) 0xF1A7492: start_thread (pthread_create.c:333) | ->26.99% (5,200,272B) 0xFF3F93D: clone (clone.S:97) | ->14.04% (2,705,578B) 0xF9F4E97: std::string::_Rep::_S_create(unsigned long, unsigned long, std::allocator const&) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | ->12.45% (2,398,667B) 0xF9F5B09: std::string::_Rep::_M_clone(std::allocator const&, unsigned long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | ->12.45% (2,398,632B) 0xF9F5BAE: std::string::reserve(unsigned long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | ->09.25% (1,781,766B) 0xF9D1FB2: std::basic_stringbuf, std::allocator >::overflow(int) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->09.25% (1,781,766B) 0xF9D5D63: std::basic_streambuf >::xsputn(char const*, long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->09.25% (1,781,229B) 0xF9CD314: std::basic_ostream >& std::__ostream_insert >(std::basic_ostream >&, char const*, long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | | ->09.15% (1,762,971B) 0xF9CD6E5: std::basic_ostream >& std::operator<< >(std::basic_ostream >&, char const*) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | | | ->09.15% (1,762,971B) 0x5109AE4: RGWOmapAppend::operate() (in /usr/lib/librgw.so.2.0.0) | | | | | | ->09.15% (1,762,971B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | | | | ->09.15% (1,762,971B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | | | | ->09.15% (1,762,971B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | | | | ->09.15% (1,762,971B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | | | | ->09.15% (1,762,971B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | | | | ->09.15% (1,762,971B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | | | | ->09.15% (1,762,971B) 0xF1A7492: start_thread (pthread_create.c:333) | | | | | | ->09.15% (1,762,971B) 0xFF3F93D: clone (clone.S:97) | | | | | | | | | | | ->00.09% (18,258B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | | | ->00.00% (537B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | ->02.82% (543,336B) 0xF9F5E38: ??? (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->02.82% (543,336B) 0xF9F5F0C: std::string::append(char const*, unsigned long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->02.64% (509,511B) 0x51DF1E4: RGWDataSyncShardControlCR::alloc_cr() (in /usr/lib/librgw.so.2.0.0) | | | | | ->02.64% (509,511B) 0x51AB72B: RGWBackoffControlCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | | | ->02.64% (509,511B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->02.64% (509,511B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->02.64% (509,511B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->02.64% (509,511B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->02.64% (509,511B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | | | ->02.64% (509,511B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | | | ->02.64% (509,511B) 0xF1A7492: start_thread (pthread_create.c:333) | | | | | ->02.64% (509,511B) 0xFF3F93D: clone (clone.S:97) | | | | | | | | | ->00.18% (33,825B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | ->00.38% (73,530B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | ->00.00% (35B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->01.29% (248,679B) 0xF9F6813: char* std::string::_S_construct(char const*, char const*, std::allocator const&, std::forward_iterator_tag) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | ->01.29% (248,679B) 0xF9F6C44: std::basic_string, std::allocator >::basic_string(char const*, std::allocator const&) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | ->01.29% (248,679B) in 743 places, all below massif's threshold (1.00%) | | | | | ->00.00% (0B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->00.30% (58,232B) in 1+ places, all below ms_print's threshold (01.00%) | ->10.19% (1,963,854B) 0x54F757B: ceph::log::Log::create_entry(int, int, unsigned long*) (in /usr/lib/librgw.so.2.0.0) | ->08.10% (1,560,344B) 0x52CD4D0: RGWRequest::log(req_state*, char const*) (in /usr/lib/librgw.so.2.0.0) | | ->06.09% (1,172,955B) in 12 places, all below massif's threshold (1.00%) | | | | | ->02.01% (387,389B) 0x52CDD4A: RGWRequest::log_format(req_state*, char const*, ...) (in /usr/lib/librgw.so.2.0.0) | | ->02.01% (387,389B) in 4 places, all below massif's threshold (1.00%) | | | ->02.09% (403,510B) in 26 places, all below massif's threshold (1.00%) | ->09.20% (1,772,000B) 0x131429: worker_thread (civetweb.c:502) | ->09.20% (1,772,000B) 0xF1A7492: start_thread (pthread_create.c:333) | ->09.20% (1,772,000B) 0xFF3F93D: clone (clone.S:97) | ->09.08% (1,749,504B) 0x510DB36: RGWCoroutine::RGWCoroutine(CephContext*) (in /usr/lib/librgw.so.2.0.0) | ->08.73% (1,680,896B) 0x51075C3: RGWOmapAppend::RGWOmapAppend(RGWAsyncRadosProcessor*, RGWRados*, rgw_bucket&, std::string const&, unsigned long) (in /usr/lib/librgw.so.2.0.0) | | ->08.73% (1,680,896B) 0x51E7CB7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | ->08.73% (1,680,896B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | ->08.73% (1,680,896B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | ->08.73% (1,680,896B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | ->08.73% (1,680,896B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | ->08.73% (1,680,896B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | ->08.73% (1,680,896B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | ->08.73% (1,680,896B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | ->08.73% (1,680,896B) 0xF1A7492: start_thread (pthread_create.c:333) | | ->08.73% (1,680,896B) 0xFF3F93D: clone (clone.S:97) | | | ->00.36% (68,608B) in 1+ places, all below ms_print's threshold (01.00%) | ->07.78% (1,499,680B) 0x50F9E51: RGWCoroutinesManager::allocate_stack() (in /usr/lib/librgw.so.2.0.0) | ->07.78% (1,499,456B) 0x50FE7AE: RGWCoroutinesStack::spawn(RGWCoroutine*, RGWCoroutine*, bool) (in /usr/lib/librgw.so.2.0.0) | | ->03.82% (735,392B) 0x51E7CD7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | | ->03.82% (735,392B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | ->03.82% (735,392B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | ->03.82% (735,392B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | ->03.82% (735,392B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | ->03.82% (735,392B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | ->03.82% (735,392B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | ->03.82% (735,392B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | ->03.82% (735,392B) 0xF1A7492: start_thread (pthread_create.c:333) | | | ->03.82% (735,392B) 0xFF3F93D: clone (clone.S:97) | | | | | ->03.82% (735,392B) 0x51D5E5F: RGWDataSyncShardCR::init_lease_cr() (in /usr/lib/librgw.so.2.0.0) | | | ->03.82% (735,392B) 0x51E7CEE: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | | ->03.82% (735,392B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | ->03.82% (735,392B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | ->03.82% (735,392B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | ->03.82% (735,392B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | ->03.82% (735,392B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | ->03.82% (735,392B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | ->03.82% (735,392B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | ->03.82% (735,392B) 0xF1A7492: start_thread (pthread_create.c:333) | | | ->03.82% (735,392B) 0xFF3F93D: clone (clone.S:97) | | | | | ->00.15% (28,672B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->00.00% (224B) in 1+ places, all below ms_print's threshold (01.00%) | ->06.90% (1,329,691B) in 661 places, all below massif's threshold (1.00%) | ->03.90% (750,605B) 0x10623B1B: ??? (in /usr/lib/librados.so.2.0.0) | ->02.20% (424,453B) 0x1051E944: ??? (in /usr/lib/librados.so.2.0.0) | | ->02.20% (424,453B) 0x1051F207: ??? (in /usr/lib/librados.so.2.0.0) | | | ->02.20% (424,453B) 0x105393B8: ??? (in /usr/lib/librados.so.2.0.0) | | | ->02.20% (424,453B) 0x10541BBB: ??? (in /usr/lib/librados.so.2.0.0) | | | ->02.20% (424,453B) 0xF1A7492: start_thread (pthread_create.c:333) | | | ->02.20% (424,453B) 0xFF3F93D: clone (clone.S:97) | | | | | ->00.00% (0B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->01.68% (324,300B) 0x105486F9: ??? (in /usr/lib/librados.so.2.0.0) | | ->01.64% (316,294B) 0x102C6DDA: ??? (in /usr/lib/librados.so.2.0.0) | | | ->01.57% (301,561B) 0x102CBD9E: ??? (in /usr/lib/librados.so.2.0.0) | | | | ->01.57% (301,561B) 0x102CDDBD: ??? (in /usr/lib/librados.so.2.0.0) | | | | | ->01.57% (301,561B) 0x102CE0DB: ??? (in /usr/lib/librados.so.2.0.0) | | | | | | ->01.57% (301,561B) in 5 places, all below massif's threshold (1.00%) | | | | | | | | | | | ->00.00% (0B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | | | ->00.00% (0B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | ->00.08% (14,733B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | ->00.04% (8,006B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->00.01% (1,852B) in 1+ places, all below ms_print's threshold (01.00%) | ->01.62% (311,296B) 0x56119DC: ceph::buffer::list::append(char const*, unsigned int) (in /usr/lib/librgw.so.2.0.0) | ->01.62% (311,296B) 0x56183AC: ceph::buffer::list::iterator_impl::copy_all(ceph::buffer::list&) (in /usr/lib/librgw.so.2.0.0) | | ->01.62% (311,296B) 0x52C5A82: RGWCache::get_system_obj(RGWObjectCtx&, RGWRados::SystemObject::Read::GetObjState&, RGWObjVersionTracker*, rgw_obj&, ceph::buffer::list&, long, long, std::map, std::allocator > >*, rgw_cache_entry_info*) (in /usr/lib/librgw.so.2.0.0) | | | ->01.62% (311,296B) 0x51044ED: RGWAsyncGetSystemObj::_send_request() (in /usr/lib/librgw.so.2.0.0) | | | | ->01.62% (311,296B) 0x5105840: RGWAsyncRadosProcessor::handle_request(RGWAsyncRadosRequest*) (in /usr/lib/librgw.so.2.0.0) | | | | ->01.62% (311,296B) 0x510590B: RGWAsyncRadosProcessor::RGWWQ::_process(RGWAsyncRadosRequest*, ThreadPool::TPHandle&) (in /usr/lib/librgw.so.2.0.0) | | | | ->01.62% (311,296B) 0x55FB64D: ThreadPool::worker(ThreadPool::WorkThread*) (in /usr/lib/librgw.so.2.0.0) | | | | ->01.62% (311,296B) 0x55FC57E: ThreadPool::WorkThread::entry() (in /usr/lib/librgw.so.2.0.0) | | | | ->01.62% (311,296B) 0xF1A7492: start_thread (pthread_create.c:333) | | | | ->01.62% (311,296B) 0xFF3F93D: clone (clone.S:97) | | | | | | | ->00.00% (0B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | ->00.00% (0B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->00.00% (0B) in 1+ places, all below ms_print's threshold (01.00%) | ->01.33% (256,032B) 0xEB3140D: ??? (in /usr/lib/x86_64-linux-gnu/libcurl-gnutls.so.4.4.0) | ->01.33% (256,032B) 0xEB432E2: curl_easy_init (in /usr/lib/x86_64-linux-gnu/libcurl-gnutls.so.4.4.0) | ->01.14% (219,456B) 0xEB4A584: ??? (in /usr/lib/x86_64-linux-gnu/libcurl-gnutls.so.4.4.0) | | ->01.14% (219,456B) 0x515E76C: RGWHTTPManager::RGWHTTPManager(CephContext*, RGWCompletionManager*) (in /usr/lib/librgw.so.2.0.0) | | ->01.14% (219,456B) in 8 places, all below massif's threshold (1.00%) | | | ->00.19% (36,576B) in 1+ places, all below ms_print's threshold (01.00%) | ->01.14% (218,688B) 0x510DB0E: RGWCoroutine::RGWCoroutine(CephContext*) (in /usr/lib/librgw.so.2.0.0) | ->01.09% (210,112B) 0x51075C3: RGWOmapAppend::RGWOmapAppend(RGWAsyncRadosProcessor*, RGWRados*, rgw_bucket&, std::string const&, unsigned long) (in /usr/lib/librgw.so.2.0.0) | | ->01.09% (210,112B) 0x51E7CB7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | ->01.09% (210,112B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | ->01.09% (210,112B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | ->01.09% (210,112B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | ->01.09% (210,112B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | ->01.09% (210,112B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | ->01.09% (210,112B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | ->01.09% (210,112B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | ->01.09% (210,112B) 0xF1A7492: start_thread (pthread_create.c:333) | | ->01.09% (210,112B) 0xFF3F93D: clone (clone.S:97) | | | ->00.04% (8,576B) in 1+ places, all below ms_print's threshold (01.00%) | ->01.08% (208,896B) 0x51E570B: RGWDataSyncCR::operate() (in /usr/lib/librgw.so.2.0.0) | ->01.08% (208,896B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | ->01.08% (208,896B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | ->01.08% (208,896B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | ->01.08% (208,896B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | ->01.08% (208,896B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | ->01.08% (208,896B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | ->01.08% (208,896B) 0xF1A7492: start_thread (pthread_create.c:333) | ->01.08% (208,896B) 0xFF3F93D: clone (clone.S:97) | ->01.02% (196,056B) 0x1D82244B: PR_NewLock (in /usr/lib/x86_64-linux-gnu/libnspr4.so) | ->01.02% (196,056B) in 75 places, all below massif's threshold (1.00%) | ->01.02% (195,648B) 0xEB48E50: ??? (in /usr/lib/x86_64-linux-gnu/libcurl-gnutls.so.4.4.0) ->01.01% (194,880B) 0xEB4925F: ??? (in /usr/lib/x86_64-linux-gnu/libcurl-gnutls.so.4.4.0) | ->01.01% (194,880B) in 3 places, all below massif's threshold (1.00%) | ->00.00% (768B) in 1+ places, all below ms_print's threshold (01.00%) -------------------------------------------------------------------------------- n time(i) total(B) useful-heap(B) extra-heap(B) stacks(B) -------------------------------------------------------------------------------- 4 7,615,986,248 24,995,832 23,866,154 1,129,678 0 95.48% (23,866,154B) (heap allocation functions) malloc/new/new[], --alloc-fns, etc. ->29.67% (7,416,288B) 0x51E7C89: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | ->29.67% (7,416,288B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | ->29.67% (7,416,288B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | ->29.67% (7,416,288B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | ->29.67% (7,416,288B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | ->29.67% (7,416,288B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | ->29.67% (7,416,288B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | ->29.67% (7,416,288B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | ->29.67% (7,416,288B) 0xF1A7492: start_thread (pthread_create.c:333) | ->29.67% (7,416,288B) 0xFF3F93D: clone (clone.S:97) | ->15.19% (3,795,798B) 0xF9F4E97: std::string::_Rep::_S_create(unsigned long, unsigned long, std::allocator const&) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | ->13.61% (3,402,753B) 0xF9F5B09: std::string::_Rep::_M_clone(std::allocator const&, unsigned long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | ->13.61% (3,401,214B) 0xF9F5BAE: std::string::reserve(unsigned long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | ->10.14% (2,535,177B) 0xF9D1FB2: std::basic_stringbuf, std::allocator >::overflow(int) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->10.14% (2,535,177B) 0xF9D5D63: std::basic_streambuf >::xsputn(char const*, long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->10.14% (2,534,640B) 0xF9CD314: std::basic_ostream >& std::__ostream_insert >(std::basic_ostream >&, char const*, long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | | ->10.06% (2,513,697B) 0xF9CD6E5: std::basic_ostream >& std::operator<< >(std::basic_ostream >&, char const*) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | | | ->10.06% (2,513,697B) 0x5109AE4: RGWOmapAppend::operate() (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->10.06% (2,513,697B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->10.06% (2,513,697B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->10.06% (2,513,697B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->10.06% (2,513,697B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->10.06% (2,513,697B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->10.06% (2,513,697B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->10.06% (2,513,697B) 0xF1A7492: start_thread (pthread_create.c:333) | | | | | | | ->10.06% (2,513,697B) 0xFF3F93D: clone (clone.S:97) | | | | | | | | | | | | | ->00.00% (0B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | | | | | ->00.08% (20,943B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | | | ->00.00% (537B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | ->03.07% (766,669B) 0xF9F5E38: ??? (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->03.07% (766,669B) 0xF9F5F0C: std::string::append(char const*, unsigned long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->02.91% (726,574B) 0x51DF1E4: RGWDataSyncShardControlCR::alloc_cr() (in /usr/lib/librgw.so.2.0.0) | | | | | ->02.91% (726,574B) 0x51AB72B: RGWBackoffControlCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | | | ->02.91% (726,574B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->02.91% (726,574B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->02.91% (726,574B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->02.91% (726,574B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->02.91% (726,574B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | | | ->02.91% (726,574B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | | | ->02.91% (726,574B) 0xF1A7492: start_thread (pthread_create.c:333) | | | | | ->02.91% (726,574B) 0xFF3F93D: clone (clone.S:97) | | | | | | | | | ->00.16% (40,095B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | ->00.40% (99,368B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | ->00.01% (1,539B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->01.30% (325,642B) 0xF9F6813: char* std::string::_S_construct(char const*, char const*, std::allocator const&, std::forward_iterator_tag) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | ->01.30% (325,642B) 0xF9F6C44: std::basic_string, std::allocator >::basic_string(char const*, std::allocator const&) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | ->01.30% (325,642B) in 829 places, all below massif's threshold (1.00%) | | | | | ->00.00% (0B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->00.27% (67,403B) in 1+ places, all below ms_print's threshold (01.00%) | ->09.87% (2,467,840B) 0x510DB36: RGWCoroutine::RGWCoroutine(CephContext*) (in /usr/lib/librgw.so.2.0.0) | ->09.59% (2,397,184B) 0x51075C3: RGWOmapAppend::RGWOmapAppend(RGWAsyncRadosProcessor*, RGWRados*, rgw_bucket&, std::string const&, unsigned long) (in /usr/lib/librgw.so.2.0.0) | | ->09.59% (2,397,184B) 0x51E7CB7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | ->09.59% (2,397,184B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | ->09.59% (2,397,184B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | ->09.59% (2,397,184B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | ->09.59% (2,397,184B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | ->09.59% (2,397,184B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | ->09.59% (2,397,184B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | ->09.59% (2,397,184B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | ->09.59% (2,397,184B) 0xF1A7492: start_thread (pthread_create.c:333) | | ->09.59% (2,397,184B) 0xFF3F93D: clone (clone.S:97) | | | ->00.28% (70,656B) in 1+ places, all below ms_print's threshold (01.00%) | ->09.11% (2,276,687B) in 776 places, all below massif's threshold (1.00%) | ->08.52% (2,128,448B) 0x50F9E51: RGWCoroutinesManager::allocate_stack() (in /usr/lib/librgw.so.2.0.0) | ->08.51% (2,128,224B) 0x50FE7AE: RGWCoroutinesStack::spawn(RGWCoroutine*, RGWCoroutine*, bool) (in /usr/lib/librgw.so.2.0.0) | | ->04.20% (1,048,768B) 0x51E7CD7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | | ->04.20% (1,048,768B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | ->04.20% (1,048,768B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | ->04.20% (1,048,768B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | ->04.20% (1,048,768B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | ->04.20% (1,048,768B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | ->04.20% (1,048,768B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | ->04.20% (1,048,768B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | ->04.20% (1,048,768B) 0xF1A7492: start_thread (pthread_create.c:333) | | | ->04.20% (1,048,768B) 0xFF3F93D: clone (clone.S:97) | | | | | ->04.19% (1,048,544B) 0x51D5E5F: RGWDataSyncShardCR::init_lease_cr() (in /usr/lib/librgw.so.2.0.0) | | | ->04.19% (1,048,544B) 0x51E7CEE: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | | ->04.19% (1,048,544B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | ->04.19% (1,048,544B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | ->04.19% (1,048,544B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | ->04.19% (1,048,544B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | ->04.19% (1,048,544B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | ->04.19% (1,048,544B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | ->04.19% (1,048,544B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | ->04.19% (1,048,544B) 0xF1A7492: start_thread (pthread_create.c:333) | | | ->04.19% (1,048,544B) 0xFF3F93D: clone (clone.S:97) | | | | | ->00.12% (30,912B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->00.00% (224B) in 1+ places, all below ms_print's threshold (01.00%) | ->07.86% (1,965,433B) 0x54F757B: ceph::log::Log::create_entry(int, int, unsigned long*) (in /usr/lib/librgw.so.2.0.0) | ->05.65% (1,411,631B) 0x52CD4D0: RGWRequest::log(req_state*, char const*) (in /usr/lib/librgw.so.2.0.0) | | ->04.24% (1,060,234B) in 12 places, all below massif's threshold (1.00%) | | | | | ->01.41% (351,397B) 0x52CDD4A: RGWRequest::log_format(req_state*, char const*, ...) (in /usr/lib/librgw.so.2.0.0) | | ->01.41% (351,397B) in 4 places, all below massif's threshold (1.00%) | | | ->02.22% (553,802B) in 34 places, all below massif's threshold (1.00%) | ->07.09% (1,772,000B) 0x131429: worker_thread (civetweb.c:502) | ->07.09% (1,772,000B) 0xF1A7492: start_thread (pthread_create.c:333) | ->07.09% (1,772,000B) 0xFF3F93D: clone (clone.S:97) | ->03.22% (805,388B) 0x10623B1B: ??? (in /usr/lib/librados.so.2.0.0) | ->01.80% (449,345B) 0x1051E944: ??? (in /usr/lib/librados.so.2.0.0) | | ->01.80% (449,345B) 0x1051F207: ??? (in /usr/lib/librados.so.2.0.0) | | | ->01.80% (449,345B) 0x105393B8: ??? (in /usr/lib/librados.so.2.0.0) | | | ->01.80% (449,345B) 0x10541BBB: ??? (in /usr/lib/librados.so.2.0.0) | | | ->01.80% (449,345B) 0xF1A7492: start_thread (pthread_create.c:333) | | | ->01.80% (449,345B) 0xFF3F93D: clone (clone.S:97) | | | | | ->00.00% (0B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->01.42% (354,191B) 0x105486F9: ??? (in /usr/lib/librados.so.2.0.0) | | ->01.39% (348,110B) 0x102C6DDA: ??? (in /usr/lib/librados.so.2.0.0) | | | ->01.35% (336,959B) 0x102CBD9E: ??? (in /usr/lib/librados.so.2.0.0) | | | | ->01.35% (336,959B) 0x102CDDBD: ??? (in /usr/lib/librados.so.2.0.0) | | | | | ->01.35% (336,959B) 0x102CE0DB: ??? (in /usr/lib/librados.so.2.0.0) | | | | | | ->01.35% (336,959B) in 6 places, all below massif's threshold (1.00%) | | | | | | | | | | | ->00.00% (0B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | | | ->00.00% (0B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | ->00.04% (11,151B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | ->00.02% (6,081B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->00.01% (1,852B) in 1+ places, all below ms_print's threshold (01.00%) | ->02.38% (593,920B) 0x5614937: ceph::buffer::create_aligned(unsigned int, unsigned int) (in /usr/lib/librgw.so.2.0.0) | ->02.38% (593,920B) 0x1052B4D8: ??? (in /usr/lib/librados.so.2.0.0) | | ->02.38% (593,920B) 0x105381E2: ??? (in /usr/lib/librados.so.2.0.0) | | ->02.38% (593,920B) 0x10541BBB: ??? (in /usr/lib/librados.so.2.0.0) | | ->02.38% (593,920B) 0xF1A7492: start_thread (pthread_create.c:333) | | ->02.38% (593,920B) 0xFF3F93D: clone (clone.S:97) | | | ->00.00% (0B) in 1+ places, all below ms_print's threshold (01.00%) | ->01.34% (335,872B) 0x56119DC: ceph::buffer::list::append(char const*, unsigned int) (in /usr/lib/librgw.so.2.0.0) | ->01.23% (307,200B) 0x56183AC: ceph::buffer::list::iterator_impl::copy_all(ceph::buffer::list&) (in /usr/lib/librgw.so.2.0.0) | | ->01.23% (307,200B) 0x52C5A82: RGWCache::get_system_obj(RGWObjectCtx&, RGWRados::SystemObject::Read::GetObjState&, RGWObjVersionTracker*, rgw_obj&, ceph::buffer::list&, long, long, std::map, std::allocator > >*, rgw_cache_entry_info*) (in /usr/lib/librgw.so.2.0.0) | | | ->01.23% (307,200B) 0x51044ED: RGWAsyncGetSystemObj::_send_request() (in /usr/lib/librgw.so.2.0.0) | | | | ->01.23% (307,200B) 0x5105840: RGWAsyncRadosProcessor::handle_request(RGWAsyncRadosRequest*) (in /usr/lib/librgw.so.2.0.0) | | | | ->01.23% (307,200B) 0x510590B: RGWAsyncRadosProcessor::RGWWQ::_process(RGWAsyncRadosRequest*, ThreadPool::TPHandle&) (in /usr/lib/librgw.so.2.0.0) | | | | ->01.23% (307,200B) 0x55FB64D: ThreadPool::worker(ThreadPool::WorkThread*) (in /usr/lib/librgw.so.2.0.0) | | | | ->01.23% (307,200B) 0x55FC57E: ThreadPool::WorkThread::entry() (in /usr/lib/librgw.so.2.0.0) | | | | ->01.23% (307,200B) 0xF1A7492: start_thread (pthread_create.c:333) | | | | ->01.23% (307,200B) 0xFF3F93D: clone (clone.S:97) | | | | | | | ->00.00% (0B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | ->00.00% (0B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->00.11% (28,672B) in 1+ places, all below ms_print's threshold (01.00%) | ->01.23% (308,480B) 0x510DB0E: RGWCoroutine::RGWCoroutine(CephContext*) (in /usr/lib/librgw.so.2.0.0) ->01.20% (299,648B) 0x51075C3: RGWOmapAppend::RGWOmapAppend(RGWAsyncRadosProcessor*, RGWRados*, rgw_bucket&, std::string const&, unsigned long) (in /usr/lib/librgw.so.2.0.0) | ->01.20% (299,648B) 0x51E7CB7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | ->01.20% (299,648B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | ->01.20% (299,648B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | ->01.20% (299,648B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | ->01.20% (299,648B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | ->01.20% (299,648B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | ->01.20% (299,648B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | ->01.20% (299,648B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | ->01.20% (299,648B) 0xF1A7492: start_thread (pthread_create.c:333) | ->01.20% (299,648B) 0xFF3F93D: clone (clone.S:97) | ->00.04% (8,832B) in 1+ places, all below ms_print's threshold (01.00%) -------------------------------------------------------------------------------- n time(i) total(B) useful-heap(B) extra-heap(B) stacks(B) -------------------------------------------------------------------------------- 5 9,685,212,325 28,760,064 27,470,928 1,289,136 0 95.52% (27,470,928B) (heap allocation functions) malloc/new/new[], --alloc-fns, etc. ->31.39% (9,027,216B) 0x51E7C89: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | ->31.39% (9,027,216B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | ->31.39% (9,027,216B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | ->31.39% (9,027,216B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | ->31.39% (9,027,216B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | ->31.39% (9,027,216B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | ->31.39% (9,027,216B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | ->31.39% (9,027,216B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | ->31.39% (9,027,216B) 0xF1A7492: start_thread (pthread_create.c:333) | ->31.39% (9,027,216B) 0xFF3F93D: clone (clone.S:97) | ->15.90% (4,572,486B) 0xF9F4E97: std::string::_Rep::_S_create(unsigned long, unsigned long, std::allocator const&) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | ->14.29% (4,108,770B) 0xF9F5B09: std::string::_Rep::_M_clone(std::allocator const&, unsigned long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | ->14.28% (4,106,035B) 0xF9F5BAE: std::string::reserve(unsigned long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | ->10.71% (3,080,769B) 0xF9D1FB2: std::basic_stringbuf, std::allocator >::overflow(int) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->10.71% (3,080,769B) 0xF9D5D63: std::basic_streambuf >::xsputn(char const*, long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->10.71% (3,080,232B) 0xF9CD314: std::basic_ostream >& std::__ostream_insert >(std::basic_ostream >&, char const*, long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | | ->10.64% (3,060,363B) 0xF9CD6E5: std::basic_ostream >& std::operator<< >(std::basic_ostream >&, char const*) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | | | ->10.64% (3,060,363B) 0x5109AE4: RGWOmapAppend::operate() (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->10.64% (3,060,363B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->10.64% (3,060,363B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->10.64% (3,060,363B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->10.64% (3,060,363B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->10.64% (3,060,363B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->10.64% (3,060,363B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->10.64% (3,060,363B) 0xF1A7492: start_thread (pthread_create.c:333) | | | | | | | ->10.64% (3,060,363B) 0xFF3F93D: clone (clone.S:97) | | | | | | | | | | | | | ->00.00% (0B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | | | | | ->00.07% (19,869B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | | | ->00.00% (537B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | ->03.22% (926,047B) 0xF9F5E38: ??? (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->03.22% (926,047B) 0xF9F5F0C: std::string::append(char const*, unsigned long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->03.07% (884,345B) 0x51DF1E4: RGWDataSyncShardControlCR::alloc_cr() (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.07% (884,345B) 0x51AB72B: RGWBackoffControlCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.07% (884,345B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.07% (884,345B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.07% (884,345B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.07% (884,345B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.07% (884,345B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.07% (884,345B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.07% (884,345B) 0xF1A7492: start_thread (pthread_create.c:333) | | | | | ->03.07% (884,345B) 0xFF3F93D: clone (clone.S:97) | | | | | | | | | ->00.14% (41,702B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | ->00.34% (99,219B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | ->00.01% (2,735B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->01.34% (385,161B) 0xF9F6813: char* std::string::_S_construct(char const*, char const*, std::allocator const&, std::forward_iterator_tag) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | ->01.34% (385,161B) 0xF9F6C44: std::basic_string, std::allocator >::basic_string(char const*, std::allocator const&) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | ->01.03% (297,636B) 0x510DA32: RGWCoroutine::RGWCoroutine(CephContext*) (in /usr/lib/librgw.so.2.0.0) | | | | ->01.01% (290,649B) 0x51075C3: RGWOmapAppend::RGWOmapAppend(RGWAsyncRadosProcessor*, RGWRados*, rgw_bucket&, std::string const&, unsigned long) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.01% (290,649B) 0x51E7CB7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.01% (290,649B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.01% (290,649B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.01% (290,649B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.01% (290,649B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.01% (290,649B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.01% (290,649B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.01% (290,649B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.01% (290,649B) 0xF1A7492: start_thread (pthread_create.c:333) | | | | | ->01.01% (290,649B) 0xFF3F93D: clone (clone.S:97) | | | | | | | | | ->00.02% (6,987B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | ->00.30% (87,525B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | ->00.00% (0B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->00.27% (78,555B) in 1+ places, all below ms_print's threshold (01.00%) | ->10.39% (2,988,032B) 0x510DB36: RGWCoroutine::RGWCoroutine(CephContext*) (in /usr/lib/librgw.so.2.0.0) | ->10.15% (2,917,888B) 0x51075C3: RGWOmapAppend::RGWOmapAppend(RGWAsyncRadosProcessor*, RGWRados*, rgw_bucket&, std::string const&, unsigned long) (in /usr/lib/librgw.so.2.0.0) | | ->10.15% (2,917,888B) 0x51E7CB7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | ->10.15% (2,917,888B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | ->10.15% (2,917,888B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | ->10.15% (2,917,888B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | ->10.15% (2,917,888B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | ->10.15% (2,917,888B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | ->10.15% (2,917,888B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | ->10.15% (2,917,888B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | ->10.15% (2,917,888B) 0xF1A7492: start_thread (pthread_create.c:333) | | ->10.15% (2,917,888B) 0xFF3F93D: clone (clone.S:97) | | | ->00.24% (70,144B) in 1+ places, all below ms_print's threshold (01.00%) | ->09.00% (2,586,976B) 0x50F9E51: RGWCoroutinesManager::allocate_stack() (in /usr/lib/librgw.so.2.0.0) | ->08.99% (2,586,752B) 0x50FE7AE: RGWCoroutinesStack::spawn(RGWCoroutine*, RGWCoroutine*, bool) (in /usr/lib/librgw.so.2.0.0) | | ->04.44% (1,276,576B) 0x51E7CD7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | | ->04.44% (1,276,576B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | ->04.44% (1,276,576B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | ->04.44% (1,276,576B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | ->04.44% (1,276,576B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | ->04.44% (1,276,576B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | ->04.44% (1,276,576B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | ->04.44% (1,276,576B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | ->04.44% (1,276,576B) 0xF1A7492: start_thread (pthread_create.c:333) | | | ->04.44% (1,276,576B) 0xFF3F93D: clone (clone.S:97) | | | | | ->04.44% (1,276,576B) 0x51D5E5F: RGWDataSyncShardCR::init_lease_cr() (in /usr/lib/librgw.so.2.0.0) | | | ->04.44% (1,276,576B) 0x51E7CEE: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | | ->04.44% (1,276,576B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | ->04.44% (1,276,576B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | ->04.44% (1,276,576B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | ->04.44% (1,276,576B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | ->04.44% (1,276,576B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | ->04.44% (1,276,576B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | ->04.44% (1,276,576B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | ->04.44% (1,276,576B) 0xF1A7492: start_thread (pthread_create.c:333) | | | ->04.44% (1,276,576B) 0xFF3F93D: clone (clone.S:97) | | | | | ->00.12% (33,600B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->00.00% (224B) in 1+ places, all below ms_print's threshold (01.00%) | ->08.42% (2,421,667B) in 777 places, all below massif's threshold (1.00%) | ->07.13% (2,050,989B) 0x54F757B: ceph::log::Log::create_entry(int, int, unsigned long*) (in /usr/lib/librgw.so.2.0.0) | ->05.31% (1,526,183B) 0x52CD4D0: RGWRequest::log(req_state*, char const*) (in /usr/lib/librgw.so.2.0.0) | | ->03.98% (1,145,660B) in 12 places, all below massif's threshold (1.00%) | | | | | ->01.32% (380,523B) 0x52CDD4A: RGWRequest::log_format(req_state*, char const*, ...) (in /usr/lib/librgw.so.2.0.0) | | ->01.32% (380,523B) in 4 places, all below massif's threshold (1.00%) | | | ->01.82% (524,806B) in 34 places, all below massif's threshold (1.00%) | ->06.16% (1,772,000B) 0x131429: worker_thread (civetweb.c:502) | ->06.16% (1,772,000B) 0xF1A7492: start_thread (pthread_create.c:333) | ->06.16% (1,772,000B) 0xFF3F93D: clone (clone.S:97) | ->02.47% (711,402B) 0x10623B1B: ??? (in /usr/lib/librados.so.2.0.0) | ->01.40% (402,144B) 0x1051E944: ??? (in /usr/lib/librados.so.2.0.0) | | ->01.40% (402,144B) 0x1051F207: ??? (in /usr/lib/librados.so.2.0.0) | | | ->01.40% (402,144B) 0x105393B8: ??? (in /usr/lib/librados.so.2.0.0) | | | ->01.40% (402,144B) 0x10541BBB: ??? (in /usr/lib/librados.so.2.0.0) | | | ->01.40% (402,144B) 0xF1A7492: start_thread (pthread_create.c:333) | | | ->01.40% (402,144B) 0xFF3F93D: clone (clone.S:97) | | | | | ->00.00% (0B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->01.07% (307,406B) 0x105486F9: ??? (in /usr/lib/librados.so.2.0.0) | | ->01.05% (301,023B) 0x102C6DDA: ??? (in /usr/lib/librados.so.2.0.0) | | | ->01.01% (289,107B) 0x102CBD9E: ??? (in /usr/lib/librados.so.2.0.0) | | | | ->01.01% (289,107B) 0x102CDDBD: ??? (in /usr/lib/librados.so.2.0.0) | | | | | ->01.01% (289,107B) 0x102CE0DB: ??? (in /usr/lib/librados.so.2.0.0) | | | | | | ->01.01% (289,107B) in 6 places, all below massif's threshold (1.00%) | | | | | | | | | | | ->00.00% (0B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | | | ->00.00% (0B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | ->00.04% (11,916B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | ->00.02% (6,383B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->00.01% (1,852B) in 1+ places, all below ms_print's threshold (01.00%) | ->02.28% (655,360B) 0x5614937: ceph::buffer::create_aligned(unsigned int, unsigned int) (in /usr/lib/librgw.so.2.0.0) | ->02.28% (655,360B) 0x1052B4D8: ??? (in /usr/lib/librados.so.2.0.0) | | ->02.28% (655,360B) 0x105381E2: ??? (in /usr/lib/librados.so.2.0.0) | | ->02.28% (655,360B) 0x10541BBB: ??? (in /usr/lib/librados.so.2.0.0) | | ->02.28% (655,360B) 0xF1A7492: start_thread (pthread_create.c:333) | | ->02.28% (655,360B) 0xFF3F93D: clone (clone.S:97) | | | ->00.00% (0B) in 1+ places, all below ms_print's threshold (01.00%) | ->01.30% (373,504B) 0x510DB0E: RGWCoroutine::RGWCoroutine(CephContext*) (in /usr/lib/librgw.so.2.0.0) | ->01.27% (364,736B) 0x51075C3: RGWOmapAppend::RGWOmapAppend(RGWAsyncRadosProcessor*, RGWRados*, rgw_bucket&, std::string const&, unsigned long) (in /usr/lib/librgw.so.2.0.0) | | ->01.27% (364,736B) 0x51E7CB7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | ->01.27% (364,736B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | ->01.27% (364,736B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | ->01.27% (364,736B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | ->01.27% (364,736B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | ->01.27% (364,736B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | ->01.27% (364,736B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | ->01.27% (364,736B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | ->01.27% (364,736B) 0xF1A7492: start_thread (pthread_create.c:333) | | ->01.27% (364,736B) 0xFF3F93D: clone (clone.S:97) | | | ->00.03% (8,768B) in 1+ places, all below ms_print's threshold (01.00%) | ->01.08% (311,296B) 0x56119DC: ceph::buffer::list::append(char const*, unsigned int) (in /usr/lib/librgw.so.2.0.0) ->01.03% (294,912B) 0x56183AC: ceph::buffer::list::iterator_impl::copy_all(ceph::buffer::list&) (in /usr/lib/librgw.so.2.0.0) | ->01.03% (294,912B) 0x52C5A82: RGWCache::get_system_obj(RGWObjectCtx&, RGWRados::SystemObject::Read::GetObjState&, RGWObjVersionTracker*, rgw_obj&, ceph::buffer::list&, long, long, std::map, std::allocator > >*, rgw_cache_entry_info*) (in /usr/lib/librgw.so.2.0.0) | | ->01.03% (294,912B) 0x51044ED: RGWAsyncGetSystemObj::_send_request() (in /usr/lib/librgw.so.2.0.0) | | | ->01.03% (294,912B) 0x5105840: RGWAsyncRadosProcessor::handle_request(RGWAsyncRadosRequest*) (in /usr/lib/librgw.so.2.0.0) | | | ->01.03% (294,912B) 0x510590B: RGWAsyncRadosProcessor::RGWWQ::_process(RGWAsyncRadosRequest*, ThreadPool::TPHandle&) (in /usr/lib/librgw.so.2.0.0) | | | ->01.03% (294,912B) 0x55FB64D: ThreadPool::worker(ThreadPool::WorkThread*) (in /usr/lib/librgw.so.2.0.0) | | | ->01.03% (294,912B) 0x55FC57E: ThreadPool::WorkThread::entry() (in /usr/lib/librgw.so.2.0.0) | | | ->01.03% (294,912B) 0xF1A7492: start_thread (pthread_create.c:333) | | | ->01.03% (294,912B) 0xFF3F93D: clone (clone.S:97) | | | | | ->00.00% (0B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->00.00% (0B) in 1+ places, all below ms_print's threshold (01.00%) | ->00.06% (16,384B) in 1+ places, all below ms_print's threshold (01.00%) -------------------------------------------------------------------------------- n time(i) total(B) useful-heap(B) extra-heap(B) stacks(B) -------------------------------------------------------------------------------- 6 11,160,700,049 33,287,848 31,696,849 1,590,999 0 7 12,992,479,183 34,045,264 32,404,169 1,641,095 0 8 14,824,238,361 34,069,080 32,578,757 1,490,323 0 9 16,394,340,696 39,878,920 38,047,905 1,831,015 0 10 18,054,495,275 41,316,432 39,458,358 1,858,074 0 11 19,927,736,267 43,108,888 41,216,332 1,892,556 0 12 22,098,584,167 45,829,432 43,751,666 2,077,766 0 13 24,569,884,579 49,501,016 47,330,992 2,170,024 0 14 26,781,821,960 51,581,888 49,313,162 2,268,726 0 15 28,681,090,618 53,552,984 51,286,802 2,266,182 0 95.77% (51,286,802B) (heap allocation functions) malloc/new/new[], --alloc-fns, etc. ->37.35% (20,002,752B) 0x51E7C89: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | ->37.35% (20,002,752B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | ->37.35% (20,002,752B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | ->37.35% (20,002,752B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | ->37.35% (20,002,752B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | ->37.35% (20,002,752B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | ->37.35% (20,002,752B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | ->37.35% (20,002,752B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | ->37.35% (20,002,752B) 0xF1A7492: start_thread (pthread_create.c:333) | ->37.35% (20,002,752B) 0xFF3F93D: clone (clone.S:97) | ->18.11% (9,699,554B) 0xF9F4E97: std::string::_Rep::_S_create(unsigned long, unsigned long, std::allocator const&) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | ->16.62% (8,902,086B) 0xF9F5B09: std::string::_Rep::_M_clone(std::allocator const&, unsigned long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | ->16.62% (8,900,538B) 0xF9F5BAE: std::string::reserve(unsigned long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | ->12.70% (6,800,031B) 0xF9D1FB2: std::basic_stringbuf, std::allocator >::overflow(int) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->12.70% (6,800,031B) 0xF9D5D63: std::basic_streambuf >::xsputn(char const*, long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->12.70% (6,799,494B) 0xF9CD314: std::basic_ostream >& std::__ostream_insert >(std::basic_ostream >&, char const*, long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | | ->12.66% (6,781,236B) 0xF9CD6E5: std::basic_ostream >& std::operator<< >(std::basic_ostream >&, char const*) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | | | ->12.66% (6,781,236B) 0x5109AE4: RGWOmapAppend::operate() (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->12.66% (6,781,236B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->12.66% (6,781,236B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->12.66% (6,781,236B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->12.66% (6,781,236B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->12.66% (6,781,236B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->12.66% (6,781,236B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->12.66% (6,781,236B) 0xF1A7492: start_thread (pthread_create.c:333) | | | | | | | ->12.66% (6,781,236B) 0xFF3F93D: clone (clone.S:97) | | | | | | | | | | | | | ->00.00% (0B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | | | | | ->00.03% (18,258B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | | | ->00.00% (537B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | ->03.74% (2,003,268B) 0xF9F5E38: ??? (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->03.74% (2,003,268B) 0xF9F5F0C: std::string::append(char const*, unsigned long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->03.66% (1,959,466B) 0x51DF1E4: RGWDataSyncShardControlCR::alloc_cr() (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.66% (1,959,466B) 0x51AB72B: RGWBackoffControlCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.66% (1,959,466B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.66% (1,959,466B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.66% (1,959,466B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.66% (1,959,466B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.66% (1,959,466B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.66% (1,959,466B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.66% (1,959,466B) 0xF1A7492: start_thread (pthread_create.c:333) | | | | | ->03.66% (1,959,466B) 0xFF3F93D: clone (clone.S:97) | | | | | | | | | ->00.08% (43,802B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | ->00.18% (97,239B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | ->00.00% (1,548B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->01.37% (731,641B) 0xF9F6813: char* std::string::_S_construct(char const*, char const*, std::allocator const&, std::forward_iterator_tag) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | ->01.37% (731,641B) 0xF9F6C44: std::basic_string, std::allocator >::basic_string(char const*, std::allocator const&) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | ->01.22% (650,811B) 0x510DA32: RGWCoroutine::RGWCoroutine(CephContext*) (in /usr/lib/librgw.so.2.0.0) | | | | ->01.20% (644,028B) 0x51075C3: RGWOmapAppend::RGWOmapAppend(RGWAsyncRadosProcessor*, RGWRados*, rgw_bucket&, std::string const&, unsigned long) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.20% (644,028B) 0x51E7CB7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.20% (644,028B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.20% (644,028B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.20% (644,028B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.20% (644,028B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.20% (644,028B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.20% (644,028B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.20% (644,028B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.20% (644,028B) 0xF1A7492: start_thread (pthread_create.c:333) | | | | | ->01.20% (644,028B) 0xFF3F93D: clone (clone.S:97) | | | | | | | | | ->00.01% (6,783B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | ->00.15% (80,830B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | ->00.00% (0B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->00.12% (65,827B) in 1+ places, all below ms_print's threshold (01.00%) | ->12.20% (6,533,120B) 0x510DB36: RGWCoroutine::RGWCoroutine(CephContext*) (in /usr/lib/librgw.so.2.0.0) | ->12.07% (6,465,024B) 0x51075C3: RGWOmapAppend::RGWOmapAppend(RGWAsyncRadosProcessor*, RGWRados*, rgw_bucket&, std::string const&, unsigned long) (in /usr/lib/librgw.so.2.0.0) | | ->12.07% (6,465,024B) 0x51E7CB7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | ->12.07% (6,465,024B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | ->12.07% (6,465,024B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | ->12.07% (6,465,024B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | ->12.07% (6,465,024B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | ->12.07% (6,465,024B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | ->12.07% (6,465,024B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | ->12.07% (6,465,024B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | ->12.07% (6,465,024B) 0xF1A7492: start_thread (pthread_create.c:333) | | ->12.07% (6,465,024B) 0xFF3F93D: clone (clone.S:97) | | | ->00.13% (68,096B) in 1+ places, all below ms_print's threshold (01.00%) | ->10.64% (5,695,872B) 0x50F9E51: RGWCoroutinesManager::allocate_stack() (in /usr/lib/librgw.so.2.0.0) | ->10.64% (5,695,648B) 0x50FE7AE: RGWCoroutinesStack::spawn(RGWCoroutine*, RGWCoroutine*, bool) (in /usr/lib/librgw.so.2.0.0) | | ->05.28% (2,828,672B) 0x51E7CD7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | | ->05.28% (2,828,672B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | ->05.28% (2,828,672B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.28% (2,828,672B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.28% (2,828,672B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.28% (2,828,672B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.28% (2,828,672B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | ->05.28% (2,828,672B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | ->05.28% (2,828,672B) 0xF1A7492: start_thread (pthread_create.c:333) | | | ->05.28% (2,828,672B) 0xFF3F93D: clone (clone.S:97) | | | | | ->05.28% (2,828,672B) 0x51D5E5F: RGWDataSyncShardCR::init_lease_cr() (in /usr/lib/librgw.so.2.0.0) | | | ->05.28% (2,828,672B) 0x51E7CEE: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | | ->05.28% (2,828,672B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | ->05.28% (2,828,672B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.28% (2,828,672B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.28% (2,828,672B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.28% (2,828,672B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.28% (2,828,672B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | ->05.28% (2,828,672B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | ->05.28% (2,828,672B) 0xF1A7492: start_thread (pthread_create.c:333) | | | ->05.28% (2,828,672B) 0xFF3F93D: clone (clone.S:97) | | | | | ->00.07% (38,304B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->00.00% (224B) in 1+ places, all below ms_print's threshold (01.00%) | ->06.99% (3,743,561B) in 828 places, all below massif's threshold (1.00%) | ->03.31% (1,772,000B) 0x131429: worker_thread (civetweb.c:502) | ->03.31% (1,772,000B) 0xF1A7492: start_thread (pthread_create.c:333) | ->03.31% (1,772,000B) 0xFF3F93D: clone (clone.S:97) | ->03.07% (1,645,331B) 0x54F757B: ceph::log::Log::create_entry(int, int, unsigned long*) (in /usr/lib/librgw.so.2.0.0) | ->01.91% (1,023,938B) 0x52CD4D0: RGWRequest::log(req_state*, char const*) (in /usr/lib/librgw.so.2.0.0) | | ->01.91% (1,023,938B) in 13 places, all below massif's threshold (1.00%) | | | ->01.16% (621,393B) in 42 places, all below massif's threshold (1.00%) | ->02.57% (1,377,908B) 0x10623B1B: ??? (in /usr/lib/librados.so.2.0.0) | ->01.43% (764,499B) 0x1051E944: ??? (in /usr/lib/librados.so.2.0.0) | | ->01.43% (763,672B) 0x1051F207: ??? (in /usr/lib/librados.so.2.0.0) | | | ->01.43% (763,672B) 0x105393B8: ??? (in /usr/lib/librados.so.2.0.0) | | | ->01.43% (763,672B) 0x10541BBB: ??? (in /usr/lib/librados.so.2.0.0) | | | ->01.43% (763,672B) 0xF1A7492: start_thread (pthread_create.c:333) | | | ->01.43% (763,672B) 0xFF3F93D: clone (clone.S:97) | | | | | ->00.00% (827B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->01.14% (608,245B) 0x105486F9: ??? (in /usr/lib/librados.so.2.0.0) | | ->01.11% (594,761B) 0x102C6DDA: ??? (in /usr/lib/librados.so.2.0.0) | | | ->01.07% (572,353B) 0x102CBD9E: ??? (in /usr/lib/librados.so.2.0.0) | | | | ->01.07% (572,353B) 0x102CDDBD: ??? (in /usr/lib/librados.so.2.0.0) | | | | | ->01.07% (572,353B) 0x102CE0DB: ??? (in /usr/lib/librados.so.2.0.0) | | | | | | ->01.07% (572,353B) in 6 places, all below massif's threshold (1.00%) | | | | | | | | | | | ->00.00% (0B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | | | ->00.00% (0B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | ->00.04% (22,408B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | ->00.03% (13,484B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->00.01% (5,164B) in 1+ places, all below ms_print's threshold (01.00%) | ->01.53% (816,704B) 0x510DB0E: RGWCoroutine::RGWCoroutine(CephContext*) (in /usr/lib/librgw.so.2.0.0) ->01.51% (808,192B) 0x51075C3: RGWOmapAppend::RGWOmapAppend(RGWAsyncRadosProcessor*, RGWRados*, rgw_bucket&, std::string const&, unsigned long) (in /usr/lib/librgw.so.2.0.0) | ->01.51% (808,192B) 0x51E7CB7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | ->01.51% (808,192B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | ->01.51% (808,192B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | ->01.51% (808,192B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | ->01.51% (808,192B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | ->01.51% (808,192B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | ->01.51% (808,192B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | ->01.51% (808,192B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | ->01.51% (808,192B) 0xF1A7492: start_thread (pthread_create.c:333) | ->01.51% (808,192B) 0xFF3F93D: clone (clone.S:97) | ->00.02% (8,512B) in 1+ places, all below ms_print's threshold (01.00%) -------------------------------------------------------------------------------- n time(i) total(B) useful-heap(B) extra-heap(B) stacks(B) -------------------------------------------------------------------------------- 16 29,862,582,559 57,545,776 55,114,915 2,430,861 0 95.78% (55,114,915B) (heap allocation functions) malloc/new/new[], --alloc-fns, etc. ->37.69% (21,686,544B) 0x51E7C89: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | ->37.69% (21,686,544B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | ->37.69% (21,686,544B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | ->37.69% (21,686,544B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | ->37.69% (21,686,544B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | ->37.69% (21,686,544B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | ->37.69% (21,686,544B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | ->37.69% (21,686,544B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | ->37.69% (21,686,544B) 0xF1A7492: start_thread (pthread_create.c:333) | ->37.69% (21,686,544B) 0xFF3F93D: clone (clone.S:97) | ->18.24% (10,498,581B) 0xF9F4E97: std::string::_Rep::_S_create(unsigned long, unsigned long, std::allocator const&) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | ->16.75% (9,636,450B) 0xF9F5B09: std::string::_Rep::_M_clone(std::allocator const&, unsigned long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | ->16.74% (9,634,382B) 0xF9F5BAE: std::string::reserve(unsigned long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | ->12.81% (7,371,936B) 0xF9D1FB2: std::basic_stringbuf, std::allocator >::overflow(int) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->12.81% (7,371,936B) 0xF9D5D63: std::basic_streambuf >::xsputn(char const*, long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->12.81% (7,370,862B) 0xF9CD314: std::basic_ostream >& std::__ostream_insert >(std::basic_ostream >&, char const*, long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | | ->12.78% (7,352,067B) 0xF9CD6E5: std::basic_ostream >& std::operator<< >(std::basic_ostream >&, char const*) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | | | ->12.78% (7,352,067B) 0x5109AE4: RGWOmapAppend::operate() (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->12.78% (7,352,067B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->12.78% (7,352,067B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->12.78% (7,352,067B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->12.78% (7,352,067B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->12.78% (7,352,067B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->12.78% (7,352,067B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->12.78% (7,352,067B) 0xF1A7492: start_thread (pthread_create.c:333) | | | | | | | ->12.78% (7,352,067B) 0xFF3F93D: clone (clone.S:97) | | | | | | | | | | | | | ->00.00% (0B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | | | | | ->00.03% (18,795B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | | | ->00.00% (1,074B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | ->03.77% (2,169,993B) 0xF9F5E38: ??? (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->03.77% (2,169,993B) 0xF9F5F0C: std::string::append(char const*, unsigned long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->03.69% (2,124,613B) 0x51DF1E4: RGWDataSyncShardControlCR::alloc_cr() (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.69% (2,124,613B) 0x51AB72B: RGWBackoffControlCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.69% (2,124,613B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.69% (2,124,613B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.69% (2,124,613B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.69% (2,124,613B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.69% (2,124,613B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.69% (2,124,613B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.69% (2,124,613B) 0xF1A7492: start_thread (pthread_create.c:333) | | | | | ->03.69% (2,124,613B) 0xFF3F93D: clone (clone.S:97) | | | | | | | | | ->00.08% (45,380B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | ->00.16% (92,453B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | ->00.00% (2,068B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->01.37% (790,039B) 0xF9F6813: char* std::string::_S_construct(char const*, char const*, std::allocator const&, std::forward_iterator_tag) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | ->01.37% (790,039B) 0xF9F6C44: std::basic_string, std::allocator >::basic_string(char const*, std::allocator const&) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | ->01.23% (705,024B) 0x510DA32: RGWCoroutine::RGWCoroutine(CephContext*) (in /usr/lib/librgw.so.2.0.0) | | | | ->01.21% (698,241B) 0x51075C3: RGWOmapAppend::RGWOmapAppend(RGWAsyncRadosProcessor*, RGWRados*, rgw_bucket&, std::string const&, unsigned long) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.21% (698,241B) 0x51E7CB7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.21% (698,241B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.21% (698,241B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.21% (698,241B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.21% (698,241B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.21% (698,241B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.21% (698,241B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.21% (698,241B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.21% (698,241B) 0xF1A7492: start_thread (pthread_create.c:333) | | | | | ->01.21% (698,241B) 0xFF3F93D: clone (clone.S:97) | | | | | | | | | ->00.01% (6,783B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | ->00.15% (85,015B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | ->00.00% (0B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->00.13% (72,092B) in 1+ places, all below ms_print's threshold (01.00%) | ->12.30% (7,077,376B) 0x510DB36: RGWCoroutine::RGWCoroutine(CephContext*) (in /usr/lib/librgw.so.2.0.0) | ->12.18% (7,009,280B) 0x51075C3: RGWOmapAppend::RGWOmapAppend(RGWAsyncRadosProcessor*, RGWRados*, rgw_bucket&, std::string const&, unsigned long) (in /usr/lib/librgw.so.2.0.0) | | ->12.18% (7,009,280B) 0x51E7CB7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | ->12.18% (7,009,280B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | ->12.18% (7,009,280B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | ->12.18% (7,009,280B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | ->12.18% (7,009,280B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | ->12.18% (7,009,280B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | ->12.18% (7,009,280B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | ->12.18% (7,009,280B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | ->12.18% (7,009,280B) 0xF1A7492: start_thread (pthread_create.c:333) | | ->12.18% (7,009,280B) 0xFF3F93D: clone (clone.S:97) | | | ->00.12% (68,096B) in 1+ places, all below ms_print's threshold (01.00%) | ->10.73% (6,172,096B) 0x50F9E51: RGWCoroutinesManager::allocate_stack() (in /usr/lib/librgw.so.2.0.0) | ->10.73% (6,171,872B) 0x50FE7AE: RGWCoroutinesStack::spawn(RGWCoroutine*, RGWCoroutine*, bool) (in /usr/lib/librgw.so.2.0.0) | | ->05.33% (3,066,784B) 0x51E7CD7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | | ->05.33% (3,066,784B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | ->05.33% (3,066,784B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.33% (3,066,784B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.33% (3,066,784B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.33% (3,066,784B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.33% (3,066,784B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | ->05.33% (3,066,784B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | ->05.33% (3,066,784B) 0xF1A7492: start_thread (pthread_create.c:333) | | | ->05.33% (3,066,784B) 0xFF3F93D: clone (clone.S:97) | | | | | ->05.33% (3,066,784B) 0x51D5E5F: RGWDataSyncShardCR::init_lease_cr() (in /usr/lib/librgw.so.2.0.0) | | | ->05.33% (3,066,784B) 0x51E7CEE: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | | ->05.33% (3,066,784B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | ->05.33% (3,066,784B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.33% (3,066,784B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.33% (3,066,784B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.33% (3,066,784B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.33% (3,066,784B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | ->05.33% (3,066,784B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | ->05.33% (3,066,784B) 0xF1A7492: start_thread (pthread_create.c:333) | | | ->05.33% (3,066,784B) 0xFF3F93D: clone (clone.S:97) | | | | | ->00.07% (38,304B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->00.00% (224B) in 1+ places, all below ms_print's threshold (01.00%) | ->05.82% (3,346,913B) in 827 places, all below massif's threshold (1.00%) | ->03.08% (1,772,000B) 0x131429: worker_thread (civetweb.c:502) | ->03.08% (1,772,000B) 0xF1A7492: start_thread (pthread_create.c:333) | ->03.08% (1,772,000B) 0xFF3F93D: clone (clone.S:97) | ->02.72% (1,565,690B) 0x10623B1B: ??? (in /usr/lib/librados.so.2.0.0) | ->01.63% (935,835B) 0x1051E944: ??? (in /usr/lib/librados.so.2.0.0) | | ->01.63% (935,835B) 0x1051F207: ??? (in /usr/lib/librados.so.2.0.0) | | | ->01.63% (935,835B) 0x105393B8: ??? (in /usr/lib/librados.so.2.0.0) | | | ->01.63% (935,835B) 0x10541BBB: ??? (in /usr/lib/librados.so.2.0.0) | | | ->01.63% (935,835B) 0xF1A7492: start_thread (pthread_create.c:333) | | | ->01.63% (935,835B) 0xFF3F93D: clone (clone.S:97) | | | | | ->00.00% (0B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->01.09% (627,540B) 0x105486F9: ??? (in /usr/lib/librados.so.2.0.0) | | ->01.07% (617,849B) 0x102C6DDA: ??? (in /usr/lib/librados.so.2.0.0) | | | ->01.04% (599,634B) 0x102CBD9E: ??? (in /usr/lib/librados.so.2.0.0) | | | | ->01.04% (599,634B) 0x102CDDBD: ??? (in /usr/lib/librados.so.2.0.0) | | | | | ->01.04% (599,634B) 0x102CE0DB: ??? (in /usr/lib/librados.so.2.0.0) | | | | | | ->01.04% (599,634B) in 6 places, all below massif's threshold (1.00%) | | | | | | | | | | | ->00.00% (0B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | | | ->00.00% (0B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | ->00.03% (18,215B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | ->00.02% (9,691B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->00.00% (2,315B) in 1+ places, all below ms_print's threshold (01.00%) | ->02.66% (1,529,347B) 0x54F757B: ceph::log::Log::create_entry(int, int, unsigned long*) (in /usr/lib/librgw.so.2.0.0) | ->01.66% (953,856B) 0x52CD4D0: RGWRequest::log(req_state*, char const*) (in /usr/lib/librgw.so.2.0.0) | | ->01.66% (953,856B) in 13 places, all below massif's threshold (1.00%) | | | ->01.00% (575,491B) in 42 places, all below massif's threshold (1.00%) | ->01.54% (884,736B) 0x510DB0E: RGWCoroutine::RGWCoroutine(CephContext*) (in /usr/lib/librgw.so.2.0.0) | ->01.52% (876,224B) 0x51075C3: RGWOmapAppend::RGWOmapAppend(RGWAsyncRadosProcessor*, RGWRados*, rgw_bucket&, std::string const&, unsigned long) (in /usr/lib/librgw.so.2.0.0) | | ->01.52% (876,224B) 0x51E7CB7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | ->01.52% (876,224B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | ->01.52% (876,224B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | ->01.52% (876,224B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | ->01.52% (876,224B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | ->01.52% (876,224B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | ->01.52% (876,224B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | ->01.52% (876,224B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | ->01.52% (876,224B) 0xF1A7492: start_thread (pthread_create.c:333) | | ->01.52% (876,224B) 0xFF3F93D: clone (clone.S:97) | | | ->00.01% (8,512B) in 1+ places, all below ms_print's threshold (01.00%) | ->01.01% (581,632B) 0x5614937: ceph::buffer::create_aligned(unsigned int, unsigned int) (in /usr/lib/librgw.so.2.0.0) ->01.01% (581,632B) 0x1052B4D8: ??? (in /usr/lib/librados.so.2.0.0) | ->01.01% (581,632B) 0x105381E2: ??? (in /usr/lib/librados.so.2.0.0) | ->01.01% (581,632B) 0x10541BBB: ??? (in /usr/lib/librados.so.2.0.0) | ->01.01% (581,632B) 0xF1A7492: start_thread (pthread_create.c:333) | ->01.01% (581,632B) 0xFF3F93D: clone (clone.S:97) | ->00.00% (0B) in 1+ places, all below ms_print's threshold (01.00%) -------------------------------------------------------------------------------- n time(i) total(B) useful-heap(B) extra-heap(B) stacks(B) -------------------------------------------------------------------------------- 17 31,749,785,773 63,717,720 61,036,724 2,680,996 0 95.79% (61,036,724B) (heap allocation functions) malloc/new/new[], --alloc-fns, etc. ->38.66% (24,634,368B) 0x51E7C89: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | ->38.66% (24,634,368B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | ->38.66% (24,634,368B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | ->38.66% (24,634,368B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | ->38.66% (24,634,368B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | ->38.66% (24,634,368B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | ->38.66% (24,634,368B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | ->38.66% (24,634,368B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | ->38.66% (24,634,368B) 0xF1A7492: start_thread (pthread_create.c:333) | ->38.66% (24,634,368B) 0xFF3F93D: clone (clone.S:97) | ->18.60% (11,848,932B) 0xF9F4E97: std::string::_Rep::_S_create(unsigned long, unsigned long, std::allocator const&) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | ->17.11% (10,902,490B) 0xF9F5B09: std::string::_Rep::_M_clone(std::allocator const&, unsigned long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | ->17.11% (10,901,401B) 0xF9F5BAE: std::string::reserve(unsigned long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | ->13.14% (8,371,830B) 0xF9D1FB2: std::basic_stringbuf, std::allocator >::overflow(int) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->13.14% (8,371,830B) 0xF9D5D63: std::basic_streambuf >::xsputn(char const*, long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->13.14% (8,371,293B) 0xF9CD314: std::basic_ostream >& std::__ostream_insert >(std::basic_ostream >&, char const*, long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | | ->13.11% (8,351,424B) 0xF9CD6E5: std::basic_ostream >& std::operator<< >(std::basic_ostream >&, char const*) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | | | ->13.11% (8,351,424B) 0x5109AE4: RGWOmapAppend::operate() (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->13.11% (8,351,424B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->13.11% (8,351,424B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->13.11% (8,351,424B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->13.11% (8,351,424B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->13.11% (8,351,424B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->13.11% (8,351,424B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->13.11% (8,351,424B) 0xF1A7492: start_thread (pthread_create.c:333) | | | | | | | ->13.11% (8,351,424B) 0xFF3F93D: clone (clone.S:97) | | | | | | | | | | | | | ->00.00% (0B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | | | | | ->00.03% (19,869B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | | | ->00.00% (537B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | ->03.86% (2,456,391B) 0xF9F5E38: ??? (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->03.86% (2,456,391B) 0xF9F5F0C: std::string::append(char const*, unsigned long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->03.79% (2,414,185B) 0x51DF1E4: RGWDataSyncShardControlCR::alloc_cr() (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.79% (2,414,185B) 0x51AB72B: RGWBackoffControlCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.79% (2,414,185B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.79% (2,414,185B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.79% (2,414,185B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.79% (2,414,185B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.79% (2,414,185B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.79% (2,414,185B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.79% (2,414,185B) 0xF1A7492: start_thread (pthread_create.c:333) | | | | | ->03.79% (2,414,185B) 0xFF3F93D: clone (clone.S:97) | | | | | | | | | ->00.07% (42,206B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | ->00.11% (73,180B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | ->00.00% (1,089B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->01.39% (882,772B) 0xF9F6813: char* std::string::_S_construct(char const*, char const*, std::allocator const&, std::forward_iterator_tag) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | ->01.39% (882,772B) 0xF9F6C44: std::basic_string, std::allocator >::basic_string(char const*, std::allocator const&) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | ->01.26% (800,190B) 0x510DA32: RGWCoroutine::RGWCoroutine(CephContext*) (in /usr/lib/librgw.so.2.0.0) | | | | ->01.24% (793,152B) 0x51075C3: RGWOmapAppend::RGWOmapAppend(RGWAsyncRadosProcessor*, RGWRados*, rgw_bucket&, std::string const&, unsigned long) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.24% (793,152B) 0x51E7CB7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.24% (793,152B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.24% (793,152B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.24% (793,152B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.24% (793,152B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.24% (793,152B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.24% (793,152B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.24% (793,152B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.24% (793,152B) 0xF1A7492: start_thread (pthread_create.c:333) | | | | | ->01.24% (793,152B) 0xFF3F93D: clone (clone.S:97) | | | | | | | | | ->00.01% (7,038B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | ->00.13% (82,582B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | ->00.00% (0B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->00.10% (63,670B) in 1+ places, all below ms_print's threshold (01.00%) | ->12.61% (8,032,768B) 0x510DB36: RGWCoroutine::RGWCoroutine(CephContext*) (in /usr/lib/librgw.so.2.0.0) | ->12.50% (7,962,112B) 0x51075C3: RGWOmapAppend::RGWOmapAppend(RGWAsyncRadosProcessor*, RGWRados*, rgw_bucket&, std::string const&, unsigned long) (in /usr/lib/librgw.so.2.0.0) | | ->12.50% (7,962,112B) 0x51E7CB7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | ->12.50% (7,962,112B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | ->12.50% (7,962,112B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | ->12.50% (7,962,112B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | ->12.50% (7,962,112B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | ->12.50% (7,962,112B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | ->12.50% (7,962,112B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | ->12.50% (7,962,112B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | ->12.50% (7,962,112B) 0xF1A7492: start_thread (pthread_create.c:333) | | ->12.50% (7,962,112B) 0xFF3F93D: clone (clone.S:97) | | | ->00.11% (70,656B) in 1+ places, all below ms_print's threshold (01.00%) | ->11.00% (7,005,824B) 0x50F9E51: RGWCoroutinesManager::allocate_stack() (in /usr/lib/librgw.so.2.0.0) | ->10.99% (7,005,600B) 0x50FE7AE: RGWCoroutinesStack::spawn(RGWCoroutine*, RGWCoroutine*, bool) (in /usr/lib/librgw.so.2.0.0) | | ->05.47% (3,483,648B) 0x51E7CD7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | | ->05.47% (3,483,648B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | ->05.47% (3,483,648B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.47% (3,483,648B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.47% (3,483,648B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.47% (3,483,648B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.47% (3,483,648B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | ->05.47% (3,483,648B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | ->05.47% (3,483,648B) 0xF1A7492: start_thread (pthread_create.c:333) | | | ->05.47% (3,483,648B) 0xFF3F93D: clone (clone.S:97) | | | | | ->05.47% (3,483,648B) 0x51D5E5F: RGWDataSyncShardCR::init_lease_cr() (in /usr/lib/librgw.so.2.0.0) | | | ->05.47% (3,483,648B) 0x51E7CEE: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | | ->05.47% (3,483,648B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | ->05.47% (3,483,648B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.47% (3,483,648B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.47% (3,483,648B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.47% (3,483,648B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.47% (3,483,648B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | ->05.47% (3,483,648B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | ->05.47% (3,483,648B) 0xF1A7492: start_thread (pthread_create.c:333) | | | ->05.47% (3,483,648B) 0xFF3F93D: clone (clone.S:97) | | | | | ->00.06% (38,304B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->00.00% (224B) in 1+ places, all below ms_print's threshold (01.00%) | ->05.79% (3,690,450B) in 829 places, all below massif's threshold (1.00%) | ->02.78% (1,772,000B) 0x131429: worker_thread (civetweb.c:502) | ->02.78% (1,772,000B) 0xF1A7492: start_thread (pthread_create.c:333) | ->02.78% (1,772,000B) 0xFF3F93D: clone (clone.S:97) | ->02.66% (1,696,888B) 0x54F757B: ceph::log::Log::create_entry(int, int, unsigned long*) (in /usr/lib/librgw.so.2.0.0) | ->01.70% (1,083,343B) 0x52CD4D0: RGWRequest::log(req_state*, char const*) (in /usr/lib/librgw.so.2.0.0) | | ->01.70% (1,083,343B) in 13 places, all below massif's threshold (1.00%) | | | ->00.96% (613,545B) in 1+ places, all below ms_print's threshold (01.00%) | ->02.12% (1,351,334B) 0x10623B1B: ??? (in /usr/lib/librados.so.2.0.0) | ->01.23% (782,290B) 0x1051E944: ??? (in /usr/lib/librados.so.2.0.0) | | ->01.22% (779,353B) 0x1051F207: ??? (in /usr/lib/librados.so.2.0.0) | | | ->01.22% (779,353B) 0x105393B8: ??? (in /usr/lib/librados.so.2.0.0) | | | ->01.22% (779,353B) 0x10541BBB: ??? (in /usr/lib/librados.so.2.0.0) | | | ->01.22% (779,353B) 0xF1A7492: start_thread (pthread_create.c:333) | | | ->01.22% (779,353B) 0xFF3F93D: clone (clone.S:97) | | | | | ->00.00% (2,937B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->00.89% (569,044B) in 1+ places, all below ms_print's threshold (01.00%) | ->01.58% (1,004,160B) 0x510DB0E: RGWCoroutine::RGWCoroutine(CephContext*) (in /usr/lib/librgw.so.2.0.0) ->01.56% (995,328B) 0x51075C3: RGWOmapAppend::RGWOmapAppend(RGWAsyncRadosProcessor*, RGWRados*, rgw_bucket&, std::string const&, unsigned long) (in /usr/lib/librgw.so.2.0.0) | ->01.56% (995,328B) 0x51E7CB7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | ->01.56% (995,328B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | ->01.56% (995,328B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | ->01.56% (995,328B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | ->01.56% (995,328B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | ->01.56% (995,328B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | ->01.56% (995,328B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | ->01.56% (995,328B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | ->01.56% (995,328B) 0xF1A7492: start_thread (pthread_create.c:333) | ->01.56% (995,328B) 0xFF3F93D: clone (clone.S:97) | ->00.01% (8,832B) in 1+ places, all below ms_print's threshold (01.00%) -------------------------------------------------------------------------------- n time(i) total(B) useful-heap(B) extra-heap(B) stacks(B) -------------------------------------------------------------------------------- 18 33,759,897,793 72,713,520 69,644,776 3,068,744 0 95.78% (69,644,776B) (heap allocation functions) malloc/new/new[], --alloc-fns, etc. ->39.20% (28,504,080B) 0x51E7C89: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | ->39.20% (28,504,080B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | ->39.20% (28,504,080B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | ->39.20% (28,504,080B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | ->39.20% (28,504,080B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | ->39.20% (28,504,080B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | ->39.20% (28,504,080B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | ->39.20% (28,504,080B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | ->39.20% (28,504,080B) 0xF1A7492: start_thread (pthread_create.c:333) | ->39.20% (28,504,080B) 0xFF3F93D: clone (clone.S:97) | ->18.94% (13,771,421B) 0xF9F4E97: std::string::_Rep::_S_create(unsigned long, unsigned long, std::allocator const&) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | ->17.47% (12,703,377B) 0xF9F5B09: std::string::_Rep::_M_clone(std::allocator const&, unsigned long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | ->17.47% (12,703,282B) 0xF9F5BAE: std::string::reserve(unsigned long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | ->13.32% (9,682,110B) 0xF9D1FB2: std::basic_stringbuf, std::allocator >::overflow(int) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->13.32% (9,682,110B) 0xF9D5D63: std::basic_streambuf >::xsputn(char const*, long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->13.31% (9,681,573B) 0xF9CD314: std::basic_ostream >& std::__ostream_insert >(std::basic_ostream >&, char const*, long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | | ->13.29% (9,663,315B) 0xF9CD6E5: std::basic_ostream >& std::operator<< >(std::basic_ostream >&, char const*) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | | | ->13.29% (9,663,315B) 0x5109AE4: RGWOmapAppend::operate() (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->13.29% (9,663,315B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->13.29% (9,663,315B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->13.29% (9,663,315B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->13.29% (9,663,315B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->13.29% (9,663,315B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->13.29% (9,663,315B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->13.29% (9,663,315B) 0xF1A7492: start_thread (pthread_create.c:333) | | | | | | | ->13.29% (9,663,315B) 0xFF3F93D: clone (clone.S:97) | | | | | | | | | | | | | ->00.00% (0B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | | | | | ->00.03% (18,258B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | | | ->00.00% (537B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | ->04.05% (2,945,911B) 0xF9F5E38: ??? (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->04.05% (2,945,911B) 0xF9F5F0C: std::string::append(char const*, unsigned long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->03.84% (2,794,213B) 0x51DF1E4: RGWDataSyncShardControlCR::alloc_cr() (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.84% (2,794,213B) 0x51AB72B: RGWBackoffControlCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.84% (2,794,213B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.84% (2,794,213B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.84% (2,794,213B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.84% (2,794,213B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.84% (2,794,213B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.84% (2,794,213B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.84% (2,794,213B) 0xF1A7492: start_thread (pthread_create.c:333) | | | | | ->03.84% (2,794,213B) 0xFF3F93D: clone (clone.S:97) | | | | | | | | | ->00.21% (151,698B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | ->00.10% (75,261B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | ->00.00% (95B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->01.39% (1,007,198B) 0xF9F6813: char* std::string::_S_construct(char const*, char const*, std::allocator const&, std::forward_iterator_tag) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | ->01.39% (1,007,198B) 0xF9F6C44: std::basic_string, std::allocator >::basic_string(char const*, std::allocator const&) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | ->01.27% (924,528B) 0x510DA32: RGWCoroutine::RGWCoroutine(CephContext*) (in /usr/lib/librgw.so.2.0.0) | | | | ->01.26% (917,745B) 0x51075C3: RGWOmapAppend::RGWOmapAppend(RGWAsyncRadosProcessor*, RGWRados*, rgw_bucket&, std::string const&, unsigned long) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.26% (917,745B) 0x51E7CB7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.26% (917,745B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.26% (917,745B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.26% (917,745B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.26% (917,745B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.26% (917,745B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.26% (917,745B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.26% (917,745B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.26% (917,745B) 0xF1A7492: start_thread (pthread_create.c:333) | | | | | ->01.26% (917,745B) 0xFF3F93D: clone (clone.S:97) | | | | | | | | | ->00.01% (6,783B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | ->00.11% (82,670B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | ->00.00% (0B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->00.08% (60,846B) in 1+ places, all below ms_print's threshold (01.00%) | ->12.76% (9,281,024B) 0x510DB36: RGWCoroutine::RGWCoroutine(CephContext*) (in /usr/lib/librgw.so.2.0.0) | ->12.67% (9,212,928B) 0x51075C3: RGWOmapAppend::RGWOmapAppend(RGWAsyncRadosProcessor*, RGWRados*, rgw_bucket&, std::string const&, unsigned long) (in /usr/lib/librgw.so.2.0.0) | | ->12.67% (9,212,928B) 0x51E7CB7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | ->12.67% (9,212,928B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | ->12.67% (9,212,928B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | ->12.67% (9,212,928B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | ->12.67% (9,212,928B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | ->12.67% (9,212,928B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | ->12.67% (9,212,928B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | ->12.67% (9,212,928B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | ->12.67% (9,212,928B) 0xF1A7492: start_thread (pthread_create.c:333) | | ->12.67% (9,212,928B) 0xFF3F93D: clone (clone.S:97) | | | ->00.09% (68,096B) in 1+ places, all below ms_print's threshold (01.00%) | ->11.14% (8,100,288B) 0x50F9E51: RGWCoroutinesManager::allocate_stack() (in /usr/lib/librgw.so.2.0.0) | ->11.14% (8,100,064B) 0x50FE7AE: RGWCoroutinesStack::spawn(RGWCoroutine*, RGWCoroutine*, bool) (in /usr/lib/librgw.so.2.0.0) | | ->05.54% (4,030,880B) 0x51E7CD7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | | ->05.54% (4,030,880B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | ->05.54% (4,030,880B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.54% (4,030,880B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.54% (4,030,880B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.54% (4,030,880B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.54% (4,030,880B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | ->05.54% (4,030,880B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | ->05.54% (4,030,880B) 0xF1A7492: start_thread (pthread_create.c:333) | | | ->05.54% (4,030,880B) 0xFF3F93D: clone (clone.S:97) | | | | | ->05.54% (4,030,880B) 0x51D5E5F: RGWDataSyncShardCR::init_lease_cr() (in /usr/lib/librgw.so.2.0.0) | | | ->05.54% (4,030,880B) 0x51E7CEE: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | | ->05.54% (4,030,880B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | ->05.54% (4,030,880B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.54% (4,030,880B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.54% (4,030,880B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.54% (4,030,880B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.54% (4,030,880B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | ->05.54% (4,030,880B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | ->05.54% (4,030,880B) 0xF1A7492: start_thread (pthread_create.c:333) | | | ->05.54% (4,030,880B) 0xFF3F93D: clone (clone.S:97) | | | | | ->00.05% (38,304B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->00.00% (224B) in 1+ places, all below ms_print's threshold (01.00%) | ->04.47% (3,253,342B) in 828 places, all below massif's threshold (1.00%) | ->02.44% (1,772,000B) 0x131429: worker_thread (civetweb.c:502) | ->02.44% (1,772,000B) 0xF1A7492: start_thread (pthread_create.c:333) | ->02.44% (1,772,000B) 0xFF3F93D: clone (clone.S:97) | ->02.21% (1,607,241B) 0x10623B1B: ??? (in /usr/lib/librados.so.2.0.0) | ->01.26% (917,768B) 0x1051E944: ??? (in /usr/lib/librados.so.2.0.0) | | ->01.26% (917,768B) 0x1051F207: ??? (in /usr/lib/librados.so.2.0.0) | | | ->01.26% (917,768B) 0x105393B8: ??? (in /usr/lib/librados.so.2.0.0) | | | ->01.26% (917,768B) 0x10541BBB: ??? (in /usr/lib/librados.so.2.0.0) | | | ->01.26% (917,768B) 0xF1A7492: start_thread (pthread_create.c:333) | | | ->01.26% (917,768B) 0xFF3F93D: clone (clone.S:97) | | | | | ->00.00% (0B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->00.95% (689,473B) in 1+ places, all below ms_print's threshold (01.00%) | ->02.01% (1,465,068B) 0x54F757B: ceph::log::Log::create_entry(int, int, unsigned long*) (in /usr/lib/librgw.so.2.0.0) | ->01.31% (951,955B) 0x52CD4D0: RGWRequest::log(req_state*, char const*) (in /usr/lib/librgw.so.2.0.0) | | ->01.31% (951,955B) in 13 places, all below massif's threshold (1.00%) | | | ->00.71% (513,113B) in 1+ places, all below ms_print's threshold (01.00%) | ->01.60% (1,160,192B) 0x510DB0E: RGWCoroutine::RGWCoroutine(CephContext*) (in /usr/lib/librgw.so.2.0.0) | ->01.58% (1,151,680B) 0x51075C3: RGWOmapAppend::RGWOmapAppend(RGWAsyncRadosProcessor*, RGWRados*, rgw_bucket&, std::string const&, unsigned long) (in /usr/lib/librgw.so.2.0.0) | | ->01.58% (1,151,680B) 0x51E7CB7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | ->01.58% (1,151,680B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | ->01.58% (1,151,680B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | ->01.58% (1,151,680B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | ->01.58% (1,151,680B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | ->01.58% (1,151,680B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | ->01.58% (1,151,680B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | ->01.58% (1,151,680B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | ->01.58% (1,151,680B) 0xF1A7492: start_thread (pthread_create.c:333) | | ->01.58% (1,151,680B) 0xFF3F93D: clone (clone.S:97) | | | ->00.01% (8,512B) in 1+ places, all below ms_print's threshold (01.00%) | ->01.00% (730,120B) 0x5103628: std::pair, bool> std::_Rb_tree, std::less, std::allocator >::_M_insert_unique(RGWCoroutinesStack* const&) (in /usr/lib/librgw.so.2.0.0) ->01.00% (730,120B) in 3 places, all below massif's threshold (1.00%) -------------------------------------------------------------------------------- n time(i) total(B) useful-heap(B) extra-heap(B) stacks(B) -------------------------------------------------------------------------------- 19 34,779,264,854 78,175,720 74,852,335 3,323,385 0 95.75% (74,852,335B) (heap allocation functions) malloc/new/new[], --alloc-fns, etc. ->38.15% (29,826,720B) 0x51E7C89: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | ->38.15% (29,826,720B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | ->38.15% (29,826,720B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | ->38.15% (29,826,720B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | ->38.15% (29,826,720B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | ->38.15% (29,826,720B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | ->38.15% (29,826,720B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | ->38.15% (29,826,720B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | ->38.15% (29,826,720B) 0xF1A7492: start_thread (pthread_create.c:333) | ->38.15% (29,826,720B) 0xFF3F93D: clone (clone.S:97) | ->18.88% (14,761,983B) 0xF9F4E97: std::string::_Rep::_S_create(unsigned long, unsigned long, std::allocator const&) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | ->17.19% (13,439,060B) 0xF9F5B09: std::string::_Rep::_M_clone(std::allocator const&, unsigned long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | ->17.17% (13,420,395B) 0xF9F5BAE: std::string::reserve(unsigned long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | ->13.07% (10,219,752B) 0xF9D1FB2: std::basic_stringbuf, std::allocator >::overflow(int) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->13.07% (10,219,752B) 0xF9D5D63: std::basic_streambuf >::xsputn(char const*, long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->13.07% (10,218,141B) 0xF9CD314: std::basic_ostream >& std::__ostream_insert >(std::basic_ostream >&, char const*, long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | | ->12.93% (10,111,173B) 0xF9CD6E5: std::basic_ostream >& std::operator<< >(std::basic_ostream >&, char const*) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | | | ->12.93% (10,111,173B) 0x5109AE4: RGWOmapAppend::operate() (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->12.93% (10,111,173B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->12.93% (10,111,173B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->12.93% (10,111,173B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->12.93% (10,111,173B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->12.93% (10,111,173B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->12.93% (10,111,173B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->12.93% (10,111,173B) 0xF1A7492: start_thread (pthread_create.c:333) | | | | | | | ->12.93% (10,111,173B) 0xFF3F93D: clone (clone.S:97) | | | | | | | | | | | | | ->00.00% (0B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | | | | | ->00.14% (106,968B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | | | ->00.00% (1,611B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | ->03.85% (3,011,203B) 0xF9F5E38: ??? (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->03.85% (3,011,203B) 0xF9F5F0C: std::string::append(char const*, unsigned long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->03.74% (2,924,946B) 0x51DF1E4: RGWDataSyncShardControlCR::alloc_cr() (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.74% (2,924,946B) 0x51AB72B: RGWBackoffControlCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.74% (2,924,946B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.74% (2,924,946B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.74% (2,924,946B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.74% (2,924,946B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.74% (2,924,946B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.74% (2,924,946B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.74% (2,924,946B) 0xF1A7492: start_thread (pthread_create.c:333) | | | | | ->03.74% (2,924,946B) 0xFF3F93D: clone (clone.S:97) | | | | | | | | | ->00.11% (86,257B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | ->00.24% (189,440B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | ->00.02% (18,665B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->01.48% (1,158,068B) 0xF9F6813: char* std::string::_S_construct(char const*, char const*, std::allocator const&, std::forward_iterator_tag) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | ->01.48% (1,158,068B) 0xF9F6C44: std::basic_string, std::allocator >::basic_string(char const*, std::allocator const&) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | ->01.25% (976,650B) 0x510DA32: RGWCoroutine::RGWCoroutine(CephContext*) (in /usr/lib/librgw.so.2.0.0) | | | | ->01.23% (960,330B) 0x51075C3: RGWOmapAppend::RGWOmapAppend(RGWAsyncRadosProcessor*, RGWRados*, rgw_bucket&, std::string const&, unsigned long) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.23% (960,330B) 0x51E7CB7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.23% (960,330B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.23% (960,330B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.23% (960,330B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.23% (960,330B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.23% (960,330B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.23% (960,330B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.23% (960,330B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.23% (960,330B) 0xF1A7492: start_thread (pthread_create.c:333) | | | | | ->01.23% (960,330B) 0xFF3F93D: clone (clone.S:97) | | | | | | | | | ->00.02% (16,320B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | ->00.23% (181,418B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | ->00.00% (0B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->00.21% (164,855B) in 1+ places, all below ms_print's threshold (01.00%) | ->12.54% (9,804,288B) 0x510DB36: RGWCoroutine::RGWCoroutine(CephContext*) (in /usr/lib/librgw.so.2.0.0) | ->12.33% (9,640,448B) 0x51075C3: RGWOmapAppend::RGWOmapAppend(RGWAsyncRadosProcessor*, RGWRados*, rgw_bucket&, std::string const&, unsigned long) (in /usr/lib/librgw.so.2.0.0) | | ->12.33% (9,640,448B) 0x51E7CB7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | ->12.33% (9,640,448B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | ->12.33% (9,640,448B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | ->12.33% (9,640,448B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | ->12.33% (9,640,448B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | ->12.33% (9,640,448B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | ->12.33% (9,640,448B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | ->12.33% (9,640,448B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | ->12.33% (9,640,448B) 0xF1A7492: start_thread (pthread_create.c:333) | | ->12.33% (9,640,448B) 0xFF3F93D: clone (clone.S:97) | | | ->00.21% (163,840B) in 1+ places, all below ms_print's threshold (01.00%) | ->10.84% (8,474,368B) 0x50F9E51: RGWCoroutinesManager::allocate_stack() (in /usr/lib/librgw.so.2.0.0) | ->10.84% (8,474,144B) 0x50FE7AE: RGWCoroutinesStack::spawn(RGWCoroutine*, RGWCoroutine*, bool) (in /usr/lib/librgw.so.2.0.0) | | ->05.40% (4,217,920B) 0x51E7CD7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | | ->05.40% (4,217,920B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | ->05.40% (4,217,920B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.40% (4,217,920B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.40% (4,217,920B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.40% (4,217,920B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.40% (4,217,920B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | ->05.40% (4,217,920B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | ->05.40% (4,217,920B) 0xF1A7492: start_thread (pthread_create.c:333) | | | ->05.40% (4,217,920B) 0xFF3F93D: clone (clone.S:97) | | | | | ->05.40% (4,217,920B) 0x51D5E5F: RGWDataSyncShardCR::init_lease_cr() (in /usr/lib/librgw.so.2.0.0) | | | ->05.40% (4,217,920B) 0x51E7CEE: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | | ->05.40% (4,217,920B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | ->05.40% (4,217,920B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.40% (4,217,920B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.40% (4,217,920B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.40% (4,217,920B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.40% (4,217,920B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | ->05.40% (4,217,920B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | ->05.40% (4,217,920B) 0xF1A7492: start_thread (pthread_create.c:333) | | | ->05.40% (4,217,920B) 0xFF3F93D: clone (clone.S:97) | | | | | ->00.05% (38,304B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->00.00% (224B) in 1+ places, all below ms_print's threshold (01.00%) | ->06.57% (5,137,874B) in 830 places, all below massif's threshold (1.00%) | ->02.53% (1,980,865B) 0x54F757B: ceph::log::Log::create_entry(int, int, unsigned long*) (in /usr/lib/librgw.so.2.0.0) | ->01.60% (1,252,251B) 0x52CD4D0: RGWRequest::log(req_state*, char const*) (in /usr/lib/librgw.so.2.0.0) | | ->01.60% (1,252,251B) in 13 places, all below massif's threshold (1.00%) | | | ->00.93% (728,614B) in 1+ places, all below ms_print's threshold (01.00%) | ->02.27% (1,772,000B) 0x131429: worker_thread (civetweb.c:502) | ->02.27% (1,772,000B) 0xF1A7492: start_thread (pthread_create.c:333) | ->02.27% (1,772,000B) 0xFF3F93D: clone (clone.S:97) | ->01.57% (1,225,600B) 0x510DB0E: RGWCoroutine::RGWCoroutine(CephContext*) (in /usr/lib/librgw.so.2.0.0) | ->01.54% (1,205,120B) 0x51075C3: RGWOmapAppend::RGWOmapAppend(RGWAsyncRadosProcessor*, RGWRados*, rgw_bucket&, std::string const&, unsigned long) (in /usr/lib/librgw.so.2.0.0) | | ->01.54% (1,205,120B) 0x51E7CB7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | ->01.54% (1,205,120B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | ->01.54% (1,205,120B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | ->01.54% (1,205,120B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | ->01.54% (1,205,120B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | ->01.54% (1,205,120B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | ->01.54% (1,205,120B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | ->01.54% (1,205,120B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | ->01.54% (1,205,120B) 0xF1A7492: start_thread (pthread_create.c:333) | | ->01.54% (1,205,120B) 0xFF3F93D: clone (clone.S:97) | | | ->00.03% (20,480B) in 1+ places, all below ms_print's threshold (01.00%) | ->01.36% (1,060,864B) 0x5614937: ceph::buffer::create_aligned(unsigned int, unsigned int) (in /usr/lib/librgw.so.2.0.0) | ->01.36% (1,060,864B) 0x1052B4D8: ??? (in /usr/lib/librados.so.2.0.0) | | ->01.36% (1,060,864B) 0x105381E2: ??? (in /usr/lib/librados.so.2.0.0) | | ->01.36% (1,060,864B) 0x10541BBB: ??? (in /usr/lib/librados.so.2.0.0) | | ->01.36% (1,060,864B) 0xF1A7492: start_thread (pthread_create.c:333) | | ->01.36% (1,060,864B) 0xFF3F93D: clone (clone.S:97) | | | ->00.00% (0B) in 1+ places, all below ms_print's threshold (01.00%) | ->01.03% (807,773B) 0x10623B1B: ??? (in /usr/lib/librados.so.2.0.0) ->01.03% (807,773B) in 42 places, all below massif's threshold (1.00%) -------------------------------------------------------------------------------- n time(i) total(B) useful-heap(B) extra-heap(B) stacks(B) -------------------------------------------------------------------------------- 20 36,860,324,378 81,436,568 77,984,276 3,452,292 0 95.76% (77,984,276B) (heap allocation functions) malloc/new/new[], --alloc-fns, etc. ->38.52% (31,367,952B) 0x51E7C89: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | ->38.52% (31,367,952B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | ->38.52% (31,367,952B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | ->38.52% (31,367,952B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | ->38.52% (31,367,952B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | ->38.52% (31,367,952B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | ->38.52% (31,367,952B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | ->38.52% (31,367,952B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | ->38.52% (31,367,952B) 0xF1A7492: start_thread (pthread_create.c:333) | ->38.52% (31,367,952B) 0xFF3F93D: clone (clone.S:97) | ->18.92% (15,406,212B) 0xF9F4E97: std::string::_Rep::_S_create(unsigned long, unsigned long, std::allocator const&) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | ->17.21% (14,018,361B) 0xF9F5B09: std::string::_Rep::_M_clone(std::allocator const&, unsigned long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | ->17.19% (13,996,386B) 0xF9F5BAE: std::string::reserve(unsigned long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | ->13.15% (10,706,706B) 0xF9D1FB2: std::basic_stringbuf, std::allocator >::overflow(int) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->13.15% (10,706,706B) 0xF9D5D63: std::basic_streambuf >::xsputn(char const*, long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->13.15% (10,706,169B) 0xF9CD314: std::basic_ostream >& std::__ostream_insert >(std::basic_ostream >&, char const*, long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | | ->13.06% (10,634,211B) 0xF9CD6E5: std::basic_ostream >& std::operator<< >(std::basic_ostream >&, char const*) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | | | ->13.06% (10,634,211B) 0x5109AE4: RGWOmapAppend::operate() (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->13.06% (10,634,211B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->13.06% (10,634,211B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->13.06% (10,634,211B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->13.06% (10,634,211B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->13.06% (10,634,211B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->13.06% (10,634,211B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->13.06% (10,634,211B) 0xF1A7492: start_thread (pthread_create.c:333) | | | | | | | ->13.06% (10,634,211B) 0xFF3F93D: clone (clone.S:97) | | | | | | | | | | | | | ->00.00% (0B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | | | | | ->00.09% (71,958B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | | | ->00.00% (537B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | ->03.87% (3,151,754B) 0xF9F5E38: ??? (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->03.87% (3,151,754B) 0xF9F5F0C: std::string::append(char const*, unsigned long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->03.78% (3,075,009B) 0x51DF1E4: RGWDataSyncShardControlCR::alloc_cr() (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.78% (3,075,009B) 0x51AB72B: RGWBackoffControlCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.78% (3,075,009B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.78% (3,075,009B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.78% (3,075,009B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.78% (3,075,009B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.78% (3,075,009B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.78% (3,075,009B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.78% (3,075,009B) 0xF1A7492: start_thread (pthread_create.c:333) | | | | | ->03.78% (3,075,009B) 0xFF3F93D: clone (clone.S:97) | | | | | | | | | ->00.09% (76,745B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | ->00.17% (137,926B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | ->00.03% (21,975B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->01.47% (1,200,290B) 0xF9F6813: char* std::string::_S_construct(char const*, char const*, std::allocator const&, std::forward_iterator_tag) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | ->01.47% (1,200,290B) 0xF9F6C44: std::basic_string, std::allocator >::basic_string(char const*, std::allocator const&) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | ->01.26% (1,023,621B) 0x510DA32: RGWCoroutine::RGWCoroutine(CephContext*) (in /usr/lib/librgw.so.2.0.0) | | | | ->01.24% (1,009,953B) 0x51075C3: RGWOmapAppend::RGWOmapAppend(RGWAsyncRadosProcessor*, RGWRados*, rgw_bucket&, std::string const&, unsigned long) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.24% (1,009,953B) 0x51E7CB7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.24% (1,009,953B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.24% (1,009,953B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.24% (1,009,953B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.24% (1,009,953B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.24% (1,009,953B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.24% (1,009,953B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.24% (1,009,953B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.24% (1,009,953B) 0xF1A7492: start_thread (pthread_create.c:333) | | | | | ->01.24% (1,009,953B) 0xFF3F93D: clone (clone.S:97) | | | | | | | | | ->00.02% (13,668B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | ->00.22% (176,669B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | ->00.00% (0B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->00.23% (187,561B) in 1+ places, all below ms_print's threshold (01.00%) | ->12.62% (10,275,840B) 0x510DB36: RGWCoroutine::RGWCoroutine(CephContext*) (in /usr/lib/librgw.so.2.0.0) | ->12.45% (10,138,624B) 0x51075C3: RGWOmapAppend::RGWOmapAppend(RGWAsyncRadosProcessor*, RGWRados*, rgw_bucket&, std::string const&, unsigned long) (in /usr/lib/librgw.so.2.0.0) | | ->12.45% (10,138,624B) 0x51E7CB7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | ->12.45% (10,138,624B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | ->12.45% (10,138,624B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | ->12.45% (10,138,624B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | ->12.45% (10,138,624B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | ->12.45% (10,138,624B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | ->12.45% (10,138,624B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | ->12.45% (10,138,624B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | ->12.45% (10,138,624B) 0xF1A7492: start_thread (pthread_create.c:333) | | ->12.45% (10,138,624B) 0xFF3F93D: clone (clone.S:97) | | | ->00.17% (137,216B) in 1+ places, all below ms_print's threshold (01.00%) | ->10.94% (8,910,272B) 0x50F9E51: RGWCoroutinesManager::allocate_stack() (in /usr/lib/librgw.so.2.0.0) | ->10.94% (8,910,048B) 0x50FE7AE: RGWCoroutinesStack::spawn(RGWCoroutine*, RGWCoroutine*, bool) (in /usr/lib/librgw.so.2.0.0) | | ->05.45% (4,435,872B) 0x51E7CD7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | | ->05.45% (4,435,872B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | ->05.45% (4,435,872B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.45% (4,435,872B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.45% (4,435,872B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.45% (4,435,872B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.45% (4,435,872B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | ->05.45% (4,435,872B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | ->05.45% (4,435,872B) 0xF1A7492: start_thread (pthread_create.c:333) | | | ->05.45% (4,435,872B) 0xFF3F93D: clone (clone.S:97) | | | | | ->05.45% (4,435,872B) 0x51D5E5F: RGWDataSyncShardCR::init_lease_cr() (in /usr/lib/librgw.so.2.0.0) | | | ->05.45% (4,435,872B) 0x51E7CEE: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | | ->05.45% (4,435,872B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | ->05.45% (4,435,872B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.45% (4,435,872B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.45% (4,435,872B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.45% (4,435,872B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.45% (4,435,872B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | ->05.45% (4,435,872B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | ->05.45% (4,435,872B) 0xF1A7492: start_thread (pthread_create.c:333) | | | ->05.45% (4,435,872B) 0xFF3F93D: clone (clone.S:97) | | | | | ->00.05% (38,304B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->00.00% (224B) in 1+ places, all below ms_print's threshold (01.00%) | ->07.38% (6,008,454B) in 831 places, all below massif's threshold (1.00%) | ->02.48% (2,021,018B) 0x54F757B: ceph::log::Log::create_entry(int, int, unsigned long*) (in /usr/lib/librgw.so.2.0.0) | ->01.38% (1,127,265B) 0x52CD4D0: RGWRequest::log(req_state*, char const*) (in /usr/lib/librgw.so.2.0.0) | | ->01.38% (1,127,265B) in 13 places, all below massif's threshold (1.00%) | | | ->01.10% (893,753B) in 42 places, all below massif's threshold (1.00%) | ->02.18% (1,772,000B) 0x131429: worker_thread (civetweb.c:502) | ->02.18% (1,772,000B) 0xF1A7492: start_thread (pthread_create.c:333) | ->02.18% (1,772,000B) 0xFF3F93D: clone (clone.S:97) | ->01.58% (1,284,544B) 0x510DB0E: RGWCoroutine::RGWCoroutine(CephContext*) (in /usr/lib/librgw.so.2.0.0) | ->01.56% (1,267,392B) 0x51075C3: RGWOmapAppend::RGWOmapAppend(RGWAsyncRadosProcessor*, RGWRados*, rgw_bucket&, std::string const&, unsigned long) (in /usr/lib/librgw.so.2.0.0) | | ->01.56% (1,267,392B) 0x51E7CB7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | ->01.56% (1,267,392B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | ->01.56% (1,267,392B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | ->01.56% (1,267,392B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | ->01.56% (1,267,392B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | ->01.56% (1,267,392B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | ->01.56% (1,267,392B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | ->01.56% (1,267,392B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | ->01.56% (1,267,392B) 0xF1A7492: start_thread (pthread_create.c:333) | | ->01.56% (1,267,392B) 0xFF3F93D: clone (clone.S:97) | | | ->00.02% (17,152B) in 1+ places, all below ms_print's threshold (01.00%) | ->01.15% (937,984B) 0x5614937: ceph::buffer::create_aligned(unsigned int, unsigned int) (in /usr/lib/librgw.so.2.0.0) ->01.15% (937,984B) 0x1052B4D8: ??? (in /usr/lib/librados.so.2.0.0) | ->01.15% (937,984B) 0x105381E2: ??? (in /usr/lib/librados.so.2.0.0) | ->01.15% (937,984B) 0x10541BBB: ??? (in /usr/lib/librados.so.2.0.0) | ->01.15% (937,984B) 0xF1A7492: start_thread (pthread_create.c:333) | ->01.15% (937,984B) 0xFF3F93D: clone (clone.S:97) | ->00.00% (0B) in 1+ places, all below ms_print's threshold (01.00%) -------------------------------------------------------------------------------- n time(i) total(B) useful-heap(B) extra-heap(B) stacks(B) -------------------------------------------------------------------------------- 21 38,576,788,137 82,510,072 79,090,421 3,419,651 0 22 40,960,688,844 90,130,256 86,387,205 3,743,051 0 95.85% (86,387,205B) (heap allocation functions) malloc/new/new[], --alloc-fns, etc. ->40.16% (36,199,152B) 0x51E7C89: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | ->40.16% (36,199,152B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | ->40.16% (36,199,152B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | ->40.16% (36,199,152B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | ->40.16% (36,199,152B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | ->40.16% (36,199,152B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | ->40.16% (36,199,152B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | ->40.16% (36,199,152B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | ->40.16% (36,199,152B) 0xF1A7492: start_thread (pthread_create.c:333) | ->40.16% (36,199,152B) 0xFF3F93D: clone (clone.S:97) | ->19.20% (17,307,995B) 0xF9F4E97: std::string::_Rep::_S_create(unsigned long, unsigned long, std::allocator const&) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | ->17.73% (15,978,185B) 0xF9F5B09: std::string::_Rep::_M_clone(std::allocator const&, unsigned long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | ->17.73% (15,976,565B) 0xF9F5BAE: std::string::reserve(unsigned long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | ->13.64% (12,290,856B) 0xF9D1FB2: std::basic_stringbuf, std::allocator >::overflow(int) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->13.64% (12,290,856B) 0xF9D5D63: std::basic_streambuf >::xsputn(char const*, long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->13.64% (12,290,319B) 0xF9CD314: std::basic_ostream >& std::__ostream_insert >(std::basic_ostream >&, char const*, long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | | ->13.62% (12,272,061B) 0xF9CD6E5: std::basic_ostream >& std::operator<< >(std::basic_ostream >&, char const*) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | | | ->13.62% (12,272,061B) 0x5109AE4: RGWOmapAppend::operate() (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->13.62% (12,272,061B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->13.62% (12,272,061B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->13.62% (12,272,061B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->13.62% (12,272,061B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->13.62% (12,272,061B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->13.62% (12,272,061B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->13.62% (12,272,061B) 0xF1A7492: start_thread (pthread_create.c:333) | | | | | | | ->13.62% (12,272,061B) 0xFF3F93D: clone (clone.S:97) | | | | | | | | | | | | | ->00.00% (0B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | | | | | ->00.02% (18,258B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | | | ->00.00% (537B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | ->03.99% (3,592,361B) 0xF9F5E38: ??? (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->03.99% (3,592,361B) 0xF9F5F0C: std::string::append(char const*, unsigned long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->03.94% (3,548,529B) 0x51DF1E4: RGWDataSyncShardControlCR::alloc_cr() (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.94% (3,548,529B) 0x51AB72B: RGWBackoffControlCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.94% (3,548,529B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.94% (3,548,529B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.94% (3,548,529B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.94% (3,548,529B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.94% (3,548,529B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.94% (3,548,529B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.94% (3,548,529B) 0xF1A7492: start_thread (pthread_create.c:333) | | | | | ->03.94% (3,548,529B) 0xFF3F93D: clone (clone.S:97) | | | | | | | | | ->00.05% (43,832B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | ->00.10% (93,348B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | ->00.00% (1,620B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->01.40% (1,259,742B) 0xF9F6813: char* std::string::_S_construct(char const*, char const*, std::allocator const&, std::forward_iterator_tag) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | ->01.40% (1,259,742B) 0xF9F6C44: std::basic_string, std::allocator >::basic_string(char const*, std::allocator const&) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | ->01.30% (1,172,286B) 0x510DA32: RGWCoroutine::RGWCoroutine(CephContext*) (in /usr/lib/librgw.so.2.0.0) | | | | ->01.29% (1,165,503B) 0x51075C3: RGWOmapAppend::RGWOmapAppend(RGWAsyncRadosProcessor*, RGWRados*, rgw_bucket&, std::string const&, unsigned long) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.29% (1,165,503B) 0x51E7CB7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.29% (1,165,503B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.29% (1,165,503B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.29% (1,165,503B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.29% (1,165,503B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.29% (1,165,503B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.29% (1,165,503B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.29% (1,165,503B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.29% (1,165,503B) 0xF1A7492: start_thread (pthread_create.c:333) | | | | | ->01.29% (1,165,503B) 0xFF3F93D: clone (clone.S:97) | | | | | | | | | ->00.01% (6,783B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | ->00.10% (87,456B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | ->00.00% (0B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->00.08% (70,068B) in 1+ places, all below ms_print's threshold (01.00%) | ->13.06% (11,768,320B) 0x510DB36: RGWCoroutine::RGWCoroutine(CephContext*) (in /usr/lib/librgw.so.2.0.0) | ->12.98% (11,700,224B) 0x51075C3: RGWOmapAppend::RGWOmapAppend(RGWAsyncRadosProcessor*, RGWRados*, rgw_bucket&, std::string const&, unsigned long) (in /usr/lib/librgw.so.2.0.0) | | ->12.98% (11,700,224B) 0x51E7CB7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | ->12.98% (11,700,224B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | ->12.98% (11,700,224B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | ->12.98% (11,700,224B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | ->12.98% (11,700,224B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | ->12.98% (11,700,224B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | ->12.98% (11,700,224B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | ->12.98% (11,700,224B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | ->12.98% (11,700,224B) 0xF1A7492: start_thread (pthread_create.c:333) | | ->12.98% (11,700,224B) 0xFF3F93D: clone (clone.S:97) | | | ->00.08% (68,096B) in 1+ places, all below ms_print's threshold (01.00%) | ->11.40% (10,276,672B) 0x50F9E51: RGWCoroutinesManager::allocate_stack() (in /usr/lib/librgw.so.2.0.0) | ->11.40% (10,276,448B) 0x50FE7AE: RGWCoroutinesStack::spawn(RGWCoroutine*, RGWCoroutine*, bool) (in /usr/lib/librgw.so.2.0.0) | | ->05.68% (5,119,072B) 0x51E7CD7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | | ->05.68% (5,119,072B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | ->05.68% (5,119,072B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.68% (5,119,072B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.68% (5,119,072B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.68% (5,119,072B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.68% (5,119,072B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | ->05.68% (5,119,072B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | ->05.68% (5,119,072B) 0xF1A7492: start_thread (pthread_create.c:333) | | | ->05.68% (5,119,072B) 0xFF3F93D: clone (clone.S:97) | | | | | ->05.68% (5,119,072B) 0x51D5E5F: RGWDataSyncShardCR::init_lease_cr() (in /usr/lib/librgw.so.2.0.0) | | | ->05.68% (5,119,072B) 0x51E7CEE: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | | ->05.68% (5,119,072B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | ->05.68% (5,119,072B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.68% (5,119,072B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.68% (5,119,072B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.68% (5,119,072B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.68% (5,119,072B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | ->05.68% (5,119,072B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | ->05.68% (5,119,072B) 0xF1A7492: start_thread (pthread_create.c:333) | | | ->05.68% (5,119,072B) 0xFF3F93D: clone (clone.S:97) | | | | | ->00.04% (38,304B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->00.00% (224B) in 1+ places, all below ms_print's threshold (01.00%) | ->04.18% (3,770,741B) in 830 places, all below massif's threshold (1.00%) | ->01.97% (1,772,000B) 0x131429: worker_thread (civetweb.c:502) | ->01.97% (1,772,000B) 0xF1A7492: start_thread (pthread_create.c:333) | ->01.97% (1,772,000B) 0xFF3F93D: clone (clone.S:97) | ->01.94% (1,747,776B) 0x54F757B: ceph::log::Log::create_entry(int, int, unsigned long*) (in /usr/lib/librgw.so.2.0.0) | ->01.33% (1,199,641B) 0x52CD4D0: RGWRequest::log(req_state*, char const*) (in /usr/lib/librgw.so.2.0.0) | | ->01.33% (1,199,641B) in 13 places, all below massif's threshold (1.00%) | | | ->00.61% (548,135B) in 1+ places, all below ms_print's threshold (01.00%) | ->01.63% (1,471,104B) 0x510DB0E: RGWCoroutine::RGWCoroutine(CephContext*) (in /usr/lib/librgw.so.2.0.0) | ->01.62% (1,462,592B) 0x51075C3: RGWOmapAppend::RGWOmapAppend(RGWAsyncRadosProcessor*, RGWRados*, rgw_bucket&, std::string const&, unsigned long) (in /usr/lib/librgw.so.2.0.0) | | ->01.62% (1,462,592B) 0x51E7CB7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | ->01.62% (1,462,592B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | ->01.62% (1,462,592B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | ->01.62% (1,462,592B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | ->01.62% (1,462,592B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | ->01.62% (1,462,592B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | ->01.62% (1,462,592B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | ->01.62% (1,462,592B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | ->01.62% (1,462,592B) 0xF1A7492: start_thread (pthread_create.c:333) | | ->01.62% (1,462,592B) 0xFF3F93D: clone (clone.S:97) | | | ->00.01% (8,512B) in 1+ places, all below ms_print's threshold (01.00%) | ->01.27% (1,149,005B) 0x10623B1B: ??? (in /usr/lib/librados.so.2.0.0) | ->01.27% (1,149,005B) in 42 places, all below massif's threshold (1.00%) | ->01.03% (924,440B) 0x5103628: std::pair, bool> std::_Rb_tree, std::less, std::allocator >::_M_insert_unique(RGWCoroutinesStack* const&) (in /usr/lib/librgw.so.2.0.0) ->01.02% (919,280B) 0x50FE628: RGWCoroutinesManager::schedule(RGWCoroutinesEnv*, RGWCoroutinesStack*) (in /usr/lib/librgw.so.2.0.0) | ->01.02% (919,280B) 0x50FE805: RGWCoroutinesStack::spawn(RGWCoroutine*, RGWCoroutine*, bool) (in /usr/lib/librgw.so.2.0.0) | ->01.01% (914,120B) 0x51E7CD7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | ->01.01% (914,120B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | ->01.01% (914,120B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | ->01.01% (914,120B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | ->01.01% (914,120B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | ->01.01% (914,120B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | ->01.01% (914,120B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | ->01.01% (914,120B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | ->01.01% (914,120B) 0xF1A7492: start_thread (pthread_create.c:333) | | ->01.01% (914,120B) 0xFF3F93D: clone (clone.S:97) | | | ->00.01% (5,160B) in 1+ places, all below ms_print's threshold (01.00%) | ->00.01% (5,160B) in 1+ places, all below ms_print's threshold (01.00%) -------------------------------------------------------------------------------- n time(i) total(B) useful-heap(B) extra-heap(B) stacks(B) -------------------------------------------------------------------------------- 23 42,238,288,347 95,899,648 91,923,988 3,975,660 0 95.85% (91,923,988B) (heap allocation functions) malloc/new/new[], --alloc-fns, etc. ->40.16% (38,508,624B) 0x51E7C89: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | ->40.16% (38,508,624B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | ->40.16% (38,508,624B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | ->40.16% (38,508,624B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | ->40.16% (38,508,624B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | ->40.16% (38,508,624B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | ->40.16% (38,508,624B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | ->40.16% (38,508,624B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | ->40.16% (38,508,624B) 0xF1A7492: start_thread (pthread_create.c:333) | ->40.16% (38,508,624B) 0xFF3F93D: clone (clone.S:97) | ->19.26% (18,468,329B) 0xF9F4E97: std::string::_Rep::_S_create(unsigned long, unsigned long, std::allocator const&) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | ->17.77% (17,043,551B) 0xF9F5B09: std::string::_Rep::_M_clone(std::allocator const&, unsigned long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | ->17.77% (17,038,287B) 0xF9F5BAE: std::string::reserve(unsigned long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | ->13.66% (13,101,189B) 0xF9D1FB2: std::basic_stringbuf, std::allocator >::overflow(int) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->13.66% (13,101,189B) 0xF9D5D63: std::basic_streambuf >::xsputn(char const*, long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->13.66% (13,100,652B) 0xF9CD314: std::basic_ostream >& std::__ostream_insert >(std::basic_ostream >&, char const*, long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | | ->13.61% (13,055,007B) 0xF9CD6E5: std::basic_ostream >& std::operator<< >(std::basic_ostream >&, char const*) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | | | ->13.61% (13,055,007B) 0x5109AE4: RGWOmapAppend::operate() (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->13.61% (13,055,007B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->13.61% (13,055,007B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->13.61% (13,055,007B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->13.61% (13,055,007B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->13.61% (13,055,007B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->13.61% (13,055,007B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->13.61% (13,055,007B) 0xF1A7492: start_thread (pthread_create.c:333) | | | | | | | ->13.61% (13,055,007B) 0xFF3F93D: clone (clone.S:97) | | | | | | | | | | | | | ->00.00% (0B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | | | | | ->00.05% (45,645B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | | | ->00.00% (537B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | ->03.98% (3,820,540B) 0xF9F5E38: ??? (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->03.98% (3,820,540B) 0xF9F5F0C: std::string::append(char const*, unsigned long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->03.94% (3,774,901B) 0x51DF1E4: RGWDataSyncShardControlCR::alloc_cr() (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.94% (3,774,901B) 0x51AB72B: RGWBackoffControlCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.94% (3,774,901B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.94% (3,774,901B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.94% (3,774,901B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.94% (3,774,901B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.94% (3,774,901B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.94% (3,774,901B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.94% (3,774,901B) 0xF1A7492: start_thread (pthread_create.c:333) | | | | | ->03.94% (3,774,901B) 0xFF3F93D: clone (clone.S:97) | | | | | | | | | ->00.05% (45,639B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | ->00.12% (116,558B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | ->00.01% (5,264B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->01.41% (1,347,477B) 0xF9F6813: char* std::string::_S_construct(char const*, char const*, std::allocator const&, std::forward_iterator_tag) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | ->01.41% (1,347,477B) 0xF9F6C44: std::basic_string, std::allocator >::basic_string(char const*, std::allocator const&) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | ->01.30% (1,250,265B) 0x510DA32: RGWCoroutine::RGWCoroutine(CephContext*) (in /usr/lib/librgw.so.2.0.0) | | | | ->01.29% (1,239,861B) 0x51075C3: RGWOmapAppend::RGWOmapAppend(RGWAsyncRadosProcessor*, RGWRados*, rgw_bucket&, std::string const&, unsigned long) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.29% (1,239,861B) 0x51E7CB7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.29% (1,239,861B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.29% (1,239,861B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.29% (1,239,861B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.29% (1,239,861B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.29% (1,239,861B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.29% (1,239,861B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.29% (1,239,861B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.29% (1,239,861B) 0xF1A7492: start_thread (pthread_create.c:333) | | | | | ->01.29% (1,239,861B) 0xFF3F93D: clone (clone.S:97) | | | | | | | | | ->00.01% (10,404B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | ->00.10% (97,212B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | ->00.00% (0B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->00.08% (77,301B) in 1+ places, all below ms_print's threshold (01.00%) | ->13.09% (12,551,168B) 0x510DB36: RGWCoroutine::RGWCoroutine(CephContext*) (in /usr/lib/librgw.so.2.0.0) | ->12.98% (12,446,720B) 0x51075C3: RGWOmapAppend::RGWOmapAppend(RGWAsyncRadosProcessor*, RGWRados*, rgw_bucket&, std::string const&, unsigned long) (in /usr/lib/librgw.so.2.0.0) | | ->12.98% (12,446,720B) 0x51E7CB7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | ->12.98% (12,446,720B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | ->12.98% (12,446,720B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | ->12.98% (12,446,720B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | ->12.98% (12,446,720B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | ->12.98% (12,446,720B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | ->12.98% (12,446,720B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | ->12.98% (12,446,720B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | ->12.98% (12,446,720B) 0xF1A7492: start_thread (pthread_create.c:333) | | ->12.98% (12,446,720B) 0xFF3F93D: clone (clone.S:97) | | | ->00.11% (104,448B) in 1+ places, all below ms_print's threshold (01.00%) | ->11.40% (10,929,856B) 0x50F9E51: RGWCoroutinesManager::allocate_stack() (in /usr/lib/librgw.so.2.0.0) | ->11.40% (10,929,632B) 0x50FE7AE: RGWCoroutinesStack::spawn(RGWCoroutine*, RGWCoroutine*, bool) (in /usr/lib/librgw.so.2.0.0) | | ->05.68% (5,445,664B) 0x51E7CD7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | | ->05.68% (5,445,664B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | ->05.68% (5,445,664B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.68% (5,445,664B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.68% (5,445,664B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.68% (5,445,664B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.68% (5,445,664B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | ->05.68% (5,445,664B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | ->05.68% (5,445,664B) 0xF1A7492: start_thread (pthread_create.c:333) | | | ->05.68% (5,445,664B) 0xFF3F93D: clone (clone.S:97) | | | | | ->05.68% (5,445,664B) 0x51D5E5F: RGWDataSyncShardCR::init_lease_cr() (in /usr/lib/librgw.so.2.0.0) | | | ->05.68% (5,445,664B) 0x51E7CEE: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | | ->05.68% (5,445,664B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | ->05.68% (5,445,664B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.68% (5,445,664B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.68% (5,445,664B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.68% (5,445,664B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.68% (5,445,664B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | ->05.68% (5,445,664B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | ->05.68% (5,445,664B) 0xF1A7492: start_thread (pthread_create.c:333) | | | ->05.68% (5,445,664B) 0xFF3F93D: clone (clone.S:97) | | | | | ->00.04% (38,304B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->00.00% (224B) in 1+ places, all below ms_print's threshold (01.00%) | ->05.27% (5,057,470B) in 831 places, all below massif's threshold (1.00%) | ->02.17% (2,084,101B) 0x54F757B: ceph::log::Log::create_entry(int, int, unsigned long*) (in /usr/lib/librgw.so.2.0.0) | ->01.30% (1,250,767B) 0x52CD4D0: RGWRequest::log(req_state*, char const*) (in /usr/lib/librgw.so.2.0.0) | | ->01.30% (1,250,767B) in 13 places, all below massif's threshold (1.00%) | | | ->00.87% (833,334B) in 1+ places, all below ms_print's threshold (01.00%) | ->01.85% (1,772,000B) 0x131429: worker_thread (civetweb.c:502) | ->01.85% (1,772,000B) 0xF1A7492: start_thread (pthread_create.c:333) | ->01.85% (1,772,000B) 0xFF3F93D: clone (clone.S:97) | ->01.64% (1,568,960B) 0x510DB0E: RGWCoroutine::RGWCoroutine(CephContext*) (in /usr/lib/librgw.so.2.0.0) | ->01.62% (1,555,904B) 0x51075C3: RGWOmapAppend::RGWOmapAppend(RGWAsyncRadosProcessor*, RGWRados*, rgw_bucket&, std::string const&, unsigned long) (in /usr/lib/librgw.so.2.0.0) | | ->01.62% (1,555,904B) 0x51E7CB7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | ->01.62% (1,555,904B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | ->01.62% (1,555,904B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | ->01.62% (1,555,904B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | ->01.62% (1,555,904B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | ->01.62% (1,555,904B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | ->01.62% (1,555,904B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | ->01.62% (1,555,904B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | ->01.62% (1,555,904B) 0xF1A7492: start_thread (pthread_create.c:333) | | ->01.62% (1,555,904B) 0xFF3F93D: clone (clone.S:97) | | | ->00.01% (13,056B) in 1+ places, all below ms_print's threshold (01.00%) | ->01.03% (983,480B) 0x5103628: std::pair, bool> std::_Rb_tree, std::less, std::allocator >::_M_insert_unique(RGWCoroutinesStack* const&) (in /usr/lib/librgw.so.2.0.0) ->01.02% (978,320B) 0x50FE628: RGWCoroutinesManager::schedule(RGWCoroutinesEnv*, RGWCoroutinesStack*) (in /usr/lib/librgw.so.2.0.0) | ->01.02% (978,320B) 0x50FE805: RGWCoroutinesStack::spawn(RGWCoroutine*, RGWCoroutine*, bool) (in /usr/lib/librgw.so.2.0.0) | ->01.01% (972,440B) 0x51E7CD7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | ->01.01% (972,440B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | ->01.01% (972,440B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | ->01.01% (972,440B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | ->01.01% (972,440B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | ->01.01% (972,440B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | ->01.01% (972,440B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | ->01.01% (972,440B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | ->01.01% (972,440B) 0xF1A7492: start_thread (pthread_create.c:333) | | ->01.01% (972,440B) 0xFF3F93D: clone (clone.S:97) | | | ->00.01% (5,880B) in 1+ places, all below ms_print's threshold (01.00%) | ->00.01% (5,160B) in 1+ places, all below ms_print's threshold (01.00%) -------------------------------------------------------------------------------- n time(i) total(B) useful-heap(B) extra-heap(B) stacks(B) -------------------------------------------------------------------------------- 24 44,632,414,168 102,200,248 97,969,356 4,230,892 0 95.86% (97,969,356B) (heap allocation functions) malloc/new/new[], --alloc-fns, etc. ->40.56% (41,448,528B) 0x51E7C89: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | ->40.56% (41,448,528B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | ->40.56% (41,448,528B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | ->40.56% (41,448,528B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | ->40.56% (41,448,528B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | ->40.56% (41,448,528B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | ->40.56% (41,448,528B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | ->40.56% (41,448,528B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | ->40.56% (41,448,528B) 0xF1A7492: start_thread (pthread_create.c:333) | ->40.56% (41,448,528B) 0xFF3F93D: clone (clone.S:97) | ->19.42% (19,844,504B) 0xF9F4E97: std::string::_Rep::_S_create(unsigned long, unsigned long, std::allocator const&) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | ->17.94% (18,334,108B) 0xF9F5B09: std::string::_Rep::_M_clone(std::allocator const&, unsigned long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | ->17.94% (18,330,405B) 0xF9F5BAE: std::string::reserve(unsigned long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | ->13.79% (14,093,028B) 0xF9D1FB2: std::basic_stringbuf, std::allocator >::overflow(int) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->13.79% (14,093,028B) 0xF9D5D63: std::basic_streambuf >::xsputn(char const*, long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->13.79% (14,092,491B) 0xF9CD314: std::basic_ostream >& std::__ostream_insert >(std::basic_ostream >&, char const*, long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | | ->13.75% (14,051,679B) 0xF9CD6E5: std::basic_ostream >& std::operator<< >(std::basic_ostream >&, char const*) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | | | ->13.75% (14,051,679B) 0x5109AE4: RGWOmapAppend::operate() (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->13.75% (14,051,679B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->13.75% (14,051,679B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->13.75% (14,051,679B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->13.75% (14,051,679B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->13.75% (14,051,679B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->13.75% (14,051,679B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->13.75% (14,051,679B) 0xF1A7492: start_thread (pthread_create.c:333) | | | | | | | ->13.75% (14,051,679B) 0xFF3F93D: clone (clone.S:97) | | | | | | | | | | | | | ->00.00% (0B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | | | | | ->00.04% (40,812B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | | | ->00.00% (537B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | ->04.02% (4,108,030B) 0xF9F5E38: ??? (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->04.02% (4,108,030B) 0xF9F5F0C: std::string::append(char const*, unsigned long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->03.98% (4,064,901B) 0x51DF1E4: RGWDataSyncShardControlCR::alloc_cr() (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.98% (4,064,901B) 0x51AB72B: RGWBackoffControlCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.98% (4,064,901B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.98% (4,064,901B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.98% (4,064,901B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.98% (4,064,901B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.98% (4,064,901B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.98% (4,064,901B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.98% (4,064,901B) 0xF1A7492: start_thread (pthread_create.c:333) | | | | | ->03.98% (4,064,901B) 0xFF3F93D: clone (clone.S:97) | | | | | | | | | ->00.04% (43,129B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | ->00.13% (129,347B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | ->00.00% (3,703B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->01.41% (1,437,452B) 0xF9F6813: char* std::string::_S_construct(char const*, char const*, std::allocator const&, std::forward_iterator_tag) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | ->01.41% (1,437,452B) 0xF9F6C44: std::basic_string, std::allocator >::basic_string(char const*, std::allocator const&) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | ->01.32% (1,344,054B) 0x510DA32: RGWCoroutine::RGWCoroutine(CephContext*) (in /usr/lib/librgw.so.2.0.0) | | | | ->01.31% (1,334,517B) 0x51075C3: RGWOmapAppend::RGWOmapAppend(RGWAsyncRadosProcessor*, RGWRados*, rgw_bucket&, std::string const&, unsigned long) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.31% (1,334,517B) 0x51E7CB7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.31% (1,334,517B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.31% (1,334,517B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.31% (1,334,517B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.31% (1,334,517B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.31% (1,334,517B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.31% (1,334,517B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.31% (1,334,517B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.31% (1,334,517B) 0xF1A7492: start_thread (pthread_create.c:333) | | | | | ->01.31% (1,334,517B) 0xFF3F93D: clone (clone.S:97) | | | | | | | | | ->00.01% (9,537B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | ->00.09% (93,398B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | ->00.00% (0B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->00.07% (72,944B) in 1+ places, all below ms_print's threshold (01.00%) | ->13.20% (13,492,736B) 0x510DB36: RGWCoroutine::RGWCoroutine(CephContext*) (in /usr/lib/librgw.so.2.0.0) | ->13.11% (13,396,992B) 0x51075C3: RGWOmapAppend::RGWOmapAppend(RGWAsyncRadosProcessor*, RGWRados*, rgw_bucket&, std::string const&, unsigned long) (in /usr/lib/librgw.so.2.0.0) | | ->13.11% (13,396,992B) 0x51E7CB7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | ->13.11% (13,396,992B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | ->13.11% (13,396,992B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | ->13.11% (13,396,992B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | ->13.11% (13,396,992B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | ->13.11% (13,396,992B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | ->13.11% (13,396,992B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | ->13.11% (13,396,992B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | ->13.11% (13,396,992B) 0xF1A7492: start_thread (pthread_create.c:333) | | ->13.11% (13,396,992B) 0xFF3F93D: clone (clone.S:97) | | | ->00.09% (95,744B) in 1+ places, all below ms_print's threshold (01.00%) | ->11.51% (11,761,344B) 0x50F9E51: RGWCoroutinesManager::allocate_stack() (in /usr/lib/librgw.so.2.0.0) | ->11.51% (11,761,120B) 0x50FE7AE: RGWCoroutinesStack::spawn(RGWCoroutine*, RGWCoroutine*, bool) (in /usr/lib/librgw.so.2.0.0) | | ->05.74% (5,861,408B) 0x51E7CD7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | | ->05.74% (5,861,408B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | ->05.74% (5,861,408B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.74% (5,861,408B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.74% (5,861,408B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.74% (5,861,408B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.74% (5,861,408B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | ->05.74% (5,861,408B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | ->05.74% (5,861,408B) 0xF1A7492: start_thread (pthread_create.c:333) | | | ->05.74% (5,861,408B) 0xFF3F93D: clone (clone.S:97) | | | | | ->05.74% (5,861,408B) 0x51D5E5F: RGWDataSyncShardCR::init_lease_cr() (in /usr/lib/librgw.so.2.0.0) | | | ->05.74% (5,861,408B) 0x51E7CEE: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | | ->05.74% (5,861,408B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | ->05.74% (5,861,408B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.74% (5,861,408B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.74% (5,861,408B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.74% (5,861,408B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.74% (5,861,408B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | ->05.74% (5,861,408B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | ->05.74% (5,861,408B) 0xF1A7492: start_thread (pthread_create.c:333) | | | ->05.74% (5,861,408B) 0xFF3F93D: clone (clone.S:97) | | | | | ->00.04% (38,304B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->00.00% (224B) in 1+ places, all below ms_print's threshold (01.00%) | ->04.70% (4,806,780B) in 831 places, all below massif's threshold (1.00%) | ->02.05% (2,099,408B) 0x54F757B: ceph::log::Log::create_entry(int, int, unsigned long*) (in /usr/lib/librgw.so.2.0.0) | ->01.31% (1,336,779B) 0x52CD4D0: RGWRequest::log(req_state*, char const*) (in /usr/lib/librgw.so.2.0.0) | | ->01.31% (1,336,779B) in 13 places, all below massif's threshold (1.00%) | | | ->00.75% (762,629B) in 1+ places, all below ms_print's threshold (01.00%) | ->01.73% (1,772,000B) 0x131429: worker_thread (civetweb.c:502) | ->01.73% (1,772,000B) 0xF1A7492: start_thread (pthread_create.c:333) | ->01.73% (1,772,000B) 0xFF3F93D: clone (clone.S:97) | ->01.65% (1,686,656B) 0x510DB0E: RGWCoroutine::RGWCoroutine(CephContext*) (in /usr/lib/librgw.so.2.0.0) | ->01.64% (1,674,688B) 0x51075C3: RGWOmapAppend::RGWOmapAppend(RGWAsyncRadosProcessor*, RGWRados*, rgw_bucket&, std::string const&, unsigned long) (in /usr/lib/librgw.so.2.0.0) | | ->01.64% (1,674,688B) 0x51E7CB7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | ->01.64% (1,674,688B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | ->01.64% (1,674,688B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | ->01.64% (1,674,688B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | ->01.64% (1,674,688B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | ->01.64% (1,674,688B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | ->01.64% (1,674,688B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | ->01.64% (1,674,688B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | ->01.64% (1,674,688B) 0xF1A7492: start_thread (pthread_create.c:333) | | ->01.64% (1,674,688B) 0xFF3F93D: clone (clone.S:97) | | | ->00.01% (11,968B) in 1+ places, all below ms_print's threshold (01.00%) | ->01.03% (1,057,400B) 0x5103628: std::pair, bool> std::_Rb_tree, std::less, std::allocator >::_M_insert_unique(RGWCoroutinesStack* const&) (in /usr/lib/librgw.so.2.0.0) ->01.03% (1,052,240B) 0x50FE628: RGWCoroutinesManager::schedule(RGWCoroutinesEnv*, RGWCoroutinesStack*) (in /usr/lib/librgw.so.2.0.0) | ->01.03% (1,052,240B) 0x50FE805: RGWCoroutinesStack::spawn(RGWCoroutine*, RGWCoroutine*, bool) (in /usr/lib/librgw.so.2.0.0) | ->01.02% (1,046,680B) 0x51E7CD7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | ->01.02% (1,046,680B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | ->01.02% (1,046,680B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | ->01.02% (1,046,680B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | ->01.02% (1,046,680B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | ->01.02% (1,046,680B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | ->01.02% (1,046,680B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | ->01.02% (1,046,680B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | ->01.02% (1,046,680B) 0xF1A7492: start_thread (pthread_create.c:333) | | ->01.02% (1,046,680B) 0xFF3F93D: clone (clone.S:97) | | | ->00.01% (5,560B) in 1+ places, all below ms_print's threshold (01.00%) | ->00.01% (5,160B) in 1+ places, all below ms_print's threshold (01.00%) -------------------------------------------------------------------------------- n time(i) total(B) useful-heap(B) extra-heap(B) stacks(B) -------------------------------------------------------------------------------- 25 46,356,708,559 111,125,496 106,460,010 4,665,486 0 95.80% (106,460,010B) (heap allocation functions) malloc/new/new[], --alloc-fns, etc. ->39.79% (44,212,608B) 0x51E7C89: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | ->39.79% (44,212,608B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | ->39.79% (44,212,608B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | ->39.79% (44,212,608B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | ->39.79% (44,212,608B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | ->39.79% (44,212,608B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | ->39.79% (44,212,608B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | ->39.79% (44,212,608B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | ->39.79% (44,212,608B) 0xF1A7492: start_thread (pthread_create.c:333) | ->39.79% (44,212,608B) 0xFF3F93D: clone (clone.S:97) | ->19.39% (21,547,337B) 0xF9F4E97: std::string::_Rep::_S_create(unsigned long, unsigned long, std::allocator const&) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | ->17.74% (19,715,453B) 0xF9F5B09: std::string::_Rep::_M_clone(std::allocator const&, unsigned long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | ->17.72% (19,694,497B) 0xF9F5BAE: std::string::reserve(unsigned long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | ->13.60% (15,111,180B) 0xF9D1FB2: std::basic_stringbuf, std::allocator >::overflow(int) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->13.60% (15,111,180B) 0xF9D5D63: std::basic_streambuf >::xsputn(char const*, long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->13.60% (15,110,643B) 0xF9CD314: std::basic_ostream >& std::__ostream_insert >(std::basic_ostream >&, char const*, long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | | ->13.49% (14,987,670B) 0xF9CD6E5: std::basic_ostream >& std::operator<< >(std::basic_ostream >&, char const*) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | | | ->13.49% (14,987,670B) 0x5109AE4: RGWOmapAppend::operate() (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->13.49% (14,987,670B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->13.49% (14,987,670B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->13.49% (14,987,670B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->13.49% (14,987,670B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->13.49% (14,987,670B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->13.49% (14,987,670B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->13.49% (14,987,670B) 0xF1A7492: start_thread (pthread_create.c:333) | | | | | | | ->13.49% (14,987,670B) 0xFF3F93D: clone (clone.S:97) | | | | | | | | | | | | | ->00.00% (0B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | | | | | ->00.11% (122,973B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | | | ->00.00% (537B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | ->03.97% (4,415,712B) 0xF9F5E38: ??? (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->03.97% (4,415,712B) 0xF9F5F0C: std::string::append(char const*, unsigned long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->03.90% (4,333,954B) 0x51DF1E4: RGWDataSyncShardControlCR::alloc_cr() (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.90% (4,333,954B) 0x51AB72B: RGWBackoffControlCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.90% (4,333,954B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.90% (4,333,954B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.90% (4,333,954B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.90% (4,333,954B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.90% (4,333,954B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.90% (4,333,954B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.90% (4,333,954B) 0xF1A7492: start_thread (pthread_create.c:333) | | | | | ->03.90% (4,333,954B) 0xFF3F93D: clone (clone.S:97) | | | | | | | | | ->00.07% (81,758B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | ->00.15% (167,605B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | ->00.02% (20,956B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->01.48% (1,640,806B) 0xF9F6813: char* std::string::_S_construct(char const*, char const*, std::allocator const&, std::forward_iterator_tag) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | ->01.48% (1,640,806B) 0xF9F6C44: std::basic_string, std::allocator >::basic_string(char const*, std::allocator const&) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | ->01.30% (1,442,229B) 0x510DA32: RGWCoroutine::RGWCoroutine(CephContext*) (in /usr/lib/librgw.so.2.0.0) | | | | ->01.28% (1,423,512B) 0x51075C3: RGWOmapAppend::RGWOmapAppend(RGWAsyncRadosProcessor*, RGWRados*, rgw_bucket&, std::string const&, unsigned long) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.28% (1,423,512B) 0x51E7CB7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.28% (1,423,512B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.28% (1,423,512B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.28% (1,423,512B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.28% (1,423,512B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.28% (1,423,512B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.28% (1,423,512B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.28% (1,423,512B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.28% (1,423,512B) 0xF1A7492: start_thread (pthread_create.c:333) | | | | | ->01.28% (1,423,512B) 0xFF3F93D: clone (clone.S:97) | | | | | | | | | ->00.02% (18,717B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | ->00.18% (198,577B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | ->00.00% (0B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->00.17% (191,078B) in 1+ places, all below ms_print's threshold (01.00%) | ->13.03% (14,478,336B) 0x510DB36: RGWCoroutine::RGWCoroutine(CephContext*) (in /usr/lib/librgw.so.2.0.0) | ->12.86% (14,290,432B) 0x51075C3: RGWOmapAppend::RGWOmapAppend(RGWAsyncRadosProcessor*, RGWRados*, rgw_bucket&, std::string const&, unsigned long) (in /usr/lib/librgw.so.2.0.0) | | ->12.86% (14,290,432B) 0x51E7CB7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | ->12.86% (14,290,432B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | ->12.86% (14,290,432B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | ->12.86% (14,290,432B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | ->12.86% (14,290,432B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | ->12.86% (14,290,432B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | ->12.86% (14,290,432B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | ->12.86% (14,290,432B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | ->12.86% (14,290,432B) 0xF1A7492: start_thread (pthread_create.c:333) | | ->12.86% (14,290,432B) 0xFF3F93D: clone (clone.S:97) | | | ->00.17% (187,904B) in 1+ places, all below ms_print's threshold (01.00%) | ->11.29% (12,541,760B) 0x50F9E51: RGWCoroutinesManager::allocate_stack() (in /usr/lib/librgw.so.2.0.0) | ->11.29% (12,541,536B) 0x50FE7AE: RGWCoroutinesStack::spawn(RGWCoroutine*, RGWCoroutine*, bool) (in /usr/lib/librgw.so.2.0.0) | | ->05.63% (6,252,288B) 0x51E7CD7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | | ->05.63% (6,252,288B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | ->05.63% (6,252,288B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.63% (6,252,288B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.63% (6,252,288B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.63% (6,252,288B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.63% (6,252,288B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | ->05.63% (6,252,288B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | ->05.63% (6,252,288B) 0xF1A7492: start_thread (pthread_create.c:333) | | | ->05.63% (6,252,288B) 0xFF3F93D: clone (clone.S:97) | | | | | ->05.63% (6,252,288B) 0x51D5E5F: RGWDataSyncShardCR::init_lease_cr() (in /usr/lib/librgw.so.2.0.0) | | | ->05.63% (6,252,288B) 0x51E7CEE: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | | ->05.63% (6,252,288B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | ->05.63% (6,252,288B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.63% (6,252,288B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.63% (6,252,288B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.63% (6,252,288B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.63% (6,252,288B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | ->05.63% (6,252,288B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | ->05.63% (6,252,288B) 0xF1A7492: start_thread (pthread_create.c:333) | | | ->05.63% (6,252,288B) 0xFF3F93D: clone (clone.S:97) | | | | | ->00.03% (36,960B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->00.00% (224B) in 1+ places, all below ms_print's threshold (01.00%) | ->06.27% (6,966,622B) in 832 places, all below massif's threshold (1.00%) | ->01.80% (2,002,131B) 0x54F757B: ceph::log::Log::create_entry(int, int, unsigned long*) (in /usr/lib/librgw.so.2.0.0) | ->01.22% (1,351,670B) 0x52CD4D0: RGWRequest::log(req_state*, char const*) (in /usr/lib/librgw.so.2.0.0) | | ->01.22% (1,351,670B) in 13 places, all below massif's threshold (1.00%) | | | ->00.59% (650,461B) in 1+ places, all below ms_print's threshold (01.00%) | ->01.63% (1,809,856B) 0x510DB0E: RGWCoroutine::RGWCoroutine(CephContext*) (in /usr/lib/librgw.so.2.0.0) | ->01.61% (1,786,368B) 0x51075C3: RGWOmapAppend::RGWOmapAppend(RGWAsyncRadosProcessor*, RGWRados*, rgw_bucket&, std::string const&, unsigned long) (in /usr/lib/librgw.so.2.0.0) | | ->01.61% (1,786,368B) 0x51E7CB7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | ->01.61% (1,786,368B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | ->01.61% (1,786,368B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | ->01.61% (1,786,368B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | ->01.61% (1,786,368B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | ->01.61% (1,786,368B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | ->01.61% (1,786,368B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | ->01.61% (1,786,368B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | ->01.61% (1,786,368B) 0xF1A7492: start_thread (pthread_create.c:333) | | ->01.61% (1,786,368B) 0xFF3F93D: clone (clone.S:97) | | | ->00.02% (23,488B) in 1+ places, all below ms_print's threshold (01.00%) | ->01.59% (1,772,000B) 0x131429: worker_thread (civetweb.c:502) | ->01.59% (1,772,000B) 0xF1A7492: start_thread (pthread_create.c:333) | ->01.59% (1,772,000B) 0xFF3F93D: clone (clone.S:97) | ->01.02% (1,129,360B) 0x5103628: std::pair, bool> std::_Rb_tree, std::less, std::allocator >::_M_insert_unique(RGWCoroutinesStack* const&) (in /usr/lib/librgw.so.2.0.0) ->01.01% (1,124,200B) 0x50FE628: RGWCoroutinesManager::schedule(RGWCoroutinesEnv*, RGWCoroutinesStack*) (in /usr/lib/librgw.so.2.0.0) | ->01.01% (1,124,200B) 0x50FE805: RGWCoroutinesStack::spawn(RGWCoroutine*, RGWCoroutine*, bool) (in /usr/lib/librgw.so.2.0.0) | ->01.00% (1,116,480B) 0x51E7CD7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | ->01.00% (1,116,480B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | ->01.00% (1,116,480B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | ->01.00% (1,116,480B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | ->01.00% (1,116,480B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | ->01.00% (1,116,480B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | ->01.00% (1,116,480B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | ->01.00% (1,116,480B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | ->01.00% (1,116,480B) 0xF1A7492: start_thread (pthread_create.c:333) | | ->01.00% (1,116,480B) 0xFF3F93D: clone (clone.S:97) | | | ->00.01% (7,720B) in 1+ places, all below ms_print's threshold (01.00%) | ->00.00% (5,160B) in 1+ places, all below ms_print's threshold (01.00%) -------------------------------------------------------------------------------- n time(i) total(B) useful-heap(B) extra-heap(B) stacks(B) -------------------------------------------------------------------------------- 26 47,995,438,056 114,638,024 109,868,326 4,769,698 0 95.84% (109,868,326B) (heap allocation functions) malloc/new/new[], --alloc-fns, etc. ->39.23% (44,968,176B) 0x51E7C89: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | ->39.23% (44,968,176B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | ->39.23% (44,968,176B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | ->39.23% (44,968,176B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | ->39.23% (44,968,176B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | ->39.23% (44,968,176B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | ->39.23% (44,968,176B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | ->39.23% (44,968,176B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | ->39.23% (44,968,176B) 0xF1A7492: start_thread (pthread_create.c:333) | ->39.23% (44,968,176B) 0xFF3F93D: clone (clone.S:97) | ->19.04% (21,827,813B) 0xF9F4E97: std::string::_Rep::_S_create(unsigned long, unsigned long, std::allocator const&) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | ->17.41% (19,961,669B) 0xF9F5B09: std::string::_Rep::_M_clone(std::allocator const&, unsigned long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | ->17.39% (19,933,733B) 0xF9F5BAE: std::string::reserve(unsigned long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | ->13.36% (15,318,462B) 0xF9D1FB2: std::basic_stringbuf, std::allocator >::overflow(int) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->13.36% (15,318,462B) 0xF9D5D63: std::basic_streambuf >::xsputn(char const*, long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->13.36% (15,317,925B) 0xF9CD314: std::basic_ostream >& std::__ostream_insert >(std::basic_ostream >&, char const*, long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | | ->13.30% (15,249,726B) 0xF9CD6E5: std::basic_ostream >& std::operator<< >(std::basic_ostream >&, char const*) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | | | ->13.29% (15,240,597B) 0x5109AE4: RGWOmapAppend::operate() (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->13.29% (15,240,597B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->13.29% (15,240,597B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->13.29% (15,240,597B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->13.29% (15,240,597B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->13.29% (15,240,597B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->13.29% (15,240,597B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->13.29% (15,240,597B) 0xF1A7492: start_thread (pthread_create.c:333) | | | | | | | ->13.29% (15,240,597B) 0xFF3F93D: clone (clone.S:97) | | | | | | | | | | | | | ->00.01% (9,129B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | | | | | ->00.06% (68,199B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | | | ->00.00% (537B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | ->03.92% (4,496,785B) 0xF9F5E38: ??? (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->03.92% (4,496,785B) 0xF9F5F0C: std::string::append(char const*, unsigned long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->03.85% (4,408,041B) 0x51DF1E4: RGWDataSyncShardControlCR::alloc_cr() (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.85% (4,408,041B) 0x51AB72B: RGWBackoffControlCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.85% (4,408,041B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.85% (4,408,041B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.85% (4,408,041B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.85% (4,408,041B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.85% (4,408,041B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.85% (4,408,041B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.85% (4,408,041B) 0xF1A7492: start_thread (pthread_create.c:333) | | | | | ->03.85% (4,408,041B) 0xFF3F93D: clone (clone.S:97) | | | | | | | | | ->00.08% (88,744B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | ->00.10% (118,486B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | ->00.02% (27,936B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->01.45% (1,657,460B) 0xF9F6813: char* std::string::_S_construct(char const*, char const*, std::allocator const&, std::forward_iterator_tag) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | ->01.45% (1,656,890B) 0xF9F6C44: std::basic_string, std::allocator >::basic_string(char const*, std::allocator const&) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | ->01.28% (1,464,108B) 0x510DA32: RGWCoroutine::RGWCoroutine(CephContext*) (in /usr/lib/librgw.so.2.0.0) | | | | ->01.26% (1,447,839B) 0x51075C3: RGWOmapAppend::RGWOmapAppend(RGWAsyncRadosProcessor*, RGWRados*, rgw_bucket&, std::string const&, unsigned long) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.26% (1,447,839B) 0x51E7CB7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.26% (1,447,839B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.26% (1,447,839B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.26% (1,447,839B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.26% (1,447,839B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.26% (1,447,839B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.26% (1,447,839B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.26% (1,447,839B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.26% (1,447,839B) 0xF1A7492: start_thread (pthread_create.c:333) | | | | | ->01.26% (1,447,839B) 0xFF3F93D: clone (clone.S:97) | | | | | | | | | ->00.01% (16,269B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | ->00.17% (192,782B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | ->00.00% (570B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->00.18% (208,684B) in 1+ places, all below ms_print's threshold (01.00%) | ->12.82% (14,697,472B) 0x510DB36: RGWCoroutine::RGWCoroutine(CephContext*) (in /usr/lib/librgw.so.2.0.0) | ->12.68% (14,534,656B) 0x51075C3: RGWOmapAppend::RGWOmapAppend(RGWAsyncRadosProcessor*, RGWRados*, rgw_bucket&, std::string const&, unsigned long) (in /usr/lib/librgw.so.2.0.0) | | ->12.68% (14,534,656B) 0x51E7CB7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | ->12.68% (14,534,656B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | ->12.68% (14,534,656B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | ->12.68% (14,534,656B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | ->12.68% (14,534,656B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | ->12.68% (14,534,656B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | ->12.68% (14,534,656B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | ->12.68% (14,534,656B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | ->12.68% (14,534,656B) 0xF1A7492: start_thread (pthread_create.c:333) | | ->12.68% (14,534,656B) 0xFF3F93D: clone (clone.S:97) | | | ->00.14% (162,816B) in 1+ places, all below ms_print's threshold (01.00%) | ->11.13% (12,759,712B) 0x50F9E51: RGWCoroutinesManager::allocate_stack() (in /usr/lib/librgw.so.2.0.0) | ->11.13% (12,759,488B) 0x50FE7AE: RGWCoroutinesStack::spawn(RGWCoroutine*, RGWCoroutine*, bool) (in /usr/lib/librgw.so.2.0.0) | | ->05.55% (6,359,136B) 0x51E7CD7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | | ->05.55% (6,359,136B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | ->05.55% (6,359,136B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.55% (6,359,136B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.55% (6,359,136B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.55% (6,359,136B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.55% (6,359,136B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | ->05.55% (6,359,136B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | ->05.55% (6,359,136B) 0xF1A7492: start_thread (pthread_create.c:333) | | | ->05.55% (6,359,136B) 0xFF3F93D: clone (clone.S:97) | | | | | ->05.55% (6,359,136B) 0x51D5E5F: RGWDataSyncShardCR::init_lease_cr() (in /usr/lib/librgw.so.2.0.0) | | | ->05.55% (6,359,136B) 0x51E7CEE: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | | ->05.55% (6,359,136B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | ->05.55% (6,359,136B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.55% (6,359,136B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.55% (6,359,136B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.55% (6,359,136B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.55% (6,359,136B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | ->05.55% (6,359,136B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | ->05.55% (6,359,136B) 0xF1A7492: start_thread (pthread_create.c:333) | | | ->05.55% (6,359,136B) 0xFF3F93D: clone (clone.S:97) | | | | | ->00.04% (41,216B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->00.00% (224B) in 1+ places, all below ms_print's threshold (01.00%) | ->06.07% (6,962,735B) in 845 places, all below massif's threshold (1.00%) | ->01.73% (1,983,266B) 0x54F757B: ceph::log::Log::create_entry(int, int, unsigned long*) (in /usr/lib/librgw.so.2.0.0) | ->01.02% (1,164,447B) 0x52CD4D0: RGWRequest::log(req_state*, char const*) (in /usr/lib/librgw.so.2.0.0) | | ->01.02% (1,164,447B) in 13 places, all below massif's threshold (1.00%) | | | ->00.71% (818,819B) in 1+ places, all below ms_print's threshold (01.00%) | ->01.67% (1,912,200B) 0x5614937: ceph::buffer::create_aligned(unsigned int, unsigned int) (in /usr/lib/librgw.so.2.0.0) | ->01.09% (1,248,648B) 0x5614B83: ceph::buffer::copy(char const*, unsigned int) (in /usr/lib/librgw.so.2.0.0) | | ->01.09% (1,248,648B) 0x5614BB6: ceph::buffer::ptr::ptr(char const*, unsigned int) (in /usr/lib/librgw.so.2.0.0) | | ->01.09% (1,248,648B) 0x52E01AE: RGWRESTStreamRWRequest::receive_data(void*, unsigned long) (in /usr/lib/librgw.so.2.0.0) | | ->01.09% (1,248,648B) 0x515B39E: ??? (in /usr/lib/librgw.so.2.0.0) | | ->01.09% (1,248,648B) 0xEB2A230: ??? (in /usr/lib/x86_64-linux-gnu/libcurl-gnutls.so.4.4.0) | | ->01.09% (1,248,648B) 0xEB42388: ??? (in /usr/lib/x86_64-linux-gnu/libcurl-gnutls.so.4.4.0) | | ->01.09% (1,248,648B) 0xEB4C6D4: ??? (in /usr/lib/x86_64-linux-gnu/libcurl-gnutls.so.4.4.0) | | ->01.09% (1,248,648B) 0xEB4D30F: curl_multi_perform (in /usr/lib/x86_64-linux-gnu/libcurl-gnutls.so.4.4.0) | | ->01.07% (1,221,147B) 0x515EDF5: RGWHTTPManager::process_requests(bool, bool*) (in /usr/lib/librgw.so.2.0.0) | | | ->01.07% (1,221,147B) 0x515F26E: RGWHTTPManager::complete_requests() (in /usr/lib/librgw.so.2.0.0) | | | ->01.07% (1,221,147B) 0x52E6727: RGWRESTStreamRWRequest::get_resource(RGWAccessKey&, std::map, std::allocator > >&, std::string const&, RGWHTTPManager*) (in /usr/lib/librgw.so.2.0.0) | | | ->01.07% (1,221,147B) 0x52E6AA1: RGWRESTStreamRWRequest::get_obj(RGWAccessKey&, std::map, std::allocator > >&, rgw_obj&) (in /usr/lib/librgw.so.2.0.0) | | | ->01.07% (1,221,147B) 0x52F185C: RGWRESTConn::get_obj(rgw_user const&, req_info*, rgw_obj&, std::chrono::time_point > > const*, std::chrono::time_point > > const*, unsigned int, unsigned long, bool, RGWGetDataCB*, RGWRESTStreamReadRequest**) (in /usr/lib/librgw.so.2.0.0) | | | ->01.07% (1,221,147B) 0x529AED3: RGWRados::fetch_remote_obj(RGWObjectCtx&, rgw_user const&, std::string const&, std::string const&, bool, req_info*, std::string const&, rgw_obj&, rgw_obj&, RGWBucketInfo&, RGWBucketInfo&, std::chrono::time_point > >*, std::chrono::time_point > >*, std::chrono::time_point > > const*, std::chrono::time_point > > const*, bool, char const*, char const*, RGWRados::AttrsMod, bool, std::map, std::allocator > >&, RGWObjCategory, unsigned long, std::chrono::time_point > >, std::string*, std::string*, std::string*, rgw_err*, void (*)(long, void*), void*) (in /usr/lib/librgw.so.2.0.0) | | | ->01.07% (1,221,147B) 0x5107AD2: RGWAsyncFetchRemoteObj::_send_request() (in /usr/lib/librgw.so.2.0.0) | | | ->01.07% (1,221,147B) 0x5105840: RGWAsyncRadosProcessor::handle_request(RGWAsyncRadosRequest*) (in /usr/lib/librgw.so.2.0.0) | | | ->01.07% (1,221,147B) 0x510590B: RGWAsyncRadosProcessor::RGWWQ::_process(RGWAsyncRadosRequest*, ThreadPool::TPHandle&) (in /usr/lib/librgw.so.2.0.0) | | | ->01.07% (1,221,147B) 0x55FB64D: ThreadPool::worker(ThreadPool::WorkThread*) (in /usr/lib/librgw.so.2.0.0) | | | ->01.07% (1,221,147B) 0x55FC57E: ThreadPool::WorkThread::entry() (in /usr/lib/librgw.so.2.0.0) | | | ->01.07% (1,221,147B) 0xF1A7492: start_thread (pthread_create.c:333) | | | ->01.07% (1,221,147B) 0xFF3F93D: clone (clone.S:97) | | | | | ->00.02% (27,501B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->00.58% (663,552B) in 1+ places, all below ms_print's threshold (01.00%) | ->01.60% (1,837,312B) 0x510DB0E: RGWCoroutine::RGWCoroutine(CephContext*) (in /usr/lib/librgw.so.2.0.0) | ->01.58% (1,816,896B) 0x51075C3: RGWOmapAppend::RGWOmapAppend(RGWAsyncRadosProcessor*, RGWRados*, rgw_bucket&, std::string const&, unsigned long) (in /usr/lib/librgw.so.2.0.0) | | ->01.58% (1,816,896B) 0x51E7CB7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | ->01.58% (1,816,896B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | ->01.58% (1,816,896B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | ->01.58% (1,816,896B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | ->01.58% (1,816,896B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | ->01.58% (1,816,896B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | ->01.58% (1,816,896B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | ->01.58% (1,816,896B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | ->01.58% (1,816,896B) 0xF1A7492: start_thread (pthread_create.c:333) | | ->01.58% (1,816,896B) 0xFF3F93D: clone (clone.S:97) | | | ->00.02% (20,416B) in 1+ places, all below ms_print's threshold (01.00%) | ->01.55% (1,772,000B) 0x131429: worker_thread (civetweb.c:502) | ->01.55% (1,772,000B) 0xF1A7492: start_thread (pthread_create.c:333) | ->01.55% (1,772,000B) 0xFF3F93D: clone (clone.S:97) | ->01.00% (1,147,640B) 0x5103628: std::pair, bool> std::_Rb_tree, std::less, std::allocator >::_M_insert_unique(RGWCoroutinesStack* const&) (in /usr/lib/librgw.so.2.0.0) ->01.00% (1,147,640B) in 3 places, all below massif's threshold (1.00%) -------------------------------------------------------------------------------- n time(i) total(B) useful-heap(B) extra-heap(B) stacks(B) -------------------------------------------------------------------------------- 27 49,110,352,468 112,672,352 107,962,532 4,709,820 0 28 50,782,695,696 114,594,696 109,818,859 4,775,837 0 29 52,222,087,947 116,938,352 112,058,853 4,879,499 0 30 53,796,538,031 119,051,640 114,113,485 4,938,155 0 31 55,997,845,220 126,747,344 121,473,249 5,274,095 0 95.84% (121,473,249B) (heap allocation functions) malloc/new/new[], --alloc-fns, etc. ->40.26% (51,026,976B) 0x51E7C89: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | ->40.26% (51,026,976B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | ->40.26% (51,026,976B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | ->40.26% (51,026,976B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | ->40.26% (51,026,976B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | ->40.26% (51,026,976B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | ->40.26% (51,026,976B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | ->40.26% (51,026,976B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | ->40.26% (51,026,976B) 0xF1A7492: start_thread (pthread_create.c:333) | ->40.26% (51,026,976B) 0xFF3F93D: clone (clone.S:97) | ->19.52% (24,736,635B) 0xF9F4E97: std::string::_Rep::_S_create(unsigned long, unsigned long, std::allocator const&) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | ->17.91% (22,700,267B) 0xF9F5B09: std::string::_Rep::_M_clone(std::allocator const&, unsigned long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | ->17.89% (22,679,431B) 0xF9F5BAE: std::string::reserve(unsigned long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | ->13.73% (17,397,189B) 0xF9D1FB2: std::basic_stringbuf, std::allocator >::overflow(int) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->13.73% (17,397,189B) 0xF9D5D63: std::basic_streambuf >::xsputn(char const*, long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->13.73% (17,396,115B) 0xF9CD314: std::basic_ostream >& std::__ostream_insert >(std::basic_ostream >&, char const*, long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | | ->13.65% (17,306,436B) 0xF9CD6E5: std::basic_ostream >& std::operator<< >(std::basic_ostream >&, char const*) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | | | ->13.65% (17,298,918B) 0x5109AE4: RGWOmapAppend::operate() (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->13.65% (17,298,918B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->13.65% (17,298,918B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->13.65% (17,298,918B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->13.65% (17,298,918B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->13.65% (17,298,918B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->13.65% (17,298,918B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->13.65% (17,298,918B) 0xF1A7492: start_thread (pthread_create.c:333) | | | | | | | ->13.65% (17,298,918B) 0xFF3F93D: clone (clone.S:97) | | | | | | | | | | | | | ->00.01% (7,518B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | | | | | ->00.07% (89,679B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | | | ->00.00% (1,074B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | ->04.01% (5,083,198B) 0xF9F5E38: ??? (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->04.01% (5,083,198B) 0xF9F5F0C: std::string::append(char const*, unsigned long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->03.95% (5,001,046B) 0x51DF1E4: RGWDataSyncShardControlCR::alloc_cr() (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.95% (5,001,046B) 0x51AB72B: RGWBackoffControlCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.95% (5,001,046B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.95% (5,001,046B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.95% (5,001,046B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.95% (5,001,046B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.95% (5,001,046B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.95% (5,001,046B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | | | ->03.95% (5,001,046B) 0xF1A7492: start_thread (pthread_create.c:333) | | | | | ->03.95% (5,001,046B) 0xFF3F93D: clone (clone.S:97) | | | | | | | | | ->00.06% (82,152B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | ->00.16% (199,044B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | ->00.02% (20,836B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->01.46% (1,849,811B) 0xF9F6813: char* std::string::_S_construct(char const*, char const*, std::allocator const&, std::forward_iterator_tag) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | ->01.46% (1,849,640B) 0xF9F6C44: std::basic_string, std::allocator >::basic_string(char const*, std::allocator const&) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | ->01.31% (1,660,458B) 0x510DA32: RGWCoroutine::RGWCoroutine(CephContext*) (in /usr/lib/librgw.so.2.0.0) | | | | ->01.30% (1,642,914B) 0x51075C3: RGWOmapAppend::RGWOmapAppend(RGWAsyncRadosProcessor*, RGWRados*, rgw_bucket&, std::string const&, unsigned long) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.30% (1,642,914B) 0x51E7CB7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.30% (1,642,914B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.30% (1,642,914B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.30% (1,642,914B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.30% (1,642,914B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.30% (1,642,914B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.30% (1,642,914B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.30% (1,642,914B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.30% (1,642,914B) 0xF1A7492: start_thread (pthread_create.c:333) | | | | | ->01.30% (1,642,914B) 0xFF3F93D: clone (clone.S:97) | | | | | | | | | ->00.01% (17,544B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | ->00.15% (189,182B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | ->00.00% (171B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->00.15% (186,557B) in 1+ places, all below ms_print's threshold (01.00%) | ->13.15% (16,668,672B) 0x510DB36: RGWCoroutine::RGWCoroutine(CephContext*) (in /usr/lib/librgw.so.2.0.0) | ->13.01% (16,493,056B) 0x51075C3: RGWOmapAppend::RGWOmapAppend(RGWAsyncRadosProcessor*, RGWRados*, rgw_bucket&, std::string const&, unsigned long) (in /usr/lib/librgw.so.2.0.0) | | ->13.01% (16,493,056B) 0x51E7CB7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | ->13.01% (16,493,056B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | ->13.01% (16,493,056B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | ->13.01% (16,493,056B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | ->13.01% (16,493,056B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | ->13.01% (16,493,056B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | ->13.01% (16,493,056B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | ->13.01% (16,493,056B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | ->13.01% (16,493,056B) 0xF1A7492: start_thread (pthread_create.c:333) | | ->13.01% (16,493,056B) 0xFF3F93D: clone (clone.S:97) | | | ->00.14% (175,616B) in 1+ places, all below ms_print's threshold (01.00%) | ->11.42% (14,471,744B) 0x50F9E51: RGWCoroutinesManager::allocate_stack() (in /usr/lib/librgw.so.2.0.0) | ->11.42% (14,471,520B) 0x50FE7AE: RGWCoroutinesStack::spawn(RGWCoroutine*, RGWCoroutine*, bool) (in /usr/lib/librgw.so.2.0.0) | | ->05.69% (7,215,936B) 0x51E7CD7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | | ->05.69% (7,215,936B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | ->05.69% (7,215,936B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.69% (7,215,936B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.69% (7,215,936B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.69% (7,215,936B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.69% (7,215,936B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | ->05.69% (7,215,936B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | ->05.69% (7,215,936B) 0xF1A7492: start_thread (pthread_create.c:333) | | | ->05.69% (7,215,936B) 0xFF3F93D: clone (clone.S:97) | | | | | ->05.69% (7,215,936B) 0x51D5E5F: RGWDataSyncShardCR::init_lease_cr() (in /usr/lib/librgw.so.2.0.0) | | | ->05.69% (7,215,936B) 0x51E7CEE: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | | ->05.69% (7,215,936B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | ->05.69% (7,215,936B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.69% (7,215,936B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.69% (7,215,936B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.69% (7,215,936B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.69% (7,215,936B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | ->05.69% (7,215,936B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | ->05.69% (7,215,936B) 0xF1A7492: start_thread (pthread_create.c:333) | | | ->05.69% (7,215,936B) 0xFF3F93D: clone (clone.S:97) | | | | | ->00.03% (39,648B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->00.00% (224B) in 1+ places, all below ms_print's threshold (01.00%) | ->05.77% (7,315,838B) in 848 places, all below massif's threshold (1.00%) | ->01.65% (2,096,872B) 0x54F757B: ceph::log::Log::create_entry(int, int, unsigned long*) (in /usr/lib/librgw.so.2.0.0) | ->01.05% (1,330,390B) 0x52CD4D0: RGWRequest::log(req_state*, char const*) (in /usr/lib/librgw.so.2.0.0) | | ->01.05% (1,330,390B) in 13 places, all below massif's threshold (1.00%) | | | ->00.60% (766,482B) in 1+ places, all below ms_print's threshold (01.00%) | ->01.64% (2,083,712B) 0x510DB0E: RGWCoroutine::RGWCoroutine(CephContext*) (in /usr/lib/librgw.so.2.0.0) | ->01.63% (2,061,696B) 0x51075C3: RGWOmapAppend::RGWOmapAppend(RGWAsyncRadosProcessor*, RGWRados*, rgw_bucket&, std::string const&, unsigned long) (in /usr/lib/librgw.so.2.0.0) | | ->01.63% (2,061,696B) 0x51E7CB7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | ->01.63% (2,061,696B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | ->01.63% (2,061,696B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | ->01.63% (2,061,696B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | ->01.63% (2,061,696B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | ->01.63% (2,061,696B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | ->01.63% (2,061,696B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | ->01.63% (2,061,696B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | ->01.63% (2,061,696B) 0xF1A7492: start_thread (pthread_create.c:333) | | ->01.63% (2,061,696B) 0xFF3F93D: clone (clone.S:97) | | | ->00.02% (22,016B) in 1+ places, all below ms_print's threshold (01.00%) | ->01.40% (1,772,000B) 0x131429: worker_thread (civetweb.c:502) | ->01.40% (1,772,000B) 0xF1A7492: start_thread (pthread_create.c:333) | ->01.40% (1,772,000B) 0xFF3F93D: clone (clone.S:97) | ->01.03% (1,300,800B) 0x5103628: std::pair, bool> std::_Rb_tree, std::less, std::allocator >::_M_insert_unique(RGWCoroutinesStack* const&) (in /usr/lib/librgw.so.2.0.0) ->01.02% (1,295,640B) 0x50FE628: RGWCoroutinesManager::schedule(RGWCoroutinesEnv*, RGWCoroutinesStack*) (in /usr/lib/librgw.so.2.0.0) | ->01.02% (1,295,640B) 0x50FE805: RGWCoroutinesStack::spawn(RGWCoroutine*, RGWCoroutine*, bool) (in /usr/lib/librgw.so.2.0.0) | ->01.02% (1,288,560B) 0x51E7CD7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | ->01.02% (1,288,560B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | ->01.02% (1,288,560B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | ->01.02% (1,288,560B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | ->01.02% (1,288,560B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | ->01.02% (1,288,560B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | ->01.02% (1,288,560B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | ->01.02% (1,288,560B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | ->01.02% (1,288,560B) 0xF1A7492: start_thread (pthread_create.c:333) | | ->01.02% (1,288,560B) 0xFF3F93D: clone (clone.S:97) | | | ->00.01% (7,080B) in 1+ places, all below ms_print's threshold (01.00%) | ->00.00% (5,160B) in 1+ places, all below ms_print's threshold (01.00%) -------------------------------------------------------------------------------- n time(i) total(B) useful-heap(B) extra-heap(B) stacks(B) -------------------------------------------------------------------------------- 32 57,548,675,534 128,781,216 123,447,168 5,334,048 0 33 59,778,475,299 129,076,040 123,716,824 5,359,216 0 34 62,085,421,221 133,020,520 127,471,544 5,548,976 0 35 64,471,330,243 137,649,376 131,995,025 5,654,351 0 95.89% (131,995,025B) (heap allocation functions) malloc/new/new[], --alloc-fns, etc. ->41.07% (56,531,376B) 0x51E7C89: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | ->41.07% (56,531,376B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | ->41.07% (56,531,376B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | ->41.07% (56,531,376B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | ->41.07% (56,531,376B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | ->41.07% (56,531,376B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | ->41.07% (56,531,376B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | ->41.07% (56,531,376B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | ->41.07% (56,531,376B) 0xF1A7492: start_thread (pthread_create.c:333) | ->41.07% (56,531,376B) 0xFF3F93D: clone (clone.S:97) | ->19.65% (27,042,463B) 0xF9F4E97: std::string::_Rep::_S_create(unsigned long, unsigned long, std::allocator const&) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | ->18.14% (24,962,958B) 0xF9F5B09: std::string::_Rep::_M_clone(std::allocator const&, unsigned long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | ->18.13% (24,953,421B) 0xF9F5BAE: std::string::reserve(unsigned long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | ->13.99% (19,254,135B) 0xF9D1FB2: std::basic_stringbuf, std::allocator >::overflow(int) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->13.99% (19,254,135B) 0xF9D5D63: std::basic_streambuf >::xsputn(char const*, long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->13.99% (19,253,598B) 0xF9CD314: std::basic_ostream >& std::__ostream_insert >(std::basic_ostream >&, char const*, long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | | ->13.92% (19,164,993B) 0xF9CD6E5: std::basic_ostream >& std::operator<< >(std::basic_ostream >&, char const*) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | | | ->13.92% (19,164,993B) 0x5109AE4: RGWOmapAppend::operate() (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->13.92% (19,164,993B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->13.92% (19,164,993B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->13.92% (19,164,993B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->13.92% (19,164,993B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->13.92% (19,164,993B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->13.92% (19,164,993B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->13.92% (19,164,993B) 0xF1A7492: start_thread (pthread_create.c:333) | | | | | | | ->13.92% (19,164,993B) 0xFF3F93D: clone (clone.S:97) | | | | | | | | | | | | | ->00.00% (0B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | | | | | ->00.06% (88,605B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | | | ->00.00% (537B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | ->04.07% (5,607,321B) 0xF9F5E38: ??? (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->04.07% (5,607,321B) 0xF9F5F0C: std::string::append(char const*, unsigned long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->04.03% (5,540,797B) 0x51DF1E4: RGWDataSyncShardControlCR::alloc_cr() (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.03% (5,540,797B) 0x51AB72B: RGWBackoffControlCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.03% (5,540,797B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.03% (5,540,797B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.03% (5,540,797B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.03% (5,540,797B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.03% (5,540,797B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.03% (5,540,797B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.03% (5,540,797B) 0xF1A7492: start_thread (pthread_create.c:333) | | | | | ->04.03% (5,540,797B) 0xFF3F93D: clone (clone.S:97) | | | | | | | | | ->00.05% (66,524B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | ->00.07% (91,965B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | ->00.01% (9,537B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->01.43% (1,967,302B) 0xF9F6813: char* std::string::_S_construct(char const*, char const*, std::allocator const&, std::forward_iterator_tag) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | ->01.43% (1,967,302B) 0xF9F6C44: std::basic_string, std::allocator >::basic_string(char const*, std::allocator const&) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | ->01.33% (1,833,246B) 0x510DA32: RGWCoroutine::RGWCoroutine(CephContext*) (in /usr/lib/librgw.so.2.0.0) | | | | ->01.32% (1,820,139B) 0x51075C3: RGWOmapAppend::RGWOmapAppend(RGWAsyncRadosProcessor*, RGWRados*, rgw_bucket&, std::string const&, unsigned long) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.32% (1,820,139B) 0x51E7CB7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.32% (1,820,139B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.32% (1,820,139B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.32% (1,820,139B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.32% (1,820,139B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.32% (1,820,139B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.32% (1,820,139B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.32% (1,820,139B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.32% (1,820,139B) 0xF1A7492: start_thread (pthread_create.c:333) | | | | | ->01.32% (1,820,139B) 0xFF3F93D: clone (clone.S:97) | | | | | | | | | ->00.01% (13,107B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | ->00.10% (134,056B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | ->00.00% (0B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->00.08% (112,203B) in 1+ places, all below ms_print's threshold (01.00%) | ->13.37% (18,403,328B) 0x510DB36: RGWCoroutine::RGWCoroutine(CephContext*) (in /usr/lib/librgw.so.2.0.0) | ->13.27% (18,272,256B) 0x51075C3: RGWOmapAppend::RGWOmapAppend(RGWAsyncRadosProcessor*, RGWRados*, rgw_bucket&, std::string const&, unsigned long) (in /usr/lib/librgw.so.2.0.0) | | ->13.27% (18,272,256B) 0x51E7CB7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | ->13.27% (18,272,256B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | ->13.27% (18,272,256B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | ->13.27% (18,272,256B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | ->13.27% (18,272,256B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | ->13.27% (18,272,256B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | ->13.27% (18,272,256B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | ->13.27% (18,272,256B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | ->13.27% (18,272,256B) 0xF1A7492: start_thread (pthread_create.c:333) | | ->13.27% (18,272,256B) 0xFF3F93D: clone (clone.S:97) | | | ->00.10% (131,072B) in 1+ places, all below ms_print's threshold (01.00%) | ->11.64% (16,024,288B) 0x50F9E51: RGWCoroutinesManager::allocate_stack() (in /usr/lib/librgw.so.2.0.0) | ->11.64% (16,024,064B) 0x50FE7AE: RGWCoroutinesStack::spawn(RGWCoroutine*, RGWCoroutine*, bool) (in /usr/lib/librgw.so.2.0.0) | | ->05.81% (7,994,336B) 0x51E7CD7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | | ->05.81% (7,994,336B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | ->05.81% (7,994,336B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.81% (7,994,336B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.81% (7,994,336B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.81% (7,994,336B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.81% (7,994,336B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | ->05.81% (7,994,336B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | ->05.81% (7,994,336B) 0xF1A7492: start_thread (pthread_create.c:333) | | | ->05.81% (7,994,336B) 0xFF3F93D: clone (clone.S:97) | | | | | ->05.81% (7,994,336B) 0x51D5E5F: RGWDataSyncShardCR::init_lease_cr() (in /usr/lib/librgw.so.2.0.0) | | | ->05.81% (7,994,336B) 0x51E7CEE: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | | ->05.81% (7,994,336B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | ->05.81% (7,994,336B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.81% (7,994,336B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.81% (7,994,336B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.81% (7,994,336B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.81% (7,994,336B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | ->05.81% (7,994,336B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | ->05.81% (7,994,336B) 0xF1A7492: start_thread (pthread_create.c:333) | | | ->05.81% (7,994,336B) 0xFF3F93D: clone (clone.S:97) | | | | | ->00.03% (35,392B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->00.00% (224B) in 1+ places, all below ms_print's threshold (01.00%) | ->03.95% (5,441,871B) in 861 places, all below massif's threshold (1.00%) | ->01.67% (2,300,544B) 0x510DB0E: RGWCoroutine::RGWCoroutine(CephContext*) (in /usr/lib/librgw.so.2.0.0) | ->01.66% (2,284,096B) 0x51075C3: RGWOmapAppend::RGWOmapAppend(RGWAsyncRadosProcessor*, RGWRados*, rgw_bucket&, std::string const&, unsigned long) (in /usr/lib/librgw.so.2.0.0) | | ->01.66% (2,284,096B) 0x51E7CB7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | ->01.66% (2,284,096B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | ->01.66% (2,284,096B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | ->01.66% (2,284,096B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | ->01.66% (2,284,096B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | ->01.66% (2,284,096B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | ->01.66% (2,284,096B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | ->01.66% (2,284,096B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | ->01.66% (2,284,096B) 0xF1A7492: start_thread (pthread_create.c:333) | | ->01.66% (2,284,096B) 0xFF3F93D: clone (clone.S:97) | | | ->00.01% (16,448B) in 1+ places, all below ms_print's threshold (01.00%) | ->01.29% (1,772,000B) 0x131429: worker_thread (civetweb.c:502) | ->01.29% (1,772,000B) 0xF1A7492: start_thread (pthread_create.c:333) | ->01.29% (1,772,000B) 0xFF3F93D: clone (clone.S:97) | ->01.18% (1,624,402B) 0x54F757B: ceph::log::Log::create_entry(int, int, unsigned long*) (in /usr/lib/librgw.so.2.0.0) | ->01.18% (1,624,402B) in 51 places, all below massif's threshold (1.00%) | ->01.05% (1,439,480B) 0x5103628: std::pair, bool> std::_Rb_tree, std::less, std::allocator >::_M_insert_unique(RGWCoroutinesStack* const&) (in /usr/lib/librgw.so.2.0.0) | ->01.04% (1,434,320B) 0x50FE628: RGWCoroutinesManager::schedule(RGWCoroutinesEnv*, RGWCoroutinesStack*) (in /usr/lib/librgw.so.2.0.0) | | ->01.04% (1,434,320B) 0x50FE805: RGWCoroutinesStack::spawn(RGWCoroutine*, RGWCoroutine*, bool) (in /usr/lib/librgw.so.2.0.0) | | ->01.04% (1,427,560B) 0x51E7CD7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | | ->01.04% (1,427,560B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | ->01.04% (1,427,560B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | ->01.04% (1,427,560B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | ->01.04% (1,427,560B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | ->01.04% (1,427,560B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | ->01.04% (1,427,560B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | ->01.04% (1,427,560B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | ->01.04% (1,427,560B) 0xF1A7492: start_thread (pthread_create.c:333) | | | ->01.04% (1,427,560B) 0xFF3F93D: clone (clone.S:97) | | | | | ->00.00% (6,760B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->00.00% (5,160B) in 1+ places, all below ms_print's threshold (01.00%) | ->01.03% (1,415,273B) 0x10623B1B: ??? (in /usr/lib/librados.so.2.0.0) ->01.03% (1,415,273B) in 42 places, all below massif's threshold (1.00%) -------------------------------------------------------------------------------- n time(i) total(B) useful-heap(B) extra-heap(B) stacks(B) -------------------------------------------------------------------------------- 36 67,177,195,108 143,564,152 137,620,947 5,943,205 0 95.86% (137,620,947B) (heap allocation functions) malloc/new/new[], --alloc-fns, etc. ->41.05% (58,926,384B) 0x51E7C89: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | ->41.05% (58,926,384B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | ->41.05% (58,926,384B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | ->41.05% (58,926,384B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | ->41.05% (58,926,384B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | ->41.05% (58,926,384B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | ->41.05% (58,926,384B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | ->41.05% (58,926,384B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | ->41.05% (58,926,384B) 0xF1A7492: start_thread (pthread_create.c:333) | ->41.05% (58,926,384B) 0xFF3F93D: clone (clone.S:97) | ->19.74% (28,344,067B) 0xF9F4E97: std::string::_Rep::_S_create(unsigned long, unsigned long, std::allocator const&) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | ->18.15% (26,057,456B) 0xF9F5B09: std::string::_Rep::_M_clone(std::allocator const&, unsigned long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | ->18.13% (26,033,399B) 0xF9F5BAE: std::string::reserve(unsigned long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | ->13.97% (20,058,024B) 0xF9D1FB2: std::basic_stringbuf, std::allocator >::overflow(int) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->13.97% (20,058,024B) 0xF9D5D63: std::basic_streambuf >::xsputn(char const*, long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->13.97% (20,057,487B) 0xF9CD314: std::basic_ostream >& std::__ostream_insert >(std::basic_ostream >&, char const*, long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | | ->13.91% (19,973,178B) 0xF9CD6E5: std::basic_ostream >& std::operator<< >(std::basic_ostream >&, char const*) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | | | ->13.91% (19,973,178B) 0x5109AE4: RGWOmapAppend::operate() (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->13.91% (19,973,178B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->13.91% (19,973,178B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->13.91% (19,973,178B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->13.91% (19,973,178B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->13.91% (19,973,178B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->13.91% (19,973,178B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->13.91% (19,973,178B) 0xF1A7492: start_thread (pthread_create.c:333) | | | | | | | ->13.91% (19,973,178B) 0xFF3F93D: clone (clone.S:97) | | | | | | | | | | | | | ->00.00% (0B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | | | | | ->00.06% (84,309B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | | | ->00.00% (537B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | ->04.08% (5,860,496B) 0xF9F5E38: ??? (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->04.08% (5,860,496B) 0xF9F5F0C: std::string::append(char const*, unsigned long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->04.02% (5,775,353B) 0x51DF1E4: RGWDataSyncShardControlCR::alloc_cr() (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.02% (5,775,353B) 0x51AB72B: RGWBackoffControlCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.02% (5,775,353B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.02% (5,775,353B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.02% (5,775,353B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.02% (5,775,353B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.02% (5,775,353B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.02% (5,775,353B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.02% (5,775,353B) 0xF1A7492: start_thread (pthread_create.c:333) | | | | | ->04.02% (5,775,353B) 0xFF3F93D: clone (clone.S:97) | | | | | | | | | ->00.06% (85,143B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | ->00.08% (114,879B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | ->00.02% (24,057B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->01.46% (2,091,067B) 0xF9F6813: char* std::string::_S_construct(char const*, char const*, std::allocator const&, std::forward_iterator_tag) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | ->01.46% (2,091,067B) 0xF9F6C44: std::basic_string, std::allocator >::basic_string(char const*, std::allocator const&) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | ->01.33% (1,911,582B) 0x510DA32: RGWCoroutine::RGWCoroutine(CephContext*) (in /usr/lib/librgw.so.2.0.0) | | | | ->01.32% (1,897,251B) 0x51075C3: RGWOmapAppend::RGWOmapAppend(RGWAsyncRadosProcessor*, RGWRados*, rgw_bucket&, std::string const&, unsigned long) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.32% (1,897,251B) 0x51E7CB7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.32% (1,897,251B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.32% (1,897,251B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.32% (1,897,251B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.32% (1,897,251B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.32% (1,897,251B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.32% (1,897,251B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.32% (1,897,251B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.32% (1,897,251B) 0xF1A7492: start_thread (pthread_create.c:333) | | | | | ->01.32% (1,897,251B) 0xFF3F93D: clone (clone.S:97) | | | | | | | | | ->00.01% (14,331B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | ->00.13% (179,485B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | ->00.00% (0B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->00.14% (195,544B) in 1+ places, all below ms_print's threshold (01.00%) | ->13.37% (19,189,760B) 0x510DB36: RGWCoroutine::RGWCoroutine(CephContext*) (in /usr/lib/librgw.so.2.0.0) | ->13.27% (19,046,400B) 0x51075C3: RGWOmapAppend::RGWOmapAppend(RGWAsyncRadosProcessor*, RGWRados*, rgw_bucket&, std::string const&, unsigned long) (in /usr/lib/librgw.so.2.0.0) | | ->13.27% (19,046,400B) 0x51E7CB7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | ->13.27% (19,046,400B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | ->13.27% (19,046,400B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | ->13.27% (19,046,400B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | ->13.27% (19,046,400B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | ->13.27% (19,046,400B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | ->13.27% (19,046,400B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | ->13.27% (19,046,400B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | ->13.27% (19,046,400B) 0xF1A7492: start_thread (pthread_create.c:333) | | ->13.27% (19,046,400B) 0xFF3F93D: clone (clone.S:97) | | | ->00.10% (143,360B) in 1+ places, all below ms_print's threshold (01.00%) | ->11.63% (16,701,664B) 0x50F9E51: RGWCoroutinesManager::allocate_stack() (in /usr/lib/librgw.so.2.0.0) | ->11.63% (16,701,440B) 0x50FE7AE: RGWCoroutinesStack::spawn(RGWCoroutine*, RGWCoroutine*, bool) (in /usr/lib/librgw.so.2.0.0) | | ->05.80% (8,333,024B) 0x51E7CD7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | | ->05.80% (8,333,024B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | ->05.80% (8,333,024B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.80% (8,333,024B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.80% (8,333,024B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.80% (8,333,024B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.80% (8,333,024B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | ->05.80% (8,333,024B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | ->05.80% (8,333,024B) 0xF1A7492: start_thread (pthread_create.c:333) | | | ->05.80% (8,333,024B) 0xFF3F93D: clone (clone.S:97) | | | | | ->05.80% (8,333,024B) 0x51D5E5F: RGWDataSyncShardCR::init_lease_cr() (in /usr/lib/librgw.so.2.0.0) | | | ->05.80% (8,333,024B) 0x51E7CEE: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | | ->05.80% (8,333,024B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | ->05.80% (8,333,024B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.80% (8,333,024B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.80% (8,333,024B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.80% (8,333,024B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.80% (8,333,024B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | ->05.80% (8,333,024B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | ->05.80% (8,333,024B) 0xF1A7492: start_thread (pthread_create.c:333) | | | ->05.80% (8,333,024B) 0xFF3F93D: clone (clone.S:97) | | | | | ->00.02% (35,392B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->00.00% (224B) in 1+ places, all below ms_print's threshold (01.00%) | ->04.66% (6,685,360B) in 863 places, all below massif's threshold (1.00%) | ->01.67% (2,398,848B) 0x510DB0E: RGWCoroutine::RGWCoroutine(CephContext*) (in /usr/lib/librgw.so.2.0.0) | ->01.66% (2,380,864B) 0x51075C3: RGWOmapAppend::RGWOmapAppend(RGWAsyncRadosProcessor*, RGWRados*, rgw_bucket&, std::string const&, unsigned long) (in /usr/lib/librgw.so.2.0.0) | | ->01.66% (2,380,864B) 0x51E7CB7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | ->01.66% (2,380,864B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | ->01.66% (2,380,864B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | ->01.66% (2,380,864B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | ->01.66% (2,380,864B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | ->01.66% (2,380,864B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | ->01.66% (2,380,864B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | ->01.66% (2,380,864B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | ->01.66% (2,380,864B) 0xF1A7492: start_thread (pthread_create.c:333) | | ->01.66% (2,380,864B) 0xFF3F93D: clone (clone.S:97) | | | ->00.01% (17,984B) in 1+ places, all below ms_print's threshold (01.00%) | ->01.46% (2,102,744B) 0x54F757B: ceph::log::Log::create_entry(int, int, unsigned long*) (in /usr/lib/librgw.so.2.0.0) | ->01.46% (2,102,744B) in 51 places, all below massif's threshold (1.00%) | ->01.23% (1,772,000B) 0x131429: worker_thread (civetweb.c:502) | ->01.23% (1,772,000B) 0xF1A7492: start_thread (pthread_create.c:333) | ->01.23% (1,772,000B) 0xFF3F93D: clone (clone.S:97) | ->01.04% (1,500,120B) 0x5103628: std::pair, bool> std::_Rb_tree, std::less, std::allocator >::_M_insert_unique(RGWCoroutinesStack* const&) (in /usr/lib/librgw.so.2.0.0) ->01.04% (1,494,960B) 0x50FE628: RGWCoroutinesManager::schedule(RGWCoroutinesEnv*, RGWCoroutinesStack*) (in /usr/lib/librgw.so.2.0.0) | ->01.04% (1,494,960B) 0x50FE805: RGWCoroutinesStack::spawn(RGWCoroutine*, RGWCoroutine*, bool) (in /usr/lib/librgw.so.2.0.0) | ->01.04% (1,488,040B) 0x51E7CD7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | ->01.04% (1,488,040B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | ->01.04% (1,488,040B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | ->01.04% (1,488,040B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | ->01.04% (1,488,040B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | ->01.04% (1,488,040B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | ->01.04% (1,488,040B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | ->01.04% (1,488,040B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | ->01.04% (1,488,040B) 0xF1A7492: start_thread (pthread_create.c:333) | | ->01.04% (1,488,040B) 0xFF3F93D: clone (clone.S:97) | | | ->00.00% (6,920B) in 1+ places, all below ms_print's threshold (01.00%) | ->00.00% (5,160B) in 1+ places, all below ms_print's threshold (01.00%) -------------------------------------------------------------------------------- n time(i) total(B) useful-heap(B) extra-heap(B) stacks(B) -------------------------------------------------------------------------------- 37 68,773,477,908 145,169,792 139,203,853 5,965,939 0 38 70,744,537,833 149,432,944 143,351,021 6,081,923 0 95.93% (143,351,021B) (heap allocation functions) malloc/new/new[], --alloc-fns, etc. ->41.83% (62,507,808B) 0x51E7C89: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | ->41.83% (62,507,808B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | ->41.83% (62,507,808B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | ->41.83% (62,507,808B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | ->41.83% (62,507,808B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | ->41.83% (62,507,808B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | ->41.83% (62,507,808B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | ->41.83% (62,507,808B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | ->41.83% (62,507,808B) 0xF1A7492: start_thread (pthread_create.c:333) | ->41.83% (62,507,808B) 0xFF3F93D: clone (clone.S:97) | ->19.84% (29,641,083B) 0xF9F4E97: std::string::_Rep::_S_create(unsigned long, unsigned long, std::allocator const&) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | ->18.38% (27,465,075B) 0xF9F5B09: std::string::_Rep::_M_clone(std::allocator const&, unsigned long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | ->18.38% (27,462,604B) 0xF9F5BAE: std::string::reserve(unsigned long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | ->14.20% (21,212,037B) 0xF9D1FB2: std::basic_stringbuf, std::allocator >::overflow(int) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->14.20% (21,212,037B) 0xF9D5D63: std::basic_streambuf >::xsputn(char const*, long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->14.19% (21,211,500B) 0xF9CD314: std::basic_ostream >& std::__ostream_insert >(std::basic_ostream >&, char const*, long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | | ->14.18% (21,191,094B) 0xF9CD6E5: std::basic_ostream >& std::operator<< >(std::basic_ostream >&, char const*) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | | | ->14.18% (21,191,094B) 0x5109AE4: RGWOmapAppend::operate() (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->14.18% (21,191,094B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->14.18% (21,191,094B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->14.18% (21,191,094B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->14.18% (21,191,094B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->14.18% (21,191,094B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->14.18% (21,191,094B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->14.18% (21,191,094B) 0xF1A7492: start_thread (pthread_create.c:333) | | | | | | | ->14.18% (21,191,094B) 0xFF3F93D: clone (clone.S:97) | | | | | | | | | | | | | ->00.00% (0B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | | | | | ->00.01% (20,406B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | | | ->00.00% (537B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | ->04.13% (6,173,469B) 0xF9F5E38: ??? (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->04.13% (6,173,469B) 0xF9F5F0C: std::string::append(char const*, unsigned long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->04.10% (6,125,628B) 0x51DF1E4: RGWDataSyncShardControlCR::alloc_cr() (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.10% (6,125,628B) 0x51AB72B: RGWBackoffControlCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.10% (6,125,628B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.10% (6,125,628B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.10% (6,125,628B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.10% (6,125,628B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.10% (6,125,628B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.10% (6,125,628B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.10% (6,125,628B) 0xF1A7492: start_thread (pthread_create.c:333) | | | | | ->04.10% (6,125,628B) 0xFF3F93D: clone (clone.S:97) | | | | | | | | | ->00.03% (47,841B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | ->00.05% (77,098B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | ->00.00% (2,471B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->01.41% (2,110,082B) 0xF9F6813: char* std::string::_S_construct(char const*, char const*, std::allocator const&, std::forward_iterator_tag) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | ->01.41% (2,110,082B) 0xF9F6C44: std::basic_string, std::allocator >::basic_string(char const*, std::allocator const&) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | ->01.35% (2,019,549B) 0x510DA32: RGWCoroutine::RGWCoroutine(CephContext*) (in /usr/lib/librgw.so.2.0.0) | | | | ->01.35% (2,012,562B) 0x51075C3: RGWOmapAppend::RGWOmapAppend(RGWAsyncRadosProcessor*, RGWRados*, rgw_bucket&, std::string const&, unsigned long) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.35% (2,012,562B) 0x51E7CB7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.35% (2,012,562B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.35% (2,012,562B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.35% (2,012,562B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.35% (2,012,562B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.35% (2,012,562B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.35% (2,012,562B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.35% (2,012,562B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.35% (2,012,562B) 0xF1A7492: start_thread (pthread_create.c:333) | | | | | ->01.35% (2,012,562B) 0xFF3F93D: clone (clone.S:97) | | | | | | | | | ->00.00% (6,987B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | ->00.06% (90,533B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | ->00.00% (0B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->00.04% (65,926B) in 1+ places, all below ms_print's threshold (01.00%) | ->13.57% (20,273,664B) 0x510DB36: RGWCoroutine::RGWCoroutine(CephContext*) (in /usr/lib/librgw.so.2.0.0) | ->13.52% (20,204,032B) 0x51075C3: RGWOmapAppend::RGWOmapAppend(RGWAsyncRadosProcessor*, RGWRados*, rgw_bucket&, std::string const&, unsigned long) (in /usr/lib/librgw.so.2.0.0) | | ->13.52% (20,204,032B) 0x51E7CB7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | ->13.52% (20,204,032B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | ->13.52% (20,204,032B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | ->13.52% (20,204,032B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | ->13.52% (20,204,032B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | ->13.52% (20,204,032B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | ->13.52% (20,204,032B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | ->13.52% (20,204,032B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | ->13.52% (20,204,032B) 0xF1A7492: start_thread (pthread_create.c:333) | | ->13.52% (20,204,032B) 0xFF3F93D: clone (clone.S:97) | | | ->00.05% (69,632B) in 1+ places, all below ms_print's threshold (01.00%) | ->11.85% (17,714,592B) 0x50F9E51: RGWCoroutinesManager::allocate_stack() (in /usr/lib/librgw.so.2.0.0) | ->11.85% (17,714,368B) 0x50FE7AE: RGWCoroutinesStack::spawn(RGWCoroutine*, RGWCoroutine*, bool) (in /usr/lib/librgw.so.2.0.0) | | ->05.92% (8,839,488B) 0x51E7CD7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | | ->05.92% (8,839,488B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | ->05.92% (8,839,488B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.92% (8,839,488B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.92% (8,839,488B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.92% (8,839,488B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.92% (8,839,488B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | ->05.92% (8,839,488B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | ->05.92% (8,839,488B) 0xF1A7492: start_thread (pthread_create.c:333) | | | ->05.92% (8,839,488B) 0xFF3F93D: clone (clone.S:97) | | | | | ->05.92% (8,839,488B) 0x51D5E5F: RGWDataSyncShardCR::init_lease_cr() (in /usr/lib/librgw.so.2.0.0) | | | ->05.92% (8,839,488B) 0x51E7CEE: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | | ->05.92% (8,839,488B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | ->05.92% (8,839,488B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.92% (8,839,488B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.92% (8,839,488B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.92% (8,839,488B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.92% (8,839,488B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | ->05.92% (8,839,488B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | ->05.92% (8,839,488B) 0xF1A7492: start_thread (pthread_create.c:333) | | | ->05.92% (8,839,488B) 0xFF3F93D: clone (clone.S:97) | | | | | ->00.02% (35,392B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->00.00% (224B) in 1+ places, all below ms_print's threshold (01.00%) | ->02.80% (4,190,529B) in 862 places, all below massif's threshold (1.00%) | ->01.70% (2,534,336B) 0x510DB0E: RGWCoroutine::RGWCoroutine(CephContext*) (in /usr/lib/librgw.so.2.0.0) | ->01.69% (2,525,568B) 0x51075C3: RGWOmapAppend::RGWOmapAppend(RGWAsyncRadosProcessor*, RGWRados*, rgw_bucket&, std::string const&, unsigned long) (in /usr/lib/librgw.so.2.0.0) | | ->01.69% (2,525,568B) 0x51E7CB7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | ->01.69% (2,525,568B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | ->01.69% (2,525,568B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | ->01.69% (2,525,568B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | ->01.69% (2,525,568B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | ->01.69% (2,525,568B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | ->01.69% (2,525,568B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | ->01.69% (2,525,568B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | ->01.69% (2,525,568B) 0xF1A7492: start_thread (pthread_create.c:333) | | ->01.69% (2,525,568B) 0xFF3F93D: clone (clone.S:97) | | | ->00.01% (8,768B) in 1+ places, all below ms_print's threshold (01.00%) | ->01.19% (1,772,000B) 0x131429: worker_thread (civetweb.c:502) | ->01.19% (1,772,000B) 0xF1A7492: start_thread (pthread_create.c:333) | ->01.19% (1,772,000B) 0xFF3F93D: clone (clone.S:97) | ->01.09% (1,631,726B) 0x10623B1B: ??? (in /usr/lib/librados.so.2.0.0) | ->01.09% (1,631,726B) in 42 places, all below massif's threshold (1.00%) | ->01.06% (1,588,840B) 0x5103628: std::pair, bool> std::_Rb_tree, std::less, std::allocator >::_M_insert_unique(RGWCoroutinesStack* const&) (in /usr/lib/librgw.so.2.0.0) | ->01.06% (1,583,680B) 0x50FE628: RGWCoroutinesManager::schedule(RGWCoroutinesEnv*, RGWCoroutinesStack*) (in /usr/lib/librgw.so.2.0.0) | | ->01.06% (1,583,680B) 0x50FE805: RGWCoroutinesStack::spawn(RGWCoroutine*, RGWCoroutine*, bool) (in /usr/lib/librgw.so.2.0.0) | | ->01.06% (1,578,480B) 0x51E7CD7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | | ->01.06% (1,578,480B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | ->01.06% (1,578,480B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | ->01.06% (1,578,480B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | ->01.06% (1,578,480B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | ->01.06% (1,578,480B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | ->01.06% (1,578,480B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | ->01.06% (1,578,480B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | ->01.06% (1,578,480B) 0xF1A7492: start_thread (pthread_create.c:333) | | | ->01.06% (1,578,480B) 0xFF3F93D: clone (clone.S:97) | | | | | ->00.00% (5,200B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->00.00% (5,160B) in 1+ places, all below ms_print's threshold (01.00%) | ->01.00% (1,496,443B) 0x54F757B: ceph::log::Log::create_entry(int, int, unsigned long*) (in /usr/lib/librgw.so.2.0.0) ->01.00% (1,496,443B) in 51 places, all below massif's threshold (1.00%) -------------------------------------------------------------------------------- n time(i) total(B) useful-heap(B) extra-heap(B) stacks(B) -------------------------------------------------------------------------------- 39 72,506,239,645 155,678,952 149,334,120 6,344,832 0 95.92% (149,334,120B) (heap allocation functions) malloc/new/new[], --alloc-fns, etc. ->41.85% (65,153,088B) 0x51E7C89: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | ->41.85% (65,153,088B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | ->41.85% (65,153,088B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | ->41.85% (65,153,088B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | ->41.85% (65,153,088B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | ->41.85% (65,153,088B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | ->41.85% (65,153,088B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | ->41.85% (65,153,088B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | ->41.85% (65,153,088B) 0xF1A7492: start_thread (pthread_create.c:333) | ->41.85% (65,153,088B) 0xFF3F93D: clone (clone.S:97) | ->19.86% (30,910,647B) 0xF9F4E97: std::string::_Rep::_S_create(unsigned long, unsigned long, std::allocator const&) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | ->18.40% (28,644,986B) 0xF9F5B09: std::string::_Rep::_M_clone(std::allocator const&, unsigned long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | ->18.40% (28,642,137B) 0xF9F5BAE: std::string::reserve(unsigned long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | ->14.20% (22,108,290B) 0xF9D1FB2: std::basic_stringbuf, std::allocator >::overflow(int) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->14.20% (22,108,290B) 0xF9D5D63: std::basic_streambuf >::xsputn(char const*, long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->14.20% (22,107,753B) 0xF9CD314: std::basic_ostream >& std::__ostream_insert >(std::basic_ostream >&, char const*, long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | | ->14.19% (22,087,884B) 0xF9CD6E5: std::basic_ostream >& std::operator<< >(std::basic_ostream >&, char const*) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | | | ->14.19% (22,087,884B) 0x5109AE4: RGWOmapAppend::operate() (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->14.19% (22,087,884B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->14.19% (22,087,884B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->14.19% (22,087,884B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->14.19% (22,087,884B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->14.19% (22,087,884B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->14.19% (22,087,884B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->14.19% (22,087,884B) 0xF1A7492: start_thread (pthread_create.c:333) | | | | | | | ->14.19% (22,087,884B) 0xFF3F93D: clone (clone.S:97) | | | | | | | | | | | | | ->00.00% (0B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | | | | | ->00.01% (19,869B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | | | ->00.00% (537B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | ->04.13% (6,432,668B) 0xF9F5E38: ??? (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->04.13% (6,432,668B) 0xF9F5F0C: std::string::append(char const*, unsigned long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->04.10% (6,383,920B) 0x51DF1E4: RGWDataSyncShardControlCR::alloc_cr() (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.10% (6,383,920B) 0x51AB72B: RGWBackoffControlCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.10% (6,383,920B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.10% (6,383,920B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.10% (6,383,920B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.10% (6,383,920B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.10% (6,383,920B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.10% (6,383,920B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.10% (6,383,920B) 0xF1A7492: start_thread (pthread_create.c:333) | | | | | ->04.10% (6,383,920B) 0xFF3F93D: clone (clone.S:97) | | | | | | | | | ->00.03% (48,748B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | ->00.06% (101,179B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | ->00.00% (2,849B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->01.41% (2,196,371B) 0xF9F6813: char* std::string::_S_construct(char const*, char const*, std::allocator const&, std::forward_iterator_tag) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | ->01.41% (2,196,371B) 0xF9F6C44: std::basic_string, std::allocator >::basic_string(char const*, std::allocator const&) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | ->01.35% (2,104,668B) 0x510DA32: RGWCoroutine::RGWCoroutine(CephContext*) (in /usr/lib/librgw.so.2.0.0) | | | | ->01.35% (2,097,732B) 0x51075C3: RGWOmapAppend::RGWOmapAppend(RGWAsyncRadosProcessor*, RGWRados*, rgw_bucket&, std::string const&, unsigned long) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.35% (2,097,732B) 0x51E7CB7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.35% (2,097,732B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.35% (2,097,732B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.35% (2,097,732B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.35% (2,097,732B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.35% (2,097,732B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.35% (2,097,732B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.35% (2,097,732B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.35% (2,097,732B) 0xF1A7492: start_thread (pthread_create.c:333) | | | | | ->01.35% (2,097,732B) 0xFF3F93D: clone (clone.S:97) | | | | | | | | | ->00.00% (6,936B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | ->00.06% (91,703B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | ->00.00% (0B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->00.04% (69,290B) in 1+ places, all below ms_print's threshold (01.00%) | ->13.57% (21,128,192B) 0x510DB36: RGWCoroutine::RGWCoroutine(CephContext*) (in /usr/lib/librgw.so.2.0.0) | ->13.53% (21,059,072B) 0x51075C3: RGWOmapAppend::RGWOmapAppend(RGWAsyncRadosProcessor*, RGWRados*, rgw_bucket&, std::string const&, unsigned long) (in /usr/lib/librgw.so.2.0.0) | | ->13.53% (21,059,072B) 0x51E7CB7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | ->13.53% (21,059,072B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | ->13.53% (21,059,072B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | ->13.53% (21,059,072B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | ->13.53% (21,059,072B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | ->13.53% (21,059,072B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | ->13.53% (21,059,072B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | ->13.53% (21,059,072B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | ->13.53% (21,059,072B) 0xF1A7492: start_thread (pthread_create.c:333) | | ->13.53% (21,059,072B) 0xFF3F93D: clone (clone.S:97) | | | ->00.04% (69,120B) in 1+ places, all below ms_print's threshold (01.00%) | ->11.86% (18,462,752B) 0x50F9E51: RGWCoroutinesManager::allocate_stack() (in /usr/lib/librgw.so.2.0.0) | ->11.86% (18,462,528B) 0x50FE7AE: RGWCoroutinesStack::spawn(RGWCoroutine*, RGWCoroutine*, bool) (in /usr/lib/librgw.so.2.0.0) | | ->05.92% (9,213,568B) 0x51E7CD7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | | ->05.92% (9,213,568B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | ->05.92% (9,213,568B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.92% (9,213,568B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.92% (9,213,568B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.92% (9,213,568B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.92% (9,213,568B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | ->05.92% (9,213,568B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | ->05.92% (9,213,568B) 0xF1A7492: start_thread (pthread_create.c:333) | | | ->05.92% (9,213,568B) 0xFF3F93D: clone (clone.S:97) | | | | | ->05.92% (9,213,568B) 0x51D5E5F: RGWDataSyncShardCR::init_lease_cr() (in /usr/lib/librgw.so.2.0.0) | | | ->05.92% (9,213,568B) 0x51E7CEE: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | | ->05.92% (9,213,568B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | ->05.92% (9,213,568B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.92% (9,213,568B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.92% (9,213,568B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.92% (9,213,568B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.92% (9,213,568B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | ->05.92% (9,213,568B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | ->05.92% (9,213,568B) 0xF1A7492: start_thread (pthread_create.c:333) | | | ->05.92% (9,213,568B) 0xFF3F93D: clone (clone.S:97) | | | | | ->00.02% (35,392B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->00.00% (224B) in 1+ places, all below ms_print's threshold (01.00%) | ->04.89% (7,610,649B) in 864 places, all below massif's threshold (1.00%) | ->01.70% (2,641,152B) 0x510DB0E: RGWCoroutine::RGWCoroutine(CephContext*) (in /usr/lib/librgw.so.2.0.0) | ->01.69% (2,632,448B) 0x51075C3: RGWOmapAppend::RGWOmapAppend(RGWAsyncRadosProcessor*, RGWRados*, rgw_bucket&, std::string const&, unsigned long) (in /usr/lib/librgw.so.2.0.0) | | ->01.69% (2,632,448B) 0x51E7CB7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | ->01.69% (2,632,448B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | ->01.69% (2,632,448B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | ->01.69% (2,632,448B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | ->01.69% (2,632,448B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | ->01.69% (2,632,448B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | ->01.69% (2,632,448B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | ->01.69% (2,632,448B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | ->01.69% (2,632,448B) 0xF1A7492: start_thread (pthread_create.c:333) | | ->01.69% (2,632,448B) 0xFF3F93D: clone (clone.S:97) | | | ->00.01% (8,704B) in 1+ places, all below ms_print's threshold (01.00%) | ->01.14% (1,772,000B) 0x131429: worker_thread (civetweb.c:502) | ->01.14% (1,772,000B) 0xF1A7492: start_thread (pthread_create.c:333) | ->01.14% (1,772,000B) 0xFF3F93D: clone (clone.S:97) | ->01.06% (1,655,640B) 0x5103628: std::pair, bool> std::_Rb_tree, std::less, std::allocator >::_M_insert_unique(RGWCoroutinesStack* const&) (in /usr/lib/librgw.so.2.0.0) ->01.06% (1,650,480B) 0x50FE628: RGWCoroutinesManager::schedule(RGWCoroutinesEnv*, RGWCoroutinesStack*) (in /usr/lib/librgw.so.2.0.0) | ->01.06% (1,650,480B) 0x50FE805: RGWCoroutinesStack::spawn(RGWCoroutine*, RGWCoroutine*, bool) (in /usr/lib/librgw.so.2.0.0) | ->01.06% (1,645,280B) 0x51E7CD7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | ->01.06% (1,645,280B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | ->01.06% (1,645,280B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | ->01.06% (1,645,280B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | ->01.06% (1,645,280B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | ->01.06% (1,645,280B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | ->01.06% (1,645,280B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | ->01.06% (1,645,280B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | ->01.06% (1,645,280B) 0xF1A7492: start_thread (pthread_create.c:333) | | ->01.06% (1,645,280B) 0xFF3F93D: clone (clone.S:97) | | | ->00.00% (5,200B) in 1+ places, all below ms_print's threshold (01.00%) | ->00.00% (5,160B) in 1+ places, all below ms_print's threshold (01.00%) -------------------------------------------------------------------------------- n time(i) total(B) useful-heap(B) extra-heap(B) stacks(B) -------------------------------------------------------------------------------- 40 73,925,239,173 160,588,232 154,039,898 6,548,334 0 95.92% (154,039,898B) (heap allocation functions) malloc/new/new[], --alloc-fns, etc. ->41.95% (67,369,104B) 0x51E7C89: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | ->41.95% (67,369,104B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | ->41.95% (67,369,104B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | ->41.95% (67,369,104B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | ->41.95% (67,369,104B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | ->41.95% (67,369,104B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | ->41.95% (67,369,104B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | ->41.95% (67,369,104B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | ->41.95% (67,369,104B) 0xF1A7492: start_thread (pthread_create.c:333) | ->41.95% (67,369,104B) 0xFF3F93D: clone (clone.S:97) | ->19.88% (31,932,043B) 0xF9F4E97: std::string::_Rep::_S_create(unsigned long, unsigned long, std::allocator const&) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | ->18.43% (29,596,337B) 0xF9F5B09: std::string::_Rep::_M_clone(std::allocator const&, unsigned long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | ->18.43% (29,593,182B) 0xF9F5BAE: std::string::reserve(unsigned long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | ->14.23% (22,859,553B) 0xF9D1FB2: std::basic_stringbuf, std::allocator >::overflow(int) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->14.23% (22,859,553B) 0xF9D5D63: std::basic_streambuf >::xsputn(char const*, long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->14.23% (22,859,016B) 0xF9CD314: std::basic_ostream >& std::__ostream_insert >(std::basic_ostream >&, char const*, long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | | ->14.22% (22,839,147B) 0xF9CD6E5: std::basic_ostream >& std::operator<< >(std::basic_ostream >&, char const*) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | | | ->14.22% (22,839,147B) 0x5109AE4: RGWOmapAppend::operate() (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->14.22% (22,839,147B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->14.22% (22,839,147B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->14.22% (22,839,147B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->14.22% (22,839,147B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->14.22% (22,839,147B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->14.22% (22,839,147B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->14.22% (22,839,147B) 0xF1A7492: start_thread (pthread_create.c:333) | | | | | | | ->14.22% (22,839,147B) 0xFF3F93D: clone (clone.S:97) | | | | | | | | | | | | | ->00.00% (0B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | | | | | ->00.01% (19,869B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | | | ->00.00% (537B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | ->04.14% (6,650,858B) 0xF9F5E38: ??? (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->04.14% (6,650,858B) 0xF9F5F0C: std::string::append(char const*, unsigned long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->04.11% (6,600,113B) 0x51DF1E4: RGWDataSyncShardControlCR::alloc_cr() (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.11% (6,600,113B) 0x51AB72B: RGWBackoffControlCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.11% (6,600,113B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.11% (6,600,113B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.11% (6,600,113B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.11% (6,600,113B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.11% (6,600,113B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.11% (6,600,113B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.11% (6,600,113B) 0xF1A7492: start_thread (pthread_create.c:333) | | | | | ->04.11% (6,600,113B) 0xFF3F93D: clone (clone.S:97) | | | | | | | | | ->00.03% (50,745B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | ->00.05% (82,771B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | ->00.00% (3,155B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->01.41% (2,267,165B) 0xF9F6813: char* std::string::_S_construct(char const*, char const*, std::allocator const&, std::forward_iterator_tag) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | ->01.41% (2,267,165B) 0xF9F6C44: std::basic_string, std::allocator >::basic_string(char const*, std::allocator const&) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | ->01.36% (2,176,017B) 0x510DA32: RGWCoroutine::RGWCoroutine(CephContext*) (in /usr/lib/librgw.so.2.0.0) | | | | ->01.35% (2,169,081B) 0x51075C3: RGWOmapAppend::RGWOmapAppend(RGWAsyncRadosProcessor*, RGWRados*, rgw_bucket&, std::string const&, unsigned long) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.35% (2,169,081B) 0x51E7CB7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.35% (2,169,081B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.35% (2,169,081B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.35% (2,169,081B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.35% (2,169,081B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.35% (2,169,081B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.35% (2,169,081B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.35% (2,169,081B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.35% (2,169,081B) 0xF1A7492: start_thread (pthread_create.c:333) | | | | | ->01.35% (2,169,081B) 0xFF3F93D: clone (clone.S:97) | | | | | | | | | ->00.00% (6,936B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | ->00.06% (91,148B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | ->00.00% (0B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->00.04% (68,541B) in 1+ places, all below ms_print's threshold (01.00%) | ->13.60% (21,844,480B) 0x510DB36: RGWCoroutine::RGWCoroutine(CephContext*) (in /usr/lib/librgw.so.2.0.0) | ->13.56% (21,775,360B) 0x51075C3: RGWOmapAppend::RGWOmapAppend(RGWAsyncRadosProcessor*, RGWRados*, rgw_bucket&, std::string const&, unsigned long) (in /usr/lib/librgw.so.2.0.0) | | ->13.56% (21,775,360B) 0x51E7CB7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | ->13.56% (21,775,360B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | ->13.56% (21,775,360B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | ->13.56% (21,775,360B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | ->13.56% (21,775,360B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | ->13.56% (21,775,360B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | ->13.56% (21,775,360B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | ->13.56% (21,775,360B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | ->13.56% (21,775,360B) 0xF1A7492: start_thread (pthread_create.c:333) | | ->13.56% (21,775,360B) 0xFF3F93D: clone (clone.S:97) | | | ->00.04% (69,120B) in 1+ places, all below ms_print's threshold (01.00%) | ->11.89% (19,089,504B) 0x50F9E51: RGWCoroutinesManager::allocate_stack() (in /usr/lib/librgw.so.2.0.0) | ->11.89% (19,089,280B) 0x50FE7AE: RGWCoroutinesStack::spawn(RGWCoroutine*, RGWCoroutine*, bool) (in /usr/lib/librgw.so.2.0.0) | | ->05.93% (9,526,944B) 0x51E7CD7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | | ->05.93% (9,526,944B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | ->05.93% (9,526,944B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.93% (9,526,944B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.93% (9,526,944B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.93% (9,526,944B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.93% (9,526,944B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | ->05.93% (9,526,944B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | ->05.93% (9,526,944B) 0xF1A7492: start_thread (pthread_create.c:333) | | | ->05.93% (9,526,944B) 0xFF3F93D: clone (clone.S:97) | | | | | ->05.93% (9,526,944B) 0x51D5E5F: RGWDataSyncShardCR::init_lease_cr() (in /usr/lib/librgw.so.2.0.0) | | | ->05.93% (9,526,944B) 0x51E7CEE: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | | ->05.93% (9,526,944B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | ->05.93% (9,526,944B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.93% (9,526,944B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.93% (9,526,944B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.93% (9,526,944B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.93% (9,526,944B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | ->05.93% (9,526,944B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | ->05.93% (9,526,944B) 0xF1A7492: start_thread (pthread_create.c:333) | | | ->05.93% (9,526,944B) 0xFF3F93D: clone (clone.S:97) | | | | | ->00.02% (35,392B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->00.00% (224B) in 1+ places, all below ms_print's threshold (01.00%) | ->03.66% (5,877,681B) in 863 places, all below massif's threshold (1.00%) | ->01.70% (2,730,688B) 0x510DB0E: RGWCoroutine::RGWCoroutine(CephContext*) (in /usr/lib/librgw.so.2.0.0) | ->01.70% (2,721,984B) 0x51075C3: RGWOmapAppend::RGWOmapAppend(RGWAsyncRadosProcessor*, RGWRados*, rgw_bucket&, std::string const&, unsigned long) (in /usr/lib/librgw.so.2.0.0) | | ->01.70% (2,721,984B) 0x51E7CB7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | ->01.70% (2,721,984B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | ->01.70% (2,721,984B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | ->01.70% (2,721,984B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | ->01.70% (2,721,984B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | ->01.70% (2,721,984B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | ->01.70% (2,721,984B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | ->01.70% (2,721,984B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | ->01.70% (2,721,984B) 0xF1A7492: start_thread (pthread_create.c:333) | | ->01.70% (2,721,984B) 0xFF3F93D: clone (clone.S:97) | | | ->00.01% (8,704B) in 1+ places, all below ms_print's threshold (01.00%) | ->01.10% (1,772,000B) 0x131429: worker_thread (civetweb.c:502) | ->01.10% (1,772,000B) 0xF1A7492: start_thread (pthread_create.c:333) | ->01.10% (1,772,000B) 0xFF3F93D: clone (clone.S:97) | ->01.07% (1,712,798B) 0x10623B1B: ??? (in /usr/lib/librados.so.2.0.0) | ->01.07% (1,712,798B) in 42 places, all below massif's threshold (1.00%) | ->01.07% (1,711,600B) 0x5103628: std::pair, bool> std::_Rb_tree, std::less, std::allocator >::_M_insert_unique(RGWCoroutinesStack* const&) (in /usr/lib/librgw.so.2.0.0) ->01.06% (1,706,440B) 0x50FE628: RGWCoroutinesManager::schedule(RGWCoroutinesEnv*, RGWCoroutinesStack*) (in /usr/lib/librgw.so.2.0.0) | ->01.06% (1,706,440B) 0x50FE805: RGWCoroutinesStack::spawn(RGWCoroutine*, RGWCoroutine*, bool) (in /usr/lib/librgw.so.2.0.0) | ->01.06% (1,701,240B) 0x51E7CD7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | ->01.06% (1,701,240B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | ->01.06% (1,701,240B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | ->01.06% (1,701,240B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | ->01.06% (1,701,240B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | ->01.06% (1,701,240B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | ->01.06% (1,701,240B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | ->01.06% (1,701,240B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | ->01.06% (1,701,240B) 0xF1A7492: start_thread (pthread_create.c:333) | | ->01.06% (1,701,240B) 0xFF3F93D: clone (clone.S:97) | | | ->00.00% (5,200B) in 1+ places, all below ms_print's threshold (01.00%) | ->00.00% (5,160B) in 1+ places, all below ms_print's threshold (01.00%) -------------------------------------------------------------------------------- n time(i) total(B) useful-heap(B) extra-heap(B) stacks(B) -------------------------------------------------------------------------------- 41 75,991,667,097 170,552,272 163,521,019 7,031,253 0 95.88% (163,521,019B) (heap allocation functions) malloc/new/new[], --alloc-fns, etc. ->41.40% (70,600,464B) 0x51E7C89: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | ->41.40% (70,600,464B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | ->41.40% (70,600,464B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | ->41.40% (70,600,464B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | ->41.40% (70,600,464B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | ->41.40% (70,600,464B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | ->41.40% (70,600,464B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | ->41.40% (70,600,464B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | ->41.40% (70,600,464B) 0xF1A7492: start_thread (pthread_create.c:333) | ->41.40% (70,600,464B) 0xFF3F93D: clone (clone.S:97) | ->19.87% (33,896,932B) 0xF9F4E97: std::string::_Rep::_S_create(unsigned long, unsigned long, std::allocator const&) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | ->18.29% (31,202,336B) 0xF9F5B09: std::string::_Rep::_M_clone(std::allocator const&, unsigned long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | ->18.28% (31,178,879B) 0xF9F5BAE: std::string::reserve(unsigned long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | ->14.08% (24,011,418B) 0xF9D1FB2: std::basic_stringbuf, std::allocator >::overflow(int) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->14.08% (24,011,418B) 0xF9D5D63: std::basic_streambuf >::xsputn(char const*, long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->14.08% (24,010,881B) 0xF9CD314: std::basic_ostream >& std::__ostream_insert >(std::basic_ostream >&, char const*, long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | | ->14.03% (23,929,257B) 0xF9CD6E5: std::basic_ostream >& std::operator<< >(std::basic_ostream >&, char const*) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | | | ->14.03% (23,928,720B) 0x5109AE4: RGWOmapAppend::operate() (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->14.03% (23,928,720B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->14.03% (23,928,720B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->14.03% (23,928,720B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->14.03% (23,928,720B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->14.03% (23,928,720B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->14.03% (23,928,720B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->14.03% (23,928,720B) 0xF1A7492: start_thread (pthread_create.c:333) | | | | | | | ->14.03% (23,928,720B) 0xFF3F93D: clone (clone.S:97) | | | | | | | | | | | | | ->00.00% (537B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | | | | | ->00.05% (81,624B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | | | ->00.00% (537B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | ->04.11% (7,005,285B) 0xF9F5E38: ??? (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->04.11% (7,005,285B) 0xF9F5F0C: std::string::append(char const*, unsigned long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->04.05% (6,915,477B) 0x51DF1E4: RGWDataSyncShardControlCR::alloc_cr() (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.05% (6,915,477B) 0x51AB72B: RGWBackoffControlCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.05% (6,915,477B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.05% (6,915,477B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.05% (6,915,477B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.05% (6,915,477B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.05% (6,915,477B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.05% (6,915,477B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.05% (6,915,477B) 0xF1A7492: start_thread (pthread_create.c:333) | | | | | ->04.05% (6,915,477B) 0xFF3F93D: clone (clone.S:97) | | | | | | | | | ->00.05% (89,808B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | ->00.10% (162,176B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | ->00.01% (23,457B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->01.46% (2,491,479B) 0xF9F6813: char* std::string::_S_construct(char const*, char const*, std::allocator const&, std::forward_iterator_tag) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | ->01.46% (2,491,479B) 0xF9F6C44: std::basic_string, std::allocator >::basic_string(char const*, std::allocator const&) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | ->01.34% (2,289,747B) 0x510DA32: RGWCoroutine::RGWCoroutine(CephContext*) (in /usr/lib/librgw.so.2.0.0) | | | | ->01.33% (2,273,121B) 0x51075C3: RGWOmapAppend::RGWOmapAppend(RGWAsyncRadosProcessor*, RGWRados*, rgw_bucket&, std::string const&, unsigned long) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.33% (2,273,121B) 0x51E7CB7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.33% (2,273,121B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.33% (2,273,121B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.33% (2,273,121B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.33% (2,273,121B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.33% (2,273,121B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.33% (2,273,121B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.33% (2,273,121B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.33% (2,273,121B) 0xF1A7492: start_thread (pthread_create.c:333) | | | | | ->01.33% (2,273,121B) 0xFF3F93D: clone (clone.S:97) | | | | | | | | | ->00.01% (16,626B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | ->00.12% (201,732B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | ->00.00% (0B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->00.12% (203,117B) in 1+ places, all below ms_print's threshold (01.00%) | ->13.48% (22,986,240B) 0x510DB36: RGWCoroutine::RGWCoroutine(CephContext*) (in /usr/lib/librgw.so.2.0.0) | ->13.38% (22,819,840B) 0x51075C3: RGWOmapAppend::RGWOmapAppend(RGWAsyncRadosProcessor*, RGWRados*, rgw_bucket&, std::string const&, unsigned long) (in /usr/lib/librgw.so.2.0.0) | | ->13.38% (22,819,840B) 0x51E7CB7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | ->13.38% (22,819,840B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | ->13.38% (22,819,840B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | ->13.38% (22,819,840B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | ->13.38% (22,819,840B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | ->13.38% (22,819,840B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | ->13.38% (22,819,840B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | ->13.38% (22,819,840B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | ->13.38% (22,819,840B) 0xF1A7492: start_thread (pthread_create.c:333) | | ->13.38% (22,819,840B) 0xFF3F93D: clone (clone.S:97) | | | ->00.10% (166,400B) in 1+ places, all below ms_print's threshold (01.00%) | ->11.73% (20,003,648B) 0x50F9E51: RGWCoroutinesManager::allocate_stack() (in /usr/lib/librgw.so.2.0.0) | ->11.73% (20,003,424B) 0x50FE7AE: RGWCoroutinesStack::spawn(RGWCoroutine*, RGWCoroutine*, bool) (in /usr/lib/librgw.so.2.0.0) | | ->05.85% (9,983,904B) 0x51E7CD7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | | ->05.85% (9,983,904B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | ->05.85% (9,983,904B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.85% (9,983,904B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.85% (9,983,904B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.85% (9,983,904B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.85% (9,983,904B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | ->05.85% (9,983,904B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | ->05.85% (9,983,904B) 0xF1A7492: start_thread (pthread_create.c:333) | | | ->05.85% (9,983,904B) 0xFF3F93D: clone (clone.S:97) | | | | | ->05.85% (9,983,904B) 0x51D5E5F: RGWDataSyncShardCR::init_lease_cr() (in /usr/lib/librgw.so.2.0.0) | | | ->05.85% (9,983,904B) 0x51E7CEE: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | | ->05.85% (9,983,904B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | ->05.85% (9,983,904B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.85% (9,983,904B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.85% (9,983,904B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.85% (9,983,904B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.85% (9,983,904B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | ->05.85% (9,983,904B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | ->05.85% (9,983,904B) 0xF1A7492: start_thread (pthread_create.c:333) | | | ->05.85% (9,983,904B) 0xFF3F93D: clone (clone.S:97) | | | | | ->00.02% (35,616B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->00.00% (224B) in 1+ places, all below ms_print's threshold (01.00%) | ->04.58% (7,810,868B) in 863 places, all below massif's threshold (1.00%) | ->01.68% (2,873,408B) 0x510DB0E: RGWCoroutine::RGWCoroutine(CephContext*) (in /usr/lib/librgw.so.2.0.0) | ->01.67% (2,852,544B) 0x51075C3: RGWOmapAppend::RGWOmapAppend(RGWAsyncRadosProcessor*, RGWRados*, rgw_bucket&, std::string const&, unsigned long) (in /usr/lib/librgw.so.2.0.0) | | ->01.67% (2,852,544B) 0x51E7CB7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | ->01.67% (2,852,544B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | ->01.67% (2,852,544B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | ->01.67% (2,852,544B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | ->01.67% (2,852,544B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | ->01.67% (2,852,544B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | ->01.67% (2,852,544B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | ->01.67% (2,852,544B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | ->01.67% (2,852,544B) 0xF1A7492: start_thread (pthread_create.c:333) | | ->01.67% (2,852,544B) 0xFF3F93D: clone (clone.S:97) | | | ->00.01% (20,864B) in 1+ places, all below ms_print's threshold (01.00%) | ->01.05% (1,794,680B) 0x5103628: std::pair, bool> std::_Rb_tree, std::less, std::allocator >::_M_insert_unique(RGWCoroutinesStack* const&) (in /usr/lib/librgw.so.2.0.0) | ->01.05% (1,789,520B) 0x50FE628: RGWCoroutinesManager::schedule(RGWCoroutinesEnv*, RGWCoroutinesStack*) (in /usr/lib/librgw.so.2.0.0) | | ->01.05% (1,789,520B) 0x50FE805: RGWCoroutinesStack::spawn(RGWCoroutine*, RGWCoroutine*, bool) (in /usr/lib/librgw.so.2.0.0) | | ->01.05% (1,782,840B) 0x51E7CD7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | | ->01.05% (1,782,840B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | ->01.05% (1,782,840B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | ->01.05% (1,782,840B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | ->01.05% (1,782,840B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | ->01.05% (1,782,840B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | ->01.05% (1,782,840B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | ->01.05% (1,782,840B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | ->01.05% (1,782,840B) 0xF1A7492: start_thread (pthread_create.c:333) | | | ->01.05% (1,782,840B) 0xFF3F93D: clone (clone.S:97) | | | | | ->00.00% (6,680B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->00.00% (5,160B) in 1+ places, all below ms_print's threshold (01.00%) | ->01.05% (1,782,779B) 0x54F757B: ceph::log::Log::create_entry(int, int, unsigned long*) (in /usr/lib/librgw.so.2.0.0) | ->01.05% (1,782,779B) in 51 places, all below massif's threshold (1.00%) | ->01.04% (1,772,000B) 0x131429: worker_thread (civetweb.c:502) ->01.04% (1,772,000B) 0xF1A7492: start_thread (pthread_create.c:333) ->01.04% (1,772,000B) 0xFF3F93D: clone (clone.S:97) -------------------------------------------------------------------------------- n time(i) total(B) useful-heap(B) extra-heap(B) stacks(B) -------------------------------------------------------------------------------- 42 77,854,462,950 172,259,016 165,187,815 7,071,201 0 95.90% (165,187,815B) (heap allocation functions) malloc/new/new[], --alloc-fns, etc. ->41.53% (71,544,528B) 0x51E7C89: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | ->41.53% (71,544,528B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | ->41.53% (71,544,528B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | ->41.53% (71,544,528B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | ->41.53% (71,544,528B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | ->41.53% (71,544,528B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | ->41.53% (71,544,528B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | ->41.53% (71,544,528B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | ->41.53% (71,544,528B) 0xF1A7492: start_thread (pthread_create.c:333) | ->41.53% (71,544,528B) 0xFF3F93D: clone (clone.S:97) | ->19.87% (34,221,391B) 0xF9F4E97: std::string::_Rep::_S_create(unsigned long, unsigned long, std::allocator const&) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | ->18.31% (31,548,230B) 0xF9F5B09: std::string::_Rep::_M_clone(std::allocator const&, unsigned long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | ->18.30% (31,528,274B) 0xF9F5BAE: std::string::reserve(unsigned long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | ->14.11% (24,309,990B) 0xF9D1FB2: std::basic_stringbuf, std::allocator >::overflow(int) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->14.11% (24,309,990B) 0xF9D5D63: std::basic_streambuf >::xsputn(char const*, long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->14.11% (24,308,916B) 0xF9CD314: std::basic_ostream >& std::__ostream_insert >(std::basic_ostream >&, char const*, long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | | ->14.08% (24,255,216B) 0xF9CD6E5: std::basic_ostream >& std::operator<< >(std::basic_ostream >&, char const*) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | | | ->14.08% (24,254,679B) 0x5109AE4: RGWOmapAppend::operate() (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->14.08% (24,254,679B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->14.08% (24,254,679B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->14.08% (24,254,679B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->14.08% (24,254,679B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->14.08% (24,254,679B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->14.08% (24,254,679B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->14.08% (24,254,679B) 0xF1A7492: start_thread (pthread_create.c:333) | | | | | | | ->14.08% (24,254,679B) 0xFF3F93D: clone (clone.S:97) | | | | | | | | | | | | | ->00.00% (537B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | | | | | ->00.03% (53,700B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | | | ->00.00% (1,074B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | ->04.13% (7,115,137B) 0xF9F5E38: ??? (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->04.13% (7,115,137B) 0xF9F5F0C: std::string::append(char const*, unsigned long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->04.07% (7,008,057B) 0x51DF1E4: RGWDataSyncShardControlCR::alloc_cr() (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.07% (7,008,057B) 0x51AB72B: RGWBackoffControlCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.07% (7,008,057B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.07% (7,008,057B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.07% (7,008,057B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.07% (7,008,057B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.07% (7,008,057B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.07% (7,008,057B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.07% (7,008,057B) 0xF1A7492: start_thread (pthread_create.c:333) | | | | | ->04.07% (7,008,057B) 0xFF3F93D: clone (clone.S:97) | | | | | | | | | ->00.06% (107,080B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | ->00.06% (103,147B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | ->00.01% (19,956B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->01.44% (2,485,827B) 0xF9F6813: char* std::string::_S_construct(char const*, char const*, std::allocator const&, std::forward_iterator_tag) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | ->01.44% (2,485,827B) 0xF9F6C44: std::basic_string, std::allocator >::basic_string(char const*, std::allocator const&) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | ->01.34% (2,315,196B) 0x510DA32: RGWCoroutine::RGWCoroutine(CephContext*) (in /usr/lib/librgw.so.2.0.0) | | | | ->01.34% (2,303,517B) 0x51075C3: RGWOmapAppend::RGWOmapAppend(RGWAsyncRadosProcessor*, RGWRados*, rgw_bucket&, std::string const&, unsigned long) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.34% (2,303,517B) 0x51E7CB7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.34% (2,303,517B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.34% (2,303,517B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.34% (2,303,517B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.34% (2,303,517B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.34% (2,303,517B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.34% (2,303,517B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.34% (2,303,517B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.34% (2,303,517B) 0xF1A7492: start_thread (pthread_create.c:333) | | | | | ->01.34% (2,303,517B) 0xFF3F93D: clone (clone.S:97) | | | | | | | | | ->00.01% (11,679B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | ->00.10% (170,631B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | ->00.00% (0B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->00.11% (187,334B) in 1+ places, all below ms_print's threshold (01.00%) | ->13.49% (23,241,728B) 0x510DB36: RGWCoroutine::RGWCoroutine(CephContext*) (in /usr/lib/librgw.so.2.0.0) | ->13.42% (23,124,992B) 0x51075C3: RGWOmapAppend::RGWOmapAppend(RGWAsyncRadosProcessor*, RGWRados*, rgw_bucket&, std::string const&, unsigned long) (in /usr/lib/librgw.so.2.0.0) | | ->13.42% (23,124,992B) 0x51E7CB7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | ->13.42% (23,124,992B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | ->13.42% (23,124,992B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | ->13.42% (23,124,992B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | ->13.42% (23,124,992B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | ->13.42% (23,124,992B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | ->13.42% (23,124,992B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | ->13.42% (23,124,992B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | ->13.42% (23,124,992B) 0xF1A7492: start_thread (pthread_create.c:333) | | ->13.42% (23,124,992B) 0xFF3F93D: clone (clone.S:97) | | | ->00.07% (116,736B) in 1+ places, all below ms_print's threshold (01.00%) | ->11.77% (20,272,224B) 0x50F9E51: RGWCoroutinesManager::allocate_stack() (in /usr/lib/librgw.so.2.0.0) | ->11.77% (20,272,000B) 0x50FE7AE: RGWCoroutinesStack::spawn(RGWCoroutine*, RGWCoroutine*, bool) (in /usr/lib/librgw.so.2.0.0) | | ->05.87% (10,117,408B) 0x51E7CD7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | | ->05.87% (10,117,408B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | ->05.87% (10,117,408B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.87% (10,117,408B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.87% (10,117,408B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.87% (10,117,408B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.87% (10,117,408B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | ->05.87% (10,117,408B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | ->05.87% (10,117,408B) 0xF1A7492: start_thread (pthread_create.c:333) | | | ->05.87% (10,117,408B) 0xFF3F93D: clone (clone.S:97) | | | | | ->05.87% (10,117,408B) 0x51D5E5F: RGWDataSyncShardCR::init_lease_cr() (in /usr/lib/librgw.so.2.0.0) | | | ->05.87% (10,117,408B) 0x51E7CEE: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | | ->05.87% (10,117,408B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | ->05.87% (10,117,408B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.87% (10,117,408B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.87% (10,117,408B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.87% (10,117,408B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.87% (10,117,408B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | ->05.87% (10,117,408B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | ->05.87% (10,117,408B) 0xF1A7492: start_thread (pthread_create.c:333) | | | ->05.87% (10,117,408B) 0xFF3F93D: clone (clone.S:97) | | | | | ->00.02% (37,184B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->00.00% (224B) in 1+ places, all below ms_print's threshold (01.00%) | ->04.37% (7,521,070B) in 863 places, all below massif's threshold (1.00%) | ->01.69% (2,905,344B) 0x510DB0E: RGWCoroutine::RGWCoroutine(CephContext*) (in /usr/lib/librgw.so.2.0.0) | ->01.68% (2,890,688B) 0x51075C3: RGWOmapAppend::RGWOmapAppend(RGWAsyncRadosProcessor*, RGWRados*, rgw_bucket&, std::string const&, unsigned long) (in /usr/lib/librgw.so.2.0.0) | | ->01.68% (2,890,688B) 0x51E7CB7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | ->01.68% (2,890,688B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | ->01.68% (2,890,688B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | ->01.68% (2,890,688B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | ->01.68% (2,890,688B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | ->01.68% (2,890,688B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | ->01.68% (2,890,688B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | ->01.68% (2,890,688B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | ->01.68% (2,890,688B) 0xF1A7492: start_thread (pthread_create.c:333) | | ->01.68% (2,890,688B) 0xFF3F93D: clone (clone.S:97) | | | ->00.01% (14,656B) in 1+ places, all below ms_print's threshold (01.00%) | ->01.10% (1,891,570B) 0x54F757B: ceph::log::Log::create_entry(int, int, unsigned long*) (in /usr/lib/librgw.so.2.0.0) | ->01.10% (1,891,570B) in 51 places, all below massif's threshold (1.00%) | ->01.06% (1,817,960B) 0x5103628: std::pair, bool> std::_Rb_tree, std::less, std::allocator >::_M_insert_unique(RGWCoroutinesStack* const&) (in /usr/lib/librgw.so.2.0.0) | ->01.05% (1,812,800B) 0x50FE628: RGWCoroutinesManager::schedule(RGWCoroutinesEnv*, RGWCoroutinesStack*) (in /usr/lib/librgw.so.2.0.0) | | ->01.05% (1,812,800B) 0x50FE805: RGWCoroutinesStack::spawn(RGWCoroutine*, RGWCoroutine*, bool) (in /usr/lib/librgw.so.2.0.0) | | ->01.05% (1,806,680B) 0x51E7CD7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | | ->01.05% (1,806,680B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | ->01.05% (1,806,680B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | ->01.05% (1,806,680B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | ->01.05% (1,806,680B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | ->01.05% (1,806,680B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | ->01.05% (1,806,680B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | ->01.05% (1,806,680B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | ->01.05% (1,806,680B) 0xF1A7492: start_thread (pthread_create.c:333) | | | ->01.05% (1,806,680B) 0xFF3F93D: clone (clone.S:97) | | | | | ->00.00% (6,120B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->00.00% (5,160B) in 1+ places, all below ms_print's threshold (01.00%) | ->01.03% (1,772,000B) 0x131429: worker_thread (civetweb.c:502) ->01.03% (1,772,000B) 0xF1A7492: start_thread (pthread_create.c:333) ->01.03% (1,772,000B) 0xFF3F93D: clone (clone.S:97) -------------------------------------------------------------------------------- n time(i) total(B) useful-heap(B) extra-heap(B) stacks(B) -------------------------------------------------------------------------------- 43 80,489,930,758 174,624,528 167,468,963 7,155,565 0 44 82,860,115,214 179,502,384 172,188,638 7,313,746 0 95.93% (172,188,638B) (heap allocation functions) malloc/new/new[], --alloc-fns, etc. ->42.03% (75,449,088B) 0x51E7C89: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | ->42.03% (75,449,088B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | ->42.03% (75,449,088B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | ->42.03% (75,449,088B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | ->42.03% (75,449,088B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | ->42.03% (75,449,088B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | ->42.03% (75,449,088B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | ->42.03% (75,449,088B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | ->42.03% (75,449,088B) 0xF1A7492: start_thread (pthread_create.c:333) | ->42.03% (75,449,088B) 0xFF3F93D: clone (clone.S:97) | ->19.98% (35,871,383B) 0xF9F4E97: std::string::_Rep::_S_create(unsigned long, unsigned long, std::allocator const&) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | ->18.51% (33,217,876B) 0xF9F5B09: std::string::_Rep::_M_clone(std::allocator const&, unsigned long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | ->18.50% (33,209,200B) 0xF9F5BAE: std::string::reserve(unsigned long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | ->14.28% (25,631,010B) 0xF9D1FB2: std::basic_stringbuf, std::allocator >::overflow(int) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->14.28% (25,631,010B) 0xF9D5D63: std::basic_streambuf >::xsputn(char const*, long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->14.28% (25,629,936B) 0xF9CD314: std::basic_ostream >& std::__ostream_insert >(std::basic_ostream >&, char const*, long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | | ->14.25% (25,577,310B) 0xF9CD6E5: std::basic_ostream >& std::operator<< >(std::basic_ostream >&, char const*) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | | | ->14.25% (25,577,310B) 0x5109AE4: RGWOmapAppend::operate() (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->14.25% (25,577,310B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->14.25% (25,577,310B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->14.25% (25,577,310B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->14.25% (25,577,310B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->14.25% (25,577,310B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->14.25% (25,577,310B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->14.25% (25,577,310B) 0xF1A7492: start_thread (pthread_create.c:333) | | | | | | | ->14.25% (25,577,310B) 0xFF3F93D: clone (clone.S:97) | | | | | | | | | | | | | ->00.00% (0B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | | | | | ->00.03% (52,626B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | | | ->00.00% (1,074B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | ->04.15% (7,447,080B) 0xF9F5E38: ??? (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->04.15% (7,447,080B) 0xF9F5F0C: std::string::append(char const*, unsigned long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->04.12% (7,391,142B) 0x51DF1E4: RGWDataSyncShardControlCR::alloc_cr() (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.12% (7,391,142B) 0x51AB72B: RGWBackoffControlCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.12% (7,391,142B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.12% (7,391,142B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.12% (7,391,142B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.12% (7,391,142B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.12% (7,391,142B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.12% (7,391,142B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.12% (7,391,142B) 0xF1A7492: start_thread (pthread_create.c:333) | | | | | ->04.12% (7,391,142B) 0xFF3F93D: clone (clone.S:97) | | | | | | | | | ->00.03% (55,938B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | ->00.07% (131,110B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | ->00.00% (8,676B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->01.42% (2,555,819B) 0xF9F6813: char* std::string::_S_construct(char const*, char const*, std::allocator const&, std::forward_iterator_tag) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | ->01.42% (2,555,819B) 0xF9F6C44: std::basic_string, std::allocator >::basic_string(char const*, std::allocator const&) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | ->01.36% (2,441,472B) 0x510DA32: RGWCoroutine::RGWCoroutine(CephContext*) (in /usr/lib/librgw.so.2.0.0) | | | | ->01.35% (2,429,232B) 0x51075C3: RGWOmapAppend::RGWOmapAppend(RGWAsyncRadosProcessor*, RGWRados*, rgw_bucket&, std::string const&, unsigned long) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.35% (2,429,232B) 0x51E7CB7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.35% (2,429,232B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.35% (2,429,232B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.35% (2,429,232B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.35% (2,429,232B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.35% (2,429,232B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.35% (2,429,232B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.35% (2,429,232B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.35% (2,429,232B) 0xF1A7492: start_thread (pthread_create.c:333) | | | | | ->01.35% (2,429,232B) 0xFF3F93D: clone (clone.S:97) | | | | | | | | | ->00.01% (12,240B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | ->00.06% (114,347B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | ->00.00% (0B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->00.05% (97,688B) in 1+ places, all below ms_print's threshold (01.00%) | ->13.65% (24,509,440B) 0x510DB36: RGWCoroutine::RGWCoroutine(CephContext*) (in /usr/lib/librgw.so.2.0.0) | ->13.59% (24,387,072B) 0x51075C3: RGWOmapAppend::RGWOmapAppend(RGWAsyncRadosProcessor*, RGWRados*, rgw_bucket&, std::string const&, unsigned long) (in /usr/lib/librgw.so.2.0.0) | | ->13.59% (24,387,072B) 0x51E7CB7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | ->13.59% (24,387,072B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | ->13.59% (24,387,072B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | ->13.59% (24,387,072B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | ->13.59% (24,387,072B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | ->13.59% (24,387,072B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | ->13.59% (24,387,072B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | ->13.59% (24,387,072B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | ->13.59% (24,387,072B) 0xF1A7492: start_thread (pthread_create.c:333) | | ->13.59% (24,387,072B) 0xFF3F93D: clone (clone.S:97) | | | ->00.07% (122,368B) in 1+ places, all below ms_print's threshold (01.00%) | ->11.91% (21,380,352B) 0x50F9E51: RGWCoroutinesManager::allocate_stack() (in /usr/lib/librgw.so.2.0.0) | ->11.91% (21,380,128B) 0x50FE7AE: RGWCoroutinesStack::spawn(RGWCoroutine*, RGWCoroutine*, bool) (in /usr/lib/librgw.so.2.0.0) | | ->05.94% (10,669,568B) 0x51E7CD7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | | ->05.94% (10,669,568B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | ->05.94% (10,669,568B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.94% (10,669,568B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.94% (10,669,568B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.94% (10,669,568B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.94% (10,669,568B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | ->05.94% (10,669,568B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | ->05.94% (10,669,568B) 0xF1A7492: start_thread (pthread_create.c:333) | | | ->05.94% (10,669,568B) 0xFF3F93D: clone (clone.S:97) | | | | | ->05.94% (10,669,568B) 0x51D5E5F: RGWDataSyncShardCR::init_lease_cr() (in /usr/lib/librgw.so.2.0.0) | | | ->05.94% (10,669,568B) 0x51E7CEE: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | | ->05.94% (10,669,568B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | ->05.94% (10,669,568B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.94% (10,669,568B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.94% (10,669,568B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.94% (10,669,568B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.94% (10,669,568B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | ->05.94% (10,669,568B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | ->05.94% (10,669,568B) 0xF1A7492: start_thread (pthread_create.c:333) | | | ->05.94% (10,669,568B) 0xFF3F93D: clone (clone.S:97) | | | | | ->00.02% (40,992B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->00.00% (224B) in 1+ places, all below ms_print's threshold (01.00%) | ->04.37% (7,850,814B) in 864 places, all below massif's threshold (1.00%) | ->01.71% (3,063,808B) 0x510DB0E: RGWCoroutine::RGWCoroutine(CephContext*) (in /usr/lib/librgw.so.2.0.0) | ->01.70% (3,048,448B) 0x51075C3: RGWOmapAppend::RGWOmapAppend(RGWAsyncRadosProcessor*, RGWRados*, rgw_bucket&, std::string const&, unsigned long) (in /usr/lib/librgw.so.2.0.0) | | ->01.70% (3,048,448B) 0x51E7CB7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | ->01.70% (3,048,448B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | ->01.70% (3,048,448B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | ->01.70% (3,048,448B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | ->01.70% (3,048,448B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | ->01.70% (3,048,448B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | ->01.70% (3,048,448B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | ->01.70% (3,048,448B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | ->01.70% (3,048,448B) 0xF1A7492: start_thread (pthread_create.c:333) | | ->01.70% (3,048,448B) 0xFF3F93D: clone (clone.S:97) | | | ->00.01% (15,360B) in 1+ places, all below ms_print's threshold (01.00%) | ->01.20% (2,147,353B) 0x54F757B: ceph::log::Log::create_entry(int, int, unsigned long*) (in /usr/lib/librgw.so.2.0.0) | ->01.20% (2,147,353B) in 51 places, all below massif's threshold (1.00%) | ->01.07% (1,916,400B) 0x5103628: std::pair, bool> std::_Rb_tree, std::less, std::allocator >::_M_insert_unique(RGWCoroutinesStack* const&) (in /usr/lib/librgw.so.2.0.0) ->01.06% (1,911,240B) 0x50FE628: RGWCoroutinesManager::schedule(RGWCoroutinesEnv*, RGWCoroutinesStack*) (in /usr/lib/librgw.so.2.0.0) | ->01.06% (1,911,240B) 0x50FE805: RGWCoroutinesStack::spawn(RGWCoroutine*, RGWCoroutine*, bool) (in /usr/lib/librgw.so.2.0.0) | ->01.06% (1,905,280B) 0x51E7CD7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | ->01.06% (1,905,280B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | ->01.06% (1,905,280B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | ->01.06% (1,905,280B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | ->01.06% (1,905,280B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | ->01.06% (1,905,280B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | ->01.06% (1,905,280B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | ->01.06% (1,905,280B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | ->01.06% (1,905,280B) 0xF1A7492: start_thread (pthread_create.c:333) | | ->01.06% (1,905,280B) 0xFF3F93D: clone (clone.S:97) | | | ->00.00% (5,960B) in 1+ places, all below ms_print's threshold (01.00%) | ->00.00% (5,160B) in 1+ places, all below ms_print's threshold (01.00%) -------------------------------------------------------------------------------- n time(i) total(B) useful-heap(B) extra-heap(B) stacks(B) -------------------------------------------------------------------------------- 45 84,425,307,651 182,922,256 175,504,799 7,417,457 0 46 85,719,762,904 188,695,448 180,966,181 7,729,267 0 95.90% (180,966,181B) (heap allocation functions) malloc/new/new[], --alloc-fns, etc. ->41.87% (79,009,920B) 0x51E7C89: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | ->41.87% (79,009,920B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | ->41.87% (79,009,920B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | ->41.87% (79,009,920B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | ->41.87% (79,009,920B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | ->41.87% (79,009,920B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | ->41.87% (79,009,920B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | ->41.87% (79,009,920B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | ->41.87% (79,009,920B) 0xF1A7492: start_thread (pthread_create.c:333) | ->41.87% (79,009,920B) 0xFF3F93D: clone (clone.S:97) | ->19.96% (37,664,809B) 0xF9F4E97: std::string::_Rep::_S_create(unsigned long, unsigned long, std::allocator const&) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | ->18.42% (34,764,793B) 0xF9F5B09: std::string::_Rep::_M_clone(std::allocator const&, unsigned long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | ->18.41% (34,746,077B) 0xF9F5BAE: std::string::reserve(unsigned long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | ->14.22% (26,837,649B) 0xF9D1FB2: std::basic_stringbuf, std::allocator >::overflow(int) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->14.22% (26,837,649B) 0xF9D5D63: std::basic_streambuf >::xsputn(char const*, long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->14.22% (26,837,112B) 0xF9CD314: std::basic_ostream >& std::__ostream_insert >(std::basic_ostream >&, char const*, long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | | ->14.20% (26,785,560B) 0xF9CD6E5: std::basic_ostream >& std::operator<< >(std::basic_ostream >&, char const*) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | | | ->14.20% (26,785,560B) 0x5109AE4: RGWOmapAppend::operate() (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->14.20% (26,785,560B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->14.20% (26,785,560B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->14.20% (26,785,560B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->14.20% (26,785,560B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->14.20% (26,785,560B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->14.20% (26,785,560B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->14.20% (26,785,560B) 0xF1A7492: start_thread (pthread_create.c:333) | | | | | | | ->14.20% (26,785,560B) 0xFF3F93D: clone (clone.S:97) | | | | | | | | | | | | | ->00.00% (0B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | | | | | ->00.03% (51,552B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | | | ->00.00% (537B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | ->04.14% (7,819,786B) 0xF9F5E38: ??? (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->04.14% (7,819,786B) 0xF9F5F0C: std::string::append(char const*, unsigned long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->04.10% (7,740,106B) 0x51DF1E4: RGWDataSyncShardControlCR::alloc_cr() (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.10% (7,740,106B) 0x51AB72B: RGWBackoffControlCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.10% (7,740,106B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.10% (7,740,106B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.10% (7,740,106B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.10% (7,740,106B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.10% (7,740,106B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.10% (7,740,106B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.10% (7,740,106B) 0xF1A7492: start_thread (pthread_create.c:333) | | | | | ->04.10% (7,740,106B) 0xFF3F93D: clone (clone.S:97) | | | | | | | | | ->00.04% (79,680B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | ->00.05% (88,642B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | ->00.01% (18,716B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->01.44% (2,723,919B) 0xF9F6813: char* std::string::_S_construct(char const*, char const*, std::allocator const&, std::forward_iterator_tag) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | ->01.44% (2,723,919B) 0xF9F6C44: std::basic_string, std::allocator >::basic_string(char const*, std::allocator const&) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | ->01.35% (2,554,947B) 0x510DA32: RGWCoroutine::RGWCoroutine(CephContext*) (in /usr/lib/librgw.so.2.0.0) | | | | ->01.35% (2,543,880B) 0x51075C3: RGWOmapAppend::RGWOmapAppend(RGWAsyncRadosProcessor*, RGWRados*, rgw_bucket&, std::string const&, unsigned long) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.35% (2,543,880B) 0x51E7CB7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.35% (2,543,880B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.35% (2,543,880B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.35% (2,543,880B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.35% (2,543,880B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.35% (2,543,880B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.35% (2,543,880B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.35% (2,543,880B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.35% (2,543,880B) 0xF1A7492: start_thread (pthread_create.c:333) | | | | | ->01.35% (2,543,880B) 0xFF3F93D: clone (clone.S:97) | | | | | | | | | ->00.01% (11,067B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | ->00.09% (168,972B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | ->00.00% (0B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->00.09% (176,097B) in 1+ places, all below ms_print's threshold (01.00%) | ->13.59% (25,648,640B) 0x510DB36: RGWCoroutine::RGWCoroutine(CephContext*) (in /usr/lib/librgw.so.2.0.0) | ->13.53% (25,538,048B) 0x51075C3: RGWOmapAppend::RGWOmapAppend(RGWAsyncRadosProcessor*, RGWRados*, rgw_bucket&, std::string const&, unsigned long) (in /usr/lib/librgw.so.2.0.0) | | ->13.53% (25,538,048B) 0x51E7CB7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | ->13.53% (25,538,048B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | ->13.53% (25,538,048B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | ->13.53% (25,538,048B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | ->13.53% (25,538,048B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | ->13.53% (25,538,048B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | ->13.53% (25,538,048B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | ->13.53% (25,538,048B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | ->13.53% (25,538,048B) 0xF1A7492: start_thread (pthread_create.c:333) | | ->13.53% (25,538,048B) 0xFF3F93D: clone (clone.S:97) | | | ->00.06% (110,592B) in 1+ places, all below ms_print's threshold (01.00%) | ->11.86% (22,387,456B) 0x50F9E51: RGWCoroutinesManager::allocate_stack() (in /usr/lib/librgw.so.2.0.0) | ->11.86% (22,387,232B) 0x50FE7AE: RGWCoroutinesStack::spawn(RGWCoroutine*, RGWCoroutine*, bool) (in /usr/lib/librgw.so.2.0.0) | | ->05.92% (11,173,120B) 0x51E7CD7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | | ->05.92% (11,173,120B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | ->05.92% (11,173,120B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.92% (11,173,120B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.92% (11,173,120B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.92% (11,173,120B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.92% (11,173,120B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | ->05.92% (11,173,120B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | ->05.92% (11,173,120B) 0xF1A7492: start_thread (pthread_create.c:333) | | | ->05.92% (11,173,120B) 0xFF3F93D: clone (clone.S:97) | | | | | ->05.92% (11,173,120B) 0x51D5E5F: RGWDataSyncShardCR::init_lease_cr() (in /usr/lib/librgw.so.2.0.0) | | | ->05.92% (11,173,120B) 0x51E7CEE: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | | ->05.92% (11,173,120B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | ->05.92% (11,173,120B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.92% (11,173,120B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.92% (11,173,120B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.92% (11,173,120B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.92% (11,173,120B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | ->05.92% (11,173,120B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | ->05.92% (11,173,120B) 0xF1A7492: start_thread (pthread_create.c:333) | | | ->05.92% (11,173,120B) 0xFF3F93D: clone (clone.S:97) | | | | | ->00.02% (40,992B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->00.00% (224B) in 1+ places, all below ms_print's threshold (01.00%) | ->04.72% (8,901,462B) in 864 places, all below massif's threshold (1.00%) | ->01.70% (3,206,208B) 0x510DB0E: RGWCoroutine::RGWCoroutine(CephContext*) (in /usr/lib/librgw.so.2.0.0) | ->01.69% (3,192,320B) 0x51075C3: RGWOmapAppend::RGWOmapAppend(RGWAsyncRadosProcessor*, RGWRados*, rgw_bucket&, std::string const&, unsigned long) (in /usr/lib/librgw.so.2.0.0) | | ->01.69% (3,192,320B) 0x51E7CB7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | ->01.69% (3,192,320B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | ->01.69% (3,192,320B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | ->01.69% (3,192,320B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | ->01.69% (3,192,320B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | ->01.69% (3,192,320B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | ->01.69% (3,192,320B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | ->01.69% (3,192,320B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | ->01.69% (3,192,320B) 0xF1A7492: start_thread (pthread_create.c:333) | | ->01.69% (3,192,320B) 0xFF3F93D: clone (clone.S:97) | | | ->00.01% (13,888B) in 1+ places, all below ms_print's threshold (01.00%) | ->01.13% (2,141,406B) 0x54F757B: ceph::log::Log::create_entry(int, int, unsigned long*) (in /usr/lib/librgw.so.2.0.0) | ->01.13% (2,141,406B) in 51 places, all below massif's threshold (1.00%) | ->01.06% (2,006,280B) 0x5103628: std::pair, bool> std::_Rb_tree, std::less, std::allocator >::_M_insert_unique(RGWCoroutinesStack* const&) (in /usr/lib/librgw.so.2.0.0) ->01.06% (2,001,120B) 0x50FE628: RGWCoroutinesManager::schedule(RGWCoroutinesEnv*, RGWCoroutinesStack*) (in /usr/lib/librgw.so.2.0.0) | ->01.06% (2,001,120B) 0x50FE805: RGWCoroutinesStack::spawn(RGWCoroutine*, RGWCoroutine*, bool) (in /usr/lib/librgw.so.2.0.0) | ->01.06% (1,995,200B) 0x51E7CD7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | ->01.06% (1,995,200B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | ->01.06% (1,995,200B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | ->01.06% (1,995,200B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | ->01.06% (1,995,200B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | ->01.06% (1,995,200B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | ->01.06% (1,995,200B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | ->01.06% (1,995,200B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | ->01.06% (1,995,200B) 0xF1A7492: start_thread (pthread_create.c:333) | | ->01.06% (1,995,200B) 0xFF3F93D: clone (clone.S:97) | | | ->00.00% (5,920B) in 1+ places, all below ms_print's threshold (01.00%) | ->00.00% (5,160B) in 1+ places, all below ms_print's threshold (01.00%) -------------------------------------------------------------------------------- n time(i) total(B) useful-heap(B) extra-heap(B) stacks(B) -------------------------------------------------------------------------------- 47 87,191,462,819 191,052,032 183,200,122 7,851,910 0 48 88,888,273,463 194,544,160 186,619,248 7,924,912 0 49 91,151,678,051 198,513,128 190,455,335 8,057,793 0 95.94% (190,455,335B) (heap allocation functions) malloc/new/new[], --alloc-fns, etc. ->42.16% (83,700,144B) 0x51E7C89: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | ->42.16% (83,700,144B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | ->42.16% (83,700,144B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | ->42.16% (83,700,144B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | ->42.16% (83,700,144B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | ->42.16% (83,700,144B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | ->42.16% (83,700,144B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | ->42.16% (83,700,144B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | ->42.16% (83,700,144B) 0xF1A7492: start_thread (pthread_create.c:333) | ->42.16% (83,700,144B) 0xFF3F93D: clone (clone.S:97) | ->20.02% (39,750,367B) 0xF9F4E97: std::string::_Rep::_S_create(unsigned long, unsigned long, std::allocator const&) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | ->18.55% (36,828,718B) 0xF9F5B09: std::string::_Rep::_M_clone(std::allocator const&, unsigned long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | ->18.55% (36,819,216B) 0xF9F5BAE: std::string::reserve(unsigned long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | ->14.32% (28,422,873B) 0xF9D1FB2: std::basic_stringbuf, std::allocator >::overflow(int) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->14.32% (28,422,873B) 0xF9D5D63: std::basic_streambuf >::xsputn(char const*, long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->14.32% (28,422,336B) 0xF9CD314: std::basic_ostream >& std::__ostream_insert >(std::basic_ostream >&, char const*, long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | | ->14.29% (28,370,784B) 0xF9CD6E5: std::basic_ostream >& std::operator<< >(std::basic_ostream >&, char const*) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | | | ->14.29% (28,370,784B) 0x5109AE4: RGWOmapAppend::operate() (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->14.29% (28,370,784B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->14.29% (28,370,784B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->14.29% (28,370,784B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->14.29% (28,370,784B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->14.29% (28,370,784B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->14.29% (28,370,784B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->14.29% (28,370,784B) 0xF1A7492: start_thread (pthread_create.c:333) | | | | | | | ->14.29% (28,370,784B) 0xFF3F93D: clone (clone.S:97) | | | | | | | | | | | | | ->00.00% (0B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | | | | | ->00.03% (51,552B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | | | ->00.00% (537B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | ->04.16% (8,259,641B) 0xF9F5E38: ??? (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->04.16% (8,259,641B) 0xF9F5F0C: std::string::append(char const*, unsigned long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->04.13% (8,199,815B) 0x51DF1E4: RGWDataSyncShardControlCR::alloc_cr() (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.13% (8,199,815B) 0x51AB72B: RGWBackoffControlCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.13% (8,199,815B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.13% (8,199,815B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.13% (8,199,815B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.13% (8,199,815B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.13% (8,199,815B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.13% (8,199,815B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.13% (8,199,815B) 0xF1A7492: start_thread (pthread_create.c:333) | | | | | ->04.13% (8,199,815B) 0xFF3F93D: clone (clone.S:97) | | | | | | | | | ->00.03% (59,826B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | ->00.07% (136,702B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | ->00.00% (9,502B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->01.42% (2,823,991B) 0xF9F6813: char* std::string::_S_construct(char const*, char const*, std::allocator const&, std::forward_iterator_tag) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | ->01.42% (2,823,991B) 0xF9F6C44: std::basic_string, std::allocator >::basic_string(char const*, std::allocator const&) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | ->01.36% (2,706,213B) 0x510DA32: RGWCoroutine::RGWCoroutine(CephContext*) (in /usr/lib/librgw.so.2.0.0) | | | | ->01.36% (2,694,891B) 0x51075C3: RGWOmapAppend::RGWOmapAppend(RGWAsyncRadosProcessor*, RGWRados*, rgw_bucket&, std::string const&, unsigned long) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.36% (2,694,891B) 0x51E7CB7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.36% (2,694,891B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.36% (2,694,891B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.36% (2,694,891B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.36% (2,694,891B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.36% (2,694,891B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.36% (2,694,891B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.36% (2,694,891B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.36% (2,694,891B) 0xF1A7492: start_thread (pthread_create.c:333) | | | | | ->01.36% (2,694,891B) 0xFF3F93D: clone (clone.S:97) | | | | | | | | | ->00.01% (11,322B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | ->00.06% (117,778B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | ->00.00% (0B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->00.05% (97,658B) in 1+ places, all below ms_print's threshold (01.00%) | ->13.69% (27,167,232B) 0x510DB36: RGWCoroutine::RGWCoroutine(CephContext*) (in /usr/lib/librgw.so.2.0.0) | ->13.63% (27,054,080B) 0x51075C3: RGWOmapAppend::RGWOmapAppend(RGWAsyncRadosProcessor*, RGWRados*, rgw_bucket&, std::string const&, unsigned long) (in /usr/lib/librgw.so.2.0.0) | | ->13.63% (27,054,080B) 0x51E7CB7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | ->13.63% (27,054,080B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | ->13.63% (27,054,080B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | ->13.63% (27,054,080B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | ->13.63% (27,054,080B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | ->13.63% (27,054,080B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | ->13.63% (27,054,080B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | ->13.63% (27,054,080B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | ->13.63% (27,054,080B) 0xF1A7492: start_thread (pthread_create.c:333) | | ->13.63% (27,054,080B) 0xFF3F93D: clone (clone.S:97) | | | ->00.06% (113,152B) in 1+ places, all below ms_print's threshold (01.00%) | ->11.95% (23,713,760B) 0x50F9E51: RGWCoroutinesManager::allocate_stack() (in /usr/lib/librgw.so.2.0.0) | ->11.95% (23,713,536B) 0x50FE7AE: RGWCoroutinesStack::spawn(RGWCoroutine*, RGWCoroutine*, bool) (in /usr/lib/librgw.so.2.0.0) | | ->05.96% (11,836,384B) 0x51E7CD7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | | ->05.96% (11,836,384B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | ->05.96% (11,836,384B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.96% (11,836,384B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.96% (11,836,384B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.96% (11,836,384B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.96% (11,836,384B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | ->05.96% (11,836,384B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | ->05.96% (11,836,384B) 0xF1A7492: start_thread (pthread_create.c:333) | | | ->05.96% (11,836,384B) 0xFF3F93D: clone (clone.S:97) | | | | | ->05.96% (11,836,160B) 0x51D5E5F: RGWDataSyncShardCR::init_lease_cr() (in /usr/lib/librgw.so.2.0.0) | | | ->05.96% (11,836,160B) 0x51E7CEE: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | | ->05.96% (11,836,160B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | ->05.96% (11,836,160B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.96% (11,836,160B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.96% (11,836,160B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.96% (11,836,160B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.96% (11,836,160B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | ->05.96% (11,836,160B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | ->05.96% (11,836,160B) 0xF1A7492: start_thread (pthread_create.c:333) | | | ->05.96% (11,836,160B) 0xFF3F93D: clone (clone.S:97) | | | | | ->00.02% (40,992B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->00.00% (224B) in 1+ places, all below ms_print's threshold (01.00%) | ->04.30% (8,535,285B) in 864 places, all below massif's threshold (1.00%) | ->01.71% (3,396,032B) 0x510DB0E: RGWCoroutine::RGWCoroutine(CephContext*) (in /usr/lib/librgw.so.2.0.0) | ->01.70% (3,381,824B) 0x51075C3: RGWOmapAppend::RGWOmapAppend(RGWAsyncRadosProcessor*, RGWRados*, rgw_bucket&, std::string const&, unsigned long) (in /usr/lib/librgw.so.2.0.0) | | ->01.70% (3,381,824B) 0x51E7CB7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | ->01.70% (3,381,824B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | ->01.70% (3,381,824B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | ->01.70% (3,381,824B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | ->01.70% (3,381,824B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | ->01.70% (3,381,824B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | ->01.70% (3,381,824B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | ->01.70% (3,381,824B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | ->01.70% (3,381,824B) 0xF1A7492: start_thread (pthread_create.c:333) | | ->01.70% (3,381,824B) 0xFF3F93D: clone (clone.S:97) | | | ->00.01% (14,208B) in 1+ places, all below ms_print's threshold (01.00%) | ->01.07% (2,124,880B) 0x5103628: std::pair, bool> std::_Rb_tree, std::less, std::allocator >::_M_insert_unique(RGWCoroutinesStack* const&) (in /usr/lib/librgw.so.2.0.0) | ->01.07% (2,119,720B) 0x50FE628: RGWCoroutinesManager::schedule(RGWCoroutinesEnv*, RGWCoroutinesStack*) (in /usr/lib/librgw.so.2.0.0) | | ->01.07% (2,119,720B) 0x50FE805: RGWCoroutinesStack::spawn(RGWCoroutine*, RGWCoroutine*, bool) (in /usr/lib/librgw.so.2.0.0) | | ->01.06% (2,113,640B) 0x51E7CD7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | | ->01.06% (2,113,640B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | ->01.06% (2,113,640B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | ->01.06% (2,113,640B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | ->01.06% (2,113,640B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | ->01.06% (2,113,640B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | ->01.06% (2,113,640B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | ->01.06% (2,113,640B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | ->01.06% (2,113,640B) 0xF1A7492: start_thread (pthread_create.c:333) | | | ->01.06% (2,113,640B) 0xFF3F93D: clone (clone.S:97) | | | | | ->00.00% (6,080B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->00.00% (5,160B) in 1+ places, all below ms_print's threshold (01.00%) | ->01.04% (2,067,635B) 0x54F757B: ceph::log::Log::create_entry(int, int, unsigned long*) (in /usr/lib/librgw.so.2.0.0) ->01.04% (2,067,635B) in 51 places, all below massif's threshold (1.00%) -------------------------------------------------------------------------------- n time(i) total(B) useful-heap(B) extra-heap(B) stacks(B) -------------------------------------------------------------------------------- 50 92,171,046,269 199,374,800 191,291,522 8,083,278 0 51 92,755,600,661 200,509,576 192,388,007 8,121,569 0 95.95% (192,388,007B) (heap allocation functions) malloc/new/new[], --alloc-fns, etc. ->42.47% (85,151,088B) 0x51E7C89: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | ->42.47% (85,151,088B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | ->42.47% (85,151,088B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | ->42.47% (85,151,088B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | ->42.47% (85,151,088B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | ->42.47% (85,151,088B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | ->42.47% (85,151,088B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | ->42.47% (85,151,088B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | ->42.47% (85,151,088B) 0xF1A7492: start_thread (pthread_create.c:333) | ->42.47% (85,151,088B) 0xFF3F93D: clone (clone.S:97) | ->20.10% (40,302,144B) 0xF9F4E97: std::string::_Rep::_S_create(unsigned long, unsigned long, std::allocator const&) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | ->18.66% (37,417,975B) 0xF9F5B09: std::string::_Rep::_M_clone(std::allocator const&, unsigned long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | ->18.66% (37,416,341B) 0xF9F5BAE: std::string::reserve(unsigned long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | ->14.41% (28,886,841B) 0xF9D1FB2: std::basic_stringbuf, std::allocator >::overflow(int) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->14.41% (28,886,841B) 0xF9D5D63: std::basic_streambuf >::xsputn(char const*, long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->14.41% (28,886,304B) 0xF9CD314: std::basic_ostream >& std::__ostream_insert >(std::basic_ostream >&, char const*, long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | | ->14.40% (28,867,509B) 0xF9CD6E5: std::basic_ostream >& std::operator<< >(std::basic_ostream >&, char const*) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | | | ->14.40% (28,867,509B) 0x5109AE4: RGWOmapAppend::operate() (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->14.40% (28,867,509B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->14.40% (28,867,509B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->14.40% (28,867,509B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->14.40% (28,867,509B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->14.40% (28,867,509B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->14.40% (28,867,509B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->14.40% (28,867,509B) 0xF1A7492: start_thread (pthread_create.c:333) | | | | | | | ->14.40% (28,867,509B) 0xFF3F93D: clone (clone.S:97) | | | | | | | | | | | | | ->00.00% (0B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | | | | | ->00.01% (18,795B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | | | ->00.00% (537B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | ->04.22% (8,454,234B) 0xF9F5E38: ??? (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->04.22% (8,454,234B) 0xF9F5F0C: std::string::append(char const*, unsigned long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->04.16% (8,341,657B) 0x51DF1E4: RGWDataSyncShardControlCR::alloc_cr() (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.16% (8,341,657B) 0x51AB72B: RGWBackoffControlCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.16% (8,341,657B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.16% (8,341,657B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.16% (8,341,657B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.16% (8,341,657B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.16% (8,341,657B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.16% (8,341,657B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.16% (8,341,657B) 0xF1A7492: start_thread (pthread_create.c:333) | | | | | ->04.16% (8,341,657B) 0xFF3F93D: clone (clone.S:97) | | | | | | | | | ->00.06% (112,577B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | ->00.04% (75,266B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | ->00.00% (1,634B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->01.41% (2,823,733B) 0xF9F6813: char* std::string::_S_construct(char const*, char const*, std::allocator const&, std::forward_iterator_tag) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | ->01.41% (2,823,733B) 0xF9F6C44: std::basic_string, std::allocator >::basic_string(char const*, std::allocator const&) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | ->01.37% (2,748,390B) 0x510DA32: RGWCoroutine::RGWCoroutine(CephContext*) (in /usr/lib/librgw.so.2.0.0) | | | | ->01.37% (2,741,607B) 0x51075C3: RGWOmapAppend::RGWOmapAppend(RGWAsyncRadosProcessor*, RGWRados*, rgw_bucket&, std::string const&, unsigned long) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.37% (2,741,607B) 0x51E7CB7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.37% (2,741,607B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.37% (2,741,607B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.37% (2,741,607B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.37% (2,741,607B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.37% (2,741,607B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.37% (2,741,607B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.37% (2,741,607B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.37% (2,741,607B) 0xF1A7492: start_thread (pthread_create.c:333) | | | | | ->01.37% (2,741,607B) 0xFF3F93D: clone (clone.S:97) | | | | | | | | | ->00.00% (6,783B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | ->00.04% (75,343B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | ->00.00% (0B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->00.03% (60,436B) in 1+ places, all below ms_print's threshold (01.00%) | ->13.76% (27,590,656B) 0x510DB36: RGWCoroutine::RGWCoroutine(CephContext*) (in /usr/lib/librgw.so.2.0.0) | ->13.73% (27,523,072B) 0x51075C3: RGWOmapAppend::RGWOmapAppend(RGWAsyncRadosProcessor*, RGWRados*, rgw_bucket&, std::string const&, unsigned long) (in /usr/lib/librgw.so.2.0.0) | | ->13.73% (27,523,072B) 0x51E7CB7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | ->13.73% (27,523,072B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | ->13.73% (27,523,072B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | ->13.73% (27,523,072B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | ->13.73% (27,523,072B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | ->13.73% (27,523,072B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | ->13.73% (27,523,072B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | ->13.73% (27,523,072B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | ->13.73% (27,523,072B) 0xF1A7492: start_thread (pthread_create.c:333) | | ->13.73% (27,523,072B) 0xFF3F93D: clone (clone.S:97) | | | ->00.03% (67,584B) in 1+ places, all below ms_print's threshold (01.00%) | ->12.03% (24,124,352B) 0x50F9E51: RGWCoroutinesManager::allocate_stack() (in /usr/lib/librgw.so.2.0.0) | ->12.03% (24,124,128B) 0x50FE7AE: RGWCoroutinesStack::spawn(RGWCoroutine*, RGWCoroutine*, bool) (in /usr/lib/librgw.so.2.0.0) | | ->06.01% (12,041,568B) 0x51E7CD7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | | ->06.01% (12,041,568B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | ->06.01% (12,041,568B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | ->06.01% (12,041,568B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | ->06.01% (12,041,568B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | ->06.01% (12,041,568B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | ->06.01% (12,041,568B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | ->06.01% (12,041,568B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | ->06.01% (12,041,568B) 0xF1A7492: start_thread (pthread_create.c:333) | | | ->06.01% (12,041,568B) 0xFF3F93D: clone (clone.S:97) | | | | | ->06.01% (12,041,568B) 0x51D5E5F: RGWDataSyncShardCR::init_lease_cr() (in /usr/lib/librgw.so.2.0.0) | | | ->06.01% (12,041,568B) 0x51E7CEE: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | | ->06.01% (12,041,568B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | ->06.01% (12,041,568B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | ->06.01% (12,041,568B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | ->06.01% (12,041,568B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | ->06.01% (12,041,568B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | ->06.01% (12,041,568B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | ->06.01% (12,041,568B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | ->06.01% (12,041,568B) 0xF1A7492: start_thread (pthread_create.c:333) | | | ->06.01% (12,041,568B) 0xFF3F93D: clone (clone.S:97) | | | | | ->00.02% (40,992B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->00.00% (224B) in 1+ places, all below ms_print's threshold (01.00%) | ->04.79% (9,610,207B) in 865 places, all below massif's threshold (1.00%) | ->01.72% (3,448,960B) 0x510DB0E: RGWCoroutine::RGWCoroutine(CephContext*) (in /usr/lib/librgw.so.2.0.0) | ->01.72% (3,440,448B) 0x51075C3: RGWOmapAppend::RGWOmapAppend(RGWAsyncRadosProcessor*, RGWRados*, rgw_bucket&, std::string const&, unsigned long) (in /usr/lib/librgw.so.2.0.0) | | ->01.72% (3,440,448B) 0x51E7CB7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | ->01.72% (3,440,448B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | ->01.72% (3,440,448B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | ->01.72% (3,440,448B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | ->01.72% (3,440,448B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | ->01.72% (3,440,448B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | ->01.72% (3,440,448B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | ->01.72% (3,440,448B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | ->01.72% (3,440,448B) 0xF1A7492: start_thread (pthread_create.c:333) | | ->01.72% (3,440,448B) 0xFF3F93D: clone (clone.S:97) | | | ->00.00% (8,512B) in 1+ places, all below ms_print's threshold (01.00%) | ->01.08% (2,160,600B) 0x5103628: std::pair, bool> std::_Rb_tree, std::less, std::allocator >::_M_insert_unique(RGWCoroutinesStack* const&) (in /usr/lib/librgw.so.2.0.0) ->01.07% (2,155,440B) 0x50FE628: RGWCoroutinesManager::schedule(RGWCoroutinesEnv*, RGWCoroutinesStack*) (in /usr/lib/librgw.so.2.0.0) | ->01.07% (2,155,440B) 0x50FE805: RGWCoroutinesStack::spawn(RGWCoroutine*, RGWCoroutine*, bool) (in /usr/lib/librgw.so.2.0.0) | ->01.07% (2,150,280B) 0x51E7CD7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | ->01.07% (2,150,280B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | ->01.07% (2,150,280B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | ->01.07% (2,150,280B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | ->01.07% (2,150,280B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | ->01.07% (2,150,280B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | ->01.07% (2,150,280B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | ->01.07% (2,150,280B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | ->01.07% (2,150,280B) 0xF1A7492: start_thread (pthread_create.c:333) | | ->01.07% (2,150,280B) 0xFF3F93D: clone (clone.S:97) | | | ->00.00% (5,160B) in 1+ places, all below ms_print's threshold (01.00%) | ->00.00% (5,160B) in 1+ places, all below ms_print's threshold (01.00%) -------------------------------------------------------------------------------- n time(i) total(B) useful-heap(B) extra-heap(B) stacks(B) -------------------------------------------------------------------------------- 52 93,259,361,818 202,586,560 194,382,959 8,203,601 0 95.95% (194,382,959B) (heap allocation functions) malloc/new/new[], --alloc-fns, etc. ->42.46% (86,015,952B) 0x51E7C89: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | ->42.46% (86,015,952B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | ->42.46% (86,015,952B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | ->42.46% (86,015,952B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | ->42.46% (86,015,952B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | ->42.46% (86,015,952B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | ->42.46% (86,015,952B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | ->42.46% (86,015,952B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | ->42.46% (86,015,952B) 0xF1A7492: start_thread (pthread_create.c:333) | ->42.46% (86,015,952B) 0xFF3F93D: clone (clone.S:97) | ->20.09% (40,698,130B) 0xF9F4E97: std::string::_Rep::_S_create(unsigned long, unsigned long, std::allocator const&) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | ->18.65% (37,784,495B) 0xF9F5B09: std::string::_Rep::_M_clone(std::allocator const&, unsigned long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | ->18.65% (37,782,918B) 0xF9F5BAE: std::string::reserve(unsigned long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | ->14.40% (29,179,506B) 0xF9D1FB2: std::basic_stringbuf, std::allocator >::overflow(int) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->14.40% (29,179,506B) 0xF9D5D63: std::basic_streambuf >::xsputn(char const*, long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->14.40% (29,178,969B) 0xF9CD314: std::basic_ostream >& std::__ostream_insert >(std::basic_ostream >&, char const*, long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | | ->14.39% (29,160,711B) 0xF9CD6E5: std::basic_ostream >& std::operator<< >(std::basic_ostream >&, char const*) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | | | ->14.39% (29,160,711B) 0x5109AE4: RGWOmapAppend::operate() (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->14.39% (29,160,711B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->14.39% (29,160,711B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->14.39% (29,160,711B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->14.39% (29,160,711B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->14.39% (29,160,711B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->14.39% (29,160,711B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->14.39% (29,160,711B) 0xF1A7492: start_thread (pthread_create.c:333) | | | | | | | ->14.39% (29,160,711B) 0xFF3F93D: clone (clone.S:97) | | | | | | | | | | | | | ->00.00% (0B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | | | | | ->00.01% (18,258B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | | | ->00.00% (537B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | ->04.21% (8,526,613B) 0xF9F5E38: ??? (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->04.21% (8,526,613B) 0xF9F5F0C: std::string::append(char const*, unsigned long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->04.16% (8,426,363B) 0x51DF1E4: RGWDataSyncShardControlCR::alloc_cr() (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.16% (8,426,363B) 0x51AB72B: RGWBackoffControlCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.16% (8,426,363B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.16% (8,426,363B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.16% (8,426,363B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.16% (8,426,363B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.16% (8,426,363B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.16% (8,426,363B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.16% (8,426,363B) 0xF1A7492: start_thread (pthread_create.c:333) | | | | | ->04.16% (8,426,363B) 0xFF3F93D: clone (clone.S:97) | | | | | | | | | ->00.05% (100,250B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | ->00.04% (76,799B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | ->00.00% (1,577B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->01.41% (2,852,267B) 0xF9F6813: char* std::string::_S_construct(char const*, char const*, std::allocator const&, std::forward_iterator_tag) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | ->01.41% (2,852,267B) 0xF9F6C44: std::basic_string, std::allocator >::basic_string(char const*, std::allocator const&) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | ->01.37% (2,776,236B) 0x510DA32: RGWCoroutine::RGWCoroutine(CephContext*) (in /usr/lib/librgw.so.2.0.0) | | | | ->01.37% (2,769,453B) 0x51075C3: RGWOmapAppend::RGWOmapAppend(RGWAsyncRadosProcessor*, RGWRados*, rgw_bucket&, std::string const&, unsigned long) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.37% (2,769,453B) 0x51E7CB7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.37% (2,769,453B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.37% (2,769,453B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.37% (2,769,453B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.37% (2,769,453B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.37% (2,769,453B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.37% (2,769,453B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.37% (2,769,453B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.37% (2,769,453B) 0xF1A7492: start_thread (pthread_create.c:333) | | | | | ->01.37% (2,769,453B) 0xFF3F93D: clone (clone.S:97) | | | | | | | | | ->00.00% (6,783B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | ->00.04% (76,031B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | ->00.00% (0B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->00.03% (61,368B) in 1+ places, all below ms_print's threshold (01.00%) | ->13.76% (27,870,208B) 0x510DB36: RGWCoroutine::RGWCoroutine(CephContext*) (in /usr/lib/librgw.so.2.0.0) | ->13.72% (27,802,624B) 0x51075C3: RGWOmapAppend::RGWOmapAppend(RGWAsyncRadosProcessor*, RGWRados*, rgw_bucket&, std::string const&, unsigned long) (in /usr/lib/librgw.so.2.0.0) | | ->13.72% (27,802,624B) 0x51E7CB7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | ->13.72% (27,802,624B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | ->13.72% (27,802,624B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | ->13.72% (27,802,624B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | ->13.72% (27,802,624B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | ->13.72% (27,802,624B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | ->13.72% (27,802,624B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | ->13.72% (27,802,624B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | ->13.72% (27,802,624B) 0xF1A7492: start_thread (pthread_create.c:333) | | ->13.72% (27,802,624B) 0xFF3F93D: clone (clone.S:97) | | | ->00.03% (67,584B) in 1+ places, all below ms_print's threshold (01.00%) | ->12.03% (24,368,960B) 0x50F9E51: RGWCoroutinesManager::allocate_stack() (in /usr/lib/librgw.so.2.0.0) | ->12.03% (24,368,736B) 0x50FE7AE: RGWCoroutinesStack::spawn(RGWCoroutine*, RGWCoroutine*, bool) (in /usr/lib/librgw.so.2.0.0) | | ->06.00% (12,163,872B) 0x51E7CD7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | | ->06.00% (12,163,872B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | ->06.00% (12,163,872B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | ->06.00% (12,163,872B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | ->06.00% (12,163,872B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | ->06.00% (12,163,872B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | ->06.00% (12,163,872B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | ->06.00% (12,163,872B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | ->06.00% (12,163,872B) 0xF1A7492: start_thread (pthread_create.c:333) | | | ->06.00% (12,163,872B) 0xFF3F93D: clone (clone.S:97) | | | | | ->06.00% (12,163,872B) 0x51D5E5F: RGWDataSyncShardCR::init_lease_cr() (in /usr/lib/librgw.so.2.0.0) | | | ->06.00% (12,163,872B) 0x51E7CEE: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | | ->06.00% (12,163,872B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | ->06.00% (12,163,872B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | ->06.00% (12,163,872B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | ->06.00% (12,163,872B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | ->06.00% (12,163,872B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | ->06.00% (12,163,872B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | ->06.00% (12,163,872B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | ->06.00% (12,163,872B) 0xF1A7492: start_thread (pthread_create.c:333) | | | ->06.00% (12,163,872B) 0xFF3F93D: clone (clone.S:97) | | | | | ->00.02% (40,992B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->00.00% (224B) in 1+ places, all below ms_print's threshold (01.00%) | ->04.82% (9,763,365B) in 865 places, all below massif's threshold (1.00%) | ->01.72% (3,483,904B) 0x510DB0E: RGWCoroutine::RGWCoroutine(CephContext*) (in /usr/lib/librgw.so.2.0.0) | ->01.72% (3,475,392B) 0x51075C3: RGWOmapAppend::RGWOmapAppend(RGWAsyncRadosProcessor*, RGWRados*, rgw_bucket&, std::string const&, unsigned long) (in /usr/lib/librgw.so.2.0.0) | | ->01.72% (3,475,392B) 0x51E7CB7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | ->01.72% (3,475,392B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | ->01.72% (3,475,392B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | ->01.72% (3,475,392B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | ->01.72% (3,475,392B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | ->01.72% (3,475,392B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | ->01.72% (3,475,392B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | ->01.72% (3,475,392B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | ->01.72% (3,475,392B) 0xF1A7492: start_thread (pthread_create.c:333) | | ->01.72% (3,475,392B) 0xFF3F93D: clone (clone.S:97) | | | ->00.00% (8,512B) in 1+ places, all below ms_print's threshold (01.00%) | ->01.08% (2,182,440B) 0x5103628: std::pair, bool> std::_Rb_tree, std::less, std::allocator >::_M_insert_unique(RGWCoroutinesStack* const&) (in /usr/lib/librgw.so.2.0.0) ->01.07% (2,177,280B) 0x50FE628: RGWCoroutinesManager::schedule(RGWCoroutinesEnv*, RGWCoroutinesStack*) (in /usr/lib/librgw.so.2.0.0) | ->01.07% (2,177,280B) 0x50FE805: RGWCoroutinesStack::spawn(RGWCoroutine*, RGWCoroutine*, bool) (in /usr/lib/librgw.so.2.0.0) | ->01.07% (2,172,120B) 0x51E7CD7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | ->01.07% (2,172,120B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | ->01.07% (2,172,120B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | ->01.07% (2,172,120B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | ->01.07% (2,172,120B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | ->01.07% (2,172,120B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | ->01.07% (2,172,120B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | ->01.07% (2,172,120B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | ->01.07% (2,172,120B) 0xF1A7492: start_thread (pthread_create.c:333) | | ->01.07% (2,172,120B) 0xFF3F93D: clone (clone.S:97) | | | ->00.00% (5,160B) in 1+ places, all below ms_print's threshold (01.00%) | ->00.00% (5,160B) in 1+ places, all below ms_print's threshold (01.00%) -------------------------------------------------------------------------------- n time(i) total(B) useful-heap(B) extra-heap(B) stacks(B) -------------------------------------------------------------------------------- 53 93,499,219,980 204,612,568 196,267,351 8,345,217 0 95.92% (196,267,351B) (heap allocation functions) malloc/new/new[], --alloc-fns, etc. ->42.12% (86,180,688B) 0x51E7C89: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | ->42.12% (86,180,688B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | ->42.12% (86,180,688B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | ->42.12% (86,180,688B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | ->42.12% (86,180,688B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | ->42.12% (86,180,688B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | ->42.12% (86,180,688B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | ->42.12% (86,180,688B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | ->42.12% (86,180,688B) 0xF1A7492: start_thread (pthread_create.c:333) | ->42.12% (86,180,688B) 0xFF3F93D: clone (clone.S:97) | ->20.04% (41,006,544B) 0xF9F4E97: std::string::_Rep::_S_create(unsigned long, unsigned long, std::allocator const&) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | ->18.54% (37,932,748B) 0xF9F5B09: std::string::_Rep::_M_clone(std::allocator const&, unsigned long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | ->18.53% (37,920,256B) 0xF9F5BAE: std::string::reserve(unsigned long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | ->14.31% (29,275,554B) 0xF9D1FB2: std::basic_stringbuf, std::allocator >::overflow(int) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->14.31% (29,275,554B) 0xF9D5D63: std::basic_streambuf >::xsputn(char const*, long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->14.31% (29,274,480B) 0xF9CD314: std::basic_ostream >& std::__ostream_insert >(std::basic_ostream >&, char const*, long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | | ->14.28% (29,216,559B) 0xF9CD6E5: std::basic_ostream >& std::operator<< >(std::basic_ostream >&, char const*) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | | | ->14.28% (29,216,559B) 0x5109AE4: RGWOmapAppend::operate() (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->14.28% (29,216,559B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->14.28% (29,216,559B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->14.28% (29,216,559B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->14.28% (29,216,559B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->14.28% (29,216,559B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->14.28% (29,216,559B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | | | | | ->14.28% (29,216,559B) 0xF1A7492: start_thread (pthread_create.c:333) | | | | | | | ->14.28% (29,216,559B) 0xFF3F93D: clone (clone.S:97) | | | | | | | | | | | | | ->00.00% (0B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | | | | | ->00.03% (57,921B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | | | ->00.00% (1,074B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | ->04.16% (8,520,997B) 0xF9F5E38: ??? (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->04.16% (8,520,997B) 0xF9F5F0C: std::string::append(char const*, unsigned long) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | | ->04.13% (8,442,511B) 0x51DF1E4: RGWDataSyncShardControlCR::alloc_cr() (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.13% (8,442,511B) 0x51AB72B: RGWBackoffControlCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.13% (8,442,511B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.13% (8,442,511B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.13% (8,442,511B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.13% (8,442,511B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.13% (8,442,511B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.13% (8,442,511B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | | | ->04.13% (8,442,511B) 0xF1A7492: start_thread (pthread_create.c:333) | | | | | ->04.13% (8,442,511B) 0xFF3F93D: clone (clone.S:97) | | | | | | | | | ->00.04% (78,486B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | ->00.06% (123,705B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | ->00.01% (12,492B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->01.43% (2,929,085B) 0xF9F6813: char* std::string::_S_construct(char const*, char const*, std::allocator const&, std::forward_iterator_tag) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | ->01.43% (2,929,085B) 0xF9F6C44: std::basic_string, std::allocator >::basic_string(char const*, std::allocator const&) (in /usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.20) | | | ->01.36% (2,785,008B) 0x510DA32: RGWCoroutine::RGWCoroutine(CephContext*) (in /usr/lib/librgw.so.2.0.0) | | | | ->01.36% (2,774,757B) 0x51075C3: RGWOmapAppend::RGWOmapAppend(RGWAsyncRadosProcessor*, RGWRados*, rgw_bucket&, std::string const&, unsigned long) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.36% (2,774,757B) 0x51E7CB7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.36% (2,774,757B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.36% (2,774,757B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.36% (2,774,757B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.36% (2,774,757B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.36% (2,774,757B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.36% (2,774,757B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.36% (2,774,757B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | | | ->01.36% (2,774,757B) 0xF1A7492: start_thread (pthread_create.c:333) | | | | | ->01.36% (2,774,757B) 0xFF3F93D: clone (clone.S:97) | | | | | | | | | ->00.01% (10,251B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | | | ->00.07% (144,077B) in 1+ places, all below ms_print's threshold (01.00%) | | | | | ->00.00% (0B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->00.07% (144,711B) in 1+ places, all below ms_print's threshold (01.00%) | ->13.66% (27,958,272B) 0x510DB36: RGWCoroutine::RGWCoroutine(CephContext*) (in /usr/lib/librgw.so.2.0.0) | ->13.61% (27,855,872B) 0x51075C3: RGWOmapAppend::RGWOmapAppend(RGWAsyncRadosProcessor*, RGWRados*, rgw_bucket&, std::string const&, unsigned long) (in /usr/lib/librgw.so.2.0.0) | | ->13.61% (27,855,872B) 0x51E7CB7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | ->13.61% (27,855,872B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | ->13.61% (27,855,872B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | ->13.61% (27,855,872B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | ->13.61% (27,855,872B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | ->13.61% (27,855,872B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | ->13.61% (27,855,872B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | ->13.61% (27,855,872B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | ->13.61% (27,855,872B) 0xF1A7492: start_thread (pthread_create.c:333) | | ->13.61% (27,855,872B) 0xFF3F93D: clone (clone.S:97) | | | ->00.05% (102,400B) in 1+ places, all below ms_print's threshold (01.00%) | ->11.93% (24,415,552B) 0x50F9E51: RGWCoroutinesManager::allocate_stack() (in /usr/lib/librgw.so.2.0.0) | ->11.93% (24,415,328B) 0x50FE7AE: RGWCoroutinesStack::spawn(RGWCoroutine*, RGWCoroutine*, bool) (in /usr/lib/librgw.so.2.0.0) | | ->05.96% (12,187,168B) 0x51E7CD7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | | ->05.96% (12,187,168B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | ->05.96% (12,187,168B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.96% (12,187,168B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.96% (12,187,168B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.96% (12,187,168B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.96% (12,187,168B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | ->05.96% (12,187,168B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | ->05.96% (12,187,168B) 0xF1A7492: start_thread (pthread_create.c:333) | | | ->05.96% (12,187,168B) 0xFF3F93D: clone (clone.S:97) | | | | | ->05.96% (12,187,168B) 0x51D5E5F: RGWDataSyncShardCR::init_lease_cr() (in /usr/lib/librgw.so.2.0.0) | | | ->05.96% (12,187,168B) 0x51E7CEE: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | | ->05.96% (12,187,168B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | | ->05.96% (12,187,168B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.96% (12,187,168B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.96% (12,187,168B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | | ->05.96% (12,187,168B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | | ->05.96% (12,187,168B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | | ->05.96% (12,187,168B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | | ->05.96% (12,187,168B) 0xF1A7492: start_thread (pthread_create.c:333) | | | ->05.96% (12,187,168B) 0xFF3F93D: clone (clone.S:97) | | | | | ->00.02% (40,992B) in 1+ places, all below ms_print's threshold (01.00%) | | | ->00.00% (224B) in 1+ places, all below ms_print's threshold (01.00%) | ->05.39% (11,023,903B) in 865 places, all below massif's threshold (1.00%) | ->01.71% (3,494,912B) 0x510DB0E: RGWCoroutine::RGWCoroutine(CephContext*) (in /usr/lib/librgw.so.2.0.0) | ->01.70% (3,482,048B) 0x51075C3: RGWOmapAppend::RGWOmapAppend(RGWAsyncRadosProcessor*, RGWRados*, rgw_bucket&, std::string const&, unsigned long) (in /usr/lib/librgw.so.2.0.0) | | ->01.70% (3,482,048B) 0x51E7CB7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | ->01.70% (3,482,048B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | ->01.70% (3,482,048B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | ->01.70% (3,482,048B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | ->01.70% (3,482,048B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | ->01.70% (3,482,048B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | ->01.70% (3,482,048B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | ->01.70% (3,482,048B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | ->01.70% (3,482,048B) 0xF1A7492: start_thread (pthread_create.c:333) | | ->01.70% (3,482,048B) 0xFF3F93D: clone (clone.S:97) | | | ->00.01% (12,864B) in 1+ places, all below ms_print's threshold (01.00%) | ->01.07% (2,187,480B) 0x5103628: std::pair, bool> std::_Rb_tree, std::less, std::allocator >::_M_insert_unique(RGWCoroutinesStack* const&) (in /usr/lib/librgw.so.2.0.0) ->01.07% (2,182,320B) 0x50FE628: RGWCoroutinesManager::schedule(RGWCoroutinesEnv*, RGWCoroutinesStack*) (in /usr/lib/librgw.so.2.0.0) | ->01.07% (2,182,320B) 0x50FE805: RGWCoroutinesStack::spawn(RGWCoroutine*, RGWCoroutine*, bool) (in /usr/lib/librgw.so.2.0.0) | ->01.06% (2,176,280B) 0x51E7CD7: RGWDataSyncShardCR::incremental_sync() (in /usr/lib/librgw.so.2.0.0) | | ->01.06% (2,176,280B) 0x51EA08B: RGWDataSyncShardCR::operate() (in /usr/lib/librgw.so.2.0.0) | | ->01.06% (2,176,280B) 0x50F89FC: RGWCoroutinesStack::operate(RGWCoroutinesEnv*) (in /usr/lib/librgw.so.2.0.0) | | ->01.06% (2,176,280B) 0x50FBE75: RGWCoroutinesManager::run(std::list >&) (in /usr/lib/librgw.so.2.0.0) | | ->01.06% (2,176,280B) 0x50FDDCC: RGWCoroutinesManager::run(RGWCoroutine*) (in /usr/lib/librgw.so.2.0.0) | | ->01.06% (2,176,280B) 0x51C754F: RGWRemoteDataLog::run_sync(int, rgw_data_sync_status&) (in /usr/lib/librgw.so.2.0.0) | | ->01.06% (2,176,280B) 0x529FD47: RGWDataSyncProcessorThread::process() (in /usr/lib/librgw.so.2.0.0) | | ->01.06% (2,176,280B) 0x523C0F4: RGWRadosThread::Worker::entry() (in /usr/lib/librgw.so.2.0.0) | | ->01.06% (2,176,280B) 0xF1A7492: start_thread (pthread_create.c:333) | | ->01.06% (2,176,280B) 0xFF3F93D: clone (clone.S:97) | | | ->00.00% (6,040B) in 1+ places, all below ms_print's threshold (01.00%) | ->00.00% (5,160B) in 1+ places, all below ms_print's threshold (01.00%)