Actions
Bug #45611
opencrimson: centos 8 vstart failure
Status:
New
Priority:
Normal
Assignee:
-
Category:
-
Target version:
-
% Done:
0%
Source:
Tags:
Backport:
Regression:
No
Severity:
3 - minor
Reviewed:
Affected Versions:
ceph-qa-suite:
Component(RADOS):
Pull request ID:
Crash signature (v1):
Crash signature (v2):
Description
$ bin/crimson-osd -i 0
[Backtrace #0]
seastar::backtrace_buffer::append_backtrace() at /home/ceph/ceph/src/seastar/src/core/reactor.cc:741
seastar::print_with_backtrace(seastar::backtrace_buffer&) at /home/ceph/ceph/src/seastar/src/core/reactor.cc:762
seastar::internal::cpu_stall_detector::generate_trace() at /home/ceph/ceph/src/seastar/src/core/reactor.cc:1211
seastar::internal::cpu_stall_detector::maybe_report() at /home/ceph/ceph/src/seastar/src/core/reactor.cc:1057
seastar::internal::cpu_stall_detector::on_signal() at /home/ceph/ceph/src/seastar/src/core/reactor.cc:1071
seastar::reactor::block_notifier(int) at /home/ceph/ceph/src/seastar/src/core/reactor.cc:1194
.annobin_sigaction.c at sigaction.c:?
__lsan_do_recoverable_leak_check at ??:?
__lsan_do_recoverable_leak_check at ??:?
__lsan_do_recoverable_leak_check at ??:?
__lsan_do_recoverable_leak_check at ??:?
__lsan_do_recoverable_leak_check at ??:?
__lsan_do_recoverable_leak_check at ??:?
__asan_backtrace_dwarf_add at ??:?
__asan_internal_strnlen at ??:?
__asan_internal_strnlen at ??:?
__asan_internal_strnlen at ??:?
dl_iterate_phdr at /home/ceph/ceph/src/seastar/src/core/exception_hacks.cc:125 (discriminator 2)
__asan_backtrace_initialize at ??:?
__asan_backtrace_dwarf_add at ??:?
__asan_backtrace_pcinfo at ??:?
__sanitizer_get_module_and_offset_for_pc at ??:?
__sanitizer_get_module_and_offset_for_pc at ??:?
OnPrint at ??:?
__sanitizer_install_malloc_and_free_hooks at ??:?
__asan_on_error at ??:?
__asan_unpoison_intra_object_redzone at ??:?
.annobin_sigaction.c at sigaction.c:?
operator new(unsigned long) at ??:?
__gnu_cxx::new_allocator<std::_Rb_tree_node<std::pair<std::basic_string_view<char, std::char_traits<char> > const, Option const&> > >::allocate(unsigned long, void const*) at /usr/include/c++/8/ext/new_allocator.h:111
std::allocator_traits<std::allocator<std::_Rb_tree_node<std::pair<std::basic_string_view<char, std::char_traits<char> > const, Option const&> > > >::allocate(std::allocator<std::_Rb_tree_node<std::pair<std::basic_string_view<char, std::char_traits<char> > const, Option const&> > >&, unsigned long) at /usr/include/c++/8/bits/alloc_traits.h:436
std::_Rb_tree<std::basic_string_view<char, std::char_traits<char> >, std::pair<std::basic_string_view<char, std::char_traits<char> > const, Option const&>, std::_Select1st<std::pair<std::basic_string_view<char, std::char_traits<char> > const, Option const&> >, std::less<std::basic_string_view<char, std::char_traits<char> > >, std::allocator<std::pair<std::basic_string_view<char, std::char_traits<char> > const, Option const&> > >::_M_get_node() at /usr/include/c++/8/bits/stl_tree.h:588
std::_Rb_tree_node<std::pair<std::basic_string_view<char, std::char_traits<char> > const, Option const&> >* std::_Rb_tree<std::basic_string_view<char, std::char_traits<char> >, std::pair<std::basic_string_view<char, std::char_traits<char> > const, Option const&>, std::_Select1st<std::pair<std::basic_string_view<char, std::char_traits<char> > const, Option const&> >, std::less<std::basic_string_view<char, std::char_traits<char> > >, std::allocator<std::pair<std::basic_string_view<char, std::char_traits<char> > const, Option const&> > >::_M_create_node<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, Option const&>(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, Option const&) at /usr/include/c++/8/bits/stl_tree.h:642
std::pair<std::_Rb_tree_iterator<std::pair<std::basic_string_view<char, std::char_traits<char> > const, Option const&> >, bool> std::_Rb_tree<std::basic_string_view<char, std::char_traits<char> >, std::pair<std::basic_string_view<char, std::char_traits<char> > const, Option const&>, std::_Select1st<std::pair<std::basic_string_view<char, std::char_traits<char> > const, Option const&> >, std::less<std::basic_string_view<char, std::char_traits<char> > >, std::allocator<std::pair<std::basic_string_view<char, std::char_traits<char> > const, Option const&> > >::_M_emplace_unique<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, Option const&>(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, Option const&) at /usr/include/c++/8/bits/stl_tree.h:2370
std::pair<std::_Rb_tree_iterator<std::pair<std::basic_string_view<char, std::char_traits<char> > const, Option const&> >, bool> std::map<std::basic_string_view<char, std::char_traits<char> >, Option const&, std::less<std::basic_string_view<char, std::char_traits<char> > >, std::allocator<std::pair<std::basic_string_view<char, std::char_traits<char> > const, Option const&> > >::emplace<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, Option const&>(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, Option const&) at /usr/include/c++/8/bits/stl_map.h:575
md_config_t::md_config_t(ConfigValues&, ConfigTracker const&, bool) at /home/ceph/ceph/src/common/config.cc:125
crimson::common::ConfigProxy::ConfigProxy(EntityName const&, std::basic_string_view<char, std::char_traits<char> >) at /home/ceph/ceph/src/crimson/common/config_proxy.cc:18 (discriminator 1)
seastar::shared_ptr_count_for<crimson::common::ConfigProxy>::shared_ptr_count_for<EntityName&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&>(EntityName&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&) at /home/ceph/ceph/src/seastar/include/seastar/core/shared_ptr.hh:466
seastar::shared_ptr<crimson::common::ConfigProxy> seastar::shared_ptr_make_helper<crimson::common::ConfigProxy, false>::make<EntityName&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&>(EntityName&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&) at /home/ceph/ceph/src/seastar/include/seastar/core/shared_ptr.hh:629
seastar::shared_ptr<crimson::common::ConfigProxy> seastar::make_shared<crimson::common::ConfigProxy, EntityName&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&>(EntityName&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&) at /home/ceph/ceph/src/seastar/include/seastar/core/shared_ptr.hh:647
seastar::shared_ptr<crimson::common::ConfigProxy> seastar::sharded<crimson::common::ConfigProxy>::create_local_service<EntityName&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&>(EntityName&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&) at /home/ceph/ceph/src/seastar/include/seastar/core/sharded.hh:477
seastar::sharded<crimson::common::ConfigProxy>::start<EntityName&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&>(EntityName&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&)::{lambda(unsigned int)#1}::operator()(unsigned int)::{lambda()#1}::operator()()::{lambda(EntityName&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&)#1}::operator()(EntityName&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&) const at /home/ceph/ceph/src/seastar/include/seastar/core/sharded.hh:542
seastar::apply_helper<seastar::sharded<crimson::common::ConfigProxy>::start<EntityName&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&>(EntityName&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&)::{lambda(unsigned int)#1}::operator()(unsigned int)::{lambda()#1}::operator()()::{lambda(EntityName&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&)#1}, std::tuple<EntityName, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >&, std::integer_sequence<unsigned long, 0ul, 1ul> >::apply({lambda()#1}&&, seastar::sharded<crimson::common::ConfigProxy>::start<EntityName&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&>(EntityName&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&)::{lambda(unsigned int)#1}::operator()(unsigned int)::{lambda()#1}::operator()()::{lambda(EntityName&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&)#1}) at /home/ceph/ceph/src/seastar/include/seastar/core/apply.hh:36
auto seastar::apply<seastar::sharded<crimson::common::ConfigProxy>::start<EntityName&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&>(EntityName&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&)::{lambda(unsigned int)#1}::operator()(unsigned int)::{lambda()#1}::operator()()::{lambda(EntityName&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&)#1}, EntityName, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >(seastar::sharded<crimson::common::ConfigProxy>::start<EntityName&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&>(EntityName&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&)::{lambda(unsigned int)#1}::operator()(unsigned int)::{lambda()#1}::operator()()::{lambda(EntityName&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&)#1}&&, std::tuple<EntityName, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >&) at /home/ceph/ceph/src/seastar/include/seastar/core/apply.hh:51
seastar::sharded<crimson::common::ConfigProxy>::start<EntityName&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&>(EntityName&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&)::{lambda(unsigned int)#1}::operator()(unsigned int)::{lambda()#1}::operator()() at /home/ceph/ceph/src/seastar/include/seastar/core/sharded.hh:541
seastar::future<> seastar::futurize<void>::invoke<seastar::sharded<crimson::common::ConfigProxy>::start<EntityName&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&>(EntityName&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&)::{lambda(unsigned int)#1}::operator()(unsigned int)::{lambda()#1}>(seastar::sharded<crimson::common::ConfigProxy>::start<EntityName&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&>(EntityName&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&)::{lambda(unsigned int)#1}::operator()(unsigned int)::{lambda()#1}&&) at /home/ceph/ceph/src/seastar/include/seastar/core/future.hh:1657
seastar::futurize<std::result_of<seastar::sharded<crimson::common::ConfigProxy>::start<EntityName&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&>(EntityName&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&)::{lambda(unsigned int)#1}::operator()(unsigned int)::{lambda()#1} ()>::type>::type seastar::smp::submit_to<seastar::sharded<crimson::common::ConfigProxy>::start<EntityName&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&>(EntityName&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&)::{lambda(unsigned int)#1}::operator()(unsigned int)::{lambda()#1}>(unsigned int, seastar::smp_submit_to_options, seastar::futurize&&) at /home/ceph/ceph/src/seastar/include/seastar/core/smp.hh:322
seastar::futurize<std::result_of<seastar::sharded<crimson::common::ConfigProxy>::start<EntityName&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&>(EntityName&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&)::{lambda(unsigned int)#1}::operator()(unsigned int)::{lambda()#1} ()>::type>::type seastar::smp::submit_to<seastar::sharded<crimson::common::ConfigProxy>::start<EntityName&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&>(EntityName&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&)::{lambda(unsigned int)#1}::operator()(unsigned int)::{lambda()#1}>(unsigned int, seastar::futurize&&) at /home/ceph/ceph/src/seastar/include/seastar/core/smp.hh:356
seastar::sharded<crimson::common::ConfigProxy>::start<EntityName&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&>(EntityName&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&)::{lambda(unsigned int)#1}::operator()(unsigned int) at /home/ceph/ceph/src/seastar/include/seastar/core/sharded.hh:544
std::_Function_handler<seastar::future<> (unsigned int), seastar::sharded<crimson::common::ConfigProxy>::start<EntityName&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&>(EntityName&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&)::{lambda(unsigned int)#1}>::_M_invoke(std::_Any_data const&, unsigned int&&) at /usr/include/c++/8/bits/std_function.h:283
std::function<seastar::future<> (unsigned int)>::operator()(unsigned int) const at /usr/include/c++/8/bits/std_function.h:687
seastar::future<> seastar::futurize<seastar::future<> >::invoke<std::function<seastar::future<> (unsigned int)>, unsigned int>(std::function<seastar::future<> (unsigned int)>&&, unsigned int&&) at /home/ceph/ceph/src/seastar/include/seastar/core/future.hh:1683
auto seastar::futurize_invoke<std::function<seastar::future<> (unsigned int)>, unsigned int>(std::function<seastar::future<> (unsigned int)>&&, unsigned int&&) at /home/ceph/ceph/src/seastar/include/seastar/core/future.hh:1767
seastar::future<> seastar::parallel_for_each<boost::range_detail::integer_iterator<unsigned int>, std::function<seastar::future<> (unsigned int)> >(boost::range_detail::integer_iterator<unsigned int>, boost::range_detail::integer_iterator<unsigned int>, std::function<seastar::future<> (unsigned int)>&&) at /home/ceph/ceph/src/seastar/include/seastar/core/future-util.hh:158
seastar::future<> seastar::internal::parallel_for_each_impl<boost::integer_range<unsigned int>, std::function<seastar::future<> (unsigned int)> >(boost::integer_range<unsigned int>&&, std::function<seastar::future<> (unsigned int)>&&) at /home/ceph/ceph/src/seastar/include/seastar/core/future-util.hh:203
seastar::future<> seastar::futurize<seastar::future<> >::invoke<seastar::future<> (*&)(boost::integer_range<unsigned int>&&, std::function<seastar::future<> (unsigned int)>&&), boost::integer_range<unsigned int>, std::function<seastar::future<> (unsigned int)> >(seastar::future<> (*&)(boost::integer_range<unsigned int>&&, std::function<seastar::future<> (unsigned int)>&&), boost::integer_range<unsigned int>&&, std::function<seastar::future<> (unsigned int)>&&) at /home/ceph/ceph/src/seastar/include/seastar/core/future.hh:1683
auto seastar::futurize_invoke<seastar::future<> (*&)(boost::integer_range<unsigned int>&&, std::function<seastar::future<> (unsigned int)>&&), boost::integer_range<unsigned int>, std::function<seastar::future<> (unsigned int)> >(seastar::future<> (*&)(boost::integer_range<unsigned int>&&, std::function<seastar::future<> (unsigned int)>&&), boost::integer_range<unsigned int>&&, std::function<seastar::future<> (unsigned int)>&&) at /home/ceph/ceph/src/seastar/include/seastar/core/future.hh:1767
seastar::future<> seastar::parallel_for_each<boost::integer_range<unsigned int>, std::function<seastar::future<> (unsigned int)> >(boost::integer_range<unsigned int>&&, std::function<seastar::future<> (unsigned int)>&&) at /home/ceph/ceph/src/seastar/include/seastar/core/future-util.hh:215
seastar::internal::sharded_parallel_for_each(unsigned int, std::function<seastar::future<> (unsigned int)>) at /home/ceph/ceph/src/seastar/src/core/sharded.cc:32
seastar::future<> seastar::sharded<crimson::common::ConfigProxy>::start<EntityName&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&>(EntityName&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&) at /home/ceph/ceph/src/seastar/include/seastar/core/sharded.hh:554 (discriminator 2)
main::{lambda()#1}::operator()() const::{lambda()#1}::operator()() const at /home/ceph/ceph/src/crimson/osd/main.cc:157
seastar::apply_helper<main::{lambda()#1}::operator()() const::{lambda()#1}, std::tuple<>&&, std::integer_sequence<unsigned long> >::apply({lambda()#1}&&, std::tuple) at /home/ceph/ceph/src/seastar/include/seastar/core/apply.hh:36
auto seastar::apply<main::{lambda()#1}::operator()() const::{lambda()#1}>(main::{lambda()#1}::operator()() const::{lambda()#1}&&, std::tuple<>&&) at /home/ceph/ceph/src/seastar/include/seastar/core/apply.hh:44
seastar::future<> seastar::futurize<void>::apply<main::{lambda()#1}::operator()() const::{lambda()#1}>(main::{lambda()#1}::operator()() const::{lambda()#1}&&, std::tuple<>&&) at /home/ceph/ceph/src/seastar/include/seastar/core/future.hh:1647
_ZZZN7seastar5asyncIZZ4mainENKUlvE_clEvEUlvE_JEEENS_8futurizeINSt9result_ofIFNSt5decayIT_E4typeEDpNS5_IT0_E4typeEEE4typeEE4typeENS_17thread_attributesEOS6_DpOS9_ENUlRZNS0_IS2_JEEESH_SI_SJ_SL_E4workE_clESN_ENKUlvE_clEv at /home/ceph/ceph/src/seastar/include/seastar/core/thread.hh:253
_ZN7seastar20noncopyable_functionIFvvEE17direct_vtable_forIZZNS_5asyncIZZ4mainENKUlvE_clEvEUlvE_JEEENS_8futurizeINSt9result_ofIFNSt5decayIT_E4typeEDpNS9_IT0_E4typeEEE4typeEE4typeENS_17thread_attributesEOSA_DpOSD_ENUlRZNS4_IS6_JEEESL_SM_SN_SP_E4workE_clESR_EUlvE_E4callEPKS2_ at /home/ceph/ceph/src/seastar/include/seastar/util/noncopyable_function.hh:101
seastar::noncopyable_function<void ()>::operator()() const at /home/ceph/ceph/src/seastar/include/seastar/util/noncopyable_function.hh:184
seastar::thread_context::main() at /home/ceph/ceph/src/seastar/src/core/thread.cc:286
Updated by Deepika Upadhyay almost 4 years ago
caught some memory leaks using core dumps, but they seem to be related to asan/libc
==42539==ERROR: LeakSanitizer: detected memory leaks
Direct leak of 400 byte(s) in 2 object(s) allocated from:
#0 0x7f152ce9df28 in __interceptor_realloc (/lib64/libasan.so.5+0xeff28)
#1 0x562efe506407 (/usr/bin/sudo+0x15407)
Direct leak of 256 byte(s) in 1 object(s) allocated from:
#0 0x7f152ce9df28 in __interceptor_realloc (/lib64/libasan.so.5+0xeff28)
#1 0x562efe50684e (/usr/bin/sudo+0x1584e)
Indirect leak of 177 byte(s) in 6 object(s) allocated from:
#0 0x7f152ce9dae8 in __interceptor_malloc (/lib64/libasan.so.5+0xefae8)
#1 0x7f152c54413b in sudo_new_key_val_v1 (/usr/libexec/sudo/libsudo_util.so.0+0x813b)
Indirect leak of 108 byte(s) in 12 object(s) allocated from:
#0 0x7f152ce9dae8 in __interceptor_malloc (/lib64/libasan.so.5+0xefae8)
#1 0x7f152be62477 in __GI___vasprintf_chk (/lib64/libc.so.6+0x10d477)
Indirect leak of 18 byte(s) in 1 object(s) allocated from:
#0 0x7f152ce9dae8 in __interceptor_malloc (/lib64/libasan.so.5+0xefae8)
#1 0x562efe506b37 (/usr/bin/sudo+0x15b37)
SUMMARY: AddressSanitizer: 959 byte(s) leaked in 22 allocation(s).
Updated by Deepika Upadhyay almost 4 years ago
- Related to Subtask #45535: crimson: crimson-osd failure in ceph-container added
Updated by Deepika Upadhyay almost 4 years ago
How to reproduce:
1. launch a centos 8 container and build vstart with -DWITH_SEASTAR=ON
2. start a vstart base cluster with crimson:
MGR=1 MON=1 OSD=1 MDS=0 RGW=0 ../src/vstart.sh -n -x --without-dashboard --memstore --crimson --nodaemon --redirect-output --osd-args "--memory 4G"
Updated by Deepika Upadhyay almost 4 years ago
caught segfault at points
1. run with next option in gdb:
## The point of failure: /home/ceph/ceph/src/seastar/src/core/thread.cc 286 _func();
2. when tracing subroutine as well using step
## if subroutine passes failure at : <lambda()>::<lambda()>::operator()(void) const (__closure=0x60f000044948) at /home/ceph/ceph/src/crimson/osd/main.cc:157 157 sharded_conf().start(init_params.name, cluster_name).get();
Actions