Actions
Bug #44566
closedceph tell segv: librados fini vs protocolv2
Status:
Resolved
Priority:
Urgent
Assignee:
-
Category:
-
Target version:
-
% Done:
0%
Source:
Tags:
Backport:
Regression:
No
Severity:
3 - minor
Reviewed:
Affected Versions:
ceph-qa-suite:
Component(RADOS):
Pull request ID:
Crash signature (v1):
Crash signature (v2):
Description
2020-03-11T12:45:56.818 INFO:teuthology.orchestra.run.smithi112:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph --log-early -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2020-03-11T12:45:57.143 INFO:teuthology.orchestra.run.smithi112.stderr:osd.0: osd_enable_op_tracker = 'true' 2020-03-11T12:45:57.144 INFO:teuthology.orchestra.run.smithi112.stderr:osd.1: osd_enable_op_tracker = 'true' 2020-03-11T12:45:57.144 INFO:teuthology.orchestra.run.smithi112.stderr:osd.2: osd_enable_op_tracker = 'true' 2020-03-11T12:45:57.144 INFO:teuthology.orchestra.run.smithi112.stderr:Error ENXIO: problem getting command descriptions from osd.3 2020-03-11T12:45:57.144 INFO:teuthology.orchestra.run.smithi112.stderr:osd.3: problem getting command descriptions from osd.3 2020-03-11T12:45:57.145 INFO:teuthology.orchestra.run.smithi112.stderr:osd.4: osd_enable_op_tracker = 'true' 2020-03-11T12:45:57.145 INFO:teuthology.orchestra.run.smithi112.stderr:osd.5: osd_enable_op_tracker = 'true' 2020-03-11T12:45:57.145 INFO:teuthology.orchestra.run.smithi112.stderr:osd.6: osd_enable_op_tracker = 'true' 2020-03-11T12:45:57.145 INFO:teuthology.orchestra.run.smithi112.stderr:osd.7: osd_enable_op_tracker = 'true' 2020-03-11T12:45:57.177 INFO:teuthology.orchestra.run.smithi112.stderr:timeout: the monitored command dumped core
/a/sage-2020-03-11_12:18:48-rados-wip-sage3-testing-2020-03-10-2044-distro-basic-smithi/4846762
the core reveals:
Thread 12 (Thread 0x7f51a0757740 (LWP 15439)): #0 0x00007f519ff4381a in __pthread_rwlock_wrlock_full (abstime=0x0, rwlock=0x7f5193b650c0) at pthread_rwlock_common.c:595 #1 __GI___pthread_rwlock_wrlock (rwlock=0x7f5193b650c0) at pthread_rwlock_wrlock.c:27 #2 0x00007f519392290e in rtnl_link_unregister_info () from /usr/lib/x86_64-linux-gnu/libnl-route-3.so.200 #3 0x00007f51a0557b73 in _dl_fini () at dl-fini.c:138 #4 0x00007f51a0199041 in __run_exit_handlers (status=6, listp=0x7f51a0541718 <__exit_funcs>, run_list_atexit=run_list_atexit@entry=true, run_dtors=run_dtors@entry=true) at exit.c:108 #5 0x00007f51a019913a in __GI_exit (status=<optimized out>) at exit.c:139 #6 0x00000000006384f7 in ?? () #7 0x0000000000000006 in ?? () #8 0x00000000006385c0 in ?? () #9 0x0000000001e34630 in ?? () #10 0x0000000000a68b60 in ?? () #11 0x0000000000000001 in ?? () #12 0x00007f5192d1af08 in ?? () #13 0x0000000000a68b60 in ?? () #14 0x0000000000a1af20 in ?? () #15 0x0000000000000001 in ?? () #16 0x00000000006385ec in ?? () #17 0x0000000000a68b60 in ?? () #18 0x00007f51a06cd288 in ?? () #19 0x00007f51a06cd288 in ?? () #20 0x00007f51a06c5d68 in ?? () #21 0x0000000000000000 in ?? ()
and
Thread 1 (Thread 0x7f518b7fe700 (LWP 15446)):
#0 __memcmp_avx2_movbe () at ../sysdeps/x86_64/multiarch/memcmp-avx2-movbe.S:244
#1 0x00007f51957ec764 in std::char_traits<char>::compare (__n=<optimized out>, __s2=<optimized out>, __s1=<optimized out>) at /usr/include/c++/7/bits/char_traits.h:310
#2 std::basic_string_view<char, std::char_traits<char> >::compare (__str=..., this=<synthetic pointer>) at /usr/include/c++/7/string_view:268
#3 std::operator< <char, std::char_traits<char> > (__y=..., Python Exception <class 'gdb.error'> value has been optimized out:
__x=) at /usr/include/c++/7/string_view:488
#4 std::less<std::basic_string_view<char, std::char_traits<char> > >::operator() (this=<optimized out>, __y="ms_inject_delay_type", __x=<error: Cannot access memory at address 0x7f5193625ab0>) at /usr/include/c++/7/bits/stl_function.h:386
#5 std::_Rb_tree<std::basic_string_view<char, std::char_traits<char> >, std::pair<std::basic_string_view<char, std::char_traits<char> > const, Option const&>, std::_Select1st<std::pair<std::basic_string_view<char, std::char_traits<char> > const, Option const&> >, std::less<std::basic_string_view<char, std::char_traits<char> > >, std::allocator<std::pair<std::basic_string_view<char, std::char_traits<char> > const, Option const&> > >::_M_lower_bound (this=0x7f518c0049a8, __k="ms_inject_delay_type", __y=0x7f518c00bbb0, __x=0x7f518c00b470) at /usr/include/c++/7/bits/stl_tree.h:1888
#6 std::_Rb_tree<std::basic_string_view<char, std::char_traits<char> >, std::pair<std::basic_string_view<char, std::char_traits<char> > const, Option const&>, std::_Select1st<std::pair<std::basic_string_view<char, std::char_traits<char> > const, Option const&> >, std::less<std::basic_string_view<char, std::char_traits<char> > >, std::allocator<std::pair<std::basic_string_view<char, std::char_traits<char> > const, Option const&> > >::find (this=0x7f518c0049a8, __k="ms_inject_delay_type") at /usr/include/c++/7/bits/stl_tree.h:2536
#7 0x00007f519580f551 in std::map<std::basic_string_view<char, std::char_traits<char> >, Option const&, std::less<std::basic_string_view<char, std::char_traits<char> > >, std::allocator<std::pair<std::basic_string_view<char, std::char_traits<char> > const, Option const&> > >::find (__x=..., this=<optimized out>)
at /usr/include/c++/7/bits/stl_map.h:1188
#8 md_config_t::find_option (this=<optimized out>, name="ms_inject_delay_type") at ./src/common/config.cc:245
#9 0x00007f51958132ab in md_config_t::_get_val (this=0x7f518c004970, values=..., key=..., stack=0x0, err=0x0) at ./src/common/config.cc:1050
---Type <return> to continue, or q <return> to quit---
#10 0x00007f5195813592 in md_config_t::get_val_generic[abi:cxx11](ConfigValues const&, std::basic_string_view<char, std::char_traits<char> >) const (this=<optimized out>, this@entry=0x7f518c004970, values=..., key=...) at ./src/common/config.cc:1034
#11 0x00007f5195980a79 in md_config_t::with_val<std::__cxx11::basic_string<char>, AsyncConnection::maybe_start_delay_thread()::<lambda(const string&)> > (cb=..., key=..., values=..., this=0x7f518c004970) at ./src/common/config.h:201
#12 ceph::common::ConfigProxy::with_val<std::__cxx11::basic_string<char>, AsyncConnection::maybe_start_delay_thread()::<lambda(const string&)> > (cb=..., key="ms_inject_delay_type", this=0x7f518c001518) at ./src/common/config_proxy.h:150
#13 AsyncConnection::maybe_start_delay_thread (this=0x7f517802d080) at ./src/msg/async/AsyncConnection.cc:169
#14 0x00007f51959bf1ef in ProtocolV2::ready (this=this@entry=0x7f517802f540) at ./src/msg/async/ProtocolV2.cc:1309
#15 0x00007f51959c8930 in ProtocolV2::handle_server_ident (this=this@entry=0x7f517802f540, payload=...) at ./src/msg/async/ProtocolV2.cc:2142
#16 0x00007f51959d34cb in ProtocolV2::handle_frame_payload (this=this@entry=0x7f517802f540) at ./src/msg/async/ProtocolV2.cc:1261
#17 0x00007f51959d3800 in ProtocolV2::handle_read_frame_dispatch (this=this@entry=0x7f517802f540) at ./src/msg/async/ProtocolV2.cc:1159
#18 0x00007f51959d3a60 in ProtocolV2::handle_read_frame_epilogue_main (this=0x7f517802f540, buffer=..., r=<optimized out>) at ./src/msg/async/ProtocolV2.cc:1385
#19 0x00007f51959b9074 in ProtocolV2::run_continuation (this=0x7f517802f540, continuation=...) at ./src/msg/async/ProtocolV2.cc:45
#20 0x00007f519598303c in std::function<void (char*, long)>::operator()(char*, long) const (__args#1=<optimized out>, __args#0=<optimized out>, this=0x7f517802d420) at /usr/include/c++/7/bits/std_function.h:706
#21 AsyncConnection::process (this=0x7f517802d080) at ./src/msg/async/AsyncConnection.cc:457
#22 0x00007f51959d9d6d in EventCenter::process_events (this=this@entry=0x7f518c043c90, timeout_microseconds=<optimized out>, timeout_microseconds@entry=30000000, working_dur=working_dur@entry=0x7f518b7fde28) at ./src/msg/async/Event.cc:406
#23 0x00007f51959e1a08 in NetworkStack::<lambda()>::operator() (__closure=0x7f518c060b28) at ./src/msg/async/Stack.cc:53
#24 std::_Function_handler<void(), NetworkStack::add_thread(unsigned int)::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/7/bits/std_function.h:316
#25 0x00007f5194fc06ef in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6
#26 0x00007f519ff3e6db in start_thread (arg=0x7f518b7fe700) at pthread_create.c:463
#27 0x00007f51a027788f in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95
Updated by Sage Weil about 4 years ago
- Priority changed from High to Urgent
again
2020-03-13T01:06:53.231 INFO:teuthology.orchestra.run.smithi012.stderr:osd.0: osd_enable_op_tracker = 'true'
2020-03-13T01:06:53.231 INFO:teuthology.orchestra.run.smithi012.stderr:osd.1: osd_enable_op_tracker = 'true'
2020-03-13T01:06:53.232 INFO:teuthology.orchestra.run.smithi012.stderr:osd.2: osd_enable_op_tracker = 'true'
2020-03-13T01:06:53.232 INFO:teuthology.orchestra.run.smithi012.stderr:Error ENXIO: problem getting command descriptions from osd.3
2020-03-13T01:06:53.232 INFO:teuthology.orchestra.run.smithi012.stderr:osd.3: problem getting command descriptions from osd.3
2020-03-13T01:06:53.232 INFO:teuthology.orchestra.run.smithi012.stderr:osd.4: osd_enable_op_tracker = 'true'
2020-03-13T01:06:53.232 INFO:teuthology.orchestra.run.smithi012.stderr:osd.5: osd_enable_op_tracker = 'true'
2020-03-13T01:06:53.232 INFO:teuthology.orchestra.run.smithi012.stderr:osd.6: osd_enable_op_tracker = 'true'
2020-03-13T01:06:53.232 INFO:teuthology.orchestra.run.smithi012.stderr:osd.7: osd_enable_op_tracker = 'true'
2020-03-13T01:06:53.233 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7: {}
2020-03-13T01:06:53.238 INFO:teuthology.orchestra.run.smithi012.stderr:2020-03-13T01:06:53.236+0000 7f55b4ff9700 1 RefCountedObject::get 0x7f55b809f9e0 7 -> 8
2020-03-13T01:06:53.238 INFO:teuthology.orchestra.run.smithi012.stderr:terminate called after throwing an instance of 'boost::wrapexcept<boost::bad_get>'
2020-03-13T01:06:53.238 INFO:teuthology.orchestra.run.smithi012.stderr: what(): boost::bad_get: failed value get using boost::get
core shows lots of threads doing work, plus
Thread 4 (Thread 0x7f55d0868640 (LWP 109779)): #0 0x00007f55d06657a3 in _dl_sort_maps () from /lib64/ld-linux-x86-64.so.2 #1 0x00007f55d065f052 in _dl_fini () from /lib64/ld-linux-x86-64.so.2 #2 0x00007f55cf3c306c in __run_exit_handlers () from /lib64/libc.so.6 #3 0x00007f55cf3c31a0 in exit () from /lib64/libc.so.6 #4 0x00007f55d0314d87 in Py_Exit () from /lib64/libpython3.6m.so.1.0 #5 0x00007f55d0314e63 in handle_system_exit () from /lib64/libpython3.6m.so.1.0 #6 0x00007f55d0314ec6 in PyErr_PrintEx () from /lib64/libpython3.6m.so.1.0 #7 0x00007f55d0315335 in PyRun_SimpleFileExFlags () from /lib64/libpython3.6m.so.1.0 #8 0x00007f55d03163ab in Py_Main () from /lib64/libpython3.6m.so.1.0 #9 0x00005603922d6c68 in ?? () #10 0x0000000000000000 in ?? ()
/a/sage-2020-03-12_21:00:43-rados-wip-sage-testing-2020-03-12-1236-distro-basic-smithi/4850890
Updated by Sage Weil about 4 years ago
- Related to Bug #44526: sporatic cephadm bootstrap failures: 'timed out' added
Updated by Sage Weil about 4 years ago
I think this is related to the fix for #44526, https://github.com/ceph/ceph/pull/33825, which skips rados shutdown.
A simple workaround is probably to use sys._exit instead of sys.exit...
Updated by Sage Weil about 4 years ago
- Status changed from New to Fix Under Review
- Pull request ID set to 33950
Updated by Sage Weil about 4 years ago
for the record, all threads in the second instance:
(gdb) bt
#0 0x00007f55cf3c08df in raise () from /lib64/libc.so.6
#1 0x00007f55cf3aacf5 in abort () from /lib64/libc.so.6
#2 0x00007f55c33d506b in __gnu_cxx::__verbose_terminate_handler() [clone .cold.1] () from /lib64/libstdc++.so.6
#3 0x00007f55c33db4cc in __cxxabiv1::__terminate(void (*)()) () from /lib64/libstdc++.so.6
#4 0x00007f55c33db527 in std::terminate() () from /lib64/libstdc++.so.6
#5 0x00007f55c33db788 in __cxa_throw () from /lib64/libstdc++.so.6
#6 0x00007f55c4e72e3a in std::_Rb_tree<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, JSONFormattable>, std::_Select1st<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, JSONFormattable> >, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, JSONFormattable> > >::_M_emplace_hint_unique<std::piecewise_construct_t const&, std::tuple<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&>, std::tuple<> >(std::_Rb_tree_const_iterator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, JSONFormattable> >, std::piecewise_construct_t const&, std::tuple<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&>&&, std::tuple<>&&) (this=0x7f55cfee5710 <pthread_key_create>, __pos=<error reading variable: Cannot access memory at address 0x20>, __args#0=..., __args#1=..., __args#2=...)
at /usr/include/c++/8/bits/stl_tree.h:273
#7 0x75ef6d0d1e330900 in ?? ()
#8 0x00007f55b4ff8c10 in ?? ()
#9 0x00007f55c4e90e1d in std::basic_string_view<char, std::char_traits<char> >::basic_string_view (__str=<optimized out>, this=<optimized out>) at /usr/include/c++/8/string_view:99
#10 std::pair<std::basic_string_view<char, std::char_traits<char> > const, boost::variant<long ConfigValues::*, unsigned long ConfigValues::*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > ConfigValues::*, double ConfigValues::*, bool ConfigValues::*, entity_addr_t ConfigValues::*, entity_addrvec_t ConfigValues::*, uuid_d ConfigValues::*> >::pair<char const (&) [27], long ConfigValues::*, true> (__y=<optimized out>, __x=..., this=<optimized out>)
at /usr/include/c++/8/bits/stl_pair.h:342
#11 md_config_t::md_config_t (this=0x7f55c56fe618 <vtable for boost::exception+16>, values=..., tracker=..., is_daemon=<optimized out>) at /usr/src/debug/ceph-15.1.0-2106.gef7eea224a.el8.x86_64/src/common/config.cc:165
#12 0x0000000000000000 in ?? ()
(gdb) thr app all bt
Thread 15 (Thread 0x7f559f7fe700 (LWP 109799)):
#0 0x00007f55cfee447c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007f55c4e729d4 in boost::detail::variant::copy_into::internal_visit<double> (this=<synthetic pointer>, operand=@0x7f55b8059b78: 7.4109846876186982e-323) at /usr/include/c++/8/new:169
#2 boost::detail::variant::visitation_impl_invoke_impl<boost::detail::variant::copy_into, void const*, double> (storage=<optimized out>, visitor=...)
at /usr/src/debug/ceph-15.1.0-2106.gef7eea224a.el8.x86_64/build/boost/include/boost/variant/detail/visitation_impl.hpp:112
#3 boost::detail::variant::visitation_impl_invoke<boost::detail::variant::copy_into, void const*, double, boost::variant<boost::recursive_wrapper<std::vector<json_spirit::Pair_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::allocator<json_spirit::Pair_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > > >, boost::recursive_wrapper<std::vector<json_spirit::Value_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::allocator<json_spirit::Value_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > > >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, double, json_spirit::Null, unsigned long>::has_fallback_type_> (internal_which=-1207592064, t=0x0,
storage=0x7f55b8059b78, visitor=<synthetic pointer>...) at /usr/src/debug/ceph-15.1.0-2106.gef7eea224a.el8.x86_64/build/boost/include/boost/variant/detail/visitation_impl.hpp:150
#4 boost::detail::variant::visitation_impl<mpl_::int_<0>, boost::detail::variant::visitation_impl_step<boost::mpl::l_iter<boost::mpl::l_item<mpl_::long_<8l>, boost::recursive_wrapper<std::vector<json_spirit::Pair_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::allocator<json_spirit::Pair_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > > >, boost::mpl::l_item<mpl_::long_<7l>, boost::recursive_wrapper<std::vector<json_spirit::Value_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::allocator<json_spirit::Value_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > > >, boost::mpl::l_item<mpl_::long_<6l>, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, boost::mpl::l_item<mpl_::long_<5l>, bool, boost::mpl::l_item<mpl_::long_<4l>, long, boost::mpl::l_item<mpl_::long_<3l>, double, boost::mpl::l_item<mpl_::long_<2l>, json_spirit::Null, boost::mpl::l_item<mpl_::long_<1l>, unsigned long, boost::mpl::l_end> > > > > > > > >, boost::mpl::l_iter<boost::mpl::l_end> >, boost::detail::variant::copy_into, void const*, boost::variant<boost::recursive_wrapper<std::vector<json_spirit::Pair_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::allocator<json_spirit::Pair_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > > >, boost::recursive_wrapper<std::vector<json_spirit::Value_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::allocator<json_spirit::Value_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > > >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, double, json_spirit::Null, unsigned long>::has_fallback_type_> (no_backup_flag=..., storage=0x7f55b8059b78, visitor=<synthetic pointer>...,
logical_which=511, internal_which=-1207592064) at /usr/src/debug/ceph-15.1.0-2106.gef7eea224a.el8.x86_64/build/boost/include/boost/variant/detail/visitation_impl.hpp:231
#5 boost::variant<boost::recursive_wrapper<std::vector<json_spirit::Pair_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::allocator<json_spirit::Pair_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > > >, boost::recursive_wrapper<std::vector<json_spirit::Value_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::allocator<json_spirit::Value_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > > >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, double, json_spirit::Null, unsigned long>::internal_apply_visitor_impl<boost::detail::variant::copy_into, void const*> (storage=0x7f55b8059b78, visitor=<synthetic pointer>..., logical_which=511, internal_which=-1207592064)
at /usr/src/debug/ceph-15.1.0-2106.gef7eea224a.el8.x86_64/build/boost/include/boost/variant/variant.hpp:2334
#6 boost::variant<boost::recursive_wrapper<std::vector<json_spirit::Pair_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::allocator<json_spirit::Pair_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > > >, boost::recursive_wrapper<std::vector<json_spirit::Value_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::allocator<json_spirit::Value_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > > >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, double, json_spirit::Null, unsigned long>::internal_apply_visitor<boost::detail::variant::copy_into> (visitor=<synthetic pointer>..., this=0x7f55b8059b70)
at /usr/src/debug/ceph-15.1.0-2106.gef7eea224a.el8.x86_64/build/boost/include/boost/variant/variant.hpp:2355
#7 boost::variant<boost::recursive_wrapper<std::vector<json_spirit::Pair_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::allocator<json_spirit::Pair_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > > >, boost::recursive_wrapper<std::vector<json_spirit::Value_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::allocator<json_spirit::Value_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > > >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, double, json_spirit::Null, unsigned long>::variant (this=0x7f55b8059c50, operand=...) at /usr/src/debug/ceph-15.1.0-2106.gef7eea224a.el8.x86_64/build/boost/include/boost/variant/variant.hpp:1760
#8 0x15fbb7a4c53e7734 in ?? ()
#9 0x00007f559f7fde30 in ?? ()
#10 0x00007f55cfee5710 in .annobin_pthread_barrierattr_setpshared.end () from /lib64/libpthread.so.0
#11 0x00007f559f7fde58 in ?? ()
#12 0x00007f55b8059ca0 in ?? ()
#13 0x00007f55b8059b70 in ?? ()
#14 0x0000000000000001 in ?? ()
#15 0x000000005e6adcac in ?? ()
#16 0x000000000fa6bf34 in ?? ()
#17 0x0000000000000000 in ?? ()
Thread 14 (Thread 0x7f55bce19700 (LWP 109786)):
#0 0x00007f55cf485467 in epoll_wait () from /lib64/libc.so.6
#1 0x00007f55c5041384 in CachedStackStringStream::~CachedStackStringStream (this=0x7f55bce18a28, __in_chrg=<optimized out>) at /usr/include/c++/8/bits/move.h:99
#2 ceph::logging::MutableEntry::~MutableEntry (this=0x7f55bce18e58, __in_chrg=<optimized out>) at /usr/src/debug/ceph-15.1.0-2106.gef7eea224a.el8.x86_64/src/log/Entry.h:62
#3 EventCenter::process_events (this=0x7f55b805ed90, timeout_microseconds=<optimized out>, working_dur=0x7f55bce18a10) at /usr/src/debug/ceph-15.1.0-2106.gef7eea224a.el8.x86_64/src/msg/async/Event.cc:416
#4 0x75ef6d0d1e330900 in ?? ()
#5 0x00007f55b805ed10 in ?? ()
#6 0x00007f55b8060318 in ?? ()
#7 0x00007f55b805ed10 in ?? ()
#8 0x00007f55b805ed18 in ?? ()
#9 0x00007f55bce18e58 in ?? ()
#10 0x00007f55bce19700 in ?? ()
#11 0x00007f55bce18fc0 in ?? ()
#12 0x00007f55c503b38c in ceph::msgr::v2::ControlFrame<ceph::msgr::v2::ResetFrame, bool>::Encode (args#0=<error reading variable>) at /usr/src/debug/ceph-15.1.0-2106.gef7eea224a.el8.x86_64/src/msg/async/frames_v2.h:377
#13 0x0000000000000000 in ?? ()
Thread 13 (Thread 0x7f559e7fc700 (LWP 109801)):
#0 0x00007f55cfee447c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007f55c4e729d4 in boost::detail::variant::copy_into::internal_visit<double> (this=<synthetic pointer>, operand=@0x7f55b805a0d0: 1.0375378562666177e-322) at /usr/include/c++/8/new:169
#2 boost::detail::variant::visitation_impl_invoke_impl<boost::detail::variant::copy_into, void const*, double> (storage=<optimized out>, visitor=...)
at /usr/src/debug/ceph-15.1.0-2106.gef7eea224a.el8.x86_64/build/boost/include/boost/variant/detail/visitation_impl.hpp:112
#3 boost::detail::variant::visitation_impl_invoke<boost::detail::variant::copy_into, void const*, double, boost::variant<boost::recursive_wrapper<std::vector<json_spirit::Pair_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::allocator<json_spirit::Pair_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > > >, boost::recursive_wrapper<std::vector<json_spirit::Value_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::allocator<json_spirit::Value_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > > >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, double, json_spirit::Null, unsigned long>::has_fallback_type_> (internal_which=-1207589584, t=0x0,
storage=0x7f55b805a0d0, visitor=<synthetic pointer>...) at /usr/src/debug/ceph-15.1.0-2106.gef7eea224a.el8.x86_64/build/boost/include/boost/variant/detail/visitation_impl.hpp:150
#4 boost::detail::variant::visitation_impl<mpl_::int_<0>, boost::detail::variant::visitation_impl_step<boost::mpl::l_iter<boost::mpl::l_item<mpl_::long_<8l>, boost::recursive_wrapper<std::vector<json_spirit::Pair_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::allocator<json_spirit::Pair_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > > >, boost::mpl:--Type <RET> for more, q to quit, c to continue without paging--
:l_item<mpl_::long_<7l>, boost::recursive_wrapper<std::vector<json_spirit::Value_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::allocator<json_spirit::Value_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > > >, boost::mpl::l_item<mpl_::long_<6l>, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, boost::mpl::l_item<mpl_::long_<5l>, bool, boost::mpl::l_item<mpl_::long_<4l>, long, boost::mpl::l_item<mpl_::long_<3l>, double, boost::mpl::l_item<mpl_::long_<2l>, json_spirit::Null, boost::mpl::l_item<mpl_::long_<1l>, unsigned long, boost::mpl::l_end> > > > > > > > >, boost::mpl::l_iter<boost::mpl::l_end> >, boost::detail::variant::copy_into, void const*, boost::variant<boost::recursive_wrapper<std::vector<json_spirit::Pair_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::allocator<json_spirit::Pair_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > > >, boost::recursive_wrapper<std::vector<json_spirit::Value_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::allocator<json_spirit::Value_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > > >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, double, json_spirit::Null, unsigned long>::has_fallback_type_> (no_backup_flag=..., storage=0x7f55b805a0d0, visitor=<synthetic pointer>...,
logical_which=511, internal_which=-1207589584) at /usr/src/debug/ceph-15.1.0-2106.gef7eea224a.el8.x86_64/build/boost/include/boost/variant/detail/visitation_impl.hpp:231
#5 boost::variant<boost::recursive_wrapper<std::vector<json_spirit::Pair_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::allocator<json_spirit::Pair_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > > >, boost::recursive_wrapper<std::vector<json_spirit::Value_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::allocator<json_spirit::Value_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > > >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, double, json_spirit::Null, unsigned long>::internal_apply_visitor_impl<boost::detail::variant::copy_into, void const*> (storage=0x7f55b805a0d0, visitor=<synthetic pointer>..., logical_which=511, internal_which=-1207589584)
at /usr/src/debug/ceph-15.1.0-2106.gef7eea224a.el8.x86_64/build/boost/include/boost/variant/variant.hpp:2334
#6 boost::variant<boost::recursive_wrapper<std::vector<json_spirit::Pair_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::allocator<json_spirit::Pair_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > > >, boost::recursive_wrapper<std::vector<json_spirit::Value_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::allocator<json_spirit::Value_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > > >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, double, json_spirit::Null, unsigned long>::internal_apply_visitor<boost::detail::variant::copy_into> (visitor=<synthetic pointer>..., this=0x7f55b805a0c8)
at /usr/src/debug/ceph-15.1.0-2106.gef7eea224a.el8.x86_64/build/boost/include/boost/variant/variant.hpp:2355
#7 boost::variant<boost::recursive_wrapper<std::vector<json_spirit::Pair_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::allocator<json_spirit::Pair_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > > >, boost::recursive_wrapper<std::vector<json_spirit::Value_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::allocator<json_spirit::Value_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > > >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, double, json_spirit::Null, unsigned long>::variant (this=0x7f55b805a150, operand=...) at /usr/src/debug/ceph-15.1.0-2106.gef7eea224a.el8.x86_64/build/boost/include/boost/variant/variant.hpp:1760
#8 0x00007f559e7fbe40 in ?? ()
#9 0x00007f559e7fbe68 in ?? ()
#10 0x00007f55cfee5710 in .annobin_pthread_barrierattr_setpshared.end () from /lib64/libpthread.so.0
#11 0x0000000000000000 in ?? ()
Thread 12 (Thread 0x7f559ffff700 (LWP 109798)):
#0 0x00007f55cfee447c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007f55c4e729d4 in boost::detail::variant::copy_into::internal_visit<double> (this=<synthetic pointer>, operand=@0x7f55b8059670: 1.1363509854348671e-322) at /usr/include/c++/8/new:169
#2 boost::detail::variant::visitation_impl_invoke_impl<boost::detail::variant::copy_into, void const*, double> (storage=<optimized out>, visitor=...)
at /usr/src/debug/ceph-15.1.0-2106.gef7eea224a.el8.x86_64/build/boost/include/boost/variant/detail/visitation_impl.hpp:112
#3 boost::detail::variant::visitation_impl_invoke<boost::detail::variant::copy_into, void const*, double, boost::variant<boost::recursive_wrapper<std::vector<json_spirit::Pair_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::allocator<json_spirit::Pair_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > > >, boost::recursive_wrapper<std::vector<json_spirit::Value_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::allocator<json_spirit::Value_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > > >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, double, json_spirit::Null, unsigned long>::has_fallback_type_> (internal_which=-1207589424, t=0x0,
storage=0x7f55b8059670, visitor=<synthetic pointer>...) at /usr/src/debug/ceph-15.1.0-2106.gef7eea224a.el8.x86_64/build/boost/include/boost/variant/detail/visitation_impl.hpp:150
#4 boost::detail::variant::visitation_impl<mpl_::int_<0>, boost::detail::variant::visitation_impl_step<boost::mpl::l_iter<boost::mpl::l_item<mpl_::long_<8l>, boost::recursive_wrapper<std::vector<json_spirit::Pair_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::allocator<json_spirit::Pair_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > > >, boost::mpl::l_item<mpl_::long_<7l>, boost::recursive_wrapper<std::vector<json_spirit::Value_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::allocator<json_spirit::Value_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > > >, boost::mpl::l_item<mpl_::long_<6l>, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, boost::mpl::l_item<mpl_::long_<5l>, bool, boost::mpl::l_item<mpl_::long_<4l>, long, boost::mpl::l_item<mpl_::long_<3l>, double, boost::mpl::l_item<mpl_::long_<2l>, json_spirit::Null, boost::mpl::l_item<mpl_::long_<1l>, unsigned long, boost::mpl::l_end> > > > > > > > >, boost::mpl::l_iter<boost::mpl::l_end> >, boost::detail::variant::copy_into, void const*, boost::variant<boost::recursive_wrapper<std::vector<json_spirit::Pair_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::allocator<json_spirit::Pair_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > > >, boost::recursive_wrapper<std::vector<json_spirit::Value_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::allocator<json_spirit::Value_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > > >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, double, json_spirit::Null, unsigned long>::has_fallback_type_> (no_backup_flag=..., storage=0x7f55b8059670, visitor=<synthetic pointer>...,
logical_which=511, internal_which=-1207589424) at /usr/src/debug/ceph-15.1.0-2106.gef7eea224a.el8.x86_64/build/boost/include/boost/variant/detail/visitation_impl.hpp:231
#5 boost::variant<boost::recursive_wrapper<std::vector<json_spirit::Pair_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::allocator<json_spirit::Pair_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > > >, boost::recursive_wrapper<std::vector<json_spirit::Value_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::allocator<json_spirit::Value_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > > >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, double, json_spirit::Null, unsigned long>::internal_apply_visitor_impl<boost::detail::variant::copy_into, void const*> (storage=0x7f55b8059670, visitor=<synthetic pointer>..., logical_which=511, internal_which=-1207589424)
at /usr/src/debug/ceph-15.1.0-2106.gef7eea224a.el8.x86_64/build/boost/include/boost/variant/variant.hpp:2334
#6 boost::variant<boost::recursive_wrapper<std::vector<json_spirit::Pair_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::allocator<json_spirit::Pair_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > > >, boost::recursive_wrapper<std::vector<json_spirit::Value_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::allocator<json_spirit::Value_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > > >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, double, json_spirit::Null, unsigned long>::internal_apply_visitor<boost::detail::variant::copy_into> (visitor=<synthetic pointer>..., this=0x7f55b8059668)
at /usr/src/debug/ceph-15.1.0-2106.gef7eea224a.el8.x86_64/build/boost/include/boost/variant/variant.hpp:2355
#7 boost::variant<boost::recursive_wrapper<std::vector<json_spirit::Pair_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::allocator<json_spirit::Pair_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > > >, boost::recursive_wrapper<std::vector<json_spirit::Value_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::allocator<json_spirit::Value_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > > >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, double, json_spirit::Null, unsigned long>::variant (this=0x7f55b80596f4, operand=...) at /usr/src/debug/ceph-15.1.0-2106.gef7eea224a.el8.x86_64/build/boost/include/boost/variant/variant.hpp:1760
#8 0x00007f559fffee40 in ?? ()
#9 0x00007f559fffee68 in ?? ()
#10 0x00007f55cfee5710 in .annobin_pthread_barrierattr_setpshared.end () from /lib64/libpthread.so.0
#11 0x0000000000000000 in ?? ()
Thread 11 (Thread 0x7f559effd700 (LWP 109800)):
#0 0x00007f55cfee447c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007f55c4e729d4 in boost::detail::variant::copy_into::internal_visit<double> (this=<synthetic pointer>, operand=@0x7f55b8059ea8: 1.3339772437713657e-322) at /usr/include/c++/8/new:169
#2 boost::detail::variant::visitation_impl_invoke_impl<boost::detail::variant::copy_into, void const*, double> (storage=<optimized out>, visitor=...)
at /usr/src/debug/ceph-15.1.0-2106.gef7eea224a.el8.x86_64/build/boost/include/boost/variant/detail/visitation_impl.hpp:112
#3 boost::detail::variant::visitation_impl_invoke<boost::detail::variant::copy_into, void const*, double, boost::variant<boost::recursive_wrapper<std::vector<json_spirit::Pair_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::allocator<json_spirit::Pair_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > > >, boost::recursive_wrapper<std::vector<json_spirit::Value_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::allocator<json_spirit::Value_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, s--Type <RET> for more, q to quit, c to continue without paging--
td::allocator<char> > > > > > >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, double, json_spirit::Null, unsigned long>::has_fallback_type_> (internal_which=-1207589392, t=0x0,
storage=0x7f55b8059ea8, visitor=<synthetic pointer>...) at /usr/src/debug/ceph-15.1.0-2106.gef7eea224a.el8.x86_64/build/boost/include/boost/variant/detail/visitation_impl.hpp:150
#4 boost::detail::variant::visitation_impl<mpl_::int_<0>, boost::detail::variant::visitation_impl_step<boost::mpl::l_iter<boost::mpl::l_item<mpl_::long_<8l>, boost::recursive_wrapper<std::vector<json_spirit::Pair_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::allocator<json_spirit::Pair_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > > >, boost::mpl::l_item<mpl_::long_<7l>, boost::recursive_wrapper<std::vector<json_spirit::Value_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::allocator<json_spirit::Value_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > > >, boost::mpl::l_item<mpl_::long_<6l>, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, boost::mpl::l_item<mpl_::long_<5l>, bool, boost::mpl::l_item<mpl_::long_<4l>, long, boost::mpl::l_item<mpl_::long_<3l>, double, boost::mpl::l_item<mpl_::long_<2l>, json_spirit::Null, boost::mpl::l_item<mpl_::long_<1l>, unsigned long, boost::mpl::l_end> > > > > > > > >, boost::mpl::l_iter<boost::mpl::l_end> >, boost::detail::variant::copy_into, void const*, boost::variant<boost::recursive_wrapper<std::vector<json_spirit::Pair_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::allocator<json_spirit::Pair_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > > >, boost::recursive_wrapper<std::vector<json_spirit::Value_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::allocator<json_spirit::Value_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > > >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, double, json_spirit::Null, unsigned long>::has_fallback_type_> (no_backup_flag=..., storage=0x7f55b8059ea8, visitor=<synthetic pointer>...,
logical_which=511, internal_which=-1207589392) at /usr/src/debug/ceph-15.1.0-2106.gef7eea224a.el8.x86_64/build/boost/include/boost/variant/detail/visitation_impl.hpp:231
#5 boost::variant<boost::recursive_wrapper<std::vector<json_spirit::Pair_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::allocator<json_spirit::Pair_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > > >, boost::recursive_wrapper<std::vector<json_spirit::Value_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::allocator<json_spirit::Value_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > > >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, double, json_spirit::Null, unsigned long>::internal_apply_visitor_impl<boost::detail::variant::copy_into, void const*> (storage=0x7f55b8059ea8, visitor=<synthetic pointer>..., logical_which=511, internal_which=-1207589392)
at /usr/src/debug/ceph-15.1.0-2106.gef7eea224a.el8.x86_64/build/boost/include/boost/variant/variant.hpp:2334
#6 boost::variant<boost::recursive_wrapper<std::vector<json_spirit::Pair_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::allocator<json_spirit::Pair_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > > >, boost::recursive_wrapper<std::vector<json_spirit::Value_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::allocator<json_spirit::Value_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > > >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, double, json_spirit::Null, unsigned long>::internal_apply_visitor<boost::detail::variant::copy_into> (visitor=<synthetic pointer>..., this=0x7f55b8059ea0)
at /usr/src/debug/ceph-15.1.0-2106.gef7eea224a.el8.x86_64/build/boost/include/boost/variant/variant.hpp:2355
#7 boost::variant<boost::recursive_wrapper<std::vector<json_spirit::Pair_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::allocator<json_spirit::Pair_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > > >, boost::recursive_wrapper<std::vector<json_spirit::Value_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::allocator<json_spirit::Value_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > > >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, double, json_spirit::Null, unsigned long>::variant (this=0x7f55b8059f78, operand=...) at /usr/src/debug/ceph-15.1.0-2106.gef7eea224a.el8.x86_64/build/boost/include/boost/variant/variant.hpp:1760
#8 0x15fbb7a4c53fbc52 in ?? ()
#9 0x00007f559effce30 in ?? ()
#10 0x00007f55cfee5710 in .annobin_pthread_barrierattr_setpshared.end () from /lib64/libpthread.so.0
#11 0x00007f559effce58 in ?? ()
#12 0x00007f55b8059fc8 in ?? ()
#13 0x00007f55b8059ea0 in ?? ()
#14 0x0000000000000001 in ?? ()
#15 0x000000005e6adcac in ?? ()
#16 0x000000000fa80452 in ?? ()
#17 0x0000000000000000 in ?? ()
Thread 10 (Thread 0x7f55b6ffd700 (LWP 109793)):
#0 0x00007f55cf47a211 in poll () from /lib64/libc.so.6
#1 0x00007f55c4e2ce5f in std::swap<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >*> (__b=@0x7f55b6ffcea0: 0x0,
__a=<error reading variable>) at /usr/include/c++/8/bits/move.h:182
#2 std::_Vector_base<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > >::_Vector_impl::_M_swap_data (__x=..., this=0xfffffffffffffe2c) at /usr/include/c++/8/bits/stl_vector.h:114
#3 std::_Vector_base<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > >::_Vector_base (__x=..., this=0xfffffffffffffe2c) at /usr/include/c++/8/bits/stl_vector.h:268
#4 std::vector<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > >::vector (__x=..., this=0xfffffffffffffe2c) at /usr/include/c++/8/bits/stl_vector.h:477
#5 std::__detail::_BracketMatcher<std::__cxx11::regex_traits<char>, true, true>::_BracketMatcher (this=0xfffffffffffffdfc) at /usr/include/c++/8/bits/regex_compiler.h:430
#6 std::_Function_base::_Base_manager<std::__detail::_BracketMatcher<std::__cxx11::regex_traits<char>, true, true> >::_M_init_functor (__f=..., __functor=...) at /usr/include/c++/8/bits/std_function.h:249
#7 std::_Function_base::_Base_manager<std::__detail::_BracketMatcher<std::__cxx11::regex_traits<char>, true, true> >::_M_init_functor (__f=..., __functor=...) at /usr/include/c++/8/bits/std_function.h:220
#8 std::function<bool (char)>::function<std::__detail::_BracketMatcher<std::__cxx11::regex_traits<char>, true, true>, void, void>(std::__detail::_BracketMatcher<std::__cxx11::regex_traits<char>, true, true>) (this=0x7f55b6ffcda0, __f=...)
at /usr/include/c++/8/bits/std_function.h:674
#9 0x0000000000000000 in ?? ()
Thread 9 (Thread 0x7f55b5ffb700 (LWP 109795)):
#0 0x00007f55cfee447c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007f55c4e729d4 in boost::detail::variant::copy_into::internal_visit<double> (this=<synthetic pointer>, operand=@0x7f55b80a01a0: 2.0256691479491108e-322) at /usr/include/c++/8/new:169
#2 boost::detail::variant::visitation_impl_invoke_impl<boost::detail::variant::copy_into, void const*, double> (storage=<optimized out>, visitor=...)
at /usr/src/debug/ceph-15.1.0-2106.gef7eea224a.el8.x86_64/build/boost/include/boost/variant/detail/visitation_impl.hpp:112
#3 boost::detail::variant::visitation_impl_invoke<boost::detail::variant::copy_into, void const*, double, boost::variant<boost::recursive_wrapper<std::vector<json_spirit::Pair_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::allocator<json_spirit::Pair_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > > >, boost::recursive_wrapper<std::vector<json_spirit::Value_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::allocator<json_spirit::Value_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > > >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, double, json_spirit::Null, unsigned long>::has_fallback_type_> (internal_which=-1207308864, t=0x0,
storage=0x7f55b80a01a0, visitor=<synthetic pointer>...) at /usr/src/debug/ceph-15.1.0-2106.gef7eea224a.el8.x86_64/build/boost/include/boost/variant/detail/visitation_impl.hpp:150
#4 boost::detail::variant::visitation_impl<mpl_::int_<0>, boost::detail::variant::visitation_impl_step<boost::mpl::l_iter<boost::mpl::l_item<mpl_::long_<8l>, boost::recursive_wrapper<std::vector<json_spirit::Pair_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::allocator<json_spirit::Pair_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > > >, boost::mpl::l_item<mpl_::long_<7l>, boost::recursive_wrapper<std::vector<json_spirit::Value_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::allocator<json_spirit::Value_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > > >, boost::mpl::l_item<mpl_::long_<6l>, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, boost::mpl::l_item<mpl_::long_<5l>, bool, boost::mpl::l_item<mpl_::long_<4l>, long, boost::mpl::l_item<mpl_::long_<3l>, double, boost::mpl::l_item<mpl_::long_<2l>, json_spirit::Null, boost::mpl::l_item<mpl_::long_<1l>, unsigned long, boost::mpl::l_end> > > > > > > > >, boost::mpl::l_iter<boost::mpl::l_end> >, boost::detail::variant::copy_into, void const*, boost::variant<boost::recursive_wrapper<std::vector<json_spirit::Pair_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::allocator<json_spirit::Pair_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > > >, boost::recursive_wrapper<std::vector<json_spirit::Value_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::allocator<json_spirit::Value_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > > >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, double, json_spirit::Null, unsigned long>::has_fallback_type_> (no_backup_flag=..., storage=0x7f55b80a01a0, visitor=<synthetic pointer>...,
logical_which=511, internal_which=-1207308864) at /usr/src/debug/ceph-15.1.0-2106.gef7eea224a.el8.x86_64/build/boost/include/boost/variant/detail/visitation_impl.hpp:231
--Type <RET> for more, q to quit, c to continue without paging--
#5 boost::variant<boost::recursive_wrapper<std::vector<json_spirit::Pair_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::allocator<json_spirit::Pair_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > > >, boost::recursive_wrapper<std::vector<json_spirit::Value_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::allocator<json_spirit::Value_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > > >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, double, json_spirit::Null, unsigned long>::internal_apply_visitor_impl<boost::detail::variant::copy_into, void const*> (storage=0x7f55b80a01a0, visitor=<synthetic pointer>..., logical_which=511, internal_which=-1207308864)
at /usr/src/debug/ceph-15.1.0-2106.gef7eea224a.el8.x86_64/build/boost/include/boost/variant/variant.hpp:2334
#6 boost::variant<boost::recursive_wrapper<std::vector<json_spirit::Pair_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::allocator<json_spirit::Pair_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > > >, boost::recursive_wrapper<std::vector<json_spirit::Value_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::allocator<json_spirit::Value_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > > >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, double, json_spirit::Null, unsigned long>::internal_apply_visitor<boost::detail::variant::copy_into> (visitor=<synthetic pointer>..., this=0x7f55b80a0198)
at /usr/src/debug/ceph-15.1.0-2106.gef7eea224a.el8.x86_64/build/boost/include/boost/variant/variant.hpp:2355
#7 boost::variant<boost::recursive_wrapper<std::vector<json_spirit::Pair_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::allocator<json_spirit::Pair_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > > >, boost::recursive_wrapper<std::vector<json_spirit::Value_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::allocator<json_spirit::Value_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > > >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, double, json_spirit::Null, unsigned long>::variant (this=0x7f55b80a0224, operand=...) at /usr/src/debug/ceph-15.1.0-2106.gef7eea224a.el8.x86_64/build/boost/include/boost/variant/variant.hpp:1760
#8 0x00007f55b5ffad80 in ?? ()
#9 0x00007f55b8097940 in ?? ()
#10 0x00007f55cfee5710 in .annobin_pthread_barrierattr_setpshared.end () from /lib64/libpthread.so.0
#11 0x00007f55b809f1e8 in ?? ()
#12 0x00007f55b5ffad90 in ?? ()
#13 0x00007f55b809fdf0 in ?? ()
#14 0x000000000000002a in ?? ()
#15 0x0000000000000000 in ?? ()
Thread 8 (Thread 0x7f55b57fa700 (LWP 109796)):
#0 0x00007f55cfee447c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007f55c4e729d4 in boost::detail::variant::copy_into::internal_visit<double> (this=<synthetic pointer>, operand=@0x7f55b80a0348: 2.7667676167109806e-322) at /usr/include/c++/8/new:169
#2 boost::detail::variant::visitation_impl_invoke_impl<boost::detail::variant::copy_into, void const*, double> (storage=<optimized out>, visitor=...)
at /usr/src/debug/ceph-15.1.0-2106.gef7eea224a.el8.x86_64/build/boost/include/boost/variant/detail/visitation_impl.hpp:112
#3 boost::detail::variant::visitation_impl_invoke<boost::detail::variant::copy_into, void const*, double, boost::variant<boost::recursive_wrapper<std::vector<json_spirit::Pair_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::allocator<json_spirit::Pair_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > > >, boost::recursive_wrapper<std::vector<json_spirit::Value_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::allocator<json_spirit::Value_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > > >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, double, json_spirit::Null, unsigned long>::has_fallback_type_> (internal_which=-1207585392, t=0x0,
storage=0x7f55b80a0348, visitor=<synthetic pointer>...) at /usr/src/debug/ceph-15.1.0-2106.gef7eea224a.el8.x86_64/build/boost/include/boost/variant/detail/visitation_impl.hpp:150
#4 boost::detail::variant::visitation_impl<mpl_::int_<0>, boost::detail::variant::visitation_impl_step<boost::mpl::l_iter<boost::mpl::l_item<mpl_::long_<8l>, boost::recursive_wrapper<std::vector<json_spirit::Pair_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::allocator<json_spirit::Pair_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > > >, boost::mpl::l_item<mpl_::long_<7l>, boost::recursive_wrapper<std::vector<json_spirit::Value_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::allocator<json_spirit::Value_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > > >, boost::mpl::l_item<mpl_::long_<6l>, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, boost::mpl::l_item<mpl_::long_<5l>, bool, boost::mpl::l_item<mpl_::long_<4l>, long, boost::mpl::l_item<mpl_::long_<3l>, double, boost::mpl::l_item<mpl_::long_<2l>, json_spirit::Null, boost::mpl::l_item<mpl_::long_<1l>, unsigned long, boost::mpl::l_end> > > > > > > > >, boost::mpl::l_iter<boost::mpl::l_end> >, boost::detail::variant::copy_into, void const*, boost::variant<boost::recursive_wrapper<std::vector<json_spirit::Pair_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::allocator<json_spirit::Pair_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > > >, boost::recursive_wrapper<std::vector<json_spirit::Value_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::allocator<json_spirit::Value_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > > >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, double, json_spirit::Null, unsigned long>::has_fallback_type_> (no_backup_flag=..., storage=0x7f55b80a0348, visitor=<synthetic pointer>...,
logical_which=511, internal_which=-1207585392) at /usr/src/debug/ceph-15.1.0-2106.gef7eea224a.el8.x86_64/build/boost/include/boost/variant/detail/visitation_impl.hpp:231
#5 boost::variant<boost::recursive_wrapper<std::vector<json_spirit::Pair_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::allocator<json_spirit::Pair_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > > >, boost::recursive_wrapper<std::vector<json_spirit::Value_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::allocator<json_spirit::Value_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > > >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, double, json_spirit::Null, unsigned long>::internal_apply_visitor_impl<boost::detail::variant::copy_into, void const*> (storage=0x7f55b80a0348, visitor=<synthetic pointer>..., logical_which=511, internal_which=-1207585392)
at /usr/src/debug/ceph-15.1.0-2106.gef7eea224a.el8.x86_64/build/boost/include/boost/variant/variant.hpp:2334
#6 boost::variant<boost::recursive_wrapper<std::vector<json_spirit::Pair_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::allocator<json_spirit::Pair_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > > >, boost::recursive_wrapper<std::vector<json_spirit::Value_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::allocator<json_spirit::Value_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > > >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, double, json_spirit::Null, unsigned long>::internal_apply_visitor<boost::detail::variant::copy_into> (visitor=<synthetic pointer>..., this=0x7f55b80a0340)
at /usr/src/debug/ceph-15.1.0-2106.gef7eea224a.el8.x86_64/build/boost/include/boost/variant/variant.hpp:2355
#7 boost::variant<boost::recursive_wrapper<std::vector<json_spirit::Pair_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::allocator<json_spirit::Pair_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > > >, boost::recursive_wrapper<std::vector<json_spirit::Value_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::allocator<json_spirit::Value_impl<json_spirit::Config_vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > > > >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, double, json_spirit::Null, unsigned long>::variant (this=0x7f55b80a03c8, operand=...) at /usr/src/debug/ceph-15.1.0-2106.gef7eea224a.el8.x86_64/build/boost/include/boost/variant/variant.hpp:1760
#8 0x00007f55b57f9e40 in ?? ()
#9 0x0000000000000000 in ?? ()
Thread 7 (Thread 0x7f55b7fff700 (LWP 109787)):
#0 0x00007f55cf485467 in epoll_wait () from /lib64/libc.so.6
#1 0x00007f55c5041384 in CachedStackStringStream::~CachedStackStringStream (this=0x7f55b7ffea28, __in_chrg=<optimized out>) at /usr/include/c++/8/bits/move.h:99
#2 ceph::logging::MutableEntry::~MutableEntry (this=0x7f55b7ffee58, __in_chrg=<optimized out>) at /usr/src/debug/ceph-15.1.0-2106.gef7eea224a.el8.x86_64/src/log/Entry.h:62
#3 EventCenter::process_events (this=0x7f55b8067b90, timeout_microseconds=<optimized out>, working_dur=0x7f55b7ffea10) at /usr/src/debug/ceph-15.1.0-2106.gef7eea224a.el8.x86_64/src/msg/async/Event.cc:416
#4 0x75ef6d0d1e330900 in ?? ()
#5 0x00007f55b8067b10 in ?? ()
#6 0x00007f55b805fff8 in ?? ()
#7 0x00007f55b8067b10 in ?? ()
#8 0x00007f55b8067b18 in ?? ()
#9 0x00007f55b7ffee58 in ?? ()
#10 0x00007f55b7fff700 in ?? ()
#11 0x00007f55b7ffefc0 in ?? ()
#12 0x00007f55c503b38c in ceph::msgr::v2::ControlFrame<ceph::msgr::v2::ResetFrame, bool>::Encode (args#0=<error reading variable>) at /usr/src/debug/ceph-15.1.0-2106.gef7eea224a.el8.x86_64/src/msg/async/frames_v2.h:377
--Type <RET> for more, q to quit, c to continue without paging--
#13 0x0000000000000000 in ?? ()
Thread 6 (Thread 0x7f55bd61a700 (LWP 109785)):
#0 0x00007f55cf485467 in epoll_wait () from /lib64/libc.so.6
#1 0x00007f55c5041384 in CachedStackStringStream::~CachedStackStringStream (this=0x7f55bd619a28, __in_chrg=<optimized out>) at /usr/include/c++/8/bits/move.h:99
#2 ceph::logging::MutableEntry::~MutableEntry (this=0x7f55bd619e58, __in_chrg=<optimized out>) at /usr/src/debug/ceph-15.1.0-2106.gef7eea224a.el8.x86_64/src/log/Entry.h:62
#3 EventCenter::process_events (this=0x7f55b80a7fd0, timeout_microseconds=<optimized out>, working_dur=0x7f55bd619a10) at /usr/src/debug/ceph-15.1.0-2106.gef7eea224a.el8.x86_64/src/msg/async/Event.cc:416
#4 0x75ef6d0d1e330900 in ?? ()
#5 0x00007f55b80a7f50 in ?? ()
#6 0x00007f55b805ffc8 in ?? ()
#7 0x00007f55b80a7f50 in ?? ()
#8 0x00007f55b80a7f58 in ?? ()
#9 0x00007f55bd619e58 in ?? ()
#10 0x00007f55bd61a700 in ?? ()
#11 0x00007f55bd619fc0 in ?? ()
#12 0x00007f55c503b38c in ceph::msgr::v2::ControlFrame<ceph::msgr::v2::ResetFrame, bool>::Encode (args#0=<error reading variable>) at /usr/src/debug/ceph-15.1.0-2106.gef7eea224a.el8.x86_64/src/msg/async/frames_v2.h:377
#13 0x0000000000000000 in ?? ()
Thread 5 (Thread 0x7f55b77fe700 (LWP 109792)):
#0 0x00007f55cfee47ca in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007f55c4e72bf4 in std::_Rb_tree_iterator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, JSONFormattable> >::operator-- (this=<synthetic pointer>) at /usr/include/c++/8/bits/stl_tree.h:300
#2 std::_Rb_tree<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, JSONFormattable>, std::_Select1st<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, JSONFormattable> >, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, JSONFormattable> > >::_M_get_insert_unique_pos (this=0x7f55b77fde80, Python Exception <class 'OverflowError'> int too big to convert:
__k=) at /usr/include/c++/8/bits/stl_tree.h:2066
#3 0x00007f55c4e50abd in std::_Rb_tree<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::_Select1st<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > >::_M_get_insert_hint_unique_pos (this=0x7f55b8067058, __position={first = "", second = ""}, __k=<error reading variable: Cannot access memory at address 0x8>) at /usr/include/c++/8/bits/move.h:74
#4 0x0000000000000000 in ?? ()
Thread 4 (Thread 0x7f55d0868640 (LWP 109779)):
#0 0x00007f55d06657a3 in _dl_sort_maps () from /lib64/ld-linux-x86-64.so.2
#1 0x00007f55d065f052 in _dl_fini () from /lib64/ld-linux-x86-64.so.2
#2 0x00007f55cf3c306c in __run_exit_handlers () from /lib64/libc.so.6
#3 0x00007f55cf3c31a0 in exit () from /lib64/libc.so.6
#4 0x00007f55d0314d87 in Py_Exit () from /lib64/libpython3.6m.so.1.0
#5 0x00007f55d0314e63 in handle_system_exit () from /lib64/libpython3.6m.so.1.0
#6 0x00007f55d0314ec6 in PyErr_PrintEx () from /lib64/libpython3.6m.so.1.0
#7 0x00007f55d0315335 in PyRun_SimpleFileExFlags () from /lib64/libpython3.6m.so.1.0
#8 0x00007f55d03163ab in Py_Main () from /lib64/libpython3.6m.so.1.0
#9 0x00005603922d6c68 in ?? ()
#10 0x0000000000000000 in ?? ()
Thread 3 (Thread 0x7f55bde1b700 (LWP 109784)):
#0 0x00007f55cfee447c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007f55c3401870 in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /lib64/libstdc++.so.6
#2 0x00007f55c5074c57 in ceph::buffer::v15_2_0::list::decode_base64 (this=0x7f55bde1b700, e=...) at /usr/src/debug/ceph-15.1.0-2106.gef7eea224a.el8.x86_64/src/include/buffer.h:270
#3 0x0000000000000000 in ?? ()
Thread 2 (Thread 0x7f55b67fc700 (LWP 109794)):
#0 0x00007f55cfee47ca in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007f55cda1f571 in Objecter::C_Linger_Ping::~C_Linger_Ping (this=0x7f55b80a2434, __in_chrg=<optimized out>) at /usr/src/debug/ceph-15.1.0-2106.gef7eea224a.el8.x86_64/src/osdc/Objecter.h:1906
#2 0x000001fd83222377 in ?? ()
#3 0x00007f55b67fbe80 in ?? ()
#4 0x00007f55b80a23e0 in ?? ()
#5 0x0000000000000001 in ?? ()
#6 0x000000005e6adcb1 in ?? ()
#7 0x000000000fa6c828 in ?? ()
#8 0x0000000000000000 in ?? ()
Thread 1 (Thread 0x7f55b4ff9700 (LWP 109797)):
#0 0x00007f55cf3c08df in raise () from /lib64/libc.so.6
#1 0x00007f55cf3aacf5 in abort () from /lib64/libc.so.6
#2 0x00007f55c33d506b in __gnu_cxx::__verbose_terminate_handler() [clone .cold.1] () from /lib64/libstdc++.so.6
#3 0x00007f55c33db4cc in __cxxabiv1::__terminate(void (*)()) () from /lib64/libstdc++.so.6
#4 0x00007f55c33db527 in std::terminate() () from /lib64/libstdc++.so.6
#5 0x00007f55c33db788 in __cxa_throw () from /lib64/libstdc++.so.6
#6 0x00007f55c4e72e3a in std::_Rb_tree<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, JSONFormattable>, std::_Select1st<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, JSONFormattable> >, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, JSONFormattable> > >::_M_emplace_hint_unique<std::piecewise_construct_t const&, std::tuple<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&>, std::tuple<> >(std::_Rb_tree_const_iterator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, JSONFormattable> >, std::piecewise_construct_t const&, std::tuple<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&>&&, std::tuple<>&&) (this=0x7f55cfee5710 <pthread_key_create>, __pos=<error reading variable: Cannot access memory at address 0x20>, __args#0=..., __args#1=..., __args#2=...)
at /usr/include/c++/8/bits/stl_tree.h:273
#7 0x75ef6d0d1e330900 in ?? ()
--Type <RET> for more, q to quit, c to continue without paging--
#8 0x00007f55b4ff8c10 in ?? ()
#9 0x00007f55c4e90e1d in std::basic_string_view<char, std::char_traits<char> >::basic_string_view (__str=<optimized out>, this=<optimized out>) at /usr/include/c++/8/string_view:99
#10 std::pair<std::basic_string_view<char, std::char_traits<char> > const, boost::variant<long ConfigValues::*, unsigned long ConfigValues::*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > ConfigValues::*, double ConfigValues::*, bool ConfigValues::*, entity_addr_t ConfigValues::*, entity_addrvec_t ConfigValues::*, uuid_d ConfigValues::*> >::pair<char const (&) [27], long ConfigValues::*, true> (__y=<optimized out>, __x=..., this=<optimized out>)
at /usr/include/c++/8/bits/stl_pair.h:342
#11 md_config_t::md_config_t (this=0x7f55c56fe618 <vtable for boost::exception+16>, values=..., tracker=..., is_daemon=<optimized out>) at /usr/src/debug/ceph-15.1.0-2106.gef7eea224a.el8.x86_64/src/common/config.cc:165
#12 0x0000000000000000 in ?? ()
(gdb)
Updated by Sage Weil about 4 years ago
- Status changed from Fix Under Review to Resolved
Actions