Ceph » rgw

I encountered two segmentation faults in libradosgw.

```
2021-09-01T18:25:27Z {} * Caught signal (Segmentation fault) *
2021-09-01T18:25:27Z {} in thread 7f52c5692700 thread_name:radosgw
2021-09-01T18:25:27Z {} ceph version 16.2.4 (3cbe25cde3cfa028984618ad32de9edc4c1eaed0) pacific (stable)
2021-09-01T18:25:27Z {} 1: /lib/x86_64-linux-gnu/libc.so.6(+0x46210) [0x7f55750c0210]
2021-09-01T18:25:27Z {} 2: (boost::asio::detail::reactive_socket_service_base::start_op(boost::asio::detail::reactive_socket_service_base::base_implementation_type&
, int, boost::asio::detail::reactor_op, bool, bool, bool)+0x125) [0x7f55756d20e5]
2021-09-01T18:25:27Z {} 3: (boost::beast::basic_stream<boost::asio::ip::tcp, boost::asio::executor, boost::beast::unlimited_rate_policy>::ops::transfer_op<false, bo
ost::asio::const_buffers_1, boost::asio::detail::write_op<boost::beast::basic_stream<boost::asio::ip::tcp, boost::asio::executor, boost::beast::unlimited_rate_polic
y>, boost::asio::const_buffers_1, boost::asio::const_buffer const*, boost::asio::detail::transfer_all_t, spawn::detail::coro_handler<boost::asio::executor_binder<vo
id ()(), boost::asio::executor>, unsigned long> > >::async_perform(unsigned long, std::integral_constant<bool, false>)+0x318) [0x7f557575f818]
2021-09-01T18:25:27Z {} 4: (boost::beast::basic_stream<boost::asio::ip::tcp, boost::asio::executor, boost::beast::unlimited_rate_policy>::ops::transfer_op<false, bo
ost::asio::const_buffers_1, boost::asio::detail::write_op<boost::beast::basic_stream<boost::asio::ip::tcp, boost::asio::executor, boost::beast::unlimited_rate_polic
y>, boost::asio::const_buffers_1, boost::asio::const_buffer const, boost::asio::detail::transfer_all_t, spawn::detail::coro_handler<boost::asio::executor_binder<vo
id ()(), boost::asio::executor>, unsigned long> > >::operator()(boost::system::error_code, unsigned long)+0x2a7) [0x7f557575fb97]
2021-09-01T18:25:27Z {} 5: /lib/libradosgw.so.2(+0x45df80) [0x7f55756c9f80]
2021-09-01T18:25:27Z {} 6: /lib/libradosgw.so.2(+0x44c807) [0x7f55756b8807]
2021-09-01T18:25:27Z {} 7: /lib/libradosgw.so.2(+0x454359) [0x7f55756c0359]
2021-09-01T18:25:27Z {} 8: (rgw::io::AccountingFilter<rgw::io::RestfulClient>::send_body(char const*, unsigned long)+0x2f) [0x7f55756a64ef]
2021-09-01T18:25:27Z {} 9: (dump_body(req_state*, char const*, unsigned long)+0x49) [0x7f5575c2fb19]
2021-09-01T18:25:27Z {} 10: (rgw_flush_formatter_and_reset(req_state*, ceph::Formatter*)+0x153) [0x7f5575c315f3]
2021-09-01T18:25:27Z {} 11: (end_header(req_state*, RGWOp*, char const*, long, bool, bool)+0x256) [0x7f5575c36ba6]
2021-09-01T18:25:27Z {} 12: (abort_early(req_state*, RGWOp*, int, RGWHandler*, optional_yield)+0x282) [0x7f5575c370f2]
2021-09-01T18:25:27Z {} 13: (process_request(rgw::sal::RGWRadosStore*, RGWREST*, RGWRequest*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocato
r<char> > const&, rgw::auth::StrategyRegistry const&, RGWRestfulIO*, OpsLogSocket*, optional_yield, rgw::dmclock::Scheduler*, std::__cxx11::basic_string<char, std::
char_traits<char>, std::allocator<char> >, std::chrono::duration<unsigned long, std::ratio<1l, 1000000000l> >, int*)+0x1aae) [0x7f557579414e]
2021-09-01T18:25:27Z {} 14: /lib/libradosgw.so.2(+0x459619) [0x7f55756c5619]
2021-09-01T18:25:27Z {} 15: /lib/libradosgw.so.2(+0x45ab61) [0x7f55756c6b61]
2021-09-01T18:25:27Z {} 16: /lib/libradosgw.so.2(+0x45acdc) [0x7f55756c6cdc]
2021-09-01T18:25:27Z {} 17: make_fcontext()
2021-09-01T18:25:27Z {} debug 2021-09-01T18:25:27.909+0000 7f52c5692700 -1 Caught signal (Segmentation fault) *
2021-09-01T18:25:27Z {} in thread 7f52c5692700 thread_name:radosgw
```

```
de_name="10.69.1.138", pod="rook-ceph-rgw-ceph-hdd-object-store-a-6fc6858677-kh8kf"} debug 2021-09-11T10:06:37.908+0000 7fb19c7f8700 -1 ** Caught signal (Segmentat
ion fault) *
2021-09-11T10:06:37Z {filename="/var/log/pods/ceph-hdd_rook-ceph-rgw-ceph-hdd-object-store-a-6fc6858677-kh8kf_63750f06-a0fe-434e-a903-4ef3266a6464/rgw/3303.log", no
de_name="10.69.1.138", pod="rook-ceph-rgw-ceph-hdd-object-store-a-6fc6858677-kh8kf"} in thread 7fb19c7f8700 thread_name:radosgw
...
2021-09-11T10:06:37Z {filename="/var/log/pods/ceph-hdd_rook-ceph-rgw-ceph-hdd-object-store-a-6fc6858677-kh8kf_63750f06-a0fe-434e-a903-4ef3266a6464/rgw/3303.log", node_name="10.69.1.138", pod="rook-ceph-rgw-ceph-hdd-object-store-a-6fc6858677-kh8kf"} 16: /lib/libradosgw.so.2(0x44e456) [0x7fb2c8ecb456]
2021-09-11T10:06:37Z {filename="/var/log/pods/ceph-hdd_rook-ceph-rgw-ceph-hdd-object-store-a-6fc6858677-kh8kf_63750f06-a0fe-434e-a903-4ef3266a6464/rgw/3303.log", node_name="10.69.1.138", pod="rook-ceph-rgw-ceph-hdd-object-store-a-6fc6858677-kh8kf"} 17: /lib/x86_64-linux-gnu/libstdc+.so.6(+0xd6d84) [0x7fb2bef52d84]
2021-09-11T10:06:37Z {filename="/var/log/pods/ceph-hdd_rook-ceph-rgw-ceph-hdd-object-store-a-6fc6858677-kh8kf_63750f06-a0fe-434e-a903-4ef3266a6464/rgw/3303.log", node_name="10.69.1.138", pod="rook-ceph-rgw-ceph-hdd-object-store-a-6fc6858677-kh8kf"} 18: /lib/x86_64-linux-gnu/libpthread.so.0(+0x9609) [0x7fb2bf066609]
2021-09-11T10:06:37Z {filename="/var/log/pods/ceph-hdd_rook-ceph-rgw-ceph-hdd-object-store-a-6fc6858677-kh8kf_63750f06-a0fe-434e-a903-4ef3266a6464/rgw/3303.log", node_name="10.69.1.138", pod="rook-ceph-rgw-ceph-hdd-object-store-a-6fc6858677-kh8kf"} 19: clone()
2021-09-11T10:06:37Z {filename="/var/log/pods/ceph-hdd_rook-ceph-rgw-ceph-hdd-object-store-a-6fc6858677-kh8kf_63750f06-a0fe-434e-a903-4ef3266a6464/rgw/3303.log", node_name="10.69.1.138", pod="rook-ceph-rgw-ceph-hdd-object-store-a-6fc6858677-kh8kf"} NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
```

Software environment:
- Ceph: v16.2.4.1
- Rook: v1.6.3.4

These problems are in crash collector.

ceph -s
```
cluster:
id: be8f440c-8394-44e7-aec2-8366e616e29e
health: HEALTH_WARN
21 pgs not deep-scrubbed in time
21 pgs not scrubbed in time
2 daemons have recently crashed

services:
    mon: 3 daemons, quorum ad,ag,ah (age 8w)
    mgr: a(active, since 8w), standbys: b
    osd: 18 osds: 18 up (since 2w), 18 in (since 2w)
    rgw: 3 daemons active (3 hosts, 1 zones)

data:
    pools:   9 pools, 113 pgs
    objects: 56.91M objects, 5.2 TiB
    usage:   30 TiB used, 101 TiB / 131 TiB avail
    pgs:     111 active+clean
             2   active+clean+scrubbing+deep+repair

io:
    client:   457 KiB/s rd, 4.3 MiB/s wr, 44 op/s rd, 1.01k op/s wr
```

ceph crash ls-new
```
ID ENTITY NEW
2021-09-01T18:25:27.909464Z_6a42037f-ec3c-4663-ae09-5d1997865b37 client.rgw.ceph.hdd.object.store.a *
2021-09-11T10:06:37.908355Z_f82f90c3-c77e-427a-b9f0-94ecfc1b75e3 client.rgw.ceph.hdd.object.store.a *
```

ceph crash info "2021-09-01T18:25:27.909464Z_6a42037f-ec3c-4663-ae09-5d1997865b37" # corresponding to 1st segmentaion fault
``` {
"backtrace": [
"/lib/x86_64-linux-gnu/libc.so.6(+0x46210) [0x7f55750c0210]",
"(boost::asio::detail::reactive_socket_service_base::start_op(boost::asio::detail::reactive_socket_service_base::base_implementation_type&, int, boost::asio::detail::reactor_op*, bool, bool, bool)+0x125) [0x7f55756d20e5]",
"(boost::beast::basic_stream<boost::asio::ip::tcp, boost::asio::executor, boost::beast::unlimited_rate_policy>::ops::transfer_op<false, boost::asio::const_buffers_1, boost::asio::detail::write_op<boost::beast::basic_stream<boost::asio::ip::tcp, boost::asio::executor, boost::beast::unlimited_rate_policy>, boost::asio::const_buffers_1, boost::asio::const_buffer const*, boost::asio::detail::transfer_all_t, spawn::detail::coro_handler<boost::asio::executor_binder<void ()(), boost::asio::executor>, unsigned long> > >::async_perform(unsigned long, std::integral_constant<bool, false>)+0x318) [0x7f557575f818]",
"(boost::beast::basic_stream<boost::asio::ip::tcp, boost::asio::executor, boost::beast::unlimited_rate_policy>::ops::transfer_op<false, boost::asio::const_buffers_1, boost::asio::detail::write_op<boost::beast::basic_stream<boost::asio::ip::tcp, boost::asio::executor, boost::beast::unlimited_rate_policy>, boost::asio::const_buffers_1, boost::asio::const_buffer const, boost::asio::detail::transfer_all_t, spawn::detail::coro_handler<boost::asio::executor_binder<void ()(), boost::asio::executor>, unsigned long> > >::operator()(boost::system::error_code, unsigned long)+0x2a7) [0x7f557575fb97]",
"/lib/libradosgw.so.2(+0x45df80) [0x7f55756c9f80]",
"/lib/libradosgw.so.2(+0x44c807) [0x7f55756b8807]",
"/lib/libradosgw.so.2(+0x454359) [0x7f55756c0359]",
"(rgw::io::AccountingFilter<rgw::io::RestfulClient>::send_body(char const*, unsigned long)+0x2f) [0x7f55756a64ef]",
"(dump_body(req_state*, char const*, unsigned long)+0x49) [0x7f5575c2fb19]",
"(rgw_flush_formatter_and_reset(req_state*, ceph::Formatter*)+0x153) [0x7f5575c315f3]",
"(end_header(req_state*, RGWOp*, char const*, long, bool, bool)+0x256) [0x7f5575c36ba6]",
"(abort_early(req_state*, RGWOp*, int, RGWHandler*, optional_yield)+0x282) [0x7f5575c370f2]",
"(process_request(rgw::sal::RGWRadosStore*, RGWREST*, RGWRequest*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, rgw::auth::StrategyRegistry const&, RGWRestfulIO*, OpsLogSocket*, optional_yield, rgw::dmclock::Scheduler*, std::__cxx11::basic_string<char, std::char_traits<char>,std::allocator<char> >, std::chrono::duration<unsigned long, std::ratio<1l, 1000000000l> >, int*)+0x1aae) [0x7f557579414e]",
"/lib/libradosgw.so.2(+0x459619) [0x7f55756c5619]",
"/lib/libradosgw.so.2(+0x45ab61) [0x7f55756c6b61]",
"/lib/libradosgw.so.2(+0x45acdc) [0x7f55756c6cdc]",
"make_fcontext()"
],
"ceph_version": "16.2.4",
"crash_id": "2021-09-01T18:25:27.909464Z_6a42037f-ec3c-4663-ae09-5d1997865b37",
"entity_name": "client.rgw.ceph.hdd.object.store.a",
"os_id": "ubuntu",
"os_name": "Ubuntu",
"os_version": "20.04.2 LTS (Focal Fossa)",
"os_version_id": "20.04",
"process_name": "radosgw",
"stack_sig": "9aaee57a8ad4daae300c42802ffa2ba5a9bf46f05971756c6ed6cd3edb256128",
"timestamp": "2021-09-01T18:25:27.909464Z",
"utsname_hostname": "rook-ceph-rgw-ceph-hdd-object-store-a-6fc6858677-kh8kf",
"utsname_machine": "x86_64",
"utsname_release": "5.10.43-flatcar",
"utsname_sysname": "Linux",
"utsname_version": "#1 SMP Tue Jun 15 19:46:57 -00 2021"
}
```

ceph crash info "2021-09-11T10:06:37.908355Z_f82f90c3-c77e-427a-b9f0-94ecfc1b75e3" # corresponding to 2nd segmentaion fault
``` {
"backtrace": [
"/lib/x86_64-linux-gnu/libc.so.6(0x46210) [0x7fb2c88d1210]",
"(boost::asio::detail::reactive_socket_service_base::start_op(boost::asio::detail::reactive_socket_service_base::base_implementation_type&, int, boost::asio::detail::reactor_op*, bool, bool, bool)+0x125) [0x7fb2c8ee30e5]",
"/lib/libradosgw.so.2(+0x481eea) [0x7fb2c8efeeea]",
"(boost::beast::basic_stream<boost::asio::ip::tcp, boost::asio::executor, boost::beast::unlimited_rate_policy>::ops::transfer_op<true, boost::asio::mutable_buffer, boost::beast::detail::dynamic_read_ops::read_op<boost::beast::basic_stream<boost::asio::ip::tcp, boost::asio::executor, boost::beast::unlimited_rate_policy>, boost::beast::flat_static_buffer<65536ul>, boost::beast::http::detail::read_header_condition<true>, spawn::detail::coro_handler<boost::asio::executor_binder<void()(), boost::asio::executor>, unsigned long> > >::operator()(boost::system::error_code, unsigned long)+0x29f) [0x7fb2c8f67dbf]",
"(boost::beast::basic_stream<boost::asio::ip::tcp, boost::asio::executor, boost::beast::unlimited_rate_policy>::ops::transfer_op<true, boost::asio::mutable_buffer, boost::beast::detail::dynamic_read_ops::read_op<boost::beast::basic_stream<boost::asio::ip::tcp, boost::asio::executor, boost::beast::unlimited_rate_policy>, boost::beast::flat_static_buffer<65536ul>, boost::beast::http::detail::read_header_condition<true>, spawn::detail::coro_handler<boost::asio::executor_binder<void()(), boost::asio::executor>, unsigned long> > >::transfer_op<boost::beast::detail::dynamic_read_ops::read_op<boost::beast::basic_stream<boost::asio::ip::tcp, boost::asio::executor, boost::beast::unlimited_rate_policy>, boost::beast::flat_static_buffer<65536ul>, boost::beast::http::detail::read_header_condition<true>, spawn::detail::coro_handler<boost::asio::executor_binder<void ()(), boost::asio::executor>, unsigned long> > >(boost::beast::detail::dynamic_read_ops::read_op<boost::beast::basic_stream<boost::asio::ip::tcp, boost::asio::executor, boost::beast::unlimited_rate_policy>, boost::beast::flat_static_buffer<65536ul>, boost::beast::http::detail::read_header_condition<true>, spawn::detail::coro_handler<boost::asio::executor_binder<void ()(), boost::asio::executor>, unsigned long> >&&, boost::beast::basic_stream<boost::asio::ip::tcp, boost::asio::executor, boost::beast::unlimited_rate_policy>&, boost::asio::mutable_buffer const&)+0x170) [0x7fb2c8f68130]",
"(boost::beast::detail::dynamic_read_ops::read_op<boost::beast::basic_stream<boost::asio::ip::tcp, boost::asio::executor, boost::beast::unlimited_rate_policy>, boost::beast::flat_static_buffer<65536ul>, boost::beast::http::detail::read_header_condition<true>, spawn::detail::coro_handler<boost::asio::executor_binder<void ()(), boost::asio::executor>, unsigned long> >::operator()(boost::system::error_code, unsigned long, bool)+0xd8) [0x7fb2c8f679e8]",
"(boost::beast::basic_stream<boost::asio::ip::tcp, boost::asio::executor, boost::beast::unlimited_rate_policy>::ops::transfer_op<true, boost::asio::mutable_buffer, boost::beast::detail::dynamic_read_ops::read_op<boost::beast::basic_stream<boost::asio::ip::tcp, boost::asio::executor, boost::beast::unlimited_rate_policy>, boost::beast::flat_static_buffer<65536ul>, boost::beast::http::detail::read_header_condition<true>, spawn::detail::coro_handler<boost::asio::executor_binder<void()(), boost::asio::executor>, unsigned long> > >::operator()(boost::system::error_code, unsigned long)+0x137) [0x7fb2c8f67c57]",
"(boost::asio::detail::executor_function<boost::asio::detail::binder2<boost::beast::basic_stream<boost::asio::ip::tcp, boost::asio::executor, boost::beast::unlimited_rate_policy>::ops::transfer_op<true, boost::asio::mutable_buffer, boost::beast::detail::dynamic_read_ops::read_op<boost::beast::basic_stream<boost::asio::ip::tcp, boost::asio::executor, boost::beast::unlimited_rate_policy>, boost::beast::flat_static_buffer<65536ul>, boost::beast::http::detail::read_header_condition<true>, spawn::detail::coro_handler<boost::asio::executor_binder<void ()(), boost::asio::executor>, unsigned long> > >, boost::system::error_code, unsigned long>, std::allocator<void> >::do_complete(boost::asio::detail::executor_function_base, bool)+0x1ee) [0x7fb2c8f683ae]",
"(boost::asio::detail::executor_op<boost::asio::executor::function, std::allocator<void>, boost::asio::detail::scheduler_operation>::do_complete(void*, boost::asio::detail::scheduler_operation*, boost::system::error_code const&, unsigned long)+0x6a) [0x7fb2c8ede2da]",
"(boost::asio::detail::strand_executor_service::invoker<boost::asio::io_context::executor_type const>::operator()()+0x8d) [0x7fb2c8ef893d]",
"(void boost::asio::io_context::executor_type::dispatch<boost::asio::detail::strand_executor_service::invoker<boost::asio::io_context::executor_type const>, std::allocator<void> >(boost::asio::detail::strand_executor_service::invoker<boost::asio::io_context::executor_type const>&&, std::allocator<void> const&) const+0x9c) [0x7fb2c8ef8a5c]",
"(void boost::asio::detail::strand_executor_service::dispatch<boost::asio::io_context::executor_type const, boost::asio::executor::function, std::allocator<void> >(std::shared_ptr<boost::asio::detail::strand_executor_service::strand_impl> const&, boost::asio::io_context::executor_type const&, boost::asio::executor::function&&, std::allocator<void> const&)+0x1a4) [0x7fb2c8f6f634]",
"(void boost::asio::executor::dispatch<boost::asio::detail::binder2<boost::beast::basic_stream<boost::asio::ip::tcp, boost::asio::executor, boost::beast::unlimited_rate_policy>::ops::transfer_op<true, boost::asio::mutable_buffer, boost::beast::detail::dynamic_read_ops::read_op<boost::beast::basic_stream<boost::asio::ip::tcp, boost::asio::executor, boost::beast::unlimited_rate_policy>, boost::beast::flat_static_buffer<65536ul>, boost::beast::http::detail::read_header_condition<true>, spawn::detail::coro_handler<boost::asio::executor_binder<void ()(), boost::asio::executor>, unsigned long> > >, boost::system::error_code, unsigned long>, std::allocator<void> >(boost::asio::detail::binder2<boost::beast::basic_stream<boost::asio::ip::tcp, boost::asio::executor, boost::beast::unlimited_rate_policy>::ops::transfer_op<true, boost::asio::mutable_buffer, boost::beast::detail::dynamic_read_ops::read_op<boost::beast::basic_stream<boost::asio::ip::tcp, boost::asio::executor, boost::beast::unlimited_rate_policy>, boost::beast::flat_static_buffer<65536ul>, boost::beast::http::detail::read_header_condition<true>, spawn::detail::coro_handler<boost::asio::executor_binder<void ()(), boost::asio::executor>, unsigned long> > >, boost::system::error_code, unsigned long>&&, std::allocator<void> const&) const+0x155) [0x7fb2c8f68555]",
"(boost::asio::detail::reactive_socket_recv_op<boost::beast::buffers_prefix_view<boost::asio::mutable_buffer>, boost::beast::basic_stream<boost::asio::ip::tcp, boost::asio::executor, boost::beast::unlimited_rate_policy>::ops::transfer_op<true, boost::asio::mutable_buffer, boost::beast::detail::dynamic_read_ops::read_op<boost::beast::basic_stream<boost::asio::ip::tcp, boost::asio::executor, boost::beast::unlimited_rate_policy>, boost::beast::flat_static_buffer<65536ul>, boost::beast::http::detail::read_header_condition<true>, spawn::detail::coro_handler<boost::asio::executor_binder<void ()(), boost::asio::executor>, unsigned long> > >, boost::asio::detail::io_object_executor<boost::asio::executor> >::do_complete(void, boost::asio::detail::scheduler_operation*, boost::system::error_code const&, unsigned long)+0x10c) [0x7fb2c8f686bc]",
"(boost::asio::detail::scheduler::run(boost::system::error_code&)+0x506) [0x7fb2c8ee8d06]",
"/lib/libradosgw.so.2(+0x44e456) [0x7fb2c8ecb456]",
"/lib/x86_64-linux-gnu/libstdc+.so.6(+0xd6d84) [0x7fb2bef52d84]",
"/lib/x86_64-linux-gnu/libpthread.so.0(+0x9609) [0x7fb2bf066609]",
"clone()"
],
"ceph_version": "16.2.4",
"crash_id": "2021-09-11T10:06:37.908355Z_f82f90c3-c77e-427a-b9f0-94ecfc1b75e3",
"entity_name": "client.rgw.ceph.hdd.object.store.a",
"os_id": "ubuntu",
"os_name": "Ubuntu",
"os_version": "20.04.2 LTS (Focal Fossa)",
"os_version_id": "20.04",
"process_name": "radosgw",
"stack_sig": "133eb59865548ceb3a0357807e24397e2b8ad005e73b04450f29bfaef087e133",
"timestamp": "2021-09-11T10:06:37.908355Z",
"utsname_hostname": "rook-ceph-rgw-ceph-hdd-object-store-a-6fc6858677-kh8kf",
"utsname_machine": "x86_64",
"utsname_release": "5.10.43-flatcar",
"utsname_sysname": "Linux",
"utsname_version": "#1 SMP Tue Jun 15 19:46:57 -00 2021"
}
```