Ceph » rgw

following testing with openssl s_client, s3cmd and Warp, the following ec's occur under normal conditions:

    if (!ec || ec == ssl::error::stream_truncated || ec == http::error::end_of_stream) {    
        // ssl shutdown (ignoring errors)
        stream.async_shutdown(yield[ec]);
    }

quoting https://www.boost.org/doc/libs/1_82_0/doc/html/boost_asio/reference/ssl__error__stream_errors.html:

stream_truncated: The underlying stream closed before the ssl stream gracefully shut down.

as discussed, we'll revert this for main/squid until we have a chance to validate the fix. the reverts are tracked in https://tracker.ceph.com/issues/65742

unfortunately, the quincy and reef backports of https://tracker.ceph.com/issues/64719 are already done

Updating:

Managed to repro the crash repeatedly and verify that the fix PR does resolve the issue.

Details:

Before the fix PR Reproduces consistently in start env during RGW shutdown under load
(which was performed in QEs tests also)
of a downstream ver ceph-18.2.0-189 (same ver that was reported by QE) branch
* (HEAD detached at ceph-18.2.0-189) cf8c965631c ceph-volume: default type to 'bluestore' in 'raw list'
Did not repro on main (albeit on a different machine which can affect the timing)

Reproduction occurs when performing RGW shut-down:

sudo pkill -2 radosgw  # SIGINT

nice numactl -N 1 -m 1 -- warp get --host=127.0.0.1:8443 --tls=true --insecure --access-key=b2345678901234567890 --secret-key=b234567890123456789012345678901234567890 --concurrent=900 --obj.size=4K --duration=24h --objects=100000 --bucket="b01b-${RANDOM}" --noclear

The call-stack:

...
  -405> 2024-05-06T18:27:32.837+0000 7ffff7e0e040 -1 shutting down
   -12> 2024-05-06T18:27:32.842+0000 7ffec6093700 -1 *** Caught signal (Segmentation fault) **
 in thread 7ffec6093700 thread_name:radosgw

 ceph version 18.2.0-603-gcf8c965631c (cf8c965631cc7e6281a5827dae32bd53b7a88a79) reef (stable)
 1: /lib64/libpthread.so.0(+0x12cf0) [0x7ffff4573cf0]
 2: (boost::asio::detail::reactive_socket_service_base::start_op(boost::asio::detail::reactive_socket_service_base::base_implementation_type&, int, boost::asio::detail::reactor_op*, bool, b
ool, bool)+0x117) [0x555555a234d7]
 3: (void boost::asio::detail::reactive_socket_service_base::async_receive<boost::asio::mutable_buffers_1, boost::asio::ssl::detail::io_op<boost::asio::basic_stream_socket<boost::asio::ip::
tcp, boost::asio::io_context::basic_executor_type<std::allocator<void>, 0ul> >, boost::asio::ssl::detail::shutdown_op, spawn::detail::coro_handler<boost::asio::executor_binder<void (*)(), b
oost::asio::strand<boost::asio::io_context::basic_executor_type<std::allocator<void>, 0ul> > >, void> >, boost::asio::io_context::basic_executor_type<std::allocator<void>, 0ul> >(boost::asi
o::detail::reactive_socket_service_base::base_implementation_type&, boost::asio::mutable_buffers_1 const&, int, boost::asio::ssl::detail::io_op<boost::asio::basic_stream_socket<boost::asio:
:ip::tcp, boost::asio::io_context::basic_executor_type<std::allocator<void>, 0ul> >, boost::asio::ssl::detail::shutdown_op, spawn::detail::coro_handler<boost::asio::executor_binder<void (*)
(), boost::asio::strand<boost::asio::io_context::basic_executor_type<std::allocator<void>, 0ul> > >, void> >&, boost::asio::io_context::basic_executor_type<std::allocator<void>, 0ul> const&
)+0x1fc) [0x555555a3ce7c]
 4: (boost::asio::ssl::detail::io_op<boost::asio::basic_stream_socket<boost::asio::ip::tcp, boost::asio::io_context::basic_executor_type<std::allocator<void>, 0ul> >, boost::asio::ssl::deta
il::shutdown_op, spawn::detail::coro_handler<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::basic_executor_type<std::allocator<void>, 0ul> > >, void>
>::operator()(boost::system::error_code, unsigned long, int)+0x5be) [0x555555a43c3e]
 5: (boost::asio::detail::executor_op<boost::asio::detail::binder2<boost::asio::ssl::detail::io_op<boost::asio::basic_stream_socket<boost::asio::ip::tcp, boost::asio::io_context::basic_exec
utor_type<std::allocator<void>, 0ul> >, boost::asio::ssl::detail::shutdown_op, spawn::detail::coro_handler<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_conte
xt::basic_executor_type<std::allocator<void>, 0ul> > >, void> >, boost::system::error_code, unsigned long>, std::allocator<void>, boost::asio::detail::scheduler_operation>::do_complete(void
*, boost::asio::detail::scheduler_operation*, boost::system::error_code const&, unsigned long)+0x17b) [0x555555a454bb]
 6: (boost::asio::detail::strand_executor_service::invoker<boost::asio::io_context::basic_executor_type<std::allocator<void>, 4ul> const, void>::operator()()+0x85) [0x555555a57605]
 7: (void boost::asio::io_context::basic_executor_type<std::allocator<void>, 4ul>::execute<boost::asio::detail::strand_executor_service::invoker<boost::asio::io_context::basic_executor_type
<std::allocator<void>, 4ul> const, void> >(boost::asio::detail::strand_executor_service::invoker<boost::asio::io_context::basic_executor_type<std::allocator<void>, 4ul> const, void>&&) cons
t+0x167) [0x555555a57827]
 8: (void boost::asio::detail::strand_executor_service::do_execute<boost::asio::io_context::basic_executor_type<std::allocator<void>, 4ul> const, boost::asio::detail::binder2<boost::asio::s
sl::detail::io_op<boost::asio::basic_stream_socket<boost::asio::ip::tcp, boost::asio::io_context::basic_executor_type<std::allocator<void>, 0ul> >, boost::asio::ssl::detail::shutdown_op, sp
awn::detail::coro_handler<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::basic_executor_type<std::allocator<void>, 0ul> > >, void> >, boost::system::e
rror_code, unsigned long>, std::allocator<void> >(std::shared_ptr<boost::asio::detail::strand_executor_service::strand_impl> const&, boost::asio::io_context::basic_executor_type<std::alloca
tor<void>, 4ul> const&, boost::asio::detail::binder2<boost::asio::ssl::detail::io_op<boost::asio::basic_stream_socket<boost::asio::ip::tcp, boost::asio::io_context::basic_executor_type<std:
:allocator<void>, 0ul> >, boost::asio::ssl::detail::shutdown_op, spawn::detail::coro_handler<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost::asio::io_context::basic_executor_type<std::allocator<void>, 0ul> > >, void> >, boost::system::error_code, unsigned long>&&, std::allocator<void> const&)+0x149) [0x555555a59709]
 9: (boost::asio::detail::reactive_socket_recv_op<boost::asio::mutable_buffers_1, boost::asio::ssl::detail::io_op<boost::asio::basic_stream_socket<boost::asio::ip::tcp, boost::asio::io_cont
ext::basic_executor_type<std::allocator<void>, 0ul> >, boost::asio::ssl::detail::shutdown_op, spawn::detail::coro_handler<boost::asio::executor_binder<void (*)(), boost::asio::strand<boost:
:asio::io_context::basic_executor_type<std::allocator<void>, 0ul> > >, void> >, boost::asio::io_context::basic_executor_type<std::allocator<void>, 0ul> >::do_complete(void*, boost::asio::de
tail::scheduler_operation*, boost::system::error_code const&, unsigned long)+0x1bd) [0x555555a599fd]
 10: ./bin/radosgw(+0x4d5059) [0x555555a29059]
 11: ./bin/radosgw(+0x4d542a) [0x555555a2942a]
 12: /lib64/libstdc++.so.6(+0xc2b23) [0x7ffff428eb23]
 13: /lib64/libpthread.so.0(+0x81ca) [0x7ffff45691ca]
 14: clone()
 NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.

After the fix PR the seg fault does not repro anymore, and the openssl session reuse is working fine:

echo "" | openssl s_client -connect localhost:8443 --reconnect -no_ticket -tls1_2 |& grep 'Session-ID:'
    Session-ID: C84271A7E7B7A28C62EAD82391171BEEC01EA9BC69C0DA8623A0E8E9A057EAEA
    Session-ID: C84271A7E7B7A28C62EAD82391171BEEC01EA9BC69C0DA8623A0E8E9A057EAEA
    Session-ID: C84271A7E7B7A28C62EAD82391171BEEC01EA9BC69C0DA8623A0E8E9A057EAEA
    Session-ID: C84271A7E7B7A28C62EAD82391171BEEC01EA9BC69C0DA8623A0E8E9A057EAEA
    Session-ID: C84271A7E7B7A28C62EAD82391171BEEC01EA9BC69C0DA8623A0E8E9A057EAEA
    Session-ID: C84271A7E7B7A28C62EAD82391171BEEC01EA9BC69C0DA8623A0E8E9A057EAEA