root@node2:/var/log# ll core.msgr-worker-* -rw------- 1 root ceph 2538639360 Feb 27 19:57 core.msgr-worker-0.377736 -rw------- 1 root ceph 2537709568 Feb 27 19:57 core.msgr-worker-1.377732 -rw------- 1 root ceph 2564624384 Feb 27 19:57 core.msgr-worker-1.377734 -rw------- 1 root ceph 2512400384 Feb 27 19:57 core.msgr-worker-1.377742 -rw------- 1 root ceph 2469797888 Feb 27 19:57 core.msgr-worker-2.377725 -rw------- 1 root ceph 2515800064 Feb 27 19:57 core.msgr-worker-2.377738 -rw------- 1 root ceph 2513207296 Feb 27 19:57 core.msgr-worker-2.377740 root@node2:/var/log# gdb /usr/bin/ceph-osd core.msgr-worker-2.377736 GNU gdb (Ubuntu 8.1-0ubuntu3) 8.1.0.20180409-git Copyright (C) 2018 Free Software Foundation, Inc. License GPLv3+: GNU GPL version 3 or later This is free software: you are free to change and redistribute it. There is NO WARRANTY, to the extent permitted by law. Type "show copying" and "show warranty" for details. This GDB was configured as "aarch64-linux-gnu". Type "show configuration" for configuration details. For bug reporting instructions, please see: . Find the GDB manual and other documentation resources online at: . For help, type "help". Type "apropos word" to search for commands related to "word"... Reading symbols from /usr/bin/ceph-osd...Reading symbols from /usr/lib/debug/.build-id/fd/2d225baa3068ff217e8e576184a3a362e494fd.debug...done. done. /var/log/core.msgr-worker-2.377736: No such file or directory. (gdb) bt No stack. (gdb) quit root@node2:/var/log# gdb /usr/bin/ceph-osd core.msgr-worker-0.377736 GNU gdb (Ubuntu 8.1-0ubuntu3) 8.1.0.20180409-git Copyright (C) 2018 Free Software Foundation, Inc. License GPLv3+: GNU GPL version 3 or later This is free software: you are free to change and redistribute it. There is NO WARRANTY, to the extent permitted by law. Type "show copying" and "show warranty" for details. This GDB was configured as "aarch64-linux-gnu". Type "show configuration" for configuration details. For bug reporting instructions, please see: . Find the GDB manual and other documentation resources online at: . For help, type "help". Type "apropos word" to search for commands related to "word"... Reading symbols from /usr/bin/ceph-osd...Reading symbols from /usr/lib/debug/.build-id/fd/2d225baa3068ff217e8e576184a3a362e494fd.debug...done. done. [New LWP 377856] [New LWP 377736] [New LWP 377760] [New LWP 377988] [New LWP 377857] [New LWP 377858] [New LWP 377990] [New LWP 377989] [New LWP 378362] [New LWP 378922] [New LWP 378923] [New LWP 378930] [New LWP 378931] [New LWP 378934] [New LWP 378939] [New LWP 379507] [New LWP 380985] [New LWP 380991] [New LWP 380995] [New LWP 379543] [New LWP 378954] [New LWP 378940] [New LWP 378933] [New LWP 378956] [New LWP 378932] [New LWP 381006] [New LWP 381010] [New LWP 381021] [New LWP 379509] [New LWP 379540] [New LWP 379545] [New LWP 381032] [New LWP 381042] [New LWP 381029] [New LWP 380987] [New LWP 379546] [New LWP 380998] [New LWP 381016] [New LWP 378955] [New LWP 381001] [New LWP 381024] [New LWP 380996] [New LWP 380994] [New LWP 379539] [New LWP 380990] [New LWP 380992] [New LWP 381022] [New LWP 380999] [New LWP 381031] [New LWP 380986] [New LWP 381028] [New LWP 380984] [New LWP 381005] [New LWP 380988] [New LWP 379516] [New LWP 381040] [New LWP 381004] [New LWP 381019] [New LWP 381023] [New LWP 380989] [New LWP 381013] [New LWP 381000] [New LWP 381008] [New LWP 381045] [New LWP 381002] [New LWP 381003] [New LWP 381033] [New LWP 381014] [New LWP 381007] [New LWP 381017] [New LWP 381020] [New LWP 381009] [New LWP 381037] [New LWP 381044] [New LWP 381015] [New LWP 381012] [New LWP 381046] [New LWP 379544] [New LWP 381034] [New LWP 381025] [New LWP 380997] [New LWP 381038] [New LWP 381027] [New LWP 381039] [New LWP 381018] [New LWP 381035] [New LWP 381026] [New LWP 380993] [New LWP 381041] [New LWP 381011] [New LWP 381047] [New LWP 381043] [New LWP 381030] [New LWP 381036] [Thread debugging using libthread_db enabled] Using host libthread_db library "/lib/aarch64-linux-gnu/libthread_db.so.1". Core was generated by `/usr/bin/ceph-osd -f --cluster ceph --id 37 --setuser ceph --setgroup ceph'. Program terminated with signal SIGSEGV, Segmentation fault. #0 raise (sig=sig@entry=11) at ../sysdeps/unix/sysv/linux/raise.c:51 51 ../sysdeps/unix/sysv/linux/raise.c: No such file or directory. [Current thread is 1 (Thread 0xffffa10bf9f0 (LWP 377856))] (gdb) bt #0 raise (sig=sig@entry=11) at ../sysdeps/unix/sysv/linux/raise.c:51 #1 0x0000aaaabc1bba1c in reraise_fatal (signum=11) at ./src/global/signal_handler.cc:326 #2 handle_fatal_signal (signum=11) at ./src/global/signal_handler.cc:326 #3 #4 __memcpy_generic () at ../sysdeps/aarch64/multiarch/../memcpy.S:105 #5 0x0000aaaabc582d00 in memcpy (__len=52, __src=0xaaab4e327000, __dest=) at /usr/include/aarch64-linux-gnu/bits/string_fortified.h:34 #6 Infiniband::MemoryManager::Chunk::write (this=this@entry=0xaaab5560f6c0, buf=buf@entry=0xaaab4e327000 "\002", , len=) at ./src/msg/async/rdma/Infiniband.cc:738 #7 0x0000aaaabc591018 in RDMAConnectedSocketImpl::tx_copy_chunk (this=this@entry=0xaaab53527d40, tx_buffers=std::vector of length 1, capacity 1 = {...}, req_copy_len=req_copy_len@entry=243, start=..., end=...) at ./src/include/buffer.h:317 #8 0x0000aaaabc592aec in RDMAConnectedSocketImpl::submit (this=this@entry=0xaaab53527d40, more=more@entry=false) at ./src/msg/async/rdma/RDMAConnectedSocketImpl.cc:380 #9 0x0000aaaabc594c34 in RDMAConnectedSocketImpl::send (this=0xaaab53527d40, bl=..., more=) at ./src/msg/async/rdma/RDMAConnectedSocketImpl.cc:297 #10 0x0000aaaabc5304d4 in ConnectedSocket::send (more=false, bl=..., this=0xaaab575d8ee8) at /usr/include/c++/9/bits/unique_ptr.h:357 #11 AsyncConnection::_try_send (this=this@entry=0xaaab575d8d00, more=more@entry=false) at ./src/msg/async/AsyncConnection.cc:330 #12 0x0000aaaabc530ec4 in AsyncConnection::write(ceph::buffer::v14_2_0::list&, std::function, bool) (this=this@entry=0xaaab575d8d00, bl=..., callback=..., more=more@entry=false) at ./src/msg/async/AsyncConnection.cc:309 #13 0x0000aaaabc55eee0 in ProtocolV2::write (this=this@entry=0xaaab5350cc00, desc="auth request", next=..., buffer=...) at /usr/include/c++/9/bits/std_function.h:87 #14 0x0000aaaabc5679b0 in ProtocolV2::write (frame=..., next=..., desc="auth request", this=0xaaab5350cc00) at ./src/msg/async/ProtocolV2.cc:760 #15 ProtocolV2::send_auth_request (this=this@entry=0xaaab5350cc00, allowed_methods=std::vector of length 0, capacity 0) at ./src/msg/async/ProtocolV2.cc:1763 #16 0x0000aaaabc567ea8 in ProtocolV2::send_auth_request (this=0xaaab5350cc00) at ./src/msg/async/ProtocolV2.h:215 #17 ProtocolV2::post_client_banner_exchange (this=0xaaab5350cc00) at ./src/msg/async/ProtocolV2.cc:1731 #18 0x0000aaaabc55e770 in ProtocolV2::run_continuation (this=0xaaab5350cc00, continuation=...) at ./src/msg/async/ProtocolV2.cc:45 #19 0x0000aaaabc532f40 in std::function::operator()(char*, long) const (__args#1=0, __args#0=0xaaab534aa460 "ceph v2\n\020", this=0xaaab575d9110) at /usr/include/c++/9/bits/std_function.h:685 #20 AsyncConnection::process (this=0xaaab575d8d00) at ./src/msg/async/AsyncConnection.cc:457 #21 0x0000aaaabc383edc in EventCenter::process_events (this=this@entry=0xaaaafe0ee608, timeout_microseconds=, working_dur=working_dur@entry=0xffffa10bf160) at /usr/include/c++/9/bits/basic_ios.h:282 #22 0x0000aaaabc38a0f0 in NetworkStack::::operator() (__closure=0xaaaafe15e488, __closure=0xaaaafe15e488) at ./src/msg/async/Stack.cc:53 #23 std::_Function_handler >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/9/bits/std_function.h:300 #24 0x0000ffffa1d42ed4 in ?? () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #25 0x0000ffffa1ef8088 in start_thread (arg=0xffffd84b1edf) at pthread_create.c:463 #26 0x0000ffffa1b124ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 (gdb) f 6 #6 Infiniband::MemoryManager::Chunk::write (this=this@entry=0xaaab5560f6c0, buf=buf@entry=0xaaab4e327000 "\002", , len=) at ./src/msg/async/rdma/Infiniband.cc:738 738 ./src/msg/async/rdma/Infiniband.cc: No such file or directory. (gdb) bt #0 raise (sig=sig@entry=11) at ../sysdeps/unix/sysv/linux/raise.c:51 #1 0x0000aaaabc1bba1c in reraise_fatal (signum=11) at ./src/global/signal_handler.cc:326 #2 handle_fatal_signal (signum=11) at ./src/global/signal_handler.cc:326 #3 #4 __memcpy_generic () at ../sysdeps/aarch64/multiarch/../memcpy.S:105 #5 0x0000aaaabc582d00 in memcpy (__len=52, __src=0xaaab4e327000, __dest=) at /usr/include/aarch64-linux-gnu/bits/string_fortified.h:34 #6 Infiniband::MemoryManager::Chunk::write (this=this@entry=0xaaab5560f6c0, buf=buf@entry=0xaaab4e327000 "\002", , len=) at ./src/msg/async/rdma/Infiniband.cc:738 #7 0x0000aaaabc591018 in RDMAConnectedSocketImpl::tx_copy_chunk (this=this@entry=0xaaab53527d40, tx_buffers=std::vector of length 1, capacity 1 = {...}, req_copy_len=req_copy_len@entry=243, start=..., end=...) at ./src/include/buffer.h:317 #8 0x0000aaaabc592aec in RDMAConnectedSocketImpl::submit (this=this@entry=0xaaab53527d40, more=more@entry=false) at ./src/msg/async/rdma/RDMAConnectedSocketImpl.cc:380 #9 0x0000aaaabc594c34 in RDMAConnectedSocketImpl::send (this=0xaaab53527d40, bl=..., more=) at ./src/msg/async/rdma/RDMAConnectedSocketImpl.cc:297 #10 0x0000aaaabc5304d4 in ConnectedSocket::send (more=false, bl=..., this=0xaaab575d8ee8) at /usr/include/c++/9/bits/unique_ptr.h:357 #11 AsyncConnection::_try_send (this=this@entry=0xaaab575d8d00, more=more@entry=false) at ./src/msg/async/AsyncConnection.cc:330 #12 0x0000aaaabc530ec4 in AsyncConnection::write(ceph::buffer::v14_2_0::list&, std::function, bool) (this=this@entry=0xaaab575d8d00, bl=..., callback=..., more=more@entry=false) at ./src/msg/async/AsyncConnection.cc:309 #13 0x0000aaaabc55eee0 in ProtocolV2::write (this=this@entry=0xaaab5350cc00, desc="auth request", next=..., buffer=...) at /usr/include/c++/9/bits/std_function.h:87 #14 0x0000aaaabc5679b0 in ProtocolV2::write (frame=..., next=..., desc="auth request", this=0xaaab5350cc00) at ./src/msg/async/ProtocolV2.cc:760 #15 ProtocolV2::send_auth_request (this=this@entry=0xaaab5350cc00, allowed_methods=std::vector of length 0, capacity 0) at ./src/msg/async/ProtocolV2.cc:1763 #16 0x0000aaaabc567ea8 in ProtocolV2::send_auth_request (this=0xaaab5350cc00) at ./src/msg/async/ProtocolV2.h:215 #17 ProtocolV2::post_client_banner_exchange (this=0xaaab5350cc00) at ./src/msg/async/ProtocolV2.cc:1731 #18 0x0000aaaabc55e770 in ProtocolV2::run_continuation (this=0xaaab5350cc00, continuation=...) at ./src/msg/async/ProtocolV2.cc:45 #19 0x0000aaaabc532f40 in std::function::operator()(char*, long) const (__args#1=0, __args#0=0xaaab534aa460 "ceph v2\n\020", this=0xaaab575d9110) at /usr/include/c++/9/bits/std_function.h:685 #20 AsyncConnection::process (this=0xaaab575d8d00) at ./src/msg/async/AsyncConnection.cc:457 #21 0x0000aaaabc383edc in EventCenter::process_events (this=this@entry=0xaaaafe0ee608, timeout_microseconds=, working_dur=working_dur@entry=0xffffa10bf160) at /usr/include/c++/9/bits/basic_ios.h:282 #22 0x0000aaaabc38a0f0 in NetworkStack::::operator() (__closure=0xaaaafe15e488, __closure=0xaaaafe15e488) at ./src/msg/async/Stack.cc:53 #23 std::_Function_handler >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/9/bits/std_function.h:300 #24 0x0000ffffa1d42ed4 in ?? () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #25 0x0000ffffa1ef8088 in start_thread (arg=0xffffd84b1edf) at pthread_create.c:463 #26 0x0000ffffa1b124ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 (gdb) l 733 in ./src/msg/async/rdma/Infiniband.cc (gdb) bt full #0 raise (sig=sig@entry=11) at ../sysdeps/unix/sysv/linux/raise.c:51 set = {__val = {4096, 281473383641896, 88, 281473383624960, 187650279109528, 187651370344448, 187650293268480, 525377, 420, 88, 281473383624960, 281473383624960, 281473383624944, 4294967288, 14028173432846458624, 187651370344448}} pid = tid = ret = #1 0x0000aaaabc1bba1c in reraise_fatal (signum=11) at ./src/global/signal_handler.cc:326 ret = buf = "/var/lib/ceph/crash/2020-02-27T11:57:04.693428Z_b789aeae-539d-4d93-ad4b-04011bf0a66c/log", '\000' #2 handle_fatal_signal (signum=11) at ./src/global/signal_handler.cc:326 buf = "*** Caught signal (Segmentation fault) **\n in thread ffffa10bf9f0 thread_name:msgr-worker-0\n", '\000' pthread_name = "msgr-worker-0\000\000" r = bt = {static max = 100, skip = 1, array = {0xaaaabc1bb954 , 0xffffa23e567c <__kernel_rt_sigreturn>, 0xffffa1ac51c8 <__memcpy_generic+56>, 0xaaaabc582d00 , 0xaaaabc591018 >&, unsigned long, ceph::buffer::v14_2_0::list::buffers_t::buffers_iterator&, ceph::buffer::v14_2_0::list::buffers_t::buffers_iterator const&)+192>, 0xaaaabc592aec , 0xaaaabc594c34 , 0xaaaabc5304d4 , 0xaaaabc530ec4 , bool)+116>, 0xaaaabc55eee0 , std::allocator > const&, CtFun&, ceph::buffer::v14_2_0::list&)+216>, 0xaaaabc5679b0 >&)+1056>, 0xaaaabc567ea8 , 0xaaaabc55e770 &)+56>, 0xaaaabc532f40 , 0xaaaabc383edc >*)+1308>, 0xaaaabc38a0f0 >::_M_invoke(const std::_Any_data &)+296>, 0xffffa1d42ed4, 0xffffa1ef8088 , 0x0 }, size = 18, strings = 0xaaab56a85b00} oss = base = "/var/lib/ceph/crash/2020-02-27T11:57:04.693428Z_b789aeae-539d-4d93-ad4b-04011bf0a66c", '\000' #3 No symbol table info available. #4 __memcpy_generic () at ../sysdeps/aarch64/multiarch/../memcpy.S:105 No locals. #5 0x0000aaaabc582d00 in memcpy (__len=52, __src=0xaaab4e327000, __dest=) at /usr/include/aarch64-linux-gnu/bits/string_fortified.h:34 No locals. #6 Infiniband::MemoryManager::Chunk::write (this=this@entry=0xaaab5560f6c0, buf=buf@entry=0xaaab4e327000 "\002", , len=) at ./src/msg/async/rdma/Infiniband.cc:738 write_len = 52 #7 0x0000aaaabc591018 in RDMAConnectedSocketImpl::tx_copy_chunk (this=this@entry=0xaaab53527d40, tx_buffers=std::vector of length 1, capacity 1 = {...}, req_copy_len=req_copy_len@entry=243, start=..., end=...) at ./src/include/buffer.h:317 real_len = addr = 187652728057856 slice_write_len = 0 __PRETTY_FUNCTION__ = chunk_idx = 0 __func__ = "tx_copy_chunk" ---Type to continue, or q to quit--- current_chunk = 0xaaab5560f6c0 write_len = 0 #8 0x0000aaaabc592aec in RDMAConnectedSocketImpl::submit (this=this@entry=0xaaab53527d40, more=more@entry=false) at ./src/msg/async/rdma/RDMAConnectedSocketImpl.cc:380 l = {_M_device = @0xaaab53527df8} bytes = __func__ = "submit" tx_buffers = std::vector of length 1, capacity 1 = {0xaaab5560f6c0} it = {cur = 0xaaab53527db0} copy_start = {cur = 0xaaab50d62880} total_copied = 0 wait_copy_len = 243 __PRETTY_FUNCTION__ = swapped = {_buffers = {_root = {next = 0x0}, _tail = 0xaaab53527df8, _size = 1}, _carriage = 0xaaaac53627b0 , _len = 32, _memcopy_count = 0, last_p = {> = {bl = 0xaaab575d8f28, ls = 0xaaab575d8f28, p = {cur = 0xaaab575d8f28}, off = 0, p_off = 0}, }, static always_empty_bptr = {_raw = 0x0, _off = 0, _len = 0}, static CLAIM_DEFAULT = 0, static CLAIM_ALLOW_NONSHAREABLE = 1} #9 0x0000aaaabc594c34 in RDMAConnectedSocketImpl::send (this=0xaaab53527d40, bl=..., more=) at ./src/msg/async/rdma/RDMAConnectedSocketImpl.cc:297 bytes = 243 __func__ = "send" r = #10 0x0000aaaabc5304d4 in ConnectedSocket::send (more=false, bl=..., this=0xaaab575d8ee8) at /usr/include/c++/9/bits/unique_ptr.h:357 No locals. #11 AsyncConnection::_try_send (this=this@entry=0xaaab575d8d00, more=more@entry=false) at ./src/msg/async/AsyncConnection.cc:330 __func__ = "_try_send" __PRETTY_FUNCTION__ = r = #12 0x0000aaaabc530ec4 in AsyncConnection::write(ceph::buffer::v14_2_0::list&, std::function, bool) (this=this@entry=0xaaab575d8d00, bl=..., callback=..., more=more@entry=false) at ./src/msg/async/AsyncConnection.cc:309 l = r = #13 0x0000aaaabc55eee0 in ProtocolV2::write (this=this@entry=0xaaab5350cc00, desc="auth request", next=..., buffer=...) at /usr/include/c++/9/bits/std_function.h:87 __PRETTY_FUNCTION__ = r = __func__ = "write" #14 0x0000aaaabc5679b0 in ProtocolV2::write (frame=..., next=..., desc="auth request", this=0xaaab5350cc00) at ./src/msg/async/ProtocolV2.cc:760 bl = {_buffers = {_root = {next = 0xffffa10beb00}, _tail = 0xffffa10beb00, _size = 0}, _carriage = 0xaaaac53627b0 , _len = 0, _memcopy_count = 0, last_p = {> = {bl = 0xffffa10beb00, ls = 0xffffa10beb00, p = {cur = 0xffffa10beb00}, off = 0, p_off = 0}, }, static always_empty_bptr = {_raw = 0x0, _off = 0, _len = 0}, static CLAIM_DEFAULT = 0, static CLAIM_ALLOW_NONSHAREABLE = 1} bl = #15 ProtocolV2::send_auth_request (this=this@entry=0xaaab5350cc00, allowed_methods=std::vector of length 0, capacity 0) at ./src/msg/async/ProtocolV2.cc:1763 __PRETTY_FUNCTION__ = ---Type to continue, or q to quit---q Quit (gdb) (gdb) f 7 #7 0x0000aaaabc591018 in RDMAConnectedSocketImpl::tx_copy_chunk (this=this@entry=0xaaab53527d40, tx_buffers=std::vector of length 1, capacity 1 = {...}, req_copy_len=req_copy_len@entry=243, start=..., end=...) at ./src/include/buffer.h:317 317 ./src/include/buffer.h: No such file or directory. (gdb) p *tx_buffers[0] $1 = {mr = 0xaaab556103c0, qp = 0xaaaafd4b86f0, lkey = 2, bytes = 43691, offset = 4263420672, bound = 43690, buffer = 0xaaab00000001 "\006\001\064", pad = { 4249587776, 43690, 4249677768, 43690, 1533731784, 43691}, data = 0xaaab5560f700 ""} (gdb) p isge[0] No symbol "isge" in current context. (gdb) info locals real_len = addr = 187652728057856 slice_write_len = 0 __PRETTY_FUNCTION__ = chunk_idx = 0 __func__ = "tx_copy_chunk" current_chunk = 0xaaab5560f6c0 write_len = 0 (gdb) p *current_chunk $2 = {mr = 0xaaab556103c0, qp = 0xaaaafd4b86f0, lkey = 2, bytes = 43691, offset = 4263420672, bound = 43690, buffer = 0xaaab00000001 "\006\001\064", pad = { 4249587776, 43690, 4249677768, 43690, 1533731784, 43691}, data = 0xaaab5560f700 ""} (gdb) p /x addr $3 = 0xaaab4e327000 (gdb) p *tx_buffers[0] $4 = {mr = 0xaaab556103c0, qp = 0xaaaafd4b86f0, lkey = 2, bytes = 43691, offset = 4263420672, bound = 43690, buffer = 0xaaab00000001 "\006\001\064", pad = { 4249587776, 43690, 4249677768, 43690, 1533731784, 43691}, data = 0xaaab5560f700 ""} (gdb) p start->length Could not find operator->. (gdb) p start->length() Could not find operator->. (gdb) p start $5 = (ceph::buffer::v14_2_0::list::buffers_t::const_iterator &) @0xffffa10be6d0: {cur = 0xaaab50d62880} (gdb) p *start Could not find operator*. (gdb) p start-> buffers_iterator buffers_iterator buffers_iterator cur operator ceph::buffer::v14_2_0::list::buffers_t::buffers_iterator::citer_t operator!= operator* operator++ operator-> operator== (gdb) p start[0] No symbol "operator[]" in current context. (gdb) p start $6 = (ceph::buffer::v14_2_0::list::buffers_t::const_iterator &) @0xffffa10be6d0: {cur = 0xaaab50d62880} (gdb) $7 = (ceph::buffer::v14_2_0::list::buffers_t::const_iterator &) @0xffffa10be6d0: {cur = 0xaaab50d62880} (gdb) $8 = (ceph::buffer::v14_2_0::list::buffers_t::const_iterator &) @0xffffa10be6d0: {cur = 0xaaab50d62880} (gdb) quit root@node2:/var/log# ll core.msgr-worker-* -rw------- 1 root ceph 2538639360 Feb 27 19:57 core.msgr-worker-0.377736 -rw------- 1 root ceph 2537709568 Feb 27 19:57 core.msgr-worker-1.377732 -rw------- 1 root ceph 2564624384 Feb 27 19:57 core.msgr-worker-1.377734 -rw------- 1 root ceph 2512400384 Feb 27 19:57 core.msgr-worker-1.377742 -rw------- 1 root ceph 2469797888 Feb 27 19:57 core.msgr-worker-2.377725 -rw------- 1 root ceph 2515800064 Feb 27 19:57 core.msgr-worker-2.377738 -rw------- 1 root ceph 2513207296 Feb 27 19:57 core.msgr-worker-2.377740 root@node2:/var/log# gdb /usr/bin/ceph-osd core.msgr-worker-1.377732 GNU gdb (Ubuntu 8.1-0ubuntu3) 8.1.0.20180409-git Copyright (C) 2018 Free Software Foundation, Inc. License GPLv3+: GNU GPL version 3 or later This is free software: you are free to change and redistribute it. There is NO WARRANTY, to the extent permitted by law. Type "show copying" and "show warranty" for details. This GDB was configured as "aarch64-linux-gnu". Type "show configuration" for configuration details. For bug reporting instructions, please see: . Find the GDB manual and other documentation resources online at: . For help, type "help". Type "apropos word" to search for commands related to "word"... Reading symbols from /usr/bin/ceph-osd...Reading symbols from /usr/lib/debug/.build-id/fd/2d225baa3068ff217e8e576184a3a362e494fd.debug...done. done. [New LWP 377835] [New LWP 377756] [New LWP 377834] [New LWP 377836] [New LWP 378008] [New LWP 378018] [New LWP 378607] [New LWP 378606] [New LWP 377732] [New LWP 378016] [New LWP 378256] [New LWP 378615] [New LWP 378610] [New LWP 379400] [New LWP 378609] [New LWP 379399] [New LWP 380477] [New LWP 380479] [New LWP 379394] [New LWP 378611] [New LWP 378616] [New LWP 378633] [New LWP 380475] [New LWP 378612] [New LWP 379371] [New LWP 380481] [New LWP 380472] [New LWP 379368] [New LWP 380485] [New LWP 380478] [New LWP 380483] [New LWP 379393] [New LWP 380496] [New LWP 380484] [New LWP 378634] [New LWP 380494] [New LWP 380487] [New LWP 380500] [New LWP 380499] [New LWP 380503] [New LWP 380506] [New LWP 380512] [New LWP 380515] [New LWP 380510] [New LWP 380523] [New LWP 380520] [New LWP 380525] [New LWP 380530] [New LWP 380531] [New LWP 380507] [New LWP 380490] [New LWP 380486] [New LWP 380473] [New LWP 380493] [New LWP 380535] [New LWP 380524] [New LWP 380508] [New LWP 380527] [New LWP 379402] [New LWP 380488] [New LWP 378613] [New LWP 380498] [New LWP 380505] [New LWP 380529] [New LWP 380497] [New LWP 380518] [New LWP 378635] [New LWP 380532] [New LWP 380533] [New LWP 380521] [New LWP 380516] [New LWP 380534] [New LWP 380522] [New LWP 380502] [New LWP 380482] [New LWP 380474] [New LWP 380495] [New LWP 380517] [New LWP 380526] [New LWP 380519] [New LWP 380492] [New LWP 379401] [New LWP 380514] [New LWP 380504] [New LWP 379367] [New LWP 380491] [New LWP 380501] [New LWP 380509] [New LWP 380480] [New LWP 380476] [New LWP 380489] [New LWP 380511] [New LWP 380513] [New LWP 380528] [Thread debugging using libthread_db enabled] Using host libthread_db library "/lib/aarch64-linux-gnu/libthread_db.so.1". Core was generated by `/usr/bin/ceph-osd -f --cluster ceph --id 33 --setuser ceph --setgroup ceph'. Program terminated with signal SIGSEGV, Segmentation fault. #0 raise (sig=sig@entry=11) at ../sysdeps/unix/sysv/linux/raise.c:51 51 ../sysdeps/unix/sysv/linux/raise.c: No such file or directory. [Current thread is 1 (Thread 0xffff92d7e9f0 (LWP 377835))] (gdb) bt #0 raise (sig=sig@entry=11) at ../sysdeps/unix/sysv/linux/raise.c:51 #1 0x0000aaaad3d49a1c in reraise_fatal (signum=11) at ./src/global/signal_handler.cc:326 #2 handle_fatal_signal (signum=11) at ./src/global/signal_handler.cc:326 #3 #4 __memcpy_generic () at ../sysdeps/aarch64/multiarch/../memcpy.S:153 #5 0x0000aaaad4110d00 in memcpy (__len=68, __src=0xaaab4c563000, __dest=) at /usr/include/aarch64-linux-gnu/bits/string_fortified.h:34 #6 Infiniband::MemoryManager::Chunk::write (this=this@entry=0xaaab4c6adba0, buf=buf@entry=0xaaab4c563000 "\001\001$", len=) at ./src/msg/async/rdma/Infiniband.cc:738 #7 0x0000aaaad411f018 in RDMAConnectedSocketImpl::tx_copy_chunk (this=this@entry=0xaaab4f25e0a0, tx_buffers=std::vector of length 1, capacity 1 = {...}, req_copy_len=req_copy_len@entry=85, start=..., end=...) at ./src/include/buffer.h:317 #8 0x0000aaaad4120aec in RDMAConnectedSocketImpl::submit (this=this@entry=0xaaab4f25e0a0, more=more@entry=false) at ./src/msg/async/rdma/RDMAConnectedSocketImpl.cc:380 #9 0x0000aaaad4122c34 in RDMAConnectedSocketImpl::send (this=0xaaab4f25e0a0, bl=..., more=) at ./src/msg/async/rdma/RDMAConnectedSocketImpl.cc:297 #10 0x0000aaaad40be4d4 in ConnectedSocket::send (more=false, bl=..., this=0xaaab4fba93e8) at /usr/include/c++/9/bits/unique_ptr.h:357 #11 AsyncConnection::_try_send (this=this@entry=0xaaab4fba9200, more=more@entry=false) at ./src/msg/async/AsyncConnection.cc:330 #12 0x0000aaaad40beec4 in AsyncConnection::write(ceph::buffer::v14_2_0::list&, std::function, bool) (this=this@entry=0xaaab4fba9200, bl=..., callback=..., more=more@entry=false) at ./src/msg/async/AsyncConnection.cc:309 #13 0x0000aaaad40ecee0 in ProtocolV2::write (this=this@entry=0xaaab4a4a8000, desc="hello frame", next=..., buffer=...) at /usr/include/c++/9/bits/std_function.h:87 #14 0x0000aaaad40ef260 in ProtocolV2::write (frame=..., next=..., desc="hello frame", this=0xaaab4a4a8000) at ./src/msg/async/ProtocolV2.cc:760 #15 ProtocolV2::_handle_peer_banner_payload (this=0xaaab4a4a8000, buffer=..., r=) at ./src/msg/async/ProtocolV2.cc:932 #16 0x0000aaaad40ec770 in ProtocolV2::run_continuation (this=0xaaab4a4a8000, continuation=...) at ./src/msg/async/ProtocolV2.cc:45 #17 0x0000aaaad40c0f40 in std::function::operator()(char*, long) const (__args#1=0, __args#0=0xaaab4c59ebd0 "ceph v2\n\020", this=0xaaab4fba9610) at /usr/include/c++/9/bits/std_function.h:685 #18 AsyncConnection::process (this=0xaaab4fba9200) at ./src/msg/async/AsyncConnection.cc:457 #19 0x0000aaaad3f11edc in EventCenter::process_events (this=this@entry=0xaaaaf103e8c8, timeout_microseconds=, working_dur=working_dur@entry=0xffff92d7e160) at /usr/include/c++/9/bits/basic_ios.h:282 #20 0x0000aaaad3f180f0 in NetworkStack::::operator() (__closure=0xaaaaf10ae4b8, __closure=0xaaaaf10ae4b8) at ./src/msg/async/Stack.cc:53 #21 std::_Function_handler >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/9/bits/std_function.h:300 #22 0x0000ffff94202ed4 in ?? () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #23 0x0000ffff943b8088 in start_thread (arg=0xfffff1bd130f) at pthread_create.c:463 #24 0x0000ffff93fd24ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 (gdb) bt full #0 raise (sig=sig@entry=11) at ../sysdeps/unix/sysv/linux/raise.c:51 set = {__val = {4096, 281473145348840, 88, 281473145331904, 187650677101464, 187651151519744, 187650691260416, 525377, 420, 88, 281473145331904, 281473145331904, 281473145331888, 4294967288, 4353719806521434112, 187651151519744}} pid = tid = ret = #1 0x0000aaaad3d49a1c in reraise_fatal (signum=11) at ./src/global/signal_handler.cc:326 ret = buf = "/var/lib/ceph/crash/2020-02-27T11:57:05.094472Z_181da468-8c5f-44b9-8c4e-3904914fa183/log", '\000' #2 handle_fatal_signal (signum=11) at ./src/global/signal_handler.cc:326 buf = "*** Caught signal (Segmentation fault) **\n in thread ffff92d7e9f0 thread_name:msgr-worker-1\n", '\000' pthread_name = "msgr-worker-1\000\000" r = bt = {static max = 100, skip = 1, array = {0xaaaad3d49954 , 0xffff948a567c <__kernel_rt_sigreturn>, 0xffff93f85254 <__memcpy_generic+196>, 0xaaaad4110d00 , 0xaaaad411f018 >&, unsigned long, ceph::buffer::v14_2_0::list::buffers_t::buffers_iterator&, ceph::buffer::v14_2_0::list::buffers_t::buffers_iterator const&)+192>, 0xaaaad4120aec , 0xaaaad4122c34 , 0xaaaad40be4d4 , 0xaaaad40beec4 , bool)+116>, 0xaaaad40ecee0 , std::allocator > const&, CtFun&, ceph::buffer::v14_2_0::list&)+216>, 0xaaaad40ef260 &&, int)+880>, 0xaaaad40ec770 &)+56>, 0xaaaad40c0f40 , 0xaaaad3f11edc >*)+1308>, 0xaaaad3f180f0 >::_M_invoke(const std::_Any_data &)+296>, 0xffff94202ed4, 0xffff943b8088 , 0x0 }, size = 17, strings = 0xaaab4fba5000} oss = base = "/var/lib/ceph/crash/2020-02-27T11:57:05.094472Z_181da468-8c5f-44b9-8c4e-3904914fa183", '\000' #3 No symbol table info available. #4 __memcpy_generic () at ../sysdeps/aarch64/multiarch/../memcpy.S:153 No locals. #5 0x0000aaaad4110d00 in memcpy (__len=68, __src=0xaaab4c563000, __dest=) at /usr/include/aarch64-linux-gnu/bits/string_fortified.h:34 No locals. #6 Infiniband::MemoryManager::Chunk::write (this=this@entry=0xaaab4c6adba0, buf=buf@entry=0xaaab4c563000 "\001\001$", len=) at ./src/msg/async/rdma/Infiniband.cc:738 write_len = 68 #7 0x0000aaaad411f018 in RDMAConnectedSocketImpl::tx_copy_chunk (this=this@entry=0xaaab4f25e0a0, tx_buffers=std::vector of length 1, capacity 1 = {...}, req_copy_len=req_copy_len@entry=85, start=..., end=...) at ./src/include/buffer.h:317 real_len = addr = 187652696846336 slice_write_len = 0 __PRETTY_FUNCTION__ = chunk_idx = 0 __func__ = "tx_copy_chunk" ---Type to continue, or q to quit--- current_chunk = 0xaaab4c6adba0 write_len = 0 #8 0x0000aaaad4120aec in RDMAConnectedSocketImpl::submit (this=this@entry=0xaaab4f25e0a0, more=more@entry=false) at ./src/msg/async/rdma/RDMAConnectedSocketImpl.cc:380 l = {_M_device = @0xaaab4f25e158} bytes = __func__ = "submit" tx_buffers = std::vector of length 1, capacity 1 = {0xaaab4c6adba0} it = {cur = 0xaaab4f25e110} copy_start = {cur = 0xaaab4991f9e0} total_copied = 0 wait_copy_len = 85 __PRETTY_FUNCTION__ = swapped = {_buffers = {_root = {next = 0x0}, _tail = 0xaaab4f25e158, _size = 0}, _carriage = 0xffff92d7d6d0, _len = 2463618768, _memcopy_count = 65535, last_p = {> = {bl = 0xaaab4fba9428, ls = 0xaaab4fba9428, p = {cur = 0xaaab4fba9428}, off = 0, p_off = 0}, }, static always_empty_bptr = {_raw = 0x0, _off = 0, _len = 0}, static CLAIM_DEFAULT = 0, static CLAIM_ALLOW_NONSHAREABLE = 1} #9 0x0000aaaad4122c34 in RDMAConnectedSocketImpl::send (this=0xaaab4f25e0a0, bl=..., more=) at ./src/msg/async/rdma/RDMAConnectedSocketImpl.cc:297 bytes = 85 __func__ = "send" r = #10 0x0000aaaad40be4d4 in ConnectedSocket::send (more=false, bl=..., this=0xaaab4fba93e8) at /usr/include/c++/9/bits/unique_ptr.h:357 No locals. #11 AsyncConnection::_try_send (this=this@entry=0xaaab4fba9200, more=more@entry=false) at ./src/msg/async/AsyncConnection.cc:330 __func__ = "_try_send" __PRETTY_FUNCTION__ = r = #12 0x0000aaaad40beec4 in AsyncConnection::write(ceph::buffer::v14_2_0::list&, std::function, bool) (this=this@entry=0xaaab4fba9200, bl=..., callback=..., more=more@entry=false) at ./src/msg/async/AsyncConnection.cc:309 l = r = #13 0x0000aaaad40ecee0 in ProtocolV2::write (this=this@entry=0xaaab4a4a8000, desc="hello frame", next=..., buffer=...) at /usr/include/c++/9/bits/std_function.h:87 __PRETTY_FUNCTION__ = r = __func__ = "write" #14 0x0000aaaad40ef260 in ProtocolV2::write (frame=..., next=..., desc="hello frame", this=0xaaab4a4a8000) at ./src/msg/async/ProtocolV2.cc:760 bl = {_buffers = {_root = {next = 0xffff92d7dae0}, _tail = 0xffff92d7dae0, _size = 0}, _carriage = 0xaaaadcef07b0 , _len = 0, _memcopy_count = 0, last_p = {> = {bl = 0xffff92d7dae0, ls = 0xffff92d7dae0, p = {cur = 0xffff92d7dae0}, off = 0, p_off = 0}, }, static always_empty_bptr = {_raw = 0x0, _off = 0, _len = 0}, static CLAIM_DEFAULT = 0, static CLAIM_ALLOW_NONSHAREABLE = 1} bl = #15 ProtocolV2::_handle_peer_banner_payload (this=0xaaab4a4a8000, buffer=..., r=) at ./src/msg/async/ProtocolV2.cc:932 __func__ = "_handle_peer_banner_payload" peer_supported_features = ---Type to continue, or q to quit---] peer_required_features = bl = {_buffers = {_root = {next = 0xaaab4bb41b00}, _tail = 0xaaab4bb41b00, _size = 1}, _carriage = 0xaaab4bb41b08, _len = 16, _memcopy_count = 0, last_p = {> = {bl = 0xffff92d7da98, ls = 0xffff92d7da98, p = {cur = 0xffff92d7da98}, off = 0, p_off = 0}, }, static always_empty_bptr = {_raw = 0x0, _off = 0, _len = 0}, static CLAIM_DEFAULT = 0, static CLAIM_ALLOW_NONSHAREABLE = 1} ti = {bl = 0xffff92d7da98, ls = 0xffff92d7da98, p = {cur = 0xffff92d7da98}, off = 16, p_off = 0} supported_features = 0 required_features = 0 __PRETTY_FUNCTION__ = hello = {> = {> = {static SegmentsNumV = 1, segments = {_M_elems = {{_buffers = {_root = {next = 0xffff92d7db28}, _tail = 0xffff92d7db28, _size = 0}, _carriage = 0xaaaadcef07b0 , _len = 0, _memcopy_count = 0, last_p = {> = {bl = 0xffff92d7db28, ls = 0xffff92d7db28, p = {cur = 0xffff92d7db28}, off = 0, p_off = 0}, }, static always_empty_bptr = {_raw = 0x0, _off = 0, _len = 0}, static CLAIM_DEFAULT = 0, static CLAIM_ALLOW_NONSHAREABLE = 1}}}, static alignments = , preamble_filler = {pos = 0xaaab4c563020 "\004\001\001\001\034"}}, _values = std::tuple containing = {[1] = 0 '\000', [2] = {static TYPE_DEFAULT = entity_addr_t::TYPE_MSGR2, type = 0, nonce = 0, u = {sa = { sa_family = 0, sa_data = '\000' }, sin = {sin_family = 0, sin_port = 0, sin_addr = {s_addr = 0}, sin_zero = "\000\000\000\000\000\000\000"}, sin6 = {sin6_family = 0, sin6_port = 0, sin6_flowinfo = 0, sin6_addr = {__in6_u = { __u6_addr8 = '\000' , __u6_addr16 = {0, 0, 0, 0, 0, 0, 0, 0}, __u6_addr32 = {0, 0, 0, 0}}}, sin6_scope_id = 0}}}}, features = 720575940647714820}, static tag = ceph::msgr::v2::Tag::HELLO} should_gather = _dout_e = _dout_cct = _dout = #16 0x0000aaaad40ec770 in ProtocolV2::run_continuation (this=0xaaab4a4a8000, continuation=...) at ./src/msg/async/ProtocolV2.cc:45 _cont = __func__ = "run_continuation" #17 0x0000aaaad40c0f40 in std::function::operator()(char*, long) const (__args#1=0, __args#0=0xaaab4c59ebd0 "ceph v2\n\020", this=0xaaab4fba9610) at /usr/include/c++/9/bits/std_function.h:685 No locals. #18 AsyncConnection::process (this=0xaaab4fba9200) at ./src/msg/async/AsyncConnection.cc:457 buf_tmp = r = 0 l = {_M_device = @0xaaab4fba94a8} __func__ = "process" __PRETTY_FUNCTION__ = #19 0x0000aaaad3f11edc in EventCenter::process_events (this=this@entry=0xaaaaf103e8c8, timeout_microseconds=, working_dur=working_dur@entry=0xffff92d7e160) at /usr/include/c++/9/bits/basic_ios.h:282 rfired = 1 event = 0xaaaaf1090798 cb = 0xaaab4b030500 event_id = tv = {tv_sec = 0, tv_usec = 320000} numevents = 1 trigger_time = true now = ---Type to continue, or q to quit---q Quit (gdb) quit root@node2:/var/log# ll core.msgr-worker-* -rw------- 1 root ceph 2538639360 Feb 27 19:57 core.msgr-worker-0.377736 -rw------- 1 root ceph 2537709568 Feb 27 19:57 core.msgr-worker-1.377732 -rw------- 1 root ceph 2564624384 Feb 27 19:57 core.msgr-worker-1.377734 -rw------- 1 root ceph 2512400384 Feb 27 19:57 core.msgr-worker-1.377742 -rw------- 1 root ceph 2469797888 Feb 27 19:57 core.msgr-worker-2.377725 -rw------- 1 root ceph 2515800064 Feb 27 19:57 core.msgr-worker-2.377738 -rw------- 1 root ceph 2513207296 Feb 27 19:57 core.msgr-worker-2.377740 root@node2:/var/log# gdb /usr/bin/ceph-osd core.msgr-worker-1.377734 GNU gdb (Ubuntu 8.1-0ubuntu3) 8.1.0.20180409-git Copyright (C) 2018 Free Software Foundation, Inc. License GPLv3+: GNU GPL version 3 or later This is free software: you are free to change and redistribute it. There is NO WARRANTY, to the extent permitted by law. Type "show copying" and "show warranty" for details. This GDB was configured as "aarch64-linux-gnu". Type "show configuration" for configuration details. For bug reporting instructions, please see: . Find the GDB manual and other documentation resources online at: . For help, type "help". Type "apropos word" to search for commands related to "word"... Reading symbols from /usr/bin/ceph-osd...Reading symbols from /usr/lib/debug/.build-id/fd/2d225baa3068ff217e8e576184a3a362e494fd.debug... done. done. [New LWP 377845] [New LWP 377734] [New LWP 377844] [New LWP 378042] [New LWP 377846] [New LWP 378041] [New LWP 378292] [New LWP 378726] [New LWP 378715] [New LWP 378717] [New LWP 378734] [New LWP 378742] [New LWP 377758] [New LWP 379673] [New LWP 379738] [New LWP 379749] [New LWP 378732] [New LWP 378733] [New LWP 379748] [New LWP 379750] [New LWP 381112] [New LWP 379755] [New LWP 378795] [New LWP 381114] [New LWP 381115] [New LWP 381118] [New LWP 381113] [New LWP 381119] [New LWP 378744] [New LWP 381117] [New LWP 381120] [New LWP 378730] [New LWP 379739] [New LWP 381163] [New LWP 381166] [New LWP 381125] [New LWP 379684] [New LWP 378796] [New LWP 379676] [New LWP 381164] [New LWP 381157] [New LWP 381165] [New LWP 381159] [New LWP 381143] [New LWP 381137] [New LWP 381126] [New LWP 381145] [New LWP 381180] [New LWP 381150] [New LWP 381133] [New LWP 381123] [New LWP 381124] [New LWP 381132] [New LWP 378797] [New LWP 381136] [New LWP 381138] [New LWP 381135] [New LWP 381139] [New LWP 381116] [New LWP 381149] [New LWP 381168] [New LWP 381146] [New LWP 381174] [New LWP 381144] [New LWP 381152] [New LWP 381134] [New LWP 381161] [New LWP 381148] [New LWP 381175] [New LWP 381121] [New LWP 381156] [New LWP 381153] [New LWP 381122] [New LWP 381169] [New LWP 381179] [New LWP 381147] [New LWP 381167] [New LWP 381170] [New LWP 381158] [New LWP 381151] [New LWP 381177] [New LWP 381141] [New LWP 381178] [New LWP 381162] [New LWP 381176] [New LWP 381173] [New LWP 381155] [New LWP 381160] [New LWP 381172] [New LWP 381154] [New LWP 381171] [New LWP 381142] [New LWP 381140] [New LWP 378010] [Thread debugging using libthread_db enabled] Using host libthread_db library "/lib/aarch64-linux-gnu/libthread_db.so.1". Core was generated by `/usr/bin/ceph-osd -f --cluster ceph --id 35 --setuser ceph --setgroup ceph'. Program terminated with signal SIGSEGV, Segmentation fault. #0 raise (sig=sig@entry=11) at ../sysdeps/unix/sysv/linux/raise.c:51 51 ../sysdeps/unix/sysv/linux/raise.c: No such file or directory. [Current thread is 1 (Thread 0xffff9d3399f0 (LWP 377845))] (gdb) (gdb) (gdb) bt #0 raise (sig=sig@entry=11) at ../sysdeps/unix/sysv/linux/raise.c:51 #1 0x0000aaaae12c1a1c in reraise_fatal (signum=11) at ./src/global/signal_handler.cc:326 #2 handle_fatal_signal (signum=11) at ./src/global/signal_handler.cc:326 #3 #4 __memcpy_generic () at ../sysdeps/aarch64/multiarch/../memcpy.S:153 #5 0x0000aaaae1688d00 in memcpy (__len=73, __src=0xaaab5dc4c000, __dest=) at /usr/include/aarch64-linux-gnu/bits/string_fortified.h:34 #6 Infiniband::MemoryManager::Chunk::write (this=this@entry=0xaaab5c679ba0, buf=buf@entry=0xaaab5dc4c000 "\021\002)", len=) at ./src/msg/async/rdma/Infiniband.cc:738 #7 0x0000aaaae1697018 in RDMAConnectedSocketImpl::tx_copy_chunk (this=this@entry=0xaaab5e785f80, tx_buffers=std::vector of length 1, capacity 1 = {...}, req_copy_len=req_copy_len@entry=1086, start=..., end=...) at ./src/include/buffer.h:317 #8 0x0000aaaae1698aec in RDMAConnectedSocketImpl::submit (this=this@entry=0xaaab5e785f80, more=more@entry=false) at ./src/msg/async/rdma/RDMAConnectedSocketImpl.cc:380 #9 0x0000aaaae169ac34 in RDMAConnectedSocketImpl::send (this=0xaaab5e785f80, bl=..., more=) at ./src/msg/async/rdma/RDMAConnectedSocketImpl.cc:297 #10 0x0000aaaae16364d4 in ConnectedSocket::send (more=false, bl=..., this=0xaaab5d172f68) at /usr/include/c++/9/bits/unique_ptr.h:357 #11 AsyncConnection::_try_send (this=0xaaab5d172d80, more=more@entry=false) at ./src/msg/async/AsyncConnection.cc:330 #12 0x0000aaaae165e99c in ProtocolV2::write_message (this=this@entry=0xaaab5dd8e580, m=m@entry=0xaaab5f6cb500, more=more@entry=false) at ./src/msg/async/ProtocolV2.cc:538 #13 0x0000aaaae1679778 in ProtocolV2::write_event (this=0xaaab5dd8e580) at ./src/msg/async/ProtocolV2.cc:637 #14 0x0000aaaae16339e8 in AsyncConnection::handle_write (this=0xaaab5d172d80) at /usr/include/c++/9/bits/unique_ptr.h:357 #15 0x0000aaaae148a570 in EventCenter::process_events (this=this@entry=0xaaaafef728c8, timeout_microseconds=, working_dur=working_dur@entry=0xffff9d339160) at /usr/include/c++/9/bits/stl_deque.h:2110 #16 0x0000aaaae14900f0 in NetworkStack::::operator() (__closure=0xaaaafefe24b8, __closure=0xaaaafefe24b8) at ./src/msg/async/Stack.cc:53 #17 std::_Function_handler >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/9/bits/std_function.h:300 #18 0x0000ffff9e7bded4 in ?? () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #19 0x0000ffff9e973088 in start_thread (arg=0xfffff56c646f) at pthread_create.c:463 #20 0x0000ffff9e58d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 (gdb) bt full #0 raise (sig=sig@entry=11) at ../sysdeps/unix/sysv/linux/raise.c:51 set = {__val = {4096, 281473319130328, 88, 281473319113392, 187650900939672, 187651385565184, 187650915098624, 525377, 420, 88, 281473319113392, 281473319113392, 281473319113376, 4294967288, 11714035691759710208, 187651385565184}} pid = tid = ret = #1 0x0000aaaae12c1a1c in reraise_fatal (signum=11) at ./src/global/signal_handler.cc:326 ret = buf = "/var/lib/ceph/crash/2020-02-27T11:57:05.402233Z_9b8d6812-8d8c-4295-8597-6ddab051c312/log", '\000' ... #2 handle_fatal_signal (signum=11) at ./src/global/signal_handler.cc:326 buf = "*** Caught signal (Segmentation fault) **\n in thread ffff9d3399f0 thread_name:msgr-worker-1\n", '\000' pthread_name = "msgr-worker-1\000\000" r = bt = {static max = 100, skip = 1, array = {0xaaaae12c1954 , 0xffff9ee6067c <__kernel_rt_sigreturn>, 0xffff9e540254 <__memcpy_generic+196>, 0xaaaae1688d00 , 0xaaaae1697018 >&, unsigned long, ceph::buffer::v14_2_0::list::buffers_t::buffers_iterator&, ceph::buffer::v14_2_0::list::buffers_t::buffers_iterator const&)+192>, 0xaaaae1698aec , 0xaaaae169ac34 , 0xaaaae16364d4 , 0xaaaae165e99c , 0xaaaae1679778 , 0xaaaae16339e8 , 0xaaaae148a570 >*)+2992>, 0xaaaae14900f0 >::_M_invoke(const std::_Any_data &)+296>, 0xffff9e7bded4, 0xffff9e973088 , 0x0 }, size = 15, strings = 0xaaab5e70f800} oss = base = "/var/lib/ceph/crash/2020-02-27T11:57:05.402233Z_9b8d6812-8d8c-4295-8597-6ddab051c312", '\000' #3 No symbol table info available. #4 __memcpy_generic () at ../sysdeps/aarch64/multiarch/../memcpy.S:153 No locals. #5 0x0000aaaae1688d00 in memcpy (__len=73, __src=0xaaab5dc4c000, __dest=) at /usr/include/aarch64-linux-gnu/bits/string_fortified.h:34 No locals. #6 Infiniband::MemoryManager::Chunk::write (this=this@entry=0xaaab5c679ba0, buf=buf@entry=0xaaab5dc4c000 "\021\002)", len=) at ./src/msg/async/rdma/Infiniband.cc:738 write_len = 73 #7 0x0000aaaae1697018 in RDMAConnectedSocketImpl::tx_copy_chunk (this=this@entry=0xaaab5e785f80, tx_buffers=std::vector of length 1, capacity 1 = {...}, req_copy_len=req_copy_len@entry=1086, start=..., end=...) at ./src/include/buffer.h:317 real_len = addr = 187652989304832 slice_write_len = 0 __PRETTY_FUNCTION__ = chunk_idx = 0 __func__ = "tx_copy_chunk" current_chunk = 0xaaab5c679ba0 write_len = 0 #8 0x0000aaaae1698aec in RDMAConnectedSocketImpl::submit (this=this@entry=0xaaab5e785f80, more=more@entry=false) ---Type to continue, or q to quit--- at ./src/msg/async/rdma/RDMAConnectedSocketImpl.cc:380 l = {_M_device = @0xaaab5e786038} bytes = __func__ = "submit" tx_buffers = std::vector of length 1, capacity 1 = {0xaaab5c679ba0} it = {cur = 0xaaab5e785ff0} copy_start = {cur = 0xaaab5f204b60} total_copied = 0 wait_copy_len = 1086 __PRETTY_FUNCTION__ = swapped = {_buffers = {_root = {next = 0x0}, _tail = 0xaaab5e786038, _size = 281473319143952}, _carriage = 0xffff9d338a58, _len = 2637400560, _memcopy_count = 65535, last_p = {> = {bl = 0xaaab5d172fa8, ls = 0xaaab5d172fa8, p = { cur = 0xaaab5d172fa8}, off = 0, p_off = 0}, }, static always_empty_bptr = {_raw = 0x0, _off = 0, _len = 0}, static CLAIM_DEFAULT = 0, static CLAIM_ALLOW_NONSHAREABLE = 1} #9 0x0000aaaae169ac34 in RDMAConnectedSocketImpl::send (this=0xaaab5e785f80, bl=..., more=) at ./src/msg/async/rdma/RDMAConnectedSocketImpl.cc:297 bytes = 1086 __func__ = "send" r = #10 0x0000aaaae16364d4 in ConnectedSocket::send (more=false, bl=..., this=0xaaab5d172f68) at /usr/include/c++/9/bits/unique_ptr.h:357 No locals. #11 AsyncConnection::_try_send (this=0xaaab5d172d80, more=more@entry=false) at ./src/msg/async/AsyncConnection.cc:330 __func__ = "_try_send" __PRETTY_FUNCTION__ = r = #12 0x0000aaaae165e99c in ProtocolV2::write_message (this=this@entry=0xaaab5dd8e580, m=m@entry=0xaaab5f6cb500, more=more@entry=false) at ./src/msg/async/ProtocolV2.cc:538 __PRETTY_FUNCTION__ = ack_seq = header = @0xaaab5f6cb518: {seq = {v = 6}, tid = {v = 0}, type = {v = 132}, priority = {v = 196}, version = {v = 1}, front_len = {v = 996}, middle_len = { v = 0}, data_len = {v = 0}, data_off = {v = 0}, src = {type = 4 '\004', num = {v = 35}}, compat_version = {v = 1}, reserved = {v = 0}, crc = {v = 0}} footer = @0xaaab5f6cb54d: {front_crc = {v = 0}, middle_crc = {v = 0}, data_crc = {v = 0}, sig = {v = 0}, flags = 3 '\003'} header2 = {seq = {v = 6}, tid = {v = 0}, type = {v = 132}, priority = {v = 196}, version = {v = 1}, data_pre_padding_len = {v = 0}, data_off = {v = 0}, ack_seq = {v = 3}, flags = 3 '\003', compat_version = {v = 1}, reserved = {v = 0}} message = {> = {static SegmentsNumV = 4, segments = {_M_elems = {{_buffers = {_root = { next = 0xffff9d338b90}, _tail = 0xffff9d338b90, _size = 0}, _carriage = 0xaaaaea4687b0 , _len = 0, _memcopy_count = 0, last_p = {> = {bl = 0xffff9d338b90, ls = 0xffff9d338b90, p = { cur = 0xffff9d338b90}, off = 0, p_off = 0}, }, static always_empty_bptr = {_raw = 0x0, _off = 0, _len = 0}, static CLAIM_DEFAULT = 0, static CLAIM_ALLOW_NONSHAREABLE = 1}, {_buffers = {_root = {next = 0xffff9d338bd8}, _tail = 0xffff9d338bd8, _size = 0}, _carriage = 0xaaaaea4687b0 , _len = 0, _memcopy_count = 0, last_p = {> = {bl = 0xffff9d338bd8, ls = 0xffff9d338bd8, p = {cur = 0xffff9d338bd8}, off = 0, p_off = 0}, }, static always_empty_bptr = {_raw = 0x0, _off = 0, _len = 0}, static CLAIM_DEFAULT = 0, static CLAIM_ALLOW_NONSHAREABLE = 1}, {_buffers = {_root = {next = 0xffff9d338c20}, _tail = 0xffff9d338c20, _size = 0}, _carriage = 0xaaaaea4687b0 , _len = 0, _memcopy_count = 0, last_p = {> = {bl = 0xffff9d338c20, ls = 0xffff9d338c20, p = {cur = 0xffff9d338c20}, off = 0, p_off = 0}, }, static always_empty_bptr = {_raw = 0x0, _off = 0, _len = 0}, static CLAIM_DEFAULT = 0, static CLAIM_ALLOW_NONSHAREABLE = 1}, {_buffers = {_root = {next = 0xffff9d338c68}, _tail = 0xffff9d338c68, _size = 0}, ---Type to continue, or q to quit--- _carriage = 0xaaaaea4687b0 , _len = 0, _memcopy_count = 0, last_p = {> = {bl = 0xffff9d338c68, ls = 0xffff9d338c68, p = {cur = 0xffff9d338c68}, off = 0, p_off = 0}, }, static always_empty_bptr = {_raw = 0x0, _off = 0, _len = 0}, static CLAIM_DEFAULT = 0, static CLAIM_ALLOW_NONSHAREABLE = 1}}}, static alignments = , preamble_filler = {pos = 0xaaab5dc4c020 "\006"}}, len = { front = 3781662728, middle = 43690, data = 2637401296}, static tag = ceph::msgr::v2::Tag::MESSAGE} __func__ = "write_message" total_send_size = 1086 rc = #13 0x0000aaaae1679778 in ProtocolV2::write_event (this=0xaaab5dd8e580) at ./src/msg/async/ProtocolV2.cc:637 out_entry = start = {__d = {__r = 193307517723240}} more = false __func__ = "write_event" r = 0 #14 0x0000aaaae16339e8 in AsyncConnection::handle_write (this=0xaaab5d172d80) at /usr/include/c++/9/bits/unique_ptr.h:357 __func__ = "handle_write" #15 0x0000aaaae148a570 in EventCenter::process_events (this=this@entry=0xaaaafef728c8, timeout_microseconds=, working_dur=working_dur@entry=0xffff9d339160) at /usr/include/c++/9/bits/stl_deque.h:2110 e = 0xaaab5e74ca90 cur_process = std::deque with 2 elements = {0xaaab5e74ca90, 0xaaab5e74c180} tv = {tv_sec = 0, tv_usec = 304000} numevents = 3 trigger_time = true now = end_time = it = blocking = __func__ = "process_events" fired_events = std::vector of length 1, capacity 1 = {{fd = 7, mask = 1}} working_start = {__d = {__r = 193307517717480}} #16 0x0000aaaae14900f0 in NetworkStack::::operator() (__closure=0xaaaafefe24b8, __closure=0xaaaafefe24b8) at ./src/msg/async/Stack.cc:53 dur = {__r = 126840} r = tp_name = "msgr-worker-1\000\000" w = this = w = this = __func__ = tp_name = EventMaxWaitUs = should_gather = _dout_e = _dout_cct = _dout = dur = ---Type to continue, or q to quit--- r = should_gather = _dout_e = _dout_cct = _dout = should_gather = _dout_e = _dout_cct = _dout = #17 std::_Function_handler >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/9/bits/std_function.h:300 No locals. #18 0x0000ffff9e7bded4 in ?? () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 No symbol table info available. #19 0x0000ffff9e973088 in start_thread (arg=0xfffff56c646f) at pthread_create.c:463 pd = 0xfffff56c646f unwind_buf = {cancel_jmp_buf = {{jmp_buf = {281473319148016, 281474799264880, 281474799264878, 281474799264879, 0, 4096, 281474799264880, 281473342582784, 281473319148016, 1, 281473319145968, 17434927501274878535, 0, 17434927501264106475, 0, 0, 0, 0, 0, 0, 0, 0}, mask_was_saved = 0}}, priv = {pad = { 0x0, 0x0, 0x0, 0x0}, data = {prev = 0x0, cleanup = 0x0, canceltype = 0}}} not_first_call = #20 0x0000ffff9e58d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 No locals. (gdb) (gdb) quit root@node2:/var/log# ll core.msgr-worker-* -rw------- 1 root ceph 2538639360 Feb 27 19:57 core.msgr-worker-0.377736 -rw------- 1 root ceph 2537709568 Feb 27 19:57 core.msgr-worker-1.377732 -rw------- 1 root ceph 2564624384 Feb 27 19:57 core.msgr-worker-1.377734 -rw------- 1 root ceph 2512400384 Feb 27 19:57 core.msgr-worker-1.377742 -rw------- 1 root ceph 2469797888 Feb 27 19:57 core.msgr-worker-2.377725 -rw------- 1 root ceph 2515800064 Feb 27 19:57 core.msgr-worker-2.377738 -rw------- 1 root ceph 2513207296 Feb 27 19:57 core.msgr-worker-2.377740 root@node2:/var/log# gdb /usr/bin/ceph-osd core.msgr-worker-1.377742 GNU gdb (Ubuntu 8.1-0ubuntu3) 8.1.0.20180409-git Copyright (C) 2018 Free Software Foundation, Inc. License GPLv3+: GNU GPL version 3 or later This is free software: you are free to change and redistribute it. There is NO WARRANTY, to the extent permitted by law. Type "show copying" and "show warranty" for details. This GDB was configured as "aarch64-linux-gnu". Type "show configuration" for configuration details. For bug reporting instructions, please see: . Find the GDB manual and other documentation resources online at: . For help, type "help". Type "apropos word" to search for commands related to "word"... Reading symbols from /usr/bin/ceph-osd...Reading symbols from /usr/lib/debug/.build-id/fd/2d225baa3068ff217e8e576184a3a362e494fd.debug...done. done. [New LWP 377893] [New LWP 377995] [New LWP 377891] [New LWP 377765] [New LWP 379079] [New LWP 377742] [New LWP 377996] [New LWP 377997] [New LWP 379072] [New LWP 379074] [New LWP 379573] [New LWP 378408] [New LWP 379632] [New LWP 380792] [New LWP 379081] [New LWP 380797] [New LWP 380799] [New LWP 379077] [New LWP 379082] [New LWP 379633] [New LWP 380801] [New LWP 379629] [New LWP 379126] [New LWP 379630] [New LWP 380813] [New LWP 380805] [New LWP 379610] [New LWP 380821] [New LWP 380825] [New LWP 380830] [New LWP 380835] [New LWP 380838] [New LWP 379574] [New LWP 380850] [New LWP 380827] [New LWP 380851] [New LWP 379075] [New LWP 379127] [New LWP 380793] [New LWP 380817] [New LWP 380795] [New LWP 380828] [New LWP 380815] [New LWP 380855] [New LWP 380841] [New LWP 380820] [New LWP 380803] [New LWP 379073] [New LWP 377895] [New LWP 380834] [New LWP 380839] [New LWP 380840] [New LWP 380809] [New LWP 379125] [New LWP 379631] [New LWP 380845] [New LWP 380826] [New LWP 379078] [New LWP 380796] [New LWP 380843] [New LWP 380853] [New LWP 379628] [New LWP 380846] [New LWP 380833] [New LWP 380816] [New LWP 380808] [New LWP 380811] [New LWP 380852] [New LWP 380849] [New LWP 380818] [New LWP 380807] [New LWP 380831] [New LWP 380822] [New LWP 380802] [New LWP 380823] [New LWP 380844] [New LWP 380847] [New LWP 380836] [New LWP 380812] [New LWP 380806] [New LWP 380848] [New LWP 380837] [New LWP 380814] [New LWP 380794] [New LWP 380798] [New LWP 380804] [New LWP 380810] [New LWP 380842] [New LWP 380800] [New LWP 380832] [New LWP 380819] [New LWP 380824] [New LWP 380854] [New LWP 380829] [Thread debugging using libthread_db enabled] Using host libthread_db library "/lib/aarch64-linux-gnu/libthread_db.so.1". Core was generated by `/usr/bin/ceph-osd -f --cluster ceph --id 43 --setuser ceph --setgroup ceph'. Program terminated with signal SIGSEGV, Segmentation fault. #0 raise (sig=sig@entry=11) at ../sysdeps/unix/sysv/linux/raise.c:51 51 ../sysdeps/unix/sysv/linux/raise.c: No such file or directory. [Current thread is 1 (Thread 0xffffbd0e59f0 (LWP 377893))] (gdb) bt #0 raise (sig=sig@entry=11) at ../sysdeps/unix/sysv/linux/raise.c:51 #1 0x0000aaaacd0a3a1c in reraise_fatal (signum=11) at ./src/global/signal_handler.cc:326 #2 handle_fatal_signal (signum=11) at ./src/global/signal_handler.cc:326 #3 #4 __memcpy_generic () at ../sysdeps/aarch64/multiarch/../memcpy.S:153 #5 0x0000aaaacd46ad00 in memcpy (__len=73, __src=0xaaab54164000, __dest=) at /usr/include/aarch64-linux-gnu/bits/string_fortified.h:34 #6 Infiniband::MemoryManager::Chunk::write (this=this@entry=0xaaab4bf09ba0, buf=buf@entry=0xaaab54164000 "\021\002)", len=) at ./src/msg/async/rdma/Infiniband.cc:738 #7 0x0000aaaacd479018 in RDMAConnectedSocketImpl::tx_copy_chunk (this=this@entry=0xaaab4d448d80, tx_buffers=std::vector of length 1, capacity 1 = {...}, req_copy_len=req_copy_len@entry=37064, start=..., end=...) at ./src/include/buffer.h:317 #8 0x0000aaaacd47aaec in RDMAConnectedSocketImpl::submit (this=this@entry=0xaaab4d448d80, more=more@entry=false) at ./src/msg/async/rdma/RDMAConnectedSocketImpl.cc:380 #9 0x0000aaaacd47cc34 in RDMAConnectedSocketImpl::send (this=0xaaab4d448d80, bl=..., more=) at ./src/msg/async/rdma/RDMAConnectedSocketImpl.cc:297 #10 0x0000aaaacd4184d4 in ConnectedSocket::send (more=false, bl=..., this=0xaaab5016aae8) at /usr/include/c++/9/bits/unique_ptr.h:357 #11 AsyncConnection::_try_send (this=0xaaab5016a900, more=more@entry=false) at ./src/msg/async/AsyncConnection.cc:330 #12 0x0000aaaacd44099c in ProtocolV2::write_message (this=this@entry=0xaaab50176000, m=m@entry=0xaaab4d3d2880, more=more@entry=false) at ./src/msg/async/ProtocolV2.cc:538 #13 0x0000aaaacd45b778 in ProtocolV2::write_event (this=0xaaab50176000) at ./src/msg/async/ProtocolV2.cc:637 #14 0x0000aaaacd4159e8 in AsyncConnection::handle_write (this=0xaaab5016a900) at /usr/include/c++/9/bits/unique_ptr.h:357 #15 0x0000aaaacd26c570 in EventCenter::process_events (this=this@entry=0xaaaaf6d808c8, timeout_microseconds=, working_dur=working_dur@entry=0xffffbd0e5160) at /usr/include/c++/9/bits/stl_deque.h:2110 #16 0x0000aaaacd2720f0 in NetworkStack::::operator() (__closure=0xaaaaf6dfa4b8, __closure=0xaaaaf6dfa4b8) at ./src/msg/async/Stack.cc:53 #17 std::_Function_handler >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/9/bits/std_function.h:300 #18 0x0000ffffbe569ed4 in ?? () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #19 0x0000ffffbe71f088 in start_thread (arg=0xffffe1a1326f) at pthread_create.c:463 #20 0x0000ffffbe3394ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 (gdb) f 6 #6 Infiniband::MemoryManager::Chunk::write (this=this@entry=0xaaab4bf09ba0, buf=buf@entry=0xaaab54164000 "\021\002)", len=) at ./src/msg/async/rdma/Infiniband.cc:738 738 ./src/msg/async/rdma/Infiniband.cc: No such file or directory. (gdb) p *this $1 = {mr = 0xaaaaf61b8000, qp = 0xaaaaf614a6f0, lkey = 2, bytes = 43690, offset = 4142424832, bound = 43690, buffer = 0xaaab00000001 "", pad = {4128550976, 43690, 4128640968, 43690, 1354679688, 43691}, data = 0xaaab4bf09be0 ""} (gdb) quit root@node2:/var/log# gdb /usr/bin/ceph-osd core.msgr-worker-1.377734 GNU gdb (Ubuntu 8.1-0ubuntu3) 8.1.0.20180409-git Copyright (C) 2018 Free Software Foundation, Inc. License GPLv3+: GNU GPL version 3 or later This is free software: you are free to change and redistribute it. There is NO WARRANTY, to the extent permitted by law. Type "show copying" and "show warranty" for details. This GDB was configured as "aarch64-linux-gnu". Type "show configuration" for configuration details. For bug reporting instructions, please see: . Find the GDB manual and other documentation resources online at: . For help, type "help". Type "apropos word" to search for commands related to "word"... Reading symbols from /usr/bin/ceph-osd...Reading symbols from /usr/lib/debug/.build-id/fd/2d225baa3068ff217e8e576184a3a362e494fd.debug...done. done. [New LWP 377845] [New LWP 377734] [New LWP 377844] [New LWP 378042] [New LWP 377846] [New LWP 378041] [New LWP 378292] [New LWP 378726] [New LWP 378715] [New LWP 378717] [New LWP 378734] [New LWP 378742] [New LWP 377758] [New LWP 379673] [New LWP 379738] [New LWP 379749] [New LWP 378732] [New LWP 378733] [New LWP 379748] [New LWP 379750] [New LWP 381112] [New LWP 379755] [New LWP 378795] [New LWP 381114] [New LWP 381115] [New LWP 381118] [New LWP 381113] [New LWP 381119] [New LWP 378744] [New LWP 381117] [New LWP 381120] [New LWP 378730] [New LWP 379739] [New LWP 381163] [New LWP 381166] [New LWP 381125] [New LWP 379684] [New LWP 378796] [New LWP 379676] [New LWP 381164] [New LWP 381157] [New LWP 381165] [New LWP 381159] [New LWP 381143] [New LWP 381137] [New LWP 381126] [New LWP 381145] [New LWP 381180] [New LWP 381150] [New LWP 381133] [New LWP 381123] [New LWP 381124] [New LWP 381132] [New LWP 378797] [New LWP 381136] [New LWP 381138] [New LWP 381135] [New LWP 381139] [New LWP 381116] [New LWP 381149] [New LWP 381168] [New LWP 381146] [New LWP 381174] [New LWP 381144] [New LWP 381152] [New LWP 381134] [New LWP 381161] [New LWP 381148] [New LWP 381175] [New LWP 381121] [New LWP 381156] [New LWP 381153] [New LWP 381122] [New LWP 381169] [New LWP 381179] [New LWP 381147] [New LWP 381167] [New LWP 381170] [New LWP 381158] [New LWP 381151] [New LWP 381177] [New LWP 381141] [New LWP 381178] [New LWP 381162] [New LWP 381176] [New LWP 381173] [New LWP 381155] [New LWP 381160] [New LWP 381172] [New LWP 381154] [New LWP 381171] [New LWP 381142] [New LWP 381140] [New LWP 378010] [Thread debugging using libthread_db enabled] Using host libthread_db library "/lib/aarch64-linux-gnu/libthread_db.so.1". Core was generated by `/usr/bin/ceph-osd -f --cluster ceph --id 35 --setuser ceph --setgroup ceph'. Program terminated with signal SIGSEGV, Segmentation fault. #0 raise (sig=sig@entry=11) at ../sysdeps/unix/sysv/linux/raise.c:51 51 ../sysdeps/unix/sysv/linux/raise.c: No such file or directory. [Current thread is 1 (Thread 0xffff9d3399f0 (LWP 377845))] (gdb) bt #0 raise (sig=sig@entry=11) at ../sysdeps/unix/sysv/linux/raise.c:51 #1 0x0000aaaae12c1a1c in reraise_fatal (signum=11) at ./src/global/signal_handler.cc:326 #2 handle_fatal_signal (signum=11) at ./src/global/signal_handler.cc:326 #3 #4 __memcpy_generic () at ../sysdeps/aarch64/multiarch/../memcpy.S:153 #5 0x0000aaaae1688d00 in memcpy (__len=73, __src=0xaaab5dc4c000, __dest=) at /usr/include/aarch64-linux-gnu/bits/string_fortified.h:34 #6 Infiniband::MemoryManager::Chunk::write (this=this@entry=0xaaab5c679ba0, buf=buf@entry=0xaaab5dc4c000 "\021\002)", len=) at ./src/msg/async/rdma/Infiniband.cc:738 #7 0x0000aaaae1697018 in RDMAConnectedSocketImpl::tx_copy_chunk (this=this@entry=0xaaab5e785f80, tx_buffers=std::vector of length 1, capacity 1 = {...}, req_copy_len=req_copy_len@entry=1086, start=..., end=...) at ./src/include/buffer.h:317 #8 0x0000aaaae1698aec in RDMAConnectedSocketImpl::submit (this=this@entry=0xaaab5e785f80, more=more@entry=false) at ./src/msg/async/rdma/RDMAConnectedSocketImpl.cc:380 #9 0x0000aaaae169ac34 in RDMAConnectedSocketImpl::send (this=0xaaab5e785f80, bl=..., more=) at ./src/msg/async/rdma/RDMAConnectedSocketImpl.cc:297 #10 0x0000aaaae16364d4 in ConnectedSocket::send (more=false, bl=..., this=0xaaab5d172f68) at /usr/include/c++/9/bits/unique_ptr.h:357 #11 AsyncConnection::_try_send (this=0xaaab5d172d80, more=more@entry=false) at ./src/msg/async/AsyncConnection.cc:330 q #12 0x0000aaaae165e99c in ProtocolV2::write_message (this=this@entry=0xaaab5dd8e580, m=m@entry=0xaaab5f6cb500, more=more@entry=false) at ./src/msg/async/ProtocolV2.cc:538 #13 0x0000aaaae1679778 in ProtocolV2::write_event (this=0xaaab5dd8e580) at ./src/msg/async/ProtocolV2.cc:637 #14 0x0000aaaae16339e8 in AsyncConnection::handle_write (this=0xaaab5d172d80) at /usr/include/c++/9/bits/unique_ptr.h:357 #15 0x0000aaaae148a570 in EventCenter::process_events (this=this@entry=0xaaaafef728c8, timeout_microseconds=, working_dur=working_dur@entry=0xffff9d339160) at /usr/include/c++/9/bits/stl_deque.h:2110 #16 0x0000aaaae14900f0 in NetworkStack::::operator() (__closure=0xaaaafefe24b8, __closure=0xaaaafefe24b8) at ./src/msg/async/Stack.cc:53 #17 std::_Function_handler >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/9/bits/std_function.h:300 #18 0x0000ffff9e7bded4 in ?? () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #19 0x0000ffff9e973088 in start_thread (arg=0xfffff56c646f) at pthread_create.c:463 #20 0x0000ffff9e58d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 (gdb) q root@node2:/var/log# gdb /usr/bin/ceph-osd core.msgr-worker-1.377734 GNU gdb (Ubuntu 8.1-0ubuntu3) 8.1.0.20180409-git Copyright (C) 2018 Free Software Foundation, Inc. License GPLv3+: GNU GPL version 3 or later This is free software: you are free to change and redistribute it. There is NO WARRANTY, to the extent permitted by law. Type "show copying" and "show warranty" for details. This GDB was configured as "aarch64-linux-gnu". Type "show configuration" for configuration details. For bug reporting instructions, please see: . Find the GDB manual and other documentation resources online at: . For help, type "help". Type "apropos word" to search for commands related to "word"... Reading symbols from /usr/bin/ceph-osd...Reading symbols from /usr/lib/debug/.build-id/fd/2d225baa3068ff217e8e576184a3a362e494fd.debug... done. done. [New LWP 377845] [New LWP 377734] [New LWP 377844] [New LWP 378042] [New LWP 377846] [New LWP 378041] [New LWP 378292] [New LWP 378726] [New LWP 378715] [New LWP 378717] [New LWP 378734] [New LWP 378742] [New LWP 377758] [New LWP 379673] [New LWP 379738] [New LWP 379749] [New LWP 378732] [New LWP 378733] [New LWP 379748] [New LWP 379750] [New LWP 381112] [New LWP 379755] [New LWP 378795] [New LWP 381114] [New LWP 381115] [New LWP 381118] [New LWP 381113] [New LWP 381119] [New LWP 378744] [New LWP 381117] [New LWP 381120] [New LWP 378730] [New LWP 379739] [New LWP 381163] [New LWP 381166] [New LWP 381125] [New LWP 379684] [New LWP 378796] [New LWP 379676] [New LWP 381164] [New LWP 381157] [New LWP 381165] [New LWP 381159] [New LWP 381143] [New LWP 381137] [New LWP 381126] [New LWP 381145] [New LWP 381180] [New LWP 381150] [New LWP 381133] [New LWP 381123] [New LWP 381124] [New LWP 381132] [New LWP 378797] [New LWP 381136] [New LWP 381138] [New LWP 381135] [New LWP 381139] [New LWP 381116] [New LWP 381149] [New LWP 381168] [New LWP 381146] [New LWP 381174] [New LWP 381144] [New LWP 381152] [New LWP 381134] [New LWP 381161] [New LWP 381148] [New LWP 381175] [New LWP 381121] [New LWP 381156] [New LWP 381153] [New LWP 381122] [New LWP 381169] [New LWP 381179] [New LWP 381147] [New LWP 381167] [New LWP 381170] [New LWP 381158] [New LWP 381151] [New LWP 381177] [New LWP 381141] [New LWP 381178] [New LWP 381162] [New LWP 381176] [New LWP 381173] [New LWP 381155] [New LWP 381160] [New LWP 381172] [New LWP 381154] [New LWP 381171] [New LWP 381142] [New LWP 381140] [New LWP 378010] [Thread debugging using libthread_db enabled] Using host libthread_db library "/lib/aarch64-linux-gnu/libthread_db.so.1". Core was generated by `/usr/bin/ceph-osd -f --cluster ceph --id 35 --setuser ceph --setgroup ceph'. Program terminated with signal SIGSEGV, Segmentation fault. #0 raise (sig=sig@entry=11) at ../sysdeps/unix/sysv/linux/raise.c:51 51 ../sysdeps/unix/sysv/linux/raise.c: No such file or directory. [Current thread is 1 (Thread 0xffff9d3399f0 (LWP 377845))] (gdb) (gdb) f 6 #6 Infiniband::MemoryManager::Chunk::write (this=this@entry=0xaaab5c679ba0, buf=buf@entry=0xaaab5dc4c000 "\021\002)", len=) at ./src/msg/async/rdma/Infiniband.cc:738 738 ./src/msg/async/rdma/Infiniband.cc: No such file or directory. (gdb) p *this $1 = {mr = 0xaaaafe3aa000, qp = 0xaaaafe33c6f0, lkey = 2, bytes = 43690, offset = 4278641408, bound = 43690, buffer = 0xaaab00000001 "ceph v2\n\020", pad = { 4264808512, 43690, 4264898504, 43690, 1424196552, 43691}, data = 0xaaab5c679be0 ""} (gdb) quit root@node2:/var/log# gdb /usr/bin/ceph-osd core.msgr-worker-1.377734^C root@node2:/var/log# ll core.msgr-worker-* -rw------- 1 root ceph 2538639360 Feb 27 19:57 core.msgr-worker-0.377736 -rw------- 1 root ceph 2537709568 Feb 27 19:57 core.msgr-worker-1.377732 -rw------- 1 root ceph 2564624384 Feb 27 19:57 core.msgr-worker-1.377734 -rw------- 1 root ceph 2512400384 Feb 27 19:57 core.msgr-worker-1.377742 -rw------- 1 root ceph 2469797888 Feb 27 19:57 core.msgr-worker-2.377725 -rw------- 1 root ceph 2515800064 Feb 27 19:57 core.msgr-worker-2.377738 -rw------- 1 root ceph 2513207296 Feb 27 19:57 core.msgr-worker-2.377740 root@node2:/var/log# gdb /usr/bin/ceph-osd core.msgr-worker-2.377738 GNU gdb (Ubuntu 8.1-0ubuntu3) 8.1.0.20180409-git Copyright (C) 2018 Free Software Foundation, Inc. License GPLv3+: GNU GPL version 3 or later This is free software: you are free to change and redistribute it. There is NO WARRANTY, to the extent permitted by law. Type "show copying" and "show warranty" for details. This GDB was configured as "aarch64-linux-gnu". Type "show configuration" for configuration details. For bug reporting instructions, please see: . Find the GDB manual and other documentation resources online at: . For help, type "help". Type "apropos word" to search for commands related to "word"... Reading symbols from /usr/bin/ceph-osd...Reading symbols from /usr/lib/debug/.build-id/fd/2d225baa3068ff217e8e576184a3a362e494fd.debug...done. done. [New LWP 377874] [New LWP 377738] [New LWP 377872] [New LWP 377762] [New LWP 377967] [New LWP 377873] [New LWP 377968] [New LWP 378481] [New LWP 378484] [New LWP 377969] [New LWP 378485] [New LWP 378488] [New LWP 378490] [New LWP 378482] [New LWP 379145] [New LWP 378486] [New LWP 378487] [New LWP 379167] [New LWP 379169] [New LWP 378496] [New LWP 378491] [New LWP 379148] [New LWP 378497] [New LWP 378495] [New LWP 379172] [New LWP 379146] [New LWP 380215] [New LWP 380218] [New LWP 380217] [New LWP 380220] [New LWP 379170] [New LWP 380221] [New LWP 380226] [New LWP 380225] [New LWP 380243] [New LWP 380246] [New LWP 380250] [New LWP 380248] [New LWP 380222] [New LWP 380260] [New LWP 380223] [New LWP 380254] [New LWP 380227] [New LWP 380229] [New LWP 380216] [New LWP 380262] [New LWP 380228] [New LWP 380273] [New LWP 380230] [New LWP 380224] [New LWP 379171] [New LWP 380231] [New LWP 380240] [New LWP 380268] [New LWP 380242] [New LWP 379168] [New LWP 380238] [New LWP 380235] [New LWP 380258] [New LWP 380239] [New LWP 380233] [New LWP 380256] [New LWP 380249] [New LWP 380272] [New LWP 380259] [New LWP 380264] [New LWP 380232] [New LWP 380234] [New LWP 380219] [New LWP 380245] [New LWP 380267] [New LWP 380244] [New LWP 380236] [New LWP 380271] [New LWP 380278] [New LWP 380251] [New LWP 380255] [New LWP 380265] [New LWP 380275] [New LWP 380274] [New LWP 380257] [New LWP 380263] [New LWP 380270] [New LWP 380269] [New LWP 380247] [New LWP 380276] [New LWP 380252] [New LWP 380277] [New LWP 380266] [New LWP 380253] [New LWP 380237] [New LWP 380241] [New LWP 378222] [New LWP 380261] [Thread debugging using libthread_db enabled] Using host libthread_db library "/lib/aarch64-linux-gnu/libthread_db.so.1". Core was generated by `/usr/bin/ceph-osd -f --cluster ceph --id 39 --setuser ceph --setgroup ceph'. Program terminated with signal SIGSEGV, Segmentation fault. #0 raise (sig=sig@entry=11) at ../sysdeps/unix/sysv/linux/raise.c:51 51 ../sysdeps/unix/sysv/linux/raise.c: No such file or directory. [Current thread is 1 (Thread 0xffff934209f0 (LWP 377874))] (gdb) bt #0 raise (sig=sig@entry=11) at ../sysdeps/unix/sysv/linux/raise.c:51 #1 0x0000aaaabc055a1c in reraise_fatal (signum=11) at ./src/global/signal_handler.cc:326 #2 handle_fatal_signal (signum=11) at ./src/global/signal_handler.cc:326 #3 #4 0x0000aaaabc42bcf4 in RDMAConnectedSocketImpl::post_work_request (this=this@entry=0xaaab5141c400, tx_buffers=std::vector of length 1, capacity 1 = {...}) at ./src/msg/async/rdma/RDMAConnectedSocketImpl.cc:424 #5 0x0000aaaabc42c9a4 in RDMAConnectedSocketImpl::submit (this=this@entry=0xaaab5141c400, more=more@entry=false) at ./src/msg/async/rdma/RDMAConnectedSocketImpl.cc:398 #6 0x0000aaaabc42ec34 in RDMAConnectedSocketImpl::send (this=0xaaab5141c400, bl=..., more=) at ./src/msg/async/rdma/RDMAConnectedSocketImpl.cc:297 #7 0x0000aaaabc3ca4d4 in ConnectedSocket::send (more=false, bl=..., this=0xaaab52b94ee8) at /usr/include/c++/9/bits/unique_ptr.h:357 #8 AsyncConnection::_try_send (this=0xaaab52b94d00, more=more@entry=false) at ./src/msg/async/AsyncConnection.cc:330 #9 0x0000aaaabc3f299c in ProtocolV2::write_message (this=this@entry=0xaaab5829e000, m=m@entry=0xaaab58602a00, more=more@entry=false) at ./src/msg/async/ProtocolV2.cc:538 #10 0x0000aaaabc40d778 in ProtocolV2::write_event (this=0xaaab5829e000) at ./src/msg/async/ProtocolV2.cc:637 #11 0x0000aaaabc3c79e8 in AsyncConnection::handle_write (this=0xaaab52b94d00) at /usr/include/c++/9/bits/unique_ptr.h:357 #12 0x0000aaaabc21e570 in EventCenter::process_events (this=this@entry=0xaaaafad94b88, timeout_microseconds=, working_dur=working_dur@entry=0xffff93420160) at /usr/include/c++/9/bits/stl_deque.h:2110 #13 0x0000aaaabc2240f0 in NetworkStack::::operator() (__closure=0xaaaafae044e8, __closure=0xaaaafae044e8) at ./src/msg/async/Stack.cc:53 #14 std::_Function_handler >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/9/bits/std_function.h:300 #15 0x0000ffff950a5ed4 in ?? () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #16 0x0000ffff9525b088 in start_thread (arg=0xffffdbe9135f) at pthread_create.c:463 #17 0x0000ffff94e754ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 (gdb) f 4 #4 0x0000aaaabc42bcf4 in RDMAConnectedSocketImpl::post_work_request (this=this@entry=0xaaab5141c400, tx_buffers=std::vector of length 1, capacity 1 = {...}) at ./src/msg/async/rdma/RDMAConnectedSocketImpl.cc:424 424 ./src/msg/async/rdma/RDMAConnectedSocketImpl.cc: No such file or directory. (gdb) p *tx_buffers[0] $1 = {mr = 0x0, qp = 0xaaaafa15e6f0, lkey = 2, bytes = 43691, offset = 2123, bound = 43691, buffer = 0xaaab00000001 "\021\002)", pad = {4195741760, 43690, 4195831752, 43690, 1433035720, 43691}, data = 0xaaab54c8f780 ""} (gdb) quit root@node2:/var/log# ll core.msgr-worker-* -rw------- 1 root ceph 2538639360 Feb 27 19:57 core.msgr-worker-0.377736 -rw------- 1 root ceph 2537709568 Feb 27 19:57 core.msgr-worker-1.377732 -rw------- 1 root ceph 2564624384 Feb 27 19:57 core.msgr-worker-1.377734 -rw------- 1 root ceph 2512400384 Feb 27 19:57 core.msgr-worker-1.377742 -rw------- 1 root ceph 2469797888 Feb 27 19:57 core.msgr-worker-2.377725 -rw------- 1 root ceph 2515800064 Feb 27 19:57 core.msgr-worker-2.377738 -rw------- 1 root ceph 2513207296 Feb 27 19:57 core.msgr-worker-2.377740 root@node2:/var/log# gdb /usr/bin/ceph-osd core.msgr-worker-2.377740 GNU gdb (Ubuntu 8.1-0ubuntu3) 8.1.0.20180409-git Copyright (C) 2018 Free Software Foundation, Inc. License GPLv3+: GNU GPL version 3 or later This is free software: you are free to change and redistribute it. There is NO WARRANTY, to the extent permitted by law. Type "show copying" and "show warranty" for details. This GDB was configured as "aarch64-linux-gnu". Type "show configuration" for configuration details. For bug reporting instructions, please see: . Find the GDB manual and other documentation resources online at: . For help, type "help". Type "apropos word" to search for commands related to "word"... Reading symbols from /usr/bin/ceph-osd...Reading symbols from /usr/lib/debug/.build-id/fd/2d225baa3068ff217e8e576184a3a362e494fd.debug...done. done. [New LWP 377894] [New LWP 377892] [New LWP 377764] [New LWP 379320] [New LWP 377740] [New LWP 378586] [New LWP 378103] [New LWP 379324] [New LWP 379325] [New LWP 378105] [New LWP 379319] [New LWP 379333] [New LWP 379328] [New LWP 379326] [New LWP 379934] [New LWP 381128] [New LWP 379428] [New LWP 379429] [New LWP 381182] [New LWP 381184] [New LWP 381186] [New LWP 379882] [New LWP 379866] [New LWP 381203] [New LWP 379430] [New LWP 381204] [New LWP 381206] [New LWP 381223] [New LWP 379945] [New LWP 381226] [New LWP 381213] [New LWP 381215] [New LWP 381191] [New LWP 381195] [New LWP 379327] [New LWP 379334] [New LWP 381228] [New LWP 381192] [New LWP 381210] [New LWP 379864] [New LWP 381197] [New LWP 381196] [New LWP 381127] [New LWP 381188] [New LWP 379943] [New LWP 381194] [New LWP 381200] [New LWP 381220] [New LWP 381218] [New LWP 381205] [New LWP 381190] [New LWP 381222] [New LWP 381219] [New LWP 379936] [New LWP 381198] [New LWP 381236] [New LWP 381216] [New LWP 381193] [New LWP 381130] [New LWP 381207] [New LWP 381129] [New LWP 381234] [New LWP 381214] [New LWP 381208] [New LWP 381230] [New LWP 381227] [New LWP 381233] [New LWP 381199] [New LWP 381229] [New LWP 381224] [New LWP 381212] [New LWP 381202] [New LWP 381181] [New LWP 379947] [New LWP 381183] [New LWP 381235] [New LWP 377890] [New LWP 381232] [New LWP 381238] [New LWP 381221] [New LWP 381237] [New LWP 381131] [New LWP 381189] [New LWP 381217] [New LWP 379944] [New LWP 381201] [New LWP 381185] [New LWP 381239] [New LWP 381231] [New LWP 381225] [New LWP 381187] [New LWP 381211] [New LWP 381209] [New LWP 378073] [Thread debugging using libthread_db enabled] Using host libthread_db library "/lib/aarch64-linux-gnu/libthread_db.so.1". Core was generated by `/usr/bin/ceph-osd -f --cluster ceph --id 41 --setuser ceph --setgroup ceph'. Program terminated with signal SIGSEGV, Segmentation fault. #0 raise (sig=sig@entry=11) at ../sysdeps/unix/sysv/linux/raise.c:51 51 ../sysdeps/unix/sysv/linux/raise.c: No such file or directory. [Current thread is 1 (Thread 0xffffa3ce89f0 (LWP 377894))] (gdb) bt #0 raise (sig=sig@entry=11) at ../sysdeps/unix/sysv/linux/raise.c:51 #1 0x0000aaaae66dea1c in reraise_fatal (signum=11) at ./src/global/signal_handler.cc:326 #2 handle_fatal_signal (signum=11) at ./src/global/signal_handler.cc:326 #3 #4 __memcpy_generic () at ../sysdeps/aarch64/multiarch/../memcpy.S:105 #5 0x0000aaaae6aa5d00 in memcpy (__len=40, __src=0xaaab4a31a000, __dest=) at /usr/include/aarch64-linux-gnu/bits/string_fortified.h:34 #6 Infiniband::MemoryManager::Chunk::write (this=this@entry=0xaaab4ba39ba0, buf=buf@entry=0xaaab4a31a000 "\024\001\b", len=) at ./src/msg/async/rdma/Infiniband.cc:738 #7 0x0000aaaae6ab4018 in RDMAConnectedSocketImpl::tx_copy_chunk (this=this@entry=0xaaab4f1794e0, tx_buffers=std::vector of length 1, capacity 1 = {...}, req_copy_len=req_copy_len@entry=57, start=..., end=...) at ./src/include/buffer.h:317 #8 0x0000aaaae6ab5aec in RDMAConnectedSocketImpl::submit (this=this@entry=0xaaab4f1794e0, more=more@entry=false) at ./src/msg/async/rdma/RDMAConnectedSocketImpl.cc:380 #9 0x0000aaaae6ab7c34 in RDMAConnectedSocketImpl::send (this=0xaaab4f1794e0, bl=..., more=) at ./src/msg/async/rdma/RDMAConnectedSocketImpl.cc:297 #10 0x0000aaaae6a534d4 in ConnectedSocket::send (more=false, bl=..., this=0xaaab4a5781e8) at /usr/include/c++/9/bits/unique_ptr.h:357 #11 AsyncConnection::_try_send (this=0xaaab4a578000, more=false) at ./src/msg/async/AsyncConnection.cc:330 #12 0x0000aaaae6a96bf8 in ProtocolV2::write_event (this=0xaaab4f170680) at ./src/msg/async/ProtocolV2.cc:663 #13 0x0000aaaae6a509e8 in AsyncConnection::handle_write (this=0xaaab4a578000) at /usr/include/c++/9/bits/unique_ptr.h:357 #14 0x0000aaaae68a7570 in EventCenter::process_events (this=this@entry=0xaaaaf26a8b88, timeout_microseconds=, working_dur=working_dur@entry=0xffffa3ce8160) at /usr/include/c++/9/bits/stl_deque.h:2110 #15 0x0000aaaae68ad0f0 in NetworkStack::::operator() (__closure=0xaaaaf27184e8, __closure=0xaaaaf27184e8) at ./src/msg/async/Stack.cc:53 #16 std::_Function_handler >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/9/bits/std_function.h:300 #17 0x0000ffffa596ded4 in ?? () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #18 0x0000ffffa5b23088 in start_thread (arg=0xffffeb90520f) at pthread_create.c:463 #19 0x0000ffffa573d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 (gdb) f 6 #6 Infiniband::MemoryManager::Chunk::write (this=this@entry=0xaaab4ba39ba0, buf=buf@entry=0xaaab4a31a000 "\024\001\b", len=) at ./src/msg/async/rdma/Infiniband.cc:738 738 ./src/msg/async/rdma/Infiniband.cc: No such file or directory. (gdb) p *this $1 = {mr = 0x0, qp = 0xaaab4a4bdc20, lkey = 1246485952, bytes = 43691, offset = 1339179008, bound = 43691, buffer = 0xaaab4ba39bd0 "smtp_port", pad = {9, 0, 1886678387, 1919905887, 1233911924, 43691}, data = 0xaaab4ba39be0 "\360\233\243K\253\252"} (gdb) bt full #0 raise (sig=sig@entry=11) at ../sysdeps/unix/sysv/linux/raise.c:51 set = {__val = {4096, 281473429948200, 88, 281473429931264, 187650989138840, 187651175022592, 187651003297792, 525377, 420, 88, 281473429931264, 281473429931264, 281473429931248, 4294967288, 11758279843738548992, 187651175022592}} pid = tid = ret = #1 0x0000aaaae66dea1c in reraise_fatal (signum=11) at ./src/global/signal_handler.cc:326 ret = buf = "/var/lib/ceph/crash/2020-02-27T11:57:05.396605Z_9795c05d-2c82-4ba5-8d2e-270432f75ecd/log", '\000' ... #2 handle_fatal_signal (signum=11) at ./src/global/signal_handler.cc:326 buf = "*** Caught signal (Segmentation fault) **\n in thread ffffa3ce89f0 thread_name:msgr-worker-2\n", '\000' pthread_name = "msgr-worker-2\000\000" r = bt = {static max = 100, skip = 1, array = {0xaaaae66de954 , 0xffffa601067c <__kernel_rt_sigreturn>, 0xffffa56f01c8 <__memcpy_generic+56>, 0xaaaae6aa5d00 , 0xaaaae6ab4018 >&, unsigned long, ceph::buffer::v14_2_0::list::buffers_t::buffers_iterator&, ceph::buffer::v14_2_0::list::buffers_t::buffers_iterator const&)+192>, 0xaaaae6ab5aec , 0xaaaae6ab7c34 , 0xaaaae6a534d4 , 0xaaaae6a96bf8 , 0xaaaae6a509e8 , 0xaaaae68a7570 >*)+2992>, 0xaaaae68ad0f0 >::_M_invoke(const std::_Any_data &)+296>, 0xffffa596ded4, 0xffffa5b23088 , 0x0 }, size = 14, strings = 0xaaab4a3f9100} oss = base = "/var/lib/ceph/crash/2020-02-27T11:57:05.396605Z_9795c05d-2c82-4ba5-8d2e-270432f75ecd", '\000' #3 No symbol table info available. #4 __memcpy_generic () at ../sysdeps/aarch64/multiarch/../memcpy.S:105 No locals. #5 0x0000aaaae6aa5d00 in memcpy (__len=40, __src=0xaaab4a31a000, __dest=) at /usr/include/aarch64-linux-gnu/bits/string_fortified.h:34 No locals. #6 Infiniband::MemoryManager::Chunk::write (this=this@entry=0xaaab4ba39ba0, buf=buf@entry=0xaaab4a31a000 "\024\001\b", len=) at ./src/msg/async/rdma/Infiniband.cc:738 write_len = 40 #7 0x0000aaaae6ab4018 in RDMAConnectedSocketImpl::tx_copy_chunk (this=this@entry=0xaaab4f1794e0, tx_buffers=std::vector of length 1, capacity 1 = {...}, req_copy_len=req_copy_len@entry=57, start=..., end=...) at ./src/include/buffer.h:317 real_len = addr = 187652660895744 slice_write_len = 0 __PRETTY_FUNCTION__ = chunk_idx = 0 __func__ = "tx_copy_chunk" current_chunk = 0xaaab4ba39ba0 write_len = 0 #8 0x0000aaaae6ab5aec in RDMAConnectedSocketImpl::submit (this=this@entry=0xaaab4f1794e0, more=more@entry=false) at ./src/msg/async/rdma/RDMAConnectedSocketImpl.cc:380 ---Type to continue, or q to quit--- l = {_M_device = @0xaaab4f179598} bytes = __func__ = "submit" tx_buffers = std::vector of length 1, capacity 1 = {0xaaab4ba39ba0} it = {cur = 0xaaab4f179550} copy_start = {cur = 0xaaab4fcfc060} total_copied = 0 wait_copy_len = 57 __PRETTY_FUNCTION__ = swapped = {_buffers = {_root = {next = 0x0}, _tail = 0xaaab4f179598, _size = 281473429961616}, _carriage = 0xaaaae68bfef8 , _len = 0, _memcopy_count = 0, last_p = {> = {bl = 0xaaab4a578228, ls = 0xaaab4a578228, p = {cur = 0xaaab4a578228}, off = 0, p_off = 0}, }, static always_empty_bptr = {_raw = 0x0, _off = 0, _len = 0}, static CLAIM_DEFAULT = 0, static CLAIM_ALLOW_NONSHAREABLE = 1} #9 0x0000aaaae6ab7c34 in RDMAConnectedSocketImpl::send (this=0xaaab4f1794e0, bl=..., more=) at ./src/msg/async/rdma/RDMAConnectedSocketImpl.cc:297 bytes = 57 __func__ = "send" r = #10 0x0000aaaae6a534d4 in ConnectedSocket::send (more=false, bl=..., this=0xaaab4a5781e8) at /usr/include/c++/9/bits/unique_ptr.h:357 No locals. #11 AsyncConnection::_try_send (this=0xaaab4a578000, more=false) at ./src/msg/async/AsyncConnection.cc:330 __func__ = "_try_send" __PRETTY_FUNCTION__ = r = #12 0x0000aaaae6a96bf8 in ProtocolV2::write_event (this=0xaaab4f170680) at ./src/msg/async/ProtocolV2.cc:663 ack = {> = {> = { static SegmentsNumV = 1, segments = {_M_elems = {{_buffers = {_root = {next = 0xffffa3ce7d88}, _tail = 0xffffa3ce7d88, _size = 0}, _carriage = 0xaaaaef8857b0 , _len = 0, _memcopy_count = 0, last_p = {> = {bl = 0xffffa3ce7d88, ls = 0xffffa3ce7d88, p = {cur = 0xffffa3ce7d88}, off = 0, p_off = 0}, }, static always_empty_bptr = {_raw = 0x0, _off = 0, _len = 0}, static CLAIM_DEFAULT = 0, static CLAIM_ALLOW_NONSHAREABLE = 1}}}, static alignments = , preamble_filler = {pos = 0xaaab4a31a020 "\003"}}, _values = std::tuple containing = {[1] = 0}, features = 720575940647714820}, static tag = ceph::msgr::v2::Tag::ACK} left = start = {__d = {__r = 193307512992120}} more = __func__ = "write_event" r = #13 0x0000aaaae6a509e8 in AsyncConnection::handle_write (this=0xaaab4a578000) at /usr/include/c++/9/bits/unique_ptr.h:357 __func__ = "handle_write" #14 0x0000aaaae68a7570 in EventCenter::process_events (this=this@entry=0xaaaaf26a8b88, timeout_microseconds=, working_dur=working_dur@entry=0xffffa3ce8160) at /usr/include/c++/9/bits/stl_deque.h:2110 e = 0xaaab483c07e0 cur_process = std::deque with 1 element = {0xaaab483c07e0} tv = {tv_sec = 8, tv_usec = 972000} numevents = 8 trigger_time = true ---Type to continue, or q to quit---q Quit (gdb) (gdb) f 7 #7 0x0000aaaae6ab4018 in RDMAConnectedSocketImpl::tx_copy_chunk (this=this@entry=0xaaab4f1794e0, tx_buffers=std::vector of length 1, capacity 1 = {...}, req_copy_len=req_copy_len@entry=57, start=..., end=...) at ./src/include/buffer.h:317 317 ./src/include/buffer.h: No such file or directory. (gdb) p *this $2 = { = {_vptr.ConnectedSocketImpl = 0xaaaae7633a58 }, cct = 0xaaaaf1ae0000, qp = 0xaaab4a4bba00, peer_qpn = 58372, local_qpn = 228285, connected = 1, error = 0, ib = std::shared_ptr (use count 63, weak count 0) = {get() = 0xaaaaf1a726f0}, dispatcher = std::shared_ptr (use count 62, weak count 0) = {get() = 0xaaaaf1a84c70}, worker = 0xaaaaf26a8b00, buffers = std::vector of length 0, capacity 2, notify_fd = 143, pending_bl = {_buffers = {_root = {next = 0xaaab4fcfc060}, _tail = 0xaaab4fcfc0e0, _size = 2}, _carriage = 0xaaaaef8857b0 , _len = 57, _memcopy_count = 0, last_p = {> = {bl = 0xaaab4f179550, ls = 0xaaab4f179550, p = {cur = 0xaaab4f179550}, off = 0, p_off = 0}, }, static always_empty_bptr = {_raw = 0x0, _off = 0, _len = 0}, static CLAIM_DEFAULT = 0, static CLAIM_ALLOW_NONSHAREABLE = 1}, lock = { = {_M_mutex = pthread_mutex_t = {Type = Normal, Status = Acquired, possibly with no waiters, Owner ID = 377894, Robust = No, Shared = No, Protocol = None}}, }, wc = std::vector of length 0, capacity 0, is_server = true, con_handler = 0xaaab4a40d5e0, tcp_fd = 142, active = true, pending = false, post_backlog = 0} (gdb) p *this->ib $3 = {tx_queue_len = 1024, rx_queue_len = 4096, max_sge = 0, ib_physical_port = 1 '\001', memory_manager = 0xaaaaf1a5b4a0, srq = 0xaaaaf1a88fc8, device = 0xaaaaf1b0e400, pd = 0xaaaaf1b24368, device_list = 0xaaaaf26a7500, cct = 0xaaaaf1ae0000, lock = { = {_M_mutex = pthread_mutex_t = { Type = Normal, Status = Not acquired, Robust = No, Shared = No, Protocol = None}}, }, initialized = true, device_name = "mlx5_0", port_num = 1 '\001', support_srq = true} (gdb) p *this->ib->memory_manager $4 = {cct = 0xaaaaf1ae0000, send = 0xaaaaf1a54c30, device = 0xaaaaf1b0e400, pd = 0xaaaaf1b24368, rxbuf_pool_ctx = {perf_logger = 0xaaaaf1a73360, manager = 0xaaaaf1a5b4a0, n_bufs_allocated = 8192}, rxbuf_pool = {> = {> = {first = 0xaaab0618b2c0}, Segmentation fault root@node2:/var/log# gdb /usr/bin/ceph-osd core.msgr-worker-2.377740 GNU gdb (Ubuntu 8.1-0ubuntu3) 8.1.0.20180409-git Copyright (C) 2018 Free Software Foundation, Inc. License GPLv3+: GNU GPL version 3 or later This is free software: you are free to change and redistribute it. There is NO WARRANTY, to the extent permitted by law. Type "show copying" and "show warranty" for details. This GDB was configured as "aarch64-linux-gnu". Type "show configuration" for configuration details. For bug reporting instructions, please see: . Find the GDB manual and other documentation resources online at: . For help, type "help". Type "apropos word" to search for commands related to "word"... Reading symbols from /usr/bin/ceph-osd...Reading symbols from /usr/lib/debug/.build-id/fd/2d225baa3068ff217e8e576184a3a362e494fd.debug...done. done. [New LWP 377894] [New LWP 377892] [New LWP 377764] [New LWP 379320] [New LWP 377740] [New LWP 378586] [New LWP 378103] [New LWP 379324] [New LWP 379325] [New LWP 378105] [New LWP 379319] [New LWP 379333] [New LWP 379328] [New LWP 379326] [New LWP 379934] [New LWP 381128] [New LWP 379428] [New LWP 379429] [New LWP 381182] [New LWP 381184] [New LWP 381186] [New LWP 379882] [New LWP 379866] [New LWP 381203] [New LWP 379430] [New LWP 381204] [New LWP 381206] [New LWP 381223] [New LWP 379945] [New LWP 381226] [New LWP 381213] [New LWP 381215] [New LWP 381191] [New LWP 381195] [New LWP 379327] [New LWP 379334] [New LWP 381228] [New LWP 381192] [New LWP 381210] [New LWP 379864] [New LWP 381197] [New LWP 381196] [New LWP 381127] [New LWP 381188] [New LWP 379943] [New LWP 381194] [New LWP 381200] [New LWP 381220] [New LWP 381218] [New LWP 381205] [New LWP 381190] [New LWP 381222] [New LWP 381219] [New LWP 379936] [New LWP 381198] [New LWP 381236] [New LWP 381216] [New LWP 381193] [New LWP 381130] [New LWP 381207] [New LWP 381129] [New LWP 381234] [New LWP 381214] [New LWP 381208] [New LWP 381230] [New LWP 381227] [New LWP 381233] [New LWP 381199] [New LWP 381229] [New LWP 381224] [New LWP 381212] [New LWP 381202] [New LWP 381181] [New LWP 379947] [New LWP 381183] [New LWP 381235] [New LWP 377890] [New LWP 381232] [New LWP 381238] [New LWP 381221] [New LWP 381237] [New LWP 381131] [New LWP 381189] [New LWP 381217] [New LWP 379944] [New LWP 381201] [New LWP 381185] [New LWP 381239] [New LWP 381231] [New LWP 381225] [New LWP 381187] [New LWP 381211] [New LWP 381209] [New LWP 378073] [Thread debugging using libthread_db enabled] Using host libthread_db library "/lib/aarch64-linux-gnu/libthread_db.so.1". Core was generated by `/usr/bin/ceph-osd -f --cluster ceph --id 41 --setuser ceph --setgroup ceph'. Program terminated with signal SIGSEGV, Segmentation fault. #0 raise (sig=sig@entry=11) at ../sysdeps/unix/sysv/linux/raise.c:51 51 ../sysdeps/unix/sysv/linux/raise.c: No such file or directory. [Current thread is 1 (Thread 0xffffa3ce89f0 (LWP 377894))] (gdb) bt #0 raise (sig=sig@entry=11) at ../sysdeps/unix/sysv/linux/raise.c:51 #1 0x0000aaaae66dea1c in reraise_fatal (signum=11) at ./src/global/signal_handler.cc:326 #2 handle_fatal_signal (signum=11) at ./src/global/signal_handler.cc:326 #3 #4 __memcpy_generic () at ../sysdeps/aarch64/multiarch/../memcpy.S:105 #5 0x0000aaaae6aa5d00 in memcpy (__len=40, __src=0xaaab4a31a000, __dest=) at /usr/include/aarch64-linux-gnu/bits/string_fortified.h:34 #6 Infiniband::MemoryManager::Chunk::write (this=this@entry=0xaaab4ba39ba0, buf=buf@entry=0xaaab4a31a000 "\024\001\b", len=) at ./src/msg/async/rdma/Infiniband.cc:738 #7 0x0000aaaae6ab4018 in RDMAConnectedSocketImpl::tx_copy_chunk (this=this@entry=0xaaab4f1794e0, tx_buffers=std::vector of length 1, capacity 1 = {...}, req_copy_len=req_copy_len@entry=57, start=..., end=...) at ./src/include/buffer.h:317 #8 0x0000aaaae6ab5aec in RDMAConnectedSocketImpl::submit (this=this@entry=0xaaab4f1794e0, more=more@entry=false) at ./src/msg/async/rdma/RDMAConnectedSocketImpl.cc:380 #9 0x0000aaaae6ab7c34 in RDMAConnectedSocketImpl::send (this=0xaaab4f1794e0, bl=..., more=) at ./src/msg/async/rdma/RDMAConnectedSocketImpl.cc:297 #10 0x0000aaaae6a534d4 in ConnectedSocket::send (more=false, bl=..., this=0xaaab4a5781e8) at /usr/include/c++/9/bits/unique_ptr.h:357 #11 AsyncConnection::_try_send (this=0xaaab4a578000, more=false) at ./src/msg/async/AsyncConnection.cc:330 #12 0x0000aaaae6a96bf8 in ProtocolV2::write_event (this=0xaaab4f170680) at ./src/msg/async/ProtocolV2.cc:663 #13 0x0000aaaae6a509e8 in AsyncConnection::handle_write (this=0xaaab4a578000) at /usr/include/c++/9/bits/unique_ptr.h:357 #14 0x0000aaaae68a7570 in EventCenter::process_events (this=this@entry=0xaaaaf26a8b88, timeout_microseconds=, working_dur=working_dur@entry=0xffffa3ce8160) at /usr/include/c++/9/bits/stl_deque.h:2110 #15 0x0000aaaae68ad0f0 in NetworkStack::::operator() (__closure=0xaaaaf27184e8, __closure=0xaaaaf27184e8) at ./src/msg/async/Stack.cc:53 #16 std::_Function_handler >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/9/bits/std_function.h:300 #17 0x0000ffffa596ded4 in ?? () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #18 0x0000ffffa5b23088 in start_thread (arg=0xffffeb90520f) at pthread_create.c:463 #19 0x0000ffffa573d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 (gdb) f 7 #7 0x0000aaaae6ab4018 in RDMAConnectedSocketImpl::tx_copy_chunk (this=this@entry=0xaaab4f1794e0, tx_buffers=std::vector of length 1, capacity 1 = {...}, req_copy_len=req_copy_len@entry=57, start=..., end=...) at ./src/include/buffer.h:317 317 ./src/include/buffer.h: No such file or directory. (gdb) bt #0 raise (sig=sig@entry=11) at ../sysdeps/unix/sysv/linux/raise.c:51 #1 0x0000aaaae66dea1c in reraise_fatal (signum=11) at ./src/global/signal_handler.cc:326 #2 handle_fatal_signal (signum=11) at ./src/global/signal_handler.cc:326 #3 #4 __memcpy_generic () at ../sysdeps/aarch64/multiarch/../memcpy.S:105 #5 0x0000aaaae6aa5d00 in memcpy (__len=40, __src=0xaaab4a31a000, __dest=) at /usr/include/aarch64-linux-gnu/bits/string_fortified.h:34 #6 Infiniband::MemoryManager::Chunk::write (this=this@entry=0xaaab4ba39ba0, buf=buf@entry=0xaaab4a31a000 "\024\001\b", len=) at ./src/msg/async/rdma/Infiniband.cc:738 #7 0x0000aaaae6ab4018 in RDMAConnectedSocketImpl::tx_copy_chunk (this=this@entry=0xaaab4f1794e0, tx_buffers=std::vector of length 1, capacity 1 = {...}, req_copy_len=req_copy_len@entry=57, start=..., end=...) at ./src/include/buffer.h:317 #8 0x0000aaaae6ab5aec in RDMAConnectedSocketImpl::submit (this=this@entry=0xaaab4f1794e0, more=more@entry=false) at ./src/msg/async/rdma/RDMAConnectedSocketImpl.cc:380 #9 0x0000aaaae6ab7c34 in RDMAConnectedSocketImpl::send (this=0xaaab4f1794e0, bl=..., more=) at ./src/msg/async/rdma/RDMAConnectedSocketImpl.cc:297 #10 0x0000aaaae6a534d4 in ConnectedSocket::send (more=false, bl=..., this=0xaaab4a5781e8) at /usr/include/c++/9/bits/unique_ptr.h:357 #11 AsyncConnection::_try_send (this=0xaaab4a578000, more=false) at ./src/msg/async/AsyncConnection.cc:330 #12 0x0000aaaae6a96bf8 in ProtocolV2::write_event (this=0xaaab4f170680) at ./src/msg/async/ProtocolV2.cc:663 #13 0x0000aaaae6a509e8 in AsyncConnection::handle_write (this=0xaaab4a578000) at /usr/include/c++/9/bits/unique_ptr.h:357 #14 0x0000aaaae68a7570 in EventCenter::process_events (this=this@entry=0xaaaaf26a8b88, timeout_microseconds=, working_dur=working_dur@entry=0xffffa3ce8160) at /usr/include/c++/9/bits/stl_deque.h:2110 #15 0x0000aaaae68ad0f0 in NetworkStack::::operator() (__closure=0xaaaaf27184e8, __closure=0xaaaaf27184e8) at ./src/msg/async/Stack.cc:53 #16 std::_Function_handler >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/9/bits/std_function.h:300 #17 0x0000ffffa596ded4 in ?? () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #18 0x0000ffffa5b23088 in start_thread (arg=0xffffeb90520f) at pthread_create.c:463 #19 0x0000ffffa573d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 (gdb) p this->ib $1 = std::shared_ptr (use count 63, weak count 0) = {get() = 0xaaaaf1a726f0} (gdb) p this->ib-> _M_get _M_ptr __shared_ptr operator bool operator-> reset swap use_count ~shared_ptr _M_get_deleter _M_refcount get operator* operator= shared_ptr unique ~__shared_ptr (gdb) p *this->ib $2 = {tx_queue_len = 1024, rx_queue_len = 4096, max_sge = 0, ib_physical_port = 1 '\001', memory_manager = 0xaaaaf1a5b4a0, srq = 0xaaaaf1a88fc8, device = 0xaaaaf1b0e400, pd = 0xaaaaf1b24368, device_list = 0xaaaaf26a7500, cct = 0xaaaaf1ae0000, lock = { = {_M_mutex = pthread_mutex_t = { Type = Normal, Status = Not acquired, Robust = No, Shared = No, Protocol = None}}, }, initialized = true, device_name = "mlx5_0", port_num = 1 '\001', support_srq = true} (gdb) p this->ib->memory_manager $3 = (Infiniband::MemoryManager *) 0xaaaaf1a5b4a0 (gdb) p this->ib->memory_manager->send $4 = (Infiniband::MemoryManager::Cluster *) 0xaaaaf1a54c30 (gdb) p *this->ib->memory_manager->send $5 = {manager = @0xaaaaf1a5b4a0, buffer_size = 131072, num_chunk = 2048, lock = { = {_M_mutex = pthread_mutex_t = {Type = Normal, Status = Not acquired, Robust = No, Shared = No, Protocol = None}}, }, free_chunks = std::vector of length 2048, capacity 4096 = { 0xaaab027d6000, 0xaaab027d6040, 0xaaab027d6080, 0xaaab027d60c0, 0xaaab027d6100, 0xaaab027d6140, 0xaaab027d6180, 0xaaab027d61c0, 0xaaab027d6200, 0xaaab027d6240, 0xaaab027d6280, 0xaaab027d62c0, 0xaaab027d6300, 0xaaab027d6340, 0xaaab027d6380, 0xaaab027d63c0, 0xaaab027d6400, 0xaaab027d6440, 0xaaab027d6480, 0xaaab027d64c0, 0xaaab027d6500, 0xaaab027d6540, 0xaaab027d6580, 0xaaab027d65c0, 0xaaab027d6600, 0xaaab027d6640, 0xaaab027d6680, 0xaaab027d66c0, 0xaaab027d6700, 0xaaab027d6740, 0xaaab027d6780, 0xaaab027d67c0, 0xaaab027d6800, 0xaaab027d6840, 0xaaab027d6880, 0xaaab027d68c0, 0xaaab027d6900, 0xaaab027d6940, 0xaaab027d6980, 0xaaab027d69c0, 0xaaab027d6a00, 0xaaab027d6a40, 0xaaab027d6a80, 0xaaab027d6ac0, 0xaaab027d6b00, 0xaaab027d6b40, 0xaaab027d6b80, 0xaaab027d6bc0, 0xaaab027d6c00, 0xaaab027d6c40, 0xaaab027d6c80, 0xaaab027d6cc0, 0xaaab027d6d00, 0xaaab027d6d40, 0xaaab027d6d80, 0xaaab027d6dc0, 0xaaab027d6e00, 0xaaab027d6e40, 0xaaab027d6e80, 0xaaab027d6ec0, 0xaaab027d6f00, 0xaaab027d6f40, 0xaaab027d6f80, 0xaaab027d6fc0, 0xaaab027d7000, 0xaaab027d7040, 0xaaab027d7080, 0xaaab027d70c0, 0xaaab027d7100, 0xaaab027d7140, 0xaaab027d7180, 0xaaab027d71c0, 0xaaab027d7200, 0xaaab027d7240, 0xaaab027d7280, 0xaaab027d72c0, 0xaaab027d7300, 0xaaab027d7340, 0xaaab027d7380, 0xaaab027d73c0, 0xaaab027d7400, 0xaaab027d7440, 0xaaab027d7480, 0xaaab027d74c0, 0xaaab027d7500, 0xaaab027d7540, 0xaaab027d7580, 0xaaab027d75c0, 0xaaab027d7600, 0xaaab027d7640, 0xaaab027d7680, 0xaaab027d76c0, 0xaaab027d7700, 0xaaab027d7740, 0xaaab027d7780, 0xaaab027d77c0, 0xaaab027d7800, 0xaaab027d7840, 0xaaab027d7880, 0xaaab027d78c0, 0xaaab027d7900, 0xaaab027d7940, 0xaaab027d7980, 0xaaab027d79c0, 0xaaab027d7a00, 0xaaab027d7a40, 0xaaab027d7a80, 0xaaab027d7ac0, 0xaaab027d7b00, 0xaaab027d7b40, 0xaaab027d7b80, 0xaaab027d7bc0, 0xaaab027d7c00, 0xaaab027d7c40, 0xaaab027d7c80, 0xaaab027d7cc0, 0xaaab027d7d00, 0xaaab027d7d40, 0xaaab027d7d80, 0xaaab027d7dc0, 0xaaab027d7e00, 0xaaab027d7e40, 0xaaab027d7e80, 0xaaab027d7ec0, 0xaaab027d7f00, 0xaaab027d7f40, 0xaaab027d7f80, 0xaaab027d7fc0, 0xaaab027d8000, 0xaaab027d8040, 0xaaab027d8080, 0xaaab027d80c0, 0xaaab027d8100, 0xaaab027d8140, 0xaaab027d8180, 0xaaab027d81c0, 0xaaab027d8200, 0xaaab027d8240, 0xaaab027d8280, 0xaaab027d82c0, 0xaaab027d8300, 0xaaab027d8340, 0xaaab027d8380, 0xaaab027d83c0, 0xaaab027d8400, 0xaaab027d8440, 0xaaab027d8480, 0xaaab027d84c0, 0xaaab027d8500, 0xaaab027d8540, 0xaaab027d8580, 0xaaab027d85c0, 0xaaab027d8600, 0xaaab027d8640, 0xaaab027d8680, 0xaaab027d86c0, 0xaaab027d8700, 0xaaab027d8740, 0xaaab027d8780, 0xaaab027d87c0, 0xaaab027d8800, 0xaaab027d8840, 0xaaab027d8880, 0xaaab027d88c0, 0xaaab027d8900, 0xaaab027d8940, 0xaaab027d8980, 0xaaab027d89c0, 0xaaab027d8a00, 0xaaab027d8a40, 0xaaab027d8a80, 0xaaab027d8ac0, 0xaaab027d8b00, 0xaaab027d8b40, 0xaaab027d8b80, 0xaaab027d8bc0, 0xaaab027d8c00, 0xaaab027d8c40, 0xaaab027d8c80, 0xaaab027d8cc0, 0xaaab027d8d00, 0xaaab027d8d40, 0xaaab027d8d80, 0xaaab027d8dc0, 0xaaab027d8e00, 0xaaab027d8e40, 0xaaab027d8e80, 0xaaab027d8ec0, 0xaaab027d8f00, 0xaaab027d8f40, 0xaaab027d8f80, 0xaaab027d8fc0, 0xaaab027d9000, 0xaaab027d9040, 0xaaab027d9080, 0xaaab027d90c0, 0xaaab027d9100, 0xaaab027d9140, 0xaaab027d9180, 0xaaab027d91c0...}, base = 0xaaaaf27d6000 "", end = 0xaaab027d6000 "@", , chunk_base = 0xaaab027d6000} (gdb) p *this->ib->memory_manager->send->free_chunks[0] $6 = {mr = 0xaaaaf1a6dd40, qp = 0x0, lkey = 18513, bytes = 131072, offset = 0, bound = 131072, buffer = 0xaaaaf27d6000 "", pad = {0, 0, 0, 0, 0, 0}, data = 0xaaab027d6040 "@", } (gdb) p *this->ib->memory_manager->send->free_chunks[1] $7 = {mr = 0xaaaaf1a6dd40, qp = 0x0, lkey = 18513, bytes = 131072, offset = 0, bound = 131072, buffer = 0xaaaaf27f6000 "", pad = {0, 0, 0, 0, 0, 0}, data = 0xaaab027d6080 "@", } (gdb) p cu cu_data currency_symbol current_function.hpp current_rtmin cuse_lowlevel_new cuserid cur current current_maxid curshell cuse_lowlevel_setup cuserid.c cur_qp_state current_chunk current_rtmax cuse_lowlevel_main cuse_lowlevel_teardown custom_reaction.hpp (gdb) p cu cu_data currency_symbol current_function.hpp current_rtmin cuse_lowlevel_new cuserid cur current current_maxid curshell cuse_lowlevel_setup cuserid.c cur_qp_state current_chunk current_rtmax cuse_lowlevel_main cuse_lowlevel_teardown custom_reaction.hpp (gdb) p current_chunk $8 = (RDMAConnectedSocketImpl::Chunk *) 0xaaab4ba39ba0 (gdb) p *current_chunk $9 = {mr = 0x0, qp = 0xaaab4a4bdc20, lkey = 1246485952, bytes = 43691, offset = 1339179008, bound = 43691, buffer = 0xaaab4ba39bd0 "smtp_port", pad = {9, 0, 1886678387, 1919905887, 1233911924, 43691}, data = 0xaaab4ba39be0 "\360\233\243K\253\252"} (gdb) p 0xaaab4ba39ba0-0xaaab027d6000 $10 = 1227242400 (gdb) p (0xaaab4ba39ba0-0xaaab027d6000)/64 $11 = 19175662 (gdb) $12 = 19175662 (gdb) $13 = 19175662 (gdb) p 131072*2048 $14 = 268435456 (gdb) p /x 131072*2048 $15 = 0x10000000 (gdb) p 0xaaab027d6000+4096*64 $16 = 187651458162688 (gdb) p /x 0xaaab027d6000+4096*64 $17 = 0xaaab02816000 (gdb) p (RDMAConnectedSocketImpl::Chunk *)0xaaab02839ba0 $18 = (RDMAConnectedSocketImpl::Chunk *) 0xaaab02839ba0 (gdb) p *(RDMAConnectedSocketImpl::Chunk *)0xaaab02839ba0 $19 = {mr = 0x3d6e203130363031, qp = 0x383d636520393135, lkey = 792277556, bytes = 959591480, offset = 1936288800, bound = 826106671, buffer = 0x3630312f30303630 , pad = {1814048816, 1664054117, 826107439, 825243184, 909127983, 808399152}, data = 0xaaab02839be0 " sis=10624) [66,41] r=1 lpr=10624 pi=[10600,10624)/1 crt=10343'341492 lcod 0'0 mlcod 0'0 unknown NOTIFY mbc={}] start_peering_interval up [66] -> [66,41], acting [66] -> [66,41], acting_primary 66 -> "...} (gdb) bt #0 raise (sig=sig@entry=11) at ../sysdeps/unix/sysv/linux/raise.c:51 #1 0x0000aaaae66dea1c in reraise_fatal (signum=11) at ./src/global/signal_handler.cc:326 #2 handle_fatal_signal (signum=11) at ./src/global/signal_handler.cc:326 #3 #4 __memcpy_generic () at ../sysdeps/aarch64/multiarch/../memcpy.S:105 #5 0x0000aaaae6aa5d00 in memcpy (__len=40, __src=0xaaab4a31a000, __dest=) at /usr/include/aarch64-linux-gnu/bits/string_fortified.h:34 #6 Infiniband::MemoryManager::Chunk::write (this=this@entry=0xaaab4ba39ba0, buf=buf@entry=0xaaab4a31a000 "\024\001\b", len=) at ./src/msg/async/rdma/Infiniband.cc:738 #7 0x0000aaaae6ab4018 in RDMAConnectedSocketImpl::tx_copy_chunk (this=this@entry=0xaaab4f1794e0, tx_buffers=std::vector of length 1, capacity 1 = {...}, req_copy_len=req_copy_len@entry=57, start=..., end=...) at ./src/include/buffer.h:317 #8 0x0000aaaae6ab5aec in RDMAConnectedSocketImpl::submit (this=this@entry=0xaaab4f1794e0, more=more@entry=false) at ./src/msg/async/rdma/RDMAConnectedSocketImpl.cc:380 #9 0x0000aaaae6ab7c34 in RDMAConnectedSocketImpl::send (this=0xaaab4f1794e0, bl=..., more=) at ./src/msg/async/rdma/RDMAConnectedSocketImpl.cc:297 #10 0x0000aaaae6a534d4 in ConnectedSocket::send (more=false, bl=..., this=0xaaab4a5781e8) at /usr/include/c++/9/bits/unique_ptr.h:357 #11 AsyncConnection::_try_send (this=0xaaab4a578000, more=false) at ./src/msg/async/AsyncConnection.cc:330 #12 0x0000aaaae6a96bf8 in ProtocolV2::write_event (this=0xaaab4f170680) at ./src/msg/async/ProtocolV2.cc:663 #13 0x0000aaaae6a509e8 in AsyncConnection::handle_write (this=0xaaab4a578000) at /usr/include/c++/9/bits/unique_ptr.h:357 #14 0x0000aaaae68a7570 in EventCenter::process_events (this=this@entry=0xaaaaf26a8b88, timeout_microseconds=, working_dur=working_dur@entry=0xffffa3ce8160) at /usr/include/c++/9/bits/stl_deque.h:2110 #15 0x0000aaaae68ad0f0 in NetworkStack::::operator() (__closure=0xaaaaf27184e8, __closure=0xaaaaf27184e8) at ./src/msg/async/Stack.cc:53 #16 std::_Function_handler >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/9/bits/std_function.h:300 #17 0x0000ffffa596ded4 in ?? () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #18 0x0000ffffa5b23088 in start_thread (arg=0xffffeb90520f) at pthread_create.c:463 #19 0x0000ffffa573d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 (gdb) f 7 #7 0x0000aaaae6ab4018 in RDMAConnectedSocketImpl::tx_copy_chunk (this=this@entry=0xaaab4f1794e0, tx_buffers=std::vector of length 1, capacity 1 = {...}, req_copy_len=req_copy_len@entry=57, start=..., end=...) at ./src/include/buffer.h:317 317 in ./src/include/buffer.h (gdb) p tx_buffers[0] $20 = (Infiniband::MemoryManager::Chunk *) 0xaaab4ba39ba0 (gdb)