Project

General

Profile

invalid_chunk.txt

chunsong feng, 02/28/2020 07:41 AM

Download (117 KB)

 
1

    
2
root@node2:/var/log# ll core.msgr-worker-*
3
-rw------- 1 root ceph 2538639360 Feb 27 19:57 core.msgr-worker-0.377736
4
-rw------- 1 root ceph 2537709568 Feb 27 19:57 core.msgr-worker-1.377732
5
-rw------- 1 root ceph 2564624384 Feb 27 19:57 core.msgr-worker-1.377734
6
-rw------- 1 root ceph 2512400384 Feb 27 19:57 core.msgr-worker-1.377742
7
-rw------- 1 root ceph 2469797888 Feb 27 19:57 core.msgr-worker-2.377725
8
-rw------- 1 root ceph 2515800064 Feb 27 19:57 core.msgr-worker-2.377738
9
-rw------- 1 root ceph 2513207296 Feb 27 19:57 core.msgr-worker-2.377740
10
root@node2:/var/log# gdb /usr/bin/ceph-osd core.msgr-worker-2.377736
11
GNU gdb (Ubuntu 8.1-0ubuntu3) 8.1.0.20180409-git
12
Copyright (C) 2018 Free Software Foundation, Inc.
13
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
14
This is free software: you are free to change and redistribute it.
15
There is NO WARRANTY, to the extent permitted by law.  Type "show copying"
16
and "show warranty" for details.
17
This GDB was configured as "aarch64-linux-gnu".
18
Type "show configuration" for configuration details.
19
For bug reporting instructions, please see:
20
<http://www.gnu.org/software/gdb/bugs/>.
21
Find the GDB manual and other documentation resources online at:
22
<http://www.gnu.org/software/gdb/documentation/>.
23
For help, type "help".
24
Type "apropos word" to search for commands related to "word"...
25
Reading symbols from /usr/bin/ceph-osd...Reading symbols from /usr/lib/debug/.build-id/fd/2d225baa3068ff217e8e576184a3a362e494fd.debug...done.
26
done.
27
/var/log/core.msgr-worker-2.377736: No such file or directory.
28
(gdb) bt
29
No stack.
30
(gdb) quit
31
root@node2:/var/log# gdb /usr/bin/ceph-osd core.msgr-worker-0.377736
32
GNU gdb (Ubuntu 8.1-0ubuntu3) 8.1.0.20180409-git
33
Copyright (C) 2018 Free Software Foundation, Inc.
34
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
35
This is free software: you are free to change and redistribute it.
36
There is NO WARRANTY, to the extent permitted by law.  Type "show copying"
37
and "show warranty" for details.
38
This GDB was configured as "aarch64-linux-gnu".
39
Type "show configuration" for configuration details.
40
For bug reporting instructions, please see:
41
<http://www.gnu.org/software/gdb/bugs/>.
42
Find the GDB manual and other documentation resources online at:
43
<http://www.gnu.org/software/gdb/documentation/>.
44
For help, type "help".
45
Type "apropos word" to search for commands related to "word"...
46
Reading symbols from /usr/bin/ceph-osd...Reading symbols from /usr/lib/debug/.build-id/fd/2d225baa3068ff217e8e576184a3a362e494fd.debug...done.
47
done.
48
[New LWP 377856]
49
[New LWP 377736]
50
[New LWP 377760]
51
[New LWP 377988]
52
[New LWP 377857]
53
[New LWP 377858]
54
[New LWP 377990]
55
[New LWP 377989]
56
[New LWP 378362]
57
[New LWP 378922]
58
[New LWP 378923]
59
[New LWP 378930]
60
[New LWP 378931]
61
[New LWP 378934]
62
[New LWP 378939]
63
[New LWP 379507]
64
[New LWP 380985]
65
[New LWP 380991]
66
[New LWP 380995]
67
[New LWP 379543]
68
[New LWP 378954]
69
[New LWP 378940]
70
[New LWP 378933]
71
[New LWP 378956]
72
[New LWP 378932]
73
[New LWP 381006]
74
[New LWP 381010]
75
[New LWP 381021]
76
[New LWP 379509]
77
[New LWP 379540]
78
[New LWP 379545]
79
[New LWP 381032]
80
[New LWP 381042]
81
[New LWP 381029]
82
[New LWP 380987]
83
[New LWP 379546]
84
[New LWP 380998]
85
[New LWP 381016]
86
[New LWP 378955]
87
[New LWP 381001]
88
[New LWP 381024]
89
[New LWP 380996]
90
[New LWP 380994]
91
[New LWP 379539]
92
[New LWP 380990]
93
[New LWP 380992]
94
[New LWP 381022]
95
[New LWP 380999]
96
[New LWP 381031]
97
[New LWP 380986]
98
[New LWP 381028]
99
[New LWP 380984]
100
[New LWP 381005]
101
[New LWP 380988]
102
[New LWP 379516]
103
[New LWP 381040]
104
[New LWP 381004]
105
[New LWP 381019]
106
[New LWP 381023]
107
[New LWP 380989]
108
[New LWP 381013]
109
[New LWP 381000]
110
[New LWP 381008]
111
[New LWP 381045]
112
[New LWP 381002]
113
[New LWP 381003]
114
[New LWP 381033]
115
[New LWP 381014]
116
[New LWP 381007]
117
[New LWP 381017]
118
[New LWP 381020]
119
[New LWP 381009]
120
[New LWP 381037]
121
[New LWP 381044]
122
[New LWP 381015]
123
[New LWP 381012]
124
[New LWP 381046]
125
[New LWP 379544]
126
[New LWP 381034]
127
[New LWP 381025]
128
[New LWP 380997]
129
[New LWP 381038]
130
[New LWP 381027]
131
[New LWP 381039]
132
[New LWP 381018]
133
[New LWP 381035]
134
[New LWP 381026]
135
[New LWP 380993]
136
[New LWP 381041]
137
[New LWP 381011]
138
[New LWP 381047]
139
[New LWP 381043]
140
[New LWP 381030]
141
[New LWP 381036]
142
[Thread debugging using libthread_db enabled]
143
Using host libthread_db library "/lib/aarch64-linux-gnu/libthread_db.so.1".
144
Core was generated by `/usr/bin/ceph-osd -f --cluster ceph --id 37 --setuser ceph --setgroup ceph'.
145
Program terminated with signal SIGSEGV, Segmentation fault.
146
#0  raise (sig=sig@entry=11) at ../sysdeps/unix/sysv/linux/raise.c:51
147
51      ../sysdeps/unix/sysv/linux/raise.c: No such file or directory.
148
[Current thread is 1 (Thread 0xffffa10bf9f0 (LWP 377856))]
149
(gdb) bt
150
#0  raise (sig=sig@entry=11) at ../sysdeps/unix/sysv/linux/raise.c:51
151
#1  0x0000aaaabc1bba1c in reraise_fatal (signum=11) at ./src/global/signal_handler.cc:326
152
#2  handle_fatal_signal (signum=11) at ./src/global/signal_handler.cc:326
153
#3  <signal handler called>
154
#4  __memcpy_generic () at ../sysdeps/aarch64/multiarch/../memcpy.S:105
155
#5  0x0000aaaabc582d00 in memcpy (__len=52, __src=0xaaab4e327000, __dest=<optimized out>) at /usr/include/aarch64-linux-gnu/bits/string_fortified.h:34
156
#6  Infiniband::MemoryManager::Chunk::write (this=this@entry=0xaaab5560f6c0, buf=buf@entry=0xaaab4e327000 "\002", <incomplete sequence \302>, len=<optimized out>)
157
    at ./src/msg/async/rdma/Infiniband.cc:738
158
#7  0x0000aaaabc591018 in RDMAConnectedSocketImpl::tx_copy_chunk (this=this@entry=0xaaab53527d40, tx_buffers=std::vector of length 1, capacity 1 = {...},
159
    req_copy_len=req_copy_len@entry=243, start=..., end=...) at ./src/include/buffer.h:317
160
#8  0x0000aaaabc592aec in RDMAConnectedSocketImpl::submit (this=this@entry=0xaaab53527d40, more=more@entry=false)
161
    at ./src/msg/async/rdma/RDMAConnectedSocketImpl.cc:380
162
#9  0x0000aaaabc594c34 in RDMAConnectedSocketImpl::send (this=0xaaab53527d40, bl=..., more=<optimized out>) at ./src/msg/async/rdma/RDMAConnectedSocketImpl.cc:297
163
#10 0x0000aaaabc5304d4 in ConnectedSocket::send (more=false, bl=..., this=0xaaab575d8ee8) at /usr/include/c++/9/bits/unique_ptr.h:357
164
#11 AsyncConnection::_try_send (this=this@entry=0xaaab575d8d00, more=more@entry=false) at ./src/msg/async/AsyncConnection.cc:330
165
#12 0x0000aaaabc530ec4 in AsyncConnection::write(ceph::buffer::v14_2_0::list&, std::function<void (long)>, bool) (this=this@entry=0xaaab575d8d00, bl=...,
166
    callback=..., more=more@entry=false) at ./src/msg/async/AsyncConnection.cc:309
167
#13 0x0000aaaabc55eee0 in ProtocolV2::write (this=this@entry=0xaaab5350cc00, desc="auth request", next=..., buffer=...) at /usr/include/c++/9/bits/std_function.h:87
168
#14 0x0000aaaabc5679b0 in ProtocolV2::write<ceph::msgr::v2::AuthRequestFrame> (frame=..., next=..., desc="auth request", this=0xaaab5350cc00)
169
    at ./src/msg/async/ProtocolV2.cc:760
170
#15 ProtocolV2::send_auth_request (this=this@entry=0xaaab5350cc00, allowed_methods=std::vector of length 0, capacity 0) at ./src/msg/async/ProtocolV2.cc:1763
171
#16 0x0000aaaabc567ea8 in ProtocolV2::send_auth_request (this=0xaaab5350cc00) at ./src/msg/async/ProtocolV2.h:215
172
#17 ProtocolV2::post_client_banner_exchange (this=0xaaab5350cc00) at ./src/msg/async/ProtocolV2.cc:1731
173
#18 0x0000aaaabc55e770 in ProtocolV2::run_continuation (this=0xaaab5350cc00, continuation=...) at ./src/msg/async/ProtocolV2.cc:45
174
#19 0x0000aaaabc532f40 in std::function<void (char*, long)>::operator()(char*, long) const (__args#1=0, __args#0=0xaaab534aa460 "ceph v2\n\020",
175
    this=0xaaab575d9110) at /usr/include/c++/9/bits/std_function.h:685
176
#20 AsyncConnection::process (this=0xaaab575d8d00) at ./src/msg/async/AsyncConnection.cc:457
177
#21 0x0000aaaabc383edc in EventCenter::process_events (this=this@entry=0xaaaafe0ee608, timeout_microseconds=<optimized out>,
178
    working_dur=working_dur@entry=0xffffa10bf160) at /usr/include/c++/9/bits/basic_ios.h:282
179
#22 0x0000aaaabc38a0f0 in NetworkStack::<lambda()>::operator() (__closure=0xaaaafe15e488, __closure=0xaaaafe15e488) at ./src/msg/async/Stack.cc:53
180
#23 std::_Function_handler<void(), NetworkStack::add_thread(unsigned int)::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...)
181
    at /usr/include/c++/9/bits/std_function.h:300
182
#24 0x0000ffffa1d42ed4 in ?? () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
183
#25 0x0000ffffa1ef8088 in start_thread (arg=0xffffd84b1edf) at pthread_create.c:463
184
#26 0x0000ffffa1b124ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
185
(gdb) f 6
186
#6  Infiniband::MemoryManager::Chunk::write (this=this@entry=0xaaab5560f6c0, buf=buf@entry=0xaaab4e327000 "\002", <incomplete sequence \302>, len=<optimized out>)
187
    at ./src/msg/async/rdma/Infiniband.cc:738
188
738     ./src/msg/async/rdma/Infiniband.cc: No such file or directory.
189
(gdb) bt
190
#0  raise (sig=sig@entry=11) at ../sysdeps/unix/sysv/linux/raise.c:51
191
#1  0x0000aaaabc1bba1c in reraise_fatal (signum=11) at ./src/global/signal_handler.cc:326
192
#2  handle_fatal_signal (signum=11) at ./src/global/signal_handler.cc:326
193
#3  <signal handler called>
194
#4  __memcpy_generic () at ../sysdeps/aarch64/multiarch/../memcpy.S:105
195
#5  0x0000aaaabc582d00 in memcpy (__len=52, __src=0xaaab4e327000, __dest=<optimized out>) at /usr/include/aarch64-linux-gnu/bits/string_fortified.h:34
196
#6  Infiniband::MemoryManager::Chunk::write (this=this@entry=0xaaab5560f6c0, buf=buf@entry=0xaaab4e327000 "\002", <incomplete sequence \302>, len=<optimized out>)
197
    at ./src/msg/async/rdma/Infiniband.cc:738
198
#7  0x0000aaaabc591018 in RDMAConnectedSocketImpl::tx_copy_chunk (this=this@entry=0xaaab53527d40, tx_buffers=std::vector of length 1, capacity 1 = {...},
199
    req_copy_len=req_copy_len@entry=243, start=..., end=...) at ./src/include/buffer.h:317
200
#8  0x0000aaaabc592aec in RDMAConnectedSocketImpl::submit (this=this@entry=0xaaab53527d40, more=more@entry=false)
201
    at ./src/msg/async/rdma/RDMAConnectedSocketImpl.cc:380
202
#9  0x0000aaaabc594c34 in RDMAConnectedSocketImpl::send (this=0xaaab53527d40, bl=..., more=<optimized out>) at ./src/msg/async/rdma/RDMAConnectedSocketImpl.cc:297
203
#10 0x0000aaaabc5304d4 in ConnectedSocket::send (more=false, bl=..., this=0xaaab575d8ee8) at /usr/include/c++/9/bits/unique_ptr.h:357
204
#11 AsyncConnection::_try_send (this=this@entry=0xaaab575d8d00, more=more@entry=false) at ./src/msg/async/AsyncConnection.cc:330
205
#12 0x0000aaaabc530ec4 in AsyncConnection::write(ceph::buffer::v14_2_0::list&, std::function<void (long)>, bool) (this=this@entry=0xaaab575d8d00, bl=...,
206
    callback=..., more=more@entry=false) at ./src/msg/async/AsyncConnection.cc:309
207
#13 0x0000aaaabc55eee0 in ProtocolV2::write (this=this@entry=0xaaab5350cc00, desc="auth request", next=..., buffer=...) at /usr/include/c++/9/bits/std_function.h:87
208
#14 0x0000aaaabc5679b0 in ProtocolV2::write<ceph::msgr::v2::AuthRequestFrame> (frame=..., next=..., desc="auth request", this=0xaaab5350cc00)
209
    at ./src/msg/async/ProtocolV2.cc:760
210
#15 ProtocolV2::send_auth_request (this=this@entry=0xaaab5350cc00, allowed_methods=std::vector of length 0, capacity 0) at ./src/msg/async/ProtocolV2.cc:1763
211
#16 0x0000aaaabc567ea8 in ProtocolV2::send_auth_request (this=0xaaab5350cc00) at ./src/msg/async/ProtocolV2.h:215
212
#17 ProtocolV2::post_client_banner_exchange (this=0xaaab5350cc00) at ./src/msg/async/ProtocolV2.cc:1731
213
#18 0x0000aaaabc55e770 in ProtocolV2::run_continuation (this=0xaaab5350cc00, continuation=...) at ./src/msg/async/ProtocolV2.cc:45
214
#19 0x0000aaaabc532f40 in std::function<void (char*, long)>::operator()(char*, long) const (__args#1=0, __args#0=0xaaab534aa460 "ceph v2\n\020",
215
    this=0xaaab575d9110) at /usr/include/c++/9/bits/std_function.h:685
216
#20 AsyncConnection::process (this=0xaaab575d8d00) at ./src/msg/async/AsyncConnection.cc:457
217
#21 0x0000aaaabc383edc in EventCenter::process_events (this=this@entry=0xaaaafe0ee608, timeout_microseconds=<optimized out>,
218
    working_dur=working_dur@entry=0xffffa10bf160) at /usr/include/c++/9/bits/basic_ios.h:282
219
#22 0x0000aaaabc38a0f0 in NetworkStack::<lambda()>::operator() (__closure=0xaaaafe15e488, __closure=0xaaaafe15e488) at ./src/msg/async/Stack.cc:53
220
#23 std::_Function_handler<void(), NetworkStack::add_thread(unsigned int)::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...)
221
    at /usr/include/c++/9/bits/std_function.h:300
222
#24 0x0000ffffa1d42ed4 in ?? () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
223
#25 0x0000ffffa1ef8088 in start_thread (arg=0xffffd84b1edf) at pthread_create.c:463
224
#26 0x0000ffffa1b124ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
225
(gdb) l
226
733     in ./src/msg/async/rdma/Infiniband.cc
227
(gdb) bt full
228
#0  raise (sig=sig@entry=11) at ../sysdeps/unix/sysv/linux/raise.c:51
229
        set = {__val = {4096, 281473383641896, 88, 281473383624960, 187650279109528, 187651370344448, 187650293268480, 525377, 420, 88, 281473383624960,
230
            281473383624960, 281473383624944, 4294967288, 14028173432846458624, 187651370344448}}
231
        pid = <optimized out>
232
        tid = <optimized out>
233
        ret = <optimized out>
234
#1  0x0000aaaabc1bba1c in reraise_fatal (signum=11) at ./src/global/signal_handler.cc:326
235
        ret = <optimized out>
236
        buf = "/var/lib/ceph/crash/2020-02-27T11:57:04.693428Z_b789aeae-539d-4d93-ad4b-04011bf0a66c/log", '\000' <repeats 935 times>
237
#2  handle_fatal_signal (signum=11) at ./src/global/signal_handler.cc:326
238
        buf = "*** Caught signal (Segmentation fault) **\n in thread ffffa10bf9f0 thread_name:msgr-worker-0\n", '\000' <repeats 931 times>
239
        pthread_name = "msgr-worker-0\000\000"
240
        r = <optimized out>
241
        bt = {static max = 100, skip = 1, array = {0xaaaabc1bb954 <handle_fatal_signal(int)+204>, 0xffffa23e567c <__kernel_rt_sigreturn>,
242
            0xffffa1ac51c8 <__memcpy_generic+56>, 0xaaaabc582d00 <Infiniband::MemoryManager::Chunk::write(char*, unsigned int)+48>,
243
            0xaaaabc591018 <RDMAConnectedSocketImpl::tx_copy_chunk(std::vector<Infiniband::MemoryManager::Chunk*, std::allocator<Infiniband::MemoryManager::Chunk*> >&, unsigned long, ceph::buffer::v14_2_0::list::buffers_t::buffers_iterator<ceph::buffer::v14_2_0::ptr_node const>&, ceph::buffer::v14_2_0::list::buffers_t::buffers_iterator<ceph::buffer::v14_2_0::ptr_node const> const&)+192>, 0xaaaabc592aec <RDMAConnectedSocketImpl::submit(bool)+924>,
244
            0xaaaabc594c34 <RDMAConnectedSocketImpl::send(ceph::buffer::v14_2_0::list&, bool)+244>, 0xaaaabc5304d4 <AsyncConnection::_try_send(bool)+172>,
245
            0xaaaabc530ec4 <AsyncConnection::write(ceph::buffer::v14_2_0::list&, std::function<void (long)>, bool)+116>,
246
            0xaaaabc55eee0 <ProtocolV2::write(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, CtFun<ProtocolV2>&, ceph::buffer::v14_2_0::list&)+216>, 0xaaaabc5679b0 <ProtocolV2::send_auth_request(std::vector<unsigned int, std::allocator<unsigned int> >&)+1056>,
247
            0xaaaabc567ea8 <ProtocolV2::post_client_banner_exchange()+88>, 0xaaaabc55e770 <ProtocolV2::run_continuation(Ct<ProtocolV2>&)+56>,
248
            0xaaaabc532f40 <AsyncConnection::process()+1568>,
249
            0xaaaabc383edc <EventCenter::process_events(unsigned int, std::chrono::duration<unsigned long, std::ratio<1l, 1000000000l> >*)+1308>,
250
            0xaaaabc38a0f0 <std::_Function_handler<void(), NetworkStack::add_thread(unsigned int)::<lambda()> >::_M_invoke(const std::_Any_data &)+296>,
251
            0xffffa1d42ed4, 0xffffa1ef8088 <start_thread+176>, 0x0 <repeats 82 times>}, size = 18, strings = 0xaaab56a85b00}
252
        oss = <incomplete type>
253
        base = "/var/lib/ceph/crash/2020-02-27T11:57:04.693428Z_b789aeae-539d-4d93-ad4b-04011bf0a66c", '\000' <repeats 4011 times>
254
#3  <signal handler called>
255
No symbol table info available.
256
#4  __memcpy_generic () at ../sysdeps/aarch64/multiarch/../memcpy.S:105
257
No locals.
258
#5  0x0000aaaabc582d00 in memcpy (__len=52, __src=0xaaab4e327000, __dest=<optimized out>) at /usr/include/aarch64-linux-gnu/bits/string_fortified.h:34
259
No locals.
260
#6  Infiniband::MemoryManager::Chunk::write (this=this@entry=0xaaab5560f6c0, buf=buf@entry=0xaaab4e327000 "\002", <incomplete sequence \302>, len=<optimized out>)
261
    at ./src/msg/async/rdma/Infiniband.cc:738
262
        write_len = 52
263
#7  0x0000aaaabc591018 in RDMAConnectedSocketImpl::tx_copy_chunk (this=this@entry=0xaaab53527d40, tx_buffers=std::vector of length 1, capacity 1 = {...},
264
    req_copy_len=req_copy_len@entry=243, start=..., end=...) at ./src/include/buffer.h:317
265
        real_len = <optimized out>
266
        addr = 187652728057856
267
        slice_write_len = 0
268
        __PRETTY_FUNCTION__ = <optimized out>
269
        chunk_idx = 0
270
        __func__ = "tx_copy_chunk"
271
---Type <return> to continue, or q <return> to quit---
272
        current_chunk = 0xaaab5560f6c0
273
        write_len = 0
274
#8  0x0000aaaabc592aec in RDMAConnectedSocketImpl::submit (this=this@entry=0xaaab53527d40, more=more@entry=false)
275
    at ./src/msg/async/rdma/RDMAConnectedSocketImpl.cc:380
276
        l = {_M_device = @0xaaab53527df8}
277
        bytes = <optimized out>
278
        __func__ = "submit"
279
        tx_buffers = std::vector of length 1, capacity 1 = {0xaaab5560f6c0}
280
        it = {cur = 0xaaab53527db0}
281
        copy_start = {cur = 0xaaab50d62880}
282
        total_copied = 0
283
        wait_copy_len = 243
284
        __PRETTY_FUNCTION__ = <optimized out>
285
        swapped = {_buffers = {_root = {next = 0x0}, _tail = 0xaaab53527df8, _size = 1},
286
          _carriage = 0xaaaac53627b0 <ceph::buffer::v14_2_0::list::always_empty_bptr>, _len = 32, _memcopy_count = 0,
287
          last_p = {<ceph::buffer::v14_2_0::list::iterator_impl<false>> = {bl = 0xaaab575d8f28, ls = 0xaaab575d8f28, p = {cur = 0xaaab575d8f28}, off = 0,
288
              p_off = 0}, <No data fields>}, static always_empty_bptr = {_raw = 0x0, _off = 0, _len = 0}, static CLAIM_DEFAULT = 0,
289
          static CLAIM_ALLOW_NONSHAREABLE = 1}
290
#9  0x0000aaaabc594c34 in RDMAConnectedSocketImpl::send (this=0xaaab53527d40, bl=..., more=<optimized out>) at ./src/msg/async/rdma/RDMAConnectedSocketImpl.cc:297
291
        bytes = 243
292
        __func__ = "send"
293
        r = <optimized out>
294
#10 0x0000aaaabc5304d4 in ConnectedSocket::send (more=false, bl=..., this=0xaaab575d8ee8) at /usr/include/c++/9/bits/unique_ptr.h:357
295
No locals.
296
#11 AsyncConnection::_try_send (this=this@entry=0xaaab575d8d00, more=more@entry=false) at ./src/msg/async/AsyncConnection.cc:330
297
        __func__ = "_try_send"
298
        __PRETTY_FUNCTION__ = <optimized out>
299
        r = <optimized out>
300
#12 0x0000aaaabc530ec4 in AsyncConnection::write(ceph::buffer::v14_2_0::list&, std::function<void (long)>, bool) (this=this@entry=0xaaab575d8d00, bl=...,
301
    callback=..., more=more@entry=false) at ./src/msg/async/AsyncConnection.cc:309
302
        l = <optimized out>
303
        r = <optimized out>
304
#13 0x0000aaaabc55eee0 in ProtocolV2::write (this=this@entry=0xaaab5350cc00, desc="auth request", next=..., buffer=...) at /usr/include/c++/9/bits/std_function.h:87
305
        __PRETTY_FUNCTION__ = <optimized out>
306
        r = <optimized out>
307
        __func__ = "write"
308
#14 0x0000aaaabc5679b0 in ProtocolV2::write<ceph::msgr::v2::AuthRequestFrame> (frame=..., next=..., desc="auth request", this=0xaaab5350cc00)
309
    at ./src/msg/async/ProtocolV2.cc:760
310
        bl = {_buffers = {_root = {next = 0xffffa10beb00}, _tail = 0xffffa10beb00, _size = 0},
311
          _carriage = 0xaaaac53627b0 <ceph::buffer::v14_2_0::list::always_empty_bptr>, _len = 0, _memcopy_count = 0,
312
          last_p = {<ceph::buffer::v14_2_0::list::iterator_impl<false>> = {bl = 0xffffa10beb00, ls = 0xffffa10beb00, p = {cur = 0xffffa10beb00}, off = 0,
313
              p_off = 0}, <No data fields>}, static always_empty_bptr = {_raw = 0x0, _off = 0, _len = 0}, static CLAIM_DEFAULT = 0,
314
          static CLAIM_ALLOW_NONSHAREABLE = 1}
315
        bl = <optimized out>
316
#15 ProtocolV2::send_auth_request (this=this@entry=0xaaab5350cc00, allowed_methods=std::vector of length 0, capacity 0) at ./src/msg/async/ProtocolV2.cc:1763
317
        __PRETTY_FUNCTION__ = <optimized out>
318
---Type <return> to continue, or q <return> to quit---q
319
Quit
320
(gdb)
321
(gdb) f 7
322
#7  0x0000aaaabc591018 in RDMAConnectedSocketImpl::tx_copy_chunk (this=this@entry=0xaaab53527d40, tx_buffers=std::vector of length 1, capacity 1 = {...},
323
    req_copy_len=req_copy_len@entry=243, start=..., end=...) at ./src/include/buffer.h:317
324
317     ./src/include/buffer.h: No such file or directory.
325
(gdb) p *tx_buffers[0]
326
$1 = {mr = 0xaaab556103c0, qp = 0xaaaafd4b86f0, lkey = 2, bytes = 43691, offset = 4263420672, bound = 43690, buffer = 0xaaab00000001 "\006\001\064", pad = {
327
    4249587776, 43690, 4249677768, 43690, 1533731784, 43691}, data = 0xaaab5560f700 ""}
328
(gdb) p isge[0]
329
No symbol "isge" in current context.
330
(gdb) info locals
331
real_len = <optimized out>
332
addr = 187652728057856
333
slice_write_len = 0
334
__PRETTY_FUNCTION__ = <optimized out>
335
chunk_idx = 0
336
__func__ = "tx_copy_chunk"
337
current_chunk = 0xaaab5560f6c0
338
write_len = 0
339
(gdb) p *current_chunk
340
$2 = {mr = 0xaaab556103c0, qp = 0xaaaafd4b86f0, lkey = 2, bytes = 43691, offset = 4263420672, bound = 43690, buffer = 0xaaab00000001 "\006\001\064", pad = {
341
    4249587776, 43690, 4249677768, 43690, 1533731784, 43691}, data = 0xaaab5560f700 ""}
342
(gdb) p /x addr
343
$3 = 0xaaab4e327000
344
(gdb) p *tx_buffers[0]
345
$4 = {mr = 0xaaab556103c0, qp = 0xaaaafd4b86f0, lkey = 2, bytes = 43691, offset = 4263420672, bound = 43690, buffer = 0xaaab00000001 "\006\001\064", pad = {
346
    4249587776, 43690, 4249677768, 43690, 1533731784, 43691}, data = 0xaaab5560f700 ""}
347
(gdb) p start->length
348
Could not find operator->.
349
(gdb) p start->length()
350
Could not find operator->.
351
(gdb) p start
352
$5 = (ceph::buffer::v14_2_0::list::buffers_t::const_iterator &) @0xffffa10be6d0: {cur = 0xaaab50d62880}
353
(gdb) p *start
354
Could not find operator*.
355
(gdb) p start->
356
buffers_iterator
357
buffers_iterator<ceph::buffer::v14_2_0::ptr_hook const>
358
buffers_iterator<ceph::buffer::v14_2_0::ptr_hook>
359
cur
360
operator ceph::buffer::v14_2_0::list::buffers_t::buffers_iterator<ceph::buffer::v14_2_0::ptr_node const>::citer_t
361
operator!=
362
operator*
363
operator++
364
operator->
365
operator==
366
(gdb) p start[0]
367
No symbol "operator[]" in current context.
368
(gdb) p start
369
$6 = (ceph::buffer::v14_2_0::list::buffers_t::const_iterator &) @0xffffa10be6d0: {cur = 0xaaab50d62880}
370
(gdb)
371
$7 = (ceph::buffer::v14_2_0::list::buffers_t::const_iterator &) @0xffffa10be6d0: {cur = 0xaaab50d62880}
372
(gdb)
373
$8 = (ceph::buffer::v14_2_0::list::buffers_t::const_iterator &) @0xffffa10be6d0: {cur = 0xaaab50d62880}
374
(gdb) quit
375
root@node2:/var/log# ll core.msgr-worker-*
376
-rw------- 1 root ceph 2538639360 Feb 27 19:57 core.msgr-worker-0.377736
377
-rw------- 1 root ceph 2537709568 Feb 27 19:57 core.msgr-worker-1.377732
378
-rw------- 1 root ceph 2564624384 Feb 27 19:57 core.msgr-worker-1.377734
379
-rw------- 1 root ceph 2512400384 Feb 27 19:57 core.msgr-worker-1.377742
380
-rw------- 1 root ceph 2469797888 Feb 27 19:57 core.msgr-worker-2.377725
381
-rw------- 1 root ceph 2515800064 Feb 27 19:57 core.msgr-worker-2.377738
382
-rw------- 1 root ceph 2513207296 Feb 27 19:57 core.msgr-worker-2.377740
383
root@node2:/var/log# gdb /usr/bin/ceph-osd core.msgr-worker-1.377732
384
GNU gdb (Ubuntu 8.1-0ubuntu3) 8.1.0.20180409-git
385
Copyright (C) 2018 Free Software Foundation, Inc.
386
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
387
This is free software: you are free to change and redistribute it.
388
There is NO WARRANTY, to the extent permitted by law.  Type "show copying"
389
and "show warranty" for details.
390
This GDB was configured as "aarch64-linux-gnu".
391
Type "show configuration" for configuration details.
392
For bug reporting instructions, please see:
393
<http://www.gnu.org/software/gdb/bugs/>.
394
Find the GDB manual and other documentation resources online at:
395
<http://www.gnu.org/software/gdb/documentation/>.
396
For help, type "help".
397
Type "apropos word" to search for commands related to "word"...
398
Reading symbols from /usr/bin/ceph-osd...Reading symbols from /usr/lib/debug/.build-id/fd/2d225baa3068ff217e8e576184a3a362e494fd.debug...done.
399
done.
400
[New LWP 377835]
401
[New LWP 377756]
402
[New LWP 377834]
403
[New LWP 377836]
404
[New LWP 378008]
405
[New LWP 378018]
406
[New LWP 378607]
407
[New LWP 378606]
408
[New LWP 377732]
409
[New LWP 378016]
410
[New LWP 378256]
411
[New LWP 378615]
412
[New LWP 378610]
413
[New LWP 379400]
414
[New LWP 378609]
415
[New LWP 379399]
416
[New LWP 380477]
417
[New LWP 380479]
418
[New LWP 379394]
419
[New LWP 378611]
420
[New LWP 378616]
421
[New LWP 378633]
422
[New LWP 380475]
423
[New LWP 378612]
424
[New LWP 379371]
425
[New LWP 380481]
426
[New LWP 380472]
427
[New LWP 379368]
428
[New LWP 380485]
429
[New LWP 380478]
430
[New LWP 380483]
431
[New LWP 379393]
432
[New LWP 380496]
433
[New LWP 380484]
434
[New LWP 378634]
435
[New LWP 380494]
436
[New LWP 380487]
437
[New LWP 380500]
438
[New LWP 380499]
439
[New LWP 380503]
440
[New LWP 380506]
441
[New LWP 380512]
442
[New LWP 380515]
443
[New LWP 380510]
444
[New LWP 380523]
445
[New LWP 380520]
446
[New LWP 380525]
447
[New LWP 380530]
448
[New LWP 380531]
449
[New LWP 380507]
450
[New LWP 380490]
451
[New LWP 380486]
452
[New LWP 380473]
453
[New LWP 380493]
454
[New LWP 380535]
455
[New LWP 380524]
456
[New LWP 380508]
457
[New LWP 380527]
458
[New LWP 379402]
459
[New LWP 380488]
460
[New LWP 378613]
461
[New LWP 380498]
462
[New LWP 380505]
463
[New LWP 380529]
464
[New LWP 380497]
465
[New LWP 380518]
466
[New LWP 378635]
467
[New LWP 380532]
468
[New LWP 380533]
469
[New LWP 380521]
470
[New LWP 380516]
471
[New LWP 380534]
472
[New LWP 380522]
473
[New LWP 380502]
474
[New LWP 380482]
475
[New LWP 380474]
476
[New LWP 380495]
477
[New LWP 380517]
478
[New LWP 380526]
479
[New LWP 380519]
480
[New LWP 380492]
481
[New LWP 379401]
482
[New LWP 380514]
483
[New LWP 380504]
484
[New LWP 379367]
485
[New LWP 380491]
486
[New LWP 380501]
487
[New LWP 380509]
488
[New LWP 380480]
489
[New LWP 380476]
490
[New LWP 380489]
491
[New LWP 380511]
492
[New LWP 380513]
493
[New LWP 380528]
494
[Thread debugging using libthread_db enabled]
495
Using host libthread_db library "/lib/aarch64-linux-gnu/libthread_db.so.1".
496
Core was generated by `/usr/bin/ceph-osd -f --cluster ceph --id 33 --setuser ceph --setgroup ceph'.
497
Program terminated with signal SIGSEGV, Segmentation fault.
498
#0  raise (sig=sig@entry=11) at ../sysdeps/unix/sysv/linux/raise.c:51
499
51      ../sysdeps/unix/sysv/linux/raise.c: No such file or directory.
500
[Current thread is 1 (Thread 0xffff92d7e9f0 (LWP 377835))]
501
(gdb) bt
502
#0  raise (sig=sig@entry=11) at ../sysdeps/unix/sysv/linux/raise.c:51
503
#1  0x0000aaaad3d49a1c in reraise_fatal (signum=11) at ./src/global/signal_handler.cc:326
504
#2  handle_fatal_signal (signum=11) at ./src/global/signal_handler.cc:326
505
#3  <signal handler called>
506
#4  __memcpy_generic () at ../sysdeps/aarch64/multiarch/../memcpy.S:153
507
#5  0x0000aaaad4110d00 in memcpy (__len=68, __src=0xaaab4c563000, __dest=<optimized out>) at /usr/include/aarch64-linux-gnu/bits/string_fortified.h:34
508
#6  Infiniband::MemoryManager::Chunk::write (this=this@entry=0xaaab4c6adba0, buf=buf@entry=0xaaab4c563000 "\001\001$", len=<optimized out>)
509
    at ./src/msg/async/rdma/Infiniband.cc:738
510
#7  0x0000aaaad411f018 in RDMAConnectedSocketImpl::tx_copy_chunk (this=this@entry=0xaaab4f25e0a0, tx_buffers=std::vector of length 1, capacity 1 = {...},
511
    req_copy_len=req_copy_len@entry=85, start=..., end=...) at ./src/include/buffer.h:317
512
#8  0x0000aaaad4120aec in RDMAConnectedSocketImpl::submit (this=this@entry=0xaaab4f25e0a0, more=more@entry=false)
513
    at ./src/msg/async/rdma/RDMAConnectedSocketImpl.cc:380
514
#9  0x0000aaaad4122c34 in RDMAConnectedSocketImpl::send (this=0xaaab4f25e0a0, bl=..., more=<optimized out>) at ./src/msg/async/rdma/RDMAConnectedSocketImpl.cc:297
515
#10 0x0000aaaad40be4d4 in ConnectedSocket::send (more=false, bl=..., this=0xaaab4fba93e8) at /usr/include/c++/9/bits/unique_ptr.h:357
516
#11 AsyncConnection::_try_send (this=this@entry=0xaaab4fba9200, more=more@entry=false) at ./src/msg/async/AsyncConnection.cc:330
517
#12 0x0000aaaad40beec4 in AsyncConnection::write(ceph::buffer::v14_2_0::list&, std::function<void (long)>, bool) (this=this@entry=0xaaab4fba9200, bl=...,
518
    callback=..., more=more@entry=false) at ./src/msg/async/AsyncConnection.cc:309
519
#13 0x0000aaaad40ecee0 in ProtocolV2::write (this=this@entry=0xaaab4a4a8000, desc="hello frame", next=..., buffer=...) at /usr/include/c++/9/bits/std_function.h:87
520
#14 0x0000aaaad40ef260 in ProtocolV2::write<ceph::msgr::v2::HelloFrame> (frame=..., next=..., desc="hello frame", this=0xaaab4a4a8000)
521
    at ./src/msg/async/ProtocolV2.cc:760
522
#15 ProtocolV2::_handle_peer_banner_payload (this=0xaaab4a4a8000, buffer=..., r=<optimized out>) at ./src/msg/async/ProtocolV2.cc:932
523
#16 0x0000aaaad40ec770 in ProtocolV2::run_continuation (this=0xaaab4a4a8000, continuation=...) at ./src/msg/async/ProtocolV2.cc:45
524
#17 0x0000aaaad40c0f40 in std::function<void (char*, long)>::operator()(char*, long) const (__args#1=0, __args#0=0xaaab4c59ebd0 "ceph v2\n\020",
525
    this=0xaaab4fba9610) at /usr/include/c++/9/bits/std_function.h:685
526
#18 AsyncConnection::process (this=0xaaab4fba9200) at ./src/msg/async/AsyncConnection.cc:457
527
#19 0x0000aaaad3f11edc in EventCenter::process_events (this=this@entry=0xaaaaf103e8c8, timeout_microseconds=<optimized out>,
528
    working_dur=working_dur@entry=0xffff92d7e160) at /usr/include/c++/9/bits/basic_ios.h:282
529
#20 0x0000aaaad3f180f0 in NetworkStack::<lambda()>::operator() (__closure=0xaaaaf10ae4b8, __closure=0xaaaaf10ae4b8) at ./src/msg/async/Stack.cc:53
530
#21 std::_Function_handler<void(), NetworkStack::add_thread(unsigned int)::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...)
531
    at /usr/include/c++/9/bits/std_function.h:300
532
#22 0x0000ffff94202ed4 in ?? () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
533
#23 0x0000ffff943b8088 in start_thread (arg=0xfffff1bd130f) at pthread_create.c:463
534
#24 0x0000ffff93fd24ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
535
(gdb) bt full
536
#0  raise (sig=sig@entry=11) at ../sysdeps/unix/sysv/linux/raise.c:51
537
        set = {__val = {4096, 281473145348840, 88, 281473145331904, 187650677101464, 187651151519744, 187650691260416, 525377, 420, 88, 281473145331904,
538
            281473145331904, 281473145331888, 4294967288, 4353719806521434112, 187651151519744}}
539
        pid = <optimized out>
540
        tid = <optimized out>
541
        ret = <optimized out>
542
#1  0x0000aaaad3d49a1c in reraise_fatal (signum=11) at ./src/global/signal_handler.cc:326
543
        ret = <optimized out>
544
        buf = "/var/lib/ceph/crash/2020-02-27T11:57:05.094472Z_181da468-8c5f-44b9-8c4e-3904914fa183/log", '\000' <repeats 935 times>
545
#2  handle_fatal_signal (signum=11) at ./src/global/signal_handler.cc:326
546
        buf = "*** Caught signal (Segmentation fault) **\n in thread ffff92d7e9f0 thread_name:msgr-worker-1\n", '\000' <repeats 931 times>
547
        pthread_name = "msgr-worker-1\000\000"
548
        r = <optimized out>
549
        bt = {static max = 100, skip = 1, array = {0xaaaad3d49954 <handle_fatal_signal(int)+204>, 0xffff948a567c <__kernel_rt_sigreturn>,
550
            0xffff93f85254 <__memcpy_generic+196>, 0xaaaad4110d00 <Infiniband::MemoryManager::Chunk::write(char*, unsigned int)+48>,
551
            0xaaaad411f018 <RDMAConnectedSocketImpl::tx_copy_chunk(std::vector<Infiniband::MemoryManager::Chunk*, std::allocator<Infiniband::MemoryManager::Chunk*> >&, unsigned long, ceph::buffer::v14_2_0::list::buffers_t::buffers_iterator<ceph::buffer::v14_2_0::ptr_node const>&, ceph::buffer::v14_2_0::list::buffers_t::buffers_iterator<ceph::buffer::v14_2_0::ptr_node const> const&)+192>, 0xaaaad4120aec <RDMAConnectedSocketImpl::submit(bool)+924>,
552
            0xaaaad4122c34 <RDMAConnectedSocketImpl::send(ceph::buffer::v14_2_0::list&, bool)+244>, 0xaaaad40be4d4 <AsyncConnection::_try_send(bool)+172>,
553
            0xaaaad40beec4 <AsyncConnection::write(ceph::buffer::v14_2_0::list&, std::function<void (long)>, bool)+116>,
554
            0xaaaad40ecee0 <ProtocolV2::write(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, CtFun<ProtocolV2>&, ceph::buffer::v14_2_0::list&)+216>,
555
            0xaaaad40ef260 <ProtocolV2::_handle_peer_banner_payload(std::unique_ptr<ceph::buffer::v14_2_0::ptr_node, ceph::buffer::v14_2_0::ptr_node::disposer>&&, int)+880>, 0xaaaad40ec770 <ProtocolV2::run_continuation(Ct<ProtocolV2>&)+56>, 0xaaaad40c0f40 <AsyncConnection::process()+1568>,
556
            0xaaaad3f11edc <EventCenter::process_events(unsigned int, std::chrono::duration<unsigned long, std::ratio<1l, 1000000000l> >*)+1308>,
557
            0xaaaad3f180f0 <std::_Function_handler<void(), NetworkStack::add_thread(unsigned int)::<lambda()> >::_M_invoke(const std::_Any_data &)+296>,
558
            0xffff94202ed4, 0xffff943b8088 <start_thread+176>, 0x0 <repeats 83 times>}, size = 17, strings = 0xaaab4fba5000}
559
        oss = <incomplete type>
560
        base = "/var/lib/ceph/crash/2020-02-27T11:57:05.094472Z_181da468-8c5f-44b9-8c4e-3904914fa183", '\000' <repeats 4011 times>
561
#3  <signal handler called>
562
No symbol table info available.
563
#4  __memcpy_generic () at ../sysdeps/aarch64/multiarch/../memcpy.S:153
564
No locals.
565
#5  0x0000aaaad4110d00 in memcpy (__len=68, __src=0xaaab4c563000, __dest=<optimized out>) at /usr/include/aarch64-linux-gnu/bits/string_fortified.h:34
566
No locals.
567
#6  Infiniband::MemoryManager::Chunk::write (this=this@entry=0xaaab4c6adba0, buf=buf@entry=0xaaab4c563000 "\001\001$", len=<optimized out>)
568
    at ./src/msg/async/rdma/Infiniband.cc:738
569
        write_len = 68
570
#7  0x0000aaaad411f018 in RDMAConnectedSocketImpl::tx_copy_chunk (this=this@entry=0xaaab4f25e0a0, tx_buffers=std::vector of length 1, capacity 1 = {...},
571
    req_copy_len=req_copy_len@entry=85, start=..., end=...) at ./src/include/buffer.h:317
572
        real_len = <optimized out>
573
        addr = 187652696846336
574
        slice_write_len = 0
575
        __PRETTY_FUNCTION__ = <optimized out>
576
        chunk_idx = 0
577
        __func__ = "tx_copy_chunk"
578
---Type <return> to continue, or q <return> to quit---
579
        current_chunk = 0xaaab4c6adba0
580
        write_len = 0
581
#8  0x0000aaaad4120aec in RDMAConnectedSocketImpl::submit (this=this@entry=0xaaab4f25e0a0, more=more@entry=false)
582
    at ./src/msg/async/rdma/RDMAConnectedSocketImpl.cc:380
583
        l = {_M_device = @0xaaab4f25e158}
584
        bytes = <optimized out>
585
        __func__ = "submit"
586
        tx_buffers = std::vector of length 1, capacity 1 = {0xaaab4c6adba0}
587
        it = {cur = 0xaaab4f25e110}
588
        copy_start = {cur = 0xaaab4991f9e0}
589
        total_copied = 0
590
        wait_copy_len = 85
591
        __PRETTY_FUNCTION__ = <optimized out>
592
        swapped = {_buffers = {_root = {next = 0x0}, _tail = 0xaaab4f25e158, _size = 0}, _carriage = 0xffff92d7d6d0, _len = 2463618768, _memcopy_count = 65535,
593
          last_p = {<ceph::buffer::v14_2_0::list::iterator_impl<false>> = {bl = 0xaaab4fba9428, ls = 0xaaab4fba9428, p = {cur = 0xaaab4fba9428}, off = 0,
594
              p_off = 0}, <No data fields>}, static always_empty_bptr = {_raw = 0x0, _off = 0, _len = 0}, static CLAIM_DEFAULT = 0,
595
          static CLAIM_ALLOW_NONSHAREABLE = 1}
596
#9  0x0000aaaad4122c34 in RDMAConnectedSocketImpl::send (this=0xaaab4f25e0a0, bl=..., more=<optimized out>) at ./src/msg/async/rdma/RDMAConnectedSocketImpl.cc:297
597
        bytes = 85
598
        __func__ = "send"
599
        r = <optimized out>
600
#10 0x0000aaaad40be4d4 in ConnectedSocket::send (more=false, bl=..., this=0xaaab4fba93e8) at /usr/include/c++/9/bits/unique_ptr.h:357
601
No locals.
602
#11 AsyncConnection::_try_send (this=this@entry=0xaaab4fba9200, more=more@entry=false) at ./src/msg/async/AsyncConnection.cc:330
603
        __func__ = "_try_send"
604
        __PRETTY_FUNCTION__ = <optimized out>
605
        r = <optimized out>
606
#12 0x0000aaaad40beec4 in AsyncConnection::write(ceph::buffer::v14_2_0::list&, std::function<void (long)>, bool) (this=this@entry=0xaaab4fba9200, bl=...,
607
    callback=..., more=more@entry=false) at ./src/msg/async/AsyncConnection.cc:309
608
        l = <optimized out>
609
        r = <optimized out>
610
#13 0x0000aaaad40ecee0 in ProtocolV2::write (this=this@entry=0xaaab4a4a8000, desc="hello frame", next=..., buffer=...) at /usr/include/c++/9/bits/std_function.h:87
611
        __PRETTY_FUNCTION__ = <optimized out>
612
        r = <optimized out>
613
        __func__ = "write"
614
#14 0x0000aaaad40ef260 in ProtocolV2::write<ceph::msgr::v2::HelloFrame> (frame=..., next=..., desc="hello frame", this=0xaaab4a4a8000)
615
    at ./src/msg/async/ProtocolV2.cc:760
616
        bl = {_buffers = {_root = {next = 0xffff92d7dae0}, _tail = 0xffff92d7dae0, _size = 0},
617
          _carriage = 0xaaaadcef07b0 <ceph::buffer::v14_2_0::list::always_empty_bptr>, _len = 0, _memcopy_count = 0,
618
          last_p = {<ceph::buffer::v14_2_0::list::iterator_impl<false>> = {bl = 0xffff92d7dae0, ls = 0xffff92d7dae0, p = {cur = 0xffff92d7dae0}, off = 0,
619
              p_off = 0}, <No data fields>}, static always_empty_bptr = {_raw = 0x0, _off = 0, _len = 0}, static CLAIM_DEFAULT = 0,
620
          static CLAIM_ALLOW_NONSHAREABLE = 1}
621
        bl = <optimized out>
622
#15 ProtocolV2::_handle_peer_banner_payload (this=0xaaab4a4a8000, buffer=..., r=<optimized out>) at ./src/msg/async/ProtocolV2.cc:932
623
        __func__ = "_handle_peer_banner_payload"
624
        peer_supported_features = <optimized out>
625
---Type <return> to continue, or q <return> to quit---]
626
        peer_required_features = <optimized out>
627
        bl = {_buffers = {_root = {next = 0xaaab4bb41b00}, _tail = 0xaaab4bb41b00, _size = 1}, _carriage = 0xaaab4bb41b08, _len = 16, _memcopy_count = 0,
628
          last_p = {<ceph::buffer::v14_2_0::list::iterator_impl<false>> = {bl = 0xffff92d7da98, ls = 0xffff92d7da98, p = {cur = 0xffff92d7da98}, off = 0,
629
              p_off = 0}, <No data fields>}, static always_empty_bptr = {_raw = 0x0, _off = 0, _len = 0}, static CLAIM_DEFAULT = 0,
630
          static CLAIM_ALLOW_NONSHAREABLE = 1}
631
        ti = {bl = 0xffff92d7da98, ls = 0xffff92d7da98, p = {cur = 0xffff92d7da98}, off = 16, p_off = 0}
632
        supported_features = 0
633
        required_features = 0
634
        __PRETTY_FUNCTION__ = <optimized out>
635
        hello = {<ceph::msgr::v2::ControlFrame<ceph::msgr::v2::HelloFrame, unsigned char, entity_addr_t>> = {<ceph::msgr::v2::Frame<ceph::msgr::v2::HelloFrame, 8>> = {static SegmentsNumV = 1, segments = {_M_elems = {{_buffers = {_root = {next = 0xffff92d7db28}, _tail = 0xffff92d7db28, _size = 0},
636
                    _carriage = 0xaaaadcef07b0 <ceph::buffer::v14_2_0::list::always_empty_bptr>, _len = 0, _memcopy_count = 0,
637
                    last_p = {<ceph::buffer::v14_2_0::list::iterator_impl<false>> = {bl = 0xffff92d7db28, ls = 0xffff92d7db28, p = {cur = 0xffff92d7db28}, off = 0,
638
                        p_off = 0}, <No data fields>}, static always_empty_bptr = {_raw = 0x0, _off = 0, _len = 0}, static CLAIM_DEFAULT = 0,
639
                    static CLAIM_ALLOW_NONSHAREABLE = 1}}}, static alignments = <optimized out>, preamble_filler = {pos = 0xaaab4c563020 "\004\001\001\001\034"}},
640
            _values = std::tuple containing = {[1] = 0 '\000', [2] = {static TYPE_DEFAULT = entity_addr_t::TYPE_MSGR2, type = 0, nonce = 0, u = {sa = {
641
                    sa_family = 0, sa_data = '\000' <repeats 13 times>}, sin = {sin_family = 0, sin_port = 0, sin_addr = {s_addr = 0},
642
                    sin_zero = "\000\000\000\000\000\000\000"}, sin6 = {sin6_family = 0, sin6_port = 0, sin6_flowinfo = 0, sin6_addr = {__in6_u = {
643
                        __u6_addr8 = '\000' <repeats 15 times>, __u6_addr16 = {0, 0, 0, 0, 0, 0, 0, 0}, __u6_addr32 = {0, 0, 0, 0}}}, sin6_scope_id = 0}}}},
644
            features = 720575940647714820}, static tag = ceph::msgr::v2::Tag::HELLO}
645
        should_gather = <optimized out>
646
        _dout_e = <optimized out>
647
        _dout_cct = <optimized out>
648
        _dout = <optimized out>
649
#16 0x0000aaaad40ec770 in ProtocolV2::run_continuation (this=0xaaab4a4a8000, continuation=...) at ./src/msg/async/ProtocolV2.cc:45
650
        _cont = <optimized out>
651
        __func__ = "run_continuation"
652
#17 0x0000aaaad40c0f40 in std::function<void (char*, long)>::operator()(char*, long) const (__args#1=0, __args#0=0xaaab4c59ebd0 "ceph v2\n\020",
653
    this=0xaaab4fba9610) at /usr/include/c++/9/bits/std_function.h:685
654
No locals.
655
#18 AsyncConnection::process (this=0xaaab4fba9200) at ./src/msg/async/AsyncConnection.cc:457
656
        buf_tmp = <optimized out>
657
        r = 0
658
        l = {_M_device = @0xaaab4fba94a8}
659
        __func__ = "process"
660
        __PRETTY_FUNCTION__ = <optimized out>
661
#19 0x0000aaaad3f11edc in EventCenter::process_events (this=this@entry=0xaaaaf103e8c8, timeout_microseconds=<optimized out>,
662
    working_dur=working_dur@entry=0xffff92d7e160) at /usr/include/c++/9/bits/basic_ios.h:282
663
        rfired = 1
664
        event = 0xaaaaf1090798
665
        cb = 0xaaab4b030500
666
        event_id = <optimized out>
667
        tv = {tv_sec = 0, tv_usec = 320000}
668
        numevents = 1
669
        trigger_time = true
670
        now = <optimized out>
671
---Type <return> to continue, or q <return> to quit---q
672
Quit
673
(gdb) quit
674
root@node2:/var/log# ll core.msgr-worker-*
675
-rw------- 1 root ceph 2538639360 Feb 27 19:57 core.msgr-worker-0.377736
676
-rw------- 1 root ceph 2537709568 Feb 27 19:57 core.msgr-worker-1.377732
677
-rw------- 1 root ceph 2564624384 Feb 27 19:57 core.msgr-worker-1.377734
678
-rw------- 1 root ceph 2512400384 Feb 27 19:57 core.msgr-worker-1.377742
679
-rw------- 1 root ceph 2469797888 Feb 27 19:57 core.msgr-worker-2.377725
680
-rw------- 1 root ceph 2515800064 Feb 27 19:57 core.msgr-worker-2.377738
681
-rw------- 1 root ceph 2513207296 Feb 27 19:57 core.msgr-worker-2.377740
682
root@node2:/var/log# gdb /usr/bin/ceph-osd core.msgr-worker-1.377734
683
GNU gdb (Ubuntu 8.1-0ubuntu3) 8.1.0.20180409-git
684
Copyright (C) 2018 Free Software Foundation, Inc.
685
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
686
This is free software: you are free to change and redistribute it.
687
There is NO WARRANTY, to the extent permitted by law.  Type "show copying"
688
and "show warranty" for details.
689
This GDB was configured as "aarch64-linux-gnu".
690
Type "show configuration" for configuration details.
691
For bug reporting instructions, please see:
692
<http://www.gnu.org/software/gdb/bugs/>.
693
Find the GDB manual and other documentation resources online at:
694
<http://www.gnu.org/software/gdb/documentation/>.
695
For help, type "help".
696
Type "apropos word" to search for commands related to "word"...
697
Reading symbols from /usr/bin/ceph-osd...Reading symbols from /usr/lib/debug/.build-id/fd/2d225baa3068ff217e8e576184a3a362e494fd.debug...
698
done.
699
done.
700
[New LWP 377845]
701
[New LWP 377734]
702
[New LWP 377844]
703
[New LWP 378042]
704
[New LWP 377846]
705
[New LWP 378041]
706
[New LWP 378292]
707
[New LWP 378726]
708
[New LWP 378715]
709
[New LWP 378717]
710
[New LWP 378734]
711
[New LWP 378742]
712
[New LWP 377758]
713
[New LWP 379673]
714
[New LWP 379738]
715
[New LWP 379749]
716
[New LWP 378732]
717
[New LWP 378733]
718
[New LWP 379748]
719
[New LWP 379750]
720
[New LWP 381112]
721
[New LWP 379755]
722
[New LWP 378795]
723
[New LWP 381114]
724
[New LWP 381115]
725
[New LWP 381118]
726
[New LWP 381113]
727
[New LWP 381119]
728
[New LWP 378744]
729
[New LWP 381117]
730
[New LWP 381120]
731
[New LWP 378730]
732
[New LWP 379739]
733
[New LWP 381163]
734
[New LWP 381166]
735
[New LWP 381125]
736
[New LWP 379684]
737
[New LWP 378796]
738
[New LWP 379676]
739
[New LWP 381164]
740
[New LWP 381157]
741
[New LWP 381165]
742
[New LWP 381159]
743
[New LWP 381143]
744
[New LWP 381137]
745
[New LWP 381126]
746
[New LWP 381145]
747
[New LWP 381180]
748
[New LWP 381150]
749
[New LWP 381133]
750
[New LWP 381123]
751
[New LWP 381124]
752
[New LWP 381132]
753
[New LWP 378797]
754
[New LWP 381136]
755
[New LWP 381138]
756
[New LWP 381135]
757
[New LWP 381139]
758
[New LWP 381116]
759
[New LWP 381149]
760
[New LWP 381168]
761
[New LWP 381146]
762
[New LWP 381174]
763
[New LWP 381144]
764
[New LWP 381152]
765
[New LWP 381134]
766
[New LWP 381161]
767
[New LWP 381148]
768
[New LWP 381175]
769
[New LWP 381121]
770
[New LWP 381156]
771
[New LWP 381153]
772
[New LWP 381122]
773
[New LWP 381169]
774
[New LWP 381179]
775
[New LWP 381147]
776
[New LWP 381167]
777
[New LWP 381170]
778
[New LWP 381158]
779
[New LWP 381151]
780
[New LWP 381177]
781
[New LWP 381141]
782
[New LWP 381178]
783
[New LWP 381162]
784
[New LWP 381176]
785
[New LWP 381173]
786
[New LWP 381155]
787
[New LWP 381160]
788
[New LWP 381172]
789
[New LWP 381154]
790
[New LWP 381171]
791
[New LWP 381142]
792
[New LWP 381140]
793
[New LWP 378010]
794
[Thread debugging using libthread_db enabled]
795
Using host libthread_db library "/lib/aarch64-linux-gnu/libthread_db.so.1".
796

    
797
Core was generated by `/usr/bin/ceph-osd -f --cluster ceph --id 35 --setuser ceph --setgroup ceph'.
798
Program terminated with signal SIGSEGV, Segmentation fault.
799
#0  raise (sig=sig@entry=11) at ../sysdeps/unix/sysv/linux/raise.c:51
800
51      ../sysdeps/unix/sysv/linux/raise.c: No such file or directory.
801
[Current thread is 1 (Thread 0xffff9d3399f0 (LWP 377845))]
802
(gdb)
803
(gdb)
804
(gdb) bt
805
#0  raise (sig=sig@entry=11) at ../sysdeps/unix/sysv/linux/raise.c:51
806
#1  0x0000aaaae12c1a1c in reraise_fatal (signum=11) at ./src/global/signal_handler.cc:326
807
#2  handle_fatal_signal (signum=11) at ./src/global/signal_handler.cc:326
808
#3  <signal handler called>
809
#4  __memcpy_generic () at ../sysdeps/aarch64/multiarch/../memcpy.S:153
810
#5  0x0000aaaae1688d00 in memcpy (__len=73, __src=0xaaab5dc4c000, __dest=<optimized out>) at /usr/include/aarch64-linux-gnu/bits/string_fortified.h:34
811
#6  Infiniband::MemoryManager::Chunk::write (this=this@entry=0xaaab5c679ba0, buf=buf@entry=0xaaab5dc4c000 "\021\002)", len=<optimized out>)
812
    at ./src/msg/async/rdma/Infiniband.cc:738
813
#7  0x0000aaaae1697018 in RDMAConnectedSocketImpl::tx_copy_chunk (this=this@entry=0xaaab5e785f80, tx_buffers=std::vector of length 1, capacity 1 = {...},
814
    req_copy_len=req_copy_len@entry=1086, start=..., end=...) at ./src/include/buffer.h:317
815
#8  0x0000aaaae1698aec in RDMAConnectedSocketImpl::submit (this=this@entry=0xaaab5e785f80, more=more@entry=false)
816
    at ./src/msg/async/rdma/RDMAConnectedSocketImpl.cc:380
817
#9  0x0000aaaae169ac34 in RDMAConnectedSocketImpl::send (this=0xaaab5e785f80, bl=..., more=<optimized out>) at ./src/msg/async/rdma/RDMAConnectedSocketImpl.cc:297
818
#10 0x0000aaaae16364d4 in ConnectedSocket::send (more=false, bl=..., this=0xaaab5d172f68) at /usr/include/c++/9/bits/unique_ptr.h:357
819
#11 AsyncConnection::_try_send (this=0xaaab5d172d80, more=more@entry=false) at ./src/msg/async/AsyncConnection.cc:330
820
#12 0x0000aaaae165e99c in ProtocolV2::write_message (this=this@entry=0xaaab5dd8e580, m=m@entry=0xaaab5f6cb500, more=more@entry=false)
821
    at ./src/msg/async/ProtocolV2.cc:538
822
#13 0x0000aaaae1679778 in ProtocolV2::write_event (this=0xaaab5dd8e580) at ./src/msg/async/ProtocolV2.cc:637
823
#14 0x0000aaaae16339e8 in AsyncConnection::handle_write (this=0xaaab5d172d80) at /usr/include/c++/9/bits/unique_ptr.h:357
824
#15 0x0000aaaae148a570 in EventCenter::process_events (this=this@entry=0xaaaafef728c8, timeout_microseconds=<optimized out>,
825
    working_dur=working_dur@entry=0xffff9d339160) at /usr/include/c++/9/bits/stl_deque.h:2110
826
#16 0x0000aaaae14900f0 in NetworkStack::<lambda()>::operator() (__closure=0xaaaafefe24b8, __closure=0xaaaafefe24b8) at ./src/msg/async/Stack.cc:53
827
#17 std::_Function_handler<void(), NetworkStack::add_thread(unsigned int)::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...)
828
    at /usr/include/c++/9/bits/std_function.h:300
829
#18 0x0000ffff9e7bded4 in ?? () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
830
#19 0x0000ffff9e973088 in start_thread (arg=0xfffff56c646f) at pthread_create.c:463
831
#20 0x0000ffff9e58d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
832
(gdb) bt full
833
#0  raise (sig=sig@entry=11) at ../sysdeps/unix/sysv/linux/raise.c:51
834
        set = {__val = {4096, 281473319130328, 88, 281473319113392, 187650900939672, 187651385565184, 187650915098624, 525377, 420, 88, 281473319113392,
835
            281473319113392, 281473319113376, 4294967288, 11714035691759710208, 187651385565184}}
836
        pid = <optimized out>
837
        tid = <optimized out>
838
        ret = <optimized out>
839
#1  0x0000aaaae12c1a1c in reraise_fatal (signum=11) at ./src/global/signal_handler.cc:326
840
        ret = <optimized out>
841
        buf = "/var/lib/ceph/crash/2020-02-27T11:57:05.402233Z_9b8d6812-8d8c-4295-8597-6ddab051c312/log", '\000' <repeats 736 times>...
842
#2  handle_fatal_signal (signum=11) at ./src/global/signal_handler.cc:326
843
        buf = "*** Caught signal (Segmentation fault) **\n in thread ffff9d3399f0 thread_name:msgr-worker-1\n", '\000' <repeats 931 times>
844
        pthread_name = "msgr-worker-1\000\000"
845
        r = <optimized out>
846
        bt = {static max = 100, skip = 1, array = {0xaaaae12c1954 <handle_fatal_signal(int)+204>, 0xffff9ee6067c <__kernel_rt_sigreturn>,
847
            0xffff9e540254 <__memcpy_generic+196>, 0xaaaae1688d00 <Infiniband::MemoryManager::Chunk::write(char*, unsigned int)+48>,
848
            0xaaaae1697018 <RDMAConnectedSocketImpl::tx_copy_chunk(std::vector<Infiniband::MemoryManager::Chunk*, std::allocator<Infiniband::MemoryManager::Chunk*> >&, unsigned long, ceph::buffer::v14_2_0::list::buffers_t::buffers_iterator<ceph::buffer::v14_2_0::ptr_node const>&, ceph::buffer::v14_2_0::list::buffers_t::buffers_iterator<ceph::buffer::v14_2_0::ptr_node const> const&)+192>, 0xaaaae1698aec <RDMAConnectedSocketImpl::submit(bool)+924>,
849
            0xaaaae169ac34 <RDMAConnectedSocketImpl::send(ceph::buffer::v14_2_0::list&, bool)+244>, 0xaaaae16364d4 <AsyncConnection::_try_send(bool)+172>,
850
            0xaaaae165e99c <ProtocolV2::write_message(Message*, bool)+1100>, 0xaaaae1679778 <ProtocolV2::write_event()+416>,
851
            0xaaaae16339e8 <AsyncConnection::handle_write()+72>,
852
            0xaaaae148a570 <EventCenter::process_events(unsigned int, std::chrono::duration<unsigned long, std::ratio<1l, 1000000000l> >*)+2992>,
853
            0xaaaae14900f0 <std::_Function_handler<void(), NetworkStack::add_thread(unsigned int)::<lambda()> >::_M_invoke(const std::_Any_data &)+296>,
854
            0xffff9e7bded4, 0xffff9e973088 <start_thread+176>, 0x0 <repeats 85 times>}, size = 15, strings = 0xaaab5e70f800}
855
        oss = <incomplete type>
856
        base = "/var/lib/ceph/crash/2020-02-27T11:57:05.402233Z_9b8d6812-8d8c-4295-8597-6ddab051c312", '\000' <repeats 4011 times>
857
#3  <signal handler called>
858
No symbol table info available.
859
#4  __memcpy_generic () at ../sysdeps/aarch64/multiarch/../memcpy.S:153
860
No locals.
861
#5  0x0000aaaae1688d00 in memcpy (__len=73, __src=0xaaab5dc4c000, __dest=<optimized out>) at /usr/include/aarch64-linux-gnu/bits/string_fortified.h:34
862
No locals.
863
#6  Infiniband::MemoryManager::Chunk::write (this=this@entry=0xaaab5c679ba0, buf=buf@entry=0xaaab5dc4c000 "\021\002)", len=<optimized out>)
864
    at ./src/msg/async/rdma/Infiniband.cc:738
865
        write_len = 73
866
#7  0x0000aaaae1697018 in RDMAConnectedSocketImpl::tx_copy_chunk (this=this@entry=0xaaab5e785f80, tx_buffers=std::vector of length 1, capacity 1 = {...},
867
    req_copy_len=req_copy_len@entry=1086, start=..., end=...) at ./src/include/buffer.h:317
868
        real_len = <optimized out>
869
        addr = 187652989304832
870
        slice_write_len = 0
871
        __PRETTY_FUNCTION__ = <optimized out>
872
        chunk_idx = 0
873
        __func__ = "tx_copy_chunk"
874
        current_chunk = 0xaaab5c679ba0
875
        write_len = 0
876
#8  0x0000aaaae1698aec in RDMAConnectedSocketImpl::submit (this=this@entry=0xaaab5e785f80, more=more@entry=false)
877
---Type <return> to continue, or q <return> to quit---
878
    at ./src/msg/async/rdma/RDMAConnectedSocketImpl.cc:380
879
        l = {_M_device = @0xaaab5e786038}
880
        bytes = <optimized out>
881
        __func__ = "submit"
882
        tx_buffers = std::vector of length 1, capacity 1 = {0xaaab5c679ba0}
883
        it = {cur = 0xaaab5e785ff0}
884
        copy_start = {cur = 0xaaab5f204b60}
885
        total_copied = 0
886
        wait_copy_len = 1086
887
        __PRETTY_FUNCTION__ = <optimized out>
888
        swapped = {_buffers = {_root = {next = 0x0}, _tail = 0xaaab5e786038, _size = 281473319143952}, _carriage = 0xffff9d338a58, _len = 2637400560,
889
          _memcopy_count = 65535, last_p = {<ceph::buffer::v14_2_0::list::iterator_impl<false>> = {bl = 0xaaab5d172fa8, ls = 0xaaab5d172fa8, p = {
890
                cur = 0xaaab5d172fa8}, off = 0, p_off = 0}, <No data fields>}, static always_empty_bptr = {_raw = 0x0, _off = 0, _len = 0},
891
          static CLAIM_DEFAULT = 0, static CLAIM_ALLOW_NONSHAREABLE = 1}
892
#9  0x0000aaaae169ac34 in RDMAConnectedSocketImpl::send (this=0xaaab5e785f80, bl=..., more=<optimized out>) at ./src/msg/async/rdma/RDMAConnectedSocketImpl.cc:297
893
        bytes = 1086
894
        __func__ = "send"
895
        r = <optimized out>
896
#10 0x0000aaaae16364d4 in ConnectedSocket::send (more=false, bl=..., this=0xaaab5d172f68) at /usr/include/c++/9/bits/unique_ptr.h:357
897
No locals.
898
#11 AsyncConnection::_try_send (this=0xaaab5d172d80, more=more@entry=false) at ./src/msg/async/AsyncConnection.cc:330
899
        __func__ = "_try_send"
900
        __PRETTY_FUNCTION__ = <optimized out>
901
        r = <optimized out>
902
#12 0x0000aaaae165e99c in ProtocolV2::write_message (this=this@entry=0xaaab5dd8e580, m=m@entry=0xaaab5f6cb500, more=more@entry=false)
903
    at ./src/msg/async/ProtocolV2.cc:538
904
        __PRETTY_FUNCTION__ = <optimized out>
905
        ack_seq = <optimized out>
906
        header = @0xaaab5f6cb518: {seq = {v = 6}, tid = {v = 0}, type = {v = 132}, priority = {v = 196}, version = {v = 1}, front_len = {v = 996}, middle_len = {
907
            v = 0}, data_len = {v = 0}, data_off = {v = 0}, src = {type = 4 '\004', num = {v = 35}}, compat_version = {v = 1}, reserved = {v = 0}, crc = {v = 0}}
908
        footer = @0xaaab5f6cb54d: {front_crc = {v = 0}, middle_crc = {v = 0}, data_crc = {v = 0}, sig = {v = 0}, flags = 3 '\003'}
909
        header2 = {seq = {v = 6}, tid = {v = 0}, type = {v = 132}, priority = {v = 196}, version = {v = 1}, data_pre_padding_len = {v = 0}, data_off = {v = 0},
910
          ack_seq = {v = 3}, flags = 3 '\003', compat_version = {v = 1}, reserved = {v = 0}}
911
        message = {<ceph::msgr::v2::Frame<ceph::msgr::v2::MessageFrame, 8, 8, 8, 4096>> = {static SegmentsNumV = 4, segments = {_M_elems = {{_buffers = {_root = {
912
                      next = 0xffff9d338b90}, _tail = 0xffff9d338b90, _size = 0}, _carriage = 0xaaaaea4687b0 <ceph::buffer::v14_2_0::list::always_empty_bptr>,
913
                  _len = 0, _memcopy_count = 0, last_p = {<ceph::buffer::v14_2_0::list::iterator_impl<false>> = {bl = 0xffff9d338b90, ls = 0xffff9d338b90, p = {
914
                        cur = 0xffff9d338b90}, off = 0, p_off = 0}, <No data fields>}, static always_empty_bptr = {_raw = 0x0, _off = 0, _len = 0},
915
                  static CLAIM_DEFAULT = 0, static CLAIM_ALLOW_NONSHAREABLE = 1}, {_buffers = {_root = {next = 0xffff9d338bd8}, _tail = 0xffff9d338bd8, _size = 0},
916
                  _carriage = 0xaaaaea4687b0 <ceph::buffer::v14_2_0::list::always_empty_bptr>, _len = 0, _memcopy_count = 0,
917
                  last_p = {<ceph::buffer::v14_2_0::list::iterator_impl<false>> = {bl = 0xffff9d338bd8, ls = 0xffff9d338bd8, p = {cur = 0xffff9d338bd8}, off = 0,
918
                      p_off = 0}, <No data fields>}, static always_empty_bptr = {_raw = 0x0, _off = 0, _len = 0}, static CLAIM_DEFAULT = 0,
919
                  static CLAIM_ALLOW_NONSHAREABLE = 1}, {_buffers = {_root = {next = 0xffff9d338c20}, _tail = 0xffff9d338c20, _size = 0},
920
                  _carriage = 0xaaaaea4687b0 <ceph::buffer::v14_2_0::list::always_empty_bptr>, _len = 0, _memcopy_count = 0,
921
                  last_p = {<ceph::buffer::v14_2_0::list::iterator_impl<false>> = {bl = 0xffff9d338c20, ls = 0xffff9d338c20, p = {cur = 0xffff9d338c20}, off = 0,
922
                      p_off = 0}, <No data fields>}, static always_empty_bptr = {_raw = 0x0, _off = 0, _len = 0}, static CLAIM_DEFAULT = 0,
923
                  static CLAIM_ALLOW_NONSHAREABLE = 1}, {_buffers = {_root = {next = 0xffff9d338c68}, _tail = 0xffff9d338c68, _size = 0},
924
---Type <return> to continue, or q <return> to quit---
925
                  _carriage = 0xaaaaea4687b0 <ceph::buffer::v14_2_0::list::always_empty_bptr>, _len = 0, _memcopy_count = 0,
926
                  last_p = {<ceph::buffer::v14_2_0::list::iterator_impl<false>> = {bl = 0xffff9d338c68, ls = 0xffff9d338c68, p = {cur = 0xffff9d338c68}, off = 0,
927
                      p_off = 0}, <No data fields>}, static always_empty_bptr = {_raw = 0x0, _off = 0, _len = 0}, static CLAIM_DEFAULT = 0,
928
                  static CLAIM_ALLOW_NONSHAREABLE = 1}}}, static alignments = <optimized out>, preamble_filler = {pos = 0xaaab5dc4c020 "\006"}}, len = {
929
            front = 3781662728, middle = 43690, data = 2637401296}, static tag = ceph::msgr::v2::Tag::MESSAGE}
930
        __func__ = "write_message"
931
        total_send_size = 1086
932
        rc = <optimized out>
933
#13 0x0000aaaae1679778 in ProtocolV2::write_event (this=0xaaab5dd8e580) at ./src/msg/async/ProtocolV2.cc:637
934
        out_entry = <optimized out>
935
        start = {__d = {__r = 193307517723240}}
936
        more = false
937
        __func__ = "write_event"
938
        r = 0
939
#14 0x0000aaaae16339e8 in AsyncConnection::handle_write (this=0xaaab5d172d80) at /usr/include/c++/9/bits/unique_ptr.h:357
940
        __func__ = "handle_write"
941
#15 0x0000aaaae148a570 in EventCenter::process_events (this=this@entry=0xaaaafef728c8, timeout_microseconds=<optimized out>,
942
    working_dur=working_dur@entry=0xffff9d339160) at /usr/include/c++/9/bits/stl_deque.h:2110
943
        e = 0xaaab5e74ca90
944
        cur_process = std::deque with 2 elements = {0xaaab5e74ca90, 0xaaab5e74c180}
945
        tv = {tv_sec = 0, tv_usec = 304000}
946
        numevents = 3
947
        trigger_time = true
948
        now = <optimized out>
949
        end_time = <optimized out>
950
        it = <optimized out>
951
        blocking = <optimized out>
952
        __func__ = "process_events"
953
        fired_events = std::vector of length 1, capacity 1 = {{fd = 7, mask = 1}}
954
        working_start = {__d = {__r = 193307517717480}}
955
#16 0x0000aaaae14900f0 in NetworkStack::<lambda()>::operator() (__closure=0xaaaafefe24b8, __closure=0xaaaafefe24b8) at ./src/msg/async/Stack.cc:53
956
        dur = {__r = 126840}
957
        r = <optimized out>
958
        tp_name = "msgr-worker-1\000\000"
959
        w = <optimized out>
960
        this = <optimized out>
961
        w = <optimized out>
962
        this = <optimized out>
963
        __func__ = <optimized out>
964
        tp_name = <optimized out>
965
        EventMaxWaitUs = <optimized out>
966
        should_gather = <optimized out>
967
        _dout_e = <optimized out>
968
        _dout_cct = <optimized out>
969
        _dout = <optimized out>
970
        dur = <optimized out>
971
---Type <return> to continue, or q <return> to quit---
972
        r = <optimized out>
973
        should_gather = <optimized out>
974
        _dout_e = <optimized out>
975
        _dout_cct = <optimized out>
976
        _dout = <optimized out>
977
        should_gather = <optimized out>
978
        _dout_e = <optimized out>
979
        _dout_cct = <optimized out>
980
        _dout = <optimized out>
981
#17 std::_Function_handler<void(), NetworkStack::add_thread(unsigned int)::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...)
982
    at /usr/include/c++/9/bits/std_function.h:300
983
No locals.
984
#18 0x0000ffff9e7bded4 in ?? () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
985
No symbol table info available.
986
#19 0x0000ffff9e973088 in start_thread (arg=0xfffff56c646f) at pthread_create.c:463
987
        pd = 0xfffff56c646f
988
        unwind_buf = {cancel_jmp_buf = {{jmp_buf = {281473319148016, 281474799264880, 281474799264878, 281474799264879, 0, 4096, 281474799264880, 281473342582784,
989
                281473319148016, 1, 281473319145968, 17434927501274878535, 0, 17434927501264106475, 0, 0, 0, 0, 0, 0, 0, 0}, mask_was_saved = 0}}, priv = {pad = {
990
              0x0, 0x0, 0x0, 0x0}, data = {prev = 0x0, cleanup = 0x0, canceltype = 0}}}
991
        not_first_call = <optimized out>
992
#20 0x0000ffff9e58d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
993
No locals.
994
(gdb)
995
(gdb) quit
996
root@node2:/var/log# ll core.msgr-worker-*
997
-rw------- 1 root ceph 2538639360 Feb 27 19:57 core.msgr-worker-0.377736
998
-rw------- 1 root ceph 2537709568 Feb 27 19:57 core.msgr-worker-1.377732
999
-rw------- 1 root ceph 2564624384 Feb 27 19:57 core.msgr-worker-1.377734
1000
-rw------- 1 root ceph 2512400384 Feb 27 19:57 core.msgr-worker-1.377742
1001
-rw------- 1 root ceph 2469797888 Feb 27 19:57 core.msgr-worker-2.377725
1002
-rw------- 1 root ceph 2515800064 Feb 27 19:57 core.msgr-worker-2.377738
1003
-rw------- 1 root ceph 2513207296 Feb 27 19:57 core.msgr-worker-2.377740
1004
root@node2:/var/log# gdb /usr/bin/ceph-osd core.msgr-worker-1.377742
1005
GNU gdb (Ubuntu 8.1-0ubuntu3) 8.1.0.20180409-git
1006
Copyright (C) 2018 Free Software Foundation, Inc.
1007
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
1008
This is free software: you are free to change and redistribute it.
1009
There is NO WARRANTY, to the extent permitted by law.  Type "show copying"
1010
and "show warranty" for details.
1011
This GDB was configured as "aarch64-linux-gnu".
1012
Type "show configuration" for configuration details.
1013
For bug reporting instructions, please see:
1014
<http://www.gnu.org/software/gdb/bugs/>.
1015
Find the GDB manual and other documentation resources online at:
1016
<http://www.gnu.org/software/gdb/documentation/>.
1017
For help, type "help".
1018
Type "apropos word" to search for commands related to "word"...
1019
Reading symbols from /usr/bin/ceph-osd...Reading symbols from /usr/lib/debug/.build-id/fd/2d225baa3068ff217e8e576184a3a362e494fd.debug...done.
1020
done.
1021
[New LWP 377893]
1022
[New LWP 377995]
1023
[New LWP 377891]
1024
[New LWP 377765]
1025
[New LWP 379079]
1026
[New LWP 377742]
1027
[New LWP 377996]
1028
[New LWP 377997]
1029
[New LWP 379072]
1030
[New LWP 379074]
1031
[New LWP 379573]
1032
[New LWP 378408]
1033
[New LWP 379632]
1034
[New LWP 380792]
1035
[New LWP 379081]
1036
[New LWP 380797]
1037
[New LWP 380799]
1038
[New LWP 379077]
1039
[New LWP 379082]
1040
[New LWP 379633]
1041
[New LWP 380801]
1042
[New LWP 379629]
1043
[New LWP 379126]
1044
[New LWP 379630]
1045
[New LWP 380813]
1046
[New LWP 380805]
1047
[New LWP 379610]
1048
[New LWP 380821]
1049
[New LWP 380825]
1050
[New LWP 380830]
1051
[New LWP 380835]
1052
[New LWP 380838]
1053
[New LWP 379574]
1054
[New LWP 380850]
1055
[New LWP 380827]
1056
[New LWP 380851]
1057
[New LWP 379075]
1058
[New LWP 379127]
1059
[New LWP 380793]
1060
[New LWP 380817]
1061
[New LWP 380795]
1062
[New LWP 380828]
1063
[New LWP 380815]
1064
[New LWP 380855]
1065
[New LWP 380841]
1066
[New LWP 380820]
1067
[New LWP 380803]
1068
[New LWP 379073]
1069
[New LWP 377895]
1070
[New LWP 380834]
1071
[New LWP 380839]
1072
[New LWP 380840]
1073
[New LWP 380809]
1074
[New LWP 379125]
1075
[New LWP 379631]
1076
[New LWP 380845]
1077
[New LWP 380826]
1078
[New LWP 379078]
1079
[New LWP 380796]
1080
[New LWP 380843]
1081
[New LWP 380853]
1082
[New LWP 379628]
1083
[New LWP 380846]
1084
[New LWP 380833]
1085
[New LWP 380816]
1086
[New LWP 380808]
1087
[New LWP 380811]
1088
[New LWP 380852]
1089
[New LWP 380849]
1090
[New LWP 380818]
1091
[New LWP 380807]
1092
[New LWP 380831]
1093
[New LWP 380822]
1094
[New LWP 380802]
1095
[New LWP 380823]
1096
[New LWP 380844]
1097
[New LWP 380847]
1098
[New LWP 380836]
1099
[New LWP 380812]
1100
[New LWP 380806]
1101
[New LWP 380848]
1102
[New LWP 380837]
1103
[New LWP 380814]
1104
[New LWP 380794]
1105
[New LWP 380798]
1106
[New LWP 380804]
1107
[New LWP 380810]
1108
[New LWP 380842]
1109
[New LWP 380800]
1110
[New LWP 380832]
1111
[New LWP 380819]
1112
[New LWP 380824]
1113
[New LWP 380854]
1114
[New LWP 380829]
1115
[Thread debugging using libthread_db enabled]
1116
Using host libthread_db library "/lib/aarch64-linux-gnu/libthread_db.so.1".
1117
Core was generated by `/usr/bin/ceph-osd -f --cluster ceph --id 43 --setuser ceph --setgroup ceph'.
1118
Program terminated with signal SIGSEGV, Segmentation fault.
1119
#0  raise (sig=sig@entry=11) at ../sysdeps/unix/sysv/linux/raise.c:51
1120
51      ../sysdeps/unix/sysv/linux/raise.c: No such file or directory.
1121
[Current thread is 1 (Thread 0xffffbd0e59f0 (LWP 377893))]
1122
(gdb) bt
1123
#0  raise (sig=sig@entry=11) at ../sysdeps/unix/sysv/linux/raise.c:51
1124
#1  0x0000aaaacd0a3a1c in reraise_fatal (signum=11) at ./src/global/signal_handler.cc:326
1125
#2  handle_fatal_signal (signum=11) at ./src/global/signal_handler.cc:326
1126
#3  <signal handler called>
1127
#4  __memcpy_generic () at ../sysdeps/aarch64/multiarch/../memcpy.S:153
1128
#5  0x0000aaaacd46ad00 in memcpy (__len=73, __src=0xaaab54164000, __dest=<optimized out>) at /usr/include/aarch64-linux-gnu/bits/string_fortified.h:34
1129
#6  Infiniband::MemoryManager::Chunk::write (this=this@entry=0xaaab4bf09ba0, buf=buf@entry=0xaaab54164000 "\021\002)", len=<optimized out>)
1130
    at ./src/msg/async/rdma/Infiniband.cc:738
1131
#7  0x0000aaaacd479018 in RDMAConnectedSocketImpl::tx_copy_chunk (this=this@entry=0xaaab4d448d80, tx_buffers=std::vector of length 1, capacity 1 = {...},
1132
    req_copy_len=req_copy_len@entry=37064, start=..., end=...) at ./src/include/buffer.h:317
1133
#8  0x0000aaaacd47aaec in RDMAConnectedSocketImpl::submit (this=this@entry=0xaaab4d448d80, more=more@entry=false)
1134
    at ./src/msg/async/rdma/RDMAConnectedSocketImpl.cc:380
1135
#9  0x0000aaaacd47cc34 in RDMAConnectedSocketImpl::send (this=0xaaab4d448d80, bl=..., more=<optimized out>) at ./src/msg/async/rdma/RDMAConnectedSocketImpl.cc:297
1136
#10 0x0000aaaacd4184d4 in ConnectedSocket::send (more=false, bl=..., this=0xaaab5016aae8) at /usr/include/c++/9/bits/unique_ptr.h:357
1137
#11 AsyncConnection::_try_send (this=0xaaab5016a900, more=more@entry=false) at ./src/msg/async/AsyncConnection.cc:330
1138
#12 0x0000aaaacd44099c in ProtocolV2::write_message (this=this@entry=0xaaab50176000, m=m@entry=0xaaab4d3d2880, more=more@entry=false)
1139
    at ./src/msg/async/ProtocolV2.cc:538
1140
#13 0x0000aaaacd45b778 in ProtocolV2::write_event (this=0xaaab50176000) at ./src/msg/async/ProtocolV2.cc:637
1141
#14 0x0000aaaacd4159e8 in AsyncConnection::handle_write (this=0xaaab5016a900) at /usr/include/c++/9/bits/unique_ptr.h:357
1142
#15 0x0000aaaacd26c570 in EventCenter::process_events (this=this@entry=0xaaaaf6d808c8, timeout_microseconds=<optimized out>,
1143
    working_dur=working_dur@entry=0xffffbd0e5160) at /usr/include/c++/9/bits/stl_deque.h:2110
1144
#16 0x0000aaaacd2720f0 in NetworkStack::<lambda()>::operator() (__closure=0xaaaaf6dfa4b8, __closure=0xaaaaf6dfa4b8) at ./src/msg/async/Stack.cc:53
1145
#17 std::_Function_handler<void(), NetworkStack::add_thread(unsigned int)::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...)
1146
    at /usr/include/c++/9/bits/std_function.h:300
1147
#18 0x0000ffffbe569ed4 in ?? () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
1148
#19 0x0000ffffbe71f088 in start_thread (arg=0xffffe1a1326f) at pthread_create.c:463
1149
#20 0x0000ffffbe3394ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
1150
(gdb) f 6
1151
#6  Infiniband::MemoryManager::Chunk::write (this=this@entry=0xaaab4bf09ba0, buf=buf@entry=0xaaab54164000 "\021\002)", len=<optimized out>)
1152
    at ./src/msg/async/rdma/Infiniband.cc:738
1153
738     ./src/msg/async/rdma/Infiniband.cc: No such file or directory.
1154
(gdb) p *this
1155
$1 = {mr = 0xaaaaf61b8000, qp = 0xaaaaf614a6f0, lkey = 2, bytes = 43690, offset = 4142424832, bound = 43690, buffer = 0xaaab00000001 "", pad = {4128550976, 43690,
1156
    4128640968, 43690, 1354679688, 43691}, data = 0xaaab4bf09be0 ""}
1157
(gdb) quit
1158
root@node2:/var/log# gdb /usr/bin/ceph-osd core.msgr-worker-1.377734
1159
GNU gdb (Ubuntu 8.1-0ubuntu3) 8.1.0.20180409-git
1160
Copyright (C) 2018 Free Software Foundation, Inc.
1161
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
1162
This is free software: you are free to change and redistribute it.
1163
There is NO WARRANTY, to the extent permitted by law.  Type "show copying"
1164
and "show warranty" for details.
1165
This GDB was configured as "aarch64-linux-gnu".
1166
Type "show configuration" for configuration details.
1167
For bug reporting instructions, please see:
1168
<http://www.gnu.org/software/gdb/bugs/>.
1169
Find the GDB manual and other documentation resources online at:
1170
<http://www.gnu.org/software/gdb/documentation/>.
1171
For help, type "help".
1172
Type "apropos word" to search for commands related to "word"...
1173
Reading symbols from /usr/bin/ceph-osd...Reading symbols from /usr/lib/debug/.build-id/fd/2d225baa3068ff217e8e576184a3a362e494fd.debug...done.
1174
done.
1175
[New LWP 377845]
1176
[New LWP 377734]
1177
[New LWP 377844]
1178
[New LWP 378042]
1179
[New LWP 377846]
1180
[New LWP 378041]
1181
[New LWP 378292]
1182
[New LWP 378726]
1183
[New LWP 378715]
1184
[New LWP 378717]
1185
[New LWP 378734]
1186
[New LWP 378742]
1187
[New LWP 377758]
1188
[New LWP 379673]
1189
[New LWP 379738]
1190
[New LWP 379749]
1191
[New LWP 378732]
1192
[New LWP 378733]
1193
[New LWP 379748]
1194
[New LWP 379750]
1195
[New LWP 381112]
1196
[New LWP 379755]
1197
[New LWP 378795]
1198
[New LWP 381114]
1199
[New LWP 381115]
1200
[New LWP 381118]
1201
[New LWP 381113]
1202
[New LWP 381119]
1203
[New LWP 378744]
1204
[New LWP 381117]
1205
[New LWP 381120]
1206
[New LWP 378730]
1207
[New LWP 379739]
1208
[New LWP 381163]
1209
[New LWP 381166]
1210
[New LWP 381125]
1211
[New LWP 379684]
1212
[New LWP 378796]
1213
[New LWP 379676]
1214
[New LWP 381164]
1215
[New LWP 381157]
1216
[New LWP 381165]
1217
[New LWP 381159]
1218
[New LWP 381143]
1219
[New LWP 381137]
1220
[New LWP 381126]
1221
[New LWP 381145]
1222
[New LWP 381180]
1223
[New LWP 381150]
1224
[New LWP 381133]
1225
[New LWP 381123]
1226
[New LWP 381124]
1227
[New LWP 381132]
1228
[New LWP 378797]
1229
[New LWP 381136]
1230
[New LWP 381138]
1231
[New LWP 381135]
1232
[New LWP 381139]
1233
[New LWP 381116]
1234
[New LWP 381149]
1235
[New LWP 381168]
1236
[New LWP 381146]
1237
[New LWP 381174]
1238
[New LWP 381144]
1239
[New LWP 381152]
1240
[New LWP 381134]
1241
[New LWP 381161]
1242
[New LWP 381148]
1243
[New LWP 381175]
1244
[New LWP 381121]
1245
[New LWP 381156]
1246
[New LWP 381153]
1247
[New LWP 381122]
1248
[New LWP 381169]
1249
[New LWP 381179]
1250
[New LWP 381147]
1251
[New LWP 381167]
1252
[New LWP 381170]
1253
[New LWP 381158]
1254
[New LWP 381151]
1255
[New LWP 381177]
1256
[New LWP 381141]
1257
[New LWP 381178]
1258
[New LWP 381162]
1259
[New LWP 381176]
1260
[New LWP 381173]
1261
[New LWP 381155]
1262
[New LWP 381160]
1263
[New LWP 381172]
1264
[New LWP 381154]
1265
[New LWP 381171]
1266
[New LWP 381142]
1267
[New LWP 381140]
1268
[New LWP 378010]
1269
[Thread debugging using libthread_db enabled]
1270
Using host libthread_db library "/lib/aarch64-linux-gnu/libthread_db.so.1".
1271
Core was generated by `/usr/bin/ceph-osd -f --cluster ceph --id 35 --setuser ceph --setgroup ceph'.
1272
Program terminated with signal SIGSEGV, Segmentation fault.
1273
#0  raise (sig=sig@entry=11) at ../sysdeps/unix/sysv/linux/raise.c:51
1274
51      ../sysdeps/unix/sysv/linux/raise.c: No such file or directory.
1275
[Current thread is 1 (Thread 0xffff9d3399f0 (LWP 377845))]
1276
(gdb) bt
1277
#0  raise (sig=sig@entry=11) at ../sysdeps/unix/sysv/linux/raise.c:51
1278
#1  0x0000aaaae12c1a1c in reraise_fatal (signum=11) at ./src/global/signal_handler.cc:326
1279
#2  handle_fatal_signal (signum=11) at ./src/global/signal_handler.cc:326
1280
#3  <signal handler called>
1281
#4  __memcpy_generic () at ../sysdeps/aarch64/multiarch/../memcpy.S:153
1282
#5  0x0000aaaae1688d00 in memcpy (__len=73, __src=0xaaab5dc4c000, __dest=<optimized out>) at /usr/include/aarch64-linux-gnu/bits/string_fortified.h:34
1283
#6  Infiniband::MemoryManager::Chunk::write (this=this@entry=0xaaab5c679ba0, buf=buf@entry=0xaaab5dc4c000 "\021\002)", len=<optimized out>)
1284
    at ./src/msg/async/rdma/Infiniband.cc:738
1285
#7  0x0000aaaae1697018 in RDMAConnectedSocketImpl::tx_copy_chunk (this=this@entry=0xaaab5e785f80, tx_buffers=std::vector of length 1, capacity 1 = {...},
1286
    req_copy_len=req_copy_len@entry=1086, start=..., end=...) at ./src/include/buffer.h:317
1287
#8  0x0000aaaae1698aec in RDMAConnectedSocketImpl::submit (this=this@entry=0xaaab5e785f80, more=more@entry=false)
1288
    at ./src/msg/async/rdma/RDMAConnectedSocketImpl.cc:380
1289
#9  0x0000aaaae169ac34 in RDMAConnectedSocketImpl::send (this=0xaaab5e785f80, bl=..., more=<optimized out>) at ./src/msg/async/rdma/RDMAConnectedSocketImpl.cc:297
1290
#10 0x0000aaaae16364d4 in ConnectedSocket::send (more=false, bl=..., this=0xaaab5d172f68) at /usr/include/c++/9/bits/unique_ptr.h:357
1291
#11 AsyncConnection::_try_send (this=0xaaab5d172d80, more=more@entry=false) at ./src/msg/async/AsyncConnection.cc:330
1292
q
1293
#12 0x0000aaaae165e99c in ProtocolV2::write_message (this=this@entry=0xaaab5dd8e580, m=m@entry=0xaaab5f6cb500, more=more@entry=false)
1294
    at ./src/msg/async/ProtocolV2.cc:538
1295
#13 0x0000aaaae1679778 in ProtocolV2::write_event (this=0xaaab5dd8e580) at ./src/msg/async/ProtocolV2.cc:637
1296
#14 0x0000aaaae16339e8 in AsyncConnection::handle_write (this=0xaaab5d172d80) at /usr/include/c++/9/bits/unique_ptr.h:357
1297
#15 0x0000aaaae148a570 in EventCenter::process_events (this=this@entry=0xaaaafef728c8, timeout_microseconds=<optimized out>,
1298
    working_dur=working_dur@entry=0xffff9d339160) at /usr/include/c++/9/bits/stl_deque.h:2110
1299
#16 0x0000aaaae14900f0 in NetworkStack::<lambda()>::operator() (__closure=0xaaaafefe24b8, __closure=0xaaaafefe24b8) at ./src/msg/async/Stack.cc:53
1300
#17 std::_Function_handler<void(), NetworkStack::add_thread(unsigned int)::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...)
1301
    at /usr/include/c++/9/bits/std_function.h:300
1302
#18 0x0000ffff9e7bded4 in ?? () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
1303
#19 0x0000ffff9e973088 in start_thread (arg=0xfffff56c646f) at pthread_create.c:463
1304
#20 0x0000ffff9e58d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
1305
(gdb) q
1306
root@node2:/var/log# gdb /usr/bin/ceph-osd core.msgr-worker-1.377734
1307
GNU gdb (Ubuntu 8.1-0ubuntu3) 8.1.0.20180409-git
1308
Copyright (C) 2018 Free Software Foundation, Inc.
1309
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
1310
This is free software: you are free to change and redistribute it.
1311
There is NO WARRANTY, to the extent permitted by law.  Type "show copying"
1312
and "show warranty" for details.
1313
This GDB was configured as "aarch64-linux-gnu".
1314
Type "show configuration" for configuration details.
1315
For bug reporting instructions, please see:
1316
<http://www.gnu.org/software/gdb/bugs/>.
1317
Find the GDB manual and other documentation resources online at:
1318
<http://www.gnu.org/software/gdb/documentation/>.
1319
For help, type "help".
1320
Type "apropos word" to search for commands related to "word"...
1321
Reading symbols from /usr/bin/ceph-osd...Reading symbols from /usr/lib/debug/.build-id/fd/2d225baa3068ff217e8e576184a3a362e494fd.debug...
1322
done.
1323
done.
1324
[New LWP 377845]
1325
[New LWP 377734]
1326
[New LWP 377844]
1327
[New LWP 378042]
1328
[New LWP 377846]
1329
[New LWP 378041]
1330
[New LWP 378292]
1331
[New LWP 378726]
1332
[New LWP 378715]
1333
[New LWP 378717]
1334
[New LWP 378734]
1335
[New LWP 378742]
1336
[New LWP 377758]
1337
[New LWP 379673]
1338
[New LWP 379738]
1339
[New LWP 379749]
1340
[New LWP 378732]
1341
[New LWP 378733]
1342
[New LWP 379748]
1343
[New LWP 379750]
1344
[New LWP 381112]
1345
[New LWP 379755]
1346
[New LWP 378795]
1347
[New LWP 381114]
1348
[New LWP 381115]
1349
[New LWP 381118]
1350
[New LWP 381113]
1351
[New LWP 381119]
1352
[New LWP 378744]
1353
[New LWP 381117]
1354
[New LWP 381120]
1355
[New LWP 378730]
1356
[New LWP 379739]
1357
[New LWP 381163]
1358
[New LWP 381166]
1359
[New LWP 381125]
1360
[New LWP 379684]
1361
[New LWP 378796]
1362
[New LWP 379676]
1363
[New LWP 381164]
1364
[New LWP 381157]
1365
[New LWP 381165]
1366
[New LWP 381159]
1367
[New LWP 381143]
1368
[New LWP 381137]
1369
[New LWP 381126]
1370
[New LWP 381145]
1371
[New LWP 381180]
1372
[New LWP 381150]
1373
[New LWP 381133]
1374
[New LWP 381123]
1375
[New LWP 381124]
1376
[New LWP 381132]
1377
[New LWP 378797]
1378
[New LWP 381136]
1379
[New LWP 381138]
1380
[New LWP 381135]
1381
[New LWP 381139]
1382
[New LWP 381116]
1383
[New LWP 381149]
1384
[New LWP 381168]
1385
[New LWP 381146]
1386
[New LWP 381174]
1387
[New LWP 381144]
1388
[New LWP 381152]
1389
[New LWP 381134]
1390
[New LWP 381161]
1391
[New LWP 381148]
1392
[New LWP 381175]
1393
[New LWP 381121]
1394
[New LWP 381156]
1395
[New LWP 381153]
1396
[New LWP 381122]
1397
[New LWP 381169]
1398
[New LWP 381179]
1399
[New LWP 381147]
1400
[New LWP 381167]
1401
[New LWP 381170]
1402
[New LWP 381158]
1403
[New LWP 381151]
1404
[New LWP 381177]
1405
[New LWP 381141]
1406
[New LWP 381178]
1407
[New LWP 381162]
1408
[New LWP 381176]
1409
[New LWP 381173]
1410
[New LWP 381155]
1411
[New LWP 381160]
1412
[New LWP 381172]
1413
[New LWP 381154]
1414
[New LWP 381171]
1415
[New LWP 381142]
1416
[New LWP 381140]
1417
[New LWP 378010]
1418
[Thread debugging using libthread_db enabled]
1419
Using host libthread_db library "/lib/aarch64-linux-gnu/libthread_db.so.1".
1420
Core was generated by `/usr/bin/ceph-osd -f --cluster ceph --id 35 --setuser ceph --setgroup ceph'.
1421
Program terminated with signal SIGSEGV, Segmentation fault.
1422
#0  raise (sig=sig@entry=11) at ../sysdeps/unix/sysv/linux/raise.c:51
1423
51      ../sysdeps/unix/sysv/linux/raise.c: No such file or directory.
1424
[Current thread is 1 (Thread 0xffff9d3399f0 (LWP 377845))]
1425
(gdb)
1426
(gdb) f 6
1427
#6  Infiniband::MemoryManager::Chunk::write (this=this@entry=0xaaab5c679ba0, buf=buf@entry=0xaaab5dc4c000 "\021\002)", len=<optimized out>)
1428
    at ./src/msg/async/rdma/Infiniband.cc:738
1429
738     ./src/msg/async/rdma/Infiniband.cc: No such file or directory.
1430
(gdb) p *this
1431
$1 = {mr = 0xaaaafe3aa000, qp = 0xaaaafe33c6f0, lkey = 2, bytes = 43690, offset = 4278641408, bound = 43690, buffer = 0xaaab00000001 "ceph v2\n\020", pad = {
1432
    4264808512, 43690, 4264898504, 43690, 1424196552, 43691}, data = 0xaaab5c679be0 ""}
1433
(gdb) quit
1434
root@node2:/var/log# gdb /usr/bin/ceph-osd core.msgr-worker-1.377734^C
1435
root@node2:/var/log# ll core.msgr-worker-*
1436
-rw------- 1 root ceph 2538639360 Feb 27 19:57 core.msgr-worker-0.377736
1437
-rw------- 1 root ceph 2537709568 Feb 27 19:57 core.msgr-worker-1.377732
1438
-rw------- 1 root ceph 2564624384 Feb 27 19:57 core.msgr-worker-1.377734
1439
-rw------- 1 root ceph 2512400384 Feb 27 19:57 core.msgr-worker-1.377742
1440
-rw------- 1 root ceph 2469797888 Feb 27 19:57 core.msgr-worker-2.377725
1441
-rw------- 1 root ceph 2515800064 Feb 27 19:57 core.msgr-worker-2.377738
1442
-rw------- 1 root ceph 2513207296 Feb 27 19:57 core.msgr-worker-2.377740
1443
root@node2:/var/log# gdb /usr/bin/ceph-osd core.msgr-worker-2.377738
1444
GNU gdb (Ubuntu 8.1-0ubuntu3) 8.1.0.20180409-git
1445
Copyright (C) 2018 Free Software Foundation, Inc.
1446
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
1447
This is free software: you are free to change and redistribute it.
1448
There is NO WARRANTY, to the extent permitted by law.  Type "show copying"
1449
and "show warranty" for details.
1450
This GDB was configured as "aarch64-linux-gnu".
1451
Type "show configuration" for configuration details.
1452
For bug reporting instructions, please see:
1453
<http://www.gnu.org/software/gdb/bugs/>.
1454
Find the GDB manual and other documentation resources online at:
1455
<http://www.gnu.org/software/gdb/documentation/>.
1456
For help, type "help".
1457
Type "apropos word" to search for commands related to "word"...
1458
Reading symbols from /usr/bin/ceph-osd...Reading symbols from /usr/lib/debug/.build-id/fd/2d225baa3068ff217e8e576184a3a362e494fd.debug...done.
1459
done.
1460
[New LWP 377874]
1461
[New LWP 377738]
1462
[New LWP 377872]
1463
[New LWP 377762]
1464
[New LWP 377967]
1465
[New LWP 377873]
1466
[New LWP 377968]
1467
[New LWP 378481]
1468
[New LWP 378484]
1469
[New LWP 377969]
1470
[New LWP 378485]
1471
[New LWP 378488]
1472
[New LWP 378490]
1473
[New LWP 378482]
1474
[New LWP 379145]
1475
[New LWP 378486]
1476
[New LWP 378487]
1477
[New LWP 379167]
1478
[New LWP 379169]
1479
[New LWP 378496]
1480
[New LWP 378491]
1481
[New LWP 379148]
1482
[New LWP 378497]
1483
[New LWP 378495]
1484
[New LWP 379172]
1485
[New LWP 379146]
1486
[New LWP 380215]
1487
[New LWP 380218]
1488
[New LWP 380217]
1489
[New LWP 380220]
1490
[New LWP 379170]
1491
[New LWP 380221]
1492
[New LWP 380226]
1493
[New LWP 380225]
1494
[New LWP 380243]
1495
[New LWP 380246]
1496
[New LWP 380250]
1497
[New LWP 380248]
1498
[New LWP 380222]
1499
[New LWP 380260]
1500
[New LWP 380223]
1501
[New LWP 380254]
1502
[New LWP 380227]
1503
[New LWP 380229]
1504
[New LWP 380216]
1505
[New LWP 380262]
1506
[New LWP 380228]
1507
[New LWP 380273]
1508
[New LWP 380230]
1509
[New LWP 380224]
1510
[New LWP 379171]
1511
[New LWP 380231]
1512
[New LWP 380240]
1513
[New LWP 380268]
1514
[New LWP 380242]
1515
[New LWP 379168]
1516
[New LWP 380238]
1517
[New LWP 380235]
1518
[New LWP 380258]
1519
[New LWP 380239]
1520
[New LWP 380233]
1521
[New LWP 380256]
1522
[New LWP 380249]
1523
[New LWP 380272]
1524
[New LWP 380259]
1525
[New LWP 380264]
1526
[New LWP 380232]
1527
[New LWP 380234]
1528
[New LWP 380219]
1529
[New LWP 380245]
1530
[New LWP 380267]
1531
[New LWP 380244]
1532
[New LWP 380236]
1533
[New LWP 380271]
1534
[New LWP 380278]
1535
[New LWP 380251]
1536
[New LWP 380255]
1537
[New LWP 380265]
1538
[New LWP 380275]
1539
[New LWP 380274]
1540
[New LWP 380257]
1541
[New LWP 380263]
1542
[New LWP 380270]
1543
[New LWP 380269]
1544
[New LWP 380247]
1545
[New LWP 380276]
1546
[New LWP 380252]
1547
[New LWP 380277]
1548
[New LWP 380266]
1549
[New LWP 380253]
1550
[New LWP 380237]
1551
[New LWP 380241]
1552
[New LWP 378222]
1553
[New LWP 380261]
1554
[Thread debugging using libthread_db enabled]
1555
Using host libthread_db library "/lib/aarch64-linux-gnu/libthread_db.so.1".
1556
Core was generated by `/usr/bin/ceph-osd -f --cluster ceph --id 39 --setuser ceph --setgroup ceph'.
1557
Program terminated with signal SIGSEGV, Segmentation fault.
1558
#0  raise (sig=sig@entry=11) at ../sysdeps/unix/sysv/linux/raise.c:51
1559
51      ../sysdeps/unix/sysv/linux/raise.c: No such file or directory.
1560
[Current thread is 1 (Thread 0xffff934209f0 (LWP 377874))]
1561
(gdb) bt
1562
#0  raise (sig=sig@entry=11) at ../sysdeps/unix/sysv/linux/raise.c:51
1563
#1  0x0000aaaabc055a1c in reraise_fatal (signum=11) at ./src/global/signal_handler.cc:326
1564
#2  handle_fatal_signal (signum=11) at ./src/global/signal_handler.cc:326
1565
#3  <signal handler called>
1566
#4  0x0000aaaabc42bcf4 in RDMAConnectedSocketImpl::post_work_request (this=this@entry=0xaaab5141c400, tx_buffers=std::vector of length 1, capacity 1 = {...})
1567
    at ./src/msg/async/rdma/RDMAConnectedSocketImpl.cc:424
1568
#5  0x0000aaaabc42c9a4 in RDMAConnectedSocketImpl::submit (this=this@entry=0xaaab5141c400, more=more@entry=false)
1569
    at ./src/msg/async/rdma/RDMAConnectedSocketImpl.cc:398
1570
#6  0x0000aaaabc42ec34 in RDMAConnectedSocketImpl::send (this=0xaaab5141c400, bl=..., more=<optimized out>) at ./src/msg/async/rdma/RDMAConnectedSocketImpl.cc:297
1571
#7  0x0000aaaabc3ca4d4 in ConnectedSocket::send (more=false, bl=..., this=0xaaab52b94ee8) at /usr/include/c++/9/bits/unique_ptr.h:357
1572
#8  AsyncConnection::_try_send (this=0xaaab52b94d00, more=more@entry=false) at ./src/msg/async/AsyncConnection.cc:330
1573
#9  0x0000aaaabc3f299c in ProtocolV2::write_message (this=this@entry=0xaaab5829e000, m=m@entry=0xaaab58602a00, more=more@entry=false)
1574
    at ./src/msg/async/ProtocolV2.cc:538
1575
#10 0x0000aaaabc40d778 in ProtocolV2::write_event (this=0xaaab5829e000) at ./src/msg/async/ProtocolV2.cc:637
1576
#11 0x0000aaaabc3c79e8 in AsyncConnection::handle_write (this=0xaaab52b94d00) at /usr/include/c++/9/bits/unique_ptr.h:357
1577
#12 0x0000aaaabc21e570 in EventCenter::process_events (this=this@entry=0xaaaafad94b88, timeout_microseconds=<optimized out>,
1578
    working_dur=working_dur@entry=0xffff93420160) at /usr/include/c++/9/bits/stl_deque.h:2110
1579
#13 0x0000aaaabc2240f0 in NetworkStack::<lambda()>::operator() (__closure=0xaaaafae044e8, __closure=0xaaaafae044e8) at ./src/msg/async/Stack.cc:53
1580
#14 std::_Function_handler<void(), NetworkStack::add_thread(unsigned int)::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...)
1581
    at /usr/include/c++/9/bits/std_function.h:300
1582
#15 0x0000ffff950a5ed4 in ?? () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
1583
#16 0x0000ffff9525b088 in start_thread (arg=0xffffdbe9135f) at pthread_create.c:463
1584
#17 0x0000ffff94e754ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
1585
(gdb) f 4
1586
#4  0x0000aaaabc42bcf4 in RDMAConnectedSocketImpl::post_work_request (this=this@entry=0xaaab5141c400, tx_buffers=std::vector of length 1, capacity 1 = {...})
1587
    at ./src/msg/async/rdma/RDMAConnectedSocketImpl.cc:424
1588
424     ./src/msg/async/rdma/RDMAConnectedSocketImpl.cc: No such file or directory.
1589
(gdb) p *tx_buffers[0]
1590
$1 = {mr = 0x0, qp = 0xaaaafa15e6f0, lkey = 2, bytes = 43691, offset = 2123, bound = 43691, buffer = 0xaaab00000001 "\021\002)", pad = {4195741760, 43690,
1591
    4195831752, 43690, 1433035720, 43691}, data = 0xaaab54c8f780 ""}
1592
(gdb) quit
1593
root@node2:/var/log# ll core.msgr-worker-*
1594
-rw------- 1 root ceph 2538639360 Feb 27 19:57 core.msgr-worker-0.377736
1595
-rw------- 1 root ceph 2537709568 Feb 27 19:57 core.msgr-worker-1.377732
1596
-rw------- 1 root ceph 2564624384 Feb 27 19:57 core.msgr-worker-1.377734
1597
-rw------- 1 root ceph 2512400384 Feb 27 19:57 core.msgr-worker-1.377742
1598
-rw------- 1 root ceph 2469797888 Feb 27 19:57 core.msgr-worker-2.377725
1599
-rw------- 1 root ceph 2515800064 Feb 27 19:57 core.msgr-worker-2.377738
1600
-rw------- 1 root ceph 2513207296 Feb 27 19:57 core.msgr-worker-2.377740
1601
root@node2:/var/log# gdb /usr/bin/ceph-osd core.msgr-worker-2.377740
1602
GNU gdb (Ubuntu 8.1-0ubuntu3) 8.1.0.20180409-git
1603
Copyright (C) 2018 Free Software Foundation, Inc.
1604
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
1605
This is free software: you are free to change and redistribute it.
1606
There is NO WARRANTY, to the extent permitted by law.  Type "show copying"
1607
and "show warranty" for details.
1608
This GDB was configured as "aarch64-linux-gnu".
1609
Type "show configuration" for configuration details.
1610
For bug reporting instructions, please see:
1611
<http://www.gnu.org/software/gdb/bugs/>.
1612
Find the GDB manual and other documentation resources online at:
1613
<http://www.gnu.org/software/gdb/documentation/>.
1614
For help, type "help".
1615
Type "apropos word" to search for commands related to "word"...
1616
Reading symbols from /usr/bin/ceph-osd...Reading symbols from /usr/lib/debug/.build-id/fd/2d225baa3068ff217e8e576184a3a362e494fd.debug...done.
1617
done.
1618
[New LWP 377894]
1619
[New LWP 377892]
1620
[New LWP 377764]
1621
[New LWP 379320]
1622
[New LWP 377740]
1623
[New LWP 378586]
1624
[New LWP 378103]
1625
[New LWP 379324]
1626
[New LWP 379325]
1627
[New LWP 378105]
1628
[New LWP 379319]
1629
[New LWP 379333]
1630
[New LWP 379328]
1631
[New LWP 379326]
1632
[New LWP 379934]
1633
[New LWP 381128]
1634
[New LWP 379428]
1635
[New LWP 379429]
1636
[New LWP 381182]
1637
[New LWP 381184]
1638
[New LWP 381186]
1639
[New LWP 379882]
1640
[New LWP 379866]
1641
[New LWP 381203]
1642
[New LWP 379430]
1643
[New LWP 381204]
1644
[New LWP 381206]
1645
[New LWP 381223]
1646
[New LWP 379945]
1647
[New LWP 381226]
1648
[New LWP 381213]
1649
[New LWP 381215]
1650
[New LWP 381191]
1651
[New LWP 381195]
1652
[New LWP 379327]
1653
[New LWP 379334]
1654
[New LWP 381228]
1655
[New LWP 381192]
1656
[New LWP 381210]
1657
[New LWP 379864]
1658
[New LWP 381197]
1659
[New LWP 381196]
1660
[New LWP 381127]
1661
[New LWP 381188]
1662
[New LWP 379943]
1663
[New LWP 381194]
1664
[New LWP 381200]
1665
[New LWP 381220]
1666
[New LWP 381218]
1667
[New LWP 381205]
1668
[New LWP 381190]
1669
[New LWP 381222]
1670
[New LWP 381219]
1671
[New LWP 379936]
1672
[New LWP 381198]
1673
[New LWP 381236]
1674
[New LWP 381216]
1675
[New LWP 381193]
1676
[New LWP 381130]
1677
[New LWP 381207]
1678
[New LWP 381129]
1679
[New LWP 381234]
1680
[New LWP 381214]
1681
[New LWP 381208]
1682
[New LWP 381230]
1683
[New LWP 381227]
1684
[New LWP 381233]
1685
[New LWP 381199]
1686
[New LWP 381229]
1687
[New LWP 381224]
1688
[New LWP 381212]
1689
[New LWP 381202]
1690
[New LWP 381181]
1691
[New LWP 379947]
1692
[New LWP 381183]
1693
[New LWP 381235]
1694
[New LWP 377890]
1695
[New LWP 381232]
1696
[New LWP 381238]
1697
[New LWP 381221]
1698
[New LWP 381237]
1699
[New LWP 381131]
1700
[New LWP 381189]
1701
[New LWP 381217]
1702
[New LWP 379944]
1703
[New LWP 381201]
1704
[New LWP 381185]
1705
[New LWP 381239]
1706
[New LWP 381231]
1707
[New LWP 381225]
1708
[New LWP 381187]
1709
[New LWP 381211]
1710
[New LWP 381209]
1711
[New LWP 378073]
1712
[Thread debugging using libthread_db enabled]
1713
Using host libthread_db library "/lib/aarch64-linux-gnu/libthread_db.so.1".
1714
Core was generated by `/usr/bin/ceph-osd -f --cluster ceph --id 41 --setuser ceph --setgroup ceph'.
1715
Program terminated with signal SIGSEGV, Segmentation fault.
1716
#0  raise (sig=sig@entry=11) at ../sysdeps/unix/sysv/linux/raise.c:51
1717
51      ../sysdeps/unix/sysv/linux/raise.c: No such file or directory.
1718
[Current thread is 1 (Thread 0xffffa3ce89f0 (LWP 377894))]
1719
(gdb) bt
1720
#0  raise (sig=sig@entry=11) at ../sysdeps/unix/sysv/linux/raise.c:51
1721
#1  0x0000aaaae66dea1c in reraise_fatal (signum=11) at ./src/global/signal_handler.cc:326
1722
#2  handle_fatal_signal (signum=11) at ./src/global/signal_handler.cc:326
1723
#3  <signal handler called>
1724
#4  __memcpy_generic () at ../sysdeps/aarch64/multiarch/../memcpy.S:105
1725
#5  0x0000aaaae6aa5d00 in memcpy (__len=40, __src=0xaaab4a31a000, __dest=<optimized out>) at /usr/include/aarch64-linux-gnu/bits/string_fortified.h:34
1726
#6  Infiniband::MemoryManager::Chunk::write (this=this@entry=0xaaab4ba39ba0, buf=buf@entry=0xaaab4a31a000 "\024\001\b", len=<optimized out>)
1727
    at ./src/msg/async/rdma/Infiniband.cc:738
1728
#7  0x0000aaaae6ab4018 in RDMAConnectedSocketImpl::tx_copy_chunk (this=this@entry=0xaaab4f1794e0, tx_buffers=std::vector of length 1, capacity 1 = {...},
1729
    req_copy_len=req_copy_len@entry=57, start=..., end=...) at ./src/include/buffer.h:317
1730
#8  0x0000aaaae6ab5aec in RDMAConnectedSocketImpl::submit (this=this@entry=0xaaab4f1794e0, more=more@entry=false)
1731
    at ./src/msg/async/rdma/RDMAConnectedSocketImpl.cc:380
1732
#9  0x0000aaaae6ab7c34 in RDMAConnectedSocketImpl::send (this=0xaaab4f1794e0, bl=..., more=<optimized out>) at ./src/msg/async/rdma/RDMAConnectedSocketImpl.cc:297
1733
#10 0x0000aaaae6a534d4 in ConnectedSocket::send (more=false, bl=..., this=0xaaab4a5781e8) at /usr/include/c++/9/bits/unique_ptr.h:357
1734
#11 AsyncConnection::_try_send (this=0xaaab4a578000, more=false) at ./src/msg/async/AsyncConnection.cc:330
1735
#12 0x0000aaaae6a96bf8 in ProtocolV2::write_event (this=0xaaab4f170680) at ./src/msg/async/ProtocolV2.cc:663
1736
#13 0x0000aaaae6a509e8 in AsyncConnection::handle_write (this=0xaaab4a578000) at /usr/include/c++/9/bits/unique_ptr.h:357
1737
#14 0x0000aaaae68a7570 in EventCenter::process_events (this=this@entry=0xaaaaf26a8b88, timeout_microseconds=<optimized out>,
1738
    working_dur=working_dur@entry=0xffffa3ce8160) at /usr/include/c++/9/bits/stl_deque.h:2110
1739
#15 0x0000aaaae68ad0f0 in NetworkStack::<lambda()>::operator() (__closure=0xaaaaf27184e8, __closure=0xaaaaf27184e8) at ./src/msg/async/Stack.cc:53
1740
#16 std::_Function_handler<void(), NetworkStack::add_thread(unsigned int)::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...)
1741
    at /usr/include/c++/9/bits/std_function.h:300
1742
#17 0x0000ffffa596ded4 in ?? () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
1743
#18 0x0000ffffa5b23088 in start_thread (arg=0xffffeb90520f) at pthread_create.c:463
1744
#19 0x0000ffffa573d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
1745
(gdb) f 6
1746
#6  Infiniband::MemoryManager::Chunk::write (this=this@entry=0xaaab4ba39ba0, buf=buf@entry=0xaaab4a31a000 "\024\001\b", len=<optimized out>)
1747
    at ./src/msg/async/rdma/Infiniband.cc:738
1748
738     ./src/msg/async/rdma/Infiniband.cc: No such file or directory.
1749
(gdb) p  *this
1750
$1 = {mr = 0x0, qp = 0xaaab4a4bdc20, lkey = 1246485952, bytes = 43691, offset = 1339179008, bound = 43691, buffer = 0xaaab4ba39bd0 "smtp_port", pad = {9, 0,
1751
    1886678387, 1919905887, 1233911924, 43691}, data = 0xaaab4ba39be0 "\360\233\243K\253\252"}
1752
(gdb) bt full
1753
#0  raise (sig=sig@entry=11) at ../sysdeps/unix/sysv/linux/raise.c:51
1754
        set = {__val = {4096, 281473429948200, 88, 281473429931264, 187650989138840, 187651175022592, 187651003297792, 525377, 420, 88, 281473429931264,
1755
            281473429931264, 281473429931248, 4294967288, 11758279843738548992, 187651175022592}}
1756
        pid = <optimized out>
1757
        tid = <optimized out>
1758
        ret = <optimized out>
1759
#1  0x0000aaaae66dea1c in reraise_fatal (signum=11) at ./src/global/signal_handler.cc:326
1760
        ret = <optimized out>
1761
        buf = "/var/lib/ceph/crash/2020-02-27T11:57:05.396605Z_9795c05d-2c82-4ba5-8d2e-270432f75ecd/log", '\000' <repeats 144 times>...
1762
#2  handle_fatal_signal (signum=11) at ./src/global/signal_handler.cc:326
1763
        buf = "*** Caught signal (Segmentation fault) **\n in thread ffffa3ce89f0 thread_name:msgr-worker-2\n", '\000' <repeats 931 times>
1764
        pthread_name = "msgr-worker-2\000\000"
1765
        r = <optimized out>
1766
        bt = {static max = 100, skip = 1, array = {0xaaaae66de954 <handle_fatal_signal(int)+204>, 0xffffa601067c <__kernel_rt_sigreturn>,
1767
            0xffffa56f01c8 <__memcpy_generic+56>, 0xaaaae6aa5d00 <Infiniband::MemoryManager::Chunk::write(char*, unsigned int)+48>,
1768
            0xaaaae6ab4018 <RDMAConnectedSocketImpl::tx_copy_chunk(std::vector<Infiniband::MemoryManager::Chunk*, std::allocator<Infiniband::MemoryManager::Chunk*> >&, unsigned long, ceph::buffer::v14_2_0::list::buffers_t::buffers_iterator<ceph::buffer::v14_2_0::ptr_node const>&, ceph::buffer::v14_2_0::list::buffers_t::buffers_iterator<ceph::buffer::v14_2_0::ptr_node const> const&)+192>, 0xaaaae6ab5aec <RDMAConnectedSocketImpl::submit(bool)+924>,
1769
            0xaaaae6ab7c34 <RDMAConnectedSocketImpl::send(ceph::buffer::v14_2_0::list&, bool)+244>, 0xaaaae6a534d4 <AsyncConnection::_try_send(bool)+172>,
1770
            0xaaaae6a96bf8 <ProtocolV2::write_event()+1568>, 0xaaaae6a509e8 <AsyncConnection::handle_write()+72>,
1771
            0xaaaae68a7570 <EventCenter::process_events(unsigned int, std::chrono::duration<unsigned long, std::ratio<1l, 1000000000l> >*)+2992>,
1772
            0xaaaae68ad0f0 <std::_Function_handler<void(), NetworkStack::add_thread(unsigned int)::<lambda()> >::_M_invoke(const std::_Any_data &)+296>,
1773
            0xffffa596ded4, 0xffffa5b23088 <start_thread+176>, 0x0 <repeats 86 times>}, size = 14, strings = 0xaaab4a3f9100}
1774
        oss = <incomplete type>
1775
        base = "/var/lib/ceph/crash/2020-02-27T11:57:05.396605Z_9795c05d-2c82-4ba5-8d2e-270432f75ecd", '\000' <repeats 4011 times>
1776
#3  <signal handler called>
1777
No symbol table info available.
1778
#4  __memcpy_generic () at ../sysdeps/aarch64/multiarch/../memcpy.S:105
1779
No locals.
1780
#5  0x0000aaaae6aa5d00 in memcpy (__len=40, __src=0xaaab4a31a000, __dest=<optimized out>) at /usr/include/aarch64-linux-gnu/bits/string_fortified.h:34
1781
No locals.
1782
#6  Infiniband::MemoryManager::Chunk::write (this=this@entry=0xaaab4ba39ba0, buf=buf@entry=0xaaab4a31a000 "\024\001\b", len=<optimized out>)
1783
    at ./src/msg/async/rdma/Infiniband.cc:738
1784
        write_len = 40
1785
#7  0x0000aaaae6ab4018 in RDMAConnectedSocketImpl::tx_copy_chunk (this=this@entry=0xaaab4f1794e0, tx_buffers=std::vector of length 1, capacity 1 = {...},
1786
    req_copy_len=req_copy_len@entry=57, start=..., end=...) at ./src/include/buffer.h:317
1787
        real_len = <optimized out>
1788
        addr = 187652660895744
1789
        slice_write_len = 0
1790
        __PRETTY_FUNCTION__ = <optimized out>
1791
        chunk_idx = 0
1792
        __func__ = "tx_copy_chunk"
1793
        current_chunk = 0xaaab4ba39ba0
1794
        write_len = 0
1795
#8  0x0000aaaae6ab5aec in RDMAConnectedSocketImpl::submit (this=this@entry=0xaaab4f1794e0, more=more@entry=false)
1796
    at ./src/msg/async/rdma/RDMAConnectedSocketImpl.cc:380
1797
---Type <return> to continue, or q <return> to quit---
1798
        l = {_M_device = @0xaaab4f179598}
1799
        bytes = <optimized out>
1800
        __func__ = "submit"
1801
        tx_buffers = std::vector of length 1, capacity 1 = {0xaaab4ba39ba0}
1802
        it = {cur = 0xaaab4f179550}
1803
        copy_start = {cur = 0xaaab4fcfc060}
1804
        total_copied = 0
1805
        wait_copy_len = 57
1806
        __PRETTY_FUNCTION__ = <optimized out>
1807
        swapped = {_buffers = {_root = {next = 0x0}, _tail = 0xaaab4f179598, _size = 281473429961616},
1808
          _carriage = 0xaaaae68bfef8 <ceph::buffer::v14_2_0::list::append(char const*, unsigned int)+144>, _len = 0, _memcopy_count = 0,
1809
          last_p = {<ceph::buffer::v14_2_0::list::iterator_impl<false>> = {bl = 0xaaab4a578228, ls = 0xaaab4a578228, p = {cur = 0xaaab4a578228}, off = 0,
1810
              p_off = 0}, <No data fields>}, static always_empty_bptr = {_raw = 0x0, _off = 0, _len = 0}, static CLAIM_DEFAULT = 0,
1811
          static CLAIM_ALLOW_NONSHAREABLE = 1}
1812
#9  0x0000aaaae6ab7c34 in RDMAConnectedSocketImpl::send (this=0xaaab4f1794e0, bl=..., more=<optimized out>) at ./src/msg/async/rdma/RDMAConnectedSocketImpl.cc:297
1813
        bytes = 57
1814
        __func__ = "send"
1815
        r = <optimized out>
1816
#10 0x0000aaaae6a534d4 in ConnectedSocket::send (more=false, bl=..., this=0xaaab4a5781e8) at /usr/include/c++/9/bits/unique_ptr.h:357
1817
No locals.
1818
#11 AsyncConnection::_try_send (this=0xaaab4a578000, more=false) at ./src/msg/async/AsyncConnection.cc:330
1819
        __func__ = "_try_send"
1820
        __PRETTY_FUNCTION__ = <optimized out>
1821
        r = <optimized out>
1822
#12 0x0000aaaae6a96bf8 in ProtocolV2::write_event (this=0xaaab4f170680) at ./src/msg/async/ProtocolV2.cc:663
1823
        ack = {<ceph::msgr::v2::ControlFrame<ceph::msgr::v2::AckFrame, unsigned long>> = {<ceph::msgr::v2::Frame<ceph::msgr::v2::AckFrame, 8>> = {
1824
              static SegmentsNumV = 1, segments = {_M_elems = {{_buffers = {_root = {next = 0xffffa3ce7d88}, _tail = 0xffffa3ce7d88, _size = 0},
1825
                    _carriage = 0xaaaaef8857b0 <ceph::buffer::v14_2_0::list::always_empty_bptr>, _len = 0, _memcopy_count = 0,
1826
                    last_p = {<ceph::buffer::v14_2_0::list::iterator_impl<false>> = {bl = 0xffffa3ce7d88, ls = 0xffffa3ce7d88, p = {cur = 0xffffa3ce7d88}, off = 0,
1827
                        p_off = 0}, <No data fields>}, static always_empty_bptr = {_raw = 0x0, _off = 0, _len = 0}, static CLAIM_DEFAULT = 0,
1828
                    static CLAIM_ALLOW_NONSHAREABLE = 1}}}, static alignments = <optimized out>, preamble_filler = {pos = 0xaaab4a31a020 "\003"}},
1829
            _values = std::tuple containing = {[1] = 0}, features = 720575940647714820}, static tag = ceph::msgr::v2::Tag::ACK}
1830
        left = <optimized out>
1831
        start = {__d = {__r = 193307512992120}}
1832
        more = <optimized out>
1833
        __func__ = "write_event"
1834
        r = <optimized out>
1835
#13 0x0000aaaae6a509e8 in AsyncConnection::handle_write (this=0xaaab4a578000) at /usr/include/c++/9/bits/unique_ptr.h:357
1836
        __func__ = "handle_write"
1837
#14 0x0000aaaae68a7570 in EventCenter::process_events (this=this@entry=0xaaaaf26a8b88, timeout_microseconds=<optimized out>,
1838
    working_dur=working_dur@entry=0xffffa3ce8160) at /usr/include/c++/9/bits/stl_deque.h:2110
1839
        e = 0xaaab483c07e0
1840
        cur_process = std::deque with 1 element = {0xaaab483c07e0}
1841
        tv = {tv_sec = 8, tv_usec = 972000}
1842
        numevents = 8
1843
        trigger_time = true
1844
---Type <return> to continue, or q <return> to quit---q
1845
Quit
1846
(gdb)
1847
(gdb) f 7
1848
#7  0x0000aaaae6ab4018 in RDMAConnectedSocketImpl::tx_copy_chunk (this=this@entry=0xaaab4f1794e0, tx_buffers=std::vector of length 1, capacity 1 = {...},
1849
    req_copy_len=req_copy_len@entry=57, start=..., end=...) at ./src/include/buffer.h:317
1850
317     ./src/include/buffer.h: No such file or directory.
1851
(gdb) p *this
1852
$2 = {<ConnectedSocketImpl> = {_vptr.ConnectedSocketImpl = 0xaaaae7633a58 <vtable for RDMAConnectedSocketImpl+16>}, cct = 0xaaaaf1ae0000, qp = 0xaaab4a4bba00,
1853
  peer_qpn = 58372, local_qpn = 228285, connected = 1, error = 0, ib = std::shared_ptr<Infiniband> (use count 63, weak count 0) = {get() = 0xaaaaf1a726f0},
1854
  dispatcher = std::shared_ptr<RDMADispatcher> (use count 62, weak count 0) = {get() = 0xaaaaf1a84c70}, worker = 0xaaaaf26a8b00,
1855
  buffers = std::vector of length 0, capacity 2, notify_fd = 143, pending_bl = {_buffers = {_root = {next = 0xaaab4fcfc060}, _tail = 0xaaab4fcfc0e0, _size = 2},
1856
    _carriage = 0xaaaaef8857b0 <ceph::buffer::v14_2_0::list::always_empty_bptr>, _len = 57, _memcopy_count = 0,
1857
    last_p = {<ceph::buffer::v14_2_0::list::iterator_impl<false>> = {bl = 0xaaab4f179550, ls = 0xaaab4f179550, p = {cur = 0xaaab4f179550}, off = 0,
1858
        p_off = 0}, <No data fields>}, static always_empty_bptr = {_raw = 0x0, _off = 0, _len = 0}, static CLAIM_DEFAULT = 0, static CLAIM_ALLOW_NONSHAREABLE = 1},
1859
  lock = {<std::__mutex_base> = {_M_mutex = pthread_mutex_t = {Type = Normal, Status = Acquired, possibly with no waiters, Owner ID = 377894, Robust = No,
1860
        Shared = No, Protocol = None}}, <No data fields>}, wc = std::vector of length 0, capacity 0, is_server = true, con_handler = 0xaaab4a40d5e0, tcp_fd = 142,
1861
  active = true, pending = false, post_backlog = 0}
1862
(gdb) p *this->ib
1863
$3 = {tx_queue_len = 1024, rx_queue_len = 4096, max_sge = 0, ib_physical_port = 1 '\001', memory_manager = 0xaaaaf1a5b4a0, srq = 0xaaaaf1a88fc8,
1864
  device = 0xaaaaf1b0e400, pd = 0xaaaaf1b24368, device_list = 0xaaaaf26a7500, cct = 0xaaaaf1ae0000, lock = {<std::__mutex_base> = {_M_mutex = pthread_mutex_t = {
1865
        Type = Normal, Status = Not acquired, Robust = No, Shared = No, Protocol = None}}, <No data fields>}, initialized = true, device_name = "mlx5_0",
1866
  port_num = 1 '\001', support_srq = true}
1867
(gdb) p *this->ib->memory_manager
1868
$4 = {cct = 0xaaaaf1ae0000, send = 0xaaaaf1a54c30, device = 0xaaaaf1b0e400, pd = 0xaaaaf1b24368, rxbuf_pool_ctx = {perf_logger = 0xaaaaf1a73360,
1869
    manager = 0xaaaaf1a5b4a0, n_bufs_allocated = 8192},
1870
  rxbuf_pool = {<boost::pool<Infiniband::MemoryManager::PoolAllocator>> = {<boost::simple_segregated_storage<unsigned long>> = {first = 0xaaab0618b2c0},
1871
Segmentation fault
1872
root@node2:/var/log# gdb /usr/bin/ceph-osd core.msgr-worker-2.377740
1873
GNU gdb (Ubuntu 8.1-0ubuntu3) 8.1.0.20180409-git
1874
Copyright (C) 2018 Free Software Foundation, Inc.
1875
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
1876
This is free software: you are free to change and redistribute it.
1877
There is NO WARRANTY, to the extent permitted by law.  Type "show copying"
1878
and "show warranty" for details.
1879
This GDB was configured as "aarch64-linux-gnu".
1880
Type "show configuration" for configuration details.
1881
For bug reporting instructions, please see:
1882
<http://www.gnu.org/software/gdb/bugs/>.
1883
Find the GDB manual and other documentation resources online at:
1884
<http://www.gnu.org/software/gdb/documentation/>.
1885
For help, type "help".
1886
Type "apropos word" to search for commands related to "word"...
1887
Reading symbols from /usr/bin/ceph-osd...Reading symbols from /usr/lib/debug/.build-id/fd/2d225baa3068ff217e8e576184a3a362e494fd.debug...done.
1888
done.
1889
[New LWP 377894]
1890
[New LWP 377892]
1891
[New LWP 377764]
1892
[New LWP 379320]
1893
[New LWP 377740]
1894
[New LWP 378586]
1895
[New LWP 378103]
1896
[New LWP 379324]
1897
[New LWP 379325]
1898
[New LWP 378105]
1899
[New LWP 379319]
1900
[New LWP 379333]
1901
[New LWP 379328]
1902
[New LWP 379326]
1903
[New LWP 379934]
1904
[New LWP 381128]
1905
[New LWP 379428]
1906
[New LWP 379429]
1907
[New LWP 381182]
1908
[New LWP 381184]
1909
[New LWP 381186]
1910
[New LWP 379882]
1911
[New LWP 379866]
1912
[New LWP 381203]
1913
[New LWP 379430]
1914
[New LWP 381204]
1915
[New LWP 381206]
1916
[New LWP 381223]
1917
[New LWP 379945]
1918
[New LWP 381226]
1919
[New LWP 381213]
1920
[New LWP 381215]
1921
[New LWP 381191]
1922
[New LWP 381195]
1923
[New LWP 379327]
1924
[New LWP 379334]
1925
[New LWP 381228]
1926
[New LWP 381192]
1927
[New LWP 381210]
1928
[New LWP 379864]
1929
[New LWP 381197]
1930
[New LWP 381196]
1931
[New LWP 381127]
1932
[New LWP 381188]
1933
[New LWP 379943]
1934
[New LWP 381194]
1935
[New LWP 381200]
1936
[New LWP 381220]
1937
[New LWP 381218]
1938
[New LWP 381205]
1939
[New LWP 381190]
1940
[New LWP 381222]
1941
[New LWP 381219]
1942
[New LWP 379936]
1943
[New LWP 381198]
1944
[New LWP 381236]
1945
[New LWP 381216]
1946
[New LWP 381193]
1947
[New LWP 381130]
1948
[New LWP 381207]
1949
[New LWP 381129]
1950
[New LWP 381234]
1951
[New LWP 381214]
1952
[New LWP 381208]
1953
[New LWP 381230]
1954
[New LWP 381227]
1955
[New LWP 381233]
1956
[New LWP 381199]
1957
[New LWP 381229]
1958
[New LWP 381224]
1959
[New LWP 381212]
1960
[New LWP 381202]
1961
[New LWP 381181]
1962
[New LWP 379947]
1963
[New LWP 381183]
1964
[New LWP 381235]
1965
[New LWP 377890]
1966
[New LWP 381232]
1967
[New LWP 381238]
1968
[New LWP 381221]
1969
[New LWP 381237]
1970
[New LWP 381131]
1971
[New LWP 381189]
1972
[New LWP 381217]
1973
[New LWP 379944]
1974
[New LWP 381201]
1975
[New LWP 381185]
1976
[New LWP 381239]
1977
[New LWP 381231]
1978
[New LWP 381225]
1979
[New LWP 381187]
1980
[New LWP 381211]
1981
[New LWP 381209]
1982
[New LWP 378073]
1983
[Thread debugging using libthread_db enabled]
1984
Using host libthread_db library "/lib/aarch64-linux-gnu/libthread_db.so.1".
1985
Core was generated by `/usr/bin/ceph-osd -f --cluster ceph --id 41 --setuser ceph --setgroup ceph'.
1986
Program terminated with signal SIGSEGV, Segmentation fault.
1987
#0  raise (sig=sig@entry=11) at ../sysdeps/unix/sysv/linux/raise.c:51
1988
51      ../sysdeps/unix/sysv/linux/raise.c: No such file or directory.
1989
[Current thread is 1 (Thread 0xffffa3ce89f0 (LWP 377894))]
1990
(gdb) bt
1991
#0  raise (sig=sig@entry=11) at ../sysdeps/unix/sysv/linux/raise.c:51
1992
#1  0x0000aaaae66dea1c in reraise_fatal (signum=11) at ./src/global/signal_handler.cc:326
1993
#2  handle_fatal_signal (signum=11) at ./src/global/signal_handler.cc:326
1994
#3  <signal handler called>
1995
#4  __memcpy_generic () at ../sysdeps/aarch64/multiarch/../memcpy.S:105
1996
#5  0x0000aaaae6aa5d00 in memcpy (__len=40, __src=0xaaab4a31a000, __dest=<optimized out>) at /usr/include/aarch64-linux-gnu/bits/string_fortified.h:34
1997
#6  Infiniband::MemoryManager::Chunk::write (this=this@entry=0xaaab4ba39ba0, buf=buf@entry=0xaaab4a31a000 "\024\001\b", len=<optimized out>)
1998
    at ./src/msg/async/rdma/Infiniband.cc:738
1999
#7  0x0000aaaae6ab4018 in RDMAConnectedSocketImpl::tx_copy_chunk (this=this@entry=0xaaab4f1794e0, tx_buffers=std::vector of length 1, capacity 1 = {...},
2000
    req_copy_len=req_copy_len@entry=57, start=..., end=...) at ./src/include/buffer.h:317
2001
#8  0x0000aaaae6ab5aec in RDMAConnectedSocketImpl::submit (this=this@entry=0xaaab4f1794e0, more=more@entry=false)
2002
    at ./src/msg/async/rdma/RDMAConnectedSocketImpl.cc:380
2003
#9  0x0000aaaae6ab7c34 in RDMAConnectedSocketImpl::send (this=0xaaab4f1794e0, bl=..., more=<optimized out>) at ./src/msg/async/rdma/RDMAConnectedSocketImpl.cc:297
2004
#10 0x0000aaaae6a534d4 in ConnectedSocket::send (more=false, bl=..., this=0xaaab4a5781e8) at /usr/include/c++/9/bits/unique_ptr.h:357
2005
#11 AsyncConnection::_try_send (this=0xaaab4a578000, more=false) at ./src/msg/async/AsyncConnection.cc:330
2006
#12 0x0000aaaae6a96bf8 in ProtocolV2::write_event (this=0xaaab4f170680) at ./src/msg/async/ProtocolV2.cc:663
2007
#13 0x0000aaaae6a509e8 in AsyncConnection::handle_write (this=0xaaab4a578000) at /usr/include/c++/9/bits/unique_ptr.h:357
2008
#14 0x0000aaaae68a7570 in EventCenter::process_events (this=this@entry=0xaaaaf26a8b88, timeout_microseconds=<optimized out>,
2009
    working_dur=working_dur@entry=0xffffa3ce8160) at /usr/include/c++/9/bits/stl_deque.h:2110
2010
#15 0x0000aaaae68ad0f0 in NetworkStack::<lambda()>::operator() (__closure=0xaaaaf27184e8, __closure=0xaaaaf27184e8) at ./src/msg/async/Stack.cc:53
2011
#16 std::_Function_handler<void(), NetworkStack::add_thread(unsigned int)::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...)
2012
    at /usr/include/c++/9/bits/std_function.h:300
2013
#17 0x0000ffffa596ded4 in ?? () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
2014
#18 0x0000ffffa5b23088 in start_thread (arg=0xffffeb90520f) at pthread_create.c:463
2015
#19 0x0000ffffa573d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
2016
(gdb) f 7
2017
#7  0x0000aaaae6ab4018 in RDMAConnectedSocketImpl::tx_copy_chunk (this=this@entry=0xaaab4f1794e0, tx_buffers=std::vector of length 1, capacity 1 = {...},
2018
    req_copy_len=req_copy_len@entry=57, start=..., end=...) at ./src/include/buffer.h:317
2019
317     ./src/include/buffer.h: No such file or directory.
2020
(gdb) bt
2021
#0  raise (sig=sig@entry=11) at ../sysdeps/unix/sysv/linux/raise.c:51
2022
#1  0x0000aaaae66dea1c in reraise_fatal (signum=11) at ./src/global/signal_handler.cc:326
2023
#2  handle_fatal_signal (signum=11) at ./src/global/signal_handler.cc:326
2024
#3  <signal handler called>
2025
#4  __memcpy_generic () at ../sysdeps/aarch64/multiarch/../memcpy.S:105
2026
#5  0x0000aaaae6aa5d00 in memcpy (__len=40, __src=0xaaab4a31a000, __dest=<optimized out>) at /usr/include/aarch64-linux-gnu/bits/string_fortified.h:34
2027
#6  Infiniband::MemoryManager::Chunk::write (this=this@entry=0xaaab4ba39ba0, buf=buf@entry=0xaaab4a31a000 "\024\001\b", len=<optimized out>)
2028
    at ./src/msg/async/rdma/Infiniband.cc:738
2029
#7  0x0000aaaae6ab4018 in RDMAConnectedSocketImpl::tx_copy_chunk (this=this@entry=0xaaab4f1794e0, tx_buffers=std::vector of length 1, capacity 1 = {...},
2030
    req_copy_len=req_copy_len@entry=57, start=..., end=...) at ./src/include/buffer.h:317
2031
#8  0x0000aaaae6ab5aec in RDMAConnectedSocketImpl::submit (this=this@entry=0xaaab4f1794e0, more=more@entry=false)
2032
    at ./src/msg/async/rdma/RDMAConnectedSocketImpl.cc:380
2033
#9  0x0000aaaae6ab7c34 in RDMAConnectedSocketImpl::send (this=0xaaab4f1794e0, bl=..., more=<optimized out>) at ./src/msg/async/rdma/RDMAConnectedSocketImpl.cc:297
2034
#10 0x0000aaaae6a534d4 in ConnectedSocket::send (more=false, bl=..., this=0xaaab4a5781e8) at /usr/include/c++/9/bits/unique_ptr.h:357
2035
#11 AsyncConnection::_try_send (this=0xaaab4a578000, more=false) at ./src/msg/async/AsyncConnection.cc:330
2036
#12 0x0000aaaae6a96bf8 in ProtocolV2::write_event (this=0xaaab4f170680) at ./src/msg/async/ProtocolV2.cc:663
2037
#13 0x0000aaaae6a509e8 in AsyncConnection::handle_write (this=0xaaab4a578000) at /usr/include/c++/9/bits/unique_ptr.h:357
2038
#14 0x0000aaaae68a7570 in EventCenter::process_events (this=this@entry=0xaaaaf26a8b88, timeout_microseconds=<optimized out>,
2039
    working_dur=working_dur@entry=0xffffa3ce8160) at /usr/include/c++/9/bits/stl_deque.h:2110
2040
#15 0x0000aaaae68ad0f0 in NetworkStack::<lambda()>::operator() (__closure=0xaaaaf27184e8, __closure=0xaaaaf27184e8) at ./src/msg/async/Stack.cc:53
2041
#16 std::_Function_handler<void(), NetworkStack::add_thread(unsigned int)::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...)
2042
    at /usr/include/c++/9/bits/std_function.h:300
2043
#17 0x0000ffffa596ded4 in ?? () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
2044
#18 0x0000ffffa5b23088 in start_thread (arg=0xffffeb90520f) at pthread_create.c:463
2045
#19 0x0000ffffa573d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
2046
(gdb) p this->ib
2047
$1 = std::shared_ptr<Infiniband> (use count 63, weak count 0) = {get() = 0xaaaaf1a726f0}
2048
(gdb) p this->ib->
2049
_M_get          _M_ptr          __shared_ptr    operator bool   operator->      reset           swap            use_count       ~shared_ptr
2050
_M_get_deleter  _M_refcount     get             operator*       operator=       shared_ptr      unique          ~__shared_ptr
2051
(gdb) p *this->ib
2052
$2 = {tx_queue_len = 1024, rx_queue_len = 4096, max_sge = 0, ib_physical_port = 1 '\001', memory_manager = 0xaaaaf1a5b4a0, srq = 0xaaaaf1a88fc8,
2053
  device = 0xaaaaf1b0e400, pd = 0xaaaaf1b24368, device_list = 0xaaaaf26a7500, cct = 0xaaaaf1ae0000, lock = {<std::__mutex_base> = {_M_mutex = pthread_mutex_t = {
2054
        Type = Normal, Status = Not acquired, Robust = No, Shared = No, Protocol = None}}, <No data fields>}, initialized = true, device_name = "mlx5_0",
2055
  port_num = 1 '\001', support_srq = true}
2056
(gdb) p this->ib->memory_manager
2057
$3 = (Infiniband::MemoryManager *) 0xaaaaf1a5b4a0
2058
(gdb) p this->ib->memory_manager->send
2059
$4 = (Infiniband::MemoryManager::Cluster *) 0xaaaaf1a54c30
2060
(gdb) p *this->ib->memory_manager->send
2061
$5 = {manager = @0xaaaaf1a5b4a0, buffer_size = 131072, num_chunk = 2048, lock = {<std::__mutex_base> = {_M_mutex = pthread_mutex_t = {Type = Normal,
2062
        Status = Not acquired, Robust = No, Shared = No, Protocol = None}}, <No data fields>}, free_chunks = std::vector of length 2048, capacity 4096 = {
2063
    0xaaab027d6000, 0xaaab027d6040, 0xaaab027d6080, 0xaaab027d60c0, 0xaaab027d6100, 0xaaab027d6140, 0xaaab027d6180, 0xaaab027d61c0, 0xaaab027d6200, 0xaaab027d6240,
2064
    0xaaab027d6280, 0xaaab027d62c0, 0xaaab027d6300, 0xaaab027d6340, 0xaaab027d6380, 0xaaab027d63c0, 0xaaab027d6400, 0xaaab027d6440, 0xaaab027d6480, 0xaaab027d64c0,
2065
    0xaaab027d6500, 0xaaab027d6540, 0xaaab027d6580, 0xaaab027d65c0, 0xaaab027d6600, 0xaaab027d6640, 0xaaab027d6680, 0xaaab027d66c0, 0xaaab027d6700, 0xaaab027d6740,
2066
    0xaaab027d6780, 0xaaab027d67c0, 0xaaab027d6800, 0xaaab027d6840, 0xaaab027d6880, 0xaaab027d68c0, 0xaaab027d6900, 0xaaab027d6940, 0xaaab027d6980, 0xaaab027d69c0,
2067
    0xaaab027d6a00, 0xaaab027d6a40, 0xaaab027d6a80, 0xaaab027d6ac0, 0xaaab027d6b00, 0xaaab027d6b40, 0xaaab027d6b80, 0xaaab027d6bc0, 0xaaab027d6c00, 0xaaab027d6c40,
2068
    0xaaab027d6c80, 0xaaab027d6cc0, 0xaaab027d6d00, 0xaaab027d6d40, 0xaaab027d6d80, 0xaaab027d6dc0, 0xaaab027d6e00, 0xaaab027d6e40, 0xaaab027d6e80, 0xaaab027d6ec0,
2069
    0xaaab027d6f00, 0xaaab027d6f40, 0xaaab027d6f80, 0xaaab027d6fc0, 0xaaab027d7000, 0xaaab027d7040, 0xaaab027d7080, 0xaaab027d70c0, 0xaaab027d7100, 0xaaab027d7140,
2070
    0xaaab027d7180, 0xaaab027d71c0, 0xaaab027d7200, 0xaaab027d7240, 0xaaab027d7280, 0xaaab027d72c0, 0xaaab027d7300, 0xaaab027d7340, 0xaaab027d7380, 0xaaab027d73c0,
2071
    0xaaab027d7400, 0xaaab027d7440, 0xaaab027d7480, 0xaaab027d74c0, 0xaaab027d7500, 0xaaab027d7540, 0xaaab027d7580, 0xaaab027d75c0, 0xaaab027d7600, 0xaaab027d7640,
2072
    0xaaab027d7680, 0xaaab027d76c0, 0xaaab027d7700, 0xaaab027d7740, 0xaaab027d7780, 0xaaab027d77c0, 0xaaab027d7800, 0xaaab027d7840, 0xaaab027d7880, 0xaaab027d78c0,
2073
    0xaaab027d7900, 0xaaab027d7940, 0xaaab027d7980, 0xaaab027d79c0, 0xaaab027d7a00, 0xaaab027d7a40, 0xaaab027d7a80, 0xaaab027d7ac0, 0xaaab027d7b00, 0xaaab027d7b40,
2074
    0xaaab027d7b80, 0xaaab027d7bc0, 0xaaab027d7c00, 0xaaab027d7c40, 0xaaab027d7c80, 0xaaab027d7cc0, 0xaaab027d7d00, 0xaaab027d7d40, 0xaaab027d7d80, 0xaaab027d7dc0,
2075
    0xaaab027d7e00, 0xaaab027d7e40, 0xaaab027d7e80, 0xaaab027d7ec0, 0xaaab027d7f00, 0xaaab027d7f40, 0xaaab027d7f80, 0xaaab027d7fc0, 0xaaab027d8000, 0xaaab027d8040,
2076
    0xaaab027d8080, 0xaaab027d80c0, 0xaaab027d8100, 0xaaab027d8140, 0xaaab027d8180, 0xaaab027d81c0, 0xaaab027d8200, 0xaaab027d8240, 0xaaab027d8280, 0xaaab027d82c0,
2077
    0xaaab027d8300, 0xaaab027d8340, 0xaaab027d8380, 0xaaab027d83c0, 0xaaab027d8400, 0xaaab027d8440, 0xaaab027d8480, 0xaaab027d84c0, 0xaaab027d8500, 0xaaab027d8540,
2078
    0xaaab027d8580, 0xaaab027d85c0, 0xaaab027d8600, 0xaaab027d8640, 0xaaab027d8680, 0xaaab027d86c0, 0xaaab027d8700, 0xaaab027d8740, 0xaaab027d8780, 0xaaab027d87c0,
2079
    0xaaab027d8800, 0xaaab027d8840, 0xaaab027d8880, 0xaaab027d88c0, 0xaaab027d8900, 0xaaab027d8940, 0xaaab027d8980, 0xaaab027d89c0, 0xaaab027d8a00, 0xaaab027d8a40,
2080
    0xaaab027d8a80, 0xaaab027d8ac0, 0xaaab027d8b00, 0xaaab027d8b40, 0xaaab027d8b80, 0xaaab027d8bc0, 0xaaab027d8c00, 0xaaab027d8c40, 0xaaab027d8c80, 0xaaab027d8cc0,
2081
    0xaaab027d8d00, 0xaaab027d8d40, 0xaaab027d8d80, 0xaaab027d8dc0, 0xaaab027d8e00, 0xaaab027d8e40, 0xaaab027d8e80, 0xaaab027d8ec0, 0xaaab027d8f00, 0xaaab027d8f40,
2082
    0xaaab027d8f80, 0xaaab027d8fc0, 0xaaab027d9000, 0xaaab027d9040, 0xaaab027d9080, 0xaaab027d90c0, 0xaaab027d9100, 0xaaab027d9140, 0xaaab027d9180,
2083
    0xaaab027d91c0...}, base = 0xaaaaf27d6000 "", end = 0xaaab027d6000 "@", <incomplete sequence \361\252\252>, chunk_base = 0xaaab027d6000}
2084
(gdb) p *this->ib->memory_manager->send->free_chunks[0]
2085
$6 = {mr = 0xaaaaf1a6dd40, qp = 0x0, lkey = 18513, bytes = 131072, offset = 0, bound = 131072, buffer = 0xaaaaf27d6000 "", pad = {0, 0, 0, 0, 0, 0},
2086
  data = 0xaaab027d6040 "@", <incomplete sequence \361\252\252>}
2087
(gdb) p *this->ib->memory_manager->send->free_chunks[1]
2088
$7 = {mr = 0xaaaaf1a6dd40, qp = 0x0, lkey = 18513, bytes = 131072, offset = 0, bound = 131072, buffer = 0xaaaaf27f6000 "", pad = {0, 0, 0, 0, 0, 0},
2089
  data = 0xaaab027d6080 "@", <incomplete sequence \361\252\252>}
2090
(gdb) p cu
2091
cu_data                 currency_symbol         current_function.hpp    current_rtmin           cuse_lowlevel_new       cuserid
2092
cur                     current                 current_maxid           curshell                cuse_lowlevel_setup     cuserid.c
2093
cur_qp_state            current_chunk           current_rtmax           cuse_lowlevel_main      cuse_lowlevel_teardown  custom_reaction.hpp
2094
(gdb) p cu
2095
cu_data                 currency_symbol         current_function.hpp    current_rtmin           cuse_lowlevel_new       cuserid
2096
cur                     current                 current_maxid           curshell                cuse_lowlevel_setup     cuserid.c
2097
cur_qp_state            current_chunk           current_rtmax           cuse_lowlevel_main      cuse_lowlevel_teardown  custom_reaction.hpp
2098
(gdb) p current_chunk
2099
$8 = (RDMAConnectedSocketImpl::Chunk *) 0xaaab4ba39ba0
2100
(gdb) p *current_chunk
2101
$9 = {mr = 0x0, qp = 0xaaab4a4bdc20, lkey = 1246485952, bytes = 43691, offset = 1339179008, bound = 43691, buffer = 0xaaab4ba39bd0 "smtp_port", pad = {9, 0,
2102
    1886678387, 1919905887, 1233911924, 43691}, data = 0xaaab4ba39be0 "\360\233\243K\253\252"}
2103
(gdb) p 0xaaab4ba39ba0-0xaaab027d6000
2104
$10 = 1227242400
2105
(gdb) p (0xaaab4ba39ba0-0xaaab027d6000)/64
2106
$11 = 19175662
2107
(gdb)
2108
$12 = 19175662
2109
(gdb)
2110
$13 = 19175662
2111
(gdb) p 131072*2048
2112
$14 = 268435456
2113
(gdb) p /x 131072*2048
2114
$15 = 0x10000000
2115
(gdb) p 0xaaab027d6000+4096*64
2116
$16 = 187651458162688
2117
(gdb) p /x 0xaaab027d6000+4096*64
2118
$17 = 0xaaab02816000
2119
(gdb) p (RDMAConnectedSocketImpl::Chunk *)0xaaab02839ba0
2120
$18 = (RDMAConnectedSocketImpl::Chunk *) 0xaaab02839ba0
2121
(gdb) p *(RDMAConnectedSocketImpl::Chunk *)0xaaab02839ba0
2122
$19 = {mr = 0x3d6e203130363031, qp = 0x383d636520393135, lkey = 792277556, bytes = 959591480, offset = 1936288800, bound = 826106671,
2123
  buffer = 0x3630312f30303630 <error: Cannot access memory at address 0x3630312f30303630>, pad = {1814048816, 1664054117, 826107439, 825243184, 909127983,
2124
    808399152},
2125
  data = 0xaaab02839be0 " sis=10624) [66,41] r=1 lpr=10624 pi=[10600,10624)/1 crt=10343'341492 lcod 0'0 mlcod 0'0 unknown NOTIFY mbc={}] start_peering_interval up [66] -> [66,41], acting [66] -> [66,41], acting_primary 66 -> "...}
2126
(gdb) bt
2127
#0  raise (sig=sig@entry=11) at ../sysdeps/unix/sysv/linux/raise.c:51
2128
#1  0x0000aaaae66dea1c in reraise_fatal (signum=11) at ./src/global/signal_handler.cc:326
2129
#2  handle_fatal_signal (signum=11) at ./src/global/signal_handler.cc:326
2130
#3  <signal handler called>
2131
#4  __memcpy_generic () at ../sysdeps/aarch64/multiarch/../memcpy.S:105
2132
#5  0x0000aaaae6aa5d00 in memcpy (__len=40, __src=0xaaab4a31a000, __dest=<optimized out>) at /usr/include/aarch64-linux-gnu/bits/string_fortified.h:34
2133
#6  Infiniband::MemoryManager::Chunk::write (this=this@entry=0xaaab4ba39ba0, buf=buf@entry=0xaaab4a31a000 "\024\001\b", len=<optimized out>)
2134
    at ./src/msg/async/rdma/Infiniband.cc:738
2135
#7  0x0000aaaae6ab4018 in RDMAConnectedSocketImpl::tx_copy_chunk (this=this@entry=0xaaab4f1794e0, tx_buffers=std::vector of length 1, capacity 1 = {...},
2136
    req_copy_len=req_copy_len@entry=57, start=..., end=...) at ./src/include/buffer.h:317
2137
#8  0x0000aaaae6ab5aec in RDMAConnectedSocketImpl::submit (this=this@entry=0xaaab4f1794e0, more=more@entry=false)
2138
    at ./src/msg/async/rdma/RDMAConnectedSocketImpl.cc:380
2139
#9  0x0000aaaae6ab7c34 in RDMAConnectedSocketImpl::send (this=0xaaab4f1794e0, bl=..., more=<optimized out>) at ./src/msg/async/rdma/RDMAConnectedSocketImpl.cc:297
2140
#10 0x0000aaaae6a534d4 in ConnectedSocket::send (more=false, bl=..., this=0xaaab4a5781e8) at /usr/include/c++/9/bits/unique_ptr.h:357
2141
#11 AsyncConnection::_try_send (this=0xaaab4a578000, more=false) at ./src/msg/async/AsyncConnection.cc:330
2142
#12 0x0000aaaae6a96bf8 in ProtocolV2::write_event (this=0xaaab4f170680) at ./src/msg/async/ProtocolV2.cc:663
2143
#13 0x0000aaaae6a509e8 in AsyncConnection::handle_write (this=0xaaab4a578000) at /usr/include/c++/9/bits/unique_ptr.h:357
2144
#14 0x0000aaaae68a7570 in EventCenter::process_events (this=this@entry=0xaaaaf26a8b88, timeout_microseconds=<optimized out>,
2145
    working_dur=working_dur@entry=0xffffa3ce8160) at /usr/include/c++/9/bits/stl_deque.h:2110
2146
#15 0x0000aaaae68ad0f0 in NetworkStack::<lambda()>::operator() (__closure=0xaaaaf27184e8, __closure=0xaaaaf27184e8) at ./src/msg/async/Stack.cc:53
2147
#16 std::_Function_handler<void(), NetworkStack::add_thread(unsigned int)::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...)
2148
    at /usr/include/c++/9/bits/std_function.h:300
2149
#17 0x0000ffffa596ded4 in ?? () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
2150
#18 0x0000ffffa5b23088 in start_thread (arg=0xffffeb90520f) at pthread_create.c:463
2151
#19 0x0000ffffa573d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
2152
(gdb) f 7
2153
#7  0x0000aaaae6ab4018 in RDMAConnectedSocketImpl::tx_copy_chunk (this=this@entry=0xaaab4f1794e0, tx_buffers=std::vector of length 1, capacity 1 = {...},
2154
    req_copy_len=req_copy_len@entry=57, start=..., end=...) at ./src/include/buffer.h:317
2155
317     in ./src/include/buffer.h
2156
(gdb) p tx_buffers[0]
2157
$20 = (Infiniband::MemoryManager::Chunk *) 0xaaab4ba39ba0
2158
(gdb)