Project

General

Profile

ceph_rdma_bug.txt

add asset in handle_tx_event - chunsong feng, 02/28/2020 07:42 AM

Download (119 KB)

 
1

    
2
(gdb) thread apply all bt
3

    
4
Thread 94 (Thread 0xffff7345d9f0 (LWP 126909)):
5
#0  0x0000ffff8b1f922c in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0xaaaafbb859a8) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
6
#1  __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafbb85950, cond=0xaaaafbb85980) at pthread_cond_wait.c:502
7
#2  __pthread_cond_wait (cond=0xaaaafbb85980, mutex=0xaaaafbb85950) at pthread_cond_wait.c:655
8
#3  0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
9
#4  0x0000aaaad28449ac in DispatchQueue::entry (this=0xaaaafbb85940) at ./src/msg/DispatchQueue.cc:210
10
#5  0x0000aaaad26977f0 in DispatchQueue::DispatchThread::entry (this=<optimized out>) at ./src/msg/DispatchQueue.h:101
11
#6  0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4df) at pthread_create.c:463
12
#7  0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
13

    
14
Thread 93 (Thread 0xffff74c609f0 (LWP 126892)):
15
#0  0x0000ffff8b1f922c in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0xaaaafbb85218) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
16
#1  __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafbb851c0, cond=0xaaaafbb851f0) at pthread_cond_wait.c:502
17
#2  __pthread_cond_wait (cond=0xaaaafbb851f0, mutex=0xaaaafbb851c0) at pthread_cond_wait.c:655
18
#3  0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
19
#4  0x0000aaaad284471c in DispatchQueue::run_local_delivery (this=0xaaaafbb85040) at ./src/msg/DispatchQueue.cc:117
20
#5  0x0000aaaad26978b0 in DispatchQueue::LocalDeliveryThread::entry (this=<optimized out>) at ./src/msg/DispatchQueue.h:115
21
#6  0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4ff) at pthread_create.c:463
22
#7  0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
23

    
24
Thread 92 (Thread 0xffff5e4339f0 (LWP 126952)):
25
#0  0x0000ffff8b1f922c in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0xaaaafafbadb0) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
26
#1  __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafafbad58, cond=0xaaaafafbad88) at pthread_cond_wait.c:502
27
#2  __pthread_cond_wait (cond=0xaaaafafbad88, mutex=0xaaaafafbad58) at pthread_cond_wait.c:655
28
#3  0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
29

    
30

    
31

    
32

    
33

    
34

    
35

    
36

    
37

    
38

    
39
#4  0x0000aaaad1ee9ab4 in OSD::ShardedOpWQ::_process (this=0xaaaafaf6ef10, thread_index=34, hb=0xaaab5091a080) at ./src/osd/OSD.cc:10423
40
#5  0x0000aaaad2523928 in ShardedThreadPool::shardedthreadpool_worker (this=0xaaaafaf6ea48, thread_index=34) at ./src/common/WorkQueue.cc:311
41
#6  0x0000aaaad2526268 in ShardedThreadPool::WorkThreadSharded::entry (this=<optimized out>) at ./src/common/WorkQueue.h:706
42
#7  0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4bf) at pthread_create.c:463
43
#8  0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
44

    
45
Thread 91 (Thread 0xffff5dc329f0 (LWP 126953)):
46
#0  0x0000ffff8b1f922c in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0xaaaafafbb030) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
47
#1  __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafafbafd8, cond=0xaaaafafbb008) at pthread_cond_wait.c:502
48
#2  __pthread_cond_wait (cond=0xaaaafafbb008, mutex=0xaaaafafbafd8) at pthread_cond_wait.c:655
49
#3  0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
50
#4  0x0000aaaad1ee9ab4 in OSD::ShardedOpWQ::_process (this=0xaaaafaf6ef10, thread_index=35, hb=0xaaab5091a0d0) at ./src/osd/OSD.cc:10423
51
#5  0x0000aaaad2523928 in ShardedThreadPool::shardedthreadpool_worker (this=0xaaaafaf6ea48, thread_index=35) at ./src/common/WorkQueue.cc:311
52
#6  0x0000aaaad2526268 in ShardedThreadPool::WorkThreadSharded::entry (this=<optimized out>) at ./src/common/WorkQueue.h:706
53
#7  0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4bf) at pthread_create.c:463
54
#8  0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
55

    
56
Thread 90 (Thread 0xffff5d4319f0 (LWP 126954)):
57
#0  0x0000ffff8b1f922c in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0xaaaafafbb2b0) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
58
#1  __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafafbb258, cond=0xaaaafafbb288) at pthread_cond_wait.c:502
59
---Type <return> to continue, or q <return> to quit---
60
#2  __pthread_cond_wait (cond=0xaaaafafbb288, mutex=0xaaaafafbb258) at pthread_cond_wait.c:655
61
#3  0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
62
#4  0x0000aaaad1ee9ab4 in OSD::ShardedOpWQ::_process (this=0xaaaafaf6ef10, thread_index=36, hb=0xaaab5091a1c0) at ./src/osd/OSD.cc:10423
63
#5  0x0000aaaad2523928 in ShardedThreadPool::shardedthreadpool_worker (this=0xaaaafaf6ea48, thread_index=36) at ./src/common/WorkQueue.cc:311
64
#6  0x0000aaaad2526268 in ShardedThreadPool::WorkThreadSharded::entry (this=<optimized out>) at ./src/common/WorkQueue.h:706
65
#7  0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4bf) at pthread_create.c:463
66
#8  0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
67

    
68
Thread 89 (Thread 0xffff5cc309f0 (LWP 126955)):
69
#0  0x0000ffff8b1f922c in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0xaaaafafbb530) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
70
#1  __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafafbb4d8, cond=0xaaaafafbb508) at pthread_cond_wait.c:502
71
#2  __pthread_cond_wait (cond=0xaaaafafbb508, mutex=0xaaaafafbb4d8) at pthread_cond_wait.c:655
72
#3  0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
73
#4  0x0000aaaad1ee9ab4 in OSD::ShardedOpWQ::_process (this=0xaaaafaf6ef10, thread_index=37, hb=0xaaab50919900) at ./src/osd/OSD.cc:10423
74
#5  0x0000aaaad2523928 in ShardedThreadPool::shardedthreadpool_worker (this=0xaaaafaf6ea48, thread_index=37) at ./src/common/WorkQueue.cc:311
75
#6  0x0000aaaad2526268 in ShardedThreadPool::WorkThreadSharded::entry (this=<optimized out>) at ./src/common/WorkQueue.h:706
76
#7  0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4bf) at pthread_create.c:463
77
#8  0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
78

    
79
Thread 88 (Thread 0xffff604379f0 (LWP 126948)):
80
#0  0x0000ffff8b1f922c in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0xaaaafafba3b0) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
81
#1  __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafafba358, cond=0xaaaafafba388) at pthread_cond_wait.c:502
82
#2  __pthread_cond_wait (cond=0xaaaafafba388, mutex=0xaaaafafba358) at pthread_cond_wait.c:655
83
#3  0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
84
#4  0x0000aaaad1ee9ab4 in OSD::ShardedOpWQ::_process (this=0xaaaafaf6ef10, thread_index=30, hb=0xaaab5091a120) at ./src/osd/OSD.cc:10423
85
#5  0x0000aaaad2523928 in ShardedThreadPool::shardedthreadpool_worker (this=0xaaaafaf6ea48, thread_index=30) at ./src/common/WorkQueue.cc:311
86
#6  0x0000aaaad2526268 in ShardedThreadPool::WorkThreadSharded::entry (this=<optimized out>) at ./src/common/WorkQueue.h:706
87
#7  0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4bf) at pthread_create.c:463
88
#8  0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
89

    
90
Thread 87 (Thread 0xffff674459f0 (LWP 126934)):
91
#0  0x0000ffff8b1f922c in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0xaaaafafbb2b0) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
92
#1  __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafafbb258, cond=0xaaaafafbb288) at pthread_cond_wait.c:502
93
#2  __pthread_cond_wait (cond=0xaaaafafbb288, mutex=0xaaaafafbb258) at pthread_cond_wait.c:655
94
#3  0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
95
#4  0x0000aaaad1ee9ab4 in OSD::ShardedOpWQ::_process (this=0xaaaafaf6ef10, thread_index=16, hb=0xaaab5091b750) at ./src/osd/OSD.cc:10423
96
#5  0x0000aaaad2523928 in ShardedThreadPool::shardedthreadpool_worker (this=0xaaaafaf6ea48, thread_index=16) at ./src/common/WorkQueue.cc:311
97
#6  0x0000aaaad2526268 in ShardedThreadPool::WorkThreadSharded::entry (this=<optimized out>) at ./src/common/WorkQueue.h:706
98
#7  0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4bf) at pthread_create.c:463
99
#8  0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
100

    
101
Thread 86 (Thread 0xffff6b44d9f0 (LWP 126926)):
102
#0  0x0000ffff8b1f922c in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0xaaaafafb9eb0) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
103
#1  __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafafb9e58, cond=0xaaaafafb9e88) at pthread_cond_wait.c:502
104
#2  __pthread_cond_wait (cond=0xaaaafafb9e88, mutex=0xaaaafafb9e58) at pthread_cond_wait.c:655
105
#3  0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
106
---Type <return> to continue, or q <return> to quit---
107
#4  0x0000aaaad1ee9ab4 in OSD::ShardedOpWQ::_process (this=0xaaaafaf6ef10, thread_index=8, hb=0xaaab5091b160) at ./src/osd/OSD.cc:10423
108
#5  0x0000aaaad2523928 in ShardedThreadPool::shardedthreadpool_worker (this=0xaaaafaf6ea48, thread_index=8) at ./src/common/WorkQueue.cc:311
109
#6  0x0000aaaad2526268 in ShardedThreadPool::WorkThreadSharded::entry (this=<optimized out>) at ./src/common/WorkQueue.h:706
110
#7  0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4bf) at pthread_create.c:463
111
#8  0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
112

    
113
Thread 85 (Thread 0xffff684479f0 (LWP 126932)):
114
#0  0x0000ffff8b1f922c in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0xaaaafafbadb0) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
115
#1  __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafafbad58, cond=0xaaaafafbad88) at pthread_cond_wait.c:502
116
#2  __pthread_cond_wait (cond=0xaaaafafbad88, mutex=0xaaaafafbad58) at pthread_cond_wait.c:655
117
#3  0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
118
#4  0x0000aaaad1ee9ab4 in OSD::ShardedOpWQ::_process (this=0xaaaafaf6ef10, thread_index=14, hb=0xaaab5091b340) at ./src/osd/OSD.cc:10423
119
#5  0x0000aaaad2523928 in ShardedThreadPool::shardedthreadpool_worker (this=0xaaaafaf6ea48, thread_index=14) at ./src/common/WorkQueue.cc:311
120
#6  0x0000aaaad2526268 in ShardedThreadPool::WorkThreadSharded::entry (this=<optimized out>) at ./src/common/WorkQueue.h:706
121
#7  0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4bf) at pthread_create.c:463
122
#8  0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
123

    
124
Thread 84 (Thread 0xffff83c7e9f0 (LWP 125796)):
125
#0  0x0000ffff8b1f922c in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0xaaaafaf6fb90) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
126
#1  __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafaf6fb20, cond=0xaaaafaf6fb68) at pthread_cond_wait.c:502
127
#2  __pthread_cond_wait (cond=0xaaaafaf6fb68, mutex=0xaaaafaf6fb20) at pthread_cond_wait.c:655
128
#3  0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
129
#4  0x0000aaaad251d438 in SafeTimer::timer_thread (this=0xaaaafaf6fb50) at ./src/common/Timer.cc:106
130
#5  0x0000aaaad251e7f0 in SafeTimerThread::entry (this=<optimized out>) at ./src/common/Timer.cc:30
131
#6  0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c55f) at pthread_create.c:463
132
#7  0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
133

    
134
Thread 83 (Thread 0xffff654419f0 (LWP 126938)):
135
#0  0x0000ffff8b1f922c in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0xaaaafafb8ab0) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
136
#1  __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafafb8a58, cond=0xaaaafafb8a88) at pthread_cond_wait.c:502
137
#2  __pthread_cond_wait (cond=0xaaaafafb8a88, mutex=0xaaaafafb8a58) at pthread_cond_wait.c:655
138
#3  0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
139
#4  0x0000aaaad1ee9ab4 in OSD::ShardedOpWQ::_process (this=0xaaaafaf6ef10, thread_index=20, hb=0xaaab5091b7a0) at ./src/osd/OSD.cc:10423
140
#5  0x0000aaaad2523928 in ShardedThreadPool::shardedthreadpool_worker (this=0xaaaafaf6ea48, thread_index=20) at ./src/common/WorkQueue.cc:311
141
#6  0x0000aaaad2526268 in ShardedThreadPool::WorkThreadSharded::entry (this=<optimized out>) at ./src/common/WorkQueue.h:706
142
#7  0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4bf) at pthread_create.c:463
143
#8  0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
144

    
145
Thread 82 (Thread 0xffff75c629f0 (LWP 126890)):
146
#0  0x0000ffff8b1f922c in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0xaaaafbb83718) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
147
#1  __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafbb836c0, cond=0xaaaafbb836f0) at pthread_cond_wait.c:502
148
#2  __pthread_cond_wait (cond=0xaaaafbb836f0, mutex=0xaaaafbb836c0) at pthread_cond_wait.c:655
149
#3  0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
150
#4  0x0000aaaad284471c in DispatchQueue::run_local_delivery (this=0xaaaafbb83540) at ./src/msg/DispatchQueue.cc:117
151
#5  0x0000aaaad26978b0 in DispatchQueue::LocalDeliveryThread::entry (this=<optimized out>) at ./src/msg/DispatchQueue.h:115
152
#6  0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4ff) at pthread_create.c:463
153
---Type <return> to continue, or q <return> to quit---
154
#7  0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
155

    
156
Thread 81 (Thread 0xffff72c5c9f0 (LWP 126910)):
157
#0  0x0000ffff8b1f922c in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0xaaaafbb85b18) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
158
#1  __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafbb85ac0, cond=0xaaaafbb85af0) at pthread_cond_wait.c:502
159
#2  __pthread_cond_wait (cond=0xaaaafbb85af0, mutex=0xaaaafbb85ac0) at pthread_cond_wait.c:655
160
#3  0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
161
#4  0x0000aaaad284471c in DispatchQueue::run_local_delivery (this=0xaaaafbb85940) at ./src/msg/DispatchQueue.cc:117
162
#5  0x0000aaaad26978b0 in DispatchQueue::LocalDeliveryThread::entry (this=<optimized out>) at ./src/msg/DispatchQueue.h:115
163
#6  0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4ff) at pthread_create.c:463
164
#7  0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
165

    
166
Thread 80 (Thread 0xffff69c4a9f0 (LWP 126929)):
167
#0  0x0000ffff8b1f922c in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0xaaaafafba630) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
168
#1  __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafafba5d8, cond=0xaaaafafba608) at pthread_cond_wait.c:502
169
#2  __pthread_cond_wait (cond=0xaaaafafba608, mutex=0xaaaafafba5d8) at pthread_cond_wait.c:655
170
#3  0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
171
#4  0x0000aaaad1ee9ab4 in OSD::ShardedOpWQ::_process (this=0xaaaafaf6ef10, thread_index=11, hb=0xaaab5091b2f0) at ./src/osd/OSD.cc:10423
172
#5  0x0000aaaad2523928 in ShardedThreadPool::shardedthreadpool_worker (this=0xaaaafaf6ea48, thread_index=11) at ./src/common/WorkQueue.cc:311
173
#6  0x0000aaaad2526268 in ShardedThreadPool::WorkThreadSharded::entry (this=<optimized out>) at ./src/common/WorkQueue.h:706
174
#7  0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4bf) at pthread_create.c:463
175
#8  0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
176

    
177
Thread 79 (Thread 0xffff6f4559f0 (LWP 126918)):
178
#0  0x0000ffff8b1f922c in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0xaaaafafb8ab0) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
179
#1  __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafafb8a58, cond=0xaaaafafb8a88) at pthread_cond_wait.c:502
180
#2  __pthread_cond_wait (cond=0xaaaafafb8a88, mutex=0xaaaafafb8a58) at pthread_cond_wait.c:655
181
#3  0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
182
#4  0x0000aaaad1ee9ab4 in OSD::ShardedOpWQ::_process (this=0xaaaafaf6ef10, thread_index=0, hb=0xaaaafbc10940) at ./src/osd/OSD.cc:10423
183
#5  0x0000aaaad2523928 in ShardedThreadPool::shardedthreadpool_worker (this=0xaaaafaf6ea48, thread_index=0) at ./src/common/WorkQueue.cc:311
184
#6  0x0000aaaad2526268 in ShardedThreadPool::WorkThreadSharded::entry (this=<optimized out>) at ./src/common/WorkQueue.h:706
185
#7  0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4bf) at pthread_create.c:463
186
#8  0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
187

    
188
Thread 78 (Thread 0xffff64c409f0 (LWP 126939)):
189
#0  0x0000ffff8b1f922c in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0xaaaafafb8d30) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
190
#1  __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafafb8cd8, cond=0xaaaafafb8d08) at pthread_cond_wait.c:502
191
#2  __pthread_cond_wait (cond=0xaaaafafb8d08, mutex=0xaaaafafb8cd8) at pthread_cond_wait.c:655
192
#3  0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
193
#4  0x0000aaaad1ee9ab4 in OSD::ShardedOpWQ::_process (this=0xaaaafaf6ef10, thread_index=21, hb=0xaaab50919d60) at ./src/osd/OSD.cc:10423
194
#5  0x0000aaaad2523928 in ShardedThreadPool::shardedthreadpool_worker (this=0xaaaafaf6ea48, thread_index=21) at ./src/common/WorkQueue.cc:311
195
#6  0x0000aaaad2526268 in ShardedThreadPool::WorkThreadSharded::entry (this=<optimized out>) at ./src/common/WorkQueue.h:706
196
#7  0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4bf) at pthread_create.c:463
197
#8  0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
198

    
199
Thread 77 (Thread 0xffff5fc369f0 (LWP 126949)):
200
---Type <return> to continue, or q <return> to quit---
201
#0  0x0000ffff8b1f922c in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0xaaaafafba630) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
202
#1  __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafafba5d8, cond=0xaaaafafba608) at pthread_cond_wait.c:502
203
#2  __pthread_cond_wait (cond=0xaaaafafba608, mutex=0xaaaafafba5d8) at pthread_cond_wait.c:655
204
#3  0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
205
#4  0x0000aaaad1ee9ab4 in OSD::ShardedOpWQ::_process (this=0xaaaafaf6ef10, thread_index=31, hb=0xaaab509198b0) at ./src/osd/OSD.cc:10423
206
#5  0x0000aaaad2523928 in ShardedThreadPool::shardedthreadpool_worker (this=0xaaaafaf6ea48, thread_index=31) at ./src/common/WorkQueue.cc:311
207
#6  0x0000aaaad2526268 in ShardedThreadPool::WorkThreadSharded::entry (this=<optimized out>) at ./src/common/WorkQueue.h:706
208
#7  0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4bf) at pthread_create.c:463
209
#8  0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
210

    
211
Thread 76 (Thread 0xffff5bc2e9f0 (LWP 126957)):
212
#0  0x0000ffff8b1f922c in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0xaaaafafbba34) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
213
#1  __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafafbb9d8, cond=0xaaaafafbba08) at pthread_cond_wait.c:502
214
#2  __pthread_cond_wait (cond=0xaaaafafbba08, mutex=0xaaaafafbb9d8) at pthread_cond_wait.c:655
215
#3  0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
216
#4  0x0000aaaad1ee9ab4 in OSD::ShardedOpWQ::_process (this=0xaaaafaf6ef10, thread_index=39, hb=0xaaab509199a0) at ./src/osd/OSD.cc:10423
217
#5  0x0000aaaad2523928 in ShardedThreadPool::shardedthreadpool_worker (this=0xaaaafaf6ea48, thread_index=39) at ./src/common/WorkQueue.cc:311
218
#6  0x0000aaaad2526268 in ShardedThreadPool::WorkThreadSharded::entry (this=<optimized out>) at ./src/common/WorkQueue.h:706
219
#7  0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4bf) at pthread_create.c:463
220
#8  0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
221

    
222
Thread 75 (Thread 0xffff893b89f0 (LWP 124384)):
223
#0  0x0000ffff8b1fc944 in __libc_write (fd=fd@entry=98, buf=buf@entry=0xffff893b7d90, nbytes=nbytes@entry=65) at ../sysdeps/unix/sysv/linux/write.c:27
224
#1  0x0000aaaad28a35e0 in Infiniband::QueuePair::send_cm_meta (this=0xaaab4fe50820, cct=0xaaaafafd4000, socket_fd=98) at ./src/msg/async/rdma/Infiniband.cc:444
225
#2  0x0000aaaad28ad9d4 in RDMAConnectedSocketImpl::handle_connection (this=0xaaab4f3607e0) at ./src/msg/async/rdma/RDMAConnectedSocketImpl.cc:152
226

    
227
#3  0x0000aaaad269d1ac in EventCenter::process_events (this=this@entry=0xaaaafbb9cb88, timeout_microseconds=<optimized out>,
228
    working_dur=working_dur@entry=0xffff893b8160) at /usr/include/c++/9/bits/basic_ios.h:282
229
#4  0x0000aaaad26a33c0 in NetworkStack::<lambda()>::operator() (__closure=0xaaaafbc0c518, __closure=0xaaaafbc0c518) at ./src/msg/async/Stack.cc:53
230
#5  std::_Function_handler<void(), NetworkStack::add_thread(unsigned int)::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...)
231
    at /usr/include/c++/9/bits/std_function.h:300
232
#6  0x0000ffff8b03ded4 in ?? () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
233
#7  0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9aeff) at pthread_create.c:463
234
#8  0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
235

    
236
Thread 74 (Thread 0xffff8a3ba9f0 (LWP 124377)):
237
#0  0x0000ffff8b1fc944 in __libc_write (fd=<optimized out>, buf=0xffff8a3b8710, nbytes=408) at ../sysdeps/unix/sysv/linux/write.c:27
238
#1  0x0000ffff8b1abfbc in ibv_exp_cmd_create_qp () from /usr/lib/aarch64-linux-gnu/libibverbs.so.1
239
#2  0x0000ffff86b67c5c in create_qp (context=0xaaaafbc98300, attrx=0xb8, attrx@entry=0xffff8a3b99a0, is_exp=1, is_exp@entry=0) at src/verbs.c:2244
240
#3  0x0000ffff86b690a4 in mlx5_create_qp (pd=<optimized out>, attr=0xffff8a3b9b08) at src/verbs.c:2370
241
#4  0x0000ffff8b1aa76c in ibv_create_qp () from /usr/lib/aarch64-linux-gnu/libibverbs.so.1
242
#5  0x0000aaaad28a6f60 in Infiniband::QueuePair::init (this=this@entry=0xaaaafaf558e0) at ./src/msg/async/rdma/Infiniband.cc:334
243
#6  0x0000aaaad28a9038 in Infiniband::create_queue_pair (this=<optimized out>, cct=cct@entry=0xaaaafafd4000, tx=0xaaaafbc0d560, rx=0xaaaafbc0d530, type=4210454592,
244
    type@entry=IBV_QPT_RC, cm_id=cm_id@entry=0x0) at ./src/msg/async/rdma/Infiniband.cc:1192
245
#7  0x0000aaaad28a9b50 in RDMAConnectedSocketImpl::RDMAConnectedSocketImpl (this=0xaaab4f361c20, cct=0xaaaafafd4000,
246
    ib=std::shared_ptr<Infiniband> (use count 48, weak count 0) = {...}, rdma_dispatcher=..., w=<optimized out>) at ./src/msg/async/rdma/RDMAStack.h:105
247
#8  0x0000aaaad28b1d8c in RDMAServerSocketImpl::accept (this=0xaaaafaf3f700, sock=0xffff8a3b9db8, opt=..., out=0xffff8a3b9e10, w=<optimized out>)
248
---Type <return> to continue, or q <return> to quit---
249
    at ./src/msg/async/rdma/RDMAServerSocketImpl.cc:115
250
#9  0x0000aaaad26956a0 in ServerSocket::accept (w=0xaaaafbb9c840, out=0xffff8a3b9e10, opt=..., sock=0xffff8a3b9db8, this=0xaaaafaf02eb0)
251
    at /usr/include/c++/9/bits/unique_ptr.h:357
252
#10 Processor::accept (this=0xaaab0bcef2c0) at ./src/msg/async/AsyncMessenger.cc:192
253
#11 0x0000aaaad269d1ac in EventCenter::process_events (this=this@entry=0xaaaafbb9c608, timeout_microseconds=<optimized out>,
254
    working_dur=working_dur@entry=0xffff8a3ba160) at /usr/include/c++/9/bits/basic_ios.h:282
255
#12 0x0000aaaad26a33c0 in NetworkStack::<lambda()>::operator() (__closure=0xaaaafbc0c4b8, __closure=0xaaaafbc0c4b8) at ./src/msg/async/Stack.cc:53
256
#13 std::_Function_handler<void(), NetworkStack::add_thread(unsigned int)::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...)
257
    at /usr/include/c++/9/bits/std_function.h:300
258
#14 0x0000ffff8b03ded4 in ?? () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
259
#15 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9aeff) at pthread_create.c:463
260
#16 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
261

    
262
Thread 73 (Thread 0xffff68c489f0 (LWP 126931)):
263
#0  0x0000ffff8b1f922c in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0xaaaafafbab34) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
264
#1  __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafafbaad8, cond=0xaaaafafbab08) at pthread_cond_wait.c:502
265
#2  __pthread_cond_wait (cond=0xaaaafafbab08, mutex=0xaaaafafbaad8) at pthread_cond_wait.c:655
266
#3  0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
267
#4  0x0000aaaad1ee9ab4 in OSD::ShardedOpWQ::_process (this=0xaaaafaf6ef10, thread_index=13, hb=0xaaab5091b2a0) at ./src/osd/OSD.cc:10423
268
#5  0x0000aaaad2523928 in ShardedThreadPool::shardedthreadpool_worker (this=0xaaaafaf6ea48, thread_index=13) at ./src/common/WorkQueue.cc:311
269
#6  0x0000aaaad2526268 in ShardedThreadPool::WorkThreadSharded::entry (this=<optimized out>) at ./src/common/WorkQueue.h:706
270
#7  0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4bf) at pthread_create.c:463
271
#8  0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
272

    
273
Thread 72 (Thread 0xffff6443f9f0 (LWP 126940)):
274
#0  0x0000ffff8b1f922c in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0xaaaafafb8fb4) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
275
#1  __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafafb8f58, cond=0xaaaafafb8f88) at pthread_cond_wait.c:502
276
#2  __pthread_cond_wait (cond=0xaaaafafb8f88, mutex=0xaaaafafb8f58) at pthread_cond_wait.c:655
277
#3  0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
278
#4  0x0000aaaad1ee9ab4 in OSD::ShardedOpWQ::_process (this=0xaaaafaf6ef10, thread_index=22, hb=0xaaab50919db0) at ./src/osd/OSD.cc:10423
279
#5  0x0000aaaad2523928 in ShardedThreadPool::shardedthreadpool_worker (this=0xaaaafaf6ea48, thread_index=22) at ./src/common/WorkQueue.cc:311
280
#6  0x0000aaaad2526268 in ShardedThreadPool::WorkThreadSharded::entry (this=<optimized out>) at ./src/common/WorkQueue.h:706
281
#7  0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4bf) at pthread_create.c:463
282
#8  0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
283

    
284
Thread 71 (Thread 0xffff6cc509f0 (LWP 126923)):
285
#0  0x0000ffff8b1f922c in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0xaaaafafb9730) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
286
#1  __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafafb96d8, cond=0xaaaafafb9708) at pthread_cond_wait.c:502
287
#2  __pthread_cond_wait (cond=0xaaaafafb9708, mutex=0xaaaafafb96d8) at pthread_cond_wait.c:655
288
#3  0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
289
#4  0x0000aaaad1ee9ab4 in OSD::ShardedOpWQ::_process (this=0xaaaafaf6ef10, thread_index=5, hb=0xaaab5091b070) at ./src/osd/OSD.cc:10423
290
#5  0x0000aaaad2523928 in ShardedThreadPool::shardedthreadpool_worker (this=0xaaaafaf6ea48, thread_index=5) at ./src/common/WorkQueue.cc:311
291
#6  0x0000aaaad2526268 in ShardedThreadPool::WorkThreadSharded::entry (this=<optimized out>) at ./src/common/WorkQueue.h:706
292
#7  0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4bf) at pthread_create.c:463
293
#8  0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
294

    
295
---Type <return> to continue, or q <return> to quit---
296
Thread 70 (Thread 0xffff63c3e9f0 (LWP 126941)):
297
#0  0x0000ffff8b1f922c in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0xaaaafafb9230) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
298
#1  __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafafb91d8, cond=0xaaaafafb9208) at pthread_cond_wait.c:502
299
#2  __pthread_cond_wait (cond=0xaaaafafb9208, mutex=0xaaaafafb91d8) at pthread_cond_wait.c:655
300
#3  0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
301
#4  0x0000aaaad1ee9ab4 in OSD::ShardedOpWQ::_process (this=0xaaaafaf6ef10, thread_index=23, hb=0xaaab50919e00) at ./src/osd/OSD.cc:10423
302
#5  0x0000aaaad2523928 in ShardedThreadPool::shardedthreadpool_worker (this=0xaaaafaf6ea48, thread_index=23) at ./src/common/WorkQueue.cc:311
303
#6  0x0000aaaad2526268 in ShardedThreadPool::WorkThreadSharded::entry (this=<optimized out>) at ./src/common/WorkQueue.h:706
304
#7  0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4bf) at pthread_create.c:463
305
#8  0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
306

    
307
Thread 69 (Thread 0xffff61c3a9f0 (LWP 126945)):
308
#0  0x0000ffff8b1f922c in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0xaaaafafb9c34) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
309
#1  __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafafb9bd8, cond=0xaaaafafb9c08) at pthread_cond_wait.c:502
310
#2  __pthread_cond_wait (cond=0xaaaafafb9c08, mutex=0xaaaafafb9bd8) at pthread_cond_wait.c:655
311
#3  0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
312
#4  0x0000aaaad1ee9ab4 in OSD::ShardedOpWQ::_process (this=0xaaaafaf6ef10, thread_index=27, hb=0xaaab50919e50) at ./src/osd/OSD.cc:10423
313
#5  0x0000aaaad2523928 in ShardedThreadPool::shardedthreadpool_worker (this=0xaaaafaf6ea48, thread_index=27) at ./src/common/WorkQueue.cc:311
314
#6  0x0000aaaad2526268 in ShardedThreadPool::WorkThreadSharded::entry (this=<optimized out>) at ./src/common/WorkQueue.h:706
315
#7  0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4bf) at pthread_create.c:463
316
#8  0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
317

    
318
Thread 68 (Thread 0xffff6fc569f0 (LWP 126917)):
319
#0  0x0000ffff8b1f922c in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0xaaaafaf6f724) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
320
#1  __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafaf6f6c8, cond=0xaaaafaf6f6f8) at pthread_cond_wait.c:502
321
#2  __pthread_cond_wait (cond=0xaaaafaf6f6f8, mutex=0xaaaafaf6f6c8) at pthread_cond_wait.c:655
322
#3  0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
323
#4  0x0000aaaad1e928e4 in OSDService::agent_entry (this=0xaaaafaf6f3c8) at ./src/osd/OSD.cc:560
324
#5  0x0000aaaad1f47c40 in OSDService::AgentThread::entry (this=<optimized out>) at ./src/osd/OSD.h:383
325
#6  0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c56f) at pthread_create.c:463
326
#7  0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
327

    
328
Thread 67 (Thread 0xffff60c389f0 (LWP 126947)):
329
#0  0x0000ffff8b1f922c in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0xaaaafafba130) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
330
#1  __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafafba0d8, cond=0xaaaafafba108) at pthread_cond_wait.c:502
331
#2  __pthread_cond_wait (cond=0xaaaafafba108, mutex=0xaaaafafba0d8) at pthread_cond_wait.c:655
332
#3  0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
333
#4  0x0000aaaad1ee9ab4 in OSD::ShardedOpWQ::_process (this=0xaaaafaf6ef10, thread_index=29, hb=0xaaab50919f90) at ./src/osd/OSD.cc:10423
334
#5  0x0000aaaad2523928 in ShardedThreadPool::shardedthreadpool_worker (this=0xaaaafaf6ea48, thread_index=29) at ./src/common/WorkQueue.cc:311
335
#6  0x0000aaaad2526268 in ShardedThreadPool::WorkThreadSharded::entry (this=<optimized out>) at ./src/common/WorkQueue.h:706
336
#7  0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4bf) at pthread_create.c:463
337
#8  0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
338

    
339
Thread 66 (Thread 0xffff614399f0 (LWP 126946)):
340
#0  0x0000ffff8b1f922c in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0xaaaafafb9eb0) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
341
#1  __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafafb9e58, cond=0xaaaafafb9e88) at pthread_cond_wait.c:502
342
---Type <return> to continue, or q <return> to quit---
343
#2  __pthread_cond_wait (cond=0xaaaafafb9e88, mutex=0xaaaafafb9e58) at pthread_cond_wait.c:655
344
#3  0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
345
#4  0x0000aaaad1ee9ab4 in OSD::ShardedOpWQ::_process (this=0xaaaafaf6ef10, thread_index=28, hb=0xaaab50919ea0) at ./src/osd/OSD.cc:10423
346
#5  0x0000aaaad2523928 in ShardedThreadPool::shardedthreadpool_worker (this=0xaaaafaf6ea48, thread_index=28) at ./src/common/WorkQueue.cc:311
347
#6  0x0000aaaad2526268 in ShardedThreadPool::WorkThreadSharded::entry (this=<optimized out>) at ./src/common/WorkQueue.h:706
348
#7  0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4bf) at pthread_create.c:463
349
#8  0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
350

    
351
Thread 65 (Thread 0xffff6ec549f0 (LWP 126919)):
352
#0  0x0000ffff8b1f922c in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0xaaaafafb8d30) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
353
#1  __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafafb8cd8, cond=0xaaaafafb8d08) at pthread_cond_wait.c:502
354
#2  __pthread_cond_wait (cond=0xaaaafafb8d08, mutex=0xaaaafafb8cd8) at pthread_cond_wait.c:655
355
#3  0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
356
#4  0x0000aaaad1ee9ab4 in OSD::ShardedOpWQ::_process (this=0xaaaafaf6ef10, thread_index=1, hb=0xaaaafbc109e0) at ./src/osd/OSD.cc:10423
357
#5  0x0000aaaad2523928 in ShardedThreadPool::shardedthreadpool_worker (this=0xaaaafaf6ea48, thread_index=1) at ./src/common/WorkQueue.cc:311
358
#6  0x0000aaaad2526268 in ShardedThreadPool::WorkThreadSharded::entry (this=<optimized out>) at ./src/common/WorkQueue.h:706
359
#7  0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4bf) at pthread_create.c:463
360
#8  0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
361

    
362
Thread 64 (Thread 0xffff62c3c9f0 (LWP 126943)):
363
#0  0x0000ffff8b1f922c in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0xaaaafafb9730) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
364

    
365
#1  __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafafb96d8, cond=0xaaaafafb9708) at pthread_cond_wait.c:502
366
#2  __pthread_cond_wait (cond=0xaaaafafb9708, mutex=0xaaaafafb96d8) at pthread_cond_wait.c:655
367
#3  0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
368
#4  0x0000aaaad1ee9ab4 in OSD::ShardedOpWQ::_process (this=0xaaaafaf6ef10, thread_index=25, hb=0xaaab50919f40) at ./src/osd/OSD.cc:10423
369
#5  0x0000aaaad2523928 in ShardedThreadPool::shardedthreadpool_worker (this=0xaaaafaf6ea48, thread_index=25) at ./src/common/WorkQueue.cc:311
370
#6  0x0000aaaad2526268 in ShardedThreadPool::WorkThreadSharded::entry (this=<optimized out>) at ./src/common/WorkQueue.h:706
371
#7  0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4bf) at pthread_create.c:463
372
#8  0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
373

    
374
Thread 63 (Thread 0xffff6343d9f0 (LWP 126942)):
375
#0  0x0000ffff8b1f922c in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0xaaaafafb94b0) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
376
#1  __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafafb9458, cond=0xaaaafafb9488) at pthread_cond_wait.c:502
377
#2  __pthread_cond_wait (cond=0xaaaafafb9488, mutex=0xaaaafafb9458) at pthread_cond_wait.c:655
378
#3  0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
379
#4  0x0000aaaad1ee9ab4 in OSD::ShardedOpWQ::_process (this=0xaaaafaf6ef10, thread_index=24, hb=0xaaab50919ef0) at ./src/osd/OSD.cc:10423
380
#5  0x0000aaaad2523928 in ShardedThreadPool::shardedthreadpool_worker (this=0xaaaafaf6ea48, thread_index=24) at ./src/common/WorkQueue.cc:311
381
#6  0x0000aaaad2526268 in ShardedThreadPool::WorkThreadSharded::entry (this=<optimized out>) at ./src/common/WorkQueue.h:706
382
#7  0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4bf) at pthread_create.c:463
383
#8  0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
384

    
385
Thread 62 (Thread 0xffff81c7a9f0 (LWP 126261)):
386
#0  0x0000ffff8b1f922c in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0xaaab4be125c0) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
387
#1  __pthread_cond_wait_common (abstime=0x0, mutex=0xaaab4be12568, cond=0xaaab4be12598) at pthread_cond_wait.c:502
388
#2  __pthread_cond_wait (cond=0xaaab4be12598, mutex=0xaaab4be12568) at pthread_cond_wait.c:655
389
#3  0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
390
---Type <return> to continue, or q <return> to quit---
391
#4  0x0000aaaad24bb69c in KernelDevice::_discard_thread (this=0xaaab4be12380) at ./src/os/bluestore/KernelDevice.cc:642
392
#5  0x0000aaaad24c9c68 in KernelDevice::DiscardThread::entry (this=<optimized out>) at ./src/os/bluestore/KernelDevice.h:74
393
#6  0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9951f) at pthread_create.c:463
394
#7  0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
395

    
396
Thread 61 (Thread 0xffff6bc4e9f0 (LWP 126925)):
397
#0  0x0000ffff8b1f922c in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0xaaaafafb9c34) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
398
#1  __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafafb9bd8, cond=0xaaaafafb9c08) at pthread_cond_wait.c:502
399
#2  __pthread_cond_wait (cond=0xaaaafafb9c08, mutex=0xaaaafafb9bd8) at pthread_cond_wait.c:655
400
#3  0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
401
#4  0x0000aaaad1ee9ab4 in OSD::ShardedOpWQ::_process (this=0xaaaafaf6ef10, thread_index=7, hb=0xaaab5091b0c0) at ./src/osd/OSD.cc:10423
402
#5  0x0000aaaad2523928 in ShardedThreadPool::shardedthreadpool_worker (this=0xaaaafaf6ea48, thread_index=7) at ./src/common/WorkQueue.cc:311
403
#6  0x0000aaaad2526268 in ShardedThreadPool::WorkThreadSharded::entry (this=<optimized out>) at ./src/common/WorkQueue.h:706
404
#7  0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4bf) at pthread_create.c:463
405
#8  0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
406

    
407
Thread 60 (Thread 0xffff6c44f9f0 (LWP 126924)):
408
#0  0x0000ffff8b1f922c in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0xaaaafafb99b4) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
409
#1  __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafafb9958, cond=0xaaaafafb9988) at pthread_cond_wait.c:502
410
#2  __pthread_cond_wait (cond=0xaaaafafb9988, mutex=0xaaaafafb9958) at pthread_cond_wait.c:655
411
#3  0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
412
#4  0x0000aaaad1ee9ab4 in OSD::ShardedOpWQ::_process (this=0xaaaafaf6ef10, thread_index=6, hb=0xaaab5091b110) at ./src/osd/OSD.cc:10423
413
#5  0x0000aaaad2523928 in ShardedThreadPool::shardedthreadpool_worker (this=0xaaaafaf6ea48, thread_index=6) at ./src/common/WorkQueue.cc:311
414
#6  0x0000aaaad2526268 in ShardedThreadPool::WorkThreadSharded::entry (this=<optimized out>) at ./src/common/WorkQueue.h:706
415
#7  0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4bf) at pthread_create.c:463
416
#8  0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
417

    
418
Thread 59 (Thread 0xffff6dc529f0 (LWP 126921)):
419
#0  0x0000ffff8b1f922c in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0xaaaafafb9230) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
420
#1  __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafafb91d8, cond=0xaaaafafb9208) at pthread_cond_wait.c:502
421
#2  __pthread_cond_wait (cond=0xaaaafafb9208, mutex=0xaaaafafb91d8) at pthread_cond_wait.c:655
422
#3  0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
423
#4  0x0000aaaad1ee9ab4 in OSD::ShardedOpWQ::_process (this=0xaaaafaf6ef10, thread_index=3, hb=0xaaaafbc10a80) at ./src/osd/OSD.cc:10423
424
#5  0x0000aaaad2523928 in ShardedThreadPool::shardedthreadpool_worker (this=0xaaaafaf6ea48, thread_index=3) at ./src/common/WorkQueue.cc:311
425
#6  0x0000aaaad2526268 in ShardedThreadPool::WorkThreadSharded::entry (this=<optimized out>) at ./src/common/WorkQueue.h:706
426
#7  0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4bf) at pthread_create.c:463
427
#8  0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
428

    
429
Thread 58 (Thread 0xffff80c789f0 (LWP 126001)):
430
#0  0x0000ffff8b1f922c in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0xaaaafaf8bec8) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
431
#1  __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafaf8be70, cond=0xaaaafaf8bea0) at pthread_cond_wait.c:502
432
#2  __pthread_cond_wait (cond=0xaaaafaf8bea0, mutex=0xaaaafaf8be70) at pthread_cond_wait.c:655
433
#3  0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
434
#4  0x0000aaaad2c7cb08 in rocksdb::ThreadPoolImpl::Impl::BGThread(unsigned long) ()
435
#5  0x0000aaaad2c7ce58 in rocksdb::ThreadPoolImpl::Impl::BGThreadWrapper(void*) ()
436
#6  0x0000aaaad2c8438c in void* std::__invoke_impl<void*, void* (*)(void*), rocksdb::BGThreadMetadata*>(std::__invoke_other, void* (*&&)(void*), rocksdb::BGThreadMet---Type <return> to continue, or q <return> to quit---
437
adata*&&) ()
438
#7  0x0000aaaad2c842c4 in std::__invoke_result<void* (*)(void*), rocksdb::BGThreadMetadata*>::type std::__invoke<void* (*)(void*), rocksdb::BGThreadMetadata*>(void* (*&&)(void*), rocksdb::BGThreadMetadata*&&) ()
439
#8  0x0000aaaad2c84218 in void* std::thread::_Invoker<std::tuple<void* (*)(void*), rocksdb::BGThreadMetadata*> >::_M_invoke<0ul, 1ul>(std::_Index_tuple<0ul, 1ul>)
440
    ()
441
#9  0x0000aaaad2c841ac in std::thread::_Invoker<std::tuple<void* (*)(void*), rocksdb::BGThreadMetadata*> >::operator()() ()
442
#10 0x0000aaaad2c84178 in std::thread::_State_impl<std::thread::_Invoker<std::tuple<void* (*)(void*), rocksdb::BGThreadMetadata*> > >::_M_run() ()
443
#11 0x0000ffff8b03ded4 in ?? () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
444
#12 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9aacf) at pthread_create.c:463
445
#13 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
446

    
447
Thread 57 (Thread 0xffff804779f0 (LWP 126002)):
448
#0  0x0000ffff8b1f922c in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0xaaaafaf8bec8) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
449
#1  __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafaf8be70, cond=0xaaaafaf8bea0) at pthread_cond_wait.c:502
450
#2  __pthread_cond_wait (cond=0xaaaafaf8bea0, mutex=0xaaaafaf8be70) at pthread_cond_wait.c:655
451
#3  0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
452
#4  0x0000aaaad2c7cb08 in rocksdb::ThreadPoolImpl::Impl::BGThread(unsigned long) ()
453
#5  0x0000aaaad2c7ce58 in rocksdb::ThreadPoolImpl::Impl::BGThreadWrapper(void*) ()
454
#6  0x0000aaaad2c8438c in void* std::__invoke_impl<void*, void* (*)(void*), rocksdb::BGThreadMetadata*>(std::__invoke_other, void* (*&&)(void*), rocksdb::BGThreadMetadata*&&) ()
455
#7  0x0000aaaad2c842c4 in std::__invoke_result<void* (*)(void*), rocksdb::BGThreadMetadata*>::type std::__invoke<void* (*)(void*), rocksdb::BGThreadMetadata*>(void* (*&&)(void*), rocksdb::BGThreadMetadata*&&) ()
456
#8  0x0000aaaad2c84218 in void* std::thread::_Invoker<std::tuple<void* (*)(void*), rocksdb::BGThreadMetadata*> >::_M_invoke<0ul, 1ul>(std::_Index_tuple<0ul, 1ul>)
457
    ()
458
#9  0x0000aaaad2c841ac in std::thread::_Invoker<std::tuple<void* (*)(void*), rocksdb::BGThreadMetadata*> >::operator()() ()
459
#10 0x0000aaaad2c84178 in std::thread::_State_impl<std::thread::_Invoker<std::tuple<void* (*)(void*), rocksdb::BGThreadMetadata*> > >::_M_run() ()
460
#11 0x0000ffff8b03ded4 in ?? () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
461
#12 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9aacf) at pthread_create.c:463
462
#13 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
463

    
464
Thread 56 (Thread 0xffff5b42d9f0 (LWP 126958)):
465
#0  0x0000ffff8b1f95d4 in futex_abstimed_wait_cancelable (private=<optimized out>, abstime=0xffff5b42d178, expected=0, futex_word=0xaaaafaf6ec90)
466
    at ../sysdeps/unix/sysv/linux/futex-internal.h:205
467
#1  __pthread_cond_wait_common (abstime=0xffff5b42d178, mutex=0xaaaafaf6ec08, cond=0xaaaafaf6ec68) at pthread_cond_wait.c:539
468
#2  __pthread_cond_timedwait (cond=0xaaaafaf6ec68, mutex=0xaaaafaf6ec08, abstime=0xffff5b42d178) at pthread_cond_wait.c:667
469
#3  0x0000aaaad1eb9474 in __gthread_cond_timedwait (__abs_timeout=0xffff5b42d178, __mutex=<optimized out>, __cond=0xaaaafaf6ec68)
470
    at /usr/include/aarch64-linux-gnu/c++/9/bits/gthr-default.h:872
471

    
472

    
473

    
474

    
475
#4  std::condition_variable::__wait_until_impl<std::chrono::duration<long, std::ratio<1l, 1000000000l> > > (__atime=<synthetic pointer>..., __lock=...,
476
    this=0xaaaafaf6ec68) at /usr/include/c++/9/condition_variable:188
477
#5  std::condition_variable::wait_until<std::chrono::_V2::steady_clock, std::chrono::duration<long, std::ratio<1l, 1000000000l> > > (__atime=..., __lock=...,
478
    this=0xaaaafaf6ec68) at /usr/include/c++/9/condition_variable:121
479
#6  std::condition_variable::wait_for<unsigned long, std::ratio<1l, 1000000000l> > (__rtime=<synthetic pointer>..., __lock=..., this=0xaaaafaf6ec68)
480
    at /usr/include/c++/9/condition_variable:152
481
#7  OSD::heartbeat_entry (this=0xaaaafaf6e000) at ./src/osd/OSD.cc:5591
482
#8  0x0000aaaad1f47c68 in OSD::T_Heartbeat::entry (this=<optimized out>) at ./src/osd/OSD.h:1503
483
#9  0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c5df) at pthread_create.c:463
484
---Type <return> to continue, or q <return> to quit---
485
#10 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
486

    
487
Thread 55 (Thread 0xffff754619f0 (LWP 126891)):
488
#0  0x0000ffff8b1f922c in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0xaaaafbb850ac) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
489
#1  __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafbb85050, cond=0xaaaafbb85080) at pthread_cond_wait.c:502
490
#2  __pthread_cond_wait (cond=0xaaaafbb85080, mutex=0xaaaafbb85050) at pthread_cond_wait.c:655
491
#3  0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
492
#4  0x0000aaaad28449ac in DispatchQueue::entry (this=0xaaaafbb85040) at ./src/msg/DispatchQueue.cc:210
493
#5  0x0000aaaad26977f0 in DispatchQueue::DispatchThread::entry (this=<optimized out>) at ./src/msg/DispatchQueue.h:101
494
#6  0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4df) at pthread_create.c:463
495
#7  0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
496

    
497
Thread 54 (Thread 0xffff66c449f0 (LWP 126935)):
498
#0  0x0000ffff8b1f922c in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0xaaaafafbb530) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
499
#1  __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafafbb4d8, cond=0xaaaafafbb508) at pthread_cond_wait.c:502
500
#2  __pthread_cond_wait (cond=0xaaaafafbb508, mutex=0xaaaafafbb4d8) at pthread_cond_wait.c:655
501
#3  0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
502
#4  0x0000aaaad1ee9ab4 in OSD::ShardedOpWQ::_process (this=0xaaaafaf6ef10, thread_index=17, hb=0xaaab5091b6b0) at ./src/osd/OSD.cc:10423
503
#5  0x0000aaaad2523928 in ShardedThreadPool::shardedthreadpool_worker (this=0xaaaafaf6ea48, thread_index=17) at ./src/common/WorkQueue.cc:311
504
#6  0x0000aaaad2526268 in ShardedThreadPool::WorkThreadSharded::entry (this=<optimized out>) at ./src/common/WorkQueue.h:706
505
#7  0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4bf) at pthread_create.c:463
506
#8  0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
507

    
508
Thread 53 (Thread 0xffff65c429f0 (LWP 126937)):
509
#0  0x0000ffff8b1f922c in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0xaaaafafbba34) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
510
#1  __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafafbb9d8, cond=0xaaaafafbba08) at pthread_cond_wait.c:502
511
#2  __pthread_cond_wait (cond=0xaaaafafbba08, mutex=0xaaaafafbb9d8) at pthread_cond_wait.c:655
512
#3  0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
513
#4  0x0000aaaad1ee9ab4 in OSD::ShardedOpWQ::_process (this=0xaaaafaf6ef10, thread_index=19, hb=0xaaab50919cc0) at ./src/osd/OSD.cc:10423
514
#5  0x0000aaaad2523928 in ShardedThreadPool::shardedthreadpool_worker (this=0xaaaafaf6ea48, thread_index=19) at ./src/common/WorkQueue.cc:311
515
#6  0x0000aaaad2526268 in ShardedThreadPool::WorkThreadSharded::entry (this=<optimized out>) at ./src/common/WorkQueue.h:706
516
#7  0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4bf) at pthread_create.c:463
517
#8  0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
518

    
519
Thread 52 (Thread 0xffff6ac4c9f0 (LWP 126927)):
520
#0  0x0000ffff8b1f922c in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0xaaaafafba130) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
521
#1  __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafafba0d8, cond=0xaaaafafba108) at pthread_cond_wait.c:502
522
#2  __pthread_cond_wait (cond=0xaaaafafba108, mutex=0xaaaafafba0d8) at pthread_cond_wait.c:655
523
#3  0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
524
#4  0x0000aaaad1ee9ab4 in OSD::ShardedOpWQ::_process (this=0xaaaafaf6ef10, thread_index=9, hb=0xaaab5091b200) at ./src/osd/OSD.cc:10423
525
#5  0x0000aaaad2523928 in ShardedThreadPool::shardedthreadpool_worker (this=0xaaaafaf6ea48, thread_index=9) at ./src/common/WorkQueue.cc:311
526
#6  0x0000aaaad2526268 in ShardedThreadPool::WorkThreadSharded::entry (this=<optimized out>) at ./src/common/WorkQueue.h:706
527
#7  0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4bf) at pthread_create.c:463
528
#8  0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
529

    
530
Thread 51 (Thread 0xffff5ec349f0 (LWP 126951)):
531
---Type <return> to continue, or q <return> to quit---
532
#0  0x0000ffff8b1f922c in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0xaaaafafbab34) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
533
#1  __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafafbaad8, cond=0xaaaafafbab08) at pthread_cond_wait.c:502
534
#2  __pthread_cond_wait (cond=0xaaaafafbab08, mutex=0xaaaafafbaad8) at pthread_cond_wait.c:655
535
#3  0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
536

    
537
#4  0x0000aaaad1ee9ab4 in OSD::ShardedOpWQ::_process (this=0xaaaafaf6ef10, thread_index=33, hb=0xaaab5091a030) at ./src/osd/OSD.cc:10423
538
#5  0x0000aaaad2523928 in ShardedThreadPool::shardedthreadpool_worker (this=0xaaaafaf6ea48, thread_index=33) at ./src/common/WorkQueue.cc:311
539
#6  0x0000aaaad2526268 in ShardedThreadPool::WorkThreadSharded::entry (this=<optimized out>) at ./src/common/WorkQueue.h:706
540
#7  0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4bf) at pthread_create.c:463
541
#8  0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
542

    
543
Thread 50 (Thread 0xffff5f4359f0 (LWP 126950)):
544
#0  0x0000ffff8b1f922c in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0xaaaafafba8b0) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
545
#1  __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafafba858, cond=0xaaaafafba888) at pthread_cond_wait.c:502
546
#2  __pthread_cond_wait (cond=0xaaaafafba888, mutex=0xaaaafafba858) at pthread_cond_wait.c:655
547
#3  0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
548
#4  0x0000aaaad1ee9ab4 in OSD::ShardedOpWQ::_process (this=0xaaaafaf6ef10, thread_index=32, hb=0xaaab5091b390) at ./src/osd/OSD.cc:10423
549
#5  0x0000aaaad2523928 in ShardedThreadPool::shardedthreadpool_worker (this=0xaaaafaf6ea48, thread_index=32) at ./src/common/WorkQueue.cc:311
550
#6  0x0000aaaad2526268 in ShardedThreadPool::WorkThreadSharded::entry (this=<optimized out>) at ./src/common/WorkQueue.h:706
551
#7  0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4bf) at pthread_create.c:463
552
#8  0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
553

    
554
Thread 49 (Thread 0xffff5c42f9f0 (LWP 126956)):
555
#0  0x0000ffff8b1f922c in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0xaaaafafbb7b0) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
556
#1  __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafafbb758, cond=0xaaaafafbb788) at pthread_cond_wait.c:502
557
#2  __pthread_cond_wait (cond=0xaaaafafbb788, mutex=0xaaaafafbb758) at pthread_cond_wait.c:655
558
#3  0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
559
#4  0x0000aaaad1ee9ab4 in OSD::ShardedOpWQ::_process (this=0xaaaafaf6ef10, thread_index=38, hb=0xaaab50919950) at ./src/osd/OSD.cc:10423
560
#5  0x0000aaaad2523928 in ShardedThreadPool::shardedthreadpool_worker (this=0xaaaafaf6ea48, thread_index=38) at ./src/common/WorkQueue.cc:311
561
#6  0x0000aaaad2526268 in ShardedThreadPool::WorkThreadSharded::entry (this=<optimized out>) at ./src/common/WorkQueue.h:706
562
#7  0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4bf) at pthread_create.c:463
563
#8  0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
564

    
565
Thread 48 (Thread 0xffff7245b9f0 (LWP 126912)):
566
#0  0x0000ffff8b1f922c in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0xaaaafaf6fc78) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
567
#1  __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafaf6fc20, cond=0xaaaafaf6fc50) at pthread_cond_wait.c:502
568
#2  __pthread_cond_wait (cond=0xaaaafaf6fc50, mutex=0xaaaafaf6fc20) at pthread_cond_wait.c:655
569
#3  0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
570

    
571
#4  0x0000aaaad24efa80 in Finisher::finisher_thread_entry (this=0xaaaafaf6fc18) at ./src/common/Finisher.cc:86
572
#5  0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4ef) at pthread_create.c:463
573
#6  0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
574

    
575
Thread 47 (Thread 0xffff67c469f0 (LWP 126933)):
576
#0  0x0000ffff8b1f922c in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0xaaaafafbb030) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
577
#1  __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafafbafd8, cond=0xaaaafafbb008) at pthread_cond_wait.c:502
578
#2  __pthread_cond_wait (cond=0xaaaafafbb008, mutex=0xaaaafafbafd8) at pthread_cond_wait.c:655
579
#3  0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
580
---Type <return> to continue, or q <return> to quit---
581
#4  0x0000aaaad1ee9ab4 in OSD::ShardedOpWQ::_process (this=0xaaaafaf6ef10, thread_index=15, hb=0xaaab5091b3e0) at ./src/osd/OSD.cc:10423
582
#5  0x0000aaaad2523928 in ShardedThreadPool::shardedthreadpool_worker (this=0xaaaafaf6ea48, thread_index=15) at ./src/common/WorkQueue.cc:311
583
#6  0x0000aaaad2526268 in ShardedThreadPool::WorkThreadSharded::entry (this=<optimized out>) at ./src/common/WorkQueue.h:706
584
#7  0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4bf) at pthread_create.c:463
585
#8  0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
586

    
587
Thread 46 (Thread 0xffff764639f0 (LWP 126889)):
588
#0  0x0000ffff8b1f922c in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0xaaaafbb835a8) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
589
#1  __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafbb83550, cond=0xaaaafbb83580) at pthread_cond_wait.c:502
590
#2  __pthread_cond_wait (cond=0xaaaafbb83580, mutex=0xaaaafbb83550) at pthread_cond_wait.c:655
591
#3  0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
592
#4  0x0000aaaad28449ac in DispatchQueue::entry (this=0xaaaafbb83540) at ./src/msg/DispatchQueue.cc:210
593
#5  0x0000aaaad26977f0 in DispatchQueue::DispatchThread::entry (this=<optimized out>) at ./src/msg/DispatchQueue.h:101
594
#6  0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4df) at pthread_create.c:463
595
#7  0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
596

    
597
Thread 45 (Thread 0xffff7d76f9f0 (LWP 126880)):
598
#0  0x0000ffff8b1f922c in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0xaaaafbb82ca8) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
599
#1  __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafbb82c50, cond=0xaaaafbb82c80) at pthread_cond_wait.c:502
600
#2  __pthread_cond_wait (cond=0xaaaafbb82c80, mutex=0xaaaafbb82c50) at pthread_cond_wait.c:655
601
#3  0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
602
#4  0x0000aaaad28449ac in DispatchQueue::entry (this=0xaaaafbb82c40) at ./src/msg/DispatchQueue.cc:210
603
#5  0x0000aaaad26977f0 in DispatchQueue::DispatchThread::entry (this=<optimized out>) at ./src/msg/DispatchQueue.h:101
604
#6  0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c47f) at pthread_create.c:463
605
#7  0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
606

    
607
Thread 44 (Thread 0xffff6d4519f0 (LWP 126922)):
608
#0  0x0000ffff8b1f922c in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0xaaaafafb94b0) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
609
#1  __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafafb9458, cond=0xaaaafafb9488) at pthread_cond_wait.c:502
610
#2  __pthread_cond_wait (cond=0xaaaafafb9488, mutex=0xaaaafafb9458) at pthread_cond_wait.c:655
611
#3  0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
612
#4  0x0000aaaad1ee9ab4 in OSD::ShardedOpWQ::_process (this=0xaaaafaf6ef10, thread_index=4, hb=0xaaaafbc10ad0) at ./src/osd/OSD.cc:10423
613
#5  0x0000aaaad2523928 in ShardedThreadPool::shardedthreadpool_worker (this=0xaaaafaf6ea48, thread_index=4) at ./src/common/WorkQueue.cc:311
614
#6  0x0000aaaad2526268 in ShardedThreadPool::WorkThreadSharded::entry (this=<optimized out>) at ./src/common/WorkQueue.h:706
615
#7  0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4bf) at pthread_create.c:463
616
#8  0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
617

    
618
Thread 43 (Thread 0xffff7b76b9f0 (LWP 126884)):
619
#0  0x0000ffff8b1f95d4 in futex_abstimed_wait_cancelable (private=<optimized out>, abstime=0xffff7b76b1a8, expected=0, futex_word=0xaaaafaf6e46c)
620
    at ../sysdeps/unix/sysv/linux/futex-internal.h:205
621
#1  __pthread_cond_wait_common (abstime=0xffff7b76b1a8, mutex=0xaaaafaf6e3c0, cond=0xaaaafaf6e440) at pthread_cond_wait.c:539
622
#2  __pthread_cond_timedwait (cond=0xaaaafaf6e440, mutex=0xaaaafaf6e3c0, abstime=0xffff7b76b1a8) at pthread_cond_wait.c:667
623
#3  0x0000aaaad251d910 in __gthread_cond_timedwait (__abs_timeout=0xffff7b76b1a8, __mutex=<optimized out>, __cond=0xaaaafaf6e440)
624
    at /usr/include/aarch64-linux-gnu/c++/9/bits/gthr-default.h:872
625
#4  std::condition_variable::__wait_until_impl<std::chrono::duration<unsigned long, std::ratio<1l, 1000000000l> > > (__atime=<synthetic pointer>..., __lock=...,
626
    this=0xaaaafaf6e440) at /usr/include/c++/9/condition_variable:188
627
---Type <return> to continue, or q <return> to quit---
628
#5  std::condition_variable::wait_until<ceph::time_detail::real_clock, std::chrono::duration<unsigned long, std::ratio<1l, 1000000000l> > > (__atime=...,
629
    __lock=..., this=0xaaaafaf6e440) at /usr/include/c++/9/condition_variable:121
630
#6  SafeTimer::timer_thread (this=0xaaaafaf6e428) at ./src/common/Timer.cc:108
631
#7  0x0000aaaad251e7f0 in SafeTimerThread::entry (this=<optimized out>) at ./src/common/Timer.cc:30
632
#8  0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c52f) at pthread_create.c:463
633
#9  0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
634

    
635
Thread 42 (Thread 0xffff7445f9f0 (LWP 126893)):
636
#0  0x0000ffff8b1f922c in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0xaaaafbb847a8) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
637
#1  __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafbb84750, cond=0xaaaafbb84780) at pthread_cond_wait.c:502
638
#2  __pthread_cond_wait (cond=0xaaaafbb84780, mutex=0xaaaafbb84750) at pthread_cond_wait.c:655
639
#3  0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
640
#4  0x0000aaaad28449ac in DispatchQueue::entry (this=0xaaaafbb84740) at ./src/msg/DispatchQueue.cc:210
641
#5  0x0000aaaad26977f0 in DispatchQueue::DispatchThread::entry (this=<optimized out>) at ./src/msg/DispatchQueue.h:101
642
#6  0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4df) at pthread_create.c:463
643
#7  0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
644

    
645
Thread 41 (Thread 0xffff7af6a9f0 (LWP 126885)):
646
#0  0x0000ffff8b1f922c in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0xaaaafbb823a8) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
647
#1  __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafbb82350, cond=0xaaaafbb82380) at pthread_cond_wait.c:502
648
#2  __pthread_cond_wait (cond=0xaaaafbb82380, mutex=0xaaaafbb82350) at pthread_cond_wait.c:655
649
#3  0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
650
#4  0x0000aaaad28449ac in DispatchQueue::entry (this=0xaaaafbb82340) at ./src/msg/DispatchQueue.cc:210
651
#5  0x0000aaaad26977f0 in DispatchQueue::DispatchThread::entry (this=<optimized out>) at ./src/msg/DispatchQueue.h:101
652
#6  0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4df) at pthread_create.c:463
653
#7  0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
654

    
655
Thread 40 (Thread 0xffff71c5a9f0 (LWP 126913)):
656
#0  0x0000ffff8b1f922c in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0xaaaafb001920) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
657
#1  __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafb0018c8, cond=0xaaaafb0018f8) at pthread_cond_wait.c:502
658
#2  __pthread_cond_wait (cond=0xaaaafb0018f8, mutex=0xaaaafb0018c8) at pthread_cond_wait.c:655
659
#3  0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
660
#4  0x0000aaaad24efa80 in Finisher::finisher_thread_entry (this=0xaaaafb0018c0) at ./src/common/Finisher.cc:86
661
#5  0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4ef) at pthread_create.c:463
662
#6  0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
663

    
664
Thread 39 (Thread 0xffff73c5e9f0 (LWP 126894)):
665
#0  0x0000ffff8b1f922c in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0xaaaafbb84918) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
666
#1  __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafbb848c0, cond=0xaaaafbb848f0) at pthread_cond_wait.c:502
667
#2  __pthread_cond_wait (cond=0xaaaafbb848f0, mutex=0xaaaafbb848c0) at pthread_cond_wait.c:655
668
#3  0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
669
#4  0x0000aaaad284471c in DispatchQueue::run_local_delivery (this=0xaaaafbb84740) at ./src/msg/DispatchQueue.cc:117
670
#5  0x0000aaaad26978b0 in DispatchQueue::LocalDeliveryThread::entry (this=<optimized out>) at ./src/msg/DispatchQueue.h:115
671
#6  0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4ff) at pthread_create.c:463
672
#7  0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
673

    
674
---Type <return> to continue, or q <return> to quit---
675
Thread 38 (Thread 0xffff8247b9f0 (LWP 125801)):
676
#0  0x0000ffff8b1f922c in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0xaaab4be12240) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
677
#1  __pthread_cond_wait_common (abstime=0x0, mutex=0xaaab4be121e8, cond=0xaaab4be12218) at pthread_cond_wait.c:502
678
#2  __pthread_cond_wait (cond=0xaaab4be12218, mutex=0xaaab4be121e8) at pthread_cond_wait.c:655
679
#3  0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
680
#4  0x0000aaaad24bb69c in KernelDevice::_discard_thread (this=0xaaab4be12000) at ./src/os/bluestore/KernelDevice.cc:642
681
#5  0x0000aaaad24c9c68 in KernelDevice::DiscardThread::entry (this=<optimized out>) at ./src/os/bluestore/KernelDevice.h:74
682
#6  0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9a23f) at pthread_create.c:463
683
#7  0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
684

    
685
Thread 37 (Thread 0xffff85ec59f0 (LWP 125791)):
686
#0  0x0000ffff8b1f95d4 in futex_abstimed_wait_cancelable (private=<optimized out>, abstime=0xffff85ec51a8, expected=0, futex_word=0xaaaafaf6e088)
687
    at ../sysdeps/unix/sysv/linux/futex-internal.h:205
688
#1  __pthread_cond_wait_common (abstime=0xffff85ec51a8, mutex=0xaaaafaf6e018, cond=0xaaaafaf6e060) at pthread_cond_wait.c:539
689
#2  __pthread_cond_timedwait (cond=0xaaaafaf6e060, mutex=0xaaaafaf6e018, abstime=0xffff85ec51a8) at pthread_cond_wait.c:667
690
#3  0x0000aaaad251d910 in __gthread_cond_timedwait (__abs_timeout=0xffff85ec51a8, __mutex=<optimized out>, __cond=0xaaaafaf6e060)
691
    at /usr/include/aarch64-linux-gnu/c++/9/bits/gthr-default.h:872
692
#4  std::condition_variable::__wait_until_impl<std::chrono::duration<unsigned long, std::ratio<1l, 1000000000l> > > (__atime=<synthetic pointer>..., __lock=...,
693
    this=0xaaaafaf6e060) at /usr/include/c++/9/condition_variable:188
694
#5  std::condition_variable::wait_until<ceph::time_detail::real_clock, std::chrono::duration<unsigned long, std::ratio<1l, 1000000000l> > > (__atime=...,
695
    __lock=..., this=0xaaaafaf6e060) at /usr/include/c++/9/condition_variable:121
696
#6  SafeTimer::timer_thread (this=0xaaaafaf6e048) at ./src/common/Timer.cc:108
697
#7  0x0000aaaad251e7f0 in SafeTimerThread::entry (this=<optimized out>) at ./src/common/Timer.cc:30
698
#8  0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c55f) at pthread_create.c:463
699
#9  0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
700

    
701
Thread 36 (Thread 0xffff664439f0 (LWP 126936)):
702
#0  0x0000ffff8b1f922c in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0xaaaafafbb7b0) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
703
#1  __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafafbb758, cond=0xaaaafafbb788) at pthread_cond_wait.c:502
704
#2  __pthread_cond_wait (cond=0xaaaafafbb788, mutex=0xaaaafafbb758) at pthread_cond_wait.c:655
705
#3  0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
706
#4  0x0000aaaad1ee9ab4 in OSD::ShardedOpWQ::_process (this=0xaaaafaf6ef10, thread_index=18, hb=0xaaab5091b700) at ./src/osd/OSD.cc:10423
707
#5  0x0000aaaad2523928 in ShardedThreadPool::shardedthreadpool_worker (this=0xaaaafaf6ea48, thread_index=18) at ./src/common/WorkQueue.cc:311
708
#6  0x0000aaaad2526268 in ShardedThreadPool::WorkThreadSharded::entry (this=<optimized out>) at ./src/common/WorkQueue.h:706
709
#7  0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4bf) at pthread_create.c:463
710
#8  0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
711

    
712
Thread 35 (Thread 0xffff694499f0 (LWP 126930)):
713
#0  0x0000ffff8b1f922c in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0xaaaafafba8b0) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
714
#1  __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafafba858, cond=0xaaaafafba888) at pthread_cond_wait.c:502
715
#2  __pthread_cond_wait (cond=0xaaaafafba888, mutex=0xaaaafafba858) at pthread_cond_wait.c:655
716
#3  0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
717
#4  0x0000aaaad1ee9ab4 in OSD::ShardedOpWQ::_process (this=0xaaaafaf6ef10, thread_index=12, hb=0xaaab5091b1b0) at ./src/osd/OSD.cc:10423
718
#5  0x0000aaaad2523928 in ShardedThreadPool::shardedthreadpool_worker (this=0xaaaafaf6ea48, thread_index=12) at ./src/common/WorkQueue.cc:311
719
#6  0x0000aaaad2526268 in ShardedThreadPool::WorkThreadSharded::entry (this=<optimized out>) at ./src/common/WorkQueue.h:706
720
#7  0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4bf) at pthread_create.c:463
721
---Type <return> to continue, or q <return> to quit---
722
#8  0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
723

    
724
Thread 34 (Thread 0xffff6243b9f0 (LWP 126944)):
725
#0  0x0000ffff8b1f922c in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0xaaaafafb99b4) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
726
#1  __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafafb9958, cond=0xaaaafafb9988) at pthread_cond_wait.c:502
727
#2  __pthread_cond_wait (cond=0xaaaafafb9988, mutex=0xaaaafafb9958) at pthread_cond_wait.c:655
728
#3  0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
729
#4  0x0000aaaad1ee9ab4 in OSD::ShardedOpWQ::_process (this=0xaaaafaf6ef10, thread_index=26, hb=0xaaab50919fe0) at ./src/osd/OSD.cc:10423
730
#5  0x0000aaaad2523928 in ShardedThreadPool::shardedthreadpool_worker (this=0xaaaafaf6ea48, thread_index=26) at ./src/common/WorkQueue.cc:311
731
#6  0x0000aaaad2526268 in ShardedThreadPool::WorkThreadSharded::entry (this=<optimized out>) at ./src/common/WorkQueue.h:706
732
#7  0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4bf) at pthread_create.c:463
733
#8  0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
734

    
735
Thread 33 (Thread 0xffff84c809f0 (LWP 125792)):
736
#0  0x0000ffff8b1f95d4 in futex_abstimed_wait_cancelable (private=<optimized out>, abstime=0xffff84c801a8, expected=0, futex_word=0xaaaafaf6e17c)
737
    at ../sysdeps/unix/sysv/linux/futex-internal.h:205
738
#1  __pthread_cond_wait_common (abstime=0xffff84c801a8, mutex=0xaaaafaf6e108, cond=0xaaaafaf6e150) at pthread_cond_wait.c:539
739
#2  __pthread_cond_timedwait (cond=0xaaaafaf6e150, mutex=0xaaaafaf6e108, abstime=0xffff84c801a8) at pthread_cond_wait.c:667
740
#3  0x0000aaaad251d910 in __gthread_cond_timedwait (__abs_timeout=0xffff84c801a8, __mutex=<optimized out>, __cond=0xaaaafaf6e150)
741
    at /usr/include/aarch64-linux-gnu/c++/9/bits/gthr-default.h:872
742
#4  std::condition_variable::__wait_until_impl<std::chrono::duration<unsigned long, std::ratio<1l, 1000000000l> > > (__atime=<synthetic pointer>..., __lock=...,
743
    this=0xaaaafaf6e150) at /usr/include/c++/9/condition_variable:188
744
#5  std::condition_variable::wait_until<ceph::time_detail::real_clock, std::chrono::duration<unsigned long, std::ratio<1l, 1000000000l> > > (__atime=...,
745
    __lock=..., this=0xaaaafaf6e150) at /usr/include/c++/9/condition_variable:121
746
#6  SafeTimer::timer_thread (this=0xaaaafaf6e138) at ./src/common/Timer.cc:108
747
#7  0x0000aaaad251e7f0 in SafeTimerThread::entry (this=<optimized out>) at ./src/common/Timer.cc:30
748
#8  0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c55f) at pthread_create.c:463
749
#9  0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
750

    
751
Thread 32 (Thread 0xffff6a44b9f0 (LWP 126928)):
752
#0  0x0000ffff8b1f922c in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0xaaaafafba3b0) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
753
#1  __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafafba358, cond=0xaaaafafba388) at pthread_cond_wait.c:502
754
#2  __pthread_cond_wait (cond=0xaaaafafba388, mutex=0xaaaafafba358) at pthread_cond_wait.c:655
755
#3  0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
756
#4  0x0000aaaad1ee9ab4 in OSD::ShardedOpWQ::_process (this=0xaaaafaf6ef10, thread_index=10, hb=0xaaab5091b250) at ./src/osd/OSD.cc:10423
757
#5  0x0000aaaad2523928 in ShardedThreadPool::shardedthreadpool_worker (this=0xaaaafaf6ea48, thread_index=10) at ./src/common/WorkQueue.cc:311
758
#6  0x0000aaaad2526268 in ShardedThreadPool::WorkThreadSharded::entry (this=<optimized out>) at ./src/common/WorkQueue.h:706
759
#7  0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4bf) at pthread_create.c:463
760
#8  0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
761

    
762
Thread 31 (Thread 0xffff704579f0 (LWP 126916)):
763
#0  0x0000ffff8b1f922c in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0xaaaafaf70880) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
764
#1  __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafaf70828, cond=0xaaaafaf70858) at pthread_cond_wait.c:502
765
#2  __pthread_cond_wait (cond=0xaaaafaf70858, mutex=0xaaaafaf70828) at pthread_cond_wait.c:655
766
#3  0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
767
#4  0x0000aaaad1f338f8 in ceph::timer_detail::timer<ceph::time_detail::mono_clock>::timer_thread (this=0xaaaafaf707e8)
768
---Type <return> to continue, or q <return> to quit---
769
    at ./obj-aarch64-linux-gnu/boost/include/boost/intrusive/detail/rbtree_node.hpp:77
770
#5  0x0000ffff8b03ded4 in ?? () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
771
#6  0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c79f) at pthread_create.c:463
772
#7  0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
773

    
774
Thread 30 (Thread 0xffff70c589f0 (LWP 126915)):
775
#0  0x0000ffff8b1f922c in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0xaaaafaf6f848) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
776
#1  __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafaf6f7d8, cond=0xaaaafaf6f820) at pthread_cond_wait.c:502
777
#2  __pthread_cond_wait (cond=0xaaaafaf6f820, mutex=0xaaaafaf6f7d8) at pthread_cond_wait.c:655
778
#3  0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
779
#4  0x0000aaaad251d438 in SafeTimer::timer_thread (this=0xaaaafaf6f808) at ./src/common/Timer.cc:106
780
#5  0x0000aaaad251e7f0 in SafeTimerThread::entry (this=<optimized out>) at ./src/common/Timer.cc:30
781
#6  0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4ef) at pthread_create.c:463
782
#7  0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
783

    
784
Thread 29 (Thread 0xffff714599f0 (LWP 126914)):
785
#0  0x0000ffff8b1f922c in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0xaaaafaf6f998) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
786
#1  __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafaf6f928, cond=0xaaaafaf6f970) at pthread_cond_wait.c:502
787
#2  __pthread_cond_wait (cond=0xaaaafaf6f970, mutex=0xaaaafaf6f928) at pthread_cond_wait.c:655
788
#3  0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
789
#4  0x0000aaaad251d438 in SafeTimer::timer_thread (this=0xaaaafaf6f958) at ./src/common/Timer.cc:106
790
#5  0x0000aaaad251e7f0 in SafeTimerThread::entry (this=<optimized out>) at ./src/common/Timer.cc:30
791
#6  0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4ef) at pthread_create.c:463
792
#7  0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
793

    
794
Thread 28 (Thread 0xffff6e4539f0 (LWP 126920)):
795
#0  0x0000ffff8b1f922c in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0xaaaafafb8fb4) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
796
#1  __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafafb8f58, cond=0xaaaafafb8f88) at pthread_cond_wait.c:502
797
#2  __pthread_cond_wait (cond=0xaaaafafb8f88, mutex=0xaaaafafb8f58) at pthread_cond_wait.c:655
798
#3  0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
799
#4  0x0000aaaad1ee9ab4 in OSD::ShardedOpWQ::_process (this=0xaaaafaf6ef10, thread_index=2, hb=0xaaaafbc10a30) at ./src/osd/OSD.cc:10423
800
#5  0x0000aaaad2523928 in ShardedThreadPool::shardedthreadpool_worker (this=0xaaaafaf6ea48, thread_index=2) at ./src/common/WorkQueue.cc:311
801
#6  0x0000aaaad2526268 in ShardedThreadPool::WorkThreadSharded::entry (this=<optimized out>) at ./src/common/WorkQueue.h:706
802
#7  0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4bf) at pthread_create.c:463
803
#8  0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
804

    
805
Thread 27 (Thread 0xffff7a7699f0 (LWP 126886)):
806
#0  0x0000ffff8b1f922c in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0xaaaafbb82518) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
807
#1  __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafbb824c0, cond=0xaaaafbb824f0) at pthread_cond_wait.c:502
808
#2  __pthread_cond_wait (cond=0xaaaafbb824f0, mutex=0xaaaafbb824c0) at pthread_cond_wait.c:655
809
#3  0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
810
#4  0x0000aaaad284471c in DispatchQueue::run_local_delivery (this=0xaaaafbb82340) at ./src/msg/DispatchQueue.cc:117
811
#5  0x0000aaaad26978b0 in DispatchQueue::LocalDeliveryThread::entry (this=<optimized out>) at ./src/msg/DispatchQueue.h:115
812
#6  0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4ff) at pthread_create.c:463
813
#7  0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
814

    
815
---Type <return> to continue, or q <return> to quit---
816
Thread 26 (Thread 0xffff8ac81010 (LWP 124314)):
817
#0  0x0000ffff8b1f922c in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0xaaaafbb83108) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
818
#1  __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafbb82f70, cond=0xaaaafbb830e0) at pthread_cond_wait.c:502
819
#2  __pthread_cond_wait (cond=0xaaaafbb830e0, mutex=0xaaaafbb82f70) at pthread_cond_wait.c:655
820
#3  0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
821
#4  0x0000aaaad269475c in AsyncMessenger::wait (this=0xaaaafbb82900) at ./src/msg/async/AsyncMessenger.cc:538
822
#5  0x0000aaaad1e5f58c in main (argc=<optimized out>, argv=<optimized out>) at ./src/ceph_osd.cc:729
823

    
824
Thread 25 (Thread 0xffff7c76d9f0 (LWP 126882)):
825
#0  0x0000ffff8b1f95d4 in futex_abstimed_wait_cancelable (private=<optimized out>, abstime=0xffff7c76d1a8, expected=0, futex_word=0xffffe7b9d888)
826
    at ../sysdeps/unix/sysv/linux/futex-internal.h:205
827
#1  __pthread_cond_wait_common (abstime=0xffff7c76d1a8, mutex=0xffffe7b9d818, cond=0xffffe7b9d860) at pthread_cond_wait.c:539
828
#2  __pthread_cond_timedwait (cond=0xffffe7b9d860, mutex=0xffffe7b9d818, abstime=0xffff7c76d1a8) at pthread_cond_wait.c:667
829
#3  0x0000aaaad251d910 in __gthread_cond_timedwait (__abs_timeout=0xffff7c76d1a8, __mutex=<optimized out>, __cond=0xffffe7b9d860)
830
    at /usr/include/aarch64-linux-gnu/c++/9/bits/gthr-default.h:872
831
#4  std::condition_variable::__wait_until_impl<std::chrono::duration<unsigned long, std::ratio<1l, 1000000000l> > > (__atime=<synthetic pointer>..., __lock=...,
832
    this=0xffffe7b9d860) at /usr/include/c++/9/condition_variable:188
833
#5  std::condition_variable::wait_until<ceph::time_detail::real_clock, std::chrono::duration<unsigned long, std::ratio<1l, 1000000000l> > > (__atime=...,
834
    __lock=..., this=0xffffe7b9d860) at /usr/include/c++/9/condition_variable:121
835
#6  SafeTimer::timer_thread (this=0xffffe7b9d848) at ./src/common/Timer.cc:108
836
#7  0x0000aaaad251e7f0 in SafeTimerThread::entry (this=<optimized out>) at ./src/common/Timer.cc:30
837
#8  0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4bf) at pthread_create.c:463
838
#9  0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
839

    
840
Thread 24 (Thread 0xffff76c649f0 (LWP 126888)):
841
#0  0x0000ffff8b1f922c in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0xaaaafbb84018) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
842
#1  __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafbb83fc0, cond=0xaaaafbb83ff0) at pthread_cond_wait.c:502
843
#2  __pthread_cond_wait (cond=0xaaaafbb83ff0, mutex=0xaaaafbb83fc0) at pthread_cond_wait.c:655
844
#3  0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
845
#4  0x0000aaaad284471c in DispatchQueue::run_local_delivery (this=0xaaaafbb83e40) at ./src/msg/DispatchQueue.cc:117
846
#5  0x0000aaaad26978b0 in DispatchQueue::LocalDeliveryThread::entry (this=<optimized out>) at ./src/msg/DispatchQueue.h:115
847
#6  0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4ff) at pthread_create.c:463
848
#7  0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
849

    
850
Thread 23 (Thread 0xffff784519f0 (LWP 126887)):
851
#0  0x0000ffff8b1f922c in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0xaaaafbb83ea8) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
852
#1  __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafbb83e50, cond=0xaaaafbb83e80) at pthread_cond_wait.c:502
853
#2  __pthread_cond_wait (cond=0xaaaafbb83e80, mutex=0xaaaafbb83e50) at pthread_cond_wait.c:655
854
#3  0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
855
#4  0x0000aaaad28449ac in DispatchQueue::entry (this=0xaaaafbb83e40) at ./src/msg/DispatchQueue.cc:210
856
#5  0x0000aaaad26977f0 in DispatchQueue::DispatchThread::entry (this=<optimized out>) at ./src/msg/DispatchQueue.h:101
857
#6  0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4df) at pthread_create.c:463
858
#7  0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
859

    
860
Thread 22 (Thread 0xffff7bf6c9f0 (LWP 126883)):
861
#0  0x0000ffff8b1f922c in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0xffffe7b9d968) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
862
---Type <return> to continue, or q <return> to quit---
863
#1  __pthread_cond_wait_common (abstime=0x0, mutex=0xffffe7b9d910, cond=0xffffe7b9d940) at pthread_cond_wait.c:502
864
#2  __pthread_cond_wait (cond=0xffffe7b9d940, mutex=0xffffe7b9d910) at pthread_cond_wait.c:655
865
#3  0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
866
#4  0x0000aaaad24efa80 in Finisher::finisher_thread_entry (this=0xffffe7b9d908) at ./src/common/Finisher.cc:86
867
#5  0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4bf) at pthread_create.c:463
868
#6  0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
869

    
870
Thread 21 (Thread 0xffff8ac349f0 (LWP 124338)):
871
#0  0x0000ffff8b1f922c in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0xaaaafaf040f4) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
872
#1  __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafaf04038, cond=0xaaaafaf040c8) at pthread_cond_wait.c:502
873
#2  __pthread_cond_wait (cond=0xaaaafaf040c8, mutex=0xaaaafaf04038) at pthread_cond_wait.c:655
874
#3  0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
875
#4  0x0000aaaad26c26ec in ceph::logging::Log::entry (this=0xaaaafaf04000) at ./src/log/Log.cc:435
876
#5  0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9b02f) at pthread_create.c:463
877
#6  0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
878

    
879
Thread 20 (Thread 0xffff774659f0 (LWP 126377)):
880
#0  0x0000ffff8b1f95d4 in futex_abstimed_wait_cancelable (private=<optimized out>, abstime=0xffff77464f58, expected=0, futex_word=0xaaaafb001cf0)
881
    at ../sysdeps/unix/sysv/linux/futex-internal.h:205
882
#1  __pthread_cond_wait_common (abstime=0xffff77464f58, mutex=0xaaaafb001c78, cond=0xaaaafb001cc8) at pthread_cond_wait.c:539
883
#2  __pthread_cond_timedwait (cond=0xaaaafb001cc8, mutex=0xaaaafb001c78, abstime=0xffff77464f58) at pthread_cond_wait.c:667
884
#3  0x0000aaaad2bd5f54 in rocksdb::port::CondVar::TimedWait(unsigned long) ()
885
#4  0x0000aaaad2ac818c in rocksdb::InstrumentedCondVar::TimedWaitInternal(unsigned long) ()
886
#5  0x0000aaaad2ac80a0 in rocksdb::InstrumentedCondVar::TimedWait(unsigned long) ()
887
#6  0x0000aaaad290e284 in rocksdb::RepeatableThread::wait(unsigned long) ()
888
#7  0x0000aaaad290e458 in rocksdb::RepeatableThread::thread() ()
889
#8  0x0000aaaad290deb4 in rocksdb::RepeatableThread::RepeatableThread(std::function<void ()>, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, rocksdb::Env*, unsigned long, unsigned long)::{lambda()#1}::operator()() const ()
890
#9  0x0000aaaad2955904 in void std::__invoke_impl<void, rocksdb::RepeatableThread::RepeatableThread(std::function<void ()>, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, rocksdb::Env*, unsigned long, unsigned long)::{lambda()#1}>(std::__invoke_other, rocksdb::RepeatableThread::RepeatableThread(std::function<void ()>, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, rocksdb::Env*, unsigned long, unsigned long)::{lambda()#1}&&) ()
891
#10 0x0000aaaad2955888 in std::__invoke_result<rocksdb::RepeatableThread::RepeatableThread(std::function<void ()>, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, rocksdb::Env*, unsigned long, unsigned long)::{lambda()#1}>::type std::__invoke<rocksdb::RepeatableThread::RepeatableThread(std::function<void ()>, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, rocksdb::Env*, unsigned long, unsigned long)::{lambda()#1}>(std::__invoke_result&&, (rocksdb::RepeatableThread::RepeatableThread(std::function<void ()>, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, rocksdb::Env*, unsigned long, unsigned long)::{lambda()#1}&&)...) ()
892
#11 0x0000aaaad2955810 in void std::thread::_Invoker<std::tuple<rocksdb::RepeatableThread::RepeatableThread(std::function<void ()>, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, rocksdb::Env*, unsigned long, unsigned long)::{lambda()#1}> >::_M_invoke<0ul>(std::_Index_tuple<0ul>) ()
893
#12 0x0000aaaad29557c8 in std::thread::_Invoker<std::tuple<rocksdb::RepeatableThread::RepeatableThread(std::function<void ()>, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, rocksdb::Env*, unsigned long, unsigned long)::{lambda()#1}> >::operator()() ()
894
#13 0x0000aaaad2955794 in std::thread::_State_impl<std::thread::_Invoker<std::tuple<rocksdb::RepeatableThread::RepeatableThread(std::function<void ()>, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, rocksdb::Env*, unsigned long, unsigned long)::{lambda()#1}> > >::_M_run() ()
895
#14 0x0000ffff8b03ded4 in ?? () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
896
#15 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9b02f) at pthread_create.c:463
897
#16 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
898
---Type <return> to continue, or q <return> to quit---
899

    
900
Thread 19 (Thread 0xffff7cf6e9f0 (LWP 126881)):
901
#0  0x0000ffff8b1f922c in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0xaaaafbb82e18) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
902
#1  __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafbb82dc0, cond=0xaaaafbb82df0) at pthread_cond_wait.c:502
903
#2  __pthread_cond_wait (cond=0xaaaafbb82df0, mutex=0xaaaafbb82dc0) at pthread_cond_wait.c:655
904
#3  0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
905
#4  0x0000aaaad284471c in DispatchQueue::run_local_delivery (this=0xaaaafbb82c40) at ./src/msg/DispatchQueue.cc:117
906
#5  0x0000aaaad26978b0 in DispatchQueue::LocalDeliveryThread::entry (this=<optimized out>) at ./src/msg/DispatchQueue.h:115
907
#6  0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c49f) at pthread_create.c:463
908
#7  0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
909

    
910
Thread 18 (Thread 0xffff7e45d9f0 (LWP 126389)):
911
#0  0x0000ffff8b1f95d4 in futex_abstimed_wait_cancelable (private=<optimized out>, abstime=0xffff7e45d1b8, expected=0, futex_word=0xaaab4be04b28)
912
    at ../sysdeps/unix/sysv/linux/futex-internal.h:205
913
#1  __pthread_cond_wait_common (abstime=0xffff7e45d1b8, mutex=0xaaab4be04b30, cond=0xaaab4be04b00) at pthread_cond_wait.c:539
914

    
915

    
916

    
917

    
918

    
919

    
920

    
921

    
922

    
923
#2  __pthread_cond_timedwait (cond=0xaaab4be04b00, mutex=0xaaab4be04b30, abstime=0xffff7e45d1b8) at pthread_cond_wait.c:667
924
#3  0x0000aaaad23c85d4 in __gthread_cond_timedwait (__abs_timeout=0xffff7e45d1b8, __mutex=<optimized out>, __cond=0xaaab4be04b00)
925
    at /usr/include/aarch64-linux-gnu/c++/9/bits/gthr-default.h:872
926
#4  std::condition_variable::__wait_until_impl<std::chrono::duration<long, std::ratio<1l, 1000000000l> > > (__atime=<synthetic pointer>..., __lock=...,
927
    this=0xaaab4be04b00) at /usr/include/c++/9/condition_variable:188
928
#5  std::condition_variable::wait_until<std::chrono::_V2::steady_clock, std::chrono::duration<long, std::ratio<1l, 1000000000l> > > (__atime=..., __lock=...,
929
    this=0xaaab4be04b00) at /usr/include/c++/9/condition_variable:121
930
#6  std::condition_variable::wait_for<unsigned long, std::ratio<1l, 1000000000l> > (__rtime=<synthetic pointer>..., __lock=..., this=0xaaab4be04b00)
931
    at /usr/include/c++/9/condition_variable:152
932
#7  BlueStore::MempoolThread::entry (this=0xaaab4be04ad8) at ./src/os/bluestore/BlueStore.cc:3882
933
#8  0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4ff) at pthread_create.c:463
934
#9  0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
935

    
936
Thread 17 (Thread 0xffff7ec5e9f0 (LWP 126383)):
937
#0  0x0000ffff8b1f922c in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0xaaab4be047ec) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
938
#1  __pthread_cond_wait_common (abstime=0x0, mutex=0xaaab4be04790, cond=0xaaab4be047c0) at pthread_cond_wait.c:502
939
#2  __pthread_cond_wait (cond=0xaaab4be047c0, mutex=0xaaab4be04790) at pthread_cond_wait.c:655
940
#3  0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
941
#4  0x0000aaaad23cd480 in BlueStore::_kv_finalize_thread (this=0xaaab4be04000) at ./src/os/bluestore/BlueStore.cc:11990
942
#5  0x0000aaaad2403e50 in BlueStore::KVFinalizeThread::entry (this=<optimized out>) at ./src/os/bluestore/BlueStore.h:1912
943
#6  0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c47f) at pthread_create.c:463
944
#7  0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
945

    
946
Thread 16 (Thread 0xffff794539f0 (LWP 126381)):
947
#0  0x0000ffff8b1f922c in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0xaaab4be044cc) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
948
#1  __pthread_cond_wait_common (abstime=0x0, mutex=0xaaab4be04470, cond=0xaaab4be044a0) at pthread_cond_wait.c:502
949
#2  __pthread_cond_wait (cond=0xaaab4be044a0, mutex=0xaaab4be04470) at pthread_cond_wait.c:655
950
#3  0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
951
#4  0x0000aaaad24efa80 in Finisher::finisher_thread_entry (this=0xaaab4be04468) at ./src/common/Finisher.cc:86
952
#5  0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c3ff) at pthread_create.c:463
953
#6  0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
954
---Type <return> to continue, or q <return> to quit---
955

    
956
Thread 15 (Thread 0xffff79c549f0 (LWP 126382)):
957
#0  0x0000ffff8b1f922c in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0xaaab4be04614) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
958
#1  __pthread_cond_wait_common (abstime=0x0, mutex=0xaaab4be045b8, cond=0xaaab4be045e8) at pthread_cond_wait.c:502
959
#2  __pthread_cond_wait (cond=0xaaab4be045e8, mutex=0xaaab4be045b8) at pthread_cond_wait.c:655
960
#3  0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
961
#4  0x0000aaaad23d7cec in BlueStore::_kv_sync_thread (this=0xaaab4be04000) at ./src/os/bluestore/BlueStore.cc:11723
962
#5  0x0000aaaad2404af0 in BlueStore::KVSyncThread::entry (this=<optimized out>) at ./src/os/bluestore/BlueStore.h:1904
963
#6  0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c47f) at pthread_create.c:463
964
#7  0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
965

    
966
Thread 14 (Thread 0xffff7f4759f0 (LWP 126334)):
967
#0  0x0000ffff8b1f922c in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0xaaab4be12c98) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
968
#1  __pthread_cond_wait_common (abstime=0x0, mutex=0xaaab4be12b88, cond=0xaaab4be12c70) at pthread_cond_wait.c:502
969
#2  __pthread_cond_wait (cond=0xaaab4be12c70, mutex=0xaaab4be12b88) at pthread_cond_wait.c:655
970
#3  0x0000aaaad2bd5e70 in rocksdb::port::CondVar::Wait() ()
971
#4  0x0000aaaad2ac7ff4 in rocksdb::InstrumentedCondVar::WaitInternal() ()
972
#5  0x0000aaaad2ac7f88 in rocksdb::InstrumentedCondVar::Wait() ()
973
#6  0x0000aaaad2b8be80 in rocksdb::DeleteScheduler::BackgroundEmptyTrash() ()
974
#7  0x0000aaaad2b921fc in void std::__invoke_impl<void, void (rocksdb::DeleteScheduler::*)(), rocksdb::DeleteScheduler*>(std::__invoke_memfun_deref, void (rocksdb::DeleteScheduler::*&&)(), rocksdb::DeleteScheduler*&&) ()
975
#8  0x0000aaaad2b920f0 in std::__invoke_result<void (rocksdb::DeleteScheduler::*)(), rocksdb::DeleteScheduler*>::type std::__invoke<void (rocksdb::DeleteScheduler::*)(), rocksdb::DeleteScheduler*>(void (rocksdb::DeleteScheduler::*&&)(), rocksdb::DeleteScheduler*&&) ()
976
#9  0x0000aaaad2b92040 in void std::thread::_Invoker<std::tuple<void (rocksdb::DeleteScheduler::*)(), rocksdb::DeleteScheduler*> >::_M_invoke<0ul, 1ul>(std::_Index_tuple<0ul, 1ul>) ()
977
#10 0x0000aaaad2b91fdc in std::thread::_Invoker<std::tuple<void (rocksdb::DeleteScheduler::*)(), rocksdb::DeleteScheduler*> >::operator()() ()
978
#11 0x0000aaaad2b91fa8 in std::thread::_State_impl<std::thread::_Invoker<std::tuple<void (rocksdb::DeleteScheduler::*)(), rocksdb::DeleteScheduler*> > >::_M_run()
979
    ()
980
#12 0x0000ffff8b03ded4 in ?? () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
981
#13 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9abaf) at pthread_create.c:463
982
#14 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
983

    
984
Thread 13 (Thread 0xffff89bb99f0 (LWP 124381)):
985
#0  0x0000ffff8ae0d600 in __GI_epoll_pwait (epfd=<optimized out>, events=0xaaaafbbda000, maxevents=<optimized out>, timeout=-750510080, set=set@entry=0x0)
986
    at ../sysdeps/unix/sysv/linux/epoll_pwait.c:42
987
#1  0x0000ffff8ae0d750 in epoll_wait (epfd=<optimized out>, events=<optimized out>, maxevents=<optimized out>, timeout=<optimized out>)
988
    at ../sysdeps/unix/sysv/linux/epoll_wait.c:32
989
#2  0x0000aaaad26a6574 in EpollDriver::event_wait (this=0xaaaafafcfec0, fired_events=std::vector of length 0, capacity 0, tvp=<optimized out>)
990
    at ./src/msg/async/EventEpoll.cc:123
991
#3  0x0000aaaad269cd64 in EventCenter::process_events (this=this@entry=0xaaaafbb9c8c8, timeout_microseconds=<optimized out>,
992
    working_dur=working_dur@entry=0xffff89bb9160) at ./src/msg/async/Event.cc:392
993
#4  0x0000aaaad26a33c0 in NetworkStack::<lambda()>::operator() (__closure=0xaaaafbc0c4e8, __closure=0xaaaafbc0c4e8) at ./src/msg/async/Stack.cc:53
994
#5  std::_Function_handler<void(), NetworkStack::add_thread(unsigned int)::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...)
995
    at /usr/include/c++/9/bits/std_function.h:300
996
#6  0x0000ffff8b03ded4 in ?? () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
997
#7  0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9aeff) at pthread_create.c:463
998
---Type <return> to continue, or q <return> to quit---
999
#8  0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
1000

    
1001
Thread 12 (Thread 0xffff78c529f0 (LWP 126378)):
1002
#0  0x0000ffff8b1f95d4 in futex_abstimed_wait_cancelable (private=<optimized out>, abstime=0xffff78c51f58, expected=0, futex_word=0xaaaafb002710)
1003
    at ../sysdeps/unix/sysv/linux/futex-internal.h:205
1004
#1  __pthread_cond_wait_common (abstime=0xffff78c51f58, mutex=0xaaaafb002698, cond=0xaaaafb0026e8) at pthread_cond_wait.c:539
1005
#2  __pthread_cond_timedwait (cond=0xaaaafb0026e8, mutex=0xaaaafb002698, abstime=0xffff78c51f58) at pthread_cond_wait.c:667
1006
#3  0x0000aaaad2bd5f54 in rocksdb::port::CondVar::TimedWait(unsigned long) ()
1007
#4  0x0000aaaad2ac818c in rocksdb::InstrumentedCondVar::TimedWaitInternal(unsigned long) ()
1008
#5  0x0000aaaad2ac80a0 in rocksdb::InstrumentedCondVar::TimedWait(unsigned long) ()
1009
#6  0x0000aaaad290e284 in rocksdb::RepeatableThread::wait(unsigned long) ()
1010
#7  0x0000aaaad290e458 in rocksdb::RepeatableThread::thread() ()
1011
#8  0x0000aaaad290deb4 in rocksdb::RepeatableThread::RepeatableThread(std::function<void ()>, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, rocksdb::Env*, unsigned long, unsigned long)::{lambda()#1}::operator()() const ()
1012
#9  0x0000aaaad2955904 in void std::__invoke_impl<void, rocksdb::RepeatableThread::RepeatableThread(std::function<void ()>, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, rocksdb::Env*, unsigned long, unsigned long)::{lambda()#1}>(std::__invoke_other, rocksdb::RepeatableThread::RepeatableThread(std::function<void ()>, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, rocksdb::Env*, unsigned long, unsigned long)::{lambda()#1}&&) ()
1013
#10 0x0000aaaad2955888 in std::__invoke_result<rocksdb::RepeatableThread::RepeatableThread(std::function<void ()>, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, rocksdb::Env*, unsigned long, unsigned long)::{lambda()#1}>::type std::__invoke<rocksdb::RepeatableThread::RepeatableThread(std::function<void ()>, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, rocksdb::Env*, unsigned long, unsigned long)::{lambda()#1}>(std::__invoke_result&&, (rocksdb::RepeatableThread::RepeatableThread(std::function<void ()>, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, rocksdb::Env*, unsigned long, unsigned long)::{lambda()#1}&&)...) ()
1014
#11 0x0000aaaad2955810 in void std::thread::_Invoker<std::tuple<rocksdb::RepeatableThread::RepeatableThread(std::function<void ()>, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, rocksdb::Env*, unsigned long, unsigned long)::{lambda()#1}> >::_M_invoke<0ul>(std::_Index_tuple<0ul>) ()
1015
#12 0x0000aaaad29557c8 in std::thread::_Invoker<std::tuple<rocksdb::RepeatableThread::RepeatableThread(std::function<void ()>, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, rocksdb::Env*, unsigned long, unsigned long)::{lambda()#1}> >::operator()() ()
1016
#13 0x0000aaaad2955794 in std::thread::_State_impl<std::thread::_Invoker<std::tuple<rocksdb::RepeatableThread::RepeatableThread(std::function<void ()>, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, rocksdb::Env*, unsigned long, unsigned long)::{lambda()#1}> > >::_M_run() ()
1017
#14 0x0000ffff8b03ded4 in ?? () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
1018
#15 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9b02f) at pthread_create.c:463
1019
#16 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
1020

    
1021
Thread 11 (Thread 0xffff814799f0 (LWP 126260)):
1022
#0  0x0000ffff8b5166c0 in ?? () from /lib/aarch64-linux-gnu/libaio.so.1
1023
#1  0x0000000000000001 in ?? ()
1024
Backtrace stopped: previous frame identical to this frame (corrupt stack?)
1025

    
1026
Thread 10 (Thread 0xffff7fc769f0 (LWP 126003)):
1027
#0  0x0000ffff8b1f922c in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0xaaaafaf8bce8) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
1028
#1  __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafaf8bc90, cond=0xaaaafaf8bcc0) at pthread_cond_wait.c:502
1029
#2  __pthread_cond_wait (cond=0xaaaafaf8bcc0, mutex=0xaaaafaf8bc90) at pthread_cond_wait.c:655
1030
#3  0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
1031
#4  0x0000aaaad2c7cb08 in rocksdb::ThreadPoolImpl::Impl::BGThread(unsigned long) ()
1032
#5  0x0000aaaad2c7ce58 in rocksdb::ThreadPoolImpl::Impl::BGThreadWrapper(void*) ()
1033
#6  0x0000aaaad2c8438c in void* std::__invoke_impl<void*, void* (*)(void*), rocksdb::BGThreadMetadata*>(std::__invoke_other, void* (*&&)(void*), rocksdb::BGThreadMet---Type <return> to continue, or q <return> to quit---
1034
adata*&&) ()
1035
#7  0x0000aaaad2c842c4 in std::__invoke_result<void* (*)(void*), rocksdb::BGThreadMetadata*>::type std::__invoke<void* (*)(void*), rocksdb::BGThreadMetadata*>(void* (*&&)(void*), rocksdb::BGThreadMetadata*&&) ()
1036
#8  0x0000aaaad2c84218 in void* std::thread::_Invoker<std::tuple<void* (*)(void*), rocksdb::BGThreadMetadata*> >::_M_invoke<0ul, 1ul>(std::_Index_tuple<0ul, 1ul>)
1037
    ()
1038
#9  0x0000aaaad2c841ac in std::thread::_Invoker<std::tuple<void* (*)(void*), rocksdb::BGThreadMetadata*> >::operator()() ()
1039
#10 0x0000aaaad2c84178 in std::thread::_State_impl<std::thread::_Invoker<std::tuple<void* (*)(void*), rocksdb::BGThreadMetadata*> > >::_M_run() ()
1040
#11 0x0000ffff8b03ded4 in ?? () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
1041
#12 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9aacf) at pthread_create.c:463
1042
#13 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
1043

    
1044
Thread 9 (Thread 0xffff82c7c9f0 (LWP 125800)):
1045
#0  0x0000ffff8b5166c0 in ?? () from /lib/aarch64-linux-gnu/libaio.so.1
1046
#1  0x0000000000000001 in ?? ()
1047
Backtrace stopped: previous frame identical to this frame (corrupt stack?)
1048

    
1049
Thread 8 (Thread 0xffff8347d9f0 (LWP 125797)):
1050
#0  0x0000ffff8b1f922c in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0xaaaafaf6f244) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
1051
#1  __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafaf6f1e8, cond=0xaaaafaf6f218) at pthread_cond_wait.c:502
1052
#2  __pthread_cond_wait (cond=0xaaaafaf6f218, mutex=0xaaaafaf6f1e8) at pthread_cond_wait.c:655
1053
#3  0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
1054
#4  0x0000aaaad24efa80 in Finisher::finisher_thread_entry (this=0xaaaafaf6f1e0) at ./src/common/Finisher.cc:86
1055
#5  0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c55f) at pthread_create.c:463
1056
#6  0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
1057

    
1058
Thread 7 (Thread 0xffff8447f9f0 (LWP 125794)):
1059
#0  0x0000ffff8b1f922c in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0xaaaafaf6fa90) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
1060
#1  __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafaf6fa20, cond=0xaaaafaf6fa68) at pthread_cond_wait.c:502
1061
#2  __pthread_cond_wait (cond=0xaaaafaf6fa68, mutex=0xaaaafaf6fa20) at pthread_cond_wait.c:655
1062
#3  0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
1063
#4  0x0000aaaad251d438 in SafeTimer::timer_thread (this=0xaaaafaf6fa50) at ./src/common/Timer.cc:106
1064
#5  0x0000aaaad251e7f0 in SafeTimerThread::entry (this=<optimized out>) at ./src/common/Timer.cc:30
1065
#6  0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c55f) at pthread_create.c:463
1066
#7  0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
1067

    
1068
Thread 6 (Thread 0xffff87bb59f0 (LWP 125785)):
1069
#0  0x0000ffff8b1f95d4 in futex_abstimed_wait_cancelable (private=<optimized out>, abstime=0xffff87bb51b8, expected=0, futex_word=0xaaaafbc9421c)
1070
    at ../sysdeps/unix/sysv/linux/futex-internal.h:205
1071
#1  __pthread_cond_wait_common (abstime=0xffff87bb51b8, mutex=0xaaaafbc941c0, cond=0xaaaafbc941f0) at pthread_cond_wait.c:539
1072

    
1073

    
1074
#2  __pthread_cond_timedwait (cond=0xaaaafbc941f0, mutex=0xaaaafbc941c0, abstime=0xffff87bb51b8) at pthread_cond_wait.c:667
1075
#3  0x0000aaaad21f4624 in __gthread_cond_timedwait (__abs_timeout=0xffff87bb51b8, __mutex=<optimized out>, __cond=0xaaaafbc941f0)
1076
    at /usr/include/aarch64-linux-gnu/c++/9/bits/gthr-default.h:872
1077
#4  std::condition_variable::__wait_until_impl<std::chrono::duration<unsigned long, std::ratio<1l, 1000000000l> > > (__atime=<synthetic pointer>..., __lock=...,
1078
    this=0xaaaafbc941f0) at /usr/include/c++/9/condition_variable:188
1079
#5  std::condition_variable::wait_until<ceph::time_detail::coarse_mono_clock, std::chrono::duration<unsigned long, std::ratio<1l, 1000000000l> > > (__atime=...,
1080
    __lock=..., this=0xaaaafbc941f0) at /usr/include/c++/9/condition_variable:121
1081
---Type <return> to continue, or q <return> to quit---
1082
#6  ceph::timer_detail::timer<ceph::time_detail::coarse_mono_clock>::timer_thread (this=0xaaaafbc94180) at ./src/common/ceph_timer.h:146
1083
#7  0x0000ffff8b03ded4 in ?? () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
1084
#8  0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c89f) at pthread_create.c:463
1085
#9  0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
1086

    
1087
Thread 5 (Thread 0xffff856c49f0 (LWP 125781)):
1088
#0  0x0000ffff8addf728 in __GI___nanosleep (requested_time=0xffff856c40b8, requested_time@entry=0xffff856c40c8, remaining=remaining@entry=0x0)
1089
    at ../sysdeps/unix/sysv/linux/nanosleep.c:28
1090

    
1091

    
1092

    
1093

    
1094
#1  0x0000ffff8ae06fec in usleep (useconds=useconds@entry=256000) at ../sysdeps/posix/usleep.c:32
1095
#2  0x0000aaaad21acca4 in OpHistoryServiceThread::entry (this=0xaaaafaf6ee70) at ./src/common/TrackedOp.cc:44
1096
#3  0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c9ef) at pthread_create.c:463
1097
#4  0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
1098

    
1099
Thread 4 (Thread 0xffff873b49f0 (LWP 125359)):
1100
#0  0x0000ffff8ae04048 in __GI___poll (fds=fds@entry=0xffff873b3f68, nfds=281473015820424, nfds@entry=4, timeout=timeout@entry=-1)
1101
    at ../sysdeps/unix/sysv/linux/poll.c:41
1102
#1  0x0000ffff8ae1e6bc in __poll_chk (fds=fds@entry=0xffff873b3f68, nfds=nfds@entry=4, timeout=timeout@entry=-1, fdslen=fdslen@entry=264) at poll_chk.c:27
1103
#2  0x0000aaaad24d487c in poll (__timeout=-1, __nfds=4, __fds=0xffff873b3f68) at /usr/include/aarch64-linux-gnu/bits/poll2.h:41
1104
#3  SignalHandler::entry (this=0xaaaafaf789a0) at ./src/global/signal_handler.cc:488
1105
#4  0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9cbcf) at pthread_create.c:463
1106
#5  0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
1107

    
1108
Thread 3 (Thread 0xffff883b69f0 (LWP 124765)):
1109
#0  0x0000ffff8ae04048 in __GI___poll (fds=fds@entry=0xffff883b6198, nfds=281473014030036, nfds@entry=2, timeout=timeout@entry=-1)
1110
    at ../sysdeps/unix/sysv/linux/poll.c:41
1111
#1  0x0000aaaad252d514 in poll (__timeout=-1, __nfds=2, __fds=0xffff883b6198) at /usr/include/aarch64-linux-gnu/bits/poll2.h:46
1112
#2  AdminSocket::entry (this=0xaaaafaf78160) at ./src/common/admin_socket.cc:240
1113
#3  0x0000ffff8b03ded4 in ?? () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
1114
#4  0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9cadf) at pthread_create.c:463
1115
#5  0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
1116

    
1117
Thread 2 (Thread 0xffff88bb79f0 (LWP 124763)):
1118
#0  0x0000ffff8b1f95d4 in futex_abstimed_wait_cancelable (private=<optimized out>, abstime=0xffff88bb71d8, expected=0, futex_word=0xaaaafaf61c08)
1119
    at ../sysdeps/unix/sysv/linux/futex-internal.h:205
1120
#1  __pthread_cond_wait_common (abstime=0xffff88bb71d8, mutex=0xaaaafaf61bb0, cond=0xaaaafaf61be0) at pthread_cond_wait.c:539
1121

    
1122

    
1123

    
1124
#2  __pthread_cond_timedwait (cond=0xaaaafaf61be0, mutex=0xaaaafaf61bb0, abstime=0xffff88bb71d8) at pthread_cond_wait.c:667
1125
#3  0x0000aaaad254a954 in __gthread_cond_timedwait (__abs_timeout=0xffff88bb71d8, __mutex=<optimized out>, __cond=0xaaaafaf61be0)
1126
    at /usr/include/aarch64-linux-gnu/c++/9/bits/gthr-default.h:872
1127
#4  std::condition_variable::__wait_until_impl<std::chrono::duration<long, std::ratio<1l, 1000000000l> > > (__atime=<synthetic pointer>..., __lock=...,
1128
    this=0xaaaafaf61be0) at /usr/include/c++/9/condition_variable:188
1129
#5  std::condition_variable::wait_until<std::chrono::_V2::steady_clock, std::chrono::duration<long, std::ratio<1l, 1000000000l> > > (__atime=..., __lock=...,
1130
    this=0xaaaafaf61be0) at /usr/include/c++/9/condition_variable:121
1131
#6  std::condition_variable::wait_for<unsigned long, std::ratio<1l, 1000000000l> > (__rtime=<synthetic pointer>..., __lock=..., this=0xaaaafaf61be0)
1132
    at /usr/include/c++/9/condition_variable:152
1133
#7  CephContextServiceThread::entry (this=0xaaaafaf61b90) at ./src/common/ceph_context.cc:206
1134
#8  0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9ca7f) at pthread_create.c:463
1135
---Type <return> to continue, or q <return> to quit---
1136
#9  0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
1137

    
1138
Thread 1 (Thread 0xffff866e89f0 (LWP 124715)):
1139
#0  raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:51
1140
#1  0x0000aaaad24d3cec in reraise_fatal (signum=6) at ./src/global/signal_handler.cc:326
1141
#2  handle_fatal_signal (signum=6) at ./src/global/signal_handler.cc:326
1142
#3  <signal handler called>
1143
#4  __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:51
1144
#5  0x0000ffff8ad708b4 in __GI_abort () at abort.c:79
1145
#6  0x0000aaaad1e2c1b0 in ceph::__ceph_assert_fail (assertion=<optimized out>, file=<optimized out>, line=<optimized out>, func=<optimized out>)
1146
    at ./src/common/assert.cc:73
1147
#7  0x0000aaaad1e2c320 in ceph::__ceph_assert_fail (ctx=...) at ./src/common/assert.cc:78
1148
#8  0x0000aaaad26aeb0c in RDMADispatcher::handle_tx_event (this=this@entry=0xaaaafaf78c70, cqe=cqe@entry=0xffff866e7a90, n=n@entry=1)
1149
    at ./src/msg/async/rdma/RDMAStack.cc:558
1150
#9  0x0000aaaad26b07bc in RDMADispatcher::polling (this=0xaaaafaf78c70) at ./src/msg/async/rdma/RDMAStack.cc:282
1151
#10 0x0000ffff8b03ded4 in ?? () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
1152
#11 0x0000ffff8b1f3088 in start_thread (arg=0xffff893b7bff) at pthread_create.c:463
1153
#12 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
1154
(gdb)
1155
(gdb)
1156
(gdb)
1157
(gdb)
1158
(gdb)
1159
(gdb)
1160
(gdb)
1161
(gdb)
1162
(gdb)
1163
(gdb)
1164
(gdb)
1165
(gdb)
1166
(gdb)
1167
(gdb) bt
1168
#0  raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:51
1169
#1  0x0000aaaad24d3cec in reraise_fatal (signum=6) at ./src/global/signal_handler.cc:326
1170
#2  handle_fatal_signal (signum=6) at ./src/global/signal_handler.cc:326
1171
#3  <signal handler called>
1172
#4  __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:51
1173
#5  0x0000ffff8ad708b4 in __GI_abort () at abort.c:79
1174
#6  0x0000aaaad1e2c1b0 in ceph::__ceph_assert_fail (assertion=<optimized out>, file=<optimized out>, line=<optimized out>, func=<optimized out>)
1175
    at ./src/common/assert.cc:73
1176
#7  0x0000aaaad1e2c320 in ceph::__ceph_assert_fail (ctx=...) at ./src/common/assert.cc:78
1177
#8  0x0000aaaad26aeb0c in RDMADispatcher::handle_tx_event (this=this@entry=0xaaaafaf78c70, cqe=cqe@entry=0xffff866e7a90, n=n@entry=1)
1178
    at ./src/msg/async/rdma/RDMAStack.cc:558
1179
#9  0x0000aaaad26b07bc in RDMADispatcher::polling (this=0xaaaafaf78c70) at ./src/msg/async/rdma/RDMAStack.cc:282
1180
#10 0x0000ffff8b03ded4 in ?? () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6
1181
#11 0x0000ffff8b1f3088 in start_thread (arg=0xffff893b7bff) at pthread_create.c:463
1182
#12 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78
1183
(gdb) f 8
1184
#8  0x0000aaaad26aeb0c in RDMADispatcher::handle_tx_event (this=this@entry=0xaaaafaf78c70, cqe=cqe@entry=0xffff866e7a90, n=n@entry=1)
1185
    at ./src/msg/async/rdma/RDMAStack.cc:558
1186
558     in ./src/msg/async/rdma/RDMAStack.cc
1187
(gdb) l
1188
553     in ./src/msg/async/rdma/RDMAStack.cc
1189
(gdb) directory ~/chunsong/ceph
1190
Source directories searched: /root/chunsong/ceph:$cdir:$cwd
1191
(gdb) l
1192
553
1193
554         //TX completion may come either from
1194
555         // 1) regular send message, WCE wr_id points to chunk
1195
556         // 2) 'fin' message, wr_id points to the QP
1196
557         if (ib->get_memory_manager()->is_tx_buffer(chunk->buffer)) {
1197
558           ceph_assert(valid);
1198
559           tx_chunks.push_back(chunk);
1199
560         } else if (reinterpret_cast<QueuePair*>(response->wr_id)->get_local_qp_number() == response->qp_num ) {
1200
561           ceph_assert(valid == false);
1201
562           ldout(cct, 1) << __func__ << " sending of the disconnect msg completed" << dendl;
1202
(gdb) p chunk
1203
$1 = (Infiniband::MemoryManager::Chunk *) 0xaaab4fe52080
1204
(gdb) p *chunk
1205
$2 = {mr = 0xaaaafafd4000, qp = 0xaaaafaf666f0, lkey = 2, bytes = 43691, offset = 4224287488, bound = 43690, buffer = 0xaaab00000001 "", pad = {4210454592, 43690,
1206
    4210544584, 43690, 1386260360, 43691}, data = 0xaaab4fe520c0 ""}
1207
(gdb) p ib->
1208
_M_enable_shared_from_this_with<Infiniband>              operator->
1209
_M_get                                                   operator=
1210
_M_get_deleter                                           reset
1211
_M_ptr                                                   shared_ptr
1212
_M_refcount                                              shared_ptr<std::allocator<Infiniband>, CephContext*&>
1213
__shared_ptr                                             swap
1214
__shared_ptr<std::allocator<Infiniband>, CephContext*&>  unique
1215
get                                                      use_count
1216
operator bool                                            ~__shared_ptr
1217
operator*                                                ~shared_ptr
1218
(gdb) p *ib
1219
$3 = {tx_queue_len = 1024, rx_queue_len = 4096, max_sge = 0, ib_physical_port = 1 '\001', memory_manager = 0xaaaafaf4f550, srq = 0xaaaafaf7cfc8,
1220
  device = 0xaaaafb001200, pd = 0xaaaafb018368, device_list = 0xaaaafbb9b500, cct = 0xaaaafafd4000, lock = {<std::__mutex_base> = {_M_mutex = pthread_mutex_t = {
1221
        Type = Normal, Status = Not acquired, Robust = No, Shared = No, Protocol = None}}, <No data fields>}, initialized = true, device_name = "mlx5_0",
1222
  port_num = 1 '\001', support_srq = true}
1223
(gdb) p ib->memory_manager
1224
$4 = (Infiniband::MemoryManager *) 0xaaaafaf4f550
1225
(gdb) p ib->memory_manager->send
1226
$5 = (Infiniband::MemoryManager::Cluster *) 0xaaaafaf48c30
1227
(gdb) p *ib->memory_manager->send
1228
$6 = {manager = @0xaaaafaf4f550, buffer_size = 131072, num_chunk = 2048, lock = {<std::__mutex_base> = {_M_mutex = pthread_mutex_t = {Type = Normal,
1229
        Status = Not acquired, Robust = No, Shared = No, Protocol = None}}, <No data fields>}, free_chunks = std::vector of length 2048, capacity 2048 = {
1230
    0xaaab0bcca000, 0xaaab0bcca040, 0xaaab0bcca080, 0xaaab0bcca0c0, 0xaaab0bcca100, 0xaaab0bcca140, 0xaaab0bcca180, 0xaaab0bcca1c0, 0xaaab0bcca200, 0xaaab0bcca240,
1231
    0xaaab0bcca280, 0xaaab0bcca2c0, 0xaaab0bcca300, 0xaaab0bcca340, 0xaaab0bcca380, 0xaaab0bcca3c0, 0xaaab0bcca400, 0xaaab0bcca440, 0xaaab0bcca480, 0xaaab0bcca4c0,
1232
    0xaaab0bcca500, 0xaaab0bcca540, 0xaaab0bcca580, 0xaaab0bcca5c0, 0xaaab0bcca600, 0xaaab0bcca640, 0xaaab0bcca680, 0xaaab0bcca6c0, 0xaaab0bcca700, 0xaaab0bcca740,
1233
    0xaaab0bcca780, 0xaaab0bcca7c0, 0xaaab0bcca800, 0xaaab0bcca840, 0xaaab0bcca880, 0xaaab0bcca8c0, 0xaaab0bcca900, 0xaaab0bcca940, 0xaaab0bcca980, 0xaaab0bcca9c0,
1234
    0xaaab0bccaa00, 0xaaab0bccaa40, 0xaaab0bccaa80, 0xaaab0bccaac0, 0xaaab0bccab00, 0xaaab0bccab40, 0xaaab0bccab80, 0xaaab0bccabc0, 0xaaab0bccac00, 0xaaab0bccac40,
1235
    0xaaab0bccac80, 0xaaab0bccacc0, 0xaaab0bccad00, 0xaaab0bccad40, 0xaaab0bccad80, 0xaaab0bccadc0, 0xaaab0bccae00, 0xaaab0bccae40, 0xaaab0bccae80, 0xaaab0bccaec0,
1236
    0xaaab0bccaf00, 0xaaab0bccaf40, 0xaaab0bccaf80, 0xaaab0bccafc0, 0xaaab0bccb000, 0xaaab0bccb040, 0xaaab0bccb080, 0xaaab0bccb0c0, 0xaaab0bccb100, 0xaaab0bccb140,
1237
    0xaaab0bccb180, 0xaaab0bccb1c0, 0xaaab0bccb200, 0xaaab0bccb240, 0xaaab0bccb280, 0xaaab0bccb2c0, 0xaaab0bccb300, 0xaaab0bccb340, 0xaaab0bccb380, 0xaaab0bccb3c0,
1238
    0xaaab0bccb400, 0xaaab0bccb440, 0xaaab0bccb480, 0xaaab0bccb4c0, 0xaaab0bccb500, 0xaaab0bccb540, 0xaaab0bccb580, 0xaaab0bccb5c0, 0xaaab0bccb600, 0xaaab0bccb640,
1239
    0xaaab0bccb680, 0xaaab0bccb6c0, 0xaaab0bccb700, 0xaaab0bccb740, 0xaaab0bccb780, 0xaaab0bccb7c0, 0xaaab0bccb800, 0xaaab0bccb840, 0xaaab0bccb880, 0xaaab0bccb8c0,
1240
    0xaaab0bccb900, 0xaaab0bccb940, 0xaaab0bccb980, 0xaaab0bccb9c0, 0xaaab0bccba00, 0xaaab0bccba40, 0xaaab0bccba80, 0xaaab0bccbac0, 0xaaab0bccbb00, 0xaaab0bccbb40,
1241
    0xaaab0bccbb80, 0xaaab0bccbbc0, 0xaaab0bccbc00, 0xaaab0bccbc40, 0xaaab0bccbc80, 0xaaab0bccbcc0, 0xaaab0bccbd00, 0xaaab0bccbd40, 0xaaab0bccbd80, 0xaaab0bccbdc0,
1242
    0xaaab0bccbe00, 0xaaab0bccbe40, 0xaaab0bccbe80, 0xaaab0bccbec0, 0xaaab0bccbf00, 0xaaab0bccbf40, 0xaaab0bccbf80, 0xaaab0bccbfc0, 0xaaab0bccc000, 0xaaab0bccc040,
1243
    0xaaab0bccc080, 0xaaab0bccc0c0, 0xaaab0bccc100, 0xaaab0bccc140, 0xaaab0bccc180, 0xaaab0bccc1c0, 0xaaab0bccc200, 0xaaab0bccc240, 0xaaab0bccc280, 0xaaab0bccc2c0,
1244
    0xaaab0bccc300, 0xaaab0bccc340, 0xaaab0bccc380, 0xaaab0bccc3c0, 0xaaab0bccc400, 0xaaab0bccc440, 0xaaab0bccc480, 0xaaab0bccc4c0, 0xaaab0bccc500, 0xaaab0bccc540,
1245
    0xaaab0bccc580, 0xaaab0bccc5c0, 0xaaab0bccc600, 0xaaab0bccc640, 0xaaab0bccc680, 0xaaab0bccc6c0, 0xaaab0bccc700, 0xaaab0bccc740, 0xaaab0bccc780, 0xaaab0bccc7c0,
1246
    0xaaab0bccc800, 0xaaab0bccc840, 0xaaab0bccc880, 0xaaab0bccc8c0, 0xaaab0bccc900, 0xaaab0bccc940, 0xaaab0bccc980, 0xaaab0bccc9c0, 0xaaab0bccca00, 0xaaab0bccca40,
1247
    0xaaab0bccca80, 0xaaab0bcccac0, 0xaaab0bcccb00, 0xaaab0bcccb40, 0xaaab0bcccb80, 0xaaab0bcccbc0, 0xaaab0bcccc00, 0xaaab0bcccc40, 0xaaab0bcccc80, 0xaaab0bccccc0,
1248
    0xaaab0bcccd00, 0xaaab0bcccd40, 0xaaab0bcccd80, 0xaaab0bcccdc0, 0xaaab0bccce00, 0xaaab0bccce40, 0xaaab0bccce80, 0xaaab0bcccec0, 0xaaab0bcccf00, 0xaaab0bcccf40,
1249
    0xaaab0bcccf80, 0xaaab0bcccfc0, 0xaaab0bccd000, 0xaaab0bccd040, 0xaaab0bccd080, 0xaaab0bccd0c0, 0xaaab0bccd100, 0xaaab0bccd140, 0xaaab0bccd180,
1250
    0xaaab0bccd1c0...}, base = 0xaaaafbcca000 "", end = 0xaaab0bcca000 "\320\035\366", <incomplete sequence \372\252\252>, chunk_base = 0xaaab0bcca000}
1251
(failed reverse-i-search)`pch': Quitb->memory_manager->send
1252
(gdb) p *chunk
1253
$7 = {mr = 0xaaaafafd4000, qp = 0xaaaafaf666f0, lkey = 2, bytes = 43691, offset = 4224287488, bound = 43690, buffer = 0xaaab00000001 "", pad = {4210454592, 43690,
1254
    4210544584, 43690, 1386260360, 43691}, data = 0xaaab4fe520c0 ""}
1255
(gdb) p chunk
1256
$8 = (Infiniband::MemoryManager::Chunk *) 0xaaab4fe52080
1257
(gdb) ceph Quit