Project

General

Profile

gdb_lockup_3.txt

Brad House, 11/18/2014 12:43 PM

Download (19.5 KB)

 
1
root@proxmox1:/# gdb -ex "set pagination 0" -ex "thread apply all bt" --batch /usr/bin/kvm 27034
2
[Thread debugging using libthread_db enabled]
3
Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1".
4
[New Thread 0x7f485742d700 (LWP 6395)]
5
[New Thread 0x7f4855e25700 (LWP 6394)]
6
[New Thread 0x7f47cfdff700 (LWP 27112)]
7
[New Thread 0x7f47cffff700 (LWP 27111)]
8
[New Thread 0x7f47cfbff700 (LWP 27108)]
9
[New Thread 0x7f4855523700 (LWP 27106)]
10
[New Thread 0x7f4855d24700 (LWP 27105)]
11
[New Thread 0x7f4855f26700 (LWP 27104)]
12
[New Thread 0x7f4856027700 (LWP 27101)]
13
[New Thread 0x7f4856128700 (LWP 27100)]
14
[New Thread 0x7f4856229700 (LWP 27099)]
15
[New Thread 0x7f4856a2a700 (LWP 27098)]
16
[New Thread 0x7f485722b700 (LWP 27097)]
17
[New Thread 0x7f4857c2e700 (LWP 27092)]
18
[New Thread 0x7f485842f700 (LWP 27091)]
19
[New Thread 0x7f4858c30700 (LWP 27090)]
20
[New Thread 0x7f4859431700 (LWP 27089)]
21
[New Thread 0x7f4859e3e700 (LWP 27088)]
22
[New Thread 0x7f485ada4700 (LWP 27087)]
23

    
24
warning: no loadable sections found in added symbol-file system-supplied DSO at 0x7fff369fc000
25
__lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:136
26
136	../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S: No such file or directory.
27

    
28
Thread 20 (Thread 0x7f485ada4700 (LWP 27087)):
29
#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
30
#1  0x00007f486647425b in ceph::log::Log::entry (this=0x7f486a0bf3e0) at log/Log.cc:323
31
#2  0x00007f48627dab50 in start_thread (arg=<optimized out>) at pthread_create.c:304
32
#3  0x00007f48625247bd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
33
#4  0x0000000000000000 in ?? ()
34

    
35
Thread 19 (Thread 0x7f4859e3e700 (LWP 27088)):
36
#0  pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:216
37
#1  0x00007f4866279f72 in WaitUntil (when=..., mutex=..., this=0x7f486a0d9d18) at ./common/Cond.h:71
38
#2  WaitInterval (interval=..., mutex=..., cct=<optimized out>, this=0x7f486a0d9d18) at ./common/Cond.h:79
39
#3  CephContextServiceThread::entry (this=0x7f486a0d9ca0) at common/ceph_context.cc:58
40
#4  0x00007f48627dab50 in start_thread (arg=<optimized out>) at pthread_create.c:304
41
#5  0x00007f48625247bd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
42
#6  0x0000000000000000 in ?? ()
43

    
44
Thread 18 (Thread 0x7f4859431700 (LWP 27089)):
45
#0  __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:136
46
#1  0x00007f48627dd3a9 in _L_lock_926 () from /lib/x86_64-linux-gnu/libpthread.so.0
47
#2  0x00007f48627dd1cb in __pthread_mutex_lock (mutex=0x7f486a0c2440) at pthread_mutex_lock.c:61
48
#3  0x00007f486623f3ab in Mutex::Lock (this=0x7f486a0c2430, no_lockdep=<optimized out>) at common/Mutex.cc:89
49
#4  0x00007f48661c176e in Locker (m=..., this=0x7f48594308a0) at ./common/Mutex.h:120
50
#5  librados::RadosClient::ms_dispatch (this=0x7f486a0c1c10, m=0x7f4850079000) at librados/RadosClient.cc:352
51
#6  0x00007f486635eb3b in ms_deliver_dispatch (m=0x7f4850079000, this=0x7f486a0db8e0) at msg/Messenger.h:587
52
#7  DispatchQueue::entry (this=0x7f486a0db9d8) at msg/DispatchQueue.cc:124
53
#8  0x00007f48663ca37d in DispatchQueue::DispatchThread::entry (this=<optimized out>) at msg/DispatchQueue.h:104
54
#9  0x00007f48627dab50 in start_thread (arg=<optimized out>) at pthread_create.c:304
55
#10 0x00007f48625247bd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
56
#11 0x0000000000000000 in ?? ()
57

    
58
Thread 17 (Thread 0x7f4858c30700 (LWP 27090)):
59
#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
60
#1  0x00007f48663c5ba8 in Wait (mutex=..., this=0x7f486a0dbe28) at ./common/Cond.h:55
61
#2  SimpleMessenger::reaper_entry (this=0x7f486a0db8e0) at msg/SimpleMessenger.cc:210
62
#3  0x00007f48663caa9d in SimpleMessenger::ReaperThread::entry (this=<optimized out>) at msg/SimpleMessenger.h:225
63
#4  0x00007f48627dab50 in start_thread (arg=<optimized out>) at pthread_create.c:304
64
#5  0x00007f48625247bd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
65
#6  0x0000000000000000 in ?? ()
66

    
67
Thread 16 (Thread 0x7f485842f700 (LWP 27091)):
68
#0  pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:216
69
#1  0x00007f486625f1b8 in WaitUntil (when=..., mutex=..., this=0x7f486a0c2030) at common/Cond.h:71
70
#2  SafeTimer::timer_thread (this=0x7f486a0c2020) at common/Timer.cc:114
71
#3  0x00007f48662600ed in SafeTimerThread::entry (this=<optimized out>) at common/Timer.cc:38
72
#4  0x00007f48627dab50 in start_thread (arg=<optimized out>) at pthread_create.c:304
73
#5  0x00007f48625247bd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
74
#6  0x0000000000000000 in ?? ()
75

    
76
Thread 15 (Thread 0x7f4857c2e700 (LWP 27092)):
77
#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
78
#1  0x00007f4866260b42 in Wait (mutex=..., this=0x7f486a0c2148) at ./common/Cond.h:55
79
#2  Finisher::finisher_thread_entry (this=0x7f486a0c20e8) at common/Finisher.cc:78
80
#3  0x00007f48627dab50 in start_thread (arg=<optimized out>) at pthread_create.c:304
81
#4  0x00007f48625247bd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
82
#5  0x0000000000000000 in ?? ()
83

    
84
Thread 14 (Thread 0x7f485722b700 (LWP 27097)):
85
#0  __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:136
86
#1  0x00007f48627e4c4e in _L_cond_lock_1001 () from /lib/x86_64-linux-gnu/libpthread.so.0
87
#2  0x00007f48627e4a7b in __pthread_mutex_cond_lock (mutex=0x7f486a0c2440) at ../nptl/pthread_mutex_lock.c:61
88
#3  0x00007f48627df76f in pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:303
89
#4  0x00007f486625f1b8 in WaitUntil (when=..., mutex=..., this=0x7f486a0c2528) at common/Cond.h:71
90
#5  SafeTimer::timer_thread (this=0x7f486a0c2518) at common/Timer.cc:114
91
#6  0x00007f48662600ed in SafeTimerThread::entry (this=<optimized out>) at common/Timer.cc:38
92
#7  0x00007f48627dab50 in start_thread (arg=<optimized out>) at pthread_create.c:304
93
#8  0x00007f48625247bd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
94
#9  0x0000000000000000 in ?? ()
95

    
96
Thread 13 (Thread 0x7f4856a2a700 (LWP 27098)):
97
#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
98
#1  0x00007f4866260b42 in Wait (mutex=..., this=0x7f486a0c2668) at ./common/Cond.h:55
99
#2  Finisher::finisher_thread_entry (this=0x7f486a0c2608) at common/Finisher.cc:78
100
#3  0x00007f48627dab50 in start_thread (arg=<optimized out>) at pthread_create.c:304
101
#4  0x00007f48625247bd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
102
#5  0x0000000000000000 in ?? ()
103

    
104
Thread 12 (Thread 0x7f4856229700 (LWP 27099)):
105
#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
106
#1  0x00007f48663b581a in Wait (mutex=..., this=0x7f4850007f50) at ./common/Cond.h:55
107
#2  Pipe::writer (this=0x7f4850007d40) at msg/Pipe.cc:1730
108
#3  0x00007f48663c1c5d in Pipe::Writer::entry (this=<optimized out>) at msg/Pipe.h:61
109
#4  0x00007f48627dab50 in start_thread (arg=<optimized out>) at pthread_create.c:304
110
#5  0x00007f48625247bd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
111
#6  0x0000000000000000 in ?? ()
112

    
113
Thread 11 (Thread 0x7f4856128700 (LWP 27100)):
114
#0  0x00007f4862519b73 in *__GI___poll (fds=<optimized out>, nfds=<optimized out>, timeout=900000) at ../sysdeps/unix/sysv/linux/poll.c:87
115
#1  0x00007f48663aa87c in Pipe::tcp_read_wait (this=this@entry=0x7f4850007d40) at msg/Pipe.cc:2282
116
#2  0x00007f48663aab10 in Pipe::tcp_read (this=this@entry=0x7f4850007d40, buf=<optimized out>, buf@entry=0x7f4856127b6f "\377", len=len@entry=1) at msg/Pipe.cc:2255
117
#3  0x00007f48663bca9f in Pipe::reader (this=0x7f4850007d40) at msg/Pipe.cc:1421
118
#4  0x00007f48663c1d7d in Pipe::Reader::entry (this=<optimized out>) at msg/Pipe.h:49
119
#5  0x00007f48627dab50 in start_thread (arg=<optimized out>) at pthread_create.c:304
120
#6  0x00007f48625247bd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
121
#7  0x0000000000000000 in ?? ()
122

    
123
Thread 10 (Thread 0x7f4856027700 (LWP 27101)):
124
#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
125
#1  0x00007f48663b581a in Wait (mutex=..., this=0x7f485000a590) at ./common/Cond.h:55
126
#2  Pipe::writer (this=0x7f485000a380) at msg/Pipe.cc:1730
127
#3  0x00007f48663c1c5d in Pipe::Writer::entry (this=<optimized out>) at msg/Pipe.h:61
128
#4  0x00007f48627dab50 in start_thread (arg=<optimized out>) at pthread_create.c:304
129
#5  0x00007f48625247bd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
130
#6  0x0000000000000000 in ?? ()
131

    
132
Thread 9 (Thread 0x7f4855f26700 (LWP 27104)):
133
#0  0x00007f4862519b73 in *__GI___poll (fds=<optimized out>, nfds=<optimized out>, timeout=900000) at ../sysdeps/unix/sysv/linux/poll.c:87
134
#1  0x00007f48663aa87c in Pipe::tcp_read_wait (this=this@entry=0x7f485000a380) at msg/Pipe.cc:2282
135
#2  0x00007f48663aab10 in Pipe::tcp_read (this=this@entry=0x7f485000a380, buf=<optimized out>, buf@entry=0x7f4855f25b6f "\377", len=len@entry=1) at msg/Pipe.cc:2255
136
#3  0x00007f48663bca9f in Pipe::reader (this=0x7f485000a380) at msg/Pipe.cc:1421
137
#4  0x00007f48663c1d7d in Pipe::Reader::entry (this=<optimized out>) at msg/Pipe.h:49
138
#5  0x00007f48627dab50 in start_thread (arg=<optimized out>) at pthread_create.c:304
139
#6  0x00007f48625247bd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
140
#7  0x0000000000000000 in ?? ()
141

    
142
Thread 8 (Thread 0x7f4855d24700 (LWP 27105)):
143
#0  __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:136
144
#1  0x00007f48627dd3a9 in _L_lock_926 () from /lib/x86_64-linux-gnu/libpthread.so.0
145
#2  0x00007f48627dd1cb in __pthread_mutex_lock (mutex=mutex@entry=0x7f4868303040) at pthread_mutex_lock.c:61
146
#3  0x00007f4867e598d9 in qemu_mutex_lock (mutex=mutex@entry=0x7f4868303040) at util/qemu-thread-posix.c:76
147
#4  0x00007f4867b99100 in qemu_mutex_lock_iothread () at /root/pve-qemu-kvm/qemu-kvm/cpus.c:1044
148
#5  0x00007f4867babb2d in kvm_cpu_exec (cpu=cpu@entry=0x7f4850010fe0) at /root/pve-qemu-kvm/qemu-kvm/kvm-all.c:1716
149
#6  0x00007f4867b97e0c in qemu_kvm_cpu_thread_fn (arg=0x7f4850010fe0) at /root/pve-qemu-kvm/qemu-kvm/cpus.c:874
150
#7  0x00007f48627dab50 in start_thread (arg=<optimized out>) at pthread_create.c:304
151
#8  0x00007f48625247bd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
152
#9  0x0000000000000000 in ?? ()
153

    
154
Thread 7 (Thread 0x7f4855523700 (LWP 27106)):
155
#0  0x00007f486251d807 in ioctl () at ../sysdeps/unix/syscall-template.S:82
156
#1  0x00007f4867bab9e9 in kvm_vcpu_ioctl (cpu=cpu@entry=0x7f4850054e80, type=type@entry=44672) at /root/pve-qemu-kvm/qemu-kvm/kvm-all.c:1845
157
#2  0x00007f4867babb25 in kvm_cpu_exec (cpu=cpu@entry=0x7f4850054e80) at /root/pve-qemu-kvm/qemu-kvm/kvm-all.c:1714
158
#3  0x00007f4867b97e0c in qemu_kvm_cpu_thread_fn (arg=0x7f4850054e80) at /root/pve-qemu-kvm/qemu-kvm/cpus.c:874
159
#4  0x00007f48627dab50 in start_thread (arg=<optimized out>) at pthread_create.c:304
160
#5  0x00007f48625247bd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
161
#6  0x0000000000000000 in ?? ()
162

    
163
Thread 6 (Thread 0x7f47cfbff700 (LWP 27108)):
164
#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
165
#1  0x00007f4867e59af9 in qemu_cond_wait (cond=cond@entry=0x7f48500dd310, mutex=mutex@entry=0x7f48500dd340) at util/qemu-thread-posix.c:135
166
#2  0x00007f4867df599b in vnc_worker_thread_loop (queue=queue@entry=0x7f48500dd310) at ui/vnc-jobs.c:222
167
#3  0x00007f4867df5d70 in vnc_worker_thread (arg=0x7f48500dd310) at ui/vnc-jobs.c:323
168
#4  0x00007f48627dab50 in start_thread (arg=<optimized out>) at pthread_create.c:304
169
#5  0x00007f48625247bd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
170
#6  0x0000000000000000 in ?? ()
171

    
172
Thread 5 (Thread 0x7f47cffff700 (LWP 27111)):
173
#0  __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:136
174
#1  0x00007f48627dd3df in _L_lock_1145 () from /lib/x86_64-linux-gnu/libpthread.so.0
175
#2  0x00007f48627dd32a in __pthread_mutex_lock (mutex=0x7f486a0d0440) at pthread_mutex_lock.c:101
176
#3  0x00007f4860fc4909 in PR_Lock () from /usr/lib/x86_64-linux-gnu/libnspr4.so
177
#4  0x00007f485a382545 in ?? () from /usr/lib/x86_64-linux-gnu/nss/libsoftokn3.so
178
#5  0x00007f485a376a0a in ?? () from /usr/lib/x86_64-linux-gnu/nss/libsoftokn3.so
179
#6  0x00007f485a3778ba in ?? () from /usr/lib/x86_64-linux-gnu/nss/libsoftokn3.so
180
#7  0x00007f4861cbf966 in ?? () from /usr/lib/x86_64-linux-gnu/libnss3.so
181
#8  0x00007f4861cbfa91 in ?? () from /usr/lib/x86_64-linux-gnu/libnss3.so
182
#9  0x00007f4861cbfe0f in PK11_CreateContextBySymKey () from /usr/lib/x86_64-linux-gnu/libnss3.so
183
#10 0x00007f48663ee9c3 in nss_aes_operation (op=260, secret=..., in=..., out=..., error=...) at auth/Crypto.cc:155
184
#11 0x00007f48663ed881 in CryptoKey::encrypt (this=0x7f486a115c28, cct=0x7f486a0bccf0, in=..., out=..., error=...) at auth/Crypto.cc:358
185
#12 0x00007f48663ed03e in encode_encrypt_enc_bl<ceph::buffer::list> (cct=cct@entry=0x7f486a0bccf0, t=..., key=..., out=..., error=...) at auth/cephx/CephxProtocol.h:465
186
#13 0x00007f48663ed172 in encode_encrypt<ceph::buffer::list> (cct=0x7f486a0bccf0, t=..., key=..., out=..., error=...) at auth/cephx/CephxProtocol.h:490
187
#14 0x00007f48663ec659 in CephxSessionHandler::sign_message (this=0x7f486a115c10, m=0x7f486a10fdc0) at auth/cephx/CephxSessionHandler.cc:48
188
#15 0x00007f48663b5ffb in Pipe::writer (this=0x7f486a1637b0) at msg/Pipe.cc:1693
189
#16 0x00007f48663c1c5d in Pipe::Writer::entry (this=<optimized out>) at msg/Pipe.h:61
190
#17 0x00007f48627dab50 in start_thread (arg=<optimized out>) at pthread_create.c:304
191
#18 0x00007f48625247bd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
192
#19 0x0000000000000000 in ?? ()
193

    
194
Thread 4 (Thread 0x7f47cfdff700 (LWP 27112)):
195
#0  __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:136
196
#1  0x00007f48627dd3a9 in _L_lock_926 () from /lib/x86_64-linux-gnu/libpthread.so.0
197
#2  0x00007f48627dd1cb in __pthread_mutex_lock (mutex=0x7f486a1638f8) at pthread_mutex_lock.c:61
198
#3  0x00007f486623f3ab in Mutex::Lock (this=0x7f486a1638e8, no_lockdep=<optimized out>) at common/Mutex.cc:89
199
#4  0x00007f48663bd6ae in Pipe::reader (this=0x7f486a1637b0) at msg/Pipe.cc:1485
200
#5  0x00007f48663c1d7d in Pipe::Reader::entry (this=<optimized out>) at msg/Pipe.h:49
201
#6  0x00007f48627dab50 in start_thread (arg=<optimized out>) at pthread_create.c:304
202
#7  0x00007f48625247bd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
203
#8  0x0000000000000000 in ?? ()
204

    
205
Thread 3 (Thread 0x7f4855e25700 (LWP 6394)):
206
#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
207
#1  0x00007f48663b581a in Wait (mutex=..., this=0x7f47b85b49b0) at ./common/Cond.h:55
208
#2  Pipe::writer (this=0x7f47b85b47a0) at msg/Pipe.cc:1730
209
#3  0x00007f48663c1c5d in Pipe::Writer::entry (this=<optimized out>) at msg/Pipe.h:61
210
#4  0x00007f48627dab50 in start_thread (arg=<optimized out>) at pthread_create.c:304
211
#5  0x00007f48625247bd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
212
#6  0x0000000000000000 in ?? ()
213

    
214
Thread 2 (Thread 0x7f485742d700 (LWP 6395)):
215
#0  0x00007f4862519b73 in *__GI___poll (fds=<optimized out>, nfds=<optimized out>, timeout=900000) at ../sysdeps/unix/sysv/linux/poll.c:87
216
#1  0x00007f48663aa87c in Pipe::tcp_read_wait (this=this@entry=0x7f47b85b47a0) at msg/Pipe.cc:2282
217
#2  0x00007f48663aab10 in Pipe::tcp_read (this=this@entry=0x7f47b85b47a0, buf=<optimized out>, buf@entry=0x7f485742cb6f "\377\250\277[\270G\177", len=len@entry=1) at msg/Pipe.cc:2255
218
#3  0x00007f48663bca9f in Pipe::reader (this=0x7f47b85b47a0) at msg/Pipe.cc:1421
219
#4  0x00007f48663c1d7d in Pipe::Reader::entry (this=<optimized out>) at msg/Pipe.h:49
220
#5  0x00007f48627dab50 in start_thread (arg=<optimized out>) at pthread_create.c:304
221
#6  0x00007f48625247bd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
222
#7  0x0000000000000000 in ?? ()
223

    
224
Thread 1 (Thread 0x7f486799f8e0 (LWP 27034)):
225
#0  __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:136
226
#1  0x00007f48627dd3a9 in _L_lock_926 () from /lib/x86_64-linux-gnu/libpthread.so.0
227
#2  0x00007f48627dd1cb in __pthread_mutex_lock (mutex=0x7f486a1638f8) at pthread_mutex_lock.c:61
228
#3  0x00007f486623f3ab in Mutex::Lock (this=0x7f486a1638e8, no_lockdep=<optimized out>) at common/Mutex.cc:89
229
#4  0x00007f48663c6c3a in SimpleMessenger::submit_message (this=this@entry=0x7f486a0db8e0, m=m@entry=0x7f47b8016ee0, con=con@entry=0x7f486a116d10, dest_addr=..., dest_type=4, lazy=lazy@entry=false) at msg/SimpleMessenger.cc:429
230
#5  0x00007f48663c76a9 in SimpleMessenger::_send_message (this=0x7f486a0db8e0, m=0x7f47b8016ee0, con=0x7f486a116d10, lazy=false) at msg/SimpleMessenger.cc:138
231
#6  0x00007f4866fdb1b5 in send_message (con=..., m=0x7f47b8016ee0, this=<optimized out>) at ./msg/Messenger.h:411
232
#7  Objecter::send_op (this=this@entry=0x7f486a0dc330, op=op@entry=0x7f47b8004e30) at osdc/Objecter.cc:1678
233
#8  0x00007f4866fdb981 in Objecter::_op_submit (this=0x7f486a0dc330, op=0x7f47b8004e30) at osdc/Objecter.cc:1340
234
#9  0x00007f48661cb8d8 in librados::IoCtxImpl::aio_operate_read (this=0x7f4850007050, oid=..., o=0x7f47b85b4c80, c=<optimized out>, flags=0, pbl=0x0) at librados/IoCtxImpl.cc:594
235
#10 0x00007f48661a0c98 in librados::IoCtx::aio_operate (this=0x7f4850006b58, oid=..., c=0x7f47b80059e0, o=0x7f486a496d40, flags=0, pbl=<optimized out>) at librados/librados.cc:1106
236
#11 0x00007f4866fa2e65 in librbd::AioRead::send (this=0x7f486a0ffc10) at librbd/AioRequest.cc:102
237
#12 0x00007f4866fb523a in librbd::aio_read (ictx=ictx@entry=0x7f4850006a10, image_extents=..., buf=buf@entry=0x7f486a101200 "", pbl=pbl@entry=0x0, c=c@entry=0x7f486a101010) at librbd/internal.cc:3153
238
#13 0x00007f4866fb5794 in librbd::aio_read (ictx=0x7f4850006a10, off=19674722304, len=4096, buf=0x7f486a101200 "", bl=0x0, c=0x7f486a101010) at librbd/internal.cc:3091
239
#14 0x00007f4867e4af4b in rbd_start_aio (bs=<optimized out>, sector_num=<optimized out>, qiov=<optimized out>, nb_sectors=8, cb=<optimized out>, opaque=<optimized out>, cmd=cmd@entry=RBD_AIO_READ) at block/rbd.c:668
240
#15 0x00007f4867e4b1e0 in qemu_rbd_aio_readv (bs=<optimized out>, sector_num=<optimized out>, qiov=<optimized out>, nb_sectors=<optimized out>, cb=<optimized out>, opaque=<optimized out>) at block/rbd.c:702
241
#16 0x00007f4867dfaa9e in bdrv_co_io_em (is_write=false, iov=0x7f486a497240, nb_sectors=8, sector_num=38427192, bs=0x7f486a0bb110) at block.c:4862
242
#17 bdrv_co_readv_em (bs=0x7f486a0bb110, sector_num=38427192, nb_sectors=8, iov=0x7f486a497240) at block.c:4879
243
#18 0x00007f4867dffd53 in bdrv_aligned_preadv (bs=bs@entry=0x7f486a0bb110, req=req@entry=0x7f486a497300, offset=offset@entry=19674722304, bytes=bytes@entry=4096, align=align@entry=512, qiov=qiov@entry=0x7f486a35b5a0, flags=flags@entry=0) at block.c:3081
244
#19 0x00007f4867e00267 in bdrv_co_do_preadv (bs=0x7f486a0bb110, offset=19674722304, bytes=4096, qiov=0x7f486a35b5a0, flags=0) at block.c:3162
245
#20 0x00007f4867dffc6b in bdrv_aligned_preadv (bs=bs@entry=0x7f486a0b8590, req=req@entry=0x7f486a4974a0, offset=offset@entry=19674722304, bytes=bytes@entry=4096, align=align@entry=512, qiov=qiov@entry=0x7f486a35b5a0, flags=flags@entry=0) at block.c:3056
246
#21 0x00007f4867e00267 in bdrv_co_do_preadv (bs=bs@entry=0x7f486a0b8590, offset=19674722304, bytes=4096, qiov=0x7f486a35b5a0, flags=0) at block.c:3162
247
#22 0x00007f4867e011f7 in bdrv_co_do_readv (flags=<optimized out>, qiov=<optimized out>, nb_sectors=<optimized out>, sector_num=<optimized out>, bs=0x7f486a0b8590) at block.c:3184
248
#23 bdrv_co_do_rw (opaque=0x7f47cb214750) at block.c:4710
249
#24 0x00007f4867e12caa in coroutine_trampoline (i0=<optimized out>, i1=<optimized out>) at coroutine-ucontext.c:118
250
#25 0x00007f486248ad10 in ?? () from /lib/x86_64-linux-gnu/libc.so.6
251
#26 0x00007fff369941c0 in ?? ()
252
#27 0x0000000000000000 in ?? ()