Project

General

Profile

Bug #23595 » gdb-threads-sender-node-O0-build.txt

Niklas Hambuechen, 04/09/2018 12:03 PM

 
(gdb) info threads
Id Target Id Frame
* 1 Thread 0x7fdb87938e00 (LWP 131171) "ceph-osd" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
2 Thread 0x7fdb83ae0700 (LWP 131176) "log" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
3 Thread 0x7fdb82a9f700 (LWP 131189) "msgr-worker-0" 0x00007fdb84e8ba13 in epoll_wait () at ../sysdeps/unix/syscall-template.S:84
4 Thread 0x7fdb8229e700 (LWP 131190) "msgr-worker-1" 0x00007fdb84e8ba13 in epoll_wait () at ../sysdeps/unix/syscall-template.S:84
5 Thread 0x7fdb81a9d700 (LWP 131191) "msgr-worker-2" 0x00007fdb84e8ba13 in epoll_wait () at ../sysdeps/unix/syscall-template.S:84
6 Thread 0x7fdb80afe700 (LWP 131197) "service" pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225
7 Thread 0x7fdb802fd700 (LWP 131199) "admin_socket" 0x00007fdb84e7f74d in poll () at ../sysdeps/unix/syscall-template.S:84
8 Thread 0x7fdb7f2af700 (LWP 131519) "ceph-osd" pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225
9 Thread 0x7fdb7eaae700 (LWP 131521) "safe_timer" pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225
10 Thread 0x7fdb7e2ad700 (LWP 131522) "safe_timer" pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225
11 Thread 0x7fdb7daac700 (LWP 131523) "safe_timer" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
12 Thread 0x7fdb7d2ab700 (LWP 131524) "safe_timer" pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225
13 Thread 0x7fdb7caaa700 (LWP 131526) "bstore_aio" 0x00007fdb86e9e64a in ?? () from target:/lib/x86_64-linux-gnu/libaio.so.1
14 Thread 0x7fdb7c2a9700 (LWP 131528) "bstore_aio" 0x00007fdb86e9e64a in ?? () from target:/lib/x86_64-linux-gnu/libaio.so.1
15 Thread 0x7fdb74299700 (LWP 131827) "dfin" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
16 Thread 0x7fdb74a9a700 (LWP 131828) "finisher" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
17 Thread 0x7fdb7529b700 (LWP 131829) "bstore_kv_sync" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
18 Thread 0x7fdb75a9c700 (LWP 131830) "bstore_kv_final" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
19 Thread 0x7fdb7baa8700 (LWP 131832) "bstore_mempool" pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225
20 Thread 0x7fdb7931d700 (LWP 131877) "ms_dispatch" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
21 Thread 0x7fdb78b1c700 (LWP 131878) "ms_local" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
22 Thread 0x7fdb7831b700 (LWP 131879) "ms_dispatch" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
23 Thread 0x7fdb77b1a700 (LWP 131880) "ms_local" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
24 Thread 0x7fdb77319700 (LWP 131881) "ms_dispatch" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
25 Thread 0x7fdb76b18700 (LWP 131882) "ms_local" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
26 Thread 0x7fdb76317700 (LWP 131883) "ms_dispatch" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
27 Thread 0x7fdb73a98700 (LWP 131884) "ms_local" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
28 Thread 0x7fdb73297700 (LWP 131885) "ms_dispatch" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
29 Thread 0x7fdb72a96700 (LWP 131886) "ms_local" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
30 Thread 0x7fdb72295700 (LWP 131887) "ms_dispatch" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
31 Thread 0x7fdb71a94700 (LWP 131888) "ms_local" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
32 Thread 0x7fdb71293700 (LWP 131889) "ms_dispatch" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
33 Thread 0x7fdb70a92700 (LWP 131890) "ms_local" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
34 Thread 0x7fdb70291700 (LWP 131891) "safe_timer" pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225
35 Thread 0x7fdb6fa90700 (LWP 131892) "fn_anonymous" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
36 Thread 0x7fdb6f28f700 (LWP 131893) "safe_timer" pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225
37 Thread 0x7fdb6ea8e700 (LWP 131894) "tp_peering" pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225
38 Thread 0x7fdb6e28d700 (LWP 131895) "tp_peering" pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225
39 Thread 0x7fdb6da8c700 (LWP 131896) "tp_osd_tp" pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225
40 Thread 0x7fdb6d28b700 (LWP 131897) "tp_osd_tp" pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225
41 Thread 0x7fdb6ca8a700 (LWP 131898) "tp_osd_tp" pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225
42 Thread 0x7fdb6c289700 (LWP 131899) "tp_osd_tp" pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225
43 Thread 0x7fdb6ba88700 (LWP 131900) "tp_osd_tp" pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225
44 Thread 0x7fdb6b287700 (LWP 131901) "tp_osd_disk" pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225
45 Thread 0x7fdb6aa86700 (LWP 131902) "tp_osd_cmd" pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225
46 Thread 0x7fdb6a285700 (LWP 131903) "osd_srv_heartbt" pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225
47 Thread 0x7fdb69a84700 (LWP 131904) "fn_anonymous" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
48 Thread 0x7fdb69283700 (LWP 131905) "fn_anonymous" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
49 Thread 0x7fdb68a82700 (LWP 131906) "safe_timer" pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225
50 Thread 0x7fdb68281700 (LWP 131907) "safe_timer" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
51 Thread 0x7fdb67a80700 (LWP 131908) "safe_timer" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
52 Thread 0x7fdb6727f700 (LWP 131909) "safe_timer" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
53 Thread 0x7fdb66a7e700 (LWP 131910) "osd_srv_agent" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
54 Thread 0x7fdb6627d700 (LWP 131911) "signal_handler" 0x00007fdb84e7f74d in poll () at ../sysdeps/unix/syscall-template.S:84
(gdb) thread apply all bt

Thread 54 (Thread 0x7fdb6627d700 (LWP 131911)):
#0 0x00007fdb84e7f74d in poll () at ../sysdeps/unix/syscall-template.S:84
#1 0x0000564c64bd4882 in SignalHandler::entry (this=0x564c8371b800) at /raid/src/ceph-deb/ceph-12.2.4/src/global/signal_handler.cc:281
#2 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c8371b800) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79
#3 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c8371b800) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59
#4 0x00007fdb85e146ba in start_thread (arg=0x7fdb6627d700) at pthread_create.c:333
#5 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109

Thread 53 (Thread 0x7fdb66a7e700 (LWP 131910)):
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1 0x0000564c64302818 in Cond::Wait (this=0x564c708f4600, mutex=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:48
#2 0x0000564c6428dc39 in OSDService::agent_entry (this=0x564c708f3dc8) at /raid/src/ceph-deb/ceph-12.2.4/src/osd/OSD.cc:610
#3 0x0000564c643181ac in OSDService::AgentThread::entry (this=0x564c708f46c0) at /raid/src/ceph-deb/ceph-12.2.4/src/osd/OSD.h:665
#4 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c708f46c0) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79
#5 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c708f46c0) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59
#6 0x00007fdb85e146ba in start_thread (arg=0x7fdb66a7e700) at pthread_create.c:333
#7 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109

Thread 52 (Thread 0x7fdb6727f700 (LWP 131909)):
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1 0x0000564c64302818 in Cond::Wait (this=0x564c708f5460, mutex=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:48
#2 0x0000564c64c2fada in SafeTimer::timer_thread (this=0x564c708f5450) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Timer.cc:108
#3 0x0000564c64c310e4 in SafeTimerThread::entry (this=0x564c81b6fed0) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Timer.cc:30
#4 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c81b6fed0) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79
#5 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c81b6fed0) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59
#6 0x00007fdb85e146ba in start_thread (arg=0x7fdb6727f700) at pthread_create.c:333
#7 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109

Thread 51 (Thread 0x7fdb67a80700 (LWP 131908)):
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1 0x0000564c64302818 in Cond::Wait (this=0x564c708f5328, mutex=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:48
#2 0x0000564c64c2fada in SafeTimer::timer_thread (this=0x564c708f5318) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Timer.cc:108
#3 0x0000564c64c310e4 in SafeTimerThread::entry (this=0x564c81b6fea0) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Timer.cc:30
#4 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c81b6fea0) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79
#5 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c81b6fea0) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59
#6 0x00007fdb85e146ba in start_thread (arg=0x7fdb67a80700) at pthread_create.c:333
#7 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109

Thread 50 (Thread 0x7fdb68281700 (LWP 131907)):
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1 0x0000564c64302818 in Cond::Wait (this=0x564c708f4778, mutex=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:48
#2 0x0000564c64c2fada in SafeTimer::timer_thread (this=0x564c708f4768) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Timer.cc:108
#3 0x0000564c64c310e4 in SafeTimerThread::entry (this=0x564c81b6fe70) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Timer.cc:30
#4 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c81b6fe70) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79
#5 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c81b6fe70) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59
---Type <return> to continue, or q <return> to quit---
#6 0x00007fdb85e146ba in start_thread (arg=0x7fdb68281700) at pthread_create.c:333
#7 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109

Thread 49 (Thread 0x7fdb68a82700 (LWP 131906)):
#0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225
#1 0x0000564c64302902 in Cond::WaitUntil (this=0x564c708f4a78, mutex=..., when=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:64
#2 0x0000564c64c2fb2c in SafeTimer::timer_thread (this=0x564c708f4a68) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Timer.cc:110
#3 0x0000564c64c310e4 in SafeTimerThread::entry (this=0x564c81b6fe40) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Timer.cc:30
#4 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c81b6fe40) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79
#5 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c81b6fe40) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59
#6 0x00007fdb85e146ba in start_thread (arg=0x7fdb68a82700) at pthread_create.c:333
#7 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109

Thread 48 (Thread 0x7fdb69283700 (LWP 131905)):
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1 0x0000564c64302818 in Cond::Wait (this=0x564c708f48e8, mutex=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:48
#2 0x0000564c64c34d40 in Finisher::finisher_thread_entry (this=0x564c708f4870) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Finisher.cc:101
#3 0x0000564c64314d62 in Finisher::FinisherThread::entry (this=0x564c708f49c8) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Finisher.h:65
#4 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c708f49c8) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79
#5 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c708f49c8) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59
#6 0x00007fdb85e146ba in start_thread (arg=0x7fdb69283700) at pthread_create.c:333
#7 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109

Thread 47 (Thread 0x7fdb69a84700 (LWP 131904)):
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1 0x0000564c64302818 in Cond::Wait (this=0x564c708f4e38, mutex=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:48
#2 0x0000564c64c34d40 in Finisher::finisher_thread_entry (this=0x564c708f4dc0) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Finisher.cc:101
#3 0x0000564c64314d62 in Finisher::FinisherThread::entry (this=0x564c708f4f18) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Finisher.h:65
#4 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c708f4f18) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79
#5 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c708f4f18) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59
#6 0x00007fdb85e146ba in start_thread (arg=0x7fdb69a84700) at pthread_create.c:333
#7 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109

Thread 46 (Thread 0x7fdb6a285700 (LWP 131903)):
#0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225
#1 0x0000564c64302902 in Cond::WaitUntil (this=0x564c708f3340, mutex=..., when=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:64
#2 0x0000564c64302980 in Cond::WaitInterval (this=0x564c708f3340, mutex=..., interval=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:73
#3 0x0000564c642bf081 in OSD::heartbeat_entry (this=0x564c708f2000) at /raid/src/ceph-deb/ceph-12.2.4/src/osd/OSD.cc:5087
#4 0x0000564c64319f84 in OSD::T_Heartbeat::entry (this=0x564c708f33f0) at /raid/src/ceph-deb/ceph-12.2.4/src/osd/OSD.h:1536
#5 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c708f33f0) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79
#6 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c708f33f0) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59
#7 0x00007fdb85e146ba in start_thread (arg=0x7fdb6a285700) at pthread_create.c:333
#8 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109

Thread 45 (Thread 0x7fdb6aa86700 (LWP 131902)):
#0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225
---Type <return> to continue, or q <return> to quit---
#1 0x0000564c64302902 in Cond::WaitUntil (this=0x564c708f3050, mutex=..., when=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:64
#2 0x0000564c64302980 in Cond::WaitInterval (this=0x564c708f3050, mutex=..., interval=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:73
#3 0x0000564c64c3cfd1 in ThreadPool::worker (this=0x564c708f2f70, wt=0x564c81b6fd20) at /raid/src/ceph-deb/ceph-12.2.4/src/common/WorkQueue.cc:143
#4 0x0000564c64c413eb in ThreadPool::WorkThread::entry (this=0x564c81b6fd20) at /raid/src/ceph-deb/ceph-12.2.4/src/common/WorkQueue.h:448
#5 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c81b6fd20) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79
#6 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c81b6fd20) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59
#7 0x00007fdb85e146ba in start_thread (arg=0x7fdb6aa86700) at pthread_create.c:333
#8 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109

Thread 44 (Thread 0x7fdb6b287700 (LWP 131901)):
#0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225
#1 0x0000564c64302902 in Cond::WaitUntil (this=0x564c708f2e38, mutex=..., when=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:64
#2 0x0000564c64302980 in Cond::WaitInterval (this=0x564c708f2e38, mutex=..., interval=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:73
#3 0x0000564c64c3cfd1 in ThreadPool::worker (this=0x564c708f2d58, wt=0x564c81b6fcc0) at /raid/src/ceph-deb/ceph-12.2.4/src/common/WorkQueue.cc:143
#4 0x0000564c64c413eb in ThreadPool::WorkThread::entry (this=0x564c81b6fcc0) at /raid/src/ceph-deb/ceph-12.2.4/src/common/WorkQueue.h:448
#5 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c81b6fcc0) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79
#6 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c81b6fcc0) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59
#7 0x00007fdb85e146ba in start_thread (arg=0x7fdb6b287700) at pthread_create.c:333
#8 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109

Thread 43 (Thread 0x7fdb6ba88700 (LWP 131900)):
#0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225
#1 0x0000564c64302902 in Cond::WaitUntil (this=0x564c70481570, mutex=..., when=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:64
#2 0x0000564c64302980 in Cond::WaitInterval (this=0x564c70481570, mutex=..., interval=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:73
#3 0x0000564c642f30e2 in OSD::ShardedOpWQ::_process (this=0x564c708f3620, thread_index=4, hb=0x564c83722140) at /raid/src/ceph-deb/ceph-12.2.4/src/osd/OSD.cc:10370
#4 0x0000564c64c3f94d in ShardedThreadPool::shardedthreadpool_worker (this=0x564c708f2bd0, thread_index=4)
at /raid/src/ceph-deb/ceph-12.2.4/src/common/WorkQueue.cc:339
#5 0x0000564c64c41461 in ShardedThreadPool::WorkThreadSharded::entry (this=0x564c80538a00) at /raid/src/ceph-deb/ceph-12.2.4/src/common/WorkQueue.h:689
#6 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c80538a00) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79
#7 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c80538a00) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59
#8 0x00007fdb85e146ba in start_thread (arg=0x7fdb6ba88700) at pthread_create.c:333
#9 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109

Thread 42 (Thread 0x7fdb6c289700 (LWP 131899)):
#0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225
#1 0x0000564c64302902 in Cond::WaitUntil (this=0x564c704813f0, mutex=..., when=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:64
#2 0x0000564c64302980 in Cond::WaitInterval (this=0x564c704813f0, mutex=..., interval=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:73
#3 0x0000564c642f30e2 in OSD::ShardedOpWQ::_process (this=0x564c708f3620, thread_index=3, hb=0x564c83722050) at /raid/src/ceph-deb/ceph-12.2.4/src/osd/OSD.cc:10370
#4 0x0000564c64c3f94d in ShardedThreadPool::shardedthreadpool_worker (this=0x564c708f2bd0, thread_index=3)
at /raid/src/ceph-deb/ceph-12.2.4/src/common/WorkQueue.cc:339
#5 0x0000564c64c41461 in ShardedThreadPool::WorkThreadSharded::entry (this=0x564c7fc2f600) at /raid/src/ceph-deb/ceph-12.2.4/src/common/WorkQueue.h:689
#6 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c7fc2f600) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79
#7 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c7fc2f600) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59
#8 0x00007fdb85e146ba in start_thread (arg=0x7fdb6c289700) at pthread_create.c:333
#9 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109

---Type <return> to continue, or q <return> to quit---
Thread 41 (Thread 0x7fdb6ca8a700 (LWP 131898)):
#0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225
#1 0x0000564c64302902 in Cond::WaitUntil (this=0x564c70481270, mutex=..., when=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:64
#2 0x0000564c64302980 in Cond::WaitInterval (this=0x564c70481270, mutex=..., interval=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:73
#3 0x0000564c642f30e2 in OSD::ShardedOpWQ::_process (this=0x564c708f3620, thread_index=2, hb=0x564c837221e0) at /raid/src/ceph-deb/ceph-12.2.4/src/osd/OSD.cc:10370
#4 0x0000564c64c3f94d in ShardedThreadPool::shardedthreadpool_worker (this=0x564c708f2bd0, thread_index=2)
at /raid/src/ceph-deb/ceph-12.2.4/src/common/WorkQueue.cc:339
#5 0x0000564c64c41461 in ShardedThreadPool::WorkThreadSharded::entry (this=0x564c804d3ec0) at /raid/src/ceph-deb/ceph-12.2.4/src/common/WorkQueue.h:689
#6 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c804d3ec0) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79
#7 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c804d3ec0) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59
#8 0x00007fdb85e146ba in start_thread (arg=0x7fdb6ca8a700) at pthread_create.c:333
#9 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109

Thread 40 (Thread 0x7fdb6d28b700 (LWP 131897)):
#0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225
#1 0x0000564c64302902 in Cond::WaitUntil (this=0x564c704810f0, mutex=..., when=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:64
#2 0x0000564c64302980 in Cond::WaitInterval (this=0x564c704810f0, mutex=..., interval=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:73
#3 0x0000564c642f30e2 in OSD::ShardedOpWQ::_process (this=0x564c708f3620, thread_index=1, hb=0x564c837220a0) at /raid/src/ceph-deb/ceph-12.2.4/src/osd/OSD.cc:10370
#4 0x0000564c64c3f94d in ShardedThreadPool::shardedthreadpool_worker (this=0x564c708f2bd0, thread_index=1)
at /raid/src/ceph-deb/ceph-12.2.4/src/common/WorkQueue.cc:339
#5 0x0000564c64c41461 in ShardedThreadPool::WorkThreadSharded::entry (this=0x564c7e578480) at /raid/src/ceph-deb/ceph-12.2.4/src/common/WorkQueue.h:689
#6 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c7e578480) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79
#7 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c7e578480) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59
#8 0x00007fdb85e146ba in start_thread (arg=0x7fdb6d28b700) at pthread_create.c:333
#9 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109

Thread 39 (Thread 0x7fdb6da8c700 (LWP 131896)):
#0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225
#1 0x0000564c64302902 in Cond::WaitUntil (this=0x564c70480f70, mutex=..., when=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:64
#2 0x0000564c64302980 in Cond::WaitInterval (this=0x564c70480f70, mutex=..., interval=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:73
#3 0x0000564c642f30e2 in OSD::ShardedOpWQ::_process (this=0x564c708f3620, thread_index=0, hb=0x564c837220f0) at /raid/src/ceph-deb/ceph-12.2.4/src/osd/OSD.cc:10370
#4 0x0000564c64c3f94d in ShardedThreadPool::shardedthreadpool_worker (this=0x564c708f2bd0, thread_index=0)
at /raid/src/ceph-deb/ceph-12.2.4/src/common/WorkQueue.cc:339
#5 0x0000564c64c41461 in ShardedThreadPool::WorkThreadSharded::entry (this=0x564c826c5780) at /raid/src/ceph-deb/ceph-12.2.4/src/common/WorkQueue.h:689
#6 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c826c5780) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79
#7 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c826c5780) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59
#8 0x00007fdb85e146ba in start_thread (arg=0x7fdb6da8c700) at pthread_create.c:333
#9 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109

Thread 38 (Thread 0x7fdb6e28d700 (LWP 131895)):
#0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225
#1 0x0000564c64302902 in Cond::WaitUntil (this=0x564c708f2a98, mutex=..., when=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:64
#2 0x0000564c64302980 in Cond::WaitInterval (this=0x564c708f2a98, mutex=..., interval=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:73
#3 0x0000564c64c3cfd1 in ThreadPool::worker (this=0x564c708f29b8, wt=0x564c81b6fc60) at /raid/src/ceph-deb/ceph-12.2.4/src/common/WorkQueue.cc:143
#4 0x0000564c64c413eb in ThreadPool::WorkThread::entry (this=0x564c81b6fc60) at /raid/src/ceph-deb/ceph-12.2.4/src/common/WorkQueue.h:448
#5 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c81b6fc60) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79
---Type <return> to continue, or q <return> to quit---
#6 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c81b6fc60) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59
#7 0x00007fdb85e146ba in start_thread (arg=0x7fdb6e28d700) at pthread_create.c:333
#8 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109

Thread 37 (Thread 0x7fdb6ea8e700 (LWP 131894)):
#0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225
#1 0x0000564c64302902 in Cond::WaitUntil (this=0x564c708f2a98, mutex=..., when=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:64
#2 0x0000564c64302980 in Cond::WaitInterval (this=0x564c708f2a98, mutex=..., interval=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:73
#3 0x0000564c64c3cfd1 in ThreadPool::worker (this=0x564c708f29b8, wt=0x564c81b6fc30) at /raid/src/ceph-deb/ceph-12.2.4/src/common/WorkQueue.cc:143
#4 0x0000564c64c413eb in ThreadPool::WorkThread::entry (this=0x564c81b6fc30) at /raid/src/ceph-deb/ceph-12.2.4/src/common/WorkQueue.h:448
#5 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c81b6fc30) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79
#6 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c81b6fc30) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59
#7 0x00007fdb85e146ba in start_thread (arg=0x7fdb6ea8e700) at pthread_create.c:333
#8 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109

Thread 36 (Thread 0x7fdb6f28f700 (LWP 131893)):
#0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225
#1 0x0000564c64302902 in Cond::WaitUntil (this=0x564c708f2480, mutex=..., when=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:64
#2 0x0000564c64c2fb2c in SafeTimer::timer_thread (this=0x564c708f2470) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Timer.cc:110
#3 0x0000564c64c310e4 in SafeTimerThread::entry (this=0x564c81b6fba0) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Timer.cc:30
#4 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c81b6fba0) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79
#5 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c81b6fba0) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59
#6 0x00007fdb85e146ba in start_thread (arg=0x7fdb6f28f700) at pthread_create.c:333
#7 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109

Thread 35 (Thread 0x7fdb6fa90700 (LWP 131892)):
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1 0x0000564c64302818 in Cond::Wait (this=0x7fff606e6690, mutex=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:48
#2 0x0000564c64c34d40 in Finisher::finisher_thread_entry (this=0x7fff606e6618) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Finisher.cc:101
#3 0x0000564c64314d62 in Finisher::FinisherThread::entry (this=0x7fff606e6770) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Finisher.h:65
#4 0x0000564c64e054df in Thread::entry_wrapper (this=0x7fff606e6770) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79
#5 0x0000564c64e05414 in Thread::_entry_func (arg=0x7fff606e6770) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59
#6 0x00007fdb85e146ba in start_thread (arg=0x7fdb6fa90700) at pthread_create.c:333
#7 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109

Thread 34 (Thread 0x7fdb70291700 (LWP 131891)):
#0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225
#1 0x0000564c64302902 in Cond::WaitUntil (this=0x7fff606e6560, mutex=..., when=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:64
#2 0x0000564c64c2fb2c in SafeTimer::timer_thread (this=0x7fff606e6550) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Timer.cc:110
#3 0x0000564c64c310e4 in SafeTimerThread::entry (this=0x564c81b6fa80) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Timer.cc:30
#4 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c81b6fa80) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79
#5 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c81b6fa80) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59
#6 0x00007fdb85e146ba in start_thread (arg=0x7fdb70291700) at pthread_create.c:333
#7 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109

Thread 33 (Thread 0x7fdb70a92700 (LWP 131890)):
---Type <return> to continue, or q <return> to quit---
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1 0x0000564c64302818 in Cond::Wait (this=0x564c707453c8, mutex=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:48
#2 0x0000564c650362df in DispatchQueue::run_local_delivery (this=0x564c70745180) at /raid/src/ceph-deb/ceph-12.2.4/src/msg/DispatchQueue.cc:113
#3 0x0000564c64d2886c in DispatchQueue::LocalDeliveryThread::entry (this=0x564c70745428) at /raid/src/ceph-deb/ceph-12.2.4/src/msg/DispatchQueue.h:115
#4 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c70745428) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79
#5 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c70745428) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59
#6 0x00007fdb85e146ba in start_thread (arg=0x7fdb70a92700) at pthread_create.c:333
#7 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109

Thread 32 (Thread 0x7fdb71293700 (LWP 131889)):
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1 0x0000564c64302818 in Cond::Wait (this=0x564c70745200, mutex=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:48
#2 0x0000564c65036cb3 in DispatchQueue::entry (this=0x564c70745180) at /raid/src/ceph-deb/ceph-12.2.4/src/msg/DispatchQueue.cc:208
#3 0x0000564c64d2880a in DispatchQueue::DispatchThread::entry (this=0x564c70745328) at /raid/src/ceph-deb/ceph-12.2.4/src/msg/DispatchQueue.h:101
#4 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c70745328) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79
#5 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c70745328) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59
#6 0x00007fdb85e146ba in start_thread (arg=0x7fdb71293700) at pthread_create.c:333
#7 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109

Thread 31 (Thread 0x7fdb71a94700 (LWP 131888)):
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1 0x0000564c64302818 in Cond::Wait (this=0x564c707443c8, mutex=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:48
#2 0x0000564c650362df in DispatchQueue::run_local_delivery (this=0x564c70744180) at /raid/src/ceph-deb/ceph-12.2.4/src/msg/DispatchQueue.cc:113
#3 0x0000564c64d2886c in DispatchQueue::LocalDeliveryThread::entry (this=0x564c70744428) at /raid/src/ceph-deb/ceph-12.2.4/src/msg/DispatchQueue.h:115
#4 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c70744428) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79
#5 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c70744428) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59
#6 0x00007fdb85e146ba in start_thread (arg=0x7fdb71a94700) at pthread_create.c:333
#7 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109

Thread 30 (Thread 0x7fdb72295700 (LWP 131887)):
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1 0x0000564c64302818 in Cond::Wait (this=0x564c70744200, mutex=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:48
#2 0x0000564c65036cb3 in DispatchQueue::entry (this=0x564c70744180) at /raid/src/ceph-deb/ceph-12.2.4/src/msg/DispatchQueue.cc:208
#3 0x0000564c64d2880a in DispatchQueue::DispatchThread::entry (this=0x564c70744328) at /raid/src/ceph-deb/ceph-12.2.4/src/msg/DispatchQueue.h:101
#4 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c70744328) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79
#5 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c70744328) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59
#6 0x00007fdb85e146ba in start_thread (arg=0x7fdb72295700) at pthread_create.c:333
#7 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109

Thread 29 (Thread 0x7fdb72a96700 (LWP 131886)):
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1 0x0000564c64302818 in Cond::Wait (this=0x564c70744bc8, mutex=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:48
#2 0x0000564c650362df in DispatchQueue::run_local_delivery (this=0x564c70744980) at /raid/src/ceph-deb/ceph-12.2.4/src/msg/DispatchQueue.cc:113
#3 0x0000564c64d2886c in DispatchQueue::LocalDeliveryThread::entry (this=0x564c70744c28) at /raid/src/ceph-deb/ceph-12.2.4/src/msg/DispatchQueue.h:115
#4 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c70744c28) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79
#5 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c70744c28) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59
---Type <return> to continue, or q <return> to quit---
#6 0x00007fdb85e146ba in start_thread (arg=0x7fdb72a96700) at pthread_create.c:333
#7 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109

Thread 28 (Thread 0x7fdb73297700 (LWP 131885)):
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1 0x0000564c64302818 in Cond::Wait (this=0x564c70744a00, mutex=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:48
#2 0x0000564c65036cb3 in DispatchQueue::entry (this=0x564c70744980) at /raid/src/ceph-deb/ceph-12.2.4/src/msg/DispatchQueue.cc:208
#3 0x0000564c64d2880a in DispatchQueue::DispatchThread::entry (this=0x564c70744b28) at /raid/src/ceph-deb/ceph-12.2.4/src/msg/DispatchQueue.h:101
#4 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c70744b28) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79
#5 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c70744b28) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59
#6 0x00007fdb85e146ba in start_thread (arg=0x7fdb73297700) at pthread_create.c:333
#7 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109

Thread 27 (Thread 0x7fdb73a98700 (LWP 131884)):
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1 0x0000564c64302818 in Cond::Wait (this=0x564c707433c8, mutex=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:48
#2 0x0000564c650362df in DispatchQueue::run_local_delivery (this=0x564c70743180) at /raid/src/ceph-deb/ceph-12.2.4/src/msg/DispatchQueue.cc:113
#3 0x0000564c64d2886c in DispatchQueue::LocalDeliveryThread::entry (this=0x564c70743428) at /raid/src/ceph-deb/ceph-12.2.4/src/msg/DispatchQueue.h:115
#4 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c70743428) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79
#5 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c70743428) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59
#6 0x00007fdb85e146ba in start_thread (arg=0x7fdb73a98700) at pthread_create.c:333
#7 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109

Thread 26 (Thread 0x7fdb76317700 (LWP 131883)):
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1 0x0000564c64302818 in Cond::Wait (this=0x564c70743200, mutex=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:48
#2 0x0000564c65036cb3 in DispatchQueue::entry (this=0x564c70743180) at /raid/src/ceph-deb/ceph-12.2.4/src/msg/DispatchQueue.cc:208
#3 0x0000564c64d2880a in DispatchQueue::DispatchThread::entry (this=0x564c70743328) at /raid/src/ceph-deb/ceph-12.2.4/src/msg/DispatchQueue.h:101
#4 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c70743328) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79
#5 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c70743328) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59
#6 0x00007fdb85e146ba in start_thread (arg=0x7fdb76317700) at pthread_create.c:333
#7 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109

Thread 25 (Thread 0x7fdb76b18700 (LWP 131882)):
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1 0x0000564c64302818 in Cond::Wait (this=0x564c70743bc8, mutex=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:48
#2 0x0000564c650362df in DispatchQueue::run_local_delivery (this=0x564c70743980) at /raid/src/ceph-deb/ceph-12.2.4/src/msg/DispatchQueue.cc:113
#3 0x0000564c64d2886c in DispatchQueue::LocalDeliveryThread::entry (this=0x564c70743c28) at /raid/src/ceph-deb/ceph-12.2.4/src/msg/DispatchQueue.h:115
#4 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c70743c28) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79
#5 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c70743c28) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59
#6 0x00007fdb85e146ba in start_thread (arg=0x7fdb76b18700) at pthread_create.c:333
#7 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109

Thread 24 (Thread 0x7fdb77319700 (LWP 131881)):
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1 0x0000564c64302818 in Cond::Wait (this=0x564c70743a00, mutex=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:48
---Type <return> to continue, or q <return> to quit---
#2 0x0000564c65036cb3 in DispatchQueue::entry (this=0x564c70743980) at /raid/src/ceph-deb/ceph-12.2.4/src/msg/DispatchQueue.cc:208
#3 0x0000564c64d2880a in DispatchQueue::DispatchThread::entry (this=0x564c70743b28) at /raid/src/ceph-deb/ceph-12.2.4/src/msg/DispatchQueue.h:101
#4 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c70743b28) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79
#5 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c70743b28) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59
#6 0x00007fdb85e146ba in start_thread (arg=0x7fdb77319700) at pthread_create.c:333
#7 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109

Thread 23 (Thread 0x7fdb77b1a700 (LWP 131880)):
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1 0x0000564c64302818 in Cond::Wait (this=0x564c70742bc8, mutex=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:48
#2 0x0000564c650362df in DispatchQueue::run_local_delivery (this=0x564c70742980) at /raid/src/ceph-deb/ceph-12.2.4/src/msg/DispatchQueue.cc:113
#3 0x0000564c64d2886c in DispatchQueue::LocalDeliveryThread::entry (this=0x564c70742c28) at /raid/src/ceph-deb/ceph-12.2.4/src/msg/DispatchQueue.h:115
#4 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c70742c28) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79
#5 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c70742c28) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59
#6 0x00007fdb85e146ba in start_thread (arg=0x7fdb77b1a700) at pthread_create.c:333
#7 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109

Thread 22 (Thread 0x7fdb7831b700 (LWP 131879)):
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1 0x0000564c64302818 in Cond::Wait (this=0x564c70742a00, mutex=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:48
#2 0x0000564c65036cb3 in DispatchQueue::entry (this=0x564c70742980) at /raid/src/ceph-deb/ceph-12.2.4/src/msg/DispatchQueue.cc:208
#3 0x0000564c64d2880a in DispatchQueue::DispatchThread::entry (this=0x564c70742b28) at /raid/src/ceph-deb/ceph-12.2.4/src/msg/DispatchQueue.h:101
#4 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c70742b28) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79
#5 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c70742b28) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59
#6 0x00007fdb85e146ba in start_thread (arg=0x7fdb7831b700) at pthread_create.c:333
#7 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109

Thread 21 (Thread 0x7fdb78b1c700 (LWP 131878)):
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1 0x0000564c64302818 in Cond::Wait (this=0x564c707423c8, mutex=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:48
#2 0x0000564c650362df in DispatchQueue::run_local_delivery (this=0x564c70742180) at /raid/src/ceph-deb/ceph-12.2.4/src/msg/DispatchQueue.cc:113
#3 0x0000564c64d2886c in DispatchQueue::LocalDeliveryThread::entry (this=0x564c70742428) at /raid/src/ceph-deb/ceph-12.2.4/src/msg/DispatchQueue.h:115
#4 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c70742428) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79
#5 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c70742428) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59
#6 0x00007fdb85e146ba in start_thread (arg=0x7fdb78b1c700) at pthread_create.c:333
#7 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109

Thread 20 (Thread 0x7fdb7931d700 (LWP 131877)):
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1 0x0000564c64302818 in Cond::Wait (this=0x564c70742200, mutex=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:48
#2 0x0000564c65036cb3 in DispatchQueue::entry (this=0x564c70742180) at /raid/src/ceph-deb/ceph-12.2.4/src/msg/DispatchQueue.cc:208
#3 0x0000564c64d2880a in DispatchQueue::DispatchThread::entry (this=0x564c70742328) at /raid/src/ceph-deb/ceph-12.2.4/src/msg/DispatchQueue.h:101
#4 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c70742328) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79
#5 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c70742328) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59
#6 0x00007fdb85e146ba in start_thread (arg=0x7fdb7931d700) at pthread_create.c:333
#7 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109
---Type <return> to continue, or q <return> to quit---

Thread 19 (Thread 0x7fdb7baa8700 (LWP 131832)):
#0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225
#1 0x0000564c64302902 in Cond::WaitUntil (this=0x564c70730bf8, mutex=..., when=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:64
#2 0x0000564c64302980 in Cond::WaitInterval (this=0x564c70730bf8, mutex=..., interval=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:73
#3 0x0000564c649fb112 in BlueStore::MempoolThread::entry (this=0x564c70730bc8) at /raid/src/ceph-deb/ceph-12.2.4/src/os/bluestore/BlueStore.cc:3374
#4 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c70730bc8) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79
#5 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c70730bc8) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59
#6 0x00007fdb85e146ba in start_thread (arg=0x7fdb7baa8700) at pthread_create.c:333
#7 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109

Thread 18 (Thread 0x7fdb75a9c700 (LWP 131830)):
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1 0x00007fdb8572091c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from target:/usr/lib/x86_64-linux-gnu/libstdc++.so.6
#2 0x0000564c64a3b3aa in BlueStore::_kv_finalize_thread (this=0x564c70730000) at /raid/src/ceph-deb/ceph-12.2.4/src/os/bluestore/BlueStore.cc:8706
#3 0x0000564c64a718a0 in BlueStore::KVFinalizeThread::entry (this=0x564c70730898) at /raid/src/ceph-deb/ceph-12.2.4/src/os/bluestore/BlueStore.h:1790
#4 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c70730898) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79
#5 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c70730898) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59
#6 0x00007fdb85e146ba in start_thread (arg=0x7fdb75a9c700) at pthread_create.c:333
#7 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109

Thread 17 (Thread 0x7fdb7529b700 (LWP 131829)):
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1 0x00007fdb8572091c in std::condition_variable::wait(std::unique_lock<std::mutex>&) () from target:/usr/lib/x86_64-linux-gnu/libstdc++.so.6
#2 0x0000564c64a380cc in BlueStore::_kv_sync_thread (this=0x564c70730000) at /raid/src/ceph-deb/ceph-12.2.4/src/os/bluestore/BlueStore.cc:8457
#3 0x0000564c64a7183e in BlueStore::KVSyncThread::entry (this=0x564c70730678) at /raid/src/ceph-deb/ceph-12.2.4/src/os/bluestore/BlueStore.h:1782
#4 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c70730678) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79
#5 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c70730678) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59
#6 0x00007fdb85e146ba in start_thread (arg=0x7fdb7529b700) at pthread_create.c:333
#7 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109

Thread 16 (Thread 0x7fdb74a9a700 (LWP 131828)):
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1 0x0000564c64302818 in Cond::Wait (this=0x564c704718d8, mutex=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:48
#2 0x0000564c64c34d40 in Finisher::finisher_thread_entry (this=0x564c70471860) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Finisher.cc:101
#3 0x0000564c64314d62 in Finisher::FinisherThread::entry (this=0x564c704719b8) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Finisher.h:65
#4 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c704719b8) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79
#5 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c704719b8) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59
#6 0x00007fdb85e146ba in start_thread (arg=0x7fdb74a9a700) at pthread_create.c:333
#7 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109

Thread 15 (Thread 0x7fdb74299700 (LWP 131827)):
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1 0x0000564c64302818 in Cond::Wait (this=0x564c70730548, mutex=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:48
#2 0x0000564c64c34d40 in Finisher::finisher_thread_entry (this=0x564c707304d0) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Finisher.cc:101
#3 0x0000564c64314d62 in Finisher::FinisherThread::entry (this=0x564c70730628) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Finisher.h:65
---Type <return> to continue, or q <return> to quit---
#4 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c70730628) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79
#5 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c70730628) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59
#6 0x00007fdb85e146ba in start_thread (arg=0x7fdb74299700) at pthread_create.c:333
#7 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109

Thread 14 (Thread 0x7fdb7c2a9700 (LWP 131528)):
#0 0x00007fdb86e9e64a in ?? () from target:/lib/x86_64-linux-gnu/libaio.so.1
#1 0x0000564c64bcc9cb in aio_queue_t::get_next_completed (this=0x564c70499378, timeout_ms=250, paio=0x7fdb7c2a7440, max=16)
at /raid/src/ceph-deb/ceph-12.2.4/src/os/bluestore/aio.cc:78
#2 0x0000564c64baaa1e in KernelDevice::_aio_thread (this=0x564c70499200) at /raid/src/ceph-deb/ceph-12.2.4/src/os/bluestore/KernelDevice.cc:350
#3 0x0000564c64bb2062 in KernelDevice::AioCompletionThread::entry (this=0x564c704993a0) at /raid/src/ceph-deb/ceph-12.2.4/src/os/bluestore/KernelDevice.h:49
#4 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c704993a0) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79
#5 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c704993a0) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59
#6 0x00007fdb85e146ba in start_thread (arg=0x7fdb7c2a9700) at pthread_create.c:333
#7 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109

Thread 13 (Thread 0x7fdb7caaa700 (LWP 131526)):
#0 0x00007fdb86e9e64a in ?? () from target:/lib/x86_64-linux-gnu/libaio.so.1
#1 0x0000564c64bcc9cb in aio_queue_t::get_next_completed (this=0x564c70499138, timeout_ms=250, paio=0x7fdb7caa8440, max=16)
at /raid/src/ceph-deb/ceph-12.2.4/src/os/bluestore/aio.cc:78
#2 0x0000564c64baaa1e in KernelDevice::_aio_thread (this=0x564c70498fc0) at /raid/src/ceph-deb/ceph-12.2.4/src/os/bluestore/KernelDevice.cc:350
#3 0x0000564c64bb2062 in KernelDevice::AioCompletionThread::entry (this=0x564c70499160) at /raid/src/ceph-deb/ceph-12.2.4/src/os/bluestore/KernelDevice.h:49
#4 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c70499160) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79
#5 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c70499160) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59
#6 0x00007fdb85e146ba in start_thread (arg=0x7fdb7caaa700) at pthread_create.c:333
#7 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109

Thread 12 (Thread 0x7fdb7d2ab700 (LWP 131524)):
#0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225
#1 0x0000564c64302902 in Cond::WaitUntil (this=0x564c708f4d00, mutex=..., when=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:64
#2 0x0000564c64c2fb2c in SafeTimer::timer_thread (this=0x564c708f4cf0) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Timer.cc:110
#3 0x0000564c64c310e4 in SafeTimerThread::entry (this=0x564c70779e30) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Timer.cc:30
#4 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c70779e30) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79
#5 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c70779e30) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59
#6 0x00007fdb85e146ba in start_thread (arg=0x7fdb7d2ab700) at pthread_create.c:333
#7 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109

Thread 11 (Thread 0x7fdb7daac700 (LWP 131523)):
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1 0x0000564c64302818 in Cond::Wait (this=0x564c708f4bb8, mutex=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:48
#2 0x0000564c64c2fada in SafeTimer::timer_thread (this=0x564c708f4ba8) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Timer.cc:108
#3 0x0000564c64c310e4 in SafeTimerThread::entry (this=0x564c70779e00) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Timer.cc:30
#4 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c70779e00) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79
#5 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c70779e00) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59
#6 0x00007fdb85e146ba in start_thread (arg=0x7fdb7daac700) at pthread_create.c:333
#7 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109
---Type <return> to continue, or q <return> to quit---

Thread 10 (Thread 0x7fdb7e2ad700 (LWP 131522)):
#0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225
#1 0x0000564c64302902 in Cond::WaitUntil (this=0x564c708f21d0, mutex=..., when=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:64
#2 0x0000564c64c2fb2c in SafeTimer::timer_thread (this=0x564c708f21c0) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Timer.cc:110
#3 0x0000564c64c310e4 in SafeTimerThread::entry (this=0x564c70779dd0) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Timer.cc:30
#4 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c70779dd0) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79
#5 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c70779dd0) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59
#6 0x00007fdb85e146ba in start_thread (arg=0x7fdb7e2ad700) at pthread_create.c:333
#7 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109

Thread 9 (Thread 0x7fdb7eaae700 (LWP 131521)):
#0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225
#1 0x0000564c64302902 in Cond::WaitUntil (this=0x564c708f2098, mutex=..., when=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:64
#2 0x0000564c64c2fb2c in SafeTimer::timer_thread (this=0x564c708f2088) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Timer.cc:110
#3 0x0000564c64c310e4 in SafeTimerThread::entry (this=0x564c70779d40) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Timer.cc:30
#4 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c70779d40) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79
#5 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c70779d40) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59
#6 0x00007fdb85e146ba in start_thread (arg=0x7fdb7eaae700) at pthread_create.c:333
#7 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109

Thread 8 (Thread 0x7fdb7f2af700 (LWP 131519)):
#0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225
#1 0x0000564c64287ec8 in __gthread_cond_timedwait (__cond=0x564c70843a80, __mutex=0x564c70843a58, __abs_timeout=0x7fdb7f2ad680)
at /usr/include/x86_64-linux-gnu/c++/5/bits/gthr-default.h:871
#2 0x0000564c6439ece2 in std::condition_variable::__wait_until_impl<std::chrono::duration<unsigned long, std::ratio<1l, 1000000000l> > > (this=0x564c70843a80,
__lock=..., __atime=...) at /usr/include/c++/5/condition_variable:165
#3 0x0000564c6437e9f2 in std::condition_variable::wait_until<ceph::time_detail::mono_clock, std::chrono::duration<unsigned long, std::ratio<1l, 1000000000l> > > (
this=0x564c70843a80, __lock=..., __atime=...) at /usr/include/c++/5/condition_variable:118
#4 0x0000564c6435a5c9 in ceph::timer_detail::timer<ceph::time_detail::mono_clock>::timer_thread (this=0x564c70843a18)
at /raid/src/ceph-deb/ceph-12.2.4/src/common/ceph_timer.h:144
#5 0x0000564c643f133b in std::_Mem_fn_base<void (ceph::timer_detail::timer<ceph::time_detail::mono_clock>::*)(), true>::operator()<, void>(ceph::timer_detail::timer<ceph::time_detail::mono_clock>*) const (this=0x564c707237b0, __object=0x564c70843a18) at /usr/include/c++/5/functional:600
#6 0x0000564c643eeff9 in std::_Bind_simple<std::_Mem_fn<void (ceph::timer_detail::timer<ceph::time_detail::mono_clock>::*)()> (ceph::timer_detail::timer<ceph::time_detail::mono_clock>*)>::_M_invoke<0ul>(std::_Index_tuple<0ul>) (this=0x564c707237a8) at /usr/include/c++/5/functional:1531
#7 0x0000564c643ec86e in std::_Bind_simple<std::_Mem_fn<void (ceph::timer_detail::timer<ceph::time_detail::mono_clock>::*)()> (ceph::timer_detail::timer<ceph::time_detail::mono_clock>*)>::operator()() (this=0x564c707237a8) at /usr/include/c++/5/functional:1520
#8 0x0000564c643e919a in std::thread::_Impl<std::_Bind_simple<std::_Mem_fn<void (ceph::timer_detail::timer<ceph::time_detail::mono_clock>::*)()> (ceph::timer_detail::timer<ceph::time_detail::mono_clock>*)> >::_M_run() (this=0x564c70723790) at /usr/include/c++/5/thread:115
#9 0x00007fdb85725c80 in ?? () from target:/usr/lib/x86_64-linux-gnu/libstdc++.so.6
#10 0x00007fdb85e146ba in start_thread (arg=0x7fdb7f2af700) at pthread_create.c:333
#11 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109

Thread 7 (Thread 0x7fdb802fd700 (LWP 131199)):
#0 0x00007fdb84e7f74d in poll () at ../sysdeps/unix/syscall-template.S:84
#1 0x0000564c64c04b3c in AdminSocket::entry (this=0x564c70490540) at /raid/src/ceph-deb/ceph-12.2.4/src/common/admin_socket.cc:250
---Type <return> to continue, or q <return> to quit---
#2 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c70490540) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79
#3 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c70490540) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59
#4 0x00007fdb85e146ba in start_thread (arg=0x7fdb802fd700) at pthread_create.c:333
#5 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109

Thread 6 (Thread 0x7fdb80afe700 (LWP 131197)):
#0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225
#1 0x0000564c64302902 in Cond::WaitUntil (this=0x564c70488368, mutex=..., when=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:64
#2 0x0000564c64302980 in Cond::WaitInterval (this=0x564c70488368, mutex=..., interval=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:73
#3 0x0000564c64dfbf0c in CephContextServiceThread::entry (this=0x564c704882d0) at /raid/src/ceph-deb/ceph-12.2.4/src/common/ceph_context.cc:135
#4 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c704882d0) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79
#5 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c704882d0) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59
#6 0x00007fdb85e146ba in start_thread (arg=0x7fdb80afe700) at pthread_create.c:333
#7 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109

Thread 5 (Thread 0x7fdb81a9d700 (LWP 131191)):
#0 0x00007fdb84e8ba13 in epoll_wait () at ../sysdeps/unix/syscall-template.S:84
#1 0x0000564c64f9b558 in EpollDriver::event_wait (this=0x564c70778300, fired_events=..., tvp=0x7fdb81a9b3d0)
at /raid/src/ceph-deb/ceph-12.2.4/src/msg/async/EventEpoll.cc:114
#2 0x0000564c64d49a0c in EventCenter::process_events (this=0x564c70498740, timeout_microseconds=30000000, working_dur=0x7fdb81a9b5b0)
at /raid/src/ceph-deb/ceph-12.2.4/src/msg/async/Event.cc:395
#3 0x0000564c64d510a8 in NetworkStack::<lambda()>::operator()(void) const (__closure=0x564c70746d98) at /raid/src/ceph-deb/ceph-12.2.4/src/msg/async/Stack.cc:51
#4 0x0000564c64d52623 in std::_Function_handler<void(), NetworkStack::add_thread(unsigned int)::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...)
at /usr/include/c++/5/functional:1871
#5 0x0000564c64337bda in std::function<void ()>::operator()() const (this=0x564c70746d98) at /usr/include/c++/5/functional:2267
#6 0x0000564c64d5b3d2 in std::_Bind_simple<std::function<void ()> ()>::_M_invoke<>(std::_Index_tuple<>) (this=0x564c70746d98) at /usr/include/c++/5/functional:1531
#7 0x0000564c64d5b328 in std::_Bind_simple<std::function<void ()> ()>::operator()() (this=0x564c70746d98) at /usr/include/c++/5/functional:1520
#8 0x0000564c64d5b2b8 in std::thread::_Impl<std::_Bind_simple<std::function<void ()> ()> >::_M_run() (this=0x564c70746d80) at /usr/include/c++/5/thread:115
#9 0x00007fdb85725c80 in ?? () from target:/usr/lib/x86_64-linux-gnu/libstdc++.so.6
#10 0x00007fdb85e146ba in start_thread (arg=0x7fdb81a9d700) at pthread_create.c:333
#11 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109

Thread 4 (Thread 0x7fdb8229e700 (LWP 131190)):
#0 0x00007fdb84e8ba13 in epoll_wait () at ../sysdeps/unix/syscall-template.S:84
#1 0x0000564c64f9b558 in EpollDriver::event_wait (this=0x564c70778120, fired_events=..., tvp=0x7fdb8229c3d0)
at /raid/src/ceph-deb/ceph-12.2.4/src/msg/async/EventEpoll.cc:114
#2 0x0000564c64d49a0c in EventCenter::process_events (this=0x564c70498bc0, timeout_microseconds=30000000, working_dur=0x7fdb8229c5b0)
at /raid/src/ceph-deb/ceph-12.2.4/src/msg/async/Event.cc:395
#3 0x0000564c64d510a8 in NetworkStack::<lambda()>::operator()(void) const (__closure=0x564c70746d48) at /raid/src/ceph-deb/ceph-12.2.4/src/msg/async/Stack.cc:51
#4 0x0000564c64d52623 in std::_Function_handler<void(), NetworkStack::add_thread(unsigned int)::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...)
at /usr/include/c++/5/functional:1871
#5 0x0000564c64337bda in std::function<void ()>::operator()() const (this=0x564c70746d48) at /usr/include/c++/5/functional:2267
#6 0x0000564c64d5b3d2 in std::_Bind_simple<std::function<void ()> ()>::_M_invoke<>(std::_Index_tuple<>) (this=0x564c70746d48) at /usr/include/c++/5/functional:1531
#7 0x0000564c64d5b328 in std::_Bind_simple<std::function<void ()> ()>::operator()() (this=0x564c70746d48) at /usr/include/c++/5/functional:1520
#8 0x0000564c64d5b2b8 in std::thread::_Impl<std::_Bind_simple<std::function<void ()> ()> >::_M_run() (this=0x564c70746d30) at /usr/include/c++/5/thread:115
#9 0x00007fdb85725c80 in ?? () from target:/usr/lib/x86_64-linux-gnu/libstdc++.so.6
---Type <return> to continue, or q <return> to quit---
#10 0x00007fdb85e146ba in start_thread (arg=0x7fdb8229e700) at pthread_create.c:333
#11 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109

Thread 3 (Thread 0x7fdb82a9f700 (LWP 131189)):
#0 0x00007fdb84e8ba13 in epoll_wait () at ../sysdeps/unix/syscall-template.S:84
#1 0x0000564c64f9b558 in EpollDriver::event_wait (this=0x564c706cc090, fired_events=..., tvp=0x7fdb82a9d3d0)
at /raid/src/ceph-deb/ceph-12.2.4/src/msg/async/EventEpoll.cc:114
#2 0x0000564c64d49a0c in EventCenter::process_events (this=0x564c70498980, timeout_microseconds=30000000, working_dur=0x7fdb82a9d5b0)
at /raid/src/ceph-deb/ceph-12.2.4/src/msg/async/Event.cc:395
#3 0x0000564c64d510a8 in NetworkStack::<lambda()>::operator()(void) const (__closure=0x564c70746cf8) at /raid/src/ceph-deb/ceph-12.2.4/src/msg/async/Stack.cc:51
#4 0x0000564c64d52623 in std::_Function_handler<void(), NetworkStack::add_thread(unsigned int)::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...)
at /usr/include/c++/5/functional:1871
#5 0x0000564c64337bda in std::function<void ()>::operator()() const (this=0x564c70746cf8) at /usr/include/c++/5/functional:2267
#6 0x0000564c64d5b3d2 in std::_Bind_simple<std::function<void ()> ()>::_M_invoke<>(std::_Index_tuple<>) (this=0x564c70746cf8) at /usr/include/c++/5/functional:1531
#7 0x0000564c64d5b328 in std::_Bind_simple<std::function<void ()> ()>::operator()() (this=0x564c70746cf8) at /usr/include/c++/5/functional:1520
#8 0x0000564c64d5b2b8 in std::thread::_Impl<std::_Bind_simple<std::function<void ()> ()> >::_M_run() (this=0x564c70746ce0) at /usr/include/c++/5/thread:115
#9 0x00007fdb85725c80 in ?? () from target:/usr/lib/x86_64-linux-gnu/libstdc++.so.6
#10 0x00007fdb85e146ba in start_thread (arg=0x7fdb82a9f700) at pthread_create.c:333
#11 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109

Thread 2 (Thread 0x7fdb83ae0700 (LWP 131176)):
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1 0x0000564c64c4ddfe in ceph::logging::Log::entry (this=0x564c70490000) at /raid/src/ceph-deb/ceph-12.2.4/src/log/Log.cc:459
#2 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c70490000) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79
#3 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c70490000) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59
#4 0x00007fdb85e146ba in start_thread (arg=0x7fdb83ae0700) at pthread_create.c:333
#5 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109

Thread 1 (Thread 0x7fdb87938e00 (LWP 131171)):
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1 0x0000564c64302818 in Cond::Wait (this=0x564c70742760, mutex=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:48
#2 0x0000564c64d3bb71 in AsyncMessenger::wait (this=0x564c70742000) at /raid/src/ceph-deb/ceph-12.2.4/src/msg/async/AsyncMessenger.cc:481
#3 0x0000564c6427b57e in main (argc=10, argv=0x7fff606e7ab8) at /raid/src/ceph-deb/ceph-12.2.4/src/ceph_osd.cc:656
(2-2/2)