(gdb) info threads Id Target Id Frame * 1 Thread 0x7fdb87938e00 (LWP 131171) "ceph-osd" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 2 Thread 0x7fdb83ae0700 (LWP 131176) "log" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 3 Thread 0x7fdb82a9f700 (LWP 131189) "msgr-worker-0" 0x00007fdb84e8ba13 in epoll_wait () at ../sysdeps/unix/syscall-template.S:84 4 Thread 0x7fdb8229e700 (LWP 131190) "msgr-worker-1" 0x00007fdb84e8ba13 in epoll_wait () at ../sysdeps/unix/syscall-template.S:84 5 Thread 0x7fdb81a9d700 (LWP 131191) "msgr-worker-2" 0x00007fdb84e8ba13 in epoll_wait () at ../sysdeps/unix/syscall-template.S:84 6 Thread 0x7fdb80afe700 (LWP 131197) "service" pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 7 Thread 0x7fdb802fd700 (LWP 131199) "admin_socket" 0x00007fdb84e7f74d in poll () at ../sysdeps/unix/syscall-template.S:84 8 Thread 0x7fdb7f2af700 (LWP 131519) "ceph-osd" pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 9 Thread 0x7fdb7eaae700 (LWP 131521) "safe_timer" pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 10 Thread 0x7fdb7e2ad700 (LWP 131522) "safe_timer" pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 11 Thread 0x7fdb7daac700 (LWP 131523) "safe_timer" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 12 Thread 0x7fdb7d2ab700 (LWP 131524) "safe_timer" pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 13 Thread 0x7fdb7caaa700 (LWP 131526) "bstore_aio" 0x00007fdb86e9e64a in ?? () from target:/lib/x86_64-linux-gnu/libaio.so.1 14 Thread 0x7fdb7c2a9700 (LWP 131528) "bstore_aio" 0x00007fdb86e9e64a in ?? () from target:/lib/x86_64-linux-gnu/libaio.so.1 15 Thread 0x7fdb74299700 (LWP 131827) "dfin" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 16 Thread 0x7fdb74a9a700 (LWP 131828) "finisher" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 17 Thread 0x7fdb7529b700 (LWP 131829) "bstore_kv_sync" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 18 Thread 0x7fdb75a9c700 (LWP 131830) "bstore_kv_final" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 19 Thread 0x7fdb7baa8700 (LWP 131832) "bstore_mempool" pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 20 Thread 0x7fdb7931d700 (LWP 131877) "ms_dispatch" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 21 Thread 0x7fdb78b1c700 (LWP 131878) "ms_local" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 22 Thread 0x7fdb7831b700 (LWP 131879) "ms_dispatch" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 23 Thread 0x7fdb77b1a700 (LWP 131880) "ms_local" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 24 Thread 0x7fdb77319700 (LWP 131881) "ms_dispatch" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 25 Thread 0x7fdb76b18700 (LWP 131882) "ms_local" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 26 Thread 0x7fdb76317700 (LWP 131883) "ms_dispatch" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 27 Thread 0x7fdb73a98700 (LWP 131884) "ms_local" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 28 Thread 0x7fdb73297700 (LWP 131885) "ms_dispatch" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 29 Thread 0x7fdb72a96700 (LWP 131886) "ms_local" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 30 Thread 0x7fdb72295700 (LWP 131887) "ms_dispatch" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 31 Thread 0x7fdb71a94700 (LWP 131888) "ms_local" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 32 Thread 0x7fdb71293700 (LWP 131889) "ms_dispatch" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 33 Thread 0x7fdb70a92700 (LWP 131890) "ms_local" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 34 Thread 0x7fdb70291700 (LWP 131891) "safe_timer" pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 35 Thread 0x7fdb6fa90700 (LWP 131892) "fn_anonymous" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 36 Thread 0x7fdb6f28f700 (LWP 131893) "safe_timer" pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 37 Thread 0x7fdb6ea8e700 (LWP 131894) "tp_peering" pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 38 Thread 0x7fdb6e28d700 (LWP 131895) "tp_peering" pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 39 Thread 0x7fdb6da8c700 (LWP 131896) "tp_osd_tp" pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 40 Thread 0x7fdb6d28b700 (LWP 131897) "tp_osd_tp" pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 41 Thread 0x7fdb6ca8a700 (LWP 131898) "tp_osd_tp" pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 42 Thread 0x7fdb6c289700 (LWP 131899) "tp_osd_tp" pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 43 Thread 0x7fdb6ba88700 (LWP 131900) "tp_osd_tp" pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 44 Thread 0x7fdb6b287700 (LWP 131901) "tp_osd_disk" pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 45 Thread 0x7fdb6aa86700 (LWP 131902) "tp_osd_cmd" pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 46 Thread 0x7fdb6a285700 (LWP 131903) "osd_srv_heartbt" pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 47 Thread 0x7fdb69a84700 (LWP 131904) "fn_anonymous" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 48 Thread 0x7fdb69283700 (LWP 131905) "fn_anonymous" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 49 Thread 0x7fdb68a82700 (LWP 131906) "safe_timer" pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 50 Thread 0x7fdb68281700 (LWP 131907) "safe_timer" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 51 Thread 0x7fdb67a80700 (LWP 131908) "safe_timer" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 52 Thread 0x7fdb6727f700 (LWP 131909) "safe_timer" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 53 Thread 0x7fdb66a7e700 (LWP 131910) "osd_srv_agent" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 54 Thread 0x7fdb6627d700 (LWP 131911) "signal_handler" 0x00007fdb84e7f74d in poll () at ../sysdeps/unix/syscall-template.S:84 (gdb) thread apply all bt Thread 54 (Thread 0x7fdb6627d700 (LWP 131911)): #0 0x00007fdb84e7f74d in poll () at ../sysdeps/unix/syscall-template.S:84 #1 0x0000564c64bd4882 in SignalHandler::entry (this=0x564c8371b800) at /raid/src/ceph-deb/ceph-12.2.4/src/global/signal_handler.cc:281 #2 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c8371b800) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79 #3 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c8371b800) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59 #4 0x00007fdb85e146ba in start_thread (arg=0x7fdb6627d700) at pthread_create.c:333 #5 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 53 (Thread 0x7fdb66a7e700 (LWP 131910)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x0000564c64302818 in Cond::Wait (this=0x564c708f4600, mutex=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:48 #2 0x0000564c6428dc39 in OSDService::agent_entry (this=0x564c708f3dc8) at /raid/src/ceph-deb/ceph-12.2.4/src/osd/OSD.cc:610 #3 0x0000564c643181ac in OSDService::AgentThread::entry (this=0x564c708f46c0) at /raid/src/ceph-deb/ceph-12.2.4/src/osd/OSD.h:665 #4 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c708f46c0) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79 #5 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c708f46c0) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59 #6 0x00007fdb85e146ba in start_thread (arg=0x7fdb66a7e700) at pthread_create.c:333 #7 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 52 (Thread 0x7fdb6727f700 (LWP 131909)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x0000564c64302818 in Cond::Wait (this=0x564c708f5460, mutex=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:48 #2 0x0000564c64c2fada in SafeTimer::timer_thread (this=0x564c708f5450) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Timer.cc:108 #3 0x0000564c64c310e4 in SafeTimerThread::entry (this=0x564c81b6fed0) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Timer.cc:30 #4 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c81b6fed0) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79 #5 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c81b6fed0) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59 #6 0x00007fdb85e146ba in start_thread (arg=0x7fdb6727f700) at pthread_create.c:333 #7 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 51 (Thread 0x7fdb67a80700 (LWP 131908)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x0000564c64302818 in Cond::Wait (this=0x564c708f5328, mutex=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:48 #2 0x0000564c64c2fada in SafeTimer::timer_thread (this=0x564c708f5318) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Timer.cc:108 #3 0x0000564c64c310e4 in SafeTimerThread::entry (this=0x564c81b6fea0) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Timer.cc:30 #4 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c81b6fea0) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79 #5 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c81b6fea0) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59 #6 0x00007fdb85e146ba in start_thread (arg=0x7fdb67a80700) at pthread_create.c:333 #7 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 50 (Thread 0x7fdb68281700 (LWP 131907)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x0000564c64302818 in Cond::Wait (this=0x564c708f4778, mutex=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:48 #2 0x0000564c64c2fada in SafeTimer::timer_thread (this=0x564c708f4768) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Timer.cc:108 #3 0x0000564c64c310e4 in SafeTimerThread::entry (this=0x564c81b6fe70) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Timer.cc:30 #4 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c81b6fe70) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79 #5 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c81b6fe70) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59 ---Type to continue, or q to quit--- #6 0x00007fdb85e146ba in start_thread (arg=0x7fdb68281700) at pthread_create.c:333 #7 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 49 (Thread 0x7fdb68a82700 (LWP 131906)): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 #1 0x0000564c64302902 in Cond::WaitUntil (this=0x564c708f4a78, mutex=..., when=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:64 #2 0x0000564c64c2fb2c in SafeTimer::timer_thread (this=0x564c708f4a68) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Timer.cc:110 #3 0x0000564c64c310e4 in SafeTimerThread::entry (this=0x564c81b6fe40) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Timer.cc:30 #4 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c81b6fe40) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79 #5 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c81b6fe40) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59 #6 0x00007fdb85e146ba in start_thread (arg=0x7fdb68a82700) at pthread_create.c:333 #7 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 48 (Thread 0x7fdb69283700 (LWP 131905)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x0000564c64302818 in Cond::Wait (this=0x564c708f48e8, mutex=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:48 #2 0x0000564c64c34d40 in Finisher::finisher_thread_entry (this=0x564c708f4870) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Finisher.cc:101 #3 0x0000564c64314d62 in Finisher::FinisherThread::entry (this=0x564c708f49c8) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Finisher.h:65 #4 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c708f49c8) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79 #5 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c708f49c8) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59 #6 0x00007fdb85e146ba in start_thread (arg=0x7fdb69283700) at pthread_create.c:333 #7 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 47 (Thread 0x7fdb69a84700 (LWP 131904)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x0000564c64302818 in Cond::Wait (this=0x564c708f4e38, mutex=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:48 #2 0x0000564c64c34d40 in Finisher::finisher_thread_entry (this=0x564c708f4dc0) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Finisher.cc:101 #3 0x0000564c64314d62 in Finisher::FinisherThread::entry (this=0x564c708f4f18) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Finisher.h:65 #4 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c708f4f18) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79 #5 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c708f4f18) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59 #6 0x00007fdb85e146ba in start_thread (arg=0x7fdb69a84700) at pthread_create.c:333 #7 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 46 (Thread 0x7fdb6a285700 (LWP 131903)): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 #1 0x0000564c64302902 in Cond::WaitUntil (this=0x564c708f3340, mutex=..., when=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:64 #2 0x0000564c64302980 in Cond::WaitInterval (this=0x564c708f3340, mutex=..., interval=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:73 #3 0x0000564c642bf081 in OSD::heartbeat_entry (this=0x564c708f2000) at /raid/src/ceph-deb/ceph-12.2.4/src/osd/OSD.cc:5087 #4 0x0000564c64319f84 in OSD::T_Heartbeat::entry (this=0x564c708f33f0) at /raid/src/ceph-deb/ceph-12.2.4/src/osd/OSD.h:1536 #5 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c708f33f0) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79 #6 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c708f33f0) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59 #7 0x00007fdb85e146ba in start_thread (arg=0x7fdb6a285700) at pthread_create.c:333 #8 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 45 (Thread 0x7fdb6aa86700 (LWP 131902)): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 ---Type to continue, or q to quit--- #1 0x0000564c64302902 in Cond::WaitUntil (this=0x564c708f3050, mutex=..., when=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:64 #2 0x0000564c64302980 in Cond::WaitInterval (this=0x564c708f3050, mutex=..., interval=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:73 #3 0x0000564c64c3cfd1 in ThreadPool::worker (this=0x564c708f2f70, wt=0x564c81b6fd20) at /raid/src/ceph-deb/ceph-12.2.4/src/common/WorkQueue.cc:143 #4 0x0000564c64c413eb in ThreadPool::WorkThread::entry (this=0x564c81b6fd20) at /raid/src/ceph-deb/ceph-12.2.4/src/common/WorkQueue.h:448 #5 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c81b6fd20) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79 #6 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c81b6fd20) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59 #7 0x00007fdb85e146ba in start_thread (arg=0x7fdb6aa86700) at pthread_create.c:333 #8 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 44 (Thread 0x7fdb6b287700 (LWP 131901)): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 #1 0x0000564c64302902 in Cond::WaitUntil (this=0x564c708f2e38, mutex=..., when=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:64 #2 0x0000564c64302980 in Cond::WaitInterval (this=0x564c708f2e38, mutex=..., interval=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:73 #3 0x0000564c64c3cfd1 in ThreadPool::worker (this=0x564c708f2d58, wt=0x564c81b6fcc0) at /raid/src/ceph-deb/ceph-12.2.4/src/common/WorkQueue.cc:143 #4 0x0000564c64c413eb in ThreadPool::WorkThread::entry (this=0x564c81b6fcc0) at /raid/src/ceph-deb/ceph-12.2.4/src/common/WorkQueue.h:448 #5 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c81b6fcc0) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79 #6 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c81b6fcc0) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59 #7 0x00007fdb85e146ba in start_thread (arg=0x7fdb6b287700) at pthread_create.c:333 #8 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 43 (Thread 0x7fdb6ba88700 (LWP 131900)): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 #1 0x0000564c64302902 in Cond::WaitUntil (this=0x564c70481570, mutex=..., when=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:64 #2 0x0000564c64302980 in Cond::WaitInterval (this=0x564c70481570, mutex=..., interval=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:73 #3 0x0000564c642f30e2 in OSD::ShardedOpWQ::_process (this=0x564c708f3620, thread_index=4, hb=0x564c83722140) at /raid/src/ceph-deb/ceph-12.2.4/src/osd/OSD.cc:10370 #4 0x0000564c64c3f94d in ShardedThreadPool::shardedthreadpool_worker (this=0x564c708f2bd0, thread_index=4) at /raid/src/ceph-deb/ceph-12.2.4/src/common/WorkQueue.cc:339 #5 0x0000564c64c41461 in ShardedThreadPool::WorkThreadSharded::entry (this=0x564c80538a00) at /raid/src/ceph-deb/ceph-12.2.4/src/common/WorkQueue.h:689 #6 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c80538a00) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79 #7 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c80538a00) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59 #8 0x00007fdb85e146ba in start_thread (arg=0x7fdb6ba88700) at pthread_create.c:333 #9 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 42 (Thread 0x7fdb6c289700 (LWP 131899)): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 #1 0x0000564c64302902 in Cond::WaitUntil (this=0x564c704813f0, mutex=..., when=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:64 #2 0x0000564c64302980 in Cond::WaitInterval (this=0x564c704813f0, mutex=..., interval=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:73 #3 0x0000564c642f30e2 in OSD::ShardedOpWQ::_process (this=0x564c708f3620, thread_index=3, hb=0x564c83722050) at /raid/src/ceph-deb/ceph-12.2.4/src/osd/OSD.cc:10370 #4 0x0000564c64c3f94d in ShardedThreadPool::shardedthreadpool_worker (this=0x564c708f2bd0, thread_index=3) at /raid/src/ceph-deb/ceph-12.2.4/src/common/WorkQueue.cc:339 #5 0x0000564c64c41461 in ShardedThreadPool::WorkThreadSharded::entry (this=0x564c7fc2f600) at /raid/src/ceph-deb/ceph-12.2.4/src/common/WorkQueue.h:689 #6 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c7fc2f600) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79 #7 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c7fc2f600) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59 #8 0x00007fdb85e146ba in start_thread (arg=0x7fdb6c289700) at pthread_create.c:333 #9 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 ---Type to continue, or q to quit--- Thread 41 (Thread 0x7fdb6ca8a700 (LWP 131898)): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 #1 0x0000564c64302902 in Cond::WaitUntil (this=0x564c70481270, mutex=..., when=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:64 #2 0x0000564c64302980 in Cond::WaitInterval (this=0x564c70481270, mutex=..., interval=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:73 #3 0x0000564c642f30e2 in OSD::ShardedOpWQ::_process (this=0x564c708f3620, thread_index=2, hb=0x564c837221e0) at /raid/src/ceph-deb/ceph-12.2.4/src/osd/OSD.cc:10370 #4 0x0000564c64c3f94d in ShardedThreadPool::shardedthreadpool_worker (this=0x564c708f2bd0, thread_index=2) at /raid/src/ceph-deb/ceph-12.2.4/src/common/WorkQueue.cc:339 #5 0x0000564c64c41461 in ShardedThreadPool::WorkThreadSharded::entry (this=0x564c804d3ec0) at /raid/src/ceph-deb/ceph-12.2.4/src/common/WorkQueue.h:689 #6 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c804d3ec0) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79 #7 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c804d3ec0) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59 #8 0x00007fdb85e146ba in start_thread (arg=0x7fdb6ca8a700) at pthread_create.c:333 #9 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 40 (Thread 0x7fdb6d28b700 (LWP 131897)): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 #1 0x0000564c64302902 in Cond::WaitUntil (this=0x564c704810f0, mutex=..., when=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:64 #2 0x0000564c64302980 in Cond::WaitInterval (this=0x564c704810f0, mutex=..., interval=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:73 #3 0x0000564c642f30e2 in OSD::ShardedOpWQ::_process (this=0x564c708f3620, thread_index=1, hb=0x564c837220a0) at /raid/src/ceph-deb/ceph-12.2.4/src/osd/OSD.cc:10370 #4 0x0000564c64c3f94d in ShardedThreadPool::shardedthreadpool_worker (this=0x564c708f2bd0, thread_index=1) at /raid/src/ceph-deb/ceph-12.2.4/src/common/WorkQueue.cc:339 #5 0x0000564c64c41461 in ShardedThreadPool::WorkThreadSharded::entry (this=0x564c7e578480) at /raid/src/ceph-deb/ceph-12.2.4/src/common/WorkQueue.h:689 #6 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c7e578480) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79 #7 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c7e578480) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59 #8 0x00007fdb85e146ba in start_thread (arg=0x7fdb6d28b700) at pthread_create.c:333 #9 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 39 (Thread 0x7fdb6da8c700 (LWP 131896)): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 #1 0x0000564c64302902 in Cond::WaitUntil (this=0x564c70480f70, mutex=..., when=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:64 #2 0x0000564c64302980 in Cond::WaitInterval (this=0x564c70480f70, mutex=..., interval=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:73 #3 0x0000564c642f30e2 in OSD::ShardedOpWQ::_process (this=0x564c708f3620, thread_index=0, hb=0x564c837220f0) at /raid/src/ceph-deb/ceph-12.2.4/src/osd/OSD.cc:10370 #4 0x0000564c64c3f94d in ShardedThreadPool::shardedthreadpool_worker (this=0x564c708f2bd0, thread_index=0) at /raid/src/ceph-deb/ceph-12.2.4/src/common/WorkQueue.cc:339 #5 0x0000564c64c41461 in ShardedThreadPool::WorkThreadSharded::entry (this=0x564c826c5780) at /raid/src/ceph-deb/ceph-12.2.4/src/common/WorkQueue.h:689 #6 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c826c5780) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79 #7 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c826c5780) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59 #8 0x00007fdb85e146ba in start_thread (arg=0x7fdb6da8c700) at pthread_create.c:333 #9 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 38 (Thread 0x7fdb6e28d700 (LWP 131895)): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 #1 0x0000564c64302902 in Cond::WaitUntil (this=0x564c708f2a98, mutex=..., when=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:64 #2 0x0000564c64302980 in Cond::WaitInterval (this=0x564c708f2a98, mutex=..., interval=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:73 #3 0x0000564c64c3cfd1 in ThreadPool::worker (this=0x564c708f29b8, wt=0x564c81b6fc60) at /raid/src/ceph-deb/ceph-12.2.4/src/common/WorkQueue.cc:143 #4 0x0000564c64c413eb in ThreadPool::WorkThread::entry (this=0x564c81b6fc60) at /raid/src/ceph-deb/ceph-12.2.4/src/common/WorkQueue.h:448 #5 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c81b6fc60) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79 ---Type to continue, or q to quit--- #6 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c81b6fc60) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59 #7 0x00007fdb85e146ba in start_thread (arg=0x7fdb6e28d700) at pthread_create.c:333 #8 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 37 (Thread 0x7fdb6ea8e700 (LWP 131894)): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 #1 0x0000564c64302902 in Cond::WaitUntil (this=0x564c708f2a98, mutex=..., when=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:64 #2 0x0000564c64302980 in Cond::WaitInterval (this=0x564c708f2a98, mutex=..., interval=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:73 #3 0x0000564c64c3cfd1 in ThreadPool::worker (this=0x564c708f29b8, wt=0x564c81b6fc30) at /raid/src/ceph-deb/ceph-12.2.4/src/common/WorkQueue.cc:143 #4 0x0000564c64c413eb in ThreadPool::WorkThread::entry (this=0x564c81b6fc30) at /raid/src/ceph-deb/ceph-12.2.4/src/common/WorkQueue.h:448 #5 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c81b6fc30) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79 #6 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c81b6fc30) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59 #7 0x00007fdb85e146ba in start_thread (arg=0x7fdb6ea8e700) at pthread_create.c:333 #8 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 36 (Thread 0x7fdb6f28f700 (LWP 131893)): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 #1 0x0000564c64302902 in Cond::WaitUntil (this=0x564c708f2480, mutex=..., when=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:64 #2 0x0000564c64c2fb2c in SafeTimer::timer_thread (this=0x564c708f2470) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Timer.cc:110 #3 0x0000564c64c310e4 in SafeTimerThread::entry (this=0x564c81b6fba0) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Timer.cc:30 #4 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c81b6fba0) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79 #5 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c81b6fba0) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59 #6 0x00007fdb85e146ba in start_thread (arg=0x7fdb6f28f700) at pthread_create.c:333 #7 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 35 (Thread 0x7fdb6fa90700 (LWP 131892)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x0000564c64302818 in Cond::Wait (this=0x7fff606e6690, mutex=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:48 #2 0x0000564c64c34d40 in Finisher::finisher_thread_entry (this=0x7fff606e6618) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Finisher.cc:101 #3 0x0000564c64314d62 in Finisher::FinisherThread::entry (this=0x7fff606e6770) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Finisher.h:65 #4 0x0000564c64e054df in Thread::entry_wrapper (this=0x7fff606e6770) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79 #5 0x0000564c64e05414 in Thread::_entry_func (arg=0x7fff606e6770) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59 #6 0x00007fdb85e146ba in start_thread (arg=0x7fdb6fa90700) at pthread_create.c:333 #7 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 34 (Thread 0x7fdb70291700 (LWP 131891)): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 #1 0x0000564c64302902 in Cond::WaitUntil (this=0x7fff606e6560, mutex=..., when=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:64 #2 0x0000564c64c2fb2c in SafeTimer::timer_thread (this=0x7fff606e6550) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Timer.cc:110 #3 0x0000564c64c310e4 in SafeTimerThread::entry (this=0x564c81b6fa80) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Timer.cc:30 #4 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c81b6fa80) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79 #5 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c81b6fa80) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59 #6 0x00007fdb85e146ba in start_thread (arg=0x7fdb70291700) at pthread_create.c:333 #7 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 33 (Thread 0x7fdb70a92700 (LWP 131890)): ---Type to continue, or q to quit--- #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x0000564c64302818 in Cond::Wait (this=0x564c707453c8, mutex=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:48 #2 0x0000564c650362df in DispatchQueue::run_local_delivery (this=0x564c70745180) at /raid/src/ceph-deb/ceph-12.2.4/src/msg/DispatchQueue.cc:113 #3 0x0000564c64d2886c in DispatchQueue::LocalDeliveryThread::entry (this=0x564c70745428) at /raid/src/ceph-deb/ceph-12.2.4/src/msg/DispatchQueue.h:115 #4 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c70745428) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79 #5 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c70745428) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59 #6 0x00007fdb85e146ba in start_thread (arg=0x7fdb70a92700) at pthread_create.c:333 #7 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 32 (Thread 0x7fdb71293700 (LWP 131889)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x0000564c64302818 in Cond::Wait (this=0x564c70745200, mutex=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:48 #2 0x0000564c65036cb3 in DispatchQueue::entry (this=0x564c70745180) at /raid/src/ceph-deb/ceph-12.2.4/src/msg/DispatchQueue.cc:208 #3 0x0000564c64d2880a in DispatchQueue::DispatchThread::entry (this=0x564c70745328) at /raid/src/ceph-deb/ceph-12.2.4/src/msg/DispatchQueue.h:101 #4 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c70745328) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79 #5 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c70745328) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59 #6 0x00007fdb85e146ba in start_thread (arg=0x7fdb71293700) at pthread_create.c:333 #7 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 31 (Thread 0x7fdb71a94700 (LWP 131888)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x0000564c64302818 in Cond::Wait (this=0x564c707443c8, mutex=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:48 #2 0x0000564c650362df in DispatchQueue::run_local_delivery (this=0x564c70744180) at /raid/src/ceph-deb/ceph-12.2.4/src/msg/DispatchQueue.cc:113 #3 0x0000564c64d2886c in DispatchQueue::LocalDeliveryThread::entry (this=0x564c70744428) at /raid/src/ceph-deb/ceph-12.2.4/src/msg/DispatchQueue.h:115 #4 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c70744428) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79 #5 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c70744428) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59 #6 0x00007fdb85e146ba in start_thread (arg=0x7fdb71a94700) at pthread_create.c:333 #7 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 30 (Thread 0x7fdb72295700 (LWP 131887)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x0000564c64302818 in Cond::Wait (this=0x564c70744200, mutex=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:48 #2 0x0000564c65036cb3 in DispatchQueue::entry (this=0x564c70744180) at /raid/src/ceph-deb/ceph-12.2.4/src/msg/DispatchQueue.cc:208 #3 0x0000564c64d2880a in DispatchQueue::DispatchThread::entry (this=0x564c70744328) at /raid/src/ceph-deb/ceph-12.2.4/src/msg/DispatchQueue.h:101 #4 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c70744328) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79 #5 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c70744328) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59 #6 0x00007fdb85e146ba in start_thread (arg=0x7fdb72295700) at pthread_create.c:333 #7 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 29 (Thread 0x7fdb72a96700 (LWP 131886)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x0000564c64302818 in Cond::Wait (this=0x564c70744bc8, mutex=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:48 #2 0x0000564c650362df in DispatchQueue::run_local_delivery (this=0x564c70744980) at /raid/src/ceph-deb/ceph-12.2.4/src/msg/DispatchQueue.cc:113 #3 0x0000564c64d2886c in DispatchQueue::LocalDeliveryThread::entry (this=0x564c70744c28) at /raid/src/ceph-deb/ceph-12.2.4/src/msg/DispatchQueue.h:115 #4 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c70744c28) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79 #5 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c70744c28) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59 ---Type to continue, or q to quit--- #6 0x00007fdb85e146ba in start_thread (arg=0x7fdb72a96700) at pthread_create.c:333 #7 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 28 (Thread 0x7fdb73297700 (LWP 131885)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x0000564c64302818 in Cond::Wait (this=0x564c70744a00, mutex=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:48 #2 0x0000564c65036cb3 in DispatchQueue::entry (this=0x564c70744980) at /raid/src/ceph-deb/ceph-12.2.4/src/msg/DispatchQueue.cc:208 #3 0x0000564c64d2880a in DispatchQueue::DispatchThread::entry (this=0x564c70744b28) at /raid/src/ceph-deb/ceph-12.2.4/src/msg/DispatchQueue.h:101 #4 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c70744b28) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79 #5 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c70744b28) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59 #6 0x00007fdb85e146ba in start_thread (arg=0x7fdb73297700) at pthread_create.c:333 #7 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 27 (Thread 0x7fdb73a98700 (LWP 131884)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x0000564c64302818 in Cond::Wait (this=0x564c707433c8, mutex=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:48 #2 0x0000564c650362df in DispatchQueue::run_local_delivery (this=0x564c70743180) at /raid/src/ceph-deb/ceph-12.2.4/src/msg/DispatchQueue.cc:113 #3 0x0000564c64d2886c in DispatchQueue::LocalDeliveryThread::entry (this=0x564c70743428) at /raid/src/ceph-deb/ceph-12.2.4/src/msg/DispatchQueue.h:115 #4 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c70743428) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79 #5 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c70743428) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59 #6 0x00007fdb85e146ba in start_thread (arg=0x7fdb73a98700) at pthread_create.c:333 #7 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 26 (Thread 0x7fdb76317700 (LWP 131883)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x0000564c64302818 in Cond::Wait (this=0x564c70743200, mutex=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:48 #2 0x0000564c65036cb3 in DispatchQueue::entry (this=0x564c70743180) at /raid/src/ceph-deb/ceph-12.2.4/src/msg/DispatchQueue.cc:208 #3 0x0000564c64d2880a in DispatchQueue::DispatchThread::entry (this=0x564c70743328) at /raid/src/ceph-deb/ceph-12.2.4/src/msg/DispatchQueue.h:101 #4 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c70743328) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79 #5 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c70743328) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59 #6 0x00007fdb85e146ba in start_thread (arg=0x7fdb76317700) at pthread_create.c:333 #7 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 25 (Thread 0x7fdb76b18700 (LWP 131882)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x0000564c64302818 in Cond::Wait (this=0x564c70743bc8, mutex=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:48 #2 0x0000564c650362df in DispatchQueue::run_local_delivery (this=0x564c70743980) at /raid/src/ceph-deb/ceph-12.2.4/src/msg/DispatchQueue.cc:113 #3 0x0000564c64d2886c in DispatchQueue::LocalDeliveryThread::entry (this=0x564c70743c28) at /raid/src/ceph-deb/ceph-12.2.4/src/msg/DispatchQueue.h:115 #4 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c70743c28) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79 #5 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c70743c28) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59 #6 0x00007fdb85e146ba in start_thread (arg=0x7fdb76b18700) at pthread_create.c:333 #7 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 24 (Thread 0x7fdb77319700 (LWP 131881)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x0000564c64302818 in Cond::Wait (this=0x564c70743a00, mutex=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:48 ---Type to continue, or q to quit--- #2 0x0000564c65036cb3 in DispatchQueue::entry (this=0x564c70743980) at /raid/src/ceph-deb/ceph-12.2.4/src/msg/DispatchQueue.cc:208 #3 0x0000564c64d2880a in DispatchQueue::DispatchThread::entry (this=0x564c70743b28) at /raid/src/ceph-deb/ceph-12.2.4/src/msg/DispatchQueue.h:101 #4 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c70743b28) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79 #5 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c70743b28) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59 #6 0x00007fdb85e146ba in start_thread (arg=0x7fdb77319700) at pthread_create.c:333 #7 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 23 (Thread 0x7fdb77b1a700 (LWP 131880)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x0000564c64302818 in Cond::Wait (this=0x564c70742bc8, mutex=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:48 #2 0x0000564c650362df in DispatchQueue::run_local_delivery (this=0x564c70742980) at /raid/src/ceph-deb/ceph-12.2.4/src/msg/DispatchQueue.cc:113 #3 0x0000564c64d2886c in DispatchQueue::LocalDeliveryThread::entry (this=0x564c70742c28) at /raid/src/ceph-deb/ceph-12.2.4/src/msg/DispatchQueue.h:115 #4 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c70742c28) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79 #5 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c70742c28) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59 #6 0x00007fdb85e146ba in start_thread (arg=0x7fdb77b1a700) at pthread_create.c:333 #7 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 22 (Thread 0x7fdb7831b700 (LWP 131879)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x0000564c64302818 in Cond::Wait (this=0x564c70742a00, mutex=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:48 #2 0x0000564c65036cb3 in DispatchQueue::entry (this=0x564c70742980) at /raid/src/ceph-deb/ceph-12.2.4/src/msg/DispatchQueue.cc:208 #3 0x0000564c64d2880a in DispatchQueue::DispatchThread::entry (this=0x564c70742b28) at /raid/src/ceph-deb/ceph-12.2.4/src/msg/DispatchQueue.h:101 #4 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c70742b28) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79 #5 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c70742b28) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59 #6 0x00007fdb85e146ba in start_thread (arg=0x7fdb7831b700) at pthread_create.c:333 #7 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 21 (Thread 0x7fdb78b1c700 (LWP 131878)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x0000564c64302818 in Cond::Wait (this=0x564c707423c8, mutex=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:48 #2 0x0000564c650362df in DispatchQueue::run_local_delivery (this=0x564c70742180) at /raid/src/ceph-deb/ceph-12.2.4/src/msg/DispatchQueue.cc:113 #3 0x0000564c64d2886c in DispatchQueue::LocalDeliveryThread::entry (this=0x564c70742428) at /raid/src/ceph-deb/ceph-12.2.4/src/msg/DispatchQueue.h:115 #4 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c70742428) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79 #5 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c70742428) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59 #6 0x00007fdb85e146ba in start_thread (arg=0x7fdb78b1c700) at pthread_create.c:333 #7 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 20 (Thread 0x7fdb7931d700 (LWP 131877)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x0000564c64302818 in Cond::Wait (this=0x564c70742200, mutex=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:48 #2 0x0000564c65036cb3 in DispatchQueue::entry (this=0x564c70742180) at /raid/src/ceph-deb/ceph-12.2.4/src/msg/DispatchQueue.cc:208 #3 0x0000564c64d2880a in DispatchQueue::DispatchThread::entry (this=0x564c70742328) at /raid/src/ceph-deb/ceph-12.2.4/src/msg/DispatchQueue.h:101 #4 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c70742328) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79 #5 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c70742328) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59 #6 0x00007fdb85e146ba in start_thread (arg=0x7fdb7931d700) at pthread_create.c:333 #7 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 ---Type to continue, or q to quit--- Thread 19 (Thread 0x7fdb7baa8700 (LWP 131832)): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 #1 0x0000564c64302902 in Cond::WaitUntil (this=0x564c70730bf8, mutex=..., when=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:64 #2 0x0000564c64302980 in Cond::WaitInterval (this=0x564c70730bf8, mutex=..., interval=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:73 #3 0x0000564c649fb112 in BlueStore::MempoolThread::entry (this=0x564c70730bc8) at /raid/src/ceph-deb/ceph-12.2.4/src/os/bluestore/BlueStore.cc:3374 #4 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c70730bc8) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79 #5 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c70730bc8) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59 #6 0x00007fdb85e146ba in start_thread (arg=0x7fdb7baa8700) at pthread_create.c:333 #7 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 18 (Thread 0x7fdb75a9c700 (LWP 131830)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x00007fdb8572091c in std::condition_variable::wait(std::unique_lock&) () from target:/usr/lib/x86_64-linux-gnu/libstdc++.so.6 #2 0x0000564c64a3b3aa in BlueStore::_kv_finalize_thread (this=0x564c70730000) at /raid/src/ceph-deb/ceph-12.2.4/src/os/bluestore/BlueStore.cc:8706 #3 0x0000564c64a718a0 in BlueStore::KVFinalizeThread::entry (this=0x564c70730898) at /raid/src/ceph-deb/ceph-12.2.4/src/os/bluestore/BlueStore.h:1790 #4 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c70730898) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79 #5 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c70730898) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59 #6 0x00007fdb85e146ba in start_thread (arg=0x7fdb75a9c700) at pthread_create.c:333 #7 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 17 (Thread 0x7fdb7529b700 (LWP 131829)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x00007fdb8572091c in std::condition_variable::wait(std::unique_lock&) () from target:/usr/lib/x86_64-linux-gnu/libstdc++.so.6 #2 0x0000564c64a380cc in BlueStore::_kv_sync_thread (this=0x564c70730000) at /raid/src/ceph-deb/ceph-12.2.4/src/os/bluestore/BlueStore.cc:8457 #3 0x0000564c64a7183e in BlueStore::KVSyncThread::entry (this=0x564c70730678) at /raid/src/ceph-deb/ceph-12.2.4/src/os/bluestore/BlueStore.h:1782 #4 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c70730678) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79 #5 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c70730678) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59 #6 0x00007fdb85e146ba in start_thread (arg=0x7fdb7529b700) at pthread_create.c:333 #7 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 16 (Thread 0x7fdb74a9a700 (LWP 131828)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x0000564c64302818 in Cond::Wait (this=0x564c704718d8, mutex=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:48 #2 0x0000564c64c34d40 in Finisher::finisher_thread_entry (this=0x564c70471860) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Finisher.cc:101 #3 0x0000564c64314d62 in Finisher::FinisherThread::entry (this=0x564c704719b8) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Finisher.h:65 #4 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c704719b8) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79 #5 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c704719b8) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59 #6 0x00007fdb85e146ba in start_thread (arg=0x7fdb74a9a700) at pthread_create.c:333 #7 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 15 (Thread 0x7fdb74299700 (LWP 131827)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x0000564c64302818 in Cond::Wait (this=0x564c70730548, mutex=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:48 #2 0x0000564c64c34d40 in Finisher::finisher_thread_entry (this=0x564c707304d0) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Finisher.cc:101 #3 0x0000564c64314d62 in Finisher::FinisherThread::entry (this=0x564c70730628) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Finisher.h:65 ---Type to continue, or q to quit--- #4 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c70730628) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79 #5 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c70730628) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59 #6 0x00007fdb85e146ba in start_thread (arg=0x7fdb74299700) at pthread_create.c:333 #7 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 14 (Thread 0x7fdb7c2a9700 (LWP 131528)): #0 0x00007fdb86e9e64a in ?? () from target:/lib/x86_64-linux-gnu/libaio.so.1 #1 0x0000564c64bcc9cb in aio_queue_t::get_next_completed (this=0x564c70499378, timeout_ms=250, paio=0x7fdb7c2a7440, max=16) at /raid/src/ceph-deb/ceph-12.2.4/src/os/bluestore/aio.cc:78 #2 0x0000564c64baaa1e in KernelDevice::_aio_thread (this=0x564c70499200) at /raid/src/ceph-deb/ceph-12.2.4/src/os/bluestore/KernelDevice.cc:350 #3 0x0000564c64bb2062 in KernelDevice::AioCompletionThread::entry (this=0x564c704993a0) at /raid/src/ceph-deb/ceph-12.2.4/src/os/bluestore/KernelDevice.h:49 #4 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c704993a0) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79 #5 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c704993a0) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59 #6 0x00007fdb85e146ba in start_thread (arg=0x7fdb7c2a9700) at pthread_create.c:333 #7 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 13 (Thread 0x7fdb7caaa700 (LWP 131526)): #0 0x00007fdb86e9e64a in ?? () from target:/lib/x86_64-linux-gnu/libaio.so.1 #1 0x0000564c64bcc9cb in aio_queue_t::get_next_completed (this=0x564c70499138, timeout_ms=250, paio=0x7fdb7caa8440, max=16) at /raid/src/ceph-deb/ceph-12.2.4/src/os/bluestore/aio.cc:78 #2 0x0000564c64baaa1e in KernelDevice::_aio_thread (this=0x564c70498fc0) at /raid/src/ceph-deb/ceph-12.2.4/src/os/bluestore/KernelDevice.cc:350 #3 0x0000564c64bb2062 in KernelDevice::AioCompletionThread::entry (this=0x564c70499160) at /raid/src/ceph-deb/ceph-12.2.4/src/os/bluestore/KernelDevice.h:49 #4 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c70499160) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79 #5 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c70499160) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59 #6 0x00007fdb85e146ba in start_thread (arg=0x7fdb7caaa700) at pthread_create.c:333 #7 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 12 (Thread 0x7fdb7d2ab700 (LWP 131524)): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 #1 0x0000564c64302902 in Cond::WaitUntil (this=0x564c708f4d00, mutex=..., when=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:64 #2 0x0000564c64c2fb2c in SafeTimer::timer_thread (this=0x564c708f4cf0) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Timer.cc:110 #3 0x0000564c64c310e4 in SafeTimerThread::entry (this=0x564c70779e30) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Timer.cc:30 #4 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c70779e30) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79 #5 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c70779e30) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59 #6 0x00007fdb85e146ba in start_thread (arg=0x7fdb7d2ab700) at pthread_create.c:333 #7 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 11 (Thread 0x7fdb7daac700 (LWP 131523)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x0000564c64302818 in Cond::Wait (this=0x564c708f4bb8, mutex=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:48 #2 0x0000564c64c2fada in SafeTimer::timer_thread (this=0x564c708f4ba8) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Timer.cc:108 #3 0x0000564c64c310e4 in SafeTimerThread::entry (this=0x564c70779e00) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Timer.cc:30 #4 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c70779e00) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79 #5 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c70779e00) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59 #6 0x00007fdb85e146ba in start_thread (arg=0x7fdb7daac700) at pthread_create.c:333 #7 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 ---Type to continue, or q to quit--- Thread 10 (Thread 0x7fdb7e2ad700 (LWP 131522)): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 #1 0x0000564c64302902 in Cond::WaitUntil (this=0x564c708f21d0, mutex=..., when=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:64 #2 0x0000564c64c2fb2c in SafeTimer::timer_thread (this=0x564c708f21c0) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Timer.cc:110 #3 0x0000564c64c310e4 in SafeTimerThread::entry (this=0x564c70779dd0) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Timer.cc:30 #4 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c70779dd0) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79 #5 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c70779dd0) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59 #6 0x00007fdb85e146ba in start_thread (arg=0x7fdb7e2ad700) at pthread_create.c:333 #7 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 9 (Thread 0x7fdb7eaae700 (LWP 131521)): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 #1 0x0000564c64302902 in Cond::WaitUntil (this=0x564c708f2098, mutex=..., when=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:64 #2 0x0000564c64c2fb2c in SafeTimer::timer_thread (this=0x564c708f2088) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Timer.cc:110 #3 0x0000564c64c310e4 in SafeTimerThread::entry (this=0x564c70779d40) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Timer.cc:30 #4 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c70779d40) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79 #5 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c70779d40) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59 #6 0x00007fdb85e146ba in start_thread (arg=0x7fdb7eaae700) at pthread_create.c:333 #7 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 8 (Thread 0x7fdb7f2af700 (LWP 131519)): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 #1 0x0000564c64287ec8 in __gthread_cond_timedwait (__cond=0x564c70843a80, __mutex=0x564c70843a58, __abs_timeout=0x7fdb7f2ad680) at /usr/include/x86_64-linux-gnu/c++/5/bits/gthr-default.h:871 #2 0x0000564c6439ece2 in std::condition_variable::__wait_until_impl > > (this=0x564c70843a80, __lock=..., __atime=...) at /usr/include/c++/5/condition_variable:165 #3 0x0000564c6437e9f2 in std::condition_variable::wait_until > > ( this=0x564c70843a80, __lock=..., __atime=...) at /usr/include/c++/5/condition_variable:118 #4 0x0000564c6435a5c9 in ceph::timer_detail::timer::timer_thread (this=0x564c70843a18) at /raid/src/ceph-deb/ceph-12.2.4/src/common/ceph_timer.h:144 #5 0x0000564c643f133b in std::_Mem_fn_base::*)(), true>::operator()<, void>(ceph::timer_detail::timer*) const (this=0x564c707237b0, __object=0x564c70843a18) at /usr/include/c++/5/functional:600 #6 0x0000564c643eeff9 in std::_Bind_simple::*)()> (ceph::timer_detail::timer*)>::_M_invoke<0ul>(std::_Index_tuple<0ul>) (this=0x564c707237a8) at /usr/include/c++/5/functional:1531 #7 0x0000564c643ec86e in std::_Bind_simple::*)()> (ceph::timer_detail::timer*)>::operator()() (this=0x564c707237a8) at /usr/include/c++/5/functional:1520 #8 0x0000564c643e919a in std::thread::_Impl::*)()> (ceph::timer_detail::timer*)> >::_M_run() (this=0x564c70723790) at /usr/include/c++/5/thread:115 #9 0x00007fdb85725c80 in ?? () from target:/usr/lib/x86_64-linux-gnu/libstdc++.so.6 #10 0x00007fdb85e146ba in start_thread (arg=0x7fdb7f2af700) at pthread_create.c:333 #11 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 7 (Thread 0x7fdb802fd700 (LWP 131199)): #0 0x00007fdb84e7f74d in poll () at ../sysdeps/unix/syscall-template.S:84 #1 0x0000564c64c04b3c in AdminSocket::entry (this=0x564c70490540) at /raid/src/ceph-deb/ceph-12.2.4/src/common/admin_socket.cc:250 ---Type to continue, or q to quit--- #2 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c70490540) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79 #3 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c70490540) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59 #4 0x00007fdb85e146ba in start_thread (arg=0x7fdb802fd700) at pthread_create.c:333 #5 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 6 (Thread 0x7fdb80afe700 (LWP 131197)): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 #1 0x0000564c64302902 in Cond::WaitUntil (this=0x564c70488368, mutex=..., when=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:64 #2 0x0000564c64302980 in Cond::WaitInterval (this=0x564c70488368, mutex=..., interval=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:73 #3 0x0000564c64dfbf0c in CephContextServiceThread::entry (this=0x564c704882d0) at /raid/src/ceph-deb/ceph-12.2.4/src/common/ceph_context.cc:135 #4 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c704882d0) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79 #5 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c704882d0) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59 #6 0x00007fdb85e146ba in start_thread (arg=0x7fdb80afe700) at pthread_create.c:333 #7 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 5 (Thread 0x7fdb81a9d700 (LWP 131191)): #0 0x00007fdb84e8ba13 in epoll_wait () at ../sysdeps/unix/syscall-template.S:84 #1 0x0000564c64f9b558 in EpollDriver::event_wait (this=0x564c70778300, fired_events=..., tvp=0x7fdb81a9b3d0) at /raid/src/ceph-deb/ceph-12.2.4/src/msg/async/EventEpoll.cc:114 #2 0x0000564c64d49a0c in EventCenter::process_events (this=0x564c70498740, timeout_microseconds=30000000, working_dur=0x7fdb81a9b5b0) at /raid/src/ceph-deb/ceph-12.2.4/src/msg/async/Event.cc:395 #3 0x0000564c64d510a8 in NetworkStack::::operator()(void) const (__closure=0x564c70746d98) at /raid/src/ceph-deb/ceph-12.2.4/src/msg/async/Stack.cc:51 #4 0x0000564c64d52623 in std::_Function_handler >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/5/functional:1871 #5 0x0000564c64337bda in std::function::operator()() const (this=0x564c70746d98) at /usr/include/c++/5/functional:2267 #6 0x0000564c64d5b3d2 in std::_Bind_simple ()>::_M_invoke<>(std::_Index_tuple<>) (this=0x564c70746d98) at /usr/include/c++/5/functional:1531 #7 0x0000564c64d5b328 in std::_Bind_simple ()>::operator()() (this=0x564c70746d98) at /usr/include/c++/5/functional:1520 #8 0x0000564c64d5b2b8 in std::thread::_Impl ()> >::_M_run() (this=0x564c70746d80) at /usr/include/c++/5/thread:115 #9 0x00007fdb85725c80 in ?? () from target:/usr/lib/x86_64-linux-gnu/libstdc++.so.6 #10 0x00007fdb85e146ba in start_thread (arg=0x7fdb81a9d700) at pthread_create.c:333 #11 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 4 (Thread 0x7fdb8229e700 (LWP 131190)): #0 0x00007fdb84e8ba13 in epoll_wait () at ../sysdeps/unix/syscall-template.S:84 #1 0x0000564c64f9b558 in EpollDriver::event_wait (this=0x564c70778120, fired_events=..., tvp=0x7fdb8229c3d0) at /raid/src/ceph-deb/ceph-12.2.4/src/msg/async/EventEpoll.cc:114 #2 0x0000564c64d49a0c in EventCenter::process_events (this=0x564c70498bc0, timeout_microseconds=30000000, working_dur=0x7fdb8229c5b0) at /raid/src/ceph-deb/ceph-12.2.4/src/msg/async/Event.cc:395 #3 0x0000564c64d510a8 in NetworkStack::::operator()(void) const (__closure=0x564c70746d48) at /raid/src/ceph-deb/ceph-12.2.4/src/msg/async/Stack.cc:51 #4 0x0000564c64d52623 in std::_Function_handler >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/5/functional:1871 #5 0x0000564c64337bda in std::function::operator()() const (this=0x564c70746d48) at /usr/include/c++/5/functional:2267 #6 0x0000564c64d5b3d2 in std::_Bind_simple ()>::_M_invoke<>(std::_Index_tuple<>) (this=0x564c70746d48) at /usr/include/c++/5/functional:1531 #7 0x0000564c64d5b328 in std::_Bind_simple ()>::operator()() (this=0x564c70746d48) at /usr/include/c++/5/functional:1520 #8 0x0000564c64d5b2b8 in std::thread::_Impl ()> >::_M_run() (this=0x564c70746d30) at /usr/include/c++/5/thread:115 #9 0x00007fdb85725c80 in ?? () from target:/usr/lib/x86_64-linux-gnu/libstdc++.so.6 ---Type to continue, or q to quit--- #10 0x00007fdb85e146ba in start_thread (arg=0x7fdb8229e700) at pthread_create.c:333 #11 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 3 (Thread 0x7fdb82a9f700 (LWP 131189)): #0 0x00007fdb84e8ba13 in epoll_wait () at ../sysdeps/unix/syscall-template.S:84 #1 0x0000564c64f9b558 in EpollDriver::event_wait (this=0x564c706cc090, fired_events=..., tvp=0x7fdb82a9d3d0) at /raid/src/ceph-deb/ceph-12.2.4/src/msg/async/EventEpoll.cc:114 #2 0x0000564c64d49a0c in EventCenter::process_events (this=0x564c70498980, timeout_microseconds=30000000, working_dur=0x7fdb82a9d5b0) at /raid/src/ceph-deb/ceph-12.2.4/src/msg/async/Event.cc:395 #3 0x0000564c64d510a8 in NetworkStack::::operator()(void) const (__closure=0x564c70746cf8) at /raid/src/ceph-deb/ceph-12.2.4/src/msg/async/Stack.cc:51 #4 0x0000564c64d52623 in std::_Function_handler >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/5/functional:1871 #5 0x0000564c64337bda in std::function::operator()() const (this=0x564c70746cf8) at /usr/include/c++/5/functional:2267 #6 0x0000564c64d5b3d2 in std::_Bind_simple ()>::_M_invoke<>(std::_Index_tuple<>) (this=0x564c70746cf8) at /usr/include/c++/5/functional:1531 #7 0x0000564c64d5b328 in std::_Bind_simple ()>::operator()() (this=0x564c70746cf8) at /usr/include/c++/5/functional:1520 #8 0x0000564c64d5b2b8 in std::thread::_Impl ()> >::_M_run() (this=0x564c70746ce0) at /usr/include/c++/5/thread:115 #9 0x00007fdb85725c80 in ?? () from target:/usr/lib/x86_64-linux-gnu/libstdc++.so.6 #10 0x00007fdb85e146ba in start_thread (arg=0x7fdb82a9f700) at pthread_create.c:333 #11 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 2 (Thread 0x7fdb83ae0700 (LWP 131176)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x0000564c64c4ddfe in ceph::logging::Log::entry (this=0x564c70490000) at /raid/src/ceph-deb/ceph-12.2.4/src/log/Log.cc:459 #2 0x0000564c64e054df in Thread::entry_wrapper (this=0x564c70490000) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:79 #3 0x0000564c64e05414 in Thread::_entry_func (arg=0x564c70490000) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Thread.cc:59 #4 0x00007fdb85e146ba in start_thread (arg=0x7fdb83ae0700) at pthread_create.c:333 #5 0x00007fdb84e8b41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 1 (Thread 0x7fdb87938e00 (LWP 131171)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x0000564c64302818 in Cond::Wait (this=0x564c70742760, mutex=...) at /raid/src/ceph-deb/ceph-12.2.4/src/common/Cond.h:48 #2 0x0000564c64d3bb71 in AsyncMessenger::wait (this=0x564c70742000) at /raid/src/ceph-deb/ceph-12.2.4/src/msg/async/AsyncMessenger.cc:481 #3 0x0000564c6427b57e in main (argc=10, argv=0x7fff606e7ab8) at /raid/src/ceph-deb/ceph-12.2.4/src/ceph_osd.cc:656