(gdb) info threads Id Target Id Frame * 1 Thread 0x7f52594dfe00 (LWP 7659) "ceph-osd" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 2 Thread 0x7f5255683700 (LWP 7663) "log" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 3 Thread 0x7f5254642700 (LWP 7664) "msgr-worker-0" 0x00007f5256a2ea13 in epoll_wait () at ../sysdeps/unix/syscall-template.S:84 4 Thread 0x7f5253e41700 (LWP 7665) "msgr-worker-1" 0x00007f5256a2ea13 in epoll_wait () at ../sysdeps/unix/syscall-template.S:84 5 Thread 0x7f5253640700 (LWP 7666) "msgr-worker-2" 0x00007f5256a2ea13 in epoll_wait () at ../sysdeps/unix/syscall-template.S:84 6 Thread 0x7f52526a1700 (LWP 7667) "service" pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 7 Thread 0x7f5251ea0700 (LWP 7668) "admin_socket" 0x00007f5256a2274d in poll () at ../sysdeps/unix/syscall-template.S:84 8 Thread 0x7f5250e52700 (LWP 7673) "ceph-osd" pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 9 Thread 0x7f5250651700 (LWP 7674) "safe_timer" pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 10 Thread 0x7f524fe50700 (LWP 7675) "safe_timer" pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 11 Thread 0x7f524f64f700 (LWP 7676) "safe_timer" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 12 Thread 0x7f524ee4e700 (LWP 7677) "safe_timer" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 13 Thread 0x7f524e64d700 (LWP 7678) "bstore_aio" 0x00007f5258a4164a in ?? () from target:/lib/x86_64-linux-gnu/libaio.so.1 14 Thread 0x7f524de4c700 (LWP 7679) "bstore_aio" 0x00007f5258a4164a in ?? () from target:/lib/x86_64-linux-gnu/libaio.so.1 15 Thread 0x7f5245e3c700 (LWP 7697) "dfin" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 16 Thread 0x7f524663d700 (LWP 7698) "finisher" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 17 Thread 0x7f5246e3e700 (LWP 7699) "bstore_kv_sync" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 18 Thread 0x7f524763f700 (LWP 7700) "bstore_kv_final" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 19 Thread 0x7f524d64b700 (LWP 7701) "bstore_mempool" pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 20 Thread 0x7f524aec0700 (LWP 7702) "ms_dispatch" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 21 Thread 0x7f524a6bf700 (LWP 7703) "ms_local" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 22 Thread 0x7f5249ebe700 (LWP 7704) "ms_dispatch" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 23 Thread 0x7f52496bd700 (LWP 7705) "ms_local" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 24 Thread 0x7f5248ebc700 (LWP 7706) "ms_dispatch" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 25 Thread 0x7f52486bb700 (LWP 7707) "ms_local" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 26 Thread 0x7f5247eba700 (LWP 7708) "ms_dispatch" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 27 Thread 0x7f524563b700 (LWP 7709) "ms_local" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 28 Thread 0x7f5244e3a700 (LWP 7710) "ms_dispatch" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 29 Thread 0x7f5244639700 (LWP 7711) "ms_local" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 30 Thread 0x7f5243e38700 (LWP 7712) "ms_dispatch" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 31 Thread 0x7f5243637700 (LWP 7713) "ms_local" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 32 Thread 0x7f5242e36700 (LWP 7714) "ms_dispatch" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 33 Thread 0x7f5242635700 (LWP 7715) "ms_local" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 34 Thread 0x7f5241e34700 (LWP 7716) "safe_timer" pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 35 Thread 0x7f5241633700 (LWP 7717) "fn_anonymous" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 36 Thread 0x7f5240e32700 (LWP 7718) "safe_timer" pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 37 Thread 0x7f5240631700 (LWP 7719) "tp_peering" pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 38 Thread 0x7f523fe30700 (LWP 7720) "tp_peering" pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 39 Thread 0x7f523f62f700 (LWP 7721) "tp_osd_tp" pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 40 Thread 0x7f523ee2e700 (LWP 7722) "tp_osd_tp" pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 41 Thread 0x7f523e62d700 (LWP 7723) "tp_osd_tp" pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 42 Thread 0x7f523de2c700 (LWP 7724) "tp_osd_tp" pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 43 Thread 0x7f523d62b700 (LWP 7725) "tp_osd_tp" pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 44 Thread 0x7f523ce2a700 (LWP 7726) "tp_osd_disk" pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 45 Thread 0x7f523c629700 (LWP 7727) "tp_osd_cmd" pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 46 Thread 0x7f523be28700 (LWP 7728) "osd_srv_heartbt" pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 47 Thread 0x7f523b627700 (LWP 7729) "fn_anonymous" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 48 Thread 0x7f523ae26700 (LWP 7730) "fn_anonymous" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 49 Thread 0x7f523a625700 (LWP 7731) "safe_timer" pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 50 Thread 0x7f5239e24700 (LWP 7732) "safe_timer" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 51 Thread 0x7f5239623700 (LWP 7733) "safe_timer" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 52 Thread 0x7f5238e22700 (LWP 7734) "safe_timer" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 53 Thread 0x7f5238621700 (LWP 7735) "osd_srv_agent" pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 54 Thread 0x7f5237e20700 (LWP 7736) "signal_handler" 0x00007f5256a2274d in poll () at ../sysdeps/unix/syscall-template.S:84 (gdb) thread apply all bt Thread 54 (Thread 0x7f5237e20700 (LWP 7736)): #0 0x00007f5256a2274d in poll () at ../sysdeps/unix/syscall-template.S:84 #1 0x00007f5256a400be in __poll_chk (fds=, nfds=, timeout=, fdslen=) at poll_chk.c:27 #2 0x0000561eb39b9631 in poll (__timeout=-1, __nfds=4, __fds=0x7f5237e1e5f0) at /usr/include/x86_64-linux-gnu/bits/poll2.h:41 #3 SignalHandler::entry (this=0x561ed1788840) at /build/ceph-12.2.4/src/global/signal_handler.cc:281 #4 0x00007f52579b76ba in start_thread (arg=0x7f5237e20700) at pthread_create.c:333 #5 0x00007f5256a2e41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 53 (Thread 0x7f5238621700 (LWP 7735)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x0000561eb3442a52 in Cond::Wait (this=0x561ebed5c600, mutex=...) at /build/ceph-12.2.4/src/common/Cond.h:48 #2 0x0000561eb33c6d42 in OSDService::agent_entry (this=0x561ebed5bdc8) at /build/ceph-12.2.4/src/osd/OSD.cc:629 #3 0x0000561eb3449e0d in OSDService::AgentThread::entry (this=) at /build/ceph-12.2.4/src/osd/OSD.h:665 #4 0x00007f52579b76ba in start_thread (arg=0x7f5238621700) at pthread_create.c:333 #5 0x00007f5256a2e41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 52 (Thread 0x7f5238e22700 (LWP 7734)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x0000561eb39f7c55 in Cond::Wait (mutex=..., this=0x561ebed5d460) at /build/ceph-12.2.4/src/common/Cond.h:48 #2 SafeTimer::timer_thread (this=0x561ebed5d450) at /build/ceph-12.2.4/src/common/Timer.cc:108 #3 0x0000561eb39f8d9d in SafeTimerThread::entry (this=) at /build/ceph-12.2.4/src/common/Timer.cc:30 #4 0x00007f52579b76ba in start_thread (arg=0x7f5238e22700) at pthread_create.c:333 #5 0x00007f5256a2e41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 51 (Thread 0x7f5239623700 (LWP 7733)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x0000561eb39f7c55 in Cond::Wait (mutex=..., this=0x561ebed5d328) at /build/ceph-12.2.4/src/common/Cond.h:48 #2 SafeTimer::timer_thread (this=0x561ebed5d318) at /build/ceph-12.2.4/src/common/Timer.cc:108 #3 0x0000561eb39f8d9d in SafeTimerThread::entry (this=) at /build/ceph-12.2.4/src/common/Timer.cc:30 #4 0x00007f52579b76ba in start_thread (arg=0x7f5239623700) at pthread_create.c:333 #5 0x00007f5256a2e41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 50 (Thread 0x7f5239e24700 (LWP 7732)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x0000561eb39f7c55 in Cond::Wait (mutex=..., this=0x561ebed5c778) at /build/ceph-12.2.4/src/common/Cond.h:48 #2 SafeTimer::timer_thread (this=0x561ebed5c768) at /build/ceph-12.2.4/src/common/Timer.cc:108 #3 0x0000561eb39f8d9d in SafeTimerThread::entry (this=) at /build/ceph-12.2.4/src/common/Timer.cc:30 #4 0x00007f52579b76ba in start_thread (arg=0x7f5239e24700) at pthread_create.c:333 #5 0x00007f5256a2e41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 49 (Thread 0x7f523a625700 (LWP 7731)): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 #1 0x0000561eb39f801f in Cond::WaitUntil (when=..., mutex=..., this=0x561ebed5ca78) at /build/ceph-12.2.4/src/common/Cond.h:64 #2 SafeTimer::timer_thread (this=0x561ebed5ca68) at /build/ceph-12.2.4/src/common/Timer.cc:110 #3 0x0000561eb39f8d9d in SafeTimerThread::entry (this=) at /build/ceph-12.2.4/src/common/Timer.cc:30 ---Type to continue, or q to quit--- #4 0x00007f52579b76ba in start_thread (arg=0x7f523a625700) at pthread_create.c:333 #5 0x00007f5256a2e41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 48 (Thread 0x7f523ae26700 (LWP 7730)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x0000561eb39fa479 in Cond::Wait (mutex=..., this=0x561ebed5c8e8) at /build/ceph-12.2.4/src/common/Cond.h:48 #2 Finisher::finisher_thread_entry (this=0x561ebed5c870) at /build/ceph-12.2.4/src/common/Finisher.cc:101 #3 0x00007f52579b76ba in start_thread (arg=0x7f523ae26700) at pthread_create.c:333 #4 0x00007f5256a2e41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 47 (Thread 0x7f523b627700 (LWP 7729)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x0000561eb39fa479 in Cond::Wait (mutex=..., this=0x561ebed5ce38) at /build/ceph-12.2.4/src/common/Cond.h:48 #2 Finisher::finisher_thread_entry (this=0x561ebed5cdc0) at /build/ceph-12.2.4/src/common/Finisher.cc:101 #3 0x00007f52579b76ba in start_thread (arg=0x7f523b627700) at pthread_create.c:333 #4 0x00007f5256a2e41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 46 (Thread 0x7f523be28700 (LWP 7728)): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 #1 0x0000561eb33e8b98 in Cond::WaitUntil (when=..., mutex=..., this=0x561ebed5b340) at /build/ceph-12.2.4/src/common/Cond.h:64 #2 Cond::WaitInterval (interval=..., mutex=..., this=0x561ebed5b340) at /build/ceph-12.2.4/src/common/Cond.h:73 #3 OSD::heartbeat_entry (this=0x561ebed5a000) at /build/ceph-12.2.4/src/osd/OSD.cc:5087 #4 0x0000561eb3467e5d in OSD::T_Heartbeat::entry (this=) at /build/ceph-12.2.4/src/osd/OSD.h:1536 #5 0x00007f52579b76ba in start_thread (arg=0x7f523be28700) at pthread_create.c:333 #6 0x00007f5256a2e41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 45 (Thread 0x7f523c629700 (LWP 7727)): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 #1 0x0000561eb3a021db in Cond::WaitUntil (when=..., mutex=..., this=0x561ebed5b050) at /build/ceph-12.2.4/src/common/Cond.h:64 #2 Cond::WaitInterval (interval=..., mutex=..., this=0x561ebed5b050) at /build/ceph-12.2.4/src/common/Cond.h:73 #3 ThreadPool::worker (this=0x561ebed5af70, wt=0x561ed10fb1d0) at /build/ceph-12.2.4/src/common/WorkQueue.cc:143 #4 0x0000561eb3a03880 in ThreadPool::WorkThread::entry (this=) at /build/ceph-12.2.4/src/common/WorkQueue.h:448 #5 0x00007f52579b76ba in start_thread (arg=0x7f523c629700) at pthread_create.c:333 #6 0x00007f5256a2e41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 44 (Thread 0x7f523ce2a700 (LWP 7726)): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 #1 0x0000561eb3a021db in Cond::WaitUntil (when=..., mutex=..., this=0x561ebed5ae38) at /build/ceph-12.2.4/src/common/Cond.h:64 #2 Cond::WaitInterval (interval=..., mutex=..., this=0x561ebed5ae38) at /build/ceph-12.2.4/src/common/Cond.h:73 #3 ThreadPool::worker (this=0x561ebed5ad58, wt=0x561ed10fb170) at /build/ceph-12.2.4/src/common/WorkQueue.cc:143 #4 0x0000561eb3a03880 in ThreadPool::WorkThread::entry (this=) at /build/ceph-12.2.4/src/common/WorkQueue.h:448 #5 0x00007f52579b76ba in start_thread (arg=0x7f523ce2a700) at pthread_create.c:333 #6 0x00007f5256a2e41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 43 (Thread 0x7f523d62b700 (LWP 7725)): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 ---Type to continue, or q to quit--- #1 0x0000561eb341e9ac in Cond::WaitUntil (when=..., mutex=..., this=0x561ebe8e9570) at /build/ceph-12.2.4/src/common/Cond.h:64 #2 Cond::WaitInterval (interval=..., mutex=..., this=0x561ebe8e9570) at /build/ceph-12.2.4/src/common/Cond.h:73 #3 OSD::ShardedOpWQ::_process (this=0x561ebed5b620, thread_index=, hb=0x561ed17ac1e0) at /build/ceph-12.2.4/src/osd/OSD.cc:10370 #4 0x0000561eb3a00664 in ShardedThreadPool::shardedthreadpool_worker (this=0x561ebed5abd0, thread_index=4) at /build/ceph-12.2.4/src/common/WorkQueue.cc:339 #5 0x0000561eb3a036a0 in ShardedThreadPool::WorkThreadSharded::entry (this=) at /build/ceph-12.2.4/src/common/WorkQueue.h:689 #6 0x00007f52579b76ba in start_thread (arg=0x7f523d62b700) at pthread_create.c:333 #7 0x00007f5256a2e41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 42 (Thread 0x7f523de2c700 (LWP 7724)): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 #1 0x0000561eb341e9ac in Cond::WaitUntil (when=..., mutex=..., this=0x561ebe8e93f0) at /build/ceph-12.2.4/src/common/Cond.h:64 #2 Cond::WaitInterval (interval=..., mutex=..., this=0x561ebe8e93f0) at /build/ceph-12.2.4/src/common/Cond.h:73 #3 OSD::ShardedOpWQ::_process (this=0x561ebed5b620, thread_index=, hb=0x561ed17ac190) at /build/ceph-12.2.4/src/osd/OSD.cc:10370 #4 0x0000561eb3a00664 in ShardedThreadPool::shardedthreadpool_worker (this=0x561ebed5abd0, thread_index=3) at /build/ceph-12.2.4/src/common/WorkQueue.cc:339 #5 0x0000561eb3a036a0 in ShardedThreadPool::WorkThreadSharded::entry (this=) at /build/ceph-12.2.4/src/common/WorkQueue.h:689 #6 0x00007f52579b76ba in start_thread (arg=0x7f523de2c700) at pthread_create.c:333 #7 0x00007f5256a2e41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 41 (Thread 0x7f523e62d700 (LWP 7723)): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 #1 0x0000561eb341e9ac in Cond::WaitUntil (when=..., mutex=..., this=0x561ebe8e9270) at /build/ceph-12.2.4/src/common/Cond.h:64 #2 Cond::WaitInterval (interval=..., mutex=..., this=0x561ebe8e9270) at /build/ceph-12.2.4/src/common/Cond.h:73 #3 OSD::ShardedOpWQ::_process (this=0x561ebed5b620, thread_index=, hb=0x561ed17ac140) at /build/ceph-12.2.4/src/osd/OSD.cc:10370 #4 0x0000561eb3a00664 in ShardedThreadPool::shardedthreadpool_worker (this=0x561ebed5abd0, thread_index=2) at /build/ceph-12.2.4/src/common/WorkQueue.cc:339 #5 0x0000561eb3a036a0 in ShardedThreadPool::WorkThreadSharded::entry (this=) at /build/ceph-12.2.4/src/common/WorkQueue.h:689 #6 0x00007f52579b76ba in start_thread (arg=0x7f523e62d700) at pthread_create.c:333 #7 0x00007f5256a2e41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 40 (Thread 0x7f523ee2e700 (LWP 7722)): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 #1 0x0000561eb341e9ac in Cond::WaitUntil (when=..., mutex=..., this=0x561ebe8e90f0) at /build/ceph-12.2.4/src/common/Cond.h:64 #2 Cond::WaitInterval (interval=..., mutex=..., this=0x561ebe8e90f0) at /build/ceph-12.2.4/src/common/Cond.h:73 #3 OSD::ShardedOpWQ::_process (this=0x561ebed5b620, thread_index=, hb=0x561ed17ac0f0) at /build/ceph-12.2.4/src/osd/OSD.cc:10370 #4 0x0000561eb3a00664 in ShardedThreadPool::shardedthreadpool_worker (this=0x561ebed5abd0, thread_index=1) at /build/ceph-12.2.4/src/common/WorkQueue.cc:339 #5 0x0000561eb3a036a0 in ShardedThreadPool::WorkThreadSharded::entry (this=) at /build/ceph-12.2.4/src/common/WorkQueue.h:689 #6 0x00007f52579b76ba in start_thread (arg=0x7f523ee2e700) at pthread_create.c:333 #7 0x00007f5256a2e41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 39 (Thread 0x7f523f62f700 (LWP 7721)): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 #1 0x0000561eb341e9ac in Cond::WaitUntil (when=..., mutex=..., this=0x561ebe8e8f70) at /build/ceph-12.2.4/src/common/Cond.h:64 #2 Cond::WaitInterval (interval=..., mutex=..., this=0x561ebe8e8f70) at /build/ceph-12.2.4/src/common/Cond.h:73 #3 OSD::ShardedOpWQ::_process (this=0x561ebed5b620, thread_index=, hb=0x561ed17ac0a0) at /build/ceph-12.2.4/src/osd/OSD.cc:10370 #4 0x0000561eb3a00664 in ShardedThreadPool::shardedthreadpool_worker (this=0x561ebed5abd0, thread_index=0) at /build/ceph-12.2.4/src/common/WorkQueue.cc:339 #5 0x0000561eb3a036a0 in ShardedThreadPool::WorkThreadSharded::entry (this=) at /build/ceph-12.2.4/src/common/WorkQueue.h:689 #6 0x00007f52579b76ba in start_thread (arg=0x7f523f62f700) at pthread_create.c:333 ---Type to continue, or q to quit--- #7 0x00007f5256a2e41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 38 (Thread 0x7f523fe30700 (LWP 7720)): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 #1 0x0000561eb3a021db in Cond::WaitUntil (when=..., mutex=..., this=0x561ebed5aa98) at /build/ceph-12.2.4/src/common/Cond.h:64 #2 Cond::WaitInterval (interval=..., mutex=..., this=0x561ebed5aa98) at /build/ceph-12.2.4/src/common/Cond.h:73 #3 ThreadPool::worker (this=0x561ebed5a9b8, wt=0x561ed10fb110) at /build/ceph-12.2.4/src/common/WorkQueue.cc:143 #4 0x0000561eb3a03880 in ThreadPool::WorkThread::entry (this=) at /build/ceph-12.2.4/src/common/WorkQueue.h:448 #5 0x00007f52579b76ba in start_thread (arg=0x7f523fe30700) at pthread_create.c:333 #6 0x00007f5256a2e41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 37 (Thread 0x7f5240631700 (LWP 7719)): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 #1 0x0000561eb3a021db in Cond::WaitUntil (when=..., mutex=..., this=0x561ebed5aa98) at /build/ceph-12.2.4/src/common/Cond.h:64 #2 Cond::WaitInterval (interval=..., mutex=..., this=0x561ebed5aa98) at /build/ceph-12.2.4/src/common/Cond.h:73 #3 ThreadPool::worker (this=0x561ebed5a9b8, wt=0x561ed10fb0e0) at /build/ceph-12.2.4/src/common/WorkQueue.cc:143 #4 0x0000561eb3a03880 in ThreadPool::WorkThread::entry (this=) at /build/ceph-12.2.4/src/common/WorkQueue.h:448 #5 0x00007f52579b76ba in start_thread (arg=0x7f5240631700) at pthread_create.c:333 #6 0x00007f5256a2e41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 36 (Thread 0x7f5240e32700 (LWP 7718)): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 #1 0x0000561eb39f801f in Cond::WaitUntil (when=..., mutex=..., this=0x561ebed5a480) at /build/ceph-12.2.4/src/common/Cond.h:64 #2 SafeTimer::timer_thread (this=0x561ebed5a470) at /build/ceph-12.2.4/src/common/Timer.cc:110 #3 0x0000561eb39f8d9d in SafeTimerThread::entry (this=) at /build/ceph-12.2.4/src/common/Timer.cc:30 #4 0x00007f52579b76ba in start_thread (arg=0x7f5240e32700) at pthread_create.c:333 #5 0x00007f5256a2e41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 35 (Thread 0x7f5241633700 (LWP 7717)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x0000561eb39fa479 in Cond::Wait (mutex=..., this=0x7fffd64a9d70) at /build/ceph-12.2.4/src/common/Cond.h:48 #2 Finisher::finisher_thread_entry (this=0x7fffd64a9cf8) at /build/ceph-12.2.4/src/common/Finisher.cc:101 #3 0x00007f52579b76ba in start_thread (arg=0x7f5241633700) at pthread_create.c:333 #4 0x00007f5256a2e41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 34 (Thread 0x7f5241e34700 (LWP 7716)): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 #1 0x0000561eb39f801f in Cond::WaitUntil (when=..., mutex=..., this=0x7fffd64a9c40) at /build/ceph-12.2.4/src/common/Cond.h:64 #2 SafeTimer::timer_thread (this=0x7fffd64a9c30) at /build/ceph-12.2.4/src/common/Timer.cc:110 #3 0x0000561eb39f8d9d in SafeTimerThread::entry (this=) at /build/ceph-12.2.4/src/common/Timer.cc:30 #4 0x00007f52579b76ba in start_thread (arg=0x7f5241e34700) at pthread_create.c:333 #5 0x00007f5256a2e41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 33 (Thread 0x7f5242635700 (LWP 7715)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x0000561eb3cc5ef3 in Cond::Wait (mutex=..., this=0x561ebebad3c8) at /build/ceph-12.2.4/src/common/Cond.h:48 ---Type to continue, or q to quit--- #2 DispatchQueue::run_local_delivery (this=0x561ebebad180) at /build/ceph-12.2.4/src/msg/DispatchQueue.cc:113 #3 0x0000561eb3a8e4cd in DispatchQueue::LocalDeliveryThread::entry (this=) at /build/ceph-12.2.4/src/msg/DispatchQueue.h:115 #4 0x00007f52579b76ba in start_thread (arg=0x7f5242635700) at pthread_create.c:333 #5 0x00007f5256a2e41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 32 (Thread 0x7f5242e36700 (LWP 7714)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x0000561eb3cc30ba in Cond::Wait (mutex=..., this=0x561ebebad200) at /build/ceph-12.2.4/src/common/Cond.h:48 #2 DispatchQueue::entry (this=0x561ebebad180) at /build/ceph-12.2.4/src/msg/DispatchQueue.cc:208 #3 0x0000561eb3a8e3ed in DispatchQueue::DispatchThread::entry (this=) at /build/ceph-12.2.4/src/msg/DispatchQueue.h:101 #4 0x00007f52579b76ba in start_thread (arg=0x7f5242e36700) at pthread_create.c:333 #5 0x00007f5256a2e41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 31 (Thread 0x7f5243637700 (LWP 7713)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x0000561eb3cc5ef3 in Cond::Wait (mutex=..., this=0x561ebebac3c8) at /build/ceph-12.2.4/src/common/Cond.h:48 #2 DispatchQueue::run_local_delivery (this=0x561ebebac180) at /build/ceph-12.2.4/src/msg/DispatchQueue.cc:113 #3 0x0000561eb3a8e4cd in DispatchQueue::LocalDeliveryThread::entry (this=) at /build/ceph-12.2.4/src/msg/DispatchQueue.h:115 #4 0x00007f52579b76ba in start_thread (arg=0x7f5243637700) at pthread_create.c:333 #5 0x00007f5256a2e41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 30 (Thread 0x7f5243e38700 (LWP 7712)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x0000561eb3cc30ba in Cond::Wait (mutex=..., this=0x561ebebac200) at /build/ceph-12.2.4/src/common/Cond.h:48 #2 DispatchQueue::entry (this=0x561ebebac180) at /build/ceph-12.2.4/src/msg/DispatchQueue.cc:208 #3 0x0000561eb3a8e3ed in DispatchQueue::DispatchThread::entry (this=) at /build/ceph-12.2.4/src/msg/DispatchQueue.h:101 #4 0x00007f52579b76ba in start_thread (arg=0x7f5243e38700) at pthread_create.c:333 #5 0x00007f5256a2e41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 29 (Thread 0x7f5244639700 (LWP 7711)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x0000561eb3cc5ef3 in Cond::Wait (mutex=..., this=0x561ebebacbc8) at /build/ceph-12.2.4/src/common/Cond.h:48 #2 DispatchQueue::run_local_delivery (this=0x561ebebac980) at /build/ceph-12.2.4/src/msg/DispatchQueue.cc:113 #3 0x0000561eb3a8e4cd in DispatchQueue::LocalDeliveryThread::entry (this=) at /build/ceph-12.2.4/src/msg/DispatchQueue.h:115 #4 0x00007f52579b76ba in start_thread (arg=0x7f5244639700) at pthread_create.c:333 #5 0x00007f5256a2e41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 28 (Thread 0x7f5244e3a700 (LWP 7710)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x0000561eb3cc30ba in Cond::Wait (mutex=..., this=0x561ebebaca00) at /build/ceph-12.2.4/src/common/Cond.h:48 #2 DispatchQueue::entry (this=0x561ebebac980) at /build/ceph-12.2.4/src/msg/DispatchQueue.cc:208 #3 0x0000561eb3a8e3ed in DispatchQueue::DispatchThread::entry (this=) at /build/ceph-12.2.4/src/msg/DispatchQueue.h:101 #4 0x00007f52579b76ba in start_thread (arg=0x7f5244e3a700) at pthread_create.c:333 #5 0x00007f5256a2e41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 27 (Thread 0x7f524563b700 (LWP 7709)): ---Type to continue, or q to quit--- #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x0000561eb3cc5ef3 in Cond::Wait (mutex=..., this=0x561ebebab3c8) at /build/ceph-12.2.4/src/common/Cond.h:48 #2 DispatchQueue::run_local_delivery (this=0x561ebebab180) at /build/ceph-12.2.4/src/msg/DispatchQueue.cc:113 #3 0x0000561eb3a8e4cd in DispatchQueue::LocalDeliveryThread::entry (this=) at /build/ceph-12.2.4/src/msg/DispatchQueue.h:115 #4 0x00007f52579b76ba in start_thread (arg=0x7f524563b700) at pthread_create.c:333 #5 0x00007f5256a2e41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 26 (Thread 0x7f5247eba700 (LWP 7708)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x0000561eb3cc30ba in Cond::Wait (mutex=..., this=0x561ebebab200) at /build/ceph-12.2.4/src/common/Cond.h:48 #2 DispatchQueue::entry (this=0x561ebebab180) at /build/ceph-12.2.4/src/msg/DispatchQueue.cc:208 #3 0x0000561eb3a8e3ed in DispatchQueue::DispatchThread::entry (this=) at /build/ceph-12.2.4/src/msg/DispatchQueue.h:101 #4 0x00007f52579b76ba in start_thread (arg=0x7f5247eba700) at pthread_create.c:333 #5 0x00007f5256a2e41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 25 (Thread 0x7f52486bb700 (LWP 7707)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x0000561eb3cc5ef3 in Cond::Wait (mutex=..., this=0x561ebebabbc8) at /build/ceph-12.2.4/src/common/Cond.h:48 #2 DispatchQueue::run_local_delivery (this=0x561ebebab980) at /build/ceph-12.2.4/src/msg/DispatchQueue.cc:113 #3 0x0000561eb3a8e4cd in DispatchQueue::LocalDeliveryThread::entry (this=) at /build/ceph-12.2.4/src/msg/DispatchQueue.h:115 #4 0x00007f52579b76ba in start_thread (arg=0x7f52486bb700) at pthread_create.c:333 #5 0x00007f5256a2e41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 24 (Thread 0x7f5248ebc700 (LWP 7706)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x0000561eb3cc30ba in Cond::Wait (mutex=..., this=0x561ebebaba00) at /build/ceph-12.2.4/src/common/Cond.h:48 #2 DispatchQueue::entry (this=0x561ebebab980) at /build/ceph-12.2.4/src/msg/DispatchQueue.cc:208 #3 0x0000561eb3a8e3ed in DispatchQueue::DispatchThread::entry (this=) at /build/ceph-12.2.4/src/msg/DispatchQueue.h:101 #4 0x00007f52579b76ba in start_thread (arg=0x7f5248ebc700) at pthread_create.c:333 #5 0x00007f5256a2e41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 23 (Thread 0x7f52496bd700 (LWP 7705)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x0000561eb3cc5ef3 in Cond::Wait (mutex=..., this=0x561ebebaabc8) at /build/ceph-12.2.4/src/common/Cond.h:48 #2 DispatchQueue::run_local_delivery (this=0x561ebebaa980) at /build/ceph-12.2.4/src/msg/DispatchQueue.cc:113 #3 0x0000561eb3a8e4cd in DispatchQueue::LocalDeliveryThread::entry (this=) at /build/ceph-12.2.4/src/msg/DispatchQueue.h:115 #4 0x00007f52579b76ba in start_thread (arg=0x7f52496bd700) at pthread_create.c:333 #5 0x00007f5256a2e41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 22 (Thread 0x7f5249ebe700 (LWP 7704)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x0000561eb3cc30ba in Cond::Wait (mutex=..., this=0x561ebebaaa00) at /build/ceph-12.2.4/src/common/Cond.h:48 #2 DispatchQueue::entry (this=0x561ebebaa980) at /build/ceph-12.2.4/src/msg/DispatchQueue.cc:208 #3 0x0000561eb3a8e3ed in DispatchQueue::DispatchThread::entry (this=) at /build/ceph-12.2.4/src/msg/DispatchQueue.h:101 #4 0x00007f52579b76ba in start_thread (arg=0x7f5249ebe700) at pthread_create.c:333 #5 0x00007f5256a2e41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 ---Type to continue, or q to quit--- Thread 21 (Thread 0x7f524a6bf700 (LWP 7703)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x0000561eb3cc5ef3 in Cond::Wait (mutex=..., this=0x561ebebaa3c8) at /build/ceph-12.2.4/src/common/Cond.h:48 #2 DispatchQueue::run_local_delivery (this=0x561ebebaa180) at /build/ceph-12.2.4/src/msg/DispatchQueue.cc:113 #3 0x0000561eb3a8e4cd in DispatchQueue::LocalDeliveryThread::entry (this=) at /build/ceph-12.2.4/src/msg/DispatchQueue.h:115 #4 0x00007f52579b76ba in start_thread (arg=0x7f524a6bf700) at pthread_create.c:333 #5 0x00007f5256a2e41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 20 (Thread 0x7f524aec0700 (LWP 7702)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x0000561eb3cc30ba in Cond::Wait (mutex=..., this=0x561ebebaa200) at /build/ceph-12.2.4/src/common/Cond.h:48 #2 DispatchQueue::entry (this=0x561ebebaa180) at /build/ceph-12.2.4/src/msg/DispatchQueue.cc:208 #3 0x0000561eb3a8e3ed in DispatchQueue::DispatchThread::entry (this=) at /build/ceph-12.2.4/src/msg/DispatchQueue.h:101 #4 0x00007f52579b76ba in start_thread (arg=0x7f524aec0700) at pthread_create.c:333 #5 0x00007f5256a2e41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 19 (Thread 0x7f524d64b700 (LWP 7701)): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 #1 0x0000561eb383df0d in Cond::WaitUntil (when=..., mutex=..., this=0x561ebeb98bf8) at /build/ceph-12.2.4/src/common/Cond.h:64 #2 Cond::WaitInterval (interval=..., mutex=..., this=0x561ebeb98bf8) at /build/ceph-12.2.4/src/common/Cond.h:73 #3 BlueStore::MempoolThread::entry (this=0x561ebeb98bc8) at /build/ceph-12.2.4/src/os/bluestore/BlueStore.cc:3374 #4 0x00007f52579b76ba in start_thread (arg=0x7f524d64b700) at pthread_create.c:333 #5 0x00007f5256a2e41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 18 (Thread 0x7f524763f700 (LWP 7700)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x00007f52572c391c in std::condition_variable::wait(std::unique_lock&) () from target:/usr/lib/x86_64-linux-gnu/libstdc++.so.6 #2 0x0000561eb3874ac0 in BlueStore::_kv_finalize_thread (this=0x561ebeb98000) at /build/ceph-12.2.4/src/os/bluestore/BlueStore.cc:8706 #3 0x0000561eb38cd83d in BlueStore::KVFinalizeThread::entry (this=) at /build/ceph-12.2.4/src/os/bluestore/BlueStore.h:1790 #4 0x00007f52579b76ba in start_thread (arg=0x7f524763f700) at pthread_create.c:333 #5 0x00007f5256a2e41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 17 (Thread 0x7f5246e3e700 (LWP 7699)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x00007f52572c391c in std::condition_variable::wait(std::unique_lock&) () from target:/usr/lib/x86_64-linux-gnu/libstdc++.so.6 #2 0x0000561eb389057a in BlueStore::_kv_sync_thread (this=) at /build/ceph-12.2.4/src/os/bluestore/BlueStore.cc:8457 #3 0x0000561eb38d308d in BlueStore::KVSyncThread::entry (this=) at /build/ceph-12.2.4/src/os/bluestore/BlueStore.h:1782 #4 0x00007f52579b76ba in start_thread (arg=0x7f5246e3e700) at pthread_create.c:333 #5 0x00007f5256a2e41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 16 (Thread 0x7f524663d700 (LWP 7698)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x0000561eb39fa479 in Cond::Wait (mutex=..., this=0x561ebe8daab8) at /build/ceph-12.2.4/src/common/Cond.h:48 #2 Finisher::finisher_thread_entry (this=0x561ebe8daa40) at /build/ceph-12.2.4/src/common/Finisher.cc:101 #3 0x00007f52579b76ba in start_thread (arg=0x7f524663d700) at pthread_create.c:333 ---Type to continue, or q to quit--- #4 0x00007f5256a2e41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 15 (Thread 0x7f5245e3c700 (LWP 7697)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x0000561eb39fa479 in Cond::Wait (mutex=..., this=0x561ebeb98548) at /build/ceph-12.2.4/src/common/Cond.h:48 #2 Finisher::finisher_thread_entry (this=0x561ebeb984d0) at /build/ceph-12.2.4/src/common/Finisher.cc:101 #3 0x00007f52579b76ba in start_thread (arg=0x7f5245e3c700) at pthread_create.c:333 #4 0x00007f5256a2e41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 14 (Thread 0x7f524de4c700 (LWP 7679)): #0 0x00007f5258a4164a in ?? () from target:/lib/x86_64-linux-gnu/libaio.so.1 #1 0x0000561eb39b1d57 in aio_queue_t::get_next_completed (this=this@entry=0x561ebe901378, timeout_ms=, paio=paio@entry=0x7f524de4a5d0, max=16) at /build/ceph-12.2.4/src/os/bluestore/aio.cc:78 #2 0x0000561eb399a3dc in KernelDevice::_aio_thread (this=0x561ebe901200) at /build/ceph-12.2.4/src/os/bluestore/KernelDevice.cc:350 #3 0x0000561eb39a09cd in KernelDevice::AioCompletionThread::entry (this=) at /build/ceph-12.2.4/src/os/bluestore/KernelDevice.h:49 #4 0x00007f52579b76ba in start_thread (arg=0x7f524de4c700) at pthread_create.c:333 #5 0x00007f5256a2e41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 13 (Thread 0x7f524e64d700 (LWP 7678)): #0 0x00007f5258a4164a in ?? () from target:/lib/x86_64-linux-gnu/libaio.so.1 #1 0x0000561eb39b1d57 in aio_queue_t::get_next_completed (this=this@entry=0x561ebe901138, timeout_ms=, paio=paio@entry=0x7f524e64b5d0, max=16) at /build/ceph-12.2.4/src/os/bluestore/aio.cc:78 #2 0x0000561eb399a3dc in KernelDevice::_aio_thread (this=0x561ebe900fc0) at /build/ceph-12.2.4/src/os/bluestore/KernelDevice.cc:350 #3 0x0000561eb39a09cd in KernelDevice::AioCompletionThread::entry (this=) at /build/ceph-12.2.4/src/os/bluestore/KernelDevice.h:49 #4 0x00007f52579b76ba in start_thread (arg=0x7f524e64d700) at pthread_create.c:333 #5 0x00007f5256a2e41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 12 (Thread 0x7f524ee4e700 (LWP 7677)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x0000561eb39f7c55 in Cond::Wait (mutex=..., this=0x561ebed5cd00) at /build/ceph-12.2.4/src/common/Cond.h:48 #2 SafeTimer::timer_thread (this=0x561ebed5ccf0) at /build/ceph-12.2.4/src/common/Timer.cc:108 #3 0x0000561eb39f8d9d in SafeTimerThread::entry (this=) at /build/ceph-12.2.4/src/common/Timer.cc:30 #4 0x00007f52579b76ba in start_thread (arg=0x7f524ee4e700) at pthread_create.c:333 #5 0x00007f5256a2e41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 11 (Thread 0x7f524f64f700 (LWP 7676)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x0000561eb39f7c55 in Cond::Wait (mutex=..., this=0x561ebed5cbb8) at /build/ceph-12.2.4/src/common/Cond.h:48 #2 SafeTimer::timer_thread (this=0x561ebed5cba8) at /build/ceph-12.2.4/src/common/Timer.cc:108 #3 0x0000561eb39f8d9d in SafeTimerThread::entry (this=) at /build/ceph-12.2.4/src/common/Timer.cc:30 #4 0x00007f52579b76ba in start_thread (arg=0x7f524f64f700) at pthread_create.c:333 #5 0x00007f5256a2e41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 10 (Thread 0x7f524fe50700 (LWP 7675)): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 #1 0x0000561eb39f801f in Cond::WaitUntil (when=..., mutex=..., this=0x561ebed5a1d0) at /build/ceph-12.2.4/src/common/Cond.h:64 ---Type to continue, or q to quit--- #2 SafeTimer::timer_thread (this=0x561ebed5a1c0) at /build/ceph-12.2.4/src/common/Timer.cc:110 #3 0x0000561eb39f8d9d in SafeTimerThread::entry (this=) at /build/ceph-12.2.4/src/common/Timer.cc:30 #4 0x00007f52579b76ba in start_thread (arg=0x7f524fe50700) at pthread_create.c:333 #5 0x00007f5256a2e41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 9 (Thread 0x7f5250651700 (LWP 7674)): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 #1 0x0000561eb39f801f in Cond::WaitUntil (when=..., mutex=..., this=0x561ebed5a098) at /build/ceph-12.2.4/src/common/Cond.h:64 #2 SafeTimer::timer_thread (this=0x561ebed5a088) at /build/ceph-12.2.4/src/common/Timer.cc:110 #3 0x0000561eb39f8d9d in SafeTimerThread::entry (this=) at /build/ceph-12.2.4/src/common/Timer.cc:30 #4 0x00007f52579b76ba in start_thread (arg=0x7f5250651700) at pthread_create.c:333 #5 0x00007f5256a2e41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 8 (Thread 0x7f5250e52700 (LWP 7673)): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 #1 0x0000561eb345cee2 in __gthread_cond_timedwait (__abs_timeout=0x7f5250e507e0, __mutex=, __cond=0x561ebecaba80) at /usr/include/x86_64-linux-gnu/c++/5/bits/gthr-default.h:871 #2 std::condition_variable::__wait_until_impl > > (__atime=, __lock=..., this=0x561ebecaba80) at /usr/include/c++/5/condition_variable:165 #3 std::condition_variable::wait_until > > (__atime=..., __lock=..., this=0x561ebecaba80) at /usr/include/c++/5/condition_variable:118 #4 ceph::timer_detail::timer::timer_thread (this=0x561ebecaba18) at /build/ceph-12.2.4/src/common/ceph_timer.h:144 #5 0x00007f52572c8c80 in ?? () from target:/usr/lib/x86_64-linux-gnu/libstdc++.so.6 #6 0x00007f52579b76ba in start_thread (arg=0x7f5250e52700) at pthread_create.c:333 #7 0x00007f5256a2e41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 7 (Thread 0x7f5251ea0700 (LWP 7668)): #0 0x00007f5256a2274d in poll () at ../sysdeps/unix/syscall-template.S:84 #1 0x0000561eb39e1015 in poll (__timeout=-1, __nfds=2, __fds=0x7f5251e9e6d0) at /usr/include/x86_64-linux-gnu/bits/poll2.h:46 #2 AdminSocket::entry (this=0x561ebe8f8540) at /build/ceph-12.2.4/src/common/admin_socket.cc:250 #3 0x00007f52579b76ba in start_thread (arg=0x7f5251ea0700) at pthread_create.c:333 #4 0x00007f5256a2e41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 6 (Thread 0x7f52526a1700 (LWP 7667)): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:225 #1 0x0000561eb3b05210 in Cond::WaitUntil (when=..., mutex=..., this=0x561ebe8f0368) at /build/ceph-12.2.4/src/common/Cond.h:64 #2 Cond::WaitInterval (interval=..., mutex=..., this=0x561ebe8f0368) at /build/ceph-12.2.4/src/common/Cond.h:73 #3 CephContextServiceThread::entry (this=0x561ebe8f02d0) at /build/ceph-12.2.4/src/common/ceph_context.cc:135 #4 0x00007f52579b76ba in start_thread (arg=0x7f52526a1700) at pthread_create.c:333 #5 0x00007f5256a2e41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 5 (Thread 0x7f5253640700 (LWP 7666)): #0 0x00007f5256a2ea13 in epoll_wait () at ../sysdeps/unix/syscall-template.S:84 #1 0x0000561eb3c6d789 in EpollDriver::event_wait (this=0x561ebebe0300, fired_events=..., tvp=) at /build/ceph-12.2.4/src/msg/async/EventEpoll.cc:114 #2 0x0000561eb3a9e591 in EventCenter::process_events (this=this@entry=0x561ebe900740, timeout_microseconds=, timeout_microseconds@entry=30000000, working_dur=working_dur@entry=0x7f525363e6a0) at /build/ceph-12.2.4/src/msg/async/Event.cc:395 ---Type to continue, or q to quit--- #3 0x0000561eb3aa2c98 in NetworkStack::::operator()(void) const (__closure=0x561ebebaed98) at /build/ceph-12.2.4/src/msg/async/Stack.cc:51 #4 0x00007f52572c8c80 in ?? () from target:/usr/lib/x86_64-linux-gnu/libstdc++.so.6 #5 0x00007f52579b76ba in start_thread (arg=0x7f5253640700) at pthread_create.c:333 #6 0x00007f5256a2e41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 4 (Thread 0x7f5253e41700 (LWP 7665)): #0 0x00007f5256a2ea13 in epoll_wait () at ../sysdeps/unix/syscall-template.S:84 #1 0x0000561eb3c6d789 in EpollDriver::event_wait (this=0x561ebebe0120, fired_events=..., tvp=) at /build/ceph-12.2.4/src/msg/async/EventEpoll.cc:114 #2 0x0000561eb3a9e591 in EventCenter::process_events (this=this@entry=0x561ebe900bc0, timeout_microseconds=, timeout_microseconds@entry=30000000, working_dur=working_dur@entry=0x7f5253e3f6a0) at /build/ceph-12.2.4/src/msg/async/Event.cc:395 #3 0x0000561eb3aa2c98 in NetworkStack::::operator()(void) const (__closure=0x561ebebaed48) at /build/ceph-12.2.4/src/msg/async/Stack.cc:51 #4 0x00007f52572c8c80 in ?? () from target:/usr/lib/x86_64-linux-gnu/libstdc++.so.6 #5 0x00007f52579b76ba in start_thread (arg=0x7f5253e41700) at pthread_create.c:333 #6 0x00007f5256a2e41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 3 (Thread 0x7f5254642700 (LWP 7664)): #0 0x00007f5256a2ea13 in epoll_wait () at ../sysdeps/unix/syscall-template.S:84 #1 0x0000561eb3c6d789 in EpollDriver::event_wait (this=0x561ebeb34090, fired_events=..., tvp=) at /build/ceph-12.2.4/src/msg/async/EventEpoll.cc:114 #2 0x0000561eb3a9e591 in EventCenter::process_events (this=this@entry=0x561ebe900980, timeout_microseconds=, timeout_microseconds@entry=30000000, working_dur=working_dur@entry=0x7f52546406a0) at /build/ceph-12.2.4/src/msg/async/Event.cc:395 #3 0x0000561eb3aa2c98 in NetworkStack::::operator()(void) const (__closure=0x561ebebaecf8) at /build/ceph-12.2.4/src/msg/async/Stack.cc:51 #4 0x00007f52572c8c80 in ?? () from target:/usr/lib/x86_64-linux-gnu/libstdc++.so.6 #5 0x00007f52579b76ba in start_thread (arg=0x7f5254642700) at pthread_create.c:333 #6 0x00007f5256a2e41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 2 (Thread 0x7f5255683700 (LWP 7663)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x0000561eb3a0a6db in ceph::logging::Log::entry (this=0x561ebe8f8000) at /build/ceph-12.2.4/src/log/Log.cc:459 #2 0x00007f52579b76ba in start_thread (arg=0x7f5255683700) at pthread_create.c:333 #3 0x00007f5256a2e41d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109 Thread 1 (Thread 0x7f52594dfe00 (LWP 7659)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x0000561eb3a9a067 in Cond::Wait (mutex=..., this=0x561ebebaa760) at /build/ceph-12.2.4/src/common/Cond.h:48 #2 AsyncMessenger::wait (this=0x561ebebaa000) at /build/ceph-12.2.4/src/msg/async/AsyncMessenger.cc:481 #3 0x0000561eb3336f9a in main (argc=, argv=) at /build/ceph-12.2.4/src/ceph_osd.cc:656