(gdb) thread apply all bt Thread 94 (Thread 0xffff7345d9f0 (LWP 126909)): #0 0x0000ffff8b1f922c in futex_wait_cancelable (private=, expected=0, futex_word=0xaaaafbb859a8) at ../sysdeps/unix/sysv/linux/futex-internal.h:88 #1 __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafbb85950, cond=0xaaaafbb85980) at pthread_cond_wait.c:502 #2 __pthread_cond_wait (cond=0xaaaafbb85980, mutex=0xaaaafbb85950) at pthread_cond_wait.c:655 #3 0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #4 0x0000aaaad28449ac in DispatchQueue::entry (this=0xaaaafbb85940) at ./src/msg/DispatchQueue.cc:210 #5 0x0000aaaad26977f0 in DispatchQueue::DispatchThread::entry (this=) at ./src/msg/DispatchQueue.h:101 #6 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4df) at pthread_create.c:463 #7 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 Thread 93 (Thread 0xffff74c609f0 (LWP 126892)): #0 0x0000ffff8b1f922c in futex_wait_cancelable (private=, expected=0, futex_word=0xaaaafbb85218) at ../sysdeps/unix/sysv/linux/futex-internal.h:88 #1 __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafbb851c0, cond=0xaaaafbb851f0) at pthread_cond_wait.c:502 #2 __pthread_cond_wait (cond=0xaaaafbb851f0, mutex=0xaaaafbb851c0) at pthread_cond_wait.c:655 #3 0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #4 0x0000aaaad284471c in DispatchQueue::run_local_delivery (this=0xaaaafbb85040) at ./src/msg/DispatchQueue.cc:117 #5 0x0000aaaad26978b0 in DispatchQueue::LocalDeliveryThread::entry (this=) at ./src/msg/DispatchQueue.h:115 #6 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4ff) at pthread_create.c:463 #7 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 Thread 92 (Thread 0xffff5e4339f0 (LWP 126952)): #0 0x0000ffff8b1f922c in futex_wait_cancelable (private=, expected=0, futex_word=0xaaaafafbadb0) at ../sysdeps/unix/sysv/linux/futex-internal.h:88 #1 __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafafbad58, cond=0xaaaafafbad88) at pthread_cond_wait.c:502 #2 __pthread_cond_wait (cond=0xaaaafafbad88, mutex=0xaaaafafbad58) at pthread_cond_wait.c:655 #3 0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #4 0x0000aaaad1ee9ab4 in OSD::ShardedOpWQ::_process (this=0xaaaafaf6ef10, thread_index=34, hb=0xaaab5091a080) at ./src/osd/OSD.cc:10423 #5 0x0000aaaad2523928 in ShardedThreadPool::shardedthreadpool_worker (this=0xaaaafaf6ea48, thread_index=34) at ./src/common/WorkQueue.cc:311 #6 0x0000aaaad2526268 in ShardedThreadPool::WorkThreadSharded::entry (this=) at ./src/common/WorkQueue.h:706 #7 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4bf) at pthread_create.c:463 #8 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 Thread 91 (Thread 0xffff5dc329f0 (LWP 126953)): #0 0x0000ffff8b1f922c in futex_wait_cancelable (private=, expected=0, futex_word=0xaaaafafbb030) at ../sysdeps/unix/sysv/linux/futex-internal.h:88 #1 __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafafbafd8, cond=0xaaaafafbb008) at pthread_cond_wait.c:502 #2 __pthread_cond_wait (cond=0xaaaafafbb008, mutex=0xaaaafafbafd8) at pthread_cond_wait.c:655 #3 0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #4 0x0000aaaad1ee9ab4 in OSD::ShardedOpWQ::_process (this=0xaaaafaf6ef10, thread_index=35, hb=0xaaab5091a0d0) at ./src/osd/OSD.cc:10423 #5 0x0000aaaad2523928 in ShardedThreadPool::shardedthreadpool_worker (this=0xaaaafaf6ea48, thread_index=35) at ./src/common/WorkQueue.cc:311 #6 0x0000aaaad2526268 in ShardedThreadPool::WorkThreadSharded::entry (this=) at ./src/common/WorkQueue.h:706 #7 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4bf) at pthread_create.c:463 #8 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 Thread 90 (Thread 0xffff5d4319f0 (LWP 126954)): #0 0x0000ffff8b1f922c in futex_wait_cancelable (private=, expected=0, futex_word=0xaaaafafbb2b0) at ../sysdeps/unix/sysv/linux/futex-internal.h:88 #1 __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafafbb258, cond=0xaaaafafbb288) at pthread_cond_wait.c:502 ---Type to continue, or q to quit--- #2 __pthread_cond_wait (cond=0xaaaafafbb288, mutex=0xaaaafafbb258) at pthread_cond_wait.c:655 #3 0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #4 0x0000aaaad1ee9ab4 in OSD::ShardedOpWQ::_process (this=0xaaaafaf6ef10, thread_index=36, hb=0xaaab5091a1c0) at ./src/osd/OSD.cc:10423 #5 0x0000aaaad2523928 in ShardedThreadPool::shardedthreadpool_worker (this=0xaaaafaf6ea48, thread_index=36) at ./src/common/WorkQueue.cc:311 #6 0x0000aaaad2526268 in ShardedThreadPool::WorkThreadSharded::entry (this=) at ./src/common/WorkQueue.h:706 #7 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4bf) at pthread_create.c:463 #8 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 Thread 89 (Thread 0xffff5cc309f0 (LWP 126955)): #0 0x0000ffff8b1f922c in futex_wait_cancelable (private=, expected=0, futex_word=0xaaaafafbb530) at ../sysdeps/unix/sysv/linux/futex-internal.h:88 #1 __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafafbb4d8, cond=0xaaaafafbb508) at pthread_cond_wait.c:502 #2 __pthread_cond_wait (cond=0xaaaafafbb508, mutex=0xaaaafafbb4d8) at pthread_cond_wait.c:655 #3 0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #4 0x0000aaaad1ee9ab4 in OSD::ShardedOpWQ::_process (this=0xaaaafaf6ef10, thread_index=37, hb=0xaaab50919900) at ./src/osd/OSD.cc:10423 #5 0x0000aaaad2523928 in ShardedThreadPool::shardedthreadpool_worker (this=0xaaaafaf6ea48, thread_index=37) at ./src/common/WorkQueue.cc:311 #6 0x0000aaaad2526268 in ShardedThreadPool::WorkThreadSharded::entry (this=) at ./src/common/WorkQueue.h:706 #7 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4bf) at pthread_create.c:463 #8 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 Thread 88 (Thread 0xffff604379f0 (LWP 126948)): #0 0x0000ffff8b1f922c in futex_wait_cancelable (private=, expected=0, futex_word=0xaaaafafba3b0) at ../sysdeps/unix/sysv/linux/futex-internal.h:88 #1 __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafafba358, cond=0xaaaafafba388) at pthread_cond_wait.c:502 #2 __pthread_cond_wait (cond=0xaaaafafba388, mutex=0xaaaafafba358) at pthread_cond_wait.c:655 #3 0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #4 0x0000aaaad1ee9ab4 in OSD::ShardedOpWQ::_process (this=0xaaaafaf6ef10, thread_index=30, hb=0xaaab5091a120) at ./src/osd/OSD.cc:10423 #5 0x0000aaaad2523928 in ShardedThreadPool::shardedthreadpool_worker (this=0xaaaafaf6ea48, thread_index=30) at ./src/common/WorkQueue.cc:311 #6 0x0000aaaad2526268 in ShardedThreadPool::WorkThreadSharded::entry (this=) at ./src/common/WorkQueue.h:706 #7 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4bf) at pthread_create.c:463 #8 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 Thread 87 (Thread 0xffff674459f0 (LWP 126934)): #0 0x0000ffff8b1f922c in futex_wait_cancelable (private=, expected=0, futex_word=0xaaaafafbb2b0) at ../sysdeps/unix/sysv/linux/futex-internal.h:88 #1 __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafafbb258, cond=0xaaaafafbb288) at pthread_cond_wait.c:502 #2 __pthread_cond_wait (cond=0xaaaafafbb288, mutex=0xaaaafafbb258) at pthread_cond_wait.c:655 #3 0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #4 0x0000aaaad1ee9ab4 in OSD::ShardedOpWQ::_process (this=0xaaaafaf6ef10, thread_index=16, hb=0xaaab5091b750) at ./src/osd/OSD.cc:10423 #5 0x0000aaaad2523928 in ShardedThreadPool::shardedthreadpool_worker (this=0xaaaafaf6ea48, thread_index=16) at ./src/common/WorkQueue.cc:311 #6 0x0000aaaad2526268 in ShardedThreadPool::WorkThreadSharded::entry (this=) at ./src/common/WorkQueue.h:706 #7 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4bf) at pthread_create.c:463 #8 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 Thread 86 (Thread 0xffff6b44d9f0 (LWP 126926)): #0 0x0000ffff8b1f922c in futex_wait_cancelable (private=, expected=0, futex_word=0xaaaafafb9eb0) at ../sysdeps/unix/sysv/linux/futex-internal.h:88 #1 __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafafb9e58, cond=0xaaaafafb9e88) at pthread_cond_wait.c:502 #2 __pthread_cond_wait (cond=0xaaaafafb9e88, mutex=0xaaaafafb9e58) at pthread_cond_wait.c:655 #3 0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 ---Type to continue, or q to quit--- #4 0x0000aaaad1ee9ab4 in OSD::ShardedOpWQ::_process (this=0xaaaafaf6ef10, thread_index=8, hb=0xaaab5091b160) at ./src/osd/OSD.cc:10423 #5 0x0000aaaad2523928 in ShardedThreadPool::shardedthreadpool_worker (this=0xaaaafaf6ea48, thread_index=8) at ./src/common/WorkQueue.cc:311 #6 0x0000aaaad2526268 in ShardedThreadPool::WorkThreadSharded::entry (this=) at ./src/common/WorkQueue.h:706 #7 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4bf) at pthread_create.c:463 #8 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 Thread 85 (Thread 0xffff684479f0 (LWP 126932)): #0 0x0000ffff8b1f922c in futex_wait_cancelable (private=, expected=0, futex_word=0xaaaafafbadb0) at ../sysdeps/unix/sysv/linux/futex-internal.h:88 #1 __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafafbad58, cond=0xaaaafafbad88) at pthread_cond_wait.c:502 #2 __pthread_cond_wait (cond=0xaaaafafbad88, mutex=0xaaaafafbad58) at pthread_cond_wait.c:655 #3 0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #4 0x0000aaaad1ee9ab4 in OSD::ShardedOpWQ::_process (this=0xaaaafaf6ef10, thread_index=14, hb=0xaaab5091b340) at ./src/osd/OSD.cc:10423 #5 0x0000aaaad2523928 in ShardedThreadPool::shardedthreadpool_worker (this=0xaaaafaf6ea48, thread_index=14) at ./src/common/WorkQueue.cc:311 #6 0x0000aaaad2526268 in ShardedThreadPool::WorkThreadSharded::entry (this=) at ./src/common/WorkQueue.h:706 #7 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4bf) at pthread_create.c:463 #8 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 Thread 84 (Thread 0xffff83c7e9f0 (LWP 125796)): #0 0x0000ffff8b1f922c in futex_wait_cancelable (private=, expected=0, futex_word=0xaaaafaf6fb90) at ../sysdeps/unix/sysv/linux/futex-internal.h:88 #1 __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafaf6fb20, cond=0xaaaafaf6fb68) at pthread_cond_wait.c:502 #2 __pthread_cond_wait (cond=0xaaaafaf6fb68, mutex=0xaaaafaf6fb20) at pthread_cond_wait.c:655 #3 0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #4 0x0000aaaad251d438 in SafeTimer::timer_thread (this=0xaaaafaf6fb50) at ./src/common/Timer.cc:106 #5 0x0000aaaad251e7f0 in SafeTimerThread::entry (this=) at ./src/common/Timer.cc:30 #6 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c55f) at pthread_create.c:463 #7 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 Thread 83 (Thread 0xffff654419f0 (LWP 126938)): #0 0x0000ffff8b1f922c in futex_wait_cancelable (private=, expected=0, futex_word=0xaaaafafb8ab0) at ../sysdeps/unix/sysv/linux/futex-internal.h:88 #1 __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafafb8a58, cond=0xaaaafafb8a88) at pthread_cond_wait.c:502 #2 __pthread_cond_wait (cond=0xaaaafafb8a88, mutex=0xaaaafafb8a58) at pthread_cond_wait.c:655 #3 0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #4 0x0000aaaad1ee9ab4 in OSD::ShardedOpWQ::_process (this=0xaaaafaf6ef10, thread_index=20, hb=0xaaab5091b7a0) at ./src/osd/OSD.cc:10423 #5 0x0000aaaad2523928 in ShardedThreadPool::shardedthreadpool_worker (this=0xaaaafaf6ea48, thread_index=20) at ./src/common/WorkQueue.cc:311 #6 0x0000aaaad2526268 in ShardedThreadPool::WorkThreadSharded::entry (this=) at ./src/common/WorkQueue.h:706 #7 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4bf) at pthread_create.c:463 #8 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 Thread 82 (Thread 0xffff75c629f0 (LWP 126890)): #0 0x0000ffff8b1f922c in futex_wait_cancelable (private=, expected=0, futex_word=0xaaaafbb83718) at ../sysdeps/unix/sysv/linux/futex-internal.h:88 #1 __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafbb836c0, cond=0xaaaafbb836f0) at pthread_cond_wait.c:502 #2 __pthread_cond_wait (cond=0xaaaafbb836f0, mutex=0xaaaafbb836c0) at pthread_cond_wait.c:655 #3 0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #4 0x0000aaaad284471c in DispatchQueue::run_local_delivery (this=0xaaaafbb83540) at ./src/msg/DispatchQueue.cc:117 #5 0x0000aaaad26978b0 in DispatchQueue::LocalDeliveryThread::entry (this=) at ./src/msg/DispatchQueue.h:115 #6 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4ff) at pthread_create.c:463 ---Type to continue, or q to quit--- #7 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 Thread 81 (Thread 0xffff72c5c9f0 (LWP 126910)): #0 0x0000ffff8b1f922c in futex_wait_cancelable (private=, expected=0, futex_word=0xaaaafbb85b18) at ../sysdeps/unix/sysv/linux/futex-internal.h:88 #1 __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafbb85ac0, cond=0xaaaafbb85af0) at pthread_cond_wait.c:502 #2 __pthread_cond_wait (cond=0xaaaafbb85af0, mutex=0xaaaafbb85ac0) at pthread_cond_wait.c:655 #3 0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #4 0x0000aaaad284471c in DispatchQueue::run_local_delivery (this=0xaaaafbb85940) at ./src/msg/DispatchQueue.cc:117 #5 0x0000aaaad26978b0 in DispatchQueue::LocalDeliveryThread::entry (this=) at ./src/msg/DispatchQueue.h:115 #6 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4ff) at pthread_create.c:463 #7 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 Thread 80 (Thread 0xffff69c4a9f0 (LWP 126929)): #0 0x0000ffff8b1f922c in futex_wait_cancelable (private=, expected=0, futex_word=0xaaaafafba630) at ../sysdeps/unix/sysv/linux/futex-internal.h:88 #1 __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafafba5d8, cond=0xaaaafafba608) at pthread_cond_wait.c:502 #2 __pthread_cond_wait (cond=0xaaaafafba608, mutex=0xaaaafafba5d8) at pthread_cond_wait.c:655 #3 0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #4 0x0000aaaad1ee9ab4 in OSD::ShardedOpWQ::_process (this=0xaaaafaf6ef10, thread_index=11, hb=0xaaab5091b2f0) at ./src/osd/OSD.cc:10423 #5 0x0000aaaad2523928 in ShardedThreadPool::shardedthreadpool_worker (this=0xaaaafaf6ea48, thread_index=11) at ./src/common/WorkQueue.cc:311 #6 0x0000aaaad2526268 in ShardedThreadPool::WorkThreadSharded::entry (this=) at ./src/common/WorkQueue.h:706 #7 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4bf) at pthread_create.c:463 #8 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 Thread 79 (Thread 0xffff6f4559f0 (LWP 126918)): #0 0x0000ffff8b1f922c in futex_wait_cancelable (private=, expected=0, futex_word=0xaaaafafb8ab0) at ../sysdeps/unix/sysv/linux/futex-internal.h:88 #1 __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafafb8a58, cond=0xaaaafafb8a88) at pthread_cond_wait.c:502 #2 __pthread_cond_wait (cond=0xaaaafafb8a88, mutex=0xaaaafafb8a58) at pthread_cond_wait.c:655 #3 0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #4 0x0000aaaad1ee9ab4 in OSD::ShardedOpWQ::_process (this=0xaaaafaf6ef10, thread_index=0, hb=0xaaaafbc10940) at ./src/osd/OSD.cc:10423 #5 0x0000aaaad2523928 in ShardedThreadPool::shardedthreadpool_worker (this=0xaaaafaf6ea48, thread_index=0) at ./src/common/WorkQueue.cc:311 #6 0x0000aaaad2526268 in ShardedThreadPool::WorkThreadSharded::entry (this=) at ./src/common/WorkQueue.h:706 #7 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4bf) at pthread_create.c:463 #8 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 Thread 78 (Thread 0xffff64c409f0 (LWP 126939)): #0 0x0000ffff8b1f922c in futex_wait_cancelable (private=, expected=0, futex_word=0xaaaafafb8d30) at ../sysdeps/unix/sysv/linux/futex-internal.h:88 #1 __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafafb8cd8, cond=0xaaaafafb8d08) at pthread_cond_wait.c:502 #2 __pthread_cond_wait (cond=0xaaaafafb8d08, mutex=0xaaaafafb8cd8) at pthread_cond_wait.c:655 #3 0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #4 0x0000aaaad1ee9ab4 in OSD::ShardedOpWQ::_process (this=0xaaaafaf6ef10, thread_index=21, hb=0xaaab50919d60) at ./src/osd/OSD.cc:10423 #5 0x0000aaaad2523928 in ShardedThreadPool::shardedthreadpool_worker (this=0xaaaafaf6ea48, thread_index=21) at ./src/common/WorkQueue.cc:311 #6 0x0000aaaad2526268 in ShardedThreadPool::WorkThreadSharded::entry (this=) at ./src/common/WorkQueue.h:706 #7 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4bf) at pthread_create.c:463 #8 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 Thread 77 (Thread 0xffff5fc369f0 (LWP 126949)): ---Type to continue, or q to quit--- #0 0x0000ffff8b1f922c in futex_wait_cancelable (private=, expected=0, futex_word=0xaaaafafba630) at ../sysdeps/unix/sysv/linux/futex-internal.h:88 #1 __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafafba5d8, cond=0xaaaafafba608) at pthread_cond_wait.c:502 #2 __pthread_cond_wait (cond=0xaaaafafba608, mutex=0xaaaafafba5d8) at pthread_cond_wait.c:655 #3 0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #4 0x0000aaaad1ee9ab4 in OSD::ShardedOpWQ::_process (this=0xaaaafaf6ef10, thread_index=31, hb=0xaaab509198b0) at ./src/osd/OSD.cc:10423 #5 0x0000aaaad2523928 in ShardedThreadPool::shardedthreadpool_worker (this=0xaaaafaf6ea48, thread_index=31) at ./src/common/WorkQueue.cc:311 #6 0x0000aaaad2526268 in ShardedThreadPool::WorkThreadSharded::entry (this=) at ./src/common/WorkQueue.h:706 #7 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4bf) at pthread_create.c:463 #8 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 Thread 76 (Thread 0xffff5bc2e9f0 (LWP 126957)): #0 0x0000ffff8b1f922c in futex_wait_cancelable (private=, expected=0, futex_word=0xaaaafafbba34) at ../sysdeps/unix/sysv/linux/futex-internal.h:88 #1 __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafafbb9d8, cond=0xaaaafafbba08) at pthread_cond_wait.c:502 #2 __pthread_cond_wait (cond=0xaaaafafbba08, mutex=0xaaaafafbb9d8) at pthread_cond_wait.c:655 #3 0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #4 0x0000aaaad1ee9ab4 in OSD::ShardedOpWQ::_process (this=0xaaaafaf6ef10, thread_index=39, hb=0xaaab509199a0) at ./src/osd/OSD.cc:10423 #5 0x0000aaaad2523928 in ShardedThreadPool::shardedthreadpool_worker (this=0xaaaafaf6ea48, thread_index=39) at ./src/common/WorkQueue.cc:311 #6 0x0000aaaad2526268 in ShardedThreadPool::WorkThreadSharded::entry (this=) at ./src/common/WorkQueue.h:706 #7 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4bf) at pthread_create.c:463 #8 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 Thread 75 (Thread 0xffff893b89f0 (LWP 124384)): #0 0x0000ffff8b1fc944 in __libc_write (fd=fd@entry=98, buf=buf@entry=0xffff893b7d90, nbytes=nbytes@entry=65) at ../sysdeps/unix/sysv/linux/write.c:27 #1 0x0000aaaad28a35e0 in Infiniband::QueuePair::send_cm_meta (this=0xaaab4fe50820, cct=0xaaaafafd4000, socket_fd=98) at ./src/msg/async/rdma/Infiniband.cc:444 #2 0x0000aaaad28ad9d4 in RDMAConnectedSocketImpl::handle_connection (this=0xaaab4f3607e0) at ./src/msg/async/rdma/RDMAConnectedSocketImpl.cc:152 #3 0x0000aaaad269d1ac in EventCenter::process_events (this=this@entry=0xaaaafbb9cb88, timeout_microseconds=, working_dur=working_dur@entry=0xffff893b8160) at /usr/include/c++/9/bits/basic_ios.h:282 #4 0x0000aaaad26a33c0 in NetworkStack::::operator() (__closure=0xaaaafbc0c518, __closure=0xaaaafbc0c518) at ./src/msg/async/Stack.cc:53 #5 std::_Function_handler >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/9/bits/std_function.h:300 #6 0x0000ffff8b03ded4 in ?? () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #7 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9aeff) at pthread_create.c:463 #8 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 Thread 74 (Thread 0xffff8a3ba9f0 (LWP 124377)): #0 0x0000ffff8b1fc944 in __libc_write (fd=, buf=0xffff8a3b8710, nbytes=408) at ../sysdeps/unix/sysv/linux/write.c:27 #1 0x0000ffff8b1abfbc in ibv_exp_cmd_create_qp () from /usr/lib/aarch64-linux-gnu/libibverbs.so.1 #2 0x0000ffff86b67c5c in create_qp (context=0xaaaafbc98300, attrx=0xb8, attrx@entry=0xffff8a3b99a0, is_exp=1, is_exp@entry=0) at src/verbs.c:2244 #3 0x0000ffff86b690a4 in mlx5_create_qp (pd=, attr=0xffff8a3b9b08) at src/verbs.c:2370 #4 0x0000ffff8b1aa76c in ibv_create_qp () from /usr/lib/aarch64-linux-gnu/libibverbs.so.1 #5 0x0000aaaad28a6f60 in Infiniband::QueuePair::init (this=this@entry=0xaaaafaf558e0) at ./src/msg/async/rdma/Infiniband.cc:334 #6 0x0000aaaad28a9038 in Infiniband::create_queue_pair (this=, cct=cct@entry=0xaaaafafd4000, tx=0xaaaafbc0d560, rx=0xaaaafbc0d530, type=4210454592, type@entry=IBV_QPT_RC, cm_id=cm_id@entry=0x0) at ./src/msg/async/rdma/Infiniband.cc:1192 #7 0x0000aaaad28a9b50 in RDMAConnectedSocketImpl::RDMAConnectedSocketImpl (this=0xaaab4f361c20, cct=0xaaaafafd4000, ib=std::shared_ptr (use count 48, weak count 0) = {...}, rdma_dispatcher=..., w=) at ./src/msg/async/rdma/RDMAStack.h:105 #8 0x0000aaaad28b1d8c in RDMAServerSocketImpl::accept (this=0xaaaafaf3f700, sock=0xffff8a3b9db8, opt=..., out=0xffff8a3b9e10, w=) ---Type to continue, or q to quit--- at ./src/msg/async/rdma/RDMAServerSocketImpl.cc:115 #9 0x0000aaaad26956a0 in ServerSocket::accept (w=0xaaaafbb9c840, out=0xffff8a3b9e10, opt=..., sock=0xffff8a3b9db8, this=0xaaaafaf02eb0) at /usr/include/c++/9/bits/unique_ptr.h:357 #10 Processor::accept (this=0xaaab0bcef2c0) at ./src/msg/async/AsyncMessenger.cc:192 #11 0x0000aaaad269d1ac in EventCenter::process_events (this=this@entry=0xaaaafbb9c608, timeout_microseconds=, working_dur=working_dur@entry=0xffff8a3ba160) at /usr/include/c++/9/bits/basic_ios.h:282 #12 0x0000aaaad26a33c0 in NetworkStack::::operator() (__closure=0xaaaafbc0c4b8, __closure=0xaaaafbc0c4b8) at ./src/msg/async/Stack.cc:53 #13 std::_Function_handler >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/9/bits/std_function.h:300 #14 0x0000ffff8b03ded4 in ?? () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #15 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9aeff) at pthread_create.c:463 #16 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 Thread 73 (Thread 0xffff68c489f0 (LWP 126931)): #0 0x0000ffff8b1f922c in futex_wait_cancelable (private=, expected=0, futex_word=0xaaaafafbab34) at ../sysdeps/unix/sysv/linux/futex-internal.h:88 #1 __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafafbaad8, cond=0xaaaafafbab08) at pthread_cond_wait.c:502 #2 __pthread_cond_wait (cond=0xaaaafafbab08, mutex=0xaaaafafbaad8) at pthread_cond_wait.c:655 #3 0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #4 0x0000aaaad1ee9ab4 in OSD::ShardedOpWQ::_process (this=0xaaaafaf6ef10, thread_index=13, hb=0xaaab5091b2a0) at ./src/osd/OSD.cc:10423 #5 0x0000aaaad2523928 in ShardedThreadPool::shardedthreadpool_worker (this=0xaaaafaf6ea48, thread_index=13) at ./src/common/WorkQueue.cc:311 #6 0x0000aaaad2526268 in ShardedThreadPool::WorkThreadSharded::entry (this=) at ./src/common/WorkQueue.h:706 #7 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4bf) at pthread_create.c:463 #8 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 Thread 72 (Thread 0xffff6443f9f0 (LWP 126940)): #0 0x0000ffff8b1f922c in futex_wait_cancelable (private=, expected=0, futex_word=0xaaaafafb8fb4) at ../sysdeps/unix/sysv/linux/futex-internal.h:88 #1 __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafafb8f58, cond=0xaaaafafb8f88) at pthread_cond_wait.c:502 #2 __pthread_cond_wait (cond=0xaaaafafb8f88, mutex=0xaaaafafb8f58) at pthread_cond_wait.c:655 #3 0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #4 0x0000aaaad1ee9ab4 in OSD::ShardedOpWQ::_process (this=0xaaaafaf6ef10, thread_index=22, hb=0xaaab50919db0) at ./src/osd/OSD.cc:10423 #5 0x0000aaaad2523928 in ShardedThreadPool::shardedthreadpool_worker (this=0xaaaafaf6ea48, thread_index=22) at ./src/common/WorkQueue.cc:311 #6 0x0000aaaad2526268 in ShardedThreadPool::WorkThreadSharded::entry (this=) at ./src/common/WorkQueue.h:706 #7 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4bf) at pthread_create.c:463 #8 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 Thread 71 (Thread 0xffff6cc509f0 (LWP 126923)): #0 0x0000ffff8b1f922c in futex_wait_cancelable (private=, expected=0, futex_word=0xaaaafafb9730) at ../sysdeps/unix/sysv/linux/futex-internal.h:88 #1 __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafafb96d8, cond=0xaaaafafb9708) at pthread_cond_wait.c:502 #2 __pthread_cond_wait (cond=0xaaaafafb9708, mutex=0xaaaafafb96d8) at pthread_cond_wait.c:655 #3 0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #4 0x0000aaaad1ee9ab4 in OSD::ShardedOpWQ::_process (this=0xaaaafaf6ef10, thread_index=5, hb=0xaaab5091b070) at ./src/osd/OSD.cc:10423 #5 0x0000aaaad2523928 in ShardedThreadPool::shardedthreadpool_worker (this=0xaaaafaf6ea48, thread_index=5) at ./src/common/WorkQueue.cc:311 #6 0x0000aaaad2526268 in ShardedThreadPool::WorkThreadSharded::entry (this=) at ./src/common/WorkQueue.h:706 #7 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4bf) at pthread_create.c:463 #8 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 ---Type to continue, or q to quit--- Thread 70 (Thread 0xffff63c3e9f0 (LWP 126941)): #0 0x0000ffff8b1f922c in futex_wait_cancelable (private=, expected=0, futex_word=0xaaaafafb9230) at ../sysdeps/unix/sysv/linux/futex-internal.h:88 #1 __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafafb91d8, cond=0xaaaafafb9208) at pthread_cond_wait.c:502 #2 __pthread_cond_wait (cond=0xaaaafafb9208, mutex=0xaaaafafb91d8) at pthread_cond_wait.c:655 #3 0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #4 0x0000aaaad1ee9ab4 in OSD::ShardedOpWQ::_process (this=0xaaaafaf6ef10, thread_index=23, hb=0xaaab50919e00) at ./src/osd/OSD.cc:10423 #5 0x0000aaaad2523928 in ShardedThreadPool::shardedthreadpool_worker (this=0xaaaafaf6ea48, thread_index=23) at ./src/common/WorkQueue.cc:311 #6 0x0000aaaad2526268 in ShardedThreadPool::WorkThreadSharded::entry (this=) at ./src/common/WorkQueue.h:706 #7 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4bf) at pthread_create.c:463 #8 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 Thread 69 (Thread 0xffff61c3a9f0 (LWP 126945)): #0 0x0000ffff8b1f922c in futex_wait_cancelable (private=, expected=0, futex_word=0xaaaafafb9c34) at ../sysdeps/unix/sysv/linux/futex-internal.h:88 #1 __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafafb9bd8, cond=0xaaaafafb9c08) at pthread_cond_wait.c:502 #2 __pthread_cond_wait (cond=0xaaaafafb9c08, mutex=0xaaaafafb9bd8) at pthread_cond_wait.c:655 #3 0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #4 0x0000aaaad1ee9ab4 in OSD::ShardedOpWQ::_process (this=0xaaaafaf6ef10, thread_index=27, hb=0xaaab50919e50) at ./src/osd/OSD.cc:10423 #5 0x0000aaaad2523928 in ShardedThreadPool::shardedthreadpool_worker (this=0xaaaafaf6ea48, thread_index=27) at ./src/common/WorkQueue.cc:311 #6 0x0000aaaad2526268 in ShardedThreadPool::WorkThreadSharded::entry (this=) at ./src/common/WorkQueue.h:706 #7 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4bf) at pthread_create.c:463 #8 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 Thread 68 (Thread 0xffff6fc569f0 (LWP 126917)): #0 0x0000ffff8b1f922c in futex_wait_cancelable (private=, expected=0, futex_word=0xaaaafaf6f724) at ../sysdeps/unix/sysv/linux/futex-internal.h:88 #1 __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafaf6f6c8, cond=0xaaaafaf6f6f8) at pthread_cond_wait.c:502 #2 __pthread_cond_wait (cond=0xaaaafaf6f6f8, mutex=0xaaaafaf6f6c8) at pthread_cond_wait.c:655 #3 0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #4 0x0000aaaad1e928e4 in OSDService::agent_entry (this=0xaaaafaf6f3c8) at ./src/osd/OSD.cc:560 #5 0x0000aaaad1f47c40 in OSDService::AgentThread::entry (this=) at ./src/osd/OSD.h:383 #6 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c56f) at pthread_create.c:463 #7 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 Thread 67 (Thread 0xffff60c389f0 (LWP 126947)): #0 0x0000ffff8b1f922c in futex_wait_cancelable (private=, expected=0, futex_word=0xaaaafafba130) at ../sysdeps/unix/sysv/linux/futex-internal.h:88 #1 __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafafba0d8, cond=0xaaaafafba108) at pthread_cond_wait.c:502 #2 __pthread_cond_wait (cond=0xaaaafafba108, mutex=0xaaaafafba0d8) at pthread_cond_wait.c:655 #3 0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #4 0x0000aaaad1ee9ab4 in OSD::ShardedOpWQ::_process (this=0xaaaafaf6ef10, thread_index=29, hb=0xaaab50919f90) at ./src/osd/OSD.cc:10423 #5 0x0000aaaad2523928 in ShardedThreadPool::shardedthreadpool_worker (this=0xaaaafaf6ea48, thread_index=29) at ./src/common/WorkQueue.cc:311 #6 0x0000aaaad2526268 in ShardedThreadPool::WorkThreadSharded::entry (this=) at ./src/common/WorkQueue.h:706 #7 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4bf) at pthread_create.c:463 #8 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 Thread 66 (Thread 0xffff614399f0 (LWP 126946)): #0 0x0000ffff8b1f922c in futex_wait_cancelable (private=, expected=0, futex_word=0xaaaafafb9eb0) at ../sysdeps/unix/sysv/linux/futex-internal.h:88 #1 __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafafb9e58, cond=0xaaaafafb9e88) at pthread_cond_wait.c:502 ---Type to continue, or q to quit--- #2 __pthread_cond_wait (cond=0xaaaafafb9e88, mutex=0xaaaafafb9e58) at pthread_cond_wait.c:655 #3 0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #4 0x0000aaaad1ee9ab4 in OSD::ShardedOpWQ::_process (this=0xaaaafaf6ef10, thread_index=28, hb=0xaaab50919ea0) at ./src/osd/OSD.cc:10423 #5 0x0000aaaad2523928 in ShardedThreadPool::shardedthreadpool_worker (this=0xaaaafaf6ea48, thread_index=28) at ./src/common/WorkQueue.cc:311 #6 0x0000aaaad2526268 in ShardedThreadPool::WorkThreadSharded::entry (this=) at ./src/common/WorkQueue.h:706 #7 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4bf) at pthread_create.c:463 #8 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 Thread 65 (Thread 0xffff6ec549f0 (LWP 126919)): #0 0x0000ffff8b1f922c in futex_wait_cancelable (private=, expected=0, futex_word=0xaaaafafb8d30) at ../sysdeps/unix/sysv/linux/futex-internal.h:88 #1 __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafafb8cd8, cond=0xaaaafafb8d08) at pthread_cond_wait.c:502 #2 __pthread_cond_wait (cond=0xaaaafafb8d08, mutex=0xaaaafafb8cd8) at pthread_cond_wait.c:655 #3 0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #4 0x0000aaaad1ee9ab4 in OSD::ShardedOpWQ::_process (this=0xaaaafaf6ef10, thread_index=1, hb=0xaaaafbc109e0) at ./src/osd/OSD.cc:10423 #5 0x0000aaaad2523928 in ShardedThreadPool::shardedthreadpool_worker (this=0xaaaafaf6ea48, thread_index=1) at ./src/common/WorkQueue.cc:311 #6 0x0000aaaad2526268 in ShardedThreadPool::WorkThreadSharded::entry (this=) at ./src/common/WorkQueue.h:706 #7 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4bf) at pthread_create.c:463 #8 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 Thread 64 (Thread 0xffff62c3c9f0 (LWP 126943)): #0 0x0000ffff8b1f922c in futex_wait_cancelable (private=, expected=0, futex_word=0xaaaafafb9730) at ../sysdeps/unix/sysv/linux/futex-internal.h:88 #1 __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafafb96d8, cond=0xaaaafafb9708) at pthread_cond_wait.c:502 #2 __pthread_cond_wait (cond=0xaaaafafb9708, mutex=0xaaaafafb96d8) at pthread_cond_wait.c:655 #3 0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #4 0x0000aaaad1ee9ab4 in OSD::ShardedOpWQ::_process (this=0xaaaafaf6ef10, thread_index=25, hb=0xaaab50919f40) at ./src/osd/OSD.cc:10423 #5 0x0000aaaad2523928 in ShardedThreadPool::shardedthreadpool_worker (this=0xaaaafaf6ea48, thread_index=25) at ./src/common/WorkQueue.cc:311 #6 0x0000aaaad2526268 in ShardedThreadPool::WorkThreadSharded::entry (this=) at ./src/common/WorkQueue.h:706 #7 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4bf) at pthread_create.c:463 #8 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 Thread 63 (Thread 0xffff6343d9f0 (LWP 126942)): #0 0x0000ffff8b1f922c in futex_wait_cancelable (private=, expected=0, futex_word=0xaaaafafb94b0) at ../sysdeps/unix/sysv/linux/futex-internal.h:88 #1 __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafafb9458, cond=0xaaaafafb9488) at pthread_cond_wait.c:502 #2 __pthread_cond_wait (cond=0xaaaafafb9488, mutex=0xaaaafafb9458) at pthread_cond_wait.c:655 #3 0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #4 0x0000aaaad1ee9ab4 in OSD::ShardedOpWQ::_process (this=0xaaaafaf6ef10, thread_index=24, hb=0xaaab50919ef0) at ./src/osd/OSD.cc:10423 #5 0x0000aaaad2523928 in ShardedThreadPool::shardedthreadpool_worker (this=0xaaaafaf6ea48, thread_index=24) at ./src/common/WorkQueue.cc:311 #6 0x0000aaaad2526268 in ShardedThreadPool::WorkThreadSharded::entry (this=) at ./src/common/WorkQueue.h:706 #7 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4bf) at pthread_create.c:463 #8 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 Thread 62 (Thread 0xffff81c7a9f0 (LWP 126261)): #0 0x0000ffff8b1f922c in futex_wait_cancelable (private=, expected=0, futex_word=0xaaab4be125c0) at ../sysdeps/unix/sysv/linux/futex-internal.h:88 #1 __pthread_cond_wait_common (abstime=0x0, mutex=0xaaab4be12568, cond=0xaaab4be12598) at pthread_cond_wait.c:502 #2 __pthread_cond_wait (cond=0xaaab4be12598, mutex=0xaaab4be12568) at pthread_cond_wait.c:655 #3 0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 ---Type to continue, or q to quit--- #4 0x0000aaaad24bb69c in KernelDevice::_discard_thread (this=0xaaab4be12380) at ./src/os/bluestore/KernelDevice.cc:642 #5 0x0000aaaad24c9c68 in KernelDevice::DiscardThread::entry (this=) at ./src/os/bluestore/KernelDevice.h:74 #6 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9951f) at pthread_create.c:463 #7 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 Thread 61 (Thread 0xffff6bc4e9f0 (LWP 126925)): #0 0x0000ffff8b1f922c in futex_wait_cancelable (private=, expected=0, futex_word=0xaaaafafb9c34) at ../sysdeps/unix/sysv/linux/futex-internal.h:88 #1 __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafafb9bd8, cond=0xaaaafafb9c08) at pthread_cond_wait.c:502 #2 __pthread_cond_wait (cond=0xaaaafafb9c08, mutex=0xaaaafafb9bd8) at pthread_cond_wait.c:655 #3 0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #4 0x0000aaaad1ee9ab4 in OSD::ShardedOpWQ::_process (this=0xaaaafaf6ef10, thread_index=7, hb=0xaaab5091b0c0) at ./src/osd/OSD.cc:10423 #5 0x0000aaaad2523928 in ShardedThreadPool::shardedthreadpool_worker (this=0xaaaafaf6ea48, thread_index=7) at ./src/common/WorkQueue.cc:311 #6 0x0000aaaad2526268 in ShardedThreadPool::WorkThreadSharded::entry (this=) at ./src/common/WorkQueue.h:706 #7 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4bf) at pthread_create.c:463 #8 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 Thread 60 (Thread 0xffff6c44f9f0 (LWP 126924)): #0 0x0000ffff8b1f922c in futex_wait_cancelable (private=, expected=0, futex_word=0xaaaafafb99b4) at ../sysdeps/unix/sysv/linux/futex-internal.h:88 #1 __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafafb9958, cond=0xaaaafafb9988) at pthread_cond_wait.c:502 #2 __pthread_cond_wait (cond=0xaaaafafb9988, mutex=0xaaaafafb9958) at pthread_cond_wait.c:655 #3 0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #4 0x0000aaaad1ee9ab4 in OSD::ShardedOpWQ::_process (this=0xaaaafaf6ef10, thread_index=6, hb=0xaaab5091b110) at ./src/osd/OSD.cc:10423 #5 0x0000aaaad2523928 in ShardedThreadPool::shardedthreadpool_worker (this=0xaaaafaf6ea48, thread_index=6) at ./src/common/WorkQueue.cc:311 #6 0x0000aaaad2526268 in ShardedThreadPool::WorkThreadSharded::entry (this=) at ./src/common/WorkQueue.h:706 #7 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4bf) at pthread_create.c:463 #8 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 Thread 59 (Thread 0xffff6dc529f0 (LWP 126921)): #0 0x0000ffff8b1f922c in futex_wait_cancelable (private=, expected=0, futex_word=0xaaaafafb9230) at ../sysdeps/unix/sysv/linux/futex-internal.h:88 #1 __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafafb91d8, cond=0xaaaafafb9208) at pthread_cond_wait.c:502 #2 __pthread_cond_wait (cond=0xaaaafafb9208, mutex=0xaaaafafb91d8) at pthread_cond_wait.c:655 #3 0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #4 0x0000aaaad1ee9ab4 in OSD::ShardedOpWQ::_process (this=0xaaaafaf6ef10, thread_index=3, hb=0xaaaafbc10a80) at ./src/osd/OSD.cc:10423 #5 0x0000aaaad2523928 in ShardedThreadPool::shardedthreadpool_worker (this=0xaaaafaf6ea48, thread_index=3) at ./src/common/WorkQueue.cc:311 #6 0x0000aaaad2526268 in ShardedThreadPool::WorkThreadSharded::entry (this=) at ./src/common/WorkQueue.h:706 #7 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4bf) at pthread_create.c:463 #8 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 Thread 58 (Thread 0xffff80c789f0 (LWP 126001)): #0 0x0000ffff8b1f922c in futex_wait_cancelable (private=, expected=0, futex_word=0xaaaafaf8bec8) at ../sysdeps/unix/sysv/linux/futex-internal.h:88 #1 __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafaf8be70, cond=0xaaaafaf8bea0) at pthread_cond_wait.c:502 #2 __pthread_cond_wait (cond=0xaaaafaf8bea0, mutex=0xaaaafaf8be70) at pthread_cond_wait.c:655 #3 0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #4 0x0000aaaad2c7cb08 in rocksdb::ThreadPoolImpl::Impl::BGThread(unsigned long) () #5 0x0000aaaad2c7ce58 in rocksdb::ThreadPoolImpl::Impl::BGThreadWrapper(void*) () #6 0x0000aaaad2c8438c in void* std::__invoke_impl(std::__invoke_other, void* (*&&)(void*), rocksdb::BGThreadMet---Type to continue, or q to quit--- adata*&&) () #7 0x0000aaaad2c842c4 in std::__invoke_result::type std::__invoke(void* (*&&)(void*), rocksdb::BGThreadMetadata*&&) () #8 0x0000aaaad2c84218 in void* std::thread::_Invoker >::_M_invoke<0ul, 1ul>(std::_Index_tuple<0ul, 1ul>) () #9 0x0000aaaad2c841ac in std::thread::_Invoker >::operator()() () #10 0x0000aaaad2c84178 in std::thread::_State_impl > >::_M_run() () #11 0x0000ffff8b03ded4 in ?? () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #12 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9aacf) at pthread_create.c:463 #13 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 Thread 57 (Thread 0xffff804779f0 (LWP 126002)): #0 0x0000ffff8b1f922c in futex_wait_cancelable (private=, expected=0, futex_word=0xaaaafaf8bec8) at ../sysdeps/unix/sysv/linux/futex-internal.h:88 #1 __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafaf8be70, cond=0xaaaafaf8bea0) at pthread_cond_wait.c:502 #2 __pthread_cond_wait (cond=0xaaaafaf8bea0, mutex=0xaaaafaf8be70) at pthread_cond_wait.c:655 #3 0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #4 0x0000aaaad2c7cb08 in rocksdb::ThreadPoolImpl::Impl::BGThread(unsigned long) () #5 0x0000aaaad2c7ce58 in rocksdb::ThreadPoolImpl::Impl::BGThreadWrapper(void*) () #6 0x0000aaaad2c8438c in void* std::__invoke_impl(std::__invoke_other, void* (*&&)(void*), rocksdb::BGThreadMetadata*&&) () #7 0x0000aaaad2c842c4 in std::__invoke_result::type std::__invoke(void* (*&&)(void*), rocksdb::BGThreadMetadata*&&) () #8 0x0000aaaad2c84218 in void* std::thread::_Invoker >::_M_invoke<0ul, 1ul>(std::_Index_tuple<0ul, 1ul>) () #9 0x0000aaaad2c841ac in std::thread::_Invoker >::operator()() () #10 0x0000aaaad2c84178 in std::thread::_State_impl > >::_M_run() () #11 0x0000ffff8b03ded4 in ?? () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #12 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9aacf) at pthread_create.c:463 #13 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 Thread 56 (Thread 0xffff5b42d9f0 (LWP 126958)): #0 0x0000ffff8b1f95d4 in futex_abstimed_wait_cancelable (private=, abstime=0xffff5b42d178, expected=0, futex_word=0xaaaafaf6ec90) at ../sysdeps/unix/sysv/linux/futex-internal.h:205 #1 __pthread_cond_wait_common (abstime=0xffff5b42d178, mutex=0xaaaafaf6ec08, cond=0xaaaafaf6ec68) at pthread_cond_wait.c:539 #2 __pthread_cond_timedwait (cond=0xaaaafaf6ec68, mutex=0xaaaafaf6ec08, abstime=0xffff5b42d178) at pthread_cond_wait.c:667 #3 0x0000aaaad1eb9474 in __gthread_cond_timedwait (__abs_timeout=0xffff5b42d178, __mutex=, __cond=0xaaaafaf6ec68) at /usr/include/aarch64-linux-gnu/c++/9/bits/gthr-default.h:872 #4 std::condition_variable::__wait_until_impl > > (__atime=..., __lock=..., this=0xaaaafaf6ec68) at /usr/include/c++/9/condition_variable:188 #5 std::condition_variable::wait_until > > (__atime=..., __lock=..., this=0xaaaafaf6ec68) at /usr/include/c++/9/condition_variable:121 #6 std::condition_variable::wait_for > (__rtime=..., __lock=..., this=0xaaaafaf6ec68) at /usr/include/c++/9/condition_variable:152 #7 OSD::heartbeat_entry (this=0xaaaafaf6e000) at ./src/osd/OSD.cc:5591 #8 0x0000aaaad1f47c68 in OSD::T_Heartbeat::entry (this=) at ./src/osd/OSD.h:1503 #9 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c5df) at pthread_create.c:463 ---Type to continue, or q to quit--- #10 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 Thread 55 (Thread 0xffff754619f0 (LWP 126891)): #0 0x0000ffff8b1f922c in futex_wait_cancelable (private=, expected=0, futex_word=0xaaaafbb850ac) at ../sysdeps/unix/sysv/linux/futex-internal.h:88 #1 __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafbb85050, cond=0xaaaafbb85080) at pthread_cond_wait.c:502 #2 __pthread_cond_wait (cond=0xaaaafbb85080, mutex=0xaaaafbb85050) at pthread_cond_wait.c:655 #3 0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #4 0x0000aaaad28449ac in DispatchQueue::entry (this=0xaaaafbb85040) at ./src/msg/DispatchQueue.cc:210 #5 0x0000aaaad26977f0 in DispatchQueue::DispatchThread::entry (this=) at ./src/msg/DispatchQueue.h:101 #6 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4df) at pthread_create.c:463 #7 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 Thread 54 (Thread 0xffff66c449f0 (LWP 126935)): #0 0x0000ffff8b1f922c in futex_wait_cancelable (private=, expected=0, futex_word=0xaaaafafbb530) at ../sysdeps/unix/sysv/linux/futex-internal.h:88 #1 __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafafbb4d8, cond=0xaaaafafbb508) at pthread_cond_wait.c:502 #2 __pthread_cond_wait (cond=0xaaaafafbb508, mutex=0xaaaafafbb4d8) at pthread_cond_wait.c:655 #3 0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #4 0x0000aaaad1ee9ab4 in OSD::ShardedOpWQ::_process (this=0xaaaafaf6ef10, thread_index=17, hb=0xaaab5091b6b0) at ./src/osd/OSD.cc:10423 #5 0x0000aaaad2523928 in ShardedThreadPool::shardedthreadpool_worker (this=0xaaaafaf6ea48, thread_index=17) at ./src/common/WorkQueue.cc:311 #6 0x0000aaaad2526268 in ShardedThreadPool::WorkThreadSharded::entry (this=) at ./src/common/WorkQueue.h:706 #7 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4bf) at pthread_create.c:463 #8 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 Thread 53 (Thread 0xffff65c429f0 (LWP 126937)): #0 0x0000ffff8b1f922c in futex_wait_cancelable (private=, expected=0, futex_word=0xaaaafafbba34) at ../sysdeps/unix/sysv/linux/futex-internal.h:88 #1 __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafafbb9d8, cond=0xaaaafafbba08) at pthread_cond_wait.c:502 #2 __pthread_cond_wait (cond=0xaaaafafbba08, mutex=0xaaaafafbb9d8) at pthread_cond_wait.c:655 #3 0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #4 0x0000aaaad1ee9ab4 in OSD::ShardedOpWQ::_process (this=0xaaaafaf6ef10, thread_index=19, hb=0xaaab50919cc0) at ./src/osd/OSD.cc:10423 #5 0x0000aaaad2523928 in ShardedThreadPool::shardedthreadpool_worker (this=0xaaaafaf6ea48, thread_index=19) at ./src/common/WorkQueue.cc:311 #6 0x0000aaaad2526268 in ShardedThreadPool::WorkThreadSharded::entry (this=) at ./src/common/WorkQueue.h:706 #7 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4bf) at pthread_create.c:463 #8 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 Thread 52 (Thread 0xffff6ac4c9f0 (LWP 126927)): #0 0x0000ffff8b1f922c in futex_wait_cancelable (private=, expected=0, futex_word=0xaaaafafba130) at ../sysdeps/unix/sysv/linux/futex-internal.h:88 #1 __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafafba0d8, cond=0xaaaafafba108) at pthread_cond_wait.c:502 #2 __pthread_cond_wait (cond=0xaaaafafba108, mutex=0xaaaafafba0d8) at pthread_cond_wait.c:655 #3 0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #4 0x0000aaaad1ee9ab4 in OSD::ShardedOpWQ::_process (this=0xaaaafaf6ef10, thread_index=9, hb=0xaaab5091b200) at ./src/osd/OSD.cc:10423 #5 0x0000aaaad2523928 in ShardedThreadPool::shardedthreadpool_worker (this=0xaaaafaf6ea48, thread_index=9) at ./src/common/WorkQueue.cc:311 #6 0x0000aaaad2526268 in ShardedThreadPool::WorkThreadSharded::entry (this=) at ./src/common/WorkQueue.h:706 #7 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4bf) at pthread_create.c:463 #8 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 Thread 51 (Thread 0xffff5ec349f0 (LWP 126951)): ---Type to continue, or q to quit--- #0 0x0000ffff8b1f922c in futex_wait_cancelable (private=, expected=0, futex_word=0xaaaafafbab34) at ../sysdeps/unix/sysv/linux/futex-internal.h:88 #1 __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafafbaad8, cond=0xaaaafafbab08) at pthread_cond_wait.c:502 #2 __pthread_cond_wait (cond=0xaaaafafbab08, mutex=0xaaaafafbaad8) at pthread_cond_wait.c:655 #3 0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #4 0x0000aaaad1ee9ab4 in OSD::ShardedOpWQ::_process (this=0xaaaafaf6ef10, thread_index=33, hb=0xaaab5091a030) at ./src/osd/OSD.cc:10423 #5 0x0000aaaad2523928 in ShardedThreadPool::shardedthreadpool_worker (this=0xaaaafaf6ea48, thread_index=33) at ./src/common/WorkQueue.cc:311 #6 0x0000aaaad2526268 in ShardedThreadPool::WorkThreadSharded::entry (this=) at ./src/common/WorkQueue.h:706 #7 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4bf) at pthread_create.c:463 #8 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 Thread 50 (Thread 0xffff5f4359f0 (LWP 126950)): #0 0x0000ffff8b1f922c in futex_wait_cancelable (private=, expected=0, futex_word=0xaaaafafba8b0) at ../sysdeps/unix/sysv/linux/futex-internal.h:88 #1 __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafafba858, cond=0xaaaafafba888) at pthread_cond_wait.c:502 #2 __pthread_cond_wait (cond=0xaaaafafba888, mutex=0xaaaafafba858) at pthread_cond_wait.c:655 #3 0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #4 0x0000aaaad1ee9ab4 in OSD::ShardedOpWQ::_process (this=0xaaaafaf6ef10, thread_index=32, hb=0xaaab5091b390) at ./src/osd/OSD.cc:10423 #5 0x0000aaaad2523928 in ShardedThreadPool::shardedthreadpool_worker (this=0xaaaafaf6ea48, thread_index=32) at ./src/common/WorkQueue.cc:311 #6 0x0000aaaad2526268 in ShardedThreadPool::WorkThreadSharded::entry (this=) at ./src/common/WorkQueue.h:706 #7 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4bf) at pthread_create.c:463 #8 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 Thread 49 (Thread 0xffff5c42f9f0 (LWP 126956)): #0 0x0000ffff8b1f922c in futex_wait_cancelable (private=, expected=0, futex_word=0xaaaafafbb7b0) at ../sysdeps/unix/sysv/linux/futex-internal.h:88 #1 __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafafbb758, cond=0xaaaafafbb788) at pthread_cond_wait.c:502 #2 __pthread_cond_wait (cond=0xaaaafafbb788, mutex=0xaaaafafbb758) at pthread_cond_wait.c:655 #3 0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #4 0x0000aaaad1ee9ab4 in OSD::ShardedOpWQ::_process (this=0xaaaafaf6ef10, thread_index=38, hb=0xaaab50919950) at ./src/osd/OSD.cc:10423 #5 0x0000aaaad2523928 in ShardedThreadPool::shardedthreadpool_worker (this=0xaaaafaf6ea48, thread_index=38) at ./src/common/WorkQueue.cc:311 #6 0x0000aaaad2526268 in ShardedThreadPool::WorkThreadSharded::entry (this=) at ./src/common/WorkQueue.h:706 #7 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4bf) at pthread_create.c:463 #8 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 Thread 48 (Thread 0xffff7245b9f0 (LWP 126912)): #0 0x0000ffff8b1f922c in futex_wait_cancelable (private=, expected=0, futex_word=0xaaaafaf6fc78) at ../sysdeps/unix/sysv/linux/futex-internal.h:88 #1 __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafaf6fc20, cond=0xaaaafaf6fc50) at pthread_cond_wait.c:502 #2 __pthread_cond_wait (cond=0xaaaafaf6fc50, mutex=0xaaaafaf6fc20) at pthread_cond_wait.c:655 #3 0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #4 0x0000aaaad24efa80 in Finisher::finisher_thread_entry (this=0xaaaafaf6fc18) at ./src/common/Finisher.cc:86 #5 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4ef) at pthread_create.c:463 #6 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 Thread 47 (Thread 0xffff67c469f0 (LWP 126933)): #0 0x0000ffff8b1f922c in futex_wait_cancelable (private=, expected=0, futex_word=0xaaaafafbb030) at ../sysdeps/unix/sysv/linux/futex-internal.h:88 #1 __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafafbafd8, cond=0xaaaafafbb008) at pthread_cond_wait.c:502 #2 __pthread_cond_wait (cond=0xaaaafafbb008, mutex=0xaaaafafbafd8) at pthread_cond_wait.c:655 #3 0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 ---Type to continue, or q to quit--- #4 0x0000aaaad1ee9ab4 in OSD::ShardedOpWQ::_process (this=0xaaaafaf6ef10, thread_index=15, hb=0xaaab5091b3e0) at ./src/osd/OSD.cc:10423 #5 0x0000aaaad2523928 in ShardedThreadPool::shardedthreadpool_worker (this=0xaaaafaf6ea48, thread_index=15) at ./src/common/WorkQueue.cc:311 #6 0x0000aaaad2526268 in ShardedThreadPool::WorkThreadSharded::entry (this=) at ./src/common/WorkQueue.h:706 #7 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4bf) at pthread_create.c:463 #8 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 Thread 46 (Thread 0xffff764639f0 (LWP 126889)): #0 0x0000ffff8b1f922c in futex_wait_cancelable (private=, expected=0, futex_word=0xaaaafbb835a8) at ../sysdeps/unix/sysv/linux/futex-internal.h:88 #1 __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafbb83550, cond=0xaaaafbb83580) at pthread_cond_wait.c:502 #2 __pthread_cond_wait (cond=0xaaaafbb83580, mutex=0xaaaafbb83550) at pthread_cond_wait.c:655 #3 0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #4 0x0000aaaad28449ac in DispatchQueue::entry (this=0xaaaafbb83540) at ./src/msg/DispatchQueue.cc:210 #5 0x0000aaaad26977f0 in DispatchQueue::DispatchThread::entry (this=) at ./src/msg/DispatchQueue.h:101 #6 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4df) at pthread_create.c:463 #7 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 Thread 45 (Thread 0xffff7d76f9f0 (LWP 126880)): #0 0x0000ffff8b1f922c in futex_wait_cancelable (private=, expected=0, futex_word=0xaaaafbb82ca8) at ../sysdeps/unix/sysv/linux/futex-internal.h:88 #1 __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafbb82c50, cond=0xaaaafbb82c80) at pthread_cond_wait.c:502 #2 __pthread_cond_wait (cond=0xaaaafbb82c80, mutex=0xaaaafbb82c50) at pthread_cond_wait.c:655 #3 0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #4 0x0000aaaad28449ac in DispatchQueue::entry (this=0xaaaafbb82c40) at ./src/msg/DispatchQueue.cc:210 #5 0x0000aaaad26977f0 in DispatchQueue::DispatchThread::entry (this=) at ./src/msg/DispatchQueue.h:101 #6 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c47f) at pthread_create.c:463 #7 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 Thread 44 (Thread 0xffff6d4519f0 (LWP 126922)): #0 0x0000ffff8b1f922c in futex_wait_cancelable (private=, expected=0, futex_word=0xaaaafafb94b0) at ../sysdeps/unix/sysv/linux/futex-internal.h:88 #1 __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafafb9458, cond=0xaaaafafb9488) at pthread_cond_wait.c:502 #2 __pthread_cond_wait (cond=0xaaaafafb9488, mutex=0xaaaafafb9458) at pthread_cond_wait.c:655 #3 0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #4 0x0000aaaad1ee9ab4 in OSD::ShardedOpWQ::_process (this=0xaaaafaf6ef10, thread_index=4, hb=0xaaaafbc10ad0) at ./src/osd/OSD.cc:10423 #5 0x0000aaaad2523928 in ShardedThreadPool::shardedthreadpool_worker (this=0xaaaafaf6ea48, thread_index=4) at ./src/common/WorkQueue.cc:311 #6 0x0000aaaad2526268 in ShardedThreadPool::WorkThreadSharded::entry (this=) at ./src/common/WorkQueue.h:706 #7 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4bf) at pthread_create.c:463 #8 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 Thread 43 (Thread 0xffff7b76b9f0 (LWP 126884)): #0 0x0000ffff8b1f95d4 in futex_abstimed_wait_cancelable (private=, abstime=0xffff7b76b1a8, expected=0, futex_word=0xaaaafaf6e46c) at ../sysdeps/unix/sysv/linux/futex-internal.h:205 #1 __pthread_cond_wait_common (abstime=0xffff7b76b1a8, mutex=0xaaaafaf6e3c0, cond=0xaaaafaf6e440) at pthread_cond_wait.c:539 #2 __pthread_cond_timedwait (cond=0xaaaafaf6e440, mutex=0xaaaafaf6e3c0, abstime=0xffff7b76b1a8) at pthread_cond_wait.c:667 #3 0x0000aaaad251d910 in __gthread_cond_timedwait (__abs_timeout=0xffff7b76b1a8, __mutex=, __cond=0xaaaafaf6e440) at /usr/include/aarch64-linux-gnu/c++/9/bits/gthr-default.h:872 #4 std::condition_variable::__wait_until_impl > > (__atime=..., __lock=..., this=0xaaaafaf6e440) at /usr/include/c++/9/condition_variable:188 ---Type to continue, or q to quit--- #5 std::condition_variable::wait_until > > (__atime=..., __lock=..., this=0xaaaafaf6e440) at /usr/include/c++/9/condition_variable:121 #6 SafeTimer::timer_thread (this=0xaaaafaf6e428) at ./src/common/Timer.cc:108 #7 0x0000aaaad251e7f0 in SafeTimerThread::entry (this=) at ./src/common/Timer.cc:30 #8 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c52f) at pthread_create.c:463 #9 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 Thread 42 (Thread 0xffff7445f9f0 (LWP 126893)): #0 0x0000ffff8b1f922c in futex_wait_cancelable (private=, expected=0, futex_word=0xaaaafbb847a8) at ../sysdeps/unix/sysv/linux/futex-internal.h:88 #1 __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafbb84750, cond=0xaaaafbb84780) at pthread_cond_wait.c:502 #2 __pthread_cond_wait (cond=0xaaaafbb84780, mutex=0xaaaafbb84750) at pthread_cond_wait.c:655 #3 0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #4 0x0000aaaad28449ac in DispatchQueue::entry (this=0xaaaafbb84740) at ./src/msg/DispatchQueue.cc:210 #5 0x0000aaaad26977f0 in DispatchQueue::DispatchThread::entry (this=) at ./src/msg/DispatchQueue.h:101 #6 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4df) at pthread_create.c:463 #7 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 Thread 41 (Thread 0xffff7af6a9f0 (LWP 126885)): #0 0x0000ffff8b1f922c in futex_wait_cancelable (private=, expected=0, futex_word=0xaaaafbb823a8) at ../sysdeps/unix/sysv/linux/futex-internal.h:88 #1 __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafbb82350, cond=0xaaaafbb82380) at pthread_cond_wait.c:502 #2 __pthread_cond_wait (cond=0xaaaafbb82380, mutex=0xaaaafbb82350) at pthread_cond_wait.c:655 #3 0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #4 0x0000aaaad28449ac in DispatchQueue::entry (this=0xaaaafbb82340) at ./src/msg/DispatchQueue.cc:210 #5 0x0000aaaad26977f0 in DispatchQueue::DispatchThread::entry (this=) at ./src/msg/DispatchQueue.h:101 #6 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4df) at pthread_create.c:463 #7 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 Thread 40 (Thread 0xffff71c5a9f0 (LWP 126913)): #0 0x0000ffff8b1f922c in futex_wait_cancelable (private=, expected=0, futex_word=0xaaaafb001920) at ../sysdeps/unix/sysv/linux/futex-internal.h:88 #1 __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafb0018c8, cond=0xaaaafb0018f8) at pthread_cond_wait.c:502 #2 __pthread_cond_wait (cond=0xaaaafb0018f8, mutex=0xaaaafb0018c8) at pthread_cond_wait.c:655 #3 0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #4 0x0000aaaad24efa80 in Finisher::finisher_thread_entry (this=0xaaaafb0018c0) at ./src/common/Finisher.cc:86 #5 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4ef) at pthread_create.c:463 #6 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 Thread 39 (Thread 0xffff73c5e9f0 (LWP 126894)): #0 0x0000ffff8b1f922c in futex_wait_cancelable (private=, expected=0, futex_word=0xaaaafbb84918) at ../sysdeps/unix/sysv/linux/futex-internal.h:88 #1 __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafbb848c0, cond=0xaaaafbb848f0) at pthread_cond_wait.c:502 #2 __pthread_cond_wait (cond=0xaaaafbb848f0, mutex=0xaaaafbb848c0) at pthread_cond_wait.c:655 #3 0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #4 0x0000aaaad284471c in DispatchQueue::run_local_delivery (this=0xaaaafbb84740) at ./src/msg/DispatchQueue.cc:117 #5 0x0000aaaad26978b0 in DispatchQueue::LocalDeliveryThread::entry (this=) at ./src/msg/DispatchQueue.h:115 #6 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4ff) at pthread_create.c:463 #7 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 ---Type to continue, or q to quit--- Thread 38 (Thread 0xffff8247b9f0 (LWP 125801)): #0 0x0000ffff8b1f922c in futex_wait_cancelable (private=, expected=0, futex_word=0xaaab4be12240) at ../sysdeps/unix/sysv/linux/futex-internal.h:88 #1 __pthread_cond_wait_common (abstime=0x0, mutex=0xaaab4be121e8, cond=0xaaab4be12218) at pthread_cond_wait.c:502 #2 __pthread_cond_wait (cond=0xaaab4be12218, mutex=0xaaab4be121e8) at pthread_cond_wait.c:655 #3 0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #4 0x0000aaaad24bb69c in KernelDevice::_discard_thread (this=0xaaab4be12000) at ./src/os/bluestore/KernelDevice.cc:642 #5 0x0000aaaad24c9c68 in KernelDevice::DiscardThread::entry (this=) at ./src/os/bluestore/KernelDevice.h:74 #6 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9a23f) at pthread_create.c:463 #7 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 Thread 37 (Thread 0xffff85ec59f0 (LWP 125791)): #0 0x0000ffff8b1f95d4 in futex_abstimed_wait_cancelable (private=, abstime=0xffff85ec51a8, expected=0, futex_word=0xaaaafaf6e088) at ../sysdeps/unix/sysv/linux/futex-internal.h:205 #1 __pthread_cond_wait_common (abstime=0xffff85ec51a8, mutex=0xaaaafaf6e018, cond=0xaaaafaf6e060) at pthread_cond_wait.c:539 #2 __pthread_cond_timedwait (cond=0xaaaafaf6e060, mutex=0xaaaafaf6e018, abstime=0xffff85ec51a8) at pthread_cond_wait.c:667 #3 0x0000aaaad251d910 in __gthread_cond_timedwait (__abs_timeout=0xffff85ec51a8, __mutex=, __cond=0xaaaafaf6e060) at /usr/include/aarch64-linux-gnu/c++/9/bits/gthr-default.h:872 #4 std::condition_variable::__wait_until_impl > > (__atime=..., __lock=..., this=0xaaaafaf6e060) at /usr/include/c++/9/condition_variable:188 #5 std::condition_variable::wait_until > > (__atime=..., __lock=..., this=0xaaaafaf6e060) at /usr/include/c++/9/condition_variable:121 #6 SafeTimer::timer_thread (this=0xaaaafaf6e048) at ./src/common/Timer.cc:108 #7 0x0000aaaad251e7f0 in SafeTimerThread::entry (this=) at ./src/common/Timer.cc:30 #8 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c55f) at pthread_create.c:463 #9 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 Thread 36 (Thread 0xffff664439f0 (LWP 126936)): #0 0x0000ffff8b1f922c in futex_wait_cancelable (private=, expected=0, futex_word=0xaaaafafbb7b0) at ../sysdeps/unix/sysv/linux/futex-internal.h:88 #1 __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafafbb758, cond=0xaaaafafbb788) at pthread_cond_wait.c:502 #2 __pthread_cond_wait (cond=0xaaaafafbb788, mutex=0xaaaafafbb758) at pthread_cond_wait.c:655 #3 0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #4 0x0000aaaad1ee9ab4 in OSD::ShardedOpWQ::_process (this=0xaaaafaf6ef10, thread_index=18, hb=0xaaab5091b700) at ./src/osd/OSD.cc:10423 #5 0x0000aaaad2523928 in ShardedThreadPool::shardedthreadpool_worker (this=0xaaaafaf6ea48, thread_index=18) at ./src/common/WorkQueue.cc:311 #6 0x0000aaaad2526268 in ShardedThreadPool::WorkThreadSharded::entry (this=) at ./src/common/WorkQueue.h:706 #7 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4bf) at pthread_create.c:463 #8 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 Thread 35 (Thread 0xffff694499f0 (LWP 126930)): #0 0x0000ffff8b1f922c in futex_wait_cancelable (private=, expected=0, futex_word=0xaaaafafba8b0) at ../sysdeps/unix/sysv/linux/futex-internal.h:88 #1 __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafafba858, cond=0xaaaafafba888) at pthread_cond_wait.c:502 #2 __pthread_cond_wait (cond=0xaaaafafba888, mutex=0xaaaafafba858) at pthread_cond_wait.c:655 #3 0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #4 0x0000aaaad1ee9ab4 in OSD::ShardedOpWQ::_process (this=0xaaaafaf6ef10, thread_index=12, hb=0xaaab5091b1b0) at ./src/osd/OSD.cc:10423 #5 0x0000aaaad2523928 in ShardedThreadPool::shardedthreadpool_worker (this=0xaaaafaf6ea48, thread_index=12) at ./src/common/WorkQueue.cc:311 #6 0x0000aaaad2526268 in ShardedThreadPool::WorkThreadSharded::entry (this=) at ./src/common/WorkQueue.h:706 #7 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4bf) at pthread_create.c:463 ---Type to continue, or q to quit--- #8 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 Thread 34 (Thread 0xffff6243b9f0 (LWP 126944)): #0 0x0000ffff8b1f922c in futex_wait_cancelable (private=, expected=0, futex_word=0xaaaafafb99b4) at ../sysdeps/unix/sysv/linux/futex-internal.h:88 #1 __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafafb9958, cond=0xaaaafafb9988) at pthread_cond_wait.c:502 #2 __pthread_cond_wait (cond=0xaaaafafb9988, mutex=0xaaaafafb9958) at pthread_cond_wait.c:655 #3 0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #4 0x0000aaaad1ee9ab4 in OSD::ShardedOpWQ::_process (this=0xaaaafaf6ef10, thread_index=26, hb=0xaaab50919fe0) at ./src/osd/OSD.cc:10423 #5 0x0000aaaad2523928 in ShardedThreadPool::shardedthreadpool_worker (this=0xaaaafaf6ea48, thread_index=26) at ./src/common/WorkQueue.cc:311 #6 0x0000aaaad2526268 in ShardedThreadPool::WorkThreadSharded::entry (this=) at ./src/common/WorkQueue.h:706 #7 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4bf) at pthread_create.c:463 #8 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 Thread 33 (Thread 0xffff84c809f0 (LWP 125792)): #0 0x0000ffff8b1f95d4 in futex_abstimed_wait_cancelable (private=, abstime=0xffff84c801a8, expected=0, futex_word=0xaaaafaf6e17c) at ../sysdeps/unix/sysv/linux/futex-internal.h:205 #1 __pthread_cond_wait_common (abstime=0xffff84c801a8, mutex=0xaaaafaf6e108, cond=0xaaaafaf6e150) at pthread_cond_wait.c:539 #2 __pthread_cond_timedwait (cond=0xaaaafaf6e150, mutex=0xaaaafaf6e108, abstime=0xffff84c801a8) at pthread_cond_wait.c:667 #3 0x0000aaaad251d910 in __gthread_cond_timedwait (__abs_timeout=0xffff84c801a8, __mutex=, __cond=0xaaaafaf6e150) at /usr/include/aarch64-linux-gnu/c++/9/bits/gthr-default.h:872 #4 std::condition_variable::__wait_until_impl > > (__atime=..., __lock=..., this=0xaaaafaf6e150) at /usr/include/c++/9/condition_variable:188 #5 std::condition_variable::wait_until > > (__atime=..., __lock=..., this=0xaaaafaf6e150) at /usr/include/c++/9/condition_variable:121 #6 SafeTimer::timer_thread (this=0xaaaafaf6e138) at ./src/common/Timer.cc:108 #7 0x0000aaaad251e7f0 in SafeTimerThread::entry (this=) at ./src/common/Timer.cc:30 #8 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c55f) at pthread_create.c:463 #9 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 Thread 32 (Thread 0xffff6a44b9f0 (LWP 126928)): #0 0x0000ffff8b1f922c in futex_wait_cancelable (private=, expected=0, futex_word=0xaaaafafba3b0) at ../sysdeps/unix/sysv/linux/futex-internal.h:88 #1 __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafafba358, cond=0xaaaafafba388) at pthread_cond_wait.c:502 #2 __pthread_cond_wait (cond=0xaaaafafba388, mutex=0xaaaafafba358) at pthread_cond_wait.c:655 #3 0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #4 0x0000aaaad1ee9ab4 in OSD::ShardedOpWQ::_process (this=0xaaaafaf6ef10, thread_index=10, hb=0xaaab5091b250) at ./src/osd/OSD.cc:10423 #5 0x0000aaaad2523928 in ShardedThreadPool::shardedthreadpool_worker (this=0xaaaafaf6ea48, thread_index=10) at ./src/common/WorkQueue.cc:311 #6 0x0000aaaad2526268 in ShardedThreadPool::WorkThreadSharded::entry (this=) at ./src/common/WorkQueue.h:706 #7 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4bf) at pthread_create.c:463 #8 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 Thread 31 (Thread 0xffff704579f0 (LWP 126916)): #0 0x0000ffff8b1f922c in futex_wait_cancelable (private=, expected=0, futex_word=0xaaaafaf70880) at ../sysdeps/unix/sysv/linux/futex-internal.h:88 #1 __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafaf70828, cond=0xaaaafaf70858) at pthread_cond_wait.c:502 #2 __pthread_cond_wait (cond=0xaaaafaf70858, mutex=0xaaaafaf70828) at pthread_cond_wait.c:655 #3 0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #4 0x0000aaaad1f338f8 in ceph::timer_detail::timer::timer_thread (this=0xaaaafaf707e8) ---Type to continue, or q to quit--- at ./obj-aarch64-linux-gnu/boost/include/boost/intrusive/detail/rbtree_node.hpp:77 #5 0x0000ffff8b03ded4 in ?? () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #6 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c79f) at pthread_create.c:463 #7 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 Thread 30 (Thread 0xffff70c589f0 (LWP 126915)): #0 0x0000ffff8b1f922c in futex_wait_cancelable (private=, expected=0, futex_word=0xaaaafaf6f848) at ../sysdeps/unix/sysv/linux/futex-internal.h:88 #1 __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafaf6f7d8, cond=0xaaaafaf6f820) at pthread_cond_wait.c:502 #2 __pthread_cond_wait (cond=0xaaaafaf6f820, mutex=0xaaaafaf6f7d8) at pthread_cond_wait.c:655 #3 0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #4 0x0000aaaad251d438 in SafeTimer::timer_thread (this=0xaaaafaf6f808) at ./src/common/Timer.cc:106 #5 0x0000aaaad251e7f0 in SafeTimerThread::entry (this=) at ./src/common/Timer.cc:30 #6 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4ef) at pthread_create.c:463 #7 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 Thread 29 (Thread 0xffff714599f0 (LWP 126914)): #0 0x0000ffff8b1f922c in futex_wait_cancelable (private=, expected=0, futex_word=0xaaaafaf6f998) at ../sysdeps/unix/sysv/linux/futex-internal.h:88 #1 __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafaf6f928, cond=0xaaaafaf6f970) at pthread_cond_wait.c:502 #2 __pthread_cond_wait (cond=0xaaaafaf6f970, mutex=0xaaaafaf6f928) at pthread_cond_wait.c:655 #3 0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #4 0x0000aaaad251d438 in SafeTimer::timer_thread (this=0xaaaafaf6f958) at ./src/common/Timer.cc:106 #5 0x0000aaaad251e7f0 in SafeTimerThread::entry (this=) at ./src/common/Timer.cc:30 #6 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4ef) at pthread_create.c:463 #7 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 Thread 28 (Thread 0xffff6e4539f0 (LWP 126920)): #0 0x0000ffff8b1f922c in futex_wait_cancelable (private=, expected=0, futex_word=0xaaaafafb8fb4) at ../sysdeps/unix/sysv/linux/futex-internal.h:88 #1 __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafafb8f58, cond=0xaaaafafb8f88) at pthread_cond_wait.c:502 #2 __pthread_cond_wait (cond=0xaaaafafb8f88, mutex=0xaaaafafb8f58) at pthread_cond_wait.c:655 #3 0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #4 0x0000aaaad1ee9ab4 in OSD::ShardedOpWQ::_process (this=0xaaaafaf6ef10, thread_index=2, hb=0xaaaafbc10a30) at ./src/osd/OSD.cc:10423 #5 0x0000aaaad2523928 in ShardedThreadPool::shardedthreadpool_worker (this=0xaaaafaf6ea48, thread_index=2) at ./src/common/WorkQueue.cc:311 #6 0x0000aaaad2526268 in ShardedThreadPool::WorkThreadSharded::entry (this=) at ./src/common/WorkQueue.h:706 #7 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4bf) at pthread_create.c:463 #8 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 Thread 27 (Thread 0xffff7a7699f0 (LWP 126886)): #0 0x0000ffff8b1f922c in futex_wait_cancelable (private=, expected=0, futex_word=0xaaaafbb82518) at ../sysdeps/unix/sysv/linux/futex-internal.h:88 #1 __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafbb824c0, cond=0xaaaafbb824f0) at pthread_cond_wait.c:502 #2 __pthread_cond_wait (cond=0xaaaafbb824f0, mutex=0xaaaafbb824c0) at pthread_cond_wait.c:655 #3 0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #4 0x0000aaaad284471c in DispatchQueue::run_local_delivery (this=0xaaaafbb82340) at ./src/msg/DispatchQueue.cc:117 #5 0x0000aaaad26978b0 in DispatchQueue::LocalDeliveryThread::entry (this=) at ./src/msg/DispatchQueue.h:115 #6 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4ff) at pthread_create.c:463 #7 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 ---Type to continue, or q to quit--- Thread 26 (Thread 0xffff8ac81010 (LWP 124314)): #0 0x0000ffff8b1f922c in futex_wait_cancelable (private=, expected=0, futex_word=0xaaaafbb83108) at ../sysdeps/unix/sysv/linux/futex-internal.h:88 #1 __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafbb82f70, cond=0xaaaafbb830e0) at pthread_cond_wait.c:502 #2 __pthread_cond_wait (cond=0xaaaafbb830e0, mutex=0xaaaafbb82f70) at pthread_cond_wait.c:655 #3 0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #4 0x0000aaaad269475c in AsyncMessenger::wait (this=0xaaaafbb82900) at ./src/msg/async/AsyncMessenger.cc:538 #5 0x0000aaaad1e5f58c in main (argc=, argv=) at ./src/ceph_osd.cc:729 Thread 25 (Thread 0xffff7c76d9f0 (LWP 126882)): #0 0x0000ffff8b1f95d4 in futex_abstimed_wait_cancelable (private=, abstime=0xffff7c76d1a8, expected=0, futex_word=0xffffe7b9d888) at ../sysdeps/unix/sysv/linux/futex-internal.h:205 #1 __pthread_cond_wait_common (abstime=0xffff7c76d1a8, mutex=0xffffe7b9d818, cond=0xffffe7b9d860) at pthread_cond_wait.c:539 #2 __pthread_cond_timedwait (cond=0xffffe7b9d860, mutex=0xffffe7b9d818, abstime=0xffff7c76d1a8) at pthread_cond_wait.c:667 #3 0x0000aaaad251d910 in __gthread_cond_timedwait (__abs_timeout=0xffff7c76d1a8, __mutex=, __cond=0xffffe7b9d860) at /usr/include/aarch64-linux-gnu/c++/9/bits/gthr-default.h:872 #4 std::condition_variable::__wait_until_impl > > (__atime=..., __lock=..., this=0xffffe7b9d860) at /usr/include/c++/9/condition_variable:188 #5 std::condition_variable::wait_until > > (__atime=..., __lock=..., this=0xffffe7b9d860) at /usr/include/c++/9/condition_variable:121 #6 SafeTimer::timer_thread (this=0xffffe7b9d848) at ./src/common/Timer.cc:108 #7 0x0000aaaad251e7f0 in SafeTimerThread::entry (this=) at ./src/common/Timer.cc:30 #8 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4bf) at pthread_create.c:463 #9 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 Thread 24 (Thread 0xffff76c649f0 (LWP 126888)): #0 0x0000ffff8b1f922c in futex_wait_cancelable (private=, expected=0, futex_word=0xaaaafbb84018) at ../sysdeps/unix/sysv/linux/futex-internal.h:88 #1 __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafbb83fc0, cond=0xaaaafbb83ff0) at pthread_cond_wait.c:502 #2 __pthread_cond_wait (cond=0xaaaafbb83ff0, mutex=0xaaaafbb83fc0) at pthread_cond_wait.c:655 #3 0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #4 0x0000aaaad284471c in DispatchQueue::run_local_delivery (this=0xaaaafbb83e40) at ./src/msg/DispatchQueue.cc:117 #5 0x0000aaaad26978b0 in DispatchQueue::LocalDeliveryThread::entry (this=) at ./src/msg/DispatchQueue.h:115 #6 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4ff) at pthread_create.c:463 #7 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 Thread 23 (Thread 0xffff784519f0 (LWP 126887)): #0 0x0000ffff8b1f922c in futex_wait_cancelable (private=, expected=0, futex_word=0xaaaafbb83ea8) at ../sysdeps/unix/sysv/linux/futex-internal.h:88 #1 __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafbb83e50, cond=0xaaaafbb83e80) at pthread_cond_wait.c:502 #2 __pthread_cond_wait (cond=0xaaaafbb83e80, mutex=0xaaaafbb83e50) at pthread_cond_wait.c:655 #3 0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #4 0x0000aaaad28449ac in DispatchQueue::entry (this=0xaaaafbb83e40) at ./src/msg/DispatchQueue.cc:210 #5 0x0000aaaad26977f0 in DispatchQueue::DispatchThread::entry (this=) at ./src/msg/DispatchQueue.h:101 #6 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4df) at pthread_create.c:463 #7 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 Thread 22 (Thread 0xffff7bf6c9f0 (LWP 126883)): #0 0x0000ffff8b1f922c in futex_wait_cancelable (private=, expected=0, futex_word=0xffffe7b9d968) at ../sysdeps/unix/sysv/linux/futex-internal.h:88 ---Type to continue, or q to quit--- #1 __pthread_cond_wait_common (abstime=0x0, mutex=0xffffe7b9d910, cond=0xffffe7b9d940) at pthread_cond_wait.c:502 #2 __pthread_cond_wait (cond=0xffffe7b9d940, mutex=0xffffe7b9d910) at pthread_cond_wait.c:655 #3 0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #4 0x0000aaaad24efa80 in Finisher::finisher_thread_entry (this=0xffffe7b9d908) at ./src/common/Finisher.cc:86 #5 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4bf) at pthread_create.c:463 #6 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 Thread 21 (Thread 0xffff8ac349f0 (LWP 124338)): #0 0x0000ffff8b1f922c in futex_wait_cancelable (private=, expected=0, futex_word=0xaaaafaf040f4) at ../sysdeps/unix/sysv/linux/futex-internal.h:88 #1 __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafaf04038, cond=0xaaaafaf040c8) at pthread_cond_wait.c:502 #2 __pthread_cond_wait (cond=0xaaaafaf040c8, mutex=0xaaaafaf04038) at pthread_cond_wait.c:655 #3 0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #4 0x0000aaaad26c26ec in ceph::logging::Log::entry (this=0xaaaafaf04000) at ./src/log/Log.cc:435 #5 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9b02f) at pthread_create.c:463 #6 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 Thread 20 (Thread 0xffff774659f0 (LWP 126377)): #0 0x0000ffff8b1f95d4 in futex_abstimed_wait_cancelable (private=, abstime=0xffff77464f58, expected=0, futex_word=0xaaaafb001cf0) at ../sysdeps/unix/sysv/linux/futex-internal.h:205 #1 __pthread_cond_wait_common (abstime=0xffff77464f58, mutex=0xaaaafb001c78, cond=0xaaaafb001cc8) at pthread_cond_wait.c:539 #2 __pthread_cond_timedwait (cond=0xaaaafb001cc8, mutex=0xaaaafb001c78, abstime=0xffff77464f58) at pthread_cond_wait.c:667 #3 0x0000aaaad2bd5f54 in rocksdb::port::CondVar::TimedWait(unsigned long) () #4 0x0000aaaad2ac818c in rocksdb::InstrumentedCondVar::TimedWaitInternal(unsigned long) () #5 0x0000aaaad2ac80a0 in rocksdb::InstrumentedCondVar::TimedWait(unsigned long) () #6 0x0000aaaad290e284 in rocksdb::RepeatableThread::wait(unsigned long) () #7 0x0000aaaad290e458 in rocksdb::RepeatableThread::thread() () #8 0x0000aaaad290deb4 in rocksdb::RepeatableThread::RepeatableThread(std::function, std::__cxx11::basic_string, std::allocator > const&, rocksdb::Env*, unsigned long, unsigned long)::{lambda()#1}::operator()() const () #9 0x0000aaaad2955904 in void std::__invoke_impl, std::__cxx11::basic_string, std::allocator > const&, rocksdb::Env*, unsigned long, unsigned long)::{lambda()#1}>(std::__invoke_other, rocksdb::RepeatableThread::RepeatableThread(std::function, std::__cxx11::basic_string, std::allocator > const&, rocksdb::Env*, unsigned long, unsigned long)::{lambda()#1}&&) () #10 0x0000aaaad2955888 in std::__invoke_result, std::__cxx11::basic_string, std::allocator > const&, rocksdb::Env*, unsigned long, unsigned long)::{lambda()#1}>::type std::__invoke, std::__cxx11::basic_string, std::allocator > const&, rocksdb::Env*, unsigned long, unsigned long)::{lambda()#1}>(std::__invoke_result&&, (rocksdb::RepeatableThread::RepeatableThread(std::function, std::__cxx11::basic_string, std::allocator > const&, rocksdb::Env*, unsigned long, unsigned long)::{lambda()#1}&&)...) () #11 0x0000aaaad2955810 in void std::thread::_Invoker, std::__cxx11::basic_string, std::allocator > const&, rocksdb::Env*, unsigned long, unsigned long)::{lambda()#1}> >::_M_invoke<0ul>(std::_Index_tuple<0ul>) () #12 0x0000aaaad29557c8 in std::thread::_Invoker, std::__cxx11::basic_string, std::allocator > const&, rocksdb::Env*, unsigned long, unsigned long)::{lambda()#1}> >::operator()() () #13 0x0000aaaad2955794 in std::thread::_State_impl, std::__cxx11::basic_string, std::allocator > const&, rocksdb::Env*, unsigned long, unsigned long)::{lambda()#1}> > >::_M_run() () #14 0x0000ffff8b03ded4 in ?? () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #15 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9b02f) at pthread_create.c:463 #16 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 ---Type to continue, or q to quit--- Thread 19 (Thread 0xffff7cf6e9f0 (LWP 126881)): #0 0x0000ffff8b1f922c in futex_wait_cancelable (private=, expected=0, futex_word=0xaaaafbb82e18) at ../sysdeps/unix/sysv/linux/futex-internal.h:88 #1 __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafbb82dc0, cond=0xaaaafbb82df0) at pthread_cond_wait.c:502 #2 __pthread_cond_wait (cond=0xaaaafbb82df0, mutex=0xaaaafbb82dc0) at pthread_cond_wait.c:655 #3 0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #4 0x0000aaaad284471c in DispatchQueue::run_local_delivery (this=0xaaaafbb82c40) at ./src/msg/DispatchQueue.cc:117 #5 0x0000aaaad26978b0 in DispatchQueue::LocalDeliveryThread::entry (this=) at ./src/msg/DispatchQueue.h:115 #6 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c49f) at pthread_create.c:463 #7 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 Thread 18 (Thread 0xffff7e45d9f0 (LWP 126389)): #0 0x0000ffff8b1f95d4 in futex_abstimed_wait_cancelable (private=, abstime=0xffff7e45d1b8, expected=0, futex_word=0xaaab4be04b28) at ../sysdeps/unix/sysv/linux/futex-internal.h:205 #1 __pthread_cond_wait_common (abstime=0xffff7e45d1b8, mutex=0xaaab4be04b30, cond=0xaaab4be04b00) at pthread_cond_wait.c:539 #2 __pthread_cond_timedwait (cond=0xaaab4be04b00, mutex=0xaaab4be04b30, abstime=0xffff7e45d1b8) at pthread_cond_wait.c:667 #3 0x0000aaaad23c85d4 in __gthread_cond_timedwait (__abs_timeout=0xffff7e45d1b8, __mutex=, __cond=0xaaab4be04b00) at /usr/include/aarch64-linux-gnu/c++/9/bits/gthr-default.h:872 #4 std::condition_variable::__wait_until_impl > > (__atime=..., __lock=..., this=0xaaab4be04b00) at /usr/include/c++/9/condition_variable:188 #5 std::condition_variable::wait_until > > (__atime=..., __lock=..., this=0xaaab4be04b00) at /usr/include/c++/9/condition_variable:121 #6 std::condition_variable::wait_for > (__rtime=..., __lock=..., this=0xaaab4be04b00) at /usr/include/c++/9/condition_variable:152 #7 BlueStore::MempoolThread::entry (this=0xaaab4be04ad8) at ./src/os/bluestore/BlueStore.cc:3882 #8 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c4ff) at pthread_create.c:463 #9 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 Thread 17 (Thread 0xffff7ec5e9f0 (LWP 126383)): #0 0x0000ffff8b1f922c in futex_wait_cancelable (private=, expected=0, futex_word=0xaaab4be047ec) at ../sysdeps/unix/sysv/linux/futex-internal.h:88 #1 __pthread_cond_wait_common (abstime=0x0, mutex=0xaaab4be04790, cond=0xaaab4be047c0) at pthread_cond_wait.c:502 #2 __pthread_cond_wait (cond=0xaaab4be047c0, mutex=0xaaab4be04790) at pthread_cond_wait.c:655 #3 0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #4 0x0000aaaad23cd480 in BlueStore::_kv_finalize_thread (this=0xaaab4be04000) at ./src/os/bluestore/BlueStore.cc:11990 #5 0x0000aaaad2403e50 in BlueStore::KVFinalizeThread::entry (this=) at ./src/os/bluestore/BlueStore.h:1912 #6 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c47f) at pthread_create.c:463 #7 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 Thread 16 (Thread 0xffff794539f0 (LWP 126381)): #0 0x0000ffff8b1f922c in futex_wait_cancelable (private=, expected=0, futex_word=0xaaab4be044cc) at ../sysdeps/unix/sysv/linux/futex-internal.h:88 #1 __pthread_cond_wait_common (abstime=0x0, mutex=0xaaab4be04470, cond=0xaaab4be044a0) at pthread_cond_wait.c:502 #2 __pthread_cond_wait (cond=0xaaab4be044a0, mutex=0xaaab4be04470) at pthread_cond_wait.c:655 #3 0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #4 0x0000aaaad24efa80 in Finisher::finisher_thread_entry (this=0xaaab4be04468) at ./src/common/Finisher.cc:86 #5 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c3ff) at pthread_create.c:463 #6 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 ---Type to continue, or q to quit--- Thread 15 (Thread 0xffff79c549f0 (LWP 126382)): #0 0x0000ffff8b1f922c in futex_wait_cancelable (private=, expected=0, futex_word=0xaaab4be04614) at ../sysdeps/unix/sysv/linux/futex-internal.h:88 #1 __pthread_cond_wait_common (abstime=0x0, mutex=0xaaab4be045b8, cond=0xaaab4be045e8) at pthread_cond_wait.c:502 #2 __pthread_cond_wait (cond=0xaaab4be045e8, mutex=0xaaab4be045b8) at pthread_cond_wait.c:655 #3 0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #4 0x0000aaaad23d7cec in BlueStore::_kv_sync_thread (this=0xaaab4be04000) at ./src/os/bluestore/BlueStore.cc:11723 #5 0x0000aaaad2404af0 in BlueStore::KVSyncThread::entry (this=) at ./src/os/bluestore/BlueStore.h:1904 #6 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c47f) at pthread_create.c:463 #7 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 Thread 14 (Thread 0xffff7f4759f0 (LWP 126334)): #0 0x0000ffff8b1f922c in futex_wait_cancelable (private=, expected=0, futex_word=0xaaab4be12c98) at ../sysdeps/unix/sysv/linux/futex-internal.h:88 #1 __pthread_cond_wait_common (abstime=0x0, mutex=0xaaab4be12b88, cond=0xaaab4be12c70) at pthread_cond_wait.c:502 #2 __pthread_cond_wait (cond=0xaaab4be12c70, mutex=0xaaab4be12b88) at pthread_cond_wait.c:655 #3 0x0000aaaad2bd5e70 in rocksdb::port::CondVar::Wait() () #4 0x0000aaaad2ac7ff4 in rocksdb::InstrumentedCondVar::WaitInternal() () #5 0x0000aaaad2ac7f88 in rocksdb::InstrumentedCondVar::Wait() () #6 0x0000aaaad2b8be80 in rocksdb::DeleteScheduler::BackgroundEmptyTrash() () #7 0x0000aaaad2b921fc in void std::__invoke_impl(std::__invoke_memfun_deref, void (rocksdb::DeleteScheduler::*&&)(), rocksdb::DeleteScheduler*&&) () #8 0x0000aaaad2b920f0 in std::__invoke_result::type std::__invoke(void (rocksdb::DeleteScheduler::*&&)(), rocksdb::DeleteScheduler*&&) () #9 0x0000aaaad2b92040 in void std::thread::_Invoker >::_M_invoke<0ul, 1ul>(std::_Index_tuple<0ul, 1ul>) () #10 0x0000aaaad2b91fdc in std::thread::_Invoker >::operator()() () #11 0x0000aaaad2b91fa8 in std::thread::_State_impl > >::_M_run() () #12 0x0000ffff8b03ded4 in ?? () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #13 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9abaf) at pthread_create.c:463 #14 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 Thread 13 (Thread 0xffff89bb99f0 (LWP 124381)): #0 0x0000ffff8ae0d600 in __GI_epoll_pwait (epfd=, events=0xaaaafbbda000, maxevents=, timeout=-750510080, set=set@entry=0x0) at ../sysdeps/unix/sysv/linux/epoll_pwait.c:42 #1 0x0000ffff8ae0d750 in epoll_wait (epfd=, events=, maxevents=, timeout=) at ../sysdeps/unix/sysv/linux/epoll_wait.c:32 #2 0x0000aaaad26a6574 in EpollDriver::event_wait (this=0xaaaafafcfec0, fired_events=std::vector of length 0, capacity 0, tvp=) at ./src/msg/async/EventEpoll.cc:123 #3 0x0000aaaad269cd64 in EventCenter::process_events (this=this@entry=0xaaaafbb9c8c8, timeout_microseconds=, working_dur=working_dur@entry=0xffff89bb9160) at ./src/msg/async/Event.cc:392 #4 0x0000aaaad26a33c0 in NetworkStack::::operator() (__closure=0xaaaafbc0c4e8, __closure=0xaaaafbc0c4e8) at ./src/msg/async/Stack.cc:53 #5 std::_Function_handler >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/9/bits/std_function.h:300 #6 0x0000ffff8b03ded4 in ?? () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #7 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9aeff) at pthread_create.c:463 ---Type to continue, or q to quit--- #8 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 Thread 12 (Thread 0xffff78c529f0 (LWP 126378)): #0 0x0000ffff8b1f95d4 in futex_abstimed_wait_cancelable (private=, abstime=0xffff78c51f58, expected=0, futex_word=0xaaaafb002710) at ../sysdeps/unix/sysv/linux/futex-internal.h:205 #1 __pthread_cond_wait_common (abstime=0xffff78c51f58, mutex=0xaaaafb002698, cond=0xaaaafb0026e8) at pthread_cond_wait.c:539 #2 __pthread_cond_timedwait (cond=0xaaaafb0026e8, mutex=0xaaaafb002698, abstime=0xffff78c51f58) at pthread_cond_wait.c:667 #3 0x0000aaaad2bd5f54 in rocksdb::port::CondVar::TimedWait(unsigned long) () #4 0x0000aaaad2ac818c in rocksdb::InstrumentedCondVar::TimedWaitInternal(unsigned long) () #5 0x0000aaaad2ac80a0 in rocksdb::InstrumentedCondVar::TimedWait(unsigned long) () #6 0x0000aaaad290e284 in rocksdb::RepeatableThread::wait(unsigned long) () #7 0x0000aaaad290e458 in rocksdb::RepeatableThread::thread() () #8 0x0000aaaad290deb4 in rocksdb::RepeatableThread::RepeatableThread(std::function, std::__cxx11::basic_string, std::allocator > const&, rocksdb::Env*, unsigned long, unsigned long)::{lambda()#1}::operator()() const () #9 0x0000aaaad2955904 in void std::__invoke_impl, std::__cxx11::basic_string, std::allocator > const&, rocksdb::Env*, unsigned long, unsigned long)::{lambda()#1}>(std::__invoke_other, rocksdb::RepeatableThread::RepeatableThread(std::function, std::__cxx11::basic_string, std::allocator > const&, rocksdb::Env*, unsigned long, unsigned long)::{lambda()#1}&&) () #10 0x0000aaaad2955888 in std::__invoke_result, std::__cxx11::basic_string, std::allocator > const&, rocksdb::Env*, unsigned long, unsigned long)::{lambda()#1}>::type std::__invoke, std::__cxx11::basic_string, std::allocator > const&, rocksdb::Env*, unsigned long, unsigned long)::{lambda()#1}>(std::__invoke_result&&, (rocksdb::RepeatableThread::RepeatableThread(std::function, std::__cxx11::basic_string, std::allocator > const&, rocksdb::Env*, unsigned long, unsigned long)::{lambda()#1}&&)...) () #11 0x0000aaaad2955810 in void std::thread::_Invoker, std::__cxx11::basic_string, std::allocator > const&, rocksdb::Env*, unsigned long, unsigned long)::{lambda()#1}> >::_M_invoke<0ul>(std::_Index_tuple<0ul>) () #12 0x0000aaaad29557c8 in std::thread::_Invoker, std::__cxx11::basic_string, std::allocator > const&, rocksdb::Env*, unsigned long, unsigned long)::{lambda()#1}> >::operator()() () #13 0x0000aaaad2955794 in std::thread::_State_impl, std::__cxx11::basic_string, std::allocator > const&, rocksdb::Env*, unsigned long, unsigned long)::{lambda()#1}> > >::_M_run() () #14 0x0000ffff8b03ded4 in ?? () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #15 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9b02f) at pthread_create.c:463 #16 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 Thread 11 (Thread 0xffff814799f0 (LWP 126260)): #0 0x0000ffff8b5166c0 in ?? () from /lib/aarch64-linux-gnu/libaio.so.1 #1 0x0000000000000001 in ?? () Backtrace stopped: previous frame identical to this frame (corrupt stack?) Thread 10 (Thread 0xffff7fc769f0 (LWP 126003)): #0 0x0000ffff8b1f922c in futex_wait_cancelable (private=, expected=0, futex_word=0xaaaafaf8bce8) at ../sysdeps/unix/sysv/linux/futex-internal.h:88 #1 __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafaf8bc90, cond=0xaaaafaf8bcc0) at pthread_cond_wait.c:502 #2 __pthread_cond_wait (cond=0xaaaafaf8bcc0, mutex=0xaaaafaf8bc90) at pthread_cond_wait.c:655 #3 0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #4 0x0000aaaad2c7cb08 in rocksdb::ThreadPoolImpl::Impl::BGThread(unsigned long) () #5 0x0000aaaad2c7ce58 in rocksdb::ThreadPoolImpl::Impl::BGThreadWrapper(void*) () #6 0x0000aaaad2c8438c in void* std::__invoke_impl(std::__invoke_other, void* (*&&)(void*), rocksdb::BGThreadMet---Type to continue, or q to quit--- adata*&&) () #7 0x0000aaaad2c842c4 in std::__invoke_result::type std::__invoke(void* (*&&)(void*), rocksdb::BGThreadMetadata*&&) () #8 0x0000aaaad2c84218 in void* std::thread::_Invoker >::_M_invoke<0ul, 1ul>(std::_Index_tuple<0ul, 1ul>) () #9 0x0000aaaad2c841ac in std::thread::_Invoker >::operator()() () #10 0x0000aaaad2c84178 in std::thread::_State_impl > >::_M_run() () #11 0x0000ffff8b03ded4 in ?? () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #12 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9aacf) at pthread_create.c:463 #13 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 Thread 9 (Thread 0xffff82c7c9f0 (LWP 125800)): #0 0x0000ffff8b5166c0 in ?? () from /lib/aarch64-linux-gnu/libaio.so.1 #1 0x0000000000000001 in ?? () Backtrace stopped: previous frame identical to this frame (corrupt stack?) Thread 8 (Thread 0xffff8347d9f0 (LWP 125797)): #0 0x0000ffff8b1f922c in futex_wait_cancelable (private=, expected=0, futex_word=0xaaaafaf6f244) at ../sysdeps/unix/sysv/linux/futex-internal.h:88 #1 __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafaf6f1e8, cond=0xaaaafaf6f218) at pthread_cond_wait.c:502 #2 __pthread_cond_wait (cond=0xaaaafaf6f218, mutex=0xaaaafaf6f1e8) at pthread_cond_wait.c:655 #3 0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #4 0x0000aaaad24efa80 in Finisher::finisher_thread_entry (this=0xaaaafaf6f1e0) at ./src/common/Finisher.cc:86 #5 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c55f) at pthread_create.c:463 #6 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 Thread 7 (Thread 0xffff8447f9f0 (LWP 125794)): #0 0x0000ffff8b1f922c in futex_wait_cancelable (private=, expected=0, futex_word=0xaaaafaf6fa90) at ../sysdeps/unix/sysv/linux/futex-internal.h:88 #1 __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaafaf6fa20, cond=0xaaaafaf6fa68) at pthread_cond_wait.c:502 #2 __pthread_cond_wait (cond=0xaaaafaf6fa68, mutex=0xaaaafaf6fa20) at pthread_cond_wait.c:655 #3 0x0000ffff8b037f80 in std::condition_variable::wait(std::unique_lock&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #4 0x0000aaaad251d438 in SafeTimer::timer_thread (this=0xaaaafaf6fa50) at ./src/common/Timer.cc:106 #5 0x0000aaaad251e7f0 in SafeTimerThread::entry (this=) at ./src/common/Timer.cc:30 #6 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c55f) at pthread_create.c:463 #7 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 Thread 6 (Thread 0xffff87bb59f0 (LWP 125785)): #0 0x0000ffff8b1f95d4 in futex_abstimed_wait_cancelable (private=, abstime=0xffff87bb51b8, expected=0, futex_word=0xaaaafbc9421c) at ../sysdeps/unix/sysv/linux/futex-internal.h:205 #1 __pthread_cond_wait_common (abstime=0xffff87bb51b8, mutex=0xaaaafbc941c0, cond=0xaaaafbc941f0) at pthread_cond_wait.c:539 #2 __pthread_cond_timedwait (cond=0xaaaafbc941f0, mutex=0xaaaafbc941c0, abstime=0xffff87bb51b8) at pthread_cond_wait.c:667 #3 0x0000aaaad21f4624 in __gthread_cond_timedwait (__abs_timeout=0xffff87bb51b8, __mutex=, __cond=0xaaaafbc941f0) at /usr/include/aarch64-linux-gnu/c++/9/bits/gthr-default.h:872 #4 std::condition_variable::__wait_until_impl > > (__atime=..., __lock=..., this=0xaaaafbc941f0) at /usr/include/c++/9/condition_variable:188 #5 std::condition_variable::wait_until > > (__atime=..., __lock=..., this=0xaaaafbc941f0) at /usr/include/c++/9/condition_variable:121 ---Type to continue, or q to quit--- #6 ceph::timer_detail::timer::timer_thread (this=0xaaaafbc94180) at ./src/common/ceph_timer.h:146 #7 0x0000ffff8b03ded4 in ?? () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #8 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c89f) at pthread_create.c:463 #9 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 Thread 5 (Thread 0xffff856c49f0 (LWP 125781)): #0 0x0000ffff8addf728 in __GI___nanosleep (requested_time=0xffff856c40b8, requested_time@entry=0xffff856c40c8, remaining=remaining@entry=0x0) at ../sysdeps/unix/sysv/linux/nanosleep.c:28 #1 0x0000ffff8ae06fec in usleep (useconds=useconds@entry=256000) at ../sysdeps/posix/usleep.c:32 #2 0x0000aaaad21acca4 in OpHistoryServiceThread::entry (this=0xaaaafaf6ee70) at ./src/common/TrackedOp.cc:44 #3 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9c9ef) at pthread_create.c:463 #4 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 Thread 4 (Thread 0xffff873b49f0 (LWP 125359)): #0 0x0000ffff8ae04048 in __GI___poll (fds=fds@entry=0xffff873b3f68, nfds=281473015820424, nfds@entry=4, timeout=timeout@entry=-1) at ../sysdeps/unix/sysv/linux/poll.c:41 #1 0x0000ffff8ae1e6bc in __poll_chk (fds=fds@entry=0xffff873b3f68, nfds=nfds@entry=4, timeout=timeout@entry=-1, fdslen=fdslen@entry=264) at poll_chk.c:27 #2 0x0000aaaad24d487c in poll (__timeout=-1, __nfds=4, __fds=0xffff873b3f68) at /usr/include/aarch64-linux-gnu/bits/poll2.h:41 #3 SignalHandler::entry (this=0xaaaafaf789a0) at ./src/global/signal_handler.cc:488 #4 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9cbcf) at pthread_create.c:463 #5 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 Thread 3 (Thread 0xffff883b69f0 (LWP 124765)): #0 0x0000ffff8ae04048 in __GI___poll (fds=fds@entry=0xffff883b6198, nfds=281473014030036, nfds@entry=2, timeout=timeout@entry=-1) at ../sysdeps/unix/sysv/linux/poll.c:41 #1 0x0000aaaad252d514 in poll (__timeout=-1, __nfds=2, __fds=0xffff883b6198) at /usr/include/aarch64-linux-gnu/bits/poll2.h:46 #2 AdminSocket::entry (this=0xaaaafaf78160) at ./src/common/admin_socket.cc:240 #3 0x0000ffff8b03ded4 in ?? () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #4 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9cadf) at pthread_create.c:463 #5 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 Thread 2 (Thread 0xffff88bb79f0 (LWP 124763)): #0 0x0000ffff8b1f95d4 in futex_abstimed_wait_cancelable (private=, abstime=0xffff88bb71d8, expected=0, futex_word=0xaaaafaf61c08) at ../sysdeps/unix/sysv/linux/futex-internal.h:205 #1 __pthread_cond_wait_common (abstime=0xffff88bb71d8, mutex=0xaaaafaf61bb0, cond=0xaaaafaf61be0) at pthread_cond_wait.c:539 #2 __pthread_cond_timedwait (cond=0xaaaafaf61be0, mutex=0xaaaafaf61bb0, abstime=0xffff88bb71d8) at pthread_cond_wait.c:667 #3 0x0000aaaad254a954 in __gthread_cond_timedwait (__abs_timeout=0xffff88bb71d8, __mutex=, __cond=0xaaaafaf61be0) at /usr/include/aarch64-linux-gnu/c++/9/bits/gthr-default.h:872 #4 std::condition_variable::__wait_until_impl > > (__atime=..., __lock=..., this=0xaaaafaf61be0) at /usr/include/c++/9/condition_variable:188 #5 std::condition_variable::wait_until > > (__atime=..., __lock=..., this=0xaaaafaf61be0) at /usr/include/c++/9/condition_variable:121 #6 std::condition_variable::wait_for > (__rtime=..., __lock=..., this=0xaaaafaf61be0) at /usr/include/c++/9/condition_variable:152 #7 CephContextServiceThread::entry (this=0xaaaafaf61b90) at ./src/common/ceph_context.cc:206 #8 0x0000ffff8b1f3088 in start_thread (arg=0xffffe7b9ca7f) at pthread_create.c:463 ---Type to continue, or q to quit--- #9 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 Thread 1 (Thread 0xffff866e89f0 (LWP 124715)): #0 raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:51 #1 0x0000aaaad24d3cec in reraise_fatal (signum=6) at ./src/global/signal_handler.cc:326 #2 handle_fatal_signal (signum=6) at ./src/global/signal_handler.cc:326 #3 #4 __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:51 #5 0x0000ffff8ad708b4 in __GI_abort () at abort.c:79 #6 0x0000aaaad1e2c1b0 in ceph::__ceph_assert_fail (assertion=, file=, line=, func=) at ./src/common/assert.cc:73 #7 0x0000aaaad1e2c320 in ceph::__ceph_assert_fail (ctx=...) at ./src/common/assert.cc:78 #8 0x0000aaaad26aeb0c in RDMADispatcher::handle_tx_event (this=this@entry=0xaaaafaf78c70, cqe=cqe@entry=0xffff866e7a90, n=n@entry=1) at ./src/msg/async/rdma/RDMAStack.cc:558 #9 0x0000aaaad26b07bc in RDMADispatcher::polling (this=0xaaaafaf78c70) at ./src/msg/async/rdma/RDMAStack.cc:282 #10 0x0000ffff8b03ded4 in ?? () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #11 0x0000ffff8b1f3088 in start_thread (arg=0xffff893b7bff) at pthread_create.c:463 #12 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 (gdb) (gdb) (gdb) (gdb) (gdb) (gdb) (gdb) (gdb) (gdb) (gdb) (gdb) (gdb) (gdb) (gdb) bt #0 raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:51 #1 0x0000aaaad24d3cec in reraise_fatal (signum=6) at ./src/global/signal_handler.cc:326 #2 handle_fatal_signal (signum=6) at ./src/global/signal_handler.cc:326 #3 #4 __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:51 #5 0x0000ffff8ad708b4 in __GI_abort () at abort.c:79 #6 0x0000aaaad1e2c1b0 in ceph::__ceph_assert_fail (assertion=, file=, line=, func=) at ./src/common/assert.cc:73 #7 0x0000aaaad1e2c320 in ceph::__ceph_assert_fail (ctx=...) at ./src/common/assert.cc:78 #8 0x0000aaaad26aeb0c in RDMADispatcher::handle_tx_event (this=this@entry=0xaaaafaf78c70, cqe=cqe@entry=0xffff866e7a90, n=n@entry=1) at ./src/msg/async/rdma/RDMAStack.cc:558 #9 0x0000aaaad26b07bc in RDMADispatcher::polling (this=0xaaaafaf78c70) at ./src/msg/async/rdma/RDMAStack.cc:282 #10 0x0000ffff8b03ded4 in ?? () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #11 0x0000ffff8b1f3088 in start_thread (arg=0xffff893b7bff) at pthread_create.c:463 #12 0x0000ffff8ae0d4ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 (gdb) f 8 #8 0x0000aaaad26aeb0c in RDMADispatcher::handle_tx_event (this=this@entry=0xaaaafaf78c70, cqe=cqe@entry=0xffff866e7a90, n=n@entry=1) at ./src/msg/async/rdma/RDMAStack.cc:558 558 in ./src/msg/async/rdma/RDMAStack.cc (gdb) l 553 in ./src/msg/async/rdma/RDMAStack.cc (gdb) directory ~/chunsong/ceph Source directories searched: /root/chunsong/ceph:$cdir:$cwd (gdb) l 553 554 //TX completion may come either from 555 // 1) regular send message, WCE wr_id points to chunk 556 // 2) 'fin' message, wr_id points to the QP 557 if (ib->get_memory_manager()->is_tx_buffer(chunk->buffer)) { 558 ceph_assert(valid); 559 tx_chunks.push_back(chunk); 560 } else if (reinterpret_cast(response->wr_id)->get_local_qp_number() == response->qp_num ) { 561 ceph_assert(valid == false); 562 ldout(cct, 1) << __func__ << " sending of the disconnect msg completed" << dendl; (gdb) p chunk $1 = (Infiniband::MemoryManager::Chunk *) 0xaaab4fe52080 (gdb) p *chunk $2 = {mr = 0xaaaafafd4000, qp = 0xaaaafaf666f0, lkey = 2, bytes = 43691, offset = 4224287488, bound = 43690, buffer = 0xaaab00000001 "", pad = {4210454592, 43690, 4210544584, 43690, 1386260360, 43691}, data = 0xaaab4fe520c0 ""} (gdb) p ib-> _M_enable_shared_from_this_with operator-> _M_get operator= _M_get_deleter reset _M_ptr shared_ptr _M_refcount shared_ptr, CephContext*&> __shared_ptr swap __shared_ptr, CephContext*&> unique get use_count operator bool ~__shared_ptr operator* ~shared_ptr (gdb) p *ib $3 = {tx_queue_len = 1024, rx_queue_len = 4096, max_sge = 0, ib_physical_port = 1 '\001', memory_manager = 0xaaaafaf4f550, srq = 0xaaaafaf7cfc8, device = 0xaaaafb001200, pd = 0xaaaafb018368, device_list = 0xaaaafbb9b500, cct = 0xaaaafafd4000, lock = { = {_M_mutex = pthread_mutex_t = { Type = Normal, Status = Not acquired, Robust = No, Shared = No, Protocol = None}}, }, initialized = true, device_name = "mlx5_0", port_num = 1 '\001', support_srq = true} (gdb) p ib->memory_manager $4 = (Infiniband::MemoryManager *) 0xaaaafaf4f550 (gdb) p ib->memory_manager->send $5 = (Infiniband::MemoryManager::Cluster *) 0xaaaafaf48c30 (gdb) p *ib->memory_manager->send $6 = {manager = @0xaaaafaf4f550, buffer_size = 131072, num_chunk = 2048, lock = { = {_M_mutex = pthread_mutex_t = {Type = Normal, Status = Not acquired, Robust = No, Shared = No, Protocol = None}}, }, free_chunks = std::vector of length 2048, capacity 2048 = { 0xaaab0bcca000, 0xaaab0bcca040, 0xaaab0bcca080, 0xaaab0bcca0c0, 0xaaab0bcca100, 0xaaab0bcca140, 0xaaab0bcca180, 0xaaab0bcca1c0, 0xaaab0bcca200, 0xaaab0bcca240, 0xaaab0bcca280, 0xaaab0bcca2c0, 0xaaab0bcca300, 0xaaab0bcca340, 0xaaab0bcca380, 0xaaab0bcca3c0, 0xaaab0bcca400, 0xaaab0bcca440, 0xaaab0bcca480, 0xaaab0bcca4c0, 0xaaab0bcca500, 0xaaab0bcca540, 0xaaab0bcca580, 0xaaab0bcca5c0, 0xaaab0bcca600, 0xaaab0bcca640, 0xaaab0bcca680, 0xaaab0bcca6c0, 0xaaab0bcca700, 0xaaab0bcca740, 0xaaab0bcca780, 0xaaab0bcca7c0, 0xaaab0bcca800, 0xaaab0bcca840, 0xaaab0bcca880, 0xaaab0bcca8c0, 0xaaab0bcca900, 0xaaab0bcca940, 0xaaab0bcca980, 0xaaab0bcca9c0, 0xaaab0bccaa00, 0xaaab0bccaa40, 0xaaab0bccaa80, 0xaaab0bccaac0, 0xaaab0bccab00, 0xaaab0bccab40, 0xaaab0bccab80, 0xaaab0bccabc0, 0xaaab0bccac00, 0xaaab0bccac40, 0xaaab0bccac80, 0xaaab0bccacc0, 0xaaab0bccad00, 0xaaab0bccad40, 0xaaab0bccad80, 0xaaab0bccadc0, 0xaaab0bccae00, 0xaaab0bccae40, 0xaaab0bccae80, 0xaaab0bccaec0, 0xaaab0bccaf00, 0xaaab0bccaf40, 0xaaab0bccaf80, 0xaaab0bccafc0, 0xaaab0bccb000, 0xaaab0bccb040, 0xaaab0bccb080, 0xaaab0bccb0c0, 0xaaab0bccb100, 0xaaab0bccb140, 0xaaab0bccb180, 0xaaab0bccb1c0, 0xaaab0bccb200, 0xaaab0bccb240, 0xaaab0bccb280, 0xaaab0bccb2c0, 0xaaab0bccb300, 0xaaab0bccb340, 0xaaab0bccb380, 0xaaab0bccb3c0, 0xaaab0bccb400, 0xaaab0bccb440, 0xaaab0bccb480, 0xaaab0bccb4c0, 0xaaab0bccb500, 0xaaab0bccb540, 0xaaab0bccb580, 0xaaab0bccb5c0, 0xaaab0bccb600, 0xaaab0bccb640, 0xaaab0bccb680, 0xaaab0bccb6c0, 0xaaab0bccb700, 0xaaab0bccb740, 0xaaab0bccb780, 0xaaab0bccb7c0, 0xaaab0bccb800, 0xaaab0bccb840, 0xaaab0bccb880, 0xaaab0bccb8c0, 0xaaab0bccb900, 0xaaab0bccb940, 0xaaab0bccb980, 0xaaab0bccb9c0, 0xaaab0bccba00, 0xaaab0bccba40, 0xaaab0bccba80, 0xaaab0bccbac0, 0xaaab0bccbb00, 0xaaab0bccbb40, 0xaaab0bccbb80, 0xaaab0bccbbc0, 0xaaab0bccbc00, 0xaaab0bccbc40, 0xaaab0bccbc80, 0xaaab0bccbcc0, 0xaaab0bccbd00, 0xaaab0bccbd40, 0xaaab0bccbd80, 0xaaab0bccbdc0, 0xaaab0bccbe00, 0xaaab0bccbe40, 0xaaab0bccbe80, 0xaaab0bccbec0, 0xaaab0bccbf00, 0xaaab0bccbf40, 0xaaab0bccbf80, 0xaaab0bccbfc0, 0xaaab0bccc000, 0xaaab0bccc040, 0xaaab0bccc080, 0xaaab0bccc0c0, 0xaaab0bccc100, 0xaaab0bccc140, 0xaaab0bccc180, 0xaaab0bccc1c0, 0xaaab0bccc200, 0xaaab0bccc240, 0xaaab0bccc280, 0xaaab0bccc2c0, 0xaaab0bccc300, 0xaaab0bccc340, 0xaaab0bccc380, 0xaaab0bccc3c0, 0xaaab0bccc400, 0xaaab0bccc440, 0xaaab0bccc480, 0xaaab0bccc4c0, 0xaaab0bccc500, 0xaaab0bccc540, 0xaaab0bccc580, 0xaaab0bccc5c0, 0xaaab0bccc600, 0xaaab0bccc640, 0xaaab0bccc680, 0xaaab0bccc6c0, 0xaaab0bccc700, 0xaaab0bccc740, 0xaaab0bccc780, 0xaaab0bccc7c0, 0xaaab0bccc800, 0xaaab0bccc840, 0xaaab0bccc880, 0xaaab0bccc8c0, 0xaaab0bccc900, 0xaaab0bccc940, 0xaaab0bccc980, 0xaaab0bccc9c0, 0xaaab0bccca00, 0xaaab0bccca40, 0xaaab0bccca80, 0xaaab0bcccac0, 0xaaab0bcccb00, 0xaaab0bcccb40, 0xaaab0bcccb80, 0xaaab0bcccbc0, 0xaaab0bcccc00, 0xaaab0bcccc40, 0xaaab0bcccc80, 0xaaab0bccccc0, 0xaaab0bcccd00, 0xaaab0bcccd40, 0xaaab0bcccd80, 0xaaab0bcccdc0, 0xaaab0bccce00, 0xaaab0bccce40, 0xaaab0bccce80, 0xaaab0bcccec0, 0xaaab0bcccf00, 0xaaab0bcccf40, 0xaaab0bcccf80, 0xaaab0bcccfc0, 0xaaab0bccd000, 0xaaab0bccd040, 0xaaab0bccd080, 0xaaab0bccd0c0, 0xaaab0bccd100, 0xaaab0bccd140, 0xaaab0bccd180, 0xaaab0bccd1c0...}, base = 0xaaaafbcca000 "", end = 0xaaab0bcca000 "\320\035\366", , chunk_base = 0xaaab0bcca000} (failed reverse-i-search)`pch': Quitb->memory_manager->send (gdb) p *chunk $7 = {mr = 0xaaaafafd4000, qp = 0xaaaafaf666f0, lkey = 2, bytes = 43691, offset = 4224287488, bound = 43690, buffer = 0xaaab00000001 "", pad = {4210454592, 43690, 4210544584, 43690, 1386260360, 43691}, data = 0xaaab4fe520c0 ""} (gdb) p chunk $8 = (Infiniband::MemoryManager::Chunk *) 0xaaab4fe52080 (gdb) ceph Quit