[New LWP 308188] [Thread debugging using libthread_db enabled] Using host libthread_db library "/lib64/libthread_db.so.1". Core was generated by `/usr/bin/ceph-osd'. #0 0x00007f241d86b965 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 Missing separate debuginfos, use: debuginfo-install bzip2-libs-1.0.6-13.el7.x86_64 elfutils-libelf-0.170-4.el7.x86_64 elfutils-libs-0.170-4.el7.x86_64 fuse-libs-2.9.2-11.el7.x86_64 glibc-2.17-260.el7_6.3.x86_64 gperftools-libs-2.6.1-1.el7.x86_64 leveldb-1.12.0-11.el7.x86_64 libaio-0.3.109-13.el7.x86_64 libattr-2.4.46-13.el7.x86_64 libblkid-2.23.2-59.el7.x86_64 libcap-2.22-9.el7.x86_64 libgcc-4.8.5-36.el7.x86_64 libibverbs-17.2-3.el7.x86_64 libnl3-3.2.28-4.el7.x86_64 liboath-2.4.1-9.el7.x86_64 librdmacm-17.2-3.el7.x86_64 libstdc++-4.8.5-28.el7.x86_64 libuuid-2.23.2-59.el7.x86_64 lttng-ust-2.4.1-4.el7.x86_64 lz4-1.7.5-3.el7.x86_64 openssl-libs-1.0.2k-16.el7.x86_64 snappy-1.1.0-3.el7.x86_64 systemd-libs-219-62.el7_6.2.x86_64 userspace-rcu-0.7.16-1.el7.x86_64 xz-libs-5.2.2-1.el7.x86_64 zlib-1.2.7-18.el7.x86_64 (gdb) bt #0 0x00007f241d86b965 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f241cfc5b1c in std::condition_variable::wait(std::unique_lock&) () from /lib64/libstdc++.so.6 #2 0x000055981b62a3af in ceph::logging::Log::entry() () at /usr/src/debug/ceph-15.2.8.1.0.0/src/log/Log.cc:435 #3 0x00007f241d867dd5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007f241c72cead in clone () from /lib64/libc.so.6 (gdb) thread apply all bt Thread 78 (Thread 0x7f241f7fbc00 (LWP 308188)): #0 0x00007f241d86b965 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f241cfc5b1c in std::condition_variable::wait(std::unique_lock&) () from /lib64/libstdc++.so.6 #2 0x000055981b5fd78f in AsyncMessenger::wait() () at /usr/src/debug/ceph-15.2.8.1.0.0/src/msg/async/AsyncMessenger.cc:556 #3 0x000055981ade5d13 in main () at /usr/src/debug/ceph-15.2.8.1.0.0/src/ceph_osd.cc:738 #4 0x00007f241c6513d5 in __libc_start_main () from /lib64/libc.so.6 #5 0x000055981ae1b2b5 in _start () at /opt/rh/devtoolset-8/root/usr/include/c++/8/new:169 Thread 77 (Thread 0x7f23f194b700 (LWP 312060)): #0 0x00007f241d86bd12 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x000055981ae6e1b7 in __gthread_cond_timedwait (__abs_timeout=0x7f23f19489d0, __mutex=, __cond=0x5598261c1350) at /opt/rh/devtoolset-8/root/usr/include/c++/8/x86_64-redhat-linux/bits/gthr-default.h:871 #2 __wait_until_impl > > (__atime=..., __lock=..., this=0x5598261c1350) at /opt/rh/devtoolset-8/root/usr/include/c++/8/condition_variable:178 #3 wait_until > > (__atime=..., __lock=..., this=0x5598261c1350) at /opt/rh/devtoolset-8/root/usr/include/c++/8/condition_variable:106 #4 wait_for > (__rtime=, __lock=..., this=0x5598261c1350) at /opt/rh/devtoolset-8/root/usr/include/c++/8/condition_variable:143 #5 OSD::heartbeat_entry() () at /usr/src/debug/ceph-15.2.8.1.0.0/src/osd/OSD.cc:5588 #6 0x000055981aee6d0d in OSD::T_Heartbeat::entry (this=) at /usr/src/debug/ceph-15.2.8.1.0.0/src/osd/OSD.h:1483 #7 0x00007f241d867dd5 in start_thread () from /lib64/libpthread.so.0 #8 0x00007f241c72cead in clone () from /lib64/libc.so.6 Thread 76 (Thread 0x7f23f214c700 (LWP 312059)): #0 0x00007f241d86b965 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f241cfc5b1c in std::condition_variable::wait(std::unique_lock&) () from /lib64/libstdc++.so.6 #2 0x000055981ae83184 in OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*) () at /usr/src/debug/ceph-15.2.8.1.0.0/src/osd/OSD.cc:10460 #3 0x000055981b490b66 in ShardedThreadPool::shardedthreadpool_worker(unsigned int) () at /usr/src/debug/ceph-15.2.8.1.0.0/src/common/WorkQueue.cc:311 #4 0x000055981b4936b0 in ShardedThreadPool::WorkThreadSharded::entry (this=) at /usr/src/debug/ceph-15.2.8.1.0.0/src/common/WorkQueue.h:715 #5 0x00007f241d867dd5 in start_thread () from /lib64/libpthread.so.0 #6 0x00007f241c72cead in clone () from /lib64/libc.so.6 Thread 75 (Thread 0x7f23f294d700 (LWP 312058)): #0 0x00007f241d86b965 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f241cfc5b1c in std::condition_variable::wait(std::unique_lock&) () from /lib64/libstdc++.so.6 #2 0x000055981ae83184 in OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*) () at /usr/src/debug/ceph-15.2.8.1.0.0/src/osd/OSD.cc:10460 #3 0x000055981b490b66 in ShardedThreadPool::shardedthreadpool_worker(unsigned int) () at /usr/src/debug/ceph-15.2.8.1.0.0/src/common/WorkQueue.cc:311 #4 0x000055981b4936b0 in ShardedThreadPool::WorkThreadSharded::entry (this=) at /usr/src/debug/ceph-15.2.8.1.0.0/src/common/WorkQueue.h:715 #5 0x00007f241d867dd5 in start_thread () from /lib64/libpthread.so.0 #6 0x00007f241c72cead in clone () from /lib64/libc.so.6 Thread 74 (Thread 0x7f23f314e700 (LWP 312057)): #0 0x00007f241d86b965 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f241cfc5b1c in std::condition_variable::wait(std::unique_lock&) () from /lib64/libstdc++.so.6 #2 0x000055981ae83184 in OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*) () at /usr/src/debug/ceph-15.2.8.1.0.0/src/osd/OSD.cc:10460 #3 0x000055981b490b66 in ShardedThreadPool::shardedthreadpool_worker(unsigned int) () at /usr/src/debug/ceph-15.2.8.1.0.0/src/common/WorkQueue.cc:311 #4 0x000055981b4936b0 in ShardedThreadPool::WorkThreadSharded::entry (this=) at /usr/src/debug/ceph-15.2.8.1.0.0/src/common/WorkQueue.h:715 #5 0x00007f241d867dd5 in start_thread () from /lib64/libpthread.so.0 #6 0x00007f241c72cead in clone () from /lib64/libc.so.6 Thread 73 (Thread 0x7f23f394f700 (LWP 312056)): #0 0x00007f241d86b965 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f241cfc5b1c in std::condition_variable::wait(std::unique_lock&) () from /lib64/libstdc++.so.6 #2 0x000055981ae83184 in OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*) () at /usr/src/debug/ceph-15.2.8.1.0.0/src/osd/OSD.cc:10460 #3 0x000055981b490b66 in ShardedThreadPool::shardedthreadpool_worker(unsigned int) () at /usr/src/debug/ceph-15.2.8.1.0.0/src/common/WorkQueue.cc:311 #4 0x000055981b4936b0 in ShardedThreadPool::WorkThreadSharded::entry (this=) at /usr/src/debug/ceph-15.2.8.1.0.0/src/common/WorkQueue.h:715 #5 0x00007f241d867dd5 in start_thread () from /lib64/libpthread.so.0 #6 0x00007f241c72cead in clone () from /lib64/libc.so.6 Thread 72 (Thread 0x7f23f4150700 (LWP 312055)): ---Type to continue, or q to quit--- #0 0x00007f241d86b965 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f241cfc5b1c in std::condition_variable::wait(std::unique_lock&) () from /lib64/libstdc++.so.6 #2 0x000055981ae83184 in OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*) () at /usr/src/debug/ceph-15.2.8.1.0.0/src/osd/OSD.cc:10460 #3 0x000055981b490b66 in ShardedThreadPool::shardedthreadpool_worker(unsigned int) () at /usr/src/debug/ceph-15.2.8.1.0.0/src/common/WorkQueue.cc:311 #4 0x000055981b4936b0 in ShardedThreadPool::WorkThreadSharded::entry (this=) at /usr/src/debug/ceph-15.2.8.1.0.0/src/common/WorkQueue.h:715 #5 0x00007f241d867dd5 in start_thread () from /lib64/libpthread.so.0 #6 0x00007f241c72cead in clone () from /lib64/libc.so.6 Thread 71 (Thread 0x7f23f4951700 (LWP 312054)): #0 0x00007f241d86b965 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f241cfc5b1c in std::condition_variable::wait(std::unique_lock&) () from /lib64/libstdc++.so.6 #2 0x000055981ae83184 in OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*) () at /usr/src/debug/ceph-15.2.8.1.0.0/src/osd/OSD.cc:10460 #3 0x000055981b490b66 in ShardedThreadPool::shardedthreadpool_worker(unsigned int) () at /usr/src/debug/ceph-15.2.8.1.0.0/src/common/WorkQueue.cc:311 #4 0x000055981b4936b0 in ShardedThreadPool::WorkThreadSharded::entry (this=) at /usr/src/debug/ceph-15.2.8.1.0.0/src/common/WorkQueue.h:715 #5 0x00007f241d867dd5 in start_thread () from /lib64/libpthread.so.0 #6 0x00007f241c72cead in clone () from /lib64/libc.so.6 Thread 70 (Thread 0x7f23f5152700 (LWP 312053)): #0 0x00007f241d86b965 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f241cfc5b1c in std::condition_variable::wait(std::unique_lock&) () from /lib64/libstdc++.so.6 #2 0x000055981ae83184 in OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*) () at /usr/src/debug/ceph-15.2.8.1.0.0/src/osd/OSD.cc:10460 #3 0x000055981b490b66 in ShardedThreadPool::shardedthreadpool_worker(unsigned int) () at /usr/src/debug/ceph-15.2.8.1.0.0/src/common/WorkQueue.cc:311 #4 0x000055981b4936b0 in ShardedThreadPool::WorkThreadSharded::entry (this=) at /usr/src/debug/ceph-15.2.8.1.0.0/src/common/WorkQueue.h:715 #5 0x00007f241d867dd5 in start_thread () from /lib64/libpthread.so.0 #6 0x00007f241c72cead in clone () from /lib64/libc.so.6 Thread 69 (Thread 0x7f23f5953700 (LWP 312052)): #0 0x00007f241d86b965 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f241cfc5b1c in std::condition_variable::wait(std::unique_lock&) () from /lib64/libstdc++.so.6 #2 0x000055981ae83184 in OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*) () at /usr/src/debug/ceph-15.2.8.1.0.0/src/osd/OSD.cc:10460 #3 0x000055981b490b66 in ShardedThreadPool::shardedthreadpool_worker(unsigned int) () at /usr/src/debug/ceph-15.2.8.1.0.0/src/common/WorkQueue.cc:311 #4 0x000055981b4936b0 in ShardedThreadPool::WorkThreadSharded::entry (this=) at /usr/src/debug/ceph-15.2.8.1.0.0/src/common/WorkQueue.h:715 #5 0x00007f241d867dd5 in start_thread () from /lib64/libpthread.so.0 #6 0x00007f241c72cead in clone () from /lib64/libc.so.6 Thread 68 (Thread 0x7f23f6154700 (LWP 312051)): #0 0x00007f241d86b965 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f241cfc5b1c in std::condition_variable::wait(std::unique_lock&) () from /lib64/libstdc++.so.6 #2 0x000055981ae83184 in OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*) () at /usr/src/debug/ceph-15.2.8.1.0.0/src/osd/OSD.cc:10460 #3 0x000055981b490b66 in ShardedThreadPool::shardedthreadpool_worker(unsigned int) () at /usr/src/debug/ceph-15.2.8.1.0.0/src/common/WorkQueue.cc:311 #4 0x000055981b4936b0 in ShardedThreadPool::WorkThreadSharded::entry (this=) at /usr/src/debug/ceph-15.2.8.1.0.0/src/common/WorkQueue.h:715 #5 0x00007f241d867dd5 in start_thread () from /lib64/libpthread.so.0 #6 0x00007f241c72cead in clone () from /lib64/libc.so.6 Thread 67 (Thread 0x7f23f6955700 (LWP 312050)): #0 0x00007f241d86b965 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f241cfc5b1c in std::condition_variable::wait(std::unique_lock&) () from /lib64/libstdc++.so.6 #2 0x000055981ae83184 in OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*) () at /usr/src/debug/ceph-15.2.8.1.0.0/src/osd/OSD.cc:10460 #3 0x000055981b490b66 in ShardedThreadPool::shardedthreadpool_worker(unsigned int) () at /usr/src/debug/ceph-15.2.8.1.0.0/src/common/WorkQueue.cc:311 #4 0x000055981b4936b0 in ShardedThreadPool::WorkThreadSharded::entry (this=) at /usr/src/debug/ceph-15.2.8.1.0.0/src/common/WorkQueue.h:715 #5 0x00007f241d867dd5 in start_thread () from /lib64/libpthread.so.0 #6 0x00007f241c72cead in clone () from /lib64/libc.so.6 Thread 66 (Thread 0x7f23f7156700 (LWP 312049)): #0 0x00007f241d86b965 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f241cfc5b1c in std::condition_variable::wait(std::unique_lock&) () from /lib64/libstdc++.so.6 #2 0x000055981ae83184 in OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*) () at /usr/src/debug/ceph-15.2.8.1.0.0/src/osd/OSD.cc:10460 ---Type to continue, or q to quit--- #3 0x000055981b490b66 in ShardedThreadPool::shardedthreadpool_worker(unsigned int) () at /usr/src/debug/ceph-15.2.8.1.0.0/src/common/WorkQueue.cc:311 #4 0x000055981b4936b0 in ShardedThreadPool::WorkThreadSharded::entry (this=) at /usr/src/debug/ceph-15.2.8.1.0.0/src/common/WorkQueue.h:715 #5 0x00007f241d867dd5 in start_thread () from /lib64/libpthread.so.0 #6 0x00007f241c72cead in clone () from /lib64/libc.so.6 Thread 65 (Thread 0x7f23f7957700 (LWP 312048)): #0 0x00007f241d86b965 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f241cfc5b1c in std::condition_variable::wait(std::unique_lock&) () from /lib64/libstdc++.so.6 #2 0x000055981ae83184 in OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*) () at /usr/src/debug/ceph-15.2.8.1.0.0/src/osd/OSD.cc:10460 #3 0x000055981b490b66 in ShardedThreadPool::shardedthreadpool_worker(unsigned int) () at /usr/src/debug/ceph-15.2.8.1.0.0/src/common/WorkQueue.cc:311 #4 0x000055981b4936b0 in ShardedThreadPool::WorkThreadSharded::entry (this=) at /usr/src/debug/ceph-15.2.8.1.0.0/src/common/WorkQueue.h:715 #5 0x00007f241d867dd5 in start_thread () from /lib64/libpthread.so.0 #6 0x00007f241c72cead in clone () from /lib64/libc.so.6 Thread 64 (Thread 0x7f23f8158700 (LWP 312047)): #0 0x00007f241d86b965 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f241cfc5b1c in std::condition_variable::wait(std::unique_lock&) () from /lib64/libstdc++.so.6 #2 0x000055981ae83184 in OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*) () at /usr/src/debug/ceph-15.2.8.1.0.0/src/osd/OSD.cc:10460 #3 0x000055981b490b66 in ShardedThreadPool::shardedthreadpool_worker(unsigned int) () at /usr/src/debug/ceph-15.2.8.1.0.0/src/common/WorkQueue.cc:311 #4 0x000055981b4936b0 in ShardedThreadPool::WorkThreadSharded::entry (this=) at /usr/src/debug/ceph-15.2.8.1.0.0/src/common/WorkQueue.h:715 #5 0x00007f241d867dd5 in start_thread () from /lib64/libpthread.so.0 #6 0x00007f241c72cead in clone () from /lib64/libc.so.6 Thread 63 (Thread 0x7f23f8959700 (LWP 312046)): #0 0x00007f241d86b965 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f241cfc5b1c in std::condition_variable::wait(std::unique_lock&) () from /lib64/libstdc++.so.6 #2 0x000055981ae83184 in OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*) () at /usr/src/debug/ceph-15.2.8.1.0.0/src/osd/OSD.cc:10460 #3 0x000055981b490b66 in ShardedThreadPool::shardedthreadpool_worker(unsigned int) () at /usr/src/debug/ceph-15.2.8.1.0.0/src/common/WorkQueue.cc:311 #4 0x000055981b4936b0 in ShardedThreadPool::WorkThreadSharded::entry (this=) at /usr/src/debug/ceph-15.2.8.1.0.0/src/common/WorkQueue.h:715 #5 0x00007f241d867dd5 in start_thread () from /lib64/libpthread.so.0 #6 0x00007f241c72cead in clone () from /lib64/libc.so.6 Thread 62 (Thread 0x7f23f915a700 (LWP 312045)): #0 0x00007f241d86b965 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f241cfc5b1c in std::condition_variable::wait(std::unique_lock&) () from /lib64/libstdc++.so.6 #2 0x000055981ae83184 in OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*) () at /usr/src/debug/ceph-15.2.8.1.0.0/src/osd/OSD.cc:10460 #3 0x000055981b490b66 in ShardedThreadPool::shardedthreadpool_worker(unsigned int) () at /usr/src/debug/ceph-15.2.8.1.0.0/src/common/WorkQueue.cc:311 #4 0x000055981b4936b0 in ShardedThreadPool::WorkThreadSharded::entry (this=) at /usr/src/debug/ceph-15.2.8.1.0.0/src/common/WorkQueue.h:715 #5 0x00007f241d867dd5 in start_thread () from /lib64/libpthread.so.0 #6 0x00007f241c72cead in clone () from /lib64/libc.so.6 Thread 61 (Thread 0x7f23f995b700 (LWP 312044)): #0 0x00007f241d86b965 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f241cfc5b1c in std::condition_variable::wait(std::unique_lock&) () from /lib64/libstdc++.so.6 #2 0x000055981ae3a0b7 in OSDService::agent_entry() () at /usr/src/debug/ceph-15.2.8.1.0.0/src/osd/OSD.cc:560 #3 0x000055981aee6bbd in OSDService::AgentThread::entry (this=) at /usr/src/debug/ceph-15.2.8.1.0.0/src/osd/OSD.h:382 #4 0x00007f241d867dd5 in start_thread () from /lib64/libpthread.so.0 #5 0x00007f241c72cead in clone () from /lib64/libc.so.6 Thread 60 (Thread 0x7f23fa15c700 (LWP 312043)): #0 0x00007f241d86bd12 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x000055981aea2e81 in __gthread_cond_timedwait (__abs_timeout=0x7f23fa1599e0, __mutex=, __cond=0x5598261c2db8) at /opt/rh/devtoolset-8/root/usr/include/c++/8/x86_64-redhat-linux/bits/gthr-default.h:871 #2 __wait_until_impl > > (__atime=, __lock=..., this=0x5598261c2db8) at /opt/rh/devtoolset-8/root/usr/include/c++/8/condition_variable:178 #3 wait_until > > (__atime=, __lock=..., this=0x5598261c2db8) at /opt/rh/devtoolset-8/root/usr/include/c++/8/condition_variable:119 #4 ceph::timer::timer_thread (this=0x5598261c2d50) at /usr/src/debug/ceph-15.2.8.1.0.0/src/common/ceph_timer.h:140 ---Type to continue, or q to quit--- #5 0x000055981bb0ee4f in execute_native_thread_routine () #6 0x00007f241d867dd5 in start_thread () from /lib64/libpthread.so.0 #7 0x00007f241c72cead in clone () from /lib64/libc.so.6 Thread 59 (Thread 0x7f23fa95d700 (LWP 312042)): #0 0x00007f241d86b965 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f241cfc5b1c in std::condition_variable::wait(std::unique_lock&) () from /lib64/libstdc++.so.6 #2 0x000055981b48a5d3 in SafeTimer::timer_thread() () at /usr/src/debug/ceph-15.2.8.1.0.0/src/common/Timer.cc:106 #3 0x000055981b48b97d in SafeTimerThread::entry (this=) at /usr/src/debug/ceph-15.2.8.1.0.0/src/common/Timer.cc:30 #4 0x00007f241d867dd5 in start_thread () from /lib64/libpthread.so.0 #5 0x00007f241c72cead in clone () from /lib64/libc.so.6 Thread 58 (Thread 0x7f23fb15e700 (LWP 312041)): #0 0x00007f241d86b965 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f241cfc5b1c in std::condition_variable::wait(std::unique_lock&) () from /lib64/libstdc++.so.6 #2 0x000055981b48a5d3 in SafeTimer::timer_thread() () at /usr/src/debug/ceph-15.2.8.1.0.0/src/common/Timer.cc:106 #3 0x000055981b48b97d in SafeTimerThread::entry (this=) at /usr/src/debug/ceph-15.2.8.1.0.0/src/common/Timer.cc:30 #4 0x00007f241d867dd5 in start_thread () from /lib64/libpthread.so.0 #5 0x00007f241c72cead in clone () from /lib64/libc.so.6 Thread 57 (Thread 0x7f23fb95f700 (LWP 312040)): #0 0x00007f241d86b965 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f241cfc5b1c in std::condition_variable::wait(std::unique_lock&) () from /lib64/libstdc++.so.6 #2 0x000055981b45f8ef in Finisher::finisher_thread_entry() () at /usr/src/debug/ceph-15.2.8.1.0.0/src/common/Finisher.cc:86 #3 0x00007f241d867dd5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007f241c72cead in clone () from /lib64/libc.so.6 Thread 56 (Thread 0x7f23fc160700 (LWP 312039)): #0 0x00007f241d86b965 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f241cfc5b1c in std::condition_variable::wait(std::unique_lock&) () from /lib64/libstdc++.so.6 #2 0x000055981b45f8ef in Finisher::finisher_thread_entry() () at /usr/src/debug/ceph-15.2.8.1.0.0/src/common/Finisher.cc:86 #3 0x00007f241d867dd5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007f241c72cead in clone () from /lib64/libc.so.6 Thread 55 (Thread 0x7f23fc961700 (LWP 312038)): #0 0x00007f241d86b965 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f241cfc5b1c in std::condition_variable::wait(std::unique_lock&) () from /lib64/libstdc++.so.6 #2 0x000055981b79f1f6 in DispatchQueue::run_local_delivery() () at /usr/src/debug/ceph-15.2.8.1.0.0/src/msg/DispatchQueue.cc:117 #3 0x000055981b601e4d in DispatchQueue::LocalDeliveryThread::entry (this=) at /usr/src/debug/ceph-15.2.8.1.0.0/src/msg/DispatchQueue.h:115 #4 0x00007f241d867dd5 in start_thread () from /lib64/libpthread.so.0 #5 0x00007f241c72cead in clone () from /lib64/libc.so.6 Thread 54 (Thread 0x7f23fd162700 (LWP 312037)): #0 0x00007f241d86b965 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f241cfc5b1c in std::condition_variable::wait(std::unique_lock&) () from /lib64/libstdc++.so.6 #2 0x000055981b79f8b8 in DispatchQueue::entry() () at /usr/src/debug/ceph-15.2.8.1.0.0/src/msg/DispatchQueue.cc:210 #3 0x000055981b601d8d in DispatchQueue::DispatchThread::entry (this=) at /usr/src/debug/ceph-15.2.8.1.0.0/src/msg/DispatchQueue.h:101 #4 0x00007f241d867dd5 in start_thread () from /lib64/libpthread.so.0 #5 0x00007f241c72cead in clone () from /lib64/libc.so.6 Thread 53 (Thread 0x7f23fd963700 (LWP 312036)): #0 0x00007f241d86b965 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f241cfc5b1c in std::condition_variable::wait(std::unique_lock&) () from /lib64/libstdc++.so.6 #2 0x000055981b79f1f6 in DispatchQueue::run_local_delivery() () at /usr/src/debug/ceph-15.2.8.1.0.0/src/msg/DispatchQueue.cc:117 #3 0x000055981b601e4d in DispatchQueue::LocalDeliveryThread::entry (this=) at /usr/src/debug/ceph-15.2.8.1.0.0/src/msg/DispatchQueue.h:115 #4 0x00007f241d867dd5 in start_thread () from /lib64/libpthread.so.0 #5 0x00007f241c72cead in clone () from /lib64/libc.so.6 ---Type to continue, or q to quit--- Thread 52 (Thread 0x7f23fe164700 (LWP 312035)): #0 0x00007f241d86b965 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f241cfc5b1c in std::condition_variable::wait(std::unique_lock&) () from /lib64/libstdc++.so.6 #2 0x000055981b79f8b8 in DispatchQueue::entry() () at /usr/src/debug/ceph-15.2.8.1.0.0/src/msg/DispatchQueue.cc:210 #3 0x000055981b601d8d in DispatchQueue::DispatchThread::entry (this=) at /usr/src/debug/ceph-15.2.8.1.0.0/src/msg/DispatchQueue.h:101 #4 0x00007f241d867dd5 in start_thread () from /lib64/libpthread.so.0 #5 0x00007f241c72cead in clone () from /lib64/libc.so.6 Thread 51 (Thread 0x7f23fe965700 (LWP 312034)): #0 0x00007f241d86b965 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f241cfc5b1c in std::condition_variable::wait(std::unique_lock&) () from /lib64/libstdc++.so.6 #2 0x000055981b79f1f6 in DispatchQueue::run_local_delivery() () at /usr/src/debug/ceph-15.2.8.1.0.0/src/msg/DispatchQueue.cc:117 #3 0x000055981b601e4d in DispatchQueue::LocalDeliveryThread::entry (this=) at /usr/src/debug/ceph-15.2.8.1.0.0/src/msg/DispatchQueue.h:115 #4 0x00007f241d867dd5 in start_thread () from /lib64/libpthread.so.0 #5 0x00007f241c72cead in clone () from /lib64/libc.so.6 Thread 50 (Thread 0x7f23ff166700 (LWP 312033)): #0 0x00007f241d86b965 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f241cfc5b1c in std::condition_variable::wait(std::unique_lock&) () from /lib64/libstdc++.so.6 #2 0x000055981b79f8b8 in DispatchQueue::entry() () at /usr/src/debug/ceph-15.2.8.1.0.0/src/msg/DispatchQueue.cc:210 #3 0x000055981b601d8d in DispatchQueue::DispatchThread::entry (this=) at /usr/src/debug/ceph-15.2.8.1.0.0/src/msg/DispatchQueue.h:101 #4 0x00007f241d867dd5 in start_thread () from /lib64/libpthread.so.0 #5 0x00007f241c72cead in clone () from /lib64/libc.so.6 Thread 49 (Thread 0x7f23ff967700 (LWP 312032)): #0 0x00007f241d86b965 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f241cfc5b1c in std::condition_variable::wait(std::unique_lock&) () from /lib64/libstdc++.so.6 #2 0x000055981b79f1f6 in DispatchQueue::run_local_delivery() () at /usr/src/debug/ceph-15.2.8.1.0.0/src/msg/DispatchQueue.cc:117 #3 0x000055981b601e4d in DispatchQueue::LocalDeliveryThread::entry (this=) at /usr/src/debug/ceph-15.2.8.1.0.0/src/msg/DispatchQueue.h:115 #4 0x00007f241d867dd5 in start_thread () from /lib64/libpthread.so.0 #5 0x00007f241c72cead in clone () from /lib64/libc.so.6 Thread 48 (Thread 0x7f2400168700 (LWP 312031)): #0 0x00007f241d86b965 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f241cfc5b1c in std::condition_variable::wait(std::unique_lock&) () from /lib64/libstdc++.so.6 #2 0x000055981b79f8b8 in DispatchQueue::entry() () at /usr/src/debug/ceph-15.2.8.1.0.0/src/msg/DispatchQueue.cc:210 #3 0x000055981b601d8d in DispatchQueue::DispatchThread::entry (this=) at /usr/src/debug/ceph-15.2.8.1.0.0/src/msg/DispatchQueue.h:101 #4 0x00007f241d867dd5 in start_thread () from /lib64/libpthread.so.0 #5 0x00007f241c72cead in clone () from /lib64/libc.so.6 Thread 47 (Thread 0x7f2400969700 (LWP 312030)): #0 0x00007f241d86b965 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f241cfc5b1c in std::condition_variable::wait(std::unique_lock&) () from /lib64/libstdc++.so.6 #2 0x000055981b79f1f6 in DispatchQueue::run_local_delivery() () at /usr/src/debug/ceph-15.2.8.1.0.0/src/msg/DispatchQueue.cc:117 #3 0x000055981b601e4d in DispatchQueue::LocalDeliveryThread::entry (this=) at /usr/src/debug/ceph-15.2.8.1.0.0/src/msg/DispatchQueue.h:115 #4 0x00007f241d867dd5 in start_thread () from /lib64/libpthread.so.0 #5 0x00007f241c72cead in clone () from /lib64/libc.so.6 Thread 46 (Thread 0x7f240116a700 (LWP 312029)): #0 0x00007f241d86b965 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f241cfc5b1c in std::condition_variable::wait(std::unique_lock&) () from /lib64/libstdc++.so.6 #2 0x000055981b79f8b8 in DispatchQueue::entry() () at /usr/src/debug/ceph-15.2.8.1.0.0/src/msg/DispatchQueue.cc:210 #3 0x000055981b601d8d in DispatchQueue::DispatchThread::entry (this=) at /usr/src/debug/ceph-15.2.8.1.0.0/src/msg/DispatchQueue.h:101 #4 0x00007f241d867dd5 in start_thread () from /lib64/libpthread.so.0 #5 0x00007f241c72cead in clone () from /lib64/libc.so.6 ---Type to continue, or q to quit--- Thread 45 (Thread 0x7f240196b700 (LWP 312028)): #0 0x00007f241d86b965 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f241cfc5b1c in std::condition_variable::wait(std::unique_lock&) () from /lib64/libstdc++.so.6 #2 0x000055981b79f1f6 in DispatchQueue::run_local_delivery() () at /usr/src/debug/ceph-15.2.8.1.0.0/src/msg/DispatchQueue.cc:117 #3 0x000055981b601e4d in DispatchQueue::LocalDeliveryThread::entry (this=) at /usr/src/debug/ceph-15.2.8.1.0.0/src/msg/DispatchQueue.h:115 #4 0x00007f241d867dd5 in start_thread () from /lib64/libpthread.so.0 #5 0x00007f241c72cead in clone () from /lib64/libc.so.6 Thread 44 (Thread 0x7f240216c700 (LWP 312027)): #0 0x00007f241d86b965 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f241cfc5b1c in std::condition_variable::wait(std::unique_lock&) () from /lib64/libstdc++.so.6 #2 0x000055981b79f8b8 in DispatchQueue::entry() () at /usr/src/debug/ceph-15.2.8.1.0.0/src/msg/DispatchQueue.cc:210 #3 0x000055981b601d8d in DispatchQueue::DispatchThread::entry (this=) at /usr/src/debug/ceph-15.2.8.1.0.0/src/msg/DispatchQueue.h:101 #4 0x00007f241d867dd5 in start_thread () from /lib64/libpthread.so.0 #5 0x00007f241c72cead in clone () from /lib64/libc.so.6 Thread 43 (Thread 0x7f240296d700 (LWP 312026)): #0 0x00007f241d86e4ed in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007f241d869dcb in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007f241d869c98 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x000055981b57691a in __gthread_mutex_lock (__mutex=0x5598261437a8) at /opt/rh/devtoolset-8/root/usr/include/c++/8/x86_64-redhat-linux/bits/gthr-default.h:748 #4 lock (this=0x5598261437a8) at /opt/rh/devtoolset-8/root/usr/include/c++/8/bits/std_mutex.h:103 #5 lock_guard (__m=..., this=) at /opt/rh/devtoolset-8/root/usr/include/c++/8/bits/std_mutex.h:162 #6 ceph::common::PerfCountersCollection::with_counters(std::function, std::allocator > > const&)>) const () at /usr/src/debug/ceph-15.2.8.1.0.0/src/common/perf_counters_collection.cc:52 #7 0x000055981b666ac7 in MgrClient::_send_report() () at /opt/rh/devtoolset-8/root/usr/include/c++/8/bits/std_function.h:87 #8 0x000055981b667069 in MgrClient::_send_stats (this=0x5598261c0a18) at /usr/src/debug/ceph-15.2.8.1.0.0/src/mgr/MgrClient.cc:284 #9 0x000055981ae9d0b9 in Context::complete (this=0x559848780c80, r=) at /usr/src/debug/ceph-15.2.8.1.0.0/src/include/Context.h:77 #10 0x000055981b48a588 in SafeTimer::timer_thread() () at /usr/src/debug/ceph-15.2.8.1.0.0/src/common/Timer.cc:96 #11 0x000055981b48b97d in SafeTimerThread::entry (this=) at /usr/src/debug/ceph-15.2.8.1.0.0/src/common/Timer.cc:30 #12 0x00007f241d867dd5 in start_thread () from /lib64/libpthread.so.0 #13 0x00007f241c72cead in clone () from /lib64/libc.so.6 Thread 42 (Thread 0x7f240316e700 (LWP 312025)): #0 0x00007f241d86b965 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f241cfc5b1c in std::condition_variable::wait(std::unique_lock&) () from /lib64/libstdc++.so.6 #2 0x000055981b45f8ef in Finisher::finisher_thread_entry() () at /usr/src/debug/ceph-15.2.8.1.0.0/src/common/Finisher.cc:86 #3 0x00007f241d867dd5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007f241c72cead in clone () from /lib64/libc.so.6 Thread 41 (Thread 0x7f240396f700 (LWP 312024)): #0 0x00007f241d86bd12 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x000055981b48ab1c in __gthread_cond_timedwait (__abs_timeout=0x7f240396c9d0, __mutex=, __cond=0x7ffe7393c588) at /opt/rh/devtoolset-8/root/usr/include/c++/8/x86_64-redhat-linux/bits/gthr-default.h:871 #2 __wait_until_impl > > (__atime=, __lock=..., this=0x7ffe7393c588) at /opt/rh/devtoolset-8/root/usr/include/c++/8/condition_variable:178 #3 wait_until > > (__atime=..., __lock=..., this=0x7ffe7393c588) at /opt/rh/devtoolset-8/root/usr/include/c++/8/condition_variable:119 #4 SafeTimer::timer_thread() () at /usr/src/debug/ceph-15.2.8.1.0.0/src/common/Timer.cc:108 #5 0x000055981b48b97d in SafeTimerThread::entry (this=) at /usr/src/debug/ceph-15.2.8.1.0.0/src/common/Timer.cc:30 #6 0x00007f241d867dd5 in start_thread () from /lib64/libpthread.so.0 #7 0x00007f241c72cead in clone () from /lib64/libc.so.6 Thread 40 (Thread 0x7f2404170700 (LWP 312023)): #0 0x00007f241d86b965 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f241cfc5b1c in std::condition_variable::wait(std::unique_lock&) () from /lib64/libstdc++.so.6 #2 0x000055981b79f1f6 in DispatchQueue::run_local_delivery() () at /usr/src/debug/ceph-15.2.8.1.0.0/src/msg/DispatchQueue.cc:117 #3 0x000055981b601e4d in DispatchQueue::LocalDeliveryThread::entry (this=) at /usr/src/debug/ceph-15.2.8.1.0.0/src/msg/DispatchQueue.h:115 ---Type to continue, or q to quit--- #4 0x00007f241d867dd5 in start_thread () from /lib64/libpthread.so.0 #5 0x00007f241c72cead in clone () from /lib64/libc.so.6 Thread 39 (Thread 0x7f2404971700 (LWP 312022)): #0 0x00007f241d86e4ed in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007f241d869dcb in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007f241d869c98 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x000055981b66671f in __gthread_mutex_lock (__mutex=0x5598261c0b80) at /opt/rh/devtoolset-8/root/usr/include/c++/8/x86_64-redhat-linux/bits/gthr-default.h:748 #4 lock (this=0x5598261c0b80) at /opt/rh/devtoolset-8/root/usr/include/c++/8/bits/std_mutex.h:103 #5 lock_guard (__m=..., this=) at /opt/rh/devtoolset-8/root/usr/include/c++/8/bits/std_mutex.h:162 #6 MgrClient::ms_handle_reset(Connection*) () at /usr/src/debug/ceph-15.2.8.1.0.0/src/mgr/MgrClient.cc:267 #7 0x000055981b79f732 in ms_deliver_handle_reset (con=0x55984bf19400, this=) at /usr/src/debug/ceph-15.2.8.1.0.0/src/msg/Messenger.h:772 #8 DispatchQueue::entry() () at /usr/src/debug/ceph-15.2.8.1.0.0/src/msg/DispatchQueue.cc:185 #9 0x000055981b601d8d in DispatchQueue::DispatchThread::entry (this=) at /usr/src/debug/ceph-15.2.8.1.0.0/src/msg/DispatchQueue.h:101 #10 0x00007f241d867dd5 in start_thread () from /lib64/libpthread.so.0 #11 0x00007f241c72cead in clone () from /lib64/libc.so.6 Thread 38 (Thread 0x7f2409828700 (LWP 308611)): #0 0x00007f241d86b965 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f241cfc5b1c in std::condition_variable::wait(std::unique_lock&) () from /lib64/libstdc++.so.6 #2 0x000055981b48a5d3 in SafeTimer::timer_thread() () at /usr/src/debug/ceph-15.2.8.1.0.0/src/common/Timer.cc:106 #3 0x000055981b48b97d in SafeTimerThread::entry (this=) at /usr/src/debug/ceph-15.2.8.1.0.0/src/common/Timer.cc:30 #4 0x00007f241d867dd5 in start_thread () from /lib64/libpthread.so.0 #5 0x00007f241c72cead in clone () from /lib64/libc.so.6 Thread 37 (Thread 0x7f240a029700 (LWP 308610)): #0 0x00007f241d86b965 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f241cfc5b1c in std::condition_variable::wait(std::unique_lock&) () from /lib64/libstdc++.so.6 #2 0x000055981b45f8ef in Finisher::finisher_thread_entry() () at /usr/src/debug/ceph-15.2.8.1.0.0/src/common/Finisher.cc:86 #3 0x00007f241d867dd5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007f241c72cead in clone () from /lib64/libc.so.6 Thread 36 (Thread 0x7f240a82a700 (LWP 308609)): #0 0x00007f241d86b965 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f241cfc5b1c in std::condition_variable::wait(std::unique_lock&) () from /lib64/libstdc++.so.6 #2 0x000055981b45f8ef in Finisher::finisher_thread_entry() () at /usr/src/debug/ceph-15.2.8.1.0.0/src/common/Finisher.cc:86 #3 0x00007f241d867dd5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007f241c72cead in clone () from /lib64/libc.so.6 Thread 35 (Thread 0x7f240b02b700 (LWP 308608)): #0 0x00007f241d86bd12 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x000055981b4924a4 in __gthread_cond_timedwait (__abs_timeout=0x7f240b028860, __mutex=, __cond=0x559826e96a70) at /opt/rh/devtoolset-8/root/usr/include/c++/8/x86_64-redhat-linux/bits/gthr-default.h:871 #2 __wait_until_impl > > (__atime=..., __lock=..., this=0x559826e96a70) at /opt/rh/devtoolset-8/root/usr/include/c++/8/condition_variable:178 #3 wait_until > > (__atime=..., __lock=..., this=0x559826e96a70) at /opt/rh/devtoolset-8/root/usr/include/c++/8/condition_variable:106 #4 wait_for > (__rtime=, __lock=..., this=0x559826e96a70) at /opt/rh/devtoolset-8/root/usr/include/c++/8/condition_variable:143 #5 ThreadPool::worker(ThreadPool::WorkThread*) () at /usr/src/debug/ceph-15.2.8.1.0.0/src/common/WorkQueue.cc:141 #6 0x000055981b4932c1 in ThreadPool::WorkThread::entry (this=) at /usr/src/debug/ceph-15.2.8.1.0.0/src/common/WorkQueue.h:475 #7 0x00007f241d867dd5 in start_thread () from /lib64/libpthread.so.0 #8 0x00007f241c72cead in clone () from /lib64/libc.so.6 Thread 34 (Thread 0x7f240b82c700 (LWP 308607)): #0 0x00007f241d86bd12 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x000055981b4924a4 in __gthread_cond_timedwait (__abs_timeout=0x7f240b829860, __mutex=, __cond=0x559826e96a70) at /opt/rh/devtoolset-8/root/usr/include/c++/8/x86_64-redhat-linux/bits/gthr-default.h:871 #2 __wait_until_impl > > (__atime=..., __lock=..., this=0x559826e96a70) at /opt/rh/devtoolset-8/root/usr/include/c++/8/condition_variable:178 #3 wait_until > > (__atime=..., __lock=..., this=0x559826e96a70) at /opt/rh/devtoolset-8/root/usr/include/c++/8/condition_variable:106 #4 wait_for > (__rtime=, __lock=..., this=0x559826e96a70) at /opt/rh/devtoolset-8/root/usr/include/c++/8/condition_variable:143 #5 ThreadPool::worker(ThreadPool::WorkThread*) () at /usr/src/debug/ceph-15.2.8.1.0.0/src/common/WorkQueue.cc:141 ---Type to continue, or q to quit--- #6 0x000055981b4932c1 in ThreadPool::WorkThread::entry (this=) at /usr/src/debug/ceph-15.2.8.1.0.0/src/common/WorkQueue.h:475 #7 0x00007f241d867dd5 in start_thread () from /lib64/libpthread.so.0 #8 0x00007f241c72cead in clone () from /lib64/libc.so.6 Thread 33 (Thread 0x7f240c02d700 (LWP 308606)): #0 0x00007f241d86b965 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f241cfc5b1c in std::condition_variable::wait(std::unique_lock&) () from /lib64/libstdc++.so.6 #2 0x000055981b45f8ef in Finisher::finisher_thread_entry() () at /usr/src/debug/ceph-15.2.8.1.0.0/src/common/Finisher.cc:86 #3 0x00007f241d867dd5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007f241c72cead in clone () from /lib64/libc.so.6 Thread 32 (Thread 0x7f240c82e700 (LWP 308605)): #0 0x00007f241d86b965 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f241cfc5b1c in std::condition_variable::wait(std::unique_lock&) () from /lib64/libstdc++.so.6 #2 0x000055981b3b3c11 in FileJournal::write_thread_entry() () at /usr/src/debug/ceph-15.2.8.1.0.0/src/os/filestore/FileJournal.cc:1420 #3 0x000055981b240d2d in FileJournal::Writer::entry (this=) at /usr/src/debug/ceph-15.2.8.1.0.0/src/os/filestore/FileJournal.h:513 #4 0x00007f241d867dd5 in start_thread () from /lib64/libpthread.so.0 #5 0x00007f241c72cead in clone () from /lib64/libc.so.6 Thread 31 (Thread 0x7f2407023700 (LWP 308604)): #0 0x00007f241d86bd12 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x000055981b22af2a in __gthread_cond_timedwait (__abs_timeout=0x7f2407020740, __mutex=, __cond=0x559826e96470) at /opt/rh/devtoolset-8/root/usr/include/c++/8/x86_64-redhat-linux/bits/gthr-default.h:871 #2 __wait_until_impl > > (__atime=..., __lock=..., this=0x559826e96470) at /opt/rh/devtoolset-8/root/usr/include/c++/8/condition_variable:178 #3 wait_until > > (__atime=..., __lock=..., this=0x559826e96470) at /opt/rh/devtoolset-8/root/usr/include/c++/8/condition_variable:106 #4 wait_for > (__rtime=..., __lock=..., this=0x559826e96470) at /opt/rh/devtoolset-8/root/usr/include/c++/8/condition_variable:143 #5 FileStore::sync_entry() () at /usr/src/debug/ceph-15.2.8.1.0.0/src/os/filestore/FileStore.cc:5473 #6 0x000055981b24e71d in FileStore::SyncThread::entry (this=) at /usr/src/debug/ceph-15.2.8.1.0.0/src/os/filestore/FileStore.h:506 #7 0x00007f241d867dd5 in start_thread () from /lib64/libpthread.so.0 #8 0x00007f241c72cead in clone () from /lib64/libc.so.6 Thread 30 (Thread 0x7f2406822700 (LWP 308603)): #0 0x00007f241d86b965 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f241cfc5b1c in std::condition_variable::wait(std::unique_lock&) () from /lib64/libstdc++.so.6 #2 0x000055981b266123 in wait&, boost::tuples::tuple, WBThrottle::PendingWB>*):: > (__p=..., __lock=..., this=0x559826e96750) at /usr/src/debug/ceph-15.2.8.1.0.0/src/os/filestore/WBThrottle.h:137 #3 WBThrottle::get_next_should_flush(std::unique_lock&, boost::tuples::tuple, WBThrottle::PendingWB, boost::tuples::null_type, boost::tuples::null_type, boost::tuples::null_type, boost::tuples::null_type, boost::tuples::null_type, boost::tuples::null_type, boost::tuples::null_type>*) () at /usr/src/debug/ceph-15.2.8.1.0.0/src/os/filestore/WBThrottle.cc:140 #4 0x000055981b2688a7 in WBThrottle::entry() () at /usr/src/debug/ceph-15.2.8.1.0.0/src/os/filestore/WBThrottle.cc:161 #5 0x00007f241d867dd5 in start_thread () from /lib64/libpthread.so.0 #6 0x00007f241c72cead in clone () from /lib64/libc.so.6 Thread 29 (Thread 0x7f2406021700 (LWP 308598)): #0 0x00007f241d86bd12 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x000055981b9c7962 in rocksdb::port::CondVar::TimedWait (this=this@entry=0x559826131580, abs_time_us=abs_time_us@entry=1628492394534804) at /usr/src/debug/ceph-15.2.8.1.0.0/src/rocksdb/port/port_posix.cc:120 #2 0x000055981b932f95 in rocksdb::InstrumentedCondVar::TimedWaitInternal (this=this@entry=0x559826131580, abs_time_us=abs_time_us@entry=1628492394534804) at /usr/src/debug/ceph-15.2.8.1.0.0/src/rocksdb/monitoring/instrumented_mutex.cc:66 #3 0x000055981b933048 in rocksdb::InstrumentedCondVar::TimedWait(unsigned long) () at /usr/src/debug/ceph-15.2.8.1.0.0/src/rocksdb/monitoring/instrumented_mutex.cc:55 #4 0x000055981b849713 in wait (delay=, this=0x559826131500) at /usr/src/debug/ceph-15.2.8.1.0.0/src/rocksdb/util/repeatable_thread.h:87 #5 thread (this=) at /usr/src/debug/ceph-15.2.8.1.0.0/src/rocksdb/util/repeatable_thread.h:123 #6 thread (this=0x559826131500) at /usr/src/debug/ceph-15.2.8.1.0.0/src/rocksdb/util/repeatable_thread.h:99 #7 operator() (__closure=) at /usr/src/debug/ceph-15.2.8.1.0.0/src/rocksdb/util/repeatable_thread.h:35 #8 __invoke_impl, const string&, rocksdb::Env*, uint64_t, uint64_t):: > (__f=) at /opt/rh/devtoolset-8/root/usr/include/c++/8/bits/invoke.h:60 #9 __invoke, const string&, rocksdb::Env*, uint64_t, uint64_t):: > (__fn=) at /opt/rh/devtoolset-8/root/usr/include/c++/8/bits/invoke.h:95 #10 _M_invoke<0> (this=) at /opt/rh/devtoolset-8/root/usr/include/c++/8/thread:244 #11 operator() (this=) at /opt/rh/devtoolset-8/root/usr/include/c++/8/thread:253 ---Type to continue, or q to quit--- #12 _ZNSt6thread11_State_implINS_8_InvokerISt5tupleIJZN7rocksdb16RepeatableThreadC4ESt8functionIFvvEERKSsPNS3_3EnvEmmEUlvE_EEEEE6_M_runEv (this=) at /opt/rh/devtoolset-8/root/usr/include/c++/8/thread:196 #13 0x000055981bb0ee4f in execute_native_thread_routine () #14 0x00007f241d867dd5 in start_thread () from /lib64/libpthread.so.0 #15 0x00007f241c72cead in clone () from /lib64/libc.so.6 Thread 28 (Thread 0x7f2405820700 (LWP 308597)): #0 0x00007f241d86bd12 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x000055981b9c7962 in rocksdb::port::CondVar::TimedWait (this=this@entry=0x5598261314a0, abs_time_us=abs_time_us@entry=1628492394535391) at /usr/src/debug/ceph-15.2.8.1.0.0/src/rocksdb/port/port_posix.cc:120 #2 0x000055981b932f95 in rocksdb::InstrumentedCondVar::TimedWaitInternal (this=this@entry=0x5598261314a0, abs_time_us=abs_time_us@entry=1628492394535391) at /usr/src/debug/ceph-15.2.8.1.0.0/src/rocksdb/monitoring/instrumented_mutex.cc:66 #3 0x000055981b933048 in rocksdb::InstrumentedCondVar::TimedWait(unsigned long) () at /usr/src/debug/ceph-15.2.8.1.0.0/src/rocksdb/monitoring/instrumented_mutex.cc:55 #4 0x000055981b849713 in wait (delay=, this=0x559826131420) at /usr/src/debug/ceph-15.2.8.1.0.0/src/rocksdb/util/repeatable_thread.h:87 #5 thread (this=) at /usr/src/debug/ceph-15.2.8.1.0.0/src/rocksdb/util/repeatable_thread.h:123 #6 thread (this=0x559826131420) at /usr/src/debug/ceph-15.2.8.1.0.0/src/rocksdb/util/repeatable_thread.h:99 #7 operator() (__closure=) at /usr/src/debug/ceph-15.2.8.1.0.0/src/rocksdb/util/repeatable_thread.h:35 #8 __invoke_impl, const string&, rocksdb::Env*, uint64_t, uint64_t):: > (__f=) at /opt/rh/devtoolset-8/root/usr/include/c++/8/bits/invoke.h:60 #9 __invoke, const string&, rocksdb::Env*, uint64_t, uint64_t):: > (__fn=) at /opt/rh/devtoolset-8/root/usr/include/c++/8/bits/invoke.h:95 #10 _M_invoke<0> (this=) at /opt/rh/devtoolset-8/root/usr/include/c++/8/thread:244 #11 operator() (this=) at /opt/rh/devtoolset-8/root/usr/include/c++/8/thread:253 #12 _ZNSt6thread11_State_implINS_8_InvokerISt5tupleIJZN7rocksdb16RepeatableThreadC4ESt8functionIFvvEERKSsPNS3_3EnvEmmEUlvE_EEEEE6_M_runEv (this=) at /opt/rh/devtoolset-8/root/usr/include/c++/8/thread:196 #13 0x000055981bb0ee4f in execute_native_thread_routine () #14 0x00007f241d867dd5 in start_thread () from /lib64/libpthread.so.0 #15 0x00007f241c72cead in clone () from /lib64/libc.so.6 Thread 27 (Thread 0x7f240d02f700 (LWP 308302)): #0 0x00007f241d86b965 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x000055981b9c78fd in rocksdb::port::CondVar::Wait (this=this@entry=0x559826ed21d0) at /usr/src/debug/ceph-15.2.8.1.0.0/src/rocksdb/port/port_posix.cc:106 #2 0x000055981b932e15 in rocksdb::InstrumentedCondVar::WaitInternal (this=this@entry=0x559826ed21d0) at /usr/src/debug/ceph-15.2.8.1.0.0/src/rocksdb/monitoring/instrumented_mutex.cc:48 #3 0x000055981b932ebc in rocksdb::InstrumentedCondVar::Wait() () at /usr/src/debug/ceph-15.2.8.1.0.0/src/rocksdb/monitoring/instrumented_mutex.cc:41 #4 0x000055981b9aac40 in rocksdb::DeleteScheduler::BackgroundEmptyTrash() () at /usr/src/debug/ceph-15.2.8.1.0.0/src/rocksdb/util/delete_scheduler.cc:194 #5 0x000055981bb0ee4f in execute_native_thread_routine () #6 0x00007f241d867dd5 in start_thread () from /lib64/libpthread.so.0 #7 0x00007f241c72cead in clone () from /lib64/libc.so.6 Thread 26 (Thread 0x7f240d830700 (LWP 308301)): #0 0x00007f241d86b965 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f241cfc5b1c in std::condition_variable::wait(std::unique_lock&) () from /lib64/libstdc++.so.6 #2 0x000055981ba2f0d9 in rocksdb::ThreadPoolImpl::Impl::BGThread(unsigned long) () at /usr/src/debug/ceph-15.2.8.1.0.0/src/rocksdb/util/threadpool_imp.cc:196 #3 0x000055981ba2f3fd in rocksdb::ThreadPoolImpl::Impl::BGThreadWrapper (arg=0x55982611ab10) at /usr/src/debug/ceph-15.2.8.1.0.0/src/rocksdb/util/threadpool_imp.cc:306 #4 0x000055981bb0ee4f in execute_native_thread_routine () #5 0x00007f241d867dd5 in start_thread () from /lib64/libpthread.so.0 #6 0x00007f241c72cead in clone () from /lib64/libc.so.6 Thread 25 (Thread 0x7f240e031700 (LWP 308300)): #0 0x00007f241d86b965 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f241cfc5b1c in std::condition_variable::wait(std::unique_lock&) () from /lib64/libstdc++.so.6 #2 0x000055981ba2f0d9 in rocksdb::ThreadPoolImpl::Impl::BGThread(unsigned long) () at /usr/src/debug/ceph-15.2.8.1.0.0/src/rocksdb/util/threadpool_imp.cc:196 #3 0x000055981ba2f3fd in rocksdb::ThreadPoolImpl::Impl::BGThreadWrapper (arg=0x55982611ab70) at /usr/src/debug/ceph-15.2.8.1.0.0/src/rocksdb/util/threadpool_imp.cc:306 #4 0x000055981bb0ee4f in execute_native_thread_routine () #5 0x00007f241d867dd5 in start_thread () from /lib64/libpthread.so.0 #6 0x00007f241c72cead in clone () from /lib64/libc.so.6 Thread 24 (Thread 0x7f240e832700 (LWP 308299)): #0 0x00007f241d86b965 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f241cfc5b1c in std::condition_variable::wait(std::unique_lock&) () from /lib64/libstdc++.so.6 ---Type to continue, or q to quit--- #2 0x000055981ba2f0d9 in rocksdb::ThreadPoolImpl::Impl::BGThread(unsigned long) () at /usr/src/debug/ceph-15.2.8.1.0.0/src/rocksdb/util/threadpool_imp.cc:196 #3 0x000055981ba2f3fd in rocksdb::ThreadPoolImpl::Impl::BGThreadWrapper (arg=0x55982611a340) at /usr/src/debug/ceph-15.2.8.1.0.0/src/rocksdb/util/threadpool_imp.cc:306 #4 0x000055981bb0ee4f in execute_native_thread_routine () #5 0x00007f241d867dd5 in start_thread () from /lib64/libpthread.so.0 #6 0x00007f241c72cead in clone () from /lib64/libc.so.6 Thread 23 (Thread 0x7f240f033700 (LWP 308298)): #0 0x00007f241d86b965 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f241cfc5b1c in std::condition_variable::wait(std::unique_lock&) () from /lib64/libstdc++.so.6 #2 0x000055981ba2f0d9 in rocksdb::ThreadPoolImpl::Impl::BGThread(unsigned long) () at /usr/src/debug/ceph-15.2.8.1.0.0/src/rocksdb/util/threadpool_imp.cc:196 #3 0x000055981ba2f3fd in rocksdb::ThreadPoolImpl::Impl::BGThreadWrapper (arg=0x55982611a300) at /usr/src/debug/ceph-15.2.8.1.0.0/src/rocksdb/util/threadpool_imp.cc:306 #4 0x000055981bb0ee4f in execute_native_thread_routine () #5 0x00007f241d867dd5 in start_thread () from /lib64/libpthread.so.0 #6 0x00007f241c72cead in clone () from /lib64/libc.so.6 Thread 22 (Thread 0x7f240f834700 (LWP 308297)): #0 0x00007f241d86b965 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f241cfc5b1c in std::condition_variable::wait(std::unique_lock&) () from /lib64/libstdc++.so.6 #2 0x000055981ba2f0d9 in rocksdb::ThreadPoolImpl::Impl::BGThread(unsigned long) () at /usr/src/debug/ceph-15.2.8.1.0.0/src/rocksdb/util/threadpool_imp.cc:196 #3 0x000055981ba2f3fd in rocksdb::ThreadPoolImpl::Impl::BGThreadWrapper (arg=0x55982611a2c0) at /usr/src/debug/ceph-15.2.8.1.0.0/src/rocksdb/util/threadpool_imp.cc:306 #4 0x000055981bb0ee4f in execute_native_thread_routine () #5 0x00007f241d867dd5 in start_thread () from /lib64/libpthread.so.0 #6 0x00007f241c72cead in clone () from /lib64/libc.so.6 Thread 21 (Thread 0x7f2410035700 (LWP 308296)): #0 0x00007f241d86b965 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f241cfc5b1c in std::condition_variable::wait(std::unique_lock&) () from /lib64/libstdc++.so.6 #2 0x000055981ba2f0d9 in rocksdb::ThreadPoolImpl::Impl::BGThread(unsigned long) () at /usr/src/debug/ceph-15.2.8.1.0.0/src/rocksdb/util/threadpool_imp.cc:196 #3 0x000055981ba2f3fd in rocksdb::ThreadPoolImpl::Impl::BGThreadWrapper (arg=0x55982611a260) at /usr/src/debug/ceph-15.2.8.1.0.0/src/rocksdb/util/threadpool_imp.cc:306 #4 0x000055981bb0ee4f in execute_native_thread_routine () #5 0x00007f241d867dd5 in start_thread () from /lib64/libpthread.so.0 #6 0x00007f241c72cead in clone () from /lib64/libc.so.6 Thread 20 (Thread 0x7f2410836700 (LWP 308295)): #0 0x00007f241d86b965 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f241cfc5b1c in std::condition_variable::wait(std::unique_lock&) () from /lib64/libstdc++.so.6 #2 0x000055981ba2f0d9 in rocksdb::ThreadPoolImpl::Impl::BGThread(unsigned long) () at /usr/src/debug/ceph-15.2.8.1.0.0/src/rocksdb/util/threadpool_imp.cc:196 #3 0x000055981ba2f3fd in rocksdb::ThreadPoolImpl::Impl::BGThreadWrapper (arg=0x55982611a1e0) at /usr/src/debug/ceph-15.2.8.1.0.0/src/rocksdb/util/threadpool_imp.cc:306 #4 0x000055981bb0ee4f in execute_native_thread_routine () #5 0x00007f241d867dd5 in start_thread () from /lib64/libpthread.so.0 #6 0x00007f241c72cead in clone () from /lib64/libc.so.6 Thread 19 (Thread 0x7f2411037700 (LWP 308294)): #0 0x00007f241d86b965 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f241cfc5b1c in std::condition_variable::wait(std::unique_lock&) () from /lib64/libstdc++.so.6 #2 0x000055981ba2f0d9 in rocksdb::ThreadPoolImpl::Impl::BGThread(unsigned long) () at /usr/src/debug/ceph-15.2.8.1.0.0/src/rocksdb/util/threadpool_imp.cc:196 #3 0x000055981ba2f3fd in rocksdb::ThreadPoolImpl::Impl::BGThreadWrapper (arg=0x55982611a220) at /usr/src/debug/ceph-15.2.8.1.0.0/src/rocksdb/util/threadpool_imp.cc:306 #4 0x000055981bb0ee4f in execute_native_thread_routine () #5 0x00007f241d867dd5 in start_thread () from /lib64/libpthread.so.0 #6 0x00007f241c72cead in clone () from /lib64/libc.so.6 Thread 18 (Thread 0x7f2411838700 (LWP 308293)): #0 0x00007f241d86b965 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f241cfc5b1c in std::condition_variable::wait(std::unique_lock&) () from /lib64/libstdc++.so.6 #2 0x000055981ba2f0d9 in rocksdb::ThreadPoolImpl::Impl::BGThread(unsigned long) () at /usr/src/debug/ceph-15.2.8.1.0.0/src/rocksdb/util/threadpool_imp.cc:196 #3 0x000055981ba2f3fd in rocksdb::ThreadPoolImpl::Impl::BGThreadWrapper (arg=0x55982611a1a0) at /usr/src/debug/ceph-15.2.8.1.0.0/src/rocksdb/util/threadpool_imp.cc:306 #4 0x000055981bb0ee4f in execute_native_thread_routine () ---Type to continue, or q to quit--- #5 0x00007f241d867dd5 in start_thread () from /lib64/libpthread.so.0 #6 0x00007f241c72cead in clone () from /lib64/libc.so.6 Thread 17 (Thread 0x7f2412039700 (LWP 308292)): #0 0x00007f241d86b965 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f241cfc5b1c in std::condition_variable::wait(std::unique_lock&) () from /lib64/libstdc++.so.6 #2 0x000055981ba2f0d9 in rocksdb::ThreadPoolImpl::Impl::BGThread(unsigned long) () at /usr/src/debug/ceph-15.2.8.1.0.0/src/rocksdb/util/threadpool_imp.cc:196 #3 0x000055981ba2f3fd in rocksdb::ThreadPoolImpl::Impl::BGThreadWrapper (arg=0x55982611a160) at /usr/src/debug/ceph-15.2.8.1.0.0/src/rocksdb/util/threadpool_imp.cc:306 #4 0x000055981bb0ee4f in execute_native_thread_routine () #5 0x00007f241d867dd5 in start_thread () from /lib64/libpthread.so.0 #6 0x00007f241c72cead in clone () from /lib64/libc.so.6 Thread 16 (Thread 0x7f241283a700 (LWP 308236)): #0 0x00007f241d86b965 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f241cfc5b1c in std::condition_variable::wait(std::unique_lock&) () from /lib64/libstdc++.so.6 #2 0x000055981b45f8ef in Finisher::finisher_thread_entry() () at /usr/src/debug/ceph-15.2.8.1.0.0/src/common/Finisher.cc:86 #3 0x00007f241d867dd5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007f241c72cead in clone () from /lib64/libc.so.6 Thread 15 (Thread 0x7f241303b700 (LWP 308235)): #0 0x00007f241d86b965 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f241cfc5b1c in std::condition_variable::wait(std::unique_lock&) () from /lib64/libstdc++.so.6 #2 0x000055981b48a5d3 in SafeTimer::timer_thread() () at /usr/src/debug/ceph-15.2.8.1.0.0/src/common/Timer.cc:106 #3 0x000055981b48b97d in SafeTimerThread::entry (this=) at /usr/src/debug/ceph-15.2.8.1.0.0/src/common/Timer.cc:30 #4 0x00007f241d867dd5 in start_thread () from /lib64/libpthread.so.0 #5 0x00007f241c72cead in clone () from /lib64/libc.so.6 Thread 14 (Thread 0x7f241383c700 (LWP 308234)): #0 0x00007f241d86b965 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f241cfc5b1c in std::condition_variable::wait(std::unique_lock&) () from /lib64/libstdc++.so.6 #2 0x000055981b48a5d3 in SafeTimer::timer_thread() () at /usr/src/debug/ceph-15.2.8.1.0.0/src/common/Timer.cc:106 #3 0x000055981b48b97d in SafeTimerThread::entry (this=) at /usr/src/debug/ceph-15.2.8.1.0.0/src/common/Timer.cc:30 #4 0x00007f241d867dd5 in start_thread () from /lib64/libpthread.so.0 #5 0x00007f241c72cead in clone () from /lib64/libc.so.6 Thread 13 (Thread 0x7f241403d700 (LWP 308233)): #0 0x00007f241d86e4ed in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007f241d869dcb in _L_lock_883 () from /lib64/libpthread.so.0 #2 0x00007f241d869c98 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x000055981b66439b in __gthread_mutex_lock (__mutex=0x5598261c0b80) at /opt/rh/devtoolset-8/root/usr/include/c++/8/x86_64-redhat-linux/bits/gthr-default.h:748 #4 lock (this=0x5598261c0b80) at /opt/rh/devtoolset-8/root/usr/include/c++/8/bits/std_mutex.h:103 #5 lock_guard (__m=..., this=) at /opt/rh/devtoolset-8/root/usr/include/c++/8/bits/std_mutex.h:162 #6 MgrClient::update_daemon_health(std::vector >&&) (this=0x5598261c0a18, metrics=) at /usr/src/debug/ceph-15.2.8.1.0.0/src/mgr/MgrClient.cc:613 #7 0x000055981ae6bf9a in OSD::tick_without_osd_lock() () at /usr/src/debug/ceph-15.2.8.1.0.0/src/osd/OSD.cc:5916 #8 0x000055981ae9d0b9 in Context::complete (this=0x5598489a1490, r=) at /usr/src/debug/ceph-15.2.8.1.0.0/src/include/Context.h:77 #9 0x000055981b48a588 in SafeTimer::timer_thread() () at /usr/src/debug/ceph-15.2.8.1.0.0/src/common/Timer.cc:96 #10 0x000055981b48b97d in SafeTimerThread::entry (this=) at /usr/src/debug/ceph-15.2.8.1.0.0/src/common/Timer.cc:30 #11 0x00007f241d867dd5 in start_thread () from /lib64/libpthread.so.0 #12 0x00007f241c72cead in clone () from /lib64/libc.so.6 Thread 12 (Thread 0x7f241483e700 (LWP 308232)): #0 0x00007f241d86bd12 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x000055981b48ab1c in __gthread_cond_timedwait (__abs_timeout=0x7f241483b9d0, __mutex=, __cond=0x5598261c0858) at /opt/rh/devtoolset-8/root/usr/include/c++/8/x86_64-redhat-linux/bits/gthr-default.h:871 #2 __wait_until_impl > > (__atime=, __lock=..., this=0x5598261c0858) at /opt/rh/devtoolset-8/root/usr/include/c++/8/condition_variable:178 #3 wait_until > > (__atime=..., __lock=..., this=0x5598261c0858) ---Type to continue, or q to quit--- at /opt/rh/devtoolset-8/root/usr/include/c++/8/condition_variable:119 #4 SafeTimer::timer_thread() () at /usr/src/debug/ceph-15.2.8.1.0.0/src/common/Timer.cc:108 #5 0x000055981b48b97d in SafeTimerThread::entry (this=) at /usr/src/debug/ceph-15.2.8.1.0.0/src/common/Timer.cc:30 #6 0x00007f241d867dd5 in start_thread () from /lib64/libpthread.so.0 #7 0x00007f241c72cead in clone () from /lib64/libc.so.6 Thread 11 (Thread 0x7f241503f700 (LWP 308231)): #0 0x00007f241d86bd12 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x000055981b150621 in __gthread_cond_timedwait (__abs_timeout=0x7f241503c9e0, __mutex=, __cond=0x559826eac768) at /opt/rh/devtoolset-8/root/usr/include/c++/8/x86_64-redhat-linux/bits/gthr-default.h:871 #2 __wait_until_impl > > (__atime=, __lock=..., this=0x559826eac768) at /opt/rh/devtoolset-8/root/usr/include/c++/8/condition_variable:178 #3 wait_until > > (__atime=, __lock=..., this=0x559826eac768) at /opt/rh/devtoolset-8/root/usr/include/c++/8/condition_variable:119 #4 ceph::timer::timer_thread (this=0x559826eac700) at /usr/src/debug/ceph-15.2.8.1.0.0/src/common/ceph_timer.h:140 #5 0x000055981bb0ee4f in execute_native_thread_routine () #6 0x00007f241d867dd5 in start_thread () from /lib64/libpthread.so.0 #7 0x00007f241c72cead in clone () from /lib64/libc.so.6 Thread 10 (Thread 0x7f24160bf700 (LWP 308230)): #0 0x00007f241c6f3e2d in nanosleep () from /lib64/libc.so.6 #1 0x00007f241c724704 in usleep () from /lib64/libc.so.6 #2 0x000055981b121ee3 in OpHistoryServiceThread::entry() () at /usr/src/debug/ceph-15.2.8.1.0.0/src/common/TrackedOp.cc:44 #3 0x00007f241d867dd5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007f241c72cead in clone () from /lib64/libc.so.6 Thread 9 (Thread 0x7f24168c0700 (LWP 308226)): #0 0x00007f241c72220d in poll () from /lib64/libc.so.6 #1 0x000055981b442d8a in poll (__timeout=-1, __nfds=4, __fds=0x7f24168bd7a0) at /usr/include/bits/poll2.h:41 #2 SignalHandler::entry (this=0x559826134c60) at /usr/src/debug/ceph-15.2.8.1.0.0/src/global/signal_handler.cc:488 #3 0x00007f241d867dd5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007f241c72cead in clone () from /lib64/libc.so.6 Thread 8 (Thread 0x7f24170c1700 (LWP 308221)): #0 load (__m=std::memory_order_seq_cst, this=0x55983a3df130) at /usr/src/debug/ceph-15.2.8.1.0.0/src/common/perf_counters.h:211 #1 operator std::__atomic_base::__int_type (this=0x55983a3df130) at /opt/rh/devtoolset-8/root/usr/include/c++/8/bits/atomic_base.h:259 #2 read_avg (this=0x55983a3df100) at /usr/src/debug/ceph-15.2.8.1.0.0/src/common/perf_counters.h:211 #3 ceph::common::PerfCounters::dump_formatted_generic(ceph::Formatter*, bool, bool, std::string const&) const () at /usr/src/debug/ceph-15.2.8.1.0.0/src/common/perf_counters.cc:422 #4 0x000055981b574bb6 in ceph::common::PerfCountersCollectionImpl::dump_formatted_generic (this=0x5598261437d0, f=0x559826ed2700, schema=false, histograms=false, logger="osd", counter="op_latency") at /usr/src/debug/ceph-15.2.8.1.0.0/src/common/perf_counters.cc:139 #5 0x000055981b5767f7 in dump_formatted (counter=..., logger="osd", schema=false, f=0x559826ed2700, this=0x5598261437d0) at /usr/src/debug/ceph-15.2.8.1.0.0/src/common/perf_counters.h:329 #6 ceph::common::PerfCountersCollection::dump_formatted(ceph::Formatter*, bool, std::string const&, std::string const&) () at /usr/src/debug/ceph-15.2.8.1.0.0/src/common/perf_counters_collection.cc:41 #7 0x000055981b4b38b4 in ceph::common::CephContext::_do_command(std::basic_string_view >, std::map >, std::vector >, std::vector > >, std::less, std::allocator >, std::vector >, std::vector > > > > > const&, ceph::Formatter*, std::ostream&, ceph::buffer::v15_2_0::list*) () at /usr/src/debug/ceph-15.2.8.1.0.0/src/common/ceph_context.cc:505 #8 0x000055981b4b5594 in ceph::common::CephContext::do_command(std::basic_string_view >, std::map >, std::vector >, std::vector > >, std::less, std::allocator >, std::vector >, std::vector > > > > > const&, ceph::Formatter*, std::ostream&, ceph::buffer::v15_2_0::list*) () at /usr/src/debug/ceph-15.2.8.1.0.0/src/common/ceph_context.cc:471 #9 0x000055981b4b9474 in ceph::common::CephContextHook::call (this=, command=..., cmdmap=..., f=, errss=..., out=...) at /usr/src/debug/ceph-15.2.8.1.0.0/src/common/ceph_context.cc:422 Python Exception Cannot find type const cmdmap_t::_Rep_type: #10 0x000055981aece165 in AdminSocketHook::call_async(std::basic_string_view >, std::map >, std::vector >, std::vector > >, std::less, std::allocator >, std::vector >, std::vector > > > > > const&, ceph::Formatter*, ceph::buffer::v15_2_0::list const&, std::function) (this=this@entry=0x55982611a660, command=..., cmdmap=std::map with 3 elements, f=f@entry=0x559826ed2700, inbl=..., on_finish=...) at /usr/src/debug/ceph-15.2.8.1.0.0/src/include/buffer.h:594 #11 0x000055981b495e19 in AdminSocket::execute_command(std::vector > const&, ceph::buffer::v15_2_0::list const&, std::function) () at /opt/rh/devtoolset-8/root/usr/include/c++/8/bits/std_function.h:87 ---Type to continue, or q to quit--- #12 0x000055981b496395 in AdminSocket::execute_command(std::vector > const&, ceph::buffer::v15_2_0::list const&, std::ostream&, ceph::buffer::v15_2_0::list*) () at /opt/rh/devtoolset-8/root/usr/include/c++/8/bits/std_function.h:87 #13 0x000055981b496c7a in AdminSocket::do_accept() () at /usr/src/debug/ceph-15.2.8.1.0.0/src/common/admin_socket.cc:367 #14 0x000055981b497c38 in AdminSocket::entry (this=0x559826154d80) at /usr/src/debug/ceph-15.2.8.1.0.0/src/common/admin_socket.cc:255 #15 0x000055981bb0ee4f in execute_native_thread_routine () #16 0x00007f241d867dd5 in start_thread () from /lib64/libpthread.so.0 #17 0x00007f241c72cead in clone () from /lib64/libc.so.6 Thread 7 (Thread 0x7f24178c2700 (LWP 308219)): #0 0x00007f241d86bd12 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x000055981b4b584d in __gthread_cond_timedwait (__abs_timeout=0x7f24178bfa00, __mutex=, __cond=0x559826143878) at /opt/rh/devtoolset-8/root/usr/include/c++/8/x86_64-redhat-linux/bits/gthr-default.h:871 #2 __wait_until_impl > > (__atime=..., __lock=..., this=0x559826143878) at /opt/rh/devtoolset-8/root/usr/include/c++/8/condition_variable:178 #3 wait_until > > (__atime=..., __lock=..., this=0x559826143878) at /opt/rh/devtoolset-8/root/usr/include/c++/8/condition_variable:106 #4 wait_for > (__rtime=, __lock=..., this=0x559826143878) at /opt/rh/devtoolset-8/root/usr/include/c++/8/condition_variable:143 #5 ceph::common::CephContextServiceThread::entry (this=0x559826143830) at /usr/src/debug/ceph-15.2.8.1.0.0/src/common/ceph_context.cc:213 #6 0x00007f241d867dd5 in start_thread () from /lib64/libpthread.so.0 #7 0x00007f241c72cead in clone () from /lib64/libc.so.6 Thread 6 (Thread 0x7f24180c3700 (LWP 308212)): #0 0x00007f241c72d483 in epoll_wait () from /lib64/libc.so.6 #1 0x000055981b6107de in EpollDriver::event_wait (this=0x559826d9e900, fired_events=std::vector of length 0, capacity 0, tvp=) at /usr/src/debug/ceph-15.2.8.1.0.0/src/msg/async/EventEpoll.cc:123 #2 0x000055981b606ebc in EventCenter::process_events(unsigned int, std::chrono::duration >*) () at /usr/src/debug/ceph-15.2.8.1.0.0/src/msg/async/Event.cc:392 #3 0x000055981b60d1f7 in operator() (__closure=0x559826d9e9c8, __closure=0x559826d9e9c8) at /usr/src/debug/ceph-15.2.8.1.0.0/src/msg/async/Stack.cc:53 #4 std::_Function_handler::_M_invoke(std::_Any_data const&) () at /opt/rh/devtoolset-8/root/usr/include/c++/8/bits/std_function.h:297 #5 0x000055981bb0ee4f in execute_native_thread_routine () #6 0x00007f241d867dd5 in start_thread () from /lib64/libpthread.so.0 #7 0x00007f241c72cead in clone () from /lib64/libc.so.6 Thread 5 (Thread 0x7f24188c4700 (LWP 308211)): #0 0x00007f241c72d483 in epoll_wait () from /lib64/libc.so.6 #1 0x000055981b6107de in EpollDriver::event_wait (this=0x559826d9e300, fired_events=std::vector of length 0, capacity 0, tvp=) at /usr/src/debug/ceph-15.2.8.1.0.0/src/msg/async/EventEpoll.cc:123 #2 0x000055981b606ebc in EventCenter::process_events(unsigned int, std::chrono::duration >*) () at /usr/src/debug/ceph-15.2.8.1.0.0/src/msg/async/Event.cc:392 #3 0x000055981b60d1f7 in operator() (__closure=0x559826d9ea28, __closure=0x559826d9ea28) at /usr/src/debug/ceph-15.2.8.1.0.0/src/msg/async/Stack.cc:53 #4 std::_Function_handler::_M_invoke(std::_Any_data const&) () at /opt/rh/devtoolset-8/root/usr/include/c++/8/bits/std_function.h:297 #5 0x000055981bb0ee4f in execute_native_thread_routine () #6 0x00007f241d867dd5 in start_thread () from /lib64/libpthread.so.0 #7 0x00007f241c72cead in clone () from /lib64/libc.so.6 Thread 4 (Thread 0x7f24190c5700 (LWP 308210)): #0 0x00007f241c72d483 in epoll_wait () from /lib64/libc.so.6 #1 0x000055981b6107de in EpollDriver::event_wait (this=0x559826d9ec30, fired_events=std::vector of length 0, capacity 0, tvp=) at /usr/src/debug/ceph-15.2.8.1.0.0/src/msg/async/EventEpoll.cc:123 #2 0x000055981b606ebc in EventCenter::process_events(unsigned int, std::chrono::duration >*) () at /usr/src/debug/ceph-15.2.8.1.0.0/src/msg/async/Event.cc:392 #3 0x000055981b60d1f7 in operator() (__closure=0x559826d9e578, __closure=0x559826d9e578) at /usr/src/debug/ceph-15.2.8.1.0.0/src/msg/async/Stack.cc:53 #4 std::_Function_handler::_M_invoke(std::_Any_data const&) () at /opt/rh/devtoolset-8/root/usr/include/c++/8/bits/std_function.h:297 #5 0x000055981bb0ee4f in execute_native_thread_routine () #6 0x00007f241d867dd5 in start_thread () from /lib64/libpthread.so.0 #7 0x00007f241c72cead in clone () from /lib64/libc.so.6 Thread 3 (Thread 0x7f24198c6700 (LWP 308209)): #0 0x00007f241c72d483 in epoll_wait () from /lib64/libc.so.6 #1 0x000055981b6107de in EpollDriver::event_wait (this=0x559826d9ebd0, fired_events=std::vector of length 0, capacity 0, tvp=) at /usr/src/debug/ceph-15.2.8.1.0.0/src/msg/async/EventEpoll.cc:123 #2 0x000055981b606ebc in EventCenter::process_events(unsigned int, std::chrono::duration >*) () at /usr/src/debug/ceph-15.2.8.1.0.0/src/msg/async/Event.cc:392 #3 0x000055981b60d1f7 in operator() (__closure=0x559826d9e368, __closure=0x559826d9e368) at /usr/src/debug/ceph-15.2.8.1.0.0/src/msg/async/Stack.cc:53 #4 std::_Function_handler::_M_invoke(std::_Any_data const&) () at /opt/rh/devtoolset-8/root/usr/include/c++/8/bits/std_function.h:297 #5 0x000055981bb0ee4f in execute_native_thread_routine () #6 0x00007f241d867dd5 in start_thread () from /lib64/libpthread.so.0 #7 0x00007f241c72cead in clone () from /lib64/libc.so.6 ---Type to continue, or q to quit--- Thread 2 (Thread 0x7f241a0c7700 (LWP 308208)): #0 0x00007f241c72d483 in epoll_wait () from /lib64/libc.so.6 #1 0x000055981b6107de in EpollDriver::event_wait (this=0x559826d9e9f0, fired_events=std::vector of length 0, capacity 0, tvp=) at /usr/src/debug/ceph-15.2.8.1.0.0/src/msg/async/EventEpoll.cc:123 #2 0x000055981b606ebc in EventCenter::process_events(unsigned int, std::chrono::duration >*) () at /usr/src/debug/ceph-15.2.8.1.0.0/src/msg/async/Event.cc:392 #3 0x000055981b60d1f7 in operator() (__closure=0x559826d9e248, __closure=0x559826d9e248) at /usr/src/debug/ceph-15.2.8.1.0.0/src/msg/async/Stack.cc:53 #4 std::_Function_handler::_M_invoke(std::_Any_data const&) () at /opt/rh/devtoolset-8/root/usr/include/c++/8/bits/std_function.h:297 #5 0x000055981bb0ee4f in execute_native_thread_routine () #6 0x00007f241d867dd5 in start_thread () from /lib64/libpthread.so.0 #7 0x00007f241c72cead in clone () from /lib64/libc.so.6 Thread 1 (Thread 0x7f241a8c8700 (LWP 308204)): #0 0x00007f241d86b965 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007f241cfc5b1c in std::condition_variable::wait(std::unique_lock&) () from /lib64/libstdc++.so.6 #2 0x000055981b62a3af in ceph::logging::Log::entry() () at /usr/src/debug/ceph-15.2.8.1.0.0/src/log/Log.cc:435 #3 0x00007f241d867dd5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007f241c72cead in clone () from /lib64/libc.so.6