Thread 84 (Thread 0x7f26e017a700 (LWP 14823)): #0 0x00007f26e65c5a63 in __poll (fds=, nfds=, timeout=) at ../sysdeps/unix/sysv/linux/poll.c:87 #1 0x00000000008c6d99 in Pipe::tcp_read_wait (this=) at msg/Pipe.cc:2048 #2 0x00000000008cc8ad in Pipe::tcp_read (this=0x7f26b02b74f0, buf=, len=1) at msg/Pipe.cc:2021 #3 0x00000000008dc33e in Pipe::reader (this=0x7f26b02b74f0) at msg/Pipe.cc:1276 #4 0x00000000008defed in Pipe::Reader::entry (this=) at msg/Pipe.h:47 #5 0x00007f26e7845851 in start_thread (arg=0x7f26e017a700) at pthread_create.c:301 #6 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115 Thread 83 (Thread 0x7f26bc4f9700 (LWP 14824)): #0 0x00007f26e65c5a63 in __poll (fds=, nfds=, timeout=) at ../sysdeps/unix/sysv/linux/poll.c:87 #1 0x00000000008c6d99 in Pipe::tcp_read_wait (this=) at msg/Pipe.cc:2048 #2 0x00000000008cc8ad in Pipe::tcp_read (this=0x7f26b05234e0, buf=, len=1) at msg/Pipe.cc:2021 #3 0x00000000008dc33e in Pipe::reader (this=0x7f26b05234e0) at msg/Pipe.cc:1276 #4 0x00000000008defed in Pipe::Reader::entry (this=) at msg/Pipe.h:47 #5 0x00007f26e7845851 in start_thread (arg=0x7f26bc4f9700) at pthread_create.c:301 #6 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115 Thread 82 (Thread 0x7f26aa1fc700 (LWP 14755)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162 #1 0x00000000008dc626 in Wait (this=0x7f248da2cdb0) at common/Cond.h:55 #2 Pipe::reader (this=0x7f248da2cdb0) at msg/Pipe.cc:1267 #3 0x00000000008defed in Pipe::Reader::entry (this=) at msg/Pipe.h:47 #4 0x00007f26e7845851 in start_thread (arg=0x7f26aa1fc700) at pthread_create.c:301 #5 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115 Thread 81 (Thread 0x7f26aa0fb700 (LWP 14753)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162 #1 0x00000000008d27a9 in Wait (this=0x7f26b085d6c0) at common/Cond.h:55 #2 Pipe::writer (this=0x7f26b085d6c0) at msg/Pipe.cc:1527 #3 0x00000000008defcd in Pipe::Writer::entry (this=) at msg/Pipe.h:59 #4 0x00007f26e7845851 in start_thread (arg=0x7f26aa0fb700) at pthread_create.c:301 #5 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115 Thread 80 (Thread 0x7f26e1b82700 (LWP 14441)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162 #1 0x0000000000854c67 in Wait (this=0x1ae3fc8) at common/Cond.h:55 #2 SafeTimer::timer_thread (this=0x1ae3fc8) at common/Timer.cc:113 #3 0x00000000008570ed in SafeTimerThread::entry (this=) at common/Timer.cc:38 #4 0x00007f26e7845851 in start_thread (arg=0x7f26e1b82700) at pthread_create.c:301 #5 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115 Thread 79 (Thread 0x7f26e2f84700 (LWP 14439)): #0 0x00007f26e65c7d03 in select () at ../sysdeps/unix/syscall-template.S:82 #1 0x00000000007917f1 in SignalHandler::entry (this=0x1afea80) at global/signal_handler.cc:210 #2 0x00007f26e7845851 in start_thread (arg=0x7f26e2f84700) at pthread_create.c:301 #3 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115 Thread 78 (Thread 0x7f26e83cb700 (LWP 14794)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162 #1 0x00000000008d27a9 in Wait (this=0x7f263400acd0) at common/Cond.h:55 #2 Pipe::writer (this=0x7f263400acd0) at msg/Pipe.cc:1527 #3 0x00000000008defcd in Pipe::Writer::entry (this=) at msg/Pipe.h:59 #4 0x00007f26e7845851 in start_thread (arg=0x7f26e83cb700) at pthread_create.c:301 #5 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115 Thread 77 (Thread 0x7f26e83df760 (LWP 14371)): #0 0x00007f26e78460ad in pthread_join (threadid=139804540925696, thread_return=0x0) at pthread_join.c:89 #1 0x0000000000858482 in Thread::join (this=0x1ac5070, prval=) at common/Thread.cc:121 #2 0x00000000008a837a in SimpleMessenger::wait (this=0x1ac4e60) at msg/SimpleMessenger.cc:500 #3 0x0000000000560a2a in main (argc=, argv=) at ceph_osd.cc:446 Thread 76 (Thread 0x7f26b75fe700 (LWP 14467)): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:216 #1 0x000000000084954f in WaitUntil (this=0x1ae2878, wt=0x1b3c8a0) at common/Cond.h:71 #2 WaitInterval (this=0x1ae2878, wt=0x1b3c8a0) at common/Cond.h:79 #3 ThreadPool::worker (this=0x1ae2878, wt=0x1b3c8a0) at common/WorkQueue.cc:137 #4 0x000000000084c680 in ThreadPool::WorkThread::entry (this=) at common/WorkQueue.h:316 #5 0x00007f26e7845851 in start_thread (arg=0x7f26b75fe700) at pthread_create.c:301 #6 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115 Thread 75 (Thread 0x7f26b6bfd700 (LWP 14468)): #0 encode (this=0xbfb08000, bl=...) at include/encoding.h:103 #1 pg_log_entry_t::encode (this=0xbfb08000, bl=...) at osd/osd_types.cc:1698 #2 0x000000000069eca4 in encode (this=0x1d75360, t=...) at osd/osd_types.h:1324 #3 PG::write_log (this=0x1d75360, t=...) at osd/PG.cc:2364 #4 0x0000000000620643 in OSD::process_peering_events (this=0x1ae2420, pgs=std::list = {...}, handle=...) at osd/OSD.cc:6207 #5 0x000000000065d5b0 in OSD::PeeringWQ::_process (this=, pgs=std::list = {...}, handle=) at osd/OSD.h:744 #6 0x0000000000849899 in ThreadPool::worker (this=0x1ae2878, wt=0x1b3f280) at common/WorkQueue.cc:119 #7 0x000000000084c680 in ThreadPool::WorkThread::entry (this=) at common/WorkQueue.h:316 #8 0x00007f26e7845851 in start_thread (arg=0x7f26b6bfd700) at pthread_create.c:301 #9 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115 Thread 74 (Thread 0x7f26a97f2700 (LWP 14793)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162 #1 0x00000000008d27a9 in Wait (this=0x7f26340107d0) at common/Cond.h:55 #2 Pipe::writer (this=0x7f26340107d0) at msg/Pipe.cc:1527 #3 0x00000000008defcd in Pipe::Writer::entry (this=) at msg/Pipe.h:59 #4 0x00007f26e7845851 in start_thread (arg=0x7f26a97f2700) at pthread_create.c:301 #5 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115 Thread 73 (Thread 0x7f26e4386700 (LWP 14375)): #0 0x00007f26e65c5a63 in __poll (fds=, nfds=, timeout=) at ../sysdeps/unix/sysv/linux/poll.c:87 #1 0x00000000007eda7a in AdminSocket::entry (this=0x1ac1110) at common/admin_socket.cc:228 #2 0x00007f26e7845851 in start_thread (arg=0x7f26e4386700) at pthread_create.c:301 #3 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115 Thread 72 (Thread 0x7f26d81f7700 (LWP 14799)): #0 0x00007f26e65c5a63 in __poll (fds=, nfds=, timeout=) at ../sysdeps/unix/sysv/linux/poll.c:87 #1 0x00000000008c6d99 in Pipe::tcp_read_wait (this=) at msg/Pipe.cc:2048 #2 0x00000000008cc8ad in Pipe::tcp_read (this=0x7f26340107d0, buf=, len=1) at msg/Pipe.cc:2021 #3 0x00000000008dc33e in Pipe::reader (this=0x7f26340107d0) at msg/Pipe.cc:1276 #4 0x00000000008defed in Pipe::Reader::entry (this=) at msg/Pipe.h:47 #5 0x00007f26e7845851 in start_thread (arg=0x7f26d81f7700) at pthread_create.c:301 #6 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115 Thread 71 (Thread 0x7f26a9ffa700 (LWP 14771)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162 #1 0x00000000008dc626 in Wait (this=0x7f26b0987820) at common/Cond.h:55 #2 Pipe::reader (this=0x7f26b0987820) at msg/Pipe.cc:1267 #3 0x00000000008defed in Pipe::Reader::entry (this=) at msg/Pipe.h:47 #4 0x00007f26e7845851 in start_thread (arg=0x7f26a9ffa700) at pthread_create.c:301 #5 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115 Thread 70 (Thread 0x7f26c2df5700 (LWP 14804)): #0 0x00007f26e65c5a63 in __poll (fds=, nfds=, timeout=) at ../sysdeps/unix/sysv/linux/poll.c:87 #1 0x00000000008c6d99 in Pipe::tcp_read_wait (this=) at msg/Pipe.cc:2048 #2 0x00000000008cc8ad in Pipe::tcp_read (this=0x7f26340050e0, buf=, len=1) at msg/Pipe.cc:2021 #3 0x00000000008dc33e in Pipe::reader (this=0x7f26340050e0) at msg/Pipe.cc:1276 #4 0x00000000008defed in Pipe::Reader::entry (this=) at msg/Pipe.h:47 #5 0x00007f26e7845851 in start_thread (arg=0x7f26c2df5700) at pthread_create.c:301 #6 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115 Thread 69 (Thread 0x7f26bc2f7700 (LWP 14788)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162 #1 0x00000000008d27a9 in Wait (this=0x7f26340021b0) at common/Cond.h:55 #2 Pipe::writer (this=0x7f26340021b0) at msg/Pipe.cc:1527 #3 0x00000000008defcd in Pipe::Writer::entry (this=) at msg/Pipe.h:59 #4 0x00007f26e7845851 in start_thread (arg=0x7f26bc2f7700) at pthread_create.c:301 #5 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115 Thread 68 (Thread 0x7f26a8be6700 (LWP 14822)): #0 0x00007f26e65c5a63 in __poll (fds=, nfds=, timeout=) at ../sysdeps/unix/sysv/linux/poll.c:87 #1 0x00000000008c6d99 in Pipe::tcp_read_wait (this=) at msg/Pipe.cc:2048 #2 0x00000000008cc8ad in Pipe::tcp_read (this=0x7f26b08f54e0, buf=, len=1) at msg/Pipe.cc:2021 #3 0x00000000008dc33e in Pipe::reader (this=0x7f26b08f54e0) at msg/Pipe.cc:1276 #4 0x00000000008defed in Pipe::Reader::entry (this=) at msg/Pipe.h:47 #5 0x00007f26e7845851 in start_thread (arg=0x7f26a8be6700) at pthread_create.c:301 #6 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115 Thread 67 (Thread 0x7f26a99f4700 (LWP 14770)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162 #1 0x00000000008dc626 in Wait (this=0x7f26b0b49510) at common/Cond.h:55 #2 Pipe::reader (this=0x7f26b0b49510) at msg/Pipe.cc:1267 #3 0x00000000008defed in Pipe::Reader::entry (this=) at msg/Pipe.h:47 #4 0x00007f26e7845851 in start_thread (arg=0x7f26a99f4700) at pthread_create.c:301 #5 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115 Thread 66 (Thread 0x7f26c2bf3700 (LWP 14805)): #0 0x00007f26e65c5a63 in __poll (fds=, nfds=, timeout=) at ../sysdeps/unix/sysv/linux/poll.c:87 #1 0x00000000008c6d99 in Pipe::tcp_read_wait (this=) at msg/Pipe.cc:2048 #2 0x00000000008cc8ad in Pipe::tcp_read (this=0x7f263400dcf0, buf=, len=1) at msg/Pipe.cc:2021 #3 0x00000000008dc33e in Pipe::reader (this=0x7f263400dcf0) at msg/Pipe.cc:1276 #4 0x00000000008defed in Pipe::Reader::entry (this=) at msg/Pipe.h:47 #5 0x00007f26e7845851 in start_thread (arg=0x7f26c2bf3700) at pthread_create.c:301 #6 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115 Thread 65 (Thread 0x7f26a9df8700 (LWP 14825)): #0 0x00007f26e65c5a63 in __poll (fds=, nfds=, timeout=) at ../sysdeps/unix/sysv/linux/poll.c:87 #1 0x00000000008c6d99 in Pipe::tcp_read_wait (this=) at msg/Pipe.cc:2048 #2 0x00000000008cc8ad in Pipe::tcp_read (this=0x7f26b01a0d50, buf=, len=1) at msg/Pipe.cc:2021 #3 0x00000000008dc33e in Pipe::reader (this=0x7f26b01a0d50) at msg/Pipe.cc:1276 #4 0x00000000008defed in Pipe::Reader::entry (this=) at msg/Pipe.h:47 #5 0x00007f26e7845851 in start_thread (arg=0x7f26a9df8700) at pthread_create.c:301 #6 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115 Thread 64 (Thread 0x7f26bc1f6700 (LWP 14826)): #0 0x00007f26e65c5a63 in __poll (fds=, nfds=, timeout=) at ../sysdeps/unix/sysv/linux/poll.c:87 #1 0x00000000008c6d99 in Pipe::tcp_read_wait (this=) at msg/Pipe.cc:2048 #2 0x00000000008cc8ad in Pipe::tcp_read (this=0x7f26b0c22200, buf=, len=1) at msg/Pipe.cc:2021 #3 0x00000000008dc33e in Pipe::reader (this=0x7f26b0c22200) at msg/Pipe.cc:1276 #4 0x00000000008defed in Pipe::Reader::entry (this=) at msg/Pipe.h:47 #5 0x00007f26e7845851 in start_thread (arg=0x7f26bc1f6700) at pthread_create.c:301 #6 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115 Thread 63 (Thread 0x7f26a87e2700 (LWP 14786)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162 #1 0x00000000008d27a9 in Wait (this=0x7f2634011a80) at common/Cond.h:55 #2 Pipe::writer (this=0x7f2634011a80) at msg/Pipe.cc:1527 #3 0x00000000008defcd in Pipe::Writer::entry (this=) at msg/Pipe.h:59 #4 0x00007f26e7845851 in start_thread (arg=0x7f26a87e2700) at pthread_create.c:301 #5 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115 Thread 62 (Thread 0x7f26a8ee9700 (LWP 14757)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162 #1 0x00000000008d27a9 in Wait (this=0x7f26b01a0d50) at common/Cond.h:55 #2 Pipe::writer (this=0x7f26b01a0d50) at msg/Pipe.cc:1527 #3 0x00000000008defcd in Pipe::Writer::entry (this=) at msg/Pipe.h:59 #4 0x00007f26e7845851 in start_thread (arg=0x7f26a8ee9700) at pthread_create.c:301 #5 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115 Thread 61 (Thread 0x7f26e5eb7700 (LWP 14372)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162 #1 0x0000000000894cfb in ceph::log::Log::entry (this=0x1ac0bc0) at log/Log.cc:323 #2 0x00007f26e7845851 in start_thread (arg=0x7f26e5eb7700) at pthread_create.c:301 #3 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115 Thread 60 (Thread 0x7f26c2cf4700 (LWP 14803)): #0 0x00007f26e65c5a63 in __poll (fds=, nfds=, timeout=) at ../sysdeps/unix/sysv/linux/poll.c:87 #1 0x00000000008c6d99 in Pipe::tcp_read_wait (this=) at msg/Pipe.cc:2048 #2 0x00000000008cc8ad in Pipe::tcp_read (this=0x7f2634011a80, buf=, len=1) at msg/Pipe.cc:2021 #3 0x00000000008dc33e in Pipe::reader (this=0x7f2634011a80) at msg/Pipe.cc:1276 #4 0x00000000008defed in Pipe::Reader::entry (this=) at msg/Pipe.h:47 #5 0x00007f26e7845851 in start_thread (arg=0x7f26c2cf4700) at pthread_create.c:301 #6 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115 Thread 59 (Thread 0x7f26e067f700 (LWP 14795)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162 #1 0x00000000008d27a9 in Wait (this=0x7f263400dcf0) at common/Cond.h:55 #2 Pipe::writer (this=0x7f263400dcf0) at msg/Pipe.cc:1527 #3 0x00000000008defcd in Pipe::Writer::entry (this=) at msg/Pipe.h:59 #4 0x00007f26e7845851 in start_thread (arg=0x7f26e067f700) at pthread_create.c:301 #5 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115 Thread 58 (Thread 0x7f26cc2f8700 (LWP 14801)): #0 0x00007f26e65c5a63 in __poll (fds=, nfds=, timeout=) at ../sysdeps/unix/sysv/linux/poll.c:87 #1 0x00000000008c6d99 in Pipe::tcp_read_wait (this=) at msg/Pipe.cc:2048 #2 0x00000000008cc8ad in Pipe::tcp_read (this=0x7f263400e3b0, buf=, len=1) at msg/Pipe.cc:2021 #3 0x00000000008dc33e in Pipe::reader (this=0x7f263400e3b0) at msg/Pipe.cc:1276 #4 0x00000000008defed in Pipe::Reader::entry (this=) at msg/Pipe.h:47 #5 0x00007f26e7845851 in start_thread (arg=0x7f26cc2f8700) at pthread_create.c:301 #6 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115 Thread 57 (Thread 0x7f26e047d700 (LWP 14761)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162 #1 0x00000000008d27a9 in Wait (this=0x7f26b0b49510) at common/Cond.h:55 #2 Pipe::writer (this=0x7f26b0b49510) at msg/Pipe.cc:1527 #3 0x00000000008defcd in Pipe::Writer::entry (this=) at msg/Pipe.h:59 #4 0x00007f26e7845851 in start_thread (arg=0x7f26e047d700) at pthread_create.c:301 #5 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115 Thread 56 (Thread 0x7f26bc3f8700 (LWP 14766)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162 #1 0x00000000008dc626 in Wait (this=0x7f26b057da90) at common/Cond.h:55 #2 Pipe::reader (this=0x7f26b057da90) at msg/Pipe.cc:1267 #3 0x00000000008defed in Pipe::Reader::entry (this=) at msg/Pipe.h:47 #4 0x00007f26e7845851 in start_thread (arg=0x7f26bc3f8700) at pthread_create.c:301 #5 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115 Thread 55 (Thread 0x7f26cc3f9700 (LWP 14800)): #0 0x00007f26e65c5a63 in __poll (fds=, nfds=, timeout=) at ../sysdeps/unix/sysv/linux/poll.c:87 #1 0x00000000008c6d99 in Pipe::tcp_read_wait (this=) at msg/Pipe.cc:2048 #2 0x00000000008cc8ad in Pipe::tcp_read (this=0x7f2634002440, buf=, len=1) at msg/Pipe.cc:2021 #3 0x00000000008dc33e in Pipe::reader (this=0x7f2634002440) at msg/Pipe.cc:1276 #4 0x00000000008defed in Pipe::Reader::entry (this=) at msg/Pipe.h:47 #5 0x00007f26e7845851 in start_thread (arg=0x7f26cc3f9700) at pthread_create.c:301 #6 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115 Thread 54 (Thread 0x7f26abfff700 (LWP 14472)): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:216 #1 0x00000000006250e6 in WaitUntil (this=0x1ae2420) at common/Cond.h:71 #2 WaitInterval (this=0x1ae2420) at common/Cond.h:79 #3 OSD::heartbeat_entry (this=0x1ae2420) at osd/OSD.cc:2108 #4 0x000000000065d73d in OSD::T_Heartbeat::entry (this=) at osd/OSD.h:572 #5 0x00007f26e7845851 in start_thread (arg=0x7f26abfff700) at pthread_create.c:301 #6 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115 Thread 53 (Thread 0x7f26d83f9700 (LWP 14798)): #0 0x00007f26e65c5a63 in __poll (fds=, nfds=, timeout=) at ../sysdeps/unix/sysv/linux/poll.c:87 #1 0x00000000008c6d99 in Pipe::tcp_read_wait (this=) at msg/Pipe.cc:2048 #2 0x00000000008cc8ad in Pipe::tcp_read (this=0x7f26340021b0, buf=, len=1) at msg/Pipe.cc:2021 #3 0x00000000008dc33e in Pipe::reader (this=0x7f26340021b0) at msg/Pipe.cc:1276 #4 0x00000000008defed in Pipe::Reader::entry (this=) at msg/Pipe.h:47 #5 0x00007f26e7845851 in start_thread (arg=0x7f26d83f9700) at pthread_create.c:301 #6 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115 Thread 52 (Thread 0x7f26c29f1700 (LWP 14806)): #0 0x00007f26e65c5a63 in __poll (fds=, nfds=, timeout=) at ../sysdeps/unix/sysv/linux/poll.c:87 #1 0x00000000008c6d99 in Pipe::tcp_read_wait (this=) at msg/Pipe.cc:2048 #2 0x00000000008cc8ad in Pipe::tcp_read (this=0x7f2634010350, buf=, len=1) at msg/Pipe.cc:2021 #3 0x00000000008dc33e in Pipe::reader (this=0x7f2634010350) at msg/Pipe.cc:1276 #4 0x00000000008defed in Pipe::Reader::entry (this=) at msg/Pipe.h:47 #5 0x00007f26e7845851 in start_thread (arg=0x7f26c29f1700) at pthread_create.c:301 #6 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115 Thread 51 (Thread 0x7f26cc1f7700 (LWP 14802)): #0 0x00007f26e65c5a63 in __poll (fds=, nfds=, timeout=) at ../sysdeps/unix/sysv/linux/poll.c:87 #1 0x00000000008c6d99 in Pipe::tcp_read_wait (this=) at msg/Pipe.cc:2048 #2 0x00000000008cc8ad in Pipe::tcp_read (this=0x7f263400acd0, buf=, len=1) at msg/Pipe.cc:2021 #3 0x00000000008dc33e in Pipe::reader (this=0x7f263400acd0) at msg/Pipe.cc:1276 #4 0x00000000008defed in Pipe::Reader::entry (this=) at msg/Pipe.h:47 #5 0x00007f26e7845851 in start_thread (arg=0x7f26cc1f7700) at pthread_create.c:301 #6 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115 Thread 50 (Thread 0x7f26b61fc700 (LWP 14469)): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:216 #1 0x000000000084954f in WaitUntil (this=0x1ae2a00, wt=0x1b3f460) at common/Cond.h:71 #2 WaitInterval (this=0x1ae2a00, wt=0x1b3f460) at common/Cond.h:79 #3 ThreadPool::worker (this=0x1ae2a00, wt=0x1b3f460) at common/WorkQueue.cc:137 #4 0x000000000084c680 in ThreadPool::WorkThread::entry (this=) at common/WorkQueue.h:316 #5 0x00007f26e7845851 in start_thread (arg=0x7f26b61fc700) at pthread_create.c:301 #6 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115 Thread 49 (Thread 0x7f262bbfb700 (LWP 14792)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162 #1 0x00000000008d27a9 in Wait (this=0x7f2634002440) at common/Cond.h:55 #2 Pipe::writer (this=0x7f2634002440) at msg/Pipe.cc:1527 #3 0x00000000008defcd in Pipe::Writer::entry (this=) at msg/Pipe.h:59 #4 0x00007f26e7845851 in start_thread (arg=0x7f262bbfb700) at pthread_create.c:301 #5 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115 Thread 48 (Thread 0x7f26c2af2700 (LWP 14758)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162 #1 0x00000000008d27a9 in Wait (this=0x7f26b08f54e0) at common/Cond.h:55 #2 Pipe::writer (this=0x7f26b08f54e0) at msg/Pipe.cc:1527 #3 0x00000000008defcd in Pipe::Writer::entry (this=) at msg/Pipe.h:59 #4 0x00007f26e7845851 in start_thread (arg=0x7f26c2af2700) at pthread_create.c:301 #5 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115 Thread 47 (Thread 0x7f26e057e700 (LWP 14797)): #0 0x00007f26e65c5a63 in __poll (fds=, nfds=, timeout=) at ../sysdeps/unix/sysv/linux/poll.c:87 #1 0x00000000008c6d99 in Pipe::tcp_read_wait (this=) at msg/Pipe.cc:2048 #2 0x00000000008cc8ad in Pipe::tcp_read (this=0x7f2634006b20, buf=, len=1) at msg/Pipe.cc:2021 #3 0x00000000008dc33e in Pipe::reader (this=0x7f2634006b20) at msg/Pipe.cc:1276 #4 0x00000000008defed in Pipe::Reader::entry (this=) at msg/Pipe.h:47 #5 0x00007f26e7845851 in start_thread (arg=0x7f26e057e700) at pthread_create.c:301 #6 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115 Thread 46 (Thread 0x7f26e037c700 (LWP 14796)): #0 0x00007f26e65c5a63 in __poll (fds=, nfds=, timeout=) at ../sysdeps/unix/sysv/linux/poll.c:87 #1 0x00000000008c6d99 in Pipe::tcp_read_wait (this=) at msg/Pipe.cc:2048 #2 0x00000000008cc8ad in Pipe::tcp_read (this=0x7f2634000e70, buf=, len=1) at msg/Pipe.cc:2021 #3 0x00000000008dc33e in Pipe::reader (this=0x7f2634000e70) at msg/Pipe.cc:1276 #4 0x00000000008defed in Pipe::Reader::entry (this=) at msg/Pipe.h:47 #5 0x00007f26e7845851 in start_thread (arg=0x7f26e037c700) at pthread_create.c:301 #6 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115 Thread 45 (Thread 0x7f262b9f9700 (LWP 14762)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162 #1 0x00000000008d27a9 in Wait (this=0x7f26b00d6c40) at common/Cond.h:55 #2 Pipe::writer (this=0x7f26b00d6c40) at msg/Pipe.cc:1527 #3 0x00000000008defcd in Pipe::Writer::entry (this=) at msg/Pipe.h:59 #4 0x00007f26e7845851 in start_thread (arg=0x7f262b9f9700) at pthread_create.c:301 #5 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115 Thread 44 (Thread 0x7f26c43f9700 (LWP 14750)): #0 0x00007f26e65c5a63 in __poll (fds=, nfds=, timeout=) at ../sysdeps/unix/sysv/linux/poll.c:87 #1 0x00000000008b4e38 in Accepter::entry (this=0x1ac7660) at msg/Accepter.cc:198 #2 0x00007f26e7845851 in start_thread (arg=0x7f26c43f9700) at pthread_create.c:301 #3 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115 Thread 43 (Thread 0x7f26c2ff7700 (LWP 14791)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162 #1 0x00000000008d27a9 in Wait (this=0x7f263400e3b0) at common/Cond.h:55 #2 Pipe::writer (this=0x7f263400e3b0) at msg/Pipe.cc:1527 #3 0x00000000008defcd in Pipe::Writer::entry (this=) at msg/Pipe.h:59 #4 0x00007f26e7845851 in start_thread (arg=0x7f26c2ff7700) at pthread_create.c:301 #5 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115 Thread 42 (Thread 0x7f262bdfd700 (LWP 14787)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162 #1 0x00000000008d27a9 in Wait (this=0x7f2634006b20) at common/Cond.h:55 #2 Pipe::writer (this=0x7f2634006b20) at msg/Pipe.cc:1527 #3 0x00000000008defcd in Pipe::Writer::entry (this=) at msg/Pipe.h:59 #4 0x00007f26e7845851 in start_thread (arg=0x7f262bdfd700) at pthread_create.c:301 #5 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115 Thread 41 (Thread 0x7f26b41f7700 (LWP 14752)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162 #1 0x00000000008d27a9 in Wait (this=0x7f248da2cdb0) at common/Cond.h:55 #2 Pipe::writer (this=0x7f248da2cdb0) at msg/Pipe.cc:1527 #3 0x00000000008defcd in Pipe::Writer::entry (this=) at msg/Pipe.h:59 #4 0x00007f26e7845851 in start_thread (arg=0x7f26b41f7700) at pthread_create.c:301 #5 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115 Thread 40 (Thread 0x7f262bfff700 (LWP 14785)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162 #1 0x00000000008d27a9 in Wait (this=0x7f2634000e70) at common/Cond.h:55 #2 Pipe::writer (this=0x7f2634000e70) at msg/Pipe.cc:1527 #3 0x00000000008defcd in Pipe::Writer::entry (this=) at msg/Pipe.h:59 #4 0x00007f26e7845851 in start_thread (arg=0x7f262bfff700) at pthread_create.c:301 #5 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115 Thread 39 (Thread 0x7f262bcfc700 (LWP 14790)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162 #1 0x00000000008d27a9 in Wait (this=0x7f26340050e0) at common/Cond.h:55 #2 Pipe::writer (this=0x7f26340050e0) at msg/Pipe.cc:1527 #3 0x00000000008defcd in Pipe::Writer::entry (this=) at msg/Pipe.h:59 #4 0x00007f26e7845851 in start_thread (arg=0x7f262bcfc700) at pthread_create.c:301 #5 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115 Thread 38 (Thread 0x7f26e0780700 (LWP 14748)): #0 0x00007f26e65c5a63 in __poll (fds=, nfds=, timeout=) at ../sysdeps/unix/sysv/linux/poll.c:87 #1 0x00000000008c6d99 in Pipe::tcp_read_wait (this=) at msg/Pipe.cc:2048 #2 0x00000000008cc8ad in Pipe::tcp_read (this=0x7f263400c690, buf=, len=1) at msg/Pipe.cc:2021 #3 0x00000000008dc33e in Pipe::reader (this=0x7f263400c690) at msg/Pipe.cc:1276 #4 0x00000000008defed in Pipe::Reader::entry (this=) at msg/Pipe.h:47 #5 0x00007f26e7845851 in start_thread (arg=0x7f26e0780700) at pthread_create.c:301 #6 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115 Thread 37 (Thread 0x7f26a8ce7700 (LWP 14772)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162 #1 0x00000000008dc626 in Wait (this=0x7f26b00d6c40) at common/Cond.h:55 #2 Pipe::reader (this=0x7f26b00d6c40) at msg/Pipe.cc:1267 #3 0x00000000008defed in Pipe::Reader::entry (this=) at msg/Pipe.h:47 #4 0x00007f26e7845851 in start_thread (arg=0x7f26a8ce7700) at pthread_create.c:301 #5 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115 Thread 36 (Thread 0x7f26aabfd700 (LWP 14474)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162 #1 0x0000000000854c67 in Wait (this=0x1ae3ea0) at common/Cond.h:55 #2 SafeTimer::timer_thread (this=0x1ae3ea0) at common/Timer.cc:113 #3 0x00000000008570ed in SafeTimerThread::entry (this=) at common/Timer.cc:38 #4 0x00007f26e7845851 in start_thread (arg=0x7f26aabfd700) at pthread_create.c:301 #5 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115 Thread 35 (Thread 0x7f26d82f8700 (LWP 14764)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162 #1 0x00000000008dc626 in Wait (this=0x7f26b085d6c0) at common/Cond.h:55 #2 Pipe::reader (this=0x7f26b085d6c0) at msg/Pipe.cc:1267 #3 0x00000000008defed in Pipe::Reader::entry (this=) at msg/Pipe.h:47 #4 0x00007f26e7845851 in start_thread (arg=0x7f26d82f8700) at pthread_create.c:301 #5 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115 Thread 34 (Thread 0x7f26a8de8700 (LWP 14789)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162 #1 0x00000000008d27a9 in Wait (this=0x7f2634010350) at common/Cond.h:55 #2 Pipe::writer (this=0x7f2634010350) at msg/Pipe.cc:1527 #3 0x00000000008defcd in Pipe::Writer::entry (this=) at msg/Pipe.h:59 #4 0x00007f26e7845851 in start_thread (arg=0x7f26a8de8700) at pthread_create.c:301 #5 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115 Thread 33 (Thread 0x7f26a88e3700 (LWP 14756)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162 #1 0x00000000008d27a9 in Wait (this=0x7f26b057da90) at common/Cond.h:55 #2 Pipe::writer (this=0x7f26b057da90) at msg/Pipe.cc:1527 #3 0x00000000008defcd in Pipe::Writer::entry (this=) at msg/Pipe.h:59 #4 0x00007f26e7845851 in start_thread (arg=0x7f26a88e3700) at pthread_create.c:301 #5 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115 Thread 32 (Thread 0x7f26ab5fe700 (LWP 14473)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162 #1 0x00000000007f07f3 in Wait (this=0x1ae40f0) at common/Cond.h:55 #2 Finisher::finisher_thread_entry (this=0x1ae40f0) at common/Finisher.cc:80 #3 0x00007f26e7845851 in start_thread (arg=0x7f26ab5fe700) at pthread_create.c:301 #4 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115 Thread 31 (Thread 0x7f26b43f9700 (LWP 14763)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162 #1 0x00000000008d27a9 in Wait (this=0x7f26b0c22200) at common/Cond.h:55 #2 Pipe::writer (this=0x7f26b0c22200) at msg/Pipe.cc:1527 #3 0x00000000008defcd in Pipe::Writer::entry (this=) at msg/Pipe.h:59 #4 0x00007f26e7845851 in start_thread (arg=0x7f26b43f9700) at pthread_create.c:301 #5 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115 Thread 30 (Thread 0x7f26c39f8700 (LWP 14465)): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:216 #1 0x000000000085528e in WaitUntil (this=0x7fff437c4b58) at common/Cond.h:71 #2 SafeTimer::timer_thread (this=0x7fff437c4b58) at common/Timer.cc:115 #3 0x00000000008570ed in SafeTimerThread::entry (this=) at common/Timer.cc:38 #4 0x00007f26e7845851 in start_thread (arg=0x7f26c39f8700) at pthread_create.c:301 #5 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115 Thread 29 (Thread 0x7f26a85e0700 (LWP 14754)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162 #1 0x00000000008d27a9 in Wait (this=0x7f26b0987820) at common/Cond.h:55 #2 Pipe::writer (this=0x7f26b0987820) at msg/Pipe.cc:1527 #3 0x00000000008defcd in Pipe::Writer::entry (this=) at msg/Pipe.h:59 #4 0x00007f26e7845851 in start_thread (arg=0x7f26a85e0700) at pthread_create.c:301 #5 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115 Thread 28 (Thread 0x7f26ce1fc700 (LWP 14449)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162 #1 0x00000000007f07f3 in Wait (this=0x1ae0408) at common/Cond.h:55 #2 Finisher::finisher_thread_entry (this=0x1ae0408) at common/Finisher.cc:80 #3 0x00007f26e7845851 in start_thread (arg=0x7f26ce1fc700) at pthread_create.c:301 #4 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115 Thread 27 (Thread 0x7f26c4dfa700 (LWP 14462)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162 #1 0x00000000008bb708 in Wait (this=0x1ac7688) at common/Cond.h:55 #2 DispatchQueue::entry (this=0x1ac7688) at msg/DispatchQueue.cc:117 #3 0x00000000008abcad in DispatchQueue::DispatchThread::entry (this=) at msg/DispatchQueue.h:85 #4 0x00007f26e7845851 in start_thread (arg=0x7f26c4dfa700) at pthread_create.c:301 #5 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115 Thread 26 (Thread 0x7f26a81dc700 (LWP 14760)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162 #1 0x00000000008d27a9 in Wait (this=0x7f26b05234e0) at common/Cond.h:55 #2 Pipe::writer (this=0x7f26b05234e0) at msg/Pipe.cc:1527 #3 0x00000000008defcd in Pipe::Writer::entry (this=) at msg/Pipe.h:59 #4 0x00007f26e7845851 in start_thread (arg=0x7f26a81dc700) at pthread_create.c:301 #5 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115 Thread 25 (Thread 0x7f26c61fc700 (LWP 14749)): #0 0x00007f26e65c5a63 in __poll (fds=, nfds=, timeout=) at ../sysdeps/unix/sysv/linux/poll.c:87 #1 0x00000000008b4e38 in Accepter::entry (this=0x1ac5a40) at msg/Accepter.cc:198 #2 0x00007f26e7845851 in start_thread (arg=0x7f26c61fc700) at pthread_create.c:301 #3 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115 Thread 24 (Thread 0x7f26a98f3700 (LWP 14759)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162 #1 0x00000000008d27a9 in Wait (this=0x7f26b02b74f0) at common/Cond.h:55 #2 Pipe::writer (this=0x7f26b02b74f0) at msg/Pipe.cc:1527 #3 0x00000000008defcd in Pipe::Writer::entry (this=) at msg/Pipe.h:59 #4 0x00007f26e7845851 in start_thread (arg=0x7f26a98f3700) at pthread_create.c:301 #5 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115 Thread 23 (Thread 0x7f26c75fe700 (LWP 14458)): #0 0x00007f26e65c5a63 in __poll (fds=, nfds=, timeout=) at ../sysdeps/unix/sysv/linux/poll.c:87 #1 0x00000000008b4e38 in Accepter::entry (this=0x1ac4f20) at msg/Accepter.cc:198 #2 0x00007f26e7845851 in start_thread (arg=0x7f26c75fe700) at pthread_create.c:301 #3 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115 Thread 22 (Thread 0x7f262bafa700 (LWP 14747)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162 #1 0x00000000008d27a9 in Wait (this=0x7f263400c690) at common/Cond.h:55 #2 Pipe::writer (this=0x7f263400c690) at msg/Pipe.cc:1527 #3 0x00000000008defcd in Pipe::Writer::entry (this=) at msg/Pipe.h:59 #4 0x00007f26e7845851 in start_thread (arg=0x7f262bafa700) at pthread_create.c:301 #5 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115 Thread 21 (Thread 0x7f26b57fb700 (LWP 14470)): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:216 #1 0x000000000084954f in WaitUntil (this=0x1ae2b88, wt=0x1b3f640) at common/Cond.h:71 #2 WaitInterval (this=0x1ae2b88, wt=0x1b3f640) at common/Cond.h:79 #3 ThreadPool::worker (this=0x1ae2b88, wt=0x1b3f640) at common/WorkQueue.cc:137 #4 0x000000000084c680 in ThreadPool::WorkThread::entry (this=) at common/WorkQueue.h:316 #5 0x00007f26e7845851 in start_thread (arg=0x7f26b57fb700) at pthread_create.c:301 #6 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115 Thread 20 (Thread 0x7f26ccdfa700 (LWP 14451)): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:216 #1 0x000000000085528e in WaitUntil (this=0x1ae0208) at common/Cond.h:71 #2 SafeTimer::timer_thread (this=0x1ae0208) at common/Timer.cc:115 #3 0x00000000008570ed in SafeTimerThread::entry (this=) at common/Timer.cc:38 #4 0x00007f26e7845851 in start_thread (arg=0x7f26ccdfa700) at pthread_create.c:301 #5 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115 Thread 19 (Thread 0x7f26c6bfd700 (LWP 14459)): #0 __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:136 #1 0x00007f26e7847388 in _L_lock_854 () from /lib64/libpthread-2.12.so #2 0x00007f26e7847257 in __pthread_mutex_lock (mutex=0x1ae2448) at pthread_mutex_lock.c:61 #3 0x00000000008876d1 in Mutex::Lock (this=0x1ae2438, no_lockdep=) at common/Mutex.cc:90 #4 0x0000000000636d97 in OSD::ms_dispatch (this=0x1ae2420, m=0x7f260c091010) at osd/OSD.cc:3250 #5 0x00000000008bb899 in ms_deliver_dispatch (this=0x1ac5a68) at msg/Messenger.h:553 #6 DispatchQueue::entry (this=0x1ac5a68) at msg/DispatchQueue.cc:107 #7 0x00000000008abcad in DispatchQueue::DispatchThread::entry (this=) at msg/DispatchQueue.h:85 #8 0x00007f26e7845851 in start_thread (arg=0x7f26c6bfd700) at pthread_create.c:301 #9 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115 Thread 18 (Thread 0x7f26b7fff700 (LWP 14466)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162 #1 0x00000000007f07f3 in Wait (this=0x7fff437c4c20) at common/Cond.h:55 #2 Finisher::finisher_thread_entry (this=0x7fff437c4c20) at common/Finisher.cc:80 #3 0x00007f26e7845851 in start_thread (arg=0x7f26b7fff700) at pthread_create.c:301 #4 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115 Thread 17 (Thread 0x7f26db5fe700 (LWP 14444)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162 #1 0x00000000006f668b in Wait (this=0x1aed200) at common/Cond.h:55 #2 FileJournal::write_finish_thread_entry (this=0x1aed200) at os/FileJournal.cc:1316 #3 0x000000000063ebfd in FileJournal::WriteFinisher::entry (this=) at os/FileJournal.h:303 #4 0x00007f26e7845851 in start_thread (arg=0x7f26db5fe700) at pthread_create.c:301 #5 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115 Thread 16 (Thread 0x7f26c7fff700 (LWP 14457)): #0 __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:136 #1 0x00007f26e7847388 in _L_lock_854 () from /lib64/libpthread-2.12.so #2 0x00007f26e7847257 in __pthread_mutex_lock (mutex=0x1ae2448) at pthread_mutex_lock.c:61 #3 0x00000000008876d1 in Mutex::Lock (this=0x1ae2438, no_lockdep=) at common/Mutex.cc:90 #4 0x0000000000636d97 in OSD::ms_dispatch (this=0x1ae2420, m=0x7f2624003270) at osd/OSD.cc:3250 #5 0x00000000008bb899 in ms_deliver_dispatch (this=0x1ac4f48) at msg/Messenger.h:553 #6 DispatchQueue::entry (this=0x1ac4f48) at msg/DispatchQueue.cc:107 #7 0x00000000008abcad in DispatchQueue::DispatchThread::entry (this=) at msg/DispatchQueue.h:85 #8 0x00007f26e7845851 in start_thread (arg=0x7f26c7fff700) at pthread_create.c:301 #9 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115 Thread 15 (Thread 0x7f26c57fb700 (LWP 14461)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162 #1 0x00000000008bb708 in Wait (this=0x1ac6848) at common/Cond.h:55 #2 DispatchQueue::entry (this=0x1ac6848) at msg/DispatchQueue.cc:117 #3 0x00000000008abcad in DispatchQueue::DispatchThread::entry (this=) at msg/DispatchQueue.h:85 #4 0x00007f26e7845851 in start_thread (arg=0x7f26c57fb700) at pthread_create.c:301 #5 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115 Thread 14 (Thread 0x7f26e3985700 (LWP 14438)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162 #1 0x00000000008aa331 in Wait (this=0x1ac5980) at common/Cond.h:55 #2 SimpleMessenger::reaper_entry (this=0x1ac5980) at msg/SimpleMessenger.cc:206 #3 0x00000000008ace7d in SimpleMessenger::ReaperThread::entry (this=) at msg/SimpleMessenger.h:412 #4 0x00007f26e7845851 in start_thread (arg=0x7f26e3985700) at pthread_create.c:301 #5 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115 Thread 13 (Thread 0x7f26b4dfa700 (LWP 14471)): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:216 #1 0x000000000084954f in WaitUntil (this=0x1ae2d10, wt=0x1b3f7a0) at common/Cond.h:71 #2 WaitInterval (this=0x1ae2d10, wt=0x1b3f7a0) at common/Cond.h:79 #3 ThreadPool::worker (this=0x1ae2d10, wt=0x1b3f7a0) at common/WorkQueue.cc:137 #4 0x000000000084c680 in ThreadPool::WorkThread::entry (this=) at common/WorkQueue.h:316 #5 0x00007f26e7845851 in start_thread (arg=0x7f26b4dfa700) at pthread_create.c:301 #6 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115 Thread 12 (Thread 0x7f26cd7fb700 (LWP 14450)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162 #1 0x00000000007f07f3 in Wait (this=0x1adffd8) at common/Cond.h:55 #2 Finisher::finisher_thread_entry (this=0x1adffd8) at common/Finisher.cc:80 #3 0x00007f26e7845851 in start_thread (arg=0x7f26cd7fb700) at pthread_create.c:301 #4 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115 Thread 11 (Thread 0x7f26cf5fe700 (LWP 14447)): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:216 #1 0x000000000084954f in WaitUntil (this=0x1ae0538, wt=0x1ae5600) at common/Cond.h:71 #2 WaitInterval (this=0x1ae0538, wt=0x1ae5600) at common/Cond.h:79 #3 ThreadPool::worker (this=0x1ae0538, wt=0x1ae5600) at common/WorkQueue.cc:137 #4 0x000000000084c680 in ThreadPool::WorkThread::entry (this=) at common/WorkQueue.h:316 #5 0x00007f26e7845851 in start_thread (arg=0x7f26cf5fe700) at pthread_create.c:301 #6 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115 Thread 10 (Thread 0x7f26cebfd700 (LWP 14448)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162 #1 0x0000000000705bdc in Wait (this=0x1adfb30) at common/Cond.h:55 #2 FileStore::flusher_entry (this=0x1adfb30) at os/FileStore.cc:3264 #3 0x00000000007356ed in FileStore::FlusherThread::entry (this=) at os/FileStore.h:262 #4 0x00007f26e7845851 in start_thread (arg=0x7f26cebfd700) at pthread_create.c:301 #5 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115 Thread 9 (Thread 0x7f26cffff700 (LWP 14446)): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:216 #1 0x000000000084954f in WaitUntil (this=0x1ae0538, wt=0x1aea420) at common/Cond.h:71 #2 WaitInterval (this=0x1ae0538, wt=0x1aea420) at common/Cond.h:79 #3 ThreadPool::worker (this=0x1ae0538, wt=0x1aea420) at common/WorkQueue.cc:137 #4 0x000000000084c680 in ThreadPool::WorkThread::entry (this=) at common/WorkQueue.h:316 #5 0x00007f26e7845851 in start_thread (arg=0x7f26cffff700) at pthread_create.c:301 #6 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115 Thread 8 (Thread 0x7f26e1181700 (LWP 14442)): #0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:216 #1 0x0000000000719e7e in WaitUntil (this=0x1adfb30) at common/Cond.h:71 #2 WaitInterval (this=0x1adfb30) at common/Cond.h:79 #3 FileStore::sync_entry (this=0x1adfb30) at os/FileStore.cc:3304 #4 0x000000000073632d in FileStore::SyncThread::entry (this=) at os/FileStore.h:115 #5 0x00007f26e7845851 in start_thread (arg=0x7f26e1181700) at pthread_create.c:301 #6 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115 Thread 7 (Thread 0x7f26d8dfa700 (LWP 14445)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162 #1 0x00000000007f07f3 in Wait (this=0x1adfb48) at common/Cond.h:55 #2 Finisher::finisher_thread_entry (this=0x1adfb48) at common/Finisher.cc:80 #3 0x00007f26e7845851 in start_thread (arg=0x7f26d8dfa700) at pthread_create.c:301 #4 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115 Thread 6 (Thread 0x7f26dbfff700 (LWP 14443)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162 #1 0x00000000006f9292 in Wait (this=0x1aed200) at common/Cond.h:55 #2 FileJournal::write_thread_entry (this=0x1aed200) at os/FileJournal.cc:1108 #3 0x000000000063ebdd in FileJournal::Writer::entry (this=) at os/FileJournal.h:293 #4 0x00007f26e7845851 in start_thread (arg=0x7f26dbfff700) at pthread_create.c:301 #5 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115 Thread 5 (Thread 0x7f26e2583700 (LWP 14440)): #0 __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:136 #1 0x00007f26e7847388 in _L_lock_854 () from /lib64/libpthread-2.12.so #2 0x00007f26e7847257 in __pthread_mutex_lock (mutex=0x1d754e8) at pthread_mutex_lock.c:61 #3 0x00000000008876d1 in Mutex::Lock (this=0x1d754d8, no_lockdep=) at common/Mutex.cc:90 #4 0x000000000068fcce in PG::lock (this=0x1d75360, no_lockdep=) at osd/PG.cc:97 #5 0x00000000005eb35b in OSD::_lookup_lock_pg (this=0x1ae2420, pgid=...) at osd/OSD.cc:1435 #6 0x0000000000622822 in OSD::sched_scrub (this=0x1ae2420) at osd/OSD.cc:3631 #7 0x0000000000636500 in OSD::tick (this=0x1ae2420) at osd/OSD.cc:2261 #8 0x0000000000854f23 in SafeTimer::timer_thread (this=0x1ae2490) at common/Timer.cc:105 #9 0x00000000008570ed in SafeTimerThread::entry (this=) at common/Timer.cc:38 #10 0x00007f26e7845851 in start_thread (arg=0x7f26e2583700) at pthread_create.c:301 #11 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115 Thread 4 (Thread 0x7f26d97fb700 (LWP 14435)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162 #1 0x00000000008aa331 in Wait (this=0x1ac4e60) at common/Cond.h:55 #2 SimpleMessenger::reaper_entry (this=0x1ac4e60) at msg/SimpleMessenger.cc:206 #3 0x00000000008ace7d in SimpleMessenger::ReaperThread::entry (this=) at msg/SimpleMessenger.h:412 #4 0x00007f26e7845851 in start_thread (arg=0x7f26d97fb700) at pthread_create.c:301 #5 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115 Thread 3 (Thread 0x7f26dabfd700 (LWP 14437)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162 #1 0x00000000008aa331 in Wait (this=0x1ac75a0) at common/Cond.h:55 #2 SimpleMessenger::reaper_entry (this=0x1ac75a0) at msg/SimpleMessenger.cc:206 #3 0x00000000008ace7d in SimpleMessenger::ReaperThread::entry (this=) at msg/SimpleMessenger.h:412 #4 0x00007f26e7845851 in start_thread (arg=0x7f26dabfd700) at pthread_create.c:301 #5 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115 Thread 2 (Thread 0x7f26da1fc700 (LWP 14436)): #0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162 #1 0x00000000008aa331 in Wait (this=0x1ac6760) at common/Cond.h:55 #2 SimpleMessenger::reaper_entry (this=0x1ac6760) at msg/SimpleMessenger.cc:206 #3 0x00000000008ace7d in SimpleMessenger::ReaperThread::entry (this=) at msg/SimpleMessenger.h:412 #4 0x00007f26e7845851 in start_thread (arg=0x7f26da1fc700) at pthread_create.c:301 #5 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115 Thread 1 (Thread 0x7f26e4d87700 (LWP 14374)): #0 0x00007f26e784d3cb in raise (sig=6) at ../nptl/sysdeps/unix/sysv/linux/pt-raise.c:42 #1 0x0000000000791287 in reraise_fatal (signum=6) at global/signal_handler.cc:58 #2 handle_fatal_signal (signum=6) at global/signal_handler.cc:104 #3 #4 0x00007f26e65198a5 in raise (sig=6) at ../nptl/sysdeps/unix/sysv/linux/raise.c:64 #5 0x00007f26e651b085 in abort () at abort.c:92 #6 0x00007f26e6dd2a5d in __gnu_cxx::__verbose_terminate_handler () at ../../../../libstdc++-v3/libsupc++/vterminate.cc:93 #7 0x00007f26e6dd0be6 in __cxxabiv1::__terminate (handler=) at ../../../../libstdc++-v3/libsupc++/eh_terminate.cc:38 #8 0x00007f26e6dd0c13 in std::terminate () at ../../../../libstdc++-v3/libsupc++/eh_terminate.cc:48 #9 0x00007f26e6dd0d0e in __cxxabiv1::__cxa_throw (obj=0x7f25c4006b60, tinfo=, dest=) at ../../../../libstdc++-v3/libsupc++/eh_throw.cc:83 #10 0x000000000083d699 in ceph::__ceph_assert_fail (assertion=0x1abed50 "\001", file=0x7f25c4006ae0 "\001", line=79, func=0x973180 "bool ceph::HeartbeatMap::_check(ceph::heartbeat_handle_d*, const char*, time_t)") at common/assert.cc:77 #11 0x00000000007c988b in ceph::HeartbeatMap::_check (this=, h=, who=0x97306c "is_healthy", now=) at common/HeartbeatMap.cc:79 #12 0x00000000007ca1b7 in ceph::HeartbeatMap::is_healthy (this=0x1ac1210) at common/HeartbeatMap.cc:130 #13 0x00000000007ca6a8 in ceph::HeartbeatMap::check_touch_file (this=0x1ac1210) at common/HeartbeatMap.cc:141 #14 0x000000000085377d in CephContextServiceThread::entry (this=0x1adedf0) at common/ceph_context.cc:68 #15 0x00007f26e7845851 in start_thread (arg=0x7f26e4d87700) at pthread_create.c:301 #16 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115