Project

General

Profile

Bug #4116 » gdb.txt

full back trace of all threads. - Evan Felix, 02/18/2013 10:44 AM

 

Thread 84 (Thread 0x7f26e017a700 (LWP 14823)):
#0 0x00007f26e65c5a63 in __poll (fds=<value optimized out>, nfds=<value optimized out>, timeout=<value optimized out>)
at ../sysdeps/unix/sysv/linux/poll.c:87
#1 0x00000000008c6d99 in Pipe::tcp_read_wait (this=<value optimized out>) at msg/Pipe.cc:2048
#2 0x00000000008cc8ad in Pipe::tcp_read (this=0x7f26b02b74f0, buf=<value optimized out>, len=1) at msg/Pipe.cc:2021
#3 0x00000000008dc33e in Pipe::reader (this=0x7f26b02b74f0) at msg/Pipe.cc:1276
#4 0x00000000008defed in Pipe::Reader::entry (this=<value optimized out>) at msg/Pipe.h:47
#5 0x00007f26e7845851 in start_thread (arg=0x7f26e017a700) at pthread_create.c:301
#6 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115

Thread 83 (Thread 0x7f26bc4f9700 (LWP 14824)):
#0 0x00007f26e65c5a63 in __poll (fds=<value optimized out>, nfds=<value optimized out>, timeout=<value optimized out>)
at ../sysdeps/unix/sysv/linux/poll.c:87
#1 0x00000000008c6d99 in Pipe::tcp_read_wait (this=<value optimized out>) at msg/Pipe.cc:2048
#2 0x00000000008cc8ad in Pipe::tcp_read (this=0x7f26b05234e0, buf=<value optimized out>, len=1) at msg/Pipe.cc:2021
#3 0x00000000008dc33e in Pipe::reader (this=0x7f26b05234e0) at msg/Pipe.cc:1276
#4 0x00000000008defed in Pipe::Reader::entry (this=<value optimized out>) at msg/Pipe.h:47
#5 0x00007f26e7845851 in start_thread (arg=0x7f26bc4f9700) at pthread_create.c:301
#6 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115

Thread 82 (Thread 0x7f26aa1fc700 (LWP 14755)):
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1 0x00000000008dc626 in Wait (this=0x7f248da2cdb0) at common/Cond.h:55
#2 Pipe::reader (this=0x7f248da2cdb0) at msg/Pipe.cc:1267
#3 0x00000000008defed in Pipe::Reader::entry (this=<value optimized out>) at msg/Pipe.h:47
#4 0x00007f26e7845851 in start_thread (arg=0x7f26aa1fc700) at pthread_create.c:301
#5 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115

Thread 81 (Thread 0x7f26aa0fb700 (LWP 14753)):
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1 0x00000000008d27a9 in Wait (this=0x7f26b085d6c0) at common/Cond.h:55
#2 Pipe::writer (this=0x7f26b085d6c0) at msg/Pipe.cc:1527
#3 0x00000000008defcd in Pipe::Writer::entry (this=<value optimized out>) at msg/Pipe.h:59
#4 0x00007f26e7845851 in start_thread (arg=0x7f26aa0fb700) at pthread_create.c:301
#5 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115

Thread 80 (Thread 0x7f26e1b82700 (LWP 14441)):
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1 0x0000000000854c67 in Wait (this=0x1ae3fc8) at common/Cond.h:55
#2 SafeTimer::timer_thread (this=0x1ae3fc8) at common/Timer.cc:113
#3 0x00000000008570ed in SafeTimerThread::entry (this=<value optimized out>) at common/Timer.cc:38
#4 0x00007f26e7845851 in start_thread (arg=0x7f26e1b82700) at pthread_create.c:301
#5 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115

Thread 79 (Thread 0x7f26e2f84700 (LWP 14439)):
#0 0x00007f26e65c7d03 in select () at ../sysdeps/unix/syscall-template.S:82
#1 0x00000000007917f1 in SignalHandler::entry (this=0x1afea80) at global/signal_handler.cc:210
#2 0x00007f26e7845851 in start_thread (arg=0x7f26e2f84700) at pthread_create.c:301
#3 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115

Thread 78 (Thread 0x7f26e83cb700 (LWP 14794)):
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1 0x00000000008d27a9 in Wait (this=0x7f263400acd0) at common/Cond.h:55
#2 Pipe::writer (this=0x7f263400acd0) at msg/Pipe.cc:1527
#3 0x00000000008defcd in Pipe::Writer::entry (this=<value optimized out>) at msg/Pipe.h:59
#4 0x00007f26e7845851 in start_thread (arg=0x7f26e83cb700) at pthread_create.c:301
#5 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115

Thread 77 (Thread 0x7f26e83df760 (LWP 14371)):
#0 0x00007f26e78460ad in pthread_join (threadid=139804540925696, thread_return=0x0) at pthread_join.c:89
#1 0x0000000000858482 in Thread::join (this=0x1ac5070, prval=<value optimized out>) at common/Thread.cc:121
#2 0x00000000008a837a in SimpleMessenger::wait (this=0x1ac4e60) at msg/SimpleMessenger.cc:500
#3 0x0000000000560a2a in main (argc=<value optimized out>, argv=<value optimized out>) at ceph_osd.cc:446

Thread 76 (Thread 0x7f26b75fe700 (LWP 14467)):
#0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:216
#1 0x000000000084954f in WaitUntil (this=0x1ae2878, wt=0x1b3c8a0) at common/Cond.h:71
#2 WaitInterval (this=0x1ae2878, wt=0x1b3c8a0) at common/Cond.h:79
#3 ThreadPool::worker (this=0x1ae2878, wt=0x1b3c8a0) at common/WorkQueue.cc:137
#4 0x000000000084c680 in ThreadPool::WorkThread::entry (this=<value optimized out>) at common/WorkQueue.h:316
#5 0x00007f26e7845851 in start_thread (arg=0x7f26b75fe700) at pthread_create.c:301
#6 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115

Thread 75 (Thread 0x7f26b6bfd700 (LWP 14468)):
#0 encode (this=0xbfb08000, bl=...) at include/encoding.h:103
#1 pg_log_entry_t::encode (this=0xbfb08000, bl=...) at osd/osd_types.cc:1698
#2 0x000000000069eca4 in encode (this=0x1d75360, t=...) at osd/osd_types.h:1324
#3 PG::write_log (this=0x1d75360, t=...) at osd/PG.cc:2364
#4 0x0000000000620643 in OSD::process_peering_events (this=0x1ae2420, pgs=std::list = {...}, handle=...) at osd/OSD.cc:6207
#5 0x000000000065d5b0 in OSD::PeeringWQ::_process (this=<value optimized out>, pgs=std::list = {...}, handle=<value optimized out>) at osd/OSD.h:744
#6 0x0000000000849899 in ThreadPool::worker (this=0x1ae2878, wt=0x1b3f280) at common/WorkQueue.cc:119
#7 0x000000000084c680 in ThreadPool::WorkThread::entry (this=<value optimized out>) at common/WorkQueue.h:316
#8 0x00007f26e7845851 in start_thread (arg=0x7f26b6bfd700) at pthread_create.c:301
#9 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115

Thread 74 (Thread 0x7f26a97f2700 (LWP 14793)):
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1 0x00000000008d27a9 in Wait (this=0x7f26340107d0) at common/Cond.h:55
#2 Pipe::writer (this=0x7f26340107d0) at msg/Pipe.cc:1527
#3 0x00000000008defcd in Pipe::Writer::entry (this=<value optimized out>) at msg/Pipe.h:59
#4 0x00007f26e7845851 in start_thread (arg=0x7f26a97f2700) at pthread_create.c:301
#5 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115

Thread 73 (Thread 0x7f26e4386700 (LWP 14375)):
#0 0x00007f26e65c5a63 in __poll (fds=<value optimized out>, nfds=<value optimized out>, timeout=<value optimized out>)
at ../sysdeps/unix/sysv/linux/poll.c:87
#1 0x00000000007eda7a in AdminSocket::entry (this=0x1ac1110) at common/admin_socket.cc:228
#2 0x00007f26e7845851 in start_thread (arg=0x7f26e4386700) at pthread_create.c:301
#3 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115

Thread 72 (Thread 0x7f26d81f7700 (LWP 14799)):
#0 0x00007f26e65c5a63 in __poll (fds=<value optimized out>, nfds=<value optimized out>, timeout=<value optimized out>)
at ../sysdeps/unix/sysv/linux/poll.c:87
#1 0x00000000008c6d99 in Pipe::tcp_read_wait (this=<value optimized out>) at msg/Pipe.cc:2048
#2 0x00000000008cc8ad in Pipe::tcp_read (this=0x7f26340107d0, buf=<value optimized out>, len=1) at msg/Pipe.cc:2021
#3 0x00000000008dc33e in Pipe::reader (this=0x7f26340107d0) at msg/Pipe.cc:1276
#4 0x00000000008defed in Pipe::Reader::entry (this=<value optimized out>) at msg/Pipe.h:47
#5 0x00007f26e7845851 in start_thread (arg=0x7f26d81f7700) at pthread_create.c:301
#6 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115

Thread 71 (Thread 0x7f26a9ffa700 (LWP 14771)):
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1 0x00000000008dc626 in Wait (this=0x7f26b0987820) at common/Cond.h:55
#2 Pipe::reader (this=0x7f26b0987820) at msg/Pipe.cc:1267
#3 0x00000000008defed in Pipe::Reader::entry (this=<value optimized out>) at msg/Pipe.h:47
#4 0x00007f26e7845851 in start_thread (arg=0x7f26a9ffa700) at pthread_create.c:301
#5 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115

Thread 70 (Thread 0x7f26c2df5700 (LWP 14804)):
#0 0x00007f26e65c5a63 in __poll (fds=<value optimized out>, nfds=<value optimized out>, timeout=<value optimized out>)
at ../sysdeps/unix/sysv/linux/poll.c:87
#1 0x00000000008c6d99 in Pipe::tcp_read_wait (this=<value optimized out>) at msg/Pipe.cc:2048
#2 0x00000000008cc8ad in Pipe::tcp_read (this=0x7f26340050e0, buf=<value optimized out>, len=1) at msg/Pipe.cc:2021
#3 0x00000000008dc33e in Pipe::reader (this=0x7f26340050e0) at msg/Pipe.cc:1276
#4 0x00000000008defed in Pipe::Reader::entry (this=<value optimized out>) at msg/Pipe.h:47
#5 0x00007f26e7845851 in start_thread (arg=0x7f26c2df5700) at pthread_create.c:301
#6 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115

Thread 69 (Thread 0x7f26bc2f7700 (LWP 14788)):
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1 0x00000000008d27a9 in Wait (this=0x7f26340021b0) at common/Cond.h:55
#2 Pipe::writer (this=0x7f26340021b0) at msg/Pipe.cc:1527
#3 0x00000000008defcd in Pipe::Writer::entry (this=<value optimized out>) at msg/Pipe.h:59
#4 0x00007f26e7845851 in start_thread (arg=0x7f26bc2f7700) at pthread_create.c:301
#5 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115

Thread 68 (Thread 0x7f26a8be6700 (LWP 14822)):
#0 0x00007f26e65c5a63 in __poll (fds=<value optimized out>, nfds=<value optimized out>, timeout=<value optimized out>)
at ../sysdeps/unix/sysv/linux/poll.c:87
#1 0x00000000008c6d99 in Pipe::tcp_read_wait (this=<value optimized out>) at msg/Pipe.cc:2048
#2 0x00000000008cc8ad in Pipe::tcp_read (this=0x7f26b08f54e0, buf=<value optimized out>, len=1) at msg/Pipe.cc:2021
#3 0x00000000008dc33e in Pipe::reader (this=0x7f26b08f54e0) at msg/Pipe.cc:1276
#4 0x00000000008defed in Pipe::Reader::entry (this=<value optimized out>) at msg/Pipe.h:47
#5 0x00007f26e7845851 in start_thread (arg=0x7f26a8be6700) at pthread_create.c:301
#6 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115

Thread 67 (Thread 0x7f26a99f4700 (LWP 14770)):
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1 0x00000000008dc626 in Wait (this=0x7f26b0b49510) at common/Cond.h:55
#2 Pipe::reader (this=0x7f26b0b49510) at msg/Pipe.cc:1267
#3 0x00000000008defed in Pipe::Reader::entry (this=<value optimized out>) at msg/Pipe.h:47
#4 0x00007f26e7845851 in start_thread (arg=0x7f26a99f4700) at pthread_create.c:301
#5 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115

Thread 66 (Thread 0x7f26c2bf3700 (LWP 14805)):
#0 0x00007f26e65c5a63 in __poll (fds=<value optimized out>, nfds=<value optimized out>, timeout=<value optimized out>)
at ../sysdeps/unix/sysv/linux/poll.c:87
#1 0x00000000008c6d99 in Pipe::tcp_read_wait (this=<value optimized out>) at msg/Pipe.cc:2048
#2 0x00000000008cc8ad in Pipe::tcp_read (this=0x7f263400dcf0, buf=<value optimized out>, len=1) at msg/Pipe.cc:2021
#3 0x00000000008dc33e in Pipe::reader (this=0x7f263400dcf0) at msg/Pipe.cc:1276
#4 0x00000000008defed in Pipe::Reader::entry (this=<value optimized out>) at msg/Pipe.h:47
#5 0x00007f26e7845851 in start_thread (arg=0x7f26c2bf3700) at pthread_create.c:301
#6 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115

Thread 65 (Thread 0x7f26a9df8700 (LWP 14825)):
#0 0x00007f26e65c5a63 in __poll (fds=<value optimized out>, nfds=<value optimized out>, timeout=<value optimized out>)
at ../sysdeps/unix/sysv/linux/poll.c:87
#1 0x00000000008c6d99 in Pipe::tcp_read_wait (this=<value optimized out>) at msg/Pipe.cc:2048
#2 0x00000000008cc8ad in Pipe::tcp_read (this=0x7f26b01a0d50, buf=<value optimized out>, len=1) at msg/Pipe.cc:2021
#3 0x00000000008dc33e in Pipe::reader (this=0x7f26b01a0d50) at msg/Pipe.cc:1276
#4 0x00000000008defed in Pipe::Reader::entry (this=<value optimized out>) at msg/Pipe.h:47
#5 0x00007f26e7845851 in start_thread (arg=0x7f26a9df8700) at pthread_create.c:301
#6 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115

Thread 64 (Thread 0x7f26bc1f6700 (LWP 14826)):
#0 0x00007f26e65c5a63 in __poll (fds=<value optimized out>, nfds=<value optimized out>, timeout=<value optimized out>)
at ../sysdeps/unix/sysv/linux/poll.c:87
#1 0x00000000008c6d99 in Pipe::tcp_read_wait (this=<value optimized out>) at msg/Pipe.cc:2048
#2 0x00000000008cc8ad in Pipe::tcp_read (this=0x7f26b0c22200, buf=<value optimized out>, len=1) at msg/Pipe.cc:2021
#3 0x00000000008dc33e in Pipe::reader (this=0x7f26b0c22200) at msg/Pipe.cc:1276
#4 0x00000000008defed in Pipe::Reader::entry (this=<value optimized out>) at msg/Pipe.h:47
#5 0x00007f26e7845851 in start_thread (arg=0x7f26bc1f6700) at pthread_create.c:301
#6 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115

Thread 63 (Thread 0x7f26a87e2700 (LWP 14786)):
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1 0x00000000008d27a9 in Wait (this=0x7f2634011a80) at common/Cond.h:55
#2 Pipe::writer (this=0x7f2634011a80) at msg/Pipe.cc:1527
#3 0x00000000008defcd in Pipe::Writer::entry (this=<value optimized out>) at msg/Pipe.h:59
#4 0x00007f26e7845851 in start_thread (arg=0x7f26a87e2700) at pthread_create.c:301
#5 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115

Thread 62 (Thread 0x7f26a8ee9700 (LWP 14757)):
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1 0x00000000008d27a9 in Wait (this=0x7f26b01a0d50) at common/Cond.h:55
#2 Pipe::writer (this=0x7f26b01a0d50) at msg/Pipe.cc:1527
#3 0x00000000008defcd in Pipe::Writer::entry (this=<value optimized out>) at msg/Pipe.h:59
#4 0x00007f26e7845851 in start_thread (arg=0x7f26a8ee9700) at pthread_create.c:301
#5 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115

Thread 61 (Thread 0x7f26e5eb7700 (LWP 14372)):
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1 0x0000000000894cfb in ceph::log::Log::entry (this=0x1ac0bc0) at log/Log.cc:323
#2 0x00007f26e7845851 in start_thread (arg=0x7f26e5eb7700) at pthread_create.c:301
#3 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115

Thread 60 (Thread 0x7f26c2cf4700 (LWP 14803)):
#0 0x00007f26e65c5a63 in __poll (fds=<value optimized out>, nfds=<value optimized out>, timeout=<value optimized out>)
at ../sysdeps/unix/sysv/linux/poll.c:87
#1 0x00000000008c6d99 in Pipe::tcp_read_wait (this=<value optimized out>) at msg/Pipe.cc:2048
#2 0x00000000008cc8ad in Pipe::tcp_read (this=0x7f2634011a80, buf=<value optimized out>, len=1) at msg/Pipe.cc:2021
#3 0x00000000008dc33e in Pipe::reader (this=0x7f2634011a80) at msg/Pipe.cc:1276
#4 0x00000000008defed in Pipe::Reader::entry (this=<value optimized out>) at msg/Pipe.h:47
#5 0x00007f26e7845851 in start_thread (arg=0x7f26c2cf4700) at pthread_create.c:301
#6 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115

Thread 59 (Thread 0x7f26e067f700 (LWP 14795)):
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1 0x00000000008d27a9 in Wait (this=0x7f263400dcf0) at common/Cond.h:55
#2 Pipe::writer (this=0x7f263400dcf0) at msg/Pipe.cc:1527
#3 0x00000000008defcd in Pipe::Writer::entry (this=<value optimized out>) at msg/Pipe.h:59
#4 0x00007f26e7845851 in start_thread (arg=0x7f26e067f700) at pthread_create.c:301
#5 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115

Thread 58 (Thread 0x7f26cc2f8700 (LWP 14801)):
#0 0x00007f26e65c5a63 in __poll (fds=<value optimized out>, nfds=<value optimized out>, timeout=<value optimized out>)
at ../sysdeps/unix/sysv/linux/poll.c:87
#1 0x00000000008c6d99 in Pipe::tcp_read_wait (this=<value optimized out>) at msg/Pipe.cc:2048
#2 0x00000000008cc8ad in Pipe::tcp_read (this=0x7f263400e3b0, buf=<value optimized out>, len=1) at msg/Pipe.cc:2021
#3 0x00000000008dc33e in Pipe::reader (this=0x7f263400e3b0) at msg/Pipe.cc:1276
#4 0x00000000008defed in Pipe::Reader::entry (this=<value optimized out>) at msg/Pipe.h:47
#5 0x00007f26e7845851 in start_thread (arg=0x7f26cc2f8700) at pthread_create.c:301
#6 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115

Thread 57 (Thread 0x7f26e047d700 (LWP 14761)):
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1 0x00000000008d27a9 in Wait (this=0x7f26b0b49510) at common/Cond.h:55
#2 Pipe::writer (this=0x7f26b0b49510) at msg/Pipe.cc:1527
#3 0x00000000008defcd in Pipe::Writer::entry (this=<value optimized out>) at msg/Pipe.h:59
#4 0x00007f26e7845851 in start_thread (arg=0x7f26e047d700) at pthread_create.c:301
#5 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115

Thread 56 (Thread 0x7f26bc3f8700 (LWP 14766)):
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1 0x00000000008dc626 in Wait (this=0x7f26b057da90) at common/Cond.h:55
#2 Pipe::reader (this=0x7f26b057da90) at msg/Pipe.cc:1267
#3 0x00000000008defed in Pipe::Reader::entry (this=<value optimized out>) at msg/Pipe.h:47
#4 0x00007f26e7845851 in start_thread (arg=0x7f26bc3f8700) at pthread_create.c:301
#5 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115

Thread 55 (Thread 0x7f26cc3f9700 (LWP 14800)):
#0 0x00007f26e65c5a63 in __poll (fds=<value optimized out>, nfds=<value optimized out>, timeout=<value optimized out>)
at ../sysdeps/unix/sysv/linux/poll.c:87
#1 0x00000000008c6d99 in Pipe::tcp_read_wait (this=<value optimized out>) at msg/Pipe.cc:2048
#2 0x00000000008cc8ad in Pipe::tcp_read (this=0x7f2634002440, buf=<value optimized out>, len=1) at msg/Pipe.cc:2021
#3 0x00000000008dc33e in Pipe::reader (this=0x7f2634002440) at msg/Pipe.cc:1276
#4 0x00000000008defed in Pipe::Reader::entry (this=<value optimized out>) at msg/Pipe.h:47
#5 0x00007f26e7845851 in start_thread (arg=0x7f26cc3f9700) at pthread_create.c:301
#6 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115

Thread 54 (Thread 0x7f26abfff700 (LWP 14472)):
#0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:216
#1 0x00000000006250e6 in WaitUntil (this=0x1ae2420) at common/Cond.h:71
#2 WaitInterval (this=0x1ae2420) at common/Cond.h:79
#3 OSD::heartbeat_entry (this=0x1ae2420) at osd/OSD.cc:2108
#4 0x000000000065d73d in OSD::T_Heartbeat::entry (this=<value optimized out>) at osd/OSD.h:572
#5 0x00007f26e7845851 in start_thread (arg=0x7f26abfff700) at pthread_create.c:301
#6 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115

Thread 53 (Thread 0x7f26d83f9700 (LWP 14798)):
#0 0x00007f26e65c5a63 in __poll (fds=<value optimized out>, nfds=<value optimized out>, timeout=<value optimized out>)
at ../sysdeps/unix/sysv/linux/poll.c:87
#1 0x00000000008c6d99 in Pipe::tcp_read_wait (this=<value optimized out>) at msg/Pipe.cc:2048
#2 0x00000000008cc8ad in Pipe::tcp_read (this=0x7f26340021b0, buf=<value optimized out>, len=1) at msg/Pipe.cc:2021
#3 0x00000000008dc33e in Pipe::reader (this=0x7f26340021b0) at msg/Pipe.cc:1276
#4 0x00000000008defed in Pipe::Reader::entry (this=<value optimized out>) at msg/Pipe.h:47
#5 0x00007f26e7845851 in start_thread (arg=0x7f26d83f9700) at pthread_create.c:301
#6 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115

Thread 52 (Thread 0x7f26c29f1700 (LWP 14806)):
#0 0x00007f26e65c5a63 in __poll (fds=<value optimized out>, nfds=<value optimized out>, timeout=<value optimized out>)
at ../sysdeps/unix/sysv/linux/poll.c:87
#1 0x00000000008c6d99 in Pipe::tcp_read_wait (this=<value optimized out>) at msg/Pipe.cc:2048
#2 0x00000000008cc8ad in Pipe::tcp_read (this=0x7f2634010350, buf=<value optimized out>, len=1) at msg/Pipe.cc:2021
#3 0x00000000008dc33e in Pipe::reader (this=0x7f2634010350) at msg/Pipe.cc:1276
#4 0x00000000008defed in Pipe::Reader::entry (this=<value optimized out>) at msg/Pipe.h:47
#5 0x00007f26e7845851 in start_thread (arg=0x7f26c29f1700) at pthread_create.c:301
#6 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115

Thread 51 (Thread 0x7f26cc1f7700 (LWP 14802)):
#0 0x00007f26e65c5a63 in __poll (fds=<value optimized out>, nfds=<value optimized out>, timeout=<value optimized out>)
at ../sysdeps/unix/sysv/linux/poll.c:87
#1 0x00000000008c6d99 in Pipe::tcp_read_wait (this=<value optimized out>) at msg/Pipe.cc:2048
#2 0x00000000008cc8ad in Pipe::tcp_read (this=0x7f263400acd0, buf=<value optimized out>, len=1) at msg/Pipe.cc:2021
#3 0x00000000008dc33e in Pipe::reader (this=0x7f263400acd0) at msg/Pipe.cc:1276
#4 0x00000000008defed in Pipe::Reader::entry (this=<value optimized out>) at msg/Pipe.h:47
#5 0x00007f26e7845851 in start_thread (arg=0x7f26cc1f7700) at pthread_create.c:301
#6 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115

Thread 50 (Thread 0x7f26b61fc700 (LWP 14469)):
#0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:216
#1 0x000000000084954f in WaitUntil (this=0x1ae2a00, wt=0x1b3f460) at common/Cond.h:71
#2 WaitInterval (this=0x1ae2a00, wt=0x1b3f460) at common/Cond.h:79
#3 ThreadPool::worker (this=0x1ae2a00, wt=0x1b3f460) at common/WorkQueue.cc:137
#4 0x000000000084c680 in ThreadPool::WorkThread::entry (this=<value optimized out>) at common/WorkQueue.h:316
#5 0x00007f26e7845851 in start_thread (arg=0x7f26b61fc700) at pthread_create.c:301
#6 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115

Thread 49 (Thread 0x7f262bbfb700 (LWP 14792)):
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1 0x00000000008d27a9 in Wait (this=0x7f2634002440) at common/Cond.h:55
#2 Pipe::writer (this=0x7f2634002440) at msg/Pipe.cc:1527
#3 0x00000000008defcd in Pipe::Writer::entry (this=<value optimized out>) at msg/Pipe.h:59
#4 0x00007f26e7845851 in start_thread (arg=0x7f262bbfb700) at pthread_create.c:301
#5 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115

Thread 48 (Thread 0x7f26c2af2700 (LWP 14758)):
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1 0x00000000008d27a9 in Wait (this=0x7f26b08f54e0) at common/Cond.h:55
#2 Pipe::writer (this=0x7f26b08f54e0) at msg/Pipe.cc:1527
#3 0x00000000008defcd in Pipe::Writer::entry (this=<value optimized out>) at msg/Pipe.h:59
#4 0x00007f26e7845851 in start_thread (arg=0x7f26c2af2700) at pthread_create.c:301
#5 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115

Thread 47 (Thread 0x7f26e057e700 (LWP 14797)):
#0 0x00007f26e65c5a63 in __poll (fds=<value optimized out>, nfds=<value optimized out>, timeout=<value optimized out>)
at ../sysdeps/unix/sysv/linux/poll.c:87
#1 0x00000000008c6d99 in Pipe::tcp_read_wait (this=<value optimized out>) at msg/Pipe.cc:2048
#2 0x00000000008cc8ad in Pipe::tcp_read (this=0x7f2634006b20, buf=<value optimized out>, len=1) at msg/Pipe.cc:2021
#3 0x00000000008dc33e in Pipe::reader (this=0x7f2634006b20) at msg/Pipe.cc:1276
#4 0x00000000008defed in Pipe::Reader::entry (this=<value optimized out>) at msg/Pipe.h:47
#5 0x00007f26e7845851 in start_thread (arg=0x7f26e057e700) at pthread_create.c:301
#6 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115

Thread 46 (Thread 0x7f26e037c700 (LWP 14796)):
#0 0x00007f26e65c5a63 in __poll (fds=<value optimized out>, nfds=<value optimized out>, timeout=<value optimized out>)
at ../sysdeps/unix/sysv/linux/poll.c:87
#1 0x00000000008c6d99 in Pipe::tcp_read_wait (this=<value optimized out>) at msg/Pipe.cc:2048
#2 0x00000000008cc8ad in Pipe::tcp_read (this=0x7f2634000e70, buf=<value optimized out>, len=1) at msg/Pipe.cc:2021
#3 0x00000000008dc33e in Pipe::reader (this=0x7f2634000e70) at msg/Pipe.cc:1276
#4 0x00000000008defed in Pipe::Reader::entry (this=<value optimized out>) at msg/Pipe.h:47
#5 0x00007f26e7845851 in start_thread (arg=0x7f26e037c700) at pthread_create.c:301
#6 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115

Thread 45 (Thread 0x7f262b9f9700 (LWP 14762)):
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1 0x00000000008d27a9 in Wait (this=0x7f26b00d6c40) at common/Cond.h:55
#2 Pipe::writer (this=0x7f26b00d6c40) at msg/Pipe.cc:1527
#3 0x00000000008defcd in Pipe::Writer::entry (this=<value optimized out>) at msg/Pipe.h:59
#4 0x00007f26e7845851 in start_thread (arg=0x7f262b9f9700) at pthread_create.c:301
#5 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115

Thread 44 (Thread 0x7f26c43f9700 (LWP 14750)):
#0 0x00007f26e65c5a63 in __poll (fds=<value optimized out>, nfds=<value optimized out>, timeout=<value optimized out>)
at ../sysdeps/unix/sysv/linux/poll.c:87
#1 0x00000000008b4e38 in Accepter::entry (this=0x1ac7660) at msg/Accepter.cc:198
#2 0x00007f26e7845851 in start_thread (arg=0x7f26c43f9700) at pthread_create.c:301
#3 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115

Thread 43 (Thread 0x7f26c2ff7700 (LWP 14791)):
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1 0x00000000008d27a9 in Wait (this=0x7f263400e3b0) at common/Cond.h:55
#2 Pipe::writer (this=0x7f263400e3b0) at msg/Pipe.cc:1527
#3 0x00000000008defcd in Pipe::Writer::entry (this=<value optimized out>) at msg/Pipe.h:59
#4 0x00007f26e7845851 in start_thread (arg=0x7f26c2ff7700) at pthread_create.c:301
#5 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115

Thread 42 (Thread 0x7f262bdfd700 (LWP 14787)):
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1 0x00000000008d27a9 in Wait (this=0x7f2634006b20) at common/Cond.h:55
#2 Pipe::writer (this=0x7f2634006b20) at msg/Pipe.cc:1527
#3 0x00000000008defcd in Pipe::Writer::entry (this=<value optimized out>) at msg/Pipe.h:59
#4 0x00007f26e7845851 in start_thread (arg=0x7f262bdfd700) at pthread_create.c:301
#5 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115

Thread 41 (Thread 0x7f26b41f7700 (LWP 14752)):
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1 0x00000000008d27a9 in Wait (this=0x7f248da2cdb0) at common/Cond.h:55
#2 Pipe::writer (this=0x7f248da2cdb0) at msg/Pipe.cc:1527
#3 0x00000000008defcd in Pipe::Writer::entry (this=<value optimized out>) at msg/Pipe.h:59
#4 0x00007f26e7845851 in start_thread (arg=0x7f26b41f7700) at pthread_create.c:301
#5 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115

Thread 40 (Thread 0x7f262bfff700 (LWP 14785)):
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1 0x00000000008d27a9 in Wait (this=0x7f2634000e70) at common/Cond.h:55
#2 Pipe::writer (this=0x7f2634000e70) at msg/Pipe.cc:1527
#3 0x00000000008defcd in Pipe::Writer::entry (this=<value optimized out>) at msg/Pipe.h:59
#4 0x00007f26e7845851 in start_thread (arg=0x7f262bfff700) at pthread_create.c:301
#5 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115

Thread 39 (Thread 0x7f262bcfc700 (LWP 14790)):
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1 0x00000000008d27a9 in Wait (this=0x7f26340050e0) at common/Cond.h:55
#2 Pipe::writer (this=0x7f26340050e0) at msg/Pipe.cc:1527
#3 0x00000000008defcd in Pipe::Writer::entry (this=<value optimized out>) at msg/Pipe.h:59
#4 0x00007f26e7845851 in start_thread (arg=0x7f262bcfc700) at pthread_create.c:301
#5 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115

Thread 38 (Thread 0x7f26e0780700 (LWP 14748)):
#0 0x00007f26e65c5a63 in __poll (fds=<value optimized out>, nfds=<value optimized out>, timeout=<value optimized out>)
at ../sysdeps/unix/sysv/linux/poll.c:87
#1 0x00000000008c6d99 in Pipe::tcp_read_wait (this=<value optimized out>) at msg/Pipe.cc:2048
#2 0x00000000008cc8ad in Pipe::tcp_read (this=0x7f263400c690, buf=<value optimized out>, len=1) at msg/Pipe.cc:2021
#3 0x00000000008dc33e in Pipe::reader (this=0x7f263400c690) at msg/Pipe.cc:1276
#4 0x00000000008defed in Pipe::Reader::entry (this=<value optimized out>) at msg/Pipe.h:47
#5 0x00007f26e7845851 in start_thread (arg=0x7f26e0780700) at pthread_create.c:301
#6 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115

Thread 37 (Thread 0x7f26a8ce7700 (LWP 14772)):
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1 0x00000000008dc626 in Wait (this=0x7f26b00d6c40) at common/Cond.h:55
#2 Pipe::reader (this=0x7f26b00d6c40) at msg/Pipe.cc:1267
#3 0x00000000008defed in Pipe::Reader::entry (this=<value optimized out>) at msg/Pipe.h:47
#4 0x00007f26e7845851 in start_thread (arg=0x7f26a8ce7700) at pthread_create.c:301
#5 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115

Thread 36 (Thread 0x7f26aabfd700 (LWP 14474)):
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1 0x0000000000854c67 in Wait (this=0x1ae3ea0) at common/Cond.h:55
#2 SafeTimer::timer_thread (this=0x1ae3ea0) at common/Timer.cc:113
#3 0x00000000008570ed in SafeTimerThread::entry (this=<value optimized out>) at common/Timer.cc:38
#4 0x00007f26e7845851 in start_thread (arg=0x7f26aabfd700) at pthread_create.c:301
#5 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115

Thread 35 (Thread 0x7f26d82f8700 (LWP 14764)):
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1 0x00000000008dc626 in Wait (this=0x7f26b085d6c0) at common/Cond.h:55
#2 Pipe::reader (this=0x7f26b085d6c0) at msg/Pipe.cc:1267
#3 0x00000000008defed in Pipe::Reader::entry (this=<value optimized out>) at msg/Pipe.h:47
#4 0x00007f26e7845851 in start_thread (arg=0x7f26d82f8700) at pthread_create.c:301
#5 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115

Thread 34 (Thread 0x7f26a8de8700 (LWP 14789)):
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1 0x00000000008d27a9 in Wait (this=0x7f2634010350) at common/Cond.h:55
#2 Pipe::writer (this=0x7f2634010350) at msg/Pipe.cc:1527
#3 0x00000000008defcd in Pipe::Writer::entry (this=<value optimized out>) at msg/Pipe.h:59
#4 0x00007f26e7845851 in start_thread (arg=0x7f26a8de8700) at pthread_create.c:301
#5 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115

Thread 33 (Thread 0x7f26a88e3700 (LWP 14756)):
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1 0x00000000008d27a9 in Wait (this=0x7f26b057da90) at common/Cond.h:55
#2 Pipe::writer (this=0x7f26b057da90) at msg/Pipe.cc:1527
#3 0x00000000008defcd in Pipe::Writer::entry (this=<value optimized out>) at msg/Pipe.h:59
#4 0x00007f26e7845851 in start_thread (arg=0x7f26a88e3700) at pthread_create.c:301
#5 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115

Thread 32 (Thread 0x7f26ab5fe700 (LWP 14473)):
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1 0x00000000007f07f3 in Wait (this=0x1ae40f0) at common/Cond.h:55
#2 Finisher::finisher_thread_entry (this=0x1ae40f0) at common/Finisher.cc:80
#3 0x00007f26e7845851 in start_thread (arg=0x7f26ab5fe700) at pthread_create.c:301
#4 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115

Thread 31 (Thread 0x7f26b43f9700 (LWP 14763)):
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1 0x00000000008d27a9 in Wait (this=0x7f26b0c22200) at common/Cond.h:55
#2 Pipe::writer (this=0x7f26b0c22200) at msg/Pipe.cc:1527
#3 0x00000000008defcd in Pipe::Writer::entry (this=<value optimized out>) at msg/Pipe.h:59
#4 0x00007f26e7845851 in start_thread (arg=0x7f26b43f9700) at pthread_create.c:301
#5 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115

Thread 30 (Thread 0x7f26c39f8700 (LWP 14465)):
#0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:216
#1 0x000000000085528e in WaitUntil (this=0x7fff437c4b58) at common/Cond.h:71
#2 SafeTimer::timer_thread (this=0x7fff437c4b58) at common/Timer.cc:115
#3 0x00000000008570ed in SafeTimerThread::entry (this=<value optimized out>) at common/Timer.cc:38
#4 0x00007f26e7845851 in start_thread (arg=0x7f26c39f8700) at pthread_create.c:301
#5 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115

Thread 29 (Thread 0x7f26a85e0700 (LWP 14754)):
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1 0x00000000008d27a9 in Wait (this=0x7f26b0987820) at common/Cond.h:55
#2 Pipe::writer (this=0x7f26b0987820) at msg/Pipe.cc:1527
#3 0x00000000008defcd in Pipe::Writer::entry (this=<value optimized out>) at msg/Pipe.h:59
#4 0x00007f26e7845851 in start_thread (arg=0x7f26a85e0700) at pthread_create.c:301
#5 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115

Thread 28 (Thread 0x7f26ce1fc700 (LWP 14449)):
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1 0x00000000007f07f3 in Wait (this=0x1ae0408) at common/Cond.h:55
#2 Finisher::finisher_thread_entry (this=0x1ae0408) at common/Finisher.cc:80
#3 0x00007f26e7845851 in start_thread (arg=0x7f26ce1fc700) at pthread_create.c:301
#4 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115

Thread 27 (Thread 0x7f26c4dfa700 (LWP 14462)):
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1 0x00000000008bb708 in Wait (this=0x1ac7688) at common/Cond.h:55
#2 DispatchQueue::entry (this=0x1ac7688) at msg/DispatchQueue.cc:117
#3 0x00000000008abcad in DispatchQueue::DispatchThread::entry (this=<value optimized out>) at msg/DispatchQueue.h:85
#4 0x00007f26e7845851 in start_thread (arg=0x7f26c4dfa700) at pthread_create.c:301
#5 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115

Thread 26 (Thread 0x7f26a81dc700 (LWP 14760)):
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1 0x00000000008d27a9 in Wait (this=0x7f26b05234e0) at common/Cond.h:55
#2 Pipe::writer (this=0x7f26b05234e0) at msg/Pipe.cc:1527
#3 0x00000000008defcd in Pipe::Writer::entry (this=<value optimized out>) at msg/Pipe.h:59
#4 0x00007f26e7845851 in start_thread (arg=0x7f26a81dc700) at pthread_create.c:301
#5 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115

Thread 25 (Thread 0x7f26c61fc700 (LWP 14749)):
#0 0x00007f26e65c5a63 in __poll (fds=<value optimized out>, nfds=<value optimized out>, timeout=<value optimized out>)
at ../sysdeps/unix/sysv/linux/poll.c:87
#1 0x00000000008b4e38 in Accepter::entry (this=0x1ac5a40) at msg/Accepter.cc:198
#2 0x00007f26e7845851 in start_thread (arg=0x7f26c61fc700) at pthread_create.c:301
#3 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115

Thread 24 (Thread 0x7f26a98f3700 (LWP 14759)):
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1 0x00000000008d27a9 in Wait (this=0x7f26b02b74f0) at common/Cond.h:55
#2 Pipe::writer (this=0x7f26b02b74f0) at msg/Pipe.cc:1527
#3 0x00000000008defcd in Pipe::Writer::entry (this=<value optimized out>) at msg/Pipe.h:59
#4 0x00007f26e7845851 in start_thread (arg=0x7f26a98f3700) at pthread_create.c:301
#5 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115

Thread 23 (Thread 0x7f26c75fe700 (LWP 14458)):
#0 0x00007f26e65c5a63 in __poll (fds=<value optimized out>, nfds=<value optimized out>, timeout=<value optimized out>)
at ../sysdeps/unix/sysv/linux/poll.c:87
#1 0x00000000008b4e38 in Accepter::entry (this=0x1ac4f20) at msg/Accepter.cc:198
#2 0x00007f26e7845851 in start_thread (arg=0x7f26c75fe700) at pthread_create.c:301
#3 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115

Thread 22 (Thread 0x7f262bafa700 (LWP 14747)):
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1 0x00000000008d27a9 in Wait (this=0x7f263400c690) at common/Cond.h:55
#2 Pipe::writer (this=0x7f263400c690) at msg/Pipe.cc:1527
#3 0x00000000008defcd in Pipe::Writer::entry (this=<value optimized out>) at msg/Pipe.h:59
#4 0x00007f26e7845851 in start_thread (arg=0x7f262bafa700) at pthread_create.c:301
#5 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115

Thread 21 (Thread 0x7f26b57fb700 (LWP 14470)):
#0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:216
#1 0x000000000084954f in WaitUntil (this=0x1ae2b88, wt=0x1b3f640) at common/Cond.h:71
#2 WaitInterval (this=0x1ae2b88, wt=0x1b3f640) at common/Cond.h:79
#3 ThreadPool::worker (this=0x1ae2b88, wt=0x1b3f640) at common/WorkQueue.cc:137
#4 0x000000000084c680 in ThreadPool::WorkThread::entry (this=<value optimized out>) at common/WorkQueue.h:316
#5 0x00007f26e7845851 in start_thread (arg=0x7f26b57fb700) at pthread_create.c:301
#6 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115

Thread 20 (Thread 0x7f26ccdfa700 (LWP 14451)):
#0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:216
#1 0x000000000085528e in WaitUntil (this=0x1ae0208) at common/Cond.h:71
#2 SafeTimer::timer_thread (this=0x1ae0208) at common/Timer.cc:115
#3 0x00000000008570ed in SafeTimerThread::entry (this=<value optimized out>) at common/Timer.cc:38
#4 0x00007f26e7845851 in start_thread (arg=0x7f26ccdfa700) at pthread_create.c:301
#5 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115

Thread 19 (Thread 0x7f26c6bfd700 (LWP 14459)):
#0 __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:136
#1 0x00007f26e7847388 in _L_lock_854 () from /lib64/libpthread-2.12.so
#2 0x00007f26e7847257 in __pthread_mutex_lock (mutex=0x1ae2448) at pthread_mutex_lock.c:61
#3 0x00000000008876d1 in Mutex::Lock (this=0x1ae2438, no_lockdep=<value optimized out>) at common/Mutex.cc:90
#4 0x0000000000636d97 in OSD::ms_dispatch (this=0x1ae2420, m=0x7f260c091010) at osd/OSD.cc:3250
#5 0x00000000008bb899 in ms_deliver_dispatch (this=0x1ac5a68) at msg/Messenger.h:553
#6 DispatchQueue::entry (this=0x1ac5a68) at msg/DispatchQueue.cc:107
#7 0x00000000008abcad in DispatchQueue::DispatchThread::entry (this=<value optimized out>) at msg/DispatchQueue.h:85
#8 0x00007f26e7845851 in start_thread (arg=0x7f26c6bfd700) at pthread_create.c:301
#9 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115

Thread 18 (Thread 0x7f26b7fff700 (LWP 14466)):
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1 0x00000000007f07f3 in Wait (this=0x7fff437c4c20) at common/Cond.h:55
#2 Finisher::finisher_thread_entry (this=0x7fff437c4c20) at common/Finisher.cc:80
#3 0x00007f26e7845851 in start_thread (arg=0x7f26b7fff700) at pthread_create.c:301
#4 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115

Thread 17 (Thread 0x7f26db5fe700 (LWP 14444)):
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1 0x00000000006f668b in Wait (this=0x1aed200) at common/Cond.h:55
#2 FileJournal::write_finish_thread_entry (this=0x1aed200) at os/FileJournal.cc:1316
#3 0x000000000063ebfd in FileJournal::WriteFinisher::entry (this=<value optimized out>) at os/FileJournal.h:303
#4 0x00007f26e7845851 in start_thread (arg=0x7f26db5fe700) at pthread_create.c:301
#5 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115

Thread 16 (Thread 0x7f26c7fff700 (LWP 14457)):
#0 __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:136
#1 0x00007f26e7847388 in _L_lock_854 () from /lib64/libpthread-2.12.so
#2 0x00007f26e7847257 in __pthread_mutex_lock (mutex=0x1ae2448) at pthread_mutex_lock.c:61
#3 0x00000000008876d1 in Mutex::Lock (this=0x1ae2438, no_lockdep=<value optimized out>) at common/Mutex.cc:90
#4 0x0000000000636d97 in OSD::ms_dispatch (this=0x1ae2420, m=0x7f2624003270) at osd/OSD.cc:3250
#5 0x00000000008bb899 in ms_deliver_dispatch (this=0x1ac4f48) at msg/Messenger.h:553
#6 DispatchQueue::entry (this=0x1ac4f48) at msg/DispatchQueue.cc:107
#7 0x00000000008abcad in DispatchQueue::DispatchThread::entry (this=<value optimized out>) at msg/DispatchQueue.h:85
#8 0x00007f26e7845851 in start_thread (arg=0x7f26c7fff700) at pthread_create.c:301
#9 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115

Thread 15 (Thread 0x7f26c57fb700 (LWP 14461)):
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1 0x00000000008bb708 in Wait (this=0x1ac6848) at common/Cond.h:55
#2 DispatchQueue::entry (this=0x1ac6848) at msg/DispatchQueue.cc:117
#3 0x00000000008abcad in DispatchQueue::DispatchThread::entry (this=<value optimized out>) at msg/DispatchQueue.h:85
#4 0x00007f26e7845851 in start_thread (arg=0x7f26c57fb700) at pthread_create.c:301
#5 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115

Thread 14 (Thread 0x7f26e3985700 (LWP 14438)):
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1 0x00000000008aa331 in Wait (this=0x1ac5980) at common/Cond.h:55
#2 SimpleMessenger::reaper_entry (this=0x1ac5980) at msg/SimpleMessenger.cc:206
#3 0x00000000008ace7d in SimpleMessenger::ReaperThread::entry (this=<value optimized out>) at msg/SimpleMessenger.h:412
#4 0x00007f26e7845851 in start_thread (arg=0x7f26e3985700) at pthread_create.c:301
#5 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115

Thread 13 (Thread 0x7f26b4dfa700 (LWP 14471)):
#0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:216
#1 0x000000000084954f in WaitUntil (this=0x1ae2d10, wt=0x1b3f7a0) at common/Cond.h:71
#2 WaitInterval (this=0x1ae2d10, wt=0x1b3f7a0) at common/Cond.h:79
#3 ThreadPool::worker (this=0x1ae2d10, wt=0x1b3f7a0) at common/WorkQueue.cc:137
#4 0x000000000084c680 in ThreadPool::WorkThread::entry (this=<value optimized out>) at common/WorkQueue.h:316
#5 0x00007f26e7845851 in start_thread (arg=0x7f26b4dfa700) at pthread_create.c:301
#6 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115

Thread 12 (Thread 0x7f26cd7fb700 (LWP 14450)):
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1 0x00000000007f07f3 in Wait (this=0x1adffd8) at common/Cond.h:55
#2 Finisher::finisher_thread_entry (this=0x1adffd8) at common/Finisher.cc:80
#3 0x00007f26e7845851 in start_thread (arg=0x7f26cd7fb700) at pthread_create.c:301
#4 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115

Thread 11 (Thread 0x7f26cf5fe700 (LWP 14447)):
#0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:216
#1 0x000000000084954f in WaitUntil (this=0x1ae0538, wt=0x1ae5600) at common/Cond.h:71
#2 WaitInterval (this=0x1ae0538, wt=0x1ae5600) at common/Cond.h:79
#3 ThreadPool::worker (this=0x1ae0538, wt=0x1ae5600) at common/WorkQueue.cc:137
#4 0x000000000084c680 in ThreadPool::WorkThread::entry (this=<value optimized out>) at common/WorkQueue.h:316
#5 0x00007f26e7845851 in start_thread (arg=0x7f26cf5fe700) at pthread_create.c:301
#6 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115

Thread 10 (Thread 0x7f26cebfd700 (LWP 14448)):
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1 0x0000000000705bdc in Wait (this=0x1adfb30) at common/Cond.h:55
#2 FileStore::flusher_entry (this=0x1adfb30) at os/FileStore.cc:3264
#3 0x00000000007356ed in FileStore::FlusherThread::entry (this=<value optimized out>) at os/FileStore.h:262
#4 0x00007f26e7845851 in start_thread (arg=0x7f26cebfd700) at pthread_create.c:301
#5 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115

Thread 9 (Thread 0x7f26cffff700 (LWP 14446)):
#0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:216
#1 0x000000000084954f in WaitUntil (this=0x1ae0538, wt=0x1aea420) at common/Cond.h:71
#2 WaitInterval (this=0x1ae0538, wt=0x1aea420) at common/Cond.h:79
#3 ThreadPool::worker (this=0x1ae0538, wt=0x1aea420) at common/WorkQueue.cc:137
#4 0x000000000084c680 in ThreadPool::WorkThread::entry (this=<value optimized out>) at common/WorkQueue.h:316
#5 0x00007f26e7845851 in start_thread (arg=0x7f26cffff700) at pthread_create.c:301
#6 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115

Thread 8 (Thread 0x7f26e1181700 (LWP 14442)):
#0 pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:216
#1 0x0000000000719e7e in WaitUntil (this=0x1adfb30) at common/Cond.h:71
#2 WaitInterval (this=0x1adfb30) at common/Cond.h:79
#3 FileStore::sync_entry (this=0x1adfb30) at os/FileStore.cc:3304
#4 0x000000000073632d in FileStore::SyncThread::entry (this=<value optimized out>) at os/FileStore.h:115
#5 0x00007f26e7845851 in start_thread (arg=0x7f26e1181700) at pthread_create.c:301
#6 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115

Thread 7 (Thread 0x7f26d8dfa700 (LWP 14445)):
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1 0x00000000007f07f3 in Wait (this=0x1adfb48) at common/Cond.h:55
#2 Finisher::finisher_thread_entry (this=0x1adfb48) at common/Finisher.cc:80
#3 0x00007f26e7845851 in start_thread (arg=0x7f26d8dfa700) at pthread_create.c:301
#4 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115

Thread 6 (Thread 0x7f26dbfff700 (LWP 14443)):
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1 0x00000000006f9292 in Wait (this=0x1aed200) at common/Cond.h:55
#2 FileJournal::write_thread_entry (this=0x1aed200) at os/FileJournal.cc:1108
#3 0x000000000063ebdd in FileJournal::Writer::entry (this=<value optimized out>) at os/FileJournal.h:293
#4 0x00007f26e7845851 in start_thread (arg=0x7f26dbfff700) at pthread_create.c:301
#5 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115

Thread 5 (Thread 0x7f26e2583700 (LWP 14440)):
#0 __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:136
#1 0x00007f26e7847388 in _L_lock_854 () from /lib64/libpthread-2.12.so
#2 0x00007f26e7847257 in __pthread_mutex_lock (mutex=0x1d754e8) at pthread_mutex_lock.c:61
#3 0x00000000008876d1 in Mutex::Lock (this=0x1d754d8, no_lockdep=<value optimized out>) at common/Mutex.cc:90
#4 0x000000000068fcce in PG::lock (this=0x1d75360, no_lockdep=<value optimized out>) at osd/PG.cc:97
#5 0x00000000005eb35b in OSD::_lookup_lock_pg (this=0x1ae2420, pgid=...) at osd/OSD.cc:1435
#6 0x0000000000622822 in OSD::sched_scrub (this=0x1ae2420) at osd/OSD.cc:3631
#7 0x0000000000636500 in OSD::tick (this=0x1ae2420) at osd/OSD.cc:2261
#8 0x0000000000854f23 in SafeTimer::timer_thread (this=0x1ae2490) at common/Timer.cc:105
#9 0x00000000008570ed in SafeTimerThread::entry (this=<value optimized out>) at common/Timer.cc:38
#10 0x00007f26e7845851 in start_thread (arg=0x7f26e2583700) at pthread_create.c:301
#11 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115

Thread 4 (Thread 0x7f26d97fb700 (LWP 14435)):
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1 0x00000000008aa331 in Wait (this=0x1ac4e60) at common/Cond.h:55
#2 SimpleMessenger::reaper_entry (this=0x1ac4e60) at msg/SimpleMessenger.cc:206
#3 0x00000000008ace7d in SimpleMessenger::ReaperThread::entry (this=<value optimized out>) at msg/SimpleMessenger.h:412
#4 0x00007f26e7845851 in start_thread (arg=0x7f26d97fb700) at pthread_create.c:301
#5 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115

Thread 3 (Thread 0x7f26dabfd700 (LWP 14437)):
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1 0x00000000008aa331 in Wait (this=0x1ac75a0) at common/Cond.h:55
#2 SimpleMessenger::reaper_entry (this=0x1ac75a0) at msg/SimpleMessenger.cc:206
#3 0x00000000008ace7d in SimpleMessenger::ReaperThread::entry (this=<value optimized out>) at msg/SimpleMessenger.h:412
#4 0x00007f26e7845851 in start_thread (arg=0x7f26dabfd700) at pthread_create.c:301
#5 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115

Thread 2 (Thread 0x7f26da1fc700 (LWP 14436)):
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1 0x00000000008aa331 in Wait (this=0x1ac6760) at common/Cond.h:55
#2 SimpleMessenger::reaper_entry (this=0x1ac6760) at msg/SimpleMessenger.cc:206
#3 0x00000000008ace7d in SimpleMessenger::ReaperThread::entry (this=<value optimized out>) at msg/SimpleMessenger.h:412
#4 0x00007f26e7845851 in start_thread (arg=0x7f26da1fc700) at pthread_create.c:301
#5 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115

Thread 1 (Thread 0x7f26e4d87700 (LWP 14374)):
#0 0x00007f26e784d3cb in raise (sig=6) at ../nptl/sysdeps/unix/sysv/linux/pt-raise.c:42
#1 0x0000000000791287 in reraise_fatal (signum=6) at global/signal_handler.cc:58
#2 handle_fatal_signal (signum=6) at global/signal_handler.cc:104
#3 <signal handler called>
#4 0x00007f26e65198a5 in raise (sig=6) at ../nptl/sysdeps/unix/sysv/linux/raise.c:64
#5 0x00007f26e651b085 in abort () at abort.c:92
#6 0x00007f26e6dd2a5d in __gnu_cxx::__verbose_terminate_handler () at ../../../../libstdc++-v3/libsupc++/vterminate.cc:93
#7 0x00007f26e6dd0be6 in __cxxabiv1::__terminate (handler=<value optimized out>) at ../../../../libstdc++-v3/libsupc++/eh_terminate.cc:38
#8 0x00007f26e6dd0c13 in std::terminate () at ../../../../libstdc++-v3/libsupc++/eh_terminate.cc:48
#9 0x00007f26e6dd0d0e in __cxxabiv1::__cxa_throw (obj=0x7f25c4006b60, tinfo=<value optimized out>, dest=<value optimized out>)
at ../../../../libstdc++-v3/libsupc++/eh_throw.cc:83
#10 0x000000000083d699 in ceph::__ceph_assert_fail (assertion=0x1abed50 "\001", file=0x7f25c4006ae0 "\001", line=79,
func=0x973180 "bool ceph::HeartbeatMap::_check(ceph::heartbeat_handle_d*, const char*, time_t)") at common/assert.cc:77
#11 0x00000000007c988b in ceph::HeartbeatMap::_check (this=<value optimized out>, h=<value optimized out>, who=0x97306c "is_healthy",
now=<value optimized out>) at common/HeartbeatMap.cc:79
#12 0x00000000007ca1b7 in ceph::HeartbeatMap::is_healthy (this=0x1ac1210) at common/HeartbeatMap.cc:130
#13 0x00000000007ca6a8 in ceph::HeartbeatMap::check_touch_file (this=0x1ac1210) at common/HeartbeatMap.cc:141
#14 0x000000000085377d in CephContextServiceThread::entry (this=0x1adedf0) at common/ceph_context.cc:68
#15 0x00007f26e7845851 in start_thread (arg=0x7f26e4d87700) at pthread_create.c:301
#16 0x00007f26e65cf11d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:115
(7-7/10)