Ceph

have seen several runs hang on osd shutdown. trusty. valgrind. here is the gdb thread dump:

(gdb) thr app all bt

Thread 104 (Thread 7501):
#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1  0x0000000000b06d58 in Wait (mutex=..., this=0x256f3d68) at ./common/Cond.h:55
#2  Pipe::writer (this=0x256f3b80) at msg/Pipe.cc:1723
#3  0x0000000000b0ad5d in Pipe::Writer::entry (this=<optimized out>) at msg/Pipe.h:59
#4  0x00000000052aa182 in start_thread (arg=0x23dfa700) at pthread_create.c:312
#5  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 103 (Thread 7483):
#0  0x0000000006f39ddd in poll () at ../sysdeps/unix/syscall-template.S:81
#1  0x0000000000af36ff in poll (__timeout=<optimized out>, __nfds=1, __fds=0x233efad0) at /usr/include/x86_64-linux-gnu/bits/poll2.h:46
#2  Pipe::tcp_read_wait (this=this@entry=0x25ee0d00) at msg/Pipe.cc:2277
#3  0x0000000000af3a18 in Pipe::tcp_read (this=this@entry=0x25ee0d00, buf=buf@entry=0x233efc7e "\377", len=len@entry=1) at msg/Pipe.cc:2250
#4  0x0000000000b005a5 in Pipe::reader (this=0x25ee0d00) at msg/Pipe.cc:1412
#5  0x0000000000b0a5fd in Pipe::Reader::entry (this=<optimized out>) at msg/Pipe.h:47
#6  0x00000000052aa182 in start_thread (arg=0x233f0700) at pthread_create.c:312
#7  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 102 (Thread 7500):
#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1  0x0000000000b06d58 in Wait (mutex=..., this=0x256f30e8) at ./common/Cond.h:55
#2  Pipe::writer (this=0x256f2f00) at msg/Pipe.cc:1723
#3  0x0000000000b0ad5d in Pipe::Writer::entry (this=<optimized out>) at msg/Pipe.h:59
#4  0x00000000052aa182 in start_thread (arg=0x24804700) at pthread_create.c:312
#5  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 101 (Thread 7315):
#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1  0x0000000000b06d58 in Wait (mutex=..., this=0x2384c768) at ./common/Cond.h:55
#2  Pipe::writer (this=0x2384c580) at msg/Pipe.cc:1723
#3  0x0000000000b0ad5d in Pipe::Writer::entry (this=<optimized out>) at msg/Pipe.h:59
#4  0x00000000052aa182 in start_thread (arg=0x22be9700) at pthread_create.c:312
#5  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 100 (Thread 7457):
#0  0x0000000006f39ddd in poll () at ../sysdeps/unix/syscall-template.S:81
#1  0x0000000000af36ff in poll (__timeout=<optimized out>, __nfds=1, __fds=0x24c05ad0) at /usr/include/x86_64-linux-gnu/bits/poll2.h:46
#2  Pipe::tcp_read_wait (this=this@entry=0x4125980) at msg/Pipe.cc:2277
#3  0x0000000000af3a18 in Pipe::tcp_read (this=this@entry=0x4125980, buf=buf@entry=0x24c05c7e "\377", len=len@entry=1) at msg/Pipe.cc:2250
#4  0x0000000000b005a5 in Pipe::reader (this=0x4125980) at msg/Pipe.cc:1412
#5  0x0000000000b0a5fd in Pipe::Reader::entry (this=<optimized out>) at msg/Pipe.h:47
#6  0x00000000052aa182 in start_thread (arg=0x24c06700) at pthread_create.c:312
#7  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 99 (Thread 6664):
#0  0x0000000006f39ddd in poll () at ../sysdeps/unix/syscall-template.S:81
#1  0x0000000000af36ff in poll (__timeout=<optimized out>, __nfds=1, __fds=0x24904ad0) at /usr/include/x86_64-linux-gnu/bits/poll2.h:46
#2  Pipe::tcp_read_wait (this=this@entry=0x2384a280) at msg/Pipe.cc:2277
#3  0x0000000000af3a18 in Pipe::tcp_read (this=this@entry=0x2384a280, buf=buf@entry=0x24904c7e "\377", len=len@entry=1) at msg/Pipe.cc:2250
#4  0x0000000000b005a5 in Pipe::reader (this=0x2384a280) at msg/Pipe.cc:1412
#5  0x0000000000b0a5fd in Pipe::Reader::entry (this=<optimized out>) at msg/Pipe.h:47
#6  0x00000000052aa182 in start_thread (arg=0x24905700) at pthread_create.c:312
#7  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 98 (Thread 7484):
#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1  0x0000000000b06d58 in Wait (mutex=..., this=0x25ee0ee8) at ./common/Cond.h:55
#2  Pipe::writer (this=0x25ee0d00) at msg/Pipe.cc:1723
#3  0x0000000000b0ad5d in Pipe::Writer::entry (this=<optimized out>) at msg/Pipe.h:59
#4  0x00000000052aa182 in start_thread (arg=0x25918700) at pthread_create.c:312
#5  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 97 (Thread 6662):
#0  0x0000000006f39ddd in poll () at ../sysdeps/unix/syscall-template.S:81
#1  0x0000000000af36ff in poll (__timeout=<optimized out>, __nfds=1, __fds=0x24702ad0) at /usr/include/x86_64-linux-gnu/bits/poll2.h:46
#2  Pipe::tcp_read_wait (this=this@entry=0x2384b680) at msg/Pipe.cc:2277
#3  0x0000000000af3a18 in Pipe::tcp_read (this=this@entry=0x2384b680, buf=buf@entry=0x24702c7e "\377", len=len@entry=1) at msg/Pipe.cc:2250
#4  0x0000000000b005a5 in Pipe::reader (this=0x2384b680) at msg/Pipe.cc:1412
#5  0x0000000000b0a5fd in Pipe::Reader::entry (this=<optimized out>) at msg/Pipe.h:47
#6  0x00000000052aa182 in start_thread (arg=0x24703700) at pthread_create.c:312
#7  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 96 (Thread 6661):
#0  0x0000000006f39ddd in poll () at ../sysdeps/unix/syscall-template.S:81
#1  0x0000000000af36ff in poll (__timeout=<optimized out>, __nfds=1, __fds=0x24601ad0) at /usr/include/x86_64-linux-gnu/bits/poll2.h:46
#2  Pipe::tcp_read_wait (this=this@entry=0x2384a000) at msg/Pipe.cc:2277
#3  0x0000000000af3a18 in Pipe::tcp_read (this=this@entry=0x2384a000, buf=buf@entry=0x24601c7e "\377", len=len@entry=1) at msg/Pipe.cc:2250
#4  0x0000000000b005a5 in Pipe::reader (this=0x2384a000) at msg/Pipe.cc:1412
#5  0x0000000000b0a5fd in Pipe::Reader::entry (this=<optimized out>) at msg/Pipe.h:47
#6  0x00000000052aa182 in start_thread (arg=0x24602700) at pthread_create.c:312
#7  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 95 (Thread 7596):
#0  0x0000000006f39ddd in poll () at ../sysdeps/unix/syscall-template.S:81
#1  0x0000000000af36ff in poll (__timeout=<optimized out>, __nfds=1, __fds=0x24500ad0) at /usr/include/x86_64-linux-gnu/bits/poll2.h:46
#2  Pipe::tcp_read_wait (this=this@entry=0x26994780) at msg/Pipe.cc:2277
#3  0x0000000000af3a18 in Pipe::tcp_read (this=this@entry=0x26994780, buf=buf@entry=0x24500c7e "\377", len=len@entry=1) at msg/Pipe.cc:2250
#4  0x0000000000b005a5 in Pipe::reader (this=0x26994780) at msg/Pipe.cc:1412
#5  0x0000000000b0a5fd in Pipe::Reader::entry (this=<optimized out>) at msg/Pipe.h:47
---Type <return> to continue, or q <return> to quit---
#6  0x00000000052aa182 in start_thread (arg=0x24501700) at pthread_create.c:312
#7  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 94 (Thread 6656):
#0  0x0000000006f39ddd in poll () at ../sysdeps/unix/syscall-template.S:81
#1  0x0000000000af36ff in poll (__timeout=<optimized out>, __nfds=1, __fds=0x242fead0) at /usr/include/x86_64-linux-gnu/bits/poll2.h:46
#2  Pipe::tcp_read_wait (this=this@entry=0x2384ac80) at msg/Pipe.cc:2277
#3  0x0000000000af3a18 in Pipe::tcp_read (this=this@entry=0x2384ac80, buf=buf@entry=0x242fec7e "\377", len=len@entry=1) at msg/Pipe.cc:2250
#4  0x0000000000b005a5 in Pipe::reader (this=0x2384ac80) at msg/Pipe.cc:1412
#5  0x0000000000b0a5fd in Pipe::Reader::entry (this=<optimized out>) at msg/Pipe.h:47
#6  0x00000000052aa182 in start_thread (arg=0x242ff700) at pthread_create.c:312
#7  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 93 (Thread 6655):
#0  0x0000000006f39ddd in poll () at ../sysdeps/unix/syscall-template.S:81
#1  0x0000000000af36ff in poll (__timeout=<optimized out>, __nfds=1, __fds=0x241fdad0) at /usr/include/x86_64-linux-gnu/bits/poll2.h:46
#2  Pipe::tcp_read_wait (this=this@entry=0x2384b400) at msg/Pipe.cc:2277
#3  0x0000000000af3a18 in Pipe::tcp_read (this=this@entry=0x2384b400, buf=buf@entry=0x241fdc7e "\377", len=len@entry=1) at msg/Pipe.cc:2250
#4  0x0000000000b005a5 in Pipe::reader (this=0x2384b400) at msg/Pipe.cc:1412
#5  0x0000000000b0a5fd in Pipe::Reader::entry (this=<optimized out>) at msg/Pipe.h:47
#6  0x00000000052aa182 in start_thread (arg=0x241fe700) at pthread_create.c:312
#7  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 92 (Thread 7593):
#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1  0x0000000000b06d58 in Wait (mutex=..., this=0x26994968) at ./common/Cond.h:55
#2  Pipe::writer (this=0x26994780) at msg/Pipe.cc:1723
#3  0x0000000000b0ad5d in Pipe::Writer::entry (this=<optimized out>) at msg/Pipe.h:59
#4  0x00000000052aa182 in start_thread (arg=0x240fd700) at pthread_create.c:312
#5  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 91 (Thread 6653):
#0  0x0000000006f39ddd in poll () at ../sysdeps/unix/syscall-template.S:81
#1  0x0000000000af36ff in poll (__timeout=<optimized out>, __nfds=1, __fds=0x23ffbad0) at /usr/include/x86_64-linux-gnu/bits/poll2.h:46
#2  Pipe::tcp_read_wait (this=this@entry=0x2384a500) at msg/Pipe.cc:2277
#3  0x0000000000af3a18 in Pipe::tcp_read (this=this@entry=0x2384a500, buf=buf@entry=0x23ffbc7e "\377", len=len@entry=1) at msg/Pipe.cc:2250
#4  0x0000000000b005a5 in Pipe::reader (this=0x2384a500) at msg/Pipe.cc:1412
#5  0x0000000000b0a5fd in Pipe::Reader::entry (this=<optimized out>) at msg/Pipe.h:47
#6  0x00000000052aa182 in start_thread (arg=0x23ffc700) at pthread_create.c:312
#7  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 90 (Thread 7595):
#0  0x0000000006f39ddd in poll () at ../sysdeps/unix/syscall-template.S:81
#1  0x0000000000af36ff in poll (__timeout=<optimized out>, __nfds=1, __fds=0x25b19ad0) at /usr/include/x86_64-linux-gnu/bits/poll2.h:46
#2  Pipe::tcp_read_wait (this=this@entry=0x26994500) at msg/Pipe.cc:2277
#3  0x0000000000af3a18 in Pipe::tcp_read (this=this@entry=0x26994500, buf=buf@entry=0x25b19c7e "\377", len=len@entry=1) at msg/Pipe.cc:2250
#4  0x0000000000b005a5 in Pipe::reader (this=0x26994500) at msg/Pipe.cc:1412
#5  0x0000000000b0a5fd in Pipe::Reader::entry (this=<optimized out>) at msg/Pipe.h:47
#6  0x00000000052aa182 in start_thread (arg=0x25b1a700) at pthread_create.c:312
#7  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 89 (Thread 7594):
#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1  0x0000000000b06d58 in Wait (mutex=..., this=0x269946e8) at ./common/Cond.h:55
#2  Pipe::writer (this=0x26994500) at msg/Pipe.cc:1723
#3  0x0000000000b0ad5d in Pipe::Writer::entry (this=<optimized out>) at msg/Pipe.h:59
#4  0x00000000052aa182 in start_thread (arg=0x25a19700) at pthread_create.c:312
#5  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 88 (Thread 6646):
#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1  0x0000000000b06d58 in Wait (mutex=..., this=0x2384b5e8) at ./common/Cond.h:55
#2  Pipe::writer (this=0x2384b400) at msg/Pipe.cc:1723
#3  0x0000000000b0ad5d in Pipe::Writer::entry (this=<optimized out>) at msg/Pipe.h:59
#4  0x00000000052aa182 in start_thread (arg=0x23cf9700) at pthread_create.c:312
#5  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 87 (Thread 6645):
#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1  0x0000000000b06d58 in Wait (mutex=..., this=0x2384b868) at ./common/Cond.h:55
#2  Pipe::writer (this=0x2384b680) at msg/Pipe.cc:1723
#3  0x0000000000b0ad5d in Pipe::Writer::entry (this=<optimized out>) at msg/Pipe.h:59
#4  0x00000000052aa182 in start_thread (arg=0x23bf8700) at pthread_create.c:312
#5  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 86 (Thread 7507):
#0  0x0000000006f39ddd in poll () at ../sysdeps/unix/syscall-template.S:81
#1  0x0000000000af36ff in poll (__timeout=<optimized out>, __nfds=1, __fds=0x243ffad0) at /usr/include/x86_64-linux-gnu/bits/poll2.h:46
#2  Pipe::tcp_read_wait (this=this@entry=0x256f2f00) at msg/Pipe.cc:2277
#3  0x0000000000af3a18 in Pipe::tcp_read (this=this@entry=0x256f2f00, buf=buf@entry=0x243ffc7e "\377", len=len@entry=1) at msg/Pipe.cc:2250
#4  0x0000000000b005a5 in Pipe::reader (this=0x256f2f00) at msg/Pipe.cc:1412
#5  0x0000000000b0a5fd in Pipe::Reader::entry (this=<optimized out>) at msg/Pipe.h:47
#6  0x00000000052aa182 in start_thread (arg=0x24400700) at pthread_create.c:312
#7  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 85 (Thread 7505):
#0  0x0000000006f39ddd in poll () at ../sysdeps/unix/syscall-template.S:81
#1  0x0000000000af36ff in poll (__timeout=<optimized out>, __nfds=1, __fds=0x23af6ad0) at /usr/include/x86_64-linux-gnu/bits/poll2.h:46
#2  Pipe::tcp_read_wait (this=this@entry=0x256f3b80) at msg/Pipe.cc:2277
#3  0x0000000000af3a18 in Pipe::tcp_read (this=this@entry=0x256f3b80, buf=buf@entry=0x23af6c7e "\377", len=len@entry=1) at msg/Pipe.cc:2250
---Type <return> to continue, or q <return> to quit---
#4  0x0000000000b005a5 in Pipe::reader (this=0x256f3b80) at msg/Pipe.cc:1412
#5  0x0000000000b0a5fd in Pipe::Reader::entry (this=<optimized out>) at msg/Pipe.h:47
#6  0x00000000052aa182 in start_thread (arg=0x23af7700) at pthread_create.c:312
#7  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 84 (Thread 6642):
#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1  0x0000000000b06d58 in Wait (mutex=..., this=0x2384ae68) at ./common/Cond.h:55
#2  Pipe::writer (this=0x2384ac80) at msg/Pipe.cc:1723
#3  0x0000000000b0ad5d in Pipe::Writer::entry (this=<optimized out>) at msg/Pipe.h:59
#4  0x00000000052aa182 in start_thread (arg=0x22dea700) at pthread_create.c:312
#5  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 83 (Thread 6635):
#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1  0x0000000000b06d58 in Wait (mutex=..., this=0x4125168) at ./common/Cond.h:55
#2  Pipe::writer (this=0x4124f80) at msg/Pipe.cc:1723
#3  0x0000000000b0ad5d in Pipe::Writer::entry (this=<optimized out>) at msg/Pipe.h:59
#4  0x00000000052aa182 in start_thread (arg=0x236f3700) at pthread_create.c:312
#5  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 82 (Thread 7314):
#0  0x0000000006f39ddd in poll () at ../sysdeps/unix/syscall-template.S:81
#1  0x0000000000af36ff in poll (__timeout=<optimized out>, __nfds=1, __fds=0x24e07ad0) at /usr/include/x86_64-linux-gnu/bits/poll2.h:46
#2  Pipe::tcp_read_wait (this=this@entry=0x2384c580) at msg/Pipe.cc:2277
#3  0x0000000000af3a18 in Pipe::tcp_read (this=this@entry=0x2384c580, buf=buf@entry=0x24e07c7e "\377", len=len@entry=1) at msg/Pipe.cc:2250
#4  0x0000000000b005a5 in Pipe::reader (this=0x2384c580) at msg/Pipe.cc:1412
#5  0x0000000000b0a5fd in Pipe::Reader::entry (this=<optimized out>) at msg/Pipe.h:47
#6  0x00000000052aa182 in start_thread (arg=0x24e08700) at pthread_create.c:312
#7  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 81 (Thread 7478):
#0  0x0000000006f39ddd in poll () at ../sysdeps/unix/syscall-template.S:81
#1  0x0000000000af36ff in poll (__timeout=<optimized out>, __nfds=1, __fds=0x235f1ad0) at /usr/include/x86_64-linux-gnu/bits/poll2.h:46
#2  Pipe::tcp_read_wait (this=this@entry=0x4125c00) at msg/Pipe.cc:2277
#3  0x0000000000af3a18 in Pipe::tcp_read (this=this@entry=0x4125c00, buf=buf@entry=0x235f1c7e "\377", len=len@entry=1) at msg/Pipe.cc:2250
#4  0x0000000000b005a5 in Pipe::reader (this=0x4125c00) at msg/Pipe.cc:1412
#5  0x0000000000b0a5fd in Pipe::Reader::entry (this=<optimized out>) at msg/Pipe.h:47
#6  0x00000000052aa182 in start_thread (arg=0x235f2700) at pthread_create.c:312
#7  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 80 (Thread 6629):
#0  pthread_spin_lock () at ../nptl/sysdeps/x86_64/pthread_spin_lock.S:33
#1  0x0000000000a4bd71 in ceph_spin_lock (l=0x26f31380) at ./include/Spinlock.h:45
#2  lock (this=0x26f31380) at ./include/Spinlock.h:94
#3  Locker (s=..., this=<synthetic pointer>) at ./include/Spinlock.h:105
#4  get_crc (crc=<synthetic pointer>, fromto=<synthetic pointer>, this=0x26f31360) at common/buffer.cc:162
#5  ceph::buffer::list::crc32c (this=this@entry=0x26950160, crc=2876307226, crc@entry=0) at common/buffer.cc:1662
#6  0x000000000099fbed in calc_front_crc (this=0x26950100) at msg/Message.h:469
#7  Message::encode (this=this@entry=0x26950100, features=features@entry=8796093022207, datacrc=<optimized out>) at msg/Message.cc:182
#8  0x0000000000b07330 in Pipe::writer (this=0x4125200) at msg/Pipe.cc:1672
#9  0x0000000000b0ad5d in Pipe::Writer::entry (this=<optimized out>) at msg/Pipe.h:59
#10 0x00000000052aa182 in start_thread (arg=0x232ef700) at pthread_create.c:312
#11 0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 79 (Thread 6625):
#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1  0x0000000000b06d58 in Wait (mutex=..., this=0x4124ee8) at ./common/Cond.h:55
#2  Pipe::writer (this=0x4124d00) at msg/Pipe.cc:1723
#3  0x0000000000b0ad5d in Pipe::Writer::entry (this=<optimized out>) at msg/Pipe.h:59
#4  0x00000000052aa182 in start_thread (arg=0x231ee700) at pthread_create.c:312
#5  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 78 (Thread 6620):
#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1  0x0000000000b06d58 in Wait (mutex=..., this=0x4124268) at ./common/Cond.h:55
#2  Pipe::writer (this=0x4124080) at msg/Pipe.cc:1723
#3  0x0000000000b0ad5d in Pipe::Writer::entry (this=<optimized out>) at msg/Pipe.h:59
#4  0x00000000052aa182 in start_thread (arg=0x230ed700) at pthread_create.c:312
#5  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 77 (Thread 6639):
#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1  0x0000000000b06d58 in Wait (mutex=..., this=0x2384a468) at ./common/Cond.h:55
#2  Pipe::writer (this=0x2384a280) at msg/Pipe.cc:1723
#3  0x0000000000b0ad5d in Pipe::Writer::entry (this=<optimized out>) at msg/Pipe.h:59
#4  0x00000000052aa182 in start_thread (arg=0x21cda700) at pthread_create.c:312
#5  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 76 (Thread 7537):
#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1  0x0000000000b06d58 in Wait (mutex=..., this=0x25ee0768) at ./common/Cond.h:55
#2  Pipe::writer (this=0x25ee0580) at msg/Pipe.cc:1723
#3  0x0000000000b0ad5d in Pipe::Writer::entry (this=<optimized out>) at msg/Pipe.h:59
#4  0x00000000052aa182 in start_thread (arg=0x22eeb700) at pthread_create.c:312
#5  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 75 (Thread 6638):
#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1  0x0000000000b06d58 in Wait (mutex=..., this=0x2384a6e8) at ./common/Cond.h:55
---Type <return> to continue, or q <return> to quit---
#2  Pipe::writer (this=0x2384a500) at msg/Pipe.cc:1723
#3  0x0000000000b0ad5d in Pipe::Writer::entry (this=<optimized out>) at msg/Pipe.h:59
#4  0x00000000052aa182 in start_thread (arg=0x22fec700) at pthread_create.c:312
#5  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 74 (Thread 6677):
#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1  0x0000000000b06d58 in Wait (mutex=..., this=0x4125b68) at ./common/Cond.h:55
#2  Pipe::writer (this=0x4125980) at msg/Pipe.cc:1723
#3  0x0000000000b0ad5d in Pipe::Writer::entry (this=<optimized out>) at msg/Pipe.h:59
#4  0x00000000052aa182 in start_thread (arg=0x220de700) at pthread_create.c:312
#5  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 73 (Thread 7352):
#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1  0x0000000000b06d58 in Wait (mutex=..., this=0x41226e8) at ./common/Cond.h:55
#2  Pipe::writer (this=0x4122500) at msg/Pipe.cc:1723
#3  0x0000000000b0ad5d in Pipe::Writer::entry (this=<optimized out>) at msg/Pipe.h:59
#4  0x00000000052aa182 in start_thread (arg=0x206ce700) at pthread_create.c:312
#5  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 72 (Thread 7425):
#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1  0x0000000000b06d58 in Wait (mutex=..., this=0x25ce04e8) at ./common/Cond.h:55
#2  Pipe::writer (this=0x25ce0300) at msg/Pipe.cc:1723
#3  0x0000000000b0ad5d in Pipe::Writer::entry (this=<optimized out>) at msg/Pipe.h:59
#4  0x00000000052aa182 in start_thread (arg=0x225e3700) at pthread_create.c:312
#5  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 71 (Thread 6578):
#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1  0x0000000000b06d58 in Wait (mutex=..., this=0x41244e8) at ./common/Cond.h:55
#2  Pipe::writer (this=0x4124300) at msg/Pipe.cc:1723
#3  0x0000000000b0ad5d in Pipe::Writer::entry (this=<optimized out>) at msg/Pipe.h:59
#4  0x00000000052aa182 in start_thread (arg=0x228e6700) at pthread_create.c:312
#5  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 70 (Thread 6577):
#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1  0x0000000000b06d58 in Wait (mutex=..., this=0x4123ae8) at ./common/Cond.h:55
#2  Pipe::writer (this=0x4123900) at msg/Pipe.cc:1723
#3  0x0000000000b0ad5d in Pipe::Writer::entry (this=<optimized out>) at msg/Pipe.h:59
#4  0x00000000052aa182 in start_thread (arg=0x227e5700) at pthread_create.c:312
#5  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 69 (Thread 6569):
#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1  0x0000000000b06d58 in Wait (mutex=..., this=0x41235e8) at ./common/Cond.h:55
#2  Pipe::writer (this=0x4123400) at msg/Pipe.cc:1723
#3  0x0000000000b0ad5d in Pipe::Writer::entry (this=<optimized out>) at msg/Pipe.h:59
#4  0x00000000052aa182 in start_thread (arg=0x226e4700) at pthread_create.c:312
#5  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 68 (Thread 7424):
#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1  0x0000000000b06d58 in Wait (mutex=..., this=0x238a7368) at ./common/Cond.h:55
#2  Pipe::writer (this=0x238a7180) at msg/Pipe.cc:1723
#3  0x0000000000b0ad5d in Pipe::Writer::entry (this=<optimized out>) at msg/Pipe.h:59
#4  0x00000000052aa182 in start_thread (arg=0x229e7700) at pthread_create.c:312
#5  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 67 (Thread 7469):
#0  0x0000000006f39ddd in poll () at ../sysdeps/unix/syscall-template.S:81
#1  0x0000000000af36ff in poll (__timeout=<optimized out>, __nfds=1, __fds=0x224e1ad0) at /usr/include/x86_64-linux-gnu/bits/poll2.h:46
#2  Pipe::tcp_read_wait (this=this@entry=0x4125200) at msg/Pipe.cc:2277
#3  0x0000000000af3a18 in Pipe::tcp_read (this=this@entry=0x4125200, buf=buf@entry=0x224e1c7e "\377", len=len@entry=1) at msg/Pipe.cc:2250
#4  0x0000000000b005a5 in Pipe::reader (this=0x4125200) at msg/Pipe.cc:1412
#5  0x0000000000b0a5fd in Pipe::Reader::entry (this=<optimized out>) at msg/Pipe.h:47
#6  0x00000000052aa182 in start_thread (arg=0x224e2700) at pthread_create.c:312
#7  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 66 (Thread 6563):
#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1  0x0000000000b06d58 in Wait (mutex=..., this=0x41221e8) at ./common/Cond.h:55
#2  Pipe::writer (this=0x4122000) at msg/Pipe.cc:1723
#3  0x0000000000b0ad5d in Pipe::Writer::entry (this=<optimized out>) at msg/Pipe.h:59
#4  0x00000000052aa182 in start_thread (arg=0x223e1700) at pthread_create.c:312
#5  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 65 (Thread 6637):
#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1  0x0000000000b06d58 in Wait (mutex=..., this=0x2384a1e8) at ./common/Cond.h:55
#2  Pipe::writer (this=0x2384a000) at msg/Pipe.cc:1723
#3  0x0000000000b0ad5d in Pipe::Writer::entry (this=<optimized out>) at msg/Pipe.h:59
#4  0x00000000052aa182 in start_thread (arg=0x222e0700) at pthread_create.c:312
#5  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 64 (Thread 7482):
#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1  0x0000000000b06d58 in Wait (mutex=..., this=0x25cdffe8) at ./common/Cond.h:55
---Type <return> to continue, or q <return> to quit---
#2  Pipe::writer (this=0x25cdfe00) at msg/Pipe.cc:1723
#3  0x0000000000b0ad5d in Pipe::Writer::entry (this=<optimized out>) at msg/Pipe.h:59
#4  0x00000000052aa182 in start_thread (arg=0x221df700) at pthread_create.c:312
#5  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 63 (Thread 6686):
#0  0x0000000006f39ddd in poll () at ../sysdeps/unix/syscall-template.S:81
#1  0x0000000000af36ff in poll (__timeout=<optimized out>, __nfds=1, __fds=0x24d06ad0) at /usr/include/x86_64-linux-gnu/bits/poll2.h:46
#2  Pipe::tcp_read_wait (this=this@entry=0x4124f80) at msg/Pipe.cc:2277
#3  0x0000000000af3a18 in Pipe::tcp_read (this=this@entry=0x4124f80, buf=buf@entry=0x24d06c7e "\377", len=len@entry=1) at msg/Pipe.cc:2250
#4  0x0000000000b005a5 in Pipe::reader (this=0x4124f80) at msg/Pipe.cc:1412
#5  0x0000000000b0a5fd in Pipe::Reader::entry (this=<optimized out>) at msg/Pipe.h:47
#6  0x00000000052aa182 in start_thread (arg=0x24d07700) at pthread_create.c:312
#7  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 62 (Thread 6550):
#0  0x0000000006f39ddd in poll () at ../sysdeps/unix/syscall-template.S:81
#1  0x0000000000af36ff in poll (__timeout=<optimized out>, __nfds=1, __fds=0x21fdcad0) at /usr/include/x86_64-linux-gnu/bits/poll2.h:46
#2  Pipe::tcp_read_wait (this=this@entry=0x4124080) at msg/Pipe.cc:2277
#3  0x0000000000af3a18 in Pipe::tcp_read (this=this@entry=0x4124080, buf=buf@entry=0x21fdcc7e "\377", len=len@entry=1) at msg/Pipe.cc:2250
#4  0x0000000000b005a5 in Pipe::reader (this=0x4124080) at msg/Pipe.cc:1412
#5  0x0000000000b0a5fd in Pipe::Reader::entry (this=<optimized out>) at msg/Pipe.h:47
#6  0x00000000052aa182 in start_thread (arg=0x21fdd700) at pthread_create.c:312
#7  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 61 (Thread 6549):
#0  0x0000000006f39ddd in poll () at ../sysdeps/unix/syscall-template.S:81
#1  0x0000000000af36ff in poll (__timeout=<optimized out>, __nfds=1, __fds=0x21edbad0) at /usr/include/x86_64-linux-gnu/bits/poll2.h:46
#2  Pipe::tcp_read_wait (this=this@entry=0x4124d00) at msg/Pipe.cc:2277
#3  0x0000000000af3a18 in Pipe::tcp_read (this=this@entry=0x4124d00, buf=buf@entry=0x21edbc7e "\377", len=len@entry=1) at msg/Pipe.cc:2250
#4  0x0000000000b005a5 in Pipe::reader (this=0x4124d00) at msg/Pipe.cc:1412
#5  0x0000000000b0a5fd in Pipe::Reader::entry (this=<optimized out>) at msg/Pipe.h:47
#6  0x00000000052aa182 in start_thread (arg=0x21edc700) at pthread_create.c:312
#7  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 60 (Thread 6636):
#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1  0x0000000000b06d58 in Wait (mutex=..., this=0x4125de8) at ./common/Cond.h:55
#2  Pipe::writer (this=0x4125c00) at msg/Pipe.cc:1723
#3  0x0000000000b0ad5d in Pipe::Writer::entry (this=<optimized out>) at msg/Pipe.h:59
#4  0x00000000052aa182 in start_thread (arg=0x237f4700) at pthread_create.c:312
#5  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 59 (Thread 6540):
#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1  0x0000000000b06d58 in Wait (mutex=..., this=0x4122be8) at ./common/Cond.h:55
#2  Pipe::writer (this=0x4122a00) at msg/Pipe.cc:1723
#3  0x0000000000b0ad5d in Pipe::Writer::entry (this=<optimized out>) at msg/Pipe.h:59
#4  0x00000000052aa182 in start_thread (arg=0x21bd9700) at pthread_create.c:312
#5  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 58 (Thread 6536):
#0  0x0000000006f39ddd in poll () at ../sysdeps/unix/syscall-template.S:81
#1  0x0000000000af36ff in poll (__timeout=<optimized out>, __nfds=1, __fds=0x21ad7ad0) at /usr/include/x86_64-linux-gnu/bits/poll2.h:46
#2  Pipe::tcp_read_wait (this=this@entry=0x4123900) at msg/Pipe.cc:2277
#3  0x0000000000af3a18 in Pipe::tcp_read (this=this@entry=0x4123900, buf=buf@entry=0x21ad7c7e "\377", len=len@entry=1) at msg/Pipe.cc:2250
#4  0x0000000000b005a5 in Pipe::reader (this=0x4123900) at msg/Pipe.cc:1412
#5  0x0000000000b0a5fd in Pipe::Reader::entry (this=<optimized out>) at msg/Pipe.h:47
#6  0x00000000052aa182 in start_thread (arg=0x21ad8700) at pthread_create.c:312
#7  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 57 (Thread 7536):
#0  0x0000000006f39ddd in poll () at ../sysdeps/unix/syscall-template.S:81
#1  0x0000000000af36ff in poll (__timeout=<optimized out>, __nfds=1, __fds=0x219d6ad0) at /usr/include/x86_64-linux-gnu/bits/poll2.h:46
#2  Pipe::tcp_read_wait (this=this@entry=0x25ee0580) at msg/Pipe.cc:2277
#3  0x0000000000af3a18 in Pipe::tcp_read (this=this@entry=0x25ee0580, buf=buf@entry=0x219d6c7e "\377", len=len@entry=1) at msg/Pipe.cc:2250
#4  0x0000000000b005a5 in Pipe::reader (this=0x25ee0580) at msg/Pipe.cc:1412
#5  0x0000000000b0a5fd in Pipe::Reader::entry (this=<optimized out>) at msg/Pipe.h:47
#6  0x00000000052aa182 in start_thread (arg=0x219d7700) at pthread_create.c:312
#7  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 56 (Thread 6517):
#0  0x0000000006f39ddd in poll () at ../sysdeps/unix/syscall-template.S:81
#1  0x0000000000af36ff in poll (__timeout=<optimized out>, __nfds=1, __fds=0x218d5ad0) at /usr/include/x86_64-linux-gnu/bits/poll2.h:46
#2  Pipe::tcp_read_wait (this=this@entry=0x4124300) at msg/Pipe.cc:2277
#3  0x0000000000af3a18 in Pipe::tcp_read (this=this@entry=0x4124300, buf=buf@entry=0x218d5c7e "\377", len=len@entry=1) at msg/Pipe.cc:2250
#4  0x0000000000b005a5 in Pipe::reader (this=0x4124300) at msg/Pipe.cc:1412
#5  0x0000000000b0a5fd in Pipe::Reader::entry (this=<optimized out>) at msg/Pipe.h:47
#6  0x00000000052aa182 in start_thread (arg=0x218d6700) at pthread_create.c:312
#7  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 55 (Thread 7423):
#0  0x0000000006f39ddd in poll () at ../sysdeps/unix/syscall-template.S:81
#1  0x0000000000af36ff in poll (__timeout=<optimized out>, __nfds=1, __fds=0x217d4ad0) at /usr/include/x86_64-linux-gnu/bits/poll2.h:46
#2  Pipe::tcp_read_wait (this=this@entry=0x238a7180) at msg/Pipe.cc:2277
#3  0x0000000000af3a18 in Pipe::tcp_read (this=this@entry=0x238a7180, buf=buf@entry=0x217d4c7e "\377", len=len@entry=1) at msg/Pipe.cc:2250
#4  0x0000000000b005a5 in Pipe::reader (this=0x238a7180) at msg/Pipe.cc:1412
#5  0x0000000000b0a5fd in Pipe::Reader::entry (this=<optimized out>) at msg/Pipe.h:47
#6  0x00000000052aa182 in start_thread (arg=0x217d5700) at pthread_create.c:312
#7  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111
---Type <return> to continue, or q <return> to quit---

Thread 54 (Thread 7481):
#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1  0x0000000000b06d58 in Wait (mutex=..., this=0x41230e8) at ./common/Cond.h:55
#2  Pipe::writer (this=0x4122f00) at msg/Pipe.cc:1723
#3  0x0000000000b0ad5d in Pipe::Writer::entry (this=<optimized out>) at msg/Pipe.h:59
#4  0x00000000052aa182 in start_thread (arg=0x215d3700) at pthread_create.c:312
#5  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 53 (Thread 7480):
#0  0x0000000006f39ddd in poll () at ../sysdeps/unix/syscall-template.S:81
#1  0x0000000000af36ff in poll (__timeout=<optimized out>, __nfds=1, __fds=0x22ae7ad0) at /usr/include/x86_64-linux-gnu/bits/poll2.h:46
#2  Pipe::tcp_read_wait (this=this@entry=0x25cdfe00) at msg/Pipe.cc:2277
#3  0x0000000000af3a18 in Pipe::tcp_read (this=this@entry=0x25cdfe00, buf=buf@entry=0x22ae7c7e "\377", len=len@entry=1) at msg/Pipe.cc:2250
#4  0x0000000000b005a5 in Pipe::reader (this=0x25cdfe00) at msg/Pipe.cc:1412
#5  0x0000000000b0a5fd in Pipe::Reader::entry (this=<optimized out>) at msg/Pipe.h:47
#6  0x00000000052aa182 in start_thread (arg=0x22ae8700) at pthread_create.c:312
#7  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 52 (Thread 6507):
#0  0x0000000006f39ddd in poll () at ../sysdeps/unix/syscall-template.S:81
#1  0x0000000000af36ff in poll (__timeout=<optimized out>, __nfds=1, __fds=0x214d1ad0) at /usr/include/x86_64-linux-gnu/bits/poll2.h:46
#2  Pipe::tcp_read_wait (this=this@entry=0x4123400) at msg/Pipe.cc:2277
#3  0x0000000000af3a18 in Pipe::tcp_read (this=this@entry=0x4123400, buf=buf@entry=0x214d1c7e "\377", len=len@entry=1) at msg/Pipe.cc:2250
#4  0x0000000000b005a5 in Pipe::reader (this=0x4123400) at msg/Pipe.cc:1412
#5  0x0000000000b0a5fd in Pipe::Reader::entry (this=<optimized out>) at msg/Pipe.h:47
#6  0x00000000052aa182 in start_thread (arg=0x214d2700) at pthread_create.c:312
#7  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 51 (Thread 7422):
#0  0x0000000006f39ddd in poll () at ../sysdeps/unix/syscall-template.S:81
#1  0x0000000000af36ff in poll (__timeout=<optimized out>, __nfds=1, __fds=0x213d0ad0) at /usr/include/x86_64-linux-gnu/bits/poll2.h:46
#2  Pipe::tcp_read_wait (this=this@entry=0x25ce0300) at msg/Pipe.cc:2277
#3  0x0000000000af3a18 in Pipe::tcp_read (this=this@entry=0x25ce0300, buf=buf@entry=0x213d0c7e "\377", len=len@entry=1) at msg/Pipe.cc:2250
#4  0x0000000000b005a5 in Pipe::reader (this=0x25ce0300) at msg/Pipe.cc:1412
#5  0x0000000000b0a5fd in Pipe::Reader::entry (this=<optimized out>) at msg/Pipe.h:47
#6  0x00000000052aa182 in start_thread (arg=0x213d1700) at pthread_create.c:312
#7  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 50 (Thread 6504):
#0  0x0000000006f39ddd in poll () at ../sysdeps/unix/syscall-template.S:81
#1  0x0000000000af36ff in poll (__timeout=<optimized out>, __nfds=1, __fds=0x212cfad0) at /usr/include/x86_64-linux-gnu/bits/poll2.h:46
#2  Pipe::tcp_read_wait (this=this@entry=0x4122a00) at msg/Pipe.cc:2277
#3  0x0000000000af3a18 in Pipe::tcp_read (this=this@entry=0x4122a00, buf=buf@entry=0x212cfc7e "\377", len=len@entry=1) at msg/Pipe.cc:2250
#4  0x0000000000b005a5 in Pipe::reader (this=0x4122a00) at msg/Pipe.cc:1412
#5  0x0000000000b0a5fd in Pipe::Reader::entry (this=<optimized out>) at msg/Pipe.h:47
#6  0x00000000052aa182 in start_thread (arg=0x212d0700) at pthread_create.c:312
#7  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 49 (Thread 6503):
#0  0x0000000006f39ddd in poll () at ../sysdeps/unix/syscall-template.S:81
#1  0x0000000000af36ff in poll (__timeout=<optimized out>, __nfds=1, __fds=0x1f4c9ad0) at /usr/include/x86_64-linux-gnu/bits/poll2.h:46
#2  Pipe::tcp_read_wait (this=this@entry=0x4122000) at msg/Pipe.cc:2277
#3  0x0000000000af3a18 in Pipe::tcp_read (this=this@entry=0x4122000, buf=buf@entry=0x1f4c9c7e "\377", len=len@entry=1) at msg/Pipe.cc:2250
#4  0x0000000000b005a5 in Pipe::reader (this=0x4122000) at msg/Pipe.cc:1412
#5  0x0000000000b0a5fd in Pipe::Reader::entry (this=<optimized out>) at msg/Pipe.h:47
#6  0x00000000052aa182 in start_thread (arg=0x1f4ca700) at pthread_create.c:312
#7  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 48 (Thread 7479):
#0  0x0000000006f39ddd in poll () at ../sysdeps/unix/syscall-template.S:81
#1  0x0000000000af36ff in poll (__timeout=<optimized out>, __nfds=1, __fds=0x216d3ad0) at /usr/include/x86_64-linux-gnu/bits/poll2.h:46
#2  Pipe::tcp_read_wait (this=this@entry=0x4122f00) at msg/Pipe.cc:2277
#3  0x0000000000af3a18 in Pipe::tcp_read (this=this@entry=0x4122f00, buf=buf@entry=0x216d3c7e "\377", len=len@entry=1) at msg/Pipe.cc:2250
#4  0x0000000000b005a5 in Pipe::reader (this=0x4122f00) at msg/Pipe.cc:1412
#5  0x0000000000b0a5fd in Pipe::Reader::entry (this=<optimized out>) at msg/Pipe.h:47
#6  0x00000000052aa182 in start_thread (arg=0x216d4700) at pthread_create.c:312
#7  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 47 (Thread 6411):
#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1  0x00000000006618ae in Cond::Wait (this=0x40edd28, mutex=...) at ./common/Cond.h:55
#2  0x0000000000605c2c in OSDService::agent_entry (this=0x40ed6b8) at osd/OSD.cc:493
#3  0x0000000000668d1d in OSDService::AgentThread::entry (this=<optimized out>) at osd/OSD.h:464
#4  0x00000000052aa182 in start_thread (arg=0x205cd700) at pthread_create.c:312
#5  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 46 (Thread 6410):
#0  pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
#1  0x0000000000a30b6f in WaitUntil (when=..., mutex=..., this=0x40ee2d8) at common/Cond.h:71
#2  SafeTimer::timer_thread (this=0x40ee2c8) at common/Timer.cc:114
#3  0x0000000000a313ad in SafeTimerThread::entry (this=<optimized out>) at common/Timer.cc:38
#4  0x00000000052aa182 in start_thread (arg=0x1fdcc700) at pthread_create.c:312
#5  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 45 (Thread 6476):
#0  0x0000000006f39ddd in poll () at ../sysdeps/unix/syscall-template.S:81
#1  0x00000000009624c0 in poll (__timeout=-1, __nfds=4, __fds=0x20eced10) at /usr/include/x86_64-linux-gnu/bits/poll2.h:41
#2  SignalHandler::entry (this=0x40f2300) at global/signal_handler.cc:213
#3  0x00000000052aa182 in start_thread (arg=0x20ecf700) at pthread_create.c:312
---Type <return> to continue, or q <return> to quit---
#4  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 44 (Thread 7353):
#0  0x0000000006f39ddd in poll () at ../sysdeps/unix/syscall-template.S:81
#1  0x0000000000af36ff in poll (__timeout=<optimized out>, __nfds=1, __fds=0x1f5caad0) at /usr/include/x86_64-linux-gnu/bits/poll2.h:46
#2  Pipe::tcp_read_wait (this=this@entry=0x4122500) at msg/Pipe.cc:2277
#3  0x0000000000af3a18 in Pipe::tcp_read (this=this@entry=0x4122500, buf=buf@entry=0x1f5cac7e "\377", len=len@entry=1) at msg/Pipe.cc:2250
#4  0x0000000000b005a5 in Pipe::reader (this=0x4122500) at msg/Pipe.cc:1412
#5  0x0000000000b0a5fd in Pipe::Reader::entry (this=<optimized out>) at msg/Pipe.h:47
#6  0x00000000052aa182 in start_thread (arg=0x1f5cb700) at pthread_create.c:312
#7  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 43 (Thread 6407):
#0  pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
#1  0x0000000000a30b6f in WaitUntil (when=..., mutex=..., this=0x40ede68) at common/Cond.h:71
#2  SafeTimer::timer_thread (this=0x40ede58) at common/Timer.cc:114
#3  0x0000000000a313ad in SafeTimerThread::entry (this=<optimized out>) at common/Timer.cc:38
#4  0x00000000052aa182 in start_thread (arg=0x1f3c9700) at pthread_create.c:312
#5  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 42 (Thread 6406):
#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1  0x0000000000984e38 in Wait (mutex=..., this=0x40ee188) at ./common/Cond.h:55
#2  Finisher::finisher_thread_entry (this=0x40ee128) at common/Finisher.cc:78
#3  0x00000000052aa182 in start_thread (arg=0x1ebc8700) at pthread_create.c:312
#4  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 41 (Thread 6405):
#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1  0x0000000000984e38 in Wait (mutex=..., this=0x40ee578) at ./common/Cond.h:55
#2  Finisher::finisher_thread_entry (this=0x40ee518) at common/Finisher.cc:78
#3  0x00000000052aa182 in start_thread (arg=0x1e3c7700) at pthread_create.c:312
#4  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 40 (Thread 6404):
#0  pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
#1  0x00000000006286ee in WaitUntil (when=..., mutex=..., this=0x40ecb20) at ./common/Cond.h:71
#2  WaitInterval (interval=..., mutex=..., cct=<optimized out>, this=0x40ecb20) at ./common/Cond.h:79
#3  OSD::heartbeat_entry (this=0x40ec000) at osd/OSD.cc:3037
#4  0x000000000068475d in OSD::T_Heartbeat::entry (this=<optimized out>) at osd/OSD.h:1037
#5  0x00000000052aa182 in start_thread (arg=0x1dbc6700) at pthread_create.c:312
#6  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 39 (Thread 6403):
#0  pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
#1  0x0000000000a3456c in WaitUntil (when=..., mutex=..., this=0x40ec980) at common/Cond.h:71
#2  WaitInterval (interval=..., mutex=..., cct=<optimized out>, this=0x40ec980) at common/Cond.h:79
#3  ThreadPool::worker (this=0x40ec908, wt=0x4046ea0) at common/WorkQueue.cc:143
#4  0x0000000000a358a0 in ThreadPool::WorkThread::entry (this=<optimized out>) at common/WorkQueue.h:317
#5  0x00000000052aa182 in start_thread (arg=0x1d3c5700) at pthread_create.c:312
#6  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 38 (Thread 6402):
#0  pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
#1  0x0000000000a3456c in WaitUntil (when=..., mutex=..., this=0x40ec7f8) at common/Cond.h:71
#2  WaitInterval (interval=..., mutex=..., cct=<optimized out>, this=0x40ec7f8) at common/Cond.h:79
#3  ThreadPool::worker (this=0x40ec780, wt=0x4046660) at common/WorkQueue.cc:143
#4  0x0000000000a358a0 in ThreadPool::WorkThread::entry (this=<optimized out>) at common/WorkQueue.h:317
#5  0x00000000052aa182 in start_thread (arg=0x1cbc4700) at pthread_create.c:312
#6  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 37 (Thread 6401):
#0  pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
#1  0x0000000000a3456c in WaitUntil (when=..., mutex=..., this=0x40ec670) at common/Cond.h:71
#2  WaitInterval (interval=..., mutex=..., cct=<optimized out>, this=0x40ec670) at common/Cond.h:79
#3  ThreadPool::worker (this=0x40ec5f8, wt=0x4046ee0) at common/WorkQueue.cc:143
#4  0x0000000000a358a0 in ThreadPool::WorkThread::entry (this=<optimized out>) at common/WorkQueue.h:317
#5  0x00000000052aa182 in start_thread (arg=0x1c3c3700) at pthread_create.c:312
#6  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 36 (Thread 6400):
#0  __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135
#1  0x00000000052ac657 in _L_lock_909 () from /lib/x86_64-linux-gnu/libpthread.so.0
#2  0x00000000052ac480 in __GI___pthread_mutex_lock (mutex=0x254de458) at ../nptl/pthread_mutex_lock.c:79
#3  0x00000000009fc73b in Mutex::Lock (this=this@entry=0x254de448, no_lockdep=no_lockdep@entry=false) at common/Mutex.cc:89
#4  0x000000000071d750 in PG::lock (this=this@entry=0x254de000, no_lockdep=no_lockdep@entry=false) at osd/PG.cc:220
#5  0x000000000071d8ce in PG::lock_suspend_timeout (this=0x254de000, handle=...) at osd/PG.cc:214
#6  0x000000000061ec28 in OSD::OpWQ::_process (this=0x40ece28, pg=..., handle=...) at osd/OSD.cc:7579
#7  0x000000000066133c in ThreadPool::WorkQueueVal<std::pair<boost::intrusive_ptr<PG>, std::tr1::shared_ptr<OpRequest> >, boost::intrusive_ptr<PG> >::_void_process (this=0x40ece28, handle=...) at ./common/WorkQueue.h:190
#8  0x0000000000a349b1 in ThreadPool::worker (this=0x40ec470, wt=0x40466a0) at common/WorkQueue.cc:125
#9  0x0000000000a358a0 in ThreadPool::WorkThread::entry (this=<optimized out>) at common/WorkQueue.h:317
#10 0x00000000052aa182 in start_thread (arg=0x1bbc2700) at pthread_create.c:312
#11 0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 35 (Thread 6399):
#0  0x0000000004c2ea78 in _vgr20180ZZ_libcZdsoZa_memcpyZAZAGLIBCZu2Zd14 () from /usr/lib/valgrind/vgpreload_memcheck-amd64-linux.so
#1  0x00000000066c95e0 in std::basic_streambuf<char, std::char_traits<char> >::xsputn(char const*, long) () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6
#2  0x00000000066c0db5 in std::basic_ostream<char, std::char_traits<char> >& std::__ostream_insert<char, std::char_traits<char> >(std::basic_ostream<char, std::char_traits<char> >&, char const*, long) () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6
#3  0x0000000000662614 in operator<< <std::char_traits<char> > (__s=0xbcab99 " ", __out=...) at /usr/include/c++/4.8/ostream:535
#4  operator<< (out=..., h=...) at ./osd/osd_types.h:1528
---Type <return> to continue, or q <return> to quit---
#5  0x0000000000662759 in operator<< (out=..., pgi=...) at ./osd/osd_types.h:1605
#6  0x000000000071ce24 in operator<< (out=..., pg=...) at osd/PG.cc:4835
#7  0x000000000071d663 in PG::gen_prefix (this=0x254de000) at osd/PG.cc:235
#8  0x00000000008112d9 in ReplicatedPG::gen_dbg_prefix (this=<optimized out>) at osd/ReplicatedPG.h:306
#9  0x00000000008f53b7 in _prefix (_dout=_dout@entry=0x1b3c02a0, pgb=pgb@entry=0x20fe0b40) at osd/ECBackend.cc:30
#10 0x00000000008f6ef3 in ECBackend::check_op (this=this@entry=0x20fe0b40, op=op@entry=0x22c7ef08) at osd/ECBackend.cc:1475
#11 0x00000000008f7177 in ECBackend::handle_sub_write_reply (this=this@entry=0x20fe0b40, from=..., op=...) at osd/ECBackend.cc:917
#12 0x0000000000904a4e in ECBackend::handle_message (this=0x20fe0b40, _op=...) at osd/ECBackend.cc:653
#13 0x00000000007a3acb in ReplicatedPG::do_request (this=0x254de000, op=..., handle=...) at osd/ReplicatedPG.cc:1087
#14 0x0000000000605061 in OSD::dequeue_op (this=0x40ec000, pg=..., op=..., handle=...) at osd/OSD.cc:7633
#15 0x000000000061ede4 in OSD::OpWQ::_process (this=0x40ece28, pg=..., handle=...) at osd/OSD.cc:7603
#16 0x000000000066133c in ThreadPool::WorkQueueVal<std::pair<boost::intrusive_ptr<PG>, std::tr1::shared_ptr<OpRequest> >, boost::intrusive_ptr<PG> >::_void_process (this=0x40ece28, handle=...) at ./common/WorkQueue.h:190
#17 0x0000000000a349b1 in ThreadPool::worker (this=0x40ec470, wt=0x4046700) at common/WorkQueue.cc:125
#18 0x0000000000a358a0 in ThreadPool::WorkThread::entry (this=<optimized out>) at common/WorkQueue.h:317
#19 0x00000000052aa182 in start_thread (arg=0x1b3c1700) at pthread_create.c:312
#20 0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 34 (Thread 6398):
#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1  0x0000000000984e38 in Wait (mutex=..., this=0xfff0006f0) at ./common/Cond.h:55
#2  Finisher::finisher_thread_entry (this=0xfff000690) at common/Finisher.cc:78
#3  0x00000000052aa182 in start_thread (arg=0x1abc0700) at pthread_create.c:312
#4  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 33 (Thread 6397):
#0  pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
#1  0x0000000000a30b6f in WaitUntil (when=..., mutex=..., this=0xfff0005d8) at common/Cond.h:71
#2  SafeTimer::timer_thread (this=0xfff0005c8) at common/Timer.cc:114
#3  0x0000000000a313ad in SafeTimerThread::entry (this=<optimized out>) at common/Timer.cc:38
#4  0x00000000052aa182 in start_thread (arg=0x1a3bf700) at pthread_create.c:312
#5  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 32 (Thread 6396):
#0  0x0000000006f39ddd in poll () at ../sysdeps/unix/syscall-template.S:81
#1  0x0000000000acdbd9 in poll (__timeout=-1, __nfds=1, __fds=0x19bbdc20) at /usr/include/x86_64-linux-gnu/bits/poll2.h:46
#2  Accepter::entry (this=0x40b32c0) at msg/Accepter.cc:201
#3  0x00000000052aa182 in start_thread (arg=0x19bbe700) at pthread_create.c:312
#4  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 31 (Thread 6395):
#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1  0x0000000000adf91b in Wait (mutex=..., this=0x40b3350) at ./common/Cond.h:55
#2  DispatchQueue::entry (this=0x40b32e8) at msg/DispatchQueue.cc:138
#3  0x0000000000a26aad in DispatchQueue::DispatchThread::entry (this=<optimized out>) at msg/DispatchQueue.h:104
#4  0x00000000052aa182 in start_thread (arg=0x193bd700) at pthread_create.c:312
#5  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 30 (Thread 6394):
#0  0x0000000006f39ddd in poll () at ../sysdeps/unix/syscall-template.S:81
#1  0x0000000000acdbd9 in poll (__timeout=-1, __nfds=1, __fds=0x18bbbc20) at /usr/include/x86_64-linux-gnu/bits/poll2.h:46
#2  Accepter::entry (this=0x40b3ec0) at msg/Accepter.cc:201
#3  0x00000000052aa182 in start_thread (arg=0x18bbc700) at pthread_create.c:312
#4  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 29 (Thread 6393):
#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1  0x0000000000adf91b in Wait (mutex=..., this=0x40b3f50) at ./common/Cond.h:55
#2  DispatchQueue::entry (this=0x40b3ee8) at msg/DispatchQueue.cc:138
#3  0x0000000000a26aad in DispatchQueue::DispatchThread::entry (this=<optimized out>) at msg/DispatchQueue.h:104
#4  0x00000000052aa182 in start_thread (arg=0x183bb700) at pthread_create.c:312
#5  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 28 (Thread 6392):
#0  0x0000000006f39ddd in poll () at ../sysdeps/unix/syscall-template.S:81
#1  0x0000000000acdbd9 in poll (__timeout=-1, __nfds=1, __fds=0x17bb9c20) at /usr/include/x86_64-linux-gnu/bits/poll2.h:46
#2  Accepter::entry (this=0x40b38c0) at msg/Accepter.cc:201
#3  0x00000000052aa182 in start_thread (arg=0x17bba700) at pthread_create.c:312
#4  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 27 (Thread 6391):
#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1  0x0000000000adf91b in Wait (mutex=..., this=0x40b3950) at ./common/Cond.h:55
#2  DispatchQueue::entry (this=0x40b38e8) at msg/DispatchQueue.cc:138
#3  0x0000000000a26aad in DispatchQueue::DispatchThread::entry (this=<optimized out>) at msg/DispatchQueue.h:104
#4  0x00000000052aa182 in start_thread (arg=0x173b9700) at pthread_create.c:312
#5  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 26 (Thread 6390):
#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1  0x0000000000adf91b in Wait (mutex=..., this=0x40b2750) at ./common/Cond.h:55
#2  DispatchQueue::entry (this=0x40b26e8) at msg/DispatchQueue.cc:138
#3  0x0000000000a26aad in DispatchQueue::DispatchThread::entry (this=<optimized out>) at msg/DispatchQueue.h:104
#4  0x00000000052aa182 in start_thread (arg=0x16bb8700) at pthread_create.c:312
#5  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 25 (Thread 6389):
#0  0x0000000006f39ddd in poll () at ../sysdeps/unix/syscall-template.S:81
#1  0x0000000000acdbd9 in poll (__timeout=-1, __nfds=1, __fds=0x163b6c20) at /usr/include/x86_64-linux-gnu/bits/poll2.h:46
#2  Accepter::entry (this=0x40b2cc0) at msg/Accepter.cc:201
#3  0x00000000052aa182 in start_thread (arg=0x163b7700) at pthread_create.c:312
---Type <return> to continue, or q <return> to quit---
#4  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 24 (Thread 6388):
#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1  0x0000000000adf91b in Wait (mutex=..., this=0x40b2d50) at ./common/Cond.h:55
#2  DispatchQueue::entry (this=0x40b2ce8) at msg/DispatchQueue.cc:138
#3  0x0000000000a26aad in DispatchQueue::DispatchThread::entry (this=<optimized out>) at msg/DispatchQueue.h:104
#4  0x00000000052aa182 in start_thread (arg=0x15bb6700) at pthread_create.c:312
#5  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 23 (Thread 6387):
#0  0x0000000006f39ddd in poll () at ../sysdeps/unix/syscall-template.S:81
#1  0x0000000000acdbd9 in poll (__timeout=-1, __nfds=1, __fds=0x153b4c20) at /usr/include/x86_64-linux-gnu/bits/poll2.h:46
#2  Accepter::entry (this=0x40b20c0) at msg/Accepter.cc:201
#3  0x00000000052aa182 in start_thread (arg=0x153b5700) at pthread_create.c:312
#4  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 22 (Thread 6386):
#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1  0x0000000000adf91b in Wait (mutex=..., this=0x40b2150) at ./common/Cond.h:55
#2  DispatchQueue::entry (this=0x40b20e8) at msg/DispatchQueue.cc:138
#3  0x0000000000a26aad in DispatchQueue::DispatchThread::entry (this=<optimized out>) at msg/DispatchQueue.h:104
#4  0x00000000052aa182 in start_thread (arg=0x14bb4700) at pthread_create.c:312
#5  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 21 (Thread 6332):
#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1  0x0000000000984e38 in Wait (mutex=..., this=0x4094550) at ./common/Cond.h:55
#2  Finisher::finisher_thread_entry (this=0x40944f0) at common/Finisher.cc:78
#3  0x00000000052aa182 in start_thread (arg=0x12270700) at pthread_create.c:312
#4  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 20 (Thread 6331):
#0  __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135
#1  0x00000000052b4308 in _L_cond_lock_886 () from /lib/x86_64-linux-gnu/libpthread.so.0
#2  0x00000000052b4074 in __pthread_mutex_cond_lock (mutex=0x4094c78) at ../nptl/pthread_mutex_lock.c:79
#3  0x00000000052ae4a4 in pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:259
#4  0x0000000000984e38 in Wait (mutex=..., this=0x4094cc0) at ./common/Cond.h:55
#5  Finisher::finisher_thread_entry (this=0x4094c60) at common/Finisher.cc:78
#6  0x00000000052aa182 in start_thread (arg=0x11a6f700) at pthread_create.c:312
#7  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 19 (Thread 6330):
#0  __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135
#1  0x00000000052ac657 in _L_lock_909 () from /lib/x86_64-linux-gnu/libpthread.so.0
#2  0x00000000052ac480 in __GI___pthread_mutex_lock (mutex=0x2537e958) at ../nptl/pthread_mutex_lock.c:79
#3  0x00000000009fc73b in Mutex::Lock (this=this@entry=0x2537e948, no_lockdep=no_lockdep@entry=false) at common/Mutex.cc:89
#4  0x0000000000883196 in FileStore::_do_op (this=0x4094000, osr=0x2537e880, handle=...) at os/FileStore.cc:1694
#5  0x0000000000a349b1 in ThreadPool::worker (this=0x4094d90, wt=0x4045be0) at common/WorkQueue.cc:125
#6  0x0000000000a358a0 in ThreadPool::WorkThread::entry (this=<optimized out>) at common/WorkQueue.h:317
#7  0x00000000052aa182 in start_thread (arg=0x1126e700) at pthread_create.c:312
#8  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 18 (Thread 6329):
#0  pthread_cond_broadcast@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S:98
#1  0x00000000008664e3 in Signal (this=0x4094cc0) at ./common/Cond.h:91
#2  queue (r=0, c=0x25c0ac20, this=0x4094c60) at ./common/Finisher.h:57
#3  FileStore::_finish_op (this=0x4094000, osr=<optimized out>) at os/FileStore.cc:1722
#4  0x0000000000a349ca in ThreadPool::worker (this=0x4094d90, wt=0x4045ba0) at common/WorkQueue.cc:127
#5  0x0000000000a358a0 in ThreadPool::WorkThread::entry (this=<optimized out>) at common/WorkQueue.h:317
#6  0x00000000052aa182 in start_thread (arg=0x10a6d700) at pthread_create.c:312
#7  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 17 (Thread 6328):
#0  __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:135
#1  0x00000000052ac657 in _L_lock_909 () from /lib/x86_64-linux-gnu/libpthread.so.0
#2  0x00000000052ac480 in __GI___pthread_mutex_lock (mutex=0x254de458) at ../nptl/pthread_mutex_lock.c:79
#3  0x00000000009fc73b in Mutex::Lock (this=this@entry=0x254de448, no_lockdep=no_lockdep@entry=false) at common/Mutex.cc:89
#4  0x000000000071d750 in PG::lock (this=0x254de000, no_lockdep=no_lockdep@entry=false) at osd/PG.cc:220
#5  0x00000000008143bf in ReplicatedPG::BlessedContext::finish (this=0x26b9e920, r=0) at osd/ReplicatedPG.h:268
#6  0x0000000000659389 in Context::complete (this=0x26b9e920, r=<optimized out>) at ./include/Context.h:64
#7  0x0000000000984c28 in Finisher::finisher_thread_entry (this=0x4094020) at common/Finisher.cc:56
#8  0x00000000052aa182 in start_thread (arg=0x1026c700) at pthread_create.c:312
#9  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 16 (Thread 6327):
#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1  0x00000000006618ae in Cond::Wait (this=0x40d4380, mutex=...) at ./common/Cond.h:55
#2  0x00000000009450da in FileJournal::write_finish_thread_entry (this=0x40d4000) at os/FileJournal.cc:1333
#3  0x0000000000886c3d in FileJournal::WriteFinisher::entry (this=<optimized out>) at os/FileJournal.h:344
#4  0x00000000052aa182 in start_thread (arg=0xfa6b700) at pthread_create.c:312
#5  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 15 (Thread 6326):
#0  operator< <long unsigned int, long unsigned int> (__y=<synthetic pointer>, __x=...) at /usr/include/c++/4.8/bits/stl_pair.h:222
#1  operator() (this=<optimized out>, __y=<synthetic pointer>, __x=...) at /usr/include/c++/4.8/bits/stl_function.h:235
#2  _M_lower_bound (this=<optimized out>, __k=<synthetic pointer>, __y=0x26f37b40, __x=0x26f37f40) at /usr/include/c++/4.8/bits/stl_tree.h:1157
#3  find (__k=<synthetic pointer>, this=0x26f31388) at /usr/include/c++/4.8/bits/stl_tree.h:1805
#4  find (__x=<synthetic pointer>, this=0x26f31388) at /usr/include/c++/4.8/bits/stl_map.h:837
#5  get_crc (crc=<synthetic pointer>, fromto=<synthetic pointer>, this=0x26f31360) at common/buffer.cc:164
---Type <return> to continue, or q <return> to quit---
#6  ceph::buffer::list::crc32c (this=this@entry=0x25b54078, crc=1908917301, crc@entry=0) at common/buffer.cc:1662
#7  0x00000000009464e2 in FileJournal::prepare_single_write (this=this@entry=0x40d4000, bl=..., queue_pos=@0xf269af8: 33570816, orig_ops=@0xf269ca0: 1, orig_bytes=@0xf269ca8: 265641) at os/FileJournal.cc:885
#8  0x0000000000946a46 in FileJournal::prepare_multi_write (this=this@entry=0x40d4000, bl=..., orig_ops=@0xf269ca0: 1, orig_bytes=@0xf269ca8: 265641) at os/FileJournal.cc:757
#9  0x00000000009473ff in FileJournal::write_thread_entry (this=0x40d4000) at os/FileJournal.cc:1152
#10 0x0000000000886c1d in FileJournal::Writer::entry (this=<optimized out>) at os/FileJournal.h:334
#11 0x00000000052aa182 in start_thread (arg=0xf26a700) at pthread_create.c:312
#12 0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 14 (Thread 6325):
#0  pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
#1  0x0000000000887dc4 in WaitUntil (when=..., mutex=..., this=0x4094680) at ./common/Cond.h:71
#2  Cond::WaitInterval (this=this@entry=0x4094680, cct=<optimized out>, mutex=..., interval=...) at ./common/Cond.h:79
#3  0x0000000000852216 in FileStore::sync_entry (this=0x4094000) at os/FileStore.cc:3072
#4  0x00000000008883cd in FileStore::SyncThread::entry (this=<optimized out>) at os/FileStore.h:178
#5  0x00000000052aa182 in start_thread (arg=0xd266700) at pthread_create.c:312
#6  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 13 (Thread 6324):
#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1  0x00000000008e52e8 in Wait (mutex=..., this=0x4094aa0) at ./common/Cond.h:55
#2  WBThrottle::get_next_should_flush (this=this@entry=0x4094998, next=next@entry=0xca64d40) at os/WBThrottle.cc:142
#3  0x00000000008e5d62 in WBThrottle::entry (this=0x4094998) at os/WBThrottle.cc:160
#4  0x00000000052aa182 in start_thread (arg=0xca65700) at pthread_create.c:312
#5  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 12 (Thread 6322):
#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1  0x0000000000a30754 in Wait (mutex=..., this=0x40ee400) at common/Cond.h:55
#2  SafeTimer::timer_thread (this=0x40ee3f0) at common/Timer.cc:112
#3  0x0000000000a313ad in SafeTimerThread::entry (this=<optimized out>) at common/Timer.cc:38
#4  0x00000000052aa182 in start_thread (arg=0xc264700) at pthread_create.c:312
#5  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 11 (Thread 6321):
#0  pthread_cond_timedwait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:238
#1  0x0000000000a30b6f in WaitUntil (when=..., mutex=..., this=0x40ec080) at common/Cond.h:71
#2  SafeTimer::timer_thread (this=0x40ec070) at common/Timer.cc:114
#3  0x0000000000a313ad in SafeTimerThread::entry (this=<optimized out>) at common/Timer.cc:38
#4  0x00000000052aa182 in start_thread (arg=0xb262700) at pthread_create.c:312
#5  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 10 (Thread 6320):
#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1  0x0000000000a20c9f in Wait (mutex=..., this=0x40b3718) at ./common/Cond.h:55
#2  SimpleMessenger::reaper_entry (this=0x40b3200) at msg/SimpleMessenger.cc:207
#3  0x0000000000a2747d in SimpleMessenger::ReaperThread::entry (this=<optimized out>) at msg/SimpleMessenger.h:225
#4  0x00000000052aa182 in start_thread (arg=0xaa61700) at pthread_create.c:312
#5  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 9 (Thread 6319):
#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1  0x0000000000a20c9f in Wait (mutex=..., this=0x40b3118) at ./common/Cond.h:55
#2  SimpleMessenger::reaper_entry (this=0x40b2c00) at msg/SimpleMessenger.cc:207
#3  0x0000000000a2747d in SimpleMessenger::ReaperThread::entry (this=<optimized out>) at msg/SimpleMessenger.h:225
#4  0x00000000052aa182 in start_thread (arg=0xa260700) at pthread_create.c:312
#5  0x0000000006f4712d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

Thread 8 (Thread 6318):
#0  pthread_cond_wait@@GLIBC_2.3.2 () at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185
#1  0x0000000000a20c9f in Wait (mutex=..., this=0x40b4318) at ./common/Cond.h:55
#2  SimpleMessenger::reaper_entry (this=0x40b3e00) at msg/SimpleMe