Project

General

Profile

Actions

Bug #9446

closed

mon deadlock

Added by Sage Weil over 9 years ago. Updated over 9 years ago.

Status:
Resolved
Priority:
Immediate
Assignee:
-
Category:
-
Target version:
-
% Done:

0%

Source:
Development
Tags:
Backport:
Regression:
Severity:
3 - minor
Reviewed:
Affected Versions:
ceph-qa-suite:
Pull request ID:
Crash signature (v1):
Crash signature (v2):

Description

(gdb) thr app all bt

Thread 99 (Thread 0x7f2d3a24c700 (LWP 28249)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1  0x00000000006cf5db in ceph::log::Log::entry() ()
#2  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d3a24c700) at pthread_create.c:308
#3  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#4  0x0000000000000000 in ?? ()

Thread 98 (Thread 0x7f2d39302700 (LWP 28251)):
#0  pthread_cond_timedwait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:215
#1  0x00000000007953e6 in CephContextServiceThread::entry() ()
#2  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d39302700) at pthread_create.c:308
#3  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#4  0x0000000000000000 in ?? ()

Thread 97 (Thread 0x7f2d38b01700 (LWP 28252)):
#0  __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:132
#1  0x00007f2d3c5e7065 in _L_lock_858 () from /lib/x86_64-linux-gnu/libpthread.so.0
#2  0x00007f2d3c5e6eba in __pthread_mutex_lock (mutex=0x4410c48) at pthread_mutex_lock.c:61
#3  0x0000000000741e23 in Mutex::Lock(bool) ()
#4  0x000000000056f197 in Monitor::do_admin_command(std::string, std::map<std::string, boost::variant<std::string, bool, long, double, std::vector<std::string, std::allocator<std::string> >, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_>, std::less<std::string>, std::allocator<std::pair<std::string const, boost::variant<std::string, bool, long, double, std::vector<std::string, std::allocator<std::string> >, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> > > >&, std::string, std::ostream&) ()
#5  0x00000000005a05fd in AdminHook::call(std::string, std::map<std::string, boost::variant<std::string, bool, long, double, std::vector<std::string, std::allocator<std::string> >, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_>, std::less<std::string>, std::allocator<std::pair<std::string const, boost::variant<std::string, bool, long, double, std::vector<std::string, std::allocator<std::string> >, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> > > >&, std::string, ceph::buffer::list&)
    ()
#6  0x000000000077aa2a in AdminSocket::do_accept() ()
#7  0x000000000077c050 in AdminSocket::entry() ()
#8  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d38b01700) at pthread_create.c:308
#9  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#10 0x0000000000000000 in ?? ()

Thread 96 (Thread 0x7f2d37167700 (LWP 28259)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1  0x00000000006c4400 in Finisher::finisher_thread_entry() ()
#2  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d37167700) at pthread_create.c:308
#3  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#4  0x0000000000000000 in ?? ()

---Type <return> to continue, or q <return> to quit---
Thread 95 (Thread 0x7f2d35368700 (LWP 28260)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1  0x00000000007618d9 in SimpleMessenger::reaper_entry() ()
#2  0x000000000076840d in SimpleMessenger::ReaperThread::entry() ()
#3  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d35368700) at pthread_create.c:308
#4  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#5  0x0000000000000000 in ?? ()

Thread 94 (Thread 0x7f2d34b67700 (LWP 28261)):
#0  __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:132
#1  0x00007f2d3c5ee2b8 in _L_cond_lock_874 () from /lib/x86_64-linux-gnu/libpthread.so.0
#2  0x00007f2d3c5ee088 in __pthread_mutex_cond_lock (mutex=0x4410c48)
    at ../nptl/pthread_mutex_lock.c:61
#3  0x00007f2d3c5e91b2 in pthread_cond_timedwait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:302
#4  0x000000000077223b in SafeTimer::timer_thread() ()
#5  0x0000000000774c2d in SafeTimerThread::entry() ()
#6  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d34b67700) at pthread_create.c:308
#7  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#8  0x0000000000000000 in ?? ()

Thread 93 (Thread 0x7f2d34366700 (LWP 28262)):
#0  0x00007f2d3b808f51 in std::basic_ostream<char, std::char_traits<char> >& std::__ostream_insert<char, std::char_traits<char> >(std::basic_ostream<char, std::char_traits<char> >&, char const*, long) ()
   from /usr/lib/x86_64-linux-gnu/libstdc++.so.6
#1  0x0000000000613ac9 in MDSMonitor::get_health(std::list<std::pair<health_status_t, std::string>, std::allocator<std::pair<health_status_t, std::string> > >&, std::list<std::pair<health_status_t, std::string>, std::allocator<std::pair<health_status_t, std::string> > >*) const ()
#2  0x00000000005736e1 in Monitor::get_health(std::string&, ceph::buffer::list*, ceph::Formatter*) ()
#3  0x000000000057533e in Monitor::get_cluster_status(std::basic_stringstream<char, std::char_traits<char>, std::allocator<char> >&, ceph::Formatter*) ()
#4  0x00000000005819d7 in Monitor::handle_command(MMonCommand*) ()
#5  0x000000000058acdc in Monitor::dispatch(MonSession*, Message*, bool) ()
#6  0x0000000000589068 in Monitor::_ms_dispatch(Message*) ()
#7  0x00000000005a3db2 in Monitor::ms_dispatch(Message*) ()
#8  0x0000000000865817 in Messenger::ms_deliver_dispatch(Message*) ()
#9  0x00000000008629ba in DispatchQueue::entry() ()
#10 0x000000000076742d in DispatchQueue::DispatchThread::entry() ()
#11 0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d34366700) at pthread_create.c:308
#12 0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#13 0x0000000000000000 in ?? ()

Thread 92 (Thread 0x7f2d33b65700 (LWP 28263)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1  0x000000000086415b in DispatchQueue::run_local_delivery() ()
#2  0x000000000076740d in DispatchQueue::LocalDeliveryThread::entry() ()
#3  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d33b65700) at pthread_create.c:308
#4  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#5  0x0000000000000000 in ?? ()

Thread 91 (Thread 0x7f2d33364700 (LWP 28264)):
#0  0x00007f2d3af8c313 in __GI___poll (fds=<optimized out>, nfds=<optimized out>, 
    timeout=<optimized out>) at ../sysdeps/unix/sysv/linux/poll.c:87
#1  0x000000000085dc64 in Accepter::entry() ()
#2  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d33364700) at pthread_create.c:308
#3  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#4  0x0000000000000000 in ?? ()

Thread 90 (Thread 0x7f2d32961700 (LWP 28267)):
#0  0x00007f2d3af8c313 in __GI___poll (fds=<optimized out>, nfds=<optimized out>, 
    timeout=<optimized out>) at ../sysdeps/unix/sysv/linux/poll.c:87
#1  0x000000000089633f in SignalHandler::entry() ()
---Type <return> to continue, or q <return> to quit---
#2  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d32961700) at pthread_create.c:308
#3  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#4  0x0000000000000000 in ?? ()

Thread 89 (Thread 0x7f2d31d4b700 (LWP 28282)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1  0x0000000000849c17 in LevelDBStore::compact_thread_entry() ()
#2  0x000000000084879d in LevelDBStore::CompactThread::entry() ()
#3  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d31d4b700) at pthread_create.c:308
#4  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#5  0x0000000000000000 in ?? ()

Thread 88 (Thread 0x7f2d3154a700 (LWP 28283)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1  0x00007f2d3c832f33 in ?? () from /usr/lib/x86_64-linux-gnu/libleveldb.so.1
#2  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d3154a700) at pthread_create.c:308
#3  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#4  0x0000000000000000 in ?? ()

Thread 87 (Thread 0x7f2d2caa4700 (LWP 21968)):
#0  0x00007f2d3af8c313 in __GI___poll (fds=<optimized out>, nfds=<optimized out>, 
    timeout=<optimized out>) at ../sysdeps/unix/sysv/linux/poll.c:87
#1  0x0000000000876d45 in Pipe::tcp_read_wait() ()
#2  0x0000000000877058 in Pipe::tcp_read(char*, int) ()
#3  0x0000000000889a50 in Pipe::reader() ()
#4  0x000000000088dbdd in Pipe::Reader::entry() ()
#5  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d2caa4700) at pthread_create.c:308
#6  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#7  0x0000000000000000 in ?? ()

Thread 86 (Thread 0x7f2d3780e700 (LWP 21969)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1  0x0000000000882892 in Pipe::writer() ()
#2  0x000000000088dbbd in Pipe::Writer::entry() ()
#3  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d3780e700) at pthread_create.c:308
#4  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#5  0x0000000000000000 in ?? ()

Thread 85 (Thread 0x7f2d2521e700 (LWP 22470)):
#0  0x00007f2d3af8c313 in __GI___poll (fds=<optimized out>, nfds=<optimized out>, 
    timeout=<optimized out>) at ../sysdeps/unix/sysv/linux/poll.c:87
#1  0x0000000000876d45 in Pipe::tcp_read_wait() ()
#2  0x0000000000877058 in Pipe::tcp_read(char*, int) ()
#3  0x0000000000889a50 in Pipe::reader() ()
#4  0x000000000088dbdd in Pipe::Reader::entry() ()
#5  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d2521e700) at pthread_create.c:308
#6  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#7  0x0000000000000000 in ?? ()

Thread 84 (Thread 0x7f2d2511d700 (LWP 22471)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1  0x0000000000882892 in Pipe::writer() ()
#2  0x000000000088dbbd in Pipe::Writer::entry() ()
#3  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d2511d700) at pthread_create.c:308
#4  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#5  0x0000000000000000 in ?? ()

Thread 83 (Thread 0x7f2d2c8a2700 (LWP 22475)):
#0  0x00007f2d3af8c313 in __GI___poll (fds=<optimized out>, nfds=<optimized out>, 
    timeout=<optimized out>) at ../sysdeps/unix/sysv/linux/poll.c:87
---Type <return> to continue, or q <return> to quit---
#1  0x0000000000876d45 in Pipe::tcp_read_wait() ()
#2  0x0000000000877058 in Pipe::tcp_read(char*, int) ()
#3  0x0000000000889a50 in Pipe::reader() ()
#4  0x000000000088dbdd in Pipe::Reader::entry() ()
#5  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d2c8a2700) at pthread_create.c:308
#6  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#7  0x0000000000000000 in ?? ()

Thread 82 (Thread 0x7f2d2b817700 (LWP 22476)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1  0x0000000000882892 in Pipe::writer() ()
#2  0x000000000088dbbd in Pipe::Writer::entry() ()
#3  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d2b817700) at pthread_create.c:308
#4  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#5  0x0000000000000000 in ?? ()

Thread 81 (Thread 0x7f2d35aa6700 (LWP 22477)):
#0  0x00007f2d3af8c313 in __GI___poll (fds=<optimized out>, nfds=<optimized out>, 
    timeout=<optimized out>) at ../sysdeps/unix/sysv/linux/poll.c:87
#1  0x0000000000876d45 in Pipe::tcp_read_wait() ()
#2  0x0000000000877058 in Pipe::tcp_read(char*, int) ()
#3  0x0000000000889a50 in Pipe::reader() ()
#4  0x000000000088dbdd in Pipe::Reader::entry() ()
#5  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d35aa6700) at pthread_create.c:308
#6  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#7  0x0000000000000000 in ?? ()

Thread 80 (Thread 0x7f2d25420700 (LWP 22478)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1  0x0000000000882892 in Pipe::writer() ()
#2  0x000000000088dbbd in Pipe::Writer::entry() ()
#3  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d25420700) at pthread_create.c:308
#4  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#5  0x0000000000000000 in ?? ()

Thread 79 (Thread 0x7f2d2d8b2700 (LWP 22666)):
#0  0x00007f2d3af8c313 in __GI___poll (fds=<optimized out>, nfds=<optimized out>, 
    timeout=<optimized out>) at ../sysdeps/unix/sysv/linux/poll.c:87
#1  0x0000000000876d45 in Pipe::tcp_read_wait() ()
#2  0x0000000000877058 in Pipe::tcp_read(char*, int) ()
#3  0x0000000000889a50 in Pipe::reader() ()
#4  0x000000000088dbdd in Pipe::Reader::entry() ()
#5  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d2d8b2700) at pthread_create.c:308
#6  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#7  0x0000000000000000 in ?? ()

Thread 78 (Thread 0x7f2d2ba19700 (LWP 22669)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1  0x0000000000882892 in Pipe::writer() ()
#2  0x000000000088dbbd in Pipe::Writer::entry() ()
#3  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d2ba19700) at pthread_create.c:308
#4  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#5  0x0000000000000000 in ?? ()

Thread 77 (Thread 0x7f2d27742700 (LWP 24479)):
#0  0x00007f2d3af8c313 in __GI___poll (fds=<optimized out>, nfds=<optimized out>, 
    timeout=<optimized out>) at ../sysdeps/unix/sysv/linux/poll.c:87
#1  0x0000000000876d45 in Pipe::tcp_read_wait() ()
#2  0x0000000000877058 in Pipe::tcp_read(char*, int) ()
#3  0x0000000000889a50 in Pipe::reader() ()
#4  0x000000000088dbdd in Pipe::Reader::entry() ()
---Type <return> to continue, or q <return> to quit---
#5  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d27742700) at pthread_create.c:308
#6  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#7  0x0000000000000000 in ?? ()

Thread 76 (Thread 0x7f2d2531f700 (LWP 24480)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1  0x0000000000882892 in Pipe::writer() ()
#2  0x000000000088dbbd in Pipe::Writer::entry() ()
#3  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d2531f700) at pthread_create.c:308
#4  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#5  0x0000000000000000 in ?? ()

Thread 75 (Thread 0x7f2d265e3700 (LWP 24481)):
#0  0x00007f2d3af8c313 in __GI___poll (fds=<optimized out>, nfds=<optimized out>, 
    timeout=<optimized out>) at ../sysdeps/unix/sysv/linux/poll.c:87
#1  0x0000000000876d45 in Pipe::tcp_read_wait() ()
#2  0x0000000000877058 in Pipe::tcp_read(char*, int) ()
#3  0x0000000000889a50 in Pipe::reader() ()
#4  0x000000000088dbdd in Pipe::Reader::entry() ()
#5  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d265e3700) at pthread_create.c:308
#6  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#7  0x0000000000000000 in ?? ()

Thread 74 (Thread 0x7f2d25c93700 (LWP 24482)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1  0x0000000000882892 in Pipe::writer() ()
#2  0x000000000088dbbd in Pipe::Writer::entry() ()
#3  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d25c93700) at pthread_create.c:308
#4  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#5  0x0000000000000000 in ?? ()

Thread 73 (Thread 0x7f2d36663700 (LWP 24485)):
#0  0x00007f2d3af8c313 in __GI___poll (fds=<optimized out>, nfds=<optimized out>, 
    timeout=<optimized out>) at ../sysdeps/unix/sysv/linux/poll.c:87
#1  0x0000000000876d45 in Pipe::tcp_read_wait() ()
#2  0x0000000000877058 in Pipe::tcp_read(char*, int) ()
#3  0x0000000000889a50 in Pipe::reader() ()
#4  0x000000000088dbdd in Pipe::Reader::entry() ()
#5  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d36663700) at pthread_create.c:308
#6  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#7  0x0000000000000000 in ?? ()

Thread 72 (Thread 0x7f2d36001700 (LWP 24486)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1  0x0000000000882892 in Pipe::writer() ()
#2  0x000000000088dbbd in Pipe::Writer::entry() ()
#3  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d36001700) at pthread_create.c:308
#4  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#5  0x0000000000000000 in ?? ()

Thread 71 (Thread 0x7f2d2c59f700 (LWP 24487)):
#0  0x00007f2d3af8c313 in __GI___poll (fds=<optimized out>, nfds=<optimized out>, 
    timeout=<optimized out>) at ../sysdeps/unix/sysv/linux/poll.c:87
#1  0x0000000000876d45 in Pipe::tcp_read_wait() ()
#2  0x0000000000877058 in Pipe::tcp_read(char*, int) ()
#3  0x0000000000889a50 in Pipe::reader() ()
#4  0x000000000088dbdd in Pipe::Reader::entry() ()
#5  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d2c59f700) at pthread_create.c:308
#6  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#7  0x0000000000000000 in ?? ()

---Type <return> to continue, or q <return> to quit---
Thread 70 (Thread 0x7f2d36764700 (LWP 24488)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1  0x0000000000882892 in Pipe::writer() ()
#2  0x000000000088dbbd in Pipe::Writer::entry() ()
#3  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d36764700) at pthread_create.c:308
#4  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#5  0x0000000000000000 in ?? ()

Thread 69 (Thread 0x7f2d2bb1a700 (LWP 24491)):
#0  0x00007f2d3af8c313 in __GI___poll (fds=<optimized out>, nfds=<optimized out>, 
    timeout=<optimized out>) at ../sysdeps/unix/sysv/linux/poll.c:87
#1  0x0000000000876d45 in Pipe::tcp_read_wait() ()
#2  0x0000000000877058 in Pipe::tcp_read(char*, int) ()
#3  0x0000000000889a50 in Pipe::reader() ()
#4  0x000000000088dbdd in Pipe::Reader::entry() ()
#5  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d2bb1a700) at pthread_create.c:308
#6  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#7  0x0000000000000000 in ?? ()

Thread 68 (Thread 0x7f2d2b918700 (LWP 24492)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1  0x0000000000882892 in Pipe::writer() ()
#2  0x000000000088dbbd in Pipe::Writer::entry() ()
#3  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d2b918700) at pthread_create.c:308
#4  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#5  0x0000000000000000 in ?? ()

Thread 67 (Thread 0x7f2d2422e700 (LWP 24518)):
#0  0x00007f2d3af8c313 in __GI___poll (fds=<optimized out>, nfds=<optimized out>, 
    timeout=<optimized out>) at ../sysdeps/unix/sysv/linux/poll.c:87
#1  0x0000000000876d45 in Pipe::tcp_read_wait() ()
#2  0x0000000000877058 in Pipe::tcp_read(char*, int) ()
#3  0x0000000000889a50 in Pipe::reader() ()
#4  0x000000000088dbdd in Pipe::Reader::entry() ()
#5  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d2422e700) at pthread_create.c:308
#6  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#7  0x0000000000000000 in ?? ()

Thread 66 (Thread 0x7f2d2412d700 (LWP 24519)):
#0  0x00007f2d3af8c313 in __GI___poll (fds=<optimized out>, nfds=<optimized out>, 
    timeout=<optimized out>) at ../sysdeps/unix/sysv/linux/poll.c:87
#1  0x0000000000876d45 in Pipe::tcp_read_wait() ()
#2  0x0000000000877058 in Pipe::tcp_read(char*, int) ()
#3  0x0000000000889a50 in Pipe::reader() ()
#4  0x000000000088dbdd in Pipe::Reader::entry() ()
#5  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d2412d700) at pthread_create.c:308
#6  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#7  0x0000000000000000 in ?? ()

Thread 65 (Thread 0x7f2d2402c700 (LWP 24520)):
#0  0x00007f2d3af8c313 in __GI___poll (fds=<optimized out>, nfds=<optimized out>, 
    timeout=<optimized out>) at ../sysdeps/unix/sysv/linux/poll.c:87
#1  0x0000000000876d45 in Pipe::tcp_read_wait() ()
#2  0x0000000000877058 in Pipe::tcp_read(char*, int) ()
#3  0x0000000000889a50 in Pipe::reader() ()
#4  0x000000000088dbdd in Pipe::Reader::entry() ()
#5  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d2402c700) at pthread_create.c:308
#6  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#7  0x0000000000000000 in ?? ()

Thread 64 (Thread 0x7f2d23f2b700 (LWP 24521)):
#0  0x00007f2d3af8c313 in __GI___poll (fds=<optimized out>, nfds=<optimized out>, 
---Type <return> to continue, or q <return> to quit---
    timeout=<optimized out>) at ../sysdeps/unix/sysv/linux/poll.c:87
#1  0x0000000000876d45 in Pipe::tcp_read_wait() ()
#2  0x0000000000877058 in Pipe::tcp_read(char*, int) ()
#3  0x0000000000889a50 in Pipe::reader() ()
#4  0x000000000088dbdd in Pipe::Reader::entry() ()
#5  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d23f2b700) at pthread_create.c:308
#6  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#7  0x0000000000000000 in ?? ()

Thread 63 (Thread 0x7f2d25824700 (LWP 24522)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1  0x0000000000882892 in Pipe::writer() ()
#2  0x000000000088dbbd in Pipe::Writer::entry() ()
#3  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d25824700) at pthread_create.c:308
#4  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#5  0x0000000000000000 in ?? ()

Thread 62 (Thread 0x7f2d2501c700 (LWP 24523)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1  0x0000000000882892 in Pipe::writer() ()
#2  0x000000000088dbbd in Pipe::Writer::entry() ()
#3  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d2501c700) at pthread_create.c:308
#4  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#5  0x0000000000000000 in ?? ()

Thread 61 (Thread 0x7f2d25723700 (LWP 24524)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1  0x0000000000882892 in Pipe::writer() ()
#2  0x000000000088dbbd in Pipe::Writer::entry() ()
#3  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d25723700) at pthread_create.c:308
#4  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#5  0x0000000000000000 in ?? ()

Thread 60 (Thread 0x7f2d25a26700 (LWP 24525)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1  0x0000000000882892 in Pipe::writer() ()
#2  0x000000000088dbbd in Pipe::Writer::entry() ()
#3  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d25a26700) at pthread_create.c:308
#4  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#5  0x0000000000000000 in ?? ()

Thread 59 (Thread 0x7f2d24834700 (LWP 24526)):
#0  0x00007f2d3af8c313 in __GI___poll (fds=<optimized out>, nfds=<optimized out>, 
    timeout=<optimized out>) at ../sysdeps/unix/sysv/linux/poll.c:87
#1  0x0000000000876d45 in Pipe::tcp_read_wait() ()
#2  0x0000000000877058 in Pipe::tcp_read(char*, int) ()
#3  0x0000000000889a50 in Pipe::reader() ()
#4  0x000000000088dbdd in Pipe::Reader::entry() ()
#5  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d24834700) at pthread_create.c:308
#6  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#7  0x0000000000000000 in ?? ()

Thread 58 (Thread 0x7f2d2432f700 (LWP 24527)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1  0x0000000000882892 in Pipe::writer() ()
#2  0x000000000088dbbd in Pipe::Writer::entry() ()
#3  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d2432f700) at pthread_create.c:308
#4  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#5  0x0000000000000000 in ?? ()
---Type <return> to continue, or q <return> to quit---

Thread 57 (Thread 0x7f2d2b716700 (LWP 24528)):
#0  0x00007f2d3af8c313 in __GI___poll (fds=<optimized out>, nfds=<optimized out>, 
    timeout=<optimized out>) at ../sysdeps/unix/sysv/linux/poll.c:87
#1  0x0000000000876d45 in Pipe::tcp_read_wait() ()
#2  0x0000000000877058 in Pipe::tcp_read(char*, int) ()
#3  0x0000000000889a50 in Pipe::reader() ()
#4  0x000000000088dbdd in Pipe::Reader::entry() ()
#5  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d2b716700) at pthread_create.c:308
#6  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#7  0x0000000000000000 in ?? ()

Thread 56 (Thread 0x7f2d2b40c700 (LWP 24529)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1  0x0000000000882892 in Pipe::writer() ()
#2  0x000000000088dbbd in Pipe::Writer::entry() ()
#3  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d2b40c700) at pthread_create.c:308
#4  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#5  0x0000000000000000 in ?? ()

Thread 55 (Thread 0x7f2d2bd1c700 (LWP 24530)):
#0  0x00007f2d3af8c313 in __GI___poll (fds=<optimized out>, nfds=<optimized out>, 
    timeout=<optimized out>) at ../sysdeps/unix/sysv/linux/poll.c:87
#1  0x0000000000876d45 in Pipe::tcp_read_wait() ()
#2  0x0000000000877058 in Pipe::tcp_read(char*, int) ()
#3  0x0000000000889a50 in Pipe::reader() ()
#4  0x000000000088dbdd in Pipe::Reader::entry() ()
#5  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d2bd1c700) at pthread_create.c:308
#6  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#7  0x0000000000000000 in ?? ()

Thread 54 (Thread 0x7f2d2bc1b700 (LWP 24531)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1  0x0000000000882892 in Pipe::writer() ()
#2  0x000000000088dbbd in Pipe::Writer::entry() ()
#3  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d2bc1b700) at pthread_create.c:308
#4  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#5  0x0000000000000000 in ?? ()

Thread 53 (Thread 0x7f2d36360700 (LWP 24532)):
#0  0x00007f2d3af8c313 in __GI___poll (fds=<optimized out>, nfds=<optimized out>, 
    timeout=<optimized out>) at ../sysdeps/unix/sysv/linux/poll.c:87
#1  0x0000000000876d45 in Pipe::tcp_read_wait() ()
#2  0x0000000000877058 in Pipe::tcp_read(char*, int) ()
#3  0x0000000000889a50 in Pipe::reader() ()
#4  0x000000000088dbdd in Pipe::Reader::entry() ()
#5  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d36360700) at pthread_create.c:308
#6  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#7  0x0000000000000000 in ?? ()

Thread 52 (Thread 0x7f2d3061d700 (LWP 24533)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1  0x0000000000882892 in Pipe::writer() ()
#2  0x000000000088dbbd in Pipe::Writer::entry() ()
#3  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d3061d700) at pthread_create.c:308
#4  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#5  0x0000000000000000 in ?? ()

Thread 51 (Thread 0x7f2d22b17700 (LWP 24536)):
#0  0x00007f2d3af8c313 in __GI___poll (fds=<optimized out>, nfds=<optimized out>, 
    timeout=<optimized out>) at ../sysdeps/unix/sysv/linux/poll.c:87
---Type <return> to continue, or q <return> to quit---
#1  0x0000000000876d45 in Pipe::tcp_read_wait() ()
#2  0x0000000000877058 in Pipe::tcp_read(char*, int) ()
#3  0x0000000000889a50 in Pipe::reader() ()
#4  0x000000000088dbdd in Pipe::Reader::entry() ()
#5  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d22b17700) at pthread_create.c:308
#6  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#7  0x0000000000000000 in ?? ()

Thread 50 (Thread 0x7f2d22a16700 (LWP 24537)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1  0x0000000000882892 in Pipe::writer() ()
#2  0x000000000088dbbd in Pipe::Writer::entry() ()
#3  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d22a16700) at pthread_create.c:308
#4  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#5  0x0000000000000000 in ?? ()

Thread 49 (Thread 0x7f2d2c9a3700 (LWP 24538)):
#0  0x00007f2d3af8c313 in __GI___poll (fds=<optimized out>, nfds=<optimized out>, 
    timeout=<optimized out>) at ../sysdeps/unix/sysv/linux/poll.c:87
#1  0x0000000000876d45 in Pipe::tcp_read_wait() ()
#2  0x0000000000877058 in Pipe::tcp_read(char*, int) ()
#3  0x0000000000889a50 in Pipe::reader() ()
#4  0x000000000088dbdd in Pipe::Reader::entry() ()
#5  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d2c9a3700) at pthread_create.c:308
#6  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#7  0x0000000000000000 in ?? ()

Thread 48 (Thread 0x7f2d29ac8700 (LWP 24539)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1  0x0000000000882892 in Pipe::writer() ()
#2  0x000000000088dbbd in Pipe::Writer::entry() ()
#3  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d29ac8700) at pthread_create.c:308
#4  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#5  0x0000000000000000 in ?? ()

Thread 47 (Thread 0x7f2d2be1d700 (LWP 24540)):
#0  0x00007f2d3af8c313 in __GI___poll (fds=<optimized out>, nfds=<optimized out>, 
    timeout=<optimized out>) at ../sysdeps/unix/sysv/linux/poll.c:87
#1  0x0000000000876d45 in Pipe::tcp_read_wait() ()
#2  0x0000000000877058 in Pipe::tcp_read(char*, int) ()
#3  0x0000000000889a50 in Pipe::reader() ()
#4  0x000000000088dbdd in Pipe::Reader::entry() ()
#5  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d2be1d700) at pthread_create.c:308
#6  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#7  0x0000000000000000 in ?? ()

Thread 46 (Thread 0x7f2d2d0aa700 (LWP 24541)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1  0x0000000000882892 in Pipe::writer() ()
#2  0x000000000088dbbd in Pipe::Writer::entry() ()
#3  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d2d0aa700) at pthread_create.c:308
#4  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#5  0x0000000000000000 in ?? ()

Thread 45 (Thread 0x7f2d2c7a1700 (LWP 24545)):
#0  0x00007f2d3af8c313 in __GI___poll (fds=<optimized out>, nfds=<optimized out>, 
    timeout=<optimized out>) at ../sysdeps/unix/sysv/linux/poll.c:87
#1  0x0000000000876d45 in Pipe::tcp_read_wait() ()
#2  0x0000000000877058 in Pipe::tcp_read(char*, int) ()
#3  0x0000000000889a50 in Pipe::reader() ()
#4  0x000000000088dbdd in Pipe::Reader::entry() ()
---Type <return> to continue, or q <return> to quit---
#5  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d2c7a1700) at pthread_create.c:308
#6  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#7  0x0000000000000000 in ?? ()

Thread 44 (Thread 0x7f2d23a26700 (LWP 24546)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1  0x0000000000882892 in Pipe::writer() ()
#2  0x000000000088dbbd in Pipe::Writer::entry() ()
#3  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d23a26700) at pthread_create.c:308
#4  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#5  0x0000000000000000 in ?? ()

Thread 43 (Thread 0x7f2d24733700 (LWP 24547)):
#0  0x00007f2d3af8c313 in __GI___poll (fds=<optimized out>, nfds=<optimized out>, 
    timeout=<optimized out>) at ../sysdeps/unix/sysv/linux/poll.c:87
#1  0x0000000000876d45 in Pipe::tcp_read_wait() ()
#2  0x0000000000877058 in Pipe::tcp_read(char*, int) ()
#3  0x0000000000889a50 in Pipe::reader() ()
#4  0x000000000088dbdd in Pipe::Reader::entry() ()
#5  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d24733700) at pthread_create.c:308
#6  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#7  0x0000000000000000 in ?? ()

Thread 42 (Thread 0x7f2d24632700 (LWP 24548)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1  0x0000000000882892 in Pipe::writer() ()
#2  0x000000000088dbbd in Pipe::Writer::entry() ()
#3  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d24632700) at pthread_create.c:308
#4  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#5  0x0000000000000000 in ?? ()

Thread 41 (Thread 0x7f2d23723700 (LWP 24549)):
#0  0x00007f2d3af8c313 in __GI___poll (fds=<optimized out>, nfds=<optimized out>, 
    timeout=<optimized out>) at ../sysdeps/unix/sysv/linux/poll.c:87
#1  0x0000000000876d45 in Pipe::tcp_read_wait() ()
#2  0x0000000000877058 in Pipe::tcp_read(char*, int) ()
#3  0x0000000000889a50 in Pipe::reader() ()
#4  0x000000000088dbdd in Pipe::Reader::entry() ()
#5  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d23723700) at pthread_create.c:308
#6  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#7  0x0000000000000000 in ?? ()

Thread 40 (Thread 0x7f2d23521700 (LWP 24550)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1  0x0000000000882892 in Pipe::writer() ()
#2  0x000000000088dbbd in Pipe::Writer::entry() ()
#3  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d23521700) at pthread_create.c:308
#4  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#5  0x0000000000000000 in ?? ()

Thread 39 (Thread 0x7f2d22713700 (LWP 24554)):
#0  0x00007f2d3af8c313 in __GI___poll (fds=<optimized out>, nfds=<optimized out>, 
    timeout=<optimized out>) at ../sysdeps/unix/sysv/linux/poll.c:87
#1  0x0000000000876d45 in Pipe::tcp_read_wait() ()
#2  0x0000000000877058 in Pipe::tcp_read(char*, int) ()
#3  0x0000000000889a50 in Pipe::reader() ()
#4  0x000000000088dbdd in Pipe::Reader::entry() ()
#5  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d22713700) at pthread_create.c:308
#6  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#7  0x0000000000000000 in ?? ()

---Type <return> to continue, or q <return> to quit---
Thread 38 (Thread 0x7f2d22612700 (LWP 24555)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1  0x0000000000882892 in Pipe::writer() ()
#2  0x000000000088dbbd in Pipe::Writer::entry() ()
#3  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d22612700) at pthread_create.c:308
#4  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#5  0x0000000000000000 in ?? ()

Thread 37 (Thread 0x7f2d35dbe700 (LWP 24559)):
#0  0x00007f2d3af8c313 in __GI___poll (fds=<optimized out>, nfds=<optimized out>, 
    timeout=<optimized out>) at ../sysdeps/unix/sysv/linux/poll.c:87
#1  0x0000000000876d45 in Pipe::tcp_read_wait() ()
#2  0x0000000000877058 in Pipe::tcp_read(char*, int) ()
#3  0x0000000000889a50 in Pipe::reader() ()
#4  0x000000000088dbdd in Pipe::Reader::entry() ()
#5  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d35dbe700) at pthread_create.c:308
#6  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#7  0x0000000000000000 in ?? ()

Thread 36 (Thread 0x7f2d358a4700 (LWP 24560)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1  0x0000000000882892 in Pipe::writer() ()
#2  0x000000000088dbbd in Pipe::Writer::entry() ()
#3  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d358a4700) at pthread_create.c:308
#4  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#5  0x0000000000000000 in ?? ()

Thread 35 (Thread 0x7f2d25521700 (LWP 24561)):
#0  0x00007f2d3af8c313 in __GI___poll (fds=<optimized out>, nfds=<optimized out>, 
    timeout=<optimized out>) at ../sysdeps/unix/sysv/linux/poll.c:87
#1  0x0000000000876d45 in Pipe::tcp_read_wait() ()
#2  0x0000000000877058 in Pipe::tcp_read(char*, int) ()
#3  0x0000000000889a50 in Pipe::reader() ()
#4  0x000000000088dbdd in Pipe::Reader::entry() ()
#5  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d25521700) at pthread_create.c:308
#6  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#7  0x0000000000000000 in ?? ()

Thread 34 (Thread 0x7f2d36966700 (LWP 24562)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1  0x0000000000882892 in Pipe::writer() ()
#2  0x000000000088dbbd in Pipe::Writer::entry() ()
#3  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d36966700) at pthread_create.c:308
#4  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#5  0x0000000000000000 in ?? ()

Thread 33 (Thread 0x7f2d23e2a700 (LWP 24563)):
#0  0x00007f2d3af8c313 in __GI___poll (fds=<optimized out>, nfds=<optimized out>, 
    timeout=<optimized out>) at ../sysdeps/unix/sysv/linux/poll.c:87
#1  0x0000000000876d45 in Pipe::tcp_read_wait() ()
#2  0x0000000000877058 in Pipe::tcp_read(char*, int) ()
#3  0x0000000000889a50 in Pipe::reader() ()
#4  0x000000000088dbdd in Pipe::Reader::entry() ()
#5  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d23e2a700) at pthread_create.c:308
#6  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#7  0x0000000000000000 in ?? ()

Thread 32 (Thread 0x7f2d23d29700 (LWP 24564)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1  0x0000000000882892 in Pipe::writer() ()
---Type <return> to continue, or q <return> to quit---
#2  0x000000000088dbbd in Pipe::Writer::entry() ()
#3  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d23d29700) at pthread_create.c:308
#4  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#5  0x0000000000000000 in ?? ()

Thread 31 (Thread 0x7f2d27a9c700 (LWP 24565)):
#0  0x00007f2d3af8c313 in __GI___poll (fds=<optimized out>, nfds=<optimized out>, 
    timeout=<optimized out>) at ../sysdeps/unix/sysv/linux/poll.c:87
#1  0x0000000000876d45 in Pipe::tcp_read_wait() ()
#2  0x0000000000877058 in Pipe::tcp_read(char*, int) ()
#3  0x0000000000889a50 in Pipe::reader() ()
#4  0x000000000088dbdd in Pipe::Reader::entry() ()
#5  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d27a9c700) at pthread_create.c:308
#6  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#7  0x0000000000000000 in ?? ()

Thread 30 (Thread 0x7f2d3770d700 (LWP 24566)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1  0x0000000000882892 in Pipe::writer() ()
#2  0x000000000088dbbd in Pipe::Writer::entry() ()
#3  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d3770d700) at pthread_create.c:308
#4  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#5  0x0000000000000000 in ?? ()

Thread 29 (Thread 0x7f2d25b92700 (LWP 24567)):
#0  0x00007f2d3af8c313 in __GI___poll (fds=<optimized out>, nfds=<optimized out>, 
    timeout=<optimized out>) at ../sysdeps/unix/sysv/linux/poll.c:87
#1  0x0000000000876d45 in Pipe::tcp_read_wait() ()
#2  0x0000000000877058 in Pipe::tcp_read(char*, int) ()
#3  0x0000000000889a50 in Pipe::reader() ()
#4  0x000000000088dbdd in Pipe::Reader::entry() ()
#5  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d25b92700) at pthread_create.c:308
#6  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#7  0x0000000000000000 in ?? ()

Thread 28 (Thread 0x7f2d25925700 (LWP 24568)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1  0x0000000000882892 in Pipe::writer() ()
#2  0x000000000088dbbd in Pipe::Writer::entry() ()
#3  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d25925700) at pthread_create.c:308
#4  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#5  0x0000000000000000 in ?? ()

Thread 27 (Thread 0x7f2d2f2ec700 (LWP 24571)):
#0  0x00007f2d3af8c313 in __GI___poll (fds=<optimized out>, nfds=<optimized out>, 
    timeout=<optimized out>) at ../sysdeps/unix/sysv/linux/poll.c:87
#1  0x0000000000876d45 in Pipe::tcp_read_wait() ()
#2  0x0000000000877058 in Pipe::tcp_read(char*, int) ()
#3  0x0000000000889a50 in Pipe::reader() ()
#4  0x000000000088dbdd in Pipe::Reader::entry() ()
#5  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d2f2ec700) at pthread_create.c:308
#6  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#7  0x0000000000000000 in ?? ()

Thread 26 (Thread 0x7f2d2dbac700 (LWP 24573)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1  0x0000000000882892 in Pipe::writer() ()
#2  0x000000000088dbbd in Pipe::Writer::entry() ()
#3  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d2dbac700) at pthread_create.c:308
#4  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#5  0x0000000000000000 in ?? ()
---Type <return> to continue, or q <return> to quit---

Thread 25 (Thread 0x7f2d30c5d700 (LWP 24574)):
#0  0x00007f2d3af8c313 in __GI___poll (fds=<optimized out>, nfds=<optimized out>, 
    timeout=<optimized out>) at ../sysdeps/unix/sysv/linux/poll.c:87
#1  0x0000000000876d45 in Pipe::tcp_read_wait() ()
#2  0x0000000000877058 in Pipe::tcp_read(char*, int) ()
#3  0x0000000000889a50 in Pipe::reader() ()
#4  0x000000000088dbdd in Pipe::Reader::entry() ()
#5  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d30c5d700) at pthread_create.c:308
#6  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#7  0x0000000000000000 in ?? ()

Thread 24 (Thread 0x7f2d3071e700 (LWP 24575)):
#0  0x00007f2d3af8c313 in __GI___poll (fds=<optimized out>, nfds=<optimized out>, 
    timeout=<optimized out>) at ../sysdeps/unix/sysv/linux/poll.c:87
#1  0x0000000000876d45 in Pipe::tcp_read_wait() ()
#2  0x0000000000877058 in Pipe::tcp_read(char*, int) ()
#3  0x0000000000889a50 in Pipe::reader() ()
#4  0x000000000088dbdd in Pipe::Reader::entry() ()
#5  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d3071e700) at pthread_create.c:308
#6  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#7  0x0000000000000000 in ?? ()

Thread 23 (Thread 0x7f2d2c6a0700 (LWP 24576)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1  0x0000000000882892 in Pipe::writer() ()
#2  0x000000000088dbbd in Pipe::Writer::entry() ()
#3  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d2c6a0700) at pthread_create.c:308
#4  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#5  0x0000000000000000 in ?? ()

Thread 22 (Thread 0x7f2d30b5c700 (LWP 24577)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1  0x0000000000882892 in Pipe::writer() ()
#2  0x000000000088dbbd in Pipe::Writer::entry() ()
#3  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d30b5c700) at pthread_create.c:308
#4  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#5  0x0000000000000000 in ?? ()

Thread 21 (Thread 0x7f2d359a5700 (LWP 24580)):
#0  0x00007f2d3af8c313 in __GI___poll (fds=<optimized out>, nfds=<optimized out>, 
    timeout=<optimized out>) at ../sysdeps/unix/sysv/linux/poll.c:87
#1  0x0000000000876d45 in Pipe::tcp_read_wait() ()
#2  0x0000000000877058 in Pipe::tcp_read(char*, int) ()
#3  0x0000000000889a50 in Pipe::reader() ()
#4  0x000000000088dbdd in Pipe::Reader::entry() ()
#5  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d359a5700) at pthread_create.c:308
#6  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#7  0x0000000000000000 in ?? ()

Thread 20 (Thread 0x7f2d36865700 (LWP 24581)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1  0x0000000000882892 in Pipe::writer() ()
#2  0x000000000088dbbd in Pipe::Writer::entry() ()
#3  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d36865700) at pthread_create.c:308
#4  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#5  0x0000000000000000 in ?? ()

Thread 19 (Thread 0x7f2d2d7b1700 (LWP 24583)):
#0  0x00007f2d3af8c313 in __GI___poll (fds=<optimized out>, nfds=<optimized out>, 
    timeout=<optimized out>) at ../sysdeps/unix/sysv/linux/poll.c:87
---Type <return> to continue, or q <return> to quit---
#1  0x0000000000876d45 in Pipe::tcp_read_wait() ()
#2  0x0000000000877058 in Pipe::tcp_read(char*, int) ()
#3  0x0000000000889a50 in Pipe::reader() ()
#4  0x000000000088dbdd in Pipe::Reader::entry() ()
#5  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d2d7b1700) at pthread_create.c:308
#6  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#7  0x0000000000000000 in ?? ()

Thread 18 (Thread 0x7f2d2d4ae700 (LWP 24584)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1  0x0000000000882892 in Pipe::writer() ()
#2  0x000000000088dbbd in Pipe::Writer::entry() ()
#3  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d2d4ae700) at pthread_create.c:308
#4  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#5  0x0000000000000000 in ?? ()

Thread 17 (Thread 0x7f2d2d5af700 (LWP 24586)):
#0  0x00007f2d3af8c313 in __GI___poll (fds=<optimized out>, nfds=<optimized out>, 
    timeout=<optimized out>) at ../sysdeps/unix/sysv/linux/poll.c:87
#1  0x0000000000876d45 in Pipe::tcp_read_wait() ()
#2  0x0000000000877058 in Pipe::tcp_read(char*, int) ()
#3  0x0000000000889a50 in Pipe::reader() ()
#4  0x000000000088dbdd in Pipe::Reader::entry() ()
#5  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d2d5af700) at pthread_create.c:308
#6  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#7  0x0000000000000000 in ?? ()

Thread 16 (Thread 0x7f2d2d3ad700 (LWP 24587)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1  0x0000000000882892 in Pipe::writer() ()
#2  0x000000000088dbbd in Pipe::Writer::entry() ()
#3  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d2d3ad700) at pthread_create.c:308
#4  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#5  0x0000000000000000 in ?? ()

Thread 15 (Thread 0x7f2d24935700 (LWP 24591)):
#0  0x00007f2d3af8c313 in __GI___poll (fds=<optimized out>, nfds=<optimized out>, 
    timeout=<optimized out>) at ../sysdeps/unix/sysv/linux/poll.c:87
#1  0x0000000000876d45 in Pipe::tcp_read_wait() ()
#2  0x0000000000877058 in Pipe::tcp_read(char*, int) ()
#3  0x0000000000889a50 in Pipe::reader() ()
#4  0x000000000088dbdd in Pipe::Reader::entry() ()
#5  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d24935700) at pthread_create.c:308
#6  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#7  0x0000000000000000 in ?? ()

Thread 14 (Thread 0x7f2d266e4700 (LWP 24592)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1  0x0000000000882892 in Pipe::writer() ()
#2  0x000000000088dbbd in Pipe::Writer::entry() ()
#3  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d266e4700) at pthread_create.c:308
#4  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#5  0x0000000000000000 in ?? ()

Thread 13 (Thread 0x7f2d22915700 (LWP 24593)):
#0  0x00007f2d3af8c313 in __GI___poll (fds=<optimized out>, nfds=<optimized out>, 
    timeout=<optimized out>) at ../sysdeps/unix/sysv/linux/poll.c:87
#1  0x0000000000876d45 in Pipe::tcp_read_wait() ()
#2  0x0000000000877058 in Pipe::tcp_read(char*, int) ()
#3  0x0000000000889a50 in Pipe::reader() ()
#4  0x000000000088dbdd in Pipe::Reader::entry() ()
---Type <return> to continue, or q <return> to quit---
#5  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d22915700) at pthread_create.c:308
#6  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#7  0x0000000000000000 in ?? ()

Thread 12 (Thread 0x7f2d22814700 (LWP 24594)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1  0x0000000000882892 in Pipe::writer() ()
#2  0x000000000088dbbd in Pipe::Writer::entry() ()
#3  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d22814700) at pthread_create.c:308
#4  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#5  0x0000000000000000 in ?? ()

Thread 11 (Thread 0x7f2d2af05700 (LWP 24595)):
#0  0x00007f2d3af8c313 in __GI___poll (fds=<optimized out>, nfds=<optimized out>, 
    timeout=<optimized out>) at ../sysdeps/unix/sysv/linux/poll.c:87
#1  0x0000000000876d45 in Pipe::tcp_read_wait() ()
#2  0x0000000000877058 in Pipe::tcp_read(char*, int) ()
#3  0x0000000000889a50 in Pipe::reader() ()
#4  0x000000000088dbdd in Pipe::Reader::entry() ()
#5  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d2af05700) at pthread_create.c:308
#6  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#7  0x0000000000000000 in ?? ()

Thread 10 (Thread 0x7f2d27d92700 (LWP 24596)):
#0  0x00007f2d3af8c313 in __GI___poll (fds=<optimized out>, nfds=<optimized out>, 
    timeout=<optimized out>) at ../sysdeps/unix/sysv/linux/poll.c:87
#1  0x0000000000876d45 in Pipe::tcp_read_wait() ()
#2  0x0000000000877058 in Pipe::tcp_read(char*, int) ()
#3  0x0000000000889a50 in Pipe::reader() ()
#4  0x000000000088dbdd in Pipe::Reader::entry() ()
#5  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d27d92700) at pthread_create.c:308
#6  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#7  0x0000000000000000 in ?? ()

Thread 9 (Thread 0x7f2d27641700 (LWP 24597)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1  0x0000000000882892 in Pipe::writer() ()
#2  0x000000000088dbbd in Pipe::Writer::entry() ()
#3  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d27641700) at pthread_create.c:308
#4  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#5  0x0000000000000000 in ?? ()

Thread 8 (Thread 0x7f2d27020700 (LWP 24598)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1  0x0000000000882892 in Pipe::writer() ()
#2  0x000000000088dbbd in Pipe::Writer::entry() ()
#3  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d27020700) at pthread_create.c:308
#4  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#5  0x0000000000000000 in ?? ()

Thread 7 (Thread 0x7f2d25622700 (LWP 24599)):
#0  0x00007f2d3af8c313 in __GI___poll (fds=<optimized out>, nfds=<optimized out>, 
    timeout=<optimized out>) at ../sysdeps/unix/sysv/linux/poll.c:87
#1  0x0000000000876d45 in Pipe::tcp_read_wait() ()
#2  0x0000000000877058 in Pipe::tcp_read(char*, int) ()
#3  0x0000000000889a50 in Pipe::reader() ()
#4  0x000000000088dbdd in Pipe::Reader::entry() ()
#5  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d25622700) at pthread_create.c:308
#6  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#7  0x0000000000000000 in ?? ()

---Type <return> to continue, or q <return> to quit---
Thread 6 (Thread 0x7f2d372f9700 (LWP 24600)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1  0x0000000000882892 in Pipe::writer() ()
#2  0x000000000088dbbd in Pipe::Writer::entry() ()
#3  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d372f9700) at pthread_create.c:308
#4  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#5  0x0000000000000000 in ?? ()

Thread 5 (Thread 0x7f2d23622700 (LWP 24601)):
#0  0x00007f2d3af8c313 in __GI___poll (fds=<optimized out>, nfds=<optimized out>, 
    timeout=<optimized out>) at ../sysdeps/unix/sysv/linux/poll.c:87
#1  0x0000000000876d45 in Pipe::tcp_read_wait() ()
#2  0x0000000000877058 in Pipe::tcp_read(char*, int) ()
#3  0x0000000000889a50 in Pipe::reader() ()
#4  0x000000000088dbdd in Pipe::Reader::entry() ()
#5  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d23622700) at pthread_create.c:308
#6  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#7  0x0000000000000000 in ?? ()

Thread 4 (Thread 0x7f2d23420700 (LWP 24602)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1  0x0000000000882892 in Pipe::writer() ()
#2  0x000000000088dbbd in Pipe::Writer::entry() ()
#3  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d23420700) at pthread_create.c:308
#4  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#5  0x0000000000000000 in ?? ()

Thread 3 (Thread 0x7f2d267e5700 (LWP 24603)):
#0  0x00007f2d3af8c313 in __GI___poll (fds=<optimized out>, nfds=<optimized out>, 
    timeout=<optimized out>) at ../sysdeps/unix/sysv/linux/poll.c:87
#1  0x0000000000876d45 in Pipe::tcp_read_wait() ()
#2  0x0000000000877058 in Pipe::tcp_read(char*, int) ()
#3  0x0000000000889a50 in Pipe::reader() ()
#4  0x000000000088dbdd in Pipe::Reader::entry() ()
#5  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d267e5700) at pthread_create.c:308
#6  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#7  0x0000000000000000 in ?? ()

Thread 2 (Thread 0x7f2d27b9d700 (LWP 24604)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1  0x0000000000882892 in Pipe::writer() ()
#2  0x000000000088dbbd in Pipe::Writer::entry() ()
#3  0x00007f2d3c5e4e9a in start_thread (arg=0x7f2d27b9d700) at pthread_create.c:308
#4  0x00007f2d3af97ccd in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#5  0x0000000000000000 in ?? ()

Thread 1 (Thread 0x7f2d3cec0780 (LWP 28244)):
#0  0x00007f2d3c5e6148 in pthread_join (threadid=139832117843712, thread_return=0x0)
    at pthread_join.c:89
#1  0x000000000076b572 in Thread::join(void**) ()
#2  0x0000000000862332 in DispatchQueue::wait() ()
#3  0x0000000000760fee in SimpleMessenger::wait() ()
#4  0x000000000053a4bf in main ()

Actions #1

Updated by Sage Weil over 9 years ago

another host:

Loaded symbols for /usr/lib/x86_64-linux-gnu/nss/libfreebl3.so
Reading symbols from /usr/lib/ceph/erasure-code/libec_jerasure.so...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/ceph/erasure-code/libec_jerasure.so
Reading symbols from /usr/lib/ceph/erasure-code/libec_jerasure_sse3.so...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/ceph/erasure-code/libec_jerasure_sse3.so
Reading symbols from /usr/lib/ceph/erasure-code/libec_lrc.so...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/ceph/erasure-code/libec_lrc.so
0x00007fd63bd69148 in pthread_join (threadid=140557960238848, 
    thread_return=0x0) at pthread_join.c:89
89    pthread_join.c: No such file or directory.
(gdb) thr app all bt

Thread 77 (Thread 0x7fd6399cf700 (LWP 18257)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1  0x00000000006cf5db in ceph::log::Log::entry() ()
#2  0x00007fd63bd67e9a in start_thread (arg=0x7fd6399cf700)
    at pthread_create.c:308
#3  0x00007fd63a71accd in clone ()
    at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#4  0x0000000000000000 in ?? ()

Thread 76 (Thread 0x7fd638a85700 (LWP 18258)):
#0  pthread_cond_timedwait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:215
#1  0x00000000007953e6 in CephContextServiceThread::entry() ()
#2  0x00007fd63bd67e9a in start_thread (arg=0x7fd638a85700)
    at pthread_create.c:308
#3  0x00007fd63a71accd in clone ()
    at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#4  0x0000000000000000 in ?? ()

Thread 75 (Thread 0x7fd638284700 (LWP 18259)):
#0  __lll_lock_wait ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:132
#1  0x00007fd63bd6a065 in _L_lock_858 ()
   from /lib/x86_64-linux-gnu/libpthread.so.0
#2  0x00007fd63bd69eba in __pthread_mutex_lock (mutex=0x4b04048)
    at pthread_mutex_lock.c:61
#3  0x0000000000741e23 in Mutex::Lock(bool) ()
#4  0x000000000056f197 in Monitor::do_admin_command(std::string, std::map<std::string, boost::variant<std::string, bool, long, double, std::vector<std::string, std::allocator<std::string> >, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_>, std::less<std::string>, std::allocator<std::pair<std::string const, boost::variant<std::string, bool, long, double, std::vector<std::string, std::allocator<std::string> >, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> > > >&, std::string, std::ostream&) ()
#5  0x00000000005a05fd in AdminHook::call(std::string, std::map<std::string, boost::variant<std::string, bool, long, double, std::vector<std::string, std::allocator<std::string> >, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_>, std::less<std::string>, std::allocator<std::pair<std::string const, boost::variant<std::string, bool, long, double, std::vector<std::string, std::allocator<std::string> >, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> > > >&, std::string, ceph::buffer::list&) ()
#6  0x000000000077aa2a in AdminSocket::do_accept() ()
---Type <return> to continue, or q <return> to quit---
#7  0x000000000077c050 in AdminSocket::entry() ()
#8  0x00007fd63bd67e9a in start_thread (arg=0x7fd638284700)
    at pthread_create.c:308
#9  0x00007fd63a71accd in clone ()
    at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#10 0x0000000000000000 in ?? ()

Thread 74 (Thread 0x7fd6371bb700 (LWP 18268)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1  0x00000000006c4400 in Finisher::finisher_thread_entry() ()
#2  0x00007fd63bd67e9a in start_thread (arg=0x7fd6371bb700)
    at pthread_create.c:308
#3  0x00007fd63a71accd in clone ()
    at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#4  0x0000000000000000 in ?? ()

Thread 73 (Thread 0x7fd634ca8700 (LWP 18302)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1  0x00000000007618d9 in SimpleMessenger::reaper_entry() ()
#2  0x000000000076840d in SimpleMessenger::ReaperThread::entry() ()
#3  0x00007fd63bd67e9a in start_thread (arg=0x7fd634ca8700)
    at pthread_create.c:308
#4  0x00007fd63a71accd in clone ()
    at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#5  0x0000000000000000 in ?? ()

Thread 72 (Thread 0x7fd6344a7700 (LWP 18303)):
#0  __lll_lock_wait ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S:132
#1  0x00007fd63bd712b8 in _L_cond_lock_874 ()
   from /lib/x86_64-linux-gnu/libpthread.so.0
#2  0x00007fd63bd71088 in __pthread_mutex_cond_lock (mutex=0x4b04048)
    at ../nptl/pthread_mutex_lock.c:61
#3  0x00007fd63bd6c1b2 in pthread_cond_timedwait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S:302
#4  0x000000000077223b in SafeTimer::timer_thread() ()
#5  0x0000000000774c2d in SafeTimerThread::entry() ()
#6  0x00007fd63bd67e9a in start_thread (arg=0x7fd6344a7700)
    at pthread_create.c:308
#7  0x00007fd63a71accd in clone ()
    at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#8  0x0000000000000000 in ?? ()

Thread 71 (Thread 0x7fd633ca6700 (LWP 18304)):
#0  0x00007fd63af8b7dc in std::ostream::sentry::sentry(std::ostream&) ()
   from /usr/lib/x86_64-linux-gnu/libstdc++.so.6
#1  0x00007fd63af8bee9 in std::basic_ostream<char, std::char_traits<char> >& std::__ostream_insert<char, std::char_traits<char> >(std::basic_ostream<char, std::char_traits<char> >&, char const*, long) ()
   from /usr/lib/x86_64-linux-gnu/libstdc++.so.6
#2  0x00007fd63af8c2ef in std::basic_ostream<char, std::char_traits<char> >& std::operator<< <std::char_traits<char> >(std::basic_ostream<char, std::char_traits<char> >&, char const*) () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6
#3  0x0000000000613ab9 in MDSMonitor::get_health(std::list<std::pair<health_status_t, std::string>, std::allocator<std::pair<health_status_t, std::string> > >&, std::list<std::pair<health_status_t, std::string>, std::allocator<std::pair<health_status_t, std::string> > >*) const ()
#4  0x00000000005736e1 in Monitor::get_health(std::string&, ceph::buffer::list*, ceph::Formatter*) ()
#5  0x000000000057533e in Monitor::get_cluster_status(std::basic_stringstream<char, std::char_traits<char>, std::allocator<char> >&, ceph::Formatter*) ()
#6  0x00000000005819d7 in Monitor::handle_command(MMonCommand*) ()
---Type <return> to continue, or q <return> to quit---
#7  0x000000000058acdc in Monitor::dispatch(MonSession*, Message*, bool) ()
#8  0x0000000000589068 in Monitor::_ms_dispatch(Message*) ()
#9  0x00000000005a3db2 in Monitor::ms_dispatch(Message*) ()
#10 0x0000000000865817 in Messenger::ms_deliver_dispatch(Message*) ()
#11 0x00000000008629ba in DispatchQueue::entry() ()
#12 0x000000000076742d in DispatchQueue::DispatchThread::entry() ()
#13 0x00007fd63bd67e9a in start_thread (arg=0x7fd633ca6700)
    at pthread_create.c:308
#14 0x00007fd63a71accd in clone ()
    at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#15 0x0000000000000000 in ?? ()

Thread 70 (Thread 0x7fd6334a5700 (LWP 18305)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1  0x000000000086415b in DispatchQueue::run_local_delivery() ()
#2  0x000000000076740d in DispatchQueue::LocalDeliveryThread::entry() ()
#3  0x00007fd63bd67e9a in start_thread (arg=0x7fd6334a5700)
    at pthread_create.c:308
#4  0x00007fd63a71accd in clone ()
    at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#5  0x0000000000000000 in ?? ()

Thread 69 (Thread 0x7fd632ca4700 (LWP 18306)):
#0  0x00007fd63a70f313 in __GI___poll (fds=<optimized out>, 
    nfds=<optimized out>, timeout=<optimized out>)
    at ../sysdeps/unix/sysv/linux/poll.c:87
#1  0x000000000085dc64 in Accepter::entry() ()
#2  0x00007fd63bd67e9a in start_thread (arg=0x7fd632ca4700)
    at pthread_create.c:308
#3  0x00007fd63a71accd in clone ()
    at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#4  0x0000000000000000 in ?? ()

Thread 68 (Thread 0x7fd63209f700 (LWP 18311)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1  0x0000000000882892 in Pipe::writer() ()
#2  0x000000000088dbbd in Pipe::Writer::entry() ()
#3  0x00007fd63bd67e9a in start_thread (arg=0x7fd63209f700)
    at pthread_create.c:308
#4  0x00007fd63a71accd in clone ()
    at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#5  0x0000000000000000 in ?? ()

Thread 67 (Thread 0x7fd631f9e700 (LWP 18312)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1  0x0000000000882892 in Pipe::writer() ()
#2  0x000000000088dbbd in Pipe::Writer::entry() ()
#3  0x00007fd63bd67e9a in start_thread (arg=0x7fd631f9e700)
    at pthread_create.c:308
#4  0x00007fd63a71accd in clone ()
    at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#5  0x0000000000000000 in ?? ()

Thread 66 (Thread 0x7fd631e9d700 (LWP 18313)):
#0  0x00007fd63a70f313 in __GI___poll (fds=<optimized out>, 
    nfds=<optimized out>, timeout=<optimized out>)
    at ../sysdeps/unix/sysv/linux/poll.c:87
#1  0x000000000089633f in SignalHandler::entry() ()
#2  0x00007fd63bd67e9a in start_thread (arg=0x7fd631e9d700)
    at pthread_create.c:308
#3  0x00007fd63a71accd in clone ()
---Type <return> to continue, or q <return> to quit---
    at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#4  0x0000000000000000 in ?? ()

Thread 65 (Thread 0x7fd63149a700 (LWP 18322)):
#0  0x00007fd63a70f313 in __GI___poll (fds=<optimized out>, 
    nfds=<optimized out>, timeout=<optimized out>)
    at ../sysdeps/unix/sysv/linux/poll.c:87
#1  0x0000000000876d45 in Pipe::tcp_read_wait() ()
#2  0x0000000000877058 in Pipe::tcp_read(char*, int) ()
#3  0x0000000000889a50 in Pipe::reader() ()
#4  0x000000000088dbdd in Pipe::Reader::entry() ()
#5  0x00007fd63bd67e9a in start_thread (arg=0x7fd63149a700)
    at pthread_create.c:308
#6  0x00007fd63a71accd in clone ()
    at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#7  0x0000000000000000 in ?? ()

Thread 64 (Thread 0x7fd62fd73700 (LWP 18601)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1  0x00007fd63bfb5f33 in ?? () from /usr/lib/x86_64-linux-gnu/libleveldb.so.1
#2  0x00007fd63bd67e9a in start_thread (arg=0x7fd62fd73700)
    at pthread_create.c:308
#3  0x00007fd63a71accd in clone ()
    at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#4  0x0000000000000000 in ?? ()

Thread 63 (Thread 0x7fd61dfce700 (LWP 19092)):
#0  0x00007fd63a70f313 in __GI___poll (fds=<optimized out>, 
    nfds=<optimized out>, timeout=<optimized out>)
    at ../sysdeps/unix/sysv/linux/poll.c:87
#1  0x0000000000876d45 in Pipe::tcp_read_wait() ()
#2  0x0000000000877058 in Pipe::tcp_read(char*, int) ()
#3  0x0000000000889a50 in Pipe::reader() ()
#4  0x000000000088dbdd in Pipe::Reader::entry() ()
#5  0x00007fd63bd67e9a in start_thread (arg=0x7fd61dfce700)
    at pthread_create.c:308
#6  0x00007fd63a71accd in clone ()
    at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#7  0x0000000000000000 in ?? ()

Thread 62 (Thread 0x7fd61decd700 (LWP 19093)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1  0x0000000000882892 in Pipe::writer() ()
#2  0x000000000088dbbd in Pipe::Writer::entry() ()
#3  0x00007fd63bd67e9a in start_thread (arg=0x7fd61decd700)
    at pthread_create.c:308
#4  0x00007fd63a71accd in clone ()
    at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#5  0x0000000000000000 in ?? ()

Thread 61 (Thread 0x7fd631389700 (LWP 19142)):
#0  0x00007fd63a70f313 in __GI___poll (fds=<optimized out>, 
    nfds=<optimized out>, timeout=<optimized out>)
    at ../sysdeps/unix/sysv/linux/poll.c:87
#1  0x0000000000876d45 in Pipe::tcp_read_wait() ()
#2  0x0000000000877058 in Pipe::tcp_read(char*, int) ()
#3  0x0000000000889a50 in Pipe::reader() ()
#4  0x000000000088dbdd in Pipe::Reader::entry() ()
#5  0x00007fd63bd67e9a in start_thread (arg=0x7fd631389700)
    at pthread_create.c:308
#6  0x00007fd63a71accd in clone ()
    at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
---Type <return> to continue, or q <return> to quit---
#7  0x0000000000000000 in ?? ()

Thread 60 (Thread 0x7fd63c5eb700 (LWP 19147)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1  0x0000000000882892 in Pipe::writer() ()
#2  0x000000000088dbbd in Pipe::Writer::entry() ()
#3  0x00007fd63bd67e9a in start_thread (arg=0x7fd63c5eb700)
    at pthread_create.c:308
#4  0x00007fd63a71accd in clone ()
    at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#5  0x0000000000000000 in ?? ()

Thread 59 (Thread 0x7fd61eedc700 (LWP 19189)):
#0  0x00007fd63a70f313 in __GI___poll (fds=<optimized out>, 
    nfds=<optimized out>, timeout=<optimized out>)
    at ../sysdeps/unix/sysv/linux/poll.c:87
#1  0x0000000000876d45 in Pipe::tcp_read_wait() ()
#2  0x0000000000877058 in Pipe::tcp_read(char*, int) ()
#3  0x0000000000889a50 in Pipe::reader() ()
#4  0x000000000088dbdd in Pipe::Reader::entry() ()
#5  0x00007fd63bd67e9a in start_thread (arg=0x7fd61eedc700)
    at pthread_create.c:308
#6  0x00007fd63a71accd in clone ()
    at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#7  0x0000000000000000 in ?? ()

Thread 58 (Thread 0x7fd61eddb700 (LWP 19190)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1  0x0000000000882892 in Pipe::writer() ()
#2  0x000000000088dbbd in Pipe::Writer::entry() ()
#3  0x00007fd63bd67e9a in start_thread (arg=0x7fd61eddb700)
    at pthread_create.c:308
#4  0x00007fd63a71accd in clone ()
    at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#5  0x0000000000000000 in ?? ()

Thread 57 (Thread 0x7fd60cfbe700 (LWP 19439)):
#0  0x00007fd63a70f313 in __GI___poll (fds=<optimized out>, 
    nfds=<optimized out>, timeout=<optimized out>)
    at ../sysdeps/unix/sysv/linux/poll.c:87
#1  0x0000000000876d45 in Pipe::tcp_read_wait() ()
#2  0x0000000000877058 in Pipe::tcp_read(char*, int) ()
#3  0x0000000000889a50 in Pipe::reader() ()
#4  0x000000000088dbdd in Pipe::Reader::entry() ()
#5  0x00007fd63bd67e9a in start_thread (arg=0x7fd60cfbe700)
    at pthread_create.c:308
#6  0x00007fd63a71accd in clone ()
    at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#7  0x0000000000000000 in ?? ()

Thread 56 (Thread 0x7fd61b91e700 (LWP 23526)):

#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1  0x0000000000849c17 in LevelDBStore::compact_thread_entry() ()
#2  0x000000000084879d in LevelDBStore::CompactThread::entry() ()
#3  0x00007fd63bd67e9a in start_thread (arg=0x7fd61b91e700)
    at pthread_create.c:308
#4  0x00007fd63a71accd in clone ()
    at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#5  0x0000000000000000 in ?? ()

Thread 55 (Thread 0x7fd615cb3700 (LWP 25073)):
---Type <return> to continue, or q <return> to quit---
#0  0x00007fd63a70f313 in __GI___poll (fds=<optimized out>, 
    nfds=<optimized out>, timeout=<optimized out>)
    at ../sysdeps/unix/sysv/linux/poll.c:87
#1  0x0000000000876d45 in Pipe::tcp_read_wait() ()
#2  0x0000000000877058 in Pipe::tcp_read(char*, int) ()
#3  0x0000000000889a50 in Pipe::reader() ()
#4  0x000000000088dbdd in Pipe::Reader::entry() ()
#5  0x00007fd63bd67e9a in start_thread (arg=0x7fd615cb3700)
    at pthread_create.c:308
#6  0x00007fd63a71accd in clone ()
    at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#7  0x0000000000000000 in ?? ()

Thread 54 (Thread 0x7fd614cbb700 (LWP 25075)):
#0  0x00007fd63a70f313 in __GI___poll (fds=<optimized out>, 
    nfds=<optimized out>, timeout=<optimized out>)
    at ../sysdeps/unix/sysv/linux/poll.c:87
#1  0x0000000000876d45 in Pipe::tcp_read_wait() ()
#2  0x0000000000877058 in Pipe::tcp_read(char*, int) ()
#3  0x0000000000889a50 in Pipe::reader() ()
#4  0x000000000088dbdd in Pipe::Reader::entry() ()
#5  0x00007fd63bd67e9a in start_thread (arg=0x7fd614cbb700)
    at pthread_create.c:308
#6  0x00007fd63a71accd in clone ()
    at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#7  0x0000000000000000 in ?? ()

Thread 53 (Thread 0x7fd61f51e700 (LWP 25076)):
#0  0x00007fd63a70f313 in __GI___poll (fds=<optimized out>, 
    nfds=<optimized out>, timeout=<optimized out>)
    at ../sysdeps/unix/sysv/linux/poll.c:87
#1  0x0000000000876d45 in Pipe::tcp_read_wait() ()
#2  0x0000000000877058 in Pipe::tcp_read(char*, int) ()
#3  0x0000000000889a50 in Pipe::reader() ()
#4  0x000000000088dbdd in Pipe::Reader::entry() ()
#5  0x00007fd63bd67e9a in start_thread (arg=0x7fd61f51e700)
    at pthread_create.c:308
#6  0x00007fd63a71accd in clone ()
    at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#7  0x0000000000000000 in ?? ()

Thread 52 (Thread 0x7fd613f99700 (LWP 25077)):
#0  0x00007fd63a70f313 in __GI___poll (fds=<optimized out>, 
    nfds=<optimized out>, timeout=<optimized out>)
    at ../sysdeps/unix/sysv/linux/poll.c:87
#1  0x0000000000876d45 in Pipe::tcp_read_wait() ()
#2  0x0000000000877058 in Pipe::tcp_read(char*, int) ()
#3  0x0000000000889a50 in Pipe::reader() ()
#4  0x000000000088dbdd in Pipe::Reader::entry() ()
#5  0x00007fd63bd67e9a in start_thread (arg=0x7fd613f99700)
    at pthread_create.c:308
#6  0x00007fd63a71accd in clone ()
    at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#7  0x0000000000000000 in ?? ()

Thread 51 (Thread 0x7fd6149a3700 (LWP 25079)):
#0  0x00007fd63a70f313 in __GI___poll (fds=<optimized out>, 
    nfds=<optimized out>, timeout=<optimized out>)
    at ../sysdeps/unix/sysv/linux/poll.c:87
#1  0x0000000000876d45 in Pipe::tcp_read_wait() ()
#2  0x0000000000877058 in Pipe::tcp_read(char*, int) ()
#3  0x0000000000889a50 in Pipe::reader() ()
#4  0x000000000088dbdd in Pipe::Reader::entry() ()
#5  0x00007fd63bd67e9a in start_thread (arg=0x7fd6149a3700)
---Type <return> to continue, or q <return> to quit---
    at pthread_create.c:308
#6  0x00007fd63a71accd in clone ()
    at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#7  0x0000000000000000 in ?? ()

Thread 50 (Thread 0x7fd61459f700 (LWP 25080)):
#0  0x00007fd63a70f313 in __GI___poll (fds=<optimized out>, 
    nfds=<optimized out>, timeout=<optimized out>)
    at ../sysdeps/unix/sysv/linux/poll.c:87
#1  0x0000000000876d45 in Pipe::tcp_read_wait() ()
#2  0x0000000000877058 in Pipe::tcp_read(char*, int) ()
#3  0x0000000000889a50 in Pipe::reader() ()
#4  0x000000000088dbdd in Pipe::Reader::entry() ()
#5  0x00007fd63bd67e9a in start_thread (arg=0x7fd61459f700)
    at pthread_create.c:308
#6  0x00007fd63a71accd in clone ()
    at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#7  0x0000000000000000 in ?? ()

Thread 49 (Thread 0x7fd61fbdd700 (LWP 25081)):
#0  0x00007fd63a70f313 in __GI___poll (fds=<optimized out>, 
    nfds=<optimized out>, timeout=<optimized out>)
    at ../sysdeps/unix/sysv/linux/poll.c:87
#1  0x0000000000876d45 in Pipe::tcp_read_wait() ()
#2  0x0000000000877058 in Pipe::tcp_read(char*, int) ()
#3  0x0000000000889a50 in Pipe::reader() ()
#4  0x000000000088dbdd in Pipe::Reader::entry() ()
#5  0x00007fd63bd67e9a in start_thread (arg=0x7fd61fbdd700)
    at pthread_create.c:308
#6  0x00007fd63a71accd in clone ()
    at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#7  0x0000000000000000 in ?? ()

Thread 48 (Thread 0x7fd634fdb700 (LWP 25085)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1  0x0000000000882892 in Pipe::writer() ()
#2  0x000000000088dbbd in Pipe::Writer::entry() ()
#3  0x00007fd63bd67e9a in start_thread (arg=0x7fd634fdb700)
    at pthread_create.c:308
#4  0x00007fd63a71accd in clone ()
    at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#5  0x0000000000000000 in ?? ()

Thread 47 (Thread 0x7fd61f41d700 (LWP 25088)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1  0x0000000000882892 in Pipe::writer() ()
#2  0x000000000088dbbd in Pipe::Writer::entry() ()
#3  0x00007fd63bd67e9a in start_thread (arg=0x7fd61f41d700)
    at pthread_create.c:308
#4  0x00007fd63a71accd in clone ()
    at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#5  0x0000000000000000 in ?? ()

Thread 46 (Thread 0x7fd60d8c7700 (LWP 25089)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1  0x0000000000882892 in Pipe::writer() ()
#2  0x000000000088dbbd in Pipe::Writer::entry() ()
#3  0x00007fd63bd67e9a in start_thread (arg=0x7fd60d8c7700)
    at pthread_create.c:308
#4  0x00007fd63a71accd in clone ()
    at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
---Type <return> to continue, or q <return> to quit---
#5  0x0000000000000000 in ?? ()

Thread 45 (Thread 0x7fd61f31c700 (LWP 25090)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1  0x0000000000882892 in Pipe::writer() ()
#2  0x000000000088dbbd in Pipe::Writer::entry() ()
#3  0x00007fd63bd67e9a in start_thread (arg=0x7fd61f31c700)
    at pthread_create.c:308
#4  0x00007fd63a71accd in clone ()
    at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#5  0x0000000000000000 in ?? ()

Thread 44 (Thread 0x7fd63077d700 (LWP 25091)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1  0x0000000000882892 in Pipe::writer() ()
#2  0x000000000088dbbd in Pipe::Writer::entry() ()
#3  0x00007fd63bd67e9a in start_thread (arg=0x7fd63077d700)
    at pthread_create.c:308
#4  0x00007fd63a71accd in clone ()
    at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#5  0x0000000000000000 in ?? ()

Thread 43 (Thread 0x7fd61fcde700 (LWP 25092)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1  0x0000000000882892 in Pipe::writer() ()
#2  0x000000000088dbbd in Pipe::Writer::entry() ()
#3  0x00007fd63bd67e9a in start_thread (arg=0x7fd61fcde700)
    at pthread_create.c:308
#4  0x00007fd63a71accd in clone ()
    at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#5  0x0000000000000000 in ?? ()

Thread 42 (Thread 0x7fd63067c700 (LWP 25093)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1  0x0000000000882892 in Pipe::writer() ()
#2  0x000000000088dbbd in Pipe::Writer::entry() ()
#3  0x00007fd63bd67e9a in start_thread (arg=0x7fd63067c700)
    at pthread_create.c:308
#4  0x00007fd63a71accd in clone ()
    at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#5  0x0000000000000000 in ?? ()

Thread 41 (Thread 0x7fd61348e700 (LWP 25094)):
#0  0x00007fd63a70f313 in __GI___poll (fds=<optimized out>, 
    nfds=<optimized out>, timeout=<optimized out>)
    at ../sysdeps/unix/sysv/linux/poll.c:87
#1  0x0000000000876d45 in Pipe::tcp_read_wait() ()
#2  0x0000000000877058 in Pipe::tcp_read(char*, int) ()
#3  0x0000000000889a50 in Pipe::reader() ()
#4  0x000000000088dbdd in Pipe::Reader::entry() ()
#5  0x00007fd63bd67e9a in start_thread (arg=0x7fd61348e700)
    at pthread_create.c:308
#6  0x00007fd63a71accd in clone ()
    at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#7  0x0000000000000000 in ?? ()

Thread 40 (Thread 0x7fd61ebd9700 (LWP 25096)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1  0x0000000000882892 in Pipe::writer() ()
---Type <return> to continue, or q <return> to quit---
#2  0x000000000088dbbd in Pipe::Writer::entry() ()
#3  0x00007fd63bd67e9a in start_thread (arg=0x7fd61ebd9700)
    at pthread_create.c:308
#4  0x00007fd63a71accd in clone ()
    at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#5  0x0000000000000000 in ?? ()

Thread 39 (Thread 0x7fd6352f2700 (LWP 25098)):
#0  0x00007fd63a70f313 in __GI___poll (fds=<optimized out>, 
    nfds=<optimized out>, timeout=<optimized out>)
    at ../sysdeps/unix/sysv/linux/poll.c:87
#1  0x0000000000876d45 in Pipe::tcp_read_wait() ()
#2  0x0000000000877058 in Pipe::tcp_read(char*, int) ()
#3  0x0000000000889a50 in Pipe::reader() ()
#4  0x000000000088dbdd in Pipe::Reader::entry() ()
#5  0x00007fd63bd67e9a in start_thread (arg=0x7fd6352f2700)
    at pthread_create.c:308
#6  0x00007fd63a71accd in clone ()
    at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#7  0x0000000000000000 in ?? ()

Thread 38 (Thread 0x7fd61328c700 (LWP 25099)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1  0x0000000000882892 in Pipe::writer() ()
#2  0x000000000088dbbd in Pipe::Writer::entry() ()
#3  0x00007fd63bd67e9a in start_thread (arg=0x7fd61328c700)
    at pthread_create.c:308
#4  0x00007fd63a71accd in clone ()
    at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#5  0x0000000000000000 in ?? ()

Thread 37 (Thread 0x7fd6357a5700 (LWP 25100)):
#0  0x00007fd63a70f313 in __GI___poll (fds=<optimized out>, 
    nfds=<optimized out>, timeout=<optimized out>)
    at ../sysdeps/unix/sysv/linux/poll.c:87
#1  0x0000000000876d45 in Pipe::tcp_read_wait() ()
#2  0x0000000000877058 in Pipe::tcp_read(char*, int) ()
#3  0x0000000000889a50 in Pipe::reader() ()
#4  0x000000000088dbdd in Pipe::Reader::entry() ()
#5  0x00007fd63bd67e9a in start_thread (arg=0x7fd6357a5700)
    at pthread_create.c:308
#6  0x00007fd63a71accd in clone ()
    at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#7  0x0000000000000000 in ?? ()

Thread 36 (Thread 0x7fd613e98700 (LWP 25104)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1  0x0000000000882892 in Pipe::writer() ()
#2  0x000000000088dbbd in Pipe::Writer::entry() ()
#3  0x00007fd63bd67e9a in start_thread (arg=0x7fd613e98700)
    at pthread_create.c:308
#4  0x00007fd63a71accd in clone ()
    at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#5  0x0000000000000000 in ?? ()

Thread 35 (Thread 0x7fd6146a0700 (LWP 25106)):
#0  0x00007fd63a70f313 in __GI___poll (fds=<optimized out>, 
    nfds=<optimized out>, timeout=<optimized out>)
    at ../sysdeps/unix/sysv/linux/poll.c:87
#1  0x0000000000876d45 in Pipe::tcp_read_wait() ()
#2  0x0000000000877058 in Pipe::tcp_read(char*, int) ()
#3  0x0000000000889a50 in Pipe::reader() ()
---Type <return> to continue, or q <return> to quit---
#4  0x000000000088dbdd in Pipe::Reader::entry() ()
#5  0x00007fd63bd67e9a in start_thread (arg=0x7fd6146a0700)
    at pthread_create.c:308
#6  0x00007fd63a71accd in clone ()
    at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#7  0x0000000000000000 in ?? ()

Thread 34 (Thread 0x7fd6322a1700 (LWP 25107)):
#0  0x00007fd63a70f313 in __GI___poll (fds=<optimized out>, 
    nfds=<optimized out>, timeout=<optimized out>)
    at ../sysdeps/unix/sysv/linux/poll.c:87
#1  0x0000000000876d45 in Pipe::tcp_read_wait() ()
#2  0x0000000000877058 in Pipe::tcp_read(char*, int) ()
#3  0x0000000000889a50 in Pipe::reader() ()
#4  0x000000000088dbdd in Pipe::Reader::entry() ()
#5  0x00007fd63bd67e9a in start_thread (arg=0x7fd6322a1700)
    at pthread_create.c:308
#6  0x00007fd63a71accd in clone ()
    at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#7  0x0000000000000000 in ?? ()

Thread 33 (Thread 0x7fd61af1b700 (LWP 25108)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1  0x0000000000882892 in Pipe::writer() ()
#2  0x000000000088dbbd in Pipe::Writer::entry() ()
#3  0x00007fd63bd67e9a in start_thread (arg=0x7fd61af1b700)
    at pthread_create.c:308
#4  0x00007fd63a71accd in clone ()
    at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#5  0x0000000000000000 in ?? ()

Thread 32 (Thread 0x7fd6323a2700 (LWP 25109)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1  0x0000000000882892 in Pipe::writer() ()
#2  0x000000000088dbbd in Pipe::Writer::entry() ()
#3  0x00007fd63bd67e9a in start_thread (arg=0x7fd6323a2700)
    at pthread_create.c:308
#4  0x00007fd63a71accd in clone ()
    at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#5  0x0000000000000000 in ?? ()

Thread 31 (Thread 0x7fd61dccb700 (LWP 25116)):
#0  0x00007fd63a70f313 in __GI___poll (fds=<optimized out>, 
    nfds=<optimized out>, timeout=<optimized out>)
    at ../sysdeps/unix/sysv/linux/poll.c:87
#1  0x0000000000876d45 in Pipe::tcp_read_wait() ()
#2  0x0000000000877058 in Pipe::tcp_read(char*, int) ()
#3  0x0000000000889a50 in Pipe::reader() ()
#4  0x000000000088dbdd in Pipe::Reader::entry() ()
#5  0x00007fd63bd67e9a in start_thread (arg=0x7fd61dccb700)
    at pthread_create.c:308
#6  0x00007fd63a71accd in clone ()
    at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#7  0x0000000000000000 in ?? ()

Thread 30 (Thread 0x7fd61429c700 (LWP 25117)):
#0  0x00007fd63a70f313 in __GI___poll (fds=<optimized out>, 
    nfds=<optimized out>, timeout=<optimized out>)
    at ../sysdeps/unix/sysv/linux/poll.c:87
#1  0x0000000000876d45 in Pipe::tcp_read_wait() ()
#2  0x0000000000877058 in Pipe::tcp_read(char*, int) ()
#3  0x0000000000889a50 in Pipe::reader() ()
---Type <return> to continue, or q <return> to quit---
#4  0x000000000088dbdd in Pipe::Reader::entry() ()
#5  0x00007fd63bd67e9a in start_thread (arg=0x7fd61429c700)
    at pthread_create.c:308
#6  0x00007fd63a71accd in clone ()
    at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#7  0x0000000000000000 in ?? ()

Thread 29 (Thread 0x7fd6161b8700 (LWP 25118)):
#0  0x00007fd63a70f313 in __GI___poll (fds=<optimized out>, 
    nfds=<optimized out>, timeout=<optimized out>)
    at ../sysdeps/unix/sysv/linux/poll.c:87
#1  0x0000000000876d45 in Pipe::tcp_read_wait() ()
#2  0x0000000000877058 in Pipe::tcp_read(char*, int) ()
#3  0x0000000000889a50 in Pipe::reader() ()
#4  0x000000000088dbdd in Pipe::Reader::entry() ()
#5  0x00007fd63bd67e9a in start_thread (arg=0x7fd6161b8700)
    at pthread_create.c:308
#6  0x00007fd63a71accd in clone ()
    at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#7  0x0000000000000000 in ?? ()

Thread 28 (Thread 0x7fd61e0cf700 (LWP 25119)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1  0x0000000000882892 in Pipe::writer() ()
#2  0x000000000088dbbd in Pipe::Writer::entry() ()
#3  0x00007fd63bd67e9a in start_thread (arg=0x7fd61e0cf700)
    at pthread_create.c:308
#4  0x00007fd63a71accd in clone ()
    at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#5  0x0000000000000000 in ?? ()

Thread 27 (Thread 0x7fd6159b0700 (LWP 25120)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1  0x0000000000882892 in Pipe::writer() ()
#2  0x000000000088dbbd in Pipe::Writer::entry() ()
#3  0x00007fd63bd67e9a in start_thread (arg=0x7fd6159b0700)
    at pthread_create.c:308
#4  0x00007fd63a71accd in clone ()
    at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#5  0x0000000000000000 in ?? ()

Thread 26 (Thread 0x7fd6157ae700 (LWP 25121)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1  0x0000000000882892 in Pipe::writer() ()
#2  0x000000000088dbbd in Pipe::Writer::entry() ()
#3  0x00007fd63bd67e9a in start_thread (arg=0x7fd6157ae700)
    at pthread_create.c:308
#4  0x00007fd63a71accd in clone ()
    at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#5  0x0000000000000000 in ?? ()

Thread 25 (Thread 0x7fd63097f700 (LWP 25161)):
#0  0x00007fd63a70f313 in __GI___poll (fds=<optimized out>, 
    nfds=<optimized out>, timeout=<optimized out>)
    at ../sysdeps/unix/sysv/linux/poll.c:87
#1  0x0000000000876d45 in Pipe::tcp_read_wait() ()
#2  0x0000000000877058 in Pipe::tcp_read(char*, int) ()
#3  0x0000000000889a50 in Pipe::reader() ()
#4  0x000000000088dbdd in Pipe::Reader::entry() ()
#5  0x00007fd63bd67e9a in start_thread (arg=0x7fd63097f700)
    at pthread_create.c:308
---Type <return> to continue, or q <return> to quit---
#6  0x00007fd63a71accd in clone ()
    at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#7  0x0000000000000000 in ?? ()

Thread 24 (Thread 0x7fd61e8d6700 (LWP 25162)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1  0x0000000000882892 in Pipe::writer() ()
#2  0x000000000088dbbd in Pipe::Writer::entry() ()
#3  0x00007fd63bd67e9a in start_thread (arg=0x7fd61e8d6700)
    at pthread_create.c:308
#4  0x00007fd63a71accd in clone ()
    at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#5  0x0000000000000000 in ?? ()

Thread 23 (Thread 0x7fd615ab1700 (LWP 25163)):
#0  0x00007fd63a70f313 in __GI___poll (fds=<optimized out>, 
    nfds=<optimized out>, timeout=<optimized out>)
    at ../sysdeps/unix/sysv/linux/poll.c:87
#1  0x0000000000876d45 in Pipe::tcp_read_wait() ()
#2  0x0000000000877058 in Pipe::tcp_read(char*, int) ()
#3  0x0000000000889a50 in Pipe::reader() ()
#4  0x000000000088dbdd in Pipe::Reader::entry() ()
#5  0x00007fd63bd67e9a in start_thread (arg=0x7fd615ab1700)
    at pthread_create.c:308
#6  0x00007fd63a71accd in clone ()
    at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#7  0x0000000000000000 in ?? ()

Thread 22 (Thread 0x7fd613b95700 (LWP 25164)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1  0x0000000000882892 in Pipe::writer() ()
#2  0x000000000088dbbd in Pipe::Writer::entry() ()
#3  0x00007fd63bd67e9a in start_thread (arg=0x7fd613b95700)
    at pthread_create.c:308
#4  0x00007fd63a71accd in clone ()
    at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#5  0x0000000000000000 in ?? ()

Thread 21 (Thread 0x7fd612f89700 (LWP 25219)):
#0  0x00007fd63a70f313 in __GI___poll (fds=<optimized out>, 
    nfds=<optimized out>, timeout=<optimized out>)
    at ../sysdeps/unix/sysv/linux/poll.c:87
#1  0x0000000000876d45 in Pipe::tcp_read_wait() ()
#2  0x0000000000877058 in Pipe::tcp_read(char*, int) ()
#3  0x0000000000889a50 in Pipe::reader() ()
#4  0x000000000088dbdd in Pipe::Reader::entry() ()
#5  0x00007fd63bd67e9a in start_thread (arg=0x7fd612f89700)
    at pthread_create.c:308
#6  0x00007fd63a71accd in clone ()
    at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#7  0x0000000000000000 in ?? ()

Thread 20 (Thread 0x7fd612e88700 (LWP 25220)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1  0x0000000000882892 in Pipe::writer() ()
#2  0x000000000088dbbd in Pipe::Writer::entry() ()
#3  0x00007fd63bd67e9a in start_thread (arg=0x7fd612e88700)
    at pthread_create.c:308
#4  0x00007fd63a71accd in clone ()
    at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#5  0x0000000000000000 in ?? ()
---Type <return> to continue, or q <return> to quit---

Thread 19 (Thread 0x7fd617533700 (LWP 25238)):
#0  0x00007fd63a70f313 in __GI___poll (fds=<optimized out>, 
    nfds=<optimized out>, timeout=<optimized out>)
    at ../sysdeps/unix/sysv/linux/poll.c:87
#1  0x0000000000876d45 in Pipe::tcp_read_wait() ()
#2  0x0000000000877058 in Pipe::tcp_read(char*, int) ()
#3  0x0000000000889a50 in Pipe::reader() ()
#4  0x000000000088dbdd in Pipe::Reader::entry() ()
#5  0x00007fd63bd67e9a in start_thread (arg=0x7fd617533700)
    at pthread_create.c:308
#6  0x00007fd63a71accd in clone ()
    at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#7  0x0000000000000000 in ?? ()

Thread 18 (Thread 0x7fd6353f3700 (LWP 25239)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1  0x0000000000882892 in Pipe::writer() ()
#2  0x000000000088dbbd in Pipe::Writer::entry() ()
#3  0x00007fd63bd67e9a in start_thread (arg=0x7fd6353f3700)
    at pthread_create.c:308
#4  0x00007fd63a71accd in clone ()
    at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#5  0x0000000000000000 in ?? ()

Thread 17 (Thread 0x7fd615fb6700 (LWP 25240)):
#0  0x00007fd63a70f313 in __GI___poll (fds=<optimized out>, 
    nfds=<optimized out>, timeout=<optimized out>)
    at ../sysdeps/unix/sysv/linux/poll.c:87
#1  0x0000000000876d45 in Pipe::tcp_read_wait() ()
#2  0x0000000000877058 in Pipe::tcp_read(char*, int) ()
#3  0x0000000000889a50 in Pipe::reader() ()
#4  0x000000000088dbdd in Pipe::Reader::entry() ()
#5  0x00007fd63bd67e9a in start_thread (arg=0x7fd615fb6700)
    at pthread_create.c:308
#6  0x00007fd63a71accd in clone ()
    at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#7  0x0000000000000000 in ?? ()

Thread 16 (Thread 0x7fd616a06700 (LWP 25241)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1  0x0000000000882892 in Pipe::writer() ()
#2  0x000000000088dbbd in Pipe::Writer::entry() ()
#3  0x00007fd63bd67e9a in start_thread (arg=0x7fd616a06700)
    at pthread_create.c:308
#4  0x00007fd63a71accd in clone ()
    at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#5  0x0000000000000000 in ?? ()

Thread 15 (Thread 0x7fd63169c700 (LWP 25244)):
#0  0x00007fd63a70f313 in __GI___poll (fds=<optimized out>, 
    nfds=<optimized out>, timeout=<optimized out>)
    at ../sysdeps/unix/sysv/linux/poll.c:87
#1  0x0000000000876d45 in Pipe::tcp_read_wait() ()
#2  0x0000000000877058 in Pipe::tcp_read(char*, int) ()
#3  0x0000000000889a50 in Pipe::reader() ()
#4  0x000000000088dbdd in Pipe::Reader::entry() ()
#5  0x00007fd63bd67e9a in start_thread (arg=0x7fd63169c700)
    at pthread_create.c:308
#6  0x00007fd63a71accd in clone ()
    at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#7  0x0000000000000000 in ?? ()
---Type <return> to continue, or q <return> to quit---

Thread 14 (Thread 0x7fd63159b700 (LWP 25245)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1  0x0000000000882892 in Pipe::writer() ()
#2  0x000000000088dbbd in Pipe::Writer::entry() ()
#3  0x00007fd63bd67e9a in start_thread (arg=0x7fd63159b700)
    at pthread_create.c:308
#4  0x00007fd63a71accd in clone ()
    at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#5  0x0000000000000000 in ?? ()

Thread 13 (Thread 0x7fd631288700 (LWP 25246)):
#0  0x00007fd63a70f313 in __GI___poll (fds=<optimized out>, 
    nfds=<optimized out>, timeout=<optimized out>)
    at ../sysdeps/unix/sysv/linux/poll.c:87
#1  0x0000000000876d45 in Pipe::tcp_read_wait() ()
#2  0x0000000000877058 in Pipe::tcp_read(char*, int) ()
#3  0x0000000000889a50 in Pipe::reader() ()
#4  0x000000000088dbdd in Pipe::Reader::entry() ()
#5  0x00007fd63bd67e9a in start_thread (arg=0x7fd631288700)
    at pthread_create.c:308
#6  0x00007fd63a71accd in clone ()
    at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#7  0x0000000000000000 in ?? ()

Thread 12 (Thread 0x7fd630f85700 (LWP 25247)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1  0x0000000000882892 in Pipe::writer() ()
#2  0x000000000088dbbd in Pipe::Writer::entry() ()
#3  0x00007fd63bd67e9a in start_thread (arg=0x7fd630f85700)
    at pthread_create.c:308
#4  0x00007fd63a71accd in clone ()
    at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#5  0x0000000000000000 in ?? ()

Thread 11 (Thread 0x7fd6324a3700 (LWP 25249)):
#0  0x00007fd63a70f313 in __GI___poll (fds=<optimized out>, 
    nfds=<optimized out>, timeout=<optimized out>)
    at ../sysdeps/unix/sysv/linux/poll.c:87
#1  0x0000000000876d45 in Pipe::tcp_read_wait() ()
#2  0x0000000000877058 in Pipe::tcp_read(char*, int) ()
#3  0x0000000000889a50 in Pipe::reader() ()
#4  0x000000000088dbdd in Pipe::Reader::entry() ()
#5  0x00007fd63bd67e9a in start_thread (arg=0x7fd6324a3700)
    at pthread_create.c:308
#6  0x00007fd63a71accd in clone ()
    at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#7  0x0000000000000000 in ?? ()

Thread 10 (Thread 0x7fd6321a0700 (LWP 25250)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1  0x0000000000882892 in Pipe::writer() ()
#2  0x000000000088dbbd in Pipe::Writer::entry() ()
#3  0x00007fd63bd67e9a in start_thread (arg=0x7fd6321a0700)
    at pthread_create.c:308
#4  0x00007fd63a71accd in clone ()
    at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#5  0x0000000000000000 in ?? ()

Thread 9 (Thread 0x7fd630e84700 (LWP 25266)):
#0  0x00007fd63a70f313 in __GI___poll (fds=<optimized out>, 
---Type <return> to continue, or q <return> to quit---
    nfds=<optimized out>, timeout=<optimized out>)
    at ../sysdeps/unix/sysv/linux/poll.c:87
#1  0x0000000000876d45 in Pipe::tcp_read_wait() ()
#2  0x0000000000877058 in Pipe::tcp_read(char*, int) ()
#3  0x0000000000889a50 in Pipe::reader() ()
#4  0x000000000088dbdd in Pipe::Reader::entry() ()
#5  0x00007fd63bd67e9a in start_thread (arg=0x7fd630e84700)
    at pthread_create.c:308
#6  0x00007fd63a71accd in clone ()
    at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#7  0x0000000000000000 in ?? ()

Thread 8 (Thread 0x7fd630d83700 (LWP 25267)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1  0x0000000000882892 in Pipe::writer() ()
#2  0x000000000088dbbd in Pipe::Writer::entry() ()
#3  0x00007fd63bd67e9a in start_thread (arg=0x7fd630d83700)
    at pthread_create.c:308
#4  0x00007fd63a71accd in clone ()
    at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#5  0x0000000000000000 in ?? ()

Thread 7 (Thread 0x7fd6162b9700 (LWP 25270)):
#0  0x00007fd63a70f313 in __GI___poll (fds=<optimized out>, 
    nfds=<optimized out>, timeout=<optimized out>)
    at ../sysdeps/unix/sysv/linux/poll.c:87
#1  0x0000000000876d45 in Pipe::tcp_read_wait() ()
#2  0x0000000000877058 in Pipe::tcp_read(char*, int) ()
#3  0x0000000000889a50 in Pipe::reader() ()
#4  0x000000000088dbdd in Pipe::Reader::entry() ()
#5  0x00007fd63bd67e9a in start_thread (arg=0x7fd6162b9700)
    at pthread_create.c:308
#6  0x00007fd63a71accd in clone ()
    at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#7  0x0000000000000000 in ?? ()

Thread 6 (Thread 0x7fd61439d700 (LWP 25271)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1  0x0000000000882892 in Pipe::writer() ()
#2  0x000000000088dbbd in Pipe::Writer::entry() ()
#3  0x00007fd63bd67e9a in start_thread (arg=0x7fd61439d700)
    at pthread_create.c:308
#4  0x00007fd63a71accd in clone ()
    at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#5  0x0000000000000000 in ?? ()

Thread 5 (Thread 0x7fd61d9c8700 (LWP 25275)):
#0  0x00007fd63a70f313 in __GI___poll (fds=<optimized out>, 
    nfds=<optimized out>, timeout=<optimized out>)
    at ../sysdeps/unix/sysv/linux/poll.c:87
#1  0x0000000000876d45 in Pipe::tcp_read_wait() ()
#2  0x0000000000877058 in Pipe::tcp_read(char*, int) ()
#3  0x0000000000889a50 in Pipe::reader() ()
#4  0x000000000088dbdd in Pipe::Reader::entry() ()
#5  0x00007fd63bd67e9a in start_thread (arg=0x7fd61d9c8700)
    at pthread_create.c:308
#6  0x00007fd63a71accd in clone ()
    at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#7  0x0000000000000000 in ?? ()

Thread 4 (Thread 0x7fd6160b7700 (LWP 25276)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
---Type <return> to continue, or q <return> to quit---
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1  0x0000000000882892 in Pipe::writer() ()
#2  0x000000000088dbbd in Pipe::Writer::entry() ()
#3  0x00007fd63bd67e9a in start_thread (arg=0x7fd6160b7700)
    at pthread_create.c:308
#4  0x00007fd63a71accd in clone ()
    at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#5  0x0000000000000000 in ?? ()

Thread 3 (Thread 0x7fd61e5d4700 (LWP 25285)):
#0  0x00007fd63a70f313 in __GI___poll (fds=<optimized out>, 
    nfds=<optimized out>, timeout=<optimized out>)
    at ../sysdeps/unix/sysv/linux/poll.c:87
#1  0x0000000000876d45 in Pipe::tcp_read_wait() ()
#2  0x0000000000877058 in Pipe::tcp_read(char*, int) ()
#3  0x0000000000889a50 in Pipe::reader() ()
#4  0x000000000088dbdd in Pipe::Reader::entry() ()
#5  0x00007fd63bd67e9a in start_thread (arg=0x7fd61e5d4700)
    at pthread_create.c:308
#6  0x00007fd63a71accd in clone ()
    at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#7  0x0000000000000000 in ?? ()

Thread 2 (Thread 0x7fd6148a2700 (LWP 25286)):
#0  pthread_cond_wait@@GLIBC_2.3.2 ()
    at ../nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:162
#1  0x0000000000882892 in Pipe::writer() ()
#2  0x000000000088dbbd in Pipe::Writer::entry() ()
#3  0x00007fd63bd67e9a in start_thread (arg=0x7fd6148a2700)
    at pthread_create.c:308
#4  0x00007fd63a71accd in clone ()
    at ../sysdeps/unix/sysv/linux/x86_64/clone.S:112
#5  0x0000000000000000 in ?? ()

Thread 1 (Thread 0x7fd63c643780 (LWP 18255)):
#0  0x00007fd63bd69148 in pthread_join (threadid=140557960238848, 
    thread_return=0x0) at pthread_join.c:89
#1  0x000000000076b572 in Thread::join(void**) ()
#2  0x0000000000862332 in DispatchQueue::wait() ()
#3  0x0000000000760fee in SimpleMessenger::wait() ()
#4  0x000000000053a4bf in main ()

Actions #2

Updated by Sage Weil over 9 years ago

  • Status changed from New to Resolved
Actions

Also available in: Atom PDF