Ceph

I'm seeing an occasional cosd failure where the process will die, but remains in a defunct state:

root 3165 0.0 0.0 0 0 ? Zsl 12:41 0:00 [cosd] <defunct>

The core file shows a bunch of threads running, although the stacks seem corrupted:

Thread 145 (Thread 10387):
#0 0x00007f788db6a789 in ?? ()
#1 0x0000000000000000 in ?? ()

Thread 144 (Thread 10391):
#0 0x00007f788f336f2b in ?? ()
#1 0x0000000000000000 in ?? ()

Thread 143 (Thread 11191):
#0 0x00007f788f336bac in ?? ()
#1 0x0000000200000000 in ?? ()
#2 0x000000000507a6d8 in ?? ()
#3 0x000000000507a5d8 in ?? ()
#4 0x0000000000000002 in ?? ()
#5 0x00000000007b5c9b in ObjectStore::C_DeleteTransaction::finish (this=0x507a6d0, r=0) at ../../src/os/ObjectStore.h:517
#6 0x00000000008d2a80 in SimpleMessenger::Pipe::read_message (this=0x7796ee, pm=0x7f787b420950) at ../../src/msg/SimpleMessenger.cc:2004
#7 0x00000000007770fe in std::map<int, xlist<SimpleMessenger::Pipe*>::iterator, std::less<int>, std::allocator<std::pair<int const, xlist<SimpleMessenger::Pipe*>::iterator> > >::map (this=0x507a748) at /usr/include/c++/4.5/bits/stl_map.h:151
#8 0x00000000008a2c85 in ceph_str_hash_rjenkins (str=0x8a2c85 "", length=32632) at ../../src/include/ceph_hash.cc:37
#9 0x00007f788f331d8c in ?? ()
#10 0x0000000000000000 in ?? ()

Thread 142 (Thread 11393):
#0 0x00007f788db65f03 in ?? ()
#1 0x00007f7878efbc80 in ?? ()
#2 0x00000000000dbba0 in ?? ()
#3 0x0000000000000001 in ?? ()
#4 0x00007f7878efbc90 in ?? ()
#5 0x0000000000775ffd in Throttle::~Throttle (this=0x507a1c0, __in_chrg=<value optimized out>) at ../../src/common/Throttle.h:7
#6 0x00000000008c262d in SimpleMessenger::Accepter::entry (this=0x5637ad8) at ../../src/msg/SimpleMessenger.cc:232
#7 0x00000000008d0cc5 in SimpleMessenger::Pipe::writer (this=0x5607c40) at ../../src/msg/SimpleMessenger.cc:1790
#8 0x00000000007770a4 in std::_Rb_tree<int, std::pair<int const, xlist<SimpleMessenger::Pipe*>*>, std::_Select1st<std::pair<int const, xlist<SimpleMessenger::Pipe*>*> >, std::less<int>, std::allocator<std::pair<int const, xlist<SimpleMessenger::Pipe*>*> > >::~_Rb_tree (this=0x7770a4,
__in_chrg=<value optimized out>) at /usr/include/c++/4.5/bits/stl_tree.h:614
#9 0x00000000008a2c85 in ceph_str_hash_rjenkins (str=0x8a2c85 "", length=32632) at ../../src/include/ceph_hash.cc:37
#10 0x00007f788f331d8c in ?? ()
#11 0x0000000000000000 in ?? ()

Thread 141 (Thread 11488):
#0 0x00007f788f336bac in ?? ()
#1 0x0000000500000000 in ?? ()
#2 0x0000000005603458 in ?? ()
#3 0x0000000005603358 in ?? ()
#4 0x0000000000000005 in ?? ()
#5 0x00000000007b5c9b in ObjectStore::C_DeleteTransaction::finish (this=0x5603450, r=0) at ../../src/os/ObjectStore.h:517
#6 0x00000000008d2a80 in SimpleMessenger::Pipe::read_message (this=0x7796ee, pm=0x7f78786f3950) at ../../src/msg/SimpleMessenger.cc:2004
#7 0x00000000007770fe in std::map<int, xlist<SimpleMessenger::Pipe*>::iterator, std::less<int>, std::allocator<std::pair<int const, xlist<SimpleMessenger::Pipe*>::iterator> > >::map (this=0x56034c8) at /usr/include/c++/4.5/bits/stl_map.h:151
#8 0x00000000008a2c85 in ceph_str_hash_rjenkins (str=0x8a2c85 "", length=32632) at ../../src/include/ceph_hash.cc:37
#9 0x00007f788f331d8c in ?? ()
#10 0x0000000000000000 in ?? ()

Thread 140 (Thread 11491):
#0 0x00007f788f336bac in ?? ()
#1 0x0000000200000000 in ?? ()
#2 0x0000000005968e58 in ?? ()
#3 0x0000000005968d58 in ?? ()
#4 0x0000000000000002 in ?? ()
#5 0x00000000007b5c9b in ObjectStore::C_DeleteTransaction::finish (this=0x5968e50, r=0) at ../../src/os/ObjectStore.h:517
#6 0x00000000008d2a80 in SimpleMessenger::Pipe::read_message (this=0x7796ee, pm=0x7f78784f1950) at ../../src/msg/SimpleMessenger.cc:2004
#7 0x00000000007770fe in std::map<int, xlist<SimpleMessenger::Pipe*>::iterator, std::less<int>, std::allocator<std::pair<int const, xlist<SimpleMessenger::Pipe*>::iterator> > >::map (this=0x5968ec8) at /usr/include/c++/4.5/bits/stl_map.h:151
#8 0x00000000008a2c85 in ceph_str_hash_rjenkins (str=0x8a2c85 "", length=32632) at ../../src/include/ceph_hash.cc:37
#9 0x00007f788f331d8c in ?? ()
#10 0x0000000000000000 in ?? ()

Thread 139 (Thread 11492):
#0 0x00007f788f336bac in ?? ()
#1 0x0000000100000000 in ?? ()
#2 0x0000000005968bd8 in ?? ()
#3 0x0000000005968ad8 in ?? ()
#4 0x0000000000000001 in ?? ()
#5 0x00000000007b5c9b in ObjectStore::C_DeleteTransaction::finish (this=0x5968bd0, r=0) at ../../src/os/ObjectStore.h:517
#6 0x00000000008d2a80 in SimpleMessenger::Pipe::read_message (this=0x7796ee, pm=0x7f78783f0950) at ../../src/msg/SimpleMessenger.cc:2004
#7 0x00000000007770fe in std::map<int, xlist<SimpleMessenger::Pipe*>::iterator, std::less<int>, std::allocator<std::pair<int const, xlist<SimpleMessenger::Pipe*>::iterator> > >::map (this=0x5968c48) at /usr/include/c++/4.5/bits/stl_map.h:151
#8 0x00000000008a2c85 in ceph_str_hash_rjenkins (str=0x8a2c85 "", length=32632) at ../../src/include/ceph_hash.cc:37
#9 0x00007f788f331d8c in ?? ()
#10 0x0000000000000000 in ?? ()

Thread 138 (Thread 11384):
#0 0x00007f788db65f03 in ?? ()
#1 0x0000000000775fd7 in SimpleMessenger::Accepter::~Accepter (this=0x771c76, __in_chrg=<value optimized out>) at ../../src/msg/SimpleMessenger.h:97
#2 0x00000000008c262d in SimpleMessenger::Accepter::entry (this=0x7f7878ffcbef) at ../../src/msg/SimpleMessenger.cc:232
#3 0x00000000008d0cc5 in SimpleMessenger::Pipe::writer (this=0x562d1c0) at ../../src/msg/SimpleMessenger.cc:1790
#4 0x00000000007770a4 in std::_Rb_tree<int, std::pair<int const, xlist<SimpleMessenger::Pipe*>*>, std::_Select1st<std::pair<int const, xlist<SimpleMessenger::Pipe*>*> >, std::less<int>, std::allocator<std::pair<int const, xlist<SimpleMessenger::Pipe*>*> > >::~_Rb_tree (this=0x7770a4,
__in_chrg=<value optimized out>) at /usr/include/c++/4.5/bits/stl_tree.h:614
#5 0x00000000008a2c85 in ceph_str_hash_rjenkins (str=0x8a2c85 "", length=32632) at ../../src/include/ceph_hash.cc:37
#6 0x00007f788f331d8c in ?? ()
#7 0x0000000000000000 in ?? ()

Thread 137 (Thread 9624):
#0 0x00007f788f336bac in ?? ()
---Type <return> to continue, or q <return> to quit---
#1 0x0000000000000000 in ?? ()

Thread 136 (Thread 10424):
#0 0x00007f788f3397c4 in ?? ()
#1 0x0000000002e7a768 in ?? ()
#2 0x0000000000000000 in ?? ()

Thread 135 (Thread 11377):
#0 0x00007f788db65f03 in ?? ()
#1 0x0000000000775fd7 in SimpleMessenger::Accepter::~Accepter (this=0x771c76, __in_chrg=<value optimized out>) at ../../src/msg/SimpleMessenger.h:97
#2 0x00000000008c262d in SimpleMessenger::Accepter::entry (this=0x7f7879703bef) at ../../src/msg/SimpleMessenger.cc:232
#3 0x00000000008d0cc5 in SimpleMessenger::Pipe::writer (this=0x589d1c0) at ../../src/msg/SimpleMessenger.cc:1790
#4 0x00000000007770a4 in std::_Rb_tree<int, std::pair<int const, xlist<SimpleMessenger::Pipe*>*>, std::_Select1st<std::pair<int const, xlist<SimpleMessenger::Pipe*>*> >, std::less<int>, std::allocator<std::pair<int const, xlist<SimpleMessenger::Pipe*>*> > >::~_Rb_tree (this=0x7770a4,
__in_chrg=<value optimized out>) at /usr/include/c++/4.5/bits/stl_tree.h:614
#5 0x00000000008a2c85 in ceph_str_hash_rjenkins (str=0x8a2c85 "", length=32632) at ../../src/include/ceph_hash.cc:37
#6 0x00007f788f331d8c in ?? ()
#7 0x0000000000000000 in ?? ()

Thread 134 (Thread 10979):
#0 0x00007f788db65f03 in ?? ()
#1 0x00007f787c733c80 in ?? ()
#2 0x00000000000dbba0 in ?? ()
#3 0x0000000000000001 in ?? ()
#4 0x00007f787c733c90 in ?? ()
#5 0x0000000000775ffd in Throttle::~Throttle (this=0x4b3dbc0, __in_chrg=<value optimized out>) at ../../src/common/Throttle.h:7
#6 0x00000000008c262d in SimpleMessenger::Accepter::entry (this=0x7785ce) at ../../src/msg/SimpleMessenger.cc:232
#7 0x00000000008d0cc5 in SimpleMessenger::Pipe::writer (this=0x5ae1700) at ../../src/msg/SimpleMessenger.cc:1790
#8 0x00000000007770a4 in std::_Rb_tree<int, std::pair<int const, xlist<SimpleMessenger::Pipe*>*>, std::_Select1st<std::pair<int const, xlist<SimpleMessenger::Pipe*>*> >, std::less<int>, std::allocator<std::pair<int const, xlist<SimpleMessenger::Pipe*>*> > >::~_Rb_tree (this=0x7770a4,
__in_chrg=<value optimized out>) at /usr/include/c++/4.5/bits/stl_tree.h:614
#9 0x00000000008a2c85 in ceph_str_hash_rjenkins (str=0x8a2c85 "", length=32632) at ../../src/include/ceph_hash.cc:37
#10 0x00007f788f331d8c in ?? ()
#11 0x0000000000000000 in ?? ()

Thread 133 (Thread 11130):
#0 0x00007f788db65f03 in ?? ()
#1 0x0000000000775fd7 in SimpleMessenger::Accepter::~Accepter (this=0x771c76, __in_chrg=<value optimized out>) at ../../src/msg/SimpleMessenger.h:97
#2 0x00000000008c262d in SimpleMessenger::Accepter::entry (this=0x7f787bb27bef) at ../../src/msg/SimpleMessenger.cc:232
#3 0x00000000008d0cc5 in SimpleMessenger::Pipe::writer (this=0x6142c40) at ../../src/msg/SimpleMessenger.cc:1790
#4 0x00000000007770a4 in std::_Rb_tree<int, std::pair<int const, xlist<SimpleMessenger::Pipe*>*>, std::_Select1st<std::pair<int const, xlist<SimpleMessenger::Pipe*>*> >, std::less<int>, std::allocator<std::pair<int const, xlist<SimpleMessenger::Pipe*>*> > >::~_Rb_tree (this=0x7770a4,
__in_chrg=<value optimized out>) at /usr/include/c++/4.5/bits/stl_tree.h:614
#5 0x00000000008a2c85 in ceph_str_hash_rjenkins (str=0x8a2c85 "", length=32632) at ../../src/include/ceph_hash.cc:37
#6 0x00007f788f331d8c in ?? ()
#7 0x0000000000000000 in ?? ()

Thread 132 (Thread 10388):
#0 0x00007f788f336bac in ?? ()
#1 0x0000006100000000 in ?? ()
#2 0x0000000002e7a0c0 in ?? ()
#3 0x0000000002e7a088 in ?? ()
#4 0x0000000000000061 in ?? ()
#5 0x00000000007b5c9b in ObjectStore::C_DeleteTransaction::finish (this=0x2e7a0b8, r=0) at ../../src/os/ObjectStore.h:517
#6 0x0000000000a2135a in FileJournal::_open_file (this=0x7f7887866dc0, oldsize=140155646537152, blksize=0, create=false) at ../../src/os/FileJournal.cc:284
#7 0x0000000000a0777a in std::vector<Context*, std::allocator<Context*> >::end (this=0xa0777a) at /usr/include/c++/4.5/bits/stl_vector.h:445
#8 0x00000000008a2c85 in ceph_str_hash_rjenkins (str=0x8a2c85 "", length=32632) at ../../src/include/ceph_hash.cc:37
#9 0x00007f788f331d8c in ?? ()
#10 0x0000000000000000 in ?? ()

Thread 131 (Thread 11585):
#0 0x00007f788db65f03 in ?? ()
#1 0x0000000000775fd7 in SimpleMessenger::Accepter::~Accepter (this=0x771c76, __in_chrg=<value optimized out>) at ../../src/msg/SimpleMessenger.h:97
#2 0x00000000008c262d in SimpleMessenger::Accepter::entry (this=0x7f78778e5bef) at ../../src/msg/SimpleMessenger.cc:232
#3 0x00000000008d0cc5 in SimpleMessenger::Pipe::writer (this=0x5ae1e00) at ../../src/msg/SimpleMessenger.cc:1790
#4 0x00000000007770a4 in std::_Rb_tree<int, std::pair<int const, xlist<SimpleMessenger::Pipe*>*>, std::_Select1st<std::pair<int const, xlist<SimpleMessenger::Pipe*>*> >, std::less<int>, std::allocator<std::pair<int const, xlist<SimpleMessenger::Pipe*>*> > >::~_Rb_tree (this=0x7770a4,
__in_chrg=<value optimized out>) at /usr/include/c++/4.5/bits/stl_tree.h:614
#5 0x00000000008a2c85 in ceph_str_hash_rjenkins (str=0x8a2c85 "", length=32632) at ../../src/include/ceph_hash.cc:37
#6 0x00007f788f331d8c in ?? ()
#7 0x0000000000000000 in ?? ()

Thread 130 (Thread 11399):
#0 0x00007f788db65f03 in ?? ()
#1 0x00007f7878cf9c80 in ?? ()
#2 0x00000000000dbba0 in ?? ()
#3 0x0000000000000001 in ?? ()
#4 0x00007f7878cf9c90 in ?? ()
#5 0x0000000000775ffd in Throttle::~Throttle (this=0x5662e40, __in_chrg=<value optimized out>) at ../../src/common/Throttle.h:7
#6 0x00000000008c262d in SimpleMessenger::Accepter::entry (this=0x5637758) at ../../src/msg/SimpleMessenger.cc:232
#7 0x00000000008d0cc5 in SimpleMessenger::Pipe::writer (this=0x54b0700) at ../../src/msg/SimpleMessenger.cc:1790
#8 0x00000000007770a4 in std::_Rb_tree<int, std::pair<int const, xlist<SimpleMessenger::Pipe*>*>, std::_Select1st<std::pair<int const, xlist<SimpleMessenger::Pipe*>*> >, std::less<int>, std::allocator<std::pair<int const, xlist<SimpleMessenger::Pipe*>*> > >::~_Rb_tree (this=0x7770a4,
__in_chrg=<value optimized out>) at /usr/include/c++/4.5/bits/stl_tree.h:614
#9 0x00000000008a2c85 in ceph_str_hash_rjenkins (str=0x8a2c85 "", length=32632) at ../../src/include/ceph_hash.cc:37
#10 0x00007f788f331d8c in ?? ()

#11 0x0000000000000000 in ?? ()

Thread 129 (Thread 11580):
#0 0x00007f788f336bac in ?? ()
#1 0x0000000100000000 in ?? ()
#2 0x00000000055f2e58 in ?? ()
#3 0x00000000055f2d58 in ?? ()
#4 0x0000000000000001 in ?? ()

#5 0x00000000007b5c9b in ObjectStore::C_DeleteTransaction::finish (this=0x55f2e50, r=0) at ../../src/os/ObjectStore.h:517
#6 0x00000000008d2a80 in SimpleMessenger::Pipe::read_message (this=0x7796ee, pm=0x7f7877ae7950) at ../../src/msg/SimpleMessenger.cc:2004
#7 0x00000000007770fe in std::map<int, xlist<SimpleMessenger::Pipe*>::iterator, std::less<int>, std::allocator<std::pair<int const, xlist<SimpleMessenger::Pipe*>::iterator> > >::map (this=0x55f2ec8) at /usr/include/c++/4.5/bits/stl_map.h:151
#8 0x00000000008a2c85 in ceph_str_hash_rjenkins (str=0x8a2c85 "", length=32632) at ../../src/include/ceph_hash.cc:37
#9 0x00007f788f331d8c in ?? ()
#10 0x0000000000000000 in ?? ()

Thread 128 (Thread 10393):
#0 0x00007f788f336bac in ?? ()
#1 0x0000077300000000 in ?? ()
#2 0x0000000002e7a7a0 in ?? ()
#3 0x0000000002e7a768 in ?? ()
#4 0x0000000000000773 in ?? ()
#5 0x00000000007b5c9b in ObjectStore::C_DeleteTransaction::finish (this=0x2e7a798, r=0) at ../../src/os/ObjectStore.h:517
#6 0x0000000000a2135a in FileJournal::_open_file (this=0x7f7885061dc0, oldsize=140155604573632, blksize=0, create=false) at ../../src/os/FileJournal.cc:284
#7 0x0000000000a0777a in std::vector<Context*, std::allocator<Context*> >::end (this=0xa0777a) at /usr/include/c++/4.5/bits/stl_vector.h:445
#8 0x00000000008a2c85 in ceph_str_hash_rjenkins (str=0x8a2c85 "", length=32632) at ../../src/include/ceph_hash.cc:37
#9 0x00007f788f331d8c in ?? ()
#10 0x0000000000000000 in ?? ()

Thread 127 (Thread 11259):
#0 0x00007f788f336bac in ?? ()
#1 0x0000000200000000 in ?? ()
#2 0x000000000507a458 in ?? ()
#3 0x000000000507a358 in ?? ()
#4 0x0000000000000002 in ?? ()
#5 0x00000000007b5c9b in ObjectStore::C_DeleteTransaction::finish (this=0x507a450, r=0) at ../../src/os/ObjectStore.h:517
#6 0x00000000008d2a80 in SimpleMessenger::Pipe::read_message (this=0x7796ee, pm=0x7f787b01c950) at ../../src/msg/SimpleMessenger.cc:2004
#7 0x00000000007770fe in std::map<int, xlist<SimpleMessenger::Pipe*>::iterator, std::less<int>, std::allocator<std::pair<int const, xlist<SimpleMessenger::Pipe*>::iterator> > >::map (this=0x507a4c8) at /usr/include/c++/4.5/bits/stl_map.h:151
#8 0x00000000008a2c85 in ceph_str_hash_rjenkins (str=0x8a2c85 "", length=32632) at ../../src/include/ceph_hash.cc:37
#9 0x00007f788f331d8c in ?? ()
#10 0x0000000000000000 in ?? ()

Thread 126 (Thread 11368):
#0 0x00007f788db65f03 in ?? ()
#1 0x0000000000775fd7 in SimpleMessenger::Accepter::~Accepter (this=0x771c76, __in_chrg=<value optimized out>) at ../../src/msg/SimpleMessenger.h:97
#2 0x00000000008c262d in SimpleMessenger::Accepter::entry (this=0x7f787a00cbef) at ../../src/msg/SimpleMessenger.cc:232
#3 0x00000000008d0cc5 in SimpleMessenger::Pipe::writer (this=0x56821c0) at ../../src/msg/SimpleMessenger.cc:1790
#4 0x00000000007770a4 in std::_Rb_tree<int, std::pair<int const, xlist<SimpleMessenger::Pipe*>*>, std::_Select1st<std::pair<int const, xlist<SimpleMessenger::Pipe*>*> >, std::less<int>, std::allocator<std::pair<int const, xlist<SimpleMessenger::Pipe*>*> > >::~_Rb_tree (this=0x7770a4,
__in_chrg=<value optimized out>) at /usr/include/c++/4.5/bits/stl_tree.h:614
#5 0x00000000008a2c85 in ceph_str_hash_rjenkins (str=0x8a2c85 "", length=32632) at ../../src/include/ceph_hash.cc:37
#6 0x00007f788f331d8c in ?? ()
#7 0x0000000000000000 in ?? ()

Thread 125 (Thread 11543):
#0 0x00007f788db65f03 in ?? ()
#1 0x0000000000775fd7 in SimpleMessenger::Accepter::~Accepter (this=0x771c76, __in_chrg=<value optimized out>) at ../../src/msg/SimpleMessenger.h:97
#2 0x00000000008c262d in SimpleMessenger::Accepter::entry (this=0x7f7877be8bef) at ../../src/msg/SimpleMessenger.cc:232
#3 0x00000000008d0cc5 in SimpleMessenger::Pipe::writer (this=0x55ac000) at ../../src/msg/SimpleMessenger.cc:1790
#4 0x00000000007770a4 in std::_Rb_tree<int, std::pair<int const, xlist<SimpleMessenger::Pipe*>*>, std::_Select1st<std::pair<int const, xlist<SimpleMessenger::Pipe*>*> >, std::less<int>, std::allocator<std::pair<int const, xlist<SimpleMessenger::Pipe*>*> > >::~_Rb_tree (this=0x7770a4,
__in_chrg=<value optimized out>) at /usr/include/c++/4.5/bits/stl_tree.h:614
#5 0x00000000008a2c85 in ceph_str_hash_rjenkins (str=0x8a2c85 "", length=32632) at ../../src/include/ceph_hash.cc:37
#6 0x00007f788f331d8c in ?? ()
#7 0x0000000000000000 in ?? ()

Thread 124 (Thread 10791):
#0 0x00007f788f336bac in ?? ()
#1 0x0000000400000000 in ?? ()
#2 0x0000000002e5c458 in ?? ()
#3 0x0000000002e5c358 in ?? ()
#4 0x0000000000000004 in ?? ()
#5 0x00000000007b5c9b in ObjectStore::C_DeleteTransaction::finish (this=0x2e5c450, r=0) at ../../src/os/ObjectStore.h:517
#6 0x00000000008d2a80 in SimpleMessenger::Pipe::read_message (this=0x7796ee, pm=0x7f787e551950) at ../../src/msg/SimpleMessenger.cc:2004
#7 0x00000000007770fe in std::map<int, xlist<SimpleMessenger::Pipe*>::iterator, std::less<int>, std::allocator<std::pair<int const, xlist<SimpleMessenger::Pipe*>::iterator> > >::map (this=0x2e5c4c8) at /usr/include/c++/4.5/bits/stl_map.h:151
#8 0x00000000008a2c85 in ceph_str_hash_rjenkins (str=0x8a2c85 "", length=32632) at ../../src/include/ceph_hash.cc:37
#9 0x00007f788f331d8c in ?? ()
#10 0x0000000000000000 in ?? ()

Thread 123 (Thread 10888):
#0 0x00007f788db65f03 in ?? ()
#1 0x00007f787d743c80 in ?? ()
#2 0x00000000000dbba0 in ?? ()
#3 0x0000000000000001 in ?? ()
#4 0x00007f787d743c90 in ?? ()
#5 0x0000000000775ffd in Throttle::~Throttle (this=0x4b3d6c0, __in_chrg=<value optimized out>) at ../../src/common/Throttle.h:7
#6 0x00000000008c262d in SimpleMessenger::Accepter::entry (this=0x7785ce) at ../../src/msg/SimpleMessenger.cc:232
#7 0x00000000008d0cc5 in SimpleMessenger::Pipe::writer (this=0x58a3a80) at ../../src/msg/SimpleMessenger.cc:1790
#8 0x00000000007770a4 in std::_Rb_tree<int, std::pair<int const, xlist<SimpleMessenger::Pipe*>*>, std::_Select1st<std::pair<int const, xlist<SimpleMessenger::Pipe*>*> >, std::less<int>, std::allocator<std::pair<int const, xlist<SimpleMessenger::Pipe*>*> > >::~_Rb_tree (this=0x7770a4,
__in_chrg=<value optimized out>) at /usr/include/c++/4.5/bits/stl_tree.h:614
#9 0x00000000008a2c85 in ceph_str_hash_rjenkins (str=0x8a2c85 "", length=32632) at ../../src/include/ceph_hash.cc:37
#10 0x00007f788f331d8c in ?? ()
#11 0x0000000000000000 in ?? ()

Thread 122 (Thread 10891):
#0 0x00007f788f336bac in ?? ()
#1 0x0000002d00000000 in ?? ()
#2 0x0000000004b3d6d8 in ?? ()
#3 0x0000000004b3d5d8 in ?? ()
#4 0x000000000000002d in ?? ()

#5 0x00000000007b5c9b in ObjectStore::C_DeleteTransaction::finish (this=0x4b3d6d0, r=0) at ../../src/os/ObjectStore.h:517
#6 0x00000000008d2a80 in SimpleMessenger::Pipe::read_message (this=0x7796ee, pm=0x7f787d642950) at ../../src/msg/SimpleMessenger.cc:2004
#7 0x00000000007770fe in std::map<int, xlist<SimpleMessenger::Pipe*>::iterator, std::less<int>, std::allocator<std::pair<int const, xlist<SimpleMessenger::Pipe*>::iterator> > >::map (this=0x4b3d748) at /usr/include/c++/4.5/bits/stl_map.h:151
#8 0x00000000008a2c85 in ceph_str_hash_rjenkins (str=0x8a2c85 "", length=32632) at ../../src/include/ceph_hash.cc:37
#9 0x00007f788f331d8c in ?? ()
#10 0x0000000000000000 in ?? ()

Thread 121 (Thread 11353):
#0 0x00007f788f336bac in ?? ()
#1 0x0000007e00000000 in ?? ()
#2 0x0000000005585458 in ?? ()
#3 0x0000000005585358 in ?? ()
#4 0x000000000000007e in ?? ()
#5 0x00000000007b5c9b in ObjectStore::C_DeleteTransaction::finish (this=0x5585450, r=0) at ../../src/os/ObjectStore.h:517
#6 0x00000000008d2a80 in SimpleMessenger::Pipe::read_message (this=0x7796ee, pm=0x7f787ad19950) at ../../src/msg/SimpleMessenger.cc:2004
#7 0x00000000007770fe in std::map<int, xlist<SimpleMessenger::Pipe*>::iterator, std::less<int>, std::allocator<std::pair<int const, xlist<SimpleMessenger::Pipe*>::iterator> > >::map (this=0x55854c8) at /usr/include/c++/4.5/bits/stl_map.h:151
#8 0x00000000008a2c85 in ceph_str_hash_rjenkins (str=0x8a2c85 "", length=32632) at ../../src/include/ceph_hash.cc:37
#9 0x00007f788f331d8c in ?? ()
#10 0x0000000000000000 in ?? ()

Thread 120 (Thread 11009):
#0 0x00007f788f336bac in ?? ()
#1 0x0000000300000000 in ?? ()
#2 0x0000000003fb61d8 in ?? ()
#3 0x0000000003fb60d8 in ?? ()
#4 0x0000000000000003 in ?? ()
#5 0x00000000007b5c9b in ObjectStore::C_DeleteTransaction::finish (this=0x3fb61d0, r=0) at ../../src/os/ObjectStore.h:517
#6 0x00000000008d2a80 in SimpleMessenger::Pipe::read_message (this=0x7796ee, pm=0x7f787c430950) at ../../src/msg/SimpleMessenger.cc:2004
#7 0x00000000007770fe in std::map<int, xlist<SimpleMessenger::Pipe*>::iterator, std::less<int>, std::allocator<std::pair<int const, xlist<SimpleMessenger::Pipe*>::iterator> > >::map (this=0x3fb6248) at /usr/include/c++/4.5/bits/stl_map.h:151
#8 0x00000000008a2c85 in ceph_str_hash_rjenkins (str=0x8a2c85 "", length=32632) at ../../src/include/ceph_hash.cc:37
#9 0x00007f788f331d8c in ?? ()
#10 0x0000000000000000 in ?? ()

Thread 119 (Thread 10390):
#0 0x00007f788f336f2b in ?? ()
#1 0x0000000000000000 in ?? ()

Thread 118 (Thread 11375):
#0 0x00007f788db65f03 in ?? ()
#1 0x0000000000775fd7 in SimpleMessenger::Accepter::~Accepter (this=0x771c76, __in_chrg=<value optimized out>) at ../../src/msg/SimpleMessenger.h:97
#2 0x00000000008c262d in SimpleMessenger::Accepter::entry (this=0x7f7879905bef) at ../../src/msg/SimpleMessenger.cc:232
#3 0x00000000008d0cc5 in SimpleMessenger::Pipe::writer (this=0x56821c0) at ../../src/msg/SimpleMessenger.cc:1790
#4 0x00000000007770a4 in std::_Rb_tree<int, std::pair<int const, xlist<SimpleMessenger::Pipe*>*>, std::_Select1st<std::pair<int const, xlist<SimpleMessenger::Pipe*>*> >, std::less<int>, std::allocator<std::pair<int const, xlist<SimpleMessenger::Pipe*>*> > >::~_Rb_tree (this=0x7770a4,
__in_chrg=<value optimized out>) at /usr/include/c++/4.5/bits/stl_tree.h:614
#5 0x00000000008a2c85 in ceph_str_hash_rjenkins (str=0x8a2c85 "", length=32632) at ../../src/include/ceph_hash.cc:37
#6 0x00007f788f331d8c in ?? ()
#7 0x0000000000000000 in ?? ()

Thread 117 (Thread 11379):
#0 0x00007f788f336bac in ?? ()
#1 0x0000000300000000 in ?? ()
#2 0x0000000005603bd8 in ?? ()
#3 0x0000000005603ad8 in ?? ()
#4 0x0000000000000003 in ?? ()
#5 0x00000000007b5c9b in ObjectStore::C_DeleteTransaction::finish (this=0x5603bd0, r=0) at ../../src/os/ObjectStore.h:517
#6 0x00000000008d2a80 in SimpleMessenger::Pipe::read_message (this=0x7796ee, pm=0x7f7879501950) at ../../src/msg/SimpleMessenger.cc:2004
#7 0x00000000007770fe in std::map<int, xlist<SimpleMessenger::Pipe*>::iterator, std::less<int>, std::allocator<std::pair<int const, xlist<SimpleMessenger::Pipe*>::iterator> > >::map (this=0x5603c48) at /usr/include/c++/4.5/bits/stl_map.h:151
#8 0x00000000008a2c85 in ceph_str_hash_rjenkins (str=0x8a2c85 "", length=32632) at ../../src/include/ceph_hash.cc:37
#9 0x00007f788f331d8c in ?? ()
#10 0x0000000000000000 in ?? ()

Thread 116 (Thread 10943):
#0 0x00007f788f336bac in ?? ()
#1 0x0000004500000000 in ?? ()
#2 0x0000000004b3de58 in ?? ()
#3 0x0000000004b3dd58 in ?? ()
#4 0x0000000000000045 in ?? ()
#5 0x00000000007b5c9b in ObjectStore::C_DeleteTransaction::finish (this=0x4b3de50, r=0) at ../../src/os/ObjectStore.h:517
#6 0x00000000008d2a80 in SimpleMessenger::Pipe::read_message (this=0x7796ee, pm=0x7f787ca36950) at ../../src/msg/SimpleMessenger.cc:2004
#7 0x00000000007770fe in std::map<int, xlist<SimpleMessenger::Pipe*>::iterator, std::less<int>, std::allocator<std::pair<int const, xlist<SimpleMessenger::Pipe*>::iterator> > >::map (this=0x4b3dec8) at /usr/include/c++/4.5/bits/stl_map.h:151
#8 0x00000000008a2c85 in ceph_str_hash_rjenkins (str=0x8a2c85 "", length=32632) at ../../src/include/ceph_hash.cc:37
#9 0x00007f788f331d8c in ?? ()
#10 0x0000000000000000 in ?? ()

Thread 115 (Thread 11531):
#0 0x00007f788f336bac in ?? ()
#1 0x0000000200000000 in ?? ()
#2 0x0000000005662958 in ?? ()
#3 0x0000000005662858 in ?? ()
#4 0x0000000000000002 in ?? ()
#5 0x00000000007b5c9b in ObjectStore::C_DeleteTransaction::finish (this=0x5662950, r=0) at ../../src/os/ObjectStore.h:517
#6 0x00000000008d2a80 in SimpleMessenger::Pipe::read_message (this=0x7796ee, pm=0x7f7877dea950) at ../../src/msg/SimpleMessenger.cc:2004
#7 0x00000000007770fe in std::map<int, xlist<SimpleMessenger::Pipe*>::iterator, std::less<int>, std::allocator<std::pair<int const, xlist<SimpleMessenger::Pipe*>::iterator> > >::map (this=0x56629c8) at /usr/include/c++/4.5/bits/stl_map.h:151
#8 0x00000000008a2c85 in ceph_str_hash_rjenkins (str=0x8a2c85 "", length=32632) at ../../src/include/ceph_hash.cc:37
#9 0x00007f788f331d8c in ?? ()
#10 0x0000000000000000 in ?? ()

Thread 114 (Thread 10947):
#0 0x00007f788db65f03 in ?? ()
#1 0x00007f787c935c80 in ?? ()
#2 0x00000000000dbba0 in ?? ()
#3 0x0000000000000001 in ?? ()
#4 0x00007f787c935c90 in ?? ()
#5 0x0000000000775ffd in Throttle::~Throttle (this=0x3fb6440, __in_chrg=<value optimized out>) at ../../src/common/Throttle.h:7
#6 0x00000000008c262d in SimpleMessenger::Accepter::entry (this=0x5637218) at ../../src/msg/SimpleMessenger.cc:232
#7 0x00000000008d0cc5 in SimpleMessenger::Pipe::writer (this=0x4151c40) at ../../src/msg/SimpleMessenger.cc:1790
#8 0x00000000007770a4 in std::_Rb_tree<int, std::pair<int const, xlist<SimpleMessenger::Pipe*>*>, std::_Select1st<std::pair<int const, xlist<SimpleMessenger::Pipe*>*> >, std::less<int>, std::allocator<std::pair<int const, xlist<SimpleMessenger::Pipe*>*> > >::~_Rb_tree (this=0x7770a4,
__in_chrg=<value optimized out>) at /usr/include/c++/4.5/bits/stl_tree.h:614
#9 0x00000000008a2c85 in ceph_str_hash_rjenkins (str=0x8a2c85 "", length=32632) at ../../src/include/ceph_hash.cc:37
#10 0x00007f788f331d8c in ?? ()
#11 0x0000000000000000 in ?? ()

Thread 113 (Thread 11494):
#0 0x00007f788db65f03 in ?? ()
#1 0x00007f78782efc80 in ?? ()
#2 0x00000000000dbba0 in ?? ()
#3 0x0000000000000001 in ?? ()
#4 0x00007f78782efc90 in ?? ()
#5 0x0000000000775ffd in Throttle::~Throttle (this=0x5603440, __in_chrg=<value optimized out>) at ../../src/common/Throttle.h:7
#6 0x00000000008c262d in SimpleMessenger::Accepter::entry (this=0x7785ce) at ../../src/msg/SimpleMessenger.cc:232
#7 0x00000000008d0cc5 in SimpleMessenger::Pipe::writer (this=0x55aca80) at ../../src/msg/SimpleMessenger.cc:1790
#8 0x00000000007770a4 in std::_Rb_tree<int, std::pair<int const, xlist<SimpleMessenger::Pipe*>*>, std::_Select1st<std::pair<int const, xlist<SimpleMessenger::Pipe*>*> >, std::less<int>, std::allocator<std::pair<int const, xlist<SimpleMessenger::Pipe*>*> > >::~_Rb_tree (this=0x7770a4,
__in_chrg=<value optimized out>) at /usr/include/c++/4.5/bits/stl_tree.h:614
#9 0x00000000008a2c85 in ceph_str_hash_rjenkins (str=0x8a2c85 "", length=32632) at ../../src/include/ceph_hash.cc:37
#10 0x00007f788f331d8c in ?? ()
#11 0x0000000000000000 in ?? ()

Thread 112 (Thread 10815):
#0 0x00007f788db65f03 in ?? ()
#1 0x00007f787df4bc80 in ?? ()
#2 0x00000000000dbba0 in ?? ()
#3 0x0000000000000001 in ?? ()
#4 0x00007f787df4bc90 in ?? ()
#5 0x0000000000775ffd in Throttle::~Throttle (this=0x2e5c940, __in_chrg=<value optimized out>) at ../../src/common/Throttle.h:7
#6 0x00000000008c262d in SimpleMessenger::Accepter::entry (this=0x7785ce) at ../../src/msg/SimpleMessenger.cc:232
#7 0x00000000008d0cc5 in SimpleMessenger::Pipe::writer (this=0x5682a80) at ../../src/msg/SimpleMessenger.cc:1790
#8 0x00000000007770a4 in std::_Rb_tree<int, std::pair<int const, xlist<SimpleMessenger::Pipe*>*>, std::_Select1st<std::pair<int const, xlist<SimpleMessenger::Pipe*>*> >, std::less<int>, std::allocator<std::pair<int const, xlist<SimpleMessenger::Pipe*>*> > >::~_Rb_tree (this=0x7770a4,
__in_chrg=<value optimized out>) at /usr/include/c++/4.5/bits/stl_tree.h:614
#9 0x00000000008a2c85 in ceph_str_hash_rjenkins (str=0x8a2c85 "", length=32632) at ../../src/include/ceph_hash.cc:37
#10 0x00007f788f331d8c in ?? ()
#11 0x0000000000000000 in ?? ()

Thread 111 (Thread 10823):
#0 0x00007f788db65f03 in ?? ()
#1 0x0000000000775fd7 in SimpleMessenger::Accepter::~Accepter (this=0x771c76, __in_chrg=<value optimized out>) at ../../src/msg/SimpleMessenger.h:97
#2 0x00000000008c262d in SimpleMessenger::Accepter::entry (this=0x7f787dd49bef) at ../../src/msg/SimpleMessenger.cc:232
#3 0x00000000008d0cc5 in SimpleMessenger::Pipe::writer (this=0x4a4a380) at ../../src/msg/SimpleMessenger.cc:1790
#4 0x00000000007770a4 in std::_Rb_tree<int, std::pair<int const, xlist<SimpleMessenger::Pipe*>*>, std::_Select1st<std::pair<int const, xlist<SimpleMessenger::Pipe*>*> >, std::less<int>, std::allocator<std::pair<int const, xlist<SimpleMessenger::Pipe*>*> > >::~_Rb_tree (this=0x7770a4,
__in_chrg=<value optimized out>) at /usr/include/c++/4.5/bits/stl_tree.h:614
#5 0x00000000008a2c85 in ceph_str_hash_rjenkins (str=0x8a2c85 "", length=32632) at ../../src/include/ceph_hash.cc:37
#6 0x00007f788f331d8c in ?? ()
#7 0x0000000000000000 in ?? ()

Thread 110 (Thread 9619):
#0 0x00007f788db65f03 in ?? ()
#1 0x00007f788d071e00 in ?? ()
#2 0x00000000ffffffff in ?? ()
#3 0x0000000000000002 in ?? ()
#4 0x00007f788d071e00 in ?? ()
#5 0x0000000000000052 in ?? ()
#6 0x00000000009d594a in CephXTicketManager::build_authorizer (this=0x2e3e320, service_id=0) at ../../src/auth/cephx/CephxProtocol.cc:334
#7 0x00000000008a2c85 in ceph_str_hash_rjenkins (str=0x8a2c85 "", length=32632) at ../../src/include/ceph_hash.cc:37
#8 0x00007f788f331d8c in ?? ()
#9 0x0000000000000000 in ?? ()

Thread 109 (Thread 11354):
#0 0x00007f788f336bac in ?? ()
#1 0x0000006700000000 in ?? ()
#2 0x00000000055851d8 in ?? ()
#3 0x00000000055850d8 in ?? ()
#4 0x0000000000000067 in ?? ()
#5 0x00000000007b5c9b in ObjectStore::C_DeleteTransaction::finish (this=0x55851d0, r=0) at ../../src/os/ObjectStore.h:517
#6 0x00000000008d2a80 in SimpleMessenger::Pipe::read_message (this=0x7796ee, pm=0x7f787ac18950) at ../../src/msg/SimpleMessenger.cc:2004
#7 0x00000000007770fe in std::map<int, xlist<SimpleMessenger::Pipe*>::iterator, std::less<int>, std::allocator<std::pair<int const, xlist<SimpleMessenger::Pipe*>::iterator> > >::map (this=0x5585248) at /usr/include/c++/4.5/bits/stl_map.h:151
#8 0x00000000008a2c85 in ceph_str_hash_rjenkins (str=0x8a2c85 "", length=32632) at ../../src/include/ceph_hash.cc:37
#9 0x00007f788f331d8c in ?? ()
#10 0x0000000000000000 in ?? ()

Thread 108 (Thread 11490):
#0 0x00007f788f336bac in ?? ()
#1 0x0000000200000000 in ?? ()
#2 0x00000000056031d8 in ?? ()
#3 0x00000000056030d8 in ?? ()
#4 0x0000000000000002 in ?? ()
#5 0x00000000007b5c9b in ObjectStore::C_DeleteTransaction::finish (this=0x56031d0, r=0) at ../../src/os/ObjectStore.h:517
#6 0x00000000008d2a80 in SimpleMessenger::Pipe::read_message (this=0x7796ee, pm=0x7f78785f2950) at ../../src/msg/SimpleMessenger.cc:2004
#7 0x00000000007770fe in std::map<int, xlist<SimpleMessenger::Pipe*>::iterator, std::less<int>, std::allocator<std::pair<int const, xlist<SimpleMessenger::Pipe*>::iterator, std::less<int>, std::allocator<std::pair<int const, xlist<SimpleMessenger::Pipe*>::iterator> > >::map (this=0x5603248) at /usr/include/c++/4.5/bits/stl_map.h:151
#8 0x00000000008a2c85 in ceph_str_hash_rjenkins (str=0x8a2c85 "", length=32632) at ../../src/include/ceph_hash.cc:37
#9 0x00007f788f331d8c in ?? ()
#10 0x0000000000000000 in ?? ()

Thread 107 (Thread 11540):
#0 0x00007f788f336bac in ?? ()
#1 0x0000000100000000 in ?? ()
#2 0x0000000005968958 in ?? ()
#3 0x0000000005968858 in ?? ()
#4 0x0000000000000001 in ?? ()
#5 0x00000000007b5c9b in ObjectStore::C_DeleteTransaction::finish (this=0x5968950, r=0) at ../../src/os/ObjectStore.h:517
#6 0x00000000008d2a80 in SimpleMessenger::Pipe::read_message (this=0x7796ee, pm=0x7f7877ce9950) at ../../src/msg/SimpleMessenger.cc:2004
#7 0x00000000007770fe in std::map<int, xlist<SimpleMessenger::Pipe*>::iterator, std::less<int>, std::allocator<std::pair<int const, xlist<SimpleMessenger::Pipe*>::iterator> > >::map (this=0x59689c8) at /usr/include/c++/4.5/bits/stl_map.h:151
#8 0x00000000008a2c85 in ceph_str_hash_rjenkins (str=0x8a2c85 "", length=32632) at ../../src/include/ceph_hash.cc:37
#9 0x00007f788f331d8c in ?? ()
#10 0x0000000000000000 in ?? ()

Thread 106 (Thread 11371):
#0 0x00007f788f336bac in ?? ()
#1 0x0000000200000000 in ?? ()
#2 0x00000000055f7e58 in ?? ()
#3 0x00000000055f7d58 in ?? ()
#4 0x0000000000000002 in ?? ()
#5 0x00000000007b5c9b in ObjectStore::C_DeleteTransaction::finish (this=0x55f7e50, r=0) at ../../src/os/ObjectStore.h:517
#6 0x00000000008d2a80 in SimpleMessenger::Pipe::read_message (this=0x7796ee, pm=0x7f7879e0a950) at ../../src/msg/SimpleMessenger.cc:2004
#7 0x00000000007770fe in std::map<int, xlist<SimpleMessenger::Pipe*>::iterator, std::less<int>, std::allocator<std::pair<int const, xlist<SimpleMessenger::Pipe*>::iterator> > >::map (this=0x55f7ec8) at /usr/include/c++/4.5/bits/stl_map.h:151
#8 0x00000000008a2c85 in ceph_str_hash_rjenkins (str=0x8a2c85 "", length=32632) at ../../src/include/ceph_hash.cc:37
#9 0x00007f788f331d8c in ?? ()
#10 0x0000000000000000 in ?? ()

Thread 105 (Thread 11369):
#0 0x00007f788f336bac in ?? ()
#1 0x0000000200000000 in ?? ()
#2 0x00000000055f71d8 in ?? ()
#3 0x00000000055f70d8 in ?? ()
#4 0x0000000000000002 in ?? ()
#5 0x00000000007b5c9b in ObjectStore::C_DeleteTransaction::finish (this=0x55f71d0, r=0) at ../../src/os/ObjectStore.h:517
#6 0x00000000008d2a80 in SimpleMessenger::Pipe::read_message (this=0x7796ee, pm=0x7f7879f0b950) at ../../src/msg/SimpleMessenger.cc:2004
#7 0x00000000007770fe in std::map<int, xlist<SimpleMessenger::Pipe*>::iterator, std::less<int>, std::allocator<std::pair<int const, xlist<SimpleMessenger::Pipe*>::iterator> > >::map (this=0x55f7248) at /usr/include/c++/4.5/bits/stl_map.h:151
#8 0x00000000008a2c85 in ceph_str_hash_rjenkins (str=0x8a2c85 "", length=32632) at ../../src/include/ceph_hash.cc:37
#9 0x00007f788f331d8c in ?? ()
#10 0x0000000000000000 in ?? ()

Thread 104 (Thread 11374):
#0 0x00007f788db65f03 in ?? ()
#1 0x0000000000775fd7 in SimpleMessenger::Accepter::~Accepter (this=0x771c76, __in_chrg=<value optimized out>) at ../../src/msg/SimpleMessenger.h:97
#2 0x00000000008c262d in SimpleMessenger::Accepter::entry (this=0x7f7879a06bef) at ../../src/msg/SimpleMessenger.cc:232
#3 0x00000000008d0cc5 in SimpleMessenger::Pipe::writer (this=0x4151000) at ../../src/msg/SimpleMessenger.cc:1790
#4 0x00000000007770a4 in std::_Rb_tree<int, std::pair<int const, xlist<SimpleMessenger::Pipe*>*>, std::_Select1st<std::pair<int const, xlist<SimpleMessenger::Pipe*>*> >, std::less<int>, std::allocator<std::pair<int const, xlist<SimpleMessenger::Pipe*>*> > >::~_Rb_tree (this=0x7770a4,
__in_chrg=<value optimized out>) at /usr/include/c++/4.5/bits/stl_tree.h:614
#5 0x00000000008a2c85 in ceph_str_hash_rjenkins (str=0x8a2c85 "", length=32632) at ../../src/include/ceph_hash.cc:37
#6 0x00007f788f331d8c in ?? ()
#7 0x0000000000000000 in ?? ()

Thread 103 (Thread 11355):
#0 0x00007f788db65f03 in ?? ()
#1 0x0000000000775fd7 in SimpleMessenger::Accepter::~Accepter (this=0x771c76, __in_chrg=<value optimized out>) at ../../src/msg/SimpleMessenger.h:97
#2 0x00000000008c262d in SimpleMessenger::Accepter::entry (this=0x7f787ab17bef) at ../../src/msg/SimpleMessenger.cc:232
#3 0x00000000008d0cc5 in SimpleMessenger::Pipe::writer (this=0x5074e00) at ../../src/msg/SimpleMessenger.cc:1790
#4 0x00000000007770a4 in std::_Rb_tree<int, std::pair<int const, xlist<SimpleMessenger::Pipe*>*>, std::_Select1st<std::pair<int const, xlist<SimpleMessenger::Pipe*>*> >, std::less<int>, std::allocator<std::pair<int const, xlist<SimpleMessenger::Pipe*>*> > >::~_Rb_tree (this=0x7770a4,
__in_chrg=<value optimized out>) at /usr/include/c++/4.5/bits/stl_tree.h:614
#5 0x00000000008a2c85 in ceph_str_hash_rjenkins (str=0x8a2c85 "", length=32632) at ../../src/include/ceph_hash.cc:37
#6 0x00007f788f331d8c in ?? ()
#7 0x0000000000000000 in ?? ()

Thread 102 (Thread 11109):
#0 0x00007f788f336bac in ?? ()
#1 0x0000006900000000 in ?? ()
#2 0x0000000005077bd8 in ?? ()
#3 0x0000000005077ad8 in ?? ()
#4 0x0000000000000069 in ?? ()
#5 0x00000000007b5c9b in ObjectStore::C_DeleteTransaction::finish (this=0x5077bd0, r=0) at ../../src/os/ObjectStore.h:517
#6 0x00000000008d2a80 in SimpleMessenger::Pipe::read_message (this=0x7796ee, pm=0x7f787be2a950) at ../../src/msg/SimpleMessenger.cc:2004
#7 0x00000000007770fe in std::map<int, xlist<SimpleMessenger::Pipe*>::iterator, std::less<int>, std::allocator<std::pair<int const, xlist<SimpleMessenger::Pipe*>::iterator> > >::map (this=0x5077c48) at /usr/include/c++/4.5/bits/stl_map.h:151
#8 0x00000000008a2c85 in ceph_str_hash_rjenkins (str=0x8a2c85 "", length=32632) at ../../src/include/ceph_hash.cc:37
#9 0x00007f788f331d8c in ?? ()
#10 0x0000000000000000 in ?? ()

Thread 101 (Thread 11104):
#0 0x00007f788db65f03 in ?? ()
#1 0x0000000000775fd7 in SimpleMessenger::Accepter::~Accepter (this=0x771c76, __in_chrg=<value optimized out>) at ../../src/msg/SimpleMessenger.h:97
#2 0x00000000008c262d in SimpleMessenger::Accepter::entry (this=0x7f787bf2bbef) at ../../src/msg/SimpleMessenger.cc:232
#3 0x00000000008d0cc5 in SimpleMessenger::Pipe::writer (this=0x4733000) at ../../src/msg/SimpleMessenger.cc:1790
#4 0x00000000007770a4 in std::_Rb_tree<int, std::pair<int const, xlist<SimpleMessenger::Pipe*>*>, std::_Select1st<std::pair<int const, xlist<SimpleMessenger::Pipe*>*> >, std::less<int>, std::allocator<std::pair<int const, xlist<SimpleMessenger::Pipe*>*> > >::~_Rb_tree (this=0x7770a4,
__in_chrg=<value optimized out>) at /usr/include/c++/4.5/bits/stl_tree.h:614
#5 0x00000000008a2c85 in ceph_str_hash_rjenkins (str=0x8a2c85 "", length=32632) at ../../src/include/ceph_hash.cc:37
#6 0x00007f788f331d8c in ?? ()
#7 0x0000000000000000 in ?? ()

Thread 100 (Thread 11356):
#0 0x00007f788db65f03 in ?? ()
#1 0x0000000000775fd7 in SimpleMessenger::Accepter::~Accepter (this=0x771c76, __in_chrg=<value optimized out>) at ../../src/msg/SimpleMessenger.h:97
#2 0x00000000008c262d in SimpleMessenger::Accepter::entry (this=0x7f787aa16bef) at ../../src/msg/SimpleMessenger.cc:232
#3 0x00000000008d0cc5 in SimpleMessenger::Pipe::writer (this=0x597c800) at ../../src/msg/SimpleMessenger.cc:1790
#4 0x00000000007770a4 in std::_Rb_tree<int, std::pair<int const, xlist<SimpleMessenger::Pipe*>*>, std::_Select1st<std::pair<int const, xlist<SimpleMessenger::Pipe*>*> >, std::less<int>, std::allocator<std::pair<int const, xlist<SimpleMessenger::Pipe*>*> > >::~_Rb_tree (this=0x7770a4,
__in_chrg=<value optimized out>) at /usr/include/c++/4.5/bits/stl_tree.h:614
#5 0x00000000008a2c85 in ceph_str_hash_rjenkins (str=0x8a2c85 "", length=32632) at ../../src/include/ceph_hash.cc:37
#6 0x00007f788f331d8c in ?? ()
#7 0x0000000000000000 in ?? ()

Thread 99 (Thread 11188):
#0 0x00007f788db65f03 in ?? ()
#1 0x00007f787b521c80 in ?? ()
#2 0x00000000000dbba0 in ?? ()
#3 0x0000000000000001 in ?? ()
#4 0x00007f787b521c90 in ?? ()
#5 0x0000000000775ffd in Throttle::~Throttle (this=0x507a6c0, __in_chrg=<value optimized out>) at ../../src/common/Throttle.h:7
#6 0x00000000008c262d in SimpleMessenger::Accepter::entry (this=0x5637058) at ../../src/msg/SimpleMessenger.cc:232
#7 0x00000000008d0cc5 in SimpleMessenger::Pipe::writer (this=0x530f000) at ../../src/msg/SimpleMessenger.cc:1790
#8 0x00000000007770a4 in std::_Rb_tree<int, std::pair<int const, xlist<SimpleMessenger::Pipe*>*>, std::_Select1st<std::pair<int const, xlist<SimpleMessenger::Pipe*>*> >, std::less<int>, std::allocator<std::pair<int const, xlist<SimpleMessenger::Pipe*>*> > >::~_Rb_tree (this=0x7770a4,
__in_chrg=<value optimized out>) at /usr/include/c++/4.5/bits/stl_tree.h:614
#9 0x00000000008a2c85 in ceph_str_hash_rjenkins (str=0x8a2c85 "", length=32632) at ../../src/include/ceph_hash.cc:37
#10 0x00007f788f331d8c in ?? ()
#11 0x0000000000000000 in ?? ()

Thread 98 (Thread 11014):
#0 0x00007f788f336bac in ?? ()
#1 0x0000003500000000 in ?? ()
#2 0x0000000005077e58 in ?? ()
#3 0x0000000005077d58 in ?? ()
#4 0x0000000000000035 in ?? ()
#5 0x00000000007b5c9b in ObjectStore::C_DeleteTransaction::finish (this=0x5077e50, r=0) at ../../src/os/ObjectStore.h:517
#6 0x00000000008d2a80 in SimpleMessenger::Pipe::read_message (this=0x7796ee, pm=0x7f787c22e950) at ../../src/msg/SimpleMessenger.cc:2004
#7 0x00000000007770fe in std::map<int, xlist<SimpleMessenger::Pipe*>::iterator, std::less<int>, std::allocator<std::pair<int const, xlist<SimpleMessenger::Pipe*>::iterator> > >::map (this=0x5077ec8) at /usr/include/c++/4.5/bits/stl_map.h:151
#8 0x00000000008a2c85 in ceph_str_hash_rjenkins (str=0x8a2c85 "", length=32632) at ../../src/include/ceph_hash.cc:37
#9 0x00007f788f331d8c in ?? ()
#10 0x0000000000000000 in ?? ()

Thread 97 (Thread 10940):
#0 0x00007f788db65f03 in ?? ()
#1 0x0000000000775fd7 in SimpleMessenger::Accepter::~Accepter (this=0x771c76, __in_chrg=<value optimized out>) at ../../src/msg/SimpleMessenger.h:97
#2 0x00000000008c262d in SimpleMessenger::Accepter::entry (this=0x7f787cb37bef) at ../../src/msg/SimpleMessenger.cc:232
#3 0x00000000008d0cc5 in SimpleMessenger::Pipe::writer (this=0x3fb71c0) at ../../src/msg/SimpleMessenger.cc:1790
#4 0x00000000007770a4 in std::_Rb_tree<int, std::pair<int const, xlist<SimpleMessenger::Pipe*>*>, std::_Select1st<std::pair<int const, xlist<SimpleMessenger::Pipe*>*> >, std::less<int>, std::allocator<std::pair<int const, xlist<SimpleMessenger::Pipe*>*> > >::~_Rb_tree (this=0x7770a4,
__in_chrg=<value optimized out>) at /usr/include/c++/4.5/bits/stl_tree.h:614
#5 0x00000000008a2c85 in ceph_str_hash_rjenkins (str=0x8a2c85 "", length=32632) at ../../src/include/ceph_hash.cc:37
#6 0x00007f788f331d8c in ?? ()
#7 0x0000000000000000 in ?? ()

Thread 96 (Thread 9625):
#0 0x00007f788db65f03 in ?? ()
#1 0x0000000005aef180 in ?? ()
#2 0x00000000ffffffff in ?? ()
#3 0x0000000000000001 in ?? ()
#4 0x00007f788b06dce0 in ?? ()
#5 0x00007f788b06de50 in ?? ()
#6 0x00000000008c4020 in SimpleMessenger::send_message (this=0x0, m=0x0, dest=...) at ../../src/msg/SimpleMessenger.cc:427
#7 0x00000000008a2c85 in ceph_str_hash_rjenkins (str=0x8a2c85 "", length=32632) at ../../src/include/ceph_hash.cc:37
#8 0x00007f788f331d8c in ?? ()
#9 0x0000000000000000 in ?? ()

Thread 95 (Thread 11586):
#0 0x00007f788f336bac in ?? ()
#1 0x0000000100000000 in ?? ()
#2 0x00000000056626d8 in ?? ()
#3 0x00000000056625d8 in ?? ()
#4 0x0000000000000001 in ?? ()
#5 0x00000000007b5c9b in ObjectStore::C_DeleteTransaction::finish (this=0x56626d0, r=0) at ../../src/os/ObjectStore.h:517
#6 0x00000000008d2a80 in SimpleMessenger::Pipe::read_message (this=0x0, pm=0x0) at ../../src/msg/SimpleMessenger.cc:2004
#7 0x00000000007770fe in std::map<int, xlist<SimpleMessenger::Pipe*>::iterator, std::less<int>, std::allocator<std::pair<int const, xlist<SimpleMessenger::Pipe*>::iterator> > >::map (this=0x5662748) at /usr/include/c++/4.5/bits/stl_map.h:151
#8 0x00000000008a2c85 in ceph_str_hash_rjenkins (str=0x8a2c85 "", length=32632) at ../../src/include/ceph_hash.cc:37
#9 0x00007f788f331d8c in ?? ()
#10 0x0000000000000000 in ?? ()

Thread 94 (Thread 10816):
#0 0x00007f788f336bac in ?? ()
#1 0x0000001b00000000 in ?? ()
#2 0x0000000002e5c958 in ?? ()
#3 0x0000000002e5c858 in ?? ()
#4 0x000000000000001b in ?? ()
#5 0x00000000007b5c9b in ObjectStore::C_DeleteTransaction::finish (this=0x2e5c950, r=0) at ../../src/os/ObjectStore.h:517
#6 0x00000000008d2a80 in SimpleMessenger::Pipe::read_message (this=0x7796ee, pm=0x7f787de4a950) at ../../src/msg/SimpleMessenger.cc:2004
#7 0x00000000007770fe in std::map<int, xlist<SimpleMessenger::Pipe*>::iterator, std::less<int>, std::allocator<std::pair<int const, xlist<SimpleMessenger::Pipe*>::iterator> > >::map (this=0x2e5c9c8) at /usr/include/c++/4.5/bits/stl_map.h:151
#8 0x00000000008a2c85 in ceph_str_hash_rjenkins (str=0x8a2c85 "", length=32632) at ../../src/include/ceph_hash.cc:37
#9 0x00007f788f331d8c in ?? ()
#10 0x0000000000000000 in ?? ()

Thread 93 (Thread 9622):
#0 0x00007f788db65f03 in ?? ()
#1 0x0000000000000000 in ?? ()

Thread 92 (Thread 11140):
#0 0x00007f788db65f03 in ?? ()
#1 0x00007f787b925c80 in ?? ()
#2 0x00000000000dbba0 in ?? ()
#3 0x0000000000000001 in ?? ()
#4 0x00007f787b925c90 in ?? ()
#5 0x0000000000775ffd in Throttle::~Throttle (this=0x50776c0, __in_chrg=<value optimized out>) at ../../src/common/Throttle.h:7
#6 0x00000000008c262d in SimpleMessenger::Accepter::entry (this=0x7785ce) at ../../src/msg/SimpleMessenger.cc:232
#7 0x00000000008d0cc5 in SimpleMessenger::Pipe::writer (this=0x55b5e00) at ../../src/msg/SimpleMessenger.cc:1790
#8 0x00000000007770a4 in std::_Rb_tree<int, std::pair<int const, xlist<SimpleMessenger::Pipe*>*>, std::_Select1st<std::pair<int const, xlist<SimpleMessenger::Pipe*>*> >, std::less<int>, std::allocator<std::pair<int const, xlist<SimpleMessenger::Pipe*>*> > >::~_Rb_tree (this=0x7770a4,
__in_chrg=<value optimized out>) at /usr/include/c++/4.5/bits/stl_tree.h:614
#9 0x00000000008a2c85 in ceph_str_hash_rjenkins (str=0x8a2c85 "", length=32632) at ../../src/include/ceph_hash.cc:37
#10 0x00007f788f331d8c in ?? ()
#11 0x0000000000000000 in ?? ()

Thread 91 (Thread 11366):
#0 0x00007f788f336bac in ?? ()
#1 0x0000000200000000 in ?? ()
#2 0x00000000055f76d8 in ?? ()
#3 0x00000000055f75d8 in ?? ()
#4 0x0000000000000002 in ?? ()
#5 0x00000000007b5c9b in ObjectStore::C_DeleteTransaction::finish (this=0x55f76d0, r=0) at ../../src/os/ObjectStore.h:517
#6 0x00000000008d2a80 in SimpleMessenger::Pipe::read_message (this=0x7796ee, pm=0x7f787a20e950) at ../../src/msg/SimpleMessenger.cc:2004
#7 0x00000000007770fe in std::map<int, xlist<SimpleMessenger::Pipe*>::iterator, std::less<int>, std::allocator<std::pair<int const, xlist<SimpleMessenger::Pipe*>::iterator> > >::map (this=0x55f7748) at /usr/include/c++/4.5/bits/stl_map.h:151
#8 0x00000000008a2c85 in ceph_str_hash_rjenkins (str=0x8a2c85 "", length=32632) at ../../src/include/ceph_hash.cc:37
#9 0x00007f788f331d8c in ?? ()
#10 0x0000000000000000 in ?? ()

Thread 90 (Thread 10853):
#0 0x00007f788f336bac in ?? ()
#1 0x0000002500000000 in ?? ()
#2 0x0000000004b3d958 in ?? ()
#3 0x0000000004b3d858 in ?? ()
#4 0x0000000000000025 in ?? ()
#5 0x00000000007b5c9b in ObjectStore::C_DeleteTransaction::finish (this=0x4b3d950, r=0) at ../../src/os/ObjectStore.h:517
#6 0x00000000008d2a80 in SimpleMessenger::Pipe::read_message (this=0x7796ee, pm=0x7f787d844950) at ../../src/msg/SimpleMessenger.cc:2004
#7 0x00000000007770fe in std::map<int, xlist<SimpleMessenger::Pipe*>::iterator, std::less<int>, std::allocator<std::pair<int const, xlist<SimpleMessenger::Pipe*>::iterator> > >::map (this=0x4b3d9c8) at /usr/include/c++/4.5/bits/stl_map.h:151
#8 0x00000000008a2c85 in ceph_str_hash_rjenkins (str=0x8a2c85 "", length=32632) at ../../src/include/ceph_hash.cc:37
#9 0x00007f788f331d8c in ?? ()
#10 0x0000000000000000 in ?? ()

Thread 89 (Thread 11126):
#0 0x00007f788db65f03 in ?? ()
#1 0x00007f787bd29c80 in ?? ()
#2 0x00000000000dbba0 in ?? ()
#3 0x0000000000000001 in ?? ()
#4 0x00007f787bd29c90 in ?? ()
#5 0x0000000000775ffd in Throttle::~Throttle (this=0x507abc0, __in_chrg=<value optimized out>) at ../../src/common/Throttle.h:7
#6 0x00000000008c262d in SimpleMessenger::Accepter::entry (this=0x5637598) at ../../src/msg/SimpleMessenger.cc:232
#7 0x00000000008d0cc5 in SimpleMessenger::Pipe::writer (this=0x5228000) at ../../src/msg/SimpleMessenger.cc:1790
#8 0x00000000007770a4 in std::_Rb_tree<int, std::pair<int const, xlist<SimpleMessenger::Pipe*>*>, std::_Select1st<std::pair<int const, xlist<SimpleMessenger::Pipe*>*> >, std::less<int>, std::allocator<std::pair<int const, xlist<SimpleMessenger::Pipe*>*> > >::~_Rb_tree (this=0x7770a4,
__in_chrg=<value optimized out>) at /usr/include/c++/4.5/bits/stl_tree.h:614
#9 0x00000000008a2c85 in ceph_str_hash_rjenkins (str=0x8a2c85 "", length=32632) at ../../src/include/ceph_hash.cc:37
#10 0x00007f788f331d8c in ?? ()
#11 0x0000000000000000 in ?? ()

Thread 88 (Thread 11360):
#0 0x00007f788f336bac in ?? ()
#1 0x0000000100000000 in ?? ()
#2 0x00000000055856d8 in ?? ()
#3 0x00000000055855d8 in ?? ()
#4 0x0000000000000001 in ?? ()
#5 0x00000000007b5c9b in ObjectStore::C_DeleteTransaction::finish (this=0x55856d0, r=0) at ../../src/os/ObjectStore.h:517
#6 0x00000000008d2a80 in SimpleMessenger::Pipe::read_message (this=0x7796ee, pm=0x7f787a713950) at ../../src/msg/SimpleMessenger.cc:2004
#7 0x00000000007770fe in std::map<int, xlist<SimpleMessenger::Pipe*>::iterator, std::less<int>, std::allocator<std::pair<int const, xlist<SimpleMessenger::Pipe*>::iterator> > >::map (this=0x5585748) at /usr/include/c++/4.5/bits/stl_map.h:151
#8 0x00000000008a2c85 in ceph_str_hash_rjenkins (str=0x8a2c85 "", length=32632) at ../../src/include/ceph_hash.cc:37
#9 0x00007f788f331d8c in ?? ()
#10 0x0000000000000000 in ?? ()

Thread 87 (Thread 11314):
#0 0x00007f788f336bac in ?? ()
#1 0x0000000a00000000 in ?? ()
#2 0x00000000050771d8 in ?? ()
#3 0x00000000050770d8 in ?? ()
#4 0x000000000000000a in ?? ()
#5 0x00000000007b5c9b in ObjectStore::C_DeleteTransaction::finish (this=0x50771d0, r=0) at ../../src/os/ObjectStore.h:517
#6 0x00000000008d2a80 in SimpleMessenger::Pipe::read_message (this=0x7796ee, pm=0x7f787ae1a950) at ../../src/msg/SimpleMessenger.cc:2004
#7 0x00000000007770fe in std::map<int, xlist<SimpleMessenger::Pipe*>::iterator, std::less<int>, std::allocator<std::pair<int const, xlist<SimpleMessenger::Pipe*>::iterator> > >::map (this=0x5077248) at /usr/include/c++/4.5/bits/stl_map.h:151
#8 0x00000000008a2c85 in ceph_str_hash_rjenkins (str=0x8a2c85 "", length=32632) at ../../src/include/ceph_hash.cc:37
#9 0x00007f788f331d8c in ?? ()
#10 0x0000000000000000 in ?? ()

Thread 86 (Thread 11583):
#0 0x00007f788db65f03 in ?? ()
#1 0x0000000000775fd7 in SimpleMessenger::Accepter::~Accepter (this=0x771c76, __in_chrg=<value optimized out>) at ../../src/msg/SimpleMessenger.h:97
#2 0x00000000008c262d in SimpleMessenger::Accepter::entry (this=0x7f78779e6bef) at ../../src/msg/SimpleMessenger.cc:232
#3 0x00000000008d0cc5 in SimpleMessenger::Pipe::writer (this=0x512e700) at ../../src/msg/SimpleMessenger.cc:1790

#4 0x00000000007770a4 in std::_Rb_tree<int, std::pair<int const, xlist<SimpleMessenger::Pipe*>*>, std::_Select1st<std::pair<int const, xlist<SimpleMessenger::Pipe*>*> >, std::less<int>, std::allocator<std::pair<int const, xlist<SimpleMessenger::Pipe*>*> > >::~_Rb_tree (this=0x7770a4,
__in_chrg=<value optimized out>) at /usr/include/c++/4.5/bits/stl_tree.h:614
#5 0x00000000008a2c85 in ceph_str_hash_rjenkins (str=0x8a2c85 "", length=32632) at ../../src/include/ceph_hash.cc:37
#6 0x00007f788f331d8c in ?? ()
#7 0x0000000000000000 in ?? ()

Thread 85 (Thread 10785):
#0 0x00007f788db65f03 in ?? ()
#1 0x0000000000775fd7 in SimpleMessenger::Accepter::~Accepter (this=0x771c76, __in_chrg=<value optimized out>) at ../../src/msg/SimpleMessenger.h:97
#2 0x00000000008c262d in SimpleMessenger::Accepter::entry (this=0x7f787e753bef) at ../../src/msg/SimpleMessenger.cc:232
#3 0x00000000008d0cc5 in SimpleMessenger::Pipe::writer (this=0x61b0000) at ../../src/msg/SimpleMessenger.cc:1790
#4 0x00000000007770a4 in std::_Rb_tree<int, std::pair<int const, xlist<SimpleMessenger::Pipe*>*>, std::_Select1st<std::pair<int const, xlist<SimpleMessenger::Pipe*>*> >, std::less<int>, std::allocator<std::pair<int const, xlist<SimpleMessenger::Pipe*>*> > >::~_Rb_tree (this=0x7770a4,
__in_chrg=<value optimized out>) at /usr/include/c++/4.5/bits/stl_tree.h:614
#5 0x00000000008a2c85 in ceph_str_hash_rjenkins (str=0x8a2c85 "", length=32632) at ../../src/include/ceph_hash.cc:37
#6 0x00007f788f331d8c in ?? ()
#7 0x0000000000000000 in ?? ()

Thread 84 (Thread 10425):
#0 0x00007f788f336f2b in ?? ()
#1 0x0000000000000000 in ?? ()

Thread 83 (Thread 11381):
#0 0x00007f788db65f03 in ?? ()
#1 0x0000000000775fd7 in SimpleMessenger::Accepter::~Accepter (this=0x771c76, __in_chrg=<value optimized out>) at ../../src/msg/SimpleMessenger.h:97
#2 0x00000000008c262d in SimpleMessenger::Accepter::entry (this=0x7f78792ffbef) at ../../src/msg/SimpleMessenger.cc:232
#3 0x00000000008d0cc5 in SimpleMessenger::Pipe::writer (this=0x5adc380) at ../../src/msg/SimpleMessenger.cc:1790
#4 0x00000000007770a4 in std::_Rb_tree<int, std::pair<int const, xlist<SimpleMessenger::Pipe*>*>, std::_Select1st<std::pair<int const, xlist<SimpleMessenger::Pipe*>*> >, std::less<int>, std::allocator<std::pair<int const, xlist<SimpleMessenger::Pipe*>*> > >::~_Rb_tree (this=0x7770a4,
__in_chrg=<value optimized out>) at /usr/include/c++/4.5/bits/stl_tree.h:614
#5 0x00000000008a2c85 in ceph_str_hash_rjenkins (str=0x8a2c85 "", length=32632) at ../../src/include/ceph_hash.cc:37
#6 0x00007f788f331d8c in ?? ()
#7 0x0000000000000000 in ?? ()

Thread 82 (Thread 11138):
#0 0x00007f788f336bac in ?? ()
#1 0x0000004f00000000 in ?? ()
#2 0x0000000005077958 in ?? ()
#3 0x0000000005077858 in ?? ()
#4 0x000000000000004f in ?? ()
#5 0x00000000007b5c9b in ObjectStore::C_DeleteTransaction::finish (this=0x5077950, r=0) at ../../src/os/ObjectStore.h:517
#6 0x00000000008d2a80 in SimpleMessenger::Pipe::read_message (this=0x7796ee, pm=0x7f787ba26950) at ../../src/msg/SimpleMessenger.cc:2004
#7 0x00000000007770fe in std::map<int, xlist<SimpleMessenger::Pipe*>::iterator, std::less<int>, std::allocator<std::pair<int const, xlist<SimpleMessenger::Pipe*>::iterator> > >::map (this=0x50779c8) at /usr/include/c++/4.5/bits/stl_map.h:151
#8 0x00000000008a2c85 in ceph_str_hash_rjenkins (str=0x8a2c85 "", length=32632) at ../../src/include/ceph_hash.cc:37
#9 0x00007f788f331d8c in ?? ()
#10 0x0000000000000000 in ?? ()

Thread 81 (Thread 10419):
#0 0x00007f788f336bac in ?? ()
#1 0x0000001100000000 in ?? ()
#2 0x0000000002e6d958 in ?? ()
#3 0x0000000002e6d858 in ?? ()
#4 0x0000000000000011 in ?? ()
#5 0x00000000007b5c9b in ObjectStore::C_DeleteTransaction::finish (this=0x2e6d950, r=0) at ../../src/os/ObjectStore.h:517
#6 0x00000000008d2a80 in SimpleMessenger::Pipe::read_message (this=0x7796ee, pm=0x7f788f754950) at ../../src/msg/SimpleMessenger.cc:2004
#7 0x00000000007770fe in std::map<int, xlist<SimpleMessenger::Pipe*>::iterator, std::less<int>, std::allocator<std::pair<int const, xlist<SimpleMessenger::Pipe*>::iterator> > >::map (this=0x2e6d9c8) at /usr/include/c++/4.5/bits/stl_map.h:151
#8 0x00000000008a2c85 in ceph_str_hash_rjenkins (str=0x8a2c85 "", length=32632) at ../../src/include/ceph_hash.cc:37
#9 0x00007f788f331d8c in ?? ()
#10 0x0000000000000000 in ?? ()

Thread 80 (Thread 9628):
#0 0x00007f788f336bac in ?? ()
#1 0x0000000000000000 in ?? ()

Thread 79 (Thread 10913):
#0 0x00007f788db65f03 in ?? ()
#1 0x00007f787d13dc80 in ?? ()
#2 0x00000000000dbba0 in ?? ()
#3 0x0000000000000001 in ?? ()
#4 0x00007f787d13dc90 in ?? ()
#5 0x0000000000775ffd in Throttle::~Throttle (this=0x3fb6940, __in_chrg=<value optimized out>) at ../../src/common/Throttle.h:7
#6 0x00000000008c262d in SimpleMessenger::Accepter::entry (this=0x5637918) at ../../src/msg/SimpleMessenger.cc:232
#7 0x00000000008d0cc5 in SimpleMessenger::Pipe::writer (this=0x4a4a8c0) at ../../src/msg/SimpleMessenger.cc:1790
#8 0x00000000007770a4 in std::_Rb_tree<int, std::pair<int const, xlist<SimpleMessenger::Pipe*>*>, std::_Select1st<std::pair<int const, xlist<SimpleMessenger::Pipe*>*> >, std::less<int>, std::allocator<std::pair<int const, xlist<SimpleMessenger::Pipe*>*> > >::~_Rb_tree (this=0x7770a4,
__in_chrg=<value optimized out>) at /usr/include/c++/4.5/bits/stl_tree.h:614
#9 0x00000000008a2c85 in ceph_str_hash_rjenkins (str=0x8a2c85 "", length=32632) at ../../src/include/ceph_hash.cc:37
#10 0x00007f788f331d8c in ?? ()
#11 0x0000000000000000 in ?? ()

Thread 78 (Thread 11380):
#0 0x00007f788f336bac in ?? ()
#1 0x0000000200000000 in ?? ()
#2 0x0000000005603958 in ?? ()
#3 0x0000000005603858 in ?? ()
#4 0x0000000000000002 in ?? ()
#5 0x00000000007b5c9b in ObjectStore::C_DeleteTransaction::finish (this=0x5603950, r=0) at ../../src/os/ObjectStore.h:517
#6 0x00000000008d2a80 in SimpleMessenger::Pipe::read_message (this=0x7796ee, pm=0x7f7879400950) at ../../src/msg/SimpleMessenger.cc:2004
#7 0x00000000007770fe in std::map<int, xlist<SimpleMessenger::Pipe*>::iterator, std::less<int>, std::allocator<std::pair<int const, xlist<SimpleMessenger::Pipe*>::iterator> > >::map (this=0x56039c8) at /usr/include/c++/4.5/bits/stl_map.h:151
#8 0x00000000008a2c85 in ceph_str_hash_rjenkins (str=0x8a2c85 "", length=32632) at ../../src/include/ceph_hash.cc:37
#9 0x00007f788f331d8c in ?? ()
#10 0x0000000000000000 in ?? ()

Thread 77 (Thread 10902):
#0 0x00007f788db65f03 in ?? ()
#1 0x0000000000775fd7 in SimpleMessenger::Accepter::~Accepter (this=0x771c76, __in_chrg=<value optimized out>) at ../../src/msg/SimpleMessenger.h:97
#2 0x00000000008c262d in SimpleMessenger::Accepter::entry (this=0x7f787d33fbef) at ../../src/msg/SimpleMessenger.cc:232
#3 0x00000000008d0cc5 in SimpleMessenger::Pipe::writer (this=0x6133c40) at ../../src/msg/SimpleMessenger.cc:1790
#4 0x00000000007770a4 in std::_Rb_tree<int, std::pair<int const, xlist<SimpleMessenger::Pipe*>*>, std::_Select1st<std::pair<int const, xlist<SimpleMessenger::Pipe*>*> >, std::less<int>, std::allocator<std::pair<int const, xlist<SimpleMessenger::Pipe*>*> > >::~_Rb_tree (this=0x7770a4,
__in_chrg=<value optimized out>) at /usr/include/c++/4.5/bits/stl_tree.h:614
#5 0x00000000008a2c85 in ceph_str_hash_rjenkins (str=0x8a2c85 "", length=32632) at ../../src/include/ceph_hash.cc:37
#6 0x00007f788f331d8c in ?? ()
#7 0x0000000000000000 in ?? ()

Thread 76 (Thread 9627):
#0 0x00007f788db65f03 in ?? ()
#1 0x000000000418d6f0 in ?? ()
#2 0x00000000ffffffff in ?? ()
#3 0x0000000000000001 in ?? ()
#4 0x00007f788a06bce0 in ?? ()
#5 0x00007f788a06be50 in ?? ()
#6 0x00000000008c4020 in SimpleMessenger::send_message (this=0x0, m=0x0, dest=...) at ../../src/msg/SimpleMessenger.cc:427
#7 0x00000000008a2c85 in ceph_str_hash_rjenkins (str=0x8a2c85 "", length=32632) at ../../src/include/ceph_hash.cc:37
#8 0x00007f788f331d8c in ?? ()
#9 0x0000000000000000 in ?? ()

Thread 75 (Thread 11022):
#0 0x00007f788f336bac in ?? ()
#1 0x0000000300000000 in ?? ()
#2 0x000000000507ae58 in ?? ()
#3 0x000000000507ad58 in ?? ()
#4 0x0000000000000003 in ?? ()
#5 0x00000000007b5c9b in ObjectStore::C_DeleteTransaction::finish (this=0x507ae50, r=0) at ../../src/os/ObjectStore.h:517
#6 0x00000000008d2a80 in SimpleMessenger::Pipe::read_message (this=0x7796ee, pm=0x7f787c02c950) at ../../src/msg/SimpleMessenger.cc:2004
#7 0x00000000007770fe in std::map<int, xlist<SimpleMessenger::Pipe*>::iterator, std::less<int>, std::allocator<std::pair<int const, xlist<SimpleMessenger::Pipe*>::iterator> > >::map (this=0x507aec8) at /usr/include/c++/4.5/bits/stl_map.h:151
#8 0x00000000008a2c85 in ceph_str_hash_rjenkins (str=0x8a2c85 "", length=32632) at ../../src/include/ceph_hash.cc:37
#9 0x00007f788f331d8c in ?? ()
#10 0x0000000000000000 in ?? ()

Thread 74 (Thread 11008):
#0 0x00007f788db65f03 in ?? ()
#1 0x00007f787c531c80 in ?? ()
#2 0x00000000000dbba0 in ?? ()
#3 0x0000000000000001 in ?? ()
#4 0x00007f787c531c90 in ?? ()
#5 0x0000000000775ffd in Throttle::~Throttle (this=0x3fb61c0, __in_chrg=<value optimized out>) at ../../src/common/Throttle.h:7
#6 0x00000000008c262d in SimpleMessenger::Accepter::entry (this=0x5a59918) at ../../src/msg/SimpleMessenger.cc:232
#7 0x00000000008d0cc5 in SimpleMessenger::Pipe::writer (this=0x5074700) at ../../src/msg/SimpleMessenger.cc:1790
#8 0x00000000007770a4 in std::_Rb_tree<int, std::pair<int const, xlist<SimpleMessenger::Pipe*>*>, std::_Select1st<std::pair<int const, xlist<SimpleMessenger::Pipe*>*> >, std::less<int>, std::allocator<std::pair<int const, xlist<SimpleMessenger::Pipe*>*> > >::~_Rb_tree (this=0x7770a4,
__in_chrg=<value optimized out>) at /usr/include/c++/4.5/bits/stl_tree.h:614
#9 0x00000000008a2c85 in ceph_str_hash_rjenkins (str=0x8a2c85 "", length=32632) at ../../src/include/ceph_hash.cc:37
#10 0x00007f788f331d8c in ?? ()
#11 0x0000000000000000 in ?? ()

Thread 73 (Thread 10926):
#0 0x00007f788db65f03 in ?? ()
#1 0x0000000000775fd7 in SimpleMessenger::Accepter::~Accepter (this=0x771c76, __in_chrg=<value optimized out>) at ../../src/msg/SimpleMessenger.h:97
#2 0x00000000008c262d in SimpleMessenger::Accepter::entry (this=0x7f787cf3bbef) at ../../src/msg/SimpleMessenger.cc:232
#3 0x00000000008d0cc5 in SimpleMessenger::Pipe::writer (this=0x61b0700) at ../../src/msg/SimpleMessenger.cc:1790
#4 0x00000000007770a4 in std::_Rb_tree<int, std::pair<int const, xlist<SimpleMessenger::Pipe*>*>, std::_Select1st<std::pair<int const, xlist<SimpleMessenger::Pipe*>*> >, std::less<int>, std::allocator<std::pair<int const, xlist<SimpleMessenger::Pipe*>*> > >::~_Rb_tree (this=0x7770a4,
__in_chrg=<value optimized out>) at /usr/include/c++/4.5/bits/stl_tree.h:614
#5 0x00000000008a2c85 in ceph_str_hash_rjenkins (str=0x8a2c85 "", length=32632) at ../../src/include/ceph_hash.cc:37
#6 0x00007f788f331d8c in ?? ()
#7 0x0000000000000000 in ?? ()

Thread 72 (Thread 11173):
#0 0x00007f788f336bac in ?? ()
#1 0x0000001600000000 in ?? ()
#2 0x00000000050776d8 in ?? ()
#3 0x00000000050775d8 in ?? ()
#4 0x0000000000000016 in ?? ()
#5 0x00000000007b5c9b in ObjectStore::C_DeleteTransaction::finish (this=0x50776d0, r=0) at ../../src/os/ObjectStore.h:517
#6 0x00000000008d2a80 in SimpleMessenger::Pipe::read_message (this=0x7796ee, pm=0x7f787b824950) at ../../src/msg/SimpleMessenger.cc:2004
#7 0x00000000007770fe in std::map<int, xlist<SimpleMessenger::Pipe*>::iterator, std::less<int>, std::allocator<std::pair<int const, xlist<SimpleMessenger::Pipe*>::iterator> > >::map (this=0x5077748) at /usr/include/c++/4.5/bits/stl_map.h:151
#8 0x00000000008a2c85 in ceph_str_hash_rjenkins (str=0x8a2c85 "", length=32632) at ../../src/include/ceph_hash.cc:37
#9 0x00007f788f331d8c in ?? ()
#10 0x0000000000000000 in ?? ()

Thread 71 (Thread 11499):
#0 0x00007f788db65f03 in ?? ()
#1 0x0000000000775fd7 in SimpleMessenger::Accepter::~Accepter (this=0x771c76, __in_chrg=<value optimized out>) at ../../src/msg/SimpleMessenger.h:97
#2 0x00000000008c262d in SimpleMessenger::Accepter::entry (this=0x7f7877fecbef) at ../../src/msg/SimpleMessenger.cc:232
#3 0x00000000008d0cc5 in SimpleMessenger::Pipe::writer (this=0x2ee0700) at ../../src/msg/SimpleMessenger.cc:1790
#4 0x00000000007770a4 in std::_Rb_tree<int, std::pair<int const, xlist<SimpleMessenger::Pipe*>*>, std::_Select1st<std::pair<int const, xlist<SimpleMessenger::Pipe*>*> >, std::less<int>, std::allocator<std::pair<int const, xlist<SimpleMessenger::Pipe*>*> > >::~_Rb_tree (this=0x7770a4,
__in_chrg=<value optimized out>) at /usr/include/c++/4.5/bits/stl_tree.h:614
#5 0x00000000008a2c85 in ceph_str_hash_rjenkins (str=0x8a2c85 "", length=32632) at ../../src/include/ceph_hash.cc:37
#6 0x00007f788f331d8c in ?? ()
#7 0x0000000000000000 in ?? ()

Thread 70 (Thread 11373):
#0 0x00007f788db65f03 in ?? ()

#1 0x0000000000775fd7 in SimpleMessenger::Accepter::~Accepter (this=0x771c76, __in_chrg=<value optimized out>) at ../../src/msg/SimpleMessenger.h:97
#2 0x00000000008c262d in SimpleMessenger::Accepter::entry (this=0x7f7879b07bef) at ../../src/msg/SimpleMessenger.cc:232
#3 0x00000000008d0cc5 in SimpleMessenger::Pipe::writer (this=0x56351c0) at ../../src/msg/SimpleMessenger.cc:1790
#4 0x00000000007770a4 in std::_Rb_tree<int, std::pair<int const, xlist<SimpleMessenger::Pipe*>*>, std::_Select1st<std::pair<int const, xlist<SimpleMessenger::Pipe*>*> >, std::less<int>, std::allocator<std::pair<int const, xlist<SimpleMessenger::Pipe*>*> > >::~_Rb_tree (this=0x7770a4,
__in_chrg=<value optimized out>) at /usr/include/c++/4.5/bits/stl_tree.h:614
#5 0x00000000008a2c85 in ceph_str_hash_rjenkins (str=0x8a2c85 "", length=32632) at ../../src/include/ceph_hash.cc:37
#6 0x00007f788f331d8c in ?? ()
#7 0x0000000000000000 in ?? ()

Thread 69 (Thread 11127):
#0 0x00007f788f336bac in ?? ()
#1 0x0000000300000000 in ?? ()
#2 0x000000000507abd8 in ?? ()
#3 0x000000000507aad8 in ?