Ceph

Unfortunately I can't use these because my libraries don't match....gdb is finicky. :(

Can you instead run gdb yourself and then:
go to frame 11 (Client::update_inode_file_bits): f 11
print the values of prior_size and truncate_size: p prior_size; p truncate_size

That should be all the info I need to make this go away. :)

All right, got in and found:
Identical truncate_seqs of 2.
Identical truncate_sizes of 0.
prior_size of 209715200.
I think these changes may have already been applied in-memory, which is why they're identical? Certainly 209715200 - 0 != 0....

(gdb) bt
#0 0x0000000000000000 in ?? ()
#1 0x000000000054ba52 in sigabrt_handler (signum=6) at config.cc:238
#2 <signal handler called>
#3 0x00007f6ba7aeaa75 in raise () from /lib/libc.so.6
#4 0x00007f6ba7aee5c0 in abort () from /lib/libc.so.6
#5 0x00007f6ba83a08e5 in _gnu_cxx::_verbose_terminate_handler() ()
from /usr/lib/libstdc++.so.6
#6 0x00007f6ba839ed16 in ?? () from /usr/lib/libstdc++.so.6
#7 0x00007f6ba839ed43 in std::terminate() () from /usr/lib/libstdc++.so.6
#8 0x00007f6ba839ee3e in _cxa_throw () from /usr/lib/libstdc++.so.6
#9 0x0000000000538bdc in ceph::_ceph_assert_fail (
assertion=0x571b1f "len > 0", file=0x57e2cc "osdc/Filer.cc", line=305,
func=0x57e4c0 "void Filer::file_to_extents(inodeno_t, ceph_file_layout*, uint64_t, uint64_t, std::vector<ObjectExtent, std::allocator<ObjectExtent> >&)")
at common/assert.cc:30
#10 0x00000000004f23b3 in Filer::file_to_extents (this=0x16efc50, ino=...,
layout=0x1700dfc, offset=0, len=0, extents=...) at osdc/Filer.cc:305
#11 0x000000000046c5d9 in Client::update_inode_file_bits (this=0x16f3e80,
in=0x1700dd0, truncate_seq=<value optimized out>, truncate_size=0,
size=209715200, time_warp_seq=0, ctime=..., mtime=..., atime=..., issued=0)
at client/Client.cc:385
#12 0x0000000000471a71 in Client::add_update_inode (this=0x16f3e80,
st=0x7f6ba6c90e10, from=<value optimized out>, mds=0)
at client/Client.cc:501
#13 0x000000000049bbdd in Client::insert_trace (this=0x16f3e80,
request=0x16f9130, from=..., mds=0) at client/Client.cc:745
#14 0x000000000049d4e2 in Client::handle_client_reply (this=0x16f3e80,
reply=0x16fea80) at client/Client.cc:1306
#15 0x000000000049dd9b in Client::ms_dispatch (this=0x16f3e80, m=0x16fea80)
at client/Client.cc:1375
#16 0x0000000000453bdb in Messenger::ms_deliver_dispatch (this=0x16f00f0)
at msg/Messenger.h:97
#17 SimpleMessenger::dispatch_entry (this=0x16f00f0)
at msg/SimpleMessenger.cc:342
#18 0x000000000044890f in SimpleMessenger::DispatchThread::entry (
this=0x16f0578) at msg/SimpleMessenger.h:570
#19 0x000000000045a63a in Thread::_entry_func (arg=0x3e5b)
at ./common/Thread.h:39
#20 0x00007f6ba897e9ca in start_thread () from /lib/libpthread.so.0
#21 0x00007f6ba7b9d70d in clone () from /lib/libc.so.6
#22 0x0000000000000000 in ?? ()

(gdb) f 12
#12 0x0000000000471a71 in Client::add_update_inode (this=0x16f3e80,
st=0x7f6ba6c90e10, from=<value optimized out>, mds=0)
at client/Client.cc:501
501 client/Client.cc: No such file or directory.
in client/Client.cc
(gdb) p st->truncate_seq
$1 = 2
(gdb) p in->truncate_seq
$2 = 2
(gdb) p st->truncate_size
$3 = 0
(gdb) p in->truncate_size
$4 = 0
(gdb) p in->size
$5 = 209715200
(gdb) down
#11 0x000000000046c5d9 in Client::update_inode_file_bits (this=0x16f3e80,
in=0x1700dd0, truncate_seq=<value optimized out>, truncate_size=0,
size=209715200, time_warp_seq=0, ctime=..., mtime=..., atime=..., issued=0)
at client/Client.cc:385
385 in client/Client.cc
(gdb) p prior_size
$6 = 0

And while I've got it here's the inode printout:

$4 = {ino = {val = 1099511632800}, snapid = {val = 18446744073709551614},
rdev = 0, ctime = {tv = {tv_sec = 1287272094, tv_nsec = 598244000}},
mode = 33188, uid = 0, gid = 0, nlink = 1, layout = {fl_stripe_unit = {
v = 4194304}, fl_stripe_count = {v = 1}, fl_object_size = {v = 4194304},
fl_cas_hash = {v = 0}, fl_object_stripe_unit = {v = 0}, fl_pg_preferred = {
v = 4294967295}, fl_pg_pool = {v = 0}}, size = 209715200,
truncate_seq = 2, truncate_size = 0, mtime = {tv = {tv_sec = 0,
tv_nsec = 0}}, atime = {tv = {tv_sec = 0, tv_nsec = 0}},
time_warp_seq = 0, max_size = 0, dirstat = {version = 0, mtime = {tv = {
tv_sec = 0, tv_nsec = 0}}, nfiles = 0, nsubdirs = 0}, rstat = {
version = 0, rctime = {tv = {tv_sec = 0, tv_nsec = 0}},
rbytes = 209715200, rfiles = 1, rsubdirs = 0, ranchors = 0,
rsnaprealms = 0}, version = 0, xattr_version = 0, flags = 0,
dir_contacts = {_M_t = {
M_impl = {<std::allocator<std::_Rb_tree_node<int> >> = {<_gnu_cxx::new_allocator<std::_Rb_tree_node<int> >> = {<No data fields>}, <No data fields>},
M_key_compare = {<std::binary_function<int, int, bool>> = {<No data fields>}, <No data fields>}, _M_header = {_M_color = std::_S_red,
_M_parent = 0x0, _M_left = 0x1700ec8, _M_right = 0x1700ec8},
_M_node_count = 0}}}, dir_hashed = false, dir_replicated = false,
caps = {_M_t = {
_M_impl = {<std::allocator<std::_Rb_tree_node<std::pair<int const, InodeCap*> > >> = {<_gnu_cxx::new_allocator<std::_Rb_tree_node<std::pair<int const, InodeCap*> > >> = {<No data fields>}, <No data fields>},
M_key_compare = {<std::binary_function<int, int, bool>> = {<No data fields>}, <No data fields>}, _M_header = {_M_color = std::_S_red,
_M_parent = 0x17011f0, _M_left = 0x17011f0, _M_right = 0x17011f0},
_M_node_count = 1}}}, auth_cap = 0x1701230, dirty_caps = 0,
flushing_caps = 0, flushing_cap_seq = 0, flushing_cap_tid = {
0 <repeats 22 times>}, shared_gen = 1, cache_gen = 1, snap_caps = 0,
snap_cap_refs = 0, exporting_issued = 0, exporting_mds = -1,
exporting_mseq = 0, hold_caps_until = {tv = {tv_sec = 0, tv_nsec = 0}},
cap_item = {_item = 0x1700dd0, _prev = 0x17008f0, _next = 0x0,
_list = 0x16f4268}, flushing_cap_item = {_item = 0x1700dd0, _prev = 0x0,
_next = 0x0, _list = 0x0}, last_flush_tid = 0, snaprealm = 0x16fad50,
snaprealm_item = {_item = 0x1700dd0, _prev = 0x1700940, _next = 0x0,
_list = 0x16fae08}, snapdir_parent = 0x0, cap_snaps = {_M_t = {
_M_impl = {<std::allocator<std::_Rb_tree_node<std::pair<snapid_t const, CapSnap> > >> = {<_gnu_cxx::new_allocator<std::_Rb_tree_node<std::pair<snapid_t const, CapSnap> > >> = {<No data fields>}, <No data fields>},
M_key_compare = {<std::binary_function<snapid_t, snapid_t, bool>> = {<No data fields>}, <No data fields>}, _M_header = {_M_color = std::_S_red,
_M_parent = 0x0, _M_left = 0x1701010, _M_right = 0x1701010},
_M_node_count = 0}}}, open_by_mode = {_M_t = {
_M_impl = {<std::allocator<std::_Rb_tree_node<std::pair<int const, int> > >> = {<_gnu_cxx::new_allocator<std::_Rb_tree_node<std::pair<int const, int> > >> = {<No data fields>}, <No data fields>},
_M_key_compare = {<std::binary_function<int, int, bool>> = {<No data fields>}, <No data fields>}, _M_header = {_M_color = std::_S_red,
_M_parent = 0x0, _M_left = 0x1701040, _M_right = 0x1701040},
_M_node_count = 0}}}, cap_refs = {_M_t = { {static npos = 18446744073709551615, _M_dataplus = warning: can't find linker symbol for virtual table for `std::basic_string<char, std::char_traits<char>, std::allocator<char> >::_Alloc_hider' value
warning: found `std::string::_Rep::_S_empty_rep_storage' instead

val = 1099511632800}, truncate_seq = 2, truncate_size = 0, objects = {
      _front = 0x0, _back = 0x0, _size = 0}, uncommitted = {_front = 0x0, 
      _back = 0x0, _size = 0}, dirty_tx = 0}, reported_size = 209715200, 
  wanted_max_size = 0, requested_max_size = 0, ref = 0, ll_ref = 0, dir = 0x0, 
  dn = 0x0, symlink = warning: can't find linker symbol for virtual table for `std::basic_string<char, std::char_traits<char>, std::allocator<char> >' value
warning:   found `std::string::_Rep::_S_empty_rep_storage' instead

{<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, M_p = 0x7b9e38 ""}}, dirfragtree = {_splits = {_M_t = {
_M_impl = {<std::allocator<std::_Rb_tree_node<std::pair<frag_t const, int> > >> = {<_gnu_cxx::new_allocator<std::_Rb_tree_node<std::pair<frag_t const, int> > >> = {<No data fields>}, <No data fields>},
M_key_compare = {<std::binary_function<frag_t, frag_t, bool>> = {<No data fields>}, <No data fields>}, _M_header = {_M_color = std::_S_red,
_M_parent = 0x0, _M_left = 0x1701130, _M_right = 0x1701130},
_M_node_count = 0}}}}, xattrs = {_M_t = {
_M_impl = {<std::allocator<std::_Rb_tree_node<std::pair<std::basic_string<char, std::char_traits<char>, std::allocator<char> > const, ceph::buffer::ptr> > >> = {<_gnu_cxx::new_allocator<std::_Rb_tree_node<std::pair<std::basic_string<char, std::char_traits<char>, std::allocator<char> > const, ceph::buffer::ptr> > >> = {<No data fields>}, <No data fields>},
M_key_compare = {<std::binary_function<std::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool>> = {<No data fields>}, <No data fields>}, _M_header = {_M_color = std::_S_red, _M_parent = 0x0, _M_left = 0x1701160,
_M_right = 0x1701160}, _M_node_count = 0}}}, fragmap = {_M_t = {
_M_impl = {<std::allocator<std::_Rb_tree_node<std::pair<frag_t const, int> > >> = {<_gnu_cxx::new_allocator<std::_Rb_tree_node<std::pair<frag_t const, int> > >> = {<No data fields>}, <No data fields>},
M_key_compare = {<std::binary_function<frag_t, frag_t, bool>> = {<No data fields>}, <No data fields>}, _M_header = {_M_color = std::_S_red,
_M_parent = 0x0, _M_left = 0x1701190, _M_right = 0x1701190},
_M_node_count = 0}}},
waitfor_caps = {<std::_List_base<Cond*, std::allocator<Cond*> >> = {
_M_impl = {<std::allocator<std::_List_node<Cond*> >> = {<_gnu_cxx::new_allocator<std::_List_node<Cond*> >> = {<No data fields>}, <No data fields>},
M_node = {_M_next = 0x17011b8,
_M_prev = 0x17011b8}}}, <No data fields>},
waitfor_commit = {<std::_List_base<Cond*, std::allocator<Cond*> >> = {
_M_impl = {<std::allocator<std::_List_node<Cond*> >> = {<_gnu_cxx::new_allocator<std::_List_node<Cond*> >> = {<No data fields>}, <No data fields>},
_M_node = {_M_next = 0x17011c8,
_M_prev = 0x17011c8}}}, <No data fields>},
hack_balance_reads = false}

compiled b5d9bec659daa8ba26810e7508ec473aba8ad287 but is still crashing on ls:

root@srv-9pjdl:/root/ceph.git/src# ./cfuse /mnt/ceph -f -o debug
cfuse[12667]: starting ceph client
cfuse[12667]: starting fuse
FUSE library version: 2.8.1
unique: 1, opcode: INIT (26), nodeid: 0, insize: 56
INIT: 7.13
flags=0x0000007b
max_readahead=0x00020000
   INIT: 7.12
   flags=0x00000001
   max_readahead=0x00020000
   max_write=0x00020000
   unique: 1, success, outsize: 40
unique: 2, opcode: STATFS (17), nodeid: 1, insize: 40
   unique: 2, success, outsize: 96
unique: 3, opcode: GETATTR (3), nodeid: 1, insize: 56
   unique: 3, success, outsize: 120
unique: 4, opcode: GETATTR (3), nodeid: 1, insize: 56
   unique: 4, success, outsize: 120
unique: 5, opcode: OPENDIR (27), nodeid: 1, insize: 48
   unique: 5, success, outsize: 32
unique: 6, opcode: READDIR (28), nodeid: 1, insize: 80
client/Client.cc: In function 'void Client::update_inode_file_bits(Inode*, uint64_t, uint64_t, uint64_t, uint64_t, utime_t, utime_t, utime_t, int)':
client/Client.cc:378: FAILED assert(in->is_file())
 ceph version 0.22.1 (commit:b5d9bec659daa8ba26810e7508ec473aba8ad287)
 1: (Client::add_update_inode(InodeStat*, utime_t, int)+0xcd7) [0x46f1d7]
 2: (Client::insert_trace(MetaRequest*, utime_t, int)+0xeb0) [0x483820]
 3: (Client::handle_client_reply(MClientReply*)+0x323) [0x4855a3]
 4: (Client::ms_dispatch(Message*)+0x35b) [0x49c8cb]
 5: (SimpleMessenger::dispatch_entry()+0x69a) [0x45011a]
 6: (SimpleMessenger::DispatchThread::entry()+0x4d) [0x44616d]
 7: (Thread::_entry_func(void*)+0x7) [0x45a937]
 8: (()+0x69ca) [0x7f7dfa5289ca]
 9: (clone()+0x6d) [0x7f7df974770d]
 NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
client/Client.cc: In function 'void Client::update_inode_file_bits(Inode*, uint64_t, uint64_t, uint64_t, uint64_t, utime_t, utime_t, utime_t, int)':
client/Client.cc:378: FAILED assert(in->is_file())
 ceph version 0.22.1 (commit:b5d9bec659daa8ba26810e7508ec473aba8ad287)
 1: (Client::add_update_inode(InodeStat*, utime_t, int)+0xcd7) [0x46f1d7]
 2: (Client::insert_trace(MetaRequest*, utime_t, int)+0xeb0) [0x483820]
 3: (Client::handle_client_reply(MClientReply*)+0x323) [0x4855a3]
 4: (Client::ms_dispatch(Message*)+0x35b) [0x49c8cb]
 5: (SimpleMessenger::dispatch_entry()+0x69a) [0x45011a]
 6: (SimpleMessenger::DispatchThread::entry()+0x4d) [0x44616d]
 7: (Thread::_entry_func(void*)+0x7) [0x45a937]
 8: (()+0x69ca) [0x7f7dfa5289ca]
 9: (clone()+0x6d) [0x7f7df974770d]
 NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
terminate called after throwing an instance of 'ceph::FailedAssertion'
*** Caught signal (ABRT) ***
 ceph version 0.22.1 (commit:b5d9bec659daa8ba26810e7508ec473aba8ad287)
 1: (sigabrt_handler(int)+0x7d) [0x53fc5d]
 2: (()+0x33af0) [0x7f7df9694af0]
 3: (gsignal()+0x35) [0x7f7df9694a75]
 4: (abort()+0x180) [0x7f7df96985c0]
 5: (__gnu_cxx::__verbose_terminate_handler()+0x115) [0x7f7df9f4a8e5]
 6: (()+0xcad16) [0x7f7df9f48d16]
 7: (()+0xcad43) [0x7f7df9f48d43]
 8: (()+0xcae3e) [0x7f7df9f48e3e]
 9: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x448) [0x52da88]
 10: (Client::update_inode_file_bits(Inode*, unsigned long, unsigned long, unsigned long, unsigned long, utime_t, utime_t, utime_t, int)+0x88a) [0x46b78a]
 11: (Client::add_update_inode(InodeStat*, utime_t, int)+0xcd7) [0x46f1d7]
 12: (Client::insert_trace(MetaRequest*, utime_t, int)+0xeb0) [0x483820]
 13: (Client::handle_client_reply(MClientReply*)+0x323) [0x4855a3]
 14: (Client::ms_dispatch(Message*)+0x35b) [0x49c8cb]
 15: (SimpleMessenger::dispatch_entry()+0x69a) [0x45011a]
 16: (SimpleMessenger::DispatchThread::entry()+0x4d) [0x44616d]
 17: (Thread::_entry_func(void*)+0x7) [0x45a937]
 18: (()+0x69ca) [0x7f7dfa5289ca]
 19: (clone()+0x6d) [0x7f7df974770d]
Segmentation fault

Okay, commit:4fd49203b6c757b97455f1b85d5b93c76e20e199 is a partial revert of my initial (incorrect) fix, but keeps the useful bit.

Still to figure out is why the inode had a different truncate_size to begin with. I've fixed one occurrence of an uninitialized truncate_size in commit:cf8bb4c80a620a48365804adfd5d3750e6dcac90, but I suspect the real culprit is one of the number of places that use uninitialized CInodes and dump into them from external sources. This will require more work to make sure truncation is working correctly, but what we have here should prevent cfuse from crashing.
A possible exception to this is if a directory inode somehow ends up with an uninitialized truncate_size being sent to the client, but I'm not sure and it's getting hard to focus on this stuff, so it'll have to wait until later. ;)

Confirmed this is fixed 0.23.1 (sorry for huge delay in confirmation).