Ceph » rgw

Running radosgw 12.2.5 from the ceph/daemon docker image (with ceph-radosgw-12.2.5-0.el7.x86_64) we had multiple (tens) of crashes with this same signature:

GDB Backtrace:

#5 0x00007f5be578f968 in abort () from /lib64/libc.so.6
#6 0x00007f5be82c5ec4 in ceph::__ceph_assert_fail (assertion=assertion@entry=0x7f5bf1cf294a "len+off <= bp.length()",
file=file@entry=0x7f5bf1cf2af0 "/home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos7/DIST/centos7/MACHINE_SIZE/huge/release/12.2.5/rpm/el7/BUILD/ceph-12.2.5/src/common/buffer.cc",
line=line@entry=1967,
func=func@entry=0x7f5bf1cf33a0 <ceph::buffer::list::append(ceph::buffer::ptr const&, unsigned int, unsigned int)::__PRETTY_FUNCTION__> "void ceph::buffer::list::append(const ceph::buffer::ptr&, unsigned int, unsigned int)")
at /usr/src/debug/ceph-12.2.5/src/common/assert.cc:66
#7 0x00007f5bf1c937ea in ceph::buffer::list::append (this=this@entry=0x7f5bb4a61150, bp=..., off=off@entry=0,
len=len@entry=6029312) at /usr/src/debug/ceph-12.2.5/src/common/buffer.cc:1967
#8 0x00005614d500511e in RGWPutObj_ObjStore::get_data (this=this@entry=0x5614d965ca00, bl=...)
at /usr/src/debug/ceph-12.2.5/src/rgw/rgw_rest.cc:1151
#9 0x00005614d50347ee in RGWPutObj_ObjStore_S3::get_data (this=0x5614d965ca00, bl=...)
at /usr/src/debug/ceph-12.2.5/src/rgw/rgw_rest_s3.cc:1353
#10 0x00005614d4f473b1 in RGWPutObj::execute (this=0x5614d965ca00) at /usr/src/debug/ceph-12.2.5/src/rgw/rgw_op.cc:3456
#11 0x00005614d4f69472 in rgw_process_authenticated (handler=handler@entry=0x5614def8e960,
op=@0x7f5bb4a61638: 0x5614d965ca00, req=req@entry=0x7f5bb4a62110, s=s@entry=0x7f5bb4a61a20,
skip_retarget=skip_retarget@entry=false) at /usr/src/debug/ceph-12.2.5/src/rgw/rgw_process.cc:104
#12 0x00005614d4f6a238 in process_request (store=0x5614d797a000, rest=0x7ffcc35b7d00, req=req@entry=0x7f5bb4a62110,
frontend_prefix="", auth_registry=..., client_io=client_io@entry=0x7f5bb4a62140, olog=0x0,
http_ret=http_ret@entry=0x7f5bb4a6210c) at /usr/src/debug/ceph-12.2.5/src/rgw/rgw_process.cc:207
#13 0x00005614d4e37d02 in RGWCivetWebFrontend::process (this=0x5614d7797540, conn=<optimized out>)
at /usr/src/debug/ceph-12.2.5/src/rgw/rgw_civetweb_frontend.cc:36
#14 0x00005614d4e7285f in handle_request (conn=conn@entry=0x5614d7dc2000)
at /usr/src/debug/ceph-12.2.5/src/civetweb/src/civetweb.c:9890
#15 0x00005614d4e741eb in process_new_connection (conn=<optimized out>)
at /usr/src/debug/ceph-12.2.5/src/civetweb/src/civetweb.c:12328
#16 worker_thread_run (thread_func_param=0x5614d7a7f000) at /usr/src/debug/ceph-12.2.5/src/civetweb/src/civetweb.c:12505
#17 worker_thread (thread_func_param=0x5614d7a7f000) at /usr/src/debug/ceph-12.2.5/src/civetweb/src/civetweb.c:12542
#18 0x00007f5bf0ce2e25 in start_thread () from /lib64/libpthread.so.0
#19 0x00007f5be5856bad in clone () from /lib64/libc.so.6
(gdb) fr 8
#8 0x00005614d500511e in RGWPutObj_ObjStore::get_data (this=this@entry=0x5614d965ca00, bl=...)
at /usr/src/debug/ceph-12.2.5/src/rgw/rgw_rest.cc:1151
1151 bl.append(bp, 0, len);
(gdb) list
1146 if (read_len < 0) {
1147 return read_len;
1148 }
1149
1150 len = read_len;
1151 bl.append(bp, 0, len);
1152
1153 ACCOUNTING_IO(s)->set_account(false);
1154 }
1155
(gdb) p bp._len
$1 = 4194304
(gdb) p len
$2 = 6029312
(gdb)


We have the corefile if needed as well