Ceph » rbd

This happens with 12.2.10 and 12.2.11, this backtrace is from 12.2.10. The cluster is running 12.2.10.

Thread 331 "python2.7" received signal SIGSEGV, Segmentation fault.
[Switching to Thread 0x7ffd39742700 (LWP 86403)]
Objecter::prepare_read_op (this=0x0, oid=..., oloc=..., op=..., snapid=..., pbl=pbl@entry=0x7ffed0033098, flags=0, onack=0x7ffecc00a910, objver=0x7ffecc00a7d0, data_offset=0x0, features=0, parent_trace=0x7ffd39740b57) at /var/tmp/portage/sys-cluster/ceph-12.2.10/work/ceph-12.2.10/src/osdc/Objecter.h:2266
2266        Op *o = new Op(oid, oloc, op.ops, flags | global_op_flags |
(gdb) bt
#0  Objecter::prepare_read_op (this=0x0, oid=..., oloc=..., op=..., snapid=..., pbl=pbl@entry=0x7ffed0033098, flags=0, onack=0x7ffecc00a910, objver=0x7ffecc00a7d0, data_offset=0x0, features=0, parent_trace=0x7ffd39740b57) at /var/tmp/portage/sys-cluster/ceph-12.2.10/work/ceph-12.2.10/src/osdc/Objecter.h:2266
#1  0x00007ffff175509d in librados::IoCtxImpl::aio_operate_read (this=0x7ffecc007830, oid=..., o=0x7ffed00580d0, c=0x7ffecc00a710, flags=flags@entry=0, pbl=pbl@entry=0x7ffed0033098, trace_info=0x0) at /var/tmp/portage/sys-cluster/ceph-12.2.10/work/ceph-12.2.10/src/librados/IoCtxImpl.cc:789
#2  0x00007ffff172abdc in librados::IoCtx::aio_operate (this=this@entry=0x7ffecc0203e8, oid=..., c=c@entry=0x7ffecc00a8b0, o=o@entry=0x7ffd39740c30, pbl=pbl@entry=0x7ffed0033098) at /var/tmp/portage/sys-cluster/ceph-12.2.10/work/ceph-12.2.10/src/librados/librados.cc:1553
#3  0x00007fffe51b3d09 in librbd::image::OpenRequest<librbd::ImageCtx>::send_v2_detect_header (this=this@entry=0x7ffed0033080) at /var/tmp/portage/sys-cluster/ceph-12.2.10/work/ceph-12.2.10/src/librbd/image/OpenRequest.cc:96
#4  0x00007fffe51b3e05 in librbd::image::OpenRequest<librbd::ImageCtx>::send (this=this@entry=0x7ffed0033080) at /var/tmp/portage/sys-cluster/ceph-12.2.10/work/ceph-12.2.10/src/librbd/image/OpenRequest.cc:42
#5  0x00007fffe511c64f in librbd::ImageState<librbd::ImageCtx>::send_open_unlock (this=0x7ffecc00c2d0) at /var/tmp/portage/sys-cluster/ceph-12.2.10/work/ceph-12.2.10/src/librbd/ImageState.cc:592
#6  0x00007fffe511e479 in librbd::ImageState<librbd::ImageCtx>::execute_next_action_unlock (this=this@entry=0x7ffecc00c2d0) at /var/tmp/portage/sys-cluster/ceph-12.2.10/work/ceph-12.2.10/src/librbd/ImageState.cc:521
#7  0x00007fffe511f319 in librbd::ImageState<librbd::ImageCtx>::execute_action_unlock (this=this@entry=0x7ffecc00c2d0, action=..., on_finish=on_finish@entry=0x7ffecc015a20) at /var/tmp/portage/sys-cluster/ceph-12.2.10/work/ceph-12.2.10/src/librbd/ImageState.cc:546
#8  0x00007fffe511f4bd in librbd::ImageState<librbd::ImageCtx>::open (this=0x7ffecc00c2d0, skip_open_parent=skip_open_parent@entry=true, on_finish=0x7ffecc015a20) at /var/tmp/portage/sys-cluster/ceph-12.2.10/work/ceph-12.2.10/src/librbd/ImageState.cc:271
#9  0x00007fffe51c37ec in librbd::image::RemoveRequest<librbd::ImageCtx>::open_image (this=this@entry=0x7ffecc019340) at /var/tmp/portage/sys-cluster/ceph-12.2.10/work/ceph-12.2.10/src/librbd/image/RemoveRequest.cc:62
#10 0x00007fffe51c3a1d in librbd::image::RemoveRequest<librbd::ImageCtx>::send (this=this@entry=0x7ffecc019340) at /var/tmp/portage/sys-cluster/ceph-12.2.10/work/ceph-12.2.10/src/librbd/image/RemoveRequest.cc:51
#11 0x00007fffe513566a in librbd::remove (io_ctx=..., image_name=..., image_id=..., prog_ctx=..., force=force@entry=false, from_trash_remove=from_trash_remove@entry=false) at /var/tmp/portage/sys-cluster/ceph-12.2.10/work/ceph-12.2.10/src/librbd/internal.cc:1331
#12 0x00007fffe50ffe8a in rbd_remove (p=p@entry=0x7ffecc007830, name=name@entry=0x7fffe484c354 "2a13167bd277b2206b3dc3ba721207d8a18e86da68a98ab01267cf8dae6d8315-1549518692-to_delete") at /var/tmp/portage/sys-cluster/ceph-12.2.10/work/ceph-12.2.10/src/librbd/librbd.cc:2224
#13 0x00007fffe5609325 in __pyx_pf_3rbd_3RBD_8remove (__pyx_self=<optimized out>, __pyx_v_self=<optimized out>, __pyx_v_name='2a13167bd277b2206b3dc3ba721207d8a18e86da68a98ab01267cf8dae6d8315-1549518692-to_delete', __pyx_v_ioctx=<optimized out>) at /var/tmp/portage/sys-cluster/ceph-12.2.10/work/ceph-12.2.10_build/src/pybind/rbd/pyrex/rbd.c:7683
#14 __pyx_pw_3rbd_3RBD_9remove (__pyx_self=<optimized out>, __pyx_args=<optimized out>, __pyx_kwds=<optimized out>) at /var/tmp/portage/sys-cluster/ceph-12.2.10/work/ceph-12.2.10_build/src/pybind/rbd/pyrex/rbd.c:7560
#15 0x00007ffff7a3c806 in PyObject_Call (func=func@entry=<cython_function_or_method at remote 0x7fffe58adad0>, arg=arg@entry=(<RBD at remote 0x7fffe0293e10>, <rados.Ioctx at remote 0x7fffe02f38a0>, '2a13167bd277b2206b3dc3ba721207d8a18e86da68a98ab01267cf8dae6d8315-1549518692-to_delete'), kw=kw@entry=0x0) at /var/tmp/portage/dev-lang/python-2.7.15/work/Python-2.7.15/Objects/abstract.c:2547
#16 0x00007ffff7ad6d05 in do_call (nk=0, na=<optimized out>, pp_stack=0x7ffd397416c8, func=<cython_function_or_method at remote 0x7fffe58adad0>) at /var/tmp/portage/dev-lang/python-2.7.15/work/Python-2.7.15/Python/ceval.c:4589
#17 call_function (oparg=<optimized out>, pp_stack=0x7ffd397416c8) at /var/tmp/portage/dev-lang/python-2.7.15/work/Python-2.7.15/Python/ceval.c:4394
#18 PyEval_EvalFrameEx (
    (REDACTED)
#19 0x00007ffff7ada560 in fast_function (nk=<optimized out>, na=<optimized out>, n=2, pp_stack=0x7ffd397417e8, func=<function at remote 0x7fffe483ced8>) at /var/tmp/portage/dev-lang/python-2.7.15/work/Python-2.7.15/Python/ceval.c:4457
#20 call_function (oparg=<optimized out>, pp_stack=0x7ffd397417e8) at /var/tmp/portage/dev-lang/python-2.7.15/work/Python-2.7.15/Python/ceval.c:4392
#21 PyEval_EvalFrameEx (
    (REDACTED)
#22 0x00007ffff7ada560 in fast_function (nk=<optimized out>, na=<optimized out>, n=1, pp_stack=0x7ffd39741908, func=<function at remote 0x7fffe3a6de60>) at /var/tmp/portage/dev-lang/python-2.7.15/work/Python-2.7.15/Python/ceval.c:4457
#23 call_function (oparg=<optimized out>, pp_stack=0x7ffd39741908) at /var/tmp/portage/dev-lang/python-2.7.15/work/Python-2.7.15/Python/ceval.c:4392
#24 PyEval_EvalFrameEx (
    (REDACTED)
#25 0x00007ffff7ada560 in fast_function (nk=<optimized out>, na=<optimized out>, n=1, pp_stack=0x7ffd39741a28, func=<function at remote 0x7ffff2f60230>) at /var/tmp/portage/dev-lang/python-2.7.15/work/Python-2.7.15/Python/ceval.c:4457
#26 call_function (oparg=<optimized out>, pp_stack=0x7ffd39741a28) at /var/tmp/portage/dev-lang/python-2.7.15/work/Python-2.7.15/Python/ceval.c:4392
#27 PyEval_EvalFrameEx (
    (REDACTED)
#28 0x00007ffff7adcb88 in PyEval_EvalCodeEx (co=<optimized out>, globals=<optimized out>, locals=locals@entry=0x0, args=args@entry=0x7fffe0360868, argcount=<optimized out>, kws=kws@entry=0x0, kwcount=0, defs=0x0, defcount=0, closure=0x0) at /var/tmp/portage/dev-lang/python-2.7.15/work/Python-2.7.15/Python/ceval.c:3604
#29 0x00007ffff7a62b79 in function_call (func=<function at remote 0x7ffff2f60140>,
    (REDACTED)
#30 0x00007ffff7a3c806 in PyObject_Call (func=func@entry=<function at remote 0x7ffff2f60140>,
    (REDACTED)
#31 0x00007ffff7a4bcec in instancemethod_call (func=<function at remote 0x7ffff2f60140>,
    (REDACTED)
#32 0x00007ffff7a3c806 in PyObject_Call (func=func@entry=<instancemethod at remote 0x7fffe046afa0>, arg=arg@entry=(), kw=<optimized out>) at /var/tmp/portage/dev-lang/python-2.7.15/work/Python-2.7.15/Objects/abstract.c:2547
#33 0x00007ffff7ad2647 in PyEval_CallObjectWithKeywords (func=<instancemethod at remote 0x7fffe046afa0>, arg=(), kw=<optimized out>) at /var/tmp/portage/dev-lang/python-2.7.15/work/Python-2.7.15/Python/ceval.c:4241
#34 0x00007ffff7b0cf12 in t_bootstrap (boot_raw=0x555556934e80) at /var/tmp/portage/dev-lang/python-2.7.15/work/Python-2.7.15/Modules/threadmodule.c:620
#35 0x00007ffff77d79aa in start_thread (arg=0x7ffd39742700) at pthread_create.c:463
#36 0x00007ffff75068ef in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95
(gdb) list
2261        snapid_t snapid, bufferlist *pbl, int flags,
2262        Context *onack, version_t *objver = NULL,
2263        int *data_offset = NULL,
2264        uint64_t features = 0,
2265        ZTracer::Trace *parent_trace = nullptr) {
2266        Op *o = new Op(oid, oloc, op.ops, flags | global_op_flags |
2267                       CEPH_OSD_FLAG_READ, onack, objver, data_offset, parent_trace);
2268        o->priority = op.priority;
2269        o->snapid = snapid;
2270        o->outbl = pbl;
(gdb) up
#1  0x00007ffff175509d in librados::IoCtxImpl::aio_operate_read (this=0x7ffecc007830, oid=..., o=0x7ffed00580d0, c=0x7ffecc00a710, flags=flags@entry=0, pbl=pbl@entry=0x7ffed0033098, trace_info=0x0) at /var/tmp/portage/sys-cluster/ceph-12.2.10/work/ceph-12.2.10/src/librados/IoCtxImpl.cc:789
789                      oncomplete, &c->objver, nullptr, 0, &trace);
(gdb) list
784       }
785
786       trace.event("init root span");
787       Objecter::Op *objecter_op = objecter->prepare_read_op(oid, oloc,
788                      *o, snap_seq, pbl, flags,
789                      oncomplete, &c->objver, nullptr, 0, &trace);
790       objecter->op_submit(objecter_op, &c->tid);
791       trace.event("rados operate read submitted");
792
793       return 0;