Ceph » rbd

when ceph client run test code ,creat image for rbd_open return -2 ;
but i run rbd cmd，it is ok ; it call librbd.cc::rbd_create4 directly;
[root@tom4 home]# rbd create rbd1 --size 1024
[root@tom4 home]#
[root@tom4 home]# rbd info rbd
rbd: error opening image rbd: (2) No such file or directory
[root@tom4 home]# rbd info rbd1
rbd image 'rbd1':
size 1 GiB in 256 objects
order 22 (4 MiB objects)
snapshot_count: 0
id: 1078827d650b
block_name_prefix: rbd_data.1078827d650b
format: 2
features: layering, exclusive-lock, object-map, fast-diff, deep-flatten
op_features:
flags:
create_timestamp: Thu May 20 22:00:38 2021
access_timestamp: Thu May 20 22:00:38 2021
modify_timestamp: Thu May 20 22:00:38 2021

ceph client version:ceph 14.2.21(debuginfo) and 16.2.3
ceph cluster:ceph 16.2.3 and python ceph(one mon/one osd)
ceph osd pool create rbd 64 (add rbd pool for ceph client)
so install ceph-14.2.21-debuginfo to fix root issue ;find C_SaferCond ctx in imagestate::open,return -2(nomal >=0);
from git , not find big change about it! it is one bug?
can you give me some suggestions?thanks

librbd:ImageState.cc
template <typename I>
int ImageState<I>::open(uint64_t flags) {
C_SaferCond ctx;
open(flags, &ctx);

• int r = ctx.wait();
  if (r < 0) {
  delete m_image_ctx;
  }*
  return r;
  }

template <typename I>
void ImageState<I>::open(uint64_t flags, Context *on_finish) {
CephContext *cct = m_image_ctx->cct;
ldout(cct, 20) << func << dendl;

m_lock.Lock();
  ceph_assert(m_state == STATE_UNINITIALIZED);
  m_open_flags = flags;

Action action(ACTION_TYPE_OPEN);
  action.refresh_seq = m_refresh_seq;

execute_action_unlock(action, on_finish);
}

gdb log:
Breakpoint 1, rbd_open () at /usr/src/debug/ceph-14.2.21/src/librbd/librbd.cc:3965
3965 {
Missing separate debuginfos, use: debuginfo-install bzip2-libs-1.0.6-13.el7.x86_64 elfutils-libelf-0.176-4.el7.x86_64 elfutils-libs-0.176-4.el7.x86_64 glibc-2.17-324.el7_9.x86_64 libaio-0.3.109-13.el7.x86_64 libattr-2.4.46-13.el7.x86_64 libblkid-2.23.2-65.el7_9.1.x86_64 libcap-2.22-11.el7.x86_64 libgcc-4.8.5-44.el7.x86_64 libibverbs-22.4-6.el7_9.x86_64 libnl3-3.2.28-4.el7.x86_64 librdmacm-22.4-6.el7_9.x86_64 libstdc++-4.8.5-44.el7.x86_64 libuuid-2.23.2-65.el7_9.1.x86_64 lttng-ust-2.10.0-1.el7.x86_64 nspr-4.21.0-1.el7.x86_64 nss-3.44.0-7.el7_7.x86_64 nss-softokn-3.44.0-8.el7_7.x86_64 nss-softokn-freebl-3.44.0-8.el7_7.x86_64 nss-util-3.44.0-4.el7_7.x86_64 numactl-libs-2.0.12-5.el7.x86_64 openssl-libs-1.0.2k-21.el7_9.x86_64 sqlite-3.7.17-8.el7_7.1.x86_64 systemd-libs-219-73.el7.1.x86_64 userspace-rcu-0.10.0-3.el7.x86_64 xz-libs-5.2.2-1.el7.x86_64 zlib-1.2.7-19.el7_9.x86_64
(gdb) s
3966 librados::IoCtx io_ctx;
(gdb) n
3967 librados::IoCtx::from_rados_ioctx_t(p, io_ctx);
(gdb) n
3968 TracepointProvider::initialize<tracepoint_traits>(get_cct(io_ctx));
(gdb) n
3969 librbd::ImageCtx ictx = new librbd::ImageCtx(name, "", snap_name, io_ctx,
(gdb) n
[New Thread 0x7fffcf7fe700 (LWP 41014)]
[New Thread 0x7fffceffd700 (LWP 41015)]
3970 false);
(gdb) n
3971 tracepoint(librbd, open_image_enter, ictx, ictx->name.c_str(), ictx->id.c_str(), ictx->snap_name.c_str(), ictx->read_only);
(gdb) n
*3973 int r = ictx->state->open(0);
(gdb) s
librbd::ImageState<librbd::ImageCtx>::open (this=0x10e7240, flags=flags@entry=0) at /usr/src/debug/ceph-14.2.21/src/librbd/ImageState.cc:247
247 int ImageState<I>::open(uint64_t flags) {
(gdb) p flags
$1 = 0
(gdb) n
248 C_SaferCond ctx;
(gdb) p ctx
$2 = {<Context> = {_vptr.Context = 0x7ffff6bfa320 <vtable for C_SaferCond+16>}, lock = {name = "C_SaferCond", id = 1, recursive = false, lockdep = true,
backtrace = false, m = {_data = {__lock = 0, _count = 0, __owner = 0, __nusers = 0, __kind = 2, __spins = 0, __elision = 0, __list = {_prev = 0x0, _next = 0x0}},
__size = '\000' <repeats 16 times>, "\002", '\000' <repeats 22 times>, __align = 0}, nlock = 0, locked_by = 0}, cond = {_vptr.Cond = 0xf36c18, _c = {_data = {
_lock = 28, __futex = 0, __total_seq = 15094144, __wakeup_seq = 140737323747168, __woken_seq = 11560, __mutex = 0x7fffffffdfd8, __nwaiters = 17725424,
__broadcast_seq = 0},
__size = "\034\000\000\000\000\000\000\000\200Q\346\000\000\000\000\000`G0\366\377\177\000\000(\000\000\000\000\000\000\330\337\377\377\377\177\000\000\360w\016\001\000\000\000", __align = 28}, waiter_mutex = 0x10e8038}, done = 216, rval = 0}
(gdb) n
251 int r = ctx.wait();
(gdb) n
248 C_SaferCond ctx;
(gdb) p ctx
$3 = {<Context> = {_vptr.Context = 0x7ffff6bfa320 <vtable for C_SaferCond+16>}, lock = {name = "C_SaferCond", id = -1, recursive = false, lockdep = true,
backtrace = false, _m = {_data = {__lock = 0, _count = 0, __owner = 0, __nusers = 0, __kind = 2, __spins = 0, __elision = 0, __list = {_prev = 0x0, _next = 0x0}},
__size = '\000' <repeats 16 times>, "\002", '\000' <repeats 22 times>, __align = 0}, nlock = 0, locked_by = 0}, cond = {
_vptr.Cond = 0x7ffff6bef9a0 <vtable for Cond+16>, _c = {_data = {__lock = 0, __futex = 2, __total_seq = 1, __wakeup_seq = 1, __woken_seq = 1,
__mutex = 0x7fffffffdee8, __nwaiters = 0, __broadcast_seq = 1},
__size = "\000\000\000\000\002\000\000\000\001\000\000\000\000\000\000\000\001\000\000\000\000\000\000\000\001\000\000\000\000\000\000\000\350\336\377\377\377\177\000\000\000\000\000\000\001\000\000", __align = 8589934592}, waiter_mutex = 0x7fffffffded8}, done = true,*rval = -2*}
(gdb) q

gcc -g test.c -o rbdtest -lrados -lrbd
test.c code below:
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <rados/librados.h>
#include <rbd/librbd.h>

rbd_image_t init_image(rados_ioctx_t io_ctx) {
int ret = 0;

// 5. open rbd image
    rbd_image_t image;
    const char *image_name = "ee0c15655d44599a82a6bc0848b4aaf8";
    ret = rbd_open(io_ctx, image_name, &image, NULL);
    if (ret < 0) {
        printf("couldn't open rbd image! err %d\n", ret);
        return NULL;
    } else {
        printf("opened an rbd image: sotest\n");
    }

return image;
}

int get_rbd_size(rbd_image_t image) {
int ret = 0;
uint64_t tsize = 0, size = 0;

// 6. get rbd image size
    ret = rbd_get_size(image, &size);
    if (ret < 0) {
        printf("couldn't get image size! err %d\n", ret);
        return EXIT_FAILURE;
    } else {
        printf("The size of the image is: %dMB\n", size/1024/1024);
    }

tsize = size;
    return tsize;
}

void rbd_finish_aiocb(rbd_completion_t c, void *arg) {
// int ret = rbd_aio_wait_for_complete(c);
int ret = rbd_aio_get_return_value(c);
rbd_aio_release(c);

// for aio read callback, the read data should be copied here to caller
    printf("aio callback: %d, %s\n", ret, (const char*)arg);
}

ssize_t io_write(rbd_image_t image, const char *buff) {
int off = 128;
int len = strlen(buff);
ssize_t ret;
uint64_t ofs;
ret = rbd_write(image,ofs, (size_t) len,buff);

if (ret < 0) {
        printf("write to image failed %s\n", ret);
        return ret;
    }
    printf("write %s to image end\n", buff);

return ret;
}

int aio_write(rbd_image_t image, const char *buff) {
int off = 128;
rbd_completion_t c;
int ret = rbd_aio_create_completion((void *)buff, (rbd_callback_t) rbd_finish_aiocb, &c);
if (ret < 0) {
printf("create callback failed %s\n", ret);
return ret;
}

int len = strlen(buff);
    ret = rbd_aio_write(image, off, len, buff, c);
    if (ret < 0) {
        printf("write to image failed %s\n", ret);
        return ret;
    }
    printf("write %s to image end\n", buff);

return ret;
}

ssize_t io_read(rbd_image_t image, char *buff) {
int off = 128;
int len = 10;
ssize_t ret;
ret = rbd_read(image, off, len, buff);
if (ret < 0) {
printf("read from image failed %s\n", ret);
return ret;
}
printf("read from image end\n");

return ret;
}

int aio_read(rbd_image_t image, char *buff) {
int off = 128;
int len = 10;
rbd_completion_t c;
int ret = rbd_aio_create_completion(buff, (rbd_callback_t) rbd_finish_aiocb, &c);
if (ret < 0) {
printf("create callback failed %s\n", ret);
return ret;
}
memset(buff, 0, 128);
ret = rbd_aio_read(image, off, len, buff, c);
if (ret < 0) {
printf("read from image failed %s\n", ret);
return ret;
}
printf("read from image end\n");

return ret;
}

int main (int argc, const char **argv) {
int ret;
char buff¹²⁸ = {0};
uint64_t size = 0;

/* Declare the cluster handle and required arguments. */
        rados_t cluster;
        char cluster_name[] = "ceph";
        char user_name[] = "client.admin";
        uint64_t flags = 0;

/* Initialize the cluster handle with the "ceph" cluster name and the "client.admin" user */
        int err;
        err = rados_create2(&cluster, cluster_name, user_name, flags);

if (err < 0) {
                fprintf(stderr, "%s: Couldn't create the cluster handle! %s\n", argv[0], strerror(-err));
                exit(EXIT_FAILURE);
        } else {
                printf("\nCreated a cluster handle.\n");
        }

/* Read a Ceph configuration file to configure the cluster handle. */
        err = rados_conf_read_file(cluster, "/etc/ceph/ceph.conf");
        if (err < 0) {
                fprintf(stderr, "%s: cannot read config file: %s\n", argv[0], strerror(-err));
                exit(EXIT_FAILURE);
        } else {
                printf("\nRead the config file.\n");
        }

/* Read command line arguments */
        err = rados_conf_parse_argv(cluster, argc, argv);
        if (err < 0) {
                fprintf(stderr, "%s: cannot parse command line arguments: %s\n", argv[0], strerror(-err));
                exit(EXIT_FAILURE);
        } else {
                printf("\nRead the command line arguments.\n");
        }

/* Connect to the cluster */
        err = rados_connect(cluster);
        if (err < 0) {
                fprintf(stderr, "%s: cannot connect to cluster: %s\n", argv[0], strerror(-err));
                exit(EXIT_FAILURE);
        } else {
                printf("\nConnected to the cluster.\n");
        }

rados_ioctx_t io_ctx;
        char *poolname = "rbd";

err = rados_ioctx_create(cluster, poolname, &io_ctx);
        if (err < 0) {
                fprintf(stderr, "%s: cannot open rados pool %s: %s\n", argv[0], poolname, strerror(-err));
                rados_shutdown(cluster);
                exit(EXIT_FAILURE);
        } else {
                printf("\nCreated I/O context.\n");
        }

rbd_image_t image = init_image(io_ctx);
    if (!image) {
        perror("init_image");
        rados_ioctx_destroy(io_ctx);
        rados_shutdown(cluster);
        return EXIT_FAILURE;
    }

size = get_rbd_size(image);
    printf("image size: %d\n", size);

sprintf(buff, "%s", "abcd123efg");
    //aio_write(image, buff);

aio_read(image, buff);

// 7. close image, io context and rados object
    ret = rbd_close(image);
    if (ret < 0) {
        printf("couldn't close rbd image! err %d\n", ret);
        return EXIT_FAILURE;
    } else {
        printf("closed rbd image: sotest\n");
    }
    rados_ioctx_destroy(io_ctx);
    rados_shutdown(cluster);

return 0;

}