Ceph » Orchestrator

Having this spec:

service_type: osd
service_id: hybrid
service_name: osd.hybrid
placement:
  host_pattern: host1
spec:
  data_devices:
    rotational: 1
  db_devices:
    rotational: 0
  filter_logic: AND
  objectstore: bluestore

And having the system partition being locked:

# ceph orch device ls host1
HOST   PATH          TYPE  DEVICE ID                                       SIZE  AVAILABLE  REJECT REASONS           
host1  /dev/nvme0n1  ssd   NVMENVMENVMENVMENVMENVMENVMENVME1              1600G  Yes                                 
host1  /dev/nvme1n1  ssd   NVMENVMENVMENVMENVMENVMENVMENVME4              1600G  Yes                                 
host1  /dev/sda      hdd   IDSDM_012345678901                             64.2G             Has GPT headers, locked  
host1  /dev/sdb      hdd   AAAAAAAAAAA_0000000000000b0f1                  16.0T  Yes                                 
host1  /dev/sdc      hdd   AAAAAAAAAAA_000000000000089cd                  16.0T  Yes                                 
host1  /dev/sdd      hdd   AAAAAAAAAAA_0000000000000af6d                  16.0T  Yes                                 
host1  /dev/sde      hdd   AAAAAAAAAAA_00000000000008a4d                  16.0T  Yes                                 
host1  /dev/sdf      hdd   AAAAAAAAAAA_0000000000000af9d                  16.0T  Yes                                 
host1  /dev/sdg      hdd   BBBBBBBBBBBb_00000000000044a5                  16.0T  Yes                                 
host1  /dev/sdh      hdd   BBBBBBBBBBBb_000000000000f7f9                  16.0T  Yes                                 
host1  /dev/sdi      hdd   AAAAAAAAAAA_000000000000089a1                  16.0T  Yes                                 
host1  /dev/sdj      hdd   AAAAAAAAAAA_00000000000008601                  16.0T  Yes                                 
host1  /dev/sdk      hdd   AAAAAAAAAAA_00000000000008a71                  16.0T  Yes                                 
host1  /dev/sdl      hdd   CCCCCCCCCCCC_000000000000ebdd                  16.0T  Yes                                 
host1  /dev/sdm      hdd   AAAAAAAAAAA_000000000000089bd                  16.0T  Yes                                 
host1  /dev/sdn      hdd   AAAAAAAAAAA_0000000000000fd31                  16.0T  Yes                                 
host1  /dev/sdo      hdd   AAAAAAAAAAA_0000000000000f9a9                  16.0T  Yes                                 
host1  /dev/sdp      hdd   AAAAAAAAAAA_00000000000008565                  16.0T  Yes                                 
host1  /dev/sdq      hdd   BBBBBBBBBBBb_000000000000f3e5                  16.0T  Yes                                 
host1  /dev/sdr      hdd   MG08SCA16TEY_5000039aa858002d                  16.0T  Yes                                 
host1  /dev/sds      hdd   AAAAAAAAAAA_0000000000000fa61                  16.0T  Yes                                 
host1  /dev/sdt      hdd   BBBBBBBBBBBb_00000000000046a5                  16.0T  Yes                                 
host1  /dev/sdu      hdd   BBBBBBBBBBBb_00000000000041a1                  16.0T  Yes                                 
host1  /dev/sdv      hdd   BBBBBBBBBBBb_000000000000f46d                  16.0T  Yes                                 
host1  /dev/sdw      hdd   BBBBBBBBBBBb_00000000000046a9                  16.0T  Yes                                 
host1  /dev/sdx      hdd   CCCCCCCCCCCC_000000000000eec1                  16.0T  Yes                                 
host1  /dev/sdy      hdd   BBBBBBBBBBBb_0000000000004509                  16.0T  Yes  

# mount -l | grep sda
/dev/sda2 on / type ext4 (rw,relatime) [root]
/dev/sda1 on /boot/efi type vfat (rw,relatime,fmask=0022,dmask=0022,codepage=437,iocharset=iso8859-1,shortname=mixed,errors=remount-ro) [boot-efi]

It fails to apply it:

# ceph orch ls --format yaml --service-type osd | python3 -c 'import sys, yaml, json; y=yaml.safe_load_all(sys.stdin.read()); print(json.dumps(list(y)))' | jq -r .[0].events[0] 
2021-11-18T14:09:26.296374Z service:osd.hybrid [ERROR] "Failed to apply: cephadm exited with an error code: 1, stderr:Non-zero exit code 2 from /usr/bin/docker run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint /usr/sbin/ceph-volume --privileged --group-add=disk --init -e CONTAINER_IMAGE=quay.ceph.io/ceph-ci/ceph@sha256:sha1 -e NODE_NAME=host1 -e CEPH_USE_RANDOM_NONCE=1 -e CEPH_VOLUME_OSDSPEC_AFFINITY=hybrid -v /var/run/ceph/fsid:/var/run/ceph:z -v /var/log/ceph/fsid:/var/log/ceph:z -v /var/lib/ceph/fsid/crash:/var/lib/ceph/crash:z -v /dev:/dev -v /run/udev:/run/udev -v /sys:/sys -v /run/lvm:/run/lvm -v /run/lock/lvm:/run/lock/lvm -v /:/rootfs -v /tmp/ceph-tmp1gohyav7:/etc/ceph/ceph.conf:z -v /tmp/ceph-tmpblgd2lz7:/var/lib/ceph/bootstrap-osd/ceph.keyring:z quay.ceph.io/ceph-ci/ceph@sha256:sha1 lvm batch --no-auto /dev/sda /dev/sdb /dev/sdc /dev/sdd /dev/sde /dev/sdf /dev/sdg /dev/sdh /dev/sdi /dev/sdj /dev/sdk /dev/sdl /dev/sdm /dev/sdn /dev/sdo /dev/sdp /dev/sdq /dev/sdr /dev/sds /dev/sdt /dev/sdu /dev/sdv /dev/sdw /dev/sdx /dev/sdy --db-devices /dev/nvme0n1 /dev/nvme1n1 --yes --no-systemd
/usr/bin/docker: stderr usage: ceph-volume lvm batch [-h] [--db-devices [DB_DEVICES [DB_DEVICES ...]]]
/usr/bin/docker: stderr                              [--wal-devices [WAL_DEVICES [WAL_DEVICES ...]]]
/usr/bin/docker: stderr                              [--journal-devices [JOURNAL_DEVICES [JOURNAL_DEVICES ...]]]
/usr/bin/docker: stderr                              [--auto] [--no-auto] [--bluestore] [--filestore]
/usr/bin/docker: stderr                              [--report] [--yes]
/usr/bin/docker: stderr                              [--format {json,json-pretty,pretty}] [--dmcrypt]
/usr/bin/docker: stderr                              [--crush-device-class CRUSH_DEVICE_CLASS]
/usr/bin/docker: stderr                              [--no-systemd]
/usr/bin/docker: stderr                              [--osds-per-device OSDS_PER_DEVICE]
/usr/bin/docker: stderr                              [--data-slots DATA_SLOTS]
/usr/bin/docker: stderr                              [--data-allocate-fraction DATA_ALLOCATE_FRACTION]
/usr/bin/docker: stderr                              [--block-db-size BLOCK_DB_SIZE]
/usr/bin/docker: stderr                              [--block-db-slots BLOCK_DB_SLOTS]
/usr/bin/docker: stderr                              [--block-wal-size BLOCK_WAL_SIZE]
/usr/bin/docker: stderr                              [--block-wal-slots BLOCK_WAL_SLOTS]
/usr/bin/docker: stderr                              [--journal-size JOURNAL_SIZE]
/usr/bin/docker: stderr                              [--journal-slots JOURNAL_SLOTS] [--prepare]
/usr/bin/docker: stderr                              [--osd-ids [OSD_IDS [OSD_IDS ...]]]
/usr/bin/docker: stderr                              [DEVICES [DEVICES ...]]
/usr/bin/docker: stderr ceph-volume lvm batch: error: GPT headers found, they must be removed on: /dev/sda
Traceback (most recent call last):
  File "/var/lib/ceph/fsid/cephadm.hash", line 8331, in <module>
    main()
  File "/var/lib/ceph/fsid/cephadm.hash", line 8319, in main
    r = ctx.func(ctx)
  File "/var/lib/ceph/fsid/cephadm.hash", line 1735, in _infer_config
    return func(ctx)
  File "/var/lib/ceph/fsid/cephadm.hash", line 1676, in _infer_fsid
    return func(ctx)
  File "/var/lib/ceph/fsid/cephadm.hash", line 1763, in _infer_image
    return func(ctx)
  File "/var/lib/ceph/fsid/cephadm.hash", line 1663, in _validate_fsid
    return func(ctx)
  File "/var/lib/ceph/fsid/cephadm.hash", line 5285, in command_ceph_volume
    out, err, code = call_throws(ctx, c.run_cmd())
  File "/var/lib/ceph/fsid/cephadm.hash", line 1465, in call_throws
    raise RuntimeError('Failed command: %s' % ' '.join(command))
RuntimeError: Failed command: /usr/bin/docker run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint /usr/sbin/ceph-volume --privileged --group-add=disk --init -e CONTAINER_IMAGE=quay.ceph.io/ceph-ci/ceph@sha256:sha1 -e NODE_NAME=host1 -e CEPH_USE_RANDOM_NONCE=1 -e CEPH_VOLUME_OSDSPEC_AFFINITY=hybrid -v /var/run/ceph/fsid:/var/run/ceph:z -v /var/log/ceph/fsid:/var/log/ceph:z -v /var/lib/ceph/fsid/crash:/var/lib/ceph/crash:z -v /dev:/dev -v /run/udev:/run/udev -v /sys:/sys -v /run/lvm:/run/lvm -v /run/lock/lvm:/run/lock/lvm -v /:/rootfs -v /tmp/ceph-tmp1gohyav7:/etc/ceph/ceph.conf:z -v /tmp/ceph-tmpblgd2lz7:/var/lib/ceph/bootstrap-osd/ceph.keyring:z quay.ceph.io/ceph-ci/ceph@sha256:sha1 lvm batch --no-auto /dev/sda /dev/sdb /dev/sdc /dev/sdd /dev/sde /dev/sdf /dev/sdg /dev/sdh /dev/sdi /dev/sdj /dev/sdk /dev/sdl /dev/sdm /dev/sdn /dev/sdo /dev/sdp /dev/sdq /dev/sdr /dev/sds /dev/sdt /dev/sdu /dev/sdv /dev/sdw /dev/sdx /dev/sdy --db-devices /dev/nvme0n1 /dev/nvme1n1 --yes --no-systemd"