Ceph » crimson

image: quay.ceph.io/ceph-ci/ceph:6a1e2cb4b217a41f64b4b1126aff68380a594f9e-crimson

[root@rook-ceph-tools-54474cfc96-sdg5b /]# ceph -v
ceph version 16.2.6 (ee28fb57e47e9f88813e24bbf4c14496ca299d31) pacific (stable)
[root@rook-ceph-tools-54474cfc96-sdg5b /]#

Tested with the following workaround and failed to configure the OSD's.

[ubuntu@depressa012 rook]$ git diff
diff --git a/cluster/examples/kubernetes/ceph/cluster-test.yaml b/cluster/examples/kubernetes/ceph/cluster-test.yaml
index 0855f95bd..1740371ec 100644
--- a/cluster/examples/kubernetes/ceph/cluster-test.yaml
++ b/cluster/examples/kubernetes/ceph/cluster-test.yaml
@ -28,7 +28,7 @ metadata:
spec:
dataDirHostPath: /var/lib/rook
cephVersion:
- image: quay.io/ceph/ceph:v16.2.6
image: quay.ceph.io/ceph-ci/ceph:37c6bbfffc2c6c4658da49d158819e4c7cb29f89-crimson
allowUnsupported: true
mon:
count: 1
diff --git a/cluster/examples/kubernetes/ceph/images.txt b/cluster/examples/kubernetes/ceph/images.txt
index 944cb2465..978fee27f 100644
--- a/cluster/examples/kubernetes/ceph/images.txt
++ b/cluster/examples/kubernetes/ceph/images.txt
@ -6,4 +6,4 @
quay.io/ceph/ceph:v16.2.6
quay.io/cephcsi/cephcsi:v3.4.0
quay.io/csiaddons/volumereplication-operator:v0.1.0
- rook/ceph:master
travisn/ceph:crimson-oct-25
diff --git a/cluster/examples/kubernetes/ceph/operator.yaml b/cluster/examples/kubernetes/ceph/operator.yaml
index 6e2512236..1820c2863 100644
--- a/cluster/examples/kubernetes/ceph/operator.yaml
++ b/cluster/examples/kubernetes/ceph/operator.yaml
@ -354,7 +354,7 @ spec:
serviceAccountName: rook-ceph-system
containers:
- name: rook-ceph-operator
- image: rook/ceph:master
image: travisn/ceph:crimson-oct-25
args: ["ceph", "operator"]
securityContext:
runAsNonRoot: true
@ -426,6 +426,9 @ spec: # Disable automatic orchestration when new devices are discovered
- name: ROOK_DISABLE_DEVICE_HOTPLUG
value: "false"
+
+ - name: CRIMSON_OSD_PREPARE_ENV_VARS
+ value: "ASAN_OPTIONS=verify_asan_link_order=0"

1. Provide customised regex as the values using comma. For eg. regex for rbd based volume, value will be like "(?i)rbd[0-9]+".
2. In case of more than one regex, use comma to separate between them.
   [ubuntu@depressa012 rook]$

OSD error Snippet:-
[ubuntu@depressa012 ~]$ kubectl -n rook-ceph get pod
NAME READY STATUS RESTARTS AGE
csi-cephfsplugin-ngn58 3/3 Running 0 22h
csi-cephfsplugin-provisioner-6859586d88-t46sm 6/6 Running 0 22h
csi-rbdplugin-dggrl 3/3 Running 0 22h
csi-rbdplugin-provisioner-58847898f6-knwp4 6/6 Running 0 22h
rook-ceph-mgr-a-6f99bfbdc-w757f 1/1 Running 0 22h
rook-ceph-mon-a-6c8f7d77ff-npmmr 1/1 Running 0 22h
rook-ceph-operator-757546f8c7-tdlgj 0/1 CrashLoopBackOff 197 (4m31s ago) 23h
rook-ceph-osd-0-6d7b9d6b84-cpxsl 0/1 Init:CrashLoopBackOff 271 (4m8s ago) 22h
rook-ceph-osd-prepare-minikube--1-g7f8h 0/1 Completed 0 5m52s
[ubuntu@depressa012 ~]$ kubectl -n rook-ceph logs rook-ceph-osd-0-6d7b9d6b84-cpxsl
Error from server (BadRequest): container "osd" in pod "rook-ceph-osd-0-6d7b9d6b84-cpxsl" is waiting to start: PodInitializing
[ubuntu@depressa012 ~]$ kubectl -n rook-ceph logs rook-ceph-osd-0-6d7b9d6b84-cpxsl -c activate
+ OSD_ID=0
+ OSD_UUID=8b84b74c-8f79-45f1-bad3-cf27c06d0940
+ OSD_STORE_FLAG=--bluestore
+ OSD_DATA_DIR=/var/lib/ceph/osd/ceph-0
+ CV_MODE=lvm
+ DEVICE=/dev/ceph-d5630cb0-fabb-4e07-8989-51594f2e4203/osd-block-8b84b74c-8f79-45f1-bad3-cf27c06d0940
+ ceph -n client.admin auth get-or-create osd.0 mon 'allow profile osd' mgr 'allow profile osd' osd 'allow *' -k /etc/ceph/admin-keyring-store/keyring
[osd.0]
key = AQB7/+dh9kyQKRAAfrS6zJNBwD9R2pjW5RJ4cw==
+ [[ lvm == \l\v\m ]]
+ mktemp -d
TMP_DIR=/tmp/tmp.r7LdyarzIf
+ ceph-volume lvm activate --no-systemd --bluestore 0 8b84b74c-8f79-45f1-bad3-cf27c06d0940
Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0
Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/ceph-d5630cb0-fabb-4e07-8989-51594f2e4203/osd-block-8b84b74c-8f79-45f1-bad3-cf27c06d0940 --path /var/lib/ceph/osd/ceph-0 --no-mon-config
stderr: failed to read label for /dev/ceph-d5630cb0-fabb-4e07-8989-51594f2e4203/osd-block-8b84b74c-8f79-45f1-bad3-cf27c06d0940: (2) No such file or directory
Traceback (most recent call last):
File "/usr/sbin/ceph-volume", line 11, in <module>
load_entry_point('ceph-volume==1.0.0', 'console_scripts', 'ceph-volume')()
File "/usr/lib/python3.6/site-packages/ceph_volume/main.py", line 40, in init
self.main(self.argv)
File "/usr/lib/python3.6/site-packages/ceph_volume/decorators.py", line 59, in newfunc
return f(*a, **kw)
File "/usr/lib/python3.6/site-packages/ceph_volume/main.py", line 152, in main
terminal.dispatch(self.mapper, subcommand_args)
File "/usr/lib/python3.6/site-packages/ceph_volume/terminal.py", line 194, in dispatch
instance.main()
File "/usr/lib/python3.6/site-packages/ceph_volume/devices/lvm/main.py", line 46, in main
terminal.dispatch(self.mapper, self.argv)
File "/usr/lib/python3.6/site-packages/ceph_volume/terminal.py", line 194, in dispatch
instance.main()
File "/usr/lib/python3.6/site-packages/ceph_volume/devices/lvm/activate.py", line 375, in main
self.activate(args)
File "/usr/lib/python3.6/site-packages/ceph_volume/decorators.py", line 16, in is_root
return func(*a, **kw)
File "/usr/lib/python3.6/site-packages/ceph_volume/devices/lvm/activate.py", line 299, in activate
activate_bluestore(lvs, args.no_systemd)
File "/usr/lib/python3.6/site-packages/ceph_volume/devices/lvm/activate.py", line 194, in activate_bluestore
process.run(prime_command)
File "/usr/lib/python3.6/site-packages/ceph_volume/process.py", line 162, in run
raise RuntimeError(msg)
RuntimeError: command returned non-zero exit status: 1
[ubuntu@depressa012 ~]$