Bug #61227
[crimson] ceph df stats are twice of actual values
Status:
Pending Backport
Priority:
Normal
Assignee:
-
Target version:
-
% Done:
0%
Source:
Tags:
crimson backport_processed
Backport:
reef
Regression:
No
Severity:
3 - minor
Reviewed:
Affected Versions:
ceph-qa-suite:
Pull request ID:
Crash signature (v1):
Crash signature (v2):
Description
The statistics fetched using ceph df for crimson osd pool, mainly object count and space utilized are twice of the actual values and in most cases inaccurate
similar to - https://tracker.ceph.com/issues/57539
pool creation -
# ceph osd pool create test-ceph-df-2
2023-05-17T20:26:27.324-0400 7f2e35eb3700 -1 WARNING: the following dangerous and experimental features are enabled: crimson
2023-05-17T20:26:27.325-0400 7f2e35eb3700 -1 WARNING: the following dangerous and experimental features are enabled: crimson
pool 'test-ceph-df-2' created
[root@ceph-crimson-4eygjm-node5 ~]# ceph osd pool application enable test-ceph-df-2 rados
2023-05-17T20:26:47.180-0400 7f817d72b700 -1 WARNING: the following dangerous and experimental features are enabled: crimson
2023-05-17T20:26:47.180-0400 7f817d72b700 -1 WARNING: the following dangerous and experimental features are enabled: crimson
enabled application 'rados' on pool 'test-ceph-df-2'
ceph df stats prior to io
# ceph df
2023-05-17T20:26:51.968-0400 7fc4aac4e700 -1 WARNING: the following dangerous and experimental features are enabled: crimson
2023-05-17T20:26:51.969-0400 7fc4aac4e700 -1 WARNING: the following dangerous and experimental features are enabled: crimson
--- RAW STORAGE ---
CLASS SIZE AVAIL USED RAW USED %RAW USED
TOTAL 1.5 TiB 1.5 TiB 2.6 GiB 2.6 GiB 0.17
--- POOLS ---
POOL ID PGS STORED OBJECTS USED %USED MAX AVAIL
.mgr 1 1 897 KiB 4 897 KiB 0 474 GiB
cephfs.cephfs.meta 2 32 4.6 KiB 44 4.6 KiB 0 474 GiB
cephfs.cephfs.data 3 32 0 B 0 0 B 0 474 GiB
.rgw.root 4 32 2.8 KiB 12 2.8 KiB 0 474 GiB
default.rgw.log 5 32 6.6 KiB 392 6.6 KiB 0 474 GiB
default.rgw.control 6 32 0 B 16 0 B 0 474 GiB
default.rgw.meta 7 32 764 B 4 764 B 0 474 GiB
test-ceph-df 8 32 152 MiB 38 152 MiB 0.01 474 GiB
alloc-test-pool 10 32 541 MiB 138.38k 541 MiB 0.04 474 GiB
test-ceph-df-2 11 32 0 B 0 0 B 0 474 GiB
Writing 10 objects to the new pool
# for i in {0..9}; do rados put -p test-ceph-df-2 obj$i /tmp/sdata.txt; done
2023-05-17T20:27:22.707-0400 7f0b36b74040 -1 WARNING: the following dangerous and experimental features are enabled: crimson
2023-05-17T20:27:22.707-0400 7f0b36b74040 -1 WARNING: the following dangerous and experimental features are enabled: crimson
2023-05-17T20:27:22.707-0400 7f0b36b74040 -1 WARNING: the following dangerous and experimental features are enabled: crimson
2023-05-17T20:27:22.938-0400 7fa81a62f040 -1 WARNING: the following dangerous and experimental features are enabled: crimson
2023-05-17T20:27:22.939-0400 7fa81a62f040 -1 WARNING: the following dangerous and experimental features are enabled: crimson
2023-05-17T20:27:22.939-0400 7fa81a62f040 -1 WARNING: the following dangerous and experimental features are enabled: crimson
2023-05-17T20:27:23.187-0400 7f4f25abf040 -1 WARNING: the following dangerous and experimental features are enabled: crimson
2023-05-17T20:27:23.187-0400 7f4f25abf040 -1 WARNING: the following dangerous and experimental features are enabled: crimson
2023-05-17T20:27:23.188-0400 7f4f25abf040 -1 WARNING: the following dangerous and experimental features are enabled: crimson
2023-05-17T20:27:23.422-0400 7fbfb5493040 -1 WARNING: the following dangerous and experimental features are enabled: crimson
2023-05-17T20:27:23.423-0400 7fbfb5493040 -1 WARNING: the following dangerous and experimental features are enabled: crimson
2023-05-17T20:27:23.424-0400 7fbfb5493040 -1 WARNING: the following dangerous and experimental features are enabled: crimson
2023-05-17T20:27:23.653-0400 7ff33b99a040 -1 WARNING: the following dangerous and experimental features are enabled: crimson
2023-05-17T20:27:23.654-0400 7ff33b99a040 -1 WARNING: the following dangerous and experimental features are enabled: crimson
2023-05-17T20:27:23.654-0400 7ff33b99a040 -1 WARNING: the following dangerous and experimental features are enabled: crimson
2023-05-17T20:27:23.876-0400 7f6eb7094040 -1 WARNING: the following dangerous and experimental features are enabled: crimson
2023-05-17T20:27:23.877-0400 7f6eb7094040 -1 WARNING: the following dangerous and experimental features are enabled: crimson
2023-05-17T20:27:23.877-0400 7f6eb7094040 -1 WARNING: the following dangerous and experimental features are enabled: crimson
2023-05-17T20:27:24.077-0400 7fea66cbc040 -1 WARNING: the following dangerous and experimental features are enabled: crimson
2023-05-17T20:27:24.078-0400 7fea66cbc040 -1 WARNING: the following dangerous and experimental features are enabled: crimson
2023-05-17T20:27:24.078-0400 7fea66cbc040 -1 WARNING: the following dangerous and experimental features are enabled: crimson
2023-05-17T20:27:24.311-0400 7f5fdfb9c040 -1 WARNING: the following dangerous and experimental features are enabled: crimson
2023-05-17T20:27:24.312-0400 7f5fdfb9c040 -1 WARNING: the following dangerous and experimental features are enabled: crimson
2023-05-17T20:27:24.312-0400 7f5fdfb9c040 -1 WARNING: the following dangerous and experimental features are enabled: crimson
2023-05-17T20:27:24.553-0400 7f4c14a68040 -1 WARNING: the following dangerous and experimental features are enabled: crimson
2023-05-17T20:27:24.553-0400 7f4c14a68040 -1 WARNING: the following dangerous and experimental features are enabled: crimson
2023-05-17T20:27:24.553-0400 7f4c14a68040 -1 WARNING: the following dangerous and experimental features are enabled: crimson
2023-05-17T20:27:24.745-0400 7ff7e843f040 -1 WARNING: the following dangerous and experimental features are enabled: crimson
2023-05-17T20:27:24.746-0400 7ff7e843f040 -1 WARNING: the following dangerous and experimental features are enabled: crimson
2023-05-17T20:27:24.746-0400 7ff7e843f040 -1 WARNING: the following dangerous and experimental features are enabled: crimson
ceph df output post write operation
# ceph df detail
2023-05-17T20:27:30.702-0400 7fcb6cb0f700 -1 WARNING: the following dangerous and experimental features are enabled: crimson
2023-05-17T20:27:30.702-0400 7fcb6cb0f700 -1 WARNING: the following dangerous and experimental features are enabled: crimson
--- RAW STORAGE ---
CLASS SIZE AVAIL USED RAW USED %RAW USED
TOTAL 1.5 TiB 1.5 TiB 2.7 GiB 2.7 GiB 0.18
--- POOLS ---
POOL ID PGS STORED (DATA) (OMAP) OBJECTS USED (DATA) (OMAP) %USED MAX AVAIL QUOTA OBJECTS QUOTA BYTES DIRTY USED COMPR UNDER COMPR
.mgr 1 1 897 KiB 897 KiB 0 B 4 897 KiB 897 KiB 0 B 0 474 GiB N/A N/A N/A 0 B 0 B
cephfs.cephfs.meta 2 32 4.6 KiB 4.6 KiB 0 B 44 4.6 KiB 4.6 KiB 0 B 0 474 GiB N/A N/A N/A 0 B 0 B
cephfs.cephfs.data 3 32 0 B 0 B 0 B 0 0 B 0 B 0 B 0 474 GiB N/A N/A N/A 0 B 0 B
.rgw.root 4 32 2.8 KiB 2.8 KiB 0 B 12 2.8 KiB 2.8 KiB 0 B 0 474 GiB N/A N/A N/A 0 B 0 B
default.rgw.log 5 32 6.6 KiB 6.6 KiB 0 B 392 6.6 KiB 6.6 KiB 0 B 0 474 GiB N/A N/A N/A 0 B 0 B
default.rgw.control 6 32 0 B 0 B 0 B 16 0 B 0 B 0 B 0 474 GiB N/A N/A N/A 0 B 0 B
default.rgw.meta 7 32 764 B 764 B 0 B 4 764 B 764 B 0 B 0 474 GiB N/A N/A N/A 0 B 0 B
test-ceph-df 8 32 152 MiB 152 MiB 0 B 38 152 MiB 152 MiB 0 B 0.01 474 GiB N/A N/A N/A 0 B 0 B
alloc-test-pool 10 32 541 MiB 541 MiB 0 B 138.38k 541 MiB 541 MiB 0 B 0.04 474 GiB N/A N/A N/A 0 B 0 B
test-ceph-df-2 11 32 80 MiB 80 MiB 0 B 20 80 MiB 80 MiB 0 B 0 474 GiB N/A N/A N/A 0 B 0 B
[root@ceph-crimson-4eygjm-node5 ~]# ceph df detail
2023-05-17T20:27:49.847-0400 7f13ffd7b700 -1 WARNING: the following dangerous and experimental features are enabled: crimson
2023-05-17T20:27:49.848-0400 7f13ffd7b700 -1 WARNING: the following dangerous and experimental features are enabled: crimson
--- RAW STORAGE ---
CLASS SIZE AVAIL USED RAW USED %RAW USED
TOTAL 1.5 TiB 1.5 TiB 2.7 GiB 2.7 GiB 0.18
--- POOLS ---
POOL ID PGS STORED (DATA) (OMAP) OBJECTS USED (DATA) (OMAP) %USED MAX AVAIL QUOTA OBJECTS QUOTA BYTES DIRTY USED COMPR UNDER COMPR
.mgr 1 1 897 KiB 897 KiB 0 B 4 897 KiB 897 KiB 0 B 0 474 GiB N/A N/A N/A 0 B 0 B
cephfs.cephfs.meta 2 32 4.6 KiB 4.6 KiB 0 B 44 4.6 KiB 4.6 KiB 0 B 0 474 GiB N/A N/A N/A 0 B 0 B
cephfs.cephfs.data 3 32 0 B 0 B 0 B 0 0 B 0 B 0 B 0 474 GiB N/A N/A N/A 0 B 0 B
.rgw.root 4 32 2.8 KiB 2.8 KiB 0 B 12 2.8 KiB 2.8 KiB 0 B 0 474 GiB N/A N/A N/A 0 B 0 B
default.rgw.log 5 32 6.6 KiB 6.6 KiB 0 B 392 6.6 KiB 6.6 KiB 0 B 0 474 GiB N/A N/A N/A 0 B 0 B
default.rgw.control 6 32 0 B 0 B 0 B 16 0 B 0 B 0 B 0 474 GiB N/A N/A N/A 0 B 0 B
default.rgw.meta 7 32 764 B 764 B 0 B 4 764 B 764 B 0 B 0 474 GiB N/A N/A N/A 0 B 0 B
test-ceph-df 8 32 152 MiB 152 MiB 0 B 38 152 MiB 152 MiB 0 B 0.01 474 GiB N/A N/A N/A 0 B 0 B
alloc-test-pool 10 32 541 MiB 541 MiB 0 B 138.38k 541 MiB 541 MiB 0 B 0.04 474 GiB N/A N/A N/A 0 B 0 B
test-ceph-df-2 11 32 80 MiB 80 MiB 0 B 20 80 MiB 80 MiB 0 B 0 474 GiB N/A N/A N/A 0 B 0 B
List of actual objects in the pool
# rados ls -p test-ceph-df-2
2023-05-17T20:28:02.188-0400 7fb96d71b040 -1 WARNING: the following dangerous and experimental features are enabled: crimson
2023-05-17T20:28:02.189-0400 7fb96d71b040 -1 WARNING: the following dangerous and experimental features are enabled: crimson
2023-05-17T20:28:02.189-0400 7fb96d71b040 -1 WARNING: the following dangerous and experimental features are enabled: crimson
obj4
obj2
obj6
obj0
obj3
obj7
obj8
obj5
obj9
obj1
ceph version -
# ceph versions
2023-05-17T23:09:55.937-0400 7f10d60fb700 -1 WARNING: the following dangerous and experimental features are enabled: crimson
2023-05-17T23:09:55.937-0400 7f10d60fb700 -1 WARNING: the following dangerous and experimental features are enabled: crimson
{
"mon": {
"ceph version 18.0.0-3971-gcd5bf7d9 (cd5bf7d94251de4667f79591d5832e648ab7ccaa) reef (dev)": 3
},
"rgw": {
"ceph version 18.0.0-3971-gcd5bf7d9 (cd5bf7d94251de4667f79591d5832e648ab7ccaa) reef (dev)": 2
},
"overall": {
"ceph version 18.0.0-3971-gcd5bf7d9 (cd5bf7d94251de4667f79591d5832e648ab7ccaa) reef (dev)": 5
}
}
ceph config dump -
# ceph config dump
2023-05-17T23:12:29.659-0400 7eff0ca5f700 -1 WARNING: the following dangerous and experimental features are enabled: crimson
2023-05-17T23:12:29.660-0400 7eff0ca5f700 -1 WARNING: the following dangerous and experimental features are enabled: crimson
WHO MASK LEVEL OPTION VALUE RO
global basic container_image quay.ceph.io/ceph-ci/ceph@sha256:5aa150c0acce870509798df79a285140e566d46f4fe2212af8d108c42c43a558 *
global advanced enable_experimental_unrecoverable_data_corrupting_features crimson
global advanced osd_pool_default_pg_autoscale_mode off
mon advanced auth_allow_insecure_global_id_reclaim false
mon advanced cluster_network 10.1.240.0/24 *
mon advanced mon_allow_pool_delete true
mon advanced osd_pool_default_crimson true
mon advanced public_network 10.1.240.0/23 *
mgr advanced mgr/cephadm/container_init True *
mgr advanced mgr/cephadm/migration_current 6 *
mgr advanced mgr/dashboard/ALERTMANAGER_API_HOST http://dell-r640-056.dsal.lab.eng.rdu2.redhat.com:9093 *
mgr advanced mgr/dashboard/GRAFANA_API_SSL_VERIFY false *
mgr advanced mgr/dashboard/GRAFANA_API_URL https://dell-r640-056.dsal.lab.eng.rdu2.redhat.com:3000 *
mgr advanced mgr/dashboard/PROMETHEUS_API_HOST http://dell-r640-056.dsal.lab.eng.rdu2.redhat.com:9095 *
mgr advanced mgr/dashboard/RGW_API_ACCESS_KEY PMT1KMAO6CJU7D58OWTJ *
mgr advanced mgr/dashboard/RGW_API_SECRET_KEY CO85kJENCK9O1PBYrkFzHcE41BrGS3yR4OiEEAPn *
mgr advanced mgr/dashboard/ssl_server_port 8443 *
mgr advanced mgr/orchestrator/orchestrator cephadm
osd host:dell-r640-083 basic osd_memory_target 27723883560
osd host:dell-r640-084 basic osd_memory_target 27723882987
osd host:dell-r640-087 basic osd_memory_target 26864889528
osd advanced osd_memory_target_autotune true
mds.cephfs basic mds_join_fs cephfs
client.rgw.rgw.1.dell-r640-079.oykhmo basic rgw_frontends beast port=80 *
client.rgw.rgw.1.dell-r640-085.lsnrnt basic rgw_frontends beast port=80 *
Crimson image used - https://shaman.ceph.com/builds/ceph/main/cd5bf7d94251de4667f79591d5832e648ab7ccaa/crimson/343381/
History
#1 Updated by Radoslaw Zarzynski 10 months ago
- Status changed from New to Fix Under Review
- Backport set to reef
- Pull request ID set to 51746
#2 Updated by Matan Breizman 10 months ago
- Status changed from Fix Under Review to Pending Backport
#3 Updated by Yuri Weinstein 10 months ago
#4 Updated by Ernesto Puerta 5 months ago
- Tags changed from crimson to crimson backport_processed