Ceph » bluestore

Wondering if you can try a patch from https://tracker.ceph.com/issues/46027 and check whether it's helpful in your case...

I will put this on test environment. Will see

Unfortunately, That's not the case. After 4 days some of the osds took >10GB of ram.
In example:

{
    "mempool": {
        "by_pool": {
            "bloom_filter": {
                "items": 0,
                "bytes": 0
            },
            "bluestore_alloc": {
                "items": 618237,
                "bytes": 49458960
            },
            "bluestore_cache_data": {
                "items": 2706603,
                "bytes": 256537796
            },
            "bluestore_cache_onode": {
                "items": 2707283,
                "bytes": 1775977648
            },
            "bluestore_cache_meta": {
                "items": 288856905,
                "bytes": 1867155824
            },
            "bluestore_cache_other": {
                "items": 17828824,
                "bytes": 302498676
            },
            "bluestore_Buffer": {
                "items": 286,
                "bytes": 38896
            },
            "bluestore_Extent": {
                "items": 17758438,
                "bytes": 852405024
            },
            "bluestore_Blob": {
                "items": 17758439,
                "bytes": 1846877656
            },
            "bluestore_SharedBlob": {
                "items": 17758439,
                "bytes": 1988945168
            },
            "bluestore_inline_bl": {
                "items": 75,
                "bytes": 9822
            },
            "bluestore_fsck": {
                "items": 0,
                "bytes": 0
            },
            "bluestore_txc": {
                "items": 32,
                "bytes": 23296
            },
            "bluestore_writing_deferred": {
                "items": 475,
                "bytes": 2434417
            },
            "bluestore_writing": {
                "items": 0,
                "bytes": 0
            },
            "bluefs": {
                "items": 957,
                "bytes": 27936
            },
            "bluefs_file_reader": {
                "items": 100,
                "bytes": 749944
            },
            "bluefs_file_writer": {
                "items": 4,
                "bytes": 1216
            },
            "buffer_anon": {
                "items": 1202,
                "bytes": 4023826
            },
            "buffer_meta": {
                "items": 1159,
                "bytes": 101992
            },
            "osd": {
                "items": 172,
                "bytes": 2238752
            },
            "osd_mapbl": {
                "items": 0,
                "bytes": 0
            },
            "osd_pglog": {
                "items": 873105,
                "bytes": 229538616
            },
            "osdmap": {
                "items": 20191,
                "bytes": 1143144
            },
            "osdmap_mapping": {
                "items": 0,
                "bytes": 0
            },
            "pgmap": {
                "items": 0,
                "bytes": 0
            },
            "mds_co": {
                "items": 0,
                "bytes": 0
            },
            "unittest_1": {
                "items": 0,
                "bytes": 0
            },
            "unittest_2": {
                "items": 0,
                "bytes": 0
            }
        },
        "total": {
            "items": 366890926,
            "bytes": 9180188609
        }
    }
}

ceph.conf

[global]
bluestore_rocksdb_options = "compression=kSnappyCompression,max_write_buffer_number=16,min_write_buffer_number_to_merge=3,recycle_log_file_num=16,compaction_style=kCompactionStyleLevel,write_buffer_size=67108864,target_file_size_base=67108864,max_background_compactions=31,level0_file_num_compaction_trigger=8,level0_slowdown_writes_trigger=32,level0_stop_writes_trigger=64,num_levels=5,max_bytes_for_level_base=1610612736,max_bytes_for_level_multiplier=10" 
cluster network = 10.0.20.0/24
err_to_syslog = true
fsid = 874ec41f-e6eb-4c0b-be4e-e82be057195b
log_to_syslog = true
mon host = [v2:10.0.20.11:3300,v1:10.0.20.11:6789],[v2:10.0.20.12:3300,v1:10.0.20.12:6789],[v2:10.0.20.14:3300,v1:10.0.20.14:6789]
mon initial members = ceph1-hdd,ceph2-hdd,ceph4-hdd
osd pool default crush rule = -1
osd_command_thread_suicide_timeout = 7200
osd_command_thread_timeout = 4800
osd_crush_update_on_start = true
osd_max_pg_per_osd_hard_ratio = 32
osd_max_scrubs = 10
osd_memory_target = 3221225472
osd_op_thread_suicide_timeout = 600
osd_op_thread_timeout = 300
osd_pool_default_ec_fast_read = true
osd_recovery_max_active = 1
public network = 10.0.20.0/24
[osd]
osd memory target = 3221225472

PS. I see that i duplicated osd_memory_target, that's not the problem :)

Presuming mem utilization is still that high could you please temporary set debug_bluestore to 20 for the osd in question and leave it running (without any load if possible) for 1 min.

@Igor Gajowiak

here you are:
https://cf2.cloudferro.com:8080/swift/v1/AUTH_5b9ea421deb745bfb4dab930cebe153f/ceph-sharings/ceph_17_debug_leak.txt

Without load - only normal cluster ops.

https://github.com/ceph/ceph/pull/39720 merged