--- begin dump of recent events --- -486> 2021-12-24T09:52:03.812+0000 7f55e65c8700 5 asok(0x5651d788c000) register_command assert hook 0x5651d77d85f0 -485> 2021-12-24T09:52:03.812+0000 7f55e65c8700 5 asok(0x5651d788c000) register_command abort hook 0x5651d77d85f0 -484> 2021-12-24T09:52:03.812+0000 7f55e65c8700 5 asok(0x5651d788c000) register_command leak_some_memory hook 0x5651d77d85f0 -483> 2021-12-24T09:52:03.812+0000 7f55e65c8700 5 asok(0x5651d788c000) register_command perfcounters_dump hook 0x5651d77d85f0 -482> 2021-12-24T09:52:03.812+0000 7f55e65c8700 5 asok(0x5651d788c000) register_command 1 hook 0x5651d77d85f0 -481> 2021-12-24T09:52:03.812+0000 7f55e65c8700 5 asok(0x5651d788c000) register_command perf dump hook 0x5651d77d85f0 -480> 2021-12-24T09:52:03.812+0000 7f55e65c8700 5 asok(0x5651d788c000) register_command perfcounters_schema hook 0x5651d77d85f0 -479> 2021-12-24T09:52:03.812+0000 7f55e65c8700 5 asok(0x5651d788c000) register_command perf histogram dump hook 0x5651d77d85f0 -478> 2021-12-24T09:52:03.812+0000 7f55e65c8700 5 asok(0x5651d788c000) register_command 2 hook 0x5651d77d85f0 -477> 2021-12-24T09:52:03.812+0000 7f55e65c8700 5 asok(0x5651d788c000) register_command perf schema hook 0x5651d77d85f0 -476> 2021-12-24T09:52:03.812+0000 7f55e65c8700 5 asok(0x5651d788c000) register_command perf histogram schema hook 0x5651d77d85f0 -475> 2021-12-24T09:52:03.812+0000 7f55e65c8700 5 asok(0x5651d788c000) register_command perf reset hook 0x5651d77d85f0 -474> 2021-12-24T09:52:03.812+0000 7f55e65c8700 5 asok(0x5651d788c000) register_command config show hook 0x5651d77d85f0 -473> 2021-12-24T09:52:03.812+0000 7f55e65c8700 5 asok(0x5651d788c000) register_command config help hook 0x5651d77d85f0 -472> 2021-12-24T09:52:03.812+0000 7f55e65c8700 5 asok(0x5651d788c000) register_command config set hook 0x5651d77d85f0 -471> 2021-12-24T09:52:03.812+0000 7f55e65c8700 5 asok(0x5651d788c000) register_command config unset hook 0x5651d77d85f0 -470> 2021-12-24T09:52:03.812+0000 7f55e65c8700 5 asok(0x5651d788c000) register_command config get hook 0x5651d77d85f0 -469> 2021-12-24T09:52:03.812+0000 7f55e65c8700 5 asok(0x5651d788c000) register_command config diff hook 0x5651d77d85f0 -468> 2021-12-24T09:52:03.812+0000 7f55e65c8700 5 asok(0x5651d788c000) register_command config diff get hook 0x5651d77d85f0 -467> 2021-12-24T09:52:03.812+0000 7f55e65c8700 5 asok(0x5651d788c000) register_command injectargs hook 0x5651d77d85f0 -466> 2021-12-24T09:52:03.812+0000 7f55e65c8700 5 asok(0x5651d788c000) register_command log flush hook 0x5651d77d85f0 -465> 2021-12-24T09:52:03.812+0000 7f55e65c8700 5 asok(0x5651d788c000) register_command log dump hook 0x5651d77d85f0 -464> 2021-12-24T09:52:03.812+0000 7f55e65c8700 5 asok(0x5651d788c000) register_command log reopen hook 0x5651d77d85f0 -463> 2021-12-24T09:52:03.812+0000 7f55e65c8700 5 asok(0x5651d788c000) register_command dump_mempools hook 0x5651d787a328 -462> 2021-12-24T09:52:03.820+0000 7f55e65c8700 0 set uid:gid to 167:167 (ceph:ceph) -461> 2021-12-24T09:52:03.820+0000 7f55e65c8700 0 ceph version 16.2.7 (dd0603118f56ab514f133c8d2e3adfc983942503) pacific (stable), process ceph-mon, pid 7 -460> 2021-12-24T09:52:03.820+0000 7f55e65c8700 0 pidfile_write: ignore empty --pid-file -459> 2021-12-24T09:52:03.820+0000 7f55e65c8700 5 asok(0x5651d788c000) init /var/run/ceph/ceph-mon.osd003.asok -458> 2021-12-24T09:52:03.820+0000 7f55e65c8700 5 asok(0x5651d788c000) bind_and_listen /var/run/ceph/ceph-mon.osd003.asok -457> 2021-12-24T09:52:03.820+0000 7f55e65c8700 5 asok(0x5651d788c000) register_command 0 hook 0x5651d77ddca0 -456> 2021-12-24T09:52:03.820+0000 7f55e65c8700 5 asok(0x5651d788c000) register_command version hook 0x5651d77ddca0 -455> 2021-12-24T09:52:03.820+0000 7f55e65c8700 5 asok(0x5651d788c000) register_command git_version hook 0x5651d77ddca0 -454> 2021-12-24T09:52:03.820+0000 7f55e65c8700 5 asok(0x5651d788c000) register_command help hook 0x5651d77d8650 -453> 2021-12-24T09:52:03.820+0000 7f55e65c8700 5 asok(0x5651d788c000) register_command get_command_descriptions hook 0x5651d77d8660 -452> 2021-12-24T09:52:03.820+0000 7f55d7691700 5 asok(0x5651d788c000) entry start -451> 2021-12-24T09:52:03.824+0000 7f55e65c8700 0 load: jerasure load: lrc load: isa -450> 2021-12-24T09:52:03.824+0000 7f55e65c8700 1 set rocksdb option level_compaction_dynamic_level_bytes = true -449> 2021-12-24T09:52:03.824+0000 7f55e65c8700 1 set rocksdb option write_buffer_size = 33554432 -448> 2021-12-24T09:52:03.824+0000 7f55e65c8700 1 set rocksdb option compression = kNoCompression -447> 2021-12-24T09:52:03.824+0000 7f55e65c8700 1 set rocksdb option level_compaction_dynamic_level_bytes = true -446> 2021-12-24T09:52:03.824+0000 7f55e65c8700 1 set rocksdb option write_buffer_size = 33554432 -445> 2021-12-24T09:52:03.824+0000 7f55e65c8700 1 set rocksdb option compression = kNoCompression -444> 2021-12-24T09:52:03.828+0000 7f55e65c8700 5 rocksdb: verify_sharding column families from rocksdb: [default] -443> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: RocksDB version: 6.8.1 -442> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ -441> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Compile date Dec 7 2021 -440> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: DB SUMMARY -439> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: CURRENT file: CURRENT -438> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: IDENTITY file: IDENTITY -437> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: MANIFEST file: MANIFEST-102171 size: 102772 Bytes -436> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-osd003/store.db dir, Total Num: 2, files: 103467.sst 103468.sst -435> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-osd003/store.db: 103465.log size: 3893138 ; -434> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.error_if_exists: 0 -433> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.create_if_missing: 0 -432> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.paranoid_checks: 1 -431> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.env: 0x5651d527b3a0 -430> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.fs: Posix File System -429> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.info_log: 0x5651d78c7b20 -428> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.max_file_opening_threads: 16 -427> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.statistics: (nil) -426> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.use_fsync: 0 -425> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.max_log_file_size: 0 -424> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.max_manifest_file_size: 1073741824 -423> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.log_file_time_to_roll: 0 -422> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.keep_log_file_num: 1000 -421> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.recycle_log_file_num: 0 -420> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.allow_fallocate: 1 -419> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.allow_mmap_reads: 0 -418> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.allow_mmap_writes: 0 -417> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.use_direct_reads: 0 -416> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 -415> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.create_missing_column_families: 0 -414> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.db_log_dir: -413> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-osd003/store.db -412> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.table_cache_numshardbits: 6 -411> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.max_subcompactions: 1 -410> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.max_background_flushes: -1 -409> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.WAL_ttl_seconds: 0 -408> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.WAL_size_limit_MB: 0 -407> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 -406> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.manifest_preallocation_size: 4194304 -405> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.is_fd_close_on_exec: 1 -404> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.advise_random_on_open: 1 -403> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.db_write_buffer_size: 0 -402> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.write_buffer_manager: 0x5651d853c090 -401> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.access_hint_on_compaction_start: 1 -400> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 -399> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.random_access_max_buffer_size: 1048576 -398> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.use_adaptive_mutex: 0 -397> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.rate_limiter: (nil) -396> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 -395> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.wal_recovery_mode: 2 -394> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.enable_thread_tracking: 0 -393> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.enable_pipelined_write: 0 -392> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.unordered_write: 0 -391> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.allow_concurrent_memtable_write: 1 -390> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 -389> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.write_thread_max_yield_usec: 100 -388> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.write_thread_slow_yield_usec: 3 -387> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.row_cache: None -386> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.wal_filter: None -385> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.avoid_flush_during_recovery: 0 -384> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.allow_ingest_behind: 0 -383> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.preserve_deletes: 0 -382> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.two_write_queues: 0 -381> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.manual_wal_flush: 0 -380> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.atomic_flush: 0 -379> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 -378> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.persist_stats_to_disk: 0 -377> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.write_dbid_to_manifest: 0 -376> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.log_readahead_size: 0 -375> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.sst_file_checksum_func: Unknown -374> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.max_background_jobs: 2 -373> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.max_background_compactions: -1 -372> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.avoid_flush_during_shutdown: 0 -371> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 -370> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.delayed_write_rate : 16777216 -369> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.max_total_wal_size: 0 -368> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 -367> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.stats_dump_period_sec: 600 -366> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.stats_persist_period_sec: 600 -365> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.stats_history_buffer_size: 1048576 -364> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.max_open_files: -1 -363> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.bytes_per_sync: 0 -362> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.wal_bytes_per_sync: 0 -361> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.strict_bytes_per_sync: 0 -360> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.compaction_readahead_size: 0 -359> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Compression algorithms supported: -358> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: kZSTDNotFinalCompression supported: 0 -357> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: kZSTD supported: 0 -356> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: kXpressCompression supported: 0 -355> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: kLZ4HCCompression supported: 1 -354> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: kLZ4Compression supported: 1 -353> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: kBZip2Compression supported: 0 -352> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: kZlibCompression supported: 1 -351> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: kSnappyCompression supported: 1 -350> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Fast CRC32 supported: Supported on x86 -349> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: [version_set.cc:4413] Recovering from manifest file: /var/lib/ceph/mon/ceph-osd003/store.db/MANIFEST-102171 -348> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: [column_family.cc:552] --------------- Options for column family [default]: -347> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.comparator: leveldb.BytewiseComparator -346> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.merge_operator: -345> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.compaction_filter: None -344> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.compaction_filter_factory: None -343> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.memtable_factory: SkipListFactory -342> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.table_factory: BlockBasedTable -341> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5651d77ddc90) cache_index_and_filter_blocks: 1 cache_index_and_filter_blocks_with_high_priority: 0 pin_l0_filter_and_index_blocks_in_cache: 0 pin_top_level_index_and_filter: 1 index_type: 0 data_block_index_type: 0 index_shortening: 1 data_block_hash_table_util_ratio: 0.750000 hash_index_allow_collision: 1 checksum: 1 no_block_cache: 0 block_cache: 0x5651d7815010 block_cache_name: BinnedLRUCache block_cache_options: capacity : 536870912 num_shard_bits : 4 strict_capacity_limit : 0 high_pri_pool_ratio: 0.000 block_cache_compressed: (nil) persistent_cache: (nil) block_size: 4096 block_size_deviation: 10 block_restart_interval: 16 index_block_restart_interval: 1 metadata_block_size: 4096 partition_filters: 0 use_delta_encoding: 1 filter_policy: rocksdb.BuiltinBloomFilter whole_key_filtering: 1 verify_compression: 0 read_amp_bytes_per_bit: 0 format_version: 2 enable_index_compression: 1 block_align: 0 -340> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.write_buffer_size: 33554432 -339> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.max_write_buffer_number: 2 -338> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.compression: NoCompression -337> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.bottommost_compression: Disabled -336> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.prefix_extractor: nullptr -335> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr -334> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.num_levels: 7 -333> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 -332> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 -331> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 -330> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 -329> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.bottommost_compression_opts.level: 32767 -328> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 -327> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 -326> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 -325> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.bottommost_compression_opts.enabled: false -324> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.compression_opts.window_bits: -14 -323> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.compression_opts.level: 32767 -322> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.compression_opts.strategy: 0 -321> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 -320> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 -319> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.compression_opts.enabled: false -318> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 -317> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 -316> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.level0_stop_writes_trigger: 36 -315> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.target_file_size_base: 67108864 -314> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.target_file_size_multiplier: 1 -313> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.max_bytes_for_level_base: 268435456 -312> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 -311> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 -310> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 -309> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 -308> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 -307> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 -306> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 -305> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 -304> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 -303> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 -302> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.max_compaction_bytes: 1677721600 -301> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.arena_block_size: 4194304 -300> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 -299> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 -298> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 -297> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.disable_auto_compactions: 0 -296> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.compaction_style: kCompactionStyleLevel -295> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio -294> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 -293> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 -292> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 -291> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 -290> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 -289> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize -288> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 -287> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 -286> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.table_properties_collectors: -285> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.inplace_update_support: 0 -284> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.inplace_update_num_locks: 10000 -283> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 -282> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.memtable_whole_key_filtering: 0 -281> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.memtable_huge_page_size: 0 -280> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.bloom_locality: 0 -279> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.max_successive_merges: 0 -278> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.optimize_filters_for_hits: 0 -277> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.paranoid_file_checks: 0 -276> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.force_consistency_checks: 0 -275> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.report_bg_io_stats: 0 -274> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.ttl: 2592000 -273> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: Options.periodic_compaction_seconds: 0 -272> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: [version_set.cc:4568] Recovered from manifest file:/var/lib/ceph/mon/ceph-osd003/store.db/MANIFEST-102171 succeeded,manifest_file_number is 102171, next_file_number is 103470, last_sequence is 34567354, log_number is 103465,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 -271> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: [version_set.cc:4577] Column family [default] (ID 0), log number is 103465 -270> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1640339523834530, "job": 1, "event": "recovery_started", "log_files": [103465]} -269> 2021-12-24T09:52:03.828+0000 7f55e65c8700 4 rocksdb: [db_impl/db_impl_open.cc:760] Recovering log #103465 mode 2 -268> 2021-12-24T09:52:03.832+0000 7f55e65c8700 3 rocksdb: [le/block_based/filter_policy.cc:584] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. -267> 2021-12-24T09:52:03.840+0000 7f55e65c8700 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1640339523846687, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 103470, "file_size": 2590220, "table_properties": {"data_size": 2586978, "index_size": 2100, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 0, "index_value_is_delta_encoded": 0, "filter_size": 325, "raw_key_size": 2083, "raw_average_key_size": 22, "raw_value_size": 2583969, "raw_average_value_size": 28086, "num_data_blocks": 60, "num_entries": 92, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1640339523, "oldest_key_time": 3, "file_creation_time": 0}} -266> 2021-12-24T09:52:03.840+0000 7f55e65c8700 4 rocksdb: [version_set.cc:3826] Creating manifest 103471 -265> 2021-12-24T09:52:03.840+0000 7f55e65c8700 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1640339523847792, "job": 1, "event": "recovery_finished"} -264> 2021-12-24T09:52:03.844+0000 7f55e65c8700 4 rocksdb: DB pointer 0x5651d78dd800 -263> 2021-12-24T09:52:03.844+0000 7f55cd600700 4 rocksdb: [db_impl/db_impl.cc:850] ------- DUMPING STATS ------- -262> 2021-12-24T09:52:03.844+0000 7f55cd600700 4 rocksdb: [db_impl/db_impl.cc:851] ** DB Stats ** Uptime(secs): 0.0 total, 0.0 interval Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s Interval stall: 00:00:0.000 H:M:S, 0.0 percent ** Compaction Stats [default] ** Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- L0 1/0 2.47 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 253.7 0.01 0.00 1 0.010 0 0 L6 2/0 108.93 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 Sum 3/0 111.40 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 253.7 0.01 0.00 1 0.010 0 0 Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 253.7 0.01 0.00 1 0.010 0 0 ** Compaction Stats [default] ** Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 253.7 0.01 0.00 1 0.010 0 0 Uptime(secs): 0.0 total, 0.0 interval Flush(GB): cumulative 0.002, interval 0.002 AddFile(GB): cumulative 0.000, interval 0.000 AddFile(Total Files): cumulative 0, interval 0 AddFile(L0 Files): cumulative 0, interval 0 AddFile(Keys): cumulative 0, interval 0 Cumulative compaction: 0.00 GB write, 146.20 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds Interval compaction: 0.00 GB write, 146.20 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count ** File Read Latency Histogram By Level [default] ** ** Compaction Stats [default] ** Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- L0 1/0 2.47 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 253.7 0.01 0.00 1 0.010 0 0 L6 2/0 108.93 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 Sum 3/0 111.40 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 253.7 0.01 0.00 1 0.010 0 0 Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 ** Compaction Stats [default] ** Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 253.7 0.01 0.00 1 0.010 0 0 Uptime(secs): 0.0 total, 0.0 interval Flush(GB): cumulative 0.002, interval 0.000 AddFile(GB): cumulative 0.000, interval 0.000 AddFile(Total Files): cumulative 0, interval 0 AddFile(L0 Files): cumulative 0, interval 0 AddFile(Keys): cumulative 0, interval 0 Cumulative compaction: 0.00 GB write, 145.88 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count ** File Read Latency Histogram By Level [default] ** -261> 2021-12-24T09:52:03.844+0000 7f55e65c8700 5 AuthRegistry(0x5651d8586a40) adding auth protocol: cephx -260> 2021-12-24T09:52:03.844+0000 7f55e65c8700 5 AuthRegistry(0x5651d8586a40) adding auth protocol: cephx -259> 2021-12-24T09:52:03.844+0000 7f55e65c8700 5 AuthRegistry(0x5651d8586a40) adding auth protocol: cephx -258> 2021-12-24T09:52:03.844+0000 7f55e65c8700 5 AuthRegistry(0x5651d8586a40) adding auth protocol: none -257> 2021-12-24T09:52:03.844+0000 7f55e65c8700 5 AuthRegistry(0x5651d8586a40) adding con mode: secure -256> 2021-12-24T09:52:03.844+0000 7f55e65c8700 5 AuthRegistry(0x5651d8586a40) adding con mode: crc -255> 2021-12-24T09:52:03.844+0000 7f55e65c8700 5 AuthRegistry(0x5651d8586a40) adding con mode: secure -254> 2021-12-24T09:52:03.844+0000 7f55e65c8700 5 AuthRegistry(0x5651d8586a40) adding con mode: crc -253> 2021-12-24T09:52:03.844+0000 7f55e65c8700 5 AuthRegistry(0x5651d8586a40) adding con mode: secure -252> 2021-12-24T09:52:03.844+0000 7f55e65c8700 5 AuthRegistry(0x5651d8586a40) adding con mode: crc -251> 2021-12-24T09:52:03.844+0000 7f55e65c8700 5 AuthRegistry(0x5651d8586a40) adding con mode: crc -250> 2021-12-24T09:52:03.844+0000 7f55e65c8700 5 AuthRegistry(0x5651d8586a40) adding con mode: secure -249> 2021-12-24T09:52:03.844+0000 7f55e65c8700 5 AuthRegistry(0x5651d8586a40) adding con mode: crc -248> 2021-12-24T09:52:03.844+0000 7f55e65c8700 5 AuthRegistry(0x5651d8586a40) adding con mode: secure -247> 2021-12-24T09:52:03.844+0000 7f55e65c8700 5 AuthRegistry(0x5651d8586a40) adding con mode: crc -246> 2021-12-24T09:52:03.844+0000 7f55e65c8700 5 AuthRegistry(0x5651d8586a40) adding con mode: secure -245> 2021-12-24T09:52:03.844+0000 7f55e65c8700 2 auth: KeyRing::load: loaded key file /var/lib/ceph/mon/ceph-osd003/keyring -244> 2021-12-24T09:52:03.844+0000 7f55e65c8700 0 starting mon.osd003 rank 1 at public addrs [v2:10.254.251.3:3300/0,v1:10.254.251.3:6789/0] at bind addrs [v2:10.254.251.3:3300/0,v1:10.254.251.3:6789/0] mon_data /var/lib/ceph/mon/ceph-osd003 fsid f89b88ea-4dad-11ec-a673-4d72a9026ccc -243> 2021-12-24T09:52:03.844+0000 7f55e65c8700 5 AuthRegistry(0x5651d8587340) adding auth protocol: cephx -242> 2021-12-24T09:52:03.844+0000 7f55e65c8700 5 AuthRegistry(0x5651d8587340) adding auth protocol: cephx -241> 2021-12-24T09:52:03.844+0000 7f55e65c8700 5 AuthRegistry(0x5651d8587340) adding auth protocol: cephx -240> 2021-12-24T09:52:03.844+0000 7f55e65c8700 5 AuthRegistry(0x5651d8587340) adding auth protocol: none -239> 2021-12-24T09:52:03.844+0000 7f55e65c8700 5 AuthRegistry(0x5651d8587340) adding con mode: secure -238> 2021-12-24T09:52:03.844+0000 7f55e65c8700 5 AuthRegistry(0x5651d8587340) adding con mode: crc -237> 2021-12-24T09:52:03.844+0000 7f55e65c8700 5 AuthRegistry(0x5651d8587340) adding con mode: secure -236> 2021-12-24T09:52:03.844+0000 7f55e65c8700 5 AuthRegistry(0x5651d8587340) adding con mode: crc -235> 2021-12-24T09:52:03.844+0000 7f55e65c8700 5 AuthRegistry(0x5651d8587340) adding con mode: secure -234> 2021-12-24T09:52:03.844+0000 7f55e65c8700 5 AuthRegistry(0x5651d8587340) adding con mode: crc -233> 2021-12-24T09:52:03.844+0000 7f55e65c8700 5 AuthRegistry(0x5651d8587340) adding con mode: crc -232> 2021-12-24T09:52:03.844+0000 7f55e65c8700 5 AuthRegistry(0x5651d8587340) adding con mode: secure -231> 2021-12-24T09:52:03.844+0000 7f55e65c8700 5 AuthRegistry(0x5651d8587340) adding con mode: crc -230> 2021-12-24T09:52:03.844+0000 7f55e65c8700 5 AuthRegistry(0x5651d8587340) adding con mode: secure -229> 2021-12-24T09:52:03.844+0000 7f55e65c8700 5 AuthRegistry(0x5651d8587340) adding con mode: crc -228> 2021-12-24T09:52:03.844+0000 7f55e65c8700 5 AuthRegistry(0x5651d8587340) adding con mode: secure -227> 2021-12-24T09:52:03.844+0000 7f55e65c8700 2 auth: KeyRing::load: loaded key file /var/lib/ceph/mon/ceph-osd003/keyring -226> 2021-12-24T09:52:03.844+0000 7f55e65c8700 5 adding auth protocol: cephx -225> 2021-12-24T09:52:03.844+0000 7f55e65c8700 5 adding auth protocol: cephx -224> 2021-12-24T09:52:03.844+0000 7f55e65c8700 10 log_channel(cluster) update_config to_monitors: true to_syslog: false syslog_facility: daemon prio: info to_graylog: false graylog_host: 127.0.0.1 graylog_port: 12201) -223> 2021-12-24T09:52:03.844+0000 7f55e65c8700 10 log_channel(audit) update_config to_monitors: true to_syslog: false syslog_facility: local0 prio: info to_graylog: false graylog_host: 127.0.0.1 graylog_port: 12201) -222> 2021-12-24T09:52:03.844+0000 7f55e65c8700 1 mon.osd003@-1(???) e9 preinit fsid f89b88ea-4dad-11ec-a673-4d72a9026ccc -221> 2021-12-24T09:52:03.844+0000 7f55e65c8700 5 mon.osd003@-1(???).mds e0 Unable to load 'last_metadata' -220> 2021-12-24T09:52:03.844+0000 7f55e65c8700 0 mon.osd003@-1(???).mds e1 new map -219> 2021-12-24T09:52:03.844+0000 7f55e65c8700 0 mon.osd003@-1(???).mds e1 print_map e1 enable_multiple, ever_enabled_multiple: 1,1 default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} legacy client fscid: -1 No filesystems configured -218> 2021-12-24T09:52:03.844+0000 7f55e65c8700 0 mon.osd003@-1(???).osd e53265 crush map has features 3314933000854323200, adjusting msgr requires -217> 2021-12-24T09:52:03.844+0000 7f55e65c8700 0 mon.osd003@-1(???).osd e53265 crush map has features 432629239337189376, adjusting msgr requires -216> 2021-12-24T09:52:03.844+0000 7f55e65c8700 0 mon.osd003@-1(???).osd e53265 crush map has features 432629239337189376, adjusting msgr requires -215> 2021-12-24T09:52:03.844+0000 7f55e65c8700 0 mon.osd003@-1(???).osd e53265 crush map has features 432629239337189376, adjusting msgr requires -214> 2021-12-24T09:52:03.848+0000 7f55e65c8700 1 mon.osd003@-1(???).paxosservice(auth 1504..1643) refresh upgraded, format 0 -> 3 -213> 2021-12-24T09:52:03.848+0000 7f55e65c8700 4 mon.osd003@-1(???).mgr e0 loading version 91 -212> 2021-12-24T09:52:03.848+0000 7f55e65c8700 4 mon.osd003@-1(???).mgr e91 active server: [v2:10.254.251.1:6928/2921383766,v1:10.254.251.1:6929/2921383766](3564211) -211> 2021-12-24T09:52:03.848+0000 7f55e65c8700 4 mon.osd003@-1(???).mgr e91 mkfs or daemon transitioned to available, loading commands -210> 2021-12-24T09:52:03.848+0000 7f55e65c8700 4 set_mon_vals no callback set -209> 2021-12-24T09:52:03.848+0000 7f55e65c8700 10 set_mon_vals auth_allow_insecure_global_id_reclaim = false -208> 2021-12-24T09:52:03.848+0000 7f55e65c8700 4 set_mon_vals failed to set cluster_network = 10.254.252.0/22: Configuration option 'cluster_network' may not be modified at runtime -207> 2021-12-24T09:52:03.848+0000 7f55e65c8700 10 set_mon_vals container_image = quay.io/ceph/ceph@sha256:bb6a71f7f481985f6d3b358e3b9ef64c6755b3db5aa53198e0aac38be5c8ae54 -206> 2021-12-24T09:52:03.848+0000 7f55e65c8700 10 set_mon_vals mon_allow_pool_delete = true -205> 2021-12-24T09:52:03.848+0000 7f55e65c8700 10 set_mon_vals mon_clock_drift_allowed = 0.100000 -204> 2021-12-24T09:52:03.848+0000 7f55e65c8700 10 set_mon_vals mon_pg_warn_max_object_skew = 0.000000 -203> 2021-12-24T09:52:03.848+0000 7f55e65c8700 4 set_mon_vals no callback set -202> 2021-12-24T09:52:03.848+0000 7f55e65c8700 4 set_mon_vals failed to set cluster_network = 10.254.252.0/22: Configuration option 'cluster_network' may not be modified at runtime -201> 2021-12-24T09:52:03.848+0000 7f55e65c8700 2 auth: KeyRing::load: loaded key file /var/lib/ceph/mon/ceph-osd003/keyring -200> 2021-12-24T09:52:03.848+0000 7f55e65c8700 5 asok(0x5651d788c000) register_command compact hook 0x5651d857c5f0 -199> 2021-12-24T09:52:03.848+0000 7f55e65c8700 5 asok(0x5651d788c000) register_command smart hook 0x5651d857c5f0 -198> 2021-12-24T09:52:03.848+0000 7f55e65c8700 5 asok(0x5651d788c000) register_command mon_status hook 0x5651d857c5f0 -197> 2021-12-24T09:52:03.848+0000 7f55e65c8700 5 asok(0x5651d788c000) register_command heap hook 0x5651d857c5f0 -196> 2021-12-24T09:52:03.848+0000 7f55e65c8700 5 asok(0x5651d788c000) register_command connection scores dump hook 0x5651d857c5f0 -195> 2021-12-24T09:52:03.848+0000 7f55e65c8700 5 asok(0x5651d788c000) register_command connection scores reset hook 0x5651d857c5f0 -194> 2021-12-24T09:52:03.848+0000 7f55e65c8700 5 asok(0x5651d788c000) register_command sync_force hook 0x5651d857c5f0 -193> 2021-12-24T09:52:03.848+0000 7f55e65c8700 5 asok(0x5651d788c000) register_command add_bootstrap_peer_hint hook 0x5651d857c5f0 -192> 2021-12-24T09:52:03.848+0000 7f55e65c8700 5 asok(0x5651d788c000) register_command add_bootstrap_peer_hintv hook 0x5651d857c5f0 -191> 2021-12-24T09:52:03.848+0000 7f55e65c8700 5 asok(0x5651d788c000) register_command quorum enter hook 0x5651d857c5f0 -190> 2021-12-24T09:52:03.848+0000 7f55e65c8700 5 asok(0x5651d788c000) register_command quorum exit hook 0x5651d857c5f0 -189> 2021-12-24T09:52:03.848+0000 7f55e65c8700 5 asok(0x5651d788c000) register_command ops hook 0x5651d857c5f0 -188> 2021-12-24T09:52:03.848+0000 7f55e65c8700 5 asok(0x5651d788c000) register_command sessions hook 0x5651d857c5f0 -187> 2021-12-24T09:52:03.848+0000 7f55e65c8700 5 asok(0x5651d788c000) register_command dump_historic_ops hook 0x5651d857c5f0 -186> 2021-12-24T09:52:03.848+0000 7f55e65c8700 5 AuthRegistry(0x5651d8a33420) adding auth protocol: cephx -185> 2021-12-24T09:52:03.848+0000 7f55e65c8700 5 AuthRegistry(0x5651d8a33420) adding auth protocol: cephx -184> 2021-12-24T09:52:03.848+0000 7f55e65c8700 5 AuthRegistry(0x5651d8a33420) adding auth protocol: cephx -183> 2021-12-24T09:52:03.848+0000 7f55e65c8700 5 AuthRegistry(0x5651d8a33420) adding auth protocol: none -182> 2021-12-24T09:52:03.848+0000 7f55e65c8700 5 AuthRegistry(0x5651d8a33420) adding con mode: secure -181> 2021-12-24T09:52:03.848+0000 7f55e65c8700 5 AuthRegistry(0x5651d8a33420) adding con mode: crc -180> 2021-12-24T09:52:03.848+0000 7f55e65c8700 5 AuthRegistry(0x5651d8a33420) adding con mode: secure -179> 2021-12-24T09:52:03.848+0000 7f55e65c8700 5 AuthRegistry(0x5651d8a33420) adding con mode: crc -178> 2021-12-24T09:52:03.848+0000 7f55e65c8700 5 AuthRegistry(0x5651d8a33420) adding con mode: secure -177> 2021-12-24T09:52:03.848+0000 7f55e65c8700 5 AuthRegistry(0x5651d8a33420) adding con mode: crc -176> 2021-12-24T09:52:03.848+0000 7f55e65c8700 5 AuthRegistry(0x5651d8a33420) adding con mode: crc -175> 2021-12-24T09:52:03.848+0000 7f55e65c8700 5 AuthRegistry(0x5651d8a33420) adding con mode: secure -174> 2021-12-24T09:52:03.848+0000 7f55e65c8700 5 AuthRegistry(0x5651d8a33420) adding con mode: crc -173> 2021-12-24T09:52:03.848+0000 7f55e65c8700 5 AuthRegistry(0x5651d8a33420) adding con mode: secure -172> 2021-12-24T09:52:03.848+0000 7f55e65c8700 5 AuthRegistry(0x5651d8a33420) adding con mode: crc -171> 2021-12-24T09:52:03.848+0000 7f55e65c8700 5 AuthRegistry(0x5651d8a33420) adding con mode: secure -170> 2021-12-24T09:52:03.848+0000 7f55e65c8700 2 auth: KeyRing::load: loaded key file /var/lib/ceph/mon/ceph-osd003/keyring -169> 2021-12-24T09:52:03.848+0000 7f55e65c8700 2 mon.osd003@-1(???) e9 init -168> 2021-12-24T09:52:03.848+0000 7f55e65c8700 4 mgrc handle_mgr_map Got map version 91 -167> 2021-12-24T09:52:03.848+0000 7f55e65c8700 4 mgrc handle_mgr_map Active mgr is now [v2:10.254.251.1:6928/2921383766,v1:10.254.251.1:6929/2921383766] -166> 2021-12-24T09:52:03.848+0000 7f55e65c8700 4 mgrc reconnect Starting new session with [v2:10.254.251.1:6928/2921383766,v1:10.254.251.1:6929/2921383766] -165> 2021-12-24T09:52:03.852+0000 7f55e65c8700 0 mon.osd003@-1(probing) e9 my rank is now 1 (was -1) -164> 2021-12-24T09:52:03.852+0000 7f55cc5fe700 4 mgrc handle_mgr_configure stats_period=5 -163> 2021-12-24T09:52:03.852+0000 7f55cc5fe700 4 mgrc handle_mgr_configure updated stats threshold: 5 -162> 2021-12-24T09:52:03.852+0000 7f55d1e09700 1 heartbeat_map reset_timeout 'Monitor::cpu_tp thread 0x7f55d1e09700' had timed out after 0.000000000s -161> 2021-12-24T09:52:03.852+0000 7f55d0e07700 1 heartbeat_map reset_timeout 'Monitor::cpu_tp thread 0x7f55d0e07700' had timed out after 0.000000000s -160> 2021-12-24T09:52:03.852+0000 7f55d1608700 1 heartbeat_map reset_timeout 'Monitor::cpu_tp thread 0x7f55d1608700' had timed out after 0.000000000s -159> 2021-12-24T09:52:04.456+0000 7f55cfe05700 5 mon.osd003@1(probing) e9 _ms_dispatch setting monitor caps on this connection -158> 2021-12-24T09:52:05.852+0000 7f55d260a700 4 mon.osd003@1(probing) e9 probe_timeout 0x5651d86bbd40 -157> 2021-12-24T09:52:06.288+0000 7f55cfe05700 5 mon.osd003@1(probing) e9 _ms_dispatch setting monitor caps on this connection -156> 2021-12-24T09:52:06.288+0000 7f55cfe05700 0 log_channel(cluster) log [INF] : mon.osd003 calling monitor election -155> 2021-12-24T09:52:06.288+0000 7f55cfe05700 10 log_client _send_to_mon log to self -154> 2021-12-24T09:52:06.288+0000 7f55cfe05700 10 log_client log_queue is 1 last_log 1 sent 0 num 1 unsent 1 sending 1 -153> 2021-12-24T09:52:06.288+0000 7f55cfe05700 10 log_client will send 2021-12-24T09:52:06.293634+0000 mon.osd003 (mon.1) 1 : cluster [INF] mon.osd003 calling monitor election -152> 2021-12-24T09:52:06.288+0000 7f55cfe05700 5 paxos.1).electionLogic(0) start -- can i be leader? -151> 2021-12-24T09:52:06.288+0000 7f55cfe05700 1 paxos.1).electionLogic(162) init, last seen epoch 162 -150> 2021-12-24T09:52:06.308+0000 7f55cfe05700 5 mon.osd003@1(electing) e9 _ms_dispatch setting monitor caps on this connection -149> 2021-12-24T09:52:06.308+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:06.312885+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -148> 2021-12-24T09:52:06.308+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:06.312901+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -147> 2021-12-24T09:52:06.852+0000 7f55cfe05700 5 mon.osd003@1(electing) e9 _ms_dispatch setting monitor caps on this connection -146> 2021-12-24T09:52:06.856+0000 7f55cfe05700 5 mon.osd003@1(electing) e9 _ms_dispatch setting monitor caps on this connection -145> 2021-12-24T09:52:08.016+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:08.023674+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -144> 2021-12-24T09:52:08.016+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:08.023699+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -143> 2021-12-24T09:52:08.020+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:08.025000+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -142> 2021-12-24T09:52:08.020+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:08.025014+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -141> 2021-12-24T09:52:08.020+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:08.025396+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -140> 2021-12-24T09:52:08.020+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:08.025403+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -139> 2021-12-24T09:52:08.020+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:08.025436+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -138> 2021-12-24T09:52:08.020+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:08.025441+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -137> 2021-12-24T09:52:08.020+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:08.025825+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -136> 2021-12-24T09:52:08.020+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:08.025832+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -135> 2021-12-24T09:52:08.020+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:08.027333+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -134> 2021-12-24T09:52:08.020+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:08.027342+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -133> 2021-12-24T09:52:08.024+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:08.029186+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -132> 2021-12-24T09:52:08.024+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:08.029194+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -131> 2021-12-24T09:52:08.024+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:08.029202+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -130> 2021-12-24T09:52:08.024+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:08.029209+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -129> 2021-12-24T09:52:08.024+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:08.029584+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -128> 2021-12-24T09:52:08.024+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:08.029604+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -127> 2021-12-24T09:52:08.024+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:08.029616+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -126> 2021-12-24T09:52:08.024+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:08.029622+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -125> 2021-12-24T09:52:08.024+0000 7f55cfe05700 0 mon.osd003@1(electing) e9 handle_command mon_command({"prefix":"osd pool get-quota", "pool": "volumes", "format":"json"} v 0) v1 -124> 2021-12-24T09:52:08.024+0000 7f55cfe05700 0 log_channel(audit) log [DBG] : from='client.3605351 10.254.248.13:0/1538571016' entity='client.cinder' cmd=[{"prefix":"osd pool get-quota", "pool": "volumes", "format":"json"}]: dispatch -123> 2021-12-24T09:52:08.024+0000 7f55cfe05700 10 log_client _send_to_mon log to self -122> 2021-12-24T09:52:08.024+0000 7f55cfe05700 10 log_client log_queue is 2 last_log 2 sent 1 num 2 unsent 1 sending 1 -121> 2021-12-24T09:52:08.024+0000 7f55cfe05700 10 log_client will send 2021-12-24T09:52:08.029756+0000 mon.osd003 (mon.1) 2 : audit [DBG] from='client.3605351 10.254.248.13:0/1538571016' entity='client.cinder' cmd=[{"prefix":"osd pool get-quota", "pool": "volumes", "format":"json"}]: dispatch -120> 2021-12-24T09:52:08.024+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:08.029774+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -119> 2021-12-24T09:52:08.024+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:08.029779+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -118> 2021-12-24T09:52:08.024+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:08.029798+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -117> 2021-12-24T09:52:08.024+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:08.029803+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -116> 2021-12-24T09:52:08.024+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:08.029817+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -115> 2021-12-24T09:52:08.024+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:08.029824+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -114> 2021-12-24T09:52:08.024+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:08.029832+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -113> 2021-12-24T09:52:08.024+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:08.029837+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -112> 2021-12-24T09:52:08.024+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:08.029845+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -111> 2021-12-24T09:52:08.024+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:08.029851+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -110> 2021-12-24T09:52:08.024+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:08.029858+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -109> 2021-12-24T09:52:08.024+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:08.029864+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -108> 2021-12-24T09:52:08.024+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:08.029871+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -107> 2021-12-24T09:52:08.024+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:08.029877+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -106> 2021-12-24T09:52:08.024+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:08.029885+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -105> 2021-12-24T09:52:08.024+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:08.029890+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -104> 2021-12-24T09:52:08.192+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:08.198697+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -103> 2021-12-24T09:52:08.192+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:08.198713+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -102> 2021-12-24T09:52:08.196+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:08.201182+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -101> 2021-12-24T09:52:08.196+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:08.201192+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -100> 2021-12-24T09:52:08.196+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:08.201199+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -99> 2021-12-24T09:52:08.196+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:08.201206+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -98> 2021-12-24T09:52:08.196+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:08.201214+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -97> 2021-12-24T09:52:08.196+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:08.201219+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -96> 2021-12-24T09:52:08.196+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:08.201229+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -95> 2021-12-24T09:52:08.196+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:08.201234+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -94> 2021-12-24T09:52:08.196+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:08.201242+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -93> 2021-12-24T09:52:08.196+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:08.201248+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -92> 2021-12-24T09:52:08.196+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:08.201256+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -91> 2021-12-24T09:52:08.196+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:08.201262+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -90> 2021-12-24T09:52:08.196+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:08.201269+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -89> 2021-12-24T09:52:08.196+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:08.201274+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -88> 2021-12-24T09:52:08.196+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:08.201284+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -87> 2021-12-24T09:52:08.196+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:08.201289+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -86> 2021-12-24T09:52:08.236+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:08.242783+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -85> 2021-12-24T09:52:08.236+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:08.242794+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -84> 2021-12-24T09:52:08.236+0000 7f55cfe05700 0 mon.osd003@1(electing) e9 handle_command mon_command({"prefix":"osd pool get-quota", "pool": "volumes", "format":"json"} v 0) v1 -83> 2021-12-24T09:52:08.236+0000 7f55cfe05700 0 log_channel(audit) log [DBG] : from='client.3605487 10.254.248.12:0/710334867' entity='client.cinder' cmd=[{"prefix":"osd pool get-quota", "pool": "volumes", "format":"json"}]: dispatch -82> 2021-12-24T09:52:08.236+0000 7f55cfe05700 10 log_client _send_to_mon log to self -81> 2021-12-24T09:52:08.236+0000 7f55cfe05700 10 log_client log_queue is 3 last_log 3 sent 2 num 3 unsent 1 sending 1 -80> 2021-12-24T09:52:08.236+0000 7f55cfe05700 10 log_client will send 2021-12-24T09:52:08.242888+0000 mon.osd003 (mon.1) 3 : audit [DBG] from='client.3605487 10.254.248.12:0/710334867' entity='client.cinder' cmd=[{"prefix":"osd pool get-quota", "pool": "volumes", "format":"json"}]: dispatch -79> 2021-12-24T09:52:08.236+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:08.242905+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -78> 2021-12-24T09:52:08.236+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:08.242910+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -77> 2021-12-24T09:52:08.236+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:08.242920+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -76> 2021-12-24T09:52:08.236+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:08.242925+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -75> 2021-12-24T09:52:08.572+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:08.577054+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -74> 2021-12-24T09:52:08.572+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:08.577086+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -73> 2021-12-24T09:52:09.592+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:09.596496+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -72> 2021-12-24T09:52:09.592+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:09.596524+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -71> 2021-12-24T09:52:10.572+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:10.577238+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -70> 2021-12-24T09:52:10.572+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:10.577273+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -69> 2021-12-24T09:52:10.656+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:10.660407+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -68> 2021-12-24T09:52:10.656+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:10.660417+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -67> 2021-12-24T09:52:11.308+0000 7f55d260a700 5 paxos.1).electionLogic(163) election period ended -66> 2021-12-24T09:52:11.308+0000 7f55d260a700 5 paxos.1).electionLogic(163) start -- can i be leader? -65> 2021-12-24T09:52:11.308+0000 7f55d260a700 1 paxos.1).electionLogic(163) init, last seen epoch 163, mid-election, bumping -64> 2021-12-24T09:52:11.308+0000 7f55cfe05700 5 mon.osd003@1(electing).elector(165) handle_propose from mon.4 -63> 2021-12-24T09:52:11.308+0000 7f55cfe05700 0 log_channel(cluster) log [INF] : mon.osd003 calling monitor election -62> 2021-12-24T09:52:11.308+0000 7f55cfe05700 10 log_client _send_to_mon log to self -61> 2021-12-24T09:52:11.308+0000 7f55cfe05700 10 log_client log_queue is 4 last_log 4 sent 3 num 4 unsent 1 sending 1 -60> 2021-12-24T09:52:11.308+0000 7f55cfe05700 10 log_client will send 2021-12-24T09:52:11.314928+0000 mon.osd003 (mon.1) 4 : cluster [INF] mon.osd003 calling monitor election -59> 2021-12-24T09:52:11.308+0000 7f55cfe05700 5 paxos.1).electionLogic(167) start -- can i be leader? -58> 2021-12-24T09:52:11.308+0000 7f55cfe05700 1 paxos.1).electionLogic(167) init, last seen epoch 167, mid-election, bumping -57> 2021-12-24T09:52:11.308+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:11.315939+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -56> 2021-12-24T09:52:11.308+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:11.315950+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -55> 2021-12-24T09:52:11.316+0000 7f55cfe05700 5 mon.osd003@1(electing).elector(169) handle_propose from mon.0 -54> 2021-12-24T09:52:11.316+0000 7f55cfe05700 5 paxos.1).electionLogic(169) ignoring old propose -53> 2021-12-24T09:52:11.328+0000 7f55cfe05700 5 mon.osd003@1(electing).elector(169) handle_ack from mon.4 -52> 2021-12-24T09:52:11.328+0000 7f55cfe05700 5 mon.osd003@1(electing).elector(169) so far i have { mon.1: features 4540138297136906239 mon_feature_t([kraken,luminous,mimic,osdmap-prune,nautilus,octopus,pacific,elector-pinging]) } -51> 2021-12-24T09:52:11.336+0000 7f55cfe05700 5 mon.osd003@1(electing).elector(169) handle_propose from mon.0 -50> 2021-12-24T09:52:11.336+0000 7f55cfe05700 5 paxos.1).electionLogic(171) defer to 0 -49> 2021-12-24T09:52:11.572+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:11.577314+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -48> 2021-12-24T09:52:11.572+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:11.577325+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -47> 2021-12-24T09:52:12.248+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:12.252990+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -46> 2021-12-24T09:52:12.248+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:12.253002+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -45> 2021-12-24T09:52:13.852+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:13.857022+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -44> 2021-12-24T09:52:13.852+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:13.857035+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -43> 2021-12-24T09:52:13.852+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:13.857059+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -42> 2021-12-24T09:52:13.852+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:13.857065+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -41> 2021-12-24T09:52:13.852+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:13.857749+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -40> 2021-12-24T09:52:13.852+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:13.857756+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -39> 2021-12-24T09:52:13.852+0000 7f55cfe05700 0 mon.osd003@1(electing) e9 handle_command mon_command({"prefix":"osd pool get-quota", "pool": "volumes", "format":"json"} v 0) v1 -38> 2021-12-24T09:52:13.852+0000 7f55cfe05700 0 log_channel(audit) log [DBG] : from='client.3605351 10.254.248.13:0/1538571016' entity='client.cinder' cmd=[{"prefix":"osd pool get-quota", "pool": "volumes", "format":"json"}]: dispatch -37> 2021-12-24T09:52:13.852+0000 7f55cfe05700 10 log_client _send_to_mon log to self -36> 2021-12-24T09:52:13.852+0000 7f55cfe05700 10 log_client log_queue is 5 last_log 5 sent 4 num 5 unsent 1 sending 1 -35> 2021-12-24T09:52:13.852+0000 7f55cfe05700 10 log_client will send 2021-12-24T09:52:13.857851+0000 mon.osd003 (mon.1) 5 : audit [DBG] from='client.3605351 10.254.248.13:0/1538571016' entity='client.cinder' cmd=[{"prefix":"osd pool get-quota", "pool": "volumes", "format":"json"}]: dispatch -34> 2021-12-24T09:52:13.852+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:13.857867+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -33> 2021-12-24T09:52:13.852+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:13.857872+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -32> 2021-12-24T09:52:13.852+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:13.857892+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -31> 2021-12-24T09:52:13.852+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:13.857897+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -30> 2021-12-24T09:52:14.572+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:14.577501+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -29> 2021-12-24T09:52:14.572+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:14.577512+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -28> 2021-12-24T09:52:15.572+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:15.577617+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -27> 2021-12-24T09:52:15.572+0000 7f55cfe05700 5 mon.osd003@1(electing).paxos(paxos recovering c 3042121..3042649) is_readable = 0 - now=2021-12-24T09:52:15.577631+0000 lease_expire=1970-01-01T00:00:00.000000+0000 has v0 lc 3042649 -26> 2021-12-24T09:52:16.340+0000 7f55cfe05700 5 mon.osd003@1(electing).elector(171) handle_victory from mon.0 quorum_features 4540138297136906239 mon_feature_t([kraken,luminous,mimic,osdmap-prune,nautilus,octopus,pacific,elector-pinging]) -25> 2021-12-24T09:52:16.340+0000 7f55cfe05700 5 mon.osd003@1(peon) e9 apply_quorum_to_compatset_features -24> 2021-12-24T09:52:16.340+0000 7f55cfe05700 5 mon.osd003@1(peon) e9 apply_monmap_to_compatset_features -23> 2021-12-24T09:52:16.340+0000 7f55cfe05700 4 set_mon_vals no callback set -22> 2021-12-24T09:52:16.340+0000 7f55cfe05700 4 set_mon_vals failed to set cluster_network = 10.254.252.0/22: Configuration option 'cluster_network' may not be modified at runtime -21> 2021-12-24T09:52:16.344+0000 7f55cfe05700 4 mgrc handle_mgr_map Got map version 91 -20> 2021-12-24T09:52:16.344+0000 7f55cfe05700 4 mgrc handle_mgr_map Active mgr is now [v2:10.254.251.1:6928/2921383766,v1:10.254.251.1:6929/2921383766] -19> 2021-12-24T09:52:16.344+0000 7f55cfe05700 4 set_mon_vals no callback set -18> 2021-12-24T09:52:16.344+0000 7f55cfe05700 4 set_mon_vals failed to set cluster_network = 10.254.252.0/22: Configuration option 'cluster_network' may not be modified at runtime -17> 2021-12-24T09:52:16.344+0000 7f55cfe05700 4 mgrc handle_mgr_map Got map version 91 -16> 2021-12-24T09:52:16.344+0000 7f55cfe05700 4 mgrc handle_mgr_map Active mgr is now [v2:10.254.251.1:6928/2921383766,v1:10.254.251.1:6929/2921383766] -15> 2021-12-24T09:52:16.344+0000 7f55cfe05700 4 set_mon_vals no callback set -14> 2021-12-24T09:52:16.344+0000 7f55cfe05700 4 set_mon_vals failed to set cluster_network = 10.254.252.0/22: Configuration option 'cluster_network' may not be modified at runtime -13> 2021-12-24T09:52:16.344+0000 7f55cfe05700 4 mgrc handle_mgr_map Got map version 91 -12> 2021-12-24T09:52:16.344+0000 7f55cfe05700 4 mgrc handle_mgr_map Active mgr is now [v2:10.254.251.1:6928/2921383766,v1:10.254.251.1:6929/2921383766] -11> 2021-12-24T09:52:16.344+0000 7f55cfe05700 5 mon.osd003@1(peon).monmap v9 apply_mon_features wait for service to be writeable -10> 2021-12-24T09:52:16.348+0000 7f55cfe05700 5 mon.osd003@1(peon).paxos(paxos active c 3042121..3042651) is_readable = 1 - now=2021-12-24T09:52:16.352181+0000 lease_expire=2021-12-24T09:52:21.398794+0000 has v0 lc 3042651 -9> 2021-12-24T09:52:16.348+0000 7f55cfe05700 5 mon.osd003@1(peon).paxos(paxos active c 3042121..3042651) is_readable = 1 - now=2021-12-24T09:52:16.352191+0000 lease_expire=2021-12-24T09:52:21.398794+0000 has v0 lc 3042651 -8> 2021-12-24T09:52:16.348+0000 7f55cfe05700 5 mon.osd003@1(peon).paxos(paxos active c 3042121..3042651) is_readable = 1 - now=2021-12-24T09:52:16.352225+0000 lease_expire=2021-12-24T09:52:21.398794+0000 has v0 lc 3042651 -7> 2021-12-24T09:52:16.348+0000 7f55cfe05700 5 mon.osd003@1(peon).paxos(paxos active c 3042121..3042651) is_readable = 1 - now=2021-12-24T09:52:16.352250+0000 lease_expire=2021-12-24T09:52:21.398794+0000 has v0 lc 3042651 -6> 2021-12-24T09:52:16.348+0000 7f55cfe05700 5 mon.osd003@1(peon).paxos(paxos active c 3042121..3042651) is_readable = 1 - now=2021-12-24T09:52:16.352267+0000 lease_expire=2021-12-24T09:52:21.398794+0000 has v0 lc 3042651 -5> 2021-12-24T09:52:16.348+0000 7f55cfe05700 5 mon.osd003@1(peon).paxos(paxos active c 3042121..3042651) is_readable = 1 - now=2021-12-24T09:52:16.352278+0000 lease_expire=2021-12-24T09:52:21.398794+0000 has v0 lc 3042651 -4> 2021-12-24T09:52:16.348+0000 7f55cfe05700 5 mon.osd003@1(peon).paxos(paxos active c 3042121..3042651) is_readable = 1 - now=2021-12-24T09:52:16.352296+0000 lease_expire=2021-12-24T09:52:21.398794+0000 has v0 lc 3042651 -3> 2021-12-24T09:52:16.348+0000 7f55d0606700 1 heartbeat_map reset_timeout 'Monitor::cpu_tp thread 0x7f55d0606700' had timed out after 0.000000000s -2> 2021-12-24T09:52:16.348+0000 7f55cfe05700 5 mon.osd003@1(peon).paxos(paxos active c 3042121..3042651) is_readable = 1 - now=2021-12-24T09:52:16.352309+0000 lease_expire=2021-12-24T09:52:21.398794+0000 has v0 lc 3042651 -1> 2021-12-24T09:52:16.348+0000 7f55cfe05700 5 mon.osd003@1(peon).paxos(paxos active c 3042121..3042651) is_readable = 1 - now=2021-12-24T09:52:16.352320+0000 lease_expire=2021-12-24T09:52:21.398794+0000 has v0 lc 3042651 0> 2021-12-24T09:52:16.348+0000 7f55cfe05700 -1 *** Caught signal (Segmentation fault) ** in thread 7f55cfe05700 thread_name:ms_dispatch ceph version 16.2.7 (dd0603118f56ab514f133c8d2e3adfc983942503) pacific (stable) 1: /lib64/libpthread.so.0(+0x12c20) [0x7f55db482c20] 2: (OSDMonitor::preprocess_command(boost::intrusive_ptr)+0x4949) [0x5651d4b07b09] 3: (OSDMonitor::preprocess_query(boost::intrusive_ptr)+0x2f3) [0x5651d4b24213] 4: (PaxosService::dispatch(boost::intrusive_ptr)+0x562) [0x5651d4ad1392] 5: (PaxosService::C_RetryMessage::_finish(int)+0x64) [0x5651d4a24534] 6: (C_MonOp::finish(int)+0x49) [0x5651d49c18b9] 7: (Context::complete(int)+0xd) [0x5651d49bedbd] 8: (void finish_contexts > >(ceph::common::CephContext*, std::__cxx11::list >&, int)+0xa5) [0x5651d49eb285] 9: (Paxos::handle_lease(boost::intrusive_ptr)+0x604) [0x5651d4ac7674] 10: (Paxos::dispatch(boost::intrusive_ptr)+0x4d7) [0x5651d4acb1d7] 11: (Monitor::dispatch_op(boost::intrusive_ptr)+0x1324) [0x5651d49bc5f4] 12: (Monitor::_ms_dispatch(Message*)+0x670) [0x5651d49bd080] 13: (Dispatcher::ms_dispatch2(boost::intrusive_ptr const&)+0x5c) [0x5651d49ebf4c] 14: (DispatchQueue::entry()+0x126a) [0x7f55dd98aaba] 15: (DispatchQueue::DispatchThread::entry()+0x11) [0x7f55dda3c5d1] 16: /lib64/libpthread.so.0(+0x817a) [0x7f55db47817a] 17: clone() NOTE: a copy of the executable, or `objdump -rdS ` is needed to interpret this. --- logging levels --- 0/ 5 none 0/ 1 lockdep 0/ 1 context 1/ 1 crush 1/ 5 mds 1/ 5 mds_balancer 1/ 5 mds_locker 1/ 5 mds_log 1/ 5 mds_log_expire 1/ 5 mds_migrator 0/ 1 buffer 0/ 1 timer 0/ 1 filer 0/ 1 striper 0/ 1 objecter 0/ 5 rados 0/ 5 rbd 0/ 5 rbd_mirror 0/ 5 rbd_replay 0/ 5 rbd_pwl 0/ 5 journaler 0/ 5 objectcacher 0/ 5 immutable_obj_cache 0/ 5 client 1/ 5 osd 0/ 5 optracker 0/ 5 objclass 1/ 3 filestore 1/ 3 journal 0/ 0 ms 1/ 5 mon 0/10 monc 1/ 5 paxos 0/ 5 tp 1/ 5 auth 1/ 5 crypto 1/ 1 finisher 1/ 1 reserver 1/ 5 heartbeatmap 1/ 5 perfcounter 1/ 5 rgw 1/ 5 rgw_sync 1/10 civetweb 1/ 5 javaclient 1/ 5 asok 1/ 1 throttle 0/ 0 refs 1/ 5 compressor 1/ 5 bluestore 1/ 5 bluefs 1/ 3 bdev 1/ 5 kstore 4/ 5 rocksdb 4/ 5 leveldb 4/ 5 memdb 1/ 5 fuse 2/ 5 mgr 1/ 5 mgrc 1/ 5 dpdk 1/ 5 eventtrace 1/ 5 prioritycache 0/ 5 test 0/ 5 cephfs_mirror 0/ 5 cephsqlite -2/-2 (syslog threshold) 99/99 (stderr threshold) --- pthread ID / name mapping for recent threads --- 140006477784832 / ms_dispatch 140006494570240 / rocksdb:dump_st 140006536533760 / ms_dispatch 140006544926464 / cpu_tp 140006553319168 / cpu_tp 140006561711872 / cpu_tp 140006570104576 / cpu_tp 140006578497280 / safe_timer 140006662936320 / admin_socket 140006913771264 / ceph-mon max_recent 10000 max_new 10000 log_file /var/lib/ceph/crash/2021-12-24T09:52:16.354163Z_224be6d1-daa6-45e8-b044-ffa2e459b21e/log --- end dump of recent events ---