2018-02-14 13:23:40.888793 7fc517f3ee40 0 set uid:gid to 167:167 (ceph:ceph) 2018-02-14 13:23:40.888815 7fc517f3ee40 0 ceph version 12.2.2 (cf0baeeeeba3b47f9427c6c97e2144b094b7e5ba) luminous (stable), process (unknown), pid 12059 2018-02-14 13:23:40.888856 7fc517f3ee40 0 pidfile_write: ignore empty --pid-file 2018-02-14 13:23:40.922300 7fc517f3ee40 0 load: jerasure load: lrc load: isa 2018-02-14 13:23:40.922421 7fc517f3ee40 0 set rocksdb option compression = kNoCompression 2018-02-14 13:23:40.922433 7fc517f3ee40 0 set rocksdb option write_buffer_size = 33554432 2018-02-14 13:23:40.922459 7fc517f3ee40 0 set rocksdb option compression = kNoCompression 2018-02-14 13:23:40.922466 7fc517f3ee40 0 set rocksdb option write_buffer_size = 33554432 2018-02-14 13:23:40.922609 7fc517f3ee40 4 rocksdb: RocksDB version: 5.4.0 2018-02-14 13:23:40.922618 7fc517f3ee40 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2018-02-14 13:23:40.922622 7fc517f3ee40 4 rocksdb: Compile date Nov 30 2017 2018-02-14 13:23:40.922625 7fc517f3ee40 4 rocksdb: DB SUMMARY 2018-02-14 13:23:40.922669 7fc517f3ee40 4 rocksdb: CURRENT file: CURRENT 2018-02-14 13:23:40.922674 7fc517f3ee40 4 rocksdb: IDENTITY file: IDENTITY 2018-02-14 13:23:40.922678 7fc517f3ee40 4 rocksdb: MANIFEST file: MANIFEST-128462 size: 320 Bytes 2018-02-14 13:23:40.922681 7fc517f3ee40 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-ceph-dal-mon-a1/store.db dir, Total Num: 4, files: 128453.sst 128455.sst 128458.sst 128461.sst 2018-02-14 13:23:40.922683 7fc517f3ee40 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-ceph-dal-mon-a1/store.db: 128463.log size: 52 ; 2018-02-14 13:23:40.922686 7fc517f3ee40 4 rocksdb: Options.error_if_exists: 0 2018-02-14 13:23:40.922687 7fc517f3ee40 4 rocksdb: Options.create_if_missing: 0 2018-02-14 13:23:40.922688 7fc517f3ee40 4 rocksdb: Options.paranoid_checks: 1 2018-02-14 13:23:40.922689 7fc517f3ee40 4 rocksdb: Options.env: 0x5587d76e6040 2018-02-14 13:23:40.922690 7fc517f3ee40 4 rocksdb: Options.info_log: 0x5587d8b9dba0 2018-02-14 13:23:40.922691 7fc517f3ee40 4 rocksdb: Options.max_open_files: -1 2018-02-14 13:23:40.922692 7fc517f3ee40 4 rocksdb: Options.max_file_opening_threads: 16 2018-02-14 13:23:40.922694 7fc517f3ee40 4 rocksdb: Options.use_fsync: 0 2018-02-14 13:23:40.922694 7fc517f3ee40 4 rocksdb: Options.max_log_file_size: 0 2018-02-14 13:23:40.922695 7fc517f3ee40 4 rocksdb: Options.max_manifest_file_size: 18446744073709551615 2018-02-14 13:23:40.922696 7fc517f3ee40 4 rocksdb: Options.log_file_time_to_roll: 0 2018-02-14 13:23:40.922697 7fc517f3ee40 4 rocksdb: Options.keep_log_file_num: 1000 2018-02-14 13:23:40.922698 7fc517f3ee40 4 rocksdb: Options.recycle_log_file_num: 0 2018-02-14 13:23:40.922698 7fc517f3ee40 4 rocksdb: Options.allow_fallocate: 1 2018-02-14 13:23:40.922699 7fc517f3ee40 4 rocksdb: Options.allow_mmap_reads: 0 2018-02-14 13:23:40.922700 7fc517f3ee40 4 rocksdb: Options.allow_mmap_writes: 0 2018-02-14 13:23:40.922701 7fc517f3ee40 4 rocksdb: Options.use_direct_reads: 0 2018-02-14 13:23:40.922702 7fc517f3ee40 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2018-02-14 13:23:40.922703 7fc517f3ee40 4 rocksdb: Options.create_missing_column_families: 0 2018-02-14 13:23:40.922704 7fc517f3ee40 4 rocksdb: Options.db_log_dir: 2018-02-14 13:23:40.922704 7fc517f3ee40 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-ceph-dal-mon-a1/store.db 2018-02-14 13:23:40.922705 7fc517f3ee40 4 rocksdb: Options.table_cache_numshardbits: 6 2018-02-14 13:23:40.922706 7fc517f3ee40 4 rocksdb: Options.max_subcompactions: 1 2018-02-14 13:23:40.922707 7fc517f3ee40 4 rocksdb: Options.max_background_flushes: 1 2018-02-14 13:23:40.922708 7fc517f3ee40 4 rocksdb: Options.WAL_ttl_seconds: 0 2018-02-14 13:23:40.922708 7fc517f3ee40 4 rocksdb: Options.WAL_size_limit_MB: 0 2018-02-14 13:23:40.922709 7fc517f3ee40 4 rocksdb: Options.manifest_preallocation_size: 4194304 2018-02-14 13:23:40.922710 7fc517f3ee40 4 rocksdb: Options.is_fd_close_on_exec: 1 2018-02-14 13:23:40.922711 7fc517f3ee40 4 rocksdb: Options.advise_random_on_open: 1 2018-02-14 13:23:40.922712 7fc517f3ee40 4 rocksdb: Options.db_write_buffer_size: 0 2018-02-14 13:23:40.922712 7fc517f3ee40 4 rocksdb: Options.access_hint_on_compaction_start: 1 2018-02-14 13:23:40.922713 7fc517f3ee40 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2018-02-14 13:23:40.922714 7fc517f3ee40 4 rocksdb: Options.compaction_readahead_size: 0 2018-02-14 13:23:40.922714 7fc517f3ee40 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2018-02-14 13:23:40.922715 7fc517f3ee40 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2018-02-14 13:23:40.922716 7fc517f3ee40 4 rocksdb: Options.use_adaptive_mutex: 0 2018-02-14 13:23:40.922717 7fc517f3ee40 4 rocksdb: Options.rate_limiter: (nil) 2018-02-14 13:23:40.922718 7fc517f3ee40 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2018-02-14 13:23:40.922719 7fc517f3ee40 4 rocksdb: Options.bytes_per_sync: 0 2018-02-14 13:23:40.922720 7fc517f3ee40 4 rocksdb: Options.wal_bytes_per_sync: 0 2018-02-14 13:23:40.922720 7fc517f3ee40 4 rocksdb: Options.wal_recovery_mode: 2 2018-02-14 13:23:40.922721 7fc517f3ee40 4 rocksdb: Options.enable_thread_tracking: 0 2018-02-14 13:23:40.922722 7fc517f3ee40 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2018-02-14 13:23:40.922722 7fc517f3ee40 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2018-02-14 13:23:40.922723 7fc517f3ee40 4 rocksdb: Options.write_thread_max_yield_usec: 100 2018-02-14 13:23:40.922724 7fc517f3ee40 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2018-02-14 13:23:40.922725 7fc517f3ee40 4 rocksdb: Options.row_cache: None 2018-02-14 13:23:40.922726 7fc517f3ee40 4 rocksdb: Options.wal_filter: None 2018-02-14 13:23:40.922727 7fc517f3ee40 4 rocksdb: Options.avoid_flush_during_recovery: 0 2018-02-14 13:23:40.922727 7fc517f3ee40 4 rocksdb: Options.base_background_compactions: 1 2018-02-14 13:23:40.922728 7fc517f3ee40 4 rocksdb: Options.max_background_compactions: 1 2018-02-14 13:23:40.922729 7fc517f3ee40 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2018-02-14 13:23:40.922730 7fc517f3ee40 4 rocksdb: Options.delayed_write_rate : 16777216 2018-02-14 13:23:40.922730 7fc517f3ee40 4 rocksdb: Options.max_total_wal_size: 0 2018-02-14 13:23:40.922731 7fc517f3ee40 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2018-02-14 13:23:40.922741 7fc517f3ee40 4 rocksdb: Options.stats_dump_period_sec: 600 2018-02-14 13:23:40.922742 7fc517f3ee40 4 rocksdb: Compression algorithms supported: 2018-02-14 13:23:40.922745 7fc517f3ee40 4 rocksdb: Snappy supported: 0 2018-02-14 13:23:40.922746 7fc517f3ee40 4 rocksdb: Zlib supported: 0 2018-02-14 13:23:40.922746 7fc517f3ee40 4 rocksdb: Bzip supported: 0 2018-02-14 13:23:40.922747 7fc517f3ee40 4 rocksdb: LZ4 supported: 0 2018-02-14 13:23:40.922748 7fc517f3ee40 4 rocksdb: ZSTD supported: 0 2018-02-14 13:23:40.922750 7fc517f3ee40 4 rocksdb: Fast CRC32 supported: 0 2018-02-14 13:23:40.922904 7fc517f3ee40 4 rocksdb: [/home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos7/DIST/centos7/MACHINE_SIZE/huge/release/12.2.2/rpm/el7/BUILD/ceph-12.2.2/src/rocksdb/db/version_set.cc:2609] Recovering from manifest file: MANIFEST-128462 2018-02-14 13:23:40.922991 7fc517f3ee40 4 rocksdb: [/home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos7/DIST/centos7/MACHINE_SIZE/huge/release/12.2.2/rpm/el7/BUILD/ceph-12.2.2/src/rocksdb/db/column_family.cc:407] --------------- Options for column family [default]: 2018-02-14 13:23:40.922999 7fc517f3ee40 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2018-02-14 13:23:40.923001 7fc517f3ee40 4 rocksdb: Options.merge_operator: 2018-02-14 13:23:40.923002 7fc517f3ee40 4 rocksdb: Options.compaction_filter: None 2018-02-14 13:23:40.923003 7fc517f3ee40 4 rocksdb: Options.compaction_filter_factory: None 2018-02-14 13:23:40.923004 7fc517f3ee40 4 rocksdb: Options.memtable_factory: SkipListFactory 2018-02-14 13:23:40.923005 7fc517f3ee40 4 rocksdb: Options.table_factory: BlockBasedTable 2018-02-14 13:23:40.923024 7fc517f3ee40 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5587d88d0158) cache_index_and_filter_blocks: 1 cache_index_and_filter_blocks_with_high_priority: 1 pin_l0_filter_and_index_blocks_in_cache: 1 index_type: 0 hash_index_allow_collision: 1 checksum: 1 no_block_cache: 0 block_cache: 0x5587d88f29a8 block_cache_name: LRUCache block_cache_options: capacity : 134217728 num_shard_bits : 4 strict_capacity_limit : 0 high_pri_pool_ratio: 0.000 block_cache_compressed: (nil) persistent_cache: (nil) block_size: 4096 block_size_deviation: 10 block_restart_interval: 16 index_block_restart_interval: 1 filter_policy: rocksdb.BuiltinBloomFilter whole_key_filtering: 1 format_version: 2 2018-02-14 13:23:40.923042 7fc517f3ee40 4 rocksdb: Options.write_buffer_size: 33554432 2018-02-14 13:23:40.923046 7fc517f3ee40 4 rocksdb: Options.max_write_buffer_number: 2 2018-02-14 13:23:40.923047 7fc517f3ee40 4 rocksdb: Options.compression: NoCompression 2018-02-14 13:23:40.923049 7fc517f3ee40 4 rocksdb: Options.bottommost_compression: Disabled 2018-02-14 13:23:40.923050 7fc517f3ee40 4 rocksdb: Options.prefix_extractor: nullptr 2018-02-14 13:23:40.923050 7fc517f3ee40 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2018-02-14 13:23:40.923051 7fc517f3ee40 4 rocksdb: Options.num_levels: 7 2018-02-14 13:23:40.923052 7fc517f3ee40 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2018-02-14 13:23:40.923053 7fc517f3ee40 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2018-02-14 13:23:40.923053 7fc517f3ee40 4 rocksdb: Options.compression_opts.window_bits: -14 2018-02-14 13:23:40.923054 7fc517f3ee40 4 rocksdb: Options.compression_opts.level: -1 2018-02-14 13:23:40.923055 7fc517f3ee40 4 rocksdb: Options.compression_opts.strategy: 0 2018-02-14 13:23:40.923056 7fc517f3ee40 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2018-02-14 13:23:40.923057 7fc517f3ee40 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2018-02-14 13:23:40.923057 7fc517f3ee40 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2018-02-14 13:23:40.923058 7fc517f3ee40 4 rocksdb: Options.level0_stop_writes_trigger: 36 2018-02-14 13:23:40.923059 7fc517f3ee40 4 rocksdb: Options.target_file_size_base: 67108864 2018-02-14 13:23:40.923065 7fc517f3ee40 4 rocksdb: Options.target_file_size_multiplier: 1 2018-02-14 13:23:40.923067 7fc517f3ee40 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2018-02-14 13:23:40.923068 7fc517f3ee40 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 0 2018-02-14 13:23:40.923069 7fc517f3ee40 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2018-02-14 13:23:40.923073 7fc517f3ee40 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2018-02-14 13:23:40.923074 7fc517f3ee40 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2018-02-14 13:23:40.923075 7fc517f3ee40 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2018-02-14 13:23:40.923076 7fc517f3ee40 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2018-02-14 13:23:40.923077 7fc517f3ee40 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2018-02-14 13:23:40.923078 7fc517f3ee40 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2018-02-14 13:23:40.923079 7fc517f3ee40 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2018-02-14 13:23:40.923079 7fc517f3ee40 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2018-02-14 13:23:40.923080 7fc517f3ee40 4 rocksdb: Options.max_compaction_bytes: 1677721600 2018-02-14 13:23:40.923081 7fc517f3ee40 4 rocksdb: Options.arena_block_size: 4194304 2018-02-14 13:23:40.923082 7fc517f3ee40 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2018-02-14 13:23:40.923083 7fc517f3ee40 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2018-02-14 13:23:40.923084 7fc517f3ee40 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2018-02-14 13:23:40.923085 7fc517f3ee40 4 rocksdb: Options.disable_auto_compactions: 0 2018-02-14 13:23:40.923095 7fc517f3ee40 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2018-02-14 13:23:40.923097 7fc517f3ee40 4 rocksdb: Options.compaction_pri: kByCompensatedSize 2018-02-14 13:23:40.923097 7fc517f3ee40 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2018-02-14 13:23:40.923098 7fc517f3ee40 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2018-02-14 13:23:40.923099 7fc517f3ee40 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2018-02-14 13:23:40.923100 7fc517f3ee40 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2018-02-14 13:23:40.923101 7fc517f3ee40 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2018-02-14 13:23:40.923101 7fc517f3ee40 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2018-02-14 13:23:40.923102 7fc517f3ee40 4 rocksdb: Options.table_properties_collectors: 2018-02-14 13:23:40.923103 7fc517f3ee40 4 rocksdb: Options.inplace_update_support: 0 2018-02-14 13:23:40.923104 7fc517f3ee40 4 rocksdb: Options.inplace_update_num_locks: 10000 2018-02-14 13:23:40.923105 7fc517f3ee40 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2018-02-14 13:23:40.923106 7fc517f3ee40 4 rocksdb: Options.memtable_huge_page_size: 0 2018-02-14 13:23:40.923107 7fc517f3ee40 4 rocksdb: Options.bloom_locality: 0 2018-02-14 13:23:40.923108 7fc517f3ee40 4 rocksdb: Options.max_successive_merges: 0 2018-02-14 13:23:40.923108 7fc517f3ee40 4 rocksdb: Options.optimize_filters_for_hits: 0 2018-02-14 13:23:40.923109 7fc517f3ee40 4 rocksdb: Options.paranoid_file_checks: 0 2018-02-14 13:23:40.923110 7fc517f3ee40 4 rocksdb: Options.force_consistency_checks: 0 2018-02-14 13:23:40.923117 7fc517f3ee40 4 rocksdb: Options.report_bg_io_stats: 0 2018-02-14 13:23:40.923790 7fc517f3ee40 4 rocksdb: [/home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos7/DIST/centos7/MACHINE_SIZE/huge/release/12.2.2/rpm/el7/BUILD/ceph-12.2.2/src/rocksdb/db/version_set.cc:2859] Recovered from manifest file:/var/lib/ceph/mon/ceph-ceph-dal-mon-a1/store.db/MANIFEST-128462 succeeded,manifest_file_number is 128462, next_file_number is 128464, last_sequence is 45590300, log_number is 0,prev_log_number is 0,max_column_family is 0 2018-02-14 13:23:40.923800 7fc517f3ee40 4 rocksdb: [/home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos7/DIST/centos7/MACHINE_SIZE/huge/release/12.2.2/rpm/el7/BUILD/ceph-12.2.2/src/rocksdb/db/version_set.cc:2867] Column family [default] (ID 0), log number is 128461 2018-02-14 13:23:40.923868 7fc517f3ee40 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1518643420923857, "job": 1, "event": "recovery_started", "log_files": [128463]} 2018-02-14 13:23:40.923882 7fc517f3ee40 4 rocksdb: [/home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos7/DIST/centos7/MACHINE_SIZE/huge/release/12.2.2/rpm/el7/BUILD/ceph-12.2.2/src/rocksdb/db/db_impl_open.cc:482] Recovering log #128463 mode 2 2018-02-14 13:23:41.043862 7fc517f3ee40 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1518643421043844, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 128464, "file_size": 1004, "table_properties": {"data_size": 54, "index_size": 27, "filter_size": 18, "raw_key_size": 30, "raw_average_key_size": 30, "raw_value_size": 8, "raw_average_value_size": 8, "num_data_blocks": 1, "num_entries": 1, "filter_policy_name": "rocksdb.BuiltinBloomFilter", "kDeletedKeys": "0", "kMergeOperands": "0"}} 2018-02-14 13:23:41.043900 7fc517f3ee40 4 rocksdb: [/home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos7/DIST/centos7/MACHINE_SIZE/huge/release/12.2.2/rpm/el7/BUILD/ceph-12.2.2/src/rocksdb/db/version_set.cc:2395] Creating manifest 128465 2018-02-14 13:23:41.136089 7fc517f3ee40 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1518643421136085, "job": 1, "event": "recovery_finished"} 2018-02-14 13:23:41.209366 7fc517f3ee40 4 rocksdb: [/home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos7/DIST/centos7/MACHINE_SIZE/huge/release/12.2.2/rpm/el7/BUILD/ceph-12.2.2/src/rocksdb/db/db_impl_open.cc:1063] DB pointer 0x5587d8c98000 2018-02-14 13:23:41.209583 7fc509cf8700 3 rocksdb: [/home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos7/DIST/centos7/MACHINE_SIZE/huge/release/12.2.2/rpm/el7/BUILD/ceph-12.2.2/src/rocksdb/db/db_impl.cc:447] ------- DUMPING STATS ------- 2018-02-14 13:23:41.209592 7fc509cf8700 3 rocksdb: [/home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos7/DIST/centos7/MACHINE_SIZE/huge/release/12.2.2/rpm/el7/BUILD/ceph-12.2.2/src/rocksdb/db/db_impl.cc:448] ** DB Stats ** Uptime(secs): 0.3 total, 0.3 interval Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s Interval stall: 00:00:0.000 H:M:S, 0.0 percent ** Compaction Stats [default] ** Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop ---------------------------------------------------------------------------------------------------------------------------------------------------------- L0 4/0 69.84 KB 1.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 0.0 0 1 0.120 0 0 L1 1/0 44.67 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0 0 0.000 0 0 Sum 5/0 44.73 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 0.0 0 1 0.120 0 0 Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 0.0 0 1 0.120 0 0 Uptime(secs): 0.3 total, 0.3 interval Flush(GB): cumulative 0.000, interval 0.000 AddFile(GB): cumulative 0.000, interval 0.000 AddFile(Total Files): cumulative 0, interval 0 AddFile(L0 Files): cumulative 0, interval 0 AddFile(Keys): cumulative 0, interval 0 Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.1 seconds Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.1 seconds Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count ** File Read Latency Histogram By Level [default] ** ** Compaction Stats [default] ** Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop ---------------------------------------------------------------------------------------------------------------------------------------------------------- L0 4/0 69.84 KB 1.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 0.0 0 1 0.120 0 0 L1 1/0 44.67 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0 0 0.000 0 0 Sum 5/0 44.73 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 0.0 0 1 0.120 0 0 Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0 0 0.000 0 0 Uptime(secs): 0.3 total, 0.3 interval Flush(GB): cumulative 0.000, interval 0.000 AddFile(GB): cumulative 0.000, interval 0.000 AddFile(Total Files): cumulative 0, interval 0 AddFile(L0 Files): cumulative 0, interval 0 AddFile(Keys): cumulative 0, interval 0 Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.1 seconds Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count ** File Read Latency Histogram By Level [default] ** 2018-02-14 13:23:41.209692 7fc509cf8700 4 rocksdb: [/home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos7/DIST/centos7/MACHINE_SIZE/huge/release/12.2.2/rpm/el7/BUILD/ceph-12.2.2/src/rocksdb/db/compaction_job.cc:1403] [default] [JOB 3] Compacting 4@0 + 1@1 files to L1, score 1.00 2018-02-14 13:23:41.209707 7fc509cf8700 4 rocksdb: [/home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos7/DIST/centos7/MACHINE_SIZE/huge/release/12.2.2/rpm/el7/BUILD/ceph-12.2.2/src/rocksdb/db/compaction_job.cc:1407] [default] Compaction start summary: Base version 2 Base level 0, inputs: [128464(1004B) 128461(1036B) 128458(66KB) 128455(1036B)], [128453(44MB)] 2018-02-14 13:23:41.209751 7fc509cf8700 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1518643421209713, "job": 3, "event": "compaction_started", "files_L0": [128464, 128461, 128458, 128455], "files_L1": [128453], "score": 1, "input_data_size": 46907561} 2018-02-14 13:23:41.210191 7fc517f3ee40 10 obtain_monmap 2018-02-14 13:23:41.210324 7fc517f3ee40 10 obtain_monmap read last committed monmap ver 1 2018-02-14 13:23:41.210865 7fc517f3ee40 0 starting mon.ceph-dal-mon-a1 rank 0 at public addr 10.255.0.30:6789/0 at bind addr 10.255.0.30:6789/0 mon_data /var/lib/ceph/mon/ceph-ceph-dal-mon-a1 fsid 39fac7f7-f5a8-4e84-b9bb-b0dfffd79c92 2018-02-14 13:23:41.210972 7fc517f3ee40 0 starting mon.ceph-dal-mon-a1 rank 0 at 10.255.0.30:6789/0 mon_data /var/lib/ceph/mon/ceph-ceph-dal-mon-a1 fsid 39fac7f7-f5a8-4e84-b9bb-b0dfffd79c92 2018-02-14 13:23:41.211482 7fc517f3ee40 1 mon.ceph-dal-mon-a1@-1(probing) e1 preinit fsid 39fac7f7-f5a8-4e84-b9bb-b0dfffd79c92 2018-02-14 13:23:41.211565 7fc517f3ee40 10 mon.ceph-dal-mon-a1@-1(probing) e1 check_fsid cluster_uuid contains '39fac7f7-f5a8-4e84-b9bb-b0dfffd79c92' 2018-02-14 13:23:41.211585 7fc517f3ee40 10 mon.ceph-dal-mon-a1@-1(probing) e1 features compat={},rocompat={},incompat={1=initial feature set (~v.18),3=single paxos with k/v store (v0.?),4=support erasure code pools,5=new-style osdmap encoding,6=support isa/lrc erasure code,7=support shec erasure code,8=support monmap features,9=luminous ondisk layout} 2018-02-14 13:23:41.211599 7fc517f3ee40 10 mon.ceph-dal-mon-a1@-1(probing) e1 calc_quorum_requirements required_features 153140804152475648 2018-02-14 13:23:41.211602 7fc517f3ee40 10 mon.ceph-dal-mon-a1@-1(probing) e1 required_features 153140804152475648 2018-02-14 13:23:41.211613 7fc517f3ee40 10 mon.ceph-dal-mon-a1@-1(probing) e1 has_ever_joined = 1 2018-02-14 13:23:41.211648 7fc517f3ee40 10 mon.ceph-dal-mon-a1@-1(probing) e1 sync_last_committed_floor 0 2018-02-14 13:23:41.211653 7fc517f3ee40 10 mon.ceph-dal-mon-a1@-1(probing) e1 init_paxos 2018-02-14 13:23:41.211695 7fc517f3ee40 1 mon.ceph-dal-mon-a1@-1(probing).mds e0 Unable to load 'last_metadata' 2018-02-14 13:23:41.211733 7fc517f3ee40 10 mon.ceph-dal-mon-a1@-1(probing).health init 2018-02-14 13:23:41.211738 7fc517f3ee40 10 mon.ceph-dal-mon-a1@-1(probing) e1 refresh_from_paxos 2018-02-14 13:23:41.211784 7fc517f3ee40 1 mon.ceph-dal-mon-a1@-1(probing).paxosservice(pgmap 1..2) refresh upgraded, format 0 -> 1 2018-02-14 13:23:41.211792 7fc517f3ee40 1 mon.ceph-dal-mon-a1@-1(probing).pg v0 on_upgrade discarding in-core PGMap 2018-02-14 13:23:41.211817 7fc517f3ee40 10 mon.ceph-dal-mon-a1@-1(probing).pg v0 update_from_paxos deleted, clearing in-memory PGMap 2018-02-14 13:23:41.211858 7fc517f3ee40 10 mon.ceph-dal-mon-a1@-1(probing).mds e0 update_from_paxos version 1, my e 0 2018-02-14 13:23:41.211900 7fc517f3ee40 10 mon.ceph-dal-mon-a1@-1(probing).mds e0 update_from_paxos got 1 2018-02-14 13:23:41.211918 7fc517f3ee40 4 mon.ceph-dal-mon-a1@-1(probing).mds e1 new map 2018-02-14 13:23:41.211923 7fc517f3ee40 0 mon.ceph-dal-mon-a1@-1(probing).mds e1 print_map e1 enable_multiple, ever_enabled_multiple: 0,0 compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=file layout v2} legacy client fscid: -1 No filesystems configured 2018-02-14 13:23:41.211940 7fc517f3ee40 10 mon.ceph-dal-mon-a1@-1(probing).mds e1 update_logger 2018-02-14 13:23:41.212027 7fc517f3ee40 15 mon.ceph-dal-mon-a1@-1(probing).osd e0 update_from_paxos paxos e 19702, my e 0 2018-02-14 13:23:41.212137 7fc517f3ee40 7 mon.ceph-dal-mon-a1@-1(probing).osd e0 update_from_paxos loading latest full map e19702 2018-02-14 13:23:41.212333 7fc517f3ee40 7 mon.ceph-dal-mon-a1@-1(probing).osd e19702 update_from_paxos loading creating_pgs last_scan_epoch 19701 with 0 pgs 2018-02-14 13:23:41.212348 7fc517f3ee40 10 mon.ceph-dal-mon-a1@-1(probing).osd e19702 update_from_paxos pgservice is mgrstat 2018-02-14 13:23:41.212353 7fc517f3ee40 10 mon.ceph-dal-mon-a1@-1(probing).osd e19702 adding osd.0 to down_pending_out map 2018-02-14 13:23:41.212356 7fc517f3ee40 10 mon.ceph-dal-mon-a1@-1(probing).osd e19702 adding osd.1 to down_pending_out map 2018-02-14 13:23:41.212358 7fc517f3ee40 10 mon.ceph-dal-mon-a1@-1(probing).osd e19702 adding osd.2 to down_pending_out map 2018-02-14 13:23:41.212359 7fc517f3ee40 10 mon.ceph-dal-mon-a1@-1(probing).osd e19702 adding osd.26 to down_pending_out map 2018-02-14 13:23:41.212362 7fc517f3ee40 10 mon.ceph-dal-mon-a1@-1(probing).osd e19702 check_osdmap_subs 2018-02-14 13:23:41.212369 7fc517f3ee40 10 mon.ceph-dal-mon-a1@-1(probing).osd e19702 share_map_with_random_osd no up osd on our session map 2018-02-14 13:23:41.212372 7fc517f3ee40 10 mon.ceph-dal-mon-a1@-1(probing).osd e19702 update_logger 2018-02-14 13:23:41.212380 7fc517f3ee40 0 mon.ceph-dal-mon-a1@-1(probing).osd e19702 crush map has features 288514394856620032, adjusting msgr requires 2018-02-14 13:23:41.212382 7fc517f3ee40 0 mon.ceph-dal-mon-a1@-1(probing).osd e19702 crush map has features 288514394856620032, adjusting msgr requires 2018-02-14 13:23:41.212385 7fc517f3ee40 0 mon.ceph-dal-mon-a1@-1(probing).osd e19702 crush map has features 1009090335235915776, adjusting msgr requires 2018-02-14 13:23:41.212387 7fc517f3ee40 0 mon.ceph-dal-mon-a1@-1(probing).osd e19702 crush map has features 288514119978713088, adjusting msgr requires 2018-02-14 13:23:41.212412 7fc517f3ee40 20 queue 0x5587d8cb31e0 4 [0,128) 2018-02-14 13:23:41.212415 7fc517f3ee40 20 queue 0x5587d8cb31e0 5 [0,64) 2018-02-14 13:23:41.212416 7fc517f3ee40 20 queue 0x5587d8cb31e0 6 [0,256) 2018-02-14 13:23:41.212417 7fc517f3ee40 10 mon.ceph-dal-mon-a1@-1(probing).osd e19702 start_mapping started mapping job 0x5587d8cb31e0 at 2018-02-14 13:23:41.212389 2018-02-14 13:23:41.212496 7fc517f3ee40 10 mon.ceph-dal-mon-a1@-1(probing).log v594874 update_from_paxos 2018-02-14 13:23:41.212504 7fc517f3ee40 10 mon.ceph-dal-mon-a1@-1(probing).log v594874 update_from_paxos version 594874 summary v 0 2018-02-14 13:23:41.212513 7fc517f3ee40 10 mon.ceph-dal-mon-a1@-1(probing).log v594874 update_from_paxos latest full 594873 2018-02-14 13:23:41.212552 7fc517f3ee40 7 mon.ceph-dal-mon-a1@-1(probing).log v594874 update_from_paxos loading summary e594873 2018-02-14 13:23:41.212641 7fc517f3ee40 7 mon.ceph-dal-mon-a1@-1(probing).log v594874 update_from_paxos loaded summary e594873 2018-02-14 13:23:41.212658 7fc517f3ee40 7 mon.ceph-dal-mon-a1@-1(probing).log v594874 update_from_paxos applying incremental log 594874 2018-02-14 13:09:57.805123 mon.ceph-dal-mon-a1 mon.0 10.255.0.30:6789/0 9 : cluster [WRN] Health check update: 1/3 mons down, quorum ceph-dal-mon-a1,ceph-dal-mon-a3 (MON_DOWN) 2018-02-14 13:23:41.212674 7fc517f3ee40 20 mon.ceph-dal-mon-a1@-1(probing).log v594874 update_from_paxos logging for channel 'cluster' to file '/var/log/ceph/ceph.log' 2018-02-14 13:23:41.212689 7fc517f3ee40 15 mon.ceph-dal-mon-a1@-1(probing).log v594874 update_from_paxos logging for 1 channels 2018-02-14 13:23:41.212693 7fc517f3ee40 15 mon.ceph-dal-mon-a1@-1(probing).log v594874 update_from_paxos channel 'cluster' logging 176 bytes 2018-02-14 13:23:41.212720 7fc517f3ee40 10 mon.ceph-dal-mon-a1@-1(probing).log v594874 check_subs 2018-02-14 13:23:41.212823 7fc517f3ee40 1 mon.ceph-dal-mon-a1@-1(probing).paxosservice(auth 2501..2644) refresh upgraded, format 0 -> 2 2018-02-14 13:23:41.212832 7fc517f3ee40 10 mon.ceph-dal-mon-a1@-1(probing).auth v2644 update_from_paxos 2018-02-14 13:23:41.212851 7fc517f3ee40 10 mon.ceph-dal-mon-a1@-1(probing).auth v2644 update_from_paxos version 2644 keys ver 0 latest 2641 2018-02-14 13:23:41.212886 7fc517f3ee40 7 mon.ceph-dal-mon-a1@-1(probing).auth v2644 update_from_paxos loading summary e 2641 2018-02-14 13:23:41.212891 7fc517f3ee40 7 mon.ceph-dal-mon-a1@-1(probing).auth v2644 update_from_paxos latest length 6195 2018-02-14 13:23:41.213687 7fc517f3ee40 10 mon.ceph-dal-mon-a1@-1(probing).auth v2644 update_from_paxos key server version 2641 2018-02-14 13:23:41.213707 7fc517f3ee40 20 mon.ceph-dal-mon-a1@-1(probing).auth v2644 update_from_paxos walking through version 2642 len 19 2018-02-14 13:23:41.213720 7fc517f3ee40 20 mon.ceph-dal-mon-a1@-1(probing).auth v2644 update_from_paxos walking through version 2643 len 801 2018-02-14 13:23:41.213897 7fc517f3ee40 20 mon.ceph-dal-mon-a1@-1(probing).auth v2644 update_from_paxos walking through version 2644 len 19 2018-02-14 13:23:41.213903 7fc517f3ee40 10 mon.ceph-dal-mon-a1@-1(probing).auth v2644 update_from_paxos() last_allocated_id=1724096 max_global_id=1724096 format_version 2 2018-02-14 13:23:41.213953 7fc517f3ee40 4 mon.ceph-dal-mon-a1@-1(probing).mgr e0 loading version 422 2018-02-14 13:23:41.213989 7fc517f3ee40 4 mon.ceph-dal-mon-a1@-1(probing).mgr e422 active server: -(1582487) 2018-02-14 13:23:41.214022 7fc517f3ee40 4 mon.ceph-dal-mon-a1@-1(probing).mgr e422 mkfs or daemon transitioned to available, loading commands 2018-02-14 13:23:41.214115 7fc517f3ee40 10 mon.ceph-dal-mon-a1@-1(probing).mgrstat 4284427 2018-02-14 13:23:41.214182 7fc517f3ee40 10 mon.ceph-dal-mon-a1@-1(probing).mgrstat update_from_paxos v4284427 service_map e1 2018-02-14 13:23:41.214188 7fc517f3ee40 10 mon.ceph-dal-mon-a1@-1(probing).mgrstat check_subs 2018-02-14 13:23:41.214189 7fc517f3ee40 20 mon.ceph-dal-mon-a1@-1(probing).mgrstat update_logger 2018-02-14 13:23:41.214210 7fc517f3ee40 10 mon.ceph-dal-mon-a1@-1(probing).health update_from_paxos 2018-02-14 13:23:41.214233 7fc517f3ee40 20 mon.ceph-dal-mon-a1@-1(probing).health dump:{ "quorum_health": {}, "leader_health": { "MON_DOWN": { "severity": "HEALTH_WARN", "summary": { "message": "1/3 mons down, quorum ceph-dal-mon-a1,ceph-dal-mon-a3" }, "detail": [ { "message": "mon.ceph-dal-mon-a2 (rank 1) addr 10.255.0.31:6789/0 is down (out of quorum)" } ] } } } 2018-02-14 13:23:41.214280 7fc517f3ee40 10 mon.ceph-dal-mon-a1@-1(probing).health(0) init 2018-02-14 13:23:41.214443 7fc517f3ee40 2 mon.ceph-dal-mon-a1@-1(probing) e1 init 2018-02-14 13:23:41.214612 7fc50e501700 20 _process 0x5587d8cb31e0 4 [0,128) 2018-02-14 13:23:41.214627 7fc50f503700 20 _process 0x5587d8cb31e0 5 [0,64) 2018-02-14 13:23:41.214677 7fc50ed02700 20 _process 0x5587d8cb31e0 6 [0,256) 2018-02-14 13:23:41.214712 7fc517f3ee40 10 mon.ceph-dal-mon-a1@-1(probing) e1 bootstrap 2018-02-14 13:23:41.214718 7fc517f3ee40 10 mon.ceph-dal-mon-a1@-1(probing) e1 sync_reset_requester 2018-02-14 13:23:41.214720 7fc517f3ee40 10 mon.ceph-dal-mon-a1@-1(probing) e1 unregister_cluster_logger - not registered 2018-02-14 13:23:41.214722 7fc517f3ee40 10 mon.ceph-dal-mon-a1@-1(probing) e1 cancel_probe_timeout (none scheduled) 2018-02-14 13:23:41.214724 7fc517f3ee40 0 mon.ceph-dal-mon-a1@-1(probing) e1 my rank is now 0 (was -1) 2018-02-14 13:23:41.214733 7fc517f3ee40 10 mon.ceph-dal-mon-a1@0(probing) e1 _reset 2018-02-14 13:23:41.214734 7fc517f3ee40 10 mon.ceph-dal-mon-a1@0(probing) e1 cancel_probe_timeout (none scheduled) 2018-02-14 13:23:41.214736 7fc517f3ee40 10 mon.ceph-dal-mon-a1@0(probing) e1 timecheck_finish 2018-02-14 13:23:41.214740 7fc517f3ee40 15 mon.ceph-dal-mon-a1@0(probing) e1 health_tick_stop 2018-02-14 13:23:41.214742 7fc517f3ee40 15 mon.ceph-dal-mon-a1@0(probing) e1 health_interval_stop 2018-02-14 13:23:41.214743 7fc517f3ee40 10 mon.ceph-dal-mon-a1@0(probing) e1 scrub_event_cancel 2018-02-14 13:23:41.214746 7fc517f3ee40 10 mon.ceph-dal-mon-a1@0(probing) e1 scrub_reset 2018-02-14 13:23:41.214756 7fc517f3ee40 10 mon.ceph-dal-mon-a1@0(probing) e1 cancel_probe_timeout (none scheduled) 2018-02-14 13:23:41.214759 7fc517f3ee40 10 mon.ceph-dal-mon-a1@0(probing) e1 reset_probe_timeout 0x5587d8ce2be0 after 2 seconds 2018-02-14 13:23:41.214765 7fc517f3ee40 10 mon.ceph-dal-mon-a1@0(probing) e1 probing other monitors 2018-02-14 13:23:41.215064 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(probing) e1 ms_handle_refused 0x5587d8f81000 10.255.0.31:6789/0 2018-02-14 13:23:41.215280 7fc50acfa700 10 mon.ceph-dal-mon-a1@0(probing) e1 ms_get_authorizer for mon 2018-02-14 13:23:41.215642 7fc50acfa700 10 mon.ceph-dal-mon-a1@0(probing) e1 ms_get_authorizer for mon 2018-02-14 13:23:41.216416 7fc50acfa700 10 mon.ceph-dal-mon-a1@0(probing) e1 ms_get_authorizer for mon 2018-02-14 13:23:41.216906 7fc50acfa700 0 -- 10.255.0.30:6789/0 >> 10.255.0.32:6789/0 conn(0x5587d8f82800 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=51044 cs=1 l=0).process missed message? skipped from seq 0 to 386006847 2018-02-14 13:23:41.216976 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(probing) e1 _ms_dispatch new session 0x5587d891c480 MonSession(mon.2 10.255.0.32:6789/0 is open ) features 0x1ffddff8eea4fffb 2018-02-14 13:23:41.216983 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(probing) e1 _ms_dispatch setting monitor caps on this connection 2018-02-14 13:23:41.216987 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(probing) e1 caps allow * 2018-02-14 13:23:41.216993 7fc50dd00700 20 is_capable service=mon command= read on cap allow * 2018-02-14 13:23:41.216995 7fc50dd00700 20 allow so far , doing grant allow * 2018-02-14 13:23:41.216996 7fc50dd00700 20 allow all 2018-02-14 13:23:41.216998 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(probing) e1 handle_probe mon_probe(reply 39fac7f7-f5a8-4e84-b9bb-b0dfffd79c92 name ceph-dal-mon-a3 paxos( fc 4935163 lc 4935904 )) v6 2018-02-14 13:23:41.217002 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(probing) e1 handle_probe_reply mon.2 10.255.0.32:6789/0mon_probe(reply 39fac7f7-f5a8-4e84-b9bb-b0dfffd79c92 name ceph-dal-mon-a3 paxos( fc 4935163 lc 4935904 )) v6 2018-02-14 13:23:41.217008 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(probing) e1 monmap is e1: 3 mons at {ceph-dal-mon-a1=10.255.0.30:6789/0,ceph-dal-mon-a2=10.255.0.31:6789/0,ceph-dal-mon-a3=10.255.0.32:6789/0} 2018-02-14 13:23:41.217025 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(probing) e1 peer name is ceph-dal-mon-a3 2018-02-14 13:23:41.217027 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(probing) e1 mon.ceph-dal-mon-a3 is outside the quorum 2018-02-14 13:23:41.217029 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(probing) e1 outside_quorum now ceph-dal-mon-a1,ceph-dal-mon-a3, need 2 2018-02-14 13:23:41.217032 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(probing) e1 that's enough to form a new quorum, calling election 2018-02-14 13:23:41.217034 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(probing) e1 start_election 2018-02-14 13:23:41.217035 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(probing) e1 _reset 2018-02-14 13:23:41.217036 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(probing) e1 cancel_probe_timeout 0x5587d8ce2be0 2018-02-14 13:23:41.217038 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(probing) e1 timecheck_finish 2018-02-14 13:23:41.217040 7fc50dd00700 15 mon.ceph-dal-mon-a1@0(probing) e1 health_tick_stop 2018-02-14 13:23:41.217041 7fc50dd00700 15 mon.ceph-dal-mon-a1@0(probing) e1 health_interval_stop 2018-02-14 13:23:41.217042 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(probing) e1 scrub_event_cancel 2018-02-14 13:23:41.217044 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(probing) e1 scrub_reset 2018-02-14 13:23:41.217057 7fc50dd00700 0 log_channel(cluster) log [INF] : mon.ceph-dal-mon-a1 calling new monitor election 2018-02-14 13:23:41.217099 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing).elector(0) start -- can i be leader? 2018-02-14 13:23:41.217130 7fc50dd00700 1 mon.ceph-dal-mon-a1@0(electing).elector(191) init, last seen epoch 191, mid-election, bumping 2018-02-14 13:23:41.219814 7fc50ed02700 10 osdmap epoch 19702 mapping took 0.007424 seconds 2018-02-14 13:23:41.219824 7fc50ed02700 10 mon.ceph-dal-mon-a1@0(electing).osd e19702 update_creating_pgs 0 pgs creating, 0 pools in queue 2018-02-14 13:23:41.247295 7fc50b4fb700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_verify_authorizer 10.255.0.105:0/1753894484 client protocol 0 2018-02-14 13:23:41.251920 7fc50acfa700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_verify_authorizer 10.255.0.106:0/3270640683 client protocol 0 2018-02-14 13:23:41.454031 7fc511507700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_verify_authorizer 10.255.0.51:0/3360953369 client protocol 0 2018-02-14 13:23:41.515575 7fc50b4fb700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_verify_authorizer 10.255.0.106:0/1436590248 client protocol 0 2018-02-14 13:23:41.585540 7fc50acfa700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_verify_authorizer 10.255.0.51:0/3461880882 client protocol 0 2018-02-14 13:23:41.635816 7fc511507700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_verify_authorizer 10.255.0.30:0/3376128839 client protocol 0 2018-02-14 13:23:41.784396 7fc50b4fb700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_verify_authorizer 10.255.0.105:0/3391979714 client protocol 0 2018-02-14 13:23:41.862819 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing).elector(192) bump_epoch 192 to 193 2018-02-14 13:23:41.905760 7fc509cf8700 4 rocksdb: [/home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos7/DIST/centos7/MACHINE_SIZE/huge/release/12.2.2/rpm/el7/BUILD/ceph-12.2.2/src/rocksdb/db/compaction_job.cc:1116] [default] [JOB 3] Generated table #128469: 6710 keys, 46901291 bytes 2018-02-14 13:23:41.905811 7fc509cf8700 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1518643421905775, "cf_name": "default", "job": 3, "event": "table_file_creation", "file_number": 128469, "file_size": 46901291, "table_properties": {"data_size": 46631736, "index_size": 137886, "filter_size": 130742, "raw_key_size": 144657, "raw_average_key_size": 21, "raw_value_size": 46438940, "raw_average_value_size": 6920, "num_data_blocks": 3792, "num_entries": 6710, "filter_policy_name": "rocksdb.BuiltinBloomFilter", "kDeletedKeys": "0", "kMergeOperands": "0"}} 2018-02-14 13:23:41.921775 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 join_election 2018-02-14 13:23:41.921792 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _reset 2018-02-14 13:23:41.921795 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 cancel_probe_timeout (none scheduled) 2018-02-14 13:23:41.921796 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 timecheck_finish 2018-02-14 13:23:41.921798 7fc50dd00700 15 mon.ceph-dal-mon-a1@0(electing) e1 health_tick_stop 2018-02-14 13:23:41.921800 7fc50dd00700 15 mon.ceph-dal-mon-a1@0(electing) e1 health_interval_stop 2018-02-14 13:23:41.921803 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 scrub_event_cancel 2018-02-14 13:23:41.921805 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 scrub_reset 2018-02-14 13:23:41.922105 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_handle_refused 0x5587d8f81000 10.255.0.31:6789/0 2018-02-14 13:23:41.922115 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_handle_refused 0x5587d8f81000 10.255.0.31:6789/0 2018-02-14 13:23:41.922133 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch new session 0x5587d891ca80 MonSession(mon.0 10.255.0.30:6789/0 is open ) features 0x1ffddff8eea4fffb 2018-02-14 13:23:41.922138 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch setting monitor caps on this connection 2018-02-14 13:23:41.922141 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 caps allow * 2018-02-14 13:23:41.922181 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch new session 0x5587d891cc00 MonSession(client.1684214 10.255.0.105:0/1753894484 is open ) features 0x1ffddff8eea4fffb 2018-02-14 13:23:41.922187 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 caps 2018-02-14 13:23:41.922192 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing) e1 waitlisting message auth(proto 0 35 bytes epoch 1) v1 2018-02-14 13:23:41.922204 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch new session 0x5587d891cd80 MonSession(client.1684262 10.255.0.106:0/3270640683 is open ) features 0x1ffddff8eea4fffb 2018-02-14 13:23:41.922210 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 caps 2018-02-14 13:23:41.922212 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing) e1 waitlisting message auth(proto 0 35 bytes epoch 1) v1 2018-02-14 13:23:41.922223 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch new session 0x5587d891d080 MonSession(client.1684205 10.255.0.51:0/3360953369 is open ) features 0x1ffddff8eea4fffb 2018-02-14 13:23:41.922228 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 caps 2018-02-14 13:23:41.922231 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing) e1 waitlisting message auth(proto 0 35 bytes epoch 1) v1 2018-02-14 13:23:41.922242 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch new session 0x5587d891d380 MonSession(client.? 10.255.0.106:0/1436590248 is open ) features 0x1ffddff8eea4fffb 2018-02-14 13:23:41.922247 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 caps 2018-02-14 13:23:41.922249 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing) e1 waitlisting message auth(proto 0 35 bytes epoch 0) v1 2018-02-14 13:23:41.922259 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch new session 0x5587d891d680 MonSession(client.? 10.255.0.51:0/3461880882 is open ) features 0x1ffddff8eea4fffb 2018-02-14 13:23:41.922265 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 caps 2018-02-14 13:23:41.922267 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing) e1 waitlisting message auth(proto 0 35 bytes epoch 0) v1 2018-02-14 13:23:41.922277 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch new session 0x5587d891d980 MonSession(client.1684241 10.255.0.30:0/3376128839 is open ) features 0x1ffddff8eea4fffb 2018-02-14 13:23:41.922280 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 caps 2018-02-14 13:23:41.922282 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing) e1 waitlisting message auth(proto 0 40 bytes epoch 1) v1 2018-02-14 13:23:41.922291 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch new session 0x5587d891dc80 MonSession(client.1684232 10.255.0.105:0/3391979714 is open ) features 0x1ffddff8eea4fffb 2018-02-14 13:23:41.922295 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 caps 2018-02-14 13:23:41.922296 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing) e1 waitlisting message auth(proto 0 35 bytes epoch 1) v1 2018-02-14 13:23:41.933701 7fc50acfa700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_verify_authorizer 10.255.0.53:0/287677392 client protocol 0 2018-02-14 13:23:41.934235 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch new session 0x5587d891df80 MonSession(client.1684235 10.255.0.53:0/287677392 is open ) features 0x1ffddff8eea4fffb 2018-02-14 13:23:41.934243 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 caps 2018-02-14 13:23:41.934245 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing) e1 waitlisting message auth(proto 0 35 bytes epoch 1) v1 2018-02-14 13:23:41.961204 7fc509cf8700 4 rocksdb: [/home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos7/DIST/centos7/MACHINE_SIZE/huge/release/12.2.2/rpm/el7/BUILD/ceph-12.2.2/src/rocksdb/db/compaction_job.cc:1173] [default] [JOB 3] Compacted 4@0 + 1@1 files to L1 => 46901291 bytes 2018-02-14 13:23:41.967280 7fc511507700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_verify_authorizer 10.255.0.105:0/1988410988 client protocol 0 2018-02-14 13:23:41.967559 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch new session 0x5587d891e280 MonSession(client.1684193 10.255.0.105:0/1988410988 is open ) features 0x1ffddff8eea4fffb 2018-02-14 13:23:41.967567 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 caps 2018-02-14 13:23:41.967569 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing) e1 waitlisting message auth(proto 0 35 bytes epoch 1) v1 2018-02-14 13:23:41.976901 7fc50b4fb700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_verify_authorizer 10.255.0.106:0/1162398434 client protocol 0 2018-02-14 13:23:41.977143 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch new session 0x5587d891e580 MonSession(client.1621590 10.255.0.106:0/1162398434 is open ) features 0x1ffddff8eea4fffb 2018-02-14 13:23:41.977150 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 caps 2018-02-14 13:23:41.977153 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing) e1 waitlisting message auth(proto 0 35 bytes epoch 1) v1 2018-02-14 13:23:42.014328 7fc50acfa700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_verify_authorizer 10.255.0.52:0/3152189720 client protocol 0 2018-02-14 13:23:42.014568 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch new session 0x5587d891e880 MonSession(client.? 10.255.0.52:0/3152189720 is open ) features 0x1ffddff8eea4fffb 2018-02-14 13:23:42.014575 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 caps 2018-02-14 13:23:42.014577 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing) e1 waitlisting message auth(proto 0 35 bytes epoch 0) v1 2018-02-14 13:23:42.030627 7fc509cf8700 4 rocksdb: (Original Log Time 2018/02/14-13:23:42.030524) [/home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos7/DIST/centos7/MACHINE_SIZE/huge/release/12.2.2/rpm/el7/BUILD/ceph-12.2.2/src/rocksdb/db/compaction_job.cc:621] [default] compacted to: base level 1 max bytes base 268435456 files[0 1 0 0 0 0 0] max score 0.17, MB/sec: 62.4 rd, 62.4 wr, level 1, files in(4, 1) out(1) MB in(0.1, 44.7) out(44.7), read-write-amplify(1311.7) write-amplify(655.8) OK, records in: 6729, records dropped: 19 2018-02-14 13:23:42.030637 7fc509cf8700 4 rocksdb: (Original Log Time 2018/02/14-13:23:42.030548) EVENT_LOG_v1 {"time_micros": 1518643422030537, "job": 3, "event": "compaction_finished", "compaction_time_micros": 751432, "output_level": 1, "num_output_files": 1, "total_output_size": 46901291, "num_input_records": 6729, "num_output_records": 6710, "num_subcompactions": 1, "num_single_delete_mismatches": 0, "num_single_delete_fallthrough": 0, "lsm_state": [0, 1, 0, 0, 0, 0, 0]} 2018-02-14 13:23:42.030734 7fc509cf8700 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1518643422030730, "job": 3, "event": "table_file_deletion", "file_number": 128464} 2018-02-14 13:23:42.030803 7fc509cf8700 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1518643422030799, "job": 3, "event": "table_file_deletion", "file_number": 128461} 2018-02-14 13:23:42.030868 7fc509cf8700 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1518643422030865, "job": 3, "event": "table_file_deletion", "file_number": 128458} 2018-02-14 13:23:42.030907 7fc509cf8700 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1518643422030897, "job": 3, "event": "table_file_deletion", "file_number": 128455} 2018-02-14 13:23:42.040205 7fc509cf8700 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1518643422040201, "job": 3, "event": "table_file_deletion", "file_number": 128453} 2018-02-14 13:23:42.071652 7fc511507700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_verify_authorizer 10.255.0.53:0/1148525388 client protocol 0 2018-02-14 13:23:42.072050 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch new session 0x5587d891eb80 MonSession(client.? 10.255.0.53:0/1148525388 is open ) features 0x1ffddff8eea4fffb 2018-02-14 13:23:42.072058 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 caps 2018-02-14 13:23:42.072060 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing) e1 waitlisting message auth(proto 0 35 bytes epoch 0) v1 2018-02-14 13:23:42.221649 7fc50b4fb700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_verify_authorizer 10.255.0.54:0/3194015072 client protocol 0 2018-02-14 13:23:42.222311 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch new session 0x5587d891ee80 MonSession(client.? 10.255.0.54:0/3194015072 is open ) features 0x1ffddff8eea4fffb 2018-02-14 13:23:42.222319 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 caps 2018-02-14 13:23:42.222321 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing) e1 waitlisting message auth(proto 0 35 bytes epoch 0) v1 2018-02-14 13:23:42.378921 7fc50acfa700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_verify_authorizer 10.255.0.54:0/2824825993 client protocol 0 2018-02-14 13:23:42.379262 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch new session 0x5587d891f180 MonSession(client.1684238 10.255.0.54:0/2824825993 is open ) features 0x1ffddff8eea4fffb 2018-02-14 13:23:42.379272 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 caps 2018-02-14 13:23:42.379275 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing) e1 waitlisting message auth(proto 0 35 bytes epoch 1) v1 2018-02-14 13:23:42.431000 7fc511507700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_verify_authorizer 10.255.0.53:0/2869630763 client protocol 0 2018-02-14 13:23:42.431232 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch new session 0x5587d891f480 MonSession(client.1684211 10.255.0.53:0/2869630763 is open ) features 0x1ffddff8eea4fffb 2018-02-14 13:23:42.431240 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 caps 2018-02-14 13:23:42.431243 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing) e1 waitlisting message auth(proto 0 35 bytes epoch 1) v1 2018-02-14 13:23:42.617695 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_handle_refused 0x5587d8f81000 10.255.0.31:6789/0 2018-02-14 13:23:42.958611 7fc50b4fb700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_verify_authorizer 10.255.0.107:0/322403586 client protocol 0 2018-02-14 13:23:42.958834 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch new session 0x5587d891f780 MonSession(client.263543 10.255.0.107:0/322403586 is open ) features 0x7010fb86aa42ada 2018-02-14 13:23:42.958842 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 caps 2018-02-14 13:23:42.958844 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing) e1 waitlisting message auth(proto 0 39 bytes epoch 0) 2018-02-14 13:23:43.088796 7fc50acfa700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_verify_authorizer 10.255.0.53:0/3548294640 client protocol 0 2018-02-14 13:23:43.089070 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch new session 0x5587d891fa80 MonSession(client.1684265 10.255.0.53:0/3548294640 is open ) features 0x1ffddff8eea4fffb 2018-02-14 13:23:43.089079 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 caps 2018-02-14 13:23:43.089091 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing) e1 waitlisting message auth(proto 0 35 bytes epoch 1) v1 2018-02-14 13:23:43.635028 7fc511507700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_verify_authorizer 10.255.0.107:0/2591354583 client protocol 0 2018-02-14 13:23:43.635307 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch new session 0x5587d891fd80 MonSession(client.1684217 10.255.0.107:0/2591354583 is open ) features 0x1ffddff8eea4fffb 2018-02-14 13:23:43.635314 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 caps 2018-02-14 13:23:43.635317 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing) e1 waitlisting message auth(proto 0 35 bytes epoch 1) v1 2018-02-14 13:23:43.679149 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch existing session 0x5587d891c480 for mon.2 10.255.0.32:6789/0 2018-02-14 13:23:43.679156 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 caps allow * 2018-02-14 13:23:43.679159 7fc50dd00700 20 is_capable service=mon command= read on cap allow * 2018-02-14 13:23:43.679162 7fc50dd00700 20 allow so far , doing grant allow * 2018-02-14 13:23:43.679163 7fc50dd00700 20 allow all 2018-02-14 13:23:43.679164 7fc50dd00700 20 is_capable service=mon command= exec on cap allow * 2018-02-14 13:23:43.679165 7fc50dd00700 20 allow so far , doing grant allow * 2018-02-14 13:23:43.679165 7fc50dd00700 20 allow all 2018-02-14 13:23:43.679189 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing).elector(193) handle_propose from mon.2 2018-02-14 13:23:43.679191 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing).elector(193) handle_propose required features 153140804152475648 mon_feature_t([kraken,luminous]), peer features 2305244844532236283 mon_feature_t([kraken,luminous]) 2018-02-14 13:23:43.679198 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing).elector(193) bump_epoch 193 to 501 2018-02-14 13:23:43.722825 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 join_election 2018-02-14 13:23:43.722835 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _reset 2018-02-14 13:23:43.722858 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 cancel_probe_timeout (none scheduled) 2018-02-14 13:23:43.722861 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 timecheck_finish 2018-02-14 13:23:43.722862 7fc50dd00700 15 mon.ceph-dal-mon-a1@0(electing) e1 health_tick_stop 2018-02-14 13:23:43.722864 7fc50dd00700 15 mon.ceph-dal-mon-a1@0(electing) e1 health_interval_stop 2018-02-14 13:23:43.722865 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 scrub_event_cancel 2018-02-14 13:23:43.722867 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 scrub_reset 2018-02-14 13:23:43.722881 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 start_election 2018-02-14 13:23:43.722883 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _reset 2018-02-14 13:23:43.722884 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 cancel_probe_timeout (none scheduled) 2018-02-14 13:23:43.722885 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 timecheck_finish 2018-02-14 13:23:43.722886 7fc50dd00700 15 mon.ceph-dal-mon-a1@0(electing) e1 health_tick_stop 2018-02-14 13:23:43.722887 7fc50dd00700 15 mon.ceph-dal-mon-a1@0(electing) e1 health_interval_stop 2018-02-14 13:23:43.722888 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 scrub_event_cancel 2018-02-14 13:23:43.722889 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 scrub_reset 2018-02-14 13:23:43.722896 7fc50dd00700 0 log_channel(cluster) log [INF] : mon.ceph-dal-mon-a1 calling new monitor election 2018-02-14 13:23:43.722926 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing).elector(501) start -- can i be leader? 2018-02-14 13:23:43.722940 7fc50dd00700 1 mon.ceph-dal-mon-a1@0(electing).elector(501) init, last seen epoch 501, mid-election, bumping 2018-02-14 13:23:43.762258 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing).elector(502) bump_epoch 502 to 503 2018-02-14 13:23:43.786396 7fc50b4fb700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_verify_authorizer 10.255.0.51:0/3289155469 client protocol 0 2018-02-14 13:23:43.789832 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 join_election 2018-02-14 13:23:43.789842 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _reset 2018-02-14 13:23:43.789844 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 cancel_probe_timeout (none scheduled) 2018-02-14 13:23:43.789845 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 timecheck_finish 2018-02-14 13:23:43.789846 7fc50dd00700 15 mon.ceph-dal-mon-a1@0(electing) e1 health_tick_stop 2018-02-14 13:23:43.789848 7fc50dd00700 15 mon.ceph-dal-mon-a1@0(electing) e1 health_interval_stop 2018-02-14 13:23:43.789849 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 scrub_event_cancel 2018-02-14 13:23:43.789850 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 scrub_reset 2018-02-14 13:23:43.790117 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch existing session 0x5587d891ca80 for mon.0 10.255.0.30:6789/0 2018-02-14 13:23:43.790123 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 caps allow * 2018-02-14 13:23:43.790164 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch new session 0x5587d9148300 MonSession(client.1684226 10.255.0.51:0/3289155469 is open ) features 0x1ffddff8eea4fffb 2018-02-14 13:23:43.790176 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 caps 2018-02-14 13:23:43.790178 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing) e1 waitlisting message auth(proto 0 35 bytes epoch 1) v1 2018-02-14 13:23:43.813121 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch existing session 0x5587d891c480 for mon.2 10.255.0.32:6789/0 2018-02-14 13:23:43.813131 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 caps allow * 2018-02-14 13:23:43.813134 7fc50dd00700 20 is_capable service=mon command= read on cap allow * 2018-02-14 13:23:43.813136 7fc50dd00700 20 allow so far , doing grant allow * 2018-02-14 13:23:43.813137 7fc50dd00700 20 allow all 2018-02-14 13:23:43.813138 7fc50dd00700 20 is_capable service=mon command= exec on cap allow * 2018-02-14 13:23:43.813140 7fc50dd00700 20 allow so far , doing grant allow * 2018-02-14 13:23:43.813140 7fc50dd00700 20 allow all 2018-02-14 13:23:43.813158 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing).elector(503) handle_ack from mon.2 2018-02-14 13:23:43.813163 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing).elector(503) so far i have { mon.0: features 2305244844532236283 mon_feature_t([kraken,luminous]), mon.2: features 2305244844532236283 mon_feature_t([kraken,luminous]) } 2018-02-14 13:23:43.837800 7fc50acfa700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_verify_authorizer 10.255.0.105:0/2076150628 client protocol 0 2018-02-14 13:23:43.838089 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch new session 0x5587d9148480 MonSession(client.1684247 10.255.0.105:0/2076150628 is open ) features 0x1ffddff8eea4fffb 2018-02-14 13:23:43.838096 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 caps 2018-02-14 13:23:43.838098 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing) e1 waitlisting message auth(proto 0 35 bytes epoch 1) v1 2018-02-14 13:23:43.854966 7fc511507700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_verify_authorizer 10.255.0.54:0/2718452115 client protocol 0 2018-02-14 13:23:43.855281 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch new session 0x5587d9148600 MonSession(client.1684244 10.255.0.54:0/2718452115 is open ) features 0x1ffddff8eea4fffb 2018-02-14 13:23:43.855291 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 caps 2018-02-14 13:23:43.855293 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing) e1 waitlisting message auth(proto 0 35 bytes epoch 1) v1 2018-02-14 13:23:43.892204 7fc50b4fb700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_verify_authorizer 10.255.0.107:0/690639890 client protocol 0 2018-02-14 13:23:43.892464 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch new session 0x5587d9148780 MonSession(client.1684250 10.255.0.107:0/690639890 is open ) features 0x1ffddff8eea4fffb 2018-02-14 13:23:43.892471 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 caps 2018-02-14 13:23:43.892474 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing) e1 waitlisting message auth(proto 0 35 bytes epoch 1) v1 2018-02-14 13:23:44.218874 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_handle_refused 0x5587d8f81000 10.255.0.31:6789/0 2018-02-14 13:23:44.499088 7fc50acfa700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_verify_authorizer 10.255.0.108:0/989583023 client protocol 0 2018-02-14 13:23:44.499321 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch new session 0x5587d9148900 MonSession(client.? 10.255.0.108:0/989583023 is open ) features 0x1ffddff8eea4fffb 2018-02-14 13:23:44.499329 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 caps 2018-02-14 13:23:44.499332 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing) e1 waitlisting message auth(proto 0 35 bytes epoch 0) v1 2018-02-14 13:23:44.515581 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_handle_reset 0x5587d8f2c000 10.255.0.106:0/1436590248 2018-02-14 13:23:44.515597 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 reset/close on session client.? 10.255.0.106:0/1436590248 2018-02-14 13:23:44.515605 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 remove_session 0x5587d891d380 client.? 10.255.0.106:0/1436590248 features 0x1ffddff8eea4fffb 2018-02-14 13:23:44.585902 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_handle_reset 0x5587d8d40000 10.255.0.51:0/3461880882 2018-02-14 13:23:44.585916 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 reset/close on session client.? 10.255.0.51:0/3461880882 2018-02-14 13:23:44.585923 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 remove_session 0x5587d891d680 client.? 10.255.0.51:0/3461880882 features 0x1ffddff8eea4fffb 2018-02-14 13:23:44.586270 7fc50acfa700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_verify_authorizer 10.255.0.51:0/3461880882 client protocol 0 2018-02-14 13:23:44.586598 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch new session 0x5587d9148a80 MonSession(client.? 10.255.0.51:0/3461880882 is open ) features 0x1ffddff8eea4fffb 2018-02-14 13:23:44.586606 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 caps 2018-02-14 13:23:44.586608 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing) e1 waitlisting message auth(proto 0 35 bytes epoch 0) v1 2018-02-14 13:23:44.930908 7fc511507700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_verify_authorizer 10.255.0.105:0/1919733865 client protocol 0 2018-02-14 13:23:44.931288 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch new session 0x5587d9148c00 MonSession(client.? 10.255.0.105:0/1919733865 is open ) features 0x1ffddff8eea4fffb 2018-02-14 13:23:44.931295 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 caps 2018-02-14 13:23:44.931297 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing) e1 waitlisting message auth(proto 0 35 bytes epoch 0) v1 2018-02-14 13:23:44.963810 7fc50b4fb700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_verify_authorizer 10.255.0.108:0/3175087704 client protocol 0 2018-02-14 13:23:44.964083 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch new session 0x5587d9148f00 MonSession(client.1684190 10.255.0.108:0/3175087704 is open ) features 0x1ffddff8eea4fffb 2018-02-14 13:23:44.964091 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 caps 2018-02-14 13:23:44.964093 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing) e1 waitlisting message auth(proto 0 35 bytes epoch 1) v1 2018-02-14 13:23:45.001795 7fc50acfa700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_verify_authorizer 10.255.0.49:0/59005741 client protocol 0 2018-02-14 13:23:45.002023 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch new session 0x5587d9149080 MonSession(client.1582487 10.255.0.49:0/59005741 is open ) features 0x1ffddff8eea4fffb 2018-02-14 13:23:45.002030 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 caps 2018-02-14 13:23:45.002033 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing) e1 waitlisting message auth(proto 0 42 bytes epoch 1) v1 2018-02-14 13:23:45.014276 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_handle_reset 0x5587d8da8000 10.255.0.52:0/3152189720 2018-02-14 13:23:45.014291 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 reset/close on session client.? 10.255.0.52:0/3152189720 2018-02-14 13:23:45.014295 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 remove_session 0x5587d891e880 client.? 10.255.0.52:0/3152189720 features 0x1ffddff8eea4fffb 2018-02-14 13:23:45.052696 7fc50acfa700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_verify_authorizer 10.255.0.106:0/1777238177 client protocol 0 2018-02-14 13:23:45.052952 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch new session 0x5587d9149200 MonSession(client.1684199 10.255.0.106:0/1777238177 is open ) features 0x1ffddff8eea4fffb 2018-02-14 13:23:45.052960 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 caps 2018-02-14 13:23:45.052962 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing) e1 waitlisting message auth(proto 0 35 bytes epoch 1) v1 2018-02-14 13:23:45.072151 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_handle_reset 0x5587d8da9800 10.255.0.53:0/1148525388 2018-02-14 13:23:45.072166 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 reset/close on session client.? 10.255.0.53:0/1148525388 2018-02-14 13:23:45.072169 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 remove_session 0x5587d891eb80 client.? 10.255.0.53:0/1148525388 features 0x1ffddff8eea4fffb 2018-02-14 13:23:45.072521 7fc511507700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_verify_authorizer 10.255.0.53:0/1148525388 client protocol 0 2018-02-14 13:23:45.072798 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch new session 0x5587d9149380 MonSession(client.? 10.255.0.53:0/1148525388 is open ) features 0x1ffddff8eea4fffb 2018-02-14 13:23:45.072805 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 caps 2018-02-14 13:23:45.072808 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing) e1 waitlisting message auth(proto 0 35 bytes epoch 0) v1 2018-02-14 13:23:45.222523 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_handle_reset 0x5587d8dab000 10.255.0.54:0/3194015072 2018-02-14 13:23:45.222539 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 reset/close on session client.? 10.255.0.54:0/3194015072 2018-02-14 13:23:45.222543 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 remove_session 0x5587d891ee80 client.? 10.255.0.54:0/3194015072 features 0x1ffddff8eea4fffb 2018-02-14 13:23:45.222820 7fc50b4fb700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_verify_authorizer 10.255.0.54:0/3194015072 client protocol 0 2018-02-14 13:23:45.223179 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch new session 0x5587d9149500 MonSession(client.? 10.255.0.54:0/3194015072 is open ) features 0x1ffddff8eea4fffb 2018-02-14 13:23:45.223186 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 caps 2018-02-14 13:23:45.223189 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing) e1 waitlisting message auth(proto 0 35 bytes epoch 0) v1 2018-02-14 13:23:45.322995 7fc511507700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_verify_authorizer 10.255.0.107:0/980595480 client protocol 0 2018-02-14 13:23:45.323312 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch new session 0x5587d9149680 MonSession(client.? 10.255.0.107:0/980595480 is open ) features 0x1ffddff8eea4fffb 2018-02-14 13:23:45.323319 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 caps 2018-02-14 13:23:45.323321 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing) e1 waitlisting message auth(proto 0 35 bytes epoch 0) v1 2018-02-14 13:23:46.214566 7fc510505700 11 mon.ceph-dal-mon-a1@0(electing) e1 tick 2018-02-14 13:23:46.214602 7fc510505700 20 mon.ceph-dal-mon-a1@0(electing) e1 sync_trim_providers 2018-02-14 13:23:46.214647 7fc510505700 10 mon.ceph-dal-mon-a1@0(electing) e1 session closed, dropping 0x5587d88db840 2018-02-14 13:23:46.214661 7fc510505700 10 mon.ceph-dal-mon-a1@0(electing) e1 session closed, dropping 0x5587d88dbcc0 2018-02-14 13:23:46.214736 7fc510505700 10 mon.ceph-dal-mon-a1@0(electing) e1 session closed, dropping 0x5587d8ec0b40 2018-02-14 13:23:46.214744 7fc510505700 10 mon.ceph-dal-mon-a1@0(electing) e1 session closed, dropping 0x5587d8ec1440 2018-02-14 13:23:46.214751 7fc510505700 10 mon.ceph-dal-mon-a1@0(electing) e1 session closed, dropping 0x5587d8ec0fc0 2018-02-14 13:23:46.993156 7fc50b4fb700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_verify_authorizer 10.255.0.108:0/850049828 client protocol 0 2018-02-14 13:23:46.993461 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch new session 0x5587d9149800 MonSession(client.1714098 10.255.0.108:0/850049828 is open ) features 0x1ffddff8eea4fffb 2018-02-14 13:23:46.993469 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 caps 2018-02-14 13:23:46.993471 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing) e1 waitlisting message auth(proto 0 35 bytes epoch 1) v1 2018-02-14 13:23:47.421583 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_handle_refused 0x5587d8f81000 10.255.0.31:6789/0 2018-02-14 13:23:47.499205 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_handle_reset 0x5587d91ba000 10.255.0.108:0/989583023 2018-02-14 13:23:47.499221 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 reset/close on session client.? 10.255.0.108:0/989583023 2018-02-14 13:23:47.499225 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 remove_session 0x5587d9148900 client.? 10.255.0.108:0/989583023 features 0x1ffddff8eea4fffb 2018-02-14 13:23:47.515956 7fc50acfa700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_verify_authorizer 10.255.0.54:0/1233019732 client protocol 0 2018-02-14 13:23:47.516410 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch new session 0x5587d9149980 MonSession(client.1714104 10.255.0.54:0/1233019732 is open ) features 0x1ffddff8eea4fffb 2018-02-14 13:23:47.516417 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 caps 2018-02-14 13:23:47.516419 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing) e1 waitlisting message auth(proto 0 35 bytes epoch 1) v1 2018-02-14 13:23:47.586677 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_handle_reset 0x5587d91bb800 10.255.0.51:0/3461880882 2018-02-14 13:23:47.586694 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 reset/close on session client.? 10.255.0.51:0/3461880882 2018-02-14 13:23:47.586698 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 remove_session 0x5587d9148a80 client.? 10.255.0.51:0/3461880882 features 0x1ffddff8eea4fffb 2018-02-14 13:23:47.587159 7fc50acfa700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_verify_authorizer 10.255.0.51:0/3461880882 client protocol 0 2018-02-14 13:23:47.587639 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch new session 0x5587d9149b00 MonSession(client.? 10.255.0.51:0/3461880882 is open ) features 0x1ffddff8eea4fffb 2018-02-14 13:23:47.587647 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 caps 2018-02-14 13:23:47.587650 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing) e1 waitlisting message auth(proto 0 35 bytes epoch 0) v1 2018-02-14 13:23:47.930851 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_handle_reset 0x5587d91bd000 10.255.0.105:0/1919733865 2018-02-14 13:23:47.930868 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 reset/close on session client.? 10.255.0.105:0/1919733865 2018-02-14 13:23:47.930871 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 remove_session 0x5587d9148c00 client.? 10.255.0.105:0/1919733865 features 0x1ffddff8eea4fffb 2018-02-14 13:23:48.014987 7fc511507700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_verify_authorizer 10.255.0.52:0/3152189720 client protocol 0 2018-02-14 13:23:48.015262 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch new session 0x5587d9149c80 MonSession(client.? 10.255.0.52:0/3152189720 is open ) features 0x1ffddff8eea4fffb 2018-02-14 13:23:48.015269 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 caps 2018-02-14 13:23:48.015271 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing) e1 waitlisting message auth(proto 0 35 bytes epoch 0) v1 2018-02-14 13:23:48.072809 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_handle_reset 0x5587d8fcf000 10.255.0.53:0/1148525388 2018-02-14 13:23:48.072825 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 reset/close on session client.? 10.255.0.53:0/1148525388 2018-02-14 13:23:48.072829 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 remove_session 0x5587d9149380 client.? 10.255.0.53:0/1148525388 features 0x1ffddff8eea4fffb 2018-02-14 13:23:48.224138 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_handle_reset 0x5587d8fd0800 10.255.0.54:0/3194015072 2018-02-14 13:23:48.224157 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 reset/close on session client.? 10.255.0.54:0/3194015072 2018-02-14 13:23:48.224161 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 remove_session 0x5587d9149500 client.? 10.255.0.54:0/3194015072 features 0x1ffddff8eea4fffb 2018-02-14 13:23:48.224558 7fc50b4fb700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_verify_authorizer 10.255.0.54:0/3194015072 client protocol 0 2018-02-14 13:23:48.224990 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch new session 0x5587d9149e00 MonSession(client.? 10.255.0.54:0/3194015072 is open ) features 0x1ffddff8eea4fffb 2018-02-14 13:23:48.224998 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 caps 2018-02-14 13:23:48.225000 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing) e1 waitlisting message auth(proto 0 35 bytes epoch 0) v1 2018-02-14 13:23:48.322965 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_handle_reset 0x5587d8fe2000 10.255.0.107:0/980595480 2018-02-14 13:23:48.322981 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 reset/close on session client.? 10.255.0.107:0/980595480 2018-02-14 13:23:48.322985 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 remove_session 0x5587d9149680 client.? 10.255.0.107:0/980595480 features 0x1ffddff8eea4fffb 2018-02-14 13:23:48.323364 7fc511507700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_verify_authorizer 10.255.0.107:0/980595480 client protocol 0 2018-02-14 13:23:48.323615 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch new session 0x5587d9149f80 MonSession(client.? 10.255.0.107:0/980595480 is open ) features 0x1ffddff8eea4fffb 2018-02-14 13:23:48.323622 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 caps 2018-02-14 13:23:48.323624 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing) e1 waitlisting message auth(proto 0 35 bytes epoch 0) v1 2018-02-14 13:23:48.520130 7fc50b4fb700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_verify_authorizer 10.255.0.107:0/3082328679 client protocol 0 2018-02-14 13:23:48.520445 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch new session 0x5587d914a100 MonSession(client.1684202 10.255.0.107:0/3082328679 is open ) features 0x1ffddff8eea4fffb 2018-02-14 13:23:48.520453 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 caps 2018-02-14 13:23:48.520455 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing) e1 waitlisting message auth(proto 0 35 bytes epoch 1) v1 2018-02-14 13:23:48.584605 7fc50acfa700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_verify_authorizer 10.255.0.52:0/97505378 client protocol 0 2018-02-14 13:23:48.584821 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch new session 0x5587d914a280 MonSession(client.1393717 10.255.0.52:0/97505378 is open ) features 0x7010fb86aa42ada 2018-02-14 13:23:48.584828 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 caps 2018-02-14 13:23:48.584830 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing) e1 waitlisting message auth(proto 0 39 bytes epoch 0) 2018-02-14 13:23:48.667084 7fc511507700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_verify_authorizer 10.255.0.52:0/4056696166 client protocol 0 2018-02-14 13:23:48.667362 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch new session 0x5587d914a400 MonSession(client.1684208 10.255.0.52:0/4056696166 is open ) features 0x1ffddff8eea4fffb 2018-02-14 13:23:48.667370 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 caps 2018-02-14 13:23:48.667372 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing) e1 waitlisting message auth(proto 0 35 bytes epoch 1) v1 2018-02-14 13:23:48.790173 7fc510505700 5 mon.ceph-dal-mon-a1@0(electing).elector(503) election timer expired 2018-02-14 13:23:48.790190 7fc510505700 10 mon.ceph-dal-mon-a1@0(electing).elector(503) bump_epoch 503 to 504 2018-02-14 13:23:48.864928 7fc510505700 10 mon.ceph-dal-mon-a1@0(electing) e1 join_election 2018-02-14 13:23:48.864939 7fc510505700 10 mon.ceph-dal-mon-a1@0(electing) e1 _reset 2018-02-14 13:23:48.864942 7fc510505700 10 mon.ceph-dal-mon-a1@0(electing) e1 cancel_probe_timeout (none scheduled) 2018-02-14 13:23:48.864944 7fc510505700 10 mon.ceph-dal-mon-a1@0(electing) e1 timecheck_finish 2018-02-14 13:23:48.864946 7fc510505700 15 mon.ceph-dal-mon-a1@0(electing) e1 health_tick_stop 2018-02-14 13:23:48.864947 7fc510505700 15 mon.ceph-dal-mon-a1@0(electing) e1 health_interval_stop 2018-02-14 13:23:48.864949 7fc510505700 10 mon.ceph-dal-mon-a1@0(electing) e1 scrub_event_cancel 2018-02-14 13:23:48.864951 7fc510505700 10 mon.ceph-dal-mon-a1@0(electing) e1 scrub_reset 2018-02-14 13:23:48.865014 7fc510505700 10 mon.ceph-dal-mon-a1@0(electing) e1 win_election epoch 504 quorum 0,2 features 2305244844532236283 mon_features mon_feature_t([kraken,luminous]) 2018-02-14 13:23:48.865035 7fc510505700 0 log_channel(cluster) log [INF] : mon.ceph-dal-mon-a1@0 won leader election with quorum 0,2 2018-02-14 13:23:48.906774 7fc510505700 10 mon.ceph-dal-mon-a1@0(leader).data_health(504) start_epoch epoch 504 2018-02-14 13:23:48.906820 7fc510505700 5 mon.ceph-dal-mon-a1@0(leader) e1 apply_quorum_to_compatset_features 2018-02-14 13:23:48.906840 7fc510505700 5 mon.ceph-dal-mon-a1@0(leader) e1 apply_monmap_to_compatset_features 2018-02-14 13:23:48.906847 7fc510505700 10 mon.ceph-dal-mon-a1@0(leader) e1 timecheck_finish 2018-02-14 13:23:48.906960 7fc510505700 10 mon.ceph-dal-mon-a1@0(leader) e1 resend_routed_requests 2018-02-14 13:23:48.906964 7fc510505700 10 mon.ceph-dal-mon-a1@0(leader) e1 register_cluster_logger 2018-02-14 13:23:48.906991 7fc510505700 10 mon.ceph-dal-mon-a1@0(leader) e1 timecheck_start 2018-02-14 13:23:48.906994 7fc510505700 10 mon.ceph-dal-mon-a1@0(leader) e1 timecheck_start_round curr 0 2018-02-14 13:23:48.906997 7fc510505700 10 mon.ceph-dal-mon-a1@0(leader) e1 timecheck_start_round new 1 2018-02-14 13:23:48.907000 7fc510505700 10 mon.ceph-dal-mon-a1@0(leader) e1 timecheck 2018-02-14 13:23:48.907002 7fc510505700 10 mon.ceph-dal-mon-a1@0(leader) e1 timecheck start timecheck epoch 504 round 1 2018-02-14 13:23:48.907008 7fc510505700 10 mon.ceph-dal-mon-a1@0(leader) e1 timecheck send time_check( ping e 504 r 1 ) v1 to mon.2 10.255.0.32:6789/0 2018-02-14 13:23:48.907033 7fc510505700 10 mon.ceph-dal-mon-a1@0(leader) e1 timecheck_start_round setting up next event 2018-02-14 13:23:48.907038 7fc510505700 10 mon.ceph-dal-mon-a1@0(leader) e1 timecheck_reset_event delay 300 rounds_since_clean 0 2018-02-14 13:23:48.907049 7fc510505700 15 mon.ceph-dal-mon-a1@0(leader) e1 health_tick_start 2018-02-14 13:23:48.907051 7fc510505700 15 mon.ceph-dal-mon-a1@0(leader) e1 health_tick_stop 2018-02-14 13:23:48.907054 7fc510505700 10 mon.ceph-dal-mon-a1@0(leader) e1 do_health_to_clog_interval 2018-02-14 13:23:48.907057 7fc510505700 10 mon.ceph-dal-mon-a1@0(leader) e1 do_health_to_clog (force) 2018-02-14 13:23:48.907076 7fc510505700 0 log_channel(cluster) log [WRN] : overall HEALTH_WARN noout flag(s) set; 6 backfillfull osd(s); 4 osds down; 1 host (4 osds) down; 8 nearfull osd(s); 3 pool(s) backfillfull; Degraded data redundancy: 4938712/26491706 objects degraded (18.642%), 347 pgs unclean, 347 pgs degraded, 347 pgs undersized; 1/3 mons down, quorum ceph-dal-mon-a1,ceph-dal-mon-a3 2018-02-14 13:23:48.907109 7fc510505700 15 mon.ceph-dal-mon-a1@0(leader) e1 health_interval_start 2018-02-14 13:23:48.907112 7fc510505700 15 mon.ceph-dal-mon-a1@0(leader) e1 health_interval_stop 2018-02-14 13:23:48.907113 7fc510505700 20 mon.ceph-dal-mon-a1@0(leader) e1 health_interval_calc_next_update now: 2018-02-14 13:23:48.907113, next: 2018-02-14 14:00:00.000000, interval: 3600 2018-02-14 13:23:48.907118 7fc510505700 10 mon.ceph-dal-mon-a1@0(leader) e1 scrub_event_start 2018-02-14 13:23:48.907159 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891ca80 for mon.0 10.255.0.30:6789/0 2018-02-14 13:23:48.907174 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps allow * 2018-02-14 13:23:48.907222 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891c480 for mon.2 10.255.0.32:6789/0 2018-02-14 13:23:48.907227 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps allow * 2018-02-14 13:23:48.907230 7fc50dd00700 20 is_capable service=mon command= read on cap allow * 2018-02-14 13:23:48.907231 7fc50dd00700 20 allow so far , doing grant allow * 2018-02-14 13:23:48.907232 7fc50dd00700 20 allow all 2018-02-14 13:23:48.907234 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 received forwarded message from mon.2 10.255.0.32:6789/0 via mon.2 10.255.0.32:6789/0 2018-02-14 13:23:48.907239 7fc50dd00700 20 is_capable service=mon command= exec on cap allow * 2018-02-14 13:23:48.907240 7fc50dd00700 20 allow so far , doing grant allow * 2018-02-14 13:23:48.907241 7fc50dd00700 20 allow all 2018-02-14 13:23:48.907249 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 caps are allow * 2018-02-14 13:23:48.907250 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 entity name '' type 0 2018-02-14 13:23:48.907252 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 mesg 0x5587d8fda900 from 10.255.0.32:6789/0 2018-02-14 13:23:48.907256 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d914a580 for mon.2 - 2018-02-14 13:23:48.907258 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps allow * 2018-02-14 13:23:48.907286 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891ca80 for mon.0 10.255.0.30:6789/0 2018-02-14 13:23:48.907291 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps allow * 2018-02-14 13:23:48.929284 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891c480 for mon.2 10.255.0.32:6789/0 2018-02-14 13:23:48.929290 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps allow * 2018-02-14 13:23:48.929293 7fc50dd00700 20 is_capable service=mon command= read on cap allow * 2018-02-14 13:23:48.929295 7fc50dd00700 20 allow so far , doing grant allow * 2018-02-14 13:23:48.929296 7fc50dd00700 20 allow all 2018-02-14 13:23:48.929297 7fc50dd00700 20 is_capable service=mon command= exec on cap allow * 2018-02-14 13:23:48.929298 7fc50dd00700 20 allow so far , doing grant allow * 2018-02-14 13:23:48.929299 7fc50dd00700 20 allow all 2018-02-14 13:23:48.929334 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 refresh_from_paxos 2018-02-14 13:23:48.929554 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 update_from_paxos 2018-02-14 13:23:48.929560 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 update_from_paxos version 594874 summary v 594874 2018-02-14 13:23:48.929647 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 update_from_paxos 2018-02-14 13:23:48.929731 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).mgrstat 4284427 2018-02-14 13:23:48.929820 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).mgrstat update_from_paxos v4284427 service_map e1 2018-02-14 13:23:48.929826 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).mgrstat check_subs 2018-02-14 13:23:48.929828 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader).mgrstat update_logger 2018-02-14 13:23:48.929850 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).health update_from_paxos 2018-02-14 13:23:48.929886 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader).health dump:{ "quorum_health": {}, "leader_health": { "MON_DOWN": { "severity": "HEALTH_WARN", "summary": { "message": "1/3 mons down, quorum ceph-dal-mon-a1,ceph-dal-mon-a3" }, "detail": [ { "message": "mon.ceph-dal-mon-a2 (rank 1) addr 10.255.0.31:6789/0 is down (out of quorum)" } ] } } } 2018-02-14 13:23:48.929957 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).monmap v1 create_pending monmap epoch 2 2018-02-14 13:23:48.929974 7fc50dd00700 0 log_channel(cluster) log [INF] : monmap e1: 3 mons at {ceph-dal-mon-a1=10.255.0.30:6789/0,ceph-dal-mon-a2=10.255.0.31:6789/0,ceph-dal-mon-a3=10.255.0.32:6789/0} 2018-02-14 13:23:48.930000 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).monmap v1 apply_mon_features features match current pending: mon_feature_t([kraken,luminous]) 2018-02-14 13:23:48.930025 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).mds e1 create_pending e2 2018-02-14 13:23:48.930030 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).mds e1 2018-02-14 13:23:48.930037 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).mds e1 update_logger 2018-02-14 13:23:48.930041 7fc50dd00700 0 log_channel(cluster) log [DBG] : fsmap 2018-02-14 13:23:48.930072 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).osd e19702 create_pending e 19703 2018-02-14 13:23:48.930124 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).osd e19702 create_pending did clean_temps 2018-02-14 13:23:48.930130 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).osd e19702 update_logger 2018-02-14 13:23:48.930137 7fc50dd00700 0 log_channel(cluster) log [DBG] : osdmap e19702: 32 total, 28 up, 32 in 2018-02-14 13:23:48.930159 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).osd e19702 start_mapping canceling previous mapping_job 0x5587d8cb31e0 2018-02-14 13:23:48.930168 7fc50dd00700 20 queue 0x5587d9025720 4 [0,128) 2018-02-14 13:23:48.930172 7fc50dd00700 20 queue 0x5587d9025720 5 [0,64) 2018-02-14 13:23:48.930182 7fc50dd00700 20 queue 0x5587d9025720 6 [0,256) 2018-02-14 13:23:48.930186 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).osd e19702 start_mapping started mapping job 0x5587d9025720 at 2018-02-14 13:23:48.930163 2018-02-14 13:23:48.930191 7fc50fd04700 20 _process 0x5587d9025720 4 [0,128) 2018-02-14 13:23:48.930202 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 create_pending v 594875 2018-02-14 13:23:48.930202 7fc50f503700 20 _process 0x5587d9025720 5 [0,64) 2018-02-14 13:23:48.930207 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 create_pending v 2645 2018-02-14 13:23:48.930211 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader).auth v2644 upgrade_format format 2 is current 2018-02-14 13:23:48.930212 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 AuthMonitor::on_active() 2018-02-14 13:23:48.930214 7fc50e501700 20 _process 0x5587d9025720 6 [0,256) 2018-02-14 13:23:48.930227 7fc50dd00700 0 log_channel(cluster) log [DBG] : mgrmap e422: ceph-dal-admin-a1(active, starting), standbys: ceph-dal-mon-a1 2018-02-14 13:23:48.930253 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).mgrstat 4284427 2018-02-14 13:23:48.930273 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader).mgrstat update_logger 2018-02-14 13:23:48.930278 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).health 3 2018-02-14 13:23:48.930290 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 preprocess_query log(1 entries from seq 1 at 2018-02-14 13:23:41.217059) v1 from mon.0 10.255.0.30:6789/0 2018-02-14 13:23:48.930300 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 preprocess_log log(1 entries from seq 1 at 2018-02-14 13:23:41.217059) v1 from mon.0 2018-02-14 13:23:48.930305 7fc50dd00700 20 is_capable service=log command= write on cap allow * 2018-02-14 13:23:48.930307 7fc50dd00700 20 allow so far , doing grant allow * 2018-02-14 13:23:48.930308 7fc50dd00700 20 allow all 2018-02-14 13:23:48.930312 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 prepare_update log(1 entries from seq 1 at 2018-02-14 13:23:41.217059) v1 from mon.0 10.255.0.30:6789/0 2018-02-14 13:23:48.930317 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 prepare_log log(1 entries from seq 1 at 2018-02-14 13:23:41.217059) v1 from mon.0 2018-02-14 13:23:48.930321 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 logging 2018-02-14 13:23:41.217059 mon.ceph-dal-mon-a1 mon.0 10.255.0.30:6789/0 1 : cluster [INF] mon.ceph-dal-mon-a1 calling new monitor election 2018-02-14 13:23:48.930345 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 preprocess_query log(1 entries from seq 2 at 2018-02-14 13:23:43.722898) v1 from mon.0 10.255.0.30:6789/0 2018-02-14 13:23:48.930352 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 preprocess_log log(1 entries from seq 2 at 2018-02-14 13:23:43.722898) v1 from mon.0 2018-02-14 13:23:48.930356 7fc50dd00700 20 is_capable service=log command= write on cap allow * 2018-02-14 13:23:48.930357 7fc50dd00700 20 allow so far , doing grant allow * 2018-02-14 13:23:48.930358 7fc50dd00700 20 allow all 2018-02-14 13:23:48.930359 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 prepare_update log(1 entries from seq 2 at 2018-02-14 13:23:43.722898) v1 from mon.0 10.255.0.30:6789/0 2018-02-14 13:23:48.930364 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 prepare_log log(1 entries from seq 2 at 2018-02-14 13:23:43.722898) v1 from mon.0 2018-02-14 13:23:48.930367 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 logging 2018-02-14 13:23:43.722898 mon.ceph-dal-mon-a1 mon.0 10.255.0.30:6789/0 2 : cluster [INF] mon.ceph-dal-mon-a1 calling new monitor election 2018-02-14 13:23:48.930381 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 0 35 bytes epoch 1) v1 from client.1684214 10.255.0.105:0/1753894484 2018-02-14 13:23:48.930392 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=35 2018-02-14 13:23:48.930425 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8911f80 0x5587d8fdcac0 auth_reply(proto 2 0 (0) Success) v1 2018-02-14 13:23:48.930467 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 0 35 bytes epoch 1) v1 from client.1684262 10.255.0.106:0/3270640683 2018-02-14 13:23:48.930473 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=35 2018-02-14 13:23:48.930486 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d89120a0 0x5587d88db180 auth_reply(proto 2 0 (0) Success) v1 2018-02-14 13:23:48.930512 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 0 35 bytes epoch 1) v1 from client.1684205 10.255.0.51:0/3360953369 2018-02-14 13:23:48.930521 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=35 2018-02-14 13:23:48.930533 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d89121c0 0x5587d88da880 auth_reply(proto 2 0 (0) Success) v1 2018-02-14 13:23:48.930570 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 0 40 bytes epoch 1) v1 from client.1684241 10.255.0.30:0/3376128839 2018-02-14 13:23:48.930579 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=40 2018-02-14 13:23:48.930598 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8912520 0x5587d88db600 auth_reply(proto 2 0 (0) Success) v1 2018-02-14 13:23:48.930655 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 0 35 bytes epoch 1) v1 from client.1684232 10.255.0.105:0/3391979714 2018-02-14 13:23:48.930666 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=35 2018-02-14 13:23:48.930686 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8912640 0x5587d8ec0000 auth_reply(proto 2 0 (0) Success) v1 2018-02-14 13:23:48.930719 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 0 35 bytes epoch 1) v1 from client.1684235 10.255.0.53:0/287677392 2018-02-14 13:23:48.930724 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=35 2018-02-14 13:23:48.930736 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8912760 0x5587d88dba80 auth_reply(proto 2 0 (0) Success) v1 2018-02-14 13:23:48.930762 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 0 35 bytes epoch 1) v1 from client.1684193 10.255.0.105:0/1988410988 2018-02-14 13:23:48.930768 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=35 2018-02-14 13:23:48.930800 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8912880 0x5587d8ec0480 auth_reply(proto 2 0 (0) Success) v1 2018-02-14 13:23:48.930845 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 0 35 bytes epoch 1) v1 from client.1621590 10.255.0.106:0/1162398434 2018-02-14 13:23:48.930853 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=35 2018-02-14 13:23:48.930872 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8912d00 0x5587d8ec0240 auth_reply(proto 2 0 (0) Success) v1 2018-02-14 13:23:48.930918 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 0 35 bytes epoch 1) v1 from client.1684238 10.255.0.54:0/2824825993 2018-02-14 13:23:48.930925 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=35 2018-02-14 13:23:48.930940 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8913180 0x5587d8ec0d80 auth_reply(proto 2 0 (0) Success) v1 2018-02-14 13:23:48.930968 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 0 35 bytes epoch 1) v1 from client.1684211 10.255.0.53:0/2869630763 2018-02-14 13:23:48.930973 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=35 2018-02-14 13:23:48.930984 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8913600 0x5587d88db3c0 auth_reply(proto 2 0 (0) Success) v1 2018-02-14 13:23:48.931017 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 0 39 bytes epoch 0) from client.263543 10.255.0.107:0/322403586 2018-02-14 13:23:48.931027 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=39 2018-02-14 13:23:48.931065 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8913720 0x5587d8fdd3c0 auth_reply(proto 2 0 (0) Success) v1 2018-02-14 13:23:48.931094 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 0 35 bytes epoch 1) v1 from client.1684265 10.255.0.53:0/3548294640 2018-02-14 13:23:48.931100 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=35 2018-02-14 13:23:48.931119 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8913840 0x5587d8ec1200 auth_reply(proto 2 0 (0) Success) v1 2018-02-14 13:23:48.931148 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 0 35 bytes epoch 1) v1 from client.1684217 10.255.0.107:0/2591354583 2018-02-14 13:23:48.931154 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=35 2018-02-14 13:23:48.931165 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8913960 0x5587d8ec1b00 auth_reply(proto 2 0 (0) Success) v1 2018-02-14 13:23:48.931200 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 0 35 bytes epoch 1) v1 from client.1684226 10.255.0.51:0/3289155469 2018-02-14 13:23:48.931210 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=35 2018-02-14 13:23:48.931230 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d915e000 0x5587d8ec18c0 auth_reply(proto 2 0 (0) Success) v1 2018-02-14 13:23:48.931263 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 0 35 bytes epoch 1) v1 from client.1684247 10.255.0.105:0/2076150628 2018-02-14 13:23:48.931269 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=35 2018-02-14 13:23:48.931281 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d915e120 0x5587d8ec2ac0 auth_reply(proto 2 0 (0) Success) v1 2018-02-14 13:23:48.931313 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 0 35 bytes epoch 1) v1 from client.1684244 10.255.0.54:0/2718452115 2018-02-14 13:23:48.931318 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=35 2018-02-14 13:23:48.931330 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d915e360 0x5587d8ec1d40 auth_reply(proto 2 0 (0) Success) v1 2018-02-14 13:23:48.931352 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 0 35 bytes epoch 1) v1 from client.1684250 10.255.0.107:0/690639890 2018-02-14 13:23:48.931357 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=35 2018-02-14 13:23:48.931367 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d915e480 0x5587d8ec33c0 auth_reply(proto 2 0 (0) Success) v1 2018-02-14 13:23:48.931466 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 0 35 bytes epoch 1) v1 from client.1684190 10.255.0.108:0/3175087704 2018-02-14 13:23:48.931473 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=35 2018-02-14 13:23:48.931488 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d915f8c0 0x5587d8ec3600 auth_reply(proto 2 0 (0) Success) v1 2018-02-14 13:23:48.931517 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 0 42 bytes epoch 1) v1 from client.1582487 10.255.0.49:0/59005741 2018-02-14 13:23:48.931522 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=42 2018-02-14 13:23:48.931534 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d915f9e0 0x5587d8ec2f40 auth_reply(proto 2 0 (0) Success) v1 2018-02-14 13:23:48.931561 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 0 35 bytes epoch 1) v1 from client.1684199 10.255.0.106:0/1777238177 2018-02-14 13:23:48.931566 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=35 2018-02-14 13:23:48.931577 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d915fc20 0x5587d8ec3cc0 auth_reply(proto 2 0 (0) Success) v1 2018-02-14 13:23:48.931615 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 0 35 bytes epoch 1) v1 from client.1714098 10.255.0.108:0/850049828 2018-02-14 13:23:48.931621 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=35 2018-02-14 13:23:48.931632 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d91614e0 0x5587d8ec3a80 auth_reply(proto 2 0 (0) Success) v1 2018-02-14 13:23:48.931671 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 0 35 bytes epoch 1) v1 from client.1714104 10.255.0.54:0/1233019732 2018-02-14 13:23:48.931681 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=35 2018-02-14 13:23:48.931697 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d9161720 0x5587d88db840 auth_reply(proto 2 0 (0) Success) v1 2018-02-14 13:23:48.931738 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 0 35 bytes epoch 0) v1 from client.? 10.255.0.51:0/3461880882 2018-02-14 13:23:48.931745 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=35 2018-02-14 13:23:48.931750 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 AuthMonitor::assign_global_id m=auth(proto 0 35 bytes epoch 0) v1 mon=0/3 last_allocated=1724096 max_global_id=1724096 2018-02-14 13:23:48.931757 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 next_global_id should be 1724097 2018-02-14 13:23:48.931761 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prepare_update auth(proto 0 35 bytes epoch 0) v1 from client.? 10.255.0.51:0/3461880882 2018-02-14 13:23:48.931765 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=35 2018-02-14 13:23:48.931768 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 AuthMonitor::assign_global_id m=auth(proto 0 35 bytes epoch 0) v1 mon=0/3 last_allocated=1724096 max_global_id=1724096 2018-02-14 13:23:48.931771 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 next_global_id should be 1724097 2018-02-14 13:23:48.931772 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 increasing max_global_id to 1734096 2018-02-14 13:23:48.931774 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 increasing global id, waitlisting message 2018-02-14 13:23:48.931800 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 encode_pending v 2645 2018-02-14 13:23:48.935381 7fc50e501700 10 osdmap epoch 19702 mapping took 0.005217 seconds 2018-02-14 13:23:48.935391 7fc50e501700 10 mon.ceph-dal-mon-a1@0(leader).osd e19702 update_creating_pgs 0 pgs creating, 0 pools in queue 2018-02-14 13:23:48.973776 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 0 35 bytes epoch 0) v1 from client.? 10.255.0.52:0/3152189720 2018-02-14 13:23:48.973788 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=35 2018-02-14 13:23:48.973796 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 AuthMonitor::assign_global_id m=auth(proto 0 35 bytes epoch 0) v1 mon=0/3 last_allocated=1724096 max_global_id=1734096 2018-02-14 13:23:48.973799 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 next_global_id should be 1724097 2018-02-14 13:23:48.973829 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ff87e0 0x5587d8ec2640 auth_reply(proto 2 0 (0) Success) v1 2018-02-14 13:23:48.973859 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 0 35 bytes epoch 0) v1 from client.? 10.255.0.54:0/3194015072 2018-02-14 13:23:48.973864 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=35 2018-02-14 13:23:48.973870 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 AuthMonitor::assign_global_id m=auth(proto 0 35 bytes epoch 0) v1 mon=0/3 last_allocated=1724097 max_global_id=1734096 2018-02-14 13:23:48.973872 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 next_global_id should be 1724100 2018-02-14 13:23:48.973905 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ff8b40 0x5587d8ec2d00 auth_reply(proto 2 0 (0) Success) v1 2018-02-14 13:23:48.973932 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 0 35 bytes epoch 0) v1 from client.? 10.255.0.107:0/980595480 2018-02-14 13:23:48.973937 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=35 2018-02-14 13:23:48.973943 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 AuthMonitor::assign_global_id m=auth(proto 0 35 bytes epoch 0) v1 mon=0/3 last_allocated=1724100 max_global_id=1734096 2018-02-14 13:23:48.973945 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 next_global_id should be 1724103 2018-02-14 13:23:48.973969 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ff8d80 0x5587d8ec1f80 auth_reply(proto 2 0 (0) Success) v1 2018-02-14 13:23:48.973996 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 0 35 bytes epoch 1) v1 from client.1684202 10.255.0.107:0/3082328679 2018-02-14 13:23:48.974001 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=35 2018-02-14 13:23:48.974017 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ff8ea0 0x5587d8fdafc0 auth_reply(proto 2 0 (0) Success) v1 2018-02-14 13:23:48.974045 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 0 39 bytes epoch 0) from client.1393717 10.255.0.52:0/97505378 2018-02-14 13:23:48.974050 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=39 2018-02-14 13:23:48.974074 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ff8fc0 0x5587d8ec21c0 auth_reply(proto 2 0 (0) Success) v1 2018-02-14 13:23:48.974100 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 0 35 bytes epoch 1) v1 from client.1684208 10.255.0.52:0/4056696166 2018-02-14 13:23:48.974106 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=35 2018-02-14 13:23:48.974117 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ff90e0 0x5587d8fda6c0 auth_reply(proto 2 0 (0) Success) v1 2018-02-14 13:23:48.974144 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 preprocess_query log(1 entries from seq 3 at 2018-02-14 13:23:48.865038) v1 from mon.0 10.255.0.30:6789/0 2018-02-14 13:23:48.974153 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 preprocess_log log(1 entries from seq 3 at 2018-02-14 13:23:48.865038) v1 from mon.0 2018-02-14 13:23:48.974157 7fc50dd00700 20 is_capable service=log command= write on cap allow * 2018-02-14 13:23:48.974159 7fc50dd00700 20 allow so far , doing grant allow * 2018-02-14 13:23:48.974160 7fc50dd00700 20 allow all 2018-02-14 13:23:48.974162 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 prepare_update log(1 entries from seq 3 at 2018-02-14 13:23:48.865038) v1 from mon.0 10.255.0.30:6789/0 2018-02-14 13:23:48.974167 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 prepare_log log(1 entries from seq 3 at 2018-02-14 13:23:48.865038) v1 from mon.0 2018-02-14 13:23:48.974170 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 logging 2018-02-14 13:23:48.865038 mon.ceph-dal-mon-a1 mon.0 10.255.0.30:6789/0 3 : cluster [INF] mon.ceph-dal-mon-a1@0 won leader election with quorum 0,2 2018-02-14 13:23:48.974185 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 preprocess_query log(1 entries from seq 25 at 2018-02-14 13:10:18.636291) v1 from mon.2 10.255.0.32:6789/0 2018-02-14 13:23:48.974192 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 preprocess_log log(1 entries from seq 25 at 2018-02-14 13:10:18.636291) v1 from mon.2 2018-02-14 13:23:48.974196 7fc50dd00700 20 is_capable service=log command= write on cap allow * 2018-02-14 13:23:48.974198 7fc50dd00700 20 allow so far , doing grant allow * 2018-02-14 13:23:48.974199 7fc50dd00700 20 allow all 2018-02-14 13:23:48.974200 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 prepare_update log(1 entries from seq 25 at 2018-02-14 13:10:18.636291) v1 from mon.2 10.255.0.32:6789/0 2018-02-14 13:23:48.974205 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 prepare_log log(1 entries from seq 25 at 2018-02-14 13:10:18.636291) v1 from mon.2 2018-02-14 13:23:48.974208 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 logging 2018-02-14 13:10:18.636291 mon.ceph-dal-mon-a3 mon.2 10.255.0.32:6789/0 25 : cluster [INF] mon.ceph-dal-mon-a3 calling new monitor election 2018-02-14 13:23:48.974221 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 preprocess_query log(1 entries from seq 4 at 2018-02-14 13:23:48.907083) v1 from mon.0 10.255.0.30:6789/0 2018-02-14 13:23:48.974231 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 preprocess_log log(1 entries from seq 4 at 2018-02-14 13:23:48.907083) v1 from mon.0 2018-02-14 13:23:48.974239 7fc50dd00700 20 is_capable service=log command= write on cap allow * 2018-02-14 13:23:48.974242 7fc50dd00700 20 allow so far , doing grant allow * 2018-02-14 13:23:48.974243 7fc50dd00700 20 allow all 2018-02-14 13:23:48.974246 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 prepare_update log(1 entries from seq 4 at 2018-02-14 13:23:48.907083) v1 from mon.0 10.255.0.30:6789/0 2018-02-14 13:23:48.974253 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 prepare_log log(1 entries from seq 4 at 2018-02-14 13:23:48.907083) v1 from mon.0 2018-02-14 13:23:48.974256 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 logging 2018-02-14 13:23:48.907083 mon.ceph-dal-mon-a1 mon.0 10.255.0.30:6789/0 4 : cluster [WRN] overall HEALTH_WARN noout flag(s) set; 6 backfillfull osd(s); 4 osds down; 1 host (4 osds) down; 8 nearfull osd(s); 3 pool(s) backfillfull; Degraded data redundancy: 4938712/26491706 objects degraded (18.642%), 347 pgs unclean, 347 pgs degraded, 347 pgs undersized; 1/3 mons down, quorum ceph-dal-mon-a1,ceph-dal-mon-a3 2018-02-14 13:23:48.974290 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891c480 for mon.2 10.255.0.32:6789/0 2018-02-14 13:23:48.974296 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps allow * 2018-02-14 13:23:48.974299 7fc50dd00700 20 is_capable service=mon command= read on cap allow * 2018-02-14 13:23:48.974301 7fc50dd00700 20 allow so far , doing grant allow * 2018-02-14 13:23:48.974302 7fc50dd00700 20 allow all 2018-02-14 13:23:48.974304 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 handle_timecheck time_check( pong e 504 r 1 ts 2018-02-14 13:23:48.928277 ) v1 2018-02-14 13:23:48.974310 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 handle_timecheck_leader time_check( pong e 504 r 1 ts 2018-02-14 13:23:48.928277 ) v1 2018-02-14 13:23:48.974321 7fc50dd00700 0 log_channel(cluster) log [INF] : mon.2 10.255.0.32:6789/0 2018-02-14 13:23:48.974351 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 handle_timecheck_leader from mon.2 10.255.0.32:6789/0 ts 2018-02-14 13:23:48.928277 delta -0.0460362 skew_bound 0 latency 0.0673085 2018-02-14 13:23:48.974365 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 handle_timecheck_leader got pongs from everybody (2 total) 2018-02-14 13:23:48.974369 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 timecheck_finish_round curr 1 2018-02-14 13:23:48.974371 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 timecheck_report 2018-02-14 13:23:48.974374 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 timecheck_report send report to mon.2 10.255.0.32:6789/0 2018-02-14 13:23:48.974392 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 timecheck_check_skews 2018-02-14 13:23:48.974412 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891ca80 for mon.0 10.255.0.30:6789/0 2018-02-14 13:23:48.974417 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps allow * 2018-02-14 13:23:48.974426 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 preprocess_query log(1 entries from seq 5 at 2018-02-14 13:23:48.929977) v1 from mon.0 10.255.0.30:6789/0 2018-02-14 13:23:48.974432 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 preprocess_log log(1 entries from seq 5 at 2018-02-14 13:23:48.929977) v1 from mon.0 2018-02-14 13:23:48.974436 7fc50dd00700 20 is_capable service=log command= write on cap allow * 2018-02-14 13:23:48.974438 7fc50dd00700 20 allow so far , doing grant allow * 2018-02-14 13:23:48.974438 7fc50dd00700 20 allow all 2018-02-14 13:23:48.974440 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 prepare_update log(1 entries from seq 5 at 2018-02-14 13:23:48.929977) v1 from mon.0 10.255.0.30:6789/0 2018-02-14 13:23:48.974445 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 prepare_log log(1 entries from seq 5 at 2018-02-14 13:23:48.929977) v1 from mon.0 2018-02-14 13:23:48.974449 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 logging 2018-02-14 13:23:48.929977 mon.ceph-dal-mon-a1 mon.0 10.255.0.30:6789/0 5 : cluster [INF] monmap e1: 3 mons at {ceph-dal-mon-a1=10.255.0.30:6789/0,ceph-dal-mon-a2=10.255.0.31:6789/0,ceph-dal-mon-a3=10.255.0.32:6789/0} 2018-02-14 13:23:48.974465 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891c480 for mon.2 10.255.0.32:6789/0 2018-02-14 13:23:48.974468 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps allow * 2018-02-14 13:23:48.974470 7fc50dd00700 20 is_capable service=mon command= read on cap allow * 2018-02-14 13:23:48.974471 7fc50dd00700 20 allow so far , doing grant allow * 2018-02-14 13:23:48.974472 7fc50dd00700 20 allow all 2018-02-14 13:23:48.974473 7fc50dd00700 20 is_capable service=mon command= exec on cap allow * 2018-02-14 13:23:48.974474 7fc50dd00700 20 allow so far , doing grant allow * 2018-02-14 13:23:48.974475 7fc50dd00700 20 allow all 2018-02-14 13:23:48.974497 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891ca80 for mon.0 10.255.0.30:6789/0 2018-02-14 13:23:48.974501 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps allow * 2018-02-14 13:23:48.974508 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 preprocess_query log(1 entries from seq 6 at 2018-02-14 13:23:48.930042) v1 from mon.0 10.255.0.30:6789/0 2018-02-14 13:23:48.974513 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 preprocess_log log(1 entries from seq 6 at 2018-02-14 13:23:48.930042) v1 from mon.0 2018-02-14 13:23:48.974516 7fc50dd00700 20 is_capable service=log command= write on cap allow * 2018-02-14 13:23:48.974518 7fc50dd00700 20 allow so far , doing grant allow * 2018-02-14 13:23:48.974519 7fc50dd00700 20 allow all 2018-02-14 13:23:48.974520 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 prepare_update log(1 entries from seq 6 at 2018-02-14 13:23:48.930042) v1 from mon.0 10.255.0.30:6789/0 2018-02-14 13:23:48.974525 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 prepare_log log(1 entries from seq 6 at 2018-02-14 13:23:48.930042) v1 from mon.0 2018-02-14 13:23:48.974528 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 logging 2018-02-14 13:23:48.930042 mon.ceph-dal-mon-a1 mon.0 10.255.0.30:6789/0 6 : cluster [DBG] fsmap 2018-02-14 13:23:48.974546 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891c480 for mon.2 10.255.0.32:6789/0 2018-02-14 13:23:48.974549 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps allow * 2018-02-14 13:23:48.974551 7fc50dd00700 20 is_capable service=mon command= read on cap allow * 2018-02-14 13:23:48.974553 7fc50dd00700 20 allow so far , doing grant allow * 2018-02-14 13:23:48.974554 7fc50dd00700 20 allow all 2018-02-14 13:23:48.974555 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 received forwarded message from mon.2 10.255.0.32:6789/0 via mon.2 10.255.0.32:6789/0 2018-02-14 13:23:48.974559 7fc50dd00700 20 is_capable service=mon command= exec on cap allow * 2018-02-14 13:23:48.974560 7fc50dd00700 20 allow so far , doing grant allow * 2018-02-14 13:23:48.974561 7fc50dd00700 20 allow all 2018-02-14 13:23:48.974566 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 caps are allow * 2018-02-14 13:23:48.974567 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 entity name 'mon.' type 1 2018-02-14 13:23:48.974569 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 mesg 0x5587d8fdab40 from 10.255.0.32:6789/0 2018-02-14 13:23:48.974572 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d9149680 for mon.2 - 2018-02-14 13:23:48.974574 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps allow * 2018-02-14 13:23:48.974590 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 preprocess_query log(1 entries from seq 26 at 2018-02-14 13:10:39.130901) v1 from mon.2 10.255.0.32:6789/0 2018-02-14 13:23:48.974596 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 preprocess_log log(1 entries from seq 26 at 2018-02-14 13:10:39.130901) v1 from mon.2 2018-02-14 13:23:48.974599 7fc50dd00700 20 is_capable service=log command= write on cap allow * 2018-02-14 13:23:48.974600 7fc50dd00700 20 allow so far , doing grant allow * 2018-02-14 13:23:48.974601 7fc50dd00700 20 allow all 2018-02-14 13:23:48.974602 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 prepare_update log(1 entries from seq 26 at 2018-02-14 13:10:39.130901) v1 from mon.2 10.255.0.32:6789/0 2018-02-14 13:23:48.974607 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 prepare_log log(1 entries from seq 26 at 2018-02-14 13:10:39.130901) v1 from mon.2 2018-02-14 13:23:48.974610 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 logging 2018-02-14 13:10:39.130901 mon.ceph-dal-mon-a3 mon.2 10.255.0.32:6789/0 26 : cluster [INF] mon.ceph-dal-mon-a3 calling new monitor election 2018-02-14 13:23:48.974637 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891ca80 for mon.0 10.255.0.30:6789/0 2018-02-14 13:23:48.974655 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps allow * 2018-02-14 13:23:48.974674 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 preprocess_query log(1 entries from seq 7 at 2018-02-14 13:23:48.930138) v1 from mon.0 10.255.0.30:6789/0 2018-02-14 13:23:48.974691 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 preprocess_log log(1 entries from seq 7 at 2018-02-14 13:23:48.930138) v1 from mon.0 2018-02-14 13:23:48.974700 7fc50dd00700 20 is_capable service=log command= write on cap allow * 2018-02-14 13:23:48.974703 7fc50dd00700 20 allow so far , doing grant allow * 2018-02-14 13:23:48.974704 7fc50dd00700 20 allow all 2018-02-14 13:23:48.974707 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 prepare_update log(1 entries from seq 7 at 2018-02-14 13:23:48.930138) v1 from mon.0 10.255.0.30:6789/0 2018-02-14 13:23:48.974714 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 prepare_log log(1 entries from seq 7 at 2018-02-14 13:23:48.930138) v1 from mon.0 2018-02-14 13:23:48.974717 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 logging 2018-02-14 13:23:48.930138 mon.ceph-dal-mon-a1 mon.0 10.255.0.30:6789/0 7 : cluster [DBG] osdmap e19702: 32 total, 28 up, 32 in 2018-02-14 13:23:48.974735 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891ca80 for mon.0 10.255.0.30:6789/0 2018-02-14 13:23:48.974740 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps allow * 2018-02-14 13:23:48.974748 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 preprocess_query log(1 entries from seq 8 at 2018-02-14 13:23:48.930230) v1 from mon.0 10.255.0.30:6789/0 2018-02-14 13:23:48.974754 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 preprocess_log log(1 entries from seq 8 at 2018-02-14 13:23:48.930230) v1 from mon.0 2018-02-14 13:23:48.974758 7fc50dd00700 20 is_capable service=log command= write on cap allow * 2018-02-14 13:23:48.974759 7fc50dd00700 20 allow so far , doing grant allow * 2018-02-14 13:23:48.974760 7fc50dd00700 20 allow all 2018-02-14 13:23:48.974762 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 prepare_update log(1 entries from seq 8 at 2018-02-14 13:23:48.930230) v1 from mon.0 10.255.0.30:6789/0 2018-02-14 13:23:48.974767 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 prepare_log log(1 entries from seq 8 at 2018-02-14 13:23:48.930230) v1 from mon.0 2018-02-14 13:23:48.974770 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 logging 2018-02-14 13:23:48.930230 mon.ceph-dal-mon-a1 mon.0 10.255.0.30:6789/0 8 : cluster [DBG] mgrmap e422: ceph-dal-admin-a1(active, starting), standbys: ceph-dal-mon-a1 2018-02-14 13:23:48.974801 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891ca80 for mon.0 10.255.0.30:6789/0 2018-02-14 13:23:48.974807 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps allow * 2018-02-14 13:23:48.974822 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 preprocess_query log(1 entries from seq 9 at 2018-02-14 13:23:48.974323) v1 from mon.0 10.255.0.30:6789/0 2018-02-14 13:23:48.974830 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 preprocess_log log(1 entries from seq 9 at 2018-02-14 13:23:48.974323) v1 from mon.0 2018-02-14 13:23:48.974834 7fc50dd00700 20 is_capable service=log command= write on cap allow * 2018-02-14 13:23:48.974836 7fc50dd00700 20 allow so far , doing grant allow * 2018-02-14 13:23:48.974836 7fc50dd00700 20 allow all 2018-02-14 13:23:48.974838 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 prepare_update log(1 entries from seq 9 at 2018-02-14 13:23:48.974323) v1 from mon.0 10.255.0.30:6789/0 2018-02-14 13:23:48.974843 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 prepare_log log(1 entries from seq 9 at 2018-02-14 13:23:48.974323) v1 from mon.0 2018-02-14 13:23:48.974846 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 logging 2018-02-14 13:23:48.974323 mon.ceph-dal-mon-a1 mon.0 10.255.0.30:6789/0 9 : cluster [INF] mon.2 10.255.0.32:6789/0 2018-02-14 13:23:48.974862 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891d980 for client.1684241 10.255.0.30:0/3376128839 2018-02-14 13:23:48.974867 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps 2018-02-14 13:23:48.974873 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 32 bytes epoch 0) v1 from client.1684241 10.255.0.30:0/3376128839 2018-02-14 13:23:48.974877 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=32 2018-02-14 13:23:48.975015 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d90757a0 0x5587d8fdcf40 auth_reply(proto 2 0 (0) Success) v1 2018-02-14 13:23:48.975058 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891dc80 for client.1684232 10.255.0.105:0/3391979714 2018-02-14 13:23:48.975063 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps 2018-02-14 13:23:48.975071 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 32 bytes epoch 0) v1 from client.1684232 10.255.0.105:0/3391979714 2018-02-14 13:23:48.975074 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=32 2018-02-14 13:23:48.975150 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d90757a0 0x5587d88db600 auth_reply(proto 2 0 (0) Success) v1 2018-02-14 13:23:48.975189 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891df80 for client.1684235 10.255.0.53:0/287677392 2018-02-14 13:23:48.975195 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps 2018-02-14 13:23:48.975203 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 32 bytes epoch 0) v1 from client.1684235 10.255.0.53:0/287677392 2018-02-14 13:23:48.975206 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=32 2018-02-14 13:23:48.975278 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d90757a0 0x5587d8fdd3c0 auth_reply(proto 2 0 (0) Success) v1 2018-02-14 13:23:48.975326 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891e280 for client.1684193 10.255.0.105:0/1988410988 2018-02-14 13:23:48.975331 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps 2018-02-14 13:23:48.975339 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 32 bytes epoch 0) v1 from client.1684193 10.255.0.105:0/1988410988 2018-02-14 13:23:48.975343 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=32 2018-02-14 13:23:48.975414 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d90757a0 0x5587d8ec2ac0 auth_reply(proto 2 0 (0) Success) v1 2018-02-14 13:23:48.975453 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891e580 for client.1621590 10.255.0.106:0/1162398434 2018-02-14 13:23:48.975458 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps 2018-02-14 13:23:48.975468 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 32 bytes epoch 0) v1 from client.1621590 10.255.0.106:0/1162398434 2018-02-14 13:23:48.975472 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=32 2018-02-14 13:23:48.975540 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d90757a0 0x5587d8ec1b00 auth_reply(proto 2 0 (0) Success) v1 2018-02-14 13:23:48.975578 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891f180 for client.1684238 10.255.0.54:0/2824825993 2018-02-14 13:23:48.975583 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps 2018-02-14 13:23:48.975592 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 32 bytes epoch 0) v1 from client.1684238 10.255.0.54:0/2824825993 2018-02-14 13:23:48.975597 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=32 2018-02-14 13:23:48.975665 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d90757a0 0x5587d8ec18c0 auth_reply(proto 2 0 (0) Success) v1 2018-02-14 13:23:48.975704 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891f480 for client.1684211 10.255.0.53:0/2869630763 2018-02-14 13:23:48.975710 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps 2018-02-14 13:23:48.975718 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 32 bytes epoch 0) v1 from client.1684211 10.255.0.53:0/2869630763 2018-02-14 13:23:48.975723 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=32 2018-02-14 13:23:48.975800 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d90757a0 0x5587d8ec0d80 auth_reply(proto 2 0 (0) Success) v1 2018-02-14 13:23:48.975841 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891f780 for client.263543 10.255.0.107:0/322403586 2018-02-14 13:23:48.975846 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps 2018-02-14 13:23:48.975854 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 144 bytes epoch 0) from client.263543 10.255.0.107:0/322403586 2018-02-14 13:23:48.975857 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=144 2018-02-14 13:23:48.975953 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d90757a0 0x5587d8ec1d40 auth_reply(proto 2 0 (0) Success) v1 2018-02-14 13:23:48.975992 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891fa80 for client.1684265 10.255.0.53:0/3548294640 2018-02-14 13:23:48.975998 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps 2018-02-14 13:23:48.976006 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 32 bytes epoch 0) v1 from client.1684265 10.255.0.53:0/3548294640 2018-02-14 13:23:48.976010 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=32 2018-02-14 13:23:48.976093 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ff9680 0x5587d8ec0fc0 auth_reply(proto 2 0 (0) Success) v1 2018-02-14 13:23:48.976136 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891fd80 for client.1684217 10.255.0.107:0/2591354583 2018-02-14 13:23:48.976141 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps 2018-02-14 13:23:48.976150 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 32 bytes epoch 0) v1 from client.1684217 10.255.0.107:0/2591354583 2018-02-14 13:23:48.976155 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=32 2018-02-14 13:23:48.976223 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ff9680 0x5587d8ec3cc0 auth_reply(proto 2 0 (0) Success) v1 2018-02-14 13:23:48.976273 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d9148300 for client.1684226 10.255.0.51:0/3289155469 2018-02-14 13:23:48.976278 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps 2018-02-14 13:23:48.976286 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 32 bytes epoch 0) v1 from client.1684226 10.255.0.51:0/3289155469 2018-02-14 13:23:48.976290 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=32 2018-02-14 13:23:48.976358 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ff9680 0x5587d8ec0480 auth_reply(proto 2 0 (0) Success) v1 2018-02-14 13:23:48.976398 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d9148480 for client.1684247 10.255.0.105:0/2076150628 2018-02-14 13:23:48.976403 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps 2018-02-14 13:23:48.976412 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 32 bytes epoch 0) v1 from client.1684247 10.255.0.105:0/2076150628 2018-02-14 13:23:48.976417 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=32 2018-02-14 13:23:48.976500 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ff9680 0x5587d8ec3a80 auth_reply(proto 2 0 (0) Success) v1 2018-02-14 13:23:48.976542 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d9148600 for client.1684244 10.255.0.54:0/2718452115 2018-02-14 13:23:48.976547 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps 2018-02-14 13:23:48.976554 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 32 bytes epoch 0) v1 from client.1684244 10.255.0.54:0/2718452115 2018-02-14 13:23:48.976559 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=32 2018-02-14 13:23:48.976628 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ff9680 0x5587d8ec2f40 auth_reply(proto 2 0 (0) Success) v1 2018-02-14 13:23:48.976668 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d9148780 for client.1684250 10.255.0.107:0/690639890 2018-02-14 13:23:48.976674 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps 2018-02-14 13:23:48.976681 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 32 bytes epoch 0) v1 from client.1684250 10.255.0.107:0/690639890 2018-02-14 13:23:48.976685 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=32 2018-02-14 13:23:48.976761 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ff9680 0x5587d88da880 auth_reply(proto 2 0 (0) Success) v1 2018-02-14 13:23:48.976809 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d9148f00 for client.1684190 10.255.0.108:0/3175087704 2018-02-14 13:23:48.976816 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps 2018-02-14 13:23:48.976828 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 32 bytes epoch 0) v1 from client.1684190 10.255.0.108:0/3175087704 2018-02-14 13:23:48.976833 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=32 2018-02-14 13:23:48.976908 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ff9680 0x5587d8ec3600 auth_reply(proto 2 0 (0) Success) v1 2018-02-14 13:23:48.976951 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d9149080 for client.1582487 10.255.0.49:0/59005741 2018-02-14 13:23:48.976956 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps 2018-02-14 13:23:48.976966 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 32 bytes epoch 0) v1 from client.1582487 10.255.0.49:0/59005741 2018-02-14 13:23:48.976971 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=32 2018-02-14 13:23:48.977089 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ff9680 0x5587d8ec33c0 auth_reply(proto 2 0 (0) Success) v1 2018-02-14 13:23:48.977142 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d9149200 for client.1684199 10.255.0.106:0/1777238177 2018-02-14 13:23:48.977152 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps 2018-02-14 13:23:48.977165 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 32 bytes epoch 0) v1 from client.1684199 10.255.0.106:0/1777238177 2018-02-14 13:23:48.977171 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=32 2018-02-14 13:23:48.977251 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ff9680 0x5587d88db840 auth_reply(proto 2 0 (0) Success) v1 2018-02-14 13:23:48.977306 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d9149800 for client.1714098 10.255.0.108:0/850049828 2018-02-14 13:23:48.977316 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps 2018-02-14 13:23:48.977326 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 32 bytes epoch 0) v1 from client.1714098 10.255.0.108:0/850049828 2018-02-14 13:23:48.977330 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=32 2018-02-14 13:23:48.977412 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ff9680 0x5587d8ec1200 auth_reply(proto 2 0 (0) Success) v1 2018-02-14 13:23:48.977464 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d9149980 for client.1714104 10.255.0.54:0/1233019732 2018-02-14 13:23:48.977473 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps 2018-02-14 13:23:48.977486 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 32 bytes epoch 0) v1 from client.1714104 10.255.0.54:0/1233019732 2018-02-14 13:23:48.977492 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=32 2018-02-14 13:23:48.977570 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ff9680 0x5587d8ec1440 auth_reply(proto 2 0 (0) Success) v1 2018-02-14 13:23:48.977624 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d9149c80 for client.? 10.255.0.52:0/3152189720 2018-02-14 13:23:48.977633 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps 2018-02-14 13:23:48.977648 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 32 bytes epoch 0) v1 from client.? 10.255.0.52:0/3152189720 2018-02-14 13:23:48.977654 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=32 2018-02-14 13:23:48.977735 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ff9680 0x5587d88db180 auth_reply(proto 2 0 (0) Success) v1 2018-02-14 13:23:48.977784 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d9149e00 for client.? 10.255.0.54:0/3194015072 2018-02-14 13:23:48.977793 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps 2018-02-14 13:23:48.977805 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 32 bytes epoch 0) v1 from client.? 10.255.0.54:0/3194015072 2018-02-14 13:23:48.977816 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=32 2018-02-14 13:23:48.977888 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ff9680 0x5587d8fdb440 auth_reply(proto 2 0 (0) Success) v1 2018-02-14 13:23:48.977932 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d9149f80 for client.? 10.255.0.107:0/980595480 2018-02-14 13:23:48.977937 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps 2018-02-14 13:23:48.977945 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 32 bytes epoch 0) v1 from client.? 10.255.0.107:0/980595480 2018-02-14 13:23:48.977950 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=32 2018-02-14 13:23:48.978022 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ff9680 0x5587d88db600 auth_reply(proto 2 0 (0) Success) v1 2018-02-14 13:23:48.978064 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d914a100 for client.1684202 10.255.0.107:0/3082328679 2018-02-14 13:23:48.978069 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps 2018-02-14 13:23:48.978078 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 32 bytes epoch 0) v1 from client.1684202 10.255.0.107:0/3082328679 2018-02-14 13:23:48.978083 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=32 2018-02-14 13:23:48.978152 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ff9680 0x5587d8fda240 auth_reply(proto 2 0 (0) Success) v1 2018-02-14 13:23:48.978193 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d914a280 for client.1393717 10.255.0.52:0/97505378 2018-02-14 13:23:48.978199 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps 2018-02-14 13:23:48.978217 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 144 bytes epoch 0) from client.1393717 10.255.0.52:0/97505378 2018-02-14 13:23:48.978223 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=144 2018-02-14 13:23:48.978317 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ff9680 0x5587d8fdafc0 auth_reply(proto 2 0 (0) Success) v1 2018-02-14 13:23:48.978361 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d914a400 for client.1684208 10.255.0.52:0/4056696166 2018-02-14 13:23:48.978366 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps 2018-02-14 13:23:48.978374 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 32 bytes epoch 0) v1 from client.1684208 10.255.0.52:0/4056696166 2018-02-14 13:23:48.978379 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=32 2018-02-14 13:23:48.978453 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ffa400 0x5587d8ec21c0 auth_reply(proto 2 0 (0) Success) v1 2018-02-14 13:23:48.978496 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891cc00 for client.1684214 10.255.0.105:0/1753894484 2018-02-14 13:23:48.978502 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps 2018-02-14 13:23:48.978510 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 32 bytes epoch 0) v1 from client.1684214 10.255.0.105:0/1753894484 2018-02-14 13:23:48.978515 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=32 2018-02-14 13:23:48.978585 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ffa400 0x5587d8fdb680 auth_reply(proto 2 0 (0) Success) v1 2018-02-14 13:23:48.978633 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891cd80 for client.1684262 10.255.0.106:0/3270640683 2018-02-14 13:23:48.978638 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps 2018-02-14 13:23:48.978646 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 32 bytes epoch 0) v1 from client.1684262 10.255.0.106:0/3270640683 2018-02-14 13:23:48.978650 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=32 2018-02-14 13:23:48.978718 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ffa400 0x5587d8fdcac0 auth_reply(proto 2 0 (0) Success) v1 2018-02-14 13:23:48.978759 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891d080 for client.1684205 10.255.0.51:0/3360953369 2018-02-14 13:23:48.978765 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps 2018-02-14 13:23:48.978773 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 32 bytes epoch 0) v1 from client.1684205 10.255.0.51:0/3360953369 2018-02-14 13:23:48.978787 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=32 2018-02-14 13:23:48.978872 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ffa400 0x5587d88dba80 auth_reply(proto 2 0 (0) Success) v1 2018-02-14 13:23:48.978937 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891d980 for client.1684241 10.255.0.30:0/3376128839 2018-02-14 13:23:48.978947 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps allow profile mgr 2018-02-14 13:23:48.978960 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 181 bytes epoch 0) v1 from client.1684241 10.255.0.30:0/3376128839 2018-02-14 13:23:48.978966 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=181 2018-02-14 13:23:48.979126 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ffa400 0x5587d88db3c0 auth_reply(proto 2 0 (0) Success) v1 2018-02-14 13:23:48.979177 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891dc80 for client.1684232 10.255.0.105:0/3391979714 2018-02-14 13:23:48.979187 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd 2018-02-14 13:23:48.979203 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 181 bytes epoch 0) v1 from client.1684232 10.255.0.105:0/3391979714 2018-02-14 13:23:48.979211 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=181 2018-02-14 13:23:48.979340 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ffa400 0x5587d8fdcf40 auth_reply(proto 2 0 (0) Success) v1 2018-02-14 13:23:48.979392 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891df80 for client.1684235 10.255.0.53:0/287677392 2018-02-14 13:23:48.979405 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd 2018-02-14 13:23:48.979418 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 181 bytes epoch 0) v1 from client.1684235 10.255.0.53:0/287677392 2018-02-14 13:23:48.979423 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=181 2018-02-14 13:23:48.979626 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ffa400 0x5587d8ec1b00 auth_reply(proto 2 0 (0) Success) v1 2018-02-14 13:23:48.979688 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891e280 for client.1684193 10.255.0.105:0/1988410988 2018-02-14 13:23:48.979693 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd 2018-02-14 13:23:48.979702 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 181 bytes epoch 0) v1 from client.1684193 10.255.0.105:0/1988410988 2018-02-14 13:23:48.979707 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=181 2018-02-14 13:23:48.979838 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ffa400 0x5587d8ec18c0 auth_reply(proto 2 0 (0) Success) v1 2018-02-14 13:23:48.979891 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891e580 for client.1621590 10.255.0.106:0/1162398434 2018-02-14 13:23:48.979901 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd 2018-02-14 13:23:48.979915 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 181 bytes epoch 0) v1 from client.1621590 10.255.0.106:0/1162398434 2018-02-14 13:23:48.979929 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=181 2018-02-14 13:23:48.980046 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ffa400 0x5587d8ec0d80 auth_reply(proto 2 0 (0) Success) v1 2018-02-14 13:23:48.980089 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891f180 for client.1684238 10.255.0.54:0/2824825993 2018-02-14 13:23:48.980094 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd 2018-02-14 13:23:48.980113 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 181 bytes epoch 0) v1 from client.1684238 10.255.0.54:0/2824825993 2018-02-14 13:23:48.980119 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=181 2018-02-14 13:23:48.980231 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ffa400 0x5587d8ec2d00 auth_reply(proto 2 0 (0) Success) v1 2018-02-14 13:23:48.980271 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891f480 for client.1684211 10.255.0.53:0/2869630763 2018-02-14 13:23:48.980276 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd 2018-02-14 13:23:48.980284 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 181 bytes epoch 0) v1 from client.1684211 10.255.0.53:0/2869630763 2018-02-14 13:23:48.980289 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=181 2018-02-14 13:23:48.980406 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ffa400 0x5587d8ec0fc0 auth_reply(proto 2 0 (0) Success) v1 2018-02-14 13:23:48.980443 7fc510505700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 encode_full log v 594874 2018-02-14 13:23:48.980546 7fc510505700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 encode_pending v594875 2018-02-14 13:23:48.980599 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891f780 for client.263543 10.255.0.107:0/322403586 2018-02-14 13:23:48.980606 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd 2018-02-14 13:23:48.980609 7fc50dd00700 20 is_capable service=mon command= read on cap allow profile rbd 2018-02-14 13:23:48.980615 7fc50dd00700 20 allow so far , doing grant allow profile rbd 2018-02-14 13:23:48.980629 7fc50dd00700 20 match 2018-02-14 13:23:48.980632 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 handle_subscribe mon_subscribe({monmap=2+,osdmap=19703}) v2 2018-02-14 13:23:48.980642 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).monmap v1 check_sub monmap next 2 have 1 2018-02-14 13:23:48.980647 7fc50dd00700 20 is_capable service=osd command= read on cap allow profile rbd 2018-02-14 13:23:48.980649 7fc50dd00700 20 allow so far , doing grant allow profile rbd 2018-02-14 13:23:48.980651 7fc50dd00700 20 match 2018-02-14 13:23:48.980652 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).osd e19702 check_osdmap_sub 0x5587d8e97200 next 19703 (onetime) 2018-02-14 13:23:48.980673 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891fa80 for client.1684265 10.255.0.53:0/3548294640 2018-02-14 13:23:48.980678 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd 2018-02-14 13:23:48.980686 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 181 bytes epoch 0) v1 from client.1684265 10.255.0.53:0/3548294640 2018-02-14 13:23:48.980691 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=181 2018-02-14 13:23:48.980823 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ffa400 0x5587d8ec3cc0 auth_reply(proto 2 0 (0) Success) v1 2018-02-14 13:23:48.980862 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891fd80 for client.1684217 10.255.0.107:0/2591354583 2018-02-14 13:23:48.980868 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd 2018-02-14 13:23:48.980875 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 181 bytes epoch 0) v1 from client.1684217 10.255.0.107:0/2591354583 2018-02-14 13:23:48.980879 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=181 2018-02-14 13:23:48.981004 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ffa400 0x5587d8fdd3c0 auth_reply(proto 2 0 (0) Success) v1 2018-02-14 13:23:48.981047 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d9148300 for client.1684226 10.255.0.51:0/3289155469 2018-02-14 13:23:48.981052 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd 2018-02-14 13:23:48.981059 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 181 bytes epoch 0) v1 from client.1684226 10.255.0.51:0/3289155469 2018-02-14 13:23:48.981063 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=181 2018-02-14 13:23:48.981174 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ffa400 0x5587d8ec2f40 auth_reply(proto 2 0 (0) Success) v1 2018-02-14 13:23:48.981212 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d9148480 for client.1684247 10.255.0.105:0/2076150628 2018-02-14 13:23:48.981218 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd 2018-02-14 13:23:48.981226 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 181 bytes epoch 0) v1 from client.1684247 10.255.0.105:0/2076150628 2018-02-14 13:23:48.981231 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=181 2018-02-14 13:23:48.981343 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ffa400 0x5587d88da880 auth_reply(proto 2 0 (0) Success) v1 2018-02-14 13:23:48.981384 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d9148600 for client.1684244 10.255.0.54:0/2718452115 2018-02-14 13:23:48.981389 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd 2018-02-14 13:23:48.981397 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 181 bytes epoch 0) v1 from client.1684244 10.255.0.54:0/2718452115 2018-02-14 13:23:48.981403 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=181 2018-02-14 13:23:48.981531 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ffa400 0x5587d8ec3a80 auth_reply(proto 2 0 (0) Success) v1 2018-02-14 13:23:48.981573 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d9148780 for client.1684250 10.255.0.107:0/690639890 2018-02-14 13:23:48.981578 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd 2018-02-14 13:23:48.981587 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 181 bytes epoch 0) v1 from client.1684250 10.255.0.107:0/690639890 2018-02-14 13:23:48.981590 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=181 2018-02-14 13:23:48.981717 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ffa400 0x5587d8ec2ac0 auth_reply(proto 2 0 (0) Success) v1 2018-02-14 13:23:48.981756 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d9148f00 for client.1684190 10.255.0.108:0/3175087704 2018-02-14 13:23:48.981761 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd 2018-02-14 13:23:48.981770 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 181 bytes epoch 0) v1 from client.1684190 10.255.0.108:0/3175087704 2018-02-14 13:23:48.981776 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=181 2018-02-14 13:23:48.981949 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ffa400 0x5587d8ec3600 auth_reply(proto 2 0 (0) Success) v1 2018-02-14 13:23:48.981991 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d9149080 for client.1582487 10.255.0.49:0/59005741 2018-02-14 13:23:48.981997 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps allow profile mgr 2018-02-14 13:23:48.982005 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 181 bytes epoch 0) v1 from client.1582487 10.255.0.49:0/59005741 2018-02-14 13:23:48.982011 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=181 2018-02-14 13:23:48.982164 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ffa400 0x5587d8ec0480 auth_reply(proto 2 0 (0) Success) v1 2018-02-14 13:23:48.982215 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d9149200 for client.1684199 10.255.0.106:0/1777238177 2018-02-14 13:23:48.982223 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd 2018-02-14 13:23:48.982238 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 181 bytes epoch 0) v1 from client.1684199 10.255.0.106:0/1777238177 2018-02-14 13:23:48.982248 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=181 2018-02-14 13:23:48.982369 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ffa400 0x5587d88db840 auth_reply(proto 2 0 (0) Success) v1 2018-02-14 13:23:48.982417 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d9149800 for client.1714098 10.255.0.108:0/850049828 2018-02-14 13:23:48.982426 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd 2018-02-14 13:23:48.982440 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 181 bytes epoch 0) v1 from client.1714098 10.255.0.108:0/850049828 2018-02-14 13:23:48.982447 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=181 2018-02-14 13:23:48.982562 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ffa400 0x5587d8ec1440 auth_reply(proto 2 0 (0) Success) v1 2018-02-14 13:23:48.982604 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d9149980 for client.1714104 10.255.0.54:0/1233019732 2018-02-14 13:23:48.982609 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd 2018-02-14 13:23:48.982617 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 181 bytes epoch 0) v1 from client.1714104 10.255.0.54:0/1233019732 2018-02-14 13:23:48.982622 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=181 2018-02-14 13:23:48.982836 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ffa400 0x5587d8ec1200 auth_reply(proto 2 0 (0) Success) v1 2018-02-14 13:23:48.982896 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d9149c80 for client.? 10.255.0.52:0/3152189720 2018-02-14 13:23:48.982904 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd 2018-02-14 13:23:48.982916 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 181 bytes epoch 0) v1 from client.? 10.255.0.52:0/3152189720 2018-02-14 13:23:48.982925 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=181 2018-02-14 13:23:48.983136 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ffa400 0x5587d8fdafc0 auth_reply(proto 2 0 (0) Success) v1 2018-02-14 13:23:48.983198 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d9149e00 for client.? 10.255.0.54:0/3194015072 2018-02-14 13:23:48.983206 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd 2018-02-14 13:23:48.983221 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 181 bytes epoch 0) v1 from client.? 10.255.0.54:0/3194015072 2018-02-14 13:23:48.983229 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=181 2018-02-14 13:23:48.983445 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ffa400 0x5587d88db600 auth_reply(proto 2 0 (0) Success) v1 2018-02-14 13:23:48.983507 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d9149f80 for client.? 10.255.0.107:0/980595480 2018-02-14 13:23:48.983516 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd 2018-02-14 13:23:48.983530 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 181 bytes epoch 0) v1 from client.? 10.255.0.107:0/980595480 2018-02-14 13:23:48.983541 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=181 2018-02-14 13:23:48.983746 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ffa400 0x5587d8fdb440 auth_reply(proto 2 0 (0) Success) v1 2018-02-14 13:23:48.983821 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d914a100 for client.1684202 10.255.0.107:0/3082328679 2018-02-14 13:23:48.983830 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd 2018-02-14 13:23:48.983846 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 181 bytes epoch 0) v1 from client.1684202 10.255.0.107:0/3082328679 2018-02-14 13:23:48.983855 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=181 2018-02-14 13:23:48.984053 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ffa400 0x5587d88db180 auth_reply(proto 2 0 (0) Success) v1 2018-02-14 13:23:48.984116 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d914a280 for client.1393717 10.255.0.52:0/97505378 2018-02-14 13:23:48.984124 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd 2018-02-14 13:23:48.984128 7fc50dd00700 20 is_capable service=mon command= read on cap allow profile rbd 2018-02-14 13:23:48.984131 7fc50dd00700 20 allow so far , doing grant allow profile rbd 2018-02-14 13:23:48.984138 7fc50dd00700 20 match 2018-02-14 13:23:48.984141 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 handle_subscribe mon_subscribe({monmap=2+,osdmap=19703}) v2 2018-02-14 13:23:48.984148 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).monmap v1 check_sub monmap next 2 have 1 2018-02-14 13:23:48.984157 7fc50dd00700 20 is_capable service=osd command= read on cap allow profile rbd 2018-02-14 13:23:48.984159 7fc50dd00700 20 allow so far , doing grant allow profile rbd 2018-02-14 13:23:48.984162 7fc50dd00700 20 match 2018-02-14 13:23:48.984164 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).osd e19702 check_osdmap_sub 0x5587d8e97780 next 19703 (onetime) 2018-02-14 13:23:48.984196 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d914a400 for client.1684208 10.255.0.52:0/4056696166 2018-02-14 13:23:48.984205 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd 2018-02-14 13:23:48.984217 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 181 bytes epoch 0) v1 from client.1684208 10.255.0.52:0/4056696166 2018-02-14 13:23:48.984223 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=181 2018-02-14 13:23:48.984430 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ffa400 0x5587d8fda240 auth_reply(proto 2 0 (0) Success) v1 2018-02-14 13:23:48.984490 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891cc00 for client.1684214 10.255.0.105:0/1753894484 2018-02-14 13:23:48.984498 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd 2018-02-14 13:23:48.984509 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 181 bytes epoch 0) v1 from client.1684214 10.255.0.105:0/1753894484 2018-02-14 13:23:48.984519 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=181 2018-02-14 13:23:48.984716 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ffa400 0x5587d8ec21c0 auth_reply(proto 2 0 (0) Success) v1 2018-02-14 13:23:48.984777 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891cd80 for client.1684262 10.255.0.106:0/3270640683 2018-02-14 13:23:48.984792 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd 2018-02-14 13:23:48.984806 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 181 bytes epoch 0) v1 from client.1684262 10.255.0.106:0/3270640683 2018-02-14 13:23:48.984814 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=181 2018-02-14 13:23:48.985077 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ffa400 0x5587d8fdb680 auth_reply(proto 2 0 (0) Success) v1 2018-02-14 13:23:48.985142 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891d080 for client.1684205 10.255.0.51:0/3360953369 2018-02-14 13:23:48.985152 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd 2018-02-14 13:23:48.985167 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 181 bytes epoch 0) v1 from client.1684205 10.255.0.51:0/3360953369 2018-02-14 13:23:48.985174 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=181 2018-02-14 13:23:48.985378 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ffa400 0x5587d8fdcac0 auth_reply(proto 2 0 (0) Success) v1 2018-02-14 13:23:48.985437 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891d980 for client.1684241 10.255.0.30:0/3376128839 2018-02-14 13:23:48.985446 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps allow profile mgr 2018-02-14 13:23:48.985450 7fc50dd00700 20 is_capable service=mon command= read on cap allow profile mgr 2018-02-14 13:23:48.985453 7fc50dd00700 20 allow so far , doing grant allow profile mgr 2018-02-14 13:23:48.985486 7fc50dd00700 20 match 2018-02-14 13:23:48.985492 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 handle_subscribe mon_subscribe({mgrmap=0+,monmap=2+}) v2 2018-02-14 13:23:48.985500 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader).mgr e422 Sending map to subscriber 0x5587d8eae800 10.255.0.30:0/3376128839 2018-02-14 13:23:48.985530 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).monmap v1 check_sub monmap next 2 have 1 2018-02-14 13:23:48.985562 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891dc80 for client.1684232 10.255.0.105:0/3391979714 2018-02-14 13:23:48.985570 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd 2018-02-14 13:23:48.985575 7fc50dd00700 20 is_capable service=mon command= read on cap allow profile rbd 2018-02-14 13:23:48.985577 7fc50dd00700 20 allow so far , doing grant allow profile rbd 2018-02-14 13:23:48.985584 7fc50dd00700 20 match 2018-02-14 13:23:48.985586 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 handle_subscribe mon_subscribe({mgrmap=0+,monmap=2+,osdmap=19703}) v2 2018-02-14 13:23:48.985593 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader).mgr e422 Sending map to subscriber 0x5587d8f94000 10.255.0.105:0/3391979714 2018-02-14 13:23:48.985616 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).monmap v1 check_sub monmap next 2 have 1 2018-02-14 13:23:48.985625 7fc50dd00700 20 is_capable service=osd command= read on cap allow profile rbd 2018-02-14 13:23:48.985628 7fc50dd00700 20 allow so far , doing grant allow profile rbd 2018-02-14 13:23:48.985631 7fc50dd00700 20 match 2018-02-14 13:23:48.985633 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).osd e19702 check_osdmap_sub 0x5587d9050100 next 19703 (onetime) 2018-02-14 13:23:48.985666 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891df80 for client.1684235 10.255.0.53:0/287677392 2018-02-14 13:23:48.985675 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd 2018-02-14 13:23:48.985679 7fc50dd00700 20 is_capable service=mon command= read on cap allow profile rbd 2018-02-14 13:23:48.985682 7fc50dd00700 20 allow so far , doing grant allow profile rbd 2018-02-14 13:23:48.985688 7fc50dd00700 20 match 2018-02-14 13:23:48.985690 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 handle_subscribe mon_subscribe({mgrmap=0+,monmap=2+,osdmap=19703}) v2 2018-02-14 13:23:48.985697 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader).mgr e422 Sending map to subscriber 0x5587d8f95800 10.255.0.53:0/287677392 2018-02-14 13:23:48.985720 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).monmap v1 check_sub monmap next 2 have 1 2018-02-14 13:23:48.985727 7fc50dd00700 20 is_capable service=osd command= read on cap allow profile rbd 2018-02-14 13:23:48.985730 7fc50dd00700 20 allow so far , doing grant allow profile rbd 2018-02-14 13:23:48.985732 7fc50dd00700 20 match 2018-02-14 13:23:48.985734 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).osd e19702 check_osdmap_sub 0x5587d90517c0 next 19703 (onetime) 2018-02-14 13:23:48.985762 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891e280 for client.1684193 10.255.0.105:0/1988410988 2018-02-14 13:23:48.985770 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd 2018-02-14 13:23:48.985774 7fc50dd00700 20 is_capable service=mon command= read on cap allow profile rbd 2018-02-14 13:23:48.985776 7fc50dd00700 20 allow so far , doing grant allow profile rbd 2018-02-14 13:23:48.985789 7fc50dd00700 20 match 2018-02-14 13:23:48.985791 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 handle_subscribe mon_subscribe({mgrmap=0+,monmap=2+,osdmap=19703}) v2 2018-02-14 13:23:48.985798 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader).mgr e422 Sending map to subscriber 0x5587d8f97000 10.255.0.105:0/1988410988 2018-02-14 13:23:48.985844 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).monmap v1 check_sub monmap next 2 have 1 2018-02-14 13:23:48.985852 7fc50dd00700 20 is_capable service=osd command= read on cap allow profile rbd 2018-02-14 13:23:48.985855 7fc50dd00700 20 allow so far , doing grant allow profile rbd 2018-02-14 13:23:48.985858 7fc50dd00700 20 match 2018-02-14 13:23:48.985859 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).osd e19702 check_osdmap_sub 0x5587d9051a00 next 19703 (onetime) 2018-02-14 13:23:48.985895 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891e580 for client.1621590 10.255.0.106:0/1162398434 2018-02-14 13:23:48.985902 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd 2018-02-14 13:23:48.985906 7fc50dd00700 20 is_capable service=mon command= read on cap allow profile rbd 2018-02-14 13:23:48.985908 7fc50dd00700 20 allow so far , doing grant allow profile rbd 2018-02-14 13:23:48.985915 7fc50dd00700 20 match 2018-02-14 13:23:48.985917 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 handle_subscribe mon_subscribe({mgrmap=0+,monmap=2+,osdmap=19703}) v2 2018-02-14 13:23:48.985924 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader).mgr e422 Sending map to subscriber 0x5587d8f98800 10.255.0.106:0/1162398434 2018-02-14 13:23:48.985952 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).monmap v1 check_sub monmap next 2 have 1 2018-02-14 13:23:48.985959 7fc50dd00700 20 is_capable service=osd command= read on cap allow profile rbd 2018-02-14 13:23:48.985962 7fc50dd00700 20 allow so far , doing grant allow profile rbd 2018-02-14 13:23:48.985965 7fc50dd00700 20 match 2018-02-14 13:23:48.985967 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).osd e19702 check_osdmap_sub 0x5587d90500c0 next 19703 (onetime) 2018-02-14 13:23:48.985998 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891f180 for client.1684238 10.255.0.54:0/2824825993 2018-02-14 13:23:48.986007 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd 2018-02-14 13:23:48.986011 7fc50dd00700 20 is_capable service=mon command= read on cap allow profile rbd 2018-02-14 13:23:48.986013 7fc50dd00700 20 allow so far , doing grant allow profile rbd 2018-02-14 13:23:48.986020 7fc50dd00700 20 match 2018-02-14 13:23:48.986022 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 handle_subscribe mon_subscribe({mgrmap=0+,monmap=2+,osdmap=19703}) v2 2018-02-14 13:23:48.986029 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader).mgr e422 Sending map to subscriber 0x5587d8dac800 10.255.0.54:0/2824825993 2018-02-14 13:23:48.986059 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).monmap v1 check_sub monmap next 2 have 1 2018-02-14 13:23:48.986067 7fc50dd00700 20 is_capable service=osd command= read on cap allow profile rbd 2018-02-14 13:23:48.986070 7fc50dd00700 20 allow so far , doing grant allow profile rbd 2018-02-14 13:23:48.986072 7fc50dd00700 20 match 2018-02-14 13:23:48.986074 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).osd e19702 check_osdmap_sub 0x5587d9050580 next 19703 (onetime) 2018-02-14 13:23:48.986102 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891f480 for client.1684211 10.255.0.53:0/2869630763 2018-02-14 13:23:48.986112 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd 2018-02-14 13:23:48.986116 7fc50dd00700 20 is_capable service=mon command= read on cap allow profile rbd 2018-02-14 13:23:48.986118 7fc50dd00700 20 allow so far , doing grant allow profile rbd 2018-02-14 13:23:48.986125 7fc50dd00700 20 match 2018-02-14 13:23:48.986126 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 handle_subscribe mon_subscribe({mgrmap=0+,monmap=2+,osdmap=19703}) v2 2018-02-14 13:23:48.986131 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader).mgr e422 Sending map to subscriber 0x5587d8fae000 10.255.0.53:0/2869630763 2018-02-14 13:23:48.986146 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).monmap v1 check_sub monmap next 2 have 1 2018-02-14 13:23:48.986150 7fc50dd00700 20 is_capable service=osd command= read on cap allow profile rbd 2018-02-14 13:23:48.986152 7fc50dd00700 20 allow so far , doing grant allow profile rbd 2018-02-14 13:23:48.986153 7fc50dd00700 20 match 2018-02-14 13:23:48.986155 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).osd e19702 check_osdmap_sub 0x5587d9050a80 next 19703 (onetime) 2018-02-14 13:23:48.986173 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891fa80 for client.1684265 10.255.0.53:0/3548294640 2018-02-14 13:23:48.986178 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd 2018-02-14 13:23:48.986180 7fc50dd00700 20 is_capable service=mon command= read on cap allow profile rbd 2018-02-14 13:23:48.986182 7fc50dd00700 20 allow so far , doing grant allow profile rbd 2018-02-14 13:23:48.986186 7fc50dd00700 20 match 2018-02-14 13:23:48.986187 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 handle_subscribe mon_subscribe({mgrmap=0+,monmap=2+,osdmap=19703}) v2 2018-02-14 13:23:48.986192 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader).mgr e422 Sending map to subscriber 0x5587d8fb1000 10.255.0.53:0/3548294640 2018-02-14 13:23:48.986205 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).monmap v1 check_sub monmap next 2 have 1 2018-02-14 13:23:48.986212 7fc50dd00700 20 is_capable service=osd command= read on cap allow profile rbd 2018-02-14 13:23:48.986215 7fc50dd00700 20 allow so far , doing grant allow profile rbd 2018-02-14 13:23:48.986217 7fc50dd00700 20 match 2018-02-14 13:23:48.986219 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).osd e19702 check_osdmap_sub 0x5587d9050b40 next 19703 (onetime) 2018-02-14 13:23:48.986246 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891fd80 for client.1684217 10.255.0.107:0/2591354583 2018-02-14 13:23:48.986252 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd 2018-02-14 13:23:48.986255 7fc50dd00700 20 is_capable service=mon command= read on cap allow profile rbd 2018-02-14 13:23:48.986267 7fc50dd00700 20 allow so far , doing grant allow profile rbd 2018-02-14 13:23:48.986273 7fc50dd00700 20 match 2018-02-14 13:23:48.986274 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 handle_subscribe mon_subscribe({mgrmap=0+,monmap=2+,osdmap=19703}) v2 2018-02-14 13:23:48.986279 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader).mgr e422 Sending map to subscriber 0x5587d8fb2800 10.255.0.107:0/2591354583 2018-02-14 13:23:48.986294 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).monmap v1 check_sub monmap next 2 have 1 2018-02-14 13:23:48.986298 7fc50dd00700 20 is_capable service=osd command= read on cap allow profile rbd 2018-02-14 13:23:48.986300 7fc50dd00700 20 allow so far , doing grant allow profile rbd 2018-02-14 13:23:48.986301 7fc50dd00700 20 match 2018-02-14 13:23:48.986302 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).osd e19702 check_osdmap_sub 0x5587d90501c0 next 19703 (onetime) 2018-02-14 13:23:48.986318 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d9148300 for client.1684226 10.255.0.51:0/3289155469 2018-02-14 13:23:48.986323 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd 2018-02-14 13:23:48.986325 7fc50dd00700 20 is_capable service=mon command= read on cap allow profile rbd 2018-02-14 13:23:48.986327 7fc50dd00700 20 allow so far , doing grant allow profile rbd 2018-02-14 13:23:48.986331 7fc50dd00700 20 match 2018-02-14 13:23:48.986332 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 handle_subscribe mon_subscribe({mgrmap=0+,monmap=2+,osdmap=19703}) v2 2018-02-14 13:23:48.986335 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader).mgr e422 Sending map to subscriber 0x5587d9150000 10.255.0.51:0/3289155469 2018-02-14 13:23:48.986349 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).monmap v1 check_sub monmap next 2 have 1 2018-02-14 13:23:48.986356 7fc50dd00700 20 is_capable service=osd command= read on cap allow profile rbd 2018-02-14 13:23:48.986358 7fc50dd00700 20 allow so far , doing grant allow profile rbd 2018-02-14 13:23:48.986361 7fc50dd00700 20 match 2018-02-14 13:23:48.986362 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).osd e19702 check_osdmap_sub 0x5587d9666ac0 next 19703 (onetime) 2018-02-14 13:23:48.986382 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d9148480 for client.1684247 10.255.0.105:0/2076150628 2018-02-14 13:23:48.986387 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd 2018-02-14 13:23:48.986390 7fc50dd00700 20 is_capable service=mon command= read on cap allow profile rbd 2018-02-14 13:23:48.986392 7fc50dd00700 20 allow so far , doing grant allow profile rbd 2018-02-14 13:23:48.986397 7fc50dd00700 20 match 2018-02-14 13:23:48.986398 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 handle_subscribe mon_subscribe({mgrmap=0+,monmap=2+,osdmap=19703}) v2 2018-02-14 13:23:48.986402 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader).mgr e422 Sending map to subscriber 0x5587d9151800 10.255.0.105:0/2076150628 2018-02-14 13:23:48.986417 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).monmap v1 check_sub monmap next 2 have 1 2018-02-14 13:23:48.986424 7fc50dd00700 20 is_capable service=osd command= read on cap allow profile rbd 2018-02-14 13:23:48.986427 7fc50dd00700 20 allow so far , doing grant allow profile rbd 2018-02-14 13:23:48.986429 7fc50dd00700 20 match 2018-02-14 13:23:48.986431 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).osd e19702 check_osdmap_sub 0x5587d9667680 next 19703 (onetime) 2018-02-14 13:23:48.986458 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d9148600 for client.1684244 10.255.0.54:0/2718452115 2018-02-14 13:23:48.986464 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd 2018-02-14 13:23:48.986467 7fc50dd00700 20 is_capable service=mon command= read on cap allow profile rbd 2018-02-14 13:23:48.986469 7fc50dd00700 20 allow so far , doing grant allow profile rbd 2018-02-14 13:23:48.986473 7fc50dd00700 20 match 2018-02-14 13:23:48.986475 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 handle_subscribe mon_subscribe({mgrmap=0+,monmap=2+,osdmap=19703}) v2 2018-02-14 13:23:48.986479 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader).mgr e422 Sending map to subscriber 0x5587d9153000 10.255.0.54:0/2718452115 2018-02-14 13:23:48.986492 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).monmap v1 check_sub monmap next 2 have 1 2018-02-14 13:23:48.986497 7fc50dd00700 20 is_capable service=osd command= read on cap allow profile rbd 2018-02-14 13:23:48.986498 7fc50dd00700 20 allow so far , doing grant allow profile rbd 2018-02-14 13:23:48.986500 7fc50dd00700 20 match 2018-02-14 13:23:48.986501 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).osd e19702 check_osdmap_sub 0x5587d9667640 next 19703 (onetime) 2018-02-14 13:23:48.986517 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d9148780 for client.1684250 10.255.0.107:0/690639890 2018-02-14 13:23:48.986522 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd 2018-02-14 13:23:48.986524 7fc50dd00700 20 is_capable service=mon command= read on cap allow profile rbd 2018-02-14 13:23:48.986525 7fc50dd00700 20 allow so far , doing grant allow profile rbd 2018-02-14 13:23:48.986529 7fc50dd00700 20 match 2018-02-14 13:23:48.986531 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 handle_subscribe mon_subscribe({mgrmap=0+,monmap=2+,osdmap=19703}) v2 2018-02-14 13:23:48.986535 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader).mgr e422 Sending map to subscriber 0x5587d9154800 10.255.0.107:0/690639890 2018-02-14 13:23:48.986558 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).monmap v1 check_sub monmap next 2 have 1 2018-02-14 13:23:48.986563 7fc50dd00700 20 is_capable service=osd command= read on cap allow profile rbd 2018-02-14 13:23:48.986564 7fc50dd00700 20 allow so far , doing grant allow profile rbd 2018-02-14 13:23:48.986566 7fc50dd00700 20 match 2018-02-14 13:23:48.986567 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).osd e19702 check_osdmap_sub 0x5587d9667600 next 19703 (onetime) 2018-02-14 13:23:48.986588 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d9148f00 for client.1684190 10.255.0.108:0/3175087704 2018-02-14 13:23:48.986594 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd 2018-02-14 13:23:48.986596 7fc50dd00700 20 is_capable service=mon command= read on cap allow profile rbd 2018-02-14 13:23:48.986598 7fc50dd00700 20 allow so far , doing grant allow profile rbd 2018-02-14 13:23:48.986602 7fc50dd00700 20 match 2018-02-14 13:23:48.986603 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 handle_subscribe mon_subscribe({mgrmap=0+,monmap=2+,osdmap=19703}) v2 2018-02-14 13:23:48.986608 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader).mgr e422 Sending map to subscriber 0x5587d91be800 10.255.0.108:0/3175087704 2018-02-14 13:23:48.986621 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).monmap v1 check_sub monmap next 2 have 1 2018-02-14 13:23:48.986625 7fc50dd00700 20 is_capable service=osd command= read on cap allow profile rbd 2018-02-14 13:23:48.986627 7fc50dd00700 20 allow so far , doing grant allow profile rbd 2018-02-14 13:23:48.986628 7fc50dd00700 20 match 2018-02-14 13:23:48.986630 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).osd e19702 check_osdmap_sub 0x5587d96675c0 next 19703 (onetime) 2018-02-14 13:23:48.986650 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d9149080 for client.1582487 10.255.0.49:0/59005741 2018-02-14 13:23:48.986655 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps allow profile mgr 2018-02-14 13:23:48.986657 7fc50dd00700 20 is_capable service=mon command= read on cap allow profile mgr 2018-02-14 13:23:48.986659 7fc50dd00700 20 allow so far , doing grant allow profile mgr 2018-02-14 13:23:48.986671 7fc50dd00700 20 match 2018-02-14 13:23:48.986672 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 handle_subscribe mon_subscribe({fsmap=2+,log-info=0+,mgrdigest=0+,mgrmap=0+,monmap=2+,osdmap=19703,servicemap=0+}) v2 2018-02-14 13:23:48.986677 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 handle_subscribe: MDS sub 'fsmap' 2018-02-14 13:23:48.986679 7fc50dd00700 20 is_capable service=mds command= read on cap allow profile mgr 2018-02-14 13:23:48.986681 7fc50dd00700 20 allow so far , doing grant allow profile mgr 2018-02-14 13:23:48.986683 7fc50dd00700 20 match 2018-02-14 13:23:48.986684 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader).mds e1 check_sub: fsmap 2018-02-14 13:23:48.986696 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 check_sub client wants log-info ver 0 2018-02-14 13:23:48.986700 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 _create_sub_summary 2018-02-14 13:23:48.986701 7fc50dd00700 1 mon.ceph-dal-mon-a1@0(leader).log v594874 check_sub sending message to client.1582487 10.255.0.49:0/59005741 with 1 entries (version 594874) 2018-02-14 13:23:48.986720 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).mgr e422 send_digests 2018-02-14 13:23:48.986726 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).mgr e422 send_digests sending digest to subscriber 0x5587d8fcc000 10.255.0.49:0/59005741 2018-02-14 13:23:48.986975 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader).mgr e422 Sending map to subscriber 0x5587d8fcc000 10.255.0.49:0/59005741 2018-02-14 13:23:48.986991 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).monmap v1 check_sub monmap next 2 have 1 2018-02-14 13:23:48.986996 7fc50dd00700 20 is_capable service=osd command= read on cap allow profile mgr 2018-02-14 13:23:48.986998 7fc50dd00700 20 allow so far , doing grant allow profile mgr 2018-02-14 13:23:48.987000 7fc50dd00700 20 match 2018-02-14 13:23:48.987002 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).osd e19702 check_osdmap_sub 0x5587d96671c0 next 19703 (onetime) 2018-02-14 13:23:48.987006 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).mgrstat check_sub next 0 have 1 2018-02-14 13:23:48.987033 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d9149200 for client.1684199 10.255.0.106:0/1777238177 2018-02-14 13:23:48.987039 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd 2018-02-14 13:23:48.987041 7fc50dd00700 20 is_capable service=mon command= read on cap allow profile rbd 2018-02-14 13:23:48.987042 7fc50dd00700 20 allow so far , doing grant allow profile rbd 2018-02-14 13:23:48.987047 7fc50dd00700 20 match 2018-02-14 13:23:48.987048 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 handle_subscribe mon_subscribe({mgrmap=0+,monmap=2+,osdmap=19703}) v2 2018-02-14 13:23:48.987053 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader).mgr e422 Sending map to subscriber 0x5587d8fcd800 10.255.0.106:0/1777238177 2018-02-14 13:23:48.987087 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).monmap v1 check_sub monmap next 2 have 1 2018-02-14 13:23:48.987092 7fc50dd00700 20 is_capable service=osd command= read on cap allow profile rbd 2018-02-14 13:23:48.987093 7fc50dd00700 20 allow so far , doing grant allow profile rbd 2018-02-14 13:23:48.987095 7fc50dd00700 20 match 2018-02-14 13:23:48.987096 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).osd e19702 check_osdmap_sub 0x5587d9052180 next 19703 (onetime) 2018-02-14 13:23:48.987112 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d9149800 for client.1714098 10.255.0.108:0/850049828 2018-02-14 13:23:48.987117 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd 2018-02-14 13:23:48.987119 7fc50dd00700 20 is_capable service=mon command= read on cap allow profile rbd 2018-02-14 13:23:48.987121 7fc50dd00700 20 allow so far , doing grant allow profile rbd 2018-02-14 13:23:48.987125 7fc50dd00700 20 match 2018-02-14 13:23:48.987126 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 handle_subscribe mon_subscribe({mgrmap=0+,monmap=2+,osdmap=19703}) v2 2018-02-14 13:23:48.987130 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader).mgr e422 Sending map to subscriber 0x5587d8fe3800 10.255.0.108:0/850049828 2018-02-14 13:23:48.987143 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).monmap v1 check_sub monmap next 2 have 1 2018-02-14 13:23:48.987146 7fc50dd00700 20 is_capable service=osd command= read on cap allow profile rbd 2018-02-14 13:23:48.987147 7fc50dd00700 20 allow so far , doing grant allow profile rbd 2018-02-14 13:23:48.987149 7fc50dd00700 20 match 2018-02-14 13:23:48.987150 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).osd e19702 check_osdmap_sub 0x5587d8e96640 next 19703 (onetime) 2018-02-14 13:23:48.987165 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d9149980 for client.1714104 10.255.0.54:0/1233019732 2018-02-14 13:23:48.987168 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd 2018-02-14 13:23:48.987170 7fc50dd00700 20 is_capable service=mon command= read on cap allow profile rbd 2018-02-14 13:23:48.987171 7fc50dd00700 20 allow so far , doing grant allow profile rbd 2018-02-14 13:23:48.987175 7fc50dd00700 20 match 2018-02-14 13:23:48.987176 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 handle_subscribe mon_subscribe({mgrmap=0+,monmap=2+,osdmap=19703}) v2 2018-02-14 13:23:48.987181 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader).mgr e422 Sending map to subscriber 0x5587d8fe5000 10.255.0.54:0/1233019732 2018-02-14 13:23:48.987194 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).monmap v1 check_sub monmap next 2 have 1 2018-02-14 13:23:48.987199 7fc50dd00700 20 is_capable service=osd command= read on cap allow profile rbd 2018-02-14 13:23:48.987200 7fc50dd00700 20 allow so far , doing grant allow profile rbd 2018-02-14 13:23:48.987202 7fc50dd00700 20 match 2018-02-14 13:23:48.987204 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).osd e19702 check_osdmap_sub 0x5587d90522c0 next 19703 (onetime) 2018-02-14 13:23:48.987217 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d9149c80 for client.? 10.255.0.52:0/3152189720 2018-02-14 13:23:48.987222 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd 2018-02-14 13:23:48.987225 7fc50dd00700 20 is_capable service=mon command= read on cap allow profile rbd 2018-02-14 13:23:48.987227 7fc50dd00700 20 allow so far , doing grant allow profile rbd 2018-02-14 13:23:48.987231 7fc50dd00700 20 match 2018-02-14 13:23:48.987232 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 handle_subscribe mon_subscribe({monmap=0+}) v2 2018-02-14 13:23:48.987235 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).monmap v1 check_sub monmap next 0 have 1 2018-02-14 13:23:48.987279 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d9149e00 for client.? 10.255.0.54:0/3194015072 2018-02-14 13:23:48.987284 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd 2018-02-14 13:23:48.987287 7fc50dd00700 20 is_capable service=mon command= read on cap allow profile rbd 2018-02-14 13:23:48.987288 7fc50dd00700 20 allow so far , doing grant allow profile rbd 2018-02-14 13:23:48.987294 7fc50dd00700 20 match 2018-02-14 13:23:48.987295 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 handle_subscribe mon_subscribe({monmap=0+}) v2 2018-02-14 13:23:48.987298 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).monmap v1 check_sub monmap next 0 have 1 2018-02-14 13:23:48.987327 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d9149f80 for client.? 10.255.0.107:0/980595480 2018-02-14 13:23:48.987332 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd 2018-02-14 13:23:48.987334 7fc50dd00700 20 is_capable service=mon command= read on cap allow profile rbd 2018-02-14 13:23:48.987336 7fc50dd00700 20 allow so far , doing grant allow profile rbd 2018-02-14 13:23:48.987340 7fc50dd00700 20 match 2018-02-14 13:23:48.987341 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 handle_subscribe mon_subscribe({monmap=0+}) v2 2018-02-14 13:23:48.987344 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).monmap v1 check_sub monmap next 0 have 1 2018-02-14 13:23:48.987372 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d914a100 for client.1684202 10.255.0.107:0/3082328679 2018-02-14 13:23:48.987377 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd 2018-02-14 13:23:48.987380 7fc50dd00700 20 is_capable service=mon command= read on cap allow profile rbd 2018-02-14 13:23:48.987381 7fc50dd00700 20 allow so far , doing grant allow profile rbd 2018-02-14 13:23:48.987385 7fc50dd00700 20 match 2018-02-14 13:23:48.987386 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 handle_subscribe mon_subscribe({mgrmap=0+,monmap=2+,osdmap=19703}) v2 2018-02-14 13:23:48.987390 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader).mgr e422 Sending map to subscriber 0x5587d9004000 10.255.0.107:0/3082328679 2018-02-14 13:23:48.987404 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).monmap v1 check_sub monmap next 2 have 1 2018-02-14 13:23:48.987407 7fc50dd00700 20 is_capable service=osd command= read on cap allow profile rbd 2018-02-14 13:23:48.987409 7fc50dd00700 20 allow so far , doing grant allow profile rbd 2018-02-14 13:23:48.987410 7fc50dd00700 20 match 2018-02-14 13:23:48.987411 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).osd e19702 check_osdmap_sub 0x5587d9666fc0 next 19703 (onetime) 2018-02-14 13:23:48.987426 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d914a400 for client.1684208 10.255.0.52:0/4056696166 2018-02-14 13:23:48.987430 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd 2018-02-14 13:23:48.987433 7fc50dd00700 20 is_capable service=mon command= read on cap allow profile rbd 2018-02-14 13:23:48.987434 7fc50dd00700 20 allow so far , doing grant allow profile rbd 2018-02-14 13:23:48.987438 7fc50dd00700 20 match 2018-02-14 13:23:48.987439 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 handle_subscribe mon_subscribe({mgrmap=0+,monmap=2+,osdmap=19703}) v2 2018-02-14 13:23:48.987443 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader).mgr e422 Sending map to subscriber 0x5587d9007000 10.255.0.52:0/4056696166 2018-02-14 13:23:48.987455 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).monmap v1 check_sub monmap next 2 have 1 2018-02-14 13:23:48.987458 7fc50dd00700 20 is_capable service=osd command= read on cap allow profile rbd 2018-02-14 13:23:48.987459 7fc50dd00700 20 allow so far , doing grant allow profile rbd 2018-02-14 13:23:48.987461 7fc50dd00700 20 match 2018-02-14 13:23:48.987462 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).osd e19702 check_osdmap_sub 0x5587d9666f80 next 19703 (onetime) 2018-02-14 13:23:48.987475 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891cc00 for client.1684214 10.255.0.105:0/1753894484 2018-02-14 13:23:48.987480 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd 2018-02-14 13:23:48.987482 7fc50dd00700 20 is_capable service=mon command= read on cap allow profile rbd 2018-02-14 13:23:48.987484 7fc50dd00700 20 allow so far , doing grant allow profile rbd 2018-02-14 13:23:48.987488 7fc50dd00700 20 match 2018-02-14 13:23:48.987489 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 handle_subscribe mon_subscribe({mgrmap=0+,monmap=2+,osdmap=19703}) v2 2018-02-14 13:23:48.987493 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader).mgr e422 Sending map to subscriber 0x5587d8eab800 10.255.0.105:0/1753894484 2018-02-14 13:23:48.987506 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).monmap v1 check_sub monmap next 2 have 1 2018-02-14 13:23:48.987511 7fc50dd00700 20 is_capable service=osd command= read on cap allow profile rbd 2018-02-14 13:23:48.987513 7fc50dd00700 20 allow so far , doing grant allow profile rbd 2018-02-14 13:23:48.987514 7fc50dd00700 20 match 2018-02-14 13:23:48.987515 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).osd e19702 check_osdmap_sub 0x5587d9666f40 next 19703 (onetime) 2018-02-14 13:23:48.987529 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891cd80 for client.1684262 10.255.0.106:0/3270640683 2018-02-14 13:23:48.987532 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd 2018-02-14 13:23:48.987534 7fc50dd00700 20 is_capable service=mon command= read on cap allow profile rbd 2018-02-14 13:23:48.987535 7fc50dd00700 20 allow so far , doing grant allow profile rbd 2018-02-14 13:23:48.987539 7fc50dd00700 20 match 2018-02-14 13:23:48.987540 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 handle_subscribe mon_subscribe({mgrmap=0+,monmap=2+,osdmap=19703}) v2 2018-02-14 13:23:48.987544 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader).mgr e422 Sending map to subscriber 0x5587d8ead000 10.255.0.106:0/3270640683 2018-02-14 13:23:48.987556 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).monmap v1 check_sub monmap next 2 have 1 2018-02-14 13:23:48.987558 7fc50dd00700 20 is_capable service=osd command= read on cap allow profile rbd 2018-02-14 13:23:48.987572 7fc50dd00700 20 allow so far , doing grant allow profile rbd 2018-02-14 13:23:48.987575 7fc50dd00700 20 match 2018-02-14 13:23:48.987577 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).osd e19702 check_osdmap_sub 0x5587d9666f00 next 19703 (onetime) 2018-02-14 13:23:48.987593 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891d080 for client.1684205 10.255.0.51:0/3360953369 2018-02-14 13:23:48.987598 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd 2018-02-14 13:23:48.987600 7fc50dd00700 20 is_capable service=mon command= read on cap allow profile rbd 2018-02-14 13:23:48.987602 7fc50dd00700 20 allow so far , doing grant allow profile rbd 2018-02-14 13:23:48.987606 7fc50dd00700 20 match 2018-02-14 13:23:48.987607 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 handle_subscribe mon_subscribe({mgrmap=0+,monmap=2+,osdmap=19703}) v2 2018-02-14 13:23:48.987617 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader).mgr e422 Sending map to subscriber 0x5587d8f2d800 10.255.0.51:0/3360953369 2018-02-14 13:23:48.987635 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).monmap v1 check_sub monmap next 2 have 1 2018-02-14 13:23:48.987639 7fc50dd00700 20 is_capable service=osd command= read on cap allow profile rbd 2018-02-14 13:23:48.987641 7fc50dd00700 20 allow so far , doing grant allow profile rbd 2018-02-14 13:23:48.987643 7fc50dd00700 20 match 2018-02-14 13:23:48.987644 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).osd e19702 check_osdmap_sub 0x5587d9666ec0 next 19703 (onetime) 2018-02-14 13:23:48.987659 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891d980 for client.1684241 10.255.0.30:0/3376128839 2018-02-14 13:23:48.987664 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps allow profile mgr 2018-02-14 13:23:48.987673 7fc50dd00700 4 mon.ceph-dal-mon-a1@0(leader).mgr e422 beacon from 1684241 2018-02-14 13:23:48.987675 7fc50dd00700 20 is_capable service=mgr command= exec on cap allow profile mgr 2018-02-14 13:23:48.987677 7fc50dd00700 20 allow so far , doing grant allow profile mgr 2018-02-14 13:23:48.987679 7fc50dd00700 20 match 2018-02-14 13:23:48.987684 7fc50dd00700 4 mon.ceph-dal-mon-a1@0(leader).mgr e422 beacon from 1684241 2018-02-14 13:23:48.987686 7fc50dd00700 4 mon.ceph-dal-mon-a1@0(leader).mgr e422 Standby daemon restart (mgr.ceph-dal-mon-a1) 2018-02-14 13:23:48.987691 7fc50dd00700 0 log_channel(cluster) log [DBG] : Standby manager daemon ceph-dal-mon-a1 restarted 2018-02-14 13:23:48.987717 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).mgr e422 new standby 1684241 2018-02-14 13:23:48.987722 7fc50dd00700 0 log_channel(cluster) log [DBG] : Standby manager daemon ceph-dal-mon-a1 started 2018-02-14 13:23:48.987746 7fc50dd00700 4 mon.ceph-dal-mon-a1@0(leader).mgr e422 updating map 2018-02-14 13:23:48.987765 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891ca80 for mon.0 10.255.0.30:6789/0 2018-02-14 13:23:48.987771 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps allow * 2018-02-14 13:23:48.987786 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 preprocess_query log(1 entries from seq 10 at 2018-02-14 13:23:48.987693) v1 from mon.0 10.255.0.30:6789/0 2018-02-14 13:23:48.987793 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 preprocess_log log(1 entries from seq 10 at 2018-02-14 13:23:48.987693) v1 from mon.0 2018-02-14 13:23:48.987797 7fc50dd00700 20 is_capable service=log command= write on cap allow * 2018-02-14 13:23:48.987798 7fc50dd00700 20 allow so far , doing grant allow * 2018-02-14 13:23:48.987799 7fc50dd00700 20 allow all 2018-02-14 13:23:48.987811 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891ca80 for mon.0 10.255.0.30:6789/0 2018-02-14 13:23:48.987816 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps allow * 2018-02-14 13:23:48.987824 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 preprocess_query log(1 entries from seq 11 at 2018-02-14 13:23:48.987723) v1 from mon.0 10.255.0.30:6789/0 2018-02-14 13:23:48.987831 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 preprocess_log log(1 entries from seq 11 at 2018-02-14 13:23:48.987723) v1 from mon.0 2018-02-14 13:23:48.987835 7fc50dd00700 20 is_capable service=log command= write on cap allow * 2018-02-14 13:23:48.987836 7fc50dd00700 20 allow so far , doing grant allow * 2018-02-14 13:23:48.987837 7fc50dd00700 20 allow all 2018-02-14 13:23:48.987849 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891e580 for client.1621590 10.255.0.106:0/1162398434 2018-02-14 13:23:48.987854 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd 2018-02-14 13:23:48.987864 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).osd e19702 preprocess_query pool_op(delete unmanaged snap pool 6 auid 0 tid 28 name v0) v4 from client.1621590 10.255.0.106:0/1162398434 2018-02-14 13:23:48.987879 7fc50dd00700 7 mon.ceph-dal-mon-a1@0(leader).osd e19702 prepare_update pool_op(delete unmanaged snap pool 6 auid 0 tid 28 name v0) v4 from client.1621590 10.255.0.106:0/1162398434 2018-02-14 13:23:48.987888 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).osd e19702 prepare_pool_op pool_op(delete unmanaged snap pool 6 auid 0 tid 28 name v0) v4 2018-02-14 13:23:48.991163 7fc50dd00700 -1 /home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos7/DIST/centos7/MACHINE_SIZE/huge/release/12.2.2/rpm/el7/BUILD/ceph-12.2.2/src/include/interval_set.h: In function 'void interval_set::insert(T, T, T*, T*) [with T = snapid_t]' thread 7fc50dd00700 time 2018-02-14 13:23:48.987900 /home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos7/DIST/centos7/MACHINE_SIZE/huge/release/12.2.2/rpm/el7/BUILD/ceph-12.2.2/src/include/interval_set.h: 396: FAILED assert(p->first > start+len) ceph version 12.2.2 (cf0baeeeeba3b47f9427c6c97e2144b094b7e5ba) luminous (stable) 1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x110) [0x5587cebc0330] 2: (interval_set::insert(snapid_t, snapid_t, snapid_t*, snapid_t*)+0x32b) [0x5587ced0965b] 3: (pg_pool_t::remove_unmanaged_snap(snapid_t)+0x4d) [0x5587cecf6a1d] 4: (OSDMonitor::prepare_pool_op(boost::intrusive_ptr)+0xe50) [0x5587ceaeb0f0] 5: (OSDMonitor::prepare_update(boost::intrusive_ptr)+0x336) [0x5587ceb22b16] 6: (PaxosService::dispatch(boost::intrusive_ptr)+0xaf8) [0x5587ceaaf908] 7: (Monitor::dispatch_op(boost::intrusive_ptr)+0x52f) [0x5587ce99281f] 8: (Monitor::_ms_dispatch(Message*)+0x7eb) [0x5587ce993e5b] 9: (Monitor::ms_dispatch(Message*)+0x23) [0x5587ce9bf923] 10: (DispatchQueue::entry()+0x792) [0x5587cee6c162] 11: (DispatchQueue::DispatchThread::entry()+0xd) [0x5587cec67b6d] 12: (()+0x7e25) [0x7fc5172e8e25] 13: (clone()+0x6d) [0x7fc51488334d] NOTE: a copy of the executable, or `objdump -rdS ` is needed to interpret this. --- begin dump of recent events --- -2458> 2018-02-14 13:23:40.880614 7fc517f3ee40 5 asok(0x5587d89481c0) register_command perfcounters_dump hook 0x5587d88d4190 -2457> 2018-02-14 13:23:40.880631 7fc517f3ee40 5 asok(0x5587d89481c0) register_command 1 hook 0x5587d88d4190 -2456> 2018-02-14 13:23:40.880634 7fc517f3ee40 5 asok(0x5587d89481c0) register_command perf dump hook 0x5587d88d4190 -2455> 2018-02-14 13:23:40.880647 7fc517f3ee40 5 asok(0x5587d89481c0) register_command perfcounters_schema hook 0x5587d88d4190 -2454> 2018-02-14 13:23:40.880654 7fc517f3ee40 5 asok(0x5587d89481c0) register_command perf histogram dump hook 0x5587d88d4190 -2453> 2018-02-14 13:23:40.880658 7fc517f3ee40 5 asok(0x5587d89481c0) register_command 2 hook 0x5587d88d4190 -2452> 2018-02-14 13:23:40.880669 7fc517f3ee40 5 asok(0x5587d89481c0) register_command perf schema hook 0x5587d88d4190 -2451> 2018-02-14 13:23:40.880673 7fc517f3ee40 5 asok(0x5587d89481c0) register_command perf histogram schema hook 0x5587d88d4190 -2450> 2018-02-14 13:23:40.880683 7fc517f3ee40 5 asok(0x5587d89481c0) register_command perf reset hook 0x5587d88d4190 -2449> 2018-02-14 13:23:40.880687 7fc517f3ee40 5 asok(0x5587d89481c0) register_command config show hook 0x5587d88d4190 -2448> 2018-02-14 13:23:40.880691 7fc517f3ee40 5 asok(0x5587d89481c0) register_command config help hook 0x5587d88d4190 -2447> 2018-02-14 13:23:40.880695 7fc517f3ee40 5 asok(0x5587d89481c0) register_command config set hook 0x5587d88d4190 -2446> 2018-02-14 13:23:40.880699 7fc517f3ee40 5 asok(0x5587d89481c0) register_command config get hook 0x5587d88d4190 -2445> 2018-02-14 13:23:40.880703 7fc517f3ee40 5 asok(0x5587d89481c0) register_command config diff hook 0x5587d88d4190 -2444> 2018-02-14 13:23:40.880709 7fc517f3ee40 5 asok(0x5587d89481c0) register_command config diff get hook 0x5587d88d4190 -2443> 2018-02-14 13:23:40.880713 7fc517f3ee40 5 asok(0x5587d89481c0) register_command log flush hook 0x5587d88d4190 -2442> 2018-02-14 13:23:40.880717 7fc517f3ee40 5 asok(0x5587d89481c0) register_command log dump hook 0x5587d88d4190 -2441> 2018-02-14 13:23:40.880721 7fc517f3ee40 5 asok(0x5587d89481c0) register_command log reopen hook 0x5587d88d4190 -2440> 2018-02-14 13:23:40.880728 7fc517f3ee40 5 asok(0x5587d89481c0) register_command dump_mempools hook 0x5587d89ba0c8 -2439> 2018-02-14 13:23:40.888793 7fc517f3ee40 0 set uid:gid to 167:167 (ceph:ceph) -2438> 2018-02-14 13:23:40.888815 7fc517f3ee40 0 ceph version 12.2.2 (cf0baeeeeba3b47f9427c6c97e2144b094b7e5ba) luminous (stable), process (unknown), pid 12059 -2437> 2018-02-14 13:23:40.888856 7fc517f3ee40 0 pidfile_write: ignore empty --pid-file -2436> 2018-02-14 13:23:40.915384 7fc517f3ee40 5 asok(0x5587d89481c0) init /var/run/ceph/ceph-mon.ceph-dal-mon-a1.asok -2435> 2018-02-14 13:23:40.915397 7fc517f3ee40 5 asok(0x5587d89481c0) bind_and_listen /var/run/ceph/ceph-mon.ceph-dal-mon-a1.asok -2434> 2018-02-14 13:23:40.915423 7fc517f3ee40 5 asok(0x5587d89481c0) register_command 0 hook 0x5587d88d0170 -2433> 2018-02-14 13:23:40.915429 7fc517f3ee40 5 asok(0x5587d89481c0) register_command version hook 0x5587d88d0170 -2432> 2018-02-14 13:23:40.915433 7fc517f3ee40 5 asok(0x5587d89481c0) register_command git_version hook 0x5587d88d0170 -2431> 2018-02-14 13:23:40.915439 7fc517f3ee40 5 asok(0x5587d89481c0) register_command help hook 0x5587d88d42a0 -2430> 2018-02-14 13:23:40.915443 7fc517f3ee40 5 asok(0x5587d89481c0) register_command get_command_descriptions hook 0x5587d88d42b0 -2429> 2018-02-14 13:23:40.915474 7fc512552700 5 asok(0x5587d89481c0) entry start -2428> 2018-02-14 13:23:40.922300 7fc517f3ee40 0 load: jerasure load: lrc load: isa -2427> 2018-02-14 13:23:40.922421 7fc517f3ee40 0 set rocksdb option compression = kNoCompression -2426> 2018-02-14 13:23:40.922433 7fc517f3ee40 0 set rocksdb option write_buffer_size = 33554432 -2425> 2018-02-14 13:23:40.922459 7fc517f3ee40 0 set rocksdb option compression = kNoCompression -2424> 2018-02-14 13:23:40.922466 7fc517f3ee40 0 set rocksdb option write_buffer_size = 33554432 -2423> 2018-02-14 13:23:40.922609 7fc517f3ee40 4 rocksdb: RocksDB version: 5.4.0 -2422> 2018-02-14 13:23:40.922618 7fc517f3ee40 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ -2421> 2018-02-14 13:23:40.922622 7fc517f3ee40 4 rocksdb: Compile date Nov 30 2017 -2420> 2018-02-14 13:23:40.922625 7fc517f3ee40 4 rocksdb: DB SUMMARY -2419> 2018-02-14 13:23:40.922669 7fc517f3ee40 4 rocksdb: CURRENT file: CURRENT -2418> 2018-02-14 13:23:40.922674 7fc517f3ee40 4 rocksdb: IDENTITY file: IDENTITY -2417> 2018-02-14 13:23:40.922678 7fc517f3ee40 4 rocksdb: MANIFEST file: MANIFEST-128462 size: 320 Bytes -2416> 2018-02-14 13:23:40.922681 7fc517f3ee40 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-ceph-dal-mon-a1/store.db dir, Total Num: 4, files: 128453.sst 128455.sst 128458.sst 128461.sst -2415> 2018-02-14 13:23:40.922683 7fc517f3ee40 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-ceph-dal-mon-a1/store.db: 128463.log size: 52 ; -2414> 2018-02-14 13:23:40.922686 7fc517f3ee40 4 rocksdb: Options.error_if_exists: 0 -2413> 2018-02-14 13:23:40.922687 7fc517f3ee40 4 rocksdb: Options.create_if_missing: 0 -2412> 2018-02-14 13:23:40.922688 7fc517f3ee40 4 rocksdb: Options.paranoid_checks: 1 -2411> 2018-02-14 13:23:40.922689 7fc517f3ee40 4 rocksdb: Options.env: 0x5587d76e6040 -2410> 2018-02-14 13:23:40.922690 7fc517f3ee40 4 rocksdb: Options.info_log: 0x5587d8b9dba0 -2409> 2018-02-14 13:23:40.922691 7fc517f3ee40 4 rocksdb: Options.max_open_files: -1 -2408> 2018-02-14 13:23:40.922692 7fc517f3ee40 4 rocksdb: Options.max_file_opening_threads: 16 -2407> 2018-02-14 13:23:40.922694 7fc517f3ee40 4 rocksdb: Options.use_fsync: 0 -2406> 2018-02-14 13:23:40.922694 7fc517f3ee40 4 rocksdb: Options.max_log_file_size: 0 -2405> 2018-02-14 13:23:40.922695 7fc517f3ee40 4 rocksdb: Options.max_manifest_file_size: 18446744073709551615 -2404> 2018-02-14 13:23:40.922696 7fc517f3ee40 4 rocksdb: Options.log_file_time_to_roll: 0 -2403> 2018-02-14 13:23:40.922697 7fc517f3ee40 4 rocksdb: Options.keep_log_file_num: 1000 -2402> 2018-02-14 13:23:40.922698 7fc517f3ee40 4 rocksdb: Options.recycle_log_file_num: 0 -2401> 2018-02-14 13:23:40.922698 7fc517f3ee40 4 rocksdb: Options.allow_fallocate: 1 -2400> 2018-02-14 13:23:40.922699 7fc517f3ee40 4 rocksdb: Options.allow_mmap_reads: 0 -2399> 2018-02-14 13:23:40.922700 7fc517f3ee40 4 rocksdb: Options.allow_mmap_writes: 0 -2398> 2018-02-14 13:23:40.922701 7fc517f3ee40 4 rocksdb: Options.use_direct_reads: 0 -2397> 2018-02-14 13:23:40.922702 7fc517f3ee40 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 -2396> 2018-02-14 13:23:40.922703 7fc517f3ee40 4 rocksdb: Options.create_missing_column_families: 0 -2395> 2018-02-14 13:23:40.922704 7fc517f3ee40 4 rocksdb: Options.db_log_dir: -2394> 2018-02-14 13:23:40.922704 7fc517f3ee40 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-ceph-dal-mon-a1/store.db -2393> 2018-02-14 13:23:40.922705 7fc517f3ee40 4 rocksdb: Options.table_cache_numshardbits: 6 -2392> 2018-02-14 13:23:40.922706 7fc517f3ee40 4 rocksdb: Options.max_subcompactions: 1 -2391> 2018-02-14 13:23:40.922707 7fc517f3ee40 4 rocksdb: Options.max_background_flushes: 1 -2390> 2018-02-14 13:23:40.922708 7fc517f3ee40 4 rocksdb: Options.WAL_ttl_seconds: 0 -2389> 2018-02-14 13:23:40.922708 7fc517f3ee40 4 rocksdb: Options.WAL_size_limit_MB: 0 -2388> 2018-02-14 13:23:40.922709 7fc517f3ee40 4 rocksdb: Options.manifest_preallocation_size: 4194304 -2387> 2018-02-14 13:23:40.922710 7fc517f3ee40 4 rocksdb: Options.is_fd_close_on_exec: 1 -2386> 2018-02-14 13:23:40.922711 7fc517f3ee40 4 rocksdb: Options.advise_random_on_open: 1 -2385> 2018-02-14 13:23:40.922712 7fc517f3ee40 4 rocksdb: Options.db_write_buffer_size: 0 -2384> 2018-02-14 13:23:40.922712 7fc517f3ee40 4 rocksdb: Options.access_hint_on_compaction_start: 1 -2383> 2018-02-14 13:23:40.922713 7fc517f3ee40 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 -2382> 2018-02-14 13:23:40.922714 7fc517f3ee40 4 rocksdb: Options.compaction_readahead_size: 0 -2381> 2018-02-14 13:23:40.922714 7fc517f3ee40 4 rocksdb: Options.random_access_max_buffer_size: 1048576 -2380> 2018-02-14 13:23:40.922715 7fc517f3ee40 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 -2379> 2018-02-14 13:23:40.922716 7fc517f3ee40 4 rocksdb: Options.use_adaptive_mutex: 0 -2378> 2018-02-14 13:23:40.922717 7fc517f3ee40 4 rocksdb: Options.rate_limiter: (nil) -2377> 2018-02-14 13:23:40.922718 7fc517f3ee40 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 -2376> 2018-02-14 13:23:40.922719 7fc517f3ee40 4 rocksdb: Options.bytes_per_sync: 0 -2375> 2018-02-14 13:23:40.922720 7fc517f3ee40 4 rocksdb: Options.wal_bytes_per_sync: 0 -2374> 2018-02-14 13:23:40.922720 7fc517f3ee40 4 rocksdb: Options.wal_recovery_mode: 2 -2373> 2018-02-14 13:23:40.922721 7fc517f3ee40 4 rocksdb: Options.enable_thread_tracking: 0 -2372> 2018-02-14 13:23:40.922722 7fc517f3ee40 4 rocksdb: Options.allow_concurrent_memtable_write: 1 -2371> 2018-02-14 13:23:40.922722 7fc517f3ee40 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 -2370> 2018-02-14 13:23:40.922723 7fc517f3ee40 4 rocksdb: Options.write_thread_max_yield_usec: 100 -2369> 2018-02-14 13:23:40.922724 7fc517f3ee40 4 rocksdb: Options.write_thread_slow_yield_usec: 3 -2368> 2018-02-14 13:23:40.922725 7fc517f3ee40 4 rocksdb: Options.row_cache: None -2367> 2018-02-14 13:23:40.922726 7fc517f3ee40 4 rocksdb: Options.wal_filter: None -2366> 2018-02-14 13:23:40.922727 7fc517f3ee40 4 rocksdb: Options.avoid_flush_during_recovery: 0 -2365> 2018-02-14 13:23:40.922727 7fc517f3ee40 4 rocksdb: Options.base_background_compactions: 1 -2364> 2018-02-14 13:23:40.922728 7fc517f3ee40 4 rocksdb: Options.max_background_compactions: 1 -2363> 2018-02-14 13:23:40.922729 7fc517f3ee40 4 rocksdb: Options.avoid_flush_during_shutdown: 0 -2362> 2018-02-14 13:23:40.922730 7fc517f3ee40 4 rocksdb: Options.delayed_write_rate : 16777216 -2361> 2018-02-14 13:23:40.922730 7fc517f3ee40 4 rocksdb: Options.max_total_wal_size: 0 -2360> 2018-02-14 13:23:40.922731 7fc517f3ee40 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 -2359> 2018-02-14 13:23:40.922741 7fc517f3ee40 4 rocksdb: Options.stats_dump_period_sec: 600 -2358> 2018-02-14 13:23:40.922742 7fc517f3ee40 4 rocksdb: Compression algorithms supported: -2357> 2018-02-14 13:23:40.922745 7fc517f3ee40 4 rocksdb: Snappy supported: 0 -2356> 2018-02-14 13:23:40.922746 7fc517f3ee40 4 rocksdb: Zlib supported: 0 -2355> 2018-02-14 13:23:40.922746 7fc517f3ee40 4 rocksdb: Bzip supported: 0 -2354> 2018-02-14 13:23:40.922747 7fc517f3ee40 4 rocksdb: LZ4 supported: 0 -2353> 2018-02-14 13:23:40.922748 7fc517f3ee40 4 rocksdb: ZSTD supported: 0 -2352> 2018-02-14 13:23:40.922750 7fc517f3ee40 4 rocksdb: Fast CRC32 supported: 0 -2351> 2018-02-14 13:23:40.922904 7fc517f3ee40 4 rocksdb: [/home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos7/DIST/centos7/MACHINE_SIZE/huge/release/12.2.2/rpm/el7/BUILD/ceph-12.2.2/src/rocksdb/db/version_set.cc:2609] Recovering from manifest file: MANIFEST-128462 -2350> 2018-02-14 13:23:40.922991 7fc517f3ee40 4 rocksdb: [/home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos7/DIST/centos7/MACHINE_SIZE/huge/release/12.2.2/rpm/el7/BUILD/ceph-12.2.2/src/rocksdb/db/column_family.cc:407] --------------- Options for column family [default]: -2349> 2018-02-14 13:23:40.922999 7fc517f3ee40 4 rocksdb: Options.comparator: leveldb.BytewiseComparator -2348> 2018-02-14 13:23:40.923001 7fc517f3ee40 4 rocksdb: Options.merge_operator: -2347> 2018-02-14 13:23:40.923002 7fc517f3ee40 4 rocksdb: Options.compaction_filter: None -2346> 2018-02-14 13:23:40.923003 7fc517f3ee40 4 rocksdb: Options.compaction_filter_factory: None -2345> 2018-02-14 13:23:40.923004 7fc517f3ee40 4 rocksdb: Options.memtable_factory: SkipListFactory -2344> 2018-02-14 13:23:40.923005 7fc517f3ee40 4 rocksdb: Options.table_factory: BlockBasedTable -2343> 2018-02-14 13:23:40.923024 7fc517f3ee40 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5587d88d0158) cache_index_and_filter_blocks: 1 cache_index_and_filter_blocks_with_high_priority: 1 pin_l0_filter_and_index_blocks_in_cache: 1 index_type: 0 hash_index_allow_collision: 1 checksum: 1 no_block_cache: 0 block_cache: 0x5587d88f29a8 block_cache_name: LRUCache block_cache_options: capacity : 134217728 num_shard_bits : 4 strict_capacity_limit : 0 high_pri_pool_ratio: 0.000 block_cache_compressed: (nil) persistent_cache: (nil) block_size: 4096 block_size_deviation: 10 block_restart_interval: 16 index_block_restart_interval: 1 filter_policy: rocksdb.BuiltinBloomFilter whole_key_filtering: 1 format_version: 2 -2342> 2018-02-14 13:23:40.923042 7fc517f3ee40 4 rocksdb: Options.write_buffer_size: 33554432 -2341> 2018-02-14 13:23:40.923046 7fc517f3ee40 4 rocksdb: Options.max_write_buffer_number: 2 -2340> 2018-02-14 13:23:40.923047 7fc517f3ee40 4 rocksdb: Options.compression: NoCompression -2339> 2018-02-14 13:23:40.923049 7fc517f3ee40 4 rocksdb: Options.bottommost_compression: Disabled -2338> 2018-02-14 13:23:40.923050 7fc517f3ee40 4 rocksdb: Options.prefix_extractor: nullptr -2337> 2018-02-14 13:23:40.923050 7fc517f3ee40 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr -2336> 2018-02-14 13:23:40.923051 7fc517f3ee40 4 rocksdb: Options.num_levels: 7 -2335> 2018-02-14 13:23:40.923052 7fc517f3ee40 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 -2334> 2018-02-14 13:23:40.923053 7fc517f3ee40 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 -2333> 2018-02-14 13:23:40.923053 7fc517f3ee40 4 rocksdb: Options.compression_opts.window_bits: -14 -2332> 2018-02-14 13:23:40.923054 7fc517f3ee40 4 rocksdb: Options.compression_opts.level: -1 -2331> 2018-02-14 13:23:40.923055 7fc517f3ee40 4 rocksdb: Options.compression_opts.strategy: 0 -2330> 2018-02-14 13:23:40.923056 7fc517f3ee40 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 -2329> 2018-02-14 13:23:40.923057 7fc517f3ee40 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 -2328> 2018-02-14 13:23:40.923057 7fc517f3ee40 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 -2327> 2018-02-14 13:23:40.923058 7fc517f3ee40 4 rocksdb: Options.level0_stop_writes_trigger: 36 -2326> 2018-02-14 13:23:40.923059 7fc517f3ee40 4 rocksdb: Options.target_file_size_base: 67108864 -2325> 2018-02-14 13:23:40.923065 7fc517f3ee40 4 rocksdb: Options.target_file_size_multiplier: 1 -2324> 2018-02-14 13:23:40.923067 7fc517f3ee40 4 rocksdb: Options.max_bytes_for_level_base: 268435456 -2323> 2018-02-14 13:23:40.923068 7fc517f3ee40 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 0 -2322> 2018-02-14 13:23:40.923069 7fc517f3ee40 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 -2321> 2018-02-14 13:23:40.923073 7fc517f3ee40 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 -2320> 2018-02-14 13:23:40.923074 7fc517f3ee40 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 -2319> 2018-02-14 13:23:40.923075 7fc517f3ee40 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 -2318> 2018-02-14 13:23:40.923076 7fc517f3ee40 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 -2317> 2018-02-14 13:23:40.923077 7fc517f3ee40 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 -2316> 2018-02-14 13:23:40.923078 7fc517f3ee40 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 -2315> 2018-02-14 13:23:40.923079 7fc517f3ee40 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 -2314> 2018-02-14 13:23:40.923079 7fc517f3ee40 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 -2313> 2018-02-14 13:23:40.923080 7fc517f3ee40 4 rocksdb: Options.max_compaction_bytes: 1677721600 -2312> 2018-02-14 13:23:40.923081 7fc517f3ee40 4 rocksdb: Options.arena_block_size: 4194304 -2311> 2018-02-14 13:23:40.923082 7fc517f3ee40 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 -2310> 2018-02-14 13:23:40.923083 7fc517f3ee40 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 -2309> 2018-02-14 13:23:40.923084 7fc517f3ee40 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 -2308> 2018-02-14 13:23:40.923085 7fc517f3ee40 4 rocksdb: Options.disable_auto_compactions: 0 -2307> 2018-02-14 13:23:40.923095 7fc517f3ee40 4 rocksdb: Options.compaction_style: kCompactionStyleLevel -2306> 2018-02-14 13:23:40.923097 7fc517f3ee40 4 rocksdb: Options.compaction_pri: kByCompensatedSize -2305> 2018-02-14 13:23:40.923097 7fc517f3ee40 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 -2304> 2018-02-14 13:23:40.923098 7fc517f3ee40 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 -2303> 2018-02-14 13:23:40.923099 7fc517f3ee40 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 -2302> 2018-02-14 13:23:40.923100 7fc517f3ee40 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 -2301> 2018-02-14 13:23:40.923101 7fc517f3ee40 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 -2300> 2018-02-14 13:23:40.923101 7fc517f3ee40 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 -2299> 2018-02-14 13:23:40.923102 7fc517f3ee40 4 rocksdb: Options.table_properties_collectors: -2298> 2018-02-14 13:23:40.923103 7fc517f3ee40 4 rocksdb: Options.inplace_update_support: 0 -2297> 2018-02-14 13:23:40.923104 7fc517f3ee40 4 rocksdb: Options.inplace_update_num_locks: 10000 -2296> 2018-02-14 13:23:40.923105 7fc517f3ee40 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 -2295> 2018-02-14 13:23:40.923106 7fc517f3ee40 4 rocksdb: Options.memtable_huge_page_size: 0 -2294> 2018-02-14 13:23:40.923107 7fc517f3ee40 4 rocksdb: Options.bloom_locality: 0 -2293> 2018-02-14 13:23:40.923108 7fc517f3ee40 4 rocksdb: Options.max_successive_merges: 0 -2292> 2018-02-14 13:23:40.923108 7fc517f3ee40 4 rocksdb: Options.optimize_filters_for_hits: 0 -2291> 2018-02-14 13:23:40.923109 7fc517f3ee40 4 rocksdb: Options.paranoid_file_checks: 0 -2290> 2018-02-14 13:23:40.923110 7fc517f3ee40 4 rocksdb: Options.force_consistency_checks: 0 -2289> 2018-02-14 13:23:40.923117 7fc517f3ee40 4 rocksdb: Options.report_bg_io_stats: 0 -2288> 2018-02-14 13:23:40.923790 7fc517f3ee40 4 rocksdb: [/home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos7/DIST/centos7/MACHINE_SIZE/huge/release/12.2.2/rpm/el7/BUILD/ceph-12.2.2/src/rocksdb/db/version_set.cc:2859] Recovered from manifest file:/var/lib/ceph/mon/ceph-ceph-dal-mon-a1/store.db/MANIFEST-128462 succeeded,manifest_file_number is 128462, next_file_number is 128464, last_sequence is 45590300, log_number is 0,prev_log_number is 0,max_column_family is 0 -2287> 2018-02-14 13:23:40.923800 7fc517f3ee40 4 rocksdb: [/home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos7/DIST/centos7/MACHINE_SIZE/huge/release/12.2.2/rpm/el7/BUILD/ceph-12.2.2/src/rocksdb/db/version_set.cc:2867] Column family [default] (ID 0), log number is 128461 -2286> 2018-02-14 13:23:40.923868 7fc517f3ee40 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1518643420923857, "job": 1, "event": "recovery_started", "log_files": [128463]} -2285> 2018-02-14 13:23:40.923882 7fc517f3ee40 4 rocksdb: [/home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos7/DIST/centos7/MACHINE_SIZE/huge/release/12.2.2/rpm/el7/BUILD/ceph-12.2.2/src/rocksdb/db/db_impl_open.cc:482] Recovering log #128463 mode 2 -2284> 2018-02-14 13:23:40.923964 7fc517f3ee40 5 rocksdb: [/home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos7/DIST/centos7/MACHINE_SIZE/huge/release/12.2.2/rpm/el7/BUILD/ceph-12.2.2/src/rocksdb/db/db_impl_open.cc:815] [default] [WriteLevel0TableForRecovery] Level-0 table #128464: started -2283> 2018-02-14 13:23:41.043862 7fc517f3ee40 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1518643421043844, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 128464, "file_size": 1004, "table_properties": {"data_size": 54, "index_size": 27, "filter_size": 18, "raw_key_size": 30, "raw_average_key_size": 30, "raw_value_size": 8, "raw_average_value_size": 8, "num_data_blocks": 1, "num_entries": 1, "filter_policy_name": "rocksdb.BuiltinBloomFilter", "kDeletedKeys": "0", "kMergeOperands": "0"}} -2282> 2018-02-14 13:23:41.043873 7fc517f3ee40 5 rocksdb: [/home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos7/DIST/centos7/MACHINE_SIZE/huge/release/12.2.2/rpm/el7/BUILD/ceph-12.2.2/src/rocksdb/db/db_impl_open.cc:847] [default] [WriteLevel0TableForRecovery] Level-0 table #128464: 1004 bytes OK -2281> 2018-02-14 13:23:41.043900 7fc517f3ee40 4 rocksdb: [/home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos7/DIST/centos7/MACHINE_SIZE/huge/release/12.2.2/rpm/el7/BUILD/ceph-12.2.2/src/rocksdb/db/version_set.cc:2395] Creating manifest 128465 -2280> 2018-02-14 13:23:41.136089 7fc517f3ee40 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1518643421136085, "job": 1, "event": "recovery_finished"} -2279> 2018-02-14 13:23:41.136278 7fc517f3ee40 5 rocksdb: [/home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos7/DIST/centos7/MACHINE_SIZE/huge/release/12.2.2/rpm/el7/BUILD/ceph-12.2.2/src/rocksdb/db/db_impl_files.cc:307] [JOB 2] Delete /var/lib/ceph/mon/ceph-ceph-dal-mon-a1/store.db//MANIFEST-128462 type=3 #128462 -- OK -2278> 2018-02-14 13:23:41.136308 7fc517f3ee40 5 rocksdb: [/home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos7/DIST/centos7/MACHINE_SIZE/huge/release/12.2.2/rpm/el7/BUILD/ceph-12.2.2/src/rocksdb/db/db_impl_files.cc:307] [JOB 2] Delete /var/lib/ceph/mon/ceph-ceph-dal-mon-a1/store.db//128463.log type=0 #128463 -- OK -2277> 2018-02-14 13:23:41.209366 7fc517f3ee40 4 rocksdb: [/home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos7/DIST/centos7/MACHINE_SIZE/huge/release/12.2.2/rpm/el7/BUILD/ceph-12.2.2/src/rocksdb/db/db_impl_open.cc:1063] DB pointer 0x5587d8c98000 -2276> 2018-02-14 13:23:41.209583 7fc509cf8700 3 rocksdb: [/home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos7/DIST/centos7/MACHINE_SIZE/huge/release/12.2.2/rpm/el7/BUILD/ceph-12.2.2/src/rocksdb/db/db_impl.cc:447] ------- DUMPING STATS ------- -2275> 2018-02-14 13:23:41.209592 7fc509cf8700 3 rocksdb: [/home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos7/DIST/centos7/MACHINE_SIZE/huge/release/12.2.2/rpm/el7/BUILD/ceph-12.2.2/src/rocksdb/db/db_impl.cc:448] ** DB Stats ** Uptime(secs): 0.3 total, 0.3 interval Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s Interval stall: 00:00:0.000 H:M:S, 0.0 percent ** Compaction Stats [default] ** Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop ---------------------------------------------------------------------------------------------------------------------------------------------------------- L0 4/0 69.84 KB 1.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 0.0 0 1 0.120 0 0 L1 1/0 44.67 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0 0 0.000 0 0 Sum 5/0 44.73 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 0.0 0 1 0.120 0 0 Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 0.0 0 1 0.120 0 0 Uptime(secs): 0.3 total, 0.3 interval Flush(GB): cumulative 0.000, interval 0.000 AddFile(GB): cumulative 0.000, interval 0.000 AddFile(Total Files): cumulative 0, interval 0 AddFile(L0 Files): cumulative 0, interval 0 AddFile(Keys): cumulative 0, interval 0 Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.1 seconds Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.1 seconds Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count ** File Read Latency Histogram By Level [default] ** ** Compaction Stats [default] ** Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop ---------------------------------------------------------------------------------------------------------------------------------------------------------- L0 4/0 69.84 KB 1.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 0.0 0 1 0.120 0 0 L1 1/0 44.67 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0 0 0.000 0 0 Sum 5/0 44.73 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 0.0 0 1 0.120 0 0 Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0 0 0.000 0 0 Uptime(secs): 0.3 total, 0.3 interval Flush(GB): cumulative 0.000, interval 0.000 AddFile(GB): cumulative 0.000, interval 0.000 AddFile(Total Files): cumulative 0, interval 0 AddFile(L0 Files): cumulative 0, interval 0 AddFile(Keys): cumulative 0, interval 0 Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.1 seconds Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count ** File Read Latency Histogram By Level [default] ** -2274> 2018-02-14 13:23:41.209692 7fc509cf8700 4 rocksdb: [/home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos7/DIST/centos7/MACHINE_SIZE/huge/release/12.2.2/rpm/el7/BUILD/ceph-12.2.2/src/rocksdb/db/compaction_job.cc:1403] [default] [JOB 3] Compacting 4@0 + 1@1 files to L1, score 1.00 -2273> 2018-02-14 13:23:41.209707 7fc509cf8700 4 rocksdb: [/home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos7/DIST/centos7/MACHINE_SIZE/huge/release/12.2.2/rpm/el7/BUILD/ceph-12.2.2/src/rocksdb/db/compaction_job.cc:1407] [default] Compaction start summary: Base version 2 Base level 0, inputs: [128464(1004B) 128461(1036B) 128458(66KB) 128455(1036B)], [128453(44MB)] -2272> 2018-02-14 13:23:41.209751 7fc509cf8700 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1518643421209713, "job": 3, "event": "compaction_started", "files_L0": [128464, 128461, 128458, 128455], "files_L1": [128453], "score": 1, "input_data_size": 46907561} -2271> 2018-02-14 13:23:41.210191 7fc517f3ee40 10 obtain_monmap -2270> 2018-02-14 13:23:41.210324 7fc517f3ee40 10 obtain_monmap read last committed monmap ver 1 -2269> 2018-02-14 13:23:41.210769 7fc50acfa700 2 Event(0x5587d8ba4a80 nevent=5000 time_id=1).set_owner idx=0 owner=140484266665728 -2268> 2018-02-14 13:23:41.210771 7fc50b4fb700 2 Event(0x5587d8ba4880 nevent=5000 time_id=1).set_owner idx=1 owner=140484275058432 -2267> 2018-02-14 13:23:41.210804 7fc511507700 2 Event(0x5587d8946280 nevent=5000 time_id=1).set_owner idx=2 owner=140484375770880 -2266> 2018-02-14 13:23:41.210865 7fc517f3ee40 0 starting mon.ceph-dal-mon-a1 rank 0 at public addr 10.255.0.30:6789/0 at bind addr 10.255.0.30:6789/0 mon_data /var/lib/ceph/mon/ceph-ceph-dal-mon-a1 fsid 39fac7f7-f5a8-4e84-b9bb-b0dfffd79c92 -2265> 2018-02-14 13:23:41.210939 7fc517f3ee40 1 -- 10.255.0.30:6789/0 learned_addr learned my addr 10.255.0.30:6789/0 -2264> 2018-02-14 13:23:41.210947 7fc517f3ee40 1 -- 10.255.0.30:6789/0 _finish_bind bind my_inst.addr is 10.255.0.30:6789/0 -2263> 2018-02-14 13:23:41.210972 7fc517f3ee40 0 starting mon.ceph-dal-mon-a1 rank 0 at 10.255.0.30:6789/0 mon_data /var/lib/ceph/mon/ceph-ceph-dal-mon-a1 fsid 39fac7f7-f5a8-4e84-b9bb-b0dfffd79c92 -2262> 2018-02-14 13:23:41.211004 7fc517f3ee40 5 adding auth protocol: cephx -2261> 2018-02-14 13:23:41.211009 7fc517f3ee40 5 adding auth protocol: cephx -2260> 2018-02-14 13:23:41.211046 7fc517f3ee40 10 log_channel(cluster) update_config to_monitors: true to_syslog: false syslog_facility: daemon prio: info to_graylog: false graylog_host: 127.0.0.1 graylog_port: 12201) -2259> 2018-02-14 13:23:41.211052 7fc517f3ee40 10 log_channel(audit) update_config to_monitors: true to_syslog: false syslog_facility: local0 prio: info to_graylog: false graylog_host: 127.0.0.1 graylog_port: 12201) -2258> 2018-02-14 13:23:41.211482 7fc517f3ee40 1 mon.ceph-dal-mon-a1@-1(probing) e1 preinit fsid 39fac7f7-f5a8-4e84-b9bb-b0dfffd79c92 -2257> 2018-02-14 13:23:41.211565 7fc517f3ee40 10 mon.ceph-dal-mon-a1@-1(probing) e1 check_fsid cluster_uuid contains '39fac7f7-f5a8-4e84-b9bb-b0dfffd79c92' -2256> 2018-02-14 13:23:41.211585 7fc517f3ee40 10 mon.ceph-dal-mon-a1@-1(probing) e1 features compat={},rocompat={},incompat={1=initial feature set (~v.18),3=single paxos with k/v store (v0.?),4=support erasure code pools,5=new-style osdmap encoding,6=support isa/lrc erasure code,7=support shec erasure code,8=support monmap features,9=luminous ondisk layout} -2255> 2018-02-14 13:23:41.211599 7fc517f3ee40 10 mon.ceph-dal-mon-a1@-1(probing) e1 calc_quorum_requirements required_features 153140804152475648 -2254> 2018-02-14 13:23:41.211602 7fc517f3ee40 10 mon.ceph-dal-mon-a1@-1(probing) e1 required_features 153140804152475648 -2253> 2018-02-14 13:23:41.211613 7fc517f3ee40 10 mon.ceph-dal-mon-a1@-1(probing) e1 has_ever_joined = 1 -2252> 2018-02-14 13:23:41.211648 7fc517f3ee40 10 mon.ceph-dal-mon-a1@-1(probing) e1 sync_last_committed_floor 0 -2251> 2018-02-14 13:23:41.211653 7fc517f3ee40 10 mon.ceph-dal-mon-a1@-1(probing) e1 init_paxos -2250> 2018-02-14 13:23:41.211695 7fc517f3ee40 1 mon.ceph-dal-mon-a1@-1(probing).mds e0 Unable to load 'last_metadata' -2249> 2018-02-14 13:23:41.211733 7fc517f3ee40 10 mon.ceph-dal-mon-a1@-1(probing).health init -2248> 2018-02-14 13:23:41.211738 7fc517f3ee40 10 mon.ceph-dal-mon-a1@-1(probing) e1 refresh_from_paxos -2247> 2018-02-14 13:23:41.211784 7fc517f3ee40 1 mon.ceph-dal-mon-a1@-1(probing).paxosservice(pgmap 1..2) refresh upgraded, format 0 -> 1 -2246> 2018-02-14 13:23:41.211792 7fc517f3ee40 1 mon.ceph-dal-mon-a1@-1(probing).pg v0 on_upgrade discarding in-core PGMap -2245> 2018-02-14 13:23:41.211817 7fc517f3ee40 10 mon.ceph-dal-mon-a1@-1(probing).pg v0 update_from_paxos deleted, clearing in-memory PGMap -2244> 2018-02-14 13:23:41.211858 7fc517f3ee40 10 mon.ceph-dal-mon-a1@-1(probing).mds e0 update_from_paxos version 1, my e 0 -2243> 2018-02-14 13:23:41.211900 7fc517f3ee40 10 mon.ceph-dal-mon-a1@-1(probing).mds e0 update_from_paxos got 1 -2242> 2018-02-14 13:23:41.211918 7fc517f3ee40 4 mon.ceph-dal-mon-a1@-1(probing).mds e1 new map -2241> 2018-02-14 13:23:41.211923 7fc517f3ee40 0 mon.ceph-dal-mon-a1@-1(probing).mds e1 print_map e1 enable_multiple, ever_enabled_multiple: 0,0 compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=file layout v2} legacy client fscid: -1 No filesystems configured -2240> 2018-02-14 13:23:41.211940 7fc517f3ee40 10 mon.ceph-dal-mon-a1@-1(probing).mds e1 update_logger -2239> 2018-02-14 13:23:41.212027 7fc517f3ee40 15 mon.ceph-dal-mon-a1@-1(probing).osd e0 update_from_paxos paxos e 19702, my e 0 -2238> 2018-02-14 13:23:41.212137 7fc517f3ee40 7 mon.ceph-dal-mon-a1@-1(probing).osd e0 update_from_paxos loading latest full map e19702 -2237> 2018-02-14 13:23:41.212333 7fc517f3ee40 7 mon.ceph-dal-mon-a1@-1(probing).osd e19702 update_from_paxos loading creating_pgs last_scan_epoch 19701 with 0 pgs -2236> 2018-02-14 13:23:41.212348 7fc517f3ee40 10 mon.ceph-dal-mon-a1@-1(probing).osd e19702 update_from_paxos pgservice is mgrstat -2235> 2018-02-14 13:23:41.212353 7fc517f3ee40 10 mon.ceph-dal-mon-a1@-1(probing).osd e19702 adding osd.0 to down_pending_out map -2234> 2018-02-14 13:23:41.212356 7fc517f3ee40 10 mon.ceph-dal-mon-a1@-1(probing).osd e19702 adding osd.1 to down_pending_out map -2233> 2018-02-14 13:23:41.212358 7fc517f3ee40 10 mon.ceph-dal-mon-a1@-1(probing).osd e19702 adding osd.2 to down_pending_out map -2232> 2018-02-14 13:23:41.212359 7fc517f3ee40 10 mon.ceph-dal-mon-a1@-1(probing).osd e19702 adding osd.26 to down_pending_out map -2231> 2018-02-14 13:23:41.212362 7fc517f3ee40 10 mon.ceph-dal-mon-a1@-1(probing).osd e19702 check_osdmap_subs -2230> 2018-02-14 13:23:41.212369 7fc517f3ee40 10 mon.ceph-dal-mon-a1@-1(probing).osd e19702 share_map_with_random_osd no up osd on our session map -2229> 2018-02-14 13:23:41.212372 7fc517f3ee40 10 mon.ceph-dal-mon-a1@-1(probing).osd e19702 update_logger -2228> 2018-02-14 13:23:41.212380 7fc517f3ee40 0 mon.ceph-dal-mon-a1@-1(probing).osd e19702 crush map has features 288514394856620032, adjusting msgr requires -2227> 2018-02-14 13:23:41.212382 7fc517f3ee40 0 mon.ceph-dal-mon-a1@-1(probing).osd e19702 crush map has features 288514394856620032, adjusting msgr requires -2226> 2018-02-14 13:23:41.212385 7fc517f3ee40 0 mon.ceph-dal-mon-a1@-1(probing).osd e19702 crush map has features 1009090335235915776, adjusting msgr requires -2225> 2018-02-14 13:23:41.212387 7fc517f3ee40 0 mon.ceph-dal-mon-a1@-1(probing).osd e19702 crush map has features 288514119978713088, adjusting msgr requires -2224> 2018-02-14 13:23:41.212412 7fc517f3ee40 20 queue 0x5587d8cb31e0 4 [0,128) -2223> 2018-02-14 13:23:41.212415 7fc517f3ee40 20 queue 0x5587d8cb31e0 5 [0,64) -2222> 2018-02-14 13:23:41.212416 7fc517f3ee40 20 queue 0x5587d8cb31e0 6 [0,256) -2221> 2018-02-14 13:23:41.212417 7fc517f3ee40 10 mon.ceph-dal-mon-a1@-1(probing).osd e19702 start_mapping started mapping job 0x5587d8cb31e0 at 2018-02-14 13:23:41.212389 -2220> 2018-02-14 13:23:41.212496 7fc517f3ee40 10 mon.ceph-dal-mon-a1@-1(probing).log v594874 update_from_paxos -2219> 2018-02-14 13:23:41.212504 7fc517f3ee40 10 mon.ceph-dal-mon-a1@-1(probing).log v594874 update_from_paxos version 594874 summary v 0 -2218> 2018-02-14 13:23:41.212513 7fc517f3ee40 10 mon.ceph-dal-mon-a1@-1(probing).log v594874 update_from_paxos latest full 594873 -2217> 2018-02-14 13:23:41.212552 7fc517f3ee40 7 mon.ceph-dal-mon-a1@-1(probing).log v594874 update_from_paxos loading summary e594873 -2216> 2018-02-14 13:23:41.212641 7fc517f3ee40 7 mon.ceph-dal-mon-a1@-1(probing).log v594874 update_from_paxos loaded summary e594873 -2215> 2018-02-14 13:23:41.212658 7fc517f3ee40 7 mon.ceph-dal-mon-a1@-1(probing).log v594874 update_from_paxos applying incremental log 594874 2018-02-14 13:09:57.805123 mon.ceph-dal-mon-a1 mon.0 10.255.0.30:6789/0 9 : cluster [WRN] Health check update: 1/3 mons down, quorum ceph-dal-mon-a1,ceph-dal-mon-a3 (MON_DOWN) -2214> 2018-02-14 13:23:41.212674 7fc517f3ee40 20 mon.ceph-dal-mon-a1@-1(probing).log v594874 update_from_paxos logging for channel 'cluster' to file '/var/log/ceph/ceph.log' -2213> 2018-02-14 13:23:41.212689 7fc517f3ee40 15 mon.ceph-dal-mon-a1@-1(probing).log v594874 update_from_paxos logging for 1 channels -2212> 2018-02-14 13:23:41.212693 7fc517f3ee40 15 mon.ceph-dal-mon-a1@-1(probing).log v594874 update_from_paxos channel 'cluster' logging 176 bytes -2211> 2018-02-14 13:23:41.212720 7fc517f3ee40 10 mon.ceph-dal-mon-a1@-1(probing).log v594874 check_subs -2210> 2018-02-14 13:23:41.212823 7fc517f3ee40 1 mon.ceph-dal-mon-a1@-1(probing).paxosservice(auth 2501..2644) refresh upgraded, format 0 -> 2 -2209> 2018-02-14 13:23:41.212832 7fc517f3ee40 10 mon.ceph-dal-mon-a1@-1(probing).auth v2644 update_from_paxos -2208> 2018-02-14 13:23:41.212851 7fc517f3ee40 10 mon.ceph-dal-mon-a1@-1(probing).auth v2644 update_from_paxos version 2644 keys ver 0 latest 2641 -2207> 2018-02-14 13:23:41.212886 7fc517f3ee40 7 mon.ceph-dal-mon-a1@-1(probing).auth v2644 update_from_paxos loading summary e 2641 -2206> 2018-02-14 13:23:41.212891 7fc517f3ee40 7 mon.ceph-dal-mon-a1@-1(probing).auth v2644 update_from_paxos latest length 6195 -2205> 2018-02-14 13:23:41.213687 7fc517f3ee40 10 mon.ceph-dal-mon-a1@-1(probing).auth v2644 update_from_paxos key server version 2641 -2204> 2018-02-14 13:23:41.213707 7fc517f3ee40 20 mon.ceph-dal-mon-a1@-1(probing).auth v2644 update_from_paxos walking through version 2642 len 19 -2203> 2018-02-14 13:23:41.213720 7fc517f3ee40 20 mon.ceph-dal-mon-a1@-1(probing).auth v2644 update_from_paxos walking through version 2643 len 801 -2202> 2018-02-14 13:23:41.213897 7fc517f3ee40 20 mon.ceph-dal-mon-a1@-1(probing).auth v2644 update_from_paxos walking through version 2644 len 19 -2201> 2018-02-14 13:23:41.213903 7fc517f3ee40 10 mon.ceph-dal-mon-a1@-1(probing).auth v2644 update_from_paxos() last_allocated_id=1724096 max_global_id=1724096 format_version 2 -2200> 2018-02-14 13:23:41.213953 7fc517f3ee40 4 mon.ceph-dal-mon-a1@-1(probing).mgr e0 loading version 422 -2199> 2018-02-14 13:23:41.213989 7fc517f3ee40 4 mon.ceph-dal-mon-a1@-1(probing).mgr e422 active server: -(1582487) -2198> 2018-02-14 13:23:41.214022 7fc517f3ee40 4 mon.ceph-dal-mon-a1@-1(probing).mgr e422 mkfs or daemon transitioned to available, loading commands -2197> 2018-02-14 13:23:41.214082 7fc517f3ee40 4 mgrc handle_mgr_map Got map version 422 -2196> 2018-02-14 13:23:41.214089 7fc517f3ee40 4 mgrc handle_mgr_map Active mgr is now - -2195> 2018-02-14 13:23:41.214092 7fc517f3ee40 4 mgrc reconnect No active mgr available yet -2194> 2018-02-14 13:23:41.214115 7fc517f3ee40 10 mon.ceph-dal-mon-a1@-1(probing).mgrstat 4284427 -2193> 2018-02-14 13:23:41.214182 7fc517f3ee40 10 mon.ceph-dal-mon-a1@-1(probing).mgrstat update_from_paxos v4284427 service_map e1 -2192> 2018-02-14 13:23:41.214188 7fc517f3ee40 10 mon.ceph-dal-mon-a1@-1(probing).mgrstat check_subs -2191> 2018-02-14 13:23:41.214189 7fc517f3ee40 20 mon.ceph-dal-mon-a1@-1(probing).mgrstat update_logger -2190> 2018-02-14 13:23:41.214210 7fc517f3ee40 10 mon.ceph-dal-mon-a1@-1(probing).health update_from_paxos -2189> 2018-02-14 13:23:41.214233 7fc517f3ee40 20 mon.ceph-dal-mon-a1@-1(probing).health dump:{ "quorum_health": {}, "leader_health": { "MON_DOWN": { "severity": "HEALTH_WARN", "summary": { "message": "1/3 mons down, quorum ceph-dal-mon-a1,ceph-dal-mon-a3" }, "detail": [ { "message": "mon.ceph-dal-mon-a2 (rank 1) addr 10.255.0.31:6789/0 is down (out of quorum)" } ] } } } -2188> 2018-02-14 13:23:41.214280 7fc517f3ee40 10 mon.ceph-dal-mon-a1@-1(probing).health(0) init -2187> 2018-02-14 13:23:41.214382 7fc517f3ee40 2 auth: KeyRing::load: loaded key file /var/lib/ceph/mon/ceph-ceph-dal-mon-a1/keyring -2186> 2018-02-14 13:23:41.214389 7fc517f3ee40 5 asok(0x5587d89481c0) register_command mon_status hook 0x5587d88d4b20 -2185> 2018-02-14 13:23:41.214396 7fc517f3ee40 5 asok(0x5587d89481c0) register_command quorum_status hook 0x5587d88d4b20 -2184> 2018-02-14 13:23:41.214400 7fc517f3ee40 5 asok(0x5587d89481c0) register_command sync_force hook 0x5587d88d4b20 -2183> 2018-02-14 13:23:41.214404 7fc517f3ee40 5 asok(0x5587d89481c0) register_command add_bootstrap_peer_hint hook 0x5587d88d4b20 -2182> 2018-02-14 13:23:41.214408 7fc517f3ee40 5 asok(0x5587d89481c0) register_command quorum enter hook 0x5587d88d4b20 -2181> 2018-02-14 13:23:41.214412 7fc517f3ee40 5 asok(0x5587d89481c0) register_command quorum exit hook 0x5587d88d4b20 -2180> 2018-02-14 13:23:41.214416 7fc517f3ee40 5 asok(0x5587d89481c0) register_command ops hook 0x5587d88d4b20 -2179> 2018-02-14 13:23:41.214420 7fc517f3ee40 5 asok(0x5587d89481c0) register_command sessions hook 0x5587d88d4b20 -2178> 2018-02-14 13:23:41.214433 7fc517f3ee40 1 -- 10.255.0.30:6789/0 start start -2177> 2018-02-14 13:23:41.214440 7fc517f3ee40 1 -- - start start -2176> 2018-02-14 13:23:41.214443 7fc517f3ee40 2 mon.ceph-dal-mon-a1@-1(probing) e1 init -2175> 2018-02-14 13:23:41.214578 7fc517f3ee40 1 Processor -- start -2174> 2018-02-14 13:23:41.214612 7fc50e501700 20 _process 0x5587d8cb31e0 4 [0,128) -2173> 2018-02-14 13:23:41.214627 7fc50f503700 20 _process 0x5587d8cb31e0 5 [0,64) -2172> 2018-02-14 13:23:41.214664 7fc517f3ee40 1 Processor -- start -2171> 2018-02-14 13:23:41.214677 7fc50ed02700 20 _process 0x5587d8cb31e0 6 [0,256) -2170> 2018-02-14 13:23:41.214712 7fc517f3ee40 10 mon.ceph-dal-mon-a1@-1(probing) e1 bootstrap -2169> 2018-02-14 13:23:41.214718 7fc517f3ee40 10 mon.ceph-dal-mon-a1@-1(probing) e1 sync_reset_requester -2168> 2018-02-14 13:23:41.214720 7fc517f3ee40 10 mon.ceph-dal-mon-a1@-1(probing) e1 unregister_cluster_logger - not registered -2167> 2018-02-14 13:23:41.214722 7fc517f3ee40 10 mon.ceph-dal-mon-a1@-1(probing) e1 cancel_probe_timeout (none scheduled) -2166> 2018-02-14 13:23:41.214724 7fc517f3ee40 0 mon.ceph-dal-mon-a1@-1(probing) e1 my rank is now 0 (was -1) -2165> 2018-02-14 13:23:41.214727 7fc517f3ee40 1 -- 10.255.0.30:6789/0 shutdown_connections -2164> 2018-02-14 13:23:41.214733 7fc517f3ee40 10 mon.ceph-dal-mon-a1@0(probing) e1 _reset -2163> 2018-02-14 13:23:41.214734 7fc517f3ee40 10 mon.ceph-dal-mon-a1@0(probing) e1 cancel_probe_timeout (none scheduled) -2162> 2018-02-14 13:23:41.214736 7fc517f3ee40 10 mon.ceph-dal-mon-a1@0(probing) e1 timecheck_finish -2161> 2018-02-14 13:23:41.214740 7fc517f3ee40 15 mon.ceph-dal-mon-a1@0(probing) e1 health_tick_stop -2160> 2018-02-14 13:23:41.214742 7fc517f3ee40 15 mon.ceph-dal-mon-a1@0(probing) e1 health_interval_stop -2159> 2018-02-14 13:23:41.214743 7fc517f3ee40 10 mon.ceph-dal-mon-a1@0(probing) e1 scrub_event_cancel -2158> 2018-02-14 13:23:41.214746 7fc517f3ee40 10 mon.ceph-dal-mon-a1@0(probing) e1 scrub_reset -2157> 2018-02-14 13:23:41.214756 7fc517f3ee40 10 mon.ceph-dal-mon-a1@0(probing) e1 cancel_probe_timeout (none scheduled) -2156> 2018-02-14 13:23:41.214759 7fc517f3ee40 10 mon.ceph-dal-mon-a1@0(probing) e1 reset_probe_timeout 0x5587d8ce2be0 after 2 seconds -2155> 2018-02-14 13:23:41.214765 7fc517f3ee40 10 mon.ceph-dal-mon-a1@0(probing) e1 probing other monitors -2154> 2018-02-14 13:23:41.214770 7fc517f3ee40 1 -- 10.255.0.30:6789/0 _send_message--> mon.1 10.255.0.31:6789/0 -- mon_probe(probe 39fac7f7-f5a8-4e84-b9bb-b0dfffd79c92 name ceph-dal-mon-a1) v6 -- ?+0 0x5587d895db80 -2153> 2018-02-14 13:23:41.214800 7fc517f3ee40 1 -- 10.255.0.30:6789/0 --> 10.255.0.31:6789/0 -- mon_probe(probe 39fac7f7-f5a8-4e84-b9bb-b0dfffd79c92 name ceph-dal-mon-a1) v6 -- 0x5587d895db80 con 0 -2152> 2018-02-14 13:23:41.214824 7fc517f3ee40 1 -- 10.255.0.30:6789/0 _send_message--> mon.2 10.255.0.32:6789/0 -- mon_probe(probe 39fac7f7-f5a8-4e84-b9bb-b0dfffd79c92 name ceph-dal-mon-a1) v6 -- ?+0 0x5587d895de00 -2151> 2018-02-14 13:23:41.214841 7fc517f3ee40 1 -- 10.255.0.30:6789/0 --> 10.255.0.32:6789/0 -- mon_probe(probe 39fac7f7-f5a8-4e84-b9bb-b0dfffd79c92 name ceph-dal-mon-a1) v6 -- 0x5587d895de00 con 0 -2150> 2018-02-14 13:23:41.215014 7fc511507700 1 -- 10.255.0.30:6789/0 >> 10.255.0.31:6789/0 conn(0x5587d8f81000 :-1 s=STATE_CONNECTING_RE pgs=0 cs=0 l=0)._process_connection reconnect failed -2149> 2018-02-14 13:23:41.215035 7fc511507700 2 -- 10.255.0.30:6789/0 >> 10.255.0.31:6789/0 conn(0x5587d8f81000 :-1 s=STATE_CONNECTING_RE pgs=0 cs=0 l=0)._process_connection connection refused! -2148> 2018-02-14 13:23:41.215064 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(probing) e1 ms_handle_refused 0x5587d8f81000 10.255.0.31:6789/0 -2147> 2018-02-14 13:23:41.215280 7fc50acfa700 10 mon.ceph-dal-mon-a1@0(probing) e1 ms_get_authorizer for mon -2146> 2018-02-14 13:23:41.215614 7fc50acfa700 5 -- 10.255.0.30:6789/0 >> 10.255.0.32:6789/0 conn(0x5587d8f82800 :-1 s=STATE_CONNECTING_WAIT_CONNECT_REPLY_AUTH pgs=0 cs=0 l=0).handle_connect_reply connect got RETRY_GLOBAL 28 chose new 29 -2145> 2018-02-14 13:23:41.215642 7fc50acfa700 10 mon.ceph-dal-mon-a1@0(probing) e1 ms_get_authorizer for mon -2144> 2018-02-14 13:23:41.216399 7fc50acfa700 5 -- 10.255.0.30:6789/0 >> 10.255.0.32:6789/0 conn(0x5587d8f82800 :-1 s=STATE_CONNECTING_WAIT_CONNECT_REPLY_AUTH pgs=0 cs=0 l=0).handle_connect_reply connect got RETRY_GLOBAL 28 chose new 30 -2143> 2018-02-14 13:23:41.216416 7fc50acfa700 10 mon.ceph-dal-mon-a1@0(probing) e1 ms_get_authorizer for mon -2142> 2018-02-14 13:23:41.216906 7fc50acfa700 0 -- 10.255.0.30:6789/0 >> 10.255.0.32:6789/0 conn(0x5587d8f82800 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=51044 cs=1 l=0).process missed message? skipped from seq 0 to 386006847 -2141> 2018-02-14 13:23:41.216920 7fc50acfa700 5 -- 10.255.0.30:6789/0 >> 10.255.0.32:6789/0 conn(0x5587d8f82800 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=51044 cs=1 l=0). rx mon.2 seq 386006847 0x5587d895de00 mon_probe(reply 39fac7f7-f5a8-4e84-b9bb-b0dfffd79c92 name ceph-dal-mon-a3 paxos( fc 4935163 lc 4935904 )) v6 -2140> 2018-02-14 13:23:41.216950 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== mon.2 10.255.0.32:6789/0 386006847 ==== mon_probe(reply 39fac7f7-f5a8-4e84-b9bb-b0dfffd79c92 name ceph-dal-mon-a3 paxos( fc 4935163 lc 4935904 )) v6 ==== 555+0+0 (1286339127 0 0) 0x5587d895de00 con 0x5587d8f82800 -2139> 2018-02-14 13:23:41.216976 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(probing) e1 _ms_dispatch new session 0x5587d891c480 MonSession(mon.2 10.255.0.32:6789/0 is open ) features 0x1ffddff8eea4fffb -2138> 2018-02-14 13:23:41.216983 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(probing) e1 _ms_dispatch setting monitor caps on this connection -2137> 2018-02-14 13:23:41.216987 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(probing) e1 caps allow * -2136> 2018-02-14 13:23:41.216993 7fc50dd00700 20 is_capable service=mon command= read on cap allow * -2135> 2018-02-14 13:23:41.216995 7fc50dd00700 20 allow so far , doing grant allow * -2134> 2018-02-14 13:23:41.216996 7fc50dd00700 20 allow all -2133> 2018-02-14 13:23:41.216998 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(probing) e1 handle_probe mon_probe(reply 39fac7f7-f5a8-4e84-b9bb-b0dfffd79c92 name ceph-dal-mon-a3 paxos( fc 4935163 lc 4935904 )) v6 -2132> 2018-02-14 13:23:41.217002 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(probing) e1 handle_probe_reply mon.2 10.255.0.32:6789/0mon_probe(reply 39fac7f7-f5a8-4e84-b9bb-b0dfffd79c92 name ceph-dal-mon-a3 paxos( fc 4935163 lc 4935904 )) v6 -2131> 2018-02-14 13:23:41.217008 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(probing) e1 monmap is e1: 3 mons at {ceph-dal-mon-a1=10.255.0.30:6789/0,ceph-dal-mon-a2=10.255.0.31:6789/0,ceph-dal-mon-a3=10.255.0.32:6789/0} -2130> 2018-02-14 13:23:41.217025 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(probing) e1 peer name is ceph-dal-mon-a3 -2129> 2018-02-14 13:23:41.217027 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(probing) e1 mon.ceph-dal-mon-a3 is outside the quorum -2128> 2018-02-14 13:23:41.217029 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(probing) e1 outside_quorum now ceph-dal-mon-a1,ceph-dal-mon-a3, need 2 -2127> 2018-02-14 13:23:41.217032 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(probing) e1 that's enough to form a new quorum, calling election -2126> 2018-02-14 13:23:41.217034 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(probing) e1 start_election -2125> 2018-02-14 13:23:41.217035 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(probing) e1 _reset -2124> 2018-02-14 13:23:41.217036 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(probing) e1 cancel_probe_timeout 0x5587d8ce2be0 -2123> 2018-02-14 13:23:41.217038 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(probing) e1 timecheck_finish -2122> 2018-02-14 13:23:41.217040 7fc50dd00700 15 mon.ceph-dal-mon-a1@0(probing) e1 health_tick_stop -2121> 2018-02-14 13:23:41.217041 7fc50dd00700 15 mon.ceph-dal-mon-a1@0(probing) e1 health_interval_stop -2120> 2018-02-14 13:23:41.217042 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(probing) e1 scrub_event_cancel -2119> 2018-02-14 13:23:41.217044 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(probing) e1 scrub_reset -2118> 2018-02-14 13:23:41.217057 7fc50dd00700 0 log_channel(cluster) log [INF] : mon.ceph-dal-mon-a1 calling new monitor election -2117> 2018-02-14 13:23:41.217060 7fc50dd00700 10 log_client _send_to_monlog to self -2116> 2018-02-14 13:23:41.217061 7fc50dd00700 10 log_client log_queue is 1 last_log 1 sent 0 num 1 unsent 1 sending 1 -2115> 2018-02-14 13:23:41.217063 7fc50dd00700 10 log_client will send 2018-02-14 13:23:41.217059 mon.ceph-dal-mon-a1 mon.0 10.255.0.30:6789/0 1 : cluster [INF] mon.ceph-dal-mon-a1 calling new monitor election -2114> 2018-02-14 13:23:41.217085 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.30:6789/0 -- log(1 entries from seq 1 at 2018-02-14 13:23:41.217059) v1 -- 0x5587d88dad00 con 0 -2113> 2018-02-14 13:23:41.217099 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing).elector(0) start -- can i be leader? -2112> 2018-02-14 13:23:41.217130 7fc50dd00700 1 mon.ceph-dal-mon-a1@0(electing).elector(191) init, last seen epoch 191, mid-election, bumping -2111> 2018-02-14 13:23:41.219814 7fc50ed02700 10 osdmap epoch 19702 mapping took 0.007424 seconds -2110> 2018-02-14 13:23:41.219824 7fc50ed02700 10 mon.ceph-dal-mon-a1@0(electing).osd e19702 update_creating_pgs 0 pgs creating, 0 pools in queue -2109> 2018-02-14 13:23:41.247066 7fc50b4fb700 1 -- 10.255.0.30:6789/0 >> - conn(0x5587d8eab800 :6789 s=STATE_ACCEPTING pgs=0 cs=0 l=0)._process_connection sd=27 - -2108> 2018-02-14 13:23:41.247295 7fc50b4fb700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_verify_authorizer 10.255.0.105:0/1753894484 client protocol 0 -2107> 2018-02-14 13:23:41.247335 7fc50b4fb700 2 -- 10.255.0.30:6789/0 >> 10.255.0.105:0/1753894484 conn(0x5587d8eab800 :6789 s=STATE_ACCEPTING_WAIT_SEQ pgs=1131 cs=1 l=1).handle_connect_msg accept write reply msg done -2106> 2018-02-14 13:23:41.247426 7fc50b4fb700 2 -- 10.255.0.30:6789/0 >> 10.255.0.105:0/1753894484 conn(0x5587d8eab800 :6789 s=STATE_ACCEPTING_WAIT_SEQ pgs=1131 cs=1 l=1)._process_connection accept get newly_acked_seq 0 -2105> 2018-02-14 13:23:41.247530 7fc50b4fb700 5 -- 10.255.0.30:6789/0 >> 10.255.0.105:0/1753894484 conn(0x5587d8eab800 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1131 cs=1 l=1). rx client.1684214 seq 1 0x5587d88db180 auth(proto 0 35 bytes epoch 1) v1 -2104> 2018-02-14 13:23:41.251678 7fc50acfa700 1 -- 10.255.0.30:6789/0 >> - conn(0x5587d8ead000 :6789 s=STATE_ACCEPTING pgs=0 cs=0 l=0)._process_connection sd=37 - -2103> 2018-02-14 13:23:41.251920 7fc50acfa700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_verify_authorizer 10.255.0.106:0/3270640683 client protocol 0 -2102> 2018-02-14 13:23:41.251952 7fc50acfa700 2 -- 10.255.0.30:6789/0 >> 10.255.0.106:0/3270640683 conn(0x5587d8ead000 :6789 s=STATE_ACCEPTING_WAIT_SEQ pgs=529 cs=1 l=1).handle_connect_msg accept write reply msg done -2101> 2018-02-14 13:23:41.252065 7fc50acfa700 2 -- 10.255.0.30:6789/0 >> 10.255.0.106:0/3270640683 conn(0x5587d8ead000 :6789 s=STATE_ACCEPTING_WAIT_SEQ pgs=529 cs=1 l=1)._process_connection accept get newly_acked_seq 0 -2100> 2018-02-14 13:23:41.252193 7fc50acfa700 5 -- 10.255.0.30:6789/0 >> 10.255.0.106:0/3270640683 conn(0x5587d8ead000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=529 cs=1 l=1). rx client.1684262 seq 1 0x5587d88da880 auth(proto 0 35 bytes epoch 1) v1 -2099> 2018-02-14 13:23:41.416015 7fc511507700 1 -- 10.255.0.30:6789/0 >> 10.255.0.31:6789/0 conn(0x5587d8f81000 :-1 s=STATE_CONNECTING_RE pgs=0 cs=0 l=0)._process_connection reconnect failed -2098> 2018-02-14 13:23:41.416045 7fc511507700 2 -- 10.255.0.30:6789/0 >> 10.255.0.31:6789/0 conn(0x5587d8f81000 :-1 s=STATE_CONNECTING_RE pgs=0 cs=0 l=0)._process_connection connection refused! -2097> 2018-02-14 13:23:41.453818 7fc511507700 1 -- 10.255.0.30:6789/0 >> - conn(0x5587d8f2d800 :6789 s=STATE_ACCEPTING pgs=0 cs=0 l=0)._process_connection sd=38 - -2096> 2018-02-14 13:23:41.454031 7fc511507700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_verify_authorizer 10.255.0.51:0/3360953369 client protocol 0 -2095> 2018-02-14 13:23:41.454064 7fc511507700 2 -- 10.255.0.30:6789/0 >> 10.255.0.51:0/3360953369 conn(0x5587d8f2d800 :6789 s=STATE_ACCEPTING_WAIT_SEQ pgs=1139 cs=1 l=1).handle_connect_msg accept write reply msg done -2094> 2018-02-14 13:23:41.454201 7fc511507700 2 -- 10.255.0.30:6789/0 >> 10.255.0.51:0/3360953369 conn(0x5587d8f2d800 :6789 s=STATE_ACCEPTING_WAIT_SEQ pgs=1139 cs=1 l=1)._process_connection accept get newly_acked_seq 0 -2093> 2018-02-14 13:23:41.454307 7fc511507700 5 -- 10.255.0.30:6789/0 >> 10.255.0.51:0/3360953369 conn(0x5587d8f2d800 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1139 cs=1 l=1). rx client.1684205 seq 1 0x5587d88db600 auth(proto 0 35 bytes epoch 1) v1 -2092> 2018-02-14 13:23:41.515356 7fc50b4fb700 1 -- 10.255.0.30:6789/0 >> - conn(0x5587d8f2c000 :6789 s=STATE_ACCEPTING pgs=0 cs=0 l=0)._process_connection sd=39 - -2091> 2018-02-14 13:23:41.515575 7fc50b4fb700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_verify_authorizer 10.255.0.106:0/1436590248 client protocol 0 -2090> 2018-02-14 13:23:41.515601 7fc50b4fb700 2 -- 10.255.0.30:6789/0 >> 10.255.0.106:0/1436590248 conn(0x5587d8f2c000 :6789 s=STATE_ACCEPTING_WAIT_SEQ pgs=442 cs=1 l=1).handle_connect_msg accept write reply msg done -2089> 2018-02-14 13:23:41.515704 7fc50b4fb700 2 -- 10.255.0.30:6789/0 >> 10.255.0.106:0/1436590248 conn(0x5587d8f2c000 :6789 s=STATE_ACCEPTING_WAIT_SEQ pgs=442 cs=1 l=1)._process_connection accept get newly_acked_seq 0 -2088> 2018-02-14 13:23:41.515860 7fc50b4fb700 5 -- 10.255.0.30:6789/0 >> 10.255.0.106:0/1436590248 conn(0x5587d8f2c000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=442 cs=1 l=1). rx client.? seq 1 0x5587d88db840 auth(proto 0 35 bytes epoch 0) v1 -2087> 2018-02-14 13:23:41.585325 7fc50acfa700 1 -- 10.255.0.30:6789/0 >> - conn(0x5587d8d40000 :6789 s=STATE_ACCEPTING pgs=0 cs=0 l=0)._process_connection sd=40 - -2086> 2018-02-14 13:23:41.585540 7fc50acfa700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_verify_authorizer 10.255.0.51:0/3461880882 client protocol 0 -2085> 2018-02-14 13:23:41.585562 7fc50acfa700 2 -- 10.255.0.30:6789/0 >> 10.255.0.51:0/3461880882 conn(0x5587d8d40000 :6789 s=STATE_ACCEPTING_WAIT_SEQ pgs=428 cs=1 l=1).handle_connect_msg accept write reply msg done -2084> 2018-02-14 13:23:41.585673 7fc50acfa700 2 -- 10.255.0.30:6789/0 >> 10.255.0.51:0/3461880882 conn(0x5587d8d40000 :6789 s=STATE_ACCEPTING_WAIT_SEQ pgs=428 cs=1 l=1)._process_connection accept get newly_acked_seq 0 -2083> 2018-02-14 13:23:41.585764 7fc50acfa700 5 -- 10.255.0.30:6789/0 >> 10.255.0.51:0/3461880882 conn(0x5587d8d40000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=428 cs=1 l=1). rx client.? seq 1 0x5587d88dbcc0 auth(proto 0 35 bytes epoch 0) v1 -2082> 2018-02-14 13:23:41.635732 7fc511507700 1 -- 10.255.0.30:6789/0 >> - conn(0x5587d8eae800 :6789 s=STATE_ACCEPTING pgs=0 cs=0 l=0)._process_connection sd=41 - -2081> 2018-02-14 13:23:41.635816 7fc511507700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_verify_authorizer 10.255.0.30:0/3376128839 client protocol 0 -2080> 2018-02-14 13:23:41.635839 7fc511507700 2 -- 10.255.0.30:6789/0 >> 10.255.0.30:0/3376128839 conn(0x5587d8eae800 :6789 s=STATE_ACCEPTING_WAIT_SEQ pgs=668 cs=1 l=1).handle_connect_msg accept write reply msg done -2079> 2018-02-14 13:23:41.635857 7fc511507700 2 -- 10.255.0.30:6789/0 >> 10.255.0.30:0/3376128839 conn(0x5587d8eae800 :6789 s=STATE_ACCEPTING_WAIT_SEQ pgs=668 cs=1 l=1)._process_connection accept get newly_acked_seq 0 -2078> 2018-02-14 13:23:41.635913 7fc511507700 5 -- 10.255.0.30:6789/0 >> 10.255.0.30:0/3376128839 conn(0x5587d8eae800 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=668 cs=1 l=1). rx client.1684241 seq 1 0x5587d8ec0000 auth(proto 0 40 bytes epoch 1) v1 -2077> 2018-02-14 13:23:41.784166 7fc50b4fb700 1 -- 10.255.0.30:6789/0 >> - conn(0x5587d8f94000 :6789 s=STATE_ACCEPTING pgs=0 cs=0 l=0)._process_connection sd=42 - -2076> 2018-02-14 13:23:41.784396 7fc50b4fb700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_verify_authorizer 10.255.0.105:0/3391979714 client protocol 0 -2075> 2018-02-14 13:23:41.784423 7fc50b4fb700 2 -- 10.255.0.30:6789/0 >> 10.255.0.105:0/3391979714 conn(0x5587d8f94000 :6789 s=STATE_ACCEPTING_WAIT_SEQ pgs=626 cs=1 l=1).handle_connect_msg accept write reply msg done -2074> 2018-02-14 13:23:41.784510 7fc50b4fb700 2 -- 10.255.0.30:6789/0 >> 10.255.0.105:0/3391979714 conn(0x5587d8f94000 :6789 s=STATE_ACCEPTING_WAIT_SEQ pgs=626 cs=1 l=1)._process_connection accept get newly_acked_seq 0 -2073> 2018-02-14 13:23:41.784611 7fc50b4fb700 5 -- 10.255.0.30:6789/0 >> 10.255.0.105:0/3391979714 conn(0x5587d8f94000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=626 cs=1 l=1). rx client.1684232 seq 1 0x5587d88dba80 auth(proto 0 35 bytes epoch 1) v1 -2072> 2018-02-14 13:23:41.816934 7fc511507700 1 -- 10.255.0.30:6789/0 >> 10.255.0.31:6789/0 conn(0x5587d8f81000 :-1 s=STATE_CONNECTING_RE pgs=0 cs=0 l=0)._process_connection reconnect failed -2071> 2018-02-14 13:23:41.816950 7fc511507700 2 -- 10.255.0.30:6789/0 >> 10.255.0.31:6789/0 conn(0x5587d8f81000 :-1 s=STATE_CONNECTING_RE pgs=0 cs=0 l=0)._process_connection connection refused! -2070> 2018-02-14 13:23:41.862819 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing).elector(192) bump_epoch 192 to 193 -2069> 2018-02-14 13:23:41.905760 7fc509cf8700 4 rocksdb: [/home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos7/DIST/centos7/MACHINE_SIZE/huge/release/12.2.2/rpm/el7/BUILD/ceph-12.2.2/src/rocksdb/db/compaction_job.cc:1116] [default] [JOB 3] Generated table #128469: 6710 keys, 46901291 bytes -2068> 2018-02-14 13:23:41.905811 7fc509cf8700 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1518643421905775, "cf_name": "default", "job": 3, "event": "table_file_creation", "file_number": 128469, "file_size": 46901291, "table_properties": {"data_size": 46631736, "index_size": 137886, "filter_size": 130742, "raw_key_size": 144657, "raw_average_key_size": 21, "raw_value_size": 46438940, "raw_average_value_size": 6920, "num_data_blocks": 3792, "num_entries": 6710, "filter_policy_name": "rocksdb.BuiltinBloomFilter", "kDeletedKeys": "0", "kMergeOperands": "0"}} -2067> 2018-02-14 13:23:41.921775 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 join_election -2066> 2018-02-14 13:23:41.921792 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _reset -2065> 2018-02-14 13:23:41.921795 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 cancel_probe_timeout (none scheduled) -2064> 2018-02-14 13:23:41.921796 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 timecheck_finish -2063> 2018-02-14 13:23:41.921798 7fc50dd00700 15 mon.ceph-dal-mon-a1@0(electing) e1 health_tick_stop -2062> 2018-02-14 13:23:41.921800 7fc50dd00700 15 mon.ceph-dal-mon-a1@0(electing) e1 health_interval_stop -2061> 2018-02-14 13:23:41.921803 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 scrub_event_cancel -2060> 2018-02-14 13:23:41.921805 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 scrub_reset -2059> 2018-02-14 13:23:41.922034 7fc50dd00700 1 -- 10.255.0.30:6789/0 _send_message--> mon.1 10.255.0.31:6789/0 -- election(39fac7f7-f5a8-4e84-b9bb-b0dfffd79c92 propose 193) v7 -- ?+0 0x5587d8959500 -2058> 2018-02-14 13:23:41.922044 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.31:6789/0 -- election(39fac7f7-f5a8-4e84-b9bb-b0dfffd79c92 propose 193) v7 -- 0x5587d8959500 con 0 -2057> 2018-02-14 13:23:41.922065 7fc50dd00700 1 -- 10.255.0.30:6789/0 _send_message--> mon.2 10.255.0.32:6789/0 -- election(39fac7f7-f5a8-4e84-b9bb-b0dfffd79c92 propose 193) v7 -- ?+0 0x5587d895a400 -2056> 2018-02-14 13:23:41.922072 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.32:6789/0 -- election(39fac7f7-f5a8-4e84-b9bb-b0dfffd79c92 propose 193) v7 -- 0x5587d895a400 con 0 -2055> 2018-02-14 13:23:41.922105 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_handle_refused 0x5587d8f81000 10.255.0.31:6789/0 -2054> 2018-02-14 13:23:41.922115 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_handle_refused 0x5587d8f81000 10.255.0.31:6789/0 -2053> 2018-02-14 13:23:41.922120 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== mon.0 10.255.0.30:6789/0 0 ==== log(1 entries from seq 1 at 2018-02-14 13:23:41.217059) v1 ==== 0+0+0 (0 0 0) 0x5587d88dad00 con 0x5587d8d41800 -2052> 2018-02-14 13:23:41.922133 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch new session 0x5587d891ca80 MonSession(mon.0 10.255.0.30:6789/0 is open ) features 0x1ffddff8eea4fffb -2051> 2018-02-14 13:23:41.922138 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch setting monitor caps on this connection -2050> 2018-02-14 13:23:41.922141 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 caps allow * -2049> 2018-02-14 13:23:41.922146 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing).paxos(paxos recovering c 4935163..4935904) is_readable = 0 - now=2018-02-14 13:23:41.922147 lease_expire=0.000000 has v0 lc 4935904 -2048> 2018-02-14 13:23:41.922160 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing).paxos(paxos recovering c 4935163..4935904) is_readable = 0 - now=2018-02-14 13:23:41.922161 lease_expire=0.000000 has v0 lc 4935904 -2047> 2018-02-14 13:23:41.922169 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1684214 10.255.0.105:0/1753894484 1 ==== auth(proto 0 35 bytes epoch 1) v1 ==== 65+0+0 (427311144 0 0) 0x5587d88db180 con 0x5587d8eab800 -2046> 2018-02-14 13:23:41.922181 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch new session 0x5587d891cc00 MonSession(client.1684214 10.255.0.105:0/1753894484 is open ) features 0x1ffddff8eea4fffb -2045> 2018-02-14 13:23:41.922187 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 caps -2044> 2018-02-14 13:23:41.922192 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing) e1 waitlisting message auth(proto 0 35 bytes epoch 1) v1 -2043> 2018-02-14 13:23:41.922196 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1684262 10.255.0.106:0/3270640683 1 ==== auth(proto 0 35 bytes epoch 1) v1 ==== 65+0+0 (986304302 0 0) 0x5587d88da880 con 0x5587d8ead000 -2042> 2018-02-14 13:23:41.922204 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch new session 0x5587d891cd80 MonSession(client.1684262 10.255.0.106:0/3270640683 is open ) features 0x1ffddff8eea4fffb -2041> 2018-02-14 13:23:41.922210 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 caps -2040> 2018-02-14 13:23:41.922212 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing) e1 waitlisting message auth(proto 0 35 bytes epoch 1) v1 -2039> 2018-02-14 13:23:41.922216 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1684205 10.255.0.51:0/3360953369 1 ==== auth(proto 0 35 bytes epoch 1) v1 ==== 65+0+0 (3718329114 0 0) 0x5587d88db600 con 0x5587d8f2d800 -2038> 2018-02-14 13:23:41.922223 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch new session 0x5587d891d080 MonSession(client.1684205 10.255.0.51:0/3360953369 is open ) features 0x1ffddff8eea4fffb -2037> 2018-02-14 13:23:41.922228 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 caps -2036> 2018-02-14 13:23:41.922231 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing) e1 waitlisting message auth(proto 0 35 bytes epoch 1) v1 -2035> 2018-02-14 13:23:41.922234 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.? 10.255.0.106:0/1436590248 1 ==== auth(proto 0 35 bytes epoch 0) v1 ==== 65+0+0 (3478176772 0 0) 0x5587d88db840 con 0x5587d8f2c000 -2034> 2018-02-14 13:23:41.922242 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch new session 0x5587d891d380 MonSession(client.? 10.255.0.106:0/1436590248 is open ) features 0x1ffddff8eea4fffb -2033> 2018-02-14 13:23:41.922247 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 caps -2032> 2018-02-14 13:23:41.922249 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing) e1 waitlisting message auth(proto 0 35 bytes epoch 0) v1 -2031> 2018-02-14 13:23:41.922253 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.? 10.255.0.51:0/3461880882 1 ==== auth(proto 0 35 bytes epoch 0) v1 ==== 65+0+0 (3478176772 0 0) 0x5587d88dbcc0 con 0x5587d8d40000 -2030> 2018-02-14 13:23:41.922259 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch new session 0x5587d891d680 MonSession(client.? 10.255.0.51:0/3461880882 is open ) features 0x1ffddff8eea4fffb -2029> 2018-02-14 13:23:41.922265 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 caps -2028> 2018-02-14 13:23:41.922267 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing) e1 waitlisting message auth(proto 0 35 bytes epoch 0) v1 -2027> 2018-02-14 13:23:41.922270 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1684241 10.255.0.30:0/3376128839 1 ==== auth(proto 0 40 bytes epoch 1) v1 ==== 70+0+0 (1250887805 0 0) 0x5587d8ec0000 con 0x5587d8eae800 -2026> 2018-02-14 13:23:41.922277 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch new session 0x5587d891d980 MonSession(client.1684241 10.255.0.30:0/3376128839 is open ) features 0x1ffddff8eea4fffb -2025> 2018-02-14 13:23:41.922280 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 caps -2024> 2018-02-14 13:23:41.922282 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing) e1 waitlisting message auth(proto 0 40 bytes epoch 1) v1 -2023> 2018-02-14 13:23:41.922285 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1684232 10.255.0.105:0/3391979714 1 ==== auth(proto 0 35 bytes epoch 1) v1 ==== 65+0+0 (568086745 0 0) 0x5587d88dba80 con 0x5587d8f94000 -2022> 2018-02-14 13:23:41.922291 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch new session 0x5587d891dc80 MonSession(client.1684232 10.255.0.105:0/3391979714 is open ) features 0x1ffddff8eea4fffb -2021> 2018-02-14 13:23:41.922295 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 caps -2020> 2018-02-14 13:23:41.922296 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing) e1 waitlisting message auth(proto 0 35 bytes epoch 1) v1 -2019> 2018-02-14 13:23:41.933491 7fc50acfa700 1 -- 10.255.0.30:6789/0 >> - conn(0x5587d8f95800 :6789 s=STATE_ACCEPTING pgs=0 cs=0 l=0)._process_connection sd=43 - -2018> 2018-02-14 13:23:41.933701 7fc50acfa700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_verify_authorizer 10.255.0.53:0/287677392 client protocol 0 -2017> 2018-02-14 13:23:41.933727 7fc50acfa700 2 -- 10.255.0.30:6789/0 >> 10.255.0.53:0/287677392 conn(0x5587d8f95800 :6789 s=STATE_ACCEPTING_WAIT_SEQ pgs=620 cs=1 l=1).handle_connect_msg accept write reply msg done -2016> 2018-02-14 13:23:41.934001 7fc50acfa700 2 -- 10.255.0.30:6789/0 >> 10.255.0.53:0/287677392 conn(0x5587d8f95800 :6789 s=STATE_ACCEPTING_WAIT_SEQ pgs=620 cs=1 l=1)._process_connection accept get newly_acked_seq 0 -2015> 2018-02-14 13:23:41.934151 7fc50acfa700 5 -- 10.255.0.30:6789/0 >> 10.255.0.53:0/287677392 conn(0x5587d8f95800 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=620 cs=1 l=1). rx client.1684235 seq 1 0x5587d8ec0480 auth(proto 0 35 bytes epoch 1) v1 -2014> 2018-02-14 13:23:41.934212 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1684235 10.255.0.53:0/287677392 1 ==== auth(proto 0 35 bytes epoch 1) v1 ==== 65+0+0 (1917725833 0 0) 0x5587d8ec0480 con 0x5587d8f95800 -2013> 2018-02-14 13:23:41.934235 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch new session 0x5587d891df80 MonSession(client.1684235 10.255.0.53:0/287677392 is open ) features 0x1ffddff8eea4fffb -2012> 2018-02-14 13:23:41.934243 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 caps -2011> 2018-02-14 13:23:41.934245 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing) e1 waitlisting message auth(proto 0 35 bytes epoch 1) v1 -2010> 2018-02-14 13:23:41.961204 7fc509cf8700 4 rocksdb: [/home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos7/DIST/centos7/MACHINE_SIZE/huge/release/12.2.2/rpm/el7/BUILD/ceph-12.2.2/src/rocksdb/db/compaction_job.cc:1173] [default] [JOB 3] Compacted 4@0 + 1@1 files to L1 => 46901291 bytes -2009> 2018-02-14 13:23:41.967078 7fc511507700 1 -- 10.255.0.30:6789/0 >> - conn(0x5587d8f97000 :6789 s=STATE_ACCEPTING pgs=0 cs=0 l=0)._process_connection sd=44 - -2008> 2018-02-14 13:23:41.967280 7fc511507700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_verify_authorizer 10.255.0.105:0/1988410988 client protocol 0 -2007> 2018-02-14 13:23:41.967312 7fc511507700 2 -- 10.255.0.30:6789/0 >> 10.255.0.105:0/1988410988 conn(0x5587d8f97000 :6789 s=STATE_ACCEPTING_WAIT_SEQ pgs=975 cs=1 l=1).handle_connect_msg accept write reply msg done -2006> 2018-02-14 13:23:41.967398 7fc511507700 2 -- 10.255.0.30:6789/0 >> 10.255.0.105:0/1988410988 conn(0x5587d8f97000 :6789 s=STATE_ACCEPTING_WAIT_SEQ pgs=975 cs=1 l=1)._process_connection accept get newly_acked_seq 0 -2005> 2018-02-14 13:23:41.967514 7fc511507700 5 -- 10.255.0.30:6789/0 >> 10.255.0.105:0/1988410988 conn(0x5587d8f97000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=975 cs=1 l=1). rx client.1684193 seq 1 0x5587d8ec0240 auth(proto 0 35 bytes epoch 1) v1 -2004> 2018-02-14 13:23:41.967535 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1684193 10.255.0.105:0/1988410988 1 ==== auth(proto 0 35 bytes epoch 1) v1 ==== 65+0+0 (2516890795 0 0) 0x5587d8ec0240 con 0x5587d8f97000 -2003> 2018-02-14 13:23:41.967559 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch new session 0x5587d891e280 MonSession(client.1684193 10.255.0.105:0/1988410988 is open ) features 0x1ffddff8eea4fffb -2002> 2018-02-14 13:23:41.967567 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 caps -2001> 2018-02-14 13:23:41.967569 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing) e1 waitlisting message auth(proto 0 35 bytes epoch 1) v1 -2000> 2018-02-14 13:23:41.976688 7fc50b4fb700 1 -- 10.255.0.30:6789/0 >> - conn(0x5587d8f98800 :6789 s=STATE_ACCEPTING pgs=0 cs=0 l=0)._process_connection sd=45 - -1999> 2018-02-14 13:23:41.976901 7fc50b4fb700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_verify_authorizer 10.255.0.106:0/1162398434 client protocol 0 -1998> 2018-02-14 13:23:41.976932 7fc50b4fb700 2 -- 10.255.0.30:6789/0 >> 10.255.0.106:0/1162398434 conn(0x5587d8f98800 :6789 s=STATE_ACCEPTING_WAIT_SEQ pgs=30292 cs=1 l=1).handle_connect_msg accept write reply msg done -1997> 2018-02-14 13:23:41.977012 7fc50b4fb700 2 -- 10.255.0.30:6789/0 >> 10.255.0.106:0/1162398434 conn(0x5587d8f98800 :6789 s=STATE_ACCEPTING_WAIT_SEQ pgs=30292 cs=1 l=1)._process_connection accept get newly_acked_seq 0 -1996> 2018-02-14 13:23:41.977100 7fc50b4fb700 5 -- 10.255.0.30:6789/0 >> 10.255.0.106:0/1162398434 conn(0x5587d8f98800 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=30292 cs=1 l=1). rx client.1621590 seq 1 0x5587d8ec0d80 auth(proto 0 35 bytes epoch 1) v1 -1995> 2018-02-14 13:23:41.977121 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1621590 10.255.0.106:0/1162398434 1 ==== auth(proto 0 35 bytes epoch 1) v1 ==== 65+0+0 (3541179417 0 0) 0x5587d8ec0d80 con 0x5587d8f98800 -1994> 2018-02-14 13:23:41.977143 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch new session 0x5587d891e580 MonSession(client.1621590 10.255.0.106:0/1162398434 is open ) features 0x1ffddff8eea4fffb -1993> 2018-02-14 13:23:41.977150 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 caps -1992> 2018-02-14 13:23:41.977153 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing) e1 waitlisting message auth(proto 0 35 bytes epoch 1) v1 -1991> 2018-02-14 13:23:42.014133 7fc50acfa700 1 -- 10.255.0.30:6789/0 >> - conn(0x5587d8da8000 :6789 s=STATE_ACCEPTING pgs=0 cs=0 l=0)._process_connection sd=46 - -1990> 2018-02-14 13:23:42.014328 7fc50acfa700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_verify_authorizer 10.255.0.52:0/3152189720 client protocol 0 -1989> 2018-02-14 13:23:42.014359 7fc50acfa700 2 -- 10.255.0.30:6789/0 >> 10.255.0.52:0/3152189720 conn(0x5587d8da8000 :6789 s=STATE_ACCEPTING_WAIT_SEQ pgs=445 cs=1 l=1).handle_connect_msg accept write reply msg done -1988> 2018-02-14 13:23:42.014451 7fc50acfa700 2 -- 10.255.0.30:6789/0 >> 10.255.0.52:0/3152189720 conn(0x5587d8da8000 :6789 s=STATE_ACCEPTING_WAIT_SEQ pgs=445 cs=1 l=1)._process_connection accept get newly_acked_seq 0 -1987> 2018-02-14 13:23:42.014525 7fc50acfa700 5 -- 10.255.0.30:6789/0 >> 10.255.0.52:0/3152189720 conn(0x5587d8da8000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=445 cs=1 l=1). rx client.? seq 1 0x5587d8ec0b40 auth(proto 0 35 bytes epoch 0) v1 -1986> 2018-02-14 13:23:42.014546 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.? 10.255.0.52:0/3152189720 1 ==== auth(proto 0 35 bytes epoch 0) v1 ==== 65+0+0 (3478176772 0 0) 0x5587d8ec0b40 con 0x5587d8da8000 -1985> 2018-02-14 13:23:42.014568 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch new session 0x5587d891e880 MonSession(client.? 10.255.0.52:0/3152189720 is open ) features 0x1ffddff8eea4fffb -1984> 2018-02-14 13:23:42.014575 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 caps -1983> 2018-02-14 13:23:42.014577 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing) e1 waitlisting message auth(proto 0 35 bytes epoch 0) v1 -1982> 2018-02-14 13:23:42.030627 7fc509cf8700 4 rocksdb: (Original Log Time 2018/02/14-13:23:42.030524) [/home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos7/DIST/centos7/MACHINE_SIZE/huge/release/12.2.2/rpm/el7/BUILD/ceph-12.2.2/src/rocksdb/db/compaction_job.cc:621] [default] compacted to: base level 1 max bytes base 268435456 files[0 1 0 0 0 0 0] max score 0.17, MB/sec: 62.4 rd, 62.4 wr, level 1, files in(4, 1) out(1) MB in(0.1, 44.7) out(44.7), read-write-amplify(1311.7) write-amplify(655.8) OK, records in: 6729, records dropped: 19 -1981> 2018-02-14 13:23:42.030637 7fc509cf8700 4 rocksdb: (Original Log Time 2018/02/14-13:23:42.030548) EVENT_LOG_v1 {"time_micros": 1518643422030537, "job": 3, "event": "compaction_finished", "compaction_time_micros": 751432, "output_level": 1, "num_output_files": 1, "total_output_size": 46901291, "num_input_records": 6729, "num_output_records": 6710, "num_subcompactions": 1, "num_single_delete_mismatches": 0, "num_single_delete_fallthrough": 0, "lsm_state": [0, 1, 0, 0, 0, 0, 0]} -1980> 2018-02-14 13:23:42.030723 7fc509cf8700 5 rocksdb: [/home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos7/DIST/centos7/MACHINE_SIZE/huge/release/12.2.2/rpm/el7/BUILD/ceph-12.2.2/src/rocksdb/db/db_impl_files.cc:307] [JOB 3] Delete /var/lib/ceph/mon/ceph-ceph-dal-mon-a1/store.db/128464.sst type=2 #128464 -- OK -1979> 2018-02-14 13:23:42.030734 7fc509cf8700 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1518643422030730, "job": 3, "event": "table_file_deletion", "file_number": 128464} -1978> 2018-02-14 13:23:42.030768 7fc509cf8700 5 rocksdb: [/home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos7/DIST/centos7/MACHINE_SIZE/huge/release/12.2.2/rpm/el7/BUILD/ceph-12.2.2/src/rocksdb/db/db_impl_files.cc:307] [JOB 3] Delete /var/lib/ceph/mon/ceph-ceph-dal-mon-a1/store.db/128461.sst type=2 #128461 -- OK -1977> 2018-02-14 13:23:42.030803 7fc509cf8700 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1518643422030799, "job": 3, "event": "table_file_deletion", "file_number": 128461} -1976> 2018-02-14 13:23:42.030858 7fc509cf8700 5 rocksdb: [/home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos7/DIST/centos7/MACHINE_SIZE/huge/release/12.2.2/rpm/el7/BUILD/ceph-12.2.2/src/rocksdb/db/db_impl_files.cc:307] [JOB 3] Delete /var/lib/ceph/mon/ceph-ceph-dal-mon-a1/store.db/128458.sst type=2 #128458 -- OK -1975> 2018-02-14 13:23:42.030868 7fc509cf8700 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1518643422030865, "job": 3, "event": "table_file_deletion", "file_number": 128458} -1974> 2018-02-14 13:23:42.030891 7fc509cf8700 5 rocksdb: [/home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos7/DIST/centos7/MACHINE_SIZE/huge/release/12.2.2/rpm/el7/BUILD/ceph-12.2.2/src/rocksdb/db/db_impl_files.cc:307] [JOB 3] Delete /var/lib/ceph/mon/ceph-ceph-dal-mon-a1/store.db/128455.sst type=2 #128455 -- OK -1973> 2018-02-14 13:23:42.030907 7fc509cf8700 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1518643422030897, "job": 3, "event": "table_file_deletion", "file_number": 128455} -1972> 2018-02-14 13:23:42.040191 7fc509cf8700 5 rocksdb: [/home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos7/DIST/centos7/MACHINE_SIZE/huge/release/12.2.2/rpm/el7/BUILD/ceph-12.2.2/src/rocksdb/db/db_impl_files.cc:307] [JOB 3] Delete /var/lib/ceph/mon/ceph-ceph-dal-mon-a1/store.db/128453.sst type=2 #128453 -- OK -1971> 2018-02-14 13:23:42.040205 7fc509cf8700 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1518643422040201, "job": 3, "event": "table_file_deletion", "file_number": 128453} -1970> 2018-02-14 13:23:42.071459 7fc511507700 1 -- 10.255.0.30:6789/0 >> - conn(0x5587d8da9800 :6789 s=STATE_ACCEPTING pgs=0 cs=0 l=0)._process_connection sd=9 - -1969> 2018-02-14 13:23:42.071652 7fc511507700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_verify_authorizer 10.255.0.53:0/1148525388 client protocol 0 -1968> 2018-02-14 13:23:42.071683 7fc511507700 2 -- 10.255.0.30:6789/0 >> 10.255.0.53:0/1148525388 conn(0x5587d8da9800 :6789 s=STATE_ACCEPTING_WAIT_SEQ pgs=425 cs=1 l=1).handle_connect_msg accept write reply msg done -1967> 2018-02-14 13:23:42.071866 7fc511507700 2 -- 10.255.0.30:6789/0 >> 10.255.0.53:0/1148525388 conn(0x5587d8da9800 :6789 s=STATE_ACCEPTING_WAIT_SEQ pgs=425 cs=1 l=1)._process_connection accept get newly_acked_seq 0 -1966> 2018-02-14 13:23:42.072006 7fc511507700 5 -- 10.255.0.30:6789/0 >> 10.255.0.53:0/1148525388 conn(0x5587d8da9800 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=425 cs=1 l=1). rx client.? seq 1 0x5587d8ec1440 auth(proto 0 35 bytes epoch 0) v1 -1965> 2018-02-14 13:23:42.072027 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.? 10.255.0.53:0/1148525388 1 ==== auth(proto 0 35 bytes epoch 0) v1 ==== 65+0+0 (3478176772 0 0) 0x5587d8ec1440 con 0x5587d8da9800 -1964> 2018-02-14 13:23:42.072050 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch new session 0x5587d891eb80 MonSession(client.? 10.255.0.53:0/1148525388 is open ) features 0x1ffddff8eea4fffb -1963> 2018-02-14 13:23:42.072058 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 caps -1962> 2018-02-14 13:23:42.072060 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing) e1 waitlisting message auth(proto 0 35 bytes epoch 0) v1 -1961> 2018-02-14 13:23:42.221444 7fc50b4fb700 1 -- 10.255.0.30:6789/0 >> - conn(0x5587d8dab000 :6789 s=STATE_ACCEPTING pgs=0 cs=0 l=0)._process_connection sd=10 - -1960> 2018-02-14 13:23:42.221649 7fc50b4fb700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_verify_authorizer 10.255.0.54:0/3194015072 client protocol 0 -1959> 2018-02-14 13:23:42.221680 7fc50b4fb700 2 -- 10.255.0.30:6789/0 >> 10.255.0.54:0/3194015072 conn(0x5587d8dab000 :6789 s=STATE_ACCEPTING_WAIT_SEQ pgs=428 cs=1 l=1).handle_connect_msg accept write reply msg done -1958> 2018-02-14 13:23:42.221799 7fc50b4fb700 2 -- 10.255.0.30:6789/0 >> 10.255.0.54:0/3194015072 conn(0x5587d8dab000 :6789 s=STATE_ACCEPTING_WAIT_SEQ pgs=428 cs=1 l=1)._process_connection accept get newly_acked_seq 0 -1957> 2018-02-14 13:23:42.222231 7fc50b4fb700 5 -- 10.255.0.30:6789/0 >> 10.255.0.54:0/3194015072 conn(0x5587d8dab000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=428 cs=1 l=1). rx client.? seq 1 0x5587d8ec0fc0 auth(proto 0 35 bytes epoch 0) v1 -1956> 2018-02-14 13:23:42.222287 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.? 10.255.0.54:0/3194015072 1 ==== auth(proto 0 35 bytes epoch 0) v1 ==== 65+0+0 (3478176772 0 0) 0x5587d8ec0fc0 con 0x5587d8dab000 -1955> 2018-02-14 13:23:42.222311 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch new session 0x5587d891ee80 MonSession(client.? 10.255.0.54:0/3194015072 is open ) features 0x1ffddff8eea4fffb -1954> 2018-02-14 13:23:42.222319 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 caps -1953> 2018-02-14 13:23:42.222321 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing) e1 waitlisting message auth(proto 0 35 bytes epoch 0) v1 -1952> 2018-02-14 13:23:42.378659 7fc50acfa700 1 -- 10.255.0.30:6789/0 >> - conn(0x5587d8dac800 :6789 s=STATE_ACCEPTING pgs=0 cs=0 l=0)._process_connection sd=11 - -1951> 2018-02-14 13:23:42.378921 7fc50acfa700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_verify_authorizer 10.255.0.54:0/2824825993 client protocol 0 -1950> 2018-02-14 13:23:42.378948 7fc50acfa700 2 -- 10.255.0.30:6789/0 >> 10.255.0.54:0/2824825993 conn(0x5587d8dac800 :6789 s=STATE_ACCEPTING_WAIT_SEQ pgs=639 cs=1 l=1).handle_connect_msg accept write reply msg done -1949> 2018-02-14 13:23:42.379104 7fc50acfa700 2 -- 10.255.0.30:6789/0 >> 10.255.0.54:0/2824825993 conn(0x5587d8dac800 :6789 s=STATE_ACCEPTING_WAIT_SEQ pgs=639 cs=1 l=1)._process_connection accept get newly_acked_seq 0 -1948> 2018-02-14 13:23:42.379219 7fc50acfa700 5 -- 10.255.0.30:6789/0 >> 10.255.0.54:0/2824825993 conn(0x5587d8dac800 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=639 cs=1 l=1). rx client.1684238 seq 1 0x5587d88db3c0 auth(proto 0 35 bytes epoch 1) v1 -1947> 2018-02-14 13:23:42.379239 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1684238 10.255.0.54:0/2824825993 1 ==== auth(proto 0 35 bytes epoch 1) v1 ==== 65+0+0 (2264443001 0 0) 0x5587d88db3c0 con 0x5587d8dac800 -1946> 2018-02-14 13:23:42.379262 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch new session 0x5587d891f180 MonSession(client.1684238 10.255.0.54:0/2824825993 is open ) features 0x1ffddff8eea4fffb -1945> 2018-02-14 13:23:42.379272 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 caps -1944> 2018-02-14 13:23:42.379275 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing) e1 waitlisting message auth(proto 0 35 bytes epoch 1) v1 -1943> 2018-02-14 13:23:42.430810 7fc511507700 1 -- 10.255.0.30:6789/0 >> - conn(0x5587d8fae000 :6789 s=STATE_ACCEPTING pgs=0 cs=0 l=0)._process_connection sd=12 - -1942> 2018-02-14 13:23:42.431000 7fc511507700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_verify_authorizer 10.255.0.53:0/2869630763 client protocol 0 -1941> 2018-02-14 13:23:42.431031 7fc511507700 2 -- 10.255.0.30:6789/0 >> 10.255.0.53:0/2869630763 conn(0x5587d8fae000 :6789 s=STATE_ACCEPTING_WAIT_SEQ pgs=1149 cs=1 l=1).handle_connect_msg accept write reply msg done -1940> 2018-02-14 13:23:42.431124 7fc511507700 2 -- 10.255.0.30:6789/0 >> 10.255.0.53:0/2869630763 conn(0x5587d8fae000 :6789 s=STATE_ACCEPTING_WAIT_SEQ pgs=1149 cs=1 l=1)._process_connection accept get newly_acked_seq 0 -1939> 2018-02-14 13:23:42.431192 7fc511507700 5 -- 10.255.0.30:6789/0 >> 10.255.0.53:0/2869630763 conn(0x5587d8fae000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1149 cs=1 l=1). rx client.1684211 seq 1 0x5587d8ec1680 auth(proto 0 35 bytes epoch 1) v1 -1938> 2018-02-14 13:23:42.431213 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1684211 10.255.0.53:0/2869630763 1 ==== auth(proto 0 35 bytes epoch 1) v1 ==== 65+0+0 (3989751000 0 0) 0x5587d8ec1680 con 0x5587d8fae000 -1937> 2018-02-14 13:23:42.431232 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch new session 0x5587d891f480 MonSession(client.1684211 10.255.0.53:0/2869630763 is open ) features 0x1ffddff8eea4fffb -1936> 2018-02-14 13:23:42.431240 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 caps -1935> 2018-02-14 13:23:42.431243 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing) e1 waitlisting message auth(proto 0 35 bytes epoch 1) v1 -1934> 2018-02-14 13:23:42.617634 7fc511507700 1 -- 10.255.0.30:6789/0 >> 10.255.0.31:6789/0 conn(0x5587d8f81000 :-1 s=STATE_CONNECTING_RE pgs=0 cs=0 l=0)._process_connection reconnect failed -1933> 2018-02-14 13:23:42.617650 7fc511507700 2 -- 10.255.0.30:6789/0 >> 10.255.0.31:6789/0 conn(0x5587d8f81000 :-1 s=STATE_CONNECTING_RE pgs=0 cs=0 l=0)._process_connection connection refused! -1932> 2018-02-14 13:23:42.617695 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_handle_refused 0x5587d8f81000 10.255.0.31:6789/0 -1931> 2018-02-14 13:23:42.958461 7fc50b4fb700 1 -- 10.255.0.30:6789/0 >> - conn(0x5587d8faf800 :6789 s=STATE_ACCEPTING pgs=0 cs=0 l=0)._process_connection sd=14 - -1930> 2018-02-14 13:23:42.958611 7fc50b4fb700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_verify_authorizer 10.255.0.107:0/322403586 client protocol 0 -1929> 2018-02-14 13:23:42.958644 7fc50b4fb700 2 -- 10.255.0.30:6789/0 >> 10.255.0.107:0/322403586 conn(0x5587d8faf800 :6789 s=STATE_ACCEPTING_WAIT_SEQ pgs=28149 cs=1 l=1).handle_connect_msg accept write reply msg done -1928> 2018-02-14 13:23:42.958709 7fc50b4fb700 2 -- 10.255.0.30:6789/0 >> 10.255.0.107:0/322403586 conn(0x5587d8faf800 :6789 s=STATE_ACCEPTING_WAIT_SEQ pgs=28149 cs=1 l=1)._process_connection accept get newly_acked_seq 0 -1927> 2018-02-14 13:23:42.958770 7fc50b4fb700 5 -- 10.255.0.30:6789/0 >> 10.255.0.107:0/322403586 conn(0x5587d8faf800 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=28149 cs=1 l=1). rx client.263543 seq 1 0x5587d8ec1200 auth(proto 0 39 bytes epoch 0) -1926> 2018-02-14 13:23:42.958812 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.263543 10.255.0.107:0/322403586 1 ==== auth(proto 0 39 bytes epoch 0) ==== 65+0+0 (3009775502 0 0) 0x5587d8ec1200 con 0x5587d8faf800 -1925> 2018-02-14 13:23:42.958834 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch new session 0x5587d891f780 MonSession(client.263543 10.255.0.107:0/322403586 is open ) features 0x7010fb86aa42ada -1924> 2018-02-14 13:23:42.958842 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 caps -1923> 2018-02-14 13:23:42.958844 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing) e1 waitlisting message auth(proto 0 39 bytes epoch 0) -1922> 2018-02-14 13:23:43.088572 7fc50acfa700 1 -- 10.255.0.30:6789/0 >> - conn(0x5587d8fb1000 :6789 s=STATE_ACCEPTING pgs=0 cs=0 l=0)._process_connection sd=47 - -1921> 2018-02-14 13:23:43.088796 7fc50acfa700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_verify_authorizer 10.255.0.53:0/3548294640 client protocol 0 -1920> 2018-02-14 13:23:43.088822 7fc50acfa700 2 -- 10.255.0.30:6789/0 >> 10.255.0.53:0/3548294640 conn(0x5587d8fb1000 :6789 s=STATE_ACCEPTING_WAIT_SEQ pgs=899 cs=1 l=1).handle_connect_msg accept write reply msg done -1919> 2018-02-14 13:23:43.088936 7fc50acfa700 2 -- 10.255.0.30:6789/0 >> 10.255.0.53:0/3548294640 conn(0x5587d8fb1000 :6789 s=STATE_ACCEPTING_WAIT_SEQ pgs=899 cs=1 l=1)._process_connection accept get newly_acked_seq 0 -1918> 2018-02-14 13:23:43.089027 7fc50acfa700 5 -- 10.255.0.30:6789/0 >> 10.255.0.53:0/3548294640 conn(0x5587d8fb1000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=899 cs=1 l=1). rx client.1684265 seq 1 0x5587d8ec1b00 auth(proto 0 35 bytes epoch 1) v1 -1917> 2018-02-14 13:23:43.089047 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1684265 10.255.0.53:0/3548294640 1 ==== auth(proto 0 35 bytes epoch 1) v1 ==== 65+0+0 (587085007 0 0) 0x5587d8ec1b00 con 0x5587d8fb1000 -1916> 2018-02-14 13:23:43.089070 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch new session 0x5587d891fa80 MonSession(client.1684265 10.255.0.53:0/3548294640 is open ) features 0x1ffddff8eea4fffb -1915> 2018-02-14 13:23:43.089079 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 caps -1914> 2018-02-14 13:23:43.089091 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing) e1 waitlisting message auth(proto 0 35 bytes epoch 1) v1 -1913> 2018-02-14 13:23:43.634764 7fc511507700 1 -- 10.255.0.30:6789/0 >> - conn(0x5587d8fb2800 :6789 s=STATE_ACCEPTING pgs=0 cs=0 l=0)._process_connection sd=48 - -1912> 2018-02-14 13:23:43.635028 7fc511507700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_verify_authorizer 10.255.0.107:0/2591354583 client protocol 0 -1911> 2018-02-14 13:23:43.635059 7fc511507700 2 -- 10.255.0.30:6789/0 >> 10.255.0.107:0/2591354583 conn(0x5587d8fb2800 :6789 s=STATE_ACCEPTING_WAIT_SEQ pgs=581 cs=1 l=1).handle_connect_msg accept write reply msg done -1910> 2018-02-14 13:23:43.635144 7fc511507700 2 -- 10.255.0.30:6789/0 >> 10.255.0.107:0/2591354583 conn(0x5587d8fb2800 :6789 s=STATE_ACCEPTING_WAIT_SEQ pgs=581 cs=1 l=1)._process_connection accept get newly_acked_seq 0 -1909> 2018-02-14 13:23:43.635264 7fc511507700 5 -- 10.255.0.30:6789/0 >> 10.255.0.107:0/2591354583 conn(0x5587d8fb2800 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=581 cs=1 l=1). rx client.1684217 seq 1 0x5587d8ec18c0 auth(proto 0 35 bytes epoch 1) v1 -1908> 2018-02-14 13:23:43.635283 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1684217 10.255.0.107:0/2591354583 1 ==== auth(proto 0 35 bytes epoch 1) v1 ==== 65+0+0 (22002633 0 0) 0x5587d8ec18c0 con 0x5587d8fb2800 -1907> 2018-02-14 13:23:43.635307 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch new session 0x5587d891fd80 MonSession(client.1684217 10.255.0.107:0/2591354583 is open ) features 0x1ffddff8eea4fffb -1906> 2018-02-14 13:23:43.635314 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 caps -1905> 2018-02-14 13:23:43.635317 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing) e1 waitlisting message auth(proto 0 35 bytes epoch 1) v1 -1904> 2018-02-14 13:23:43.679066 7fc50acfa700 5 -- 10.255.0.30:6789/0 >> 10.255.0.32:6789/0 conn(0x5587d8f82800 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=51044 cs=1 l=0). rx mon.2 seq 386006848 0x5587d895a400 election(39fac7f7-f5a8-4e84-b9bb-b0dfffd79c92 propose 501) v7 -1903> 2018-02-14 13:23:43.679126 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== mon.2 10.255.0.32:6789/0 386006848 ==== election(39fac7f7-f5a8-4e84-b9bb-b0dfffd79c92 propose 501) v7 ==== 561+0+0 (1861587700 0 0) 0x5587d895a400 con 0x5587d8f82800 -1902> 2018-02-14 13:23:43.679149 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch existing session 0x5587d891c480 for mon.2 10.255.0.32:6789/0 -1901> 2018-02-14 13:23:43.679156 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 caps allow * -1900> 2018-02-14 13:23:43.679159 7fc50dd00700 20 is_capable service=mon command= read on cap allow * -1899> 2018-02-14 13:23:43.679162 7fc50dd00700 20 allow so far , doing grant allow * -1898> 2018-02-14 13:23:43.679163 7fc50dd00700 20 allow all -1897> 2018-02-14 13:23:43.679164 7fc50dd00700 20 is_capable service=mon command= exec on cap allow * -1896> 2018-02-14 13:23:43.679165 7fc50dd00700 20 allow so far , doing grant allow * -1895> 2018-02-14 13:23:43.679165 7fc50dd00700 20 allow all -1894> 2018-02-14 13:23:43.679189 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing).elector(193) handle_propose from mon.2 -1893> 2018-02-14 13:23:43.679191 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing).elector(193) handle_propose required features 153140804152475648 mon_feature_t([kraken,luminous]), peer features 2305244844532236283 mon_feature_t([kraken,luminous]) -1892> 2018-02-14 13:23:43.679198 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing).elector(193) bump_epoch 193 to 501 -1891> 2018-02-14 13:23:43.722825 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 join_election -1890> 2018-02-14 13:23:43.722835 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _reset -1889> 2018-02-14 13:23:43.722858 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 cancel_probe_timeout (none scheduled) -1888> 2018-02-14 13:23:43.722861 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 timecheck_finish -1887> 2018-02-14 13:23:43.722862 7fc50dd00700 15 mon.ceph-dal-mon-a1@0(electing) e1 health_tick_stop -1886> 2018-02-14 13:23:43.722864 7fc50dd00700 15 mon.ceph-dal-mon-a1@0(electing) e1 health_interval_stop -1885> 2018-02-14 13:23:43.722865 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 scrub_event_cancel -1884> 2018-02-14 13:23:43.722867 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 scrub_reset -1883> 2018-02-14 13:23:43.722881 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 start_election -1882> 2018-02-14 13:23:43.722883 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _reset -1881> 2018-02-14 13:23:43.722884 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 cancel_probe_timeout (none scheduled) -1880> 2018-02-14 13:23:43.722885 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 timecheck_finish -1879> 2018-02-14 13:23:43.722886 7fc50dd00700 15 mon.ceph-dal-mon-a1@0(electing) e1 health_tick_stop -1878> 2018-02-14 13:23:43.722887 7fc50dd00700 15 mon.ceph-dal-mon-a1@0(electing) e1 health_interval_stop -1877> 2018-02-14 13:23:43.722888 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 scrub_event_cancel -1876> 2018-02-14 13:23:43.722889 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 scrub_reset -1875> 2018-02-14 13:23:43.722896 7fc50dd00700 0 log_channel(cluster) log [INF] : mon.ceph-dal-mon-a1 calling new monitor election -1874> 2018-02-14 13:23:43.722900 7fc50dd00700 10 log_client _send_to_monlog to self -1873> 2018-02-14 13:23:43.722901 7fc50dd00700 10 log_client log_queue is 2 last_log 2 sent 1 num 2 unsent 1 sending 1 -1872> 2018-02-14 13:23:43.722903 7fc50dd00700 10 log_client will send 2018-02-14 13:23:43.722898 mon.ceph-dal-mon-a1 mon.0 10.255.0.30:6789/0 2 : cluster [INF] mon.ceph-dal-mon-a1 calling new monitor election -1871> 2018-02-14 13:23:43.722915 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.30:6789/0 -- log(1 entries from seq 2 at 2018-02-14 13:23:43.722898) v1 -- 0x5587d8ec2400 con 0 -1870> 2018-02-14 13:23:43.722926 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing).elector(501) start -- can i be leader? -1869> 2018-02-14 13:23:43.722940 7fc50dd00700 1 mon.ceph-dal-mon-a1@0(electing).elector(501) init, last seen epoch 501, mid-election, bumping -1868> 2018-02-14 13:23:43.762258 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing).elector(502) bump_epoch 502 to 503 -1867> 2018-02-14 13:23:43.786170 7fc50b4fb700 1 -- 10.255.0.30:6789/0 >> - conn(0x5587d9150000 :6789 s=STATE_ACCEPTING pgs=0 cs=0 l=0)._process_connection sd=49 - -1866> 2018-02-14 13:23:43.786396 7fc50b4fb700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_verify_authorizer 10.255.0.51:0/3289155469 client protocol 0 -1865> 2018-02-14 13:23:43.786425 7fc50b4fb700 2 -- 10.255.0.30:6789/0 >> 10.255.0.51:0/3289155469 conn(0x5587d9150000 :6789 s=STATE_ACCEPTING_WAIT_SEQ pgs=612 cs=1 l=1).handle_connect_msg accept write reply msg done -1864> 2018-02-14 13:23:43.786583 7fc50b4fb700 2 -- 10.255.0.30:6789/0 >> 10.255.0.51:0/3289155469 conn(0x5587d9150000 :6789 s=STATE_ACCEPTING_WAIT_SEQ pgs=612 cs=1 l=1)._process_connection accept get newly_acked_seq 0 -1863> 2018-02-14 13:23:43.786657 7fc50b4fb700 5 -- 10.255.0.30:6789/0 >> 10.255.0.51:0/3289155469 conn(0x5587d9150000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=612 cs=1 l=1). rx client.1684226 seq 1 0x5587d8ec2ac0 auth(proto 0 35 bytes epoch 1) v1 -1862> 2018-02-14 13:23:43.789832 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 join_election -1861> 2018-02-14 13:23:43.789842 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _reset -1860> 2018-02-14 13:23:43.789844 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 cancel_probe_timeout (none scheduled) -1859> 2018-02-14 13:23:43.789845 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 timecheck_finish -1858> 2018-02-14 13:23:43.789846 7fc50dd00700 15 mon.ceph-dal-mon-a1@0(electing) e1 health_tick_stop -1857> 2018-02-14 13:23:43.789848 7fc50dd00700 15 mon.ceph-dal-mon-a1@0(electing) e1 health_interval_stop -1856> 2018-02-14 13:23:43.789849 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 scrub_event_cancel -1855> 2018-02-14 13:23:43.789850 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 scrub_reset -1854> 2018-02-14 13:23:43.790031 7fc50dd00700 1 -- 10.255.0.30:6789/0 _send_message--> mon.1 10.255.0.31:6789/0 -- election(39fac7f7-f5a8-4e84-b9bb-b0dfffd79c92 propose 503) v7 -- ?+0 0x5587d895a700 -1853> 2018-02-14 13:23:43.790045 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.31:6789/0 -- election(39fac7f7-f5a8-4e84-b9bb-b0dfffd79c92 propose 503) v7 -- 0x5587d895a700 con 0 -1852> 2018-02-14 13:23:43.790071 7fc50dd00700 1 -- 10.255.0.30:6789/0 _send_message--> mon.2 10.255.0.32:6789/0 -- election(39fac7f7-f5a8-4e84-b9bb-b0dfffd79c92 propose 503) v7 -- ?+0 0x5587d895aa00 -1851> 2018-02-14 13:23:43.790079 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.32:6789/0 -- election(39fac7f7-f5a8-4e84-b9bb-b0dfffd79c92 propose 503) v7 -- 0x5587d895aa00 con 0 -1850> 2018-02-14 13:23:43.790104 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== mon.0 10.255.0.30:6789/0 0 ==== log(1 entries from seq 2 at 2018-02-14 13:23:43.722898) v1 ==== 0+0+0 (0 0 0) 0x5587d8ec2400 con 0x5587d8d41800 -1849> 2018-02-14 13:23:43.790117 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch existing session 0x5587d891ca80 for mon.0 10.255.0.30:6789/0 -1848> 2018-02-14 13:23:43.790123 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 caps allow * -1847> 2018-02-14 13:23:43.790127 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing).paxos(paxos recovering c 4935163..4935904) is_readable = 0 - now=2018-02-14 13:23:43.790128 lease_expire=0.000000 has v0 lc 4935904 -1846> 2018-02-14 13:23:43.790134 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing).paxos(paxos recovering c 4935163..4935904) is_readable = 0 - now=2018-02-14 13:23:43.790135 lease_expire=0.000000 has v0 lc 4935904 -1845> 2018-02-14 13:23:43.790148 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1684226 10.255.0.51:0/3289155469 1 ==== auth(proto 0 35 bytes epoch 1) v1 ==== 65+0+0 (3445436360 0 0) 0x5587d8ec2ac0 con 0x5587d9150000 -1844> 2018-02-14 13:23:43.790164 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch new session 0x5587d9148300 MonSession(client.1684226 10.255.0.51:0/3289155469 is open ) features 0x1ffddff8eea4fffb -1843> 2018-02-14 13:23:43.790176 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 caps -1842> 2018-02-14 13:23:43.790178 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing) e1 waitlisting message auth(proto 0 35 bytes epoch 1) v1 -1841> 2018-02-14 13:23:43.813067 7fc50acfa700 5 -- 10.255.0.30:6789/0 >> 10.255.0.32:6789/0 conn(0x5587d8f82800 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=51044 cs=1 l=0). rx mon.2 seq 386006849 0x5587d895aa00 election(39fac7f7-f5a8-4e84-b9bb-b0dfffd79c92 ack 503) v7 -1840> 2018-02-14 13:23:43.813100 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== mon.2 10.255.0.32:6789/0 386006849 ==== election(39fac7f7-f5a8-4e84-b9bb-b0dfffd79c92 ack 503) v7 ==== 40011+0+0 (1046991694 0 0) 0x5587d895aa00 con 0x5587d8f82800 -1839> 2018-02-14 13:23:43.813121 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch existing session 0x5587d891c480 for mon.2 10.255.0.32:6789/0 -1838> 2018-02-14 13:23:43.813131 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 caps allow * -1837> 2018-02-14 13:23:43.813134 7fc50dd00700 20 is_capable service=mon command= read on cap allow * -1836> 2018-02-14 13:23:43.813136 7fc50dd00700 20 allow so far , doing grant allow * -1835> 2018-02-14 13:23:43.813137 7fc50dd00700 20 allow all -1834> 2018-02-14 13:23:43.813138 7fc50dd00700 20 is_capable service=mon command= exec on cap allow * -1833> 2018-02-14 13:23:43.813140 7fc50dd00700 20 allow so far , doing grant allow * -1832> 2018-02-14 13:23:43.813140 7fc50dd00700 20 allow all -1831> 2018-02-14 13:23:43.813158 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing).elector(503) handle_ack from mon.2 -1830> 2018-02-14 13:23:43.813163 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing).elector(503) so far i have { mon.0: features 2305244844532236283 mon_feature_t([kraken,luminous]), mon.2: features 2305244844532236283 mon_feature_t([kraken,luminous]) } -1829> 2018-02-14 13:23:43.837549 7fc50acfa700 1 -- 10.255.0.30:6789/0 >> - conn(0x5587d9151800 :6789 s=STATE_ACCEPTING pgs=0 cs=0 l=0)._process_connection sd=50 - -1828> 2018-02-14 13:23:43.837800 7fc50acfa700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_verify_authorizer 10.255.0.105:0/2076150628 client protocol 0 -1827> 2018-02-14 13:23:43.837832 7fc50acfa700 2 -- 10.255.0.30:6789/0 >> 10.255.0.105:0/2076150628 conn(0x5587d9151800 :6789 s=STATE_ACCEPTING_WAIT_SEQ pgs=492 cs=1 l=1).handle_connect_msg accept write reply msg done -1826> 2018-02-14 13:23:43.837920 7fc50acfa700 2 -- 10.255.0.30:6789/0 >> 10.255.0.105:0/2076150628 conn(0x5587d9151800 :6789 s=STATE_ACCEPTING_WAIT_SEQ pgs=492 cs=1 l=1)._process_connection accept get newly_acked_seq 0 -1825> 2018-02-14 13:23:43.838045 7fc50acfa700 5 -- 10.255.0.30:6789/0 >> 10.255.0.105:0/2076150628 conn(0x5587d9151800 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=492 cs=1 l=1). rx client.1684247 seq 1 0x5587d8ec1d40 auth(proto 0 35 bytes epoch 1) v1 -1824> 2018-02-14 13:23:43.838068 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1684247 10.255.0.105:0/2076150628 1 ==== auth(proto 0 35 bytes epoch 1) v1 ==== 65+0+0 (549614379 0 0) 0x5587d8ec1d40 con 0x5587d9151800 -1823> 2018-02-14 13:23:43.838089 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch new session 0x5587d9148480 MonSession(client.1684247 10.255.0.105:0/2076150628 is open ) features 0x1ffddff8eea4fffb -1822> 2018-02-14 13:23:43.838096 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 caps -1821> 2018-02-14 13:23:43.838098 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing) e1 waitlisting message auth(proto 0 35 bytes epoch 1) v1 -1820> 2018-02-14 13:23:43.854764 7fc511507700 1 -- 10.255.0.30:6789/0 >> - conn(0x5587d9153000 :6789 s=STATE_ACCEPTING pgs=0 cs=0 l=0)._process_connection sd=51 - -1819> 2018-02-14 13:23:43.854966 7fc511507700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_verify_authorizer 10.255.0.54:0/2718452115 client protocol 0 -1818> 2018-02-14 13:23:43.855009 7fc511507700 2 -- 10.255.0.30:6789/0 >> 10.255.0.54:0/2718452115 conn(0x5587d9153000 :6789 s=STATE_ACCEPTING_WAIT_SEQ pgs=517 cs=1 l=1).handle_connect_msg accept write reply msg done -1817> 2018-02-14 13:23:43.855131 7fc511507700 2 -- 10.255.0.30:6789/0 >> 10.255.0.54:0/2718452115 conn(0x5587d9153000 :6789 s=STATE_ACCEPTING_WAIT_SEQ pgs=517 cs=1 l=1)._process_connection accept get newly_acked_seq 0 -1816> 2018-02-14 13:23:43.855239 7fc511507700 5 -- 10.255.0.30:6789/0 >> 10.255.0.54:0/2718452115 conn(0x5587d9153000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=517 cs=1 l=1). rx client.1684244 seq 1 0x5587d8ec33c0 auth(proto 0 35 bytes epoch 1) v1 -1815> 2018-02-14 13:23:43.855260 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1684244 10.255.0.54:0/2718452115 1 ==== auth(proto 0 35 bytes epoch 1) v1 ==== 65+0+0 (1934626683 0 0) 0x5587d8ec33c0 con 0x5587d9153000 -1814> 2018-02-14 13:23:43.855281 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch new session 0x5587d9148600 MonSession(client.1684244 10.255.0.54:0/2718452115 is open ) features 0x1ffddff8eea4fffb -1813> 2018-02-14 13:23:43.855291 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 caps -1812> 2018-02-14 13:23:43.855293 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing) e1 waitlisting message auth(proto 0 35 bytes epoch 1) v1 -1811> 2018-02-14 13:23:43.891997 7fc50b4fb700 1 -- 10.255.0.30:6789/0 >> - conn(0x5587d9154800 :6789 s=STATE_ACCEPTING pgs=0 cs=0 l=0)._process_connection sd=52 - -1810> 2018-02-14 13:23:43.892204 7fc50b4fb700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_verify_authorizer 10.255.0.107:0/690639890 client protocol 0 -1809> 2018-02-14 13:23:43.892234 7fc50b4fb700 2 -- 10.255.0.30:6789/0 >> 10.255.0.107:0/690639890 conn(0x5587d9154800 :6789 s=STATE_ACCEPTING_WAIT_SEQ pgs=501 cs=1 l=1).handle_connect_msg accept write reply msg done -1808> 2018-02-14 13:23:43.892325 7fc50b4fb700 2 -- 10.255.0.30:6789/0 >> 10.255.0.107:0/690639890 conn(0x5587d9154800 :6789 s=STATE_ACCEPTING_WAIT_SEQ pgs=501 cs=1 l=1)._process_connection accept get newly_acked_seq 0 -1807> 2018-02-14 13:23:43.892421 7fc50b4fb700 5 -- 10.255.0.30:6789/0 >> 10.255.0.107:0/690639890 conn(0x5587d9154800 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=501 cs=1 l=1). rx client.1684250 seq 1 0x5587d8ec2d00 auth(proto 0 35 bytes epoch 1) v1 -1806> 2018-02-14 13:23:43.892442 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1684250 10.255.0.107:0/690639890 1 ==== auth(proto 0 35 bytes epoch 1) v1 ==== 65+0+0 (1511399594 0 0) 0x5587d8ec2d00 con 0x5587d9154800 -1805> 2018-02-14 13:23:43.892464 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch new session 0x5587d9148780 MonSession(client.1684250 10.255.0.107:0/690639890 is open ) features 0x1ffddff8eea4fffb -1804> 2018-02-14 13:23:43.892471 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 caps -1803> 2018-02-14 13:23:43.892474 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing) e1 waitlisting message auth(proto 0 35 bytes epoch 1) v1 -1802> 2018-02-14 13:23:44.218812 7fc511507700 1 -- 10.255.0.30:6789/0 >> 10.255.0.31:6789/0 conn(0x5587d8f81000 :-1 s=STATE_CONNECTING_RE pgs=0 cs=0 l=0)._process_connection reconnect failed -1801> 2018-02-14 13:23:44.218827 7fc511507700 2 -- 10.255.0.30:6789/0 >> 10.255.0.31:6789/0 conn(0x5587d8f81000 :-1 s=STATE_CONNECTING_RE pgs=0 cs=0 l=0)._process_connection connection refused! -1800> 2018-02-14 13:23:44.218874 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_handle_refused 0x5587d8f81000 10.255.0.31:6789/0 -1799> 2018-02-14 13:23:44.498903 7fc50acfa700 1 -- 10.255.0.30:6789/0 >> - conn(0x5587d91ba000 :6789 s=STATE_ACCEPTING pgs=0 cs=0 l=0)._process_connection sd=53 - -1798> 2018-02-14 13:23:44.499088 7fc50acfa700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_verify_authorizer 10.255.0.108:0/989583023 client protocol 0 -1797> 2018-02-14 13:23:44.499114 7fc50acfa700 2 -- 10.255.0.30:6789/0 >> 10.255.0.108:0/989583023 conn(0x5587d91ba000 :6789 s=STATE_ACCEPTING_WAIT_SEQ pgs=432 cs=1 l=1).handle_connect_msg accept write reply msg done -1796> 2018-02-14 13:23:44.499203 7fc50acfa700 2 -- 10.255.0.30:6789/0 >> 10.255.0.108:0/989583023 conn(0x5587d91ba000 :6789 s=STATE_ACCEPTING_WAIT_SEQ pgs=432 cs=1 l=1)._process_connection accept get newly_acked_seq 0 -1795> 2018-02-14 13:23:44.499279 7fc50acfa700 5 -- 10.255.0.30:6789/0 >> 10.255.0.108:0/989583023 conn(0x5587d91ba000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=432 cs=1 l=1). rx client.? seq 1 0x5587d8ec1f80 auth(proto 0 35 bytes epoch 0) v1 -1794> 2018-02-14 13:23:44.499300 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.? 10.255.0.108:0/989583023 1 ==== auth(proto 0 35 bytes epoch 0) v1 ==== 65+0+0 (3478176772 0 0) 0x5587d8ec1f80 con 0x5587d91ba000 -1793> 2018-02-14 13:23:44.499321 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch new session 0x5587d9148900 MonSession(client.? 10.255.0.108:0/989583023 is open ) features 0x1ffddff8eea4fffb -1792> 2018-02-14 13:23:44.499329 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 caps -1791> 2018-02-14 13:23:44.499332 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing) e1 waitlisting message auth(proto 0 35 bytes epoch 0) v1 -1790> 2018-02-14 13:23:44.515481 7fc50b4fb700 1 -- 10.255.0.30:6789/0 >> 10.255.0.106:0/1436590248 conn(0x5587d8f2c000 :6789 s=STATE_OPEN pgs=442 cs=1 l=1).read_bulk peer close file descriptor 39 -1789> 2018-02-14 13:23:44.515498 7fc50b4fb700 1 -- 10.255.0.30:6789/0 >> 10.255.0.106:0/1436590248 conn(0x5587d8f2c000 :6789 s=STATE_OPEN pgs=442 cs=1 l=1).read_until read failed -1788> 2018-02-14 13:23:44.515508 7fc50b4fb700 1 -- 10.255.0.30:6789/0 >> 10.255.0.106:0/1436590248 conn(0x5587d8f2c000 :6789 s=STATE_OPEN pgs=442 cs=1 l=1).process read tag failed -1787> 2018-02-14 13:23:44.515517 7fc50b4fb700 1 -- 10.255.0.30:6789/0 >> 10.255.0.106:0/1436590248 conn(0x5587d8f2c000 :6789 s=STATE_OPEN pgs=442 cs=1 l=1).fault on lossy channel, failing -1786> 2018-02-14 13:23:44.515527 7fc50b4fb700 2 -- 10.255.0.30:6789/0 >> 10.255.0.106:0/1436590248 conn(0x5587d8f2c000 :6789 s=STATE_OPEN pgs=442 cs=1 l=1)._stop -1785> 2018-02-14 13:23:44.515581 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_handle_reset 0x5587d8f2c000 10.255.0.106:0/1436590248 -1784> 2018-02-14 13:23:44.515597 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 reset/close on session client.? 10.255.0.106:0/1436590248 -1783> 2018-02-14 13:23:44.515605 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 remove_session 0x5587d891d380 client.? 10.255.0.106:0/1436590248 features 0x1ffddff8eea4fffb -1782> 2018-02-14 13:23:44.585842 7fc50acfa700 1 -- 10.255.0.30:6789/0 >> 10.255.0.51:0/3461880882 conn(0x5587d8d40000 :6789 s=STATE_OPEN pgs=428 cs=1 l=1).read_bulk peer close file descriptor 40 -1781> 2018-02-14 13:23:44.585857 7fc50acfa700 1 -- 10.255.0.30:6789/0 >> 10.255.0.51:0/3461880882 conn(0x5587d8d40000 :6789 s=STATE_OPEN pgs=428 cs=1 l=1).read_until read failed -1780> 2018-02-14 13:23:44.585862 7fc50acfa700 1 -- 10.255.0.30:6789/0 >> 10.255.0.51:0/3461880882 conn(0x5587d8d40000 :6789 s=STATE_OPEN pgs=428 cs=1 l=1).process read tag failed -1779> 2018-02-14 13:23:44.585880 7fc50acfa700 1 -- 10.255.0.30:6789/0 >> 10.255.0.51:0/3461880882 conn(0x5587d8d40000 :6789 s=STATE_OPEN pgs=428 cs=1 l=1).fault on lossy channel, failing -1778> 2018-02-14 13:23:44.585884 7fc50acfa700 2 -- 10.255.0.30:6789/0 >> 10.255.0.51:0/3461880882 conn(0x5587d8d40000 :6789 s=STATE_OPEN pgs=428 cs=1 l=1)._stop -1777> 2018-02-14 13:23:44.585902 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_handle_reset 0x5587d8d40000 10.255.0.51:0/3461880882 -1776> 2018-02-14 13:23:44.585916 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 reset/close on session client.? 10.255.0.51:0/3461880882 -1775> 2018-02-14 13:23:44.585923 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 remove_session 0x5587d891d680 client.? 10.255.0.51:0/3461880882 features 0x1ffddff8eea4fffb -1774> 2018-02-14 13:23:44.586073 7fc50acfa700 1 -- 10.255.0.30:6789/0 >> - conn(0x5587d91bb800 :6789 s=STATE_ACCEPTING pgs=0 cs=0 l=0)._process_connection sd=39 - -1773> 2018-02-14 13:23:44.586270 7fc50acfa700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_verify_authorizer 10.255.0.51:0/3461880882 client protocol 0 -1772> 2018-02-14 13:23:44.586295 7fc50acfa700 2 -- 10.255.0.30:6789/0 >> 10.255.0.51:0/3461880882 conn(0x5587d91bb800 :6789 s=STATE_ACCEPTING_WAIT_SEQ pgs=433 cs=1 l=1).handle_connect_msg accept write reply msg done -1771> 2018-02-14 13:23:44.586431 7fc50acfa700 2 -- 10.255.0.30:6789/0 >> 10.255.0.51:0/3461880882 conn(0x5587d91bb800 :6789 s=STATE_ACCEPTING_WAIT_SEQ pgs=433 cs=1 l=1)._process_connection accept get newly_acked_seq 0 -1770> 2018-02-14 13:23:44.586557 7fc50acfa700 5 -- 10.255.0.30:6789/0 >> 10.255.0.51:0/3461880882 conn(0x5587d91bb800 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=433 cs=1 l=1). rx client.? seq 1 0x5587d8ec21c0 auth(proto 0 35 bytes epoch 0) v1 -1769> 2018-02-14 13:23:44.586577 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.? 10.255.0.51:0/3461880882 1 ==== auth(proto 0 35 bytes epoch 0) v1 ==== 65+0+0 (3478176772 0 0) 0x5587d8ec21c0 con 0x5587d91bb800 -1768> 2018-02-14 13:23:44.586598 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch new session 0x5587d9148a80 MonSession(client.? 10.255.0.51:0/3461880882 is open ) features 0x1ffddff8eea4fffb -1767> 2018-02-14 13:23:44.586606 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 caps -1766> 2018-02-14 13:23:44.586608 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing) e1 waitlisting message auth(proto 0 35 bytes epoch 0) v1 -1765> 2018-02-14 13:23:44.930654 7fc511507700 1 -- 10.255.0.30:6789/0 >> - conn(0x5587d91bd000 :6789 s=STATE_ACCEPTING pgs=0 cs=0 l=0)._process_connection sd=40 - -1764> 2018-02-14 13:23:44.930908 7fc511507700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_verify_authorizer 10.255.0.105:0/1919733865 client protocol 0 -1763> 2018-02-14 13:23:44.930940 7fc511507700 2 -- 10.255.0.30:6789/0 >> 10.255.0.105:0/1919733865 conn(0x5587d91bd000 :6789 s=STATE_ACCEPTING_WAIT_SEQ pgs=427 cs=1 l=1).handle_connect_msg accept write reply msg done -1762> 2018-02-14 13:23:44.931047 7fc511507700 2 -- 10.255.0.30:6789/0 >> 10.255.0.105:0/1919733865 conn(0x5587d91bd000 :6789 s=STATE_ACCEPTING_WAIT_SEQ pgs=427 cs=1 l=1)._process_connection accept get newly_acked_seq 0 -1761> 2018-02-14 13:23:44.931214 7fc511507700 5 -- 10.255.0.30:6789/0 >> 10.255.0.105:0/1919733865 conn(0x5587d91bd000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=427 cs=1 l=1). rx client.? seq 1 0x5587d8ec3600 auth(proto 0 35 bytes epoch 0) v1 -1760> 2018-02-14 13:23:44.931267 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.? 10.255.0.105:0/1919733865 1 ==== auth(proto 0 35 bytes epoch 0) v1 ==== 65+0+0 (3478176772 0 0) 0x5587d8ec3600 con 0x5587d91bd000 -1759> 2018-02-14 13:23:44.931288 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch new session 0x5587d9148c00 MonSession(client.? 10.255.0.105:0/1919733865 is open ) features 0x1ffddff8eea4fffb -1758> 2018-02-14 13:23:44.931295 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 caps -1757> 2018-02-14 13:23:44.931297 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing) e1 waitlisting message auth(proto 0 35 bytes epoch 0) v1 -1756> 2018-02-14 13:23:44.963564 7fc50b4fb700 1 -- 10.255.0.30:6789/0 >> - conn(0x5587d91be800 :6789 s=STATE_ACCEPTING pgs=0 cs=0 l=0)._process_connection sd=54 - -1755> 2018-02-14 13:23:44.963810 7fc50b4fb700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_verify_authorizer 10.255.0.108:0/3175087704 client protocol 0 -1754> 2018-02-14 13:23:44.963840 7fc50b4fb700 2 -- 10.255.0.30:6789/0 >> 10.255.0.108:0/3175087704 conn(0x5587d91be800 :6789 s=STATE_ACCEPTING_WAIT_SEQ pgs=1094 cs=1 l=1).handle_connect_msg accept write reply msg done -1753> 2018-02-14 13:23:44.963947 7fc50b4fb700 2 -- 10.255.0.30:6789/0 >> 10.255.0.108:0/3175087704 conn(0x5587d91be800 :6789 s=STATE_ACCEPTING_WAIT_SEQ pgs=1094 cs=1 l=1)._process_connection accept get newly_acked_seq 0 -1752> 2018-02-14 13:23:44.964039 7fc50b4fb700 5 -- 10.255.0.30:6789/0 >> 10.255.0.108:0/3175087704 conn(0x5587d91be800 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1094 cs=1 l=1). rx client.1684190 seq 1 0x5587d8ec2f40 auth(proto 0 35 bytes epoch 1) v1 -1751> 2018-02-14 13:23:44.964061 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1684190 10.255.0.108:0/3175087704 1 ==== auth(proto 0 35 bytes epoch 1) v1 ==== 65+0+0 (2773034879 0 0) 0x5587d8ec2f40 con 0x5587d91be800 -1750> 2018-02-14 13:23:44.964083 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch new session 0x5587d9148f00 MonSession(client.1684190 10.255.0.108:0/3175087704 is open ) features 0x1ffddff8eea4fffb -1749> 2018-02-14 13:23:44.964091 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 caps -1748> 2018-02-14 13:23:44.964093 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing) e1 waitlisting message auth(proto 0 35 bytes epoch 1) v1 -1747> 2018-02-14 13:23:45.001629 7fc50acfa700 1 -- 10.255.0.30:6789/0 >> - conn(0x5587d8fcc000 :6789 s=STATE_ACCEPTING pgs=0 cs=0 l=0)._process_connection sd=55 - -1746> 2018-02-14 13:23:45.001795 7fc50acfa700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_verify_authorizer 10.255.0.49:0/59005741 client protocol 0 -1745> 2018-02-14 13:23:45.001833 7fc50acfa700 2 -- 10.255.0.30:6789/0 >> 10.255.0.49:0/59005741 conn(0x5587d8fcc000 :6789 s=STATE_ACCEPTING_WAIT_SEQ pgs=12332 cs=1 l=1).handle_connect_msg accept write reply msg done -1744> 2018-02-14 13:23:45.001925 7fc50acfa700 2 -- 10.255.0.30:6789/0 >> 10.255.0.49:0/59005741 conn(0x5587d8fcc000 :6789 s=STATE_ACCEPTING_WAIT_SEQ pgs=12332 cs=1 l=1)._process_connection accept get newly_acked_seq 0 -1743> 2018-02-14 13:23:45.001975 7fc50acfa700 5 -- 10.255.0.30:6789/0 >> 10.255.0.49:0/59005741 conn(0x5587d8fcc000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=12332 cs=1 l=1). rx client.1582487 seq 1 0x5587d8ec3cc0 auth(proto 0 42 bytes epoch 1) v1 -1742> 2018-02-14 13:23:45.002003 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1582487 10.255.0.49:0/59005741 1 ==== auth(proto 0 42 bytes epoch 1) v1 ==== 72+0+0 (320495188 0 0) 0x5587d8ec3cc0 con 0x5587d8fcc000 -1741> 2018-02-14 13:23:45.002023 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch new session 0x5587d9149080 MonSession(client.1582487 10.255.0.49:0/59005741 is open ) features 0x1ffddff8eea4fffb -1740> 2018-02-14 13:23:45.002030 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 caps -1739> 2018-02-14 13:23:45.002033 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing) e1 waitlisting message auth(proto 0 42 bytes epoch 1) v1 -1738> 2018-02-14 13:23:45.014205 7fc50acfa700 1 -- 10.255.0.30:6789/0 >> 10.255.0.52:0/3152189720 conn(0x5587d8da8000 :6789 s=STATE_OPEN pgs=445 cs=1 l=1).read_bulk peer close file descriptor 46 -1737> 2018-02-14 13:23:45.014221 7fc50acfa700 1 -- 10.255.0.30:6789/0 >> 10.255.0.52:0/3152189720 conn(0x5587d8da8000 :6789 s=STATE_OPEN pgs=445 cs=1 l=1).read_until read failed -1736> 2018-02-14 13:23:45.014226 7fc50acfa700 1 -- 10.255.0.30:6789/0 >> 10.255.0.52:0/3152189720 conn(0x5587d8da8000 :6789 s=STATE_OPEN pgs=445 cs=1 l=1).process read tag failed -1735> 2018-02-14 13:23:45.014231 7fc50acfa700 1 -- 10.255.0.30:6789/0 >> 10.255.0.52:0/3152189720 conn(0x5587d8da8000 :6789 s=STATE_OPEN pgs=445 cs=1 l=1).fault on lossy channel, failing -1734> 2018-02-14 13:23:45.014234 7fc50acfa700 2 -- 10.255.0.30:6789/0 >> 10.255.0.52:0/3152189720 conn(0x5587d8da8000 :6789 s=STATE_OPEN pgs=445 cs=1 l=1)._stop -1733> 2018-02-14 13:23:45.014276 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_handle_reset 0x5587d8da8000 10.255.0.52:0/3152189720 -1732> 2018-02-14 13:23:45.014291 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 reset/close on session client.? 10.255.0.52:0/3152189720 -1731> 2018-02-14 13:23:45.014295 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 remove_session 0x5587d891e880 client.? 10.255.0.52:0/3152189720 features 0x1ffddff8eea4fffb -1730> 2018-02-14 13:23:45.052504 7fc50acfa700 1 -- 10.255.0.30:6789/0 >> - conn(0x5587d8fcd800 :6789 s=STATE_ACCEPTING pgs=0 cs=0 l=0)._process_connection sd=46 - -1729> 2018-02-14 13:23:45.052696 7fc50acfa700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_verify_authorizer 10.255.0.106:0/1777238177 client protocol 0 -1728> 2018-02-14 13:23:45.052721 7fc50acfa700 2 -- 10.255.0.30:6789/0 >> 10.255.0.106:0/1777238177 conn(0x5587d8fcd800 :6789 s=STATE_ACCEPTING_WAIT_SEQ pgs=1161 cs=1 l=1).handle_connect_msg accept write reply msg done -1727> 2018-02-14 13:23:45.052811 7fc50acfa700 2 -- 10.255.0.30:6789/0 >> 10.255.0.106:0/1777238177 conn(0x5587d8fcd800 :6789 s=STATE_ACCEPTING_WAIT_SEQ pgs=1161 cs=1 l=1)._process_connection accept get newly_acked_seq 0 -1726> 2018-02-14 13:23:45.052903 7fc50acfa700 5 -- 10.255.0.30:6789/0 >> 10.255.0.106:0/1777238177 conn(0x5587d8fcd800 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1161 cs=1 l=1). rx client.1684199 seq 1 0x5587d8fda000 auth(proto 0 35 bytes epoch 1) v1 -1725> 2018-02-14 13:23:45.052931 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1684199 10.255.0.106:0/1777238177 1 ==== auth(proto 0 35 bytes epoch 1) v1 ==== 65+0+0 (824200203 0 0) 0x5587d8fda000 con 0x5587d8fcd800 -1724> 2018-02-14 13:23:45.052952 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch new session 0x5587d9149200 MonSession(client.1684199 10.255.0.106:0/1777238177 is open ) features 0x1ffddff8eea4fffb -1723> 2018-02-14 13:23:45.052960 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 caps -1722> 2018-02-14 13:23:45.052962 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing) e1 waitlisting message auth(proto 0 35 bytes epoch 1) v1 -1721> 2018-02-14 13:23:45.072067 7fc511507700 1 -- 10.255.0.30:6789/0 >> 10.255.0.53:0/1148525388 conn(0x5587d8da9800 :6789 s=STATE_OPEN pgs=425 cs=1 l=1).read_bulk peer close file descriptor 9 -1720> 2018-02-14 13:23:45.072081 7fc511507700 1 -- 10.255.0.30:6789/0 >> 10.255.0.53:0/1148525388 conn(0x5587d8da9800 :6789 s=STATE_OPEN pgs=425 cs=1 l=1).read_until read failed -1719> 2018-02-14 13:23:45.072098 7fc511507700 1 -- 10.255.0.30:6789/0 >> 10.255.0.53:0/1148525388 conn(0x5587d8da9800 :6789 s=STATE_OPEN pgs=425 cs=1 l=1).process read tag failed -1718> 2018-02-14 13:23:45.072103 7fc511507700 1 -- 10.255.0.30:6789/0 >> 10.255.0.53:0/1148525388 conn(0x5587d8da9800 :6789 s=STATE_OPEN pgs=425 cs=1 l=1).fault on lossy channel, failing -1717> 2018-02-14 13:23:45.072107 7fc511507700 2 -- 10.255.0.30:6789/0 >> 10.255.0.53:0/1148525388 conn(0x5587d8da9800 :6789 s=STATE_OPEN pgs=425 cs=1 l=1)._stop -1716> 2018-02-14 13:23:45.072151 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_handle_reset 0x5587d8da9800 10.255.0.53:0/1148525388 -1715> 2018-02-14 13:23:45.072166 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 reset/close on session client.? 10.255.0.53:0/1148525388 -1714> 2018-02-14 13:23:45.072169 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 remove_session 0x5587d891eb80 client.? 10.255.0.53:0/1148525388 features 0x1ffddff8eea4fffb -1713> 2018-02-14 13:23:45.072331 7fc511507700 1 -- 10.255.0.30:6789/0 >> - conn(0x5587d8fcf000 :6789 s=STATE_ACCEPTING pgs=0 cs=0 l=0)._process_connection sd=9 - -1712> 2018-02-14 13:23:45.072521 7fc511507700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_verify_authorizer 10.255.0.53:0/1148525388 client protocol 0 -1711> 2018-02-14 13:23:45.072549 7fc511507700 2 -- 10.255.0.30:6789/0 >> 10.255.0.53:0/1148525388 conn(0x5587d8fcf000 :6789 s=STATE_ACCEPTING_WAIT_SEQ pgs=426 cs=1 l=1).handle_connect_msg accept write reply msg done -1710> 2018-02-14 13:23:45.072657 7fc511507700 2 -- 10.255.0.30:6789/0 >> 10.255.0.53:0/1148525388 conn(0x5587d8fcf000 :6789 s=STATE_ACCEPTING_WAIT_SEQ pgs=426 cs=1 l=1)._process_connection accept get newly_acked_seq 0 -1709> 2018-02-14 13:23:45.072748 7fc511507700 5 -- 10.255.0.30:6789/0 >> 10.255.0.53:0/1148525388 conn(0x5587d8fcf000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=426 cs=1 l=1). rx client.? seq 1 0x5587d8ec3840 auth(proto 0 35 bytes epoch 0) v1 -1708> 2018-02-14 13:23:45.072769 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.? 10.255.0.53:0/1148525388 1 ==== auth(proto 0 35 bytes epoch 0) v1 ==== 65+0+0 (3478176772 0 0) 0x5587d8ec3840 con 0x5587d8fcf000 -1707> 2018-02-14 13:23:45.072798 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch new session 0x5587d9149380 MonSession(client.? 10.255.0.53:0/1148525388 is open ) features 0x1ffddff8eea4fffb -1706> 2018-02-14 13:23:45.072805 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 caps -1705> 2018-02-14 13:23:45.072808 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing) e1 waitlisting message auth(proto 0 35 bytes epoch 0) v1 -1704> 2018-02-14 13:23:45.222441 7fc50b4fb700 1 -- 10.255.0.30:6789/0 >> 10.255.0.54:0/3194015072 conn(0x5587d8dab000 :6789 s=STATE_OPEN pgs=428 cs=1 l=1).read_bulk peer close file descriptor 10 -1703> 2018-02-14 13:23:45.222469 7fc50b4fb700 1 -- 10.255.0.30:6789/0 >> 10.255.0.54:0/3194015072 conn(0x5587d8dab000 :6789 s=STATE_OPEN pgs=428 cs=1 l=1).read_until read failed -1702> 2018-02-14 13:23:45.222474 7fc50b4fb700 1 -- 10.255.0.30:6789/0 >> 10.255.0.54:0/3194015072 conn(0x5587d8dab000 :6789 s=STATE_OPEN pgs=428 cs=1 l=1).process read tag failed -1701> 2018-02-14 13:23:45.222478 7fc50b4fb700 1 -- 10.255.0.30:6789/0 >> 10.255.0.54:0/3194015072 conn(0x5587d8dab000 :6789 s=STATE_OPEN pgs=428 cs=1 l=1).fault on lossy channel, failing -1700> 2018-02-14 13:23:45.222483 7fc50b4fb700 2 -- 10.255.0.30:6789/0 >> 10.255.0.54:0/3194015072 conn(0x5587d8dab000 :6789 s=STATE_OPEN pgs=428 cs=1 l=1)._stop -1699> 2018-02-14 13:23:45.222523 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_handle_reset 0x5587d8dab000 10.255.0.54:0/3194015072 -1698> 2018-02-14 13:23:45.222539 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 reset/close on session client.? 10.255.0.54:0/3194015072 -1697> 2018-02-14 13:23:45.222543 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 remove_session 0x5587d891ee80 client.? 10.255.0.54:0/3194015072 features 0x1ffddff8eea4fffb -1696> 2018-02-14 13:23:45.222637 7fc50b4fb700 1 -- 10.255.0.30:6789/0 >> - conn(0x5587d8fd0800 :6789 s=STATE_ACCEPTING pgs=0 cs=0 l=0)._process_connection sd=10 - -1695> 2018-02-14 13:23:45.222820 7fc50b4fb700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_verify_authorizer 10.255.0.54:0/3194015072 client protocol 0 -1694> 2018-02-14 13:23:45.222846 7fc50b4fb700 2 -- 10.255.0.30:6789/0 >> 10.255.0.54:0/3194015072 conn(0x5587d8fd0800 :6789 s=STATE_ACCEPTING_WAIT_SEQ pgs=429 cs=1 l=1).handle_connect_msg accept write reply msg done -1693> 2018-02-14 13:23:45.222994 7fc50b4fb700 2 -- 10.255.0.30:6789/0 >> 10.255.0.54:0/3194015072 conn(0x5587d8fd0800 :6789 s=STATE_ACCEPTING_WAIT_SEQ pgs=429 cs=1 l=1)._process_connection accept get newly_acked_seq 0 -1692> 2018-02-14 13:23:45.223113 7fc50b4fb700 5 -- 10.255.0.30:6789/0 >> 10.255.0.54:0/3194015072 conn(0x5587d8fd0800 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=429 cs=1 l=1). rx client.? seq 1 0x5587d8ec3180 auth(proto 0 35 bytes epoch 0) v1 -1691> 2018-02-14 13:23:45.223158 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.? 10.255.0.54:0/3194015072 1 ==== auth(proto 0 35 bytes epoch 0) v1 ==== 65+0+0 (3478176772 0 0) 0x5587d8ec3180 con 0x5587d8fd0800 -1690> 2018-02-14 13:23:45.223179 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch new session 0x5587d9149500 MonSession(client.? 10.255.0.54:0/3194015072 is open ) features 0x1ffddff8eea4fffb -1689> 2018-02-14 13:23:45.223186 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 caps -1688> 2018-02-14 13:23:45.223189 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing) e1 waitlisting message auth(proto 0 35 bytes epoch 0) v1 -1687> 2018-02-14 13:23:45.322772 7fc511507700 1 -- 10.255.0.30:6789/0 >> - conn(0x5587d8fe2000 :6789 s=STATE_ACCEPTING pgs=0 cs=0 l=0)._process_connection sd=56 - -1686> 2018-02-14 13:23:45.322995 7fc511507700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_verify_authorizer 10.255.0.107:0/980595480 client protocol 0 -1685> 2018-02-14 13:23:45.323030 7fc511507700 2 -- 10.255.0.30:6789/0 >> 10.255.0.107:0/980595480 conn(0x5587d8fe2000 :6789 s=STATE_ACCEPTING_WAIT_SEQ pgs=441 cs=1 l=1).handle_connect_msg accept write reply msg done -1684> 2018-02-14 13:23:45.323153 7fc511507700 2 -- 10.255.0.30:6789/0 >> 10.255.0.107:0/980595480 conn(0x5587d8fe2000 :6789 s=STATE_ACCEPTING_WAIT_SEQ pgs=441 cs=1 l=1)._process_connection accept get newly_acked_seq 0 -1683> 2018-02-14 13:23:45.323269 7fc511507700 5 -- 10.255.0.30:6789/0 >> 10.255.0.107:0/980595480 conn(0x5587d8fe2000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=441 cs=1 l=1). rx client.? seq 1 0x5587d8ec3a80 auth(proto 0 35 bytes epoch 0) v1 -1682> 2018-02-14 13:23:45.323290 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.? 10.255.0.107:0/980595480 1 ==== auth(proto 0 35 bytes epoch 0) v1 ==== 65+0+0 (3478176772 0 0) 0x5587d8ec3a80 con 0x5587d8fe2000 -1681> 2018-02-14 13:23:45.323312 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch new session 0x5587d9149680 MonSession(client.? 10.255.0.107:0/980595480 is open ) features 0x1ffddff8eea4fffb -1680> 2018-02-14 13:23:45.323319 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 caps -1679> 2018-02-14 13:23:45.323321 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing) e1 waitlisting message auth(proto 0 35 bytes epoch 0) v1 -1678> 2018-02-14 13:23:46.214566 7fc510505700 11 mon.ceph-dal-mon-a1@0(electing) e1 tick -1677> 2018-02-14 13:23:46.214602 7fc510505700 20 mon.ceph-dal-mon-a1@0(electing) e1 sync_trim_providers -1676> 2018-02-14 13:23:46.214608 7fc510505700 5 mon.ceph-dal-mon-a1@0(electing).paxos(paxos recovering c 4935163..4935904) is_readable = 0 - now=2018-02-14 13:23:46.214608 lease_expire=0.000000 has v0 lc 4935904 -1675> 2018-02-14 13:23:46.214618 7fc510505700 5 mon.ceph-dal-mon-a1@0(electing).paxos(paxos recovering c 4935163..4935904) is_readable = 0 - now=2018-02-14 13:23:46.214618 lease_expire=0.000000 has v0 lc 4935904 -1674> 2018-02-14 13:23:46.214623 7fc510505700 5 mon.ceph-dal-mon-a1@0(electing).paxos(paxos recovering c 4935163..4935904) is_readable = 0 - now=2018-02-14 13:23:46.214624 lease_expire=0.000000 has v0 lc 4935904 -1673> 2018-02-14 13:23:46.214628 7fc510505700 5 mon.ceph-dal-mon-a1@0(electing).paxos(paxos recovering c 4935163..4935904) is_readable = 0 - now=2018-02-14 13:23:46.214628 lease_expire=0.000000 has v0 lc 4935904 -1672> 2018-02-14 13:23:46.214634 7fc510505700 5 mon.ceph-dal-mon-a1@0(electing).paxos(paxos recovering c 4935163..4935904) is_readable = 0 - now=2018-02-14 13:23:46.214635 lease_expire=0.000000 has v0 lc 4935904 -1671> 2018-02-14 13:23:46.214641 7fc510505700 5 mon.ceph-dal-mon-a1@0(electing).paxos(paxos recovering c 4935163..4935904) is_readable = 0 - now=2018-02-14 13:23:46.214641 lease_expire=0.000000 has v0 lc 4935904 -1670> 2018-02-14 13:23:46.214647 7fc510505700 10 mon.ceph-dal-mon-a1@0(electing) e1 session closed, dropping 0x5587d88db840 -1669> 2018-02-14 13:23:46.214661 7fc510505700 10 mon.ceph-dal-mon-a1@0(electing) e1 session closed, dropping 0x5587d88dbcc0 -1668> 2018-02-14 13:23:46.214670 7fc510505700 5 mon.ceph-dal-mon-a1@0(electing).paxos(paxos recovering c 4935163..4935904) is_readable = 0 - now=2018-02-14 13:23:46.214670 lease_expire=0.000000 has v0 lc 4935904 -1667> 2018-02-14 13:23:46.214675 7fc510505700 5 mon.ceph-dal-mon-a1@0(electing).paxos(paxos recovering c 4935163..4935904) is_readable = 0 - now=2018-02-14 13:23:46.214675 lease_expire=0.000000 has v0 lc 4935904 -1666> 2018-02-14 13:23:46.214689 7fc510505700 5 mon.ceph-dal-mon-a1@0(electing).paxos(paxos recovering c 4935163..4935904) is_readable = 0 - now=2018-02-14 13:23:46.214690 lease_expire=0.000000 has v0 lc 4935904 -1665> 2018-02-14 13:23:46.214696 7fc510505700 5 mon.ceph-dal-mon-a1@0(electing).paxos(paxos recovering c 4935163..4935904) is_readable = 0 - now=2018-02-14 13:23:46.214696 lease_expire=0.000000 has v0 lc 4935904 -1664> 2018-02-14 13:23:46.214702 7fc510505700 5 mon.ceph-dal-mon-a1@0(electing).paxos(paxos recovering c 4935163..4935904) is_readable = 0 - now=2018-02-14 13:23:46.214702 lease_expire=0.000000 has v0 lc 4935904 -1663> 2018-02-14 13:23:46.214708 7fc510505700 5 mon.ceph-dal-mon-a1@0(electing).paxos(paxos recovering c 4935163..4935904) is_readable = 0 - now=2018-02-14 13:23:46.214708 lease_expire=0.000000 has v0 lc 4935904 -1662> 2018-02-14 13:23:46.214714 7fc510505700 5 mon.ceph-dal-mon-a1@0(electing).paxos(paxos recovering c 4935163..4935904) is_readable = 0 - now=2018-02-14 13:23:46.214714 lease_expire=0.000000 has v0 lc 4935904 -1661> 2018-02-14 13:23:46.214719 7fc510505700 5 mon.ceph-dal-mon-a1@0(electing).paxos(paxos recovering c 4935163..4935904) is_readable = 0 - now=2018-02-14 13:23:46.214720 lease_expire=0.000000 has v0 lc 4935904 -1660> 2018-02-14 13:23:46.214725 7fc510505700 5 mon.ceph-dal-mon-a1@0(electing).paxos(paxos recovering c 4935163..4935904) is_readable = 0 - now=2018-02-14 13:23:46.214726 lease_expire=0.000000 has v0 lc 4935904 -1659> 2018-02-14 13:23:46.214731 7fc510505700 5 mon.ceph-dal-mon-a1@0(electing).paxos(paxos recovering c 4935163..4935904) is_readable = 0 - now=2018-02-14 13:23:46.214731 lease_expire=0.000000 has v0 lc 4935904 -1658> 2018-02-14 13:23:46.214736 7fc510505700 10 mon.ceph-dal-mon-a1@0(electing) e1 session closed, dropping 0x5587d8ec0b40 -1657> 2018-02-14 13:23:46.214744 7fc510505700 10 mon.ceph-dal-mon-a1@0(electing) e1 session closed, dropping 0x5587d8ec1440 -1656> 2018-02-14 13:23:46.214751 7fc510505700 10 mon.ceph-dal-mon-a1@0(electing) e1 session closed, dropping 0x5587d8ec0fc0 -1655> 2018-02-14 13:23:46.214758 7fc510505700 5 mon.ceph-dal-mon-a1@0(electing).paxos(paxos recovering c 4935163..4935904) is_readable = 0 - now=2018-02-14 13:23:46.214759 lease_expire=0.000000 has v0 lc 4935904 -1654> 2018-02-14 13:23:46.214765 7fc510505700 5 mon.ceph-dal-mon-a1@0(electing).paxos(paxos recovering c 4935163..4935904) is_readable = 0 - now=2018-02-14 13:23:46.214765 lease_expire=0.000000 has v0 lc 4935904 -1653> 2018-02-14 13:23:46.214771 7fc510505700 5 mon.ceph-dal-mon-a1@0(electing).paxos(paxos recovering c 4935163..4935904) is_readable = 0 - now=2018-02-14 13:23:46.214772 lease_expire=0.000000 has v0 lc 4935904 -1652> 2018-02-14 13:23:46.214777 7fc510505700 5 mon.ceph-dal-mon-a1@0(electing).paxos(paxos recovering c 4935163..4935904) is_readable = 0 - now=2018-02-14 13:23:46.214777 lease_expire=0.000000 has v0 lc 4935904 -1651> 2018-02-14 13:23:46.214797 7fc510505700 5 mon.ceph-dal-mon-a1@0(electing).paxos(paxos recovering c 4935163..4935904) is_readable = 0 - now=2018-02-14 13:23:46.214798 lease_expire=0.000000 has v0 lc 4935904 -1650> 2018-02-14 13:23:46.214803 7fc510505700 5 mon.ceph-dal-mon-a1@0(electing).paxos(paxos recovering c 4935163..4935904) is_readable = 0 - now=2018-02-14 13:23:46.214804 lease_expire=0.000000 has v0 lc 4935904 -1649> 2018-02-14 13:23:46.214809 7fc510505700 5 mon.ceph-dal-mon-a1@0(electing).paxos(paxos recovering c 4935163..4935904) is_readable = 0 - now=2018-02-14 13:23:46.214810 lease_expire=0.000000 has v0 lc 4935904 -1648> 2018-02-14 13:23:46.214815 7fc510505700 5 mon.ceph-dal-mon-a1@0(electing).paxos(paxos recovering c 4935163..4935904) is_readable = 0 - now=2018-02-14 13:23:46.214815 lease_expire=0.000000 has v0 lc 4935904 -1647> 2018-02-14 13:23:46.214821 7fc510505700 5 mon.ceph-dal-mon-a1@0(electing).paxos(paxos recovering c 4935163..4935904) is_readable = 0 - now=2018-02-14 13:23:46.214821 lease_expire=0.000000 has v0 lc 4935904 -1646> 2018-02-14 13:23:46.214826 7fc510505700 5 mon.ceph-dal-mon-a1@0(electing).paxos(paxos recovering c 4935163..4935904) is_readable = 0 - now=2018-02-14 13:23:46.214827 lease_expire=0.000000 has v0 lc 4935904 -1645> 2018-02-14 13:23:46.214832 7fc510505700 5 mon.ceph-dal-mon-a1@0(electing).paxos(paxos recovering c 4935163..4935904) is_readable = 0 - now=2018-02-14 13:23:46.214832 lease_expire=0.000000 has v0 lc 4935904 -1644> 2018-02-14 13:23:46.214837 7fc510505700 5 mon.ceph-dal-mon-a1@0(electing).paxos(paxos recovering c 4935163..4935904) is_readable = 0 - now=2018-02-14 13:23:46.214838 lease_expire=0.000000 has v0 lc 4935904 -1643> 2018-02-14 13:23:46.214843 7fc510505700 5 mon.ceph-dal-mon-a1@0(electing).paxos(paxos recovering c 4935163..4935904) is_readable = 0 - now=2018-02-14 13:23:46.214843 lease_expire=0.000000 has v0 lc 4935904 -1642> 2018-02-14 13:23:46.214848 7fc510505700 5 mon.ceph-dal-mon-a1@0(electing).paxos(paxos recovering c 4935163..4935904) is_readable = 0 - now=2018-02-14 13:23:46.214849 lease_expire=0.000000 has v0 lc 4935904 -1641> 2018-02-14 13:23:46.214854 7fc510505700 5 mon.ceph-dal-mon-a1@0(electing).paxos(paxos recovering c 4935163..4935904) is_readable = 0 - now=2018-02-14 13:23:46.214854 lease_expire=0.000000 has v0 lc 4935904 -1640> 2018-02-14 13:23:46.214859 7fc510505700 5 mon.ceph-dal-mon-a1@0(electing).paxos(paxos recovering c 4935163..4935904) is_readable = 0 - now=2018-02-14 13:23:46.214860 lease_expire=0.000000 has v0 lc 4935904 -1639> 2018-02-14 13:23:46.214865 7fc510505700 5 mon.ceph-dal-mon-a1@0(electing).paxos(paxos recovering c 4935163..4935904) is_readable = 0 - now=2018-02-14 13:23:46.214865 lease_expire=0.000000 has v0 lc 4935904 -1638> 2018-02-14 13:23:46.214870 7fc510505700 5 mon.ceph-dal-mon-a1@0(electing).paxos(paxos recovering c 4935163..4935904) is_readable = 0 - now=2018-02-14 13:23:46.214871 lease_expire=0.000000 has v0 lc 4935904 -1637> 2018-02-14 13:23:46.214876 7fc510505700 5 mon.ceph-dal-mon-a1@0(electing).paxos(paxos recovering c 4935163..4935904) is_readable = 0 - now=2018-02-14 13:23:46.214877 lease_expire=0.000000 has v0 lc 4935904 -1636> 2018-02-14 13:23:46.214881 7fc510505700 5 mon.ceph-dal-mon-a1@0(electing).paxos(paxos recovering c 4935163..4935904) is_readable = 0 - now=2018-02-14 13:23:46.214882 lease_expire=0.000000 has v0 lc 4935904 -1635> 2018-02-14 13:23:46.214887 7fc510505700 5 mon.ceph-dal-mon-a1@0(electing).paxos(paxos recovering c 4935163..4935904) is_readable = 0 - now=2018-02-14 13:23:46.214887 lease_expire=0.000000 has v0 lc 4935904 -1634> 2018-02-14 13:23:46.214892 7fc510505700 5 mon.ceph-dal-mon-a1@0(electing).paxos(paxos recovering c 4935163..4935904) is_readable = 0 - now=2018-02-14 13:23:46.214892 lease_expire=0.000000 has v0 lc 4935904 -1633> 2018-02-14 13:23:46.214897 7fc510505700 5 mon.ceph-dal-mon-a1@0(electing).paxos(paxos recovering c 4935163..4935904) is_readable = 0 - now=2018-02-14 13:23:46.214898 lease_expire=0.000000 has v0 lc 4935904 -1632> 2018-02-14 13:23:46.214903 7fc510505700 5 mon.ceph-dal-mon-a1@0(electing).paxos(paxos recovering c 4935163..4935904) is_readable = 0 - now=2018-02-14 13:23:46.214903 lease_expire=0.000000 has v0 lc 4935904 -1631> 2018-02-14 13:23:46.214908 7fc510505700 5 mon.ceph-dal-mon-a1@0(electing).paxos(paxos recovering c 4935163..4935904) is_readable = 0 - now=2018-02-14 13:23:46.214909 lease_expire=0.000000 has v0 lc 4935904 -1630> 2018-02-14 13:23:46.214914 7fc510505700 5 mon.ceph-dal-mon-a1@0(electing).paxos(paxos recovering c 4935163..4935904) is_readable = 0 - now=2018-02-14 13:23:46.214915 lease_expire=0.000000 has v0 lc 4935904 -1629> 2018-02-14 13:23:46.214920 7fc510505700 5 mon.ceph-dal-mon-a1@0(electing).paxos(paxos recovering c 4935163..4935904) is_readable = 0 - now=2018-02-14 13:23:46.214920 lease_expire=0.000000 has v0 lc 4935904 -1628> 2018-02-14 13:23:46.214925 7fc510505700 5 mon.ceph-dal-mon-a1@0(electing).paxos(paxos recovering c 4935163..4935904) is_readable = 0 - now=2018-02-14 13:23:46.214926 lease_expire=0.000000 has v0 lc 4935904 -1627> 2018-02-14 13:23:46.214931 7fc510505700 5 mon.ceph-dal-mon-a1@0(electing).paxos(paxos recovering c 4935163..4935904) is_readable = 0 - now=2018-02-14 13:23:46.214931 lease_expire=0.000000 has v0 lc 4935904 -1626> 2018-02-14 13:23:46.214936 7fc510505700 5 mon.ceph-dal-mon-a1@0(electing).paxos(paxos recovering c 4935163..4935904) is_readable = 0 - now=2018-02-14 13:23:46.214937 lease_expire=0.000000 has v0 lc 4935904 -1625> 2018-02-14 13:23:46.214941 7fc510505700 5 mon.ceph-dal-mon-a1@0(electing).paxos(paxos recovering c 4935163..4935904) is_readable = 0 - now=2018-02-14 13:23:46.214942 lease_expire=0.000000 has v0 lc 4935904 -1624> 2018-02-14 13:23:46.214947 7fc510505700 5 mon.ceph-dal-mon-a1@0(electing).paxos(paxos recovering c 4935163..4935904) is_readable = 0 - now=2018-02-14 13:23:46.214947 lease_expire=0.000000 has v0 lc 4935904 -1623> 2018-02-14 13:23:46.214952 7fc510505700 5 mon.ceph-dal-mon-a1@0(electing).paxos(paxos recovering c 4935163..4935904) is_readable = 0 - now=2018-02-14 13:23:46.214953 lease_expire=0.000000 has v0 lc 4935904 -1622> 2018-02-14 13:23:46.214957 7fc510505700 5 mon.ceph-dal-mon-a1@0(electing).paxos(paxos recovering c 4935163..4935904) is_readable = 0 - now=2018-02-14 13:23:46.214958 lease_expire=0.000000 has v0 lc 4935904 -1621> 2018-02-14 13:23:46.214963 7fc510505700 5 mon.ceph-dal-mon-a1@0(electing).paxos(paxos recovering c 4935163..4935904) is_readable = 0 - now=2018-02-14 13:23:46.214963 lease_expire=0.000000 has v0 lc 4935904 -1620> 2018-02-14 13:23:46.214968 7fc510505700 5 mon.ceph-dal-mon-a1@0(electing).paxos(paxos recovering c 4935163..4935904) is_readable = 0 - now=2018-02-14 13:23:46.214968 lease_expire=0.000000 has v0 lc 4935904 -1619> 2018-02-14 13:23:46.992958 7fc50b4fb700 1 -- 10.255.0.30:6789/0 >> - conn(0x5587d8fe3800 :6789 s=STATE_ACCEPTING pgs=0 cs=0 l=0)._process_connection sd=57 - -1618> 2018-02-14 13:23:46.993156 7fc50b4fb700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_verify_authorizer 10.255.0.108:0/850049828 client protocol 0 -1617> 2018-02-14 13:23:46.993187 7fc50b4fb700 2 -- 10.255.0.30:6789/0 >> 10.255.0.108:0/850049828 conn(0x5587d8fe3800 :6789 s=STATE_ACCEPTING_WAIT_SEQ pgs=382 cs=1 l=1).handle_connect_msg accept write reply msg done -1616> 2018-02-14 13:23:46.993267 7fc50b4fb700 2 -- 10.255.0.30:6789/0 >> 10.255.0.108:0/850049828 conn(0x5587d8fe3800 :6789 s=STATE_ACCEPTING_WAIT_SEQ pgs=382 cs=1 l=1)._process_connection accept get newly_acked_seq 0 -1615> 2018-02-14 13:23:46.993418 7fc50b4fb700 5 -- 10.255.0.30:6789/0 >> 10.255.0.108:0/850049828 conn(0x5587d8fe3800 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=382 cs=1 l=1). rx client.1714098 seq 1 0x5587d88db840 auth(proto 0 35 bytes epoch 1) v1 -1614> 2018-02-14 13:23:46.993439 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1714098 10.255.0.108:0/850049828 1 ==== auth(proto 0 35 bytes epoch 1) v1 ==== 65+0+0 (2611868766 0 0) 0x5587d88db840 con 0x5587d8fe3800 -1613> 2018-02-14 13:23:46.993461 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch new session 0x5587d9149800 MonSession(client.1714098 10.255.0.108:0/850049828 is open ) features 0x1ffddff8eea4fffb -1612> 2018-02-14 13:23:46.993469 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 caps -1611> 2018-02-14 13:23:46.993471 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing) e1 waitlisting message auth(proto 0 35 bytes epoch 1) v1 -1610> 2018-02-14 13:23:47.421546 7fc511507700 1 -- 10.255.0.30:6789/0 >> 10.255.0.31:6789/0 conn(0x5587d8f81000 :-1 s=STATE_CONNECTING_RE pgs=0 cs=0 l=0)._process_connection reconnect failed -1609> 2018-02-14 13:23:47.421562 7fc511507700 2 -- 10.255.0.30:6789/0 >> 10.255.0.31:6789/0 conn(0x5587d8f81000 :-1 s=STATE_CONNECTING_RE pgs=0 cs=0 l=0)._process_connection connection refused! -1608> 2018-02-14 13:23:47.421583 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_handle_refused 0x5587d8f81000 10.255.0.31:6789/0 -1607> 2018-02-14 13:23:47.499077 7fc50acfa700 1 -- 10.255.0.30:6789/0 >> 10.255.0.108:0/989583023 conn(0x5587d91ba000 :6789 s=STATE_OPEN pgs=432 cs=1 l=1).read_bulk peer close file descriptor 53 -1606> 2018-02-14 13:23:47.499092 7fc50acfa700 1 -- 10.255.0.30:6789/0 >> 10.255.0.108:0/989583023 conn(0x5587d91ba000 :6789 s=STATE_OPEN pgs=432 cs=1 l=1).read_until read failed -1605> 2018-02-14 13:23:47.499097 7fc50acfa700 1 -- 10.255.0.30:6789/0 >> 10.255.0.108:0/989583023 conn(0x5587d91ba000 :6789 s=STATE_OPEN pgs=432 cs=1 l=1).process read tag failed -1604> 2018-02-14 13:23:47.499101 7fc50acfa700 1 -- 10.255.0.30:6789/0 >> 10.255.0.108:0/989583023 conn(0x5587d91ba000 :6789 s=STATE_OPEN pgs=432 cs=1 l=1).fault on lossy channel, failing -1603> 2018-02-14 13:23:47.499105 7fc50acfa700 2 -- 10.255.0.30:6789/0 >> 10.255.0.108:0/989583023 conn(0x5587d91ba000 :6789 s=STATE_OPEN pgs=432 cs=1 l=1)._stop -1602> 2018-02-14 13:23:47.499205 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_handle_reset 0x5587d91ba000 10.255.0.108:0/989583023 -1601> 2018-02-14 13:23:47.499221 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 reset/close on session client.? 10.255.0.108:0/989583023 -1600> 2018-02-14 13:23:47.499225 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 remove_session 0x5587d9148900 client.? 10.255.0.108:0/989583023 features 0x1ffddff8eea4fffb -1599> 2018-02-14 13:23:47.515624 7fc50acfa700 1 -- 10.255.0.30:6789/0 >> - conn(0x5587d8fe5000 :6789 s=STATE_ACCEPTING pgs=0 cs=0 l=0)._process_connection sd=53 - -1598> 2018-02-14 13:23:47.515956 7fc50acfa700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_verify_authorizer 10.255.0.54:0/1233019732 client protocol 0 -1597> 2018-02-14 13:23:47.515982 7fc50acfa700 2 -- 10.255.0.30:6789/0 >> 10.255.0.54:0/1233019732 conn(0x5587d8fe5000 :6789 s=STATE_ACCEPTING_WAIT_SEQ pgs=380 cs=1 l=1).handle_connect_msg accept write reply msg done -1596> 2018-02-14 13:23:47.516185 7fc50acfa700 2 -- 10.255.0.30:6789/0 >> 10.255.0.54:0/1233019732 conn(0x5587d8fe5000 :6789 s=STATE_ACCEPTING_WAIT_SEQ pgs=380 cs=1 l=1)._process_connection accept get newly_acked_seq 0 -1595> 2018-02-14 13:23:47.516333 7fc50acfa700 5 -- 10.255.0.30:6789/0 >> 10.255.0.54:0/1233019732 conn(0x5587d8fe5000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=380 cs=1 l=1). rx client.1714104 seq 1 0x5587d8fda240 auth(proto 0 35 bytes epoch 1) v1 -1594> 2018-02-14 13:23:47.516386 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1714104 10.255.0.54:0/1233019732 1 ==== auth(proto 0 35 bytes epoch 1) v1 ==== 65+0+0 (1999407951 0 0) 0x5587d8fda240 con 0x5587d8fe5000 -1593> 2018-02-14 13:23:47.516410 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch new session 0x5587d9149980 MonSession(client.1714104 10.255.0.54:0/1233019732 is open ) features 0x1ffddff8eea4fffb -1592> 2018-02-14 13:23:47.516417 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 caps -1591> 2018-02-14 13:23:47.516419 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing) e1 waitlisting message auth(proto 0 35 bytes epoch 1) v1 -1590> 2018-02-14 13:23:47.586548 7fc50acfa700 1 -- 10.255.0.30:6789/0 >> 10.255.0.51:0/3461880882 conn(0x5587d91bb800 :6789 s=STATE_OPEN pgs=433 cs=1 l=1).read_bulk peer close file descriptor 39 -1589> 2018-02-14 13:23:47.586563 7fc50acfa700 1 -- 10.255.0.30:6789/0 >> 10.255.0.51:0/3461880882 conn(0x5587d91bb800 :6789 s=STATE_OPEN pgs=433 cs=1 l=1).read_until read failed -1588> 2018-02-14 13:23:47.586567 7fc50acfa700 1 -- 10.255.0.30:6789/0 >> 10.255.0.51:0/3461880882 conn(0x5587d91bb800 :6789 s=STATE_OPEN pgs=433 cs=1 l=1).process read tag failed -1587> 2018-02-14 13:23:47.586571 7fc50acfa700 1 -- 10.255.0.30:6789/0 >> 10.255.0.51:0/3461880882 conn(0x5587d91bb800 :6789 s=STATE_OPEN pgs=433 cs=1 l=1).fault on lossy channel, failing -1586> 2018-02-14 13:23:47.586574 7fc50acfa700 2 -- 10.255.0.30:6789/0 >> 10.255.0.51:0/3461880882 conn(0x5587d91bb800 :6789 s=STATE_OPEN pgs=433 cs=1 l=1)._stop -1585> 2018-02-14 13:23:47.586677 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_handle_reset 0x5587d91bb800 10.255.0.51:0/3461880882 -1584> 2018-02-14 13:23:47.586694 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 reset/close on session client.? 10.255.0.51:0/3461880882 -1583> 2018-02-14 13:23:47.586698 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 remove_session 0x5587d9148a80 client.? 10.255.0.51:0/3461880882 features 0x1ffddff8eea4fffb -1582> 2018-02-14 13:23:47.586846 7fc50acfa700 1 -- 10.255.0.30:6789/0 >> - conn(0x5587d8fe6800 :6789 s=STATE_ACCEPTING pgs=0 cs=0 l=0)._process_connection sd=39 - -1581> 2018-02-14 13:23:47.587159 7fc50acfa700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_verify_authorizer 10.255.0.51:0/3461880882 client protocol 0 -1580> 2018-02-14 13:23:47.587185 7fc50acfa700 2 -- 10.255.0.30:6789/0 >> 10.255.0.51:0/3461880882 conn(0x5587d8fe6800 :6789 s=STATE_ACCEPTING_WAIT_SEQ pgs=438 cs=1 l=1).handle_connect_msg accept write reply msg done -1579> 2018-02-14 13:23:47.587434 7fc50acfa700 2 -- 10.255.0.30:6789/0 >> 10.255.0.51:0/3461880882 conn(0x5587d8fe6800 :6789 s=STATE_ACCEPTING_WAIT_SEQ pgs=438 cs=1 l=1)._process_connection accept get newly_acked_seq 0 -1578> 2018-02-14 13:23:47.587577 7fc50acfa700 5 -- 10.255.0.30:6789/0 >> 10.255.0.51:0/3461880882 conn(0x5587d8fe6800 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=438 cs=1 l=1). rx client.? seq 1 0x5587d8fda480 auth(proto 0 35 bytes epoch 0) v1 -1577> 2018-02-14 13:23:47.587616 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.? 10.255.0.51:0/3461880882 1 ==== auth(proto 0 35 bytes epoch 0) v1 ==== 65+0+0 (3478176772 0 0) 0x5587d8fda480 con 0x5587d8fe6800 -1576> 2018-02-14 13:23:47.587639 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch new session 0x5587d9149b00 MonSession(client.? 10.255.0.51:0/3461880882 is open ) features 0x1ffddff8eea4fffb -1575> 2018-02-14 13:23:47.587647 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 caps -1574> 2018-02-14 13:23:47.587650 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing) e1 waitlisting message auth(proto 0 35 bytes epoch 0) v1 -1573> 2018-02-14 13:23:47.930724 7fc511507700 1 -- 10.255.0.30:6789/0 >> 10.255.0.105:0/1919733865 conn(0x5587d91bd000 :6789 s=STATE_OPEN pgs=427 cs=1 l=1).read_bulk peer close file descriptor 40 -1572> 2018-02-14 13:23:47.930738 7fc511507700 1 -- 10.255.0.30:6789/0 >> 10.255.0.105:0/1919733865 conn(0x5587d91bd000 :6789 s=STATE_OPEN pgs=427 cs=1 l=1).read_until read failed -1571> 2018-02-14 13:23:47.930743 7fc511507700 1 -- 10.255.0.30:6789/0 >> 10.255.0.105:0/1919733865 conn(0x5587d91bd000 :6789 s=STATE_OPEN pgs=427 cs=1 l=1).process read tag failed -1570> 2018-02-14 13:23:47.930770 7fc511507700 1 -- 10.255.0.30:6789/0 >> 10.255.0.105:0/1919733865 conn(0x5587d91bd000 :6789 s=STATE_OPEN pgs=427 cs=1 l=1).fault on lossy channel, failing -1569> 2018-02-14 13:23:47.930775 7fc511507700 2 -- 10.255.0.30:6789/0 >> 10.255.0.105:0/1919733865 conn(0x5587d91bd000 :6789 s=STATE_OPEN pgs=427 cs=1 l=1)._stop -1568> 2018-02-14 13:23:47.930851 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_handle_reset 0x5587d91bd000 10.255.0.105:0/1919733865 -1567> 2018-02-14 13:23:47.930868 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 reset/close on session client.? 10.255.0.105:0/1919733865 -1566> 2018-02-14 13:23:47.930871 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 remove_session 0x5587d9148c00 client.? 10.255.0.105:0/1919733865 features 0x1ffddff8eea4fffb -1565> 2018-02-14 13:23:48.014774 7fc511507700 1 -- 10.255.0.30:6789/0 >> - conn(0x5587d8da9800 :6789 s=STATE_ACCEPTING pgs=0 cs=0 l=0)._process_connection sd=40 - -1564> 2018-02-14 13:23:48.014987 7fc511507700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_verify_authorizer 10.255.0.52:0/3152189720 client protocol 0 -1563> 2018-02-14 13:23:48.015028 7fc511507700 2 -- 10.255.0.30:6789/0 >> 10.255.0.52:0/3152189720 conn(0x5587d8da9800 :6789 s=STATE_ACCEPTING_WAIT_SEQ pgs=455 cs=1 l=1).handle_connect_msg accept write reply msg done -1562> 2018-02-14 13:23:48.015131 7fc511507700 2 -- 10.255.0.30:6789/0 >> 10.255.0.52:0/3152189720 conn(0x5587d8da9800 :6789 s=STATE_ACCEPTING_WAIT_SEQ pgs=455 cs=1 l=1)._process_connection accept get newly_acked_seq 0 -1561> 2018-02-14 13:23:48.015221 7fc511507700 5 -- 10.255.0.30:6789/0 >> 10.255.0.52:0/3152189720 conn(0x5587d8da9800 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=455 cs=1 l=1). rx client.? seq 1 0x5587d8fdad80 auth(proto 0 35 bytes epoch 0) v1 -1560> 2018-02-14 13:23:48.015242 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.? 10.255.0.52:0/3152189720 1 ==== auth(proto 0 35 bytes epoch 0) v1 ==== 65+0+0 (3478176772 0 0) 0x5587d8fdad80 con 0x5587d8da9800 -1559> 2018-02-14 13:23:48.015262 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch new session 0x5587d9149c80 MonSession(client.? 10.255.0.52:0/3152189720 is open ) features 0x1ffddff8eea4fffb -1558> 2018-02-14 13:23:48.015269 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 caps -1557> 2018-02-14 13:23:48.015271 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing) e1 waitlisting message auth(proto 0 35 bytes epoch 0) v1 -1556> 2018-02-14 13:23:48.072681 7fc511507700 1 -- 10.255.0.30:6789/0 >> 10.255.0.53:0/1148525388 conn(0x5587d8fcf000 :6789 s=STATE_OPEN pgs=426 cs=1 l=1).read_bulk peer close file descriptor 9 -1555> 2018-02-14 13:23:48.072696 7fc511507700 1 -- 10.255.0.30:6789/0 >> 10.255.0.53:0/1148525388 conn(0x5587d8fcf000 :6789 s=STATE_OPEN pgs=426 cs=1 l=1).read_until read failed -1554> 2018-02-14 13:23:48.072700 7fc511507700 1 -- 10.255.0.30:6789/0 >> 10.255.0.53:0/1148525388 conn(0x5587d8fcf000 :6789 s=STATE_OPEN pgs=426 cs=1 l=1).process read tag failed -1553> 2018-02-14 13:23:48.072704 7fc511507700 1 -- 10.255.0.30:6789/0 >> 10.255.0.53:0/1148525388 conn(0x5587d8fcf000 :6789 s=STATE_OPEN pgs=426 cs=1 l=1).fault on lossy channel, failing -1552> 2018-02-14 13:23:48.072707 7fc511507700 2 -- 10.255.0.30:6789/0 >> 10.255.0.53:0/1148525388 conn(0x5587d8fcf000 :6789 s=STATE_OPEN pgs=426 cs=1 l=1)._stop -1551> 2018-02-14 13:23:48.072809 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_handle_reset 0x5587d8fcf000 10.255.0.53:0/1148525388 -1550> 2018-02-14 13:23:48.072825 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 reset/close on session client.? 10.255.0.53:0/1148525388 -1549> 2018-02-14 13:23:48.072829 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 remove_session 0x5587d9149380 client.? 10.255.0.53:0/1148525388 features 0x1ffddff8eea4fffb -1548> 2018-02-14 13:23:48.224035 7fc50b4fb700 1 -- 10.255.0.30:6789/0 >> 10.255.0.54:0/3194015072 conn(0x5587d8fd0800 :6789 s=STATE_OPEN pgs=429 cs=1 l=1).read_bulk peer close file descriptor 10 -1547> 2018-02-14 13:23:48.224049 7fc50b4fb700 1 -- 10.255.0.30:6789/0 >> 10.255.0.54:0/3194015072 conn(0x5587d8fd0800 :6789 s=STATE_OPEN pgs=429 cs=1 l=1).read_until read failed -1546> 2018-02-14 13:23:48.224059 7fc50b4fb700 1 -- 10.255.0.30:6789/0 >> 10.255.0.54:0/3194015072 conn(0x5587d8fd0800 :6789 s=STATE_OPEN pgs=429 cs=1 l=1).process read tag failed -1545> 2018-02-14 13:23:48.224068 7fc50b4fb700 1 -- 10.255.0.30:6789/0 >> 10.255.0.54:0/3194015072 conn(0x5587d8fd0800 :6789 s=STATE_OPEN pgs=429 cs=1 l=1).fault on lossy channel, failing -1544> 2018-02-14 13:23:48.224077 7fc50b4fb700 2 -- 10.255.0.30:6789/0 >> 10.255.0.54:0/3194015072 conn(0x5587d8fd0800 :6789 s=STATE_OPEN pgs=429 cs=1 l=1)._stop -1543> 2018-02-14 13:23:48.224139 7fc50acfa700 1 -- 10.255.0.30:6789/0 reap_dead start -1542> 2018-02-14 13:23:48.224138 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_handle_reset 0x5587d8fd0800 10.255.0.54:0/3194015072 -1541> 2018-02-14 13:23:48.224155 7fc50acfa700 5 -- 10.255.0.30:6789/0 reap_dead delete 0x5587d8f2c000 -1540> 2018-02-14 13:23:48.224157 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 reset/close on session client.? 10.255.0.54:0/3194015072 -1539> 2018-02-14 13:23:48.224161 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 remove_session 0x5587d9149500 client.? 10.255.0.54:0/3194015072 features 0x1ffddff8eea4fffb -1538> 2018-02-14 13:23:48.224163 7fc50acfa700 5 -- 10.255.0.30:6789/0 reap_dead delete 0x5587d8fcf000 -1537> 2018-02-14 13:23:48.224166 7fc50acfa700 5 -- 10.255.0.30:6789/0 reap_dead delete 0x5587d8fd0800 -1536> 2018-02-14 13:23:48.224169 7fc50acfa700 5 -- 10.255.0.30:6789/0 reap_dead delete 0x5587d91ba000 -1535> 2018-02-14 13:23:48.224171 7fc50acfa700 5 -- 10.255.0.30:6789/0 reap_dead delete 0x5587d91bd000 -1534> 2018-02-14 13:23:48.224321 7fc50b4fb700 1 -- 10.255.0.30:6789/0 >> - conn(0x5587d8f2c000 :6789 s=STATE_ACCEPTING pgs=0 cs=0 l=0)._process_connection sd=9 - -1533> 2018-02-14 13:23:48.224558 7fc50b4fb700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_verify_authorizer 10.255.0.54:0/3194015072 client protocol 0 -1532> 2018-02-14 13:23:48.224589 7fc50b4fb700 2 -- 10.255.0.30:6789/0 >> 10.255.0.54:0/3194015072 conn(0x5587d8f2c000 :6789 s=STATE_ACCEPTING_WAIT_SEQ pgs=432 cs=1 l=1).handle_connect_msg accept write reply msg done -1531> 2018-02-14 13:23:48.224758 7fc50b4fb700 2 -- 10.255.0.30:6789/0 >> 10.255.0.54:0/3194015072 conn(0x5587d8f2c000 :6789 s=STATE_ACCEPTING_WAIT_SEQ pgs=432 cs=1 l=1)._process_connection accept get newly_acked_seq 0 -1530> 2018-02-14 13:23:48.224948 7fc50b4fb700 5 -- 10.255.0.30:6789/0 >> 10.255.0.54:0/3194015072 conn(0x5587d8f2c000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=432 cs=1 l=1). rx client.? seq 1 0x5587d88dbcc0 auth(proto 0 35 bytes epoch 0) v1 -1529> 2018-02-14 13:23:48.224968 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.? 10.255.0.54:0/3194015072 1 ==== auth(proto 0 35 bytes epoch 0) v1 ==== 65+0+0 (3478176772 0 0) 0x5587d88dbcc0 con 0x5587d8f2c000 -1528> 2018-02-14 13:23:48.224990 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch new session 0x5587d9149e00 MonSession(client.? 10.255.0.54:0/3194015072 is open ) features 0x1ffddff8eea4fffb -1527> 2018-02-14 13:23:48.224998 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 caps -1526> 2018-02-14 13:23:48.225000 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing) e1 waitlisting message auth(proto 0 35 bytes epoch 0) v1 -1525> 2018-02-14 13:23:48.322862 7fc511507700 1 -- 10.255.0.30:6789/0 >> 10.255.0.107:0/980595480 conn(0x5587d8fe2000 :6789 s=STATE_OPEN pgs=441 cs=1 l=1).read_bulk peer close file descriptor 56 -1524> 2018-02-14 13:23:48.322878 7fc511507700 1 -- 10.255.0.30:6789/0 >> 10.255.0.107:0/980595480 conn(0x5587d8fe2000 :6789 s=STATE_OPEN pgs=441 cs=1 l=1).read_until read failed -1523> 2018-02-14 13:23:48.322888 7fc511507700 1 -- 10.255.0.30:6789/0 >> 10.255.0.107:0/980595480 conn(0x5587d8fe2000 :6789 s=STATE_OPEN pgs=441 cs=1 l=1).process read tag failed -1522> 2018-02-14 13:23:48.322897 7fc511507700 1 -- 10.255.0.30:6789/0 >> 10.255.0.107:0/980595480 conn(0x5587d8fe2000 :6789 s=STATE_OPEN pgs=441 cs=1 l=1).fault on lossy channel, failing -1521> 2018-02-14 13:23:48.322918 7fc511507700 2 -- 10.255.0.30:6789/0 >> 10.255.0.107:0/980595480 conn(0x5587d8fe2000 :6789 s=STATE_OPEN pgs=441 cs=1 l=1)._stop -1520> 2018-02-14 13:23:48.322965 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_handle_reset 0x5587d8fe2000 10.255.0.107:0/980595480 -1519> 2018-02-14 13:23:48.322981 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 reset/close on session client.? 10.255.0.107:0/980595480 -1518> 2018-02-14 13:23:48.322985 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 remove_session 0x5587d9149680 client.? 10.255.0.107:0/980595480 features 0x1ffddff8eea4fffb -1517> 2018-02-14 13:23:48.323093 7fc511507700 1 -- 10.255.0.30:6789/0 >> - conn(0x5587d8d40000 :6789 s=STATE_ACCEPTING pgs=0 cs=0 l=0)._process_connection sd=10 - -1516> 2018-02-14 13:23:48.323364 7fc511507700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_verify_authorizer 10.255.0.107:0/980595480 client protocol 0 -1515> 2018-02-14 13:23:48.323397 7fc511507700 2 -- 10.255.0.30:6789/0 >> 10.255.0.107:0/980595480 conn(0x5587d8d40000 :6789 s=STATE_ACCEPTING_WAIT_SEQ pgs=442 cs=1 l=1).handle_connect_msg accept write reply msg done -1514> 2018-02-14 13:23:48.323478 7fc511507700 2 -- 10.255.0.30:6789/0 >> 10.255.0.107:0/980595480 conn(0x5587d8d40000 :6789 s=STATE_ACCEPTING_WAIT_SEQ pgs=442 cs=1 l=1)._process_connection accept get newly_acked_seq 0 -1513> 2018-02-14 13:23:48.323575 7fc511507700 5 -- 10.255.0.30:6789/0 >> 10.255.0.107:0/980595480 conn(0x5587d8d40000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=442 cs=1 l=1). rx client.? seq 1 0x5587d8fdafc0 auth(proto 0 35 bytes epoch 0) v1 -1512> 2018-02-14 13:23:48.323595 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.? 10.255.0.107:0/980595480 1 ==== auth(proto 0 35 bytes epoch 0) v1 ==== 65+0+0 (3478176772 0 0) 0x5587d8fdafc0 con 0x5587d8d40000 -1511> 2018-02-14 13:23:48.323615 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch new session 0x5587d9149f80 MonSession(client.? 10.255.0.107:0/980595480 is open ) features 0x1ffddff8eea4fffb -1510> 2018-02-14 13:23:48.323622 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 caps -1509> 2018-02-14 13:23:48.323624 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing) e1 waitlisting message auth(proto 0 35 bytes epoch 0) v1 -1508> 2018-02-14 13:23:48.519938 7fc50b4fb700 1 -- 10.255.0.30:6789/0 >> - conn(0x5587d9004000 :6789 s=STATE_ACCEPTING pgs=0 cs=0 l=0)._process_connection sd=56 - -1507> 2018-02-14 13:23:48.520130 7fc50b4fb700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_verify_authorizer 10.255.0.107:0/3082328679 client protocol 0 -1506> 2018-02-14 13:23:48.520166 7fc50b4fb700 2 -- 10.255.0.30:6789/0 >> 10.255.0.107:0/3082328679 conn(0x5587d9004000 :6789 s=STATE_ACCEPTING_WAIT_SEQ pgs=1115 cs=1 l=1).handle_connect_msg accept write reply msg done -1505> 2018-02-14 13:23:48.520285 7fc50b4fb700 2 -- 10.255.0.30:6789/0 >> 10.255.0.107:0/3082328679 conn(0x5587d9004000 :6789 s=STATE_ACCEPTING_WAIT_SEQ pgs=1115 cs=1 l=1)._process_connection accept get newly_acked_seq 0 -1504> 2018-02-14 13:23:48.520391 7fc50b4fb700 5 -- 10.255.0.30:6789/0 >> 10.255.0.107:0/3082328679 conn(0x5587d9004000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1115 cs=1 l=1). rx client.1684202 seq 1 0x5587d8ec0b40 auth(proto 0 35 bytes epoch 1) v1 -1503> 2018-02-14 13:23:48.520412 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1684202 10.255.0.107:0/3082328679 1 ==== auth(proto 0 35 bytes epoch 1) v1 ==== 65+0+0 (1274290058 0 0) 0x5587d8ec0b40 con 0x5587d9004000 -1502> 2018-02-14 13:23:48.520445 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch new session 0x5587d914a100 MonSession(client.1684202 10.255.0.107:0/3082328679 is open ) features 0x1ffddff8eea4fffb -1501> 2018-02-14 13:23:48.520453 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 caps -1500> 2018-02-14 13:23:48.520455 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing) e1 waitlisting message auth(proto 0 35 bytes epoch 1) v1 -1499> 2018-02-14 13:23:48.584470 7fc50acfa700 1 -- 10.255.0.30:6789/0 >> - conn(0x5587d9005800 :6789 s=STATE_ACCEPTING pgs=0 cs=0 l=0)._process_connection sd=58 - -1498> 2018-02-14 13:23:48.584605 7fc50acfa700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_verify_authorizer 10.255.0.52:0/97505378 client protocol 0 -1497> 2018-02-14 13:23:48.584631 7fc50acfa700 2 -- 10.255.0.30:6789/0 >> 10.255.0.52:0/97505378 conn(0x5587d9005800 :6789 s=STATE_ACCEPTING_WAIT_SEQ pgs=6642 cs=1 l=1).handle_connect_msg accept write reply msg done -1496> 2018-02-14 13:23:48.584703 7fc50acfa700 2 -- 10.255.0.30:6789/0 >> 10.255.0.52:0/97505378 conn(0x5587d9005800 :6789 s=STATE_ACCEPTING_WAIT_SEQ pgs=6642 cs=1 l=1)._process_connection accept get newly_acked_seq 0 -1495> 2018-02-14 13:23:48.584765 7fc50acfa700 5 -- 10.255.0.30:6789/0 >> 10.255.0.52:0/97505378 conn(0x5587d9005800 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=6642 cs=1 l=1). rx client.1393717 seq 1 0x5587d8fda6c0 auth(proto 0 39 bytes epoch 0) -1494> 2018-02-14 13:23:48.584800 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1393717 10.255.0.52:0/97505378 1 ==== auth(proto 0 39 bytes epoch 0) ==== 65+0+0 (110478951 0 0) 0x5587d8fda6c0 con 0x5587d9005800 -1493> 2018-02-14 13:23:48.584821 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch new session 0x5587d914a280 MonSession(client.1393717 10.255.0.52:0/97505378 is open ) features 0x7010fb86aa42ada -1492> 2018-02-14 13:23:48.584828 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 caps -1491> 2018-02-14 13:23:48.584830 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing) e1 waitlisting message auth(proto 0 39 bytes epoch 0) -1490> 2018-02-14 13:23:48.666889 7fc511507700 1 -- 10.255.0.30:6789/0 >> - conn(0x5587d9007000 :6789 s=STATE_ACCEPTING pgs=0 cs=0 l=0)._process_connection sd=59 - -1489> 2018-02-14 13:23:48.667084 7fc511507700 10 mon.ceph-dal-mon-a1@0(electing) e1 ms_verify_authorizer 10.255.0.52:0/4056696166 client protocol 0 -1488> 2018-02-14 13:23:48.667115 7fc511507700 2 -- 10.255.0.30:6789/0 >> 10.255.0.52:0/4056696166 conn(0x5587d9007000 :6789 s=STATE_ACCEPTING_WAIT_SEQ pgs=1121 cs=1 l=1).handle_connect_msg accept write reply msg done -1487> 2018-02-14 13:23:48.667238 7fc511507700 2 -- 10.255.0.30:6789/0 >> 10.255.0.52:0/4056696166 conn(0x5587d9007000 :6789 s=STATE_ACCEPTING_WAIT_SEQ pgs=1121 cs=1 l=1)._process_connection accept get newly_acked_seq 0 -1486> 2018-02-14 13:23:48.667320 7fc511507700 5 -- 10.255.0.30:6789/0 >> 10.255.0.52:0/4056696166 conn(0x5587d9007000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1121 cs=1 l=1). rx client.1684208 seq 1 0x5587d8fdb200 auth(proto 0 35 bytes epoch 1) v1 -1485> 2018-02-14 13:23:48.667342 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1684208 10.255.0.52:0/4056696166 1 ==== auth(proto 0 35 bytes epoch 1) v1 ==== 65+0+0 (3193747592 0 0) 0x5587d8fdb200 con 0x5587d9007000 -1484> 2018-02-14 13:23:48.667362 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(electing) e1 _ms_dispatch new session 0x5587d914a400 MonSession(client.1684208 10.255.0.52:0/4056696166 is open ) features 0x1ffddff8eea4fffb -1483> 2018-02-14 13:23:48.667370 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(electing) e1 caps -1482> 2018-02-14 13:23:48.667372 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(electing) e1 waitlisting message auth(proto 0 35 bytes epoch 1) v1 -1481> 2018-02-14 13:23:48.790173 7fc510505700 5 mon.ceph-dal-mon-a1@0(electing).elector(503) election timer expired -1480> 2018-02-14 13:23:48.790190 7fc510505700 10 mon.ceph-dal-mon-a1@0(electing).elector(503) bump_epoch 503 to 504 -1479> 2018-02-14 13:23:48.864928 7fc510505700 10 mon.ceph-dal-mon-a1@0(electing) e1 join_election -1478> 2018-02-14 13:23:48.864939 7fc510505700 10 mon.ceph-dal-mon-a1@0(electing) e1 _reset -1477> 2018-02-14 13:23:48.864942 7fc510505700 10 mon.ceph-dal-mon-a1@0(electing) e1 cancel_probe_timeout (none scheduled) -1476> 2018-02-14 13:23:48.864944 7fc510505700 10 mon.ceph-dal-mon-a1@0(electing) e1 timecheck_finish -1475> 2018-02-14 13:23:48.864946 7fc510505700 15 mon.ceph-dal-mon-a1@0(electing) e1 health_tick_stop -1474> 2018-02-14 13:23:48.864947 7fc510505700 15 mon.ceph-dal-mon-a1@0(electing) e1 health_interval_stop -1473> 2018-02-14 13:23:48.864949 7fc510505700 10 mon.ceph-dal-mon-a1@0(electing) e1 scrub_event_cancel -1472> 2018-02-14 13:23:48.864951 7fc510505700 10 mon.ceph-dal-mon-a1@0(electing) e1 scrub_reset -1471> 2018-02-14 13:23:48.864975 7fc510505700 1 -- 10.255.0.30:6789/0 _send_message--> mon.2 10.255.0.32:6789/0 -- election(39fac7f7-f5a8-4e84-b9bb-b0dfffd79c92 victory 504) v7 -- ?+0 0x5587d895b300 -1470> 2018-02-14 13:23:48.864997 7fc510505700 1 -- 10.255.0.30:6789/0 --> 10.255.0.32:6789/0 -- election(39fac7f7-f5a8-4e84-b9bb-b0dfffd79c92 victory 504) v7 -- 0x5587d895b300 con 0 -1469> 2018-02-14 13:23:48.865014 7fc510505700 10 mon.ceph-dal-mon-a1@0(electing) e1 win_election epoch 504 quorum 0,2 features 2305244844532236283 mon_features mon_feature_t([kraken,luminous]) -1468> 2018-02-14 13:23:48.865035 7fc510505700 0 log_channel(cluster) log [INF] : mon.ceph-dal-mon-a1@0 won leader election with quorum 0,2 -1467> 2018-02-14 13:23:48.865039 7fc510505700 10 log_client _send_to_monlog to self -1466> 2018-02-14 13:23:48.865041 7fc510505700 10 log_client log_queue is 3 last_log 3 sent 2 num 3 unsent 1 sending 1 -1465> 2018-02-14 13:23:48.865044 7fc510505700 10 log_client will send 2018-02-14 13:23:48.865038 mon.ceph-dal-mon-a1 mon.0 10.255.0.30:6789/0 3 : cluster [INF] mon.ceph-dal-mon-a1@0 won leader election with quorum 0,2 -1464> 2018-02-14 13:23:48.865053 7fc510505700 1 -- 10.255.0.30:6789/0 --> 10.255.0.30:6789/0 -- log(1 entries from seq 3 at 2018-02-14 13:23:48.865038) v1 -- 0x5587d8fdbb00 con 0 -1463> 2018-02-14 13:23:48.865101 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== mon.0 10.255.0.30:6789/0 0 ==== log(1 entries from seq 3 at 2018-02-14 13:23:48.865038) v1 ==== 0+0+0 (0 0 0) 0x5587d8fdbb00 con 0x5587d8d41800 -1462> 2018-02-14 13:23:48.895830 7fc50acfa700 5 -- 10.255.0.30:6789/0 >> 10.255.0.32:6789/0 conn(0x5587d8f82800 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=51044 cs=1 l=0). rx mon.2 seq 386006850 0x5587d895e580 forward(log(1 entries from seq 25 at 2018-02-14 13:10:18.636291) v1 caps allow * tid 2577 con_features 2305244844532236283) v3 -1461> 2018-02-14 13:23:48.906730 7fc510505700 1 -- 10.255.0.30:6789/0 _send_message--> mon.2 10.255.0.32:6789/0 -- paxos(collect lc 4935904 fc 4935163 pn 4700 opn 0) v4 -- ?+0 0x5587d895b600 -1460> 2018-02-14 13:23:48.906752 7fc510505700 1 -- 10.255.0.30:6789/0 --> 10.255.0.32:6789/0 -- paxos(collect lc 4935904 fc 4935163 pn 4700 opn 0) v4 -- 0x5587d895b600 con 0 -1459> 2018-02-14 13:23:48.906774 7fc510505700 10 mon.ceph-dal-mon-a1@0(leader).data_health(504) start_epoch epoch 504 -1458> 2018-02-14 13:23:48.906820 7fc510505700 5 mon.ceph-dal-mon-a1@0(leader) e1 apply_quorum_to_compatset_features -1457> 2018-02-14 13:23:48.906840 7fc510505700 5 mon.ceph-dal-mon-a1@0(leader) e1 apply_monmap_to_compatset_features -1456> 2018-02-14 13:23:48.906847 7fc510505700 10 mon.ceph-dal-mon-a1@0(leader) e1 timecheck_finish -1455> 2018-02-14 13:23:48.906853 7fc510505700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos recovering c 4935163..4935904) is_readable = 0 - now=2018-02-14 13:23:48.906853 lease_expire=0.000000 has v0 lc 4935904 -1454> 2018-02-14 13:23:48.906863 7fc510505700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos recovering c 4935163..4935904) is_readable = 0 - now=2018-02-14 13:23:48.906863 lease_expire=0.000000 has v0 lc 4935904 -1453> 2018-02-14 13:23:48.906870 7fc510505700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos recovering c 4935163..4935904) is_readable = 0 - now=2018-02-14 13:23:48.906870 lease_expire=0.000000 has v0 lc 4935904 -1452> 2018-02-14 13:23:48.906875 7fc510505700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos recovering c 4935163..4935904) is_readable = 0 - now=2018-02-14 13:23:48.906876 lease_expire=0.000000 has v0 lc 4935904 -1451> 2018-02-14 13:23:48.906881 7fc510505700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos recovering c 4935163..4935904) is_readable = 0 - now=2018-02-14 13:23:48.906882 lease_expire=0.000000 has v0 lc 4935904 -1450> 2018-02-14 13:23:48.906887 7fc510505700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos recovering c 4935163..4935904) is_readable = 0 - now=2018-02-14 13:23:48.906887 lease_expire=0.000000 has v0 lc 4935904 -1449> 2018-02-14 13:23:48.906893 7fc510505700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos recovering c 4935163..4935904) is_readable = 0 - now=2018-02-14 13:23:48.906893 lease_expire=0.000000 has v0 lc 4935904 -1448> 2018-02-14 13:23:48.906898 7fc510505700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos recovering c 4935163..4935904) is_readable = 0 - now=2018-02-14 13:23:48.906898 lease_expire=0.000000 has v0 lc 4935904 -1447> 2018-02-14 13:23:48.906903 7fc510505700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos recovering c 4935163..4935904) is_readable = 0 - now=2018-02-14 13:23:48.906904 lease_expire=0.000000 has v0 lc 4935904 -1446> 2018-02-14 13:23:48.906909 7fc510505700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos recovering c 4935163..4935904) is_readable = 0 - now=2018-02-14 13:23:48.906909 lease_expire=0.000000 has v0 lc 4935904 -1445> 2018-02-14 13:23:48.906915 7fc510505700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos recovering c 4935163..4935904) is_readable = 0 - now=2018-02-14 13:23:48.906915 lease_expire=0.000000 has v0 lc 4935904 -1444> 2018-02-14 13:23:48.906920 7fc510505700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos recovering c 4935163..4935904) is_readable = 0 - now=2018-02-14 13:23:48.906920 lease_expire=0.000000 has v0 lc 4935904 -1443> 2018-02-14 13:23:48.906925 7fc510505700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos recovering c 4935163..4935904) is_readable = 0 - now=2018-02-14 13:23:48.906926 lease_expire=0.000000 has v0 lc 4935904 -1442> 2018-02-14 13:23:48.906931 7fc510505700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos recovering c 4935163..4935904) is_readable = 0 - now=2018-02-14 13:23:48.906931 lease_expire=0.000000 has v0 lc 4935904 -1441> 2018-02-14 13:23:48.906936 7fc510505700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos recovering c 4935163..4935904) is_readable = 0 - now=2018-02-14 13:23:48.906937 lease_expire=0.000000 has v0 lc 4935904 -1440> 2018-02-14 13:23:48.906942 7fc510505700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos recovering c 4935163..4935904) is_readable = 0 - now=2018-02-14 13:23:48.906942 lease_expire=0.000000 has v0 lc 4935904 -1439> 2018-02-14 13:23:48.906947 7fc510505700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos recovering c 4935163..4935904) is_readable = 0 - now=2018-02-14 13:23:48.906947 lease_expire=0.000000 has v0 lc 4935904 -1438> 2018-02-14 13:23:48.906952 7fc510505700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos recovering c 4935163..4935904) is_readable = 0 - now=2018-02-14 13:23:48.906953 lease_expire=0.000000 has v0 lc 4935904 -1437> 2018-02-14 13:23:48.906960 7fc510505700 10 mon.ceph-dal-mon-a1@0(leader) e1 resend_routed_requests -1436> 2018-02-14 13:23:48.906964 7fc510505700 10 mon.ceph-dal-mon-a1@0(leader) e1 register_cluster_logger -1435> 2018-02-14 13:23:48.906991 7fc510505700 10 mon.ceph-dal-mon-a1@0(leader) e1 timecheck_start -1434> 2018-02-14 13:23:48.906994 7fc510505700 10 mon.ceph-dal-mon-a1@0(leader) e1 timecheck_start_round curr 0 -1433> 2018-02-14 13:23:48.906997 7fc510505700 10 mon.ceph-dal-mon-a1@0(leader) e1 timecheck_start_round new 1 -1432> 2018-02-14 13:23:48.907000 7fc510505700 10 mon.ceph-dal-mon-a1@0(leader) e1 timecheck -1431> 2018-02-14 13:23:48.907002 7fc510505700 10 mon.ceph-dal-mon-a1@0(leader) e1 timecheck start timecheck epoch 504 round 1 -1430> 2018-02-14 13:23:48.907008 7fc510505700 10 mon.ceph-dal-mon-a1@0(leader) e1 timecheck send time_check( ping e 504 r 1 ) v1 to mon.2 10.255.0.32:6789/0 -1429> 2018-02-14 13:23:48.907015 7fc510505700 1 -- 10.255.0.30:6789/0 _send_message--> mon.2 10.255.0.32:6789/0 -- time_check( ping e 504 r 1 ) v1 -- ?+0 0x5587d8fdb8c0 -1428> 2018-02-14 13:23:48.907023 7fc510505700 1 -- 10.255.0.30:6789/0 --> 10.255.0.32:6789/0 -- time_check( ping e 504 r 1 ) v1 -- 0x5587d8fdb8c0 con 0 -1427> 2018-02-14 13:23:48.907033 7fc510505700 10 mon.ceph-dal-mon-a1@0(leader) e1 timecheck_start_round setting up next event -1426> 2018-02-14 13:23:48.907038 7fc510505700 10 mon.ceph-dal-mon-a1@0(leader) e1 timecheck_reset_event delay 300 rounds_since_clean 0 -1425> 2018-02-14 13:23:48.907049 7fc510505700 15 mon.ceph-dal-mon-a1@0(leader) e1 health_tick_start -1424> 2018-02-14 13:23:48.907051 7fc510505700 15 mon.ceph-dal-mon-a1@0(leader) e1 health_tick_stop -1423> 2018-02-14 13:23:48.907054 7fc510505700 10 mon.ceph-dal-mon-a1@0(leader) e1 do_health_to_clog_interval -1422> 2018-02-14 13:23:48.907057 7fc510505700 10 mon.ceph-dal-mon-a1@0(leader) e1 do_health_to_clog (force) -1421> 2018-02-14 13:23:48.907076 7fc510505700 0 log_channel(cluster) log [WRN] : overall HEALTH_WARN noout flag(s) set; 6 backfillfull osd(s); 4 osds down; 1 host (4 osds) down; 8 nearfull osd(s); 3 pool(s) backfillfull; Degraded data redundancy: 4938712/26491706 objects degraded (18.642%), 347 pgs unclean, 347 pgs degraded, 347 pgs undersized; 1/3 mons down, quorum ceph-dal-mon-a1,ceph-dal-mon-a3 -1420> 2018-02-14 13:23:48.907085 7fc510505700 10 log_client _send_to_monlog to self -1419> 2018-02-14 13:23:48.907086 7fc510505700 10 log_client log_queue is 4 last_log 4 sent 3 num 4 unsent 1 sending 1 -1418> 2018-02-14 13:23:48.907089 7fc510505700 10 log_client will send 2018-02-14 13:23:48.907083 mon.ceph-dal-mon-a1 mon.0 10.255.0.30:6789/0 4 : cluster [WRN] overall HEALTH_WARN noout flag(s) set; 6 backfillfull osd(s); 4 osds down; 1 host (4 osds) down; 8 nearfull osd(s); 3 pool(s) backfillfull; Degraded data redundancy: 4938712/26491706 objects degraded (18.642%), 347 pgs unclean, 347 pgs degraded, 347 pgs undersized; 1/3 mons down, quorum ceph-dal-mon-a1,ceph-dal-mon-a3 -1417> 2018-02-14 13:23:48.907098 7fc510505700 1 -- 10.255.0.30:6789/0 --> 10.255.0.30:6789/0 -- log(1 entries from seq 4 at 2018-02-14 13:23:48.907083) v1 -- 0x5587d8fdc1c0 con 0 -1416> 2018-02-14 13:23:48.907109 7fc510505700 15 mon.ceph-dal-mon-a1@0(leader) e1 health_interval_start -1415> 2018-02-14 13:23:48.907112 7fc510505700 15 mon.ceph-dal-mon-a1@0(leader) e1 health_interval_stop -1414> 2018-02-14 13:23:48.907113 7fc510505700 20 mon.ceph-dal-mon-a1@0(leader) e1 health_interval_calc_next_update now: 2018-02-14 13:23:48.907113, next: 2018-02-14 14:00:00.000000, interval: 3600 -1413> 2018-02-14 13:23:48.907118 7fc510505700 10 mon.ceph-dal-mon-a1@0(leader) e1 scrub_event_start -1412> 2018-02-14 13:23:48.907159 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891ca80 for mon.0 10.255.0.30:6789/0 -1411> 2018-02-14 13:23:48.907174 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps allow * -1410> 2018-02-14 13:23:48.907189 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos recovering c 4935163..4935904) is_readable = 0 - now=2018-02-14 13:23:48.907190 lease_expire=0.000000 has v0 lc 4935904 -1409> 2018-02-14 13:23:48.907200 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos recovering c 4935163..4935904) is_readable = 0 - now=2018-02-14 13:23:48.907201 lease_expire=0.000000 has v0 lc 4935904 -1408> 2018-02-14 13:23:48.907209 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== mon.2 10.255.0.32:6789/0 386006850 ==== forward(log(1 entries from seq 25 at 2018-02-14 13:10:18.636291) v1 caps allow * tid 2577 con_features 2305244844532236283) v3 ==== 355+0+0 (273235745 0 0) 0x5587d895e580 con 0x5587d8f82800 -1407> 2018-02-14 13:23:48.907222 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891c480 for mon.2 10.255.0.32:6789/0 -1406> 2018-02-14 13:23:48.907227 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps allow * -1405> 2018-02-14 13:23:48.907230 7fc50dd00700 20 is_capable service=mon command= read on cap allow * -1404> 2018-02-14 13:23:48.907231 7fc50dd00700 20 allow so far , doing grant allow * -1403> 2018-02-14 13:23:48.907232 7fc50dd00700 20 allow all -1402> 2018-02-14 13:23:48.907234 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 received forwarded message from mon.2 10.255.0.32:6789/0 via mon.2 10.255.0.32:6789/0 -1401> 2018-02-14 13:23:48.907239 7fc50dd00700 20 is_capable service=mon command= exec on cap allow * -1400> 2018-02-14 13:23:48.907240 7fc50dd00700 20 allow so far , doing grant allow * -1399> 2018-02-14 13:23:48.907241 7fc50dd00700 20 allow all -1398> 2018-02-14 13:23:48.907249 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 caps are allow * -1397> 2018-02-14 13:23:48.907250 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 entity name '' type 0 -1396> 2018-02-14 13:23:48.907252 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 mesg 0x5587d8fda900 from 10.255.0.32:6789/0 -1395> 2018-02-14 13:23:48.907256 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d914a580 for mon.2 - -1394> 2018-02-14 13:23:48.907258 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps allow * -1393> 2018-02-14 13:23:48.907259 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos recovering c 4935163..4935904) is_readable = 0 - now=2018-02-14 13:23:48.907260 lease_expire=0.000000 has v0 lc 4935904 -1392> 2018-02-14 13:23:48.907264 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos recovering c 4935163..4935904) is_readable = 0 - now=2018-02-14 13:23:48.907265 lease_expire=0.000000 has v0 lc 4935904 -1391> 2018-02-14 13:23:48.907276 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== mon.0 10.255.0.30:6789/0 0 ==== log(1 entries from seq 4 at 2018-02-14 13:23:48.907083) v1 ==== 0+0+0 (0 0 0) 0x5587d8fdc1c0 con 0x5587d8d41800 -1390> 2018-02-14 13:23:48.907286 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891ca80 for mon.0 10.255.0.30:6789/0 -1389> 2018-02-14 13:23:48.907291 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps allow * -1388> 2018-02-14 13:23:48.907293 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos recovering c 4935163..4935904) is_readable = 0 - now=2018-02-14 13:23:48.907294 lease_expire=0.000000 has v0 lc 4935904 -1387> 2018-02-14 13:23:48.907298 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos recovering c 4935163..4935904) is_readable = 0 - now=2018-02-14 13:23:48.907298 lease_expire=0.000000 has v0 lc 4935904 -1386> 2018-02-14 13:23:48.929198 7fc50acfa700 5 -- 10.255.0.30:6789/0 >> 10.255.0.32:6789/0 conn(0x5587d8f82800 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=51044 cs=1 l=0). rx mon.2 seq 386006851 0x5587d895b300 paxos(last lc 4935904 fc 4935163 pn 4700 opn 0) v4 -1385> 2018-02-14 13:23:48.929232 7fc50acfa700 5 -- 10.255.0.30:6789/0 >> 10.255.0.32:6789/0 conn(0x5587d8f82800 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=51044 cs=1 l=0). rx mon.2 seq 386006852 0x5587d8fdb8c0 time_check( pong e 504 r 1 ts 2018-02-14 13:23:48.928277 ) v1 -1384> 2018-02-14 13:23:48.929264 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== mon.2 10.255.0.32:6789/0 386006851 ==== paxos(last lc 4935904 fc 4935163 pn 4700 opn 0) v4 ==== 84+0+0 (3297345876 0 0) 0x5587d895b300 con 0x5587d8f82800 -1383> 2018-02-14 13:23:48.929284 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891c480 for mon.2 10.255.0.32:6789/0 -1382> 2018-02-14 13:23:48.929290 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps allow * -1381> 2018-02-14 13:23:48.929293 7fc50dd00700 20 is_capable service=mon command= read on cap allow * -1380> 2018-02-14 13:23:48.929295 7fc50dd00700 20 allow so far , doing grant allow * -1379> 2018-02-14 13:23:48.929296 7fc50dd00700 20 allow all -1378> 2018-02-14 13:23:48.929297 7fc50dd00700 20 is_capable service=mon command= exec on cap allow * -1377> 2018-02-14 13:23:48.929298 7fc50dd00700 20 allow so far , doing grant allow * -1376> 2018-02-14 13:23:48.929299 7fc50dd00700 20 allow all -1375> 2018-02-14 13:23:48.929314 7fc50dd00700 1 -- 10.255.0.30:6789/0 _send_message--> mon.2 10.255.0.32:6789/0 -- paxos(lease lc 4935904 fc 4935163 pn 0 opn 0) v4 -- ?+0 0x5587d895aa00 -1374> 2018-02-14 13:23:48.929321 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.32:6789/0 -- paxos(lease lc 4935904 fc 4935163 pn 0 opn 0) v4 -- 0x5587d895aa00 con 0 -1373> 2018-02-14 13:23:48.929334 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 refresh_from_paxos -1372> 2018-02-14 13:23:48.929521 7fc50acfa700 5 -- 10.255.0.30:6789/0 >> 10.255.0.32:6789/0 conn(0x5587d8f82800 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=51044 cs=1 l=0). rx mon.2 seq 386006853 0x5587d895aa00 paxos(lease_ack lc 4935904 fc 4935163 pn 0 opn 0) v4 -1371> 2018-02-14 13:23:48.929554 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 update_from_paxos -1370> 2018-02-14 13:23:48.929560 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 update_from_paxos version 594874 summary v 594874 -1369> 2018-02-14 13:23:48.929647 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 update_from_paxos -1368> 2018-02-14 13:23:48.929702 7fc50dd00700 4 mgrc handle_mgr_map Got map version 422 -1367> 2018-02-14 13:23:48.929708 7fc50dd00700 4 mgrc handle_mgr_map Active mgr is now - -1366> 2018-02-14 13:23:48.929712 7fc50dd00700 4 mgrc reconnect No active mgr available yet -1365> 2018-02-14 13:23:48.929731 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).mgrstat 4284427 -1364> 2018-02-14 13:23:48.929820 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).mgrstat update_from_paxos v4284427 service_map e1 -1363> 2018-02-14 13:23:48.929826 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).mgrstat check_subs -1362> 2018-02-14 13:23:48.929828 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader).mgrstat update_logger -1361> 2018-02-14 13:23:48.929850 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).health update_from_paxos -1360> 2018-02-14 13:23:48.929886 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader).health dump:{ "quorum_health": {}, "leader_health": { "MON_DOWN": { "severity": "HEALTH_WARN", "summary": { "message": "1/3 mons down, quorum ceph-dal-mon-a1,ceph-dal-mon-a3" }, "detail": [ { "message": "mon.ceph-dal-mon-a2 (rank 1) addr 10.255.0.31:6789/0 is down (out of quorum)" } ] } } } -1359> 2018-02-14 13:23:48.929957 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).monmap v1 create_pending monmap epoch 2 -1358> 2018-02-14 13:23:48.929974 7fc50dd00700 0 log_channel(cluster) log [INF] : monmap e1: 3 mons at {ceph-dal-mon-a1=10.255.0.30:6789/0,ceph-dal-mon-a2=10.255.0.31:6789/0,ceph-dal-mon-a3=10.255.0.32:6789/0} -1357> 2018-02-14 13:23:48.929978 7fc50dd00700 10 log_client _send_to_monlog to self -1356> 2018-02-14 13:23:48.929979 7fc50dd00700 10 log_client log_queue is 5 last_log 5 sent 4 num 5 unsent 1 sending 1 -1355> 2018-02-14 13:23:48.929981 7fc50dd00700 10 log_client will send 2018-02-14 13:23:48.929977 mon.ceph-dal-mon-a1 mon.0 10.255.0.30:6789/0 5 : cluster [INF] monmap e1: 3 mons at {ceph-dal-mon-a1=10.255.0.30:6789/0,ceph-dal-mon-a2=10.255.0.31:6789/0,ceph-dal-mon-a3=10.255.0.32:6789/0} -1354> 2018-02-14 13:23:48.929990 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.30:6789/0 -- log(1 entries from seq 5 at 2018-02-14 13:23:48.929977) v1 -- 0x5587d8ec2880 con 0 -1353> 2018-02-14 13:23:48.930000 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).monmap v1 apply_mon_features features match current pending: mon_feature_t([kraken,luminous]) -1352> 2018-02-14 13:23:48.930017 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos active c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.930017 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -1351> 2018-02-14 13:23:48.930025 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).mds e1 create_pending e2 -1350> 2018-02-14 13:23:48.930030 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).mds e1 -1349> 2018-02-14 13:23:48.930037 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).mds e1 update_logger -1348> 2018-02-14 13:23:48.930041 7fc50dd00700 0 log_channel(cluster) log [DBG] : fsmap -1347> 2018-02-14 13:23:48.930043 7fc50dd00700 10 log_client _send_to_monlog to self -1346> 2018-02-14 13:23:48.930044 7fc50dd00700 10 log_client log_queue is 6 last_log 6 sent 5 num 6 unsent 1 sending 1 -1345> 2018-02-14 13:23:48.930046 7fc50dd00700 10 log_client will send 2018-02-14 13:23:48.930042 mon.ceph-dal-mon-a1 mon.0 10.255.0.30:6789/0 6 : cluster [DBG] fsmap -1344> 2018-02-14 13:23:48.930053 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.30:6789/0 -- log(1 entries from seq 6 at 2018-02-14 13:23:48.930042) v1 -- 0x5587d8fdc880 con 0 -1343> 2018-02-14 13:23:48.930072 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).osd e19702 create_pending e 19703 -1342> 2018-02-14 13:23:48.930124 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).osd e19702 create_pending did clean_temps -1341> 2018-02-14 13:23:48.930130 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).osd e19702 update_logger -1340> 2018-02-14 13:23:48.930137 7fc50dd00700 0 log_channel(cluster) log [DBG] : osdmap e19702: 32 total, 28 up, 32 in -1339> 2018-02-14 13:23:48.930139 7fc50dd00700 10 log_client _send_to_monlog to self -1338> 2018-02-14 13:23:48.930140 7fc50dd00700 10 log_client log_queue is 7 last_log 7 sent 6 num 7 unsent 1 sending 1 -1337> 2018-02-14 13:23:48.930142 7fc50dd00700 10 log_client will send 2018-02-14 13:23:48.930138 mon.ceph-dal-mon-a1 mon.0 10.255.0.30:6789/0 7 : cluster [DBG] osdmap e19702: 32 total, 28 up, 32 in -1336> 2018-02-14 13:23:48.930150 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.30:6789/0 -- log(1 entries from seq 7 at 2018-02-14 13:23:48.930138) v1 -- 0x5587d8fdda80 con 0 -1335> 2018-02-14 13:23:48.930159 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).osd e19702 start_mapping canceling previous mapping_job 0x5587d8cb31e0 -1334> 2018-02-14 13:23:48.930168 7fc50dd00700 20 queue 0x5587d9025720 4 [0,128) -1333> 2018-02-14 13:23:48.930172 7fc50dd00700 20 queue 0x5587d9025720 5 [0,64) -1332> 2018-02-14 13:23:48.930182 7fc50dd00700 20 queue 0x5587d9025720 6 [0,256) -1331> 2018-02-14 13:23:48.930186 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).osd e19702 start_mapping started mapping job 0x5587d9025720 at 2018-02-14 13:23:48.930163 -1330> 2018-02-14 13:23:48.930191 7fc50fd04700 20 _process 0x5587d9025720 4 [0,128) -1329> 2018-02-14 13:23:48.930202 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 create_pending v 594875 -1328> 2018-02-14 13:23:48.930202 7fc50f503700 20 _process 0x5587d9025720 5 [0,64) -1327> 2018-02-14 13:23:48.930207 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 create_pending v 2645 -1326> 2018-02-14 13:23:48.930211 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader).auth v2644 upgrade_format format 2 is current -1325> 2018-02-14 13:23:48.930212 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 AuthMonitor::on_active() -1324> 2018-02-14 13:23:48.930214 7fc50e501700 20 _process 0x5587d9025720 6 [0,256) -1323> 2018-02-14 13:23:48.930227 7fc50dd00700 0 log_channel(cluster) log [DBG] : mgrmap e422: ceph-dal-admin-a1(active, starting), standbys: ceph-dal-mon-a1 -1322> 2018-02-14 13:23:48.930231 7fc50dd00700 10 log_client _send_to_monlog to self -1321> 2018-02-14 13:23:48.930232 7fc50dd00700 10 log_client log_queue is 8 last_log 8 sent 7 num 8 unsent 1 sending 1 -1320> 2018-02-14 13:23:48.930234 7fc50dd00700 10 log_client will send 2018-02-14 13:23:48.930230 mon.ceph-dal-mon-a1 mon.0 10.255.0.30:6789/0 8 : cluster [DBG] mgrmap e422: ceph-dal-admin-a1(active, starting), standbys: ceph-dal-mon-a1 -1319> 2018-02-14 13:23:48.930241 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.30:6789/0 -- log(1 entries from seq 8 at 2018-02-14 13:23:48.930230) v1 -- 0x5587d8fdd600 con 0 -1318> 2018-02-14 13:23:48.930253 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).mgrstat 4284427 -1317> 2018-02-14 13:23:48.930273 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader).mgrstat update_logger -1316> 2018-02-14 13:23:48.930278 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).health 3 -1315> 2018-02-14 13:23:48.930282 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos active c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.930282 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -1314> 2018-02-14 13:23:48.930290 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 preprocess_query log(1 entries from seq 1 at 2018-02-14 13:23:41.217059) v1 from mon.0 10.255.0.30:6789/0 -1313> 2018-02-14 13:23:48.930300 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 preprocess_log log(1 entries from seq 1 at 2018-02-14 13:23:41.217059) v1 from mon.0 -1312> 2018-02-14 13:23:48.930305 7fc50dd00700 20 is_capable service=log command= write on cap allow * -1311> 2018-02-14 13:23:48.930307 7fc50dd00700 20 allow so far , doing grant allow * -1310> 2018-02-14 13:23:48.930308 7fc50dd00700 20 allow all -1309> 2018-02-14 13:23:48.930312 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 prepare_update log(1 entries from seq 1 at 2018-02-14 13:23:41.217059) v1 from mon.0 10.255.0.30:6789/0 -1308> 2018-02-14 13:23:48.930317 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 prepare_log log(1 entries from seq 1 at 2018-02-14 13:23:41.217059) v1 from mon.0 -1307> 2018-02-14 13:23:48.930321 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 logging 2018-02-14 13:23:41.217059 mon.ceph-dal-mon-a1 mon.0 10.255.0.30:6789/0 1 : cluster [INF] mon.ceph-dal-mon-a1 calling new monitor election -1306> 2018-02-14 13:23:48.930337 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos active c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.930338 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -1305> 2018-02-14 13:23:48.930345 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 preprocess_query log(1 entries from seq 2 at 2018-02-14 13:23:43.722898) v1 from mon.0 10.255.0.30:6789/0 -1304> 2018-02-14 13:23:48.930352 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 preprocess_log log(1 entries from seq 2 at 2018-02-14 13:23:43.722898) v1 from mon.0 -1303> 2018-02-14 13:23:48.930356 7fc50dd00700 20 is_capable service=log command= write on cap allow * -1302> 2018-02-14 13:23:48.930357 7fc50dd00700 20 allow so far , doing grant allow * -1301> 2018-02-14 13:23:48.930358 7fc50dd00700 20 allow all -1300> 2018-02-14 13:23:48.930359 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 prepare_update log(1 entries from seq 2 at 2018-02-14 13:23:43.722898) v1 from mon.0 10.255.0.30:6789/0 -1299> 2018-02-14 13:23:48.930364 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 prepare_log log(1 entries from seq 2 at 2018-02-14 13:23:43.722898) v1 from mon.0 -1298> 2018-02-14 13:23:48.930367 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 logging 2018-02-14 13:23:43.722898 mon.ceph-dal-mon-a1 mon.0 10.255.0.30:6789/0 2 : cluster [INF] mon.ceph-dal-mon-a1 calling new monitor election -1297> 2018-02-14 13:23:48.930375 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos active c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.930375 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -1296> 2018-02-14 13:23:48.930381 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 0 35 bytes epoch 1) v1 from client.1684214 10.255.0.105:0/1753894484 -1295> 2018-02-14 13:23:48.930367 7fc50acfa700 5 -- 10.255.0.30:6789/0 >> 10.255.0.32:6789/0 conn(0x5587d8f82800 :-1 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=51044 cs=1 l=0). rx mon.2 seq 386006854 0x5587d895f700 forward(log(1 entries from seq 26 at 2018-02-14 13:10:39.130901) v1 caps allow * tid 2601 con_features 2305244844532236283) v3 -1294> 2018-02-14 13:23:48.930392 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=35 -1293> 2018-02-14 13:23:48.930425 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8911f80 0x5587d8fdcac0 auth_reply(proto 2 0 (0) Success) v1 -1292> 2018-02-14 13:23:48.930439 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.105:0/1753894484 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x5587d8fdcac0 con 0 -1291> 2018-02-14 13:23:48.930460 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos active c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.930461 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -1290> 2018-02-14 13:23:48.930467 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 0 35 bytes epoch 1) v1 from client.1684262 10.255.0.106:0/3270640683 -1289> 2018-02-14 13:23:48.930473 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=35 -1288> 2018-02-14 13:23:48.930486 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d89120a0 0x5587d88db180 auth_reply(proto 2 0 (0) Success) v1 -1287> 2018-02-14 13:23:48.930491 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.106:0/3270640683 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x5587d88db180 con 0 -1286> 2018-02-14 13:23:48.930505 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos active c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.930506 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -1285> 2018-02-14 13:23:48.930512 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 0 35 bytes epoch 1) v1 from client.1684205 10.255.0.51:0/3360953369 -1284> 2018-02-14 13:23:48.930521 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=35 -1283> 2018-02-14 13:23:48.930533 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d89121c0 0x5587d88da880 auth_reply(proto 2 0 (0) Success) v1 -1282> 2018-02-14 13:23:48.930540 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.51:0/3360953369 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x5587d88da880 con 0 -1281> 2018-02-14 13:23:48.930559 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos active c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.930560 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -1280> 2018-02-14 13:23:48.930570 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 0 40 bytes epoch 1) v1 from client.1684241 10.255.0.30:0/3376128839 -1279> 2018-02-14 13:23:48.930579 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=40 -1278> 2018-02-14 13:23:48.930598 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8912520 0x5587d88db600 auth_reply(proto 2 0 (0) Success) v1 -1277> 2018-02-14 13:23:48.930620 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.30:0/3376128839 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x5587d88db600 con 0 -1276> 2018-02-14 13:23:48.930643 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos active c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.930644 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -1275> 2018-02-14 13:23:48.930655 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 0 35 bytes epoch 1) v1 from client.1684232 10.255.0.105:0/3391979714 -1274> 2018-02-14 13:23:48.930666 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=35 -1273> 2018-02-14 13:23:48.930686 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8912640 0x5587d8ec0000 auth_reply(proto 2 0 (0) Success) v1 -1272> 2018-02-14 13:23:48.930696 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.105:0/3391979714 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x5587d8ec0000 con 0 -1271> 2018-02-14 13:23:48.930713 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos active c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.930714 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -1270> 2018-02-14 13:23:48.930719 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 0 35 bytes epoch 1) v1 from client.1684235 10.255.0.53:0/287677392 -1269> 2018-02-14 13:23:48.930724 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=35 -1268> 2018-02-14 13:23:48.930736 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8912760 0x5587d88dba80 auth_reply(proto 2 0 (0) Success) v1 -1267> 2018-02-14 13:23:48.930741 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.53:0/287677392 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x5587d88dba80 con 0 -1266> 2018-02-14 13:23:48.930755 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos active c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.930756 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -1265> 2018-02-14 13:23:48.930762 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 0 35 bytes epoch 1) v1 from client.1684193 10.255.0.105:0/1988410988 -1264> 2018-02-14 13:23:48.930768 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=35 -1263> 2018-02-14 13:23:48.930800 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8912880 0x5587d8ec0480 auth_reply(proto 2 0 (0) Success) v1 -1262> 2018-02-14 13:23:48.930812 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.105:0/1988410988 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x5587d8ec0480 con 0 -1261> 2018-02-14 13:23:48.930834 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos active c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.930835 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -1260> 2018-02-14 13:23:48.930845 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 0 35 bytes epoch 1) v1 from client.1621590 10.255.0.106:0/1162398434 -1259> 2018-02-14 13:23:48.930853 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=35 -1258> 2018-02-14 13:23:48.930846 7fc511507700 5 -- 10.255.0.30:6789/0 >> 10.255.0.30:0/3376128839 conn(0x5587d8eae800 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=668 cs=1 l=1). rx client.1684241 seq 2 0x5587d88db600 auth(proto 2 32 bytes epoch 0) v1 -1257> 2018-02-14 13:23:48.930872 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8912d00 0x5587d8ec0240 auth_reply(proto 2 0 (0) Success) v1 -1256> 2018-02-14 13:23:48.930884 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.106:0/1162398434 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x5587d8ec0240 con 0 -1255> 2018-02-14 13:23:48.930909 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos active c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.930911 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -1254> 2018-02-14 13:23:48.930918 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 0 35 bytes epoch 1) v1 from client.1684238 10.255.0.54:0/2824825993 -1253> 2018-02-14 13:23:48.930925 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=35 -1252> 2018-02-14 13:23:48.930940 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8913180 0x5587d8ec0d80 auth_reply(proto 2 0 (0) Success) v1 -1251> 2018-02-14 13:23:48.930946 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.54:0/2824825993 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x5587d8ec0d80 con 0 -1250> 2018-02-14 13:23:48.930938 7fc50acfa700 5 -- 10.255.0.30:6789/0 >> 10.255.0.106:0/3270640683 conn(0x5587d8ead000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=529 cs=1 l=1). rx client.1684262 seq 2 0x5587d88dba80 auth(proto 2 32 bytes epoch 0) v1 -1249> 2018-02-14 13:23:48.930961 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos active c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.930961 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -1248> 2018-02-14 13:23:48.930968 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 0 35 bytes epoch 1) v1 from client.1684211 10.255.0.53:0/2869630763 -1247> 2018-02-14 13:23:48.930973 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=35 -1246> 2018-02-14 13:23:48.930984 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8913600 0x5587d88db3c0 auth_reply(proto 2 0 (0) Success) v1 -1245> 2018-02-14 13:23:48.930991 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.53:0/2869630763 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x5587d88db3c0 con 0 -1244> 2018-02-14 13:23:48.931006 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos active c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.931008 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -1243> 2018-02-14 13:23:48.931017 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 0 39 bytes epoch 0) from client.263543 10.255.0.107:0/322403586 -1242> 2018-02-14 13:23:48.931027 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=39 -1241> 2018-02-14 13:23:48.931045 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.107:0/322403586 -- mon_map magic: 0 v1 -- 0x5587d8ec1680 con 0 -1240> 2018-02-14 13:23:48.931065 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8913720 0x5587d8fdd3c0 auth_reply(proto 2 0 (0) Success) v1 -1239> 2018-02-14 13:23:48.931073 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.107:0/322403586 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x5587d8fdd3c0 con 0 -1238> 2018-02-14 13:23:48.931087 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos active c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.931087 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -1237> 2018-02-14 13:23:48.931094 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 0 35 bytes epoch 1) v1 from client.1684265 10.255.0.53:0/3548294640 -1236> 2018-02-14 13:23:48.931100 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=35 -1235> 2018-02-14 13:23:48.931119 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8913840 0x5587d8ec1200 auth_reply(proto 2 0 (0) Success) v1 -1234> 2018-02-14 13:23:48.931127 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.53:0/3548294640 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x5587d8ec1200 con 0 -1233> 2018-02-14 13:23:48.931141 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos active c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.931142 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -1232> 2018-02-14 13:23:48.931148 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 0 35 bytes epoch 1) v1 from client.1684217 10.255.0.107:0/2591354583 -1231> 2018-02-14 13:23:48.931154 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=35 -1230> 2018-02-14 13:23:48.931165 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8913960 0x5587d8ec1b00 auth_reply(proto 2 0 (0) Success) v1 -1229> 2018-02-14 13:23:48.931170 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.107:0/2591354583 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x5587d8ec1b00 con 0 -1228> 2018-02-14 13:23:48.931168 7fc50b4fb700 5 -- 10.255.0.30:6789/0 >> 10.255.0.105:0/1753894484 conn(0x5587d8eab800 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1131 cs=1 l=1). rx client.1684214 seq 2 0x5587d8fdcac0 auth(proto 2 32 bytes epoch 0) v1 -1227> 2018-02-14 13:23:48.931189 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos active c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.931190 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -1226> 2018-02-14 13:23:48.931200 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 0 35 bytes epoch 1) v1 from client.1684226 10.255.0.51:0/3289155469 -1225> 2018-02-14 13:23:48.931210 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=35 -1224> 2018-02-14 13:23:48.931206 7fc50b4fb700 5 -- 10.255.0.30:6789/0 >> 10.255.0.105:0/3391979714 conn(0x5587d8f94000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=626 cs=1 l=1). rx client.1684232 seq 2 0x5587d8fdd3c0 auth(proto 2 32 bytes epoch 0) v1 -1223> 2018-02-14 13:23:48.931230 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d915e000 0x5587d8ec18c0 auth_reply(proto 2 0 (0) Success) v1 -1222> 2018-02-14 13:23:48.931239 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.51:0/3289155469 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x5587d8ec18c0 con 0 -1221> 2018-02-14 13:23:48.931256 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos active c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.931257 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -1220> 2018-02-14 13:23:48.931263 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 0 35 bytes epoch 1) v1 from client.1684247 10.255.0.105:0/2076150628 -1219> 2018-02-14 13:23:48.931269 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=35 -1218> 2018-02-14 13:23:48.931281 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d915e120 0x5587d8ec2ac0 auth_reply(proto 2 0 (0) Success) v1 -1217> 2018-02-14 13:23:48.931289 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.105:0/2076150628 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x5587d8ec2ac0 con 0 -1216> 2018-02-14 13:23:48.931306 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos active c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.931307 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -1215> 2018-02-14 13:23:48.931313 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 0 35 bytes epoch 1) v1 from client.1684244 10.255.0.54:0/2718452115 -1214> 2018-02-14 13:23:48.931318 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=35 -1213> 2018-02-14 13:23:48.931314 7fc511507700 5 -- 10.255.0.30:6789/0 >> 10.255.0.105:0/1988410988 conn(0x5587d8f97000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=975 cs=1 l=1). rx client.1684193 seq 2 0x5587d8ec1b00 auth(proto 2 32 bytes epoch 0) v1 -1212> 2018-02-14 13:23:48.931330 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d915e360 0x5587d8ec1d40 auth_reply(proto 2 0 (0) Success) v1 -1211> 2018-02-14 13:23:48.931335 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.54:0/2718452115 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x5587d8ec1d40 con 0 -1210> 2018-02-14 13:23:48.931347 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos active c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.931348 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -1209> 2018-02-14 13:23:48.931340 7fc50b4fb700 5 -- 10.255.0.30:6789/0 >> 10.255.0.106:0/1162398434 conn(0x5587d8f98800 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=30292 cs=1 l=1). rx client.1621590 seq 2 0x5587d8ec18c0 auth(proto 2 32 bytes epoch 0) v1 -1208> 2018-02-14 13:23:48.931347 7fc511507700 5 -- 10.255.0.30:6789/0 >> 10.255.0.51:0/3360953369 conn(0x5587d8f2d800 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1139 cs=1 l=1). rx client.1684205 seq 2 0x5587d88db3c0 auth(proto 2 32 bytes epoch 0) v1 -1207> 2018-02-14 13:23:48.931352 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 0 35 bytes epoch 1) v1 from client.1684250 10.255.0.107:0/690639890 -1206> 2018-02-14 13:23:48.931357 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=35 -1205> 2018-02-14 13:23:48.931367 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d915e480 0x5587d8ec33c0 auth_reply(proto 2 0 (0) Success) v1 -1204> 2018-02-14 13:23:48.931374 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.107:0/690639890 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x5587d8ec33c0 con 0 -1203> 2018-02-14 13:23:48.931381 7fc50b4fb700 5 -- 10.255.0.30:6789/0 >> 10.255.0.107:0/322403586 conn(0x5587d8faf800 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=28149 cs=1 l=1). rx client.263543 seq 2 0x5587d8ec0fc0 auth(proto 2 144 bytes epoch 0) -1202> 2018-02-14 13:23:48.931455 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos active c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.931458 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -1201> 2018-02-14 13:23:48.931466 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 0 35 bytes epoch 1) v1 from client.1684190 10.255.0.108:0/3175087704 -1200> 2018-02-14 13:23:48.931473 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=35 -1199> 2018-02-14 13:23:48.931488 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d915f8c0 0x5587d8ec3600 auth_reply(proto 2 0 (0) Success) v1 -1198> 2018-02-14 13:23:48.931496 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.108:0/3175087704 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x5587d8ec3600 con 0 -1197> 2018-02-14 13:23:48.931510 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos active c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.931511 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -1196> 2018-02-14 13:23:48.931517 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 0 42 bytes epoch 1) v1 from client.1582487 10.255.0.49:0/59005741 -1195> 2018-02-14 13:23:48.931522 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=42 -1194> 2018-02-14 13:23:48.931519 7fc50acfa700 5 -- 10.255.0.30:6789/0 >> 10.255.0.53:0/287677392 conn(0x5587d8f95800 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=620 cs=1 l=1). rx client.1684235 seq 2 0x5587d8ec2ac0 auth(proto 2 32 bytes epoch 0) v1 -1193> 2018-02-14 13:23:48.931534 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d915f9e0 0x5587d8ec2f40 auth_reply(proto 2 0 (0) Success) v1 -1192> 2018-02-14 13:23:48.931539 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.49:0/59005741 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x5587d8ec2f40 con 0 -1191> 2018-02-14 13:23:48.931554 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos active c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.931554 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -1190> 2018-02-14 13:23:48.931561 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 0 35 bytes epoch 1) v1 from client.1684199 10.255.0.106:0/1777238177 -1189> 2018-02-14 13:23:48.931566 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=35 -1188> 2018-02-14 13:23:48.931577 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d915fc20 0x5587d8ec3cc0 auth_reply(proto 2 0 (0) Success) v1 -1187> 2018-02-14 13:23:48.931583 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.106:0/1777238177 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x5587d8ec3cc0 con 0 -1186> 2018-02-14 13:23:48.931597 7fc511507700 5 -- 10.255.0.30:6789/0 >> 10.255.0.53:0/2869630763 conn(0x5587d8fae000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1149 cs=1 l=1). rx client.1684211 seq 2 0x5587d8ec1d40 auth(proto 2 32 bytes epoch 0) v1 -1185> 2018-02-14 13:23:48.931610 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos active c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.931610 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -1184> 2018-02-14 13:23:48.931615 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 0 35 bytes epoch 1) v1 from client.1714098 10.255.0.108:0/850049828 -1183> 2018-02-14 13:23:48.931621 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=35 -1182> 2018-02-14 13:23:48.931632 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d91614e0 0x5587d8ec3a80 auth_reply(proto 2 0 (0) Success) v1 -1181> 2018-02-14 13:23:48.931639 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.108:0/850049828 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x5587d8ec3a80 con 0 -1180> 2018-02-14 13:23:48.931649 7fc511507700 5 -- 10.255.0.30:6789/0 >> 10.255.0.107:0/2591354583 conn(0x5587d8fb2800 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=581 cs=1 l=1). rx client.1684217 seq 2 0x5587d8ec0480 auth(proto 2 32 bytes epoch 0) v1 -1179> 2018-02-14 13:23:48.931654 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos active c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.931654 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -1178> 2018-02-14 13:23:48.931671 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 0 35 bytes epoch 1) v1 from client.1714104 10.255.0.54:0/1233019732 -1177> 2018-02-14 13:23:48.931681 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=35 -1176> 2018-02-14 13:23:48.931690 7fc50acfa700 5 -- 10.255.0.30:6789/0 >> 10.255.0.53:0/3548294640 conn(0x5587d8fb1000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=899 cs=1 l=1). rx client.1684265 seq 2 0x5587d8ec3cc0 auth(proto 2 32 bytes epoch 0) v1 -1175> 2018-02-14 13:23:48.931697 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d9161720 0x5587d88db840 auth_reply(proto 2 0 (0) Success) v1 -1174> 2018-02-14 13:23:48.931708 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.54:0/1233019732 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x5587d88db840 con 0 -1173> 2018-02-14 13:23:48.931717 7fc50acfa700 5 -- 10.255.0.30:6789/0 >> 10.255.0.105:0/2076150628 conn(0x5587d9151800 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=492 cs=1 l=1). rx client.1684247 seq 2 0x5587d8ec2f40 auth(proto 2 32 bytes epoch 0) v1 -1172> 2018-02-14 13:23:48.931729 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos active c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.931731 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -1171> 2018-02-14 13:23:48.931738 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 0 35 bytes epoch 0) v1 from client.? 10.255.0.51:0/3461880882 -1170> 2018-02-14 13:23:48.931745 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=35 -1169> 2018-02-14 13:23:48.931750 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 AuthMonitor::assign_global_id m=auth(proto 0 35 bytes epoch 0) v1 mon=0/3 last_allocated=1724096 max_global_id=1724096 -1168> 2018-02-14 13:23:48.931757 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 next_global_id should be 1724097 -1167> 2018-02-14 13:23:48.931761 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prepare_update auth(proto 0 35 bytes epoch 0) v1 from client.? 10.255.0.51:0/3461880882 -1166> 2018-02-14 13:23:48.931765 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=35 -1165> 2018-02-14 13:23:48.931768 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 AuthMonitor::assign_global_id m=auth(proto 0 35 bytes epoch 0) v1 mon=0/3 last_allocated=1724096 max_global_id=1724096 -1164> 2018-02-14 13:23:48.931771 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 next_global_id should be 1724097 -1163> 2018-02-14 13:23:48.931772 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 increasing max_global_id to 1734096 -1162> 2018-02-14 13:23:48.931774 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 increasing global id, waitlisting message -1161> 2018-02-14 13:23:48.931800 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 encode_pending v 2645 -1160> 2018-02-14 13:23:48.931789 7fc50b4fb700 5 -- 10.255.0.30:6789/0 >> 10.255.0.51:0/3289155469 conn(0x5587d9150000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=612 cs=1 l=1). rx client.1684226 seq 2 0x5587d8ec3a80 auth(proto 2 32 bytes epoch 0) v1 -1159> 2018-02-14 13:23:48.931810 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos active c 4935163..4935904) queue_pending_finisher 0x5587d88d5590 -1158> 2018-02-14 13:23:48.931819 7fc50b4fb700 5 -- 10.255.0.30:6789/0 >> 10.255.0.107:0/690639890 conn(0x5587d9154800 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=501 cs=1 l=1). rx client.1684250 seq 2 0x5587d8ec3600 auth(proto 2 32 bytes epoch 0) v1 -1157> 2018-02-14 13:23:48.931920 7fc50acfa700 5 -- 10.255.0.30:6789/0 >> 10.255.0.49:0/59005741 conn(0x5587d8fcc000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=12332 cs=1 l=1). rx client.1582487 seq 2 0x5587d88db840 auth(proto 2 32 bytes epoch 0) v1 -1156> 2018-02-14 13:23:48.931946 7fc50acfa700 5 -- 10.255.0.30:6789/0 >> 10.255.0.106:0/1777238177 conn(0x5587d8fcd800 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1161 cs=1 l=1). rx client.1684199 seq 2 0x5587d8ec1200 auth(proto 2 32 bytes epoch 0) v1 -1155> 2018-02-14 13:23:48.931962 7fc50acfa700 5 -- 10.255.0.30:6789/0 >> 10.255.0.54:0/2824825993 conn(0x5587d8dac800 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=639 cs=1 l=1). rx client.1684238 seq 2 0x5587d8ec0d80 auth(proto 2 32 bytes epoch 0) v1 -1154> 2018-02-14 13:23:48.931976 7fc50b4fb700 5 -- 10.255.0.30:6789/0 >> 10.255.0.108:0/3175087704 conn(0x5587d91be800 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1094 cs=1 l=1). rx client.1684190 seq 2 0x5587d8ec33c0 auth(proto 2 32 bytes epoch 0) v1 -1153> 2018-02-14 13:23:48.932093 7fc50b4fb700 5 -- 10.255.0.30:6789/0 >> 10.255.0.108:0/850049828 conn(0x5587d8fe3800 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=382 cs=1 l=1). rx client.1714098 seq 2 0x5587d8ec1440 auth(proto 2 32 bytes epoch 0) v1 -1152> 2018-02-14 13:23:48.932294 7fc511507700 5 -- 10.255.0.30:6789/0 >> 10.255.0.54:0/2718452115 conn(0x5587d9153000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=517 cs=1 l=1). rx client.1684244 seq 2 0x5587d88da880 auth(proto 2 32 bytes epoch 0) v1 -1151> 2018-02-14 13:23:48.932536 7fc50acfa700 5 -- 10.255.0.30:6789/0 >> 10.255.0.54:0/1233019732 conn(0x5587d8fe5000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=380 cs=1 l=1). rx client.1714104 seq 2 0x5587d88db180 auth(proto 2 32 bytes epoch 0) v1 -1150> 2018-02-14 13:23:48.935381 7fc50e501700 10 osdmap epoch 19702 mapping took 0.005217 seconds -1149> 2018-02-14 13:23:48.935391 7fc50e501700 10 mon.ceph-dal-mon-a1@0(leader).osd e19702 update_creating_pgs 0 pgs creating, 0 pools in queue -1148> 2018-02-14 13:23:48.973727 7fc50dd00700 1 -- 10.255.0.30:6789/0 _send_message--> mon.2 10.255.0.32:6789/0 -- paxos(begin lc 4935904 fc 0 pn 4700 opn 0) v4 -- ?+0 0x5587d895ad00 -1147> 2018-02-14 13:23:48.973745 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.32:6789/0 -- paxos(begin lc 4935904 fc 0 pn 4700 opn 0) v4 -- 0x5587d895ad00 con 0 -1146> 2018-02-14 13:23:48.973766 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos updating c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.973767 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -1145> 2018-02-14 13:23:48.973776 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 0 35 bytes epoch 0) v1 from client.? 10.255.0.52:0/3152189720 -1144> 2018-02-14 13:23:48.973788 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=35 -1143> 2018-02-14 13:23:48.973796 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 AuthMonitor::assign_global_id m=auth(proto 0 35 bytes epoch 0) v1 mon=0/3 last_allocated=1724096 max_global_id=1734096 -1142> 2018-02-14 13:23:48.973799 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 next_global_id should be 1724097 -1141> 2018-02-14 13:23:48.973809 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.52:0/3152189720 -- mon_map magic: 0 v1 -- 0x5587d8fda240 con 0 -1140> 2018-02-14 13:23:48.973829 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ff87e0 0x5587d8ec2640 auth_reply(proto 2 0 (0) Success) v1 -1139> 2018-02-14 13:23:48.973838 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.52:0/3152189720 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x5587d8ec2640 con 0 -1138> 2018-02-14 13:23:48.973852 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos updating c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.973853 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -1137> 2018-02-14 13:23:48.973859 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 0 35 bytes epoch 0) v1 from client.? 10.255.0.54:0/3194015072 -1136> 2018-02-14 13:23:48.973864 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=35 -1135> 2018-02-14 13:23:48.973870 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 AuthMonitor::assign_global_id m=auth(proto 0 35 bytes epoch 0) v1 mon=0/3 last_allocated=1724097 max_global_id=1734096 -1134> 2018-02-14 13:23:48.973872 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 next_global_id should be 1724100 -1133> 2018-02-14 13:23:48.973887 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.54:0/3194015072 -- mon_map magic: 0 v1 -- 0x5587d8fdad80 con 0 -1132> 2018-02-14 13:23:48.973905 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ff8b40 0x5587d8ec2d00 auth_reply(proto 2 0 (0) Success) v1 -1131> 2018-02-14 13:23:48.973911 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.54:0/3194015072 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x5587d8ec2d00 con 0 -1130> 2018-02-14 13:23:48.973925 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos updating c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.973925 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -1129> 2018-02-14 13:23:48.973932 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 0 35 bytes epoch 0) v1 from client.? 10.255.0.107:0/980595480 -1128> 2018-02-14 13:23:48.973937 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=35 -1127> 2018-02-14 13:23:48.973943 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 AuthMonitor::assign_global_id m=auth(proto 0 35 bytes epoch 0) v1 mon=0/3 last_allocated=1724100 max_global_id=1734096 -1126> 2018-02-14 13:23:48.973945 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 next_global_id should be 1724103 -1125> 2018-02-14 13:23:48.973952 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.107:0/980595480 -- mon_map magic: 0 v1 -- 0x5587d88dbcc0 con 0 -1124> 2018-02-14 13:23:48.973969 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ff8d80 0x5587d8ec1f80 auth_reply(proto 2 0 (0) Success) v1 -1123> 2018-02-14 13:23:48.973976 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.107:0/980595480 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x5587d8ec1f80 con 0 -1122> 2018-02-14 13:23:48.973989 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos updating c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.973990 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -1121> 2018-02-14 13:23:48.973996 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 0 35 bytes epoch 1) v1 from client.1684202 10.255.0.107:0/3082328679 -1120> 2018-02-14 13:23:48.974001 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=35 -1119> 2018-02-14 13:23:48.974017 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ff8ea0 0x5587d8fdafc0 auth_reply(proto 2 0 (0) Success) v1 -1118> 2018-02-14 13:23:48.974024 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.107:0/3082328679 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x5587d8fdafc0 con 0 -1117> 2018-02-14 13:23:48.974038 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos updating c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.974038 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -1116> 2018-02-14 13:23:48.974045 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 0 39 bytes epoch 0) from client.1393717 10.255.0.52:0/97505378 -1115> 2018-02-14 13:23:48.974050 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=39 -1114> 2018-02-14 13:23:48.974060 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.52:0/97505378 -- mon_map magic: 0 v1 -- 0x5587d8ec0b40 con 0 -1113> 2018-02-14 13:23:48.974074 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ff8fc0 0x5587d8ec21c0 auth_reply(proto 2 0 (0) Success) v1 -1112> 2018-02-14 13:23:48.974081 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.52:0/97505378 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x5587d8ec21c0 con 0 -1111> 2018-02-14 13:23:48.974094 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos updating c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.974094 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -1110> 2018-02-14 13:23:48.974100 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 0 35 bytes epoch 1) v1 from client.1684208 10.255.0.52:0/4056696166 -1109> 2018-02-14 13:23:48.974106 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=35 -1108> 2018-02-14 13:23:48.974117 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ff90e0 0x5587d8fda6c0 auth_reply(proto 2 0 (0) Success) v1 -1107> 2018-02-14 13:23:48.974125 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.52:0/4056696166 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x5587d8fda6c0 con 0 -1106> 2018-02-14 13:23:48.974138 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos updating c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.974138 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -1105> 2018-02-14 13:23:48.974144 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 preprocess_query log(1 entries from seq 3 at 2018-02-14 13:23:48.865038) v1 from mon.0 10.255.0.30:6789/0 -1104> 2018-02-14 13:23:48.974153 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 preprocess_log log(1 entries from seq 3 at 2018-02-14 13:23:48.865038) v1 from mon.0 -1103> 2018-02-14 13:23:48.974157 7fc50dd00700 20 is_capable service=log command= write on cap allow * -1102> 2018-02-14 13:23:48.974159 7fc50dd00700 20 allow so far , doing grant allow * -1101> 2018-02-14 13:23:48.974160 7fc50dd00700 20 allow all -1100> 2018-02-14 13:23:48.974162 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 prepare_update log(1 entries from seq 3 at 2018-02-14 13:23:48.865038) v1 from mon.0 10.255.0.30:6789/0 -1099> 2018-02-14 13:23:48.974167 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 prepare_log log(1 entries from seq 3 at 2018-02-14 13:23:48.865038) v1 from mon.0 -1098> 2018-02-14 13:23:48.974170 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 logging 2018-02-14 13:23:48.865038 mon.ceph-dal-mon-a1 mon.0 10.255.0.30:6789/0 3 : cluster [INF] mon.ceph-dal-mon-a1@0 won leader election with quorum 0,2 -1097> 2018-02-14 13:23:48.974179 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos updating c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.974180 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -1096> 2018-02-14 13:23:48.974185 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 preprocess_query log(1 entries from seq 25 at 2018-02-14 13:10:18.636291) v1 from mon.2 10.255.0.32:6789/0 -1095> 2018-02-14 13:23:48.974192 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 preprocess_log log(1 entries from seq 25 at 2018-02-14 13:10:18.636291) v1 from mon.2 -1094> 2018-02-14 13:23:48.974196 7fc50dd00700 20 is_capable service=log command= write on cap allow * -1093> 2018-02-14 13:23:48.974198 7fc50dd00700 20 allow so far , doing grant allow * -1092> 2018-02-14 13:23:48.974199 7fc50dd00700 20 allow all -1091> 2018-02-14 13:23:48.974200 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 prepare_update log(1 entries from seq 25 at 2018-02-14 13:10:18.636291) v1 from mon.2 10.255.0.32:6789/0 -1090> 2018-02-14 13:23:48.974205 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 prepare_log log(1 entries from seq 25 at 2018-02-14 13:10:18.636291) v1 from mon.2 -1089> 2018-02-14 13:23:48.974208 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 logging 2018-02-14 13:10:18.636291 mon.ceph-dal-mon-a3 mon.2 10.255.0.32:6789/0 25 : cluster [INF] mon.ceph-dal-mon-a3 calling new monitor election -1088> 2018-02-14 13:23:48.974216 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos updating c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.974217 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -1087> 2018-02-14 13:23:48.974221 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 preprocess_query log(1 entries from seq 4 at 2018-02-14 13:23:48.907083) v1 from mon.0 10.255.0.30:6789/0 -1086> 2018-02-14 13:23:48.974231 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 preprocess_log log(1 entries from seq 4 at 2018-02-14 13:23:48.907083) v1 from mon.0 -1085> 2018-02-14 13:23:48.974239 7fc50dd00700 20 is_capable service=log command= write on cap allow * -1084> 2018-02-14 13:23:48.974242 7fc50dd00700 20 allow so far , doing grant allow * -1083> 2018-02-14 13:23:48.974243 7fc50dd00700 20 allow all -1082> 2018-02-14 13:23:48.974246 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 prepare_update log(1 entries from seq 4 at 2018-02-14 13:23:48.907083) v1 from mon.0 10.255.0.30:6789/0 -1081> 2018-02-14 13:23:48.974253 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 prepare_log log(1 entries from seq 4 at 2018-02-14 13:23:48.907083) v1 from mon.0 -1080> 2018-02-14 13:23:48.974256 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 logging 2018-02-14 13:23:48.907083 mon.ceph-dal-mon-a1 mon.0 10.255.0.30:6789/0 4 : cluster [WRN] overall HEALTH_WARN noout flag(s) set; 6 backfillfull osd(s); 4 osds down; 1 host (4 osds) down; 8 nearfull osd(s); 3 pool(s) backfillfull; Degraded data redundancy: 4938712/26491706 objects degraded (18.642%), 347 pgs unclean, 347 pgs degraded, 347 pgs undersized; 1/3 mons down, quorum ceph-dal-mon-a1,ceph-dal-mon-a3 -1079> 2018-02-14 13:23:48.974278 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== mon.2 10.255.0.32:6789/0 386006852 ==== time_check( pong e 504 r 1 ts 2018-02-14 13:23:48.928277 ) v1 ==== 36+0+0 (1400908744 0 0) 0x5587d8fdb8c0 con 0x5587d8f82800 -1078> 2018-02-14 13:23:48.974290 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891c480 for mon.2 10.255.0.32:6789/0 -1077> 2018-02-14 13:23:48.974296 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps allow * -1076> 2018-02-14 13:23:48.974299 7fc50dd00700 20 is_capable service=mon command= read on cap allow * -1075> 2018-02-14 13:23:48.974301 7fc50dd00700 20 allow so far , doing grant allow * -1074> 2018-02-14 13:23:48.974302 7fc50dd00700 20 allow all -1073> 2018-02-14 13:23:48.974304 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 handle_timecheck time_check( pong e 504 r 1 ts 2018-02-14 13:23:48.928277 ) v1 -1072> 2018-02-14 13:23:48.974310 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 handle_timecheck_leader time_check( pong e 504 r 1 ts 2018-02-14 13:23:48.928277 ) v1 -1071> 2018-02-14 13:23:48.974321 7fc50dd00700 0 log_channel(cluster) log [INF] : mon.2 10.255.0.32:6789/0 -1070> 2018-02-14 13:23:48.974323 7fc50dd00700 10 log_client _send_to_monlog to self -1069> 2018-02-14 13:23:48.974331 7fc50dd00700 10 log_client log_queue is 9 last_log 9 sent 8 num 9 unsent 1 sending 1 -1068> 2018-02-14 13:23:48.974334 7fc50dd00700 10 log_client will send 2018-02-14 13:23:48.974323 mon.ceph-dal-mon-a1 mon.0 10.255.0.30:6789/0 9 : cluster [INF] mon.2 10.255.0.32:6789/0 -1067> 2018-02-14 13:23:48.974342 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.30:6789/0 -- log(1 entries from seq 9 at 2018-02-14 13:23:48.974323) v1 -- 0x5587d8ec3180 con 0 -1066> 2018-02-14 13:23:48.974351 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 handle_timecheck_leader from mon.2 10.255.0.32:6789/0 ts 2018-02-14 13:23:48.928277 delta -0.0460362 skew_bound 0 latency 0.0673085 -1065> 2018-02-14 13:23:48.974365 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 handle_timecheck_leader got pongs from everybody (2 total) -1064> 2018-02-14 13:23:48.974369 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 timecheck_finish_round curr 1 -1063> 2018-02-14 13:23:48.974371 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 timecheck_report -1062> 2018-02-14 13:23:48.974374 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 timecheck_report send report to mon.2 10.255.0.32:6789/0 -1061> 2018-02-14 13:23:48.974378 7fc50dd00700 1 -- 10.255.0.30:6789/0 _send_message--> mon.2 10.255.0.32:6789/0 -- time_check( report e 504 r 2 #skews 2 #latencies 2 ) v1 -- ?+0 0x5587d8fdb200 -1060> 2018-02-14 13:23:48.974383 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.32:6789/0 -- time_check( report e 504 r 2 #skews 2 #latencies 2 ) v1 -- 0x5587d8fdb200 con 0 -1059> 2018-02-14 13:23:48.974392 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 timecheck_check_skews -1058> 2018-02-14 13:23:48.974402 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== mon.0 10.255.0.30:6789/0 0 ==== log(1 entries from seq 5 at 2018-02-14 13:23:48.929977) v1 ==== 0+0+0 (0 0 0) 0x5587d8ec2880 con 0x5587d8d41800 -1057> 2018-02-14 13:23:48.974412 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891ca80 for mon.0 10.255.0.30:6789/0 -1056> 2018-02-14 13:23:48.974417 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps allow * -1055> 2018-02-14 13:23:48.974420 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos updating c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.974421 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -1054> 2018-02-14 13:23:48.974426 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 preprocess_query log(1 entries from seq 5 at 2018-02-14 13:23:48.929977) v1 from mon.0 10.255.0.30:6789/0 -1053> 2018-02-14 13:23:48.974432 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 preprocess_log log(1 entries from seq 5 at 2018-02-14 13:23:48.929977) v1 from mon.0 -1052> 2018-02-14 13:23:48.974436 7fc50dd00700 20 is_capable service=log command= write on cap allow * -1051> 2018-02-14 13:23:48.974438 7fc50dd00700 20 allow so far , doing grant allow * -1050> 2018-02-14 13:23:48.974438 7fc50dd00700 20 allow all -1049> 2018-02-14 13:23:48.974440 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 prepare_update log(1 entries from seq 5 at 2018-02-14 13:23:48.929977) v1 from mon.0 10.255.0.30:6789/0 -1048> 2018-02-14 13:23:48.974445 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 prepare_log log(1 entries from seq 5 at 2018-02-14 13:23:48.929977) v1 from mon.0 -1047> 2018-02-14 13:23:48.974449 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 logging 2018-02-14 13:23:48.929977 mon.ceph-dal-mon-a1 mon.0 10.255.0.30:6789/0 5 : cluster [INF] monmap e1: 3 mons at {ceph-dal-mon-a1=10.255.0.30:6789/0,ceph-dal-mon-a2=10.255.0.31:6789/0,ceph-dal-mon-a3=10.255.0.32:6789/0} -1046> 2018-02-14 13:23:48.974458 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== mon.2 10.255.0.32:6789/0 386006853 ==== paxos(lease_ack lc 4935904 fc 4935163 pn 0 opn 0) v4 ==== 118+0+0 (3435705059 0 0) 0x5587d895aa00 con 0x5587d8f82800 -1045> 2018-02-14 13:23:48.974465 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891c480 for mon.2 10.255.0.32:6789/0 -1044> 2018-02-14 13:23:48.974468 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps allow * -1043> 2018-02-14 13:23:48.974470 7fc50dd00700 20 is_capable service=mon command= read on cap allow * -1042> 2018-02-14 13:23:48.974471 7fc50dd00700 20 allow so far , doing grant allow * -1041> 2018-02-14 13:23:48.974472 7fc50dd00700 20 allow all -1040> 2018-02-14 13:23:48.974473 7fc50dd00700 20 is_capable service=mon command= exec on cap allow * -1039> 2018-02-14 13:23:48.974474 7fc50dd00700 20 allow so far , doing grant allow * -1038> 2018-02-14 13:23:48.974475 7fc50dd00700 20 allow all -1037> 2018-02-14 13:23:48.974467 7fc50b4fb700 5 -- 10.255.0.30:6789/0 >> 10.255.0.107:0/3082328679 conn(0x5587d9004000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1115 cs=1 l=1). rx client.1684202 seq 2 0x5587d8fdafc0 auth(proto 2 32 bytes epoch 0) v1 -1036> 2018-02-14 13:23:48.974482 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== mon.0 10.255.0.30:6789/0 0 ==== log(1 entries from seq 6 at 2018-02-14 13:23:48.930042) v1 ==== 0+0+0 (0 0 0) 0x5587d8fdc880 con 0x5587d8d41800 -1035> 2018-02-14 13:23:48.974497 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891ca80 for mon.0 10.255.0.30:6789/0 -1034> 2018-02-14 13:23:48.974501 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps allow * -1033> 2018-02-14 13:23:48.974503 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos updating c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.974503 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -1032> 2018-02-14 13:23:48.974508 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 preprocess_query log(1 entries from seq 6 at 2018-02-14 13:23:48.930042) v1 from mon.0 10.255.0.30:6789/0 -1031> 2018-02-14 13:23:48.974513 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 preprocess_log log(1 entries from seq 6 at 2018-02-14 13:23:48.930042) v1 from mon.0 -1030> 2018-02-14 13:23:48.974516 7fc50dd00700 20 is_capable service=log command= write on cap allow * -1029> 2018-02-14 13:23:48.974518 7fc50dd00700 20 allow so far , doing grant allow * -1028> 2018-02-14 13:23:48.974519 7fc50dd00700 20 allow all -1027> 2018-02-14 13:23:48.974509 7fc50acfa700 5 -- 10.255.0.30:6789/0 >> 10.255.0.52:0/97505378 conn(0x5587d9005800 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=6642 cs=1 l=1). rx client.1393717 seq 2 0x5587d8ec21c0 auth(proto 2 144 bytes epoch 0) -1026> 2018-02-14 13:23:48.974520 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 prepare_update log(1 entries from seq 6 at 2018-02-14 13:23:48.930042) v1 from mon.0 10.255.0.30:6789/0 -1025> 2018-02-14 13:23:48.974525 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 prepare_log log(1 entries from seq 6 at 2018-02-14 13:23:48.930042) v1 from mon.0 -1024> 2018-02-14 13:23:48.974528 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 logging 2018-02-14 13:23:48.930042 mon.ceph-dal-mon-a1 mon.0 10.255.0.30:6789/0 6 : cluster [DBG] fsmap -1023> 2018-02-14 13:23:48.974536 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== mon.2 10.255.0.32:6789/0 386006854 ==== forward(log(1 entries from seq 26 at 2018-02-14 13:10:39.130901) v1 caps allow * tid 2601 con_features 2305244844532236283) v3 ==== 355+0+0 (189745656 0 0) 0x5587d895f700 con 0x5587d8f82800 -1022> 2018-02-14 13:23:48.974546 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891c480 for mon.2 10.255.0.32:6789/0 -1021> 2018-02-14 13:23:48.974549 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps allow * -1020> 2018-02-14 13:23:48.974551 7fc50dd00700 20 is_capable service=mon command= read on cap allow * -1019> 2018-02-14 13:23:48.974553 7fc50dd00700 20 allow so far , doing grant allow * -1018> 2018-02-14 13:23:48.974554 7fc50dd00700 20 allow all -1017> 2018-02-14 13:23:48.974555 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 received forwarded message from mon.2 10.255.0.32:6789/0 via mon.2 10.255.0.32:6789/0 -1016> 2018-02-14 13:23:48.974559 7fc50dd00700 20 is_capable service=mon command= exec on cap allow * -1015> 2018-02-14 13:23:48.974560 7fc50dd00700 20 allow so far , doing grant allow * -1014> 2018-02-14 13:23:48.974561 7fc50dd00700 20 allow all -1013> 2018-02-14 13:23:48.974566 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 caps are allow * -1012> 2018-02-14 13:23:48.974567 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 entity name 'mon.' type 1 -1011> 2018-02-14 13:23:48.974569 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 mesg 0x5587d8fdab40 from 10.255.0.32:6789/0 -1010> 2018-02-14 13:23:48.974572 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d9149680 for mon.2 - -1009> 2018-02-14 13:23:48.974574 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps allow * -1008> 2018-02-14 13:23:48.974585 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos updating c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.974585 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -1007> 2018-02-14 13:23:48.974590 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 preprocess_query log(1 entries from seq 26 at 2018-02-14 13:10:39.130901) v1 from mon.2 10.255.0.32:6789/0 -1006> 2018-02-14 13:23:48.974596 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 preprocess_log log(1 entries from seq 26 at 2018-02-14 13:10:39.130901) v1 from mon.2 -1005> 2018-02-14 13:23:48.974599 7fc50dd00700 20 is_capable service=log command= write on cap allow * -1004> 2018-02-14 13:23:48.974600 7fc50dd00700 20 allow so far , doing grant allow * -1003> 2018-02-14 13:23:48.974601 7fc50dd00700 20 allow all -1002> 2018-02-14 13:23:48.974602 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 prepare_update log(1 entries from seq 26 at 2018-02-14 13:10:39.130901) v1 from mon.2 10.255.0.32:6789/0 -1001> 2018-02-14 13:23:48.974607 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 prepare_log log(1 entries from seq 26 at 2018-02-14 13:10:39.130901) v1 from mon.2 -1000> 2018-02-14 13:23:48.974610 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 logging 2018-02-14 13:10:39.130901 mon.ceph-dal-mon-a3 mon.2 10.255.0.32:6789/0 26 : cluster [INF] mon.ceph-dal-mon-a3 calling new monitor election -999> 2018-02-14 13:23:48.974625 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== mon.0 10.255.0.30:6789/0 0 ==== log(1 entries from seq 7 at 2018-02-14 13:23:48.930138) v1 ==== 0+0+0 (0 0 0) 0x5587d8fdda80 con 0x5587d8d41800 -998> 2018-02-14 13:23:48.974637 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891ca80 for mon.0 10.255.0.30:6789/0 -997> 2018-02-14 13:23:48.974655 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps allow * -996> 2018-02-14 13:23:48.974662 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos updating c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.974663 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -995> 2018-02-14 13:23:48.974668 7fc511507700 5 -- 10.255.0.30:6789/0 >> 10.255.0.52:0/4056696166 conn(0x5587d9007000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1121 cs=1 l=1). rx client.1684208 seq 2 0x5587d8fdb680 auth(proto 2 32 bytes epoch 0) v1 -994> 2018-02-14 13:23:48.974674 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 preprocess_query log(1 entries from seq 7 at 2018-02-14 13:23:48.930138) v1 from mon.0 10.255.0.30:6789/0 -993> 2018-02-14 13:23:48.974691 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 preprocess_log log(1 entries from seq 7 at 2018-02-14 13:23:48.930138) v1 from mon.0 -992> 2018-02-14 13:23:48.974700 7fc50dd00700 20 is_capable service=log command= write on cap allow * -991> 2018-02-14 13:23:48.974703 7fc50dd00700 20 allow so far , doing grant allow * -990> 2018-02-14 13:23:48.974704 7fc50dd00700 20 allow all -989> 2018-02-14 13:23:48.974707 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 prepare_update log(1 entries from seq 7 at 2018-02-14 13:23:48.930138) v1 from mon.0 10.255.0.30:6789/0 -988> 2018-02-14 13:23:48.974714 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 prepare_log log(1 entries from seq 7 at 2018-02-14 13:23:48.930138) v1 from mon.0 -987> 2018-02-14 13:23:48.974717 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 logging 2018-02-14 13:23:48.930138 mon.ceph-dal-mon-a1 mon.0 10.255.0.30:6789/0 7 : cluster [DBG] osdmap e19702: 32 total, 28 up, 32 in -986> 2018-02-14 13:23:48.974726 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== mon.0 10.255.0.30:6789/0 0 ==== log(1 entries from seq 8 at 2018-02-14 13:23:48.930230) v1 ==== 0+0+0 (0 0 0) 0x5587d8fdd600 con 0x5587d8d41800 -985> 2018-02-14 13:23:48.974735 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891ca80 for mon.0 10.255.0.30:6789/0 -984> 2018-02-14 13:23:48.974740 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps allow * -983> 2018-02-14 13:23:48.974742 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos updating c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.974743 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -982> 2018-02-14 13:23:48.974748 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 preprocess_query log(1 entries from seq 8 at 2018-02-14 13:23:48.930230) v1 from mon.0 10.255.0.30:6789/0 -981> 2018-02-14 13:23:48.974754 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 preprocess_log log(1 entries from seq 8 at 2018-02-14 13:23:48.930230) v1 from mon.0 -980> 2018-02-14 13:23:48.974758 7fc50dd00700 20 is_capable service=log command= write on cap allow * -979> 2018-02-14 13:23:48.974759 7fc50dd00700 20 allow so far , doing grant allow * -978> 2018-02-14 13:23:48.974760 7fc50dd00700 20 allow all -977> 2018-02-14 13:23:48.974762 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 prepare_update log(1 entries from seq 8 at 2018-02-14 13:23:48.930230) v1 from mon.0 10.255.0.30:6789/0 -976> 2018-02-14 13:23:48.974767 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 prepare_log log(1 entries from seq 8 at 2018-02-14 13:23:48.930230) v1 from mon.0 -975> 2018-02-14 13:23:48.974770 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 logging 2018-02-14 13:23:48.930230 mon.ceph-dal-mon-a1 mon.0 10.255.0.30:6789/0 8 : cluster [DBG] mgrmap e422: ceph-dal-admin-a1(active, starting), standbys: ceph-dal-mon-a1 -974> 2018-02-14 13:23:48.974788 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== mon.0 10.255.0.30:6789/0 0 ==== log(1 entries from seq 9 at 2018-02-14 13:23:48.974323) v1 ==== 0+0+0 (0 0 0) 0x5587d8ec3180 con 0x5587d8d41800 -973> 2018-02-14 13:23:48.974801 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891ca80 for mon.0 10.255.0.30:6789/0 -972> 2018-02-14 13:23:48.974807 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps allow * -971> 2018-02-14 13:23:48.974809 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos updating c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.974810 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -970> 2018-02-14 13:23:48.974822 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 preprocess_query log(1 entries from seq 9 at 2018-02-14 13:23:48.974323) v1 from mon.0 10.255.0.30:6789/0 -969> 2018-02-14 13:23:48.974816 7fc511507700 5 -- 10.255.0.30:6789/0 >> 10.255.0.52:0/3152189720 conn(0x5587d8da9800 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=455 cs=1 l=1). rx client.? seq 2 0x5587d8fdb440 auth(proto 2 32 bytes epoch 0) v1 -968> 2018-02-14 13:23:48.974830 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 preprocess_log log(1 entries from seq 9 at 2018-02-14 13:23:48.974323) v1 from mon.0 -967> 2018-02-14 13:23:48.974834 7fc50dd00700 20 is_capable service=log command= write on cap allow * -966> 2018-02-14 13:23:48.974836 7fc50dd00700 20 allow so far , doing grant allow * -965> 2018-02-14 13:23:48.974836 7fc50dd00700 20 allow all -964> 2018-02-14 13:23:48.974838 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 prepare_update log(1 entries from seq 9 at 2018-02-14 13:23:48.974323) v1 from mon.0 10.255.0.30:6789/0 -963> 2018-02-14 13:23:48.974843 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 prepare_log log(1 entries from seq 9 at 2018-02-14 13:23:48.974323) v1 from mon.0 -962> 2018-02-14 13:23:48.974846 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 logging 2018-02-14 13:23:48.974323 mon.ceph-dal-mon-a1 mon.0 10.255.0.30:6789/0 9 : cluster [INF] mon.2 10.255.0.32:6789/0 -961> 2018-02-14 13:23:48.974849 7fc511507700 5 -- 10.255.0.30:6789/0 >> 10.255.0.107:0/980595480 conn(0x5587d8d40000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=442 cs=1 l=1). rx client.? seq 2 0x5587d8fda240 auth(proto 2 32 bytes epoch 0) v1 -960> 2018-02-14 13:23:48.974855 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1684241 10.255.0.30:0/3376128839 2 ==== auth(proto 2 32 bytes epoch 0) v1 ==== 62+0+0 (4212562530 0 0) 0x5587d88db600 con 0x5587d8eae800 -959> 2018-02-14 13:23:48.974862 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891d980 for client.1684241 10.255.0.30:0/3376128839 -958> 2018-02-14 13:23:48.974867 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps -957> 2018-02-14 13:23:48.974869 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos updating c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.974869 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -956> 2018-02-14 13:23:48.974873 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 32 bytes epoch 0) v1 from client.1684241 10.255.0.30:0/3376128839 -955> 2018-02-14 13:23:48.974877 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=32 -954> 2018-02-14 13:23:48.975015 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d90757a0 0x5587d8fdcf40 auth_reply(proto 2 0 (0) Success) v1 -953> 2018-02-14 13:23:48.975027 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.30:0/3376128839 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x5587d8fdcf40 con 0 -952> 2018-02-14 13:23:48.975047 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1684232 10.255.0.105:0/3391979714 2 ==== auth(proto 2 32 bytes epoch 0) v1 ==== 62+0+0 (47965627 0 0) 0x5587d8fdd3c0 con 0x5587d8f94000 -951> 2018-02-14 13:23:48.975058 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891dc80 for client.1684232 10.255.0.105:0/3391979714 -950> 2018-02-14 13:23:48.975063 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps -949> 2018-02-14 13:23:48.975065 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos updating c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.975066 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -948> 2018-02-14 13:23:48.975071 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 32 bytes epoch 0) v1 from client.1684232 10.255.0.105:0/3391979714 -947> 2018-02-14 13:23:48.975074 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=32 -946> 2018-02-14 13:23:48.975150 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d90757a0 0x5587d88db600 auth_reply(proto 2 0 (0) Success) v1 -945> 2018-02-14 13:23:48.975160 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.105:0/3391979714 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x5587d88db600 con 0 -944> 2018-02-14 13:23:48.975179 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1684235 10.255.0.53:0/287677392 2 ==== auth(proto 2 32 bytes epoch 0) v1 ==== 62+0+0 (1916555513 0 0) 0x5587d8ec2ac0 con 0x5587d8f95800 -943> 2018-02-14 13:23:48.975189 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891df80 for client.1684235 10.255.0.53:0/287677392 -942> 2018-02-14 13:23:48.975195 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps -941> 2018-02-14 13:23:48.975197 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos updating c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.975198 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -940> 2018-02-14 13:23:48.975203 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 32 bytes epoch 0) v1 from client.1684235 10.255.0.53:0/287677392 -939> 2018-02-14 13:23:48.975206 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=32 -938> 2018-02-14 13:23:48.975275 7fc511507700 5 -- 10.255.0.30:6789/0 >> 10.255.0.30:0/3376128839 conn(0x5587d8eae800 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=668 cs=1 l=1). rx client.1684241 seq 3 0x5587d8fdcf40 auth(proto 2 181 bytes epoch 0) v1 -937> 2018-02-14 13:23:48.975278 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d90757a0 0x5587d8fdd3c0 auth_reply(proto 2 0 (0) Success) v1 -936> 2018-02-14 13:23:48.975296 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.53:0/287677392 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x5587d8fdd3c0 con 0 -935> 2018-02-14 13:23:48.975315 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1684193 10.255.0.105:0/1988410988 2 ==== auth(proto 2 32 bytes epoch 0) v1 ==== 62+0+0 (2229067381 0 0) 0x5587d8ec1b00 con 0x5587d8f97000 -934> 2018-02-14 13:23:48.975326 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891e280 for client.1684193 10.255.0.105:0/1988410988 -933> 2018-02-14 13:23:48.975331 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps -932> 2018-02-14 13:23:48.975334 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos updating c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.975334 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -931> 2018-02-14 13:23:48.975339 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 32 bytes epoch 0) v1 from client.1684193 10.255.0.105:0/1988410988 -930> 2018-02-14 13:23:48.975343 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=32 -929> 2018-02-14 13:23:48.975346 7fc50b4fb700 5 -- 10.255.0.30:6789/0 >> 10.255.0.54:0/3194015072 conn(0x5587d8f2c000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=432 cs=1 l=1). rx client.? seq 2 0x5587d88db600 auth(proto 2 32 bytes epoch 0) v1 -928> 2018-02-14 13:23:48.975414 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d90757a0 0x5587d8ec2ac0 auth_reply(proto 2 0 (0) Success) v1 -927> 2018-02-14 13:23:48.975424 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.105:0/1988410988 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x5587d8ec2ac0 con 0 -926> 2018-02-14 13:23:48.975443 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1621590 10.255.0.106:0/1162398434 2 ==== auth(proto 2 32 bytes epoch 0) v1 ==== 62+0+0 (1329137703 0 0) 0x5587d8ec18c0 con 0x5587d8f98800 -925> 2018-02-14 13:23:48.975453 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891e580 for client.1621590 10.255.0.106:0/1162398434 -924> 2018-02-14 13:23:48.975458 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps -923> 2018-02-14 13:23:48.975463 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos updating c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.975463 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -922> 2018-02-14 13:23:48.975468 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 32 bytes epoch 0) v1 from client.1621590 10.255.0.106:0/1162398434 -921> 2018-02-14 13:23:48.975472 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=32 -920> 2018-02-14 13:23:48.975540 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d90757a0 0x5587d8ec1b00 auth_reply(proto 2 0 (0) Success) v1 -919> 2018-02-14 13:23:48.975550 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.106:0/1162398434 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x5587d8ec1b00 con 0 -918> 2018-02-14 13:23:48.975568 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1684238 10.255.0.54:0/2824825993 2 ==== auth(proto 2 32 bytes epoch 0) v1 ==== 62+0+0 (4220998679 0 0) 0x5587d8ec0d80 con 0x5587d8dac800 -917> 2018-02-14 13:23:48.975578 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891f180 for client.1684238 10.255.0.54:0/2824825993 -916> 2018-02-14 13:23:48.975583 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps -915> 2018-02-14 13:23:48.975586 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos updating c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.975587 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -914> 2018-02-14 13:23:48.975592 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 32 bytes epoch 0) v1 from client.1684238 10.255.0.54:0/2824825993 -913> 2018-02-14 13:23:48.975597 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=32 -912> 2018-02-14 13:23:48.975647 7fc50b4fb700 5 -- 10.255.0.30:6789/0 >> 10.255.0.105:0/3391979714 conn(0x5587d8f94000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=626 cs=1 l=1). rx client.1684232 seq 3 0x5587d8ec1b00 auth(proto 2 181 bytes epoch 0) v1 -911> 2018-02-14 13:23:48.975665 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d90757a0 0x5587d8ec18c0 auth_reply(proto 2 0 (0) Success) v1 -910> 2018-02-14 13:23:48.975675 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.54:0/2824825993 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x5587d8ec18c0 con 0 -909> 2018-02-14 13:23:48.975695 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1684211 10.255.0.53:0/2869630763 2 ==== auth(proto 2 32 bytes epoch 0) v1 ==== 62+0+0 (1807881714 0 0) 0x5587d8ec1d40 con 0x5587d8fae000 -908> 2018-02-14 13:23:48.975704 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891f480 for client.1684211 10.255.0.53:0/2869630763 -907> 2018-02-14 13:23:48.975710 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps -906> 2018-02-14 13:23:48.975713 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos updating c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.975713 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -905> 2018-02-14 13:23:48.975718 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 32 bytes epoch 0) v1 from client.1684211 10.255.0.53:0/2869630763 -904> 2018-02-14 13:23:48.975723 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=32 -903> 2018-02-14 13:23:48.975800 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d90757a0 0x5587d8ec0d80 auth_reply(proto 2 0 (0) Success) v1 -902> 2018-02-14 13:23:48.975810 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.53:0/2869630763 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x5587d8ec0d80 con 0 -901> 2018-02-14 13:23:48.975830 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.263543 10.255.0.107:0/322403586 2 ==== auth(proto 2 144 bytes epoch 0) ==== 170+0+0 (2647591564 0 0) 0x5587d8ec0fc0 con 0x5587d8faf800 -900> 2018-02-14 13:23:48.975841 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891f780 for client.263543 10.255.0.107:0/322403586 -899> 2018-02-14 13:23:48.975846 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps -898> 2018-02-14 13:23:48.975848 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos updating c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.975849 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -897> 2018-02-14 13:23:48.975854 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 144 bytes epoch 0) from client.263543 10.255.0.107:0/322403586 -896> 2018-02-14 13:23:48.975857 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=144 -895> 2018-02-14 13:23:48.975875 7fc50acfa700 5 -- 10.255.0.30:6789/0 >> 10.255.0.53:0/287677392 conn(0x5587d8f95800 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=620 cs=1 l=1). rx client.1684235 seq 3 0x5587d8ec18c0 auth(proto 2 181 bytes epoch 0) v1 -894> 2018-02-14 13:23:48.975884 7fc511507700 5 -- 10.255.0.30:6789/0 >> 10.255.0.105:0/1988410988 conn(0x5587d8f97000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=975 cs=1 l=1). rx client.1684193 seq 3 0x5587d8ec0d80 auth(proto 2 181 bytes epoch 0) v1 -893> 2018-02-14 13:23:48.975953 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d90757a0 0x5587d8ec1d40 auth_reply(proto 2 0 (0) Success) v1 -892> 2018-02-14 13:23:48.975963 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.107:0/322403586 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x5587d8ec1d40 con 0 -891> 2018-02-14 13:23:48.975982 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1684265 10.255.0.53:0/3548294640 2 ==== auth(proto 2 32 bytes epoch 0) v1 ==== 62+0+0 (3599928056 0 0) 0x5587d8ec3cc0 con 0x5587d8fb1000 -890> 2018-02-14 13:23:48.975992 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891fa80 for client.1684265 10.255.0.53:0/3548294640 -889> 2018-02-14 13:23:48.975998 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps -888> 2018-02-14 13:23:48.976001 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos updating c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.976001 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -887> 2018-02-14 13:23:48.975990 7fc50b4fb700 5 -- 10.255.0.30:6789/0 >> 10.255.0.106:0/1162398434 conn(0x5587d8f98800 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=30292 cs=1 l=1). rx client.1621590 seq 3 0x5587d8ec2d00 auth(proto 2 181 bytes epoch 0) v1 -886> 2018-02-14 13:23:48.976006 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 32 bytes epoch 0) v1 from client.1684265 10.255.0.53:0/3548294640 -885> 2018-02-14 13:23:48.976010 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=32 -884> 2018-02-14 13:23:48.976093 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ff9680 0x5587d8ec0fc0 auth_reply(proto 2 0 (0) Success) v1 -883> 2018-02-14 13:23:48.976103 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.53:0/3548294640 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x5587d8ec0fc0 con 0 -882> 2018-02-14 13:23:48.976125 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1684217 10.255.0.107:0/2591354583 2 ==== auth(proto 2 32 bytes epoch 0) v1 ==== 62+0+0 (2086724689 0 0) 0x5587d8ec0480 con 0x5587d8fb2800 -881> 2018-02-14 13:23:48.976136 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891fd80 for client.1684217 10.255.0.107:0/2591354583 -880> 2018-02-14 13:23:48.976141 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps -879> 2018-02-14 13:23:48.976144 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos updating c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.976145 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -878> 2018-02-14 13:23:48.976150 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 32 bytes epoch 0) v1 from client.1684217 10.255.0.107:0/2591354583 -877> 2018-02-14 13:23:48.976155 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=32 -876> 2018-02-14 13:23:48.976223 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ff9680 0x5587d8ec3cc0 auth_reply(proto 2 0 (0) Success) v1 -875> 2018-02-14 13:23:48.976233 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.107:0/2591354583 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x5587d8ec3cc0 con 0 -874> 2018-02-14 13:23:48.976238 7fc50b4fb700 5 -- 10.255.0.30:6789/0 >> 10.255.0.107:0/322403586 conn(0x5587d8faf800 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=28149 cs=1 l=1). rx client.263543 seq 3 0x5587d8e9cc00 mon_subscribe({monmap=2+,osdmap=19703}) v2 -873> 2018-02-14 13:23:48.976254 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1684226 10.255.0.51:0/3289155469 2 ==== auth(proto 2 32 bytes epoch 0) v1 ==== 62+0+0 (4078516145 0 0) 0x5587d8ec3a80 con 0x5587d9150000 -872> 2018-02-14 13:23:48.976273 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d9148300 for client.1684226 10.255.0.51:0/3289155469 -871> 2018-02-14 13:23:48.976278 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps -870> 2018-02-14 13:23:48.976281 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos updating c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.976281 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -869> 2018-02-14 13:23:48.976286 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 32 bytes epoch 0) v1 from client.1684226 10.255.0.51:0/3289155469 -868> 2018-02-14 13:23:48.976290 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=32 -867> 2018-02-14 13:23:48.976305 7fc511507700 5 -- 10.255.0.30:6789/0 >> 10.255.0.53:0/2869630763 conn(0x5587d8fae000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1149 cs=1 l=1). rx client.1684211 seq 3 0x5587d8ec3cc0 auth(proto 2 181 bytes epoch 0) v1 -866> 2018-02-14 13:23:48.976358 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ff9680 0x5587d8ec0480 auth_reply(proto 2 0 (0) Success) v1 -865> 2018-02-14 13:23:48.976368 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.51:0/3289155469 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x5587d8ec0480 con 0 -864> 2018-02-14 13:23:48.976388 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1684247 10.255.0.105:0/2076150628 2 ==== auth(proto 2 32 bytes epoch 0) v1 ==== 62+0+0 (308577622 0 0) 0x5587d8ec2f40 con 0x5587d9151800 -863> 2018-02-14 13:23:48.976398 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d9148480 for client.1684247 10.255.0.105:0/2076150628 -862> 2018-02-14 13:23:48.976403 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps -861> 2018-02-14 13:23:48.976406 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos updating c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.976407 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -860> 2018-02-14 13:23:48.976412 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 32 bytes epoch 0) v1 from client.1684247 10.255.0.105:0/2076150628 -859> 2018-02-14 13:23:48.976417 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=32 -858> 2018-02-14 13:23:48.976459 7fc50acfa700 5 -- 10.255.0.30:6789/0 >> 10.255.0.54:0/2824825993 conn(0x5587d8dac800 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=639 cs=1 l=1). rx client.1684238 seq 3 0x5587d8ec0fc0 auth(proto 2 181 bytes epoch 0) v1 -857> 2018-02-14 13:23:48.976500 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ff9680 0x5587d8ec3a80 auth_reply(proto 2 0 (0) Success) v1 -856> 2018-02-14 13:23:48.976510 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.105:0/2076150628 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x5587d8ec3a80 con 0 -855> 2018-02-14 13:23:48.976530 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1684244 10.255.0.54:0/2718452115 2 ==== auth(proto 2 32 bytes epoch 0) v1 ==== 62+0+0 (697284517 0 0) 0x5587d88da880 con 0x5587d9153000 -854> 2018-02-14 13:23:48.976525 7fc50acfa700 5 -- 10.255.0.30:6789/0 >> 10.255.0.53:0/3548294640 conn(0x5587d8fb1000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=899 cs=1 l=1). rx client.1684265 seq 3 0x5587d8fdd3c0 auth(proto 2 181 bytes epoch 0) v1 -853> 2018-02-14 13:23:48.976542 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d9148600 for client.1684244 10.255.0.54:0/2718452115 -852> 2018-02-14 13:23:48.976547 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps -851> 2018-02-14 13:23:48.976549 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos updating c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.976549 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -850> 2018-02-14 13:23:48.976554 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 32 bytes epoch 0) v1 from client.1684244 10.255.0.54:0/2718452115 -849> 2018-02-14 13:23:48.976559 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=32 -848> 2018-02-14 13:23:48.976628 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ff9680 0x5587d8ec2f40 auth_reply(proto 2 0 (0) Success) v1 -847> 2018-02-14 13:23:48.976638 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.54:0/2718452115 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x5587d8ec2f40 con 0 -846> 2018-02-14 13:23:48.976658 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1684250 10.255.0.107:0/690639890 2 ==== auth(proto 2 32 bytes epoch 0) v1 ==== 62+0+0 (614326204 0 0) 0x5587d8ec3600 con 0x5587d9154800 -845> 2018-02-14 13:23:48.976668 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d9148780 for client.1684250 10.255.0.107:0/690639890 -844> 2018-02-14 13:23:48.976674 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps -843> 2018-02-14 13:23:48.976676 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos updating c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.976676 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -842> 2018-02-14 13:23:48.976681 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 32 bytes epoch 0) v1 from client.1684250 10.255.0.107:0/690639890 -841> 2018-02-14 13:23:48.976685 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=32 -840> 2018-02-14 13:23:48.976689 7fc511507700 5 -- 10.255.0.30:6789/0 >> 10.255.0.107:0/2591354583 conn(0x5587d8fb2800 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=581 cs=1 l=1). rx client.1684217 seq 3 0x5587d8ec2f40 auth(proto 2 181 bytes epoch 0) v1 -839> 2018-02-14 13:23:48.976761 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ff9680 0x5587d88da880 auth_reply(proto 2 0 (0) Success) v1 -838> 2018-02-14 13:23:48.976772 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.107:0/690639890 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x5587d88da880 con 0 -837> 2018-02-14 13:23:48.976798 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1684190 10.255.0.108:0/3175087704 2 ==== auth(proto 2 32 bytes epoch 0) v1 ==== 62+0+0 (139109796 0 0) 0x5587d8ec33c0 con 0x5587d91be800 -836> 2018-02-14 13:23:48.976809 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d9148f00 for client.1684190 10.255.0.108:0/3175087704 -835> 2018-02-14 13:23:48.976816 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps -834> 2018-02-14 13:23:48.976819 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos updating c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.976820 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -833> 2018-02-14 13:23:48.976828 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 32 bytes epoch 0) v1 from client.1684190 10.255.0.108:0/3175087704 -832> 2018-02-14 13:23:48.976833 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=32 -831> 2018-02-14 13:23:48.976890 7fc50b4fb700 5 -- 10.255.0.30:6789/0 >> 10.255.0.51:0/3289155469 conn(0x5587d9150000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=612 cs=1 l=1). rx client.1684226 seq 3 0x5587d88da880 auth(proto 2 181 bytes epoch 0) v1 -830> 2018-02-14 13:23:48.976908 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ff9680 0x5587d8ec3600 auth_reply(proto 2 0 (0) Success) v1 -829> 2018-02-14 13:23:48.976918 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.108:0/3175087704 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x5587d8ec3600 con 0 -828> 2018-02-14 13:23:48.976941 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1582487 10.255.0.49:0/59005741 2 ==== auth(proto 2 32 bytes epoch 0) v1 ==== 62+0+0 (2325247192 0 0) 0x5587d88db840 con 0x5587d8fcc000 -827> 2018-02-14 13:23:48.976951 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d9149080 for client.1582487 10.255.0.49:0/59005741 -826> 2018-02-14 13:23:48.976956 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps -825> 2018-02-14 13:23:48.976961 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos updating c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.976961 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -824> 2018-02-14 13:23:48.976966 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 32 bytes epoch 0) v1 from client.1582487 10.255.0.49:0/59005741 -823> 2018-02-14 13:23:48.976971 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=32 -822> 2018-02-14 13:23:48.977037 7fc50acfa700 5 -- 10.255.0.30:6789/0 >> 10.255.0.105:0/2076150628 conn(0x5587d9151800 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=492 cs=1 l=1). rx client.1684247 seq 3 0x5587d8ec3a80 auth(proto 2 181 bytes epoch 0) v1 -821> 2018-02-14 13:23:48.977089 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ff9680 0x5587d8ec33c0 auth_reply(proto 2 0 (0) Success) v1 -820> 2018-02-14 13:23:48.977100 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.49:0/59005741 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x5587d8ec33c0 con 0 -819> 2018-02-14 13:23:48.977124 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1684199 10.255.0.106:0/1777238177 2 ==== auth(proto 2 32 bytes epoch 0) v1 ==== 62+0+0 (3834194588 0 0) 0x5587d8ec1200 con 0x5587d8fcd800 -818> 2018-02-14 13:23:48.977142 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d9149200 for client.1684199 10.255.0.106:0/1777238177 -817> 2018-02-14 13:23:48.977152 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps -816> 2018-02-14 13:23:48.977156 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos updating c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.977156 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -815> 2018-02-14 13:23:48.977165 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 32 bytes epoch 0) v1 from client.1684199 10.255.0.106:0/1777238177 -814> 2018-02-14 13:23:48.977171 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=32 -813> 2018-02-14 13:23:48.977229 7fc50b4fb700 5 -- 10.255.0.30:6789/0 >> 10.255.0.107:0/690639890 conn(0x5587d9154800 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=501 cs=1 l=1). rx client.1684250 seq 3 0x5587d8ec3600 auth(proto 2 181 bytes epoch 0) v1 -812> 2018-02-14 13:23:48.977251 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ff9680 0x5587d88db840 auth_reply(proto 2 0 (0) Success) v1 -811> 2018-02-14 13:23:48.977262 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.106:0/1777238177 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x5587d88db840 con 0 -810> 2018-02-14 13:23:48.977288 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1714098 10.255.0.108:0/850049828 2 ==== auth(proto 2 32 bytes epoch 0) v1 ==== 62+0+0 (894853614 0 0) 0x5587d8ec1440 con 0x5587d8fe3800 -809> 2018-02-14 13:23:48.977306 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d9149800 for client.1714098 10.255.0.108:0/850049828 -808> 2018-02-14 13:23:48.977316 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps -807> 2018-02-14 13:23:48.977320 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos updating c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.977321 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -806> 2018-02-14 13:23:48.977326 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 32 bytes epoch 0) v1 from client.1714098 10.255.0.108:0/850049828 -805> 2018-02-14 13:23:48.977330 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=32 -804> 2018-02-14 13:23:48.977353 7fc50b4fb700 5 -- 10.255.0.30:6789/0 >> 10.255.0.108:0/3175087704 conn(0x5587d91be800 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1094 cs=1 l=1). rx client.1684190 seq 3 0x5587d8ec0480 auth(proto 2 181 bytes epoch 0) v1 -803> 2018-02-14 13:23:48.977387 7fc511507700 5 -- 10.255.0.30:6789/0 >> 10.255.0.54:0/2718452115 conn(0x5587d9153000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=517 cs=1 l=1). rx client.1684244 seq 3 0x5587d8ec2ac0 auth(proto 2 181 bytes epoch 0) v1 -802> 2018-02-14 13:23:48.977412 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ff9680 0x5587d8ec1200 auth_reply(proto 2 0 (0) Success) v1 -801> 2018-02-14 13:23:48.977422 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.108:0/850049828 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x5587d8ec1200 con 0 -800> 2018-02-14 13:23:48.977445 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1714104 10.255.0.54:0/1233019732 2 ==== auth(proto 2 32 bytes epoch 0) v1 ==== 62+0+0 (3197282834 0 0) 0x5587d88db180 con 0x5587d8fe5000 -799> 2018-02-14 13:23:48.977464 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d9149980 for client.1714104 10.255.0.54:0/1233019732 -798> 2018-02-14 13:23:48.977473 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps -797> 2018-02-14 13:23:48.977477 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos updating c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.977478 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -796> 2018-02-14 13:23:48.977477 7fc50acfa700 5 -- 10.255.0.30:6789/0 >> 10.255.0.49:0/59005741 conn(0x5587d8fcc000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=12332 cs=1 l=1). rx client.1582487 seq 3 0x5587d88db840 auth(proto 2 181 bytes epoch 0) v1 -795> 2018-02-14 13:23:48.977486 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 32 bytes epoch 0) v1 from client.1714104 10.255.0.54:0/1233019732 -794> 2018-02-14 13:23:48.977492 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=32 -793> 2018-02-14 13:23:48.977570 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ff9680 0x5587d8ec1440 auth_reply(proto 2 0 (0) Success) v1 -792> 2018-02-14 13:23:48.977580 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.54:0/1233019732 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x5587d8ec1440 con 0 -791> 2018-02-14 13:23:48.977606 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.? 10.255.0.52:0/3152189720 2 ==== auth(proto 2 32 bytes epoch 0) v1 ==== 62+0+0 (1476567901 0 0) 0x5587d8fdb440 con 0x5587d8da9800 -790> 2018-02-14 13:23:48.977624 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d9149c80 for client.? 10.255.0.52:0/3152189720 -789> 2018-02-14 13:23:48.977633 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps -788> 2018-02-14 13:23:48.977637 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos updating c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.977638 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -787> 2018-02-14 13:23:48.977648 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 32 bytes epoch 0) v1 from client.? 10.255.0.52:0/3152189720 -786> 2018-02-14 13:23:48.977654 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=32 -785> 2018-02-14 13:23:48.977674 7fc50acfa700 5 -- 10.255.0.30:6789/0 >> 10.255.0.106:0/1777238177 conn(0x5587d8fcd800 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1161 cs=1 l=1). rx client.1684199 seq 3 0x5587d8ec1440 auth(proto 2 181 bytes epoch 0) v1 -784> 2018-02-14 13:23:48.977735 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ff9680 0x5587d88db180 auth_reply(proto 2 0 (0) Success) v1 -783> 2018-02-14 13:23:48.977745 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.52:0/3152189720 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x5587d88db180 con 0 -782> 2018-02-14 13:23:48.977768 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.? 10.255.0.54:0/3194015072 2 ==== auth(proto 2 32 bytes epoch 0) v1 ==== 62+0+0 (1473540303 0 0) 0x5587d88db600 con 0x5587d8f2c000 -781> 2018-02-14 13:23:48.977784 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d9149e00 for client.? 10.255.0.54:0/3194015072 -780> 2018-02-14 13:23:48.977793 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps -779> 2018-02-14 13:23:48.977798 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos updating c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.977799 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -778> 2018-02-14 13:23:48.977797 7fc50b4fb700 5 -- 10.255.0.30:6789/0 >> 10.255.0.108:0/850049828 conn(0x5587d8fe3800 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=382 cs=1 l=1). rx client.1714098 seq 3 0x5587d8ec1200 auth(proto 2 181 bytes epoch 0) v1 -777> 2018-02-14 13:23:48.977805 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 32 bytes epoch 0) v1 from client.? 10.255.0.54:0/3194015072 -776> 2018-02-14 13:23:48.977816 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=32 -775> 2018-02-14 13:23:48.977888 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ff9680 0x5587d8fdb440 auth_reply(proto 2 0 (0) Success) v1 -774> 2018-02-14 13:23:48.977899 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.54:0/3194015072 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x5587d8fdb440 con 0 -773> 2018-02-14 13:23:48.977922 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.? 10.255.0.107:0/980595480 2 ==== auth(proto 2 32 bytes epoch 0) v1 ==== 62+0+0 (1331670321 0 0) 0x5587d8fda240 con 0x5587d8d40000 -772> 2018-02-14 13:23:48.977932 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d9149f80 for client.? 10.255.0.107:0/980595480 -771> 2018-02-14 13:23:48.977937 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps -770> 2018-02-14 13:23:48.977940 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos updating c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.977940 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -769> 2018-02-14 13:23:48.977945 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 32 bytes epoch 0) v1 from client.? 10.255.0.107:0/980595480 -768> 2018-02-14 13:23:48.977950 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=32 -767> 2018-02-14 13:23:48.978022 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ff9680 0x5587d88db600 auth_reply(proto 2 0 (0) Success) v1 -766> 2018-02-14 13:23:48.978032 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.107:0/980595480 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x5587d88db600 con 0 -765> 2018-02-14 13:23:48.978053 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1684202 10.255.0.107:0/3082328679 2 ==== auth(proto 2 32 bytes epoch 0) v1 ==== 62+0+0 (2217792213 0 0) 0x5587d8fdafc0 con 0x5587d9004000 -764> 2018-02-14 13:23:48.978064 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d914a100 for client.1684202 10.255.0.107:0/3082328679 -763> 2018-02-14 13:23:48.978069 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps -762> 2018-02-14 13:23:48.978072 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos updating c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.978073 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -761> 2018-02-14 13:23:48.978078 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 32 bytes epoch 0) v1 from client.1684202 10.255.0.107:0/3082328679 -760> 2018-02-14 13:23:48.978083 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=32 -759> 2018-02-14 13:23:48.978142 7fc511507700 5 -- 10.255.0.30:6789/0 >> 10.255.0.52:0/3152189720 conn(0x5587d8da9800 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=455 cs=1 l=1). rx client.? seq 3 0x5587d88db600 auth(proto 2 181 bytes epoch 0) v1 -758> 2018-02-14 13:23:48.978152 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ff9680 0x5587d8fda240 auth_reply(proto 2 0 (0) Success) v1 -757> 2018-02-14 13:23:48.978162 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.107:0/3082328679 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x5587d8fda240 con 0 -756> 2018-02-14 13:23:48.978183 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1393717 10.255.0.52:0/97505378 2 ==== auth(proto 2 144 bytes epoch 0) ==== 170+0+0 (3757563272 0 0) 0x5587d8ec21c0 con 0x5587d9005800 -755> 2018-02-14 13:23:48.978193 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d914a280 for client.1393717 10.255.0.52:0/97505378 -754> 2018-02-14 13:23:48.978199 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps -753> 2018-02-14 13:23:48.978202 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos updating c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.978202 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -752> 2018-02-14 13:23:48.978217 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 144 bytes epoch 0) from client.1393717 10.255.0.52:0/97505378 -751> 2018-02-14 13:23:48.978223 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=144 -750> 2018-02-14 13:23:48.978317 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ff9680 0x5587d8fdafc0 auth_reply(proto 2 0 (0) Success) v1 -749> 2018-02-14 13:23:48.978328 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.52:0/97505378 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x5587d8fdafc0 con 0 -748> 2018-02-14 13:23:48.978350 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1684208 10.255.0.52:0/4056696166 2 ==== auth(proto 2 32 bytes epoch 0) v1 ==== 62+0+0 (2824886521 0 0) 0x5587d8fdb680 con 0x5587d9007000 -747> 2018-02-14 13:23:48.978361 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d914a400 for client.1684208 10.255.0.52:0/4056696166 -746> 2018-02-14 13:23:48.978366 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps -745> 2018-02-14 13:23:48.978369 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos updating c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.978369 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -744> 2018-02-14 13:23:48.978374 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 32 bytes epoch 0) v1 from client.1684208 10.255.0.52:0/4056696166 -743> 2018-02-14 13:23:48.978379 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=32 -742> 2018-02-14 13:23:48.978430 7fc50acfa700 5 -- 10.255.0.30:6789/0 >> 10.255.0.54:0/1233019732 conn(0x5587d8fe5000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=380 cs=1 l=1). rx client.1714104 seq 3 0x5587d8fdafc0 auth(proto 2 181 bytes epoch 0) v1 -741> 2018-02-14 13:23:48.978453 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ffa400 0x5587d8ec21c0 auth_reply(proto 2 0 (0) Success) v1 -740> 2018-02-14 13:23:48.978463 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.52:0/4056696166 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x5587d8ec21c0 con 0 -739> 2018-02-14 13:23:48.978468 7fc511507700 5 -- 10.255.0.30:6789/0 >> 10.255.0.107:0/980595480 conn(0x5587d8d40000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=442 cs=1 l=1). rx client.? seq 3 0x5587d88db180 auth(proto 2 181 bytes epoch 0) v1 -738> 2018-02-14 13:23:48.978485 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1684214 10.255.0.105:0/1753894484 2 ==== auth(proto 2 32 bytes epoch 0) v1 ==== 62+0+0 (785037704 0 0) 0x5587d8fdcac0 con 0x5587d8eab800 -737> 2018-02-14 13:23:48.978496 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891cc00 for client.1684214 10.255.0.105:0/1753894484 -736> 2018-02-14 13:23:48.978502 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps -735> 2018-02-14 13:23:48.978494 7fc50acfa700 5 -- 10.255.0.30:6789/0 >> 10.255.0.52:0/97505378 conn(0x5587d9005800 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=6642 cs=1 l=1). rx client.1393717 seq 3 0x5587d8947000 mon_subscribe({monmap=2+,osdmap=19703}) v2 -734> 2018-02-14 13:23:48.978504 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos updating c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.978505 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -733> 2018-02-14 13:23:48.978510 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 32 bytes epoch 0) v1 from client.1684214 10.255.0.105:0/1753894484 -732> 2018-02-14 13:23:48.978515 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=32 -731> 2018-02-14 13:23:48.978581 7fc50b4fb700 5 -- 10.255.0.30:6789/0 >> 10.255.0.107:0/3082328679 conn(0x5587d9004000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1115 cs=1 l=1). rx client.1684202 seq 3 0x5587d8fda240 auth(proto 2 181 bytes epoch 0) v1 -730> 2018-02-14 13:23:48.978585 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ffa400 0x5587d8fdb680 auth_reply(proto 2 0 (0) Success) v1 -729> 2018-02-14 13:23:48.978603 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.105:0/1753894484 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x5587d8fdb680 con 0 -728> 2018-02-14 13:23:48.978615 7fc50b4fb700 5 -- 10.255.0.30:6789/0 >> 10.255.0.54:0/3194015072 conn(0x5587d8f2c000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=432 cs=1 l=1). rx client.? seq 3 0x5587d8fdb440 auth(proto 2 181 bytes epoch 0) v1 -727> 2018-02-14 13:23:48.978623 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1684262 10.255.0.106:0/3270640683 2 ==== auth(proto 2 32 bytes epoch 0) v1 ==== 62+0+0 (2366757496 0 0) 0x5587d88dba80 con 0x5587d8ead000 -726> 2018-02-14 13:23:48.978633 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891cd80 for client.1684262 10.255.0.106:0/3270640683 -725> 2018-02-14 13:23:48.978638 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps -724> 2018-02-14 13:23:48.978640 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos updating c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.978641 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -723> 2018-02-14 13:23:48.978646 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 32 bytes epoch 0) v1 from client.1684262 10.255.0.106:0/3270640683 -722> 2018-02-14 13:23:48.978650 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=32 -721> 2018-02-14 13:23:48.978718 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ffa400 0x5587d8fdcac0 auth_reply(proto 2 0 (0) Success) v1 -720> 2018-02-14 13:23:48.978728 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.106:0/3270640683 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x5587d8fdcac0 con 0 -719> 2018-02-14 13:23:48.978749 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1684205 10.255.0.51:0/3360953369 2 ==== auth(proto 2 32 bytes epoch 0) v1 ==== 62+0+0 (3476145941 0 0) 0x5587d88db3c0 con 0x5587d8f2d800 -718> 2018-02-14 13:23:48.978759 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891d080 for client.1684205 10.255.0.51:0/3360953369 -717> 2018-02-14 13:23:48.978765 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps -716> 2018-02-14 13:23:48.978768 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos updating c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.978768 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -715> 2018-02-14 13:23:48.978773 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 32 bytes epoch 0) v1 from client.1684205 10.255.0.51:0/3360953369 -714> 2018-02-14 13:23:48.978787 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=32 -713> 2018-02-14 13:23:48.978872 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ffa400 0x5587d88dba80 auth_reply(proto 2 0 (0) Success) v1 -712> 2018-02-14 13:23:48.978871 7fc511507700 5 -- 10.255.0.30:6789/0 >> 10.255.0.52:0/4056696166 conn(0x5587d9007000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1121 cs=1 l=1). rx client.1684208 seq 3 0x5587d8ec21c0 auth(proto 2 181 bytes epoch 0) v1 -711> 2018-02-14 13:23:48.978888 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.51:0/3360953369 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x5587d88dba80 con 0 -710> 2018-02-14 13:23:48.978918 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1684241 10.255.0.30:0/3376128839 3 ==== auth(proto 2 181 bytes epoch 0) v1 ==== 211+0+0 (2037061891 0 0) 0x5587d8fdcf40 con 0x5587d8eae800 -709> 2018-02-14 13:23:48.978937 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891d980 for client.1684241 10.255.0.30:0/3376128839 -708> 2018-02-14 13:23:48.978947 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps allow profile mgr -707> 2018-02-14 13:23:48.978952 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos updating c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.978953 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -706> 2018-02-14 13:23:48.978960 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 181 bytes epoch 0) v1 from client.1684241 10.255.0.30:0/3376128839 -705> 2018-02-14 13:23:48.978966 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=181 -704> 2018-02-14 13:23:48.979022 7fc50b4fb700 5 -- 10.255.0.30:6789/0 >> 10.255.0.105:0/1753894484 conn(0x5587d8eab800 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1131 cs=1 l=1). rx client.1684214 seq 3 0x5587d8fdb680 auth(proto 2 181 bytes epoch 0) v1 -703> 2018-02-14 13:23:48.979119 7fc50acfa700 5 -- 10.255.0.30:6789/0 >> 10.255.0.106:0/3270640683 conn(0x5587d8ead000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=529 cs=1 l=1). rx client.1684262 seq 3 0x5587d8fdcac0 auth(proto 2 181 bytes epoch 0) v1 -702> 2018-02-14 13:23:48.979126 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ffa400 0x5587d88db3c0 auth_reply(proto 2 0 (0) Success) v1 -701> 2018-02-14 13:23:48.979134 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.30:0/3376128839 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x5587d88db3c0 con 0 -700> 2018-02-14 13:23:48.979160 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1684232 10.255.0.105:0/3391979714 3 ==== auth(proto 2 181 bytes epoch 0) v1 ==== 211+0+0 (1143668971 0 0) 0x5587d8ec1b00 con 0x5587d8f94000 -699> 2018-02-14 13:23:48.979177 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891dc80 for client.1684232 10.255.0.105:0/3391979714 -698> 2018-02-14 13:23:48.979187 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd -697> 2018-02-14 13:23:48.979190 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos updating c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.979191 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -696> 2018-02-14 13:23:48.979203 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 181 bytes epoch 0) v1 from client.1684232 10.255.0.105:0/3391979714 -695> 2018-02-14 13:23:48.979211 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=181 -694> 2018-02-14 13:23:48.979340 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ffa400 0x5587d8fdcf40 auth_reply(proto 2 0 (0) Success) v1 -693> 2018-02-14 13:23:48.979351 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.105:0/3391979714 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x5587d8fdcf40 con 0 -692> 2018-02-14 13:23:48.979378 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1684235 10.255.0.53:0/287677392 3 ==== auth(proto 2 181 bytes epoch 0) v1 ==== 211+0+0 (3879148054 0 0) 0x5587d8ec18c0 con 0x5587d8f95800 -691> 2018-02-14 13:23:48.979384 7fc511507700 5 -- 10.255.0.30:6789/0 >> 10.255.0.51:0/3360953369 conn(0x5587d8f2d800 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1139 cs=1 l=1). rx client.1684205 seq 3 0x5587d88db3c0 auth(proto 2 181 bytes epoch 0) v1 -690> 2018-02-14 13:23:48.979392 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891df80 for client.1684235 10.255.0.53:0/287677392 -689> 2018-02-14 13:23:48.979405 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd -688> 2018-02-14 13:23:48.979409 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos updating c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.979410 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -687> 2018-02-14 13:23:48.979418 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 181 bytes epoch 0) v1 from client.1684235 10.255.0.53:0/287677392 -686> 2018-02-14 13:23:48.979423 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=181 -685> 2018-02-14 13:23:48.979468 7fc511507700 5 -- 10.255.0.30:6789/0 >> 10.255.0.30:0/3376128839 conn(0x5587d8eae800 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=668 cs=1 l=1). rx client.1684241 seq 4 0x5587d8e9a000 mon_subscribe({mgrmap=0+,monmap=2+}) v2 -684> 2018-02-14 13:23:48.979516 7fc511507700 5 -- 10.255.0.30:6789/0 >> 10.255.0.30:0/3376128839 conn(0x5587d8eae800 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=668 cs=1 l=1). rx client.1684241 seq 5 0x5587d895ad00 mgrbeacon mgr.ceph-dal-mon-a1(39fac7f7-f5a8-4e84-b9bb-b0dfffd79c92,1684241, -, 0) v5 -683> 2018-02-14 13:23:48.979557 7fc511507700 5 -- 10.255.0.30:6789/0 >> 10.255.0.30:0/3376128839 conn(0x5587d8eae800 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=668 cs=1 l=1). rx client.1684241 seq 6 0x5587d895a400 mgrbeacon mgr.ceph-dal-mon-a1(39fac7f7-f5a8-4e84-b9bb-b0dfffd79c92,1684241, -, 0) v5 -682> 2018-02-14 13:23:48.979604 7fc511507700 5 -- 10.255.0.30:6789/0 >> 10.255.0.30:0/3376128839 conn(0x5587d8eae800 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=668 cs=1 l=1). rx client.1684241 seq 7 0x5587d895bc00 mgrbeacon mgr.ceph-dal-mon-a1(39fac7f7-f5a8-4e84-b9bb-b0dfffd79c92,1684241, -, 0) v5 -681> 2018-02-14 13:23:48.979626 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ffa400 0x5587d8ec1b00 auth_reply(proto 2 0 (0) Success) v1 -680> 2018-02-14 13:23:48.979640 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.53:0/287677392 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x5587d8ec1b00 con 0 -679> 2018-02-14 13:23:48.979647 7fc511507700 5 -- 10.255.0.30:6789/0 >> 10.255.0.30:0/3376128839 conn(0x5587d8eae800 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=668 cs=1 l=1). rx client.1684241 seq 8 0x5587d90b8000 mgrbeacon mgr.ceph-dal-mon-a1(39fac7f7-f5a8-4e84-b9bb-b0dfffd79c92,1684241, -, 0) v5 -678> 2018-02-14 13:23:48.979674 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1684193 10.255.0.105:0/1988410988 3 ==== auth(proto 2 181 bytes epoch 0) v1 ==== 211+0+0 (169167003 0 0) 0x5587d8ec0d80 con 0x5587d8f97000 -677> 2018-02-14 13:23:48.979688 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891e280 for client.1684193 10.255.0.105:0/1988410988 -676> 2018-02-14 13:23:48.979693 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd -675> 2018-02-14 13:23:48.979696 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos updating c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.979697 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -674> 2018-02-14 13:23:48.979702 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 181 bytes epoch 0) v1 from client.1684193 10.255.0.105:0/1988410988 -673> 2018-02-14 13:23:48.979707 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=181 -672> 2018-02-14 13:23:48.979838 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ffa400 0x5587d8ec18c0 auth_reply(proto 2 0 (0) Success) v1 -671> 2018-02-14 13:23:48.979849 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.105:0/1988410988 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x5587d8ec18c0 con 0 -670> 2018-02-14 13:23:48.979875 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1621590 10.255.0.106:0/1162398434 3 ==== auth(proto 2 181 bytes epoch 0) v1 ==== 211+0+0 (1398288050 0 0) 0x5587d8ec2d00 con 0x5587d8f98800 -669> 2018-02-14 13:23:48.979891 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891e580 for client.1621590 10.255.0.106:0/1162398434 -668> 2018-02-14 13:23:48.979901 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd -667> 2018-02-14 13:23:48.979905 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos updating c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.979906 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -666> 2018-02-14 13:23:48.979905 7fc50b4fb700 5 -- 10.255.0.30:6789/0 >> 10.255.0.105:0/3391979714 conn(0x5587d8f94000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=626 cs=1 l=1). rx client.1684232 seq 4 0x5587d8e9ca00 mon_subscribe({mgrmap=0+,monmap=2+,osdmap=19703}) v2 -665> 2018-02-14 13:23:48.979915 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 181 bytes epoch 0) v1 from client.1621590 10.255.0.106:0/1162398434 -664> 2018-02-14 13:23:48.979929 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=181 -663> 2018-02-14 13:23:48.980046 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ffa400 0x5587d8ec0d80 auth_reply(proto 2 0 (0) Success) v1 -662> 2018-02-14 13:23:48.980056 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.106:0/1162398434 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x5587d8ec0d80 con 0 -661> 2018-02-14 13:23:48.980078 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1684238 10.255.0.54:0/2824825993 3 ==== auth(proto 2 181 bytes epoch 0) v1 ==== 211+0+0 (2399453269 0 0) 0x5587d8ec0fc0 con 0x5587d8dac800 -660> 2018-02-14 13:23:48.980089 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891f180 for client.1684238 10.255.0.54:0/2824825993 -659> 2018-02-14 13:23:48.980094 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd -658> 2018-02-14 13:23:48.980106 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos updating c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.980106 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -657> 2018-02-14 13:23:48.980113 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 181 bytes epoch 0) v1 from client.1684238 10.255.0.54:0/2824825993 -656> 2018-02-14 13:23:48.980119 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=181 -655> 2018-02-14 13:23:48.980153 7fc50acfa700 5 -- 10.255.0.30:6789/0 >> 10.255.0.53:0/287677392 conn(0x5587d8f95800 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=620 cs=1 l=1). rx client.1684235 seq 4 0x5587d8e9d400 mon_subscribe({mgrmap=0+,monmap=2+,osdmap=19703}) v2 -654> 2018-02-14 13:23:48.980231 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ffa400 0x5587d8ec2d00 auth_reply(proto 2 0 (0) Success) v1 -653> 2018-02-14 13:23:48.980241 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.54:0/2824825993 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x5587d8ec2d00 con 0 -652> 2018-02-14 13:23:48.980261 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1684211 10.255.0.53:0/2869630763 3 ==== auth(proto 2 181 bytes epoch 0) v1 ==== 211+0+0 (2041379879 0 0) 0x5587d8ec3cc0 con 0x5587d8fae000 -651> 2018-02-14 13:23:48.980271 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891f480 for client.1684211 10.255.0.53:0/2869630763 -650> 2018-02-14 13:23:48.980276 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd -649> 2018-02-14 13:23:48.980279 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos updating c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.980279 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -648> 2018-02-14 13:23:48.980284 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 181 bytes epoch 0) v1 from client.1684211 10.255.0.53:0/2869630763 -647> 2018-02-14 13:23:48.980289 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=181 -646> 2018-02-14 13:23:48.980325 7fc511507700 5 -- 10.255.0.30:6789/0 >> 10.255.0.105:0/1988410988 conn(0x5587d8f97000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=975 cs=1 l=1). rx client.1684193 seq 4 0x5587d8947600 mon_subscribe({mgrmap=0+,monmap=2+,osdmap=19703}) v2 -645> 2018-02-14 13:23:48.980406 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ffa400 0x5587d8ec0fc0 auth_reply(proto 2 0 (0) Success) v1 -644> 2018-02-14 13:23:48.980416 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.53:0/2869630763 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x5587d8ec0fc0 con 0 -643> 2018-02-14 13:23:48.980443 7fc510505700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 encode_full log v 594874 -642> 2018-02-14 13:23:48.980442 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.263543 10.255.0.107:0/322403586 3 ==== mon_subscribe({monmap=2+,osdmap=19703}) v2 ==== 42+0+0 (765878944 0 0) 0x5587d8e9cc00 con 0x5587d8faf800 -641> 2018-02-14 13:23:48.980546 7fc510505700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 encode_pending v594875 -640> 2018-02-14 13:23:48.980587 7fc510505700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos updating c 4935163..4935904) queue_pending_finisher 0x5587d88d5370 -639> 2018-02-14 13:23:48.980599 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891f780 for client.263543 10.255.0.107:0/322403586 -638> 2018-02-14 13:23:48.980606 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd -637> 2018-02-14 13:23:48.980609 7fc50dd00700 20 is_capable service=mon command= read on cap allow profile rbd -636> 2018-02-14 13:23:48.980615 7fc50dd00700 20 allow so far , doing grant allow profile rbd -635> 2018-02-14 13:23:48.980609 7fc50b4fb700 5 -- 10.255.0.30:6789/0 >> 10.255.0.106:0/1162398434 conn(0x5587d8f98800 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=30292 cs=1 l=1). rx client.1621590 seq 4 0x5587d8945800 mon_subscribe({mgrmap=0+,monmap=2+,osdmap=19703}) v2 -634> 2018-02-14 13:23:48.980629 7fc50dd00700 20 match -633> 2018-02-14 13:23:48.980632 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 handle_subscribe mon_subscribe({monmap=2+,osdmap=19703}) v2 -632> 2018-02-14 13:23:48.980627 7fc50b4fb700 5 -- 10.255.0.30:6789/0 >> 10.255.0.106:0/1162398434 conn(0x5587d8f98800 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=30292 cs=1 l=1). rx client.1621590 seq 5 0x5587d8ec0d80 pool_op(delete unmanaged snap pool 6 auid 0 tid 28 name v0) v4 -631> 2018-02-14 13:23:48.980642 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).monmap v1 check_sub monmap next 2 have 1 -630> 2018-02-14 13:23:48.980641 7fc50b4fb700 5 -- 10.255.0.30:6789/0 >> 10.255.0.106:0/1162398434 conn(0x5587d8f98800 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=30292 cs=1 l=1). rx client.1621590 seq 6 0x5587d8fdcf40 pool_op(delete unmanaged snap pool 6 auid 0 tid 28 name v0) v4 -629> 2018-02-14 13:23:48.980647 7fc50dd00700 20 is_capable service=osd command= read on cap allow profile rbd -628> 2018-02-14 13:23:48.980649 7fc50dd00700 20 allow so far , doing grant allow profile rbd -627> 2018-02-14 13:23:48.980651 7fc50dd00700 20 match -626> 2018-02-14 13:23:48.980652 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).osd e19702 check_osdmap_sub 0x5587d8e97200 next 19703 (onetime) -625> 2018-02-14 13:23:48.980663 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1684265 10.255.0.53:0/3548294640 3 ==== auth(proto 2 181 bytes epoch 0) v1 ==== 211+0+0 (434562067 0 0) 0x5587d8fdd3c0 con 0x5587d8fb1000 -624> 2018-02-14 13:23:48.980673 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891fa80 for client.1684265 10.255.0.53:0/3548294640 -623> 2018-02-14 13:23:48.980678 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd -622> 2018-02-14 13:23:48.980681 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos updating c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.980681 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -621> 2018-02-14 13:23:48.980686 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 181 bytes epoch 0) v1 from client.1684265 10.255.0.53:0/3548294640 -620> 2018-02-14 13:23:48.980691 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=181 -619> 2018-02-14 13:23:48.980823 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ffa400 0x5587d8ec3cc0 auth_reply(proto 2 0 (0) Success) v1 -618> 2018-02-14 13:23:48.980834 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.53:0/3548294640 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x5587d8ec3cc0 con 0 -617> 2018-02-14 13:23:48.980853 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1684217 10.255.0.107:0/2591354583 3 ==== auth(proto 2 181 bytes epoch 0) v1 ==== 211+0+0 (3468925151 0 0) 0x5587d8ec2f40 con 0x5587d8fb2800 -616> 2018-02-14 13:23:48.980862 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891fd80 for client.1684217 10.255.0.107:0/2591354583 -615> 2018-02-14 13:23:48.980868 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd -614> 2018-02-14 13:23:48.980870 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos updating c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.980870 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -613> 2018-02-14 13:23:48.980875 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 181 bytes epoch 0) v1 from client.1684217 10.255.0.107:0/2591354583 -612> 2018-02-14 13:23:48.980879 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=181 -611> 2018-02-14 13:23:48.980946 7fc511507700 5 -- 10.255.0.30:6789/0 >> 10.255.0.53:0/2869630763 conn(0x5587d8fae000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1149 cs=1 l=1). rx client.1684211 seq 4 0x5587d8e9a200 mon_subscribe({mgrmap=0+,monmap=2+,osdmap=19703}) v2 -610> 2018-02-14 13:23:48.981004 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ffa400 0x5587d8fdd3c0 auth_reply(proto 2 0 (0) Success) v1 -609> 2018-02-14 13:23:48.981014 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.107:0/2591354583 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x5587d8fdd3c0 con 0 -608> 2018-02-14 13:23:48.981037 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1684226 10.255.0.51:0/3289155469 3 ==== auth(proto 2 181 bytes epoch 0) v1 ==== 211+0+0 (3905577934 0 0) 0x5587d88da880 con 0x5587d9150000 -607> 2018-02-14 13:23:48.981047 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d9148300 for client.1684226 10.255.0.51:0/3289155469 -606> 2018-02-14 13:23:48.981052 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd -605> 2018-02-14 13:23:48.981054 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos updating c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.981054 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -604> 2018-02-14 13:23:48.981059 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 181 bytes epoch 0) v1 from client.1684226 10.255.0.51:0/3289155469 -603> 2018-02-14 13:23:48.981063 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=181 -602> 2018-02-14 13:23:48.981174 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ffa400 0x5587d8ec2f40 auth_reply(proto 2 0 (0) Success) v1 -601> 2018-02-14 13:23:48.981183 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.51:0/3289155469 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x5587d8ec2f40 con 0 -600> 2018-02-14 13:23:48.981203 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1684247 10.255.0.105:0/2076150628 3 ==== auth(proto 2 181 bytes epoch 0) v1 ==== 211+0+0 (2619923869 0 0) 0x5587d8ec3a80 con 0x5587d9151800 -599> 2018-02-14 13:23:48.981212 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d9148480 for client.1684247 10.255.0.105:0/2076150628 -598> 2018-02-14 13:23:48.981218 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd -597> 2018-02-14 13:23:48.981220 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos updating c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.981221 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -596> 2018-02-14 13:23:48.981226 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 181 bytes epoch 0) v1 from client.1684247 10.255.0.105:0/2076150628 -595> 2018-02-14 13:23:48.981231 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=181 -594> 2018-02-14 13:23:48.981270 7fc50acfa700 5 -- 10.255.0.30:6789/0 >> 10.255.0.54:0/2824825993 conn(0x5587d8dac800 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=639 cs=1 l=1). rx client.1684238 seq 4 0x5587d8e9a800 mon_subscribe({mgrmap=0+,monmap=2+,osdmap=19703}) v2 -593> 2018-02-14 13:23:48.981297 7fc50acfa700 5 -- 10.255.0.30:6789/0 >> 10.255.0.53:0/3548294640 conn(0x5587d8fb1000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=899 cs=1 l=1). rx client.1684265 seq 4 0x5587d8e9dc00 mon_subscribe({mgrmap=0+,monmap=2+,osdmap=19703}) v2 -592> 2018-02-14 13:23:48.981343 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ffa400 0x5587d88da880 auth_reply(proto 2 0 (0) Success) v1 -591> 2018-02-14 13:23:48.981353 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.105:0/2076150628 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x5587d88da880 con 0 -590> 2018-02-14 13:23:48.981374 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1684244 10.255.0.54:0/2718452115 3 ==== auth(proto 2 181 bytes epoch 0) v1 ==== 211+0+0 (4224441183 0 0) 0x5587d8ec2ac0 con 0x5587d9153000 -589> 2018-02-14 13:23:48.981384 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d9148600 for client.1684244 10.255.0.54:0/2718452115 -588> 2018-02-14 13:23:48.981389 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd -587> 2018-02-14 13:23:48.981392 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos updating c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.981393 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -586> 2018-02-14 13:23:48.981397 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 181 bytes epoch 0) v1 from client.1684244 10.255.0.54:0/2718452115 -585> 2018-02-14 13:23:48.981403 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=181 -584> 2018-02-14 13:23:48.981491 7fc511507700 5 -- 10.255.0.30:6789/0 >> 10.255.0.107:0/2591354583 conn(0x5587d8fb2800 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=581 cs=1 l=1). rx client.1684217 seq 4 0x5587d8e9b600 mon_subscribe({mgrmap=0+,monmap=2+,osdmap=19703}) v2 -583> 2018-02-14 13:23:48.981531 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ffa400 0x5587d8ec3a80 auth_reply(proto 2 0 (0) Success) v1 -582> 2018-02-14 13:23:48.981541 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.54:0/2718452115 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x5587d8ec3a80 con 0 -581> 2018-02-14 13:23:48.981563 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1684250 10.255.0.107:0/690639890 3 ==== auth(proto 2 181 bytes epoch 0) v1 ==== 211+0+0 (2045289250 0 0) 0x5587d8ec3600 con 0x5587d9154800 -580> 2018-02-14 13:23:48.981573 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d9148780 for client.1684250 10.255.0.107:0/690639890 -579> 2018-02-14 13:23:48.981578 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd -578> 2018-02-14 13:23:48.981581 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos updating c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.981582 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -577> 2018-02-14 13:23:48.981587 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 181 bytes epoch 0) v1 from client.1684250 10.255.0.107:0/690639890 -576> 2018-02-14 13:23:48.981590 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=181 -575> 2018-02-14 13:23:48.981717 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ffa400 0x5587d8ec2ac0 auth_reply(proto 2 0 (0) Success) v1 -574> 2018-02-14 13:23:48.981727 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.107:0/690639890 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x5587d8ec2ac0 con 0 -573> 2018-02-14 13:23:48.981730 7fc50b4fb700 5 -- 10.255.0.30:6789/0 >> 10.255.0.51:0/3289155469 conn(0x5587d9150000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=612 cs=1 l=1). rx client.1684226 seq 4 0x5587d8e9d600 mon_subscribe({mgrmap=0+,monmap=2+,osdmap=19703}) v2 -572> 2018-02-14 13:23:48.981746 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1684190 10.255.0.108:0/3175087704 3 ==== auth(proto 2 181 bytes epoch 0) v1 ==== 211+0+0 (1784991356 0 0) 0x5587d8ec0480 con 0x5587d91be800 -571> 2018-02-14 13:23:48.981756 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d9148f00 for client.1684190 10.255.0.108:0/3175087704 -570> 2018-02-14 13:23:48.981761 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd -569> 2018-02-14 13:23:48.981765 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos updating c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.981765 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -568> 2018-02-14 13:23:48.981770 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 181 bytes epoch 0) v1 from client.1684190 10.255.0.108:0/3175087704 -567> 2018-02-14 13:23:48.981776 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=181 -566> 2018-02-14 13:23:48.981912 7fc50acfa700 5 -- 10.255.0.30:6789/0 >> 10.255.0.105:0/2076150628 conn(0x5587d9151800 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=492 cs=1 l=1). rx client.1684247 seq 4 0x5587d8945600 mon_subscribe({mgrmap=0+,monmap=2+,osdmap=19703}) v2 -565> 2018-02-14 13:23:48.981949 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ffa400 0x5587d8ec3600 auth_reply(proto 2 0 (0) Success) v1 -564> 2018-02-14 13:23:48.981961 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.108:0/3175087704 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x5587d8ec3600 con 0 -563> 2018-02-14 13:23:48.981982 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1582487 10.255.0.49:0/59005741 3 ==== auth(proto 2 181 bytes epoch 0) v1 ==== 211+0+0 (636662280 0 0) 0x5587d88db840 con 0x5587d8fcc000 -562> 2018-02-14 13:23:48.981991 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d9149080 for client.1582487 10.255.0.49:0/59005741 -561> 2018-02-14 13:23:48.981997 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps allow profile mgr -560> 2018-02-14 13:23:48.982000 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos updating c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.982000 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -559> 2018-02-14 13:23:48.982005 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 181 bytes epoch 0) v1 from client.1582487 10.255.0.49:0/59005741 -558> 2018-02-14 13:23:48.982011 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=181 -557> 2018-02-14 13:23:48.982164 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ffa400 0x5587d8ec0480 auth_reply(proto 2 0 (0) Success) v1 -556> 2018-02-14 13:23:48.982174 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.49:0/59005741 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x5587d8ec0480 con 0 -555> 2018-02-14 13:23:48.982198 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1684199 10.255.0.106:0/1777238177 3 ==== auth(proto 2 181 bytes epoch 0) v1 ==== 211+0+0 (1015469992 0 0) 0x5587d8ec1440 con 0x5587d8fcd800 -554> 2018-02-14 13:23:48.982202 7fc511507700 5 -- 10.255.0.30:6789/0 >> 10.255.0.54:0/2718452115 conn(0x5587d9153000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=517 cs=1 l=1). rx client.1684244 seq 4 0x5587d8e9a600 mon_subscribe({mgrmap=0+,monmap=2+,osdmap=19703}) v2 -553> 2018-02-14 13:23:48.982215 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d9149200 for client.1684199 10.255.0.106:0/1777238177 -552> 2018-02-14 13:23:48.982223 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd -551> 2018-02-14 13:23:48.982227 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos updating c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.982228 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -550> 2018-02-14 13:23:48.982238 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 181 bytes epoch 0) v1 from client.1684199 10.255.0.106:0/1777238177 -549> 2018-02-14 13:23:48.982248 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=181 -548> 2018-02-14 13:23:48.982296 7fc50b4fb700 5 -- 10.255.0.30:6789/0 >> 10.255.0.107:0/690639890 conn(0x5587d9154800 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=501 cs=1 l=1). rx client.1684250 seq 4 0x5587d8e9d800 mon_subscribe({mgrmap=0+,monmap=2+,osdmap=19703}) v2 -547> 2018-02-14 13:23:48.982369 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ffa400 0x5587d88db840 auth_reply(proto 2 0 (0) Success) v1 -546> 2018-02-14 13:23:48.982379 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.106:0/1777238177 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x5587d88db840 con 0 -545> 2018-02-14 13:23:48.982400 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1714098 10.255.0.108:0/850049828 3 ==== auth(proto 2 181 bytes epoch 0) v1 ==== 211+0+0 (1892171923 0 0) 0x5587d8ec1200 con 0x5587d8fe3800 -544> 2018-02-14 13:23:48.982417 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d9149800 for client.1714098 10.255.0.108:0/850049828 -543> 2018-02-14 13:23:48.982426 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd -542> 2018-02-14 13:23:48.982430 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos updating c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.982430 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -541> 2018-02-14 13:23:48.982440 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 181 bytes epoch 0) v1 from client.1714098 10.255.0.108:0/850049828 -540> 2018-02-14 13:23:48.982447 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=181 -539> 2018-02-14 13:23:48.982518 7fc50b4fb700 5 -- 10.255.0.30:6789/0 >> 10.255.0.108:0/3175087704 conn(0x5587d91be800 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1094 cs=1 l=1). rx client.1684190 seq 4 0x5587d8e9da00 mon_subscribe({mgrmap=0+,monmap=2+,osdmap=19703}) v2 -538> 2018-02-14 13:23:48.982562 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ffa400 0x5587d8ec1440 auth_reply(proto 2 0 (0) Success) v1 -537> 2018-02-14 13:23:48.982572 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.108:0/850049828 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x5587d8ec1440 con 0 -536> 2018-02-14 13:23:48.982594 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1714104 10.255.0.54:0/1233019732 3 ==== auth(proto 2 181 bytes epoch 0) v1 ==== 211+0+0 (675897030 0 0) 0x5587d8fdafc0 con 0x5587d8fe5000 -535> 2018-02-14 13:23:48.982604 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d9149980 for client.1714104 10.255.0.54:0/1233019732 -534> 2018-02-14 13:23:48.982609 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd -533> 2018-02-14 13:23:48.982612 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos updating c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.982612 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -532> 2018-02-14 13:23:48.982617 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 181 bytes epoch 0) v1 from client.1714104 10.255.0.54:0/1233019732 -531> 2018-02-14 13:23:48.982622 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=181 -530> 2018-02-14 13:23:48.982687 7fc50acfa700 5 -- 10.255.0.30:6789/0 >> 10.255.0.49:0/59005741 conn(0x5587d8fcc000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=12332 cs=1 l=1). rx client.1582487 seq 4 0x5587d90e0000 mon_subscribe({fsmap=2+,log-info=0+,mgrdigest=0+,mgrmap=0+,monmap=2+,osdmap=19703,servicemap=0+}) v2 -529> 2018-02-14 13:23:48.982836 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ffa400 0x5587d8ec1200 auth_reply(proto 2 0 (0) Success) v1 -528> 2018-02-14 13:23:48.982852 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.54:0/1233019732 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x5587d8ec1200 con 0 -527> 2018-02-14 13:23:48.982880 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.? 10.255.0.52:0/3152189720 3 ==== auth(proto 2 181 bytes epoch 0) v1 ==== 211+0+0 (1997859206 0 0) 0x5587d88db600 con 0x5587d8da9800 -526> 2018-02-14 13:23:48.982896 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d9149c80 for client.? 10.255.0.52:0/3152189720 -525> 2018-02-14 13:23:48.982904 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd -524> 2018-02-14 13:23:48.982908 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos updating c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.982909 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -523> 2018-02-14 13:23:48.982903 7fc50acfa700 5 -- 10.255.0.30:6789/0 >> 10.255.0.49:0/59005741 conn(0x5587d8fcc000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=12332 cs=1 l=1). rx client.1582487 seq 5 0x5587d895b600 mgrbeacon mgr.ceph-dal-admin-a1(39fac7f7-f5a8-4e84-b9bb-b0dfffd79c92,1582487, 10.255.0.49:6800/20575, 1) v5 -522> 2018-02-14 13:23:48.982916 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 181 bytes epoch 0) v1 from client.? 10.255.0.52:0/3152189720 -521> 2018-02-14 13:23:48.982925 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=181 -520> 2018-02-14 13:23:48.982987 7fc50acfa700 5 -- 10.255.0.30:6789/0 >> 10.255.0.106:0/1777238177 conn(0x5587d8fcd800 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1161 cs=1 l=1). rx client.1684199 seq 4 0x5587d8e9de00 mon_subscribe({mgrmap=0+,monmap=2+,osdmap=19703}) v2 -519> 2018-02-14 13:23:48.983039 7fc50b4fb700 5 -- 10.255.0.30:6789/0 >> 10.255.0.108:0/850049828 conn(0x5587d8fe3800 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=382 cs=1 l=1). rx client.1714098 seq 4 0x5587d8e9c000 mon_subscribe({mgrmap=0+,monmap=2+,osdmap=19703}) v2 -518> 2018-02-14 13:23:48.983136 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ffa400 0x5587d8fdafc0 auth_reply(proto 2 0 (0) Success) v1 -517> 2018-02-14 13:23:48.983140 7fc50acfa700 5 -- 10.255.0.30:6789/0 >> 10.255.0.49:0/59005741 conn(0x5587d8fcc000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=12332 cs=1 l=1). rx client.1582487 seq 6 0x5587d895f480 monmgrreport(3 checks) v1 -516> 2018-02-14 13:23:48.983152 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.52:0/3152189720 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x5587d8fdafc0 con 0 -515> 2018-02-14 13:23:48.983182 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.? 10.255.0.54:0/3194015072 3 ==== auth(proto 2 181 bytes epoch 0) v1 ==== 211+0+0 (1571713827 0 0) 0x5587d8fdb440 con 0x5587d8f2c000 -514> 2018-02-14 13:23:48.983198 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d9149e00 for client.? 10.255.0.54:0/3194015072 -513> 2018-02-14 13:23:48.983206 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd -512> 2018-02-14 13:23:48.983210 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos updating c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.983211 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -511> 2018-02-14 13:23:48.983221 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 181 bytes epoch 0) v1 from client.? 10.255.0.54:0/3194015072 -510> 2018-02-14 13:23:48.983229 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=181 -509> 2018-02-14 13:23:48.983264 7fc50acfa700 5 -- 10.255.0.30:6789/0 >> 10.255.0.49:0/59005741 conn(0x5587d8fcc000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=12332 cs=1 l=1). rx client.1582487 seq 7 0x5587d90b8900 mgrbeacon mgr.ceph-dal-admin-a1(39fac7f7-f5a8-4e84-b9bb-b0dfffd79c92,1582487, 10.255.0.49:6800/20575, 1) v5 -508> 2018-02-14 13:23:48.983344 7fc50acfa700 5 -- 10.255.0.30:6789/0 >> 10.255.0.49:0/59005741 conn(0x5587d8fcc000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=12332 cs=1 l=1). rx client.1582487 seq 8 0x5587d90f4000 monmgrreport(3 checks) v1 -507> 2018-02-14 13:23:48.983445 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ffa400 0x5587d88db600 auth_reply(proto 2 0 (0) Success) v1 -506> 2018-02-14 13:23:48.983460 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.54:0/3194015072 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x5587d88db600 con 0 -505> 2018-02-14 13:23:48.983461 7fc50acfa700 5 -- 10.255.0.30:6789/0 >> 10.255.0.49:0/59005741 conn(0x5587d8fcc000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=12332 cs=1 l=1). rx client.1582487 seq 9 0x5587d90b8c00 mgrbeacon mgr.ceph-dal-admin-a1(39fac7f7-f5a8-4e84-b9bb-b0dfffd79c92,1582487, 10.255.0.49:6800/20575, 1) v5 -504> 2018-02-14 13:23:48.983489 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.? 10.255.0.107:0/980595480 3 ==== auth(proto 2 181 bytes epoch 0) v1 ==== 211+0+0 (3428397772 0 0) 0x5587d88db180 con 0x5587d8d40000 -503> 2018-02-14 13:23:48.983507 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d9149f80 for client.? 10.255.0.107:0/980595480 -502> 2018-02-14 13:23:48.983516 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd -501> 2018-02-14 13:23:48.983520 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos updating c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.983521 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -500> 2018-02-14 13:23:48.983530 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 181 bytes epoch 0) v1 from client.? 10.255.0.107:0/980595480 -499> 2018-02-14 13:23:48.983527 7fc50acfa700 5 -- 10.255.0.30:6789/0 >> 10.255.0.49:0/59005741 conn(0x5587d8fcc000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=12332 cs=1 l=1). rx client.1582487 seq 10 0x5587d90f4280 monmgrreport(3 checks) v1 -498> 2018-02-14 13:23:48.983541 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=181 -497> 2018-02-14 13:23:48.983678 7fc511507700 5 -- 10.255.0.30:6789/0 >> 10.255.0.52:0/3152189720 conn(0x5587d8da9800 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=455 cs=1 l=1). rx client.? seq 4 0x5587d8e9c200 mon_subscribe({monmap=0+}) v2 -496> 2018-02-14 13:23:48.983746 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ffa400 0x5587d8fdb440 auth_reply(proto 2 0 (0) Success) v1 -495> 2018-02-14 13:23:48.983763 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.107:0/980595480 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x5587d8fdb440 con 0 -494> 2018-02-14 13:23:48.983802 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1684202 10.255.0.107:0/3082328679 3 ==== auth(proto 2 181 bytes epoch 0) v1 ==== 211+0+0 (965991588 0 0) 0x5587d8fda240 con 0x5587d9004000 -493> 2018-02-14 13:23:48.983808 7fc511507700 5 -- 10.255.0.30:6789/0 >> 10.255.0.52:0/3152189720 conn(0x5587d8da9800 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=455 cs=1 l=1). rx client.1724097 seq 5 0x5587d90e1000 mon_subscribe({mgrmap=0+}) v2 -492> 2018-02-14 13:23:48.983821 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d914a100 for client.1684202 10.255.0.107:0/3082328679 -491> 2018-02-14 13:23:48.983830 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd -490> 2018-02-14 13:23:48.983834 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos updating c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.983835 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -489> 2018-02-14 13:23:48.983846 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 181 bytes epoch 0) v1 from client.1684202 10.255.0.107:0/3082328679 -488> 2018-02-14 13:23:48.983855 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=181 -487> 2018-02-14 13:23:48.984036 7fc50acfa700 5 -- 10.255.0.30:6789/0 >> 10.255.0.49:0/59005741 conn(0x5587d8fcc000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=12332 cs=1 l=1). rx client.1582487 seq 11 0x5587d90b8f00 mgrbeacon mgr.ceph-dal-admin-a1(39fac7f7-f5a8-4e84-b9bb-b0dfffd79c92,1582487, 10.255.0.49:6800/20575, 1) v5 -486> 2018-02-14 13:23:48.984053 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ffa400 0x5587d88db180 auth_reply(proto 2 0 (0) Success) v1 -485> 2018-02-14 13:23:48.984067 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.107:0/3082328679 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x5587d88db180 con 0 -484> 2018-02-14 13:23:48.984077 7fc511507700 5 -- 10.255.0.30:6789/0 >> 10.255.0.52:0/3152189720 conn(0x5587d8da9800 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=455 cs=1 l=1). rx client.1724097 seq 6 0x5587d8e9ba00 mon_subscribe({osdmap=0}) v2 -483> 2018-02-14 13:23:48.984097 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1393717 10.255.0.52:0/97505378 3 ==== mon_subscribe({monmap=2+,osdmap=19703}) v2 ==== 42+0+0 (765878944 0 0) 0x5587d8947000 con 0x5587d9005800 -482> 2018-02-14 13:23:48.984116 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d914a280 for client.1393717 10.255.0.52:0/97505378 -481> 2018-02-14 13:23:48.984112 7fc50acfa700 5 -- 10.255.0.30:6789/0 >> 10.255.0.49:0/59005741 conn(0x5587d8fcc000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=12332 cs=1 l=1). rx client.1582487 seq 12 0x5587d90f4500 monmgrreport(3 checks) v1 -480> 2018-02-14 13:23:48.984124 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd -479> 2018-02-14 13:23:48.984128 7fc50dd00700 20 is_capable service=mon command= read on cap allow profile rbd -478> 2018-02-14 13:23:48.984131 7fc50dd00700 20 allow so far , doing grant allow profile rbd -477> 2018-02-14 13:23:48.984138 7fc50dd00700 20 match -476> 2018-02-14 13:23:48.984141 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 handle_subscribe mon_subscribe({monmap=2+,osdmap=19703}) v2 -475> 2018-02-14 13:23:48.984148 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).monmap v1 check_sub monmap next 2 have 1 -474> 2018-02-14 13:23:48.984157 7fc50dd00700 20 is_capable service=osd command= read on cap allow profile rbd -473> 2018-02-14 13:23:48.984159 7fc50dd00700 20 allow so far , doing grant allow profile rbd -472> 2018-02-14 13:23:48.984162 7fc50dd00700 20 match -471> 2018-02-14 13:23:48.984164 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).osd e19702 check_osdmap_sub 0x5587d8e97780 next 19703 (onetime) -470> 2018-02-14 13:23:48.984178 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1684208 10.255.0.52:0/4056696166 3 ==== auth(proto 2 181 bytes epoch 0) v1 ==== 211+0+0 (1181318265 0 0) 0x5587d8ec21c0 con 0x5587d9007000 -469> 2018-02-14 13:23:48.984196 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d914a400 for client.1684208 10.255.0.52:0/4056696166 -468> 2018-02-14 13:23:48.984205 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd -467> 2018-02-14 13:23:48.984208 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos updating c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.984209 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -466> 2018-02-14 13:23:48.984217 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 181 bytes epoch 0) v1 from client.1684208 10.255.0.52:0/4056696166 -465> 2018-02-14 13:23:48.984223 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=181 -464> 2018-02-14 13:23:48.984237 7fc50acfa700 5 -- 10.255.0.30:6789/0 >> 10.255.0.49:0/59005741 conn(0x5587d8fcc000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=12332 cs=1 l=1). rx client.1582487 seq 13 0x5587d90b9200 mgrbeacon mgr.ceph-dal-admin-a1(39fac7f7-f5a8-4e84-b9bb-b0dfffd79c92,1582487, 10.255.0.49:6800/20575, 1) v5 -463> 2018-02-14 13:23:48.984295 7fc50acfa700 5 -- 10.255.0.30:6789/0 >> 10.255.0.49:0/59005741 conn(0x5587d8fcc000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=12332 cs=1 l=1). rx client.1582487 seq 14 0x5587d90f4780 monmgrreport(3 checks) v1 -462> 2018-02-14 13:23:48.984342 7fc50b4fb700 5 -- 10.255.0.30:6789/0 >> 10.255.0.54:0/3194015072 conn(0x5587d8f2c000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=432 cs=1 l=1). rx client.? seq 4 0x5587d90e0a00 mon_subscribe({monmap=0+}) v2 -461> 2018-02-14 13:23:48.984413 7fc50acfa700 5 -- 10.255.0.30:6789/0 >> 10.255.0.49:0/59005741 conn(0x5587d8fcc000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=12332 cs=1 l=1). rx client.1582487 seq 15 0x5587d90b9500 mgrbeacon mgr.ceph-dal-admin-a1(39fac7f7-f5a8-4e84-b9bb-b0dfffd79c92,1582487, 10.255.0.49:6800/20575, 1) v5 -460> 2018-02-14 13:23:48.984430 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ffa400 0x5587d8fda240 auth_reply(proto 2 0 (0) Success) v1 -459> 2018-02-14 13:23:48.984443 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.52:0/4056696166 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x5587d8fda240 con 0 -458> 2018-02-14 13:23:48.984472 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1684214 10.255.0.105:0/1753894484 3 ==== auth(proto 2 181 bytes epoch 0) v1 ==== 211+0+0 (1604397602 0 0) 0x5587d8fdb680 con 0x5587d8eab800 -457> 2018-02-14 13:23:48.984490 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891cc00 for client.1684214 10.255.0.105:0/1753894484 -456> 2018-02-14 13:23:48.984489 7fc50acfa700 5 -- 10.255.0.30:6789/0 >> 10.255.0.49:0/59005741 conn(0x5587d8fcc000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=12332 cs=1 l=1). rx client.1582487 seq 16 0x5587d90f4a00 monmgrreport(3 checks) v1 -455> 2018-02-14 13:23:48.984498 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd -454> 2018-02-14 13:23:48.984501 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos updating c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.984502 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -453> 2018-02-14 13:23:48.984509 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 181 bytes epoch 0) v1 from client.1684214 10.255.0.105:0/1753894484 -452> 2018-02-14 13:23:48.984519 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=181 -451> 2018-02-14 13:23:48.984561 7fc50b4fb700 5 -- 10.255.0.30:6789/0 >> 10.255.0.54:0/3194015072 conn(0x5587d8f2c000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=432 cs=1 l=1). rx client.1724100 seq 5 0x5587d90e0c00 mon_subscribe({mgrmap=0+}) v2 -450> 2018-02-14 13:23:48.984585 7fc511507700 5 -- 10.255.0.30:6789/0 >> 10.255.0.107:0/980595480 conn(0x5587d8d40000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=442 cs=1 l=1). rx client.? seq 4 0x5587d90e1200 mon_subscribe({monmap=0+}) v2 -449> 2018-02-14 13:23:48.984600 7fc50b4fb700 5 -- 10.255.0.30:6789/0 >> 10.255.0.107:0/3082328679 conn(0x5587d9004000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1115 cs=1 l=1). rx client.1684202 seq 4 0x5587d90e0e00 mon_subscribe({mgrmap=0+,monmap=2+,osdmap=19703}) v2 -448> 2018-02-14 13:23:48.984607 7fc50acfa700 5 -- 10.255.0.30:6789/0 >> 10.255.0.49:0/59005741 conn(0x5587d8fcc000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=12332 cs=1 l=1). rx client.1582487 seq 17 0x5587d90b9800 mgrbeacon mgr.ceph-dal-admin-a1(39fac7f7-f5a8-4e84-b9bb-b0dfffd79c92,1582487, 10.255.0.49:6800/20575, 1) v5 -447> 2018-02-14 13:23:48.984668 7fc511507700 5 -- 10.255.0.30:6789/0 >> 10.255.0.107:0/980595480 conn(0x5587d8d40000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=442 cs=1 l=1). rx client.1724103 seq 5 0x5587d90e1600 mon_subscribe({mgrmap=0+}) v2 -446> 2018-02-14 13:23:48.984691 7fc50acfa700 5 -- 10.255.0.30:6789/0 >> 10.255.0.49:0/59005741 conn(0x5587d8fcc000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=12332 cs=1 l=1). rx client.1582487 seq 18 0x5587d90f4c80 monmgrreport(3 checks) v1 -445> 2018-02-14 13:23:48.984716 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ffa400 0x5587d8ec21c0 auth_reply(proto 2 0 (0) Success) v1 -444> 2018-02-14 13:23:48.984731 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.105:0/1753894484 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x5587d8ec21c0 con 0 -443> 2018-02-14 13:23:48.984760 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1684262 10.255.0.106:0/3270640683 3 ==== auth(proto 2 181 bytes epoch 0) v1 ==== 211+0+0 (43003829 0 0) 0x5587d8fdcac0 con 0x5587d8ead000 -442> 2018-02-14 13:23:48.984777 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891cd80 for client.1684262 10.255.0.106:0/3270640683 -441> 2018-02-14 13:23:48.984792 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd -440> 2018-02-14 13:23:48.984796 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos updating c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.984797 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -439> 2018-02-14 13:23:48.984806 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 181 bytes epoch 0) v1 from client.1684262 10.255.0.106:0/3270640683 -438> 2018-02-14 13:23:48.984814 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=181 -437> 2018-02-14 13:23:48.984828 7fc50acfa700 5 -- 10.255.0.30:6789/0 >> 10.255.0.49:0/59005741 conn(0x5587d8fcc000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=12332 cs=1 l=1). rx client.1582487 seq 19 0x5587d90b9b00 mgrbeacon mgr.ceph-dal-admin-a1(39fac7f7-f5a8-4e84-b9bb-b0dfffd79c92,1582487, 10.255.0.49:6800/20575, 1) v5 -436> 2018-02-14 13:23:48.984906 7fc511507700 5 -- 10.255.0.30:6789/0 >> 10.255.0.107:0/980595480 conn(0x5587d8d40000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=442 cs=1 l=1). rx client.1724103 seq 6 0x5587d90e2200 mon_subscribe({osdmap=0}) v2 -435> 2018-02-14 13:23:48.984908 7fc50acfa700 5 -- 10.255.0.30:6789/0 >> 10.255.0.49:0/59005741 conn(0x5587d8fcc000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=12332 cs=1 l=1). rx client.1582487 seq 20 0x5587d90f4f00 monmgrreport(3 checks) v1 -434> 2018-02-14 13:23:48.984956 7fc511507700 5 -- 10.255.0.30:6789/0 >> 10.255.0.52:0/4056696166 conn(0x5587d9007000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1121 cs=1 l=1). rx client.1684208 seq 4 0x5587d90e2400 mon_subscribe({mgrmap=0+,monmap=2+,osdmap=19703}) v2 -433> 2018-02-14 13:23:48.985046 7fc50acfa700 5 -- 10.255.0.30:6789/0 >> 10.255.0.49:0/59005741 conn(0x5587d8fcc000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=12332 cs=1 l=1). rx client.1582487 seq 21 0x5587d90b9e00 mgrbeacon mgr.ceph-dal-admin-a1(39fac7f7-f5a8-4e84-b9bb-b0dfffd79c92,1582487, 10.255.0.49:6800/20575, 1) v5 -432> 2018-02-14 13:23:48.985077 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ffa400 0x5587d8fdb680 auth_reply(proto 2 0 (0) Success) v1 -431> 2018-02-14 13:23:48.985080 7fc50b4fb700 5 -- 10.255.0.30:6789/0 >> 10.255.0.54:0/3194015072 conn(0x5587d8f2c000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=432 cs=1 l=1). rx client.1724100 seq 6 0x5587d90e0800 mon_subscribe({osdmap=0}) v2 -430> 2018-02-14 13:23:48.985094 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.106:0/3270640683 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x5587d8fdb680 con 0 -429> 2018-02-14 13:23:48.985123 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1684205 10.255.0.51:0/3360953369 3 ==== auth(proto 2 181 bytes epoch 0) v1 ==== 211+0+0 (920581475 0 0) 0x5587d88db3c0 con 0x5587d8f2d800 -428> 2018-02-14 13:23:48.985142 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891d080 for client.1684205 10.255.0.51:0/3360953369 -427> 2018-02-14 13:23:48.985152 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd -426> 2018-02-14 13:23:48.985156 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos updating c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.985156 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -425> 2018-02-14 13:23:48.985160 7fc50acfa700 5 -- 10.255.0.30:6789/0 >> 10.255.0.49:0/59005741 conn(0x5587d8fcc000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=12332 cs=1 l=1). rx client.1582487 seq 22 0x5587d90f5180 monmgrreport(3 checks) v1 -424> 2018-02-14 13:23:48.985167 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 preprocess_query auth(proto 2 181 bytes epoch 0) v1 from client.1684205 10.255.0.51:0/3360953369 -423> 2018-02-14 13:23:48.985174 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).auth v2644 prep_auth() blob_size=181 -422> 2018-02-14 13:23:48.985255 7fc50acfa700 5 -- 10.255.0.30:6789/0 >> 10.255.0.49:0/59005741 conn(0x5587d8fcc000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=12332 cs=1 l=1). rx client.1582487 seq 23 0x5587d90ba100 mgrbeacon mgr.ceph-dal-admin-a1(39fac7f7-f5a8-4e84-b9bb-b0dfffd79c92,1582487, 10.255.0.49:6800/20575, 1) v5 -421> 2018-02-14 13:23:48.985257 7fc50b4fb700 5 -- 10.255.0.30:6789/0 >> 10.255.0.105:0/1753894484 conn(0x5587d8eab800 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1131 cs=1 l=1). rx client.1684214 seq 4 0x5587d90e1a00 mon_subscribe({mgrmap=0+,monmap=2+,osdmap=19703}) v2 -420> 2018-02-14 13:23:48.985329 7fc50acfa700 5 -- 10.255.0.30:6789/0 >> 10.255.0.49:0/59005741 conn(0x5587d8fcc000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=12332 cs=1 l=1). rx client.1582487 seq 24 0x5587d90f5400 monmgrreport(3 checks) v1 -419> 2018-02-14 13:23:48.985378 7fc50dd00700 2 mon.ceph-dal-mon-a1@0(leader) e1 send_reply 0x5587d8ffa400 0x5587d8fdcac0 auth_reply(proto 2 0 (0) Success) v1 -418> 2018-02-14 13:23:48.985393 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.51:0/3360953369 -- auth_reply(proto 2 0 (0) Success) v1 -- 0x5587d8fdcac0 con 0 -417> 2018-02-14 13:23:48.985420 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1684241 10.255.0.30:0/3376128839 4 ==== mon_subscribe({mgrmap=0+,monmap=2+}) v2 ==== 42+0+0 (3030213120 0 0) 0x5587d8e9a000 con 0x5587d8eae800 -416> 2018-02-14 13:23:48.985420 7fc50acfa700 5 -- 10.255.0.30:6789/0 >> 10.255.0.49:0/59005741 conn(0x5587d8fcc000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=12332 cs=1 l=1). rx client.1582487 seq 25 0x5587d90ba400 mgrbeacon mgr.ceph-dal-admin-a1(39fac7f7-f5a8-4e84-b9bb-b0dfffd79c92,1582487, 10.255.0.49:6800/20575, 1) v5 -415> 2018-02-14 13:23:48.985437 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891d980 for client.1684241 10.255.0.30:0/3376128839 -414> 2018-02-14 13:23:48.985446 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps allow profile mgr -413> 2018-02-14 13:23:48.985450 7fc50dd00700 20 is_capable service=mon command= read on cap allow profile mgr -412> 2018-02-14 13:23:48.985453 7fc50dd00700 20 allow so far , doing grant allow profile mgr -411> 2018-02-14 13:23:48.985486 7fc50dd00700 20 match -410> 2018-02-14 13:23:48.985484 7fc50acfa700 5 -- 10.255.0.30:6789/0 >> 10.255.0.49:0/59005741 conn(0x5587d8fcc000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=12332 cs=1 l=1). rx client.1582487 seq 26 0x5587d90f5680 monmgrreport(3 checks) v1 -409> 2018-02-14 13:23:48.985492 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 handle_subscribe mon_subscribe({mgrmap=0+,monmap=2+}) v2 -408> 2018-02-14 13:23:48.985500 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader).mgr e422 Sending map to subscriber 0x5587d8eae800 10.255.0.30:0/3376128839 -407> 2018-02-14 13:23:48.985513 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.30:0/3376128839 -- mgrmap(e 422) v1 -- 0x5587d8ea0100 con 0 -406> 2018-02-14 13:23:48.985530 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).monmap v1 check_sub monmap next 2 have 1 -405> 2018-02-14 13:23:48.985545 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1684232 10.255.0.105:0/3391979714 4 ==== mon_subscribe({mgrmap=0+,monmap=2+,osdmap=19703}) v2 ==== 61+0+0 (3551549038 0 0) 0x5587d8e9ca00 con 0x5587d8f94000 -404> 2018-02-14 13:23:48.985562 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891dc80 for client.1684232 10.255.0.105:0/3391979714 -403> 2018-02-14 13:23:48.985570 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd -402> 2018-02-14 13:23:48.985575 7fc50dd00700 20 is_capable service=mon command= read on cap allow profile rbd -401> 2018-02-14 13:23:48.985577 7fc50dd00700 20 allow so far , doing grant allow profile rbd -400> 2018-02-14 13:23:48.985584 7fc50dd00700 20 match -399> 2018-02-14 13:23:48.985586 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 handle_subscribe mon_subscribe({mgrmap=0+,monmap=2+,osdmap=19703}) v2 -398> 2018-02-14 13:23:48.985585 7fc50acfa700 5 -- 10.255.0.30:6789/0 >> 10.255.0.49:0/59005741 conn(0x5587d8fcc000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=12332 cs=1 l=1). rx client.1582487 seq 27 0x5587d90ba700 mgrbeacon mgr.ceph-dal-admin-a1(39fac7f7-f5a8-4e84-b9bb-b0dfffd79c92,1582487, 10.255.0.49:6800/20575, 1) v5 -397> 2018-02-14 13:23:48.985593 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader).mgr e422 Sending map to subscriber 0x5587d8f94000 10.255.0.105:0/3391979714 -396> 2018-02-14 13:23:48.985601 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.105:0/3391979714 -- mgrmap(e 422) v1 -- 0x5587d8ea0940 con 0 -395> 2018-02-14 13:23:48.985616 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).monmap v1 check_sub monmap next 2 have 1 -394> 2018-02-14 13:23:48.985625 7fc50dd00700 20 is_capable service=osd command= read on cap allow profile rbd -393> 2018-02-14 13:23:48.985628 7fc50dd00700 20 allow so far , doing grant allow profile rbd -392> 2018-02-14 13:23:48.985631 7fc50dd00700 20 match -391> 2018-02-14 13:23:48.985633 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).osd e19702 check_osdmap_sub 0x5587d9050100 next 19703 (onetime) -390> 2018-02-14 13:23:48.985644 7fc50acfa700 5 -- 10.255.0.30:6789/0 >> 10.255.0.49:0/59005741 conn(0x5587d8fcc000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=12332 cs=1 l=1). rx client.1582487 seq 28 0x5587d90f5900 monmgrreport(3 checks) v1 -389> 2018-02-14 13:23:48.985644 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1684235 10.255.0.53:0/287677392 4 ==== mon_subscribe({mgrmap=0+,monmap=2+,osdmap=19703}) v2 ==== 61+0+0 (3551549038 0 0) 0x5587d8e9d400 con 0x5587d8f95800 -388> 2018-02-14 13:23:48.985666 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891df80 for client.1684235 10.255.0.53:0/287677392 -387> 2018-02-14 13:23:48.985675 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd -386> 2018-02-14 13:23:48.985679 7fc50dd00700 20 is_capable service=mon command= read on cap allow profile rbd -385> 2018-02-14 13:23:48.985682 7fc50dd00700 20 allow so far , doing grant allow profile rbd -384> 2018-02-14 13:23:48.985688 7fc50dd00700 20 match -383> 2018-02-14 13:23:48.985690 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 handle_subscribe mon_subscribe({mgrmap=0+,monmap=2+,osdmap=19703}) v2 -382> 2018-02-14 13:23:48.985697 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader).mgr e422 Sending map to subscriber 0x5587d8f95800 10.255.0.53:0/287677392 -381> 2018-02-14 13:23:48.985708 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.53:0/287677392 -- mgrmap(e 422) v1 -- 0x5587d8ea0680 con 0 -380> 2018-02-14 13:23:48.985720 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).monmap v1 check_sub monmap next 2 have 1 -379> 2018-02-14 13:23:48.985727 7fc50dd00700 20 is_capable service=osd command= read on cap allow profile rbd -378> 2018-02-14 13:23:48.985730 7fc50dd00700 20 allow so far , doing grant allow profile rbd -377> 2018-02-14 13:23:48.985732 7fc50dd00700 20 match -376> 2018-02-14 13:23:48.985734 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).osd e19702 check_osdmap_sub 0x5587d90517c0 next 19703 (onetime) -375> 2018-02-14 13:23:48.985725 7fc50acfa700 5 -- 10.255.0.30:6789/0 >> 10.255.0.49:0/59005741 conn(0x5587d8fcc000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=12332 cs=1 l=1). rx client.1582487 seq 29 0x5587d90baa00 mgrbeacon mgr.ceph-dal-admin-a1(39fac7f7-f5a8-4e84-b9bb-b0dfffd79c92,1582487, 10.255.0.49:6800/20575, 1) v5 -374> 2018-02-14 13:23:48.985745 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1684193 10.255.0.105:0/1988410988 4 ==== mon_subscribe({mgrmap=0+,monmap=2+,osdmap=19703}) v2 ==== 61+0+0 (3551549038 0 0) 0x5587d8947600 con 0x5587d8f97000 -373> 2018-02-14 13:23:48.985762 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891e280 for client.1684193 10.255.0.105:0/1988410988 -372> 2018-02-14 13:23:48.985770 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd -371> 2018-02-14 13:23:48.985774 7fc50dd00700 20 is_capable service=mon command= read on cap allow profile rbd -370> 2018-02-14 13:23:48.985776 7fc50dd00700 20 allow so far , doing grant allow profile rbd -369> 2018-02-14 13:23:48.985775 7fc50acfa700 5 -- 10.255.0.30:6789/0 >> 10.255.0.49:0/59005741 conn(0x5587d8fcc000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=12332 cs=1 l=1). rx client.1582487 seq 30 0x5587d90f5b80 monmgrreport(3 checks) v1 -368> 2018-02-14 13:23:48.985789 7fc50dd00700 20 match -367> 2018-02-14 13:23:48.985791 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 handle_subscribe mon_subscribe({mgrmap=0+,monmap=2+,osdmap=19703}) v2 -366> 2018-02-14 13:23:48.985798 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader).mgr e422 Sending map to subscriber 0x5587d8f97000 10.255.0.105:0/1988410988 -365> 2018-02-14 13:23:48.985824 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.105:0/1988410988 -- mgrmap(e 422) v1 -- 0x5587d8ea0c00 con 0 -364> 2018-02-14 13:23:48.985844 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).monmap v1 check_sub monmap next 2 have 1 -363> 2018-02-14 13:23:48.985852 7fc50dd00700 20 is_capable service=osd command= read on cap allow profile rbd -362> 2018-02-14 13:23:48.985855 7fc50dd00700 20 allow so far , doing grant allow profile rbd -361> 2018-02-14 13:23:48.985858 7fc50dd00700 20 match -360> 2018-02-14 13:23:48.985859 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).osd e19702 check_osdmap_sub 0x5587d9051a00 next 19703 (onetime) -359> 2018-02-14 13:23:48.985878 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1621590 10.255.0.106:0/1162398434 4 ==== mon_subscribe({mgrmap=0+,monmap=2+,osdmap=19703}) v2 ==== 61+0+0 (3551549038 0 0) 0x5587d8945800 con 0x5587d8f98800 -358> 2018-02-14 13:23:48.985895 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891e580 for client.1621590 10.255.0.106:0/1162398434 -357> 2018-02-14 13:23:48.985889 7fc50acfa700 5 -- 10.255.0.30:6789/0 >> 10.255.0.49:0/59005741 conn(0x5587d8fcc000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=12332 cs=1 l=1). rx client.1582487 seq 31 0x5587d90bad00 mgrbeacon mgr.ceph-dal-admin-a1(39fac7f7-f5a8-4e84-b9bb-b0dfffd79c92,1582487, 10.255.0.49:6800/20575, 1) v5 -356> 2018-02-14 13:23:48.985902 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd -355> 2018-02-14 13:23:48.985906 7fc50dd00700 20 is_capable service=mon command= read on cap allow profile rbd -354> 2018-02-14 13:23:48.985908 7fc50dd00700 20 allow so far , doing grant allow profile rbd -353> 2018-02-14 13:23:48.985915 7fc50dd00700 20 match -352> 2018-02-14 13:23:48.985917 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 handle_subscribe mon_subscribe({mgrmap=0+,monmap=2+,osdmap=19703}) v2 -351> 2018-02-14 13:23:48.985924 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader).mgr e422 Sending map to subscriber 0x5587d8f98800 10.255.0.106:0/1162398434 -350> 2018-02-14 13:23:48.985936 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.106:0/1162398434 -- mgrmap(e 422) v1 -- 0x5587d8ea0ec0 con 0 -349> 2018-02-14 13:23:48.985952 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).monmap v1 check_sub monmap next 2 have 1 -348> 2018-02-14 13:23:48.985949 7fc511507700 5 -- 10.255.0.30:6789/0 >> 10.255.0.51:0/3360953369 conn(0x5587d8f2d800 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=1139 cs=1 l=1). rx client.1684205 seq 4 0x5587d90e1400 mon_subscribe({mgrmap=0+,monmap=2+,osdmap=19703}) v2 -347> 2018-02-14 13:23:48.985959 7fc50dd00700 20 is_capable service=osd command= read on cap allow profile rbd -346> 2018-02-14 13:23:48.985962 7fc50dd00700 20 allow so far , doing grant allow profile rbd -345> 2018-02-14 13:23:48.985965 7fc50dd00700 20 match -344> 2018-02-14 13:23:48.985967 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).osd e19702 check_osdmap_sub 0x5587d90500c0 next 19703 (onetime) -343> 2018-02-14 13:23:48.985967 7fc50acfa700 5 -- 10.255.0.30:6789/0 >> 10.255.0.49:0/59005741 conn(0x5587d8fcc000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=12332 cs=1 l=1). rx client.1582487 seq 32 0x5587d90f5e00 monmgrreport(3 checks) v1 -342> 2018-02-14 13:23:48.985982 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1684238 10.255.0.54:0/2824825993 4 ==== mon_subscribe({mgrmap=0+,monmap=2+,osdmap=19703}) v2 ==== 61+0+0 (3551549038 0 0) 0x5587d8e9a800 con 0x5587d8dac800 -341> 2018-02-14 13:23:48.985998 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891f180 for client.1684238 10.255.0.54:0/2824825993 -340> 2018-02-14 13:23:48.986007 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd -339> 2018-02-14 13:23:48.986011 7fc50dd00700 20 is_capable service=mon command= read on cap allow profile rbd -338> 2018-02-14 13:23:48.986013 7fc50dd00700 20 allow so far , doing grant allow profile rbd -337> 2018-02-14 13:23:48.986020 7fc50dd00700 20 match -336> 2018-02-14 13:23:48.986022 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 handle_subscribe mon_subscribe({mgrmap=0+,monmap=2+,osdmap=19703}) v2 -335> 2018-02-14 13:23:48.986029 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader).mgr e422 Sending map to subscriber 0x5587d8dac800 10.255.0.54:0/2824825993 -334> 2018-02-14 13:23:48.986042 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.54:0/2824825993 -- mgrmap(e 422) v1 -- 0x5587d8ea0940 con 0 -333> 2018-02-14 13:23:48.986059 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).monmap v1 check_sub monmap next 2 have 1 -332> 2018-02-14 13:23:48.986067 7fc50dd00700 20 is_capable service=osd command= read on cap allow profile rbd -331> 2018-02-14 13:23:48.986070 7fc50dd00700 20 allow so far , doing grant allow profile rbd -330> 2018-02-14 13:23:48.986072 7fc50dd00700 20 match -329> 2018-02-14 13:23:48.986074 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).osd e19702 check_osdmap_sub 0x5587d9050580 next 19703 (onetime) -328> 2018-02-14 13:23:48.986066 7fc50acfa700 5 -- 10.255.0.30:6789/0 >> 10.255.0.54:0/1233019732 conn(0x5587d8fe5000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=380 cs=1 l=1). rx client.1714104 seq 4 0x5587d90e0200 mon_subscribe({mgrmap=0+,monmap=2+,osdmap=19703}) v2 -327> 2018-02-14 13:23:48.986085 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1684211 10.255.0.53:0/2869630763 4 ==== mon_subscribe({mgrmap=0+,monmap=2+,osdmap=19703}) v2 ==== 61+0+0 (3551549038 0 0) 0x5587d8e9a200 con 0x5587d8fae000 -326> 2018-02-14 13:23:48.986102 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891f480 for client.1684211 10.255.0.53:0/2869630763 -325> 2018-02-14 13:23:48.986112 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd -324> 2018-02-14 13:23:48.986116 7fc50dd00700 20 is_capable service=mon command= read on cap allow profile rbd -323> 2018-02-14 13:23:48.986118 7fc50dd00700 20 allow so far , doing grant allow profile rbd -322> 2018-02-14 13:23:48.986125 7fc50dd00700 20 match -321> 2018-02-14 13:23:48.986126 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 handle_subscribe mon_subscribe({mgrmap=0+,monmap=2+,osdmap=19703}) v2 -320> 2018-02-14 13:23:48.986131 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader).mgr e422 Sending map to subscriber 0x5587d8fae000 10.255.0.53:0/2869630763 -319> 2018-02-14 13:23:48.986136 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.53:0/2869630763 -- mgrmap(e 422) v1 -- 0x5587d8ea0ec0 con 0 -318> 2018-02-14 13:23:48.986146 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).monmap v1 check_sub monmap next 2 have 1 -317> 2018-02-14 13:23:48.986150 7fc50dd00700 20 is_capable service=osd command= read on cap allow profile rbd -316> 2018-02-14 13:23:48.986152 7fc50dd00700 20 allow so far , doing grant allow profile rbd -315> 2018-02-14 13:23:48.986153 7fc50dd00700 20 match -314> 2018-02-14 13:23:48.986155 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).osd e19702 check_osdmap_sub 0x5587d9050a80 next 19703 (onetime) -313> 2018-02-14 13:23:48.986164 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1684265 10.255.0.53:0/3548294640 4 ==== mon_subscribe({mgrmap=0+,monmap=2+,osdmap=19703}) v2 ==== 61+0+0 (3551549038 0 0) 0x5587d8e9dc00 con 0x5587d8fb1000 -312> 2018-02-14 13:23:48.986173 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891fa80 for client.1684265 10.255.0.53:0/3548294640 -311> 2018-02-14 13:23:48.986178 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd -310> 2018-02-14 13:23:48.986180 7fc50dd00700 20 is_capable service=mon command= read on cap allow profile rbd -309> 2018-02-14 13:23:48.986182 7fc50dd00700 20 allow so far , doing grant allow profile rbd -308> 2018-02-14 13:23:48.986186 7fc50dd00700 20 match -307> 2018-02-14 13:23:48.986187 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 handle_subscribe mon_subscribe({mgrmap=0+,monmap=2+,osdmap=19703}) v2 -306> 2018-02-14 13:23:48.986192 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader).mgr e422 Sending map to subscriber 0x5587d8fb1000 10.255.0.53:0/3548294640 -305> 2018-02-14 13:23:48.986196 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.53:0/3548294640 -- mgrmap(e 422) v1 -- 0x5587d8ea0100 con 0 -304> 2018-02-14 13:23:48.986205 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).monmap v1 check_sub monmap next 2 have 1 -303> 2018-02-14 13:23:48.986212 7fc50dd00700 20 is_capable service=osd command= read on cap allow profile rbd -302> 2018-02-14 13:23:48.986215 7fc50dd00700 20 allow so far , doing grant allow profile rbd -301> 2018-02-14 13:23:48.986217 7fc50dd00700 20 match -300> 2018-02-14 13:23:48.986219 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).osd e19702 check_osdmap_sub 0x5587d9050b40 next 19703 (onetime) -299> 2018-02-14 13:23:48.986230 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1684217 10.255.0.107:0/2591354583 4 ==== mon_subscribe({mgrmap=0+,monmap=2+,osdmap=19703}) v2 ==== 61+0+0 (3551549038 0 0) 0x5587d8e9b600 con 0x5587d8fb2800 -298> 2018-02-14 13:23:48.986246 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891fd80 for client.1684217 10.255.0.107:0/2591354583 -297> 2018-02-14 13:23:48.986252 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd -296> 2018-02-14 13:23:48.986255 7fc50dd00700 20 is_capable service=mon command= read on cap allow profile rbd -295> 2018-02-14 13:23:48.986267 7fc50dd00700 20 allow so far , doing grant allow profile rbd -294> 2018-02-14 13:23:48.986273 7fc50dd00700 20 match -293> 2018-02-14 13:23:48.986274 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 handle_subscribe mon_subscribe({mgrmap=0+,monmap=2+,osdmap=19703}) v2 -292> 2018-02-14 13:23:48.986279 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader).mgr e422 Sending map to subscriber 0x5587d8fb2800 10.255.0.107:0/2591354583 -291> 2018-02-14 13:23:48.986284 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.107:0/2591354583 -- mgrmap(e 422) v1 -- 0x5587d8ea0c00 con 0 -290> 2018-02-14 13:23:48.986294 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).monmap v1 check_sub monmap next 2 have 1 -289> 2018-02-14 13:23:48.986298 7fc50dd00700 20 is_capable service=osd command= read on cap allow profile rbd -288> 2018-02-14 13:23:48.986300 7fc50dd00700 20 allow so far , doing grant allow profile rbd -287> 2018-02-14 13:23:48.986301 7fc50dd00700 20 match -286> 2018-02-14 13:23:48.986302 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).osd e19702 check_osdmap_sub 0x5587d90501c0 next 19703 (onetime) -285> 2018-02-14 13:23:48.986309 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1684226 10.255.0.51:0/3289155469 4 ==== mon_subscribe({mgrmap=0+,monmap=2+,osdmap=19703}) v2 ==== 61+0+0 (3551549038 0 0) 0x5587d8e9d600 con 0x5587d9150000 -284> 2018-02-14 13:23:48.986318 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d9148300 for client.1684226 10.255.0.51:0/3289155469 -283> 2018-02-14 13:23:48.986323 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd -282> 2018-02-14 13:23:48.986325 7fc50dd00700 20 is_capable service=mon command= read on cap allow profile rbd -281> 2018-02-14 13:23:48.986327 7fc50dd00700 20 allow so far , doing grant allow profile rbd -280> 2018-02-14 13:23:48.986331 7fc50dd00700 20 match -279> 2018-02-14 13:23:48.986332 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 handle_subscribe mon_subscribe({mgrmap=0+,monmap=2+,osdmap=19703}) v2 -278> 2018-02-14 13:23:48.986335 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader).mgr e422 Sending map to subscriber 0x5587d9150000 10.255.0.51:0/3289155469 -277> 2018-02-14 13:23:48.986341 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.51:0/3289155469 -- mgrmap(e 422) v1 -- 0x5587d8ea1180 con 0 -276> 2018-02-14 13:23:48.986349 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).monmap v1 check_sub monmap next 2 have 1 -275> 2018-02-14 13:23:48.986356 7fc50dd00700 20 is_capable service=osd command= read on cap allow profile rbd -274> 2018-02-14 13:23:48.986358 7fc50dd00700 20 allow so far , doing grant allow profile rbd -273> 2018-02-14 13:23:48.986361 7fc50dd00700 20 match -272> 2018-02-14 13:23:48.986362 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).osd e19702 check_osdmap_sub 0x5587d9666ac0 next 19703 (onetime) -271> 2018-02-14 13:23:48.986372 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1684247 10.255.0.105:0/2076150628 4 ==== mon_subscribe({mgrmap=0+,monmap=2+,osdmap=19703}) v2 ==== 61+0+0 (3551549038 0 0) 0x5587d8945600 con 0x5587d9151800 -270> 2018-02-14 13:23:48.986382 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d9148480 for client.1684247 10.255.0.105:0/2076150628 -269> 2018-02-14 13:23:48.986387 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd -268> 2018-02-14 13:23:48.986390 7fc50dd00700 20 is_capable service=mon command= read on cap allow profile rbd -267> 2018-02-14 13:23:48.986392 7fc50dd00700 20 allow so far , doing grant allow profile rbd -266> 2018-02-14 13:23:48.986397 7fc50dd00700 20 match -265> 2018-02-14 13:23:48.986398 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 handle_subscribe mon_subscribe({mgrmap=0+,monmap=2+,osdmap=19703}) v2 -264> 2018-02-14 13:23:48.986402 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader).mgr e422 Sending map to subscriber 0x5587d9151800 10.255.0.105:0/2076150628 -263> 2018-02-14 13:23:48.986408 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.105:0/2076150628 -- mgrmap(e 422) v1 -- 0x5587d8ea1440 con 0 -262> 2018-02-14 13:23:48.986417 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).monmap v1 check_sub monmap next 2 have 1 -261> 2018-02-14 13:23:48.986424 7fc50dd00700 20 is_capable service=osd command= read on cap allow profile rbd -260> 2018-02-14 13:23:48.986427 7fc50dd00700 20 allow so far , doing grant allow profile rbd -259> 2018-02-14 13:23:48.986429 7fc50dd00700 20 match -258> 2018-02-14 13:23:48.986431 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).osd e19702 check_osdmap_sub 0x5587d9667680 next 19703 (onetime) -257> 2018-02-14 13:23:48.986441 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1684244 10.255.0.54:0/2718452115 4 ==== mon_subscribe({mgrmap=0+,monmap=2+,osdmap=19703}) v2 ==== 61+0+0 (3551549038 0 0) 0x5587d8e9a600 con 0x5587d9153000 -256> 2018-02-14 13:23:48.986458 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d9148600 for client.1684244 10.255.0.54:0/2718452115 -255> 2018-02-14 13:23:48.986464 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd -254> 2018-02-14 13:23:48.986467 7fc50dd00700 20 is_capable service=mon command= read on cap allow profile rbd -253> 2018-02-14 13:23:48.986469 7fc50dd00700 20 allow so far , doing grant allow profile rbd -252> 2018-02-14 13:23:48.986473 7fc50dd00700 20 match -251> 2018-02-14 13:23:48.986475 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 handle_subscribe mon_subscribe({mgrmap=0+,monmap=2+,osdmap=19703}) v2 -250> 2018-02-14 13:23:48.986479 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader).mgr e422 Sending map to subscriber 0x5587d9153000 10.255.0.54:0/2718452115 -249> 2018-02-14 13:23:48.986483 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.54:0/2718452115 -- mgrmap(e 422) v1 -- 0x5587d8ea1700 con 0 -248> 2018-02-14 13:23:48.986492 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).monmap v1 check_sub monmap next 2 have 1 -247> 2018-02-14 13:23:48.986497 7fc50dd00700 20 is_capable service=osd command= read on cap allow profile rbd -246> 2018-02-14 13:23:48.986498 7fc50dd00700 20 allow so far , doing grant allow profile rbd -245> 2018-02-14 13:23:48.986500 7fc50dd00700 20 match -244> 2018-02-14 13:23:48.986501 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).osd e19702 check_osdmap_sub 0x5587d9667640 next 19703 (onetime) -243> 2018-02-14 13:23:48.986508 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1684250 10.255.0.107:0/690639890 4 ==== mon_subscribe({mgrmap=0+,monmap=2+,osdmap=19703}) v2 ==== 61+0+0 (3551549038 0 0) 0x5587d8e9d800 con 0x5587d9154800 -242> 2018-02-14 13:23:48.986517 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d9148780 for client.1684250 10.255.0.107:0/690639890 -241> 2018-02-14 13:23:48.986522 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd -240> 2018-02-14 13:23:48.986524 7fc50dd00700 20 is_capable service=mon command= read on cap allow profile rbd -239> 2018-02-14 13:23:48.986525 7fc50dd00700 20 allow so far , doing grant allow profile rbd -238> 2018-02-14 13:23:48.986529 7fc50dd00700 20 match -237> 2018-02-14 13:23:48.986531 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 handle_subscribe mon_subscribe({mgrmap=0+,monmap=2+,osdmap=19703}) v2 -236> 2018-02-14 13:23:48.986535 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader).mgr e422 Sending map to subscriber 0x5587d9154800 10.255.0.107:0/690639890 -235> 2018-02-14 13:23:48.986548 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.107:0/690639890 -- mgrmap(e 422) v1 -- 0x5587d8ea19c0 con 0 -234> 2018-02-14 13:23:48.986558 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).monmap v1 check_sub monmap next 2 have 1 -233> 2018-02-14 13:23:48.986563 7fc50dd00700 20 is_capable service=osd command= read on cap allow profile rbd -232> 2018-02-14 13:23:48.986564 7fc50dd00700 20 allow so far , doing grant allow profile rbd -231> 2018-02-14 13:23:48.986566 7fc50dd00700 20 match -230> 2018-02-14 13:23:48.986567 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).osd e19702 check_osdmap_sub 0x5587d9667600 next 19703 (onetime) -229> 2018-02-14 13:23:48.986576 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1684190 10.255.0.108:0/3175087704 4 ==== mon_subscribe({mgrmap=0+,monmap=2+,osdmap=19703}) v2 ==== 61+0+0 (3551549038 0 0) 0x5587d8e9da00 con 0x5587d91be800 -228> 2018-02-14 13:23:48.986576 7fc50acfa700 5 -- 10.255.0.30:6789/0 >> 10.255.0.106:0/3270640683 conn(0x5587d8ead000 :6789 s=STATE_OPEN_MESSAGE_READ_FOOTER_AND_DISPATCH pgs=529 cs=1 l=1). rx client.1684262 seq 4 0x5587d90e0400 mon_subscribe({mgrmap=0+,monmap=2+,osdmap=19703}) v2 -227> 2018-02-14 13:23:48.986588 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d9148f00 for client.1684190 10.255.0.108:0/3175087704 -226> 2018-02-14 13:23:48.986594 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd -225> 2018-02-14 13:23:48.986596 7fc50dd00700 20 is_capable service=mon command= read on cap allow profile rbd -224> 2018-02-14 13:23:48.986598 7fc50dd00700 20 allow so far , doing grant allow profile rbd -223> 2018-02-14 13:23:48.986602 7fc50dd00700 20 match -222> 2018-02-14 13:23:48.986603 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 handle_subscribe mon_subscribe({mgrmap=0+,monmap=2+,osdmap=19703}) v2 -221> 2018-02-14 13:23:48.986608 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader).mgr e422 Sending map to subscriber 0x5587d91be800 10.255.0.108:0/3175087704 -220> 2018-02-14 13:23:48.986612 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.108:0/3175087704 -- mgrmap(e 422) v1 -- 0x5587d8ea1c80 con 0 -219> 2018-02-14 13:23:48.986621 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).monmap v1 check_sub monmap next 2 have 1 -218> 2018-02-14 13:23:48.986625 7fc50dd00700 20 is_capable service=osd command= read on cap allow profile rbd -217> 2018-02-14 13:23:48.986627 7fc50dd00700 20 allow so far , doing grant allow profile rbd -216> 2018-02-14 13:23:48.986628 7fc50dd00700 20 match -215> 2018-02-14 13:23:48.986630 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).osd e19702 check_osdmap_sub 0x5587d96675c0 next 19703 (onetime) -214> 2018-02-14 13:23:48.986638 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1582487 10.255.0.49:0/59005741 4 ==== mon_subscribe({fsmap=2+,log-info=0+,mgrdigest=0+,mgrmap=0+,monmap=2+,osdmap=19703,servicemap=0+}) v2 ==== 145+0+0 (3376921321 0 0) 0x5587d90e0000 con 0x5587d8fcc000 -213> 2018-02-14 13:23:48.986650 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d9149080 for client.1582487 10.255.0.49:0/59005741 -212> 2018-02-14 13:23:48.986655 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps allow profile mgr -211> 2018-02-14 13:23:48.986657 7fc50dd00700 20 is_capable service=mon command= read on cap allow profile mgr -210> 2018-02-14 13:23:48.986659 7fc50dd00700 20 allow so far , doing grant allow profile mgr -209> 2018-02-14 13:23:48.986671 7fc50dd00700 20 match -208> 2018-02-14 13:23:48.986672 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 handle_subscribe mon_subscribe({fsmap=2+,log-info=0+,mgrdigest=0+,mgrmap=0+,monmap=2+,osdmap=19703,servicemap=0+}) v2 -207> 2018-02-14 13:23:48.986677 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 handle_subscribe: MDS sub 'fsmap' -206> 2018-02-14 13:23:48.986679 7fc50dd00700 20 is_capable service=mds command= read on cap allow profile mgr -205> 2018-02-14 13:23:48.986681 7fc50dd00700 20 allow so far , doing grant allow profile mgr -204> 2018-02-14 13:23:48.986683 7fc50dd00700 20 match -203> 2018-02-14 13:23:48.986684 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader).mds e1 check_sub: fsmap -202> 2018-02-14 13:23:48.986696 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 check_sub client wants log-info ver 0 -201> 2018-02-14 13:23:48.986700 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 _create_sub_summary -200> 2018-02-14 13:23:48.986701 7fc50dd00700 1 mon.ceph-dal-mon-a1@0(leader).log v594874 check_sub sending message to client.1582487 10.255.0.49:0/59005741 with 1 entries (version 594874) -199> 2018-02-14 13:23:48.986706 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.49:0/59005741 -- log(1 entries from seq 9 at 2018-02-14 13:09:57.805123) v1 -- 0x5587d8fdd180 con 0 -198> 2018-02-14 13:23:48.986720 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).mgr e422 send_digests -197> 2018-02-14 13:23:48.986726 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).mgr e422 send_digests sending digest to subscriber 0x5587d8fcc000 10.255.0.49:0/59005741 -196> 2018-02-14 13:23:48.986944 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.49:0/59005741 -- mgrdigest v1 -- 0x5587d895f980 con 0 -195> 2018-02-14 13:23:48.986975 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader).mgr e422 Sending map to subscriber 0x5587d8fcc000 10.255.0.49:0/59005741 -194> 2018-02-14 13:23:48.986983 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.49:0/59005741 -- mgrmap(e 422) v1 -- 0x5587d8ea1180 con 0 -193> 2018-02-14 13:23:48.986991 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).monmap v1 check_sub monmap next 2 have 1 -192> 2018-02-14 13:23:48.986996 7fc50dd00700 20 is_capable service=osd command= read on cap allow profile mgr -191> 2018-02-14 13:23:48.986998 7fc50dd00700 20 allow so far , doing grant allow profile mgr -190> 2018-02-14 13:23:48.987000 7fc50dd00700 20 match -189> 2018-02-14 13:23:48.987002 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).osd e19702 check_osdmap_sub 0x5587d96671c0 next 19703 (onetime) -188> 2018-02-14 13:23:48.987006 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).mgrstat check_sub next 0 have 1 -187> 2018-02-14 13:23:48.987008 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.49:0/59005741 -- service_map(e1 0 svc) v1 -- 0x5587d8e9b600 con 0 -186> 2018-02-14 13:23:48.987023 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1684199 10.255.0.106:0/1777238177 4 ==== mon_subscribe({mgrmap=0+,monmap=2+,osdmap=19703}) v2 ==== 61+0+0 (3551549038 0 0) 0x5587d8e9de00 con 0x5587d8fcd800 -185> 2018-02-14 13:23:48.987033 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d9149200 for client.1684199 10.255.0.106:0/1777238177 -184> 2018-02-14 13:23:48.987039 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd -183> 2018-02-14 13:23:48.987041 7fc50dd00700 20 is_capable service=mon command= read on cap allow profile rbd -182> 2018-02-14 13:23:48.987042 7fc50dd00700 20 allow so far , doing grant allow profile rbd -181> 2018-02-14 13:23:48.987047 7fc50dd00700 20 match -180> 2018-02-14 13:23:48.987048 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 handle_subscribe mon_subscribe({mgrmap=0+,monmap=2+,osdmap=19703}) v2 -179> 2018-02-14 13:23:48.987053 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader).mgr e422 Sending map to subscriber 0x5587d8fcd800 10.255.0.106:0/1777238177 -178> 2018-02-14 13:23:48.987071 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.106:0/1777238177 -- mgrmap(e 422) v1 -- 0x5587d8ea19c0 con 0 -177> 2018-02-14 13:23:48.987087 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).monmap v1 check_sub monmap next 2 have 1 -176> 2018-02-14 13:23:48.987092 7fc50dd00700 20 is_capable service=osd command= read on cap allow profile rbd -175> 2018-02-14 13:23:48.987093 7fc50dd00700 20 allow so far , doing grant allow profile rbd -174> 2018-02-14 13:23:48.987095 7fc50dd00700 20 match -173> 2018-02-14 13:23:48.987096 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).osd e19702 check_osdmap_sub 0x5587d9052180 next 19703 (onetime) -172> 2018-02-14 13:23:48.987104 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1714098 10.255.0.108:0/850049828 4 ==== mon_subscribe({mgrmap=0+,monmap=2+,osdmap=19703}) v2 ==== 61+0+0 (3551549038 0 0) 0x5587d8e9c000 con 0x5587d8fe3800 -171> 2018-02-14 13:23:48.987112 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d9149800 for client.1714098 10.255.0.108:0/850049828 -170> 2018-02-14 13:23:48.987117 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd -169> 2018-02-14 13:23:48.987119 7fc50dd00700 20 is_capable service=mon command= read on cap allow profile rbd -168> 2018-02-14 13:23:48.987121 7fc50dd00700 20 allow so far , doing grant allow profile rbd -167> 2018-02-14 13:23:48.987125 7fc50dd00700 20 match -166> 2018-02-14 13:23:48.987126 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 handle_subscribe mon_subscribe({mgrmap=0+,monmap=2+,osdmap=19703}) v2 -165> 2018-02-14 13:23:48.987130 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader).mgr e422 Sending map to subscriber 0x5587d8fe3800 10.255.0.108:0/850049828 -164> 2018-02-14 13:23:48.987134 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.108:0/850049828 -- mgrmap(e 422) v1 -- 0x5587d8ea1c80 con 0 -163> 2018-02-14 13:23:48.987143 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).monmap v1 check_sub monmap next 2 have 1 -162> 2018-02-14 13:23:48.987146 7fc50dd00700 20 is_capable service=osd command= read on cap allow profile rbd -161> 2018-02-14 13:23:48.987147 7fc50dd00700 20 allow so far , doing grant allow profile rbd -160> 2018-02-14 13:23:48.987149 7fc50dd00700 20 match -159> 2018-02-14 13:23:48.987150 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).osd e19702 check_osdmap_sub 0x5587d8e96640 next 19703 (onetime) -158> 2018-02-14 13:23:48.987157 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1714104 10.255.0.54:0/1233019732 4 ==== mon_subscribe({mgrmap=0+,monmap=2+,osdmap=19703}) v2 ==== 61+0+0 (3551549038 0 0) 0x5587d90e0200 con 0x5587d8fe5000 -157> 2018-02-14 13:23:48.987165 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d9149980 for client.1714104 10.255.0.54:0/1233019732 -156> 2018-02-14 13:23:48.987168 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd -155> 2018-02-14 13:23:48.987170 7fc50dd00700 20 is_capable service=mon command= read on cap allow profile rbd -154> 2018-02-14 13:23:48.987171 7fc50dd00700 20 allow so far , doing grant allow profile rbd -153> 2018-02-14 13:23:48.987175 7fc50dd00700 20 match -152> 2018-02-14 13:23:48.987176 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 handle_subscribe mon_subscribe({mgrmap=0+,monmap=2+,osdmap=19703}) v2 -151> 2018-02-14 13:23:48.987181 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader).mgr e422 Sending map to subscriber 0x5587d8fe5000 10.255.0.54:0/1233019732 -150> 2018-02-14 13:23:48.987185 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.54:0/1233019732 -- mgrmap(e 422) v1 -- 0x5587d8ea0ec0 con 0 -149> 2018-02-14 13:23:48.987194 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).monmap v1 check_sub monmap next 2 have 1 -148> 2018-02-14 13:23:48.987199 7fc50dd00700 20 is_capable service=osd command= read on cap allow profile rbd -147> 2018-02-14 13:23:48.987200 7fc50dd00700 20 allow so far , doing grant allow profile rbd -146> 2018-02-14 13:23:48.987202 7fc50dd00700 20 match -145> 2018-02-14 13:23:48.987204 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).osd e19702 check_osdmap_sub 0x5587d90522c0 next 19703 (onetime) -144> 2018-02-14 13:23:48.987210 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.? 10.255.0.52:0/3152189720 4 ==== mon_subscribe({monmap=0+}) v2 ==== 23+0+0 (1620593354 0 0) 0x5587d8e9c200 con 0x5587d8da9800 -143> 2018-02-14 13:23:48.987217 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d9149c80 for client.? 10.255.0.52:0/3152189720 -142> 2018-02-14 13:23:48.987222 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd -141> 2018-02-14 13:23:48.987225 7fc50dd00700 20 is_capable service=mon command= read on cap allow profile rbd -140> 2018-02-14 13:23:48.987227 7fc50dd00700 20 allow so far , doing grant allow profile rbd -139> 2018-02-14 13:23:48.987231 7fc50dd00700 20 match -138> 2018-02-14 13:23:48.987232 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 handle_subscribe mon_subscribe({monmap=0+}) v2 -137> 2018-02-14 13:23:48.987235 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).monmap v1 check_sub monmap next 0 have 1 -136> 2018-02-14 13:23:48.987249 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.52:0/3152189720 -- mon_map magic: 0 v1 -- 0x5587d8fdad80 con 0 -135> 2018-02-14 13:23:48.987269 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.? 10.255.0.54:0/3194015072 4 ==== mon_subscribe({monmap=0+}) v2 ==== 23+0+0 (1620593354 0 0) 0x5587d90e0a00 con 0x5587d8f2c000 -134> 2018-02-14 13:23:48.987279 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d9149e00 for client.? 10.255.0.54:0/3194015072 -133> 2018-02-14 13:23:48.987284 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd -132> 2018-02-14 13:23:48.987287 7fc50dd00700 20 is_capable service=mon command= read on cap allow profile rbd -131> 2018-02-14 13:23:48.987288 7fc50dd00700 20 allow so far , doing grant allow profile rbd -130> 2018-02-14 13:23:48.987294 7fc50dd00700 20 match -129> 2018-02-14 13:23:48.987295 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 handle_subscribe mon_subscribe({monmap=0+}) v2 -128> 2018-02-14 13:23:48.987298 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).monmap v1 check_sub monmap next 0 have 1 -127> 2018-02-14 13:23:48.987306 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.54:0/3194015072 -- mon_map magic: 0 v1 -- 0x5587d8fda000 con 0 -126> 2018-02-14 13:23:48.987318 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.? 10.255.0.107:0/980595480 4 ==== mon_subscribe({monmap=0+}) v2 ==== 23+0+0 (1620593354 0 0) 0x5587d90e1200 con 0x5587d8d40000 -125> 2018-02-14 13:23:48.987327 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d9149f80 for client.? 10.255.0.107:0/980595480 -124> 2018-02-14 13:23:48.987332 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd -123> 2018-02-14 13:23:48.987334 7fc50dd00700 20 is_capable service=mon command= read on cap allow profile rbd -122> 2018-02-14 13:23:48.987336 7fc50dd00700 20 allow so far , doing grant allow profile rbd -121> 2018-02-14 13:23:48.987340 7fc50dd00700 20 match -120> 2018-02-14 13:23:48.987341 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 handle_subscribe mon_subscribe({monmap=0+}) v2 -119> 2018-02-14 13:23:48.987344 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).monmap v1 check_sub monmap next 0 have 1 -118> 2018-02-14 13:23:48.987351 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.107:0/980595480 -- mon_map magic: 0 v1 -- 0x5587d8fddcc0 con 0 -117> 2018-02-14 13:23:48.987363 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1684202 10.255.0.107:0/3082328679 4 ==== mon_subscribe({mgrmap=0+,monmap=2+,osdmap=19703}) v2 ==== 61+0+0 (3551549038 0 0) 0x5587d90e0e00 con 0x5587d9004000 -116> 2018-02-14 13:23:48.987372 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d914a100 for client.1684202 10.255.0.107:0/3082328679 -115> 2018-02-14 13:23:48.987377 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd -114> 2018-02-14 13:23:48.987380 7fc50dd00700 20 is_capable service=mon command= read on cap allow profile rbd -113> 2018-02-14 13:23:48.987381 7fc50dd00700 20 allow so far , doing grant allow profile rbd -112> 2018-02-14 13:23:48.987385 7fc50dd00700 20 match -111> 2018-02-14 13:23:48.987386 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 handle_subscribe mon_subscribe({mgrmap=0+,monmap=2+,osdmap=19703}) v2 -110> 2018-02-14 13:23:48.987390 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader).mgr e422 Sending map to subscriber 0x5587d9004000 10.255.0.107:0/3082328679 -109> 2018-02-14 13:23:48.987395 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.107:0/3082328679 -- mgrmap(e 422) v1 -- 0x5587d8ea0c00 con 0 -108> 2018-02-14 13:23:48.987404 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).monmap v1 check_sub monmap next 2 have 1 -107> 2018-02-14 13:23:48.987407 7fc50dd00700 20 is_capable service=osd command= read on cap allow profile rbd -106> 2018-02-14 13:23:48.987409 7fc50dd00700 20 allow so far , doing grant allow profile rbd -105> 2018-02-14 13:23:48.987410 7fc50dd00700 20 match -104> 2018-02-14 13:23:48.987411 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).osd e19702 check_osdmap_sub 0x5587d9666fc0 next 19703 (onetime) -103> 2018-02-14 13:23:48.987417 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1684208 10.255.0.52:0/4056696166 4 ==== mon_subscribe({mgrmap=0+,monmap=2+,osdmap=19703}) v2 ==== 61+0+0 (3551549038 0 0) 0x5587d90e2400 con 0x5587d9007000 -102> 2018-02-14 13:23:48.987426 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d914a400 for client.1684208 10.255.0.52:0/4056696166 -101> 2018-02-14 13:23:48.987430 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd -100> 2018-02-14 13:23:48.987433 7fc50dd00700 20 is_capable service=mon command= read on cap allow profile rbd -99> 2018-02-14 13:23:48.987434 7fc50dd00700 20 allow so far , doing grant allow profile rbd -98> 2018-02-14 13:23:48.987438 7fc50dd00700 20 match -97> 2018-02-14 13:23:48.987439 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 handle_subscribe mon_subscribe({mgrmap=0+,monmap=2+,osdmap=19703}) v2 -96> 2018-02-14 13:23:48.987443 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader).mgr e422 Sending map to subscriber 0x5587d9007000 10.255.0.52:0/4056696166 -95> 2018-02-14 13:23:48.987448 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.52:0/4056696166 -- mgrmap(e 422) v1 -- 0x5587d8ea1700 con 0 -94> 2018-02-14 13:23:48.987455 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).monmap v1 check_sub monmap next 2 have 1 -93> 2018-02-14 13:23:48.987458 7fc50dd00700 20 is_capable service=osd command= read on cap allow profile rbd -92> 2018-02-14 13:23:48.987459 7fc50dd00700 20 allow so far , doing grant allow profile rbd -91> 2018-02-14 13:23:48.987461 7fc50dd00700 20 match -90> 2018-02-14 13:23:48.987462 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).osd e19702 check_osdmap_sub 0x5587d9666f80 next 19703 (onetime) -89> 2018-02-14 13:23:48.987468 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1684214 10.255.0.105:0/1753894484 4 ==== mon_subscribe({mgrmap=0+,monmap=2+,osdmap=19703}) v2 ==== 61+0+0 (3551549038 0 0) 0x5587d90e1a00 con 0x5587d8eab800 -88> 2018-02-14 13:23:48.987475 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891cc00 for client.1684214 10.255.0.105:0/1753894484 -87> 2018-02-14 13:23:48.987480 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd -86> 2018-02-14 13:23:48.987482 7fc50dd00700 20 is_capable service=mon command= read on cap allow profile rbd -85> 2018-02-14 13:23:48.987484 7fc50dd00700 20 allow so far , doing grant allow profile rbd -84> 2018-02-14 13:23:48.987488 7fc50dd00700 20 match -83> 2018-02-14 13:23:48.987489 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 handle_subscribe mon_subscribe({mgrmap=0+,monmap=2+,osdmap=19703}) v2 -82> 2018-02-14 13:23:48.987493 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader).mgr e422 Sending map to subscriber 0x5587d8eab800 10.255.0.105:0/1753894484 -81> 2018-02-14 13:23:48.987498 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.105:0/1753894484 -- mgrmap(e 422) v1 -- 0x5587d8ea0680 con 0 -80> 2018-02-14 13:23:48.987506 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).monmap v1 check_sub monmap next 2 have 1 -79> 2018-02-14 13:23:48.987511 7fc50dd00700 20 is_capable service=osd command= read on cap allow profile rbd -78> 2018-02-14 13:23:48.987513 7fc50dd00700 20 allow so far , doing grant allow profile rbd -77> 2018-02-14 13:23:48.987514 7fc50dd00700 20 match -76> 2018-02-14 13:23:48.987515 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).osd e19702 check_osdmap_sub 0x5587d9666f40 next 19703 (onetime) -75> 2018-02-14 13:23:48.987521 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1684262 10.255.0.106:0/3270640683 4 ==== mon_subscribe({mgrmap=0+,monmap=2+,osdmap=19703}) v2 ==== 61+0+0 (3551549038 0 0) 0x5587d90e0400 con 0x5587d8ead000 -74> 2018-02-14 13:23:48.987529 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891cd80 for client.1684262 10.255.0.106:0/3270640683 -73> 2018-02-14 13:23:48.987532 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd -72> 2018-02-14 13:23:48.987534 7fc50dd00700 20 is_capable service=mon command= read on cap allow profile rbd -71> 2018-02-14 13:23:48.987535 7fc50dd00700 20 allow so far , doing grant allow profile rbd -70> 2018-02-14 13:23:48.987539 7fc50dd00700 20 match -69> 2018-02-14 13:23:48.987540 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 handle_subscribe mon_subscribe({mgrmap=0+,monmap=2+,osdmap=19703}) v2 -68> 2018-02-14 13:23:48.987544 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader).mgr e422 Sending map to subscriber 0x5587d8ead000 10.255.0.106:0/3270640683 -67> 2018-02-14 13:23:48.987548 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.106:0/3270640683 -- mgrmap(e 422) v1 -- 0x5587d8ea0940 con 0 -66> 2018-02-14 13:23:48.987556 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).monmap v1 check_sub monmap next 2 have 1 -65> 2018-02-14 13:23:48.987558 7fc50dd00700 20 is_capable service=osd command= read on cap allow profile rbd -64> 2018-02-14 13:23:48.987572 7fc50dd00700 20 allow so far , doing grant allow profile rbd -63> 2018-02-14 13:23:48.987575 7fc50dd00700 20 match -62> 2018-02-14 13:23:48.987577 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).osd e19702 check_osdmap_sub 0x5587d9666f00 next 19703 (onetime) -61> 2018-02-14 13:23:48.987583 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1684205 10.255.0.51:0/3360953369 4 ==== mon_subscribe({mgrmap=0+,monmap=2+,osdmap=19703}) v2 ==== 61+0+0 (3551549038 0 0) 0x5587d90e1400 con 0x5587d8f2d800 -60> 2018-02-14 13:23:48.987593 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891d080 for client.1684205 10.255.0.51:0/3360953369 -59> 2018-02-14 13:23:48.987598 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd -58> 2018-02-14 13:23:48.987600 7fc50dd00700 20 is_capable service=mon command= read on cap allow profile rbd -57> 2018-02-14 13:23:48.987602 7fc50dd00700 20 allow so far , doing grant allow profile rbd -56> 2018-02-14 13:23:48.987606 7fc50dd00700 20 match -55> 2018-02-14 13:23:48.987607 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader) e1 handle_subscribe mon_subscribe({mgrmap=0+,monmap=2+,osdmap=19703}) v2 -54> 2018-02-14 13:23:48.987617 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader).mgr e422 Sending map to subscriber 0x5587d8f2d800 10.255.0.51:0/3360953369 -53> 2018-02-14 13:23:48.987624 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.51:0/3360953369 -- mgrmap(e 422) v1 -- 0x5587d8ea0100 con 0 -52> 2018-02-14 13:23:48.987635 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).monmap v1 check_sub monmap next 2 have 1 -51> 2018-02-14 13:23:48.987639 7fc50dd00700 20 is_capable service=osd command= read on cap allow profile rbd -50> 2018-02-14 13:23:48.987641 7fc50dd00700 20 allow so far , doing grant allow profile rbd -49> 2018-02-14 13:23:48.987643 7fc50dd00700 20 match -48> 2018-02-14 13:23:48.987644 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).osd e19702 check_osdmap_sub 0x5587d9666ec0 next 19703 (onetime) -47> 2018-02-14 13:23:48.987650 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1684241 10.255.0.30:0/3376128839 5 ==== mgrbeacon mgr.ceph-dal-mon-a1(39fac7f7-f5a8-4e84-b9bb-b0dfffd79c92,1684241, -, 0) v5 ==== 688+0+0 (3351600542 0 0) 0x5587d895ad00 con 0x5587d8eae800 -46> 2018-02-14 13:23:48.987659 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891d980 for client.1684241 10.255.0.30:0/3376128839 -45> 2018-02-14 13:23:48.987664 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps allow profile mgr -44> 2018-02-14 13:23:48.987666 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos updating c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.987667 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -43> 2018-02-14 13:23:48.987673 7fc50dd00700 4 mon.ceph-dal-mon-a1@0(leader).mgr e422 beacon from 1684241 -42> 2018-02-14 13:23:48.987675 7fc50dd00700 20 is_capable service=mgr command= exec on cap allow profile mgr -41> 2018-02-14 13:23:48.987677 7fc50dd00700 20 allow so far , doing grant allow profile mgr -40> 2018-02-14 13:23:48.987679 7fc50dd00700 20 match -39> 2018-02-14 13:23:48.987684 7fc50dd00700 4 mon.ceph-dal-mon-a1@0(leader).mgr e422 beacon from 1684241 -38> 2018-02-14 13:23:48.987686 7fc50dd00700 4 mon.ceph-dal-mon-a1@0(leader).mgr e422 Standby daemon restart (mgr.ceph-dal-mon-a1) -37> 2018-02-14 13:23:48.987691 7fc50dd00700 0 log_channel(cluster) log [DBG] : Standby manager daemon ceph-dal-mon-a1 restarted -36> 2018-02-14 13:23:48.987694 7fc50dd00700 10 log_client _send_to_monlog to self -35> 2018-02-14 13:23:48.987695 7fc50dd00700 10 log_client log_queue is 10 last_log 10 sent 9 num 10 unsent 1 sending 1 -34> 2018-02-14 13:23:48.987697 7fc50dd00700 10 log_client will send 2018-02-14 13:23:48.987693 mon.ceph-dal-mon-a1 mon.0 10.255.0.30:6789/0 10 : cluster [DBG] Standby manager daemon ceph-dal-mon-a1 restarted -33> 2018-02-14 13:23:48.987704 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.30:6789/0 -- log(1 entries from seq 10 at 2018-02-14 13:23:48.987693) v1 -- 0x5587d8fdd3c0 con 0 -32> 2018-02-14 13:23:48.987717 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).mgr e422 new standby 1684241 -31> 2018-02-14 13:23:48.987722 7fc50dd00700 0 log_channel(cluster) log [DBG] : Standby manager daemon ceph-dal-mon-a1 started -30> 2018-02-14 13:23:48.987724 7fc50dd00700 10 log_client _send_to_monlog to self -29> 2018-02-14 13:23:48.987725 7fc50dd00700 10 log_client log_queue is 11 last_log 11 sent 10 num 11 unsent 1 sending 1 -28> 2018-02-14 13:23:48.987726 7fc50dd00700 10 log_client will send 2018-02-14 13:23:48.987723 mon.ceph-dal-mon-a1 mon.0 10.255.0.30:6789/0 11 : cluster [DBG] Standby manager daemon ceph-dal-mon-a1 started -27> 2018-02-14 13:23:48.987731 7fc50dd00700 1 -- 10.255.0.30:6789/0 --> 10.255.0.30:6789/0 -- log(1 entries from seq 11 at 2018-02-14 13:23:48.987723) v1 -- 0x5587d88db180 con 0 -26> 2018-02-14 13:23:48.987746 7fc50dd00700 4 mon.ceph-dal-mon-a1@0(leader).mgr e422 updating map -25> 2018-02-14 13:23:48.987755 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== mon.0 10.255.0.30:6789/0 0 ==== log(1 entries from seq 10 at 2018-02-14 13:23:48.987693) v1 ==== 0+0+0 (0 0 0) 0x5587d8fdd3c0 con 0x5587d8d41800 -24> 2018-02-14 13:23:48.987765 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891ca80 for mon.0 10.255.0.30:6789/0 -23> 2018-02-14 13:23:48.987771 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps allow * -22> 2018-02-14 13:23:48.987773 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos updating c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.987773 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -21> 2018-02-14 13:23:48.987786 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 preprocess_query log(1 entries from seq 10 at 2018-02-14 13:23:48.987693) v1 from mon.0 10.255.0.30:6789/0 -20> 2018-02-14 13:23:48.987793 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 preprocess_log log(1 entries from seq 10 at 2018-02-14 13:23:48.987693) v1 from mon.0 -19> 2018-02-14 13:23:48.987797 7fc50dd00700 20 is_capable service=log command= write on cap allow * -18> 2018-02-14 13:23:48.987798 7fc50dd00700 20 allow so far , doing grant allow * -17> 2018-02-14 13:23:48.987799 7fc50dd00700 20 allow all -16> 2018-02-14 13:23:48.987803 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== mon.0 10.255.0.30:6789/0 0 ==== log(1 entries from seq 11 at 2018-02-14 13:23:48.987723) v1 ==== 0+0+0 (0 0 0) 0x5587d88db180 con 0x5587d8d41800 -15> 2018-02-14 13:23:48.987811 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891ca80 for mon.0 10.255.0.30:6789/0 -14> 2018-02-14 13:23:48.987816 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps allow * -13> 2018-02-14 13:23:48.987819 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos updating c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.987819 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -12> 2018-02-14 13:23:48.987824 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 preprocess_query log(1 entries from seq 11 at 2018-02-14 13:23:48.987723) v1 from mon.0 10.255.0.30:6789/0 -11> 2018-02-14 13:23:48.987831 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).log v594874 preprocess_log log(1 entries from seq 11 at 2018-02-14 13:23:48.987723) v1 from mon.0 -10> 2018-02-14 13:23:48.987835 7fc50dd00700 20 is_capable service=log command= write on cap allow * -9> 2018-02-14 13:23:48.987836 7fc50dd00700 20 allow so far , doing grant allow * -8> 2018-02-14 13:23:48.987837 7fc50dd00700 20 allow all -7> 2018-02-14 13:23:48.987841 7fc50dd00700 1 -- 10.255.0.30:6789/0 <== client.1621590 10.255.0.106:0/1162398434 5 ==== pool_op(delete unmanaged snap pool 6 auid 0 tid 28 name v0) v4 ==== 65+0+0 (3119176652 0 0) 0x5587d8ec0d80 con 0x5587d8f98800 -6> 2018-02-14 13:23:48.987849 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 _ms_dispatch existing session 0x5587d891e580 for client.1621590 10.255.0.106:0/1162398434 -5> 2018-02-14 13:23:48.987854 7fc50dd00700 20 mon.ceph-dal-mon-a1@0(leader) e1 caps profile rbd -4> 2018-02-14 13:23:48.987856 7fc50dd00700 5 mon.ceph-dal-mon-a1@0(leader).paxos(paxos updating c 4935163..4935904) is_readable = 1 - now=2018-02-14 13:23:48.987856 lease_expire=2018-02-14 13:23:53.929312 has v0 lc 4935904 -3> 2018-02-14 13:23:48.987864 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).osd e19702 preprocess_query pool_op(delete unmanaged snap pool 6 auid 0 tid 28 name v0) v4 from client.1621590 10.255.0.106:0/1162398434 -2> 2018-02-14 13:23:48.987879 7fc50dd00700 7 mon.ceph-dal-mon-a1@0(leader).osd e19702 prepare_update pool_op(delete unmanaged snap pool 6 auid 0 tid 28 name v0) v4 from client.1621590 10.255.0.106:0/1162398434 -1> 2018-02-14 13:23:48.987888 7fc50dd00700 10 mon.ceph-dal-mon-a1@0(leader).osd e19702 prepare_pool_op pool_op(delete unmanaged snap pool 6 auid 0 tid 28 name v0) v4 0> 2018-02-14 13:23:48.991163 7fc50dd00700 -1 /home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos7/DIST/centos7/MACHINE_SIZE/huge/release/12.2.2/rpm/el7/BUILD/ceph-12.2.2/src/include/interval_set.h: In function 'void interval_set::insert(T, T, T*, T*) [with T = snapid_t]' thread 7fc50dd00700 time 2018-02-14 13:23:48.987900 /home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos7/DIST/centos7/MACHINE_SIZE/huge/release/12.2.2/rpm/el7/BUILD/ceph-12.2.2/src/include/interval_set.h: 396: FAILED assert(p->first > start+len) ceph version 12.2.2 (cf0baeeeeba3b47f9427c6c97e2144b094b7e5ba) luminous (stable) 1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x110) [0x5587cebc0330] 2: (interval_set::insert(snapid_t, snapid_t, snapid_t*, snapid_t*)+0x32b) [0x5587ced0965b] 3: (pg_pool_t::remove_unmanaged_snap(snapid_t)+0x4d) [0x5587cecf6a1d] 4: (OSDMonitor::prepare_pool_op(boost::intrusive_ptr)+0xe50) [0x5587ceaeb0f0] 5: (OSDMonitor::prepare_update(boost::intrusive_ptr)+0x336) [0x5587ceb22b16] 6: (PaxosService::dispatch(boost::intrusive_ptr)+0xaf8) [0x5587ceaaf908] 7: (Monitor::dispatch_op(boost::intrusive_ptr)+0x52f) [0x5587ce99281f] 8: (Monitor::_ms_dispatch(Message*)+0x7eb) [0x5587ce993e5b] 9: (Monitor::ms_dispatch(Message*)+0x23) [0x5587ce9bf923] 10: (DispatchQueue::entry()+0x792) [0x5587cee6c162] 11: (DispatchQueue::DispatchThread::entry()+0xd) [0x5587cec67b6d] 12: (()+0x7e25) [0x7fc5172e8e25] 13: (clone()+0x6d) [0x7fc51488334d] NOTE: a copy of the executable, or `objdump -rdS ` is needed to interpret this. --- logging levels --- 0/ 5 none 0/ 1 lockdep 0/ 1 context 1/ 1 crush 1/ 5 mds 1/ 5 mds_balancer 1/ 5 mds_locker 1/ 5 mds_log 1/ 5 mds_log_expire 1/ 5 mds_migrator 0/ 1 buffer 0/ 1 timer 0/ 1 filer 0/ 1 striper 0/ 1 objecter 0/ 5 rados 0/ 5 rbd 0/ 5 rbd_mirror 0/ 5 rbd_replay 0/ 5 journaler 0/ 5 objectcacher 0/ 5 client 1/ 5 osd 0/ 5 optracker 0/ 5 objclass 1/ 3 filestore 1/ 3 journal 0/ 5 ms 20/20 mon 0/10 monc 1/ 5 paxos 0/ 5 tp 1/ 5 auth 1/ 5 crypto 1/ 1 finisher 1/ 1 reserver 1/ 5 heartbeatmap 1/ 5 perfcounter 1/ 5 rgw 1/10 civetweb 1/ 5 javaclient 1/ 5 asok 1/ 1 throttle 0/ 0 refs 1/ 5 xio 1/ 5 compressor 1/ 5 bluestore 1/ 5 bluefs 1/ 3 bdev 1/ 5 kstore 4/ 5 rocksdb 4/ 5 leveldb 4/ 5 memdb 1/ 5 kinetic 1/ 5 fuse 1/ 5 mgr 1/ 5 mgrc 1/ 5 dpdk 1/ 5 eventtrace -2/-2 (syslog threshold) -1/-1 (stderr threshold) max_recent 10000 max_new 1000 log_file /var/log/ceph/ceph-mon.ceph-dal-mon-a1.log --- end dump of recent events --- 2018-02-14 13:23:49.004155 7fc50dd00700 -1 *** Caught signal (Aborted) ** in thread 7fc50dd00700 thread_name:ms_dispatch ceph version 12.2.2 (cf0baeeeeba3b47f9427c6c97e2144b094b7e5ba) luminous (stable) 1: (()+0x8eae11) [0x5587ceec0e11] 2: (()+0xf5e0) [0x7fc5172f05e0] 3: (gsignal()+0x37) [0x7fc5147c01f7] 4: (abort()+0x148) [0x7fc5147c18e8] 5: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x284) [0x5587cebc04a4] 6: (interval_set::insert(snapid_t, snapid_t, snapid_t*, snapid_t*)+0x32b) [0x5587ced0965b] 7: (pg_pool_t::remove_unmanaged_snap(snapid_t)+0x4d) [0x5587cecf6a1d] 8: (OSDMonitor::prepare_pool_op(boost::intrusive_ptr)+0xe50) [0x5587ceaeb0f0] 9: (OSDMonitor::prepare_update(boost::intrusive_ptr)+0x336) [0x5587ceb22b16] 10: (PaxosService::dispatch(boost::intrusive_ptr)+0xaf8) [0x5587ceaaf908] 11: (Monitor::dispatch_op(boost::intrusive_ptr)+0x52f) [0x5587ce99281f] 12: (Monitor::_ms_dispatch(Message*)+0x7eb) [0x5587ce993e5b] 13: (Monitor::ms_dispatch(Message*)+0x23) [0x5587ce9bf923] 14: (DispatchQueue::entry()+0x792) [0x5587cee6c162] 15: (DispatchQueue::DispatchThread::entry()+0xd) [0x5587cec67b6d] 16: (()+0x7e25) [0x7fc5172e8e25] 17: (clone()+0x6d) [0x7fc51488334d] NOTE: a copy of the executable, or `objdump -rdS ` is needed to interpret this.