Ceph

3 mon,mgr,msd - v17
50 osd - v16

osd v17 not booting, because of require_osd_release equals nautilus (forgot to update at previous upgrade).
"ceph osd require_osd_release pacific" command stuck, ceph-mon service ASSERT FAILed.

DETAILS:

/src/mon/OSDMonitor.cc: 11618: FAILED ceph_assert(osdmap.require_osd_release >= ceph_release_t::octopus)

ceph version 17.2.1 (2508b9f16ef63944cb33be33a271b10931071205) quincy (stable)
 1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x124) [0x7ff26673423c]
 2: /usr/lib/ceph/libceph-common.so.2(+0x2593da) [0x7ff2667343da]
 3: (OSDMonitor::prepare_command_impl(boost::intrusive_ptr<MonOpRequest>, std::map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, boost::variant<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, double, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::vector<long, std::allocator<long> >, std::vector<double, std::allocator<double> > >, std::less<void>, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, boost::variant<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, double, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::vector<long, std::allocator<long> >, std::vector<double, std::allocator<double> > > > > > const&)+0xbe8e) [0x5652021c99ae]
 4: (OSDMonitor::prepare_command(boost::intrusive_ptr<MonOpRequest>)+0x38f) [0x5652021d9dbf]
 5: (OSDMonitor::prepare_update(boost::intrusive_ptr<MonOpRequest>)+0x173) [0x5652021e3313]
 6: (PaxosService::dispatch(boost::intrusive_ptr<MonOpRequest>)+0x2ce) [0x565202160a0e]
 7: (Monitor::handle_command(boost::intrusive_ptr<MonOpRequest>)+0x1fb0) [0x5652020371f0]
 8: (Monitor::dispatch_op(boost::intrusive_ptr<MonOpRequest>)+0x5a3) [0x56520203ac83]
 9: (Monitor::_ms_dispatch(Message*)+0x41e) [0x56520203be8e]
 10: (Monitor::handle_forward(boost::intrusive_ptr<MonOpRequest>)+0xc39) [0x56520203d7a9]
 11: (Monitor::dispatch_op(boost::intrusive_ptr<MonOpRequest>)+0x1104) [0x56520203b7e4]
 12: (Monitor::_ms_dispatch(Message*)+0x41e) [0x56520203be8e]
 13: (Dispatcher::ms_dispatch2(boost::intrusive_ptr<Message> const&)+0x59) [0x56520206b6a9]
 14: (Messenger::ms_deliver_dispatch(boost::intrusive_ptr<Message> const&)+0x468) [0x7ff26697baf8]
 15: (DispatchQueue::entry()+0x5ef) [0x7ff2669791ff]
 16: (DispatchQueue::DispatchThread::entry()+0xd) [0x7ff266a3a59d]
 17: /lib/x86_64-linux-gnu/libpthread.so.0(+0x8ea7) [0x7ff26622cea7]
 18: clone()

2022-09-08T10:15:51.912+0200 7ff25e59a700 -1 ** Caught signal (Aborted) *
in thread 7ff25e59a700 thread_name:ms_dispatch

ceph versions {
"mon": {
"ceph version 17.2.1 (2508b9f16ef63944cb33be33a271b10931071205) quincy (stable)": 3
},
"mgr": {
"ceph version 17.2.1 (2508b9f16ef63944cb33be33a271b10931071205) quincy (stable)": 3
},
"osd": {
"ceph version 16.2.9 (a569859f5e07da0c4c39da81d5fb5675cd95da49) pacific (stable)": 50
},
"mds": {
"ceph version 17.2.1 (2508b9f16ef63944cb33be33a271b10931071205) quincy (stable)": 3
},
"overall": {
"ceph version 16.2.9 (a569859f5e07da0c4c39da81d5fb5675cd95da49) pacific (stable)": 50,
"ceph version 17.2.1 (2508b9f16ef63944cb33be33a271b10931071205) quincy (stable)": 9
}
}

ceph osd dump

...
flags noout,sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit
crush_version 531
full_ratio 0.95
backfillfull_ratio 0.9
nearfull_ratio 0.85
require_min_compat_client pacific
min_compat_client luminous
require_osd_release nautilus
stretch_mode_enabled false
...