Ceph » RADOS

If your cluster is in the same state, can you please share mon logs with debug_mon=20? The following code snippet in check_for_older_version() does the version checking and it would be helpful to see the logs from here.

    if (all_versions.size() > 1) {
      dout(20) << __func__ << " all_versions=" << all_versions << dendl;
      // The last entry has the largest version
      dout(20) << __func__ << " highest version daemon count " 
               << all_versions.rbegin()->second.size() << dendl;
      // Erase last element (the highest version running)
      all_versions.erase(all_versions.rbegin()->first);
      ceph_assert(all_versions.size() > 0);
      ostringstream ss;
      unsigned daemon_count = 0;
      for (auto& g : all_versions) {
        daemon_count += g.second.size();
      }
      int ver_count = all_versions.size();
      ceph_assert(!(daemon_count == 1 && ver_count != 1)); 
      ss << "There " << (daemon_count == 1 ? "is a daemon" : "are daemons")
         << " running " << (ver_count > 1 ? "multiple old versions" : "an older version")  << " of ceph";
      health_status_t status;
      if (ver_count > 1)
        status = HEALTH_ERR;
      else
        status = HEALTH_WARN;
      auto& d = checks->add("DAEMON_OLD_VERSION", status, ss.str(), all_versions.size());
      for (auto& g : all_versions) {
        ostringstream ds;
        for (auto& i : g.second) { // Daemon list
          ds << i << " ";
        }
        ds << (g.second.size() == 1 ? "is" : "are") 
           << " running an older version of ceph: " << g.first;
        d.detail.push_back(ds.str());
      }
    } else {
      old_version_first_time = ceph::coarse_mono_clock::zero();
    }
  }