Ceph » RADOS

Seems to be the same problem in:
http://pulpito.front.sepia.ceph.com/yuriw-2021-11-20_18:00:22-rados-wip-yuri6-testing-2021-11-20-0807-distro-basic-smithi/6516753

/a/yuriw-2021-12-23_16:50:03-rados-wip-yuri6-testing-2021-12-22-1410-distro-default-smithi/6582413

2021-12-23T18:37:45.913 INFO:tasks.workunit.client.0.smithi117.stderr:entity client.foo does not exist
2021-12-23T18:37:45.919 INFO:tasks.workunit.client.0.smithi117.stderr:+ expect 'ceph -k /tmp/cephtest-mon-caps-madness.foo.keyring --user foo quorum_status' 13
2021-12-23T18:37:45.919 INFO:tasks.workunit.client.0.smithi117.stderr:+ cmd='ceph -k /tmp/cephtest-mon-caps-madness.foo.keyring --user foo quorum_status'
2021-12-23T18:37:45.919 INFO:tasks.workunit.client.0.smithi117.stderr:+ expected_ret=13
2021-12-23T18:37:45.920 INFO:tasks.workunit.client.0.smithi117.stdout:ceph -k /tmp/cephtest-mon-caps-madness.foo.keyring --user foo quorum_status
2021-12-23T18:37:45.920 INFO:tasks.workunit.client.0.smithi117.stderr:+ echo ceph -k /tmp/cephtest-mon-caps-madness.foo.keyring --user foo quorum_status
2021-12-23T18:37:45.920 INFO:tasks.workunit.client.0.smithi117.stderr:+ eval ceph -k /tmp/cephtest-mon-caps-madness.foo.keyring --user foo quorum_status
2021-12-23T18:37:46.438 INFO:tasks.workunit.client.0.smithi117.stderr:+ ret=0
2021-12-23T18:37:46.438 INFO:tasks.workunit.client.0.smithi117.stderr:+ [[ 0 -ne 13 ]]
2021-12-23T18:37:46.439 INFO:tasks.workunit.client.0.smithi117.stdout:Error: Expected return 13, got 0
2021-12-23T18:37:46.440 DEBUG:teuthology.orchestra.run:got remote process result: 1
2021-12-23T18:37:46.440 INFO:tasks.workunit.client.0.smithi117.stderr:+ echo 'Error: Expected return 13, got 0'
2021-12-23T18:37:46.441 INFO:tasks.workunit.client.0.smithi117.stderr:+ [[ 1 -eq 1 ]]
2021-12-23T18:37:46.441 INFO:tasks.workunit.client.0.smithi117.stderr:+ exit 1

/a/yuriw-2022-03-11_00:13:58-rados-wip-yuri11-testing-2022-03-10-1443-octopus-distro-default-smithi/6730807

2022-03-11T02:40:17.017 INFO:tasks.workunit.client.0.smithi136.stderr:entity client.foo does not exist
2022-03-11T02:40:17.030 INFO:tasks.workunit.client.0.smithi136.stderr:+ expect 'ceph -k /tmp/cephtest-mon-caps-madness.foo.keyring --user foo quorum_status' 13
2022-03-11T02:40:17.030 INFO:tasks.workunit.client.0.smithi136.stderr:+ cmd='ceph -k /tmp/cephtest-mon-caps-madness.foo.keyring --user foo quorum_status'
2022-03-11T02:40:17.031 INFO:tasks.workunit.client.0.smithi136.stdout:ceph -k /tmp/cephtest-mon-caps-madness.foo.keyring --user foo quorum_status
2022-03-11T02:40:17.031 INFO:tasks.workunit.client.0.smithi136.stderr:+ expected_ret=13
2022-03-11T02:40:17.032 INFO:tasks.workunit.client.0.smithi136.stderr:+ echo ceph -k /tmp/cephtest-mon-caps-madness.foo.keyring --user foo quorum_status
2022-03-11T02:40:17.032 INFO:tasks.workunit.client.0.smithi136.stderr:+ eval ceph -k /tmp/cephtest-mon-caps-madness.foo.keyring --user foo quorum_status
2022-03-11T02:40:25.846 INFO:tasks.mon_thrash:Sending CONT to mon a
2022-03-11T02:40:25.847 INFO:tasks.ceph.mon.a:Sent signal 18
2022-03-11T02:40:25.847 INFO:tasks.mon_thrash.ceph_manager:waiting for quorum size 3
2022-03-11T02:40:25.848 DEBUG:teuthology.orchestra.run.smithi073:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph quorum_status
2022-03-11T02:40:26.912 INFO:tasks.workunit.client.0.smithi136.stderr:+ ret=0
2022-03-11T02:40:26.912 INFO:tasks.workunit.client.0.smithi136.stderr:+ [[ 0 -ne 13 ]]
2022-03-11T02:40:26.912 INFO:tasks.workunit.client.0.smithi136.stderr:+ echo 'Error: Expected return 13, got 0'
2022-03-11T02:40:26.913 INFO:tasks.workunit.client.0.smithi136.stdout:Error: Expected return 13, got 0
2022-03-11T02:40:26.914 DEBUG:teuthology.orchestra.run:got remote process result: 1
2022-03-11T02:40:26.914 INFO:tasks.workunit.client.0.smithi136.stderr:+ [[ 1 -eq 1 ]]
2022-03-11T02:40:26.915 INFO:tasks.workunit.client.0.smithi136.stderr:+ exit 1

/a/yuriw-2022-06-16_19:58:30-rados-wip-yuri7-testing-2022-06-16-1051-pacific-distro-default-smithi/6882914

The wrong return code is just an echo of a failure with an auth entity deletion:

2022-06-16T22:41:28.399 INFO:tasks.workunit.client.0.smithi103.stderr:+ ceph auth del client.foo
2022-06-16T22:41:39.662 INFO:tasks.workunit.client.0.smithi103.stderr:entity client.foo does not exist

It affects multiple entities in the test:

2022-06-16T22:36:25.463 INFO:tasks.workunit.client.0.smithi103.stderr:+ ceph auth del client.bazar
2022-06-16T22:36:25.931 INFO:tasks.workunit.client.0.smithi103.stderr:entity client.bazar does not exist

  } else if ((prefix == "auth del" || prefix == "auth rm") &&
             !entity_name.empty()) {
    KeyServerData::Incremental auth_inc;
    auth_inc.name = entity;
    if (!mon.key_server.contains(auth_inc.name)) {
      ss << "entity " << entity << " does not exist";
      err = 0;
      goto done;
    }
    auth_inc.op = KeyServerData::AUTH_INC_DEL;
    push_cephx_inc(auth_inc);

    ss << "updated";
    getline(ss, rs);
    wait_for_finished_proposal(op, new Monitor::C_Command(mon, op, 0, rs,
                                              get_last_committed() + 1));
    return true;
  }

Why does the key server lacks these entities?

bool KeyServer::contains(const EntityName& name) const
{
  std::scoped_lock l{lock};

  return data.contains(name);
}

Well, the exact (but not-so-useful) answer is because KeyServerData lacks it:

struct KeyServerData {
  version_t version;

  /* for each entity */
  std::map<EntityName, EntityAuth> secrets;
  KeyRing *extra_secrets;

  /* for each service type */
  version_t rotating_ver;
  std::map<uint32_t, RotatingSecrets> rotating_secrets;

  explicit KeyServerData(KeyRing *extra)
    : version(0),
      extra_secrets(extra),
      rotating_ver(0) {}

  // ...
  bool contains(const EntityName& name) const {
    return (secrets.find(name) != secrets.end());
  }

  void clear_secrets() {
    version = 0;
    secrets.clear();
    rotating_ver = 0;
    rotating_secrets.clear();
  }

  void add_auth(const EntityName& name, EntityAuth& auth) {
    secrets[name] = auth;
  }

  void remove_secret(const EntityName& name) {
    auto iter = secrets.find(name);
    if (iter == secrets.end())
      return;
    secrets.erase(iter);
  }

Is this a monitors' sync issue?

/a/yuriw-2022-11-28_16:10:10-rados-wip-yuri6-testing-2022-11-23-1348-distro-default-smithi/7093588