Project

General

Profile

Bug #55310 ยป cloud-sync-pacific.log

Baturay Soysal, 04/13/2022 06:39 AM

 
root@ceph-ss-test-oct-controller-av-01:~# docker logs -f ceph-rgw-ceph-ss-test-oct-controller-av-01-rgw1
2022-04-12 16:07:44 /opt/ceph-container/bin/entrypoint.sh: static: does not generate config
2022-04-12 16:07:44 /opt/ceph-container/bin/entrypoint.sh: SUCCESS
exec: PID 54: spawning /usr/bin/radosgw --cluster av-ceph-test --setuser ceph --setgroup ceph --default-log-to-stderr=true --err-to-stderr=true --default-log-to-file=false --foreground -n client.rgw.ceph-ss-test-oct-controller-av-01.rgw1 -k /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring
exec: Waiting 54 to quit
2022-04-12T16:07:44.647+0300 7f2a3b460400 0 deferred set uid:gid to 167:167 (ceph:ceph)
2022-04-12T16:07:44.647+0300 7f2a3b460400 0 ceph version 16.2.7 (dd0603118f56ab514f133c8d2e3adfc983942503) pacific (stable), process radosgw, pid 54
2022-04-12T16:07:44.647+0300 7f2a3b460400 0 framework: beast
2022-04-12T16:07:44.647+0300 7f2a3b460400 0 framework conf key: endpoint, val: 192.168.166.83:8081
2022-04-12T16:07:44.647+0300 7f2a3b460400 1 radosgw_Main not setting numa affinity
2022-04-12T16:07:46.835+0300 7f2a3b460400 5 sync module config (parsed representation):
{
"config": {
"source_bucket": "",
"target_path": "rgw-${zonegroup}-${sid}/${bucket}",
"connection_id": "",
"acls_id": "",
"connection": {
"id": "",
"endpoint": "https://storage.googleapis.com",
"host_style": "path",
"key": {
"access_key": "<gcs_access_key>",
"secret": "******"
}
}
},
"connections": [],
"acl_profiles": [],
"profiles": [
{
"name": "",
"config": {
"source_bucket": "",
"target_path": "rgw-${zonegroup}-${sid}/${bucket}",
"connection_id": "",
"acls_id": "",
"connection": {
"id": "",
"endpoint": "https://storage.googleapis.com",
"host_style": "path",
"key": {
"access_key": "<gcs_access_key>",
"secret": "******"
}
}
}
}
]
}

2022-04-12T16:07:49.163+0300 7f2a3b460400 2 all 8 watchers are set, enabling cache
2022-04-12T16:07:49.163+0300 7f2a3b460400 5 rgw main: tl::expected<std::pair<boost::container::flat_map<long unsigned int, logback_generation>, obj_version>, boost::system::error_code> logback_generations::read(const DoutPrefixProvider*, optional_yield):409: oid=data_loggenerations_metadata not found
2022-04-12T16:07:49.211+0300 7f2a0e091700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start
2022-04-12T16:07:52.747+0300 7f2a09888700 2 object expiration: start
2022-04-12T16:07:52.747+0300 7f2a0a089700 2 garbage collection: garbage collection: start
2022-04-12T16:07:52.751+0300 7f2a3b460400 4 rgw main: failed to read sync status info with (2) No such file or directory
2022-04-12T16:07:52.755+0300 7f2a07884700 4 rgw rados thread: failed to read sync status info with (2) No such file or directory
2022-04-12T16:07:52.755+0300 7f2a07884700 1 RGW-SYNC:meta: initialize sync (reason: period is empty)
2022-04-12T16:07:52.755+0300 7f2a07884700 1 rgw rados thread: epoch=0 in sync status comes before remote's oldest mdlog epoch=1, restarting sync
2022-04-12T16:07:52.759+0300 7f2a3b460400 5 rgw main: starting data sync thread for zone avrupa
2022-04-12T16:07:52.759+0300 7f29ff073700 5 lifecycle: schedule life cycle next start time: Tue Apr 12 21:00:00 2022
2022-04-12T16:07:52.759+0300 7f2a01077700 5 lifecycle: schedule life cycle next start time: Tue Apr 12 21:00:00 2022
2022-04-12T16:07:52.759+0300 7f29fd06f700 5 lifecycle: schedule life cycle next start time: Tue Apr 12 21:00:00 2022
2022-04-12T16:07:52.763+0300 7f2a3b460400 0 framework: beast
2022-04-12T16:07:52.763+0300 7f2a3b460400 0 framework conf key: ssl_certificate, val: config://rgw/cert/$realm/$zone.crt
2022-04-12T16:07:52.763+0300 7f2a3b460400 0 framework conf key: ssl_private_key, val: config://rgw/cert/$realm/$zone.key
2022-04-12T16:07:52.763+0300 7f2a3b460400 0 starting handler: beast
2022-04-12T16:07:52.763+0300 7f2a06882700 4 rgw rados thread: failed to read sync status info with (2) No such file or directory
2022-04-12T16:07:52.763+0300 7f2a3b460400 4 frontend listening on 192.168.166.83:8081
2022-04-12T16:07:52.763+0300 7f2a3b460400 0 set uid:gid to 167:167 (ceph:ceph)
2022-04-12T16:07:52.763+0300 7f2a3b460400 4 frontend spawning 512 threads
2022-04-12T16:07:52.799+0300 7f2a3b460400 1 mgrc service_daemon_register rgw.5727 metadata {arch=x86_64,ceph_release=pacific,ceph_version=ceph version 16.2.7 (dd0603118f56ab514f133c8d2e3adfc983942503) pacific (stable),ceph_version_short=16.2.7,container_image=gitlab.example.com:5050/openstack/registry/ceph/daemon:v6.0.6-stable-6.0-pacific-centos-8-x86_64-ceph-ansible-20220121,cpu=Intel(R) Xeon(R) Gold 6230R CPU @ 2.10GHz,distro=centos,distro_description=CentOS Linux 8,distro_version=8,frontend_config#0=beast endpoint=192.168.166.83:8081,frontend_type#0=beast,hostname=ceph-ss-test-oct-controller-av-01,id=ceph-ss-test-oct-controller-av-01.rgw1,kernel_description=#121-Ubuntu SMP Thu Mar 24 16:04:27 UTC 2022,kernel_version=5.4.0-107-generic,mem_cgroup_limit=4294967296,mem_swap_kb=4030460,mem_total_kb=8148192,num_handles=1,os=Linux,pid=54,realm_id=532f2481-54e3-4d51-95fd-19867ca818e9,realm_name=myrealm,zone_id=c81e6d95-54da-48d3-83a4-7bb7b286709f,zone_name=sync,zonegroup_id=784a05a5-2089-4564-ab1b-a55c43c5f401,zonegroup_name=cdn}
2022-04-12T16:07:53.083+0300 7f2a0a089700 2 garbage collection: garbage collection: stop
2022-04-12T16:07:53.643+0300 7f2a09888700 2 object expiration: stop
2022-04-12T16:07:53.951+0300 7f29f8065700 1 ====== starting new request req=0x7f2a3b415620 =====
2022-04-12T16:07:53.951+0300 7f29f8065700 2 req 6350979191525555842 0.000000000s initializing for trans_id = tx00000582335ecfec73e82-00625579a9-1659-sync
2022-04-12T16:07:53.951+0300 7f29f8065700 2 req 6350979191525555842 0.000000000s getting op 4
2022-04-12T16:07:53.951+0300 7f29f6061700 2 req 6350979191525555842 0.000000000s :post_period verifying requester
2022-04-12T16:07:53.951+0300 7f29f9868700 1 ====== starting new request req=0x7f2a3b313620 =====
2022-04-12T16:07:53.951+0300 7f29f305b700 1 ====== starting new request req=0x7f2a3b394620 =====
2022-04-12T16:07:53.951+0300 7f29f9868700 2 req 8268245483269231629 0.000000000s initializing for trans_id = tx0000072beb575c1fd940d-00625579a9-1659-sync
2022-04-12T16:07:53.951+0300 7f29f305b700 2 req 10560051255912296523 0.000000000s initializing for trans_id = tx00000928cd6aaee91f04b-00625579a9-1659-sync
2022-04-12T16:07:53.955+0300 7f29f9868700 2 req 8268245483269231629 0.004000003s getting op 4
2022-04-12T16:07:53.955+0300 7f29f9868700 2 req 8268245483269231629 0.004000003s :post_period verifying requester
2022-04-12T16:07:53.955+0300 7f29f9868700 5 req 8268245483269231629 0.004000003s :post_period error reading user info, uid=m7yH86NH1D44QiwaUUV9 can't authenticate
2022-04-12T16:07:53.955+0300 7f29f9868700 5 req 8268245483269231629 0.004000003s :post_period error reading user info, uid=m7yH86NH1D44QiwaUUV9 can't authenticate
2022-04-12T16:07:53.955+0300 7f29f9868700 5 req 8268245483269231629 0.004000003s :post_period Failed the auth strategy, reason=-2028
2022-04-12T16:07:53.955+0300 7f29f9868700 1 req 8268245483269231629 0.004000003s op->ERRORHANDLER: err_no=-2028 new_err_no=-2028
2022-04-12T16:07:53.955+0300 7f29f0856700 5 req 6350979191525555842 0.004000003s :post_period error reading user info, uid=m7yH86NH1D44QiwaUUV9 can't authenticate
2022-04-12T16:07:53.955+0300 7f29f0856700 5 req 6350979191525555842 0.004000003s :post_period error reading user info, uid=m7yH86NH1D44QiwaUUV9 can't authenticate
2022-04-12T16:07:53.955+0300 7f29f0856700 5 req 6350979191525555842 0.004000003s :post_period Failed the auth strategy, reason=-2028
2022-04-12T16:07:53.955+0300 7f29f0856700 1 req 6350979191525555842 0.004000003s op->ERRORHANDLER: err_no=-2028 new_err_no=-2028
2022-04-12T16:07:53.955+0300 7f29f305b700 2 req 10560051255912296523 0.004000003s getting op 4
2022-04-12T16:07:53.955+0300 7f29f305b700 2 req 10560051255912296523 0.004000003s :post_period verifying requester
2022-04-12T16:07:53.955+0300 7f29e6842700 2 req 8268245483269231629 0.004000003s :post_period op status=0
2022-04-12T16:07:53.955+0300 7f29e6842700 2 req 8268245483269231629 0.004000003s :post_period http status=403
2022-04-12T16:07:53.955+0300 7f29e6842700 1 ====== req done req=0x7f2a3b313620 op status=0 http_status=403 latency=0.004000003s ======
2022-04-12T16:07:53.955+0300 7f29e6842700 1 beast: 0x7f2a3b313620: 192.168.166.84 - - [12/Apr/2022:16:07:53.951 +0300] "POST /admin/realm/period?period=8b4c894f-cf43-4c56-9af0-1c2b54c74991&epoch=3&rgwx-zonegroup=784a05a5-2089-4564-ab1b-a55c43c5f401 HTTP/1.1" 403 113 - - - latency=0.004000003s
2022-04-12T16:07:53.955+0300 7f29f0856700 2 req 6350979191525555842 0.004000003s :post_period op status=0
2022-04-12T16:07:53.955+0300 7f29f0856700 2 req 6350979191525555842 0.004000003s :post_period http status=403
2022-04-12T16:07:53.955+0300 7f29f0856700 1 ====== req done req=0x7f2a3b415620 op status=0 http_status=403 latency=0.004000003s ======
2022-04-12T16:07:53.955+0300 7f29f305b700 5 req 10560051255912296523 0.004000003s :post_period error reading user info, uid=m7yH86NH1D44QiwaUUV9 can't authenticate
2022-04-12T16:07:53.955+0300 7f29f305b700 5 req 10560051255912296523 0.004000003s :post_period error reading user info, uid=m7yH86NH1D44QiwaUUV9 can't authenticate
2022-04-12T16:07:53.955+0300 7f29f305b700 5 req 10560051255912296523 0.004000003s :post_period Failed the auth strategy, reason=-2028
2022-04-12T16:07:53.955+0300 7f29f305b700 1 req 10560051255912296523 0.004000003s op->ERRORHANDLER: err_no=-2028 new_err_no=-2028
2022-04-12T16:07:53.955+0300 7f29f0856700 1 beast: 0x7f2a3b415620: 192.168.166.83 - - [12/Apr/2022:16:07:53.951 +0300] "POST /admin/realm/period?period=8b4c894f-cf43-4c56-9af0-1c2b54c74991&epoch=3&rgwx-zonegroup=784a05a5-2089-4564-ab1b-a55c43c5f401 HTTP/1.1" 403 113 - - - latency=0.004000003s
2022-04-12T16:07:53.955+0300 7f29ee051700 2 req 10560051255912296523 0.004000003s :post_period op status=0
2022-04-12T16:07:53.955+0300 7f29ee051700 2 req 10560051255912296523 0.004000003s :post_period http status=403
2022-04-12T16:07:53.955+0300 7f29ee051700 1 ====== req done req=0x7f2a3b394620 op status=0 http_status=403 latency=0.004000003s ======
2022-04-12T16:07:53.955+0300 7f29ee051700 1 beast: 0x7f2a3b394620: 192.168.166.85 - - [12/Apr/2022:16:07:53.951 +0300] "POST /admin/realm/period?period=8b4c894f-cf43-4c56-9af0-1c2b54c74991&epoch=3&rgwx-zonegroup=784a05a5-2089-4564-ab1b-a55c43c5f401 HTTP/1.1" 403 113 - - - latency=0.004000003s
2022-04-12T16:07:54.731+0300 7f2a07884700 1 RGW-SYNC:meta: start
2022-04-12T16:07:54.731+0300 7f2a07884700 1 RGW-SYNC:meta: realm epoch=2 period id=8b4c894f-cf43-4c56-9af0-1c2b54c74991
2022-04-12T16:07:54.735+0300 7f2a07884700 4 meta sync: cr:s=0x559e571e3180:op=0x559e59df4000:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
2022-04-12T16:07:54.735+0300 7f2a07884700 4 meta sync: cr:s=0x559e59bc08c0:op=0x559e5578a800:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
2022-04-12T16:07:54.735+0300 7f2a07884700 4 meta sync: cr:s=0x559e594b3540:op=0x559e59df4a00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
2022-04-12T16:07:54.735+0300 7f2a07884700 4 meta sync: cr:s=0x559e5988b680:op=0x559e59df5400:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
2022-04-12T16:07:54.735+0300 7f2a07884700 4 meta sync: cr:s=0x559e59bc1180:op=0x559e59d78a00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
2022-04-12T16:07:54.735+0300 7f2a07884700 4 meta sync: cr:s=0x559e596883c0:op=0x559e59dfa800:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
2022-04-12T16:07:54.735+0300 7f2a07884700 4 meta sync: cr:s=0x559e59689b80:op=0x559e59df7200:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
2022-04-12T16:07:54.735+0300 7f2a07884700 4 meta sync: cr:s=0x559e55ddeb40:op=0x559e59df9400:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
2022-04-12T16:07:54.735+0300 7f2a07884700 4 meta sync: cr:s=0x559e59bc17c0:op=0x559e59d79e00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
2022-04-12T16:07:54.735+0300 7f2a07884700 4 meta sync: cr:s=0x559e5965aa00:op=0x559e59dfb200:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
2022-04-12T16:07:54.735+0300 7f2a07884700 4 meta sync: cr:s=0x559e59bc0140:op=0x559e5578b200:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e595b1e00:op=0x559e59dfca00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e55dfd900:op=0x559e59df8a00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e59bc12c0:op=0x559e59d78000:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e59688a00:op=0x559e59e01400:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e59689cc0:op=0x559e59e02800:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e5988b900:op=0x559e59e09400:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e55dfc8c0:op=0x559e59dfde00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e571e3040:op=0x559e59e05400:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e55dfcc80:op=0x559e59d7b200:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e571e3540:op=0x559e59e00a00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e55dfdb80:op=0x559e59dfe800:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e596888c0:op=0x559e59e01e00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e55dfd2c0:op=0x559e59df6800:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e594b3b80:op=0x559e59e08a00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e55dfc280:op=0x559e59dff200:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e5965ac80:op=0x559e59e04000:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e5988bcc0:op=0x559e59e0d400:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e55dfc3c0:op=0x559e59dfd400:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e594b37c0:op=0x559e59e09e00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e5965ab40:op=0x559e59df9e00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e571e2dc0:op=0x559e59e00000:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e5988b540:op=0x559e59e08000:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
2022-04-12T16:07:54.743+0300 7f2a07884700 4 meta sync: cr:s=0x559e55dfcdc0:op=0x559e59df5e00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
2022-04-12T16:07:54.743+0300 7f2a07884700 4 meta sync: cr:s=0x559e59acc3c0:op=0x559e59d79400:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
2022-04-12T16:07:54.743+0300 7f2a07884700 4 meta sync: cr:s=0x559e55dfc780:op=0x559e59d7a800:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
2022-04-12T16:07:54.743+0300 7f2a07884700 4 meta sync: cr:s=0x559e57c2af00:op=0x559e59e17200:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
2022-04-12T16:07:54.747+0300 7f2a07884700 4 meta sync: cr:s=0x559e571e2c80:op=0x559e59e06800:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
2022-04-12T16:07:54.747+0300 7f2a07884700 4 meta sync: cr:s=0x559e59bc0dc0:op=0x559e59e11400:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
2022-04-12T16:07:54.747+0300 7f2a07884700 4 meta sync: cr:s=0x559e59bcd540:op=0x559e59e14000:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
2022-04-12T16:07:54.747+0300 7f2a07884700 4 meta sync: cr:s=0x559e57c2adc0:op=0x559e59e16800:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
2022-04-12T16:07:54.747+0300 7f2a07884700 4 meta sync: cr:s=0x559e55a08140:op=0x559e59e14a00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
2022-04-12T16:07:54.747+0300 7f2a07884700 4 meta sync: cr:s=0x559e564c1cc0:op=0x559e59e04a00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
2022-04-12T16:07:54.747+0300 7f2a07884700 4 meta sync: cr:s=0x559e57c2b040:op=0x559e59e18000:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
2022-04-12T16:07:54.747+0300 7f2a07884700 4 meta sync: cr:s=0x559e59bc1900:op=0x559e59e0e800:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
2022-04-12T16:07:54.747+0300 7f2a07884700 4 meta sync: cr:s=0x559e59bc0640:op=0x559e59e0f200:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
2022-04-12T16:07:54.751+0300 7f2a07884700 4 meta sync: cr:s=0x559e59689e00:op=0x559e59e03200:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
2022-04-12T16:07:54.751+0300 7f2a07884700 4 meta sync: cr:s=0x559e55a083c0:op=0x559e59e0de00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
2022-04-12T16:07:54.751+0300 7f2a07884700 4 meta sync: cr:s=0x559e571e3cc0:op=0x559e59e07200:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
2022-04-12T16:07:54.751+0300 7f2a07884700 4 meta sync: cr:s=0x559e5988be00:op=0x559e59e0b200:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
2022-04-12T16:07:54.751+0300 7f2a07884700 4 meta sync: cr:s=0x559e594b3680:op=0x559e59e0ca00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
2022-04-12T16:07:54.751+0300 7f2a07884700 4 meta sync: cr:s=0x559e5988ba40:op=0x559e59df8000:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
2022-04-12T16:07:54.751+0300 7f2a07884700 4 meta sync: cr:s=0x559e571e3900:op=0x559e59e05e00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
2022-04-12T16:07:54.751+0300 7f2a07884700 4 meta sync: cr:s=0x559e5988bb80:op=0x559e59e0a800:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
2022-04-12T16:07:54.751+0300 7f2a07884700 4 meta sync: cr:s=0x559e55a08780:op=0x559e59e15400:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
2022-04-12T16:07:54.751+0300 7f2a07884700 4 meta sync: cr:s=0x559e5988b7c0:op=0x559e59e0c000:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
2022-04-12T16:07:54.751+0300 7f2a07884700 4 meta sync: cr:s=0x559e595b1b80:op=0x559e59dfc000:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
2022-04-12T16:07:54.751+0300 7f2a07884700 4 meta sync: cr:s=0x559e59bc1540:op=0x559e59e10a00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
2022-04-12T16:07:54.751+0300 7f2a07884700 4 meta sync: cr:s=0x559e59bc0780:op=0x559e59e11e00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
2022-04-12T16:07:54.751+0300 7f2a07884700 4 meta sync: cr:s=0x559e59bcc000:op=0x559e59e13200:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
2022-04-12T16:07:54.755+0300 7f2a07884700 4 meta sync: cr:s=0x559e59bc1b80:op=0x559e59e10000:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
2022-04-12T16:07:54.755+0300 7f2a07884700 4 meta sync: cr:s=0x559e55a08280:op=0x559e59e15e00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
2022-04-12T16:07:54.755+0300 7f2a07884700 4 meta sync: cr:s=0x559e59bcc500:op=0x559e59e12800:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
2022-04-12T16:07:54.803+0300 7f2a07884700 4 meta sync: cr:s=0x559e54a72640:op=0x559e55789e00:18RGWMetaSyncShardCR: adjusting marker pos=user:dashboard
2022-04-12T16:07:54.811+0300 7f2a07884700 4 meta sync: cr:s=0x559e54a72640:op=0x559e55789e00:18RGWMetaSyncShardCR: adjusting marker pos=user:testuser
2022-04-12T16:07:54.835+0300 7f2a07884700 4 meta sync: cr:s=0x559e54a72640:op=0x559e55789e00:18RGWMetaSyncShardCR: adjusting marker pos=user:testuser
2022-04-12T16:07:54.835+0300 7f2a07884700 4 meta sync: cr:s=0x559e54a72640:op=0x559e55789e00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
2022-04-12T16:08:11.211+0300 7f2a0e091700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start
2022-04-12T16:08:23.959+0300 7f29f505f700 1 ====== starting new request req=0x7f2a3b313620 =====
2022-04-12T16:08:23.959+0300 7f29f505f700 2 req 8920709381089314344 0.000000000s initializing for trans_id = tx000007bccb9eb0c52c628-00625579c7-1659-sync
2022-04-12T16:08:23.959+0300 7f29f505f700 2 req 8920709381089314344 0.000000000s getting op 4
2022-04-12T16:08:23.959+0300 7f29f505f700 2 req 8920709381089314344 0.000000000s :post_period verifying requester
2022-04-12T16:08:23.959+0300 7f29f505f700 2 req 8920709381089314344 0.000000000s :post_period normalizing buckets and tenants
2022-04-12T16:08:23.959+0300 7f29f505f700 2 req 8920709381089314344 0.000000000s :post_period init permissions
2022-04-12T16:08:23.959+0300 7f29f505f700 2 req 8920709381089314344 0.000000000s :post_period recalculating target
2022-04-12T16:08:23.959+0300 7f29f505f700 2 req 8920709381089314344 0.000000000s :post_period reading permissions
2022-04-12T16:08:23.959+0300 7f29f505f700 2 req 8920709381089314344 0.000000000s :post_period init op
2022-04-12T16:08:23.959+0300 7f29f505f700 2 req 8920709381089314344 0.000000000s :post_period verifying op mask
2022-04-12T16:08:23.959+0300 7f29f505f700 2 req 8920709381089314344 0.000000000s :post_period verifying op permissions
2022-04-12T16:08:23.959+0300 7f29f505f700 2 overriding permissions due to system operation
2022-04-12T16:08:23.959+0300 7f29f505f700 2 req 8920709381089314344 0.000000000s :post_period verifying op params
2022-04-12T16:08:23.959+0300 7f29f505f700 2 req 8920709381089314344 0.000000000s :post_period pre-executing
2022-04-12T16:08:23.959+0300 7f29f505f700 2 req 8920709381089314344 0.000000000s :post_period executing
2022-04-12T16:08:23.959+0300 7f29cc80e700 1 ====== starting new request req=0x7f2a3b415620 =====
2022-04-12T16:08:23.959+0300 7f29cc80e700 2 req 8609644640538370824 0.000000000s initializing for trans_id = tx00000777b9a2f49a2a308-00625579c7-1659-sync
2022-04-12T16:08:23.959+0300 7f29cc80e700 2 req 8609644640538370824 0.000000000s getting op 4
2022-04-12T16:08:23.959+0300 7f29cc80e700 2 req 8609644640538370824 0.000000000s :post_period verifying requester
2022-04-12T16:08:23.959+0300 7f29cc80e700 2 req 8609644640538370824 0.000000000s :post_period normalizing buckets and tenants
2022-04-12T16:08:23.959+0300 7f29cc80e700 2 req 8609644640538370824 0.000000000s :post_period init permissions
2022-04-12T16:08:23.959+0300 7f29cc80e700 2 req 8609644640538370824 0.000000000s :post_period recalculating target
2022-04-12T16:08:23.959+0300 7f29cc80e700 2 req 8609644640538370824 0.000000000s :post_period reading permissions
2022-04-12T16:08:23.959+0300 7f29cc80e700 2 req 8609644640538370824 0.000000000s :post_period init op
2022-04-12T16:08:23.959+0300 7f29cc80e700 2 req 8609644640538370824 0.000000000s :post_period verifying op mask
2022-04-12T16:08:23.959+0300 7f29cc80e700 2 req 8609644640538370824 0.000000000s :post_period verifying op permissions
2022-04-12T16:08:23.959+0300 7f29cc80e700 2 overriding permissions due to system operation
2022-04-12T16:08:23.959+0300 7f29cc80e700 2 req 8609644640538370824 0.000000000s :post_period verifying op params
2022-04-12T16:08:23.959+0300 7f29cc80e700 2 req 8609644640538370824 0.000000000s :post_period pre-executing
2022-04-12T16:08:23.959+0300 7f29cc80e700 2 req 8609644640538370824 0.000000000s :post_period executing
2022-04-12T16:08:23.959+0300 7f29c8005700 1 ====== starting new request req=0x7f2a3b394620 =====
2022-04-12T16:08:23.959+0300 7f29c8005700 2 req 14231248929957790225 0.000000000s initializing for trans_id = tx00000c57f8d805c4ac211-00625579c7-1659-sync
2022-04-12T16:08:23.959+0300 7f29c8005700 2 req 14231248929957790225 0.000000000s getting op 4
2022-04-12T16:08:23.959+0300 7f29c8005700 2 req 14231248929957790225 0.000000000s :post_period verifying requester
2022-04-12T16:08:23.959+0300 7f29c8005700 2 req 14231248929957790225 0.000000000s :post_period normalizing buckets and tenants
2022-04-12T16:08:23.959+0300 7f29c8005700 2 req 14231248929957790225 0.000000000s :post_period init permissions
2022-04-12T16:08:23.959+0300 7f29c8005700 2 req 14231248929957790225 0.000000000s :post_period recalculating target
2022-04-12T16:08:23.959+0300 7f29c8005700 2 req 14231248929957790225 0.000000000s :post_period reading permissions
2022-04-12T16:08:23.959+0300 7f29c8005700 2 req 14231248929957790225 0.000000000s :post_period init op
2022-04-12T16:08:23.959+0300 7f29c8005700 2 req 14231248929957790225 0.000000000s :post_period verifying op mask
2022-04-12T16:08:23.959+0300 7f29c8005700 2 req 14231248929957790225 0.000000000s :post_period verifying op permissions
2022-04-12T16:08:23.959+0300 7f29c8005700 2 overriding permissions due to system operation
2022-04-12T16:08:23.959+0300 7f29c8005700 2 req 14231248929957790225 0.000000000s :post_period verifying op params
2022-04-12T16:08:23.959+0300 7f29c8005700 2 req 14231248929957790225 0.000000000s :post_period pre-executing
2022-04-12T16:08:23.959+0300 7f29c8005700 2 req 14231248929957790225 0.000000000s :post_period executing
2022-04-12T16:08:23.963+0300 7f29ac7ce700 4 req 8920709381089314344 0.004000003s :post_period already have epoch >= 3 for period 8b4c894f-cf43-4c56-9af0-1c2b54c74991
2022-04-12T16:08:23.963+0300 7f29ac7ce700 2 req 8920709381089314344 0.004000003s :post_period completing
2022-04-12T16:08:23.963+0300 7f29ac7ce700 2 req 8920709381089314344 0.004000003s :post_period op status=0
2022-04-12T16:08:23.963+0300 7f29ac7ce700 2 req 8920709381089314344 0.004000003s :post_period http status=200
2022-04-12T16:08:23.963+0300 7f29ac7ce700 1 ====== req done req=0x7f2a3b313620 op status=0 http_status=200 latency=0.004000003s ======
2022-04-12T16:08:23.963+0300 7f29ac7ce700 1 beast: 0x7f2a3b313620: 192.168.166.84 - rgw_zone_user [12/Apr/2022:16:08:23.959 +0300] "POST /admin/realm/period?period=8b4c894f-cf43-4c56-9af0-1c2b54c74991&epoch=3&rgwx-zonegroup=784a05a5-2089-4564-ab1b-a55c43c5f401 HTTP/1.1" 200 1868 - - - latency=0.004000003s
2022-04-12T16:08:23.963+0300 7f29a57c0700 4 req 14231248929957790225 0.004000003s :post_period already have epoch >= 3 for period 8b4c894f-cf43-4c56-9af0-1c2b54c74991
2022-04-12T16:08:23.963+0300 7f29a57c0700 2 req 14231248929957790225 0.004000003s :post_period completing
2022-04-12T16:08:23.963+0300 7f29a57c0700 2 req 14231248929957790225 0.004000003s :post_period op status=0
2022-04-12T16:08:23.963+0300 7f29a57c0700 2 req 14231248929957790225 0.004000003s :post_period http status=200
2022-04-12T16:08:23.963+0300 7f29a57c0700 1 ====== req done req=0x7f2a3b394620 op status=0 http_status=200 latency=0.004000003s ======
2022-04-12T16:08:23.963+0300 7f29a57c0700 1 beast: 0x7f2a3b394620: 192.168.166.85 - rgw_zone_user [12/Apr/2022:16:08:23.959 +0300] "POST /admin/realm/period?period=8b4c894f-cf43-4c56-9af0-1c2b54c74991&epoch=3&rgwx-zonegroup=784a05a5-2089-4564-ab1b-a55c43c5f401 HTTP/1.1" 200 1868 - - - latency=0.004000003s
2022-04-12T16:08:23.967+0300 7f299e7b2700 4 req 8609644640538370824 0.008000007s :post_period already have epoch >= 3 for period 8b4c894f-cf43-4c56-9af0-1c2b54c74991
2022-04-12T16:08:23.967+0300 7f299e7b2700 2 req 8609644640538370824 0.008000007s :post_period completing
2022-04-12T16:08:23.967+0300 7f299e7b2700 2 req 8609644640538370824 0.008000007s :post_period op status=0
2022-04-12T16:08:23.967+0300 7f299e7b2700 2 req 8609644640538370824 0.008000007s :post_period http status=200
2022-04-12T16:08:23.967+0300 7f299e7b2700 1 ====== req done req=0x7f2a3b415620 op status=0 http_status=200 latency=0.008000007s ======
2022-04-12T16:08:23.967+0300 7f299e7b2700 1 beast: 0x7f2a3b415620: 192.168.166.83 - rgw_zone_user [12/Apr/2022:16:08:23.959 +0300] "POST /admin/realm/period?period=8b4c894f-cf43-4c56-9af0-1c2b54c74991&epoch=3&rgwx-zonegroup=784a05a5-2089-4564-ab1b-a55c43c5f401 HTTP/1.1" 200 1868 - - - latency=0.008000007s
2022-04-12T16:08:33.211+0300 7f2a0e091700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start
2022-04-12T16:08:52.691+0300 7f29b5fe1700 1 ====== starting new request req=0x7f28f565f620 =====
2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s initializing for trans_id = tx00000ecd2ee05e4c9a4ca-00625579e4-1659-sync
2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s getting op 4
2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s :mdlog_notify verifying requester
2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s :mdlog_notify normalizing buckets and tenants
2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s :mdlog_notify init permissions
2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s :mdlog_notify recalculating target
2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s :mdlog_notify reading permissions
2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s :mdlog_notify init op
2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s :mdlog_notify verifying op mask
2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s :mdlog_notify verifying op permissions
2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 overriding permissions due to system operation
2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s :mdlog_notify verifying op params
2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s :mdlog_notify pre-executing
2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s :mdlog_notify executing
2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s :mdlog_notify completing
2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s :mdlog_notify op status=0
2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s :mdlog_notify http status=200
2022-04-12T16:08:52.691+0300 7f29b5fe1700 1 ====== req done req=0x7f28f565f620 op status=0 http_status=200 latency=0.000000000s ======
2022-04-12T16:08:52.691+0300 7f29b5fe1700 1 beast: 0x7f28f565f620: 192.168.166.85 - rgw_zone_user [12/Apr/2022:16:08:52.691 +0300] "POST /admin/log?type=metadata&notify&rgwx-zonegroup=784a05a5-2089-4564-ab1b-a55c43c5f401 HTTP/1.1" 200 0 - - - latency=0.000000000s
2022-04-12T16:08:52.699+0300 7f2a07884700 4 meta sync: cr:s=0x559e57c2af00:op=0x559e59e17200:18RGWMetaSyncShardCR: adjusting marker pos=
2022-04-12T16:08:54.783+0300 7f2a07884700 4 meta sync: cr:s=0x559e57c2af00:op=0x559e59e17200:18RGWMetaSyncShardCR: adjusting marker pos=1_1649768932.547706_90.1
2022-04-12T16:08:54.783+0300 7f2a07884700 4 meta sync: cr:s=0x559e57c2af00:op=0x559e59e17200:18RGWMetaSyncShardCR: adjusting marker pos=1_1649768932.547706_90.1
2022-04-12T16:08:55.215+0300 7f2a0e091700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start
2022-04-12T16:09:02.747+0300 7f299a7aa700 1 ====== starting new request req=0x7f28f55de620 =====
2022-04-12T16:09:02.747+0300 7f299a7aa700 2 req 6236571265947506715 0.000000000s initializing for trans_id = tx00000568cc0860b35981b-00625579ee-1659-sync
2022-04-12T16:09:02.747+0300 7f299a7aa700 2 req 6236571265947506715 0.000000000s getting op 4
2022-04-12T16:09:02.747+0300 7f299a7aa700 2 req 6236571265947506715 0.000000000s :datalog_notify verifying requester
2022-04-12T16:09:02.751+0300 7f299a7aa700 2 req 6236571265947506715 0.004000004s :datalog_notify normalizing buckets and tenants
2022-04-12T16:09:02.751+0300 7f299a7aa700 2 req 6236571265947506715 0.004000004s :datalog_notify init permissions
2022-04-12T16:09:02.751+0300 7f299a7aa700 2 req 6236571265947506715 0.004000004s :datalog_notify recalculating target
2022-04-12T16:09:02.751+0300 7f299a7aa700 2 req 6236571265947506715 0.004000004s :datalog_notify reading permissions
2022-04-12T16:09:02.751+0300 7f299a7aa700 2 req 6236571265947506715 0.004000004s :datalog_notify init op
2022-04-12T16:09:02.751+0300 7f299a7aa700 2 req 6236571265947506715 0.004000004s :datalog_notify verifying op mask
2022-04-12T16:09:02.751+0300 7f299a7aa700 2 req 6236571265947506715 0.004000004s :datalog_notify verifying op permissions
2022-04-12T16:09:02.751+0300 7f299a7aa700 2 overriding permissions due to system operation
2022-04-12T16:09:02.751+0300 7f299a7aa700 2 req 6236571265947506715 0.004000004s :datalog_notify verifying op params
2022-04-12T16:09:02.751+0300 7f299a7aa700 2 req 6236571265947506715 0.004000004s :datalog_notify pre-executing
2022-04-12T16:09:02.751+0300 7f299a7aa700 2 req 6236571265947506715 0.004000004s :datalog_notify executing
2022-04-12T16:09:02.751+0300 7f299a7aa700 2 req 6236571265947506715 0.004000004s :datalog_notify completing
2022-04-12T16:09:02.751+0300 7f299a7aa700 2 req 6236571265947506715 0.004000004s :datalog_notify op status=0
2022-04-12T16:09:02.751+0300 7f299a7aa700 2 req 6236571265947506715 0.004000004s :datalog_notify http status=200
2022-04-12T16:09:02.751+0300 7f299a7aa700 1 ====== req done req=0x7f28f55de620 op status=0 http_status=200 latency=0.004000004s ======
2022-04-12T16:09:02.751+0300 7f299a7aa700 1 beast: 0x7f28f55de620: 192.168.166.83 - rgw_zone_user [12/Apr/2022:16:09:02.747 +0300] "POST /admin/log?type=data&notify&source-zone=6fa70b8c-6324-4bee-8dbf-d323e8dc5f43&rgwx-zonegroup=784a05a5-2089-4564-ab1b-a55c43c5f401 HTTP/1.1" 200 0 - - - latency=0.004000004s
2022-04-12T16:09:02.751+0300 7f2a06882700 4 data sync: starting sync on testbucket:6fa70b8c-6324-4bee-8dbf-d323e8dc5f43.5105.1:3 key=testbucket:6fa70b8c-6324-4bee-8dbf-d323e8dc5f43.5105.1:3
2022-04-12T16:09:02.759+0300 7f2a06882700 5 RGW-SYNC:data:sync:shard[30]:entry:bucket_sync_sources[target=:[]):source_bucket=:[]):source_zone=6fa70b8c-6324-4bee-8dbf-d323e8dc5f43]:bucket[testbucket:6fa70b8c-6324-4bee-8dbf-d323e8dc5f43.5105.1:3<-testbucket:6fa70b8c-6324-4bee-8dbf-d323e8dc5f43.5105.1:3]:full_sync[testbucket:6fa70b8c-6324-4bee-8dbf-d323e8dc5f43.5105.1:3]:entry[amazon_case.png]: bucket sync: sync obj: 6fa70b8c-6324-4bee-8dbf-d323e8dc5f43/:testbucket[6fa70b8c-6324-4bee-8dbf-d323e8dc5f43.5105.1])/amazon_case.png[0]
2022-04-12T16:09:02.759+0300 7f2a06882700 0 e21904192857b447: sync_object: b=:testbucket[6fa70b8c-6324-4bee-8dbf-d323e8dc5f43.5105.1]) k=amazon_case.png versioned_epoch=0
2022-04-12T16:09:02.767+0300 7f2a06882700 4 rgw rados thread: AWS: download begin: z=6fa70b8c-6324-4bee-8dbf-d323e8dc5f43 b=:testbucket[6fa70b8c-6324-4bee-8dbf-d323e8dc5f43.5105.1]) k=amazon_case.png size=1880884 mtime=2022-04-12T16:09:02.643682+0300 etag=8cd436573b0ee6e9cc4f6f2cb3ba0cf1 zone_short_id=2425654257 pg_ver=0
2022-04-12T16:09:02.767+0300 7f2a06882700 0 AWS: creating bucket rgw-cdn-e21904192857b447
terminate called after throwing an instance of 'std::length_error'
what(): basic_string::_M_create
*** Caught signal (Aborted) **
in thread 7f2a06882700 thread_name:data-sync
ceph version 16.2.7 (dd0603118f56ab514f133c8d2e3adfc983942503) pacific (stable)
1: /lib64/libpthread.so.0(+0x12c20) [0x7f2a2fa66c20]
2: gsignal()
3: abort()
4: /lib64/libstdc++.so.6(+0x9009b) [0x7f2a2ea5a09b]
5: /lib64/libstdc++.so.6(+0x9653c) [0x7f2a2ea6053c]
6: /lib64/libstdc++.so.6(+0x96597) [0x7f2a2ea60597]
7: /lib64/libstdc++.so.6(+0x967f8) [0x7f2a2ea607f8]
8: /lib64/libstdc++.so.6(+0x92025) [0x7f2a2ea5c025]
9: /lib64/libradosgw.so.2(+0x48b109) [0x7f2a3a5a1109]
10: (std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::_M_mutate(unsigned long, unsigned long, char const*, unsigned long)+0x6d) [0x7f2a3a5a471d]
11: (std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::_M_append(char const*, unsigned long)+0x6b) [0x7f2a3a5a489b]
12: (rgw_obj_key::get_oid[abi:cxx11]() const+0xc6) [0x7f2a3a7c9066]
13: (RGWRESTStreamS3PutObj::send_init(rgw::sal::RGWObject*)+0xb3) [0x7f2a3aa8f4b3]
14: (RGWRESTConn::put_obj_send_init(rgw::sal::RGWObject*, rgw_http_param_pair const*, RGWRESTStreamS3PutObj**)+0x5c1) [0x7f2a3aa99891]
15: (RGWAWSStreamPutCRF::init()+0xf1) [0x7f2a3a918ec1]
16: (RGWStreamSpliceCR::operate(DoutPrefixProvider const*)+0xa2f) [0x7f2a3a9b567f]
17: (RGWCoroutinesStack::operate(DoutPrefixProvider const*, RGWCoroutinesEnv*)+0x15c) [0x7f2a3a9a0cbc]
18: (RGWCoroutinesManager::run(DoutPrefixProvider const*, std::__cxx11::list<RGWCoroutinesStack*, std::allocator<RGWCoroutinesStack*> >&)+0x296) [0x7f2a3a9a1b06]
19: (RGWCoroutinesManager::run(DoutPrefixProvider const*, RGWCoroutine*)+0x91) [0x7f2a3a9a2dd1]
20: (RGWRemoteDataLog::run_sync(DoutPrefixProvider const*, int)+0x1b4) [0x7f2a3a8b84b4]
21: (RGWDataSyncProcessorThread::process(DoutPrefixProvider const*)+0x59) [0x7f2a3aa6fe09]
22: (RGWRadosThread::Worker::entry()+0x13a) [0x7f2a3aa3368a]
23: /lib64/libpthread.so.0(+0x817a) [0x7f2a2fa5c17a]
24: clone()
2022-04-12T16:09:04.839+0300 7f2a06882700 -1 *** Caught signal (Aborted) **
in thread 7f2a06882700 thread_name:data-sync

ceph version 16.2.7 (dd0603118f56ab514f133c8d2e3adfc983942503) pacific (stable)
1: /lib64/libpthread.so.0(+0x12c20) [0x7f2a2fa66c20]
2: gsignal()
3: abort()
4: /lib64/libstdc++.so.6(+0x9009b) [0x7f2a2ea5a09b]
5: /lib64/libstdc++.so.6(+0x9653c) [0x7f2a2ea6053c]
6: /lib64/libstdc++.so.6(+0x96597) [0x7f2a2ea60597]
7: /lib64/libstdc++.so.6(+0x967f8) [0x7f2a2ea607f8]
8: /lib64/libstdc++.so.6(+0x92025) [0x7f2a2ea5c025]
9: /lib64/libradosgw.so.2(+0x48b109) [0x7f2a3a5a1109]
10: (std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::_M_mutate(unsigned long, unsigned long, char const*, unsigned long)+0x6d) [0x7f2a3a5a471d]
11: (std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::_M_append(char const*, unsigned long)+0x6b) [0x7f2a3a5a489b]
12: (rgw_obj_key::get_oid[abi:cxx11]() const+0xc6) [0x7f2a3a7c9066]
13: (RGWRESTStreamS3PutObj::send_init(rgw::sal::RGWObject*)+0xb3) [0x7f2a3aa8f4b3]
14: (RGWRESTConn::put_obj_send_init(rgw::sal::RGWObject*, rgw_http_param_pair const*, RGWRESTStreamS3PutObj**)+0x5c1) [0x7f2a3aa99891]
15: (RGWAWSStreamPutCRF::init()+0xf1) [0x7f2a3a918ec1]
16: (RGWStreamSpliceCR::operate(DoutPrefixProvider const*)+0xa2f) [0x7f2a3a9b567f]
17: (RGWCoroutinesStack::operate(DoutPrefixProvider const*, RGWCoroutinesEnv*)+0x15c) [0x7f2a3a9a0cbc]
18: (RGWCoroutinesManager::run(DoutPrefixProvider const*, std::__cxx11::list<RGWCoroutinesStack*, std::allocator<RGWCoroutinesStack*> >&)+0x296) [0x7f2a3a9a1b06]
19: (RGWCoroutinesManager::run(DoutPrefixProvider const*, RGWCoroutine*)+0x91) [0x7f2a3a9a2dd1]
20: (RGWRemoteDataLog::run_sync(DoutPrefixProvider const*, int)+0x1b4) [0x7f2a3a8b84b4]
21: (RGWDataSyncProcessorThread::process(DoutPrefixProvider const*)+0x59) [0x7f2a3aa6fe09]
22: (RGWRadosThread::Worker::entry()+0x13a) [0x7f2a3aa3368a]
23: /lib64/libpthread.so.0(+0x817a) [0x7f2a2fa5c17a]
24: clone()
NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.

--- begin dump of recent events ---
-1216> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command assert hook 0x559e54a2a760
-1215> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command abort hook 0x559e54a2a760
-1214> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command leak_some_memory hook 0x559e54a2a760
-1213> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command perfcounters_dump hook 0x559e54a2a760
-1212> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command 1 hook 0x559e54a2a760
-1211> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command perf dump hook 0x559e54a2a760
-1210> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command perfcounters_schema hook 0x559e54a2a760
-1209> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command perf histogram dump hook 0x559e54a2a760
-1208> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command 2 hook 0x559e54a2a760
-1207> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command perf schema hook 0x559e54a2a760
-1206> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command perf histogram schema hook 0x559e54a2a760
-1205> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command perf reset hook 0x559e54a2a760
-1204> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command config show hook 0x559e54a2a760
-1203> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command config help hook 0x559e54a2a760
-1202> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command config set hook 0x559e54a2a760
-1201> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command config unset hook 0x559e54a2a760
-1200> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command config get hook 0x559e54a2a760
-1199> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command config diff hook 0x559e54a2a760
-1198> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command config diff get hook 0x559e54a2a760
-1197> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command injectargs hook 0x559e54a2a760
-1196> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command log flush hook 0x559e54a2a760
-1195> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command log dump hook 0x559e54a2a760
-1194> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command log reopen hook 0x559e54a2a760
-1193> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command dump_mempools hook 0x559e54b02068
-1192> 2022-04-12T16:07:44.643+0300 7f2a3b460400 10 monclient: get_monmap_and_config
-1191> 2022-04-12T16:07:44.643+0300 7f2a3b460400 10 monclient: build_initial_monmap
-1190> 2022-04-12T16:07:44.643+0300 7f2a3b460400 10 monclient: monmap:
epoch 0
fsid 65ccd25a-39ce-4072-a5c6-a0e6654bda87
last_changed 2022-04-12T16:07:44.650033+0300
created 2022-04-12T16:07:44.650033+0300
min_mon_release 0 (unknown)
election_strategy: 1
0: [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] mon.noname-a
1: [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] mon.noname-b
2: [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] mon.noname-c

-1189> 2022-04-12T16:07:44.643+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding auth protocol: cephx
-1188> 2022-04-12T16:07:44.643+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding auth protocol: cephx
-1187> 2022-04-12T16:07:44.643+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding auth protocol: cephx
-1186> 2022-04-12T16:07:44.643+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding auth protocol: none
-1185> 2022-04-12T16:07:44.643+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: secure
-1184> 2022-04-12T16:07:44.643+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: crc
-1183> 2022-04-12T16:07:44.643+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: secure
-1182> 2022-04-12T16:07:44.643+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: crc
-1181> 2022-04-12T16:07:44.643+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: secure
-1180> 2022-04-12T16:07:44.643+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: crc
-1179> 2022-04-12T16:07:44.643+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: crc
-1178> 2022-04-12T16:07:44.643+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: secure
-1177> 2022-04-12T16:07:44.643+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: crc
-1176> 2022-04-12T16:07:44.643+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: secure
-1175> 2022-04-12T16:07:44.643+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: crc
-1174> 2022-04-12T16:07:44.643+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: secure
-1173> 2022-04-12T16:07:44.647+0300 7f2a3b460400 2 auth: KeyRing::load: loaded key file /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring
-1172> 2022-04-12T16:07:44.647+0300 7f2a3b460400 10 monclient: init
-1171> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d65c0) adding auth protocol: cephx
-1170> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d65c0) adding auth protocol: cephx
-1169> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d65c0) adding auth protocol: cephx
-1168> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d65c0) adding auth protocol: none
-1167> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d65c0) adding con mode: secure
-1166> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d65c0) adding con mode: crc
-1165> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d65c0) adding con mode: secure
-1164> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d65c0) adding con mode: crc
-1163> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d65c0) adding con mode: secure
-1162> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d65c0) adding con mode: crc
-1161> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d65c0) adding con mode: crc
-1160> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d65c0) adding con mode: secure
-1159> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d65c0) adding con mode: crc
-1158> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d65c0) adding con mode: secure
-1157> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d65c0) adding con mode: crc
-1156> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d65c0) adding con mode: secure
-1155> 2022-04-12T16:07:44.647+0300 7f2a3b460400 2 auth: KeyRing::load: loaded key file /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring
-1154> 2022-04-12T16:07:44.647+0300 7f2a3b460400 2 auth: KeyRing::load: loaded key file /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring
-1153> 2022-04-12T16:07:44.647+0300 7f2a3b460400 10 monclient: _reopen_session rank -1
-1152> 2022-04-12T16:07:44.647+0300 7f2a3b460400 10 monclient: _add_conns ranks=[1,2,0]
-1151> 2022-04-12T16:07:44.647+0300 7f2a3b460400 10 monclient(hunting): picked mon.noname-b con 0x559e54b38800 addr [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0]
-1150> 2022-04-12T16:07:44.647+0300 7f2a3b460400 10 monclient(hunting): picked mon.noname-c con 0x559e54b38c00 addr [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0]
-1149> 2022-04-12T16:07:44.647+0300 7f2a3b460400 10 monclient(hunting): picked mon.noname-a con 0x559e54b39000 addr [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0]
-1148> 2022-04-12T16:07:44.647+0300 7f2a3b460400 10 monclient(hunting): start opening mon connection
-1147> 2022-04-12T16:07:44.647+0300 7f2a3b460400 10 monclient(hunting): start opening mon connection
-1146> 2022-04-12T16:07:44.647+0300 7f2a3b460400 10 monclient(hunting): start opening mon connection
-1145> 2022-04-12T16:07:44.647+0300 7f2a3b460400 10 monclient(hunting): _renew_subs
-1144> 2022-04-12T16:07:44.647+0300 7f2a3b460400 10 monclient(hunting): authenticate will time out at 2022-04-12T16:12:44.651302+0300
-1143> 2022-04-12T16:07:44.647+0300 7f2a270c3700 10 monclient(hunting): get_auth_request con 0x559e54b39000 auth_method 0
-1142> 2022-04-12T16:07:44.647+0300 7f2a270c3700 10 monclient(hunting): get_auth_request method 2 preferred_modes [2,1]
-1141> 2022-04-12T16:07:44.647+0300 7f2a270c3700 10 monclient(hunting): _init_auth method 2
-1140> 2022-04-12T16:07:44.647+0300 7f2a270c3700 10 monclient(hunting): _init_auth creating new auth
-1139> 2022-04-12T16:07:44.647+0300 7f2a270c3700 10 monclient(hunting): handle_auth_reply_more payload 9
-1138> 2022-04-12T16:07:44.647+0300 7f2a270c3700 10 monclient(hunting): handle_auth_reply_more payload_len 9
-1137> 2022-04-12T16:07:44.647+0300 7f2a270c3700 10 monclient(hunting): handle_auth_reply_more responding with 36 bytes
-1136> 2022-04-12T16:07:44.647+0300 7f2a270c3700 10 monclient(hunting): handle_auth_done global_id 5715 payload 322
-1135> 2022-04-12T16:07:44.647+0300 7f2a270c3700 10 monclient: _finish_hunting 0
-1134> 2022-04-12T16:07:44.647+0300 7f2a270c3700 1 monclient: found mon.noname-a
-1133> 2022-04-12T16:07:44.647+0300 7f2a270c3700 10 monclient: _send_mon_message to mon.noname-a at v2:192.168.166.83:3300/0
-1132> 2022-04-12T16:07:44.647+0300 7f2a258c0700 10 monclient: handle_monmap mon_map magic: 0 v1
-1131> 2022-04-12T16:07:44.647+0300 7f2a258c0700 10 monclient: got monmap 1 from mon.noname-a (according to old e1)
-1130> 2022-04-12T16:07:44.647+0300 7f2a258c0700 10 monclient: dump:
epoch 1
fsid 65ccd25a-39ce-4072-a5c6-a0e6654bda87
last_changed 2022-04-12T15:48:27.663651+0300
created 2022-04-12T15:48:27.663651+0300
min_mon_release 16 (pacific)
election_strategy: 1
0: [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] mon.ceph-ss-test-oct-controller-av-01
1: [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] mon.ceph-ss-test-oct-controller-av-02
2: [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] mon.ceph-ss-test-oct-controller-av-03

-1129> 2022-04-12T16:07:44.647+0300 7f2a258c0700 10 monclient: _finish_auth 0
-1128> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 monclient: authenticate success, global_id 5715
-1127> 2022-04-12T16:07:44.647+0300 7f2a258c0700 10 monclient: handle_config config(0 keys) v1
-1126> 2022-04-12T16:07:44.647+0300 7f2a3b460400 10 monclient: get_monmap_and_config success
-1125> 2022-04-12T16:07:44.647+0300 7f2a3b460400 4 set_mon_vals no callback set
-1124> 2022-04-12T16:07:44.647+0300 7f2a3b460400 10 monclient: shutdown
-1123> 2022-04-12T16:07:44.647+0300 7f2a3b460400 0 deferred set uid:gid to 167:167 (ceph:ceph)
-1122> 2022-04-12T16:07:44.647+0300 7f2a3b460400 0 ceph version 16.2.7 (dd0603118f56ab514f133c8d2e3adfc983942503) pacific (stable), process radosgw, pid 54
-1121> 2022-04-12T16:07:44.647+0300 7f2a3b460400 0 framework: beast
-1120> 2022-04-12T16:07:44.647+0300 7f2a3b460400 0 framework conf key: endpoint, val: 192.168.166.83:8081
-1119> 2022-04-12T16:07:44.647+0300 7f2a3b460400 1 radosgw_Main not setting numa affinity
-1118> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 asok(0x559e54b34000) init /var/run/ceph/av-ceph-test-client.rgw.ceph-ss-test-oct-controller-av-01.rgw1.54.94138514240960.asok
-1117> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 asok(0x559e54b34000) bind_and_listen /var/run/ceph/av-ceph-test-client.rgw.ceph-ss-test-oct-controller-av-01.rgw1.54.94138514240960.asok
-1116> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command 0 hook 0x559e54a6c360
-1115> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command version hook 0x559e54a6c360
-1114> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command git_version hook 0x559e54a6c360
-1113> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command help hook 0x559e54a2a4b0
-1112> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command get_command_descriptions hook 0x559e54a2a4c0
-1111> 2022-04-12T16:07:44.651+0300 7f2a250bf700 5 asok(0x559e54b34000) entry start
-1110> 2022-04-12T16:07:44.651+0300 7f2a3b460400 10 monclient: get_monmap_and_config
-1109> 2022-04-12T16:07:44.651+0300 7f2a3b460400 10 monclient: build_initial_monmap
-1108> 2022-04-12T16:07:44.651+0300 7f2a3b460400 10 monclient: monmap:
epoch 0
fsid 00000000-0000-0000-0000-000000000000
last_changed 0.000000
created 0.000000
min_mon_release 0 (unknown)
election_strategy: 1
0: [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] mon.noname-a
1: [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] mon.noname-b
2: [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] mon.noname-c

-1107> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding auth protocol: cephx
-1106> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding auth protocol: cephx
-1105> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding auth protocol: cephx
-1104> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding auth protocol: none
-1103> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: secure
-1102> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: crc
-1101> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: secure
-1100> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: crc
-1099> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: secure
-1098> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: crc
-1097> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: crc
-1096> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: secure
-1095> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: crc
-1094> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: secure
-1093> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: crc
-1092> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: secure
-1091> 2022-04-12T16:07:44.651+0300 7f2a3b460400 2 auth: KeyRing::load: loaded key file /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring
-1090> 2022-04-12T16:07:44.651+0300 7f2a3b460400 10 monclient: init
-1089> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d6fa0) adding auth protocol: cephx
-1088> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d6fa0) adding auth protocol: cephx
-1087> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d6fa0) adding auth protocol: cephx
-1086> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d6fa0) adding auth protocol: none
-1085> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d6fa0) adding con mode: secure
-1084> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d6fa0) adding con mode: crc
-1083> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d6fa0) adding con mode: secure
-1082> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d6fa0) adding con mode: crc
-1081> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d6fa0) adding con mode: secure
-1080> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d6fa0) adding con mode: crc
-1079> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d6fa0) adding con mode: crc
-1078> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d6fa0) adding con mode: secure
-1077> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d6fa0) adding con mode: crc
-1076> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d6fa0) adding con mode: secure
-1075> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d6fa0) adding con mode: crc
-1074> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d6fa0) adding con mode: secure
-1073> 2022-04-12T16:07:44.651+0300 7f2a3b460400 2 auth: KeyRing::load: loaded key file /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring
-1072> 2022-04-12T16:07:44.651+0300 7f2a3b460400 2 auth: KeyRing::load: loaded key file /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring
-1071> 2022-04-12T16:07:44.651+0300 7f2a3b460400 10 monclient: _reopen_session rank -1
-1070> 2022-04-12T16:07:44.651+0300 7f2a3b460400 10 monclient: _add_conns ranks=[0,1,2]
-1069> 2022-04-12T16:07:44.651+0300 7f2a3b460400 10 monclient(hunting): picked mon.noname-a con 0x559e54b39000 addr [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0]
-1068> 2022-04-12T16:07:44.651+0300 7f2a3b460400 10 monclient(hunting): picked mon.noname-b con 0x559e54b38c00 addr [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0]
-1067> 2022-04-12T16:07:44.651+0300 7f2a3b460400 10 monclient(hunting): picked mon.noname-c con 0x559e54b38800 addr [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0]
-1066> 2022-04-12T16:07:44.651+0300 7f2a3b460400 10 monclient(hunting): start opening mon connection
-1065> 2022-04-12T16:07:44.651+0300 7f2a3b460400 10 monclient(hunting): start opening mon connection
-1064> 2022-04-12T16:07:44.651+0300 7f2a3b460400 10 monclient(hunting): start opening mon connection
-1063> 2022-04-12T16:07:44.651+0300 7f2a3b460400 10 monclient(hunting): _renew_subs
-1062> 2022-04-12T16:07:44.651+0300 7f2a3b460400 10 monclient(hunting): authenticate will time out at 2022-04-12T16:12:44.656631+0300
-1061> 2022-04-12T16:07:44.651+0300 7f2a268c2700 10 monclient(hunting): get_auth_request con 0x559e54b39000 auth_method 0
-1060> 2022-04-12T16:07:44.651+0300 7f2a268c2700 10 monclient(hunting): get_auth_request method 2 preferred_modes [2,1]
-1059> 2022-04-12T16:07:44.651+0300 7f2a268c2700 10 monclient(hunting): _init_auth method 2
-1058> 2022-04-12T16:07:44.651+0300 7f2a268c2700 10 monclient(hunting): _init_auth creating new auth
-1057> 2022-04-12T16:07:44.651+0300 7f2a268c2700 10 monclient(hunting): handle_auth_reply_more payload 9
-1056> 2022-04-12T16:07:44.651+0300 7f2a268c2700 10 monclient(hunting): handle_auth_reply_more payload_len 9
-1055> 2022-04-12T16:07:44.651+0300 7f2a268c2700 10 monclient(hunting): handle_auth_reply_more responding with 36 bytes
-1054> 2022-04-12T16:07:44.651+0300 7f2a268c2700 10 monclient(hunting): handle_auth_done global_id 5718 payload 322
-1053> 2022-04-12T16:07:44.651+0300 7f2a268c2700 10 monclient: _finish_hunting 0
-1052> 2022-04-12T16:07:44.651+0300 7f2a268c2700 1 monclient: found mon.noname-a
-1051> 2022-04-12T16:07:44.651+0300 7f2a268c2700 10 monclient: _send_mon_message to mon.noname-a at v2:192.168.166.83:3300/0
-1050> 2022-04-12T16:07:44.651+0300 7f2a218b8700 10 monclient: handle_monmap mon_map magic: 0 v1
-1049> 2022-04-12T16:07:44.651+0300 7f2a218b8700 10 monclient: got monmap 1 from mon.noname-a (according to old e1)
-1048> 2022-04-12T16:07:44.651+0300 7f2a218b8700 10 monclient: dump:
epoch 1
fsid 65ccd25a-39ce-4072-a5c6-a0e6654bda87
last_changed 2022-04-12T15:48:27.663651+0300
created 2022-04-12T15:48:27.663651+0300
min_mon_release 16 (pacific)
election_strategy: 1
0: [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] mon.ceph-ss-test-oct-controller-av-01
1: [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] mon.ceph-ss-test-oct-controller-av-02
2: [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] mon.ceph-ss-test-oct-controller-av-03

-1047> 2022-04-12T16:07:44.651+0300 7f2a218b8700 10 monclient: _finish_auth 0
-1046> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 monclient: authenticate success, global_id 5718
-1045> 2022-04-12T16:07:44.651+0300 7f2a218b8700 10 monclient: handle_config config(0 keys) v1
-1044> 2022-04-12T16:07:44.651+0300 7f2a3b460400 10 monclient: get_monmap_and_config success
-1043> 2022-04-12T16:07:44.651+0300 7f2a3b460400 4 set_mon_vals no callback set
-1042> 2022-04-12T16:07:44.651+0300 7f2a3b460400 10 monclient: shutdown
-1041> 2022-04-12T16:07:44.655+0300 7f2a3b460400 10 monclient: build_initial_monmap
-1040> 2022-04-12T16:07:44.655+0300 7f2a3b460400 10 monclient: monmap:
epoch 0
fsid 00000000-0000-0000-0000-000000000000
last_changed 0.000000
created 0.000000
min_mon_release 0 (unknown)
election_strategy: 1
0: [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] mon.noname-a
1: [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] mon.noname-b
2: [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] mon.noname-c

-1039> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding auth protocol: cephx
-1038> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding auth protocol: cephx
-1037> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding auth protocol: cephx
-1036> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding auth protocol: none
-1035> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: secure
-1034> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: crc
-1033> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: secure
-1032> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: crc
-1031> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: secure
-1030> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: crc
-1029> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: crc
-1028> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: secure
-1027> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: crc
-1026> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: secure
-1025> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: crc
-1024> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: secure
-1023> 2022-04-12T16:07:44.655+0300 7f2a3b460400 2 auth: KeyRing::load: loaded key file /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring
-1022> 2022-04-12T16:07:44.655+0300 7f2a3b460400 1 librados: starting msgr at
-1021> 2022-04-12T16:07:44.655+0300 7f2a3b460400 1 librados: starting objecter
-1020> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command objecter_requests hook 0x559e54a2a510
-1019> 2022-04-12T16:07:44.655+0300 7f2a3b460400 1 librados: setting wanted keys
-1018> 2022-04-12T16:07:44.655+0300 7f2a3b460400 1 librados: calling monclient init
-1017> 2022-04-12T16:07:44.655+0300 7f2a3b460400 10 monclient: init
-1016> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54a700d8) adding auth protocol: cephx
-1015> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54a700d8) adding auth protocol: cephx
-1014> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54a700d8) adding auth protocol: cephx
-1013> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54a700d8) adding auth protocol: none
-1012> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54a700d8) adding con mode: secure
-1011> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54a700d8) adding con mode: crc
-1010> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54a700d8) adding con mode: secure
-1009> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54a700d8) adding con mode: crc
-1008> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54a700d8) adding con mode: secure
-1007> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54a700d8) adding con mode: crc
-1006> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54a700d8) adding con mode: crc
-1005> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54a700d8) adding con mode: secure
-1004> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54a700d8) adding con mode: crc
-1003> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54a700d8) adding con mode: secure
-1002> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54a700d8) adding con mode: crc
-1001> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54a700d8) adding con mode: secure
-1000> 2022-04-12T16:07:44.655+0300 7f2a3b460400 2 auth: KeyRing::load: loaded key file /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring
-999> 2022-04-12T16:07:44.655+0300 7f2a3b460400 2 auth: KeyRing::load: loaded key file /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring
-998> 2022-04-12T16:07:44.655+0300 7f2a3b460400 10 monclient: _reopen_session rank -1
-997> 2022-04-12T16:07:44.655+0300 7f2a3b460400 10 monclient: _add_conns ranks=[0,2,1]
-996> 2022-04-12T16:07:44.655+0300 7f2a3b460400 10 monclient(hunting): picked mon.noname-a con 0x559e54b38800 addr [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0]
-995> 2022-04-12T16:07:44.655+0300 7f2a3b460400 10 monclient(hunting): picked mon.noname-c con 0x559e54b38c00 addr [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0]
-994> 2022-04-12T16:07:44.655+0300 7f2a3b460400 10 monclient(hunting): picked mon.noname-b con 0x559e54b39000 addr [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0]
-993> 2022-04-12T16:07:44.655+0300 7f2a3b460400 10 monclient(hunting): start opening mon connection
-992> 2022-04-12T16:07:44.655+0300 7f2a3b460400 10 monclient(hunting): start opening mon connection
-991> 2022-04-12T16:07:44.655+0300 7f2a3b460400 10 monclient(hunting): start opening mon connection
-990> 2022-04-12T16:07:44.655+0300 7f2a3b460400 10 monclient(hunting): _renew_subs
-989> 2022-04-12T16:07:44.655+0300 7f2a3b460400 10 monclient(hunting): authenticate will time out at 2022-04-12T16:12:44.659081+0300
-988> 2022-04-12T16:07:44.655+0300 7f2a268c2700 10 monclient(hunting): get_auth_request con 0x559e54b38800 auth_method 0
-987> 2022-04-12T16:07:44.655+0300 7f2a268c2700 10 monclient(hunting): get_auth_request method 2 preferred_modes [2,1]
-986> 2022-04-12T16:07:44.655+0300 7f2a268c2700 10 monclient(hunting): _init_auth method 2
-985> 2022-04-12T16:07:44.655+0300 7f2a268c2700 10 monclient(hunting): _init_auth creating new auth
-984> 2022-04-12T16:07:44.655+0300 7f2a268c2700 10 monclient(hunting): handle_auth_reply_more payload 9
-983> 2022-04-12T16:07:44.655+0300 7f2a268c2700 10 monclient(hunting): handle_auth_reply_more payload_len 9
-982> 2022-04-12T16:07:44.655+0300 7f2a268c2700 10 monclient(hunting): handle_auth_reply_more responding with 36 bytes
-981> 2022-04-12T16:07:44.655+0300 7f2a268c2700 10 monclient(hunting): handle_auth_done global_id 5721 payload 1000
-980> 2022-04-12T16:07:44.655+0300 7f2a268c2700 10 monclient: _finish_hunting 0
-979> 2022-04-12T16:07:44.655+0300 7f2a268c2700 1 monclient: found mon.noname-a
-978> 2022-04-12T16:07:44.655+0300 7f2a268c2700 10 monclient: _send_mon_message to mon.noname-a at v2:192.168.166.83:3300/0
-977> 2022-04-12T16:07:44.655+0300 7f2a200b5700 10 monclient: handle_monmap mon_map magic: 0 v1
-976> 2022-04-12T16:07:44.655+0300 7f2a200b5700 10 monclient: got monmap 1 from mon.noname-a (according to old e1)
-975> 2022-04-12T16:07:44.655+0300 7f2a200b5700 10 monclient: dump:
epoch 1
fsid 65ccd25a-39ce-4072-a5c6-a0e6654bda87
last_changed 2022-04-12T15:48:27.663651+0300
created 2022-04-12T15:48:27.663651+0300
min_mon_release 16 (pacific)
election_strategy: 1
0: [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] mon.ceph-ss-test-oct-controller-av-01
1: [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] mon.ceph-ss-test-oct-controller-av-02
2: [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] mon.ceph-ss-test-oct-controller-av-03

-974> 2022-04-12T16:07:44.655+0300 7f2a200b5700 10 monclient: _finish_auth 0
-973> 2022-04-12T16:07:44.655+0300 7f2a200b5700 10 monclient: handle_config config(0 keys) v1
-972> 2022-04-12T16:07:44.655+0300 7f2a200b5700 10 monclient: handle_monmap mon_map magic: 0 v1
-971> 2022-04-12T16:07:44.655+0300 7f2a218b8700 4 set_mon_vals no callback set
-970> 2022-04-12T16:07:44.655+0300 7f2a200b5700 10 monclient: got monmap 1 from mon.ceph-ss-test-oct-controller-av-01 (according to old e1)
-969> 2022-04-12T16:07:44.655+0300 7f2a200b5700 10 monclient: dump:
epoch 1
fsid 65ccd25a-39ce-4072-a5c6-a0e6654bda87
last_changed 2022-04-12T15:48:27.663651+0300
created 2022-04-12T15:48:27.663651+0300
min_mon_release 16 (pacific)
election_strategy: 1
0: [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] mon.ceph-ss-test-oct-controller-av-01
1: [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] mon.ceph-ss-test-oct-controller-av-02
2: [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] mon.ceph-ss-test-oct-controller-av-03

-968> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 monclient: authenticate success, global_id 5721
-967> 2022-04-12T16:07:44.655+0300 7f2a3b460400 10 monclient: _renew_subs
-966> 2022-04-12T16:07:44.655+0300 7f2a3b460400 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-965> 2022-04-12T16:07:44.655+0300 7f2a3b460400 10 monclient: _renew_subs
-964> 2022-04-12T16:07:44.655+0300 7f2a3b460400 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-963> 2022-04-12T16:07:44.655+0300 7f2a3b460400 1 librados: init done
-962> 2022-04-12T16:07:44.659+0300 7f2a200b5700 4 mgrc handle_mgr_map Got map version 29
-961> 2022-04-12T16:07:44.659+0300 7f2a200b5700 4 mgrc handle_mgr_map Active mgr is now [v2:192.168.166.84:6800/56,v1:192.168.166.84:6801/56]
-960> 2022-04-12T16:07:44.659+0300 7f2a200b5700 4 mgrc reconnect Starting new session with [v2:192.168.166.84:6800/56,v1:192.168.166.84:6801/56]
-959> 2022-04-12T16:07:44.659+0300 7f2a260c1700 10 monclient: get_auth_request con 0x559e54b39800 auth_method 0
-958> 2022-04-12T16:07:44.659+0300 7f2a270c3700 10 monclient: get_auth_request con 0x559e54b38400 auth_method 0
-957> 2022-04-12T16:07:44.659+0300 7f2a268c2700 10 monclient: get_auth_request con 0x559e55914000 auth_method 0
-956> 2022-04-12T16:07:44.663+0300 7f2a260c1700 10 monclient: get_auth_request con 0x559e55914c00 auth_method 0
-955> 2022-04-12T16:07:44.663+0300 7f2a270c3700 10 monclient: get_auth_request con 0x559e55915800 auth_method 0
-954> 2022-04-12T16:07:44.667+0300 7f2a268c2700 10 monclient: get_auth_request con 0x559e55914800 auth_method 0
-953> 2022-04-12T16:07:44.667+0300 7f2a260c1700 10 monclient: get_auth_request con 0x559e55914400 auth_method 0
-952> 2022-04-12T16:07:44.671+0300 7f2a270c3700 10 monclient: get_auth_request con 0x559e55915c00 auth_method 0
-951> 2022-04-12T16:07:44.671+0300 7f2a268c2700 10 monclient: get_auth_request con 0x559e5591c000 auth_method 0
-950> 2022-04-12T16:07:44.671+0300 7f2a260c1700 10 monclient: get_auth_request con 0x559e55915400 auth_method 0
-949> 2022-04-12T16:07:44.675+0300 7f2a270c3700 10 monclient: get_auth_request con 0x559e55915000 auth_method 0
-948> 2022-04-12T16:07:44.675+0300 7f2a268c2700 10 monclient: get_auth_request con 0x559e5591c400 auth_method 0
-947> 2022-04-12T16:07:44.675+0300 7f2a260c1700 10 monclient: get_auth_request con 0x559e5591c800 auth_method 0
-946> 2022-04-12T16:07:44.679+0300 7f2a270c3700 10 monclient: get_auth_request con 0x559e5591d000 auth_method 0
-945> 2022-04-12T16:07:44.679+0300 7f2a268c2700 10 monclient: get_auth_request con 0x559e5591d400 auth_method 0
-944> 2022-04-12T16:07:44.679+0300 7f2a260c1700 10 monclient: get_auth_request con 0x559e5591cc00 auth_method 0
-943> 2022-04-12T16:07:44.683+0300 7f2a270c3700 10 monclient: get_auth_request con 0x559e5591d800 auth_method 0
-942> 2022-04-12T16:07:44.687+0300 7f2a268c2700 10 monclient: get_auth_request con 0x559e55978000 auth_method 0
-941> 2022-04-12T16:07:44.691+0300 7f2a3b460400 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-940> 2022-04-12T16:07:44.691+0300 7f2a200b5700 10 monclient: handle_get_version_reply finishing 1 version 64
-939> 2022-04-12T16:07:44.691+0300 7f2a3b460400 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-938> 2022-04-12T16:07:45.539+0300 7f2a200b5700 10 monclient: _renew_subs
-937> 2022-04-12T16:07:45.539+0300 7f2a200b5700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-936> 2022-04-12T16:07:45.539+0300 7f2a3b460400 10 start_mon_command cmd=[{"prefix": "osd pool application enable","pool": "sync.rgw.log","app": "rgw"}]
-935> 2022-04-12T16:07:45.539+0300 7f2a3b460400 10 monclient: _send_command 1 [{"prefix": "osd pool application enable","pool": "sync.rgw.log","app": "rgw"}]
-934> 2022-04-12T16:07:45.539+0300 7f2a3b460400 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-933> 2022-04-12T16:07:46.559+0300 7f2a200b5700 10 monclient: handle_mon_command_ack 1 [{"prefix": "osd pool application enable","pool": "sync.rgw.log","app": "rgw"}]
-932> 2022-04-12T16:07:46.559+0300 7f2a200b5700 10 monclient: _finish_command 1 = system:0 enabled application 'rgw' on pool 'sync.rgw.log'
-931> 2022-04-12T16:07:46.559+0300 7f2a3b460400 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-930> 2022-04-12T16:07:46.559+0300 7f2a200b5700 10 monclient: handle_get_version_reply finishing 2 version 66
-929> 2022-04-12T16:07:46.559+0300 7f2a218b8700 10 monclient: _renew_subs
-928> 2022-04-12T16:07:46.559+0300 7f2a218b8700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-927> 2022-04-12T16:07:46.835+0300 7f2a3b460400 5 sync module config (parsed representation):
{
"config": {
"source_bucket": "",
"target_path": "rgw-${zonegroup}-${sid}/${bucket}",
"connection_id": "",
"acls_id": "",
"connection": {
"id": "",
"endpoint": "https://storage.googleapis.com",
"host_style": "path",
"key": {
"access_key": "<gcs_access_key>",
"secret": "******"
}
}
},
"connections": [],
"acl_profiles": [],
"profiles": [
{
"name": "",
"config": {
"source_bucket": "",
"target_path": "rgw-${zonegroup}-${sid}/${bucket}",
"connection_id": "",
"acls_id": "",
"connection": {
"id": "",
"endpoint": "https://storage.googleapis.com",
"host_style": "path",
"key": {
"access_key": "<gcs_access_key>",
"secret": "******"
}
}
}
}
]
}

-926> 2022-04-12T16:07:46.835+0300 7f2a3b460400 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-925> 2022-04-12T16:07:46.835+0300 7f2a200b5700 10 monclient: handle_get_version_reply finishing 3 version 66
-924> 2022-04-12T16:07:46.835+0300 7f2a3b460400 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-923> 2022-04-12T16:07:47.563+0300 7f2a200b5700 10 monclient: _renew_subs
-922> 2022-04-12T16:07:47.563+0300 7f2a200b5700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-921> 2022-04-12T16:07:47.567+0300 7f2a3b460400 10 start_mon_command cmd=[{"prefix": "osd pool application enable","pool": "sync.rgw.control","app": "rgw"}]
-920> 2022-04-12T16:07:47.567+0300 7f2a3b460400 10 monclient: _send_command 2 [{"prefix": "osd pool application enable","pool": "sync.rgw.control","app": "rgw"}]
-919> 2022-04-12T16:07:47.567+0300 7f2a3b460400 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-918> 2022-04-12T16:07:47.655+0300 7f2a1f0b3700 10 monclient: tick
-917> 2022-04-12T16:07:48.575+0300 7f2a200b5700 10 monclient: handle_mon_command_ack 2 [{"prefix": "osd pool application enable","pool": "sync.rgw.control","app": "rgw"}]
-916> 2022-04-12T16:07:48.575+0300 7f2a200b5700 10 monclient: _finish_command 2 = system:0 enabled application 'rgw' on pool 'sync.rgw.control'
-915> 2022-04-12T16:07:48.575+0300 7f2a3b460400 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-914> 2022-04-12T16:07:48.579+0300 7f2a200b5700 10 monclient: handle_get_version_reply finishing 4 version 68
-913> 2022-04-12T16:07:48.579+0300 7f2a210b7700 10 monclient: _renew_subs
-912> 2022-04-12T16:07:48.579+0300 7f2a210b7700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-911> 2022-04-12T16:07:48.655+0300 7f2a1f0b3700 10 monclient: tick
-910> 2022-04-12T16:07:49.163+0300 7f2a3b460400 2 all 8 watchers are set, enabling cache
-909> 2022-04-12T16:07:49.163+0300 7f2a3b460400 5 rgw main: tl::expected<std::pair<boost::container::flat_map<long unsigned int, logback_generation>, obj_version>, boost::system::error_code> logback_generations::read(const DoutPrefixProvider*, optional_yield):409: oid=data_loggenerations_metadata not found
-908> 2022-04-12T16:07:49.167+0300 7f2a260c1700 10 monclient: get_auth_request con 0x559e55978400 auth_method 0
-907> 2022-04-12T16:07:49.211+0300 7f2a0e091700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start
-906> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d78b0) adding auth protocol: cephx
-905> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d78b0) adding auth protocol: cephx
-904> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d78b0) adding auth protocol: cephx
-903> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d78b0) adding auth protocol: none
-902> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d78b0) adding con mode: secure
-901> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d78b0) adding con mode: crc
-900> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d78b0) adding con mode: secure
-899> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d78b0) adding con mode: crc
-898> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d78b0) adding con mode: secure
-897> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d78b0) adding con mode: crc
-896> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d78b0) adding con mode: crc
-895> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d78b0) adding con mode: secure
-894> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d78b0) adding con mode: crc
-893> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d78b0) adding con mode: secure
-892> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d78b0) adding con mode: crc
-891> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d78b0) adding con mode: secure
-890> 2022-04-12T16:07:49.219+0300 7f2a3b460400 2 auth: KeyRing::load: loaded key file /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring
-889> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command cache list hook 0x559e54a2aa10
-888> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command cache inspect hook 0x559e54a2aa10
-887> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command cache erase hook 0x559e54a2aa10
-886> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command cache zap hook 0x559e54a2aa10
-885> 2022-04-12T16:07:49.219+0300 7f2a3b460400 10 monclient: get_monmap_and_config
-884> 2022-04-12T16:07:49.219+0300 7f2a3b460400 10 monclient: build_initial_monmap
-883> 2022-04-12T16:07:49.219+0300 7f2a3b460400 10 monclient: monmap:
epoch 0
fsid 00000000-0000-0000-0000-000000000000
last_changed 0.000000
created 0.000000
min_mon_release 0 (unknown)
election_strategy: 1
0: [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] mon.noname-a
1: [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] mon.noname-b
2: [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] mon.noname-c

-882> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding auth protocol: cephx
-881> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding auth protocol: cephx
-880> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding auth protocol: cephx
-879> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding auth protocol: none
-878> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: secure
-877> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: crc
-876> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: secure
-875> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: crc
-874> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: secure
-873> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: crc
-872> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: crc
-871> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: secure
-870> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: crc
-869> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: secure
-868> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: crc
-867> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: secure
-866> 2022-04-12T16:07:49.219+0300 7f2a3b460400 2 auth: KeyRing::load: loaded key file /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring
-865> 2022-04-12T16:07:49.219+0300 7f2a3b460400 10 monclient: init
-864> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding auth protocol: cephx
-863> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding auth protocol: cephx
-862> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding auth protocol: cephx
-861> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding auth protocol: none
-860> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: secure
-859> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: crc
-858> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: secure
-857> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: crc
-856> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: secure
-855> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: crc
-854> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: crc
-853> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: secure
-852> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: crc
-851> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: secure
-850> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: crc
-849> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: secure
-848> 2022-04-12T16:07:49.219+0300 7f2a3b460400 2 auth: KeyRing::load: loaded key file /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring
-847> 2022-04-12T16:07:49.219+0300 7f2a3b460400 2 auth: KeyRing::load: loaded key file /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring
-846> 2022-04-12T16:07:49.219+0300 7f2a3b460400 10 monclient: _reopen_session rank -1
-845> 2022-04-12T16:07:49.219+0300 7f2a3b460400 10 monclient: _add_conns ranks=[1,0,2]
-844> 2022-04-12T16:07:49.219+0300 7f2a3b460400 10 monclient(hunting): picked mon.noname-b con 0x559e55990400 addr [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0]
-843> 2022-04-12T16:07:49.219+0300 7f2a3b460400 10 monclient(hunting): picked mon.noname-a con 0x559e55991000 addr [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0]
-842> 2022-04-12T16:07:49.219+0300 7f2a3b460400 10 monclient(hunting): picked mon.noname-c con 0x559e55990c00 addr [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0]
-841> 2022-04-12T16:07:49.219+0300 7f2a3b460400 10 monclient(hunting): start opening mon connection
-840> 2022-04-12T16:07:49.219+0300 7f2a3b460400 10 monclient(hunting): start opening mon connection
-839> 2022-04-12T16:07:49.219+0300 7f2a3b460400 10 monclient(hunting): start opening mon connection
-838> 2022-04-12T16:07:49.219+0300 7f2a3b460400 10 monclient(hunting): _renew_subs
-837> 2022-04-12T16:07:49.219+0300 7f2a3b460400 10 monclient(hunting): authenticate will time out at 2022-04-12T16:12:49.224469+0300
-836> 2022-04-12T16:07:49.219+0300 7f2a260c1700 10 monclient(hunting): get_auth_request con 0x559e55991000 auth_method 0
-835> 2022-04-12T16:07:49.219+0300 7f2a260c1700 10 monclient(hunting): get_auth_request method 2 preferred_modes [2,1]
-834> 2022-04-12T16:07:49.219+0300 7f2a260c1700 10 monclient(hunting): _init_auth method 2
-833> 2022-04-12T16:07:49.219+0300 7f2a260c1700 10 monclient(hunting): _init_auth creating new auth
-832> 2022-04-12T16:07:49.219+0300 7f2a260c1700 10 monclient(hunting): handle_auth_reply_more payload 9
-831> 2022-04-12T16:07:49.219+0300 7f2a260c1700 10 monclient(hunting): handle_auth_reply_more payload_len 9
-830> 2022-04-12T16:07:49.219+0300 7f2a260c1700 10 monclient(hunting): handle_auth_reply_more responding with 36 bytes
-829> 2022-04-12T16:07:49.219+0300 7f2a268c2700 10 monclient(hunting): get_auth_request con 0x559e55990400 auth_method 0
-828> 2022-04-12T16:07:49.219+0300 7f2a268c2700 10 monclient(hunting): get_auth_request method 2 preferred_modes [2,1]
-827> 2022-04-12T16:07:49.219+0300 7f2a268c2700 10 monclient(hunting): _init_auth method 2
-826> 2022-04-12T16:07:49.219+0300 7f2a268c2700 10 monclient(hunting): _init_auth creating new auth
-825> 2022-04-12T16:07:49.219+0300 7f2a260c1700 10 monclient(hunting): handle_auth_done global_id 5724 payload 322
-824> 2022-04-12T16:07:49.219+0300 7f2a260c1700 10 monclient: _finish_hunting 0
-823> 2022-04-12T16:07:49.219+0300 7f2a260c1700 1 monclient: found mon.noname-a
-822> 2022-04-12T16:07:49.219+0300 7f2a260c1700 10 monclient: _send_mon_message to mon.noname-a at v2:192.168.166.83:3300/0
-821> 2022-04-12T16:07:49.219+0300 7f2a0d890700 10 monclient: handle_monmap mon_map magic: 0 v1
-820> 2022-04-12T16:07:49.219+0300 7f2a0d890700 10 monclient: got monmap 1 from mon.noname-a (according to old e1)
-819> 2022-04-12T16:07:49.219+0300 7f2a0d890700 10 monclient: dump:
epoch 1
fsid 65ccd25a-39ce-4072-a5c6-a0e6654bda87
last_changed 2022-04-12T15:48:27.663651+0300
created 2022-04-12T15:48:27.663651+0300
min_mon_release 16 (pacific)
election_strategy: 1
0: [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] mon.ceph-ss-test-oct-controller-av-01
1: [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] mon.ceph-ss-test-oct-controller-av-02
2: [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] mon.ceph-ss-test-oct-controller-av-03

-818> 2022-04-12T16:07:49.219+0300 7f2a0d890700 10 monclient: _finish_auth 0
-817> 2022-04-12T16:07:49.219+0300 7f2a0d890700 10 monclient: handle_config config(0 keys) v1
-816> 2022-04-12T16:07:49.219+0300 7f2a0d890700 10 monclient: handle_monmap mon_map magic: 0 v1
-815> 2022-04-12T16:07:49.219+0300 7f2a0d890700 10 monclient: got monmap 1 from mon.ceph-ss-test-oct-controller-av-01 (according to old e1)
-814> 2022-04-12T16:07:49.219+0300 7f2a0d890700 10 monclient: dump:
epoch 1
fsid 65ccd25a-39ce-4072-a5c6-a0e6654bda87
last_changed 2022-04-12T15:48:27.663651+0300
created 2022-04-12T15:48:27.663651+0300
min_mon_release 16 (pacific)
election_strategy: 1
0: [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] mon.ceph-ss-test-oct-controller-av-01
1: [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] mon.ceph-ss-test-oct-controller-av-02
2: [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] mon.ceph-ss-test-oct-controller-av-03

-813> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 monclient: authenticate success, global_id 5724
-812> 2022-04-12T16:07:49.219+0300 7f2a3b460400 10 monclient: get_monmap_and_config success
-811> 2022-04-12T16:07:49.219+0300 7f2a3b460400 4 set_mon_vals no callback set
-810> 2022-04-12T16:07:49.219+0300 7f2a3b460400 10 monclient: shutdown
-809> 2022-04-12T16:07:49.223+0300 7f2a3b460400 10 monclient: build_initial_monmap
-808> 2022-04-12T16:07:49.223+0300 7f2a3b460400 10 monclient: monmap:
epoch 0
fsid 00000000-0000-0000-0000-000000000000
last_changed 0.000000
created 0.000000
min_mon_release 0 (unknown)
election_strategy: 1
0: [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] mon.noname-a
1: [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] mon.noname-b
2: [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] mon.noname-c

-807> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding auth protocol: cephx
-806> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding auth protocol: cephx
-805> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding auth protocol: cephx
-804> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding auth protocol: none
-803> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: secure
-802> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: crc
-801> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: secure
-800> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: crc
-799> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: secure
-798> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: crc
-797> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: crc
-796> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: secure
-795> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: crc
-794> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: secure
-793> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: crc
-792> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: secure
-791> 2022-04-12T16:07:49.223+0300 7f2a3b460400 2 auth: KeyRing::load: loaded key file /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring
-790> 2022-04-12T16:07:49.223+0300 7f2a3b460400 1 librados: starting msgr at
-789> 2022-04-12T16:07:49.223+0300 7f2a3b460400 1 librados: starting objecter
-788> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command objecter_requests cmddesc objecter_requests hook 0x559e54a2ba40 EEXIST
-787> 2022-04-12T16:07:49.223+0300 7f2a3b460400 1 librados: setting wanted keys
-786> 2022-04-12T16:07:49.223+0300 7f2a3b460400 1 librados: calling monclient init
-785> 2022-04-12T16:07:49.223+0300 7f2a3b460400 10 monclient: init
-784> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e559dd0d8) adding auth protocol: cephx
-783> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e559dd0d8) adding auth protocol: cephx
-782> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e559dd0d8) adding auth protocol: cephx
-781> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e559dd0d8) adding auth protocol: none
-780> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e559dd0d8) adding con mode: secure
-779> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e559dd0d8) adding con mode: crc
-778> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e559dd0d8) adding con mode: secure
-777> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e559dd0d8) adding con mode: crc
-776> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e559dd0d8) adding con mode: secure
-775> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e559dd0d8) adding con mode: crc
-774> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e559dd0d8) adding con mode: crc
-773> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e559dd0d8) adding con mode: secure
-772> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e559dd0d8) adding con mode: crc
-771> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e559dd0d8) adding con mode: secure
-770> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e559dd0d8) adding con mode: crc
-769> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e559dd0d8) adding con mode: secure
-768> 2022-04-12T16:07:49.223+0300 7f2a3b460400 2 auth: KeyRing::load: loaded key file /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring
-767> 2022-04-12T16:07:49.223+0300 7f2a3b460400 2 auth: KeyRing::load: loaded key file /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring
-766> 2022-04-12T16:07:49.223+0300 7f2a3b460400 10 monclient: _reopen_session rank -1
-765> 2022-04-12T16:07:49.223+0300 7f2a3b460400 10 monclient: _add_conns ranks=[0,2,1]
-764> 2022-04-12T16:07:49.223+0300 7f2a3b460400 10 monclient(hunting): picked mon.noname-a con 0x559e55990400 addr [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0]
-763> 2022-04-12T16:07:49.223+0300 7f2a3b460400 10 monclient(hunting): picked mon.noname-c con 0x559e55990c00 addr [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0]
-762> 2022-04-12T16:07:49.223+0300 7f2a3b460400 10 monclient(hunting): picked mon.noname-b con 0x559e55991000 addr [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0]
-761> 2022-04-12T16:07:49.223+0300 7f2a3b460400 10 monclient(hunting): start opening mon connection
-760> 2022-04-12T16:07:49.223+0300 7f2a3b460400 10 monclient(hunting): start opening mon connection
-759> 2022-04-12T16:07:49.223+0300 7f2a3b460400 10 monclient(hunting): start opening mon connection
-758> 2022-04-12T16:07:49.223+0300 7f2a3b460400 10 monclient(hunting): _renew_subs
-757> 2022-04-12T16:07:49.223+0300 7f2a3b460400 10 monclient(hunting): authenticate will time out at 2022-04-12T16:12:49.227127+0300
-756> 2022-04-12T16:07:49.223+0300 7f2a268c2700 10 monclient(hunting): get_auth_request con 0x559e55990400 auth_method 0
-755> 2022-04-12T16:07:49.223+0300 7f2a268c2700 10 monclient(hunting): get_auth_request method 2 preferred_modes [2,1]
-754> 2022-04-12T16:07:49.223+0300 7f2a268c2700 10 monclient(hunting): _init_auth method 2
-753> 2022-04-12T16:07:49.223+0300 7f2a268c2700 10 monclient(hunting): _init_auth creating new auth
-752> 2022-04-12T16:07:49.223+0300 7f2a268c2700 10 monclient(hunting): handle_auth_reply_more payload 9
-751> 2022-04-12T16:07:49.223+0300 7f2a268c2700 10 monclient(hunting): handle_auth_reply_more payload_len 9
-750> 2022-04-12T16:07:49.223+0300 7f2a268c2700 10 monclient(hunting): handle_auth_reply_more responding with 36 bytes
-749> 2022-04-12T16:07:49.223+0300 7f2a268c2700 10 monclient(hunting): handle_auth_done global_id 5727 payload 1000
-748> 2022-04-12T16:07:49.223+0300 7f2a268c2700 10 monclient: _finish_hunting 0
-747> 2022-04-12T16:07:49.223+0300 7f2a268c2700 1 monclient: found mon.noname-a
-746> 2022-04-12T16:07:49.223+0300 7f2a268c2700 10 monclient: _send_mon_message to mon.noname-a at v2:192.168.166.83:3300/0
-745> 2022-04-12T16:07:49.223+0300 7f2a0c08d700 10 monclient: handle_monmap mon_map magic: 0 v1
-744> 2022-04-12T16:07:49.223+0300 7f2a0c08d700 10 monclient: got monmap 1 from mon.noname-a (according to old e1)
-743> 2022-04-12T16:07:49.223+0300 7f2a0c08d700 10 monclient: dump:
epoch 1
fsid 65ccd25a-39ce-4072-a5c6-a0e6654bda87
last_changed 2022-04-12T15:48:27.663651+0300
created 2022-04-12T15:48:27.663651+0300
min_mon_release 16 (pacific)
election_strategy: 1
0: [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] mon.ceph-ss-test-oct-controller-av-01
1: [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] mon.ceph-ss-test-oct-controller-av-02
2: [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] mon.ceph-ss-test-oct-controller-av-03

-742> 2022-04-12T16:07:49.223+0300 7f2a0c08d700 10 monclient: _finish_auth 0
-741> 2022-04-12T16:07:49.223+0300 7f2a0c08d700 10 monclient: handle_config config(0 keys) v1
-740> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 monclient: authenticate success, global_id 5727
-739> 2022-04-12T16:07:49.223+0300 7f2a3b460400 10 monclient: _renew_subs
-738> 2022-04-12T16:07:49.223+0300 7f2a0d890700 4 set_mon_vals no callback set
-737> 2022-04-12T16:07:49.223+0300 7f2a3b460400 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-736> 2022-04-12T16:07:49.223+0300 7f2a0c08d700 10 monclient: handle_monmap mon_map magic: 0 v1
-735> 2022-04-12T16:07:49.223+0300 7f2a0c08d700 10 monclient: got monmap 1 from mon.ceph-ss-test-oct-controller-av-01 (according to old e1)
-734> 2022-04-12T16:07:49.223+0300 7f2a0c08d700 10 monclient: dump:
epoch 1
fsid 65ccd25a-39ce-4072-a5c6-a0e6654bda87
last_changed 2022-04-12T15:48:27.663651+0300
created 2022-04-12T15:48:27.663651+0300
min_mon_release 16 (pacific)
election_strategy: 1
0: [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] mon.ceph-ss-test-oct-controller-av-01
1: [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] mon.ceph-ss-test-oct-controller-av-02
2: [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] mon.ceph-ss-test-oct-controller-av-03

-733> 2022-04-12T16:07:49.223+0300 7f2a3b460400 10 monclient: _renew_subs
-732> 2022-04-12T16:07:49.223+0300 7f2a3b460400 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-731> 2022-04-12T16:07:49.223+0300 7f2a3b460400 1 librados: init done
-730> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command cr dump hook 0x559e558a04e8
-729> 2022-04-12T16:07:49.227+0300 7f2a0c08d700 4 mgrc handle_mgr_map Got map version 29
-728> 2022-04-12T16:07:49.227+0300 7f2a0c08d700 4 mgrc handle_mgr_map Active mgr is now [v2:192.168.166.84:6800/56,v1:192.168.166.84:6801/56]
-727> 2022-04-12T16:07:49.227+0300 7f2a0c08d700 4 mgrc reconnect Starting new session with [v2:192.168.166.84:6800/56,v1:192.168.166.84:6801/56]
-726> 2022-04-12T16:07:49.227+0300 7f2a3b460400 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-725> 2022-04-12T16:07:49.227+0300 7f2a0c08d700 10 monclient: handle_get_version_reply finishing 1 version 68
-724> 2022-04-12T16:07:49.227+0300 7f2a260c1700 10 monclient: get_auth_request con 0x559e559cbc00 auth_method 0
-723> 2022-04-12T16:07:49.227+0300 7f2a3b460400 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-722> 2022-04-12T16:07:49.603+0300 7f2a0c08d700 10 monclient: _renew_subs
-721> 2022-04-12T16:07:49.603+0300 7f2a0c08d700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-720> 2022-04-12T16:07:49.607+0300 7f2a3b460400 10 start_mon_command cmd=[{"prefix": "osd pool application enable","pool": "sync.rgw.meta","app": "rgw"}]
-719> 2022-04-12T16:07:49.607+0300 7f2a3b460400 10 monclient: _send_command 1 [{"prefix": "osd pool application enable","pool": "sync.rgw.meta","app": "rgw"}]
-718> 2022-04-12T16:07:49.607+0300 7f2a3b460400 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-717> 2022-04-12T16:07:49.655+0300 7f2a1f0b3700 10 monclient: tick
-716> 2022-04-12T16:07:49.655+0300 7f2a208b6700 10 monclient: _renew_subs
-715> 2022-04-12T16:07:49.655+0300 7f2a208b6700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-714> 2022-04-12T16:07:50.611+0300 7f2a0c08d700 10 monclient: handle_mon_command_ack 1 [{"prefix": "osd pool application enable","pool": "sync.rgw.meta","app": "rgw"}]
-713> 2022-04-12T16:07:50.611+0300 7f2a0c08d700 10 monclient: _finish_command 1 = system:0 enabled application 'rgw' on pool 'sync.rgw.meta'
-712> 2022-04-12T16:07:50.611+0300 7f2a3b460400 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-711> 2022-04-12T16:07:50.615+0300 7f2a0c08d700 10 monclient: handle_get_version_reply finishing 2 version 70
-710> 2022-04-12T16:07:50.615+0300 7f2a0d890700 10 monclient: _renew_subs
-709> 2022-04-12T16:07:50.615+0300 7f2a0d890700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-708> 2022-04-12T16:07:50.615+0300 7f2a3b460400 10 start_mon_command cmd=[{"prefix": "osd pool set", "pool": "sync.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]
-707> 2022-04-12T16:07:50.615+0300 7f2a3b460400 10 monclient: _send_command 2 [{"prefix": "osd pool set", "pool": "sync.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]
-706> 2022-04-12T16:07:50.615+0300 7f2a3b460400 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-705> 2022-04-12T16:07:50.655+0300 7f2a1f0b3700 10 monclient: tick
-704> 2022-04-12T16:07:51.615+0300 7f2a0c08d700 10 monclient: handle_mon_command_ack 2 [{"prefix": "osd pool set", "pool": "sync.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]
-703> 2022-04-12T16:07:51.615+0300 7f2a0c08d700 10 monclient: _finish_command 2 = system:0 set pool 11 pg_autoscale_bias to 4
-702> 2022-04-12T16:07:51.615+0300 7f2a3b460400 10 start_mon_command cmd=[{"prefix": "osd pool set", "pool": "sync.rgw.meta", "var": "pg_num_min", "val": "8"}]
-701> 2022-04-12T16:07:51.615+0300 7f2a3b460400 10 monclient: _send_command 3 [{"prefix": "osd pool set", "pool": "sync.rgw.meta", "var": "pg_num_min", "val": "8"}]
-700> 2022-04-12T16:07:51.615+0300 7f2a3b460400 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-699> 2022-04-12T16:07:51.655+0300 7f2a1f0b3700 10 monclient: tick
-698> 2022-04-12T16:07:52.223+0300 7f2a0b08b700 10 monclient: tick
-697> 2022-04-12T16:07:52.619+0300 7f2a0c08d700 10 monclient: handle_mon_command_ack 3 [{"prefix": "osd pool set", "pool": "sync.rgw.meta", "var": "pg_num_min", "val": "8"}]
-696> 2022-04-12T16:07:52.619+0300 7f2a0c08d700 10 monclient: _finish_command 3 = system:0 set pool 11 pg_num_min to 8
-695> 2022-04-12T16:07:52.619+0300 7f2a3b460400 10 start_mon_command cmd=[{"prefix": "osd pool set", "pool": "sync.rgw.meta", "var": "recovery_priority": "5"}]
-694> 2022-04-12T16:07:52.619+0300 7f2a3b460400 10 monclient: _send_command 4 [{"prefix": "osd pool set", "pool": "sync.rgw.meta", "var": "recovery_priority": "5"}]
-693> 2022-04-12T16:07:52.619+0300 7f2a3b460400 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-692> 2022-04-12T16:07:52.623+0300 7f2a0c08d700 10 monclient: handle_mon_command_ack 4 [{"prefix": "osd pool set", "pool": "sync.rgw.meta", "var": "recovery_priority": "5"}]
-691> 2022-04-12T16:07:52.623+0300 7f2a0c08d700 10 monclient: _finish_command 4 = mon:22 unparseable JSON {"prefix": "osd pool set", "pool": "sync.rgw.meta", "var": "recovery_priority": "5"}
-690> 2022-04-12T16:07:52.623+0300 7f2a270c3700 10 monclient: get_auth_request con 0x559e559f1400 auth_method 0
-689> 2022-04-12T16:07:52.627+0300 7f2a268c2700 10 monclient: get_auth_request con 0x559e559cac00 auth_method 0
-688> 2022-04-12T16:07:52.635+0300 7f2a260c1700 10 monclient: get_auth_request con 0x559e559ca000 auth_method 0
-687> 2022-04-12T16:07:52.639+0300 7f2a270c3700 10 monclient: get_auth_request con 0x559e55b21400 auth_method 0
-686> 2022-04-12T16:07:52.647+0300 7f2a268c2700 10 monclient: get_auth_request con 0x559e54b39c00 auth_method 0
-685> 2022-04-12T16:07:52.651+0300 7f2a260c1700 10 monclient: get_auth_request con 0x559e559cb000 auth_method 0
-684> 2022-04-12T16:07:52.659+0300 7f2a1f0b3700 10 monclient: tick
-683> 2022-04-12T16:07:52.659+0300 7f2a270c3700 10 monclient: get_auth_request con 0x559e55990000 auth_method 0
-682> 2022-04-12T16:07:52.671+0300 7f2a268c2700 10 monclient: get_auth_request con 0x559e559f1800 auth_method 0
-681> 2022-04-12T16:07:52.699+0300 7f2a260c1700 10 monclient: get_auth_request con 0x559e55b51800 auth_method 0
-680> 2022-04-12T16:07:52.711+0300 7f2a270c3700 10 monclient: get_auth_request con 0x559e55b50000 auth_method 0
-679> 2022-04-12T16:07:52.723+0300 7f2a268c2700 10 monclient: get_auth_request con 0x559e559f0400 auth_method 0
-678> 2022-04-12T16:07:52.731+0300 7f2a260c1700 10 monclient: get_auth_request con 0x559e559ca400 auth_method 0
-677> 2022-04-12T16:07:52.743+0300 7f2a270c3700 10 monclient: get_auth_request con 0x559e55979c00 auth_method 0
-676> 2022-04-12T16:07:52.747+0300 7f2a09888700 2 object expiration: start
-675> 2022-04-12T16:07:52.747+0300 7f2a0a089700 2 garbage collection: garbage collection: start
-674> 2022-04-12T16:07:52.747+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command sync trace show hook 0x559e5592a1c0
-673> 2022-04-12T16:07:52.747+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command sync trace history hook 0x559e5592a1c0
-672> 2022-04-12T16:07:52.747+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command sync trace active hook 0x559e5592a1c0
-671> 2022-04-12T16:07:52.747+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command sync trace active_short hook 0x559e5592a1c0
-670> 2022-04-12T16:07:52.751+0300 7f2a3b460400 4 rgw main: failed to read sync status info with (2) No such file or directory
-669> 2022-04-12T16:07:52.755+0300 7f2a07884700 4 rgw rados thread: failed to read sync status info with (2) No such file or directory
-668> 2022-04-12T16:07:52.755+0300 7f2a07884700 1 RGW-SYNC:meta: initialize sync (reason: period is empty)
-667> 2022-04-12T16:07:52.755+0300 7f2a07884700 1 rgw rados thread: epoch=0 in sync status comes before remote's oldest mdlog epoch=1, restarting sync
-666> 2022-04-12T16:07:52.759+0300 7f2a3b460400 5 rgw main: starting data sync thread for zone avrupa
-665> 2022-04-12T16:07:52.759+0300 7f29ff073700 5 lifecycle: schedule life cycle next start time: Tue Apr 12 21:00:00 2022
-664> 2022-04-12T16:07:52.759+0300 7f2a01077700 5 lifecycle: schedule life cycle next start time: Tue Apr 12 21:00:00 2022
-663> 2022-04-12T16:07:52.759+0300 7f29fd06f700 5 lifecycle: schedule life cycle next start time: Tue Apr 12 21:00:00 2022
-662> 2022-04-12T16:07:52.763+0300 7f2a3b460400 0 framework: beast
-661> 2022-04-12T16:07:52.763+0300 7f2a3b460400 0 framework conf key: ssl_certificate, val: config://rgw/cert/$realm/$zone.crt
-660> 2022-04-12T16:07:52.763+0300 7f2a3b460400 0 framework conf key: ssl_private_key, val: config://rgw/cert/$realm/$zone.key
-659> 2022-04-12T16:07:52.763+0300 7f2a3b460400 0 starting handler: beast
-658> 2022-04-12T16:07:52.763+0300 7f2a06882700 4 rgw rados thread: failed to read sync status info with (2) No such file or directory
-657> 2022-04-12T16:07:52.763+0300 7f2a3b460400 4 frontend listening on 192.168.166.83:8081
-656> 2022-04-12T16:07:52.763+0300 7f2a3b460400 0 set uid:gid to 167:167 (ceph:ceph)
-655> 2022-04-12T16:07:52.763+0300 7f2a3b460400 4 frontend spawning 512 threads
-654> 2022-04-12T16:07:52.799+0300 7f2a3b460400 1 mgrc service_daemon_register rgw.5727 metadata {arch=x86_64,ceph_release=pacific,ceph_version=ceph version 16.2.7 (dd0603118f56ab514f133c8d2e3adfc983942503) pacific (stable),ceph_version_short=16.2.7,container_image=gitlab.example.com:5050/openstack/registry/ceph/daemon:v6.0.6-stable-6.0-pacific-centos-8-x86_64-ceph-ansible-20220121,cpu=Intel(R) Xeon(R) Gold 6230R CPU @ 2.10GHz,distro=centos,distro_description=CentOS Linux 8,distro_version=8,frontend_config#0=beast endpoint=192.168.166.83:8081,frontend_type#0=beast,hostname=ceph-ss-test-oct-controller-av-01,id=ceph-ss-test-oct-controller-av-01.rgw1,kernel_description=#121-Ubuntu SMP Thu Mar 24 16:04:27 UTC 2022,kernel_version=5.4.0-107-generic,mem_cgroup_limit=4294967296,mem_swap_kb=4030460,mem_total_kb=8148192,num_handles=1,os=Linux,pid=54,realm_id=532f2481-54e3-4d51-95fd-19867ca818e9,realm_name=myrealm,zone_id=c81e6d95-54da-48d3-83a4-7bb7b286709f,zone_name=sync,zonegroup_id=784a05a5-2089-4564-ab1b-a55c43c5f401,zonegroup_name=cdn}
-653> 2022-04-12T16:07:52.807+0300 7f2a0c08d700 4 mgrc handle_mgr_configure stats_period=5
-652> 2022-04-12T16:07:52.807+0300 7f2a0c08d700 4 mgrc handle_mgr_configure updated stats threshold: 5
-651> 2022-04-12T16:07:52.807+0300 7f2a3b460400 10 monclient: get_monmap_and_config
-650> 2022-04-12T16:07:52.807+0300 7f2a3b460400 10 monclient: build_initial_monmap
-649> 2022-04-12T16:07:52.807+0300 7f2a3b460400 10 monclient: monmap:
epoch 0
fsid 00000000-0000-0000-0000-000000000000
last_changed 0.000000
created 0.000000
min_mon_release 0 (unknown)
election_strategy: 1
0: [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] mon.noname-a
1: [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] mon.noname-b
2: [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] mon.noname-c

-648> 2022-04-12T16:07:52.807+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding auth protocol: cephx
-647> 2022-04-12T16:07:52.807+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding auth protocol: cephx
-646> 2022-04-12T16:07:52.807+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding auth protocol: cephx
-645> 2022-04-12T16:07:52.807+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding auth protocol: none
-644> 2022-04-12T16:07:52.807+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: secure
-643> 2022-04-12T16:07:52.807+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: crc
-642> 2022-04-12T16:07:52.807+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: secure
-641> 2022-04-12T16:07:52.807+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: crc
-640> 2022-04-12T16:07:52.807+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: secure
-639> 2022-04-12T16:07:52.807+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: crc
-638> 2022-04-12T16:07:52.807+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: crc
-637> 2022-04-12T16:07:52.807+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: secure
-636> 2022-04-12T16:07:52.807+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: crc
-635> 2022-04-12T16:07:52.807+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: secure
-634> 2022-04-12T16:07:52.807+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: crc
-633> 2022-04-12T16:07:52.807+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: secure
-632> 2022-04-12T16:07:52.807+0300 7f2a3b460400 2 auth: KeyRing::load: loaded key file /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring
-631> 2022-04-12T16:07:52.819+0300 7f2a3b460400 10 monclient: init
-630> 2022-04-12T16:07:52.819+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding auth protocol: cephx
-629> 2022-04-12T16:07:52.819+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding auth protocol: cephx
-628> 2022-04-12T16:07:52.819+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding auth protocol: cephx
-627> 2022-04-12T16:07:52.819+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding auth protocol: none
-626> 2022-04-12T16:07:52.819+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: secure
-625> 2022-04-12T16:07:52.819+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: crc
-624> 2022-04-12T16:07:52.819+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: secure
-623> 2022-04-12T16:07:52.819+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: crc
-622> 2022-04-12T16:07:52.819+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: secure
-621> 2022-04-12T16:07:52.819+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: crc
-620> 2022-04-12T16:07:52.819+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: crc
-619> 2022-04-12T16:07:52.819+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: secure
-618> 2022-04-12T16:07:52.819+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: crc
-617> 2022-04-12T16:07:52.819+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: secure
-616> 2022-04-12T16:07:52.819+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: crc
-615> 2022-04-12T16:07:52.819+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: secure
-614> 2022-04-12T16:07:52.819+0300 7f2a3b460400 2 auth: KeyRing::load: loaded key file /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring
-613> 2022-04-12T16:07:52.819+0300 7f2a3b460400 2 auth: KeyRing::load: loaded key file /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring
-612> 2022-04-12T16:07:52.819+0300 7f2a3b460400 10 monclient: _reopen_session rank -1
-611> 2022-04-12T16:07:52.819+0300 7f2a3b460400 10 monclient: _add_conns ranks=[1,0,2]
-610> 2022-04-12T16:07:52.819+0300 7f2a3b460400 10 monclient(hunting): picked mon.noname-b con 0x559e55db1c00 addr [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0]
-609> 2022-04-12T16:07:52.819+0300 7f2a3b460400 10 monclient(hunting): picked mon.noname-a con 0x559e559f1000 addr [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0]
-608> 2022-04-12T16:07:52.819+0300 7f2a3b460400 10 monclient(hunting): picked mon.noname-c con 0x559e559f0800 addr [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0]
-607> 2022-04-12T16:07:52.819+0300 7f2a3b460400 10 monclient(hunting): start opening mon connection
-606> 2022-04-12T16:07:52.819+0300 7f2a3b460400 10 monclient(hunting): start opening mon connection
-605> 2022-04-12T16:07:52.819+0300 7f2a3b460400 10 monclient(hunting): start opening mon connection
-604> 2022-04-12T16:07:52.819+0300 7f2a3b460400 10 monclient(hunting): _renew_subs
-603> 2022-04-12T16:07:52.819+0300 7f2a3b460400 10 monclient(hunting): authenticate will time out at 2022-04-12T16:12:52.824021+0300
-602> 2022-04-12T16:07:52.819+0300 7f2a260c1700 10 monclient(hunting): get_auth_request con 0x559e55db1c00 auth_method 0
-601> 2022-04-12T16:07:52.819+0300 7f2a260c1700 10 monclient(hunting): get_auth_request method 2 preferred_modes [2,1]
-600> 2022-04-12T16:07:52.819+0300 7f2a260c1700 10 monclient(hunting): _init_auth method 2
-599> 2022-04-12T16:07:52.819+0300 7f2a260c1700 10 monclient(hunting): _init_auth creating new auth
-598> 2022-04-12T16:07:52.819+0300 7f2a268c2700 10 monclient(hunting): get_auth_request con 0x559e559f0800 auth_method 0
-597> 2022-04-12T16:07:52.819+0300 7f2a268c2700 10 monclient(hunting): get_auth_request method 2 preferred_modes [2,1]
-596> 2022-04-12T16:07:52.819+0300 7f2a268c2700 10 monclient(hunting): _init_auth method 2
-595> 2022-04-12T16:07:52.819+0300 7f2a268c2700 10 monclient(hunting): _init_auth creating new auth
-594> 2022-04-12T16:07:52.819+0300 7f2a260c1700 10 monclient(hunting): handle_auth_reply_more payload 9
-593> 2022-04-12T16:07:52.819+0300 7f2a260c1700 10 monclient(hunting): handle_auth_reply_more payload_len 9
-592> 2022-04-12T16:07:52.819+0300 7f2a260c1700 10 monclient(hunting): handle_auth_reply_more responding with 36 bytes
-591> 2022-04-12T16:07:52.819+0300 7f2a268c2700 10 monclient(hunting): handle_auth_reply_more payload 9
-590> 2022-04-12T16:07:52.819+0300 7f2a268c2700 10 monclient(hunting): handle_auth_reply_more payload_len 9
-589> 2022-04-12T16:07:52.819+0300 7f2a268c2700 10 monclient(hunting): handle_auth_reply_more responding with 36 bytes
-588> 2022-04-12T16:07:52.819+0300 7f2a260c1700 10 monclient(hunting): handle_auth_done global_id 5197 payload 322
-587> 2022-04-12T16:07:52.819+0300 7f2a260c1700 10 monclient: _finish_hunting 0
-586> 2022-04-12T16:07:52.819+0300 7f2a260c1700 1 monclient: found mon.noname-b
-585> 2022-04-12T16:07:52.819+0300 7f2a260c1700 10 monclient: _send_mon_message to mon.noname-b at v2:192.168.166.84:3300/0
-584> 2022-04-12T16:07:52.819+0300 7f28f8e67700 10 monclient: handle_monmap mon_map magic: 0 v1
-583> 2022-04-12T16:07:52.819+0300 7f28f8e67700 10 monclient: got monmap 1 from mon.noname-b (according to old e1)
-582> 2022-04-12T16:07:52.819+0300 7f28f8e67700 10 monclient: dump:
epoch 1
fsid 65ccd25a-39ce-4072-a5c6-a0e6654bda87
last_changed 2022-04-12T15:48:27.663651+0300
created 2022-04-12T15:48:27.663651+0300
min_mon_release 16 (pacific)
election_strategy: 1
0: [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] mon.ceph-ss-test-oct-controller-av-01
1: [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] mon.ceph-ss-test-oct-controller-av-02
2: [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] mon.ceph-ss-test-oct-controller-av-03

-581> 2022-04-12T16:07:52.819+0300 7f28f8e67700 10 monclient: _finish_auth 0
-580> 2022-04-12T16:07:52.819+0300 7f28f8e67700 10 monclient: handle_config config(0 keys) v1
-579> 2022-04-12T16:07:52.819+0300 7f28f8e67700 10 monclient: handle_monmap mon_map magic: 0 v1
-578> 2022-04-12T16:07:52.819+0300 7f28f8e67700 10 monclient: got monmap 1 from mon.ceph-ss-test-oct-controller-av-02 (according to old e1)
-577> 2022-04-12T16:07:52.819+0300 7f28f8e67700 10 monclient: dump:
epoch 1
fsid 65ccd25a-39ce-4072-a5c6-a0e6654bda87
last_changed 2022-04-12T15:48:27.663651+0300
created 2022-04-12T15:48:27.663651+0300
min_mon_release 16 (pacific)
election_strategy: 1
0: [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] mon.ceph-ss-test-oct-controller-av-01
1: [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] mon.ceph-ss-test-oct-controller-av-02
2: [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] mon.ceph-ss-test-oct-controller-av-03

-576> 2022-04-12T16:07:52.819+0300 7f2a3b460400 5 monclient: authenticate success, global_id 5197
-575> 2022-04-12T16:07:52.819+0300 7f2a3b460400 10 monclient: get_monmap_and_config success
-574> 2022-04-12T16:07:52.819+0300 7f2a3b460400 4 set_mon_vals no callback set
-573> 2022-04-12T16:07:52.823+0300 7f2a3b460400 10 monclient: shutdown
-572> 2022-04-12T16:07:52.823+0300 7f2a3b460400 10 monclient: build_initial_monmap
-571> 2022-04-12T16:07:52.823+0300 7f2a3b460400 10 monclient: monmap:
epoch 0
fsid 00000000-0000-0000-0000-000000000000
last_changed 0.000000
created 0.000000
min_mon_release 0 (unknown)
election_strategy: 1
0: [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] mon.noname-a
1: [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] mon.noname-b
2: [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] mon.noname-c

-570> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding auth protocol: cephx
-569> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding auth protocol: cephx
-568> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding auth protocol: cephx
-567> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding auth protocol: none
-566> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: secure
-565> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: crc
-564> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: secure
-563> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: crc
-562> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: secure
-561> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: crc
-560> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: crc
-559> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: secure
-558> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: crc
-557> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: secure
-556> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: crc
-555> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: secure
-554> 2022-04-12T16:07:52.823+0300 7f2a3b460400 2 auth: KeyRing::load: loaded key file /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring
-553> 2022-04-12T16:07:52.823+0300 7f2a3b460400 1 librados: starting msgr at
-552> 2022-04-12T16:07:52.823+0300 7f2a3b460400 1 librados: starting objecter
-551> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command objecter_requests cmddesc objecter_requests hook 0x559e58ce73c0 EEXIST
-550> 2022-04-12T16:07:52.823+0300 7f2a3b460400 1 librados: setting wanted keys
-549> 2022-04-12T16:07:52.823+0300 7f2a3b460400 1 librados: calling monclient init
-548> 2022-04-12T16:07:52.823+0300 7f2a3b460400 10 monclient: init
-547> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e594860d8) adding auth protocol: cephx
-546> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e594860d8) adding auth protocol: cephx
-545> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e594860d8) adding auth protocol: cephx
-544> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e594860d8) adding auth protocol: none
-543> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e594860d8) adding con mode: secure
-542> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e594860d8) adding con mode: crc
-541> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e594860d8) adding con mode: secure
-540> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e594860d8) adding con mode: crc
-539> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e594860d8) adding con mode: secure
-538> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e594860d8) adding con mode: crc
-537> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e594860d8) adding con mode: crc
-536> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e594860d8) adding con mode: secure
-535> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e594860d8) adding con mode: crc
-534> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e594860d8) adding con mode: secure
-533> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e594860d8) adding con mode: crc
-532> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e594860d8) adding con mode: secure
-531> 2022-04-12T16:07:52.823+0300 7f2a3b460400 2 auth: KeyRing::load: loaded key file /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring
-530> 2022-04-12T16:07:52.823+0300 7f2a3b460400 2 auth: KeyRing::load: loaded key file /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring
-529> 2022-04-12T16:07:52.823+0300 7f2a3b460400 10 monclient: _reopen_session rank -1
-528> 2022-04-12T16:07:52.823+0300 7f2a3b460400 10 monclient: _add_conns ranks=[0,2,1]
-527> 2022-04-12T16:07:52.823+0300 7f2a3b460400 10 monclient(hunting): picked mon.noname-a con 0x559e559f0800 addr [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0]
-526> 2022-04-12T16:07:52.823+0300 7f2a3b460400 10 monclient(hunting): picked mon.noname-c con 0x559e559f1000 addr [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0]
-525> 2022-04-12T16:07:52.823+0300 7f2a3b460400 10 monclient(hunting): picked mon.noname-b con 0x559e55db1c00 addr [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0]
-524> 2022-04-12T16:07:52.823+0300 7f2a3b460400 10 monclient(hunting): start opening mon connection
-523> 2022-04-12T16:07:52.823+0300 7f2a3b460400 10 monclient(hunting): start opening mon connection
-522> 2022-04-12T16:07:52.823+0300 7f2a3b460400 10 monclient(hunting): start opening mon connection
-521> 2022-04-12T16:07:52.823+0300 7f2a3b460400 10 monclient(hunting): _renew_subs
-520> 2022-04-12T16:07:52.823+0300 7f2a3b460400 10 monclient(hunting): authenticate will time out at 2022-04-12T16:12:52.827867+0300
-519> 2022-04-12T16:07:52.823+0300 7f2a260c1700 10 monclient(hunting): get_auth_request con 0x559e559f0800 auth_method 0
-518> 2022-04-12T16:07:52.823+0300 7f2a260c1700 10 monclient(hunting): get_auth_request method 2 preferred_modes [2,1]
-517> 2022-04-12T16:07:52.823+0300 7f2a260c1700 10 monclient(hunting): _init_auth method 2
-516> 2022-04-12T16:07:52.823+0300 7f2a260c1700 10 monclient(hunting): _init_auth creating new auth
-515> 2022-04-12T16:07:52.823+0300 7f2a260c1700 10 monclient(hunting): handle_auth_reply_more payload 9
-514> 2022-04-12T16:07:52.823+0300 7f2a260c1700 10 monclient(hunting): handle_auth_reply_more payload_len 9
-513> 2022-04-12T16:07:52.823+0300 7f2a260c1700 10 monclient(hunting): handle_auth_reply_more responding with 36 bytes
-512> 2022-04-12T16:07:52.823+0300 7f2a270c3700 10 monclient(hunting): get_auth_request con 0x559e559f1000 auth_method 0
-511> 2022-04-12T16:07:52.823+0300 7f2a270c3700 10 monclient(hunting): get_auth_request method 2 preferred_modes [2,1]
-510> 2022-04-12T16:07:52.823+0300 7f2a270c3700 10 monclient(hunting): _init_auth method 2
-509> 2022-04-12T16:07:52.823+0300 7f2a270c3700 10 monclient(hunting): _init_auth creating new auth
-508> 2022-04-12T16:07:52.823+0300 7f2a260c1700 10 monclient(hunting): handle_auth_done global_id 5730 payload 1000
-507> 2022-04-12T16:07:52.823+0300 7f2a260c1700 10 monclient: _finish_hunting 0
-506> 2022-04-12T16:07:52.823+0300 7f2a260c1700 1 monclient: found mon.noname-a
-505> 2022-04-12T16:07:52.823+0300 7f2a260c1700 10 monclient: _send_mon_message to mon.noname-a at v2:192.168.166.83:3300/0
-504> 2022-04-12T16:07:52.823+0300 7f28f7664700 10 monclient: handle_monmap mon_map magic: 0 v1
-503> 2022-04-12T16:07:52.827+0300 7f28f7664700 10 monclient: got monmap 1 from mon.noname-a (according to old e1)
-502> 2022-04-12T16:07:52.827+0300 7f28f7664700 10 monclient: dump:
epoch 1
fsid 65ccd25a-39ce-4072-a5c6-a0e6654bda87
last_changed 2022-04-12T15:48:27.663651+0300
created 2022-04-12T15:48:27.663651+0300
min_mon_release 16 (pacific)
election_strategy: 1
0: [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] mon.ceph-ss-test-oct-controller-av-01
1: [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] mon.ceph-ss-test-oct-controller-av-02
2: [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] mon.ceph-ss-test-oct-controller-av-03

-501> 2022-04-12T16:07:52.827+0300 7f28f7664700 10 monclient: _finish_auth 0
-500> 2022-04-12T16:07:52.827+0300 7f28f7664700 10 monclient: handle_config config(0 keys) v1
-499> 2022-04-12T16:07:52.827+0300 7f28f7664700 10 monclient: handle_monmap mon_map magic: 0 v1
-498> 2022-04-12T16:07:52.827+0300 7f28f7664700 10 monclient: got monmap 1 from mon.ceph-ss-test-oct-controller-av-01 (according to old e1)
-497> 2022-04-12T16:07:52.827+0300 7f28f7664700 10 monclient: dump:
epoch 1
fsid 65ccd25a-39ce-4072-a5c6-a0e6654bda87
last_changed 2022-04-12T15:48:27.663651+0300
created 2022-04-12T15:48:27.663651+0300
min_mon_release 16 (pacific)
election_strategy: 1
0: [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] mon.ceph-ss-test-oct-controller-av-01
1: [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] mon.ceph-ss-test-oct-controller-av-02
2: [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] mon.ceph-ss-test-oct-controller-av-03

-496> 2022-04-12T16:07:52.827+0300 7f28f8e67700 4 set_mon_vals no callback set
-495> 2022-04-12T16:07:52.827+0300 7f2a3b460400 5 monclient: authenticate success, global_id 5730
-494> 2022-04-12T16:07:52.827+0300 7f2a3b460400 10 monclient: _renew_subs
-493> 2022-04-12T16:07:52.827+0300 7f2a3b460400 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-492> 2022-04-12T16:07:52.827+0300 7f2a3b460400 10 monclient: _renew_subs
-491> 2022-04-12T16:07:52.827+0300 7f2a3b460400 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-490> 2022-04-12T16:07:52.827+0300 7f2a3b460400 1 librados: init done
-489> 2022-04-12T16:07:52.831+0300 7f28f7664700 4 mgrc handle_mgr_map Got map version 29
-488> 2022-04-12T16:07:52.831+0300 7f28f7664700 4 mgrc handle_mgr_map Active mgr is now [v2:192.168.166.84:6800/56,v1:192.168.166.84:6801/56]
-487> 2022-04-12T16:07:52.831+0300 7f28f7664700 4 mgrc reconnect Starting new session with [v2:192.168.166.84:6800/56,v1:192.168.166.84:6801/56]
-486> 2022-04-12T16:07:52.831+0300 7f2a268c2700 10 monclient: get_auth_request con 0x559e593a1000 auth_method 0
-485> 2022-04-12T16:07:52.835+0300 7f2a270c3700 10 monclient: get_auth_request con 0x559e55979800 auth_method 0
-484> 2022-04-12T16:07:52.855+0300 7f2a260c1700 10 monclient: get_auth_request con 0x559e562ab000 auth_method 0
-483> 2022-04-12T16:07:53.075+0300 7f2a270c3700 10 monclient: get_auth_request con 0x559e59cf6000 auth_method 0
-482> 2022-04-12T16:07:53.083+0300 7f2a0a089700 2 garbage collection: garbage collection: stop
-481> 2022-04-12T16:07:53.223+0300 7f2a0b08b700 10 monclient: tick
-480> 2022-04-12T16:07:53.643+0300 7f2a09888700 2 object expiration: stop
-479> 2022-04-12T16:07:53.655+0300 7f2a1f0b3700 10 monclient: tick
-478> 2022-04-12T16:07:53.951+0300 7f29f8065700 1 ====== starting new request req=0x7f2a3b415620 =====
-477> 2022-04-12T16:07:53.951+0300 7f29f8065700 2 req 6350979191525555842 0.000000000s initializing for trans_id = tx00000582335ecfec73e82-00625579a9-1659-sync
-476> 2022-04-12T16:07:53.951+0300 7f29f8065700 2 req 6350979191525555842 0.000000000s getting op 4
-475> 2022-04-12T16:07:53.951+0300 7f29f6061700 2 req 6350979191525555842 0.000000000s :post_period verifying requester
-474> 2022-04-12T16:07:53.951+0300 7f29f9868700 1 ====== starting new request req=0x7f2a3b313620 =====
-473> 2022-04-12T16:07:53.951+0300 7f29f305b700 1 ====== starting new request req=0x7f2a3b394620 =====
-472> 2022-04-12T16:07:53.951+0300 7f29f9868700 2 req 8268245483269231629 0.000000000s initializing for trans_id = tx0000072beb575c1fd940d-00625579a9-1659-sync
-471> 2022-04-12T16:07:53.951+0300 7f29f305b700 2 req 10560051255912296523 0.000000000s initializing for trans_id = tx00000928cd6aaee91f04b-00625579a9-1659-sync
-470> 2022-04-12T16:07:53.955+0300 7f29f9868700 2 req 8268245483269231629 0.004000003s getting op 4
-469> 2022-04-12T16:07:53.955+0300 7f29f9868700 2 req 8268245483269231629 0.004000003s :post_period verifying requester
-468> 2022-04-12T16:07:53.955+0300 7f29f9868700 5 req 8268245483269231629 0.004000003s :post_period error reading user info, uid=m7yH86NH1D44QiwaUUV9 can't authenticate
-467> 2022-04-12T16:07:53.955+0300 7f29f9868700 5 req 8268245483269231629 0.004000003s :post_period error reading user info, uid=m7yH86NH1D44QiwaUUV9 can't authenticate
-466> 2022-04-12T16:07:53.955+0300 7f29f9868700 5 req 8268245483269231629 0.004000003s :post_period Failed the auth strategy, reason=-2028
-465> 2022-04-12T16:07:53.955+0300 7f29f9868700 1 req 8268245483269231629 0.004000003s op->ERRORHANDLER: err_no=-2028 new_err_no=-2028
-464> 2022-04-12T16:07:53.955+0300 7f29f0856700 5 req 6350979191525555842 0.004000003s :post_period error reading user info, uid=m7yH86NH1D44QiwaUUV9 can't authenticate
-463> 2022-04-12T16:07:53.955+0300 7f29f0856700 5 req 6350979191525555842 0.004000003s :post_period error reading user info, uid=m7yH86NH1D44QiwaUUV9 can't authenticate
-462> 2022-04-12T16:07:53.955+0300 7f29f0856700 5 req 6350979191525555842 0.004000003s :post_period Failed the auth strategy, reason=-2028
-461> 2022-04-12T16:07:53.955+0300 7f29f0856700 1 req 6350979191525555842 0.004000003s op->ERRORHANDLER: err_no=-2028 new_err_no=-2028
-460> 2022-04-12T16:07:53.955+0300 7f29f305b700 2 req 10560051255912296523 0.004000003s getting op 4
-459> 2022-04-12T16:07:53.955+0300 7f29f305b700 2 req 10560051255912296523 0.004000003s :post_period verifying requester
-458> 2022-04-12T16:07:53.955+0300 7f29e6842700 2 req 8268245483269231629 0.004000003s :post_period op status=0
-457> 2022-04-12T16:07:53.955+0300 7f29e6842700 2 req 8268245483269231629 0.004000003s :post_period http status=403
-456> 2022-04-12T16:07:53.955+0300 7f29e6842700 1 ====== req done req=0x7f2a3b313620 op status=0 http_status=403 latency=0.004000003s ======
-455> 2022-04-12T16:07:53.955+0300 7f29e6842700 1 beast: 0x7f2a3b313620: 192.168.166.84 - - [12/Apr/2022:16:07:53.951 +0300] "POST /admin/realm/period?period=8b4c894f-cf43-4c56-9af0-1c2b54c74991&epoch=3&rgwx-zonegroup=784a05a5-2089-4564-ab1b-a55c43c5f401 HTTP/1.1" 403 113 - - - latency=0.004000003s
-454> 2022-04-12T16:07:53.955+0300 7f29f0856700 2 req 6350979191525555842 0.004000003s :post_period op status=0
-453> 2022-04-12T16:07:53.955+0300 7f29f0856700 2 req 6350979191525555842 0.004000003s :post_period http status=403
-452> 2022-04-12T16:07:53.955+0300 7f29f0856700 1 ====== req done req=0x7f2a3b415620 op status=0 http_status=403 latency=0.004000003s ======
-451> 2022-04-12T16:07:53.955+0300 7f29f305b700 5 req 10560051255912296523 0.004000003s :post_period error reading user info, uid=m7yH86NH1D44QiwaUUV9 can't authenticate
-450> 2022-04-12T16:07:53.955+0300 7f29f305b700 5 req 10560051255912296523 0.004000003s :post_period error reading user info, uid=m7yH86NH1D44QiwaUUV9 can't authenticate
-449> 2022-04-12T16:07:53.955+0300 7f29f305b700 5 req 10560051255912296523 0.004000003s :post_period Failed the auth strategy, reason=-2028
-448> 2022-04-12T16:07:53.955+0300 7f29f305b700 1 req 10560051255912296523 0.004000003s op->ERRORHANDLER: err_no=-2028 new_err_no=-2028
-447> 2022-04-12T16:07:53.955+0300 7f29f0856700 1 beast: 0x7f2a3b415620: 192.168.166.83 - - [12/Apr/2022:16:07:53.951 +0300] "POST /admin/realm/period?period=8b4c894f-cf43-4c56-9af0-1c2b54c74991&epoch=3&rgwx-zonegroup=784a05a5-2089-4564-ab1b-a55c43c5f401 HTTP/1.1" 403 113 - - - latency=0.004000003s
-446> 2022-04-12T16:07:53.955+0300 7f29ee051700 2 req 10560051255912296523 0.004000003s :post_period op status=0
-445> 2022-04-12T16:07:53.955+0300 7f29ee051700 2 req 10560051255912296523 0.004000003s :post_period http status=403
-444> 2022-04-12T16:07:53.955+0300 7f29ee051700 1 ====== req done req=0x7f2a3b394620 op status=0 http_status=403 latency=0.004000003s ======
-443> 2022-04-12T16:07:53.955+0300 7f29ee051700 1 beast: 0x7f2a3b394620: 192.168.166.85 - - [12/Apr/2022:16:07:53.951 +0300] "POST /admin/realm/period?period=8b4c894f-cf43-4c56-9af0-1c2b54c74991&epoch=3&rgwx-zonegroup=784a05a5-2089-4564-ab1b-a55c43c5f401 HTTP/1.1" 403 113 - - - latency=0.004000003s
-442> 2022-04-12T16:07:54.223+0300 7f2a0b08b700 10 monclient: tick
-441> 2022-04-12T16:07:54.655+0300 7f2a1f0b3700 10 monclient: tick
-440> 2022-04-12T16:07:54.655+0300 7f2a208b6700 10 monclient: _renew_subs
-439> 2022-04-12T16:07:54.655+0300 7f2a208b6700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-438> 2022-04-12T16:07:54.731+0300 7f2a07884700 1 RGW-SYNC:meta: start
-437> 2022-04-12T16:07:54.731+0300 7f2a07884700 1 RGW-SYNC:meta: realm epoch=2 period id=8b4c894f-cf43-4c56-9af0-1c2b54c74991
-436> 2022-04-12T16:07:54.735+0300 7f2a07884700 4 meta sync: cr:s=0x559e571e3180:op=0x559e59df4000:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-435> 2022-04-12T16:07:54.735+0300 7f2a07884700 4 meta sync: cr:s=0x559e59bc08c0:op=0x559e5578a800:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-434> 2022-04-12T16:07:54.735+0300 7f2a07884700 4 meta sync: cr:s=0x559e594b3540:op=0x559e59df4a00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-433> 2022-04-12T16:07:54.735+0300 7f2a07884700 4 meta sync: cr:s=0x559e5988b680:op=0x559e59df5400:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-432> 2022-04-12T16:07:54.735+0300 7f2a07884700 4 meta sync: cr:s=0x559e59bc1180:op=0x559e59d78a00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-431> 2022-04-12T16:07:54.735+0300 7f2a07884700 4 meta sync: cr:s=0x559e596883c0:op=0x559e59dfa800:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-430> 2022-04-12T16:07:54.735+0300 7f2a07884700 4 meta sync: cr:s=0x559e59689b80:op=0x559e59df7200:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-429> 2022-04-12T16:07:54.735+0300 7f2a07884700 4 meta sync: cr:s=0x559e55ddeb40:op=0x559e59df9400:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-428> 2022-04-12T16:07:54.735+0300 7f2a07884700 4 meta sync: cr:s=0x559e59bc17c0:op=0x559e59d79e00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-427> 2022-04-12T16:07:54.735+0300 7f2a07884700 4 meta sync: cr:s=0x559e5965aa00:op=0x559e59dfb200:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-426> 2022-04-12T16:07:54.735+0300 7f2a07884700 4 meta sync: cr:s=0x559e59bc0140:op=0x559e5578b200:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-425> 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e595b1e00:op=0x559e59dfca00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-424> 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e55dfd900:op=0x559e59df8a00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-423> 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e59bc12c0:op=0x559e59d78000:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-422> 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e59688a00:op=0x559e59e01400:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-421> 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e59689cc0:op=0x559e59e02800:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-420> 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e5988b900:op=0x559e59e09400:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-419> 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e55dfc8c0:op=0x559e59dfde00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-418> 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e571e3040:op=0x559e59e05400:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-417> 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e55dfcc80:op=0x559e59d7b200:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-416> 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e571e3540:op=0x559e59e00a00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-415> 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e55dfdb80:op=0x559e59dfe800:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-414> 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e596888c0:op=0x559e59e01e00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-413> 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e55dfd2c0:op=0x559e59df6800:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-412> 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e594b3b80:op=0x559e59e08a00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-411> 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e55dfc280:op=0x559e59dff200:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-410> 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e5965ac80:op=0x559e59e04000:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-409> 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e5988bcc0:op=0x559e59e0d400:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-408> 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e55dfc3c0:op=0x559e59dfd400:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-407> 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e594b37c0:op=0x559e59e09e00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-406> 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e5965ab40:op=0x559e59df9e00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-405> 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e571e2dc0:op=0x559e59e00000:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-404> 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e5988b540:op=0x559e59e08000:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-403> 2022-04-12T16:07:54.743+0300 7f2a07884700 4 meta sync: cr:s=0x559e55dfcdc0:op=0x559e59df5e00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-402> 2022-04-12T16:07:54.743+0300 7f2a07884700 4 meta sync: cr:s=0x559e59acc3c0:op=0x559e59d79400:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-401> 2022-04-12T16:07:54.743+0300 7f2a07884700 4 meta sync: cr:s=0x559e55dfc780:op=0x559e59d7a800:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-400> 2022-04-12T16:07:54.743+0300 7f2a07884700 4 meta sync: cr:s=0x559e57c2af00:op=0x559e59e17200:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-399> 2022-04-12T16:07:54.747+0300 7f2a07884700 4 meta sync: cr:s=0x559e571e2c80:op=0x559e59e06800:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-398> 2022-04-12T16:07:54.747+0300 7f2a07884700 4 meta sync: cr:s=0x559e59bc0dc0:op=0x559e59e11400:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-397> 2022-04-12T16:07:54.747+0300 7f2a07884700 4 meta sync: cr:s=0x559e59bcd540:op=0x559e59e14000:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-396> 2022-04-12T16:07:54.747+0300 7f2a07884700 4 meta sync: cr:s=0x559e57c2adc0:op=0x559e59e16800:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-395> 2022-04-12T16:07:54.747+0300 7f2a07884700 4 meta sync: cr:s=0x559e55a08140:op=0x559e59e14a00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-394> 2022-04-12T16:07:54.747+0300 7f2a07884700 4 meta sync: cr:s=0x559e564c1cc0:op=0x559e59e04a00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-393> 2022-04-12T16:07:54.747+0300 7f2a07884700 4 meta sync: cr:s=0x559e57c2b040:op=0x559e59e18000:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-392> 2022-04-12T16:07:54.747+0300 7f2a07884700 4 meta sync: cr:s=0x559e59bc1900:op=0x559e59e0e800:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-391> 2022-04-12T16:07:54.747+0300 7f2a07884700 4 meta sync: cr:s=0x559e59bc0640:op=0x559e59e0f200:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-390> 2022-04-12T16:07:54.751+0300 7f2a07884700 4 meta sync: cr:s=0x559e59689e00:op=0x559e59e03200:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-389> 2022-04-12T16:07:54.751+0300 7f2a07884700 4 meta sync: cr:s=0x559e55a083c0:op=0x559e59e0de00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-388> 2022-04-12T16:07:54.751+0300 7f2a07884700 4 meta sync: cr:s=0x559e571e3cc0:op=0x559e59e07200:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-387> 2022-04-12T16:07:54.751+0300 7f2a07884700 4 meta sync: cr:s=0x559e5988be00:op=0x559e59e0b200:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-386> 2022-04-12T16:07:54.751+0300 7f2a07884700 4 meta sync: cr:s=0x559e594b3680:op=0x559e59e0ca00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-385> 2022-04-12T16:07:54.751+0300 7f2a07884700 4 meta sync: cr:s=0x559e5988ba40:op=0x559e59df8000:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-384> 2022-04-12T16:07:54.751+0300 7f2a07884700 4 meta sync: cr:s=0x559e571e3900:op=0x559e59e05e00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-383> 2022-04-12T16:07:54.751+0300 7f2a07884700 4 meta sync: cr:s=0x559e5988bb80:op=0x559e59e0a800:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-382> 2022-04-12T16:07:54.751+0300 7f2a07884700 4 meta sync: cr:s=0x559e55a08780:op=0x559e59e15400:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-381> 2022-04-12T16:07:54.751+0300 7f2a07884700 4 meta sync: cr:s=0x559e5988b7c0:op=0x559e59e0c000:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-380> 2022-04-12T16:07:54.751+0300 7f2a07884700 4 meta sync: cr:s=0x559e595b1b80:op=0x559e59dfc000:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-379> 2022-04-12T16:07:54.751+0300 7f2a07884700 4 meta sync: cr:s=0x559e59bc1540:op=0x559e59e10a00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-378> 2022-04-12T16:07:54.751+0300 7f2a07884700 4 meta sync: cr:s=0x559e59bc0780:op=0x559e59e11e00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-377> 2022-04-12T16:07:54.751+0300 7f2a07884700 4 meta sync: cr:s=0x559e59bcc000:op=0x559e59e13200:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-376> 2022-04-12T16:07:54.755+0300 7f2a07884700 4 meta sync: cr:s=0x559e59bc1b80:op=0x559e59e10000:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-375> 2022-04-12T16:07:54.755+0300 7f2a07884700 4 meta sync: cr:s=0x559e55a08280:op=0x559e59e15e00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-374> 2022-04-12T16:07:54.755+0300 7f2a07884700 4 meta sync: cr:s=0x559e59bcc500:op=0x559e59e12800:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-373> 2022-04-12T16:07:54.803+0300 7f2a07884700 4 meta sync: cr:s=0x559e54a72640:op=0x559e55789e00:18RGWMetaSyncShardCR: adjusting marker pos=user:dashboard
-372> 2022-04-12T16:07:54.811+0300 7f2a07884700 4 meta sync: cr:s=0x559e54a72640:op=0x559e55789e00:18RGWMetaSyncShardCR: adjusting marker pos=user:testuser
-371> 2022-04-12T16:07:54.835+0300 7f2a07884700 4 meta sync: cr:s=0x559e54a72640:op=0x559e55789e00:18RGWMetaSyncShardCR: adjusting marker pos=user:testuser
-370> 2022-04-12T16:07:54.835+0300 7f2a07884700 4 meta sync: cr:s=0x559e54a72640:op=0x559e55789e00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-369> 2022-04-12T16:07:55.223+0300 7f2a0b08b700 10 monclient: tick
-368> 2022-04-12T16:07:55.655+0300 7f2a1f0b3700 10 monclient: tick
-367> 2022-04-12T16:07:55.823+0300 7f28f6662700 10 monclient: tick
-366> 2022-04-12T16:07:56.223+0300 7f2a0b08b700 10 monclient: tick
-365> 2022-04-12T16:07:56.659+0300 7f2a1f0b3700 10 monclient: tick
-364> 2022-04-12T16:07:56.827+0300 7f28f6662700 10 monclient: tick
-363> 2022-04-12T16:07:57.227+0300 7f2a0b08b700 10 monclient: tick
-362> 2022-04-12T16:07:57.655+0300 7f2a1f0b3700 10 monclient: tick
-361> 2022-04-12T16:07:57.823+0300 7f28f6662700 10 monclient: tick
-360> 2022-04-12T16:07:57.827+0300 7f28f7e65700 10 monclient: _renew_subs
-359> 2022-04-12T16:07:57.827+0300 7f28f7e65700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-358> 2022-04-12T16:07:58.223+0300 7f2a0b08b700 10 monclient: tick
-357> 2022-04-12T16:07:58.659+0300 7f2a1f0b3700 10 monclient: tick
-356> 2022-04-12T16:07:58.823+0300 7f28f6662700 10 monclient: tick
-355> 2022-04-12T16:07:59.223+0300 7f2a0b08b700 10 monclient: tick
-354> 2022-04-12T16:07:59.655+0300 7f2a208b6700 10 monclient: _renew_subs
-353> 2022-04-12T16:07:59.655+0300 7f2a208b6700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-352> 2022-04-12T16:07:59.655+0300 7f2a1f0b3700 10 monclient: tick
-351> 2022-04-12T16:07:59.823+0300 7f28f6662700 10 monclient: tick
-350> 2022-04-12T16:08:00.223+0300 7f2a0b08b700 10 monclient: tick
-349> 2022-04-12T16:08:00.659+0300 7f2a1f0b3700 10 monclient: tick
-348> 2022-04-12T16:08:00.823+0300 7f28f6662700 10 monclient: tick
-347> 2022-04-12T16:08:01.227+0300 7f2a0b08b700 10 monclient: tick
-346> 2022-04-12T16:08:01.655+0300 7f2a1f0b3700 10 monclient: tick
-345> 2022-04-12T16:08:01.827+0300 7f28f6662700 10 monclient: tick
-344> 2022-04-12T16:08:02.227+0300 7f2a0b08b700 10 monclient: tick
-343> 2022-04-12T16:08:02.659+0300 7f2a1f0b3700 10 monclient: tick
-342> 2022-04-12T16:08:02.827+0300 7f28f6662700 10 monclient: tick
-341> 2022-04-12T16:08:02.827+0300 7f28f7e65700 10 monclient: _renew_subs
-340> 2022-04-12T16:08:02.827+0300 7f28f7e65700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-339> 2022-04-12T16:08:03.223+0300 7f2a0b08b700 10 monclient: tick
-338> 2022-04-12T16:08:03.655+0300 7f2a1f0b3700 10 monclient: tick
-337> 2022-04-12T16:08:03.827+0300 7f28f6662700 10 monclient: tick
-336> 2022-04-12T16:08:04.227+0300 7f2a0b08b700 10 monclient: tick
-335> 2022-04-12T16:08:04.655+0300 7f2a208b6700 10 monclient: _renew_subs
-334> 2022-04-12T16:08:04.655+0300 7f2a208b6700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-333> 2022-04-12T16:08:04.655+0300 7f2a1f0b3700 10 monclient: tick
-332> 2022-04-12T16:08:04.823+0300 7f28f6662700 10 monclient: tick
-331> 2022-04-12T16:08:05.223+0300 7f2a0b08b700 10 monclient: tick
-330> 2022-04-12T16:08:05.659+0300 7f2a1f0b3700 10 monclient: tick
-329> 2022-04-12T16:08:05.823+0300 7f28f6662700 10 monclient: tick
-328> 2022-04-12T16:08:06.227+0300 7f2a0b08b700 10 monclient: tick
-327> 2022-04-12T16:08:06.659+0300 7f2a1f0b3700 10 monclient: tick
-326> 2022-04-12T16:08:06.823+0300 7f28f6662700 10 monclient: tick
-325> 2022-04-12T16:08:07.227+0300 7f2a0b08b700 10 monclient: tick
-324> 2022-04-12T16:08:07.655+0300 7f2a1f0b3700 10 monclient: tick
-323> 2022-04-12T16:08:07.827+0300 7f28f6662700 10 monclient: tick
-322> 2022-04-12T16:08:08.227+0300 7f2a0b08b700 10 monclient: tick
-321> 2022-04-12T16:08:08.659+0300 7f2a1f0b3700 10 monclient: tick
-320> 2022-04-12T16:08:08.823+0300 7f28f6662700 10 monclient: tick
-319> 2022-04-12T16:08:09.223+0300 7f2a0b08b700 10 monclient: tick
-318> 2022-04-12T16:08:09.659+0300 7f2a1f0b3700 10 monclient: tick
-317> 2022-04-12T16:08:09.823+0300 7f28f6662700 10 monclient: tick
-316> 2022-04-12T16:08:10.223+0300 7f2a0b08b700 10 monclient: tick
-315> 2022-04-12T16:08:10.655+0300 7f2a1f0b3700 10 monclient: tick
-314> 2022-04-12T16:08:10.827+0300 7f28f6662700 10 monclient: tick
-313> 2022-04-12T16:08:11.211+0300 7f2a0e091700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start
-312> 2022-04-12T16:08:11.223+0300 7f2a0b08b700 10 monclient: tick
-311> 2022-04-12T16:08:11.655+0300 7f2a1f0b3700 10 monclient: tick
-310> 2022-04-12T16:08:11.827+0300 7f28f6662700 10 monclient: tick
-309> 2022-04-12T16:08:12.223+0300 7f2a0b08b700 10 monclient: tick
-308> 2022-04-12T16:08:12.655+0300 7f2a1f0b3700 10 monclient: tick
-307> 2022-04-12T16:08:12.823+0300 7f28f6662700 10 monclient: tick
-306> 2022-04-12T16:08:13.223+0300 7f2a0b08b700 10 monclient: tick
-305> 2022-04-12T16:08:13.659+0300 7f2a1f0b3700 10 monclient: tick
-304> 2022-04-12T16:08:13.823+0300 7f28f6662700 10 monclient: tick
-303> 2022-04-12T16:08:14.227+0300 7f2a0b08b700 10 monclient: tick
-302> 2022-04-12T16:08:14.659+0300 7f2a1f0b3700 10 monclient: tick
-301> 2022-04-12T16:08:14.827+0300 7f28f6662700 10 monclient: tick
-300> 2022-04-12T16:08:15.227+0300 7f2a0b08b700 10 monclient: tick
-299> 2022-04-12T16:08:15.659+0300 7f2a1f0b3700 10 monclient: tick
-298> 2022-04-12T16:08:15.827+0300 7f28f6662700 10 monclient: tick
-297> 2022-04-12T16:08:16.227+0300 7f2a0b08b700 10 monclient: tick
-296> 2022-04-12T16:08:16.659+0300 7f2a1f0b3700 10 monclient: tick
-295> 2022-04-12T16:08:16.827+0300 7f28f6662700 10 monclient: tick
-294> 2022-04-12T16:08:17.227+0300 7f2a0b08b700 10 monclient: tick
-293> 2022-04-12T16:08:17.659+0300 7f2a1f0b3700 10 monclient: tick
-292> 2022-04-12T16:08:17.827+0300 7f28f6662700 10 monclient: tick
-291> 2022-04-12T16:08:18.227+0300 7f2a0b08b700 10 monclient: tick
-290> 2022-04-12T16:08:18.659+0300 7f2a1f0b3700 10 monclient: tick
-289> 2022-04-12T16:08:18.827+0300 7f28f6662700 10 monclient: tick
-288> 2022-04-12T16:08:19.227+0300 7f2a0b08b700 10 monclient: tick
-287> 2022-04-12T16:08:19.659+0300 7f2a1f0b3700 10 monclient: tick
-286> 2022-04-12T16:08:19.827+0300 7f28f6662700 10 monclient: tick
-285> 2022-04-12T16:08:20.227+0300 7f2a0b08b700 10 monclient: tick
-284> 2022-04-12T16:08:20.659+0300 7f2a1f0b3700 10 monclient: tick
-283> 2022-04-12T16:08:20.827+0300 7f28f6662700 10 monclient: tick
-282> 2022-04-12T16:08:21.227+0300 7f2a0b08b700 10 monclient: tick
-281> 2022-04-12T16:08:21.659+0300 7f2a1f0b3700 10 monclient: tick
-280> 2022-04-12T16:08:21.827+0300 7f28f6662700 10 monclient: tick
-279> 2022-04-12T16:08:22.227+0300 7f2a0b08b700 10 monclient: tick
-278> 2022-04-12T16:08:22.659+0300 7f2a1f0b3700 10 monclient: tick
-277> 2022-04-12T16:08:22.827+0300 7f28f6662700 10 monclient: tick
-276> 2022-04-12T16:08:23.227+0300 7f2a0b08b700 10 monclient: tick
-275> 2022-04-12T16:08:23.659+0300 7f2a1f0b3700 10 monclient: tick
-274> 2022-04-12T16:08:23.827+0300 7f28f6662700 10 monclient: tick
-273> 2022-04-12T16:08:23.959+0300 7f29f505f700 1 ====== starting new request req=0x7f2a3b313620 =====
-272> 2022-04-12T16:08:23.959+0300 7f29f505f700 2 req 8920709381089314344 0.000000000s initializing for trans_id = tx000007bccb9eb0c52c628-00625579c7-1659-sync
-271> 2022-04-12T16:08:23.959+0300 7f29f505f700 2 req 8920709381089314344 0.000000000s getting op 4
-270> 2022-04-12T16:08:23.959+0300 7f29f505f700 2 req 8920709381089314344 0.000000000s :post_period verifying requester
-269> 2022-04-12T16:08:23.959+0300 7f29f505f700 2 req 8920709381089314344 0.000000000s :post_period normalizing buckets and tenants
-268> 2022-04-12T16:08:23.959+0300 7f29f505f700 2 req 8920709381089314344 0.000000000s :post_period init permissions
-267> 2022-04-12T16:08:23.959+0300 7f29f505f700 2 req 8920709381089314344 0.000000000s :post_period recalculating target
-266> 2022-04-12T16:08:23.959+0300 7f29f505f700 2 req 8920709381089314344 0.000000000s :post_period reading permissions
-265> 2022-04-12T16:08:23.959+0300 7f29f505f700 2 req 8920709381089314344 0.000000000s :post_period init op
-264> 2022-04-12T16:08:23.959+0300 7f29f505f700 2 req 8920709381089314344 0.000000000s :post_period verifying op mask
-263> 2022-04-12T16:08:23.959+0300 7f29f505f700 2 req 8920709381089314344 0.000000000s :post_period verifying op permissions
-262> 2022-04-12T16:08:23.959+0300 7f29f505f700 2 overriding permissions due to system operation
-261> 2022-04-12T16:08:23.959+0300 7f29f505f700 2 req 8920709381089314344 0.000000000s :post_period verifying op params
-260> 2022-04-12T16:08:23.959+0300 7f29f505f700 2 req 8920709381089314344 0.000000000s :post_period pre-executing
-259> 2022-04-12T16:08:23.959+0300 7f29f505f700 2 req 8920709381089314344 0.000000000s :post_period executing
-258> 2022-04-12T16:08:23.959+0300 7f29cc80e700 1 ====== starting new request req=0x7f2a3b415620 =====
-257> 2022-04-12T16:08:23.959+0300 7f29cc80e700 2 req 8609644640538370824 0.000000000s initializing for trans_id = tx00000777b9a2f49a2a308-00625579c7-1659-sync
-256> 2022-04-12T16:08:23.959+0300 7f29cc80e700 2 req 8609644640538370824 0.000000000s getting op 4
-255> 2022-04-12T16:08:23.959+0300 7f29cc80e700 2 req 8609644640538370824 0.000000000s :post_period verifying requester
-254> 2022-04-12T16:08:23.959+0300 7f29cc80e700 2 req 8609644640538370824 0.000000000s :post_period normalizing buckets and tenants
-253> 2022-04-12T16:08:23.959+0300 7f29cc80e700 2 req 8609644640538370824 0.000000000s :post_period init permissions
-252> 2022-04-12T16:08:23.959+0300 7f29cc80e700 2 req 8609644640538370824 0.000000000s :post_period recalculating target
-251> 2022-04-12T16:08:23.959+0300 7f29cc80e700 2 req 8609644640538370824 0.000000000s :post_period reading permissions
-250> 2022-04-12T16:08:23.959+0300 7f29cc80e700 2 req 8609644640538370824 0.000000000s :post_period init op
-249> 2022-04-12T16:08:23.959+0300 7f29cc80e700 2 req 8609644640538370824 0.000000000s :post_period verifying op mask
-248> 2022-04-12T16:08:23.959+0300 7f29cc80e700 2 req 8609644640538370824 0.000000000s :post_period verifying op permissions
-247> 2022-04-12T16:08:23.959+0300 7f29cc80e700 2 overriding permissions due to system operation
-246> 2022-04-12T16:08:23.959+0300 7f29cc80e700 2 req 8609644640538370824 0.000000000s :post_period verifying op params
-245> 2022-04-12T16:08:23.959+0300 7f29cc80e700 2 req 8609644640538370824 0.000000000s :post_period pre-executing
-244> 2022-04-12T16:08:23.959+0300 7f29cc80e700 2 req 8609644640538370824 0.000000000s :post_period executing
-243> 2022-04-12T16:08:23.959+0300 7f29c8005700 1 ====== starting new request req=0x7f2a3b394620 =====
-242> 2022-04-12T16:08:23.959+0300 7f29c8005700 2 req 14231248929957790225 0.000000000s initializing for trans_id = tx00000c57f8d805c4ac211-00625579c7-1659-sync
-241> 2022-04-12T16:08:23.959+0300 7f29c8005700 2 req 14231248929957790225 0.000000000s getting op 4
-240> 2022-04-12T16:08:23.959+0300 7f29c8005700 2 req 14231248929957790225 0.000000000s :post_period verifying requester
-239> 2022-04-12T16:08:23.959+0300 7f29c8005700 2 req 14231248929957790225 0.000000000s :post_period normalizing buckets and tenants
-238> 2022-04-12T16:08:23.959+0300 7f29c8005700 2 req 14231248929957790225 0.000000000s :post_period init permissions
-237> 2022-04-12T16:08:23.959+0300 7f29c8005700 2 req 14231248929957790225 0.000000000s :post_period recalculating target
-236> 2022-04-12T16:08:23.959+0300 7f29c8005700 2 req 14231248929957790225 0.000000000s :post_period reading permissions
-235> 2022-04-12T16:08:23.959+0300 7f29c8005700 2 req 14231248929957790225 0.000000000s :post_period init op
-234> 2022-04-12T16:08:23.959+0300 7f29c8005700 2 req 14231248929957790225 0.000000000s :post_period verifying op mask
-233> 2022-04-12T16:08:23.959+0300 7f29c8005700 2 req 14231248929957790225 0.000000000s :post_period verifying op permissions
-232> 2022-04-12T16:08:23.959+0300 7f29c8005700 2 overriding permissions due to system operation
-231> 2022-04-12T16:08:23.959+0300 7f29c8005700 2 req 14231248929957790225 0.000000000s :post_period verifying op params
-230> 2022-04-12T16:08:23.959+0300 7f29c8005700 2 req 14231248929957790225 0.000000000s :post_period pre-executing
-229> 2022-04-12T16:08:23.959+0300 7f29c8005700 2 req 14231248929957790225 0.000000000s :post_period executing
-228> 2022-04-12T16:08:23.963+0300 7f29ac7ce700 4 req 8920709381089314344 0.004000003s :post_period already have epoch >= 3 for period 8b4c894f-cf43-4c56-9af0-1c2b54c74991
-227> 2022-04-12T16:08:23.963+0300 7f29ac7ce700 2 req 8920709381089314344 0.004000003s :post_period completing
-226> 2022-04-12T16:08:23.963+0300 7f29ac7ce700 2 req 8920709381089314344 0.004000003s :post_period op status=0
-225> 2022-04-12T16:08:23.963+0300 7f29ac7ce700 2 req 8920709381089314344 0.004000003s :post_period http status=200
-224> 2022-04-12T16:08:23.963+0300 7f29ac7ce700 1 ====== req done req=0x7f2a3b313620 op status=0 http_status=200 latency=0.004000003s ======
-223> 2022-04-12T16:08:23.963+0300 7f29ac7ce700 1 beast: 0x7f2a3b313620: 192.168.166.84 - rgw_zone_user [12/Apr/2022:16:08:23.959 +0300] "POST /admin/realm/period?period=8b4c894f-cf43-4c56-9af0-1c2b54c74991&epoch=3&rgwx-zonegroup=784a05a5-2089-4564-ab1b-a55c43c5f401 HTTP/1.1" 200 1868 - - - latency=0.004000003s
-222> 2022-04-12T16:08:23.963+0300 7f29a57c0700 4 req 14231248929957790225 0.004000003s :post_period already have epoch >= 3 for period 8b4c894f-cf43-4c56-9af0-1c2b54c74991
-221> 2022-04-12T16:08:23.963+0300 7f29a57c0700 2 req 14231248929957790225 0.004000003s :post_period completing
-220> 2022-04-12T16:08:23.963+0300 7f29a57c0700 2 req 14231248929957790225 0.004000003s :post_period op status=0
-219> 2022-04-12T16:08:23.963+0300 7f29a57c0700 2 req 14231248929957790225 0.004000003s :post_period http status=200
-218> 2022-04-12T16:08:23.963+0300 7f29a57c0700 1 ====== req done req=0x7f2a3b394620 op status=0 http_status=200 latency=0.004000003s ======
-217> 2022-04-12T16:08:23.963+0300 7f29a57c0700 1 beast: 0x7f2a3b394620: 192.168.166.85 - rgw_zone_user [12/Apr/2022:16:08:23.959 +0300] "POST /admin/realm/period?period=8b4c894f-cf43-4c56-9af0-1c2b54c74991&epoch=3&rgwx-zonegroup=784a05a5-2089-4564-ab1b-a55c43c5f401 HTTP/1.1" 200 1868 - - - latency=0.004000003s
-216> 2022-04-12T16:08:23.967+0300 7f299e7b2700 4 req 8609644640538370824 0.008000007s :post_period already have epoch >= 3 for period 8b4c894f-cf43-4c56-9af0-1c2b54c74991
-215> 2022-04-12T16:08:23.967+0300 7f299e7b2700 2 req 8609644640538370824 0.008000007s :post_period completing
-214> 2022-04-12T16:08:23.967+0300 7f299e7b2700 2 req 8609644640538370824 0.008000007s :post_period op status=0
-213> 2022-04-12T16:08:23.967+0300 7f299e7b2700 2 req 8609644640538370824 0.008000007s :post_period http status=200
-212> 2022-04-12T16:08:23.967+0300 7f299e7b2700 1 ====== req done req=0x7f2a3b415620 op status=0 http_status=200 latency=0.008000007s ======
-211> 2022-04-12T16:08:23.967+0300 7f299e7b2700 1 beast: 0x7f2a3b415620: 192.168.166.83 - rgw_zone_user [12/Apr/2022:16:08:23.959 +0300] "POST /admin/realm/period?period=8b4c894f-cf43-4c56-9af0-1c2b54c74991&epoch=3&rgwx-zonegroup=784a05a5-2089-4564-ab1b-a55c43c5f401 HTTP/1.1" 200 1868 - - - latency=0.008000007s
-210> 2022-04-12T16:08:24.227+0300 7f2a0b08b700 10 monclient: tick
-209> 2022-04-12T16:08:24.663+0300 7f2a1f0b3700 10 monclient: tick
-208> 2022-04-12T16:08:24.827+0300 7f28f6662700 10 monclient: tick
-207> 2022-04-12T16:08:25.227+0300 7f2a0b08b700 10 monclient: tick
-206> 2022-04-12T16:08:25.663+0300 7f2a1f0b3700 10 monclient: tick
-205> 2022-04-12T16:08:25.827+0300 7f28f6662700 10 monclient: tick
-204> 2022-04-12T16:08:26.231+0300 7f2a0b08b700 10 monclient: tick
-203> 2022-04-12T16:08:26.663+0300 7f2a1f0b3700 10 monclient: tick
-202> 2022-04-12T16:08:26.831+0300 7f28f6662700 10 monclient: tick
-201> 2022-04-12T16:08:27.227+0300 7f2a0b08b700 10 monclient: tick
-200> 2022-04-12T16:08:27.663+0300 7f2a1f0b3700 10 monclient: tick
-199> 2022-04-12T16:08:27.827+0300 7f28f6662700 10 monclient: tick
-198> 2022-04-12T16:08:28.227+0300 7f2a0b08b700 10 monclient: tick
-197> 2022-04-12T16:08:28.659+0300 7f2a1f0b3700 10 monclient: tick
-196> 2022-04-12T16:08:28.827+0300 7f28f6662700 10 monclient: tick
-195> 2022-04-12T16:08:29.227+0300 7f2a0b08b700 10 monclient: tick
-194> 2022-04-12T16:08:29.663+0300 7f2a1f0b3700 10 monclient: tick
-193> 2022-04-12T16:08:29.827+0300 7f28f6662700 10 monclient: tick
-192> 2022-04-12T16:08:30.227+0300 7f2a0b08b700 10 monclient: tick
-191> 2022-04-12T16:08:30.663+0300 7f2a1f0b3700 10 monclient: tick
-190> 2022-04-12T16:08:30.827+0300 7f28f6662700 10 monclient: tick
-189> 2022-04-12T16:08:31.227+0300 7f2a0b08b700 10 monclient: tick
-188> 2022-04-12T16:08:31.659+0300 7f2a1f0b3700 10 monclient: tick
-187> 2022-04-12T16:08:31.831+0300 7f28f6662700 10 monclient: tick
-186> 2022-04-12T16:08:32.227+0300 7f2a0b08b700 10 monclient: tick
-185> 2022-04-12T16:08:32.659+0300 7f2a1f0b3700 10 monclient: tick
-184> 2022-04-12T16:08:32.827+0300 7f28f6662700 10 monclient: tick
-183> 2022-04-12T16:08:33.211+0300 7f2a0e091700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start
-182> 2022-04-12T16:08:33.227+0300 7f2a0b08b700 10 monclient: tick
-181> 2022-04-12T16:08:33.659+0300 7f2a1f0b3700 10 monclient: tick
-180> 2022-04-12T16:08:33.831+0300 7f28f6662700 10 monclient: tick
-179> 2022-04-12T16:08:34.227+0300 7f2a0b08b700 10 monclient: tick
-178> 2022-04-12T16:08:34.659+0300 7f2a1f0b3700 10 monclient: tick
-177> 2022-04-12T16:08:34.827+0300 7f28f6662700 10 monclient: tick
-176> 2022-04-12T16:08:35.227+0300 7f2a0b08b700 10 monclient: tick
-175> 2022-04-12T16:08:35.659+0300 7f2a1f0b3700 10 monclient: tick
-174> 2022-04-12T16:08:35.827+0300 7f28f6662700 10 monclient: tick
-173> 2022-04-12T16:08:36.227+0300 7f2a0b08b700 10 monclient: tick
-172> 2022-04-12T16:08:36.659+0300 7f2a1f0b3700 10 monclient: tick
-171> 2022-04-12T16:08:36.827+0300 7f28f6662700 10 monclient: tick
-170> 2022-04-12T16:08:37.231+0300 7f2a0b08b700 10 monclient: tick
-169> 2022-04-12T16:08:37.659+0300 7f2a1f0b3700 10 monclient: tick
-168> 2022-04-12T16:08:37.831+0300 7f28f6662700 10 monclient: tick
-167> 2022-04-12T16:08:38.227+0300 7f2a0b08b700 10 monclient: tick
-166> 2022-04-12T16:08:38.663+0300 7f2a1f0b3700 10 monclient: tick
-165> 2022-04-12T16:08:38.831+0300 7f28f6662700 10 monclient: tick
-164> 2022-04-12T16:08:39.227+0300 7f2a0b08b700 10 monclient: tick
-163> 2022-04-12T16:08:39.659+0300 7f2a1f0b3700 10 monclient: tick
-162> 2022-04-12T16:08:39.827+0300 7f28f6662700 10 monclient: tick
-161> 2022-04-12T16:08:40.227+0300 7f2a0b08b700 10 monclient: tick
-160> 2022-04-12T16:08:40.659+0300 7f2a1f0b3700 10 monclient: tick
-159> 2022-04-12T16:08:40.827+0300 7f28f6662700 10 monclient: tick
-158> 2022-04-12T16:08:41.231+0300 7f2a0b08b700 10 monclient: tick
-157> 2022-04-12T16:08:41.659+0300 7f2a1f0b3700 10 monclient: tick
-156> 2022-04-12T16:08:41.831+0300 7f28f6662700 10 monclient: tick
-155> 2022-04-12T16:08:42.227+0300 7f2a0b08b700 10 monclient: tick
-154> 2022-04-12T16:08:42.659+0300 7f2a1f0b3700 10 monclient: tick
-153> 2022-04-12T16:08:42.827+0300 7f28f6662700 10 monclient: tick
-152> 2022-04-12T16:08:43.227+0300 7f2a0b08b700 10 monclient: tick
-151> 2022-04-12T16:08:43.659+0300 7f2a1f0b3700 10 monclient: tick
-150> 2022-04-12T16:08:43.831+0300 7f28f6662700 10 monclient: tick
-149> 2022-04-12T16:08:44.227+0300 7f2a0b08b700 10 monclient: tick
-148> 2022-04-12T16:08:44.659+0300 7f2a1f0b3700 10 monclient: tick
-147> 2022-04-12T16:08:44.831+0300 7f28f6662700 10 monclient: tick
-146> 2022-04-12T16:08:45.231+0300 7f2a0b08b700 10 monclient: tick
-145> 2022-04-12T16:08:45.663+0300 7f2a1f0b3700 10 monclient: tick
-144> 2022-04-12T16:08:45.831+0300 7f28f6662700 10 monclient: tick
-143> 2022-04-12T16:08:46.231+0300 7f2a0b08b700 10 monclient: tick
-142> 2022-04-12T16:08:46.663+0300 7f2a1f0b3700 10 monclient: tick
-141> 2022-04-12T16:08:46.827+0300 7f28f6662700 10 monclient: tick
-140> 2022-04-12T16:08:47.231+0300 7f2a0b08b700 10 monclient: tick
-139> 2022-04-12T16:08:47.663+0300 7f2a1f0b3700 10 monclient: tick
-138> 2022-04-12T16:08:47.831+0300 7f28f6662700 10 monclient: tick
-137> 2022-04-12T16:08:48.231+0300 7f2a0b08b700 10 monclient: tick
-136> 2022-04-12T16:08:48.663+0300 7f2a1f0b3700 10 monclient: tick
-135> 2022-04-12T16:08:48.831+0300 7f28f6662700 10 monclient: tick
-134> 2022-04-12T16:08:49.231+0300 7f2a0b08b700 10 monclient: tick
-133> 2022-04-12T16:08:49.663+0300 7f2a1f0b3700 10 monclient: tick
-132> 2022-04-12T16:08:49.667+0300 7f2a208b6700 10 monclient: _renew_subs
-131> 2022-04-12T16:08:49.667+0300 7f2a208b6700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-130> 2022-04-12T16:08:49.831+0300 7f28f6662700 10 monclient: tick
-129> 2022-04-12T16:08:50.231+0300 7f2a0b08b700 10 monclient: tick
-128> 2022-04-12T16:08:50.663+0300 7f2a1f0b3700 10 monclient: tick
-127> 2022-04-12T16:08:50.831+0300 7f28f6662700 10 monclient: tick
-126> 2022-04-12T16:08:51.231+0300 7f2a0b08b700 10 monclient: tick
-125> 2022-04-12T16:08:51.663+0300 7f2a1f0b3700 10 monclient: tick
-124> 2022-04-12T16:08:51.831+0300 7f28f6662700 10 monclient: tick
-123> 2022-04-12T16:08:52.231+0300 7f2a0b08b700 10 monclient: tick
-122> 2022-04-12T16:08:52.663+0300 7f2a1f0b3700 10 monclient: tick
-121> 2022-04-12T16:08:52.691+0300 7f29b5fe1700 1 ====== starting new request req=0x7f28f565f620 =====
-120> 2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s initializing for trans_id = tx00000ecd2ee05e4c9a4ca-00625579e4-1659-sync
-119> 2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s getting op 4
-118> 2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s :mdlog_notify verifying requester
-117> 2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s :mdlog_notify normalizing buckets and tenants
-116> 2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s :mdlog_notify init permissions
-115> 2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s :mdlog_notify recalculating target
-114> 2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s :mdlog_notify reading permissions
-113> 2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s :mdlog_notify init op
-112> 2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s :mdlog_notify verifying op mask
-111> 2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s :mdlog_notify verifying op permissions
-110> 2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 overriding permissions due to system operation
-109> 2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s :mdlog_notify verifying op params
-108> 2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s :mdlog_notify pre-executing
-107> 2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s :mdlog_notify executing
-106> 2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s :mdlog_notify completing
-105> 2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s :mdlog_notify op status=0
-104> 2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s :mdlog_notify http status=200
-103> 2022-04-12T16:08:52.691+0300 7f29b5fe1700 1 ====== req done req=0x7f28f565f620 op status=0 http_status=200 latency=0.000000000s ======
-102> 2022-04-12T16:08:52.691+0300 7f29b5fe1700 1 beast: 0x7f28f565f620: 192.168.166.85 - rgw_zone_user [12/Apr/2022:16:08:52.691 +0300] "POST /admin/log?type=metadata&notify&rgwx-zonegroup=784a05a5-2089-4564-ab1b-a55c43c5f401 HTTP/1.1" 200 0 - - - latency=0.000000000s
-101> 2022-04-12T16:08:52.699+0300 7f2a07884700 4 meta sync: cr:s=0x559e57c2af00:op=0x559e59e17200:18RGWMetaSyncShardCR: adjusting marker pos=
-100> 2022-04-12T16:08:52.759+0300 7f2a1409d700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-99> 2022-04-12T16:08:52.759+0300 7f2a200b5700 10 monclient: handle_get_version_reply finishing 5 version 76
-98> 2022-04-12T16:08:52.759+0300 7f2a1409d700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-97> 2022-04-12T16:08:52.831+0300 7f28f6662700 10 monclient: tick
-96> 2022-04-12T16:08:52.831+0300 7f28f7e65700 10 monclient: _renew_subs
-95> 2022-04-12T16:08:52.831+0300 7f28f7e65700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-94> 2022-04-12T16:08:53.231+0300 7f2a0b08b700 10 monclient: tick
-93> 2022-04-12T16:08:53.663+0300 7f2a1f0b3700 10 monclient: tick
-92> 2022-04-12T16:08:53.751+0300 7f2a1409d700 10 start_mon_command cmd=[{"prefix": "osd pool application enable","pool": "sync.rgw.buckets.index","app": "rgw"}]
-91> 2022-04-12T16:08:53.751+0300 7f2a1409d700 10 monclient: _send_command 3 [{"prefix": "osd pool application enable","pool": "sync.rgw.buckets.index","app": "rgw"}]
-90> 2022-04-12T16:08:53.751+0300 7f2a1409d700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-89> 2022-04-12T16:08:53.831+0300 7f28f6662700 10 monclient: tick
-88> 2022-04-12T16:08:54.231+0300 7f2a0b08b700 10 monclient: tick
-87> 2022-04-12T16:08:54.663+0300 7f2a1f0b3700 10 monclient: tick
-86> 2022-04-12T16:08:54.667+0300 7f2a208b6700 10 monclient: _renew_subs
-85> 2022-04-12T16:08:54.667+0300 7f2a208b6700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-84> 2022-04-12T16:08:54.759+0300 7f2a200b5700 10 monclient: handle_mon_command_ack 3 [{"prefix": "osd pool application enable","pool": "sync.rgw.buckets.index","app": "rgw"}]
-83> 2022-04-12T16:08:54.759+0300 7f2a200b5700 10 monclient: _finish_command 3 = system:0 enabled application 'rgw' on pool 'sync.rgw.buckets.index'
-82> 2022-04-12T16:08:54.759+0300 7f2a1409d700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-81> 2022-04-12T16:08:54.763+0300 7f2a200b5700 10 monclient: handle_get_version_reply finishing 6 version 78
-80> 2022-04-12T16:08:54.763+0300 7f2a1409d700 10 start_mon_command cmd=[{"prefix": "osd pool set", "pool": "sync.rgw.buckets.index", "var": "pg_autoscale_bias", "val": "4"}]
-79> 2022-04-12T16:08:54.763+0300 7f2a1409d700 10 monclient: _send_command 4 [{"prefix": "osd pool set", "pool": "sync.rgw.buckets.index", "var": "pg_autoscale_bias", "val": "4"}]
-78> 2022-04-12T16:08:54.763+0300 7f2a1409d700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-77> 2022-04-12T16:08:54.783+0300 7f2a07884700 4 meta sync: cr:s=0x559e57c2af00:op=0x559e59e17200:18RGWMetaSyncShardCR: adjusting marker pos=1_1649768932.547706_90.1
-76> 2022-04-12T16:08:54.783+0300 7f2a07884700 4 meta sync: cr:s=0x559e57c2af00:op=0x559e59e17200:18RGWMetaSyncShardCR: adjusting marker pos=1_1649768932.547706_90.1
-75> 2022-04-12T16:08:54.831+0300 7f28f6662700 10 monclient: tick
-74> 2022-04-12T16:08:55.215+0300 7f2a0e091700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start
-73> 2022-04-12T16:08:55.231+0300 7f2a0b08b700 10 monclient: tick
-72> 2022-04-12T16:08:55.663+0300 7f2a1f0b3700 10 monclient: tick
-71> 2022-04-12T16:08:55.767+0300 7f2a200b5700 10 monclient: handle_mon_command_ack 4 [{"prefix": "osd pool set", "pool": "sync.rgw.buckets.index", "var": "pg_autoscale_bias", "val": "4"}]
-70> 2022-04-12T16:08:55.767+0300 7f2a200b5700 10 monclient: _finish_command 4 = system:0 set pool 13 pg_autoscale_bias to 4
-69> 2022-04-12T16:08:55.767+0300 7f2a1409d700 10 start_mon_command cmd=[{"prefix": "osd pool set", "pool": "sync.rgw.buckets.index", "var": "pg_num_min", "val": "8"}]
-68> 2022-04-12T16:08:55.767+0300 7f2a1409d700 10 monclient: _send_command 5 [{"prefix": "osd pool set", "pool": "sync.rgw.buckets.index", "var": "pg_num_min", "val": "8"}]
-67> 2022-04-12T16:08:55.767+0300 7f2a1409d700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-66> 2022-04-12T16:08:55.831+0300 7f28f6662700 10 monclient: tick
-65> 2022-04-12T16:08:56.231+0300 7f2a0b08b700 10 monclient: tick
-64> 2022-04-12T16:08:56.663+0300 7f2a1f0b3700 10 monclient: tick
-63> 2022-04-12T16:08:56.775+0300 7f2a200b5700 10 monclient: handle_mon_command_ack 5 [{"prefix": "osd pool set", "pool": "sync.rgw.buckets.index", "var": "pg_num_min", "val": "8"}]
-62> 2022-04-12T16:08:56.775+0300 7f2a200b5700 10 monclient: _finish_command 5 = system:0 set pool 13 pg_num_min to 8
-61> 2022-04-12T16:08:56.775+0300 7f2a1409d700 10 start_mon_command cmd=[{"prefix": "osd pool set", "pool": "sync.rgw.buckets.index", "var": "recovery_priority": "5"}]
-60> 2022-04-12T16:08:56.775+0300 7f2a1409d700 10 monclient: _send_command 6 [{"prefix": "osd pool set", "pool": "sync.rgw.buckets.index", "var": "recovery_priority": "5"}]
-59> 2022-04-12T16:08:56.775+0300 7f2a1409d700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-58> 2022-04-12T16:08:56.775+0300 7f2a200b5700 10 monclient: handle_mon_command_ack 6 [{"prefix": "osd pool set", "pool": "sync.rgw.buckets.index", "var": "recovery_priority": "5"}]
-57> 2022-04-12T16:08:56.775+0300 7f2a200b5700 10 monclient: _finish_command 6 = mon:22 unparseable JSON {"prefix": "osd pool set", "pool": "sync.rgw.buckets.index", "var": "recovery_priority": "5"}
-56> 2022-04-12T16:08:56.831+0300 7f28f6662700 10 monclient: tick
-55> 2022-04-12T16:08:57.231+0300 7f2a0b08b700 10 monclient: tick
-54> 2022-04-12T16:08:57.663+0300 7f2a1f0b3700 10 monclient: tick
-53> 2022-04-12T16:08:57.831+0300 7f28f6662700 10 monclient: tick
-52> 2022-04-12T16:08:57.831+0300 7f28f7e65700 10 monclient: _renew_subs
-51> 2022-04-12T16:08:57.831+0300 7f28f7e65700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-50> 2022-04-12T16:08:58.231+0300 7f2a0b08b700 10 monclient: tick
-49> 2022-04-12T16:08:58.667+0300 7f2a1f0b3700 10 monclient: tick
-48> 2022-04-12T16:08:58.831+0300 7f28f6662700 10 monclient: tick
-47> 2022-04-12T16:08:59.231+0300 7f2a0b08b700 10 monclient: tick
-46> 2022-04-12T16:08:59.667+0300 7f2a1f0b3700 10 monclient: tick
-45> 2022-04-12T16:08:59.667+0300 7f2a208b6700 10 monclient: _renew_subs
-44> 2022-04-12T16:08:59.667+0300 7f2a208b6700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-43> 2022-04-12T16:08:59.831+0300 7f28f6662700 10 monclient: tick
-42> 2022-04-12T16:09:00.231+0300 7f2a0b08b700 10 monclient: tick
-41> 2022-04-12T16:09:00.663+0300 7f2a1f0b3700 10 monclient: tick
-40> 2022-04-12T16:09:00.831+0300 7f28f6662700 10 monclient: tick
-39> 2022-04-12T16:09:01.235+0300 7f2a0b08b700 10 monclient: tick
-38> 2022-04-12T16:09:01.663+0300 7f2a1f0b3700 10 monclient: tick
-37> 2022-04-12T16:09:01.835+0300 7f28f6662700 10 monclient: tick
-36> 2022-04-12T16:09:02.231+0300 7f2a0b08b700 10 monclient: tick
-35> 2022-04-12T16:09:02.663+0300 7f2a1f0b3700 10 monclient: tick
-34> 2022-04-12T16:09:02.747+0300 7f299a7aa700 1 ====== starting new request req=0x7f28f55de620 =====
-33> 2022-04-12T16:09:02.747+0300 7f299a7aa700 2 req 6236571265947506715 0.000000000s initializing for trans_id = tx00000568cc0860b35981b-00625579ee-1659-sync
-32> 2022-04-12T16:09:02.747+0300 7f299a7aa700 2 req 6236571265947506715 0.000000000s getting op 4
-31> 2022-04-12T16:09:02.747+0300 7f299a7aa700 2 req 6236571265947506715 0.000000000s :datalog_notify verifying requester
-30> 2022-04-12T16:09:02.751+0300 7f299a7aa700 2 req 6236571265947506715 0.004000004s :datalog_notify normalizing buckets and tenants
-29> 2022-04-12T16:09:02.751+0300 7f299a7aa700 2 req 6236571265947506715 0.004000004s :datalog_notify init permissions
-28> 2022-04-12T16:09:02.751+0300 7f299a7aa700 2 req 6236571265947506715 0.004000004s :datalog_notify recalculating target
-27> 2022-04-12T16:09:02.751+0300 7f299a7aa700 2 req 6236571265947506715 0.004000004s :datalog_notify reading permissions
-26> 2022-04-12T16:09:02.751+0300 7f299a7aa700 2 req 6236571265947506715 0.004000004s :datalog_notify init op
-25> 2022-04-12T16:09:02.751+0300 7f299a7aa700 2 req 6236571265947506715 0.004000004s :datalog_notify verifying op mask
-24> 2022-04-12T16:09:02.751+0300 7f299a7aa700 2 req 6236571265947506715 0.004000004s :datalog_notify verifying op permissions
-23> 2022-04-12T16:09:02.751+0300 7f299a7aa700 2 overriding permissions due to system operation
-22> 2022-04-12T16:09:02.751+0300 7f299a7aa700 2 req 6236571265947506715 0.004000004s :datalog_notify verifying op params
-21> 2022-04-12T16:09:02.751+0300 7f299a7aa700 2 req 6236571265947506715 0.004000004s :datalog_notify pre-executing
-20> 2022-04-12T16:09:02.751+0300 7f299a7aa700 2 req 6236571265947506715 0.004000004s :datalog_notify executing
-19> 2022-04-12T16:09:02.751+0300 7f299a7aa700 2 req 6236571265947506715 0.004000004s :datalog_notify completing
-18> 2022-04-12T16:09:02.751+0300 7f299a7aa700 2 req 6236571265947506715 0.004000004s :datalog_notify op status=0
-17> 2022-04-12T16:09:02.751+0300 7f299a7aa700 2 req 6236571265947506715 0.004000004s :datalog_notify http status=200
-16> 2022-04-12T16:09:02.751+0300 7f299a7aa700 1 ====== req done req=0x7f28f55de620 op status=0 http_status=200 latency=0.004000004s ======
-15> 2022-04-12T16:09:02.751+0300 7f299a7aa700 1 beast: 0x7f28f55de620: 192.168.166.83 - rgw_zone_user [12/Apr/2022:16:09:02.747 +0300] "POST /admin/log?type=data&notify&source-zone=6fa70b8c-6324-4bee-8dbf-d323e8dc5f43&rgwx-zonegroup=784a05a5-2089-4564-ab1b-a55c43c5f401 HTTP/1.1" 200 0 - - - latency=0.004000004s
-14> 2022-04-12T16:09:02.751+0300 7f2a06882700 4 data sync: starting sync on testbucket:6fa70b8c-6324-4bee-8dbf-d323e8dc5f43.5105.1:3 key=testbucket:6fa70b8c-6324-4bee-8dbf-d323e8dc5f43.5105.1:3
-13> 2022-04-12T16:09:02.759+0300 7f2a06882700 5 RGW-SYNC:data:sync:shard[30]:entry:bucket_sync_sources[target=:[]):source_bucket=:[]):source_zone=6fa70b8c-6324-4bee-8dbf-d323e8dc5f43]:bucket[testbucket:6fa70b8c-6324-4bee-8dbf-d323e8dc5f43.5105.1:3<-testbucket:6fa70b8c-6324-4bee-8dbf-d323e8dc5f43.5105.1:3]:full_sync[testbucket:6fa70b8c-6324-4bee-8dbf-d323e8dc5f43.5105.1:3]:entry[amazon_case.png]: bucket sync: sync obj: 6fa70b8c-6324-4bee-8dbf-d323e8dc5f43/:testbucket[6fa70b8c-6324-4bee-8dbf-d323e8dc5f43.5105.1])/amazon_case.png[0]
-12> 2022-04-12T16:09:02.759+0300 7f2a06882700 0 e21904192857b447: sync_object: b=:testbucket[6fa70b8c-6324-4bee-8dbf-d323e8dc5f43.5105.1]) k=amazon_case.png versioned_epoch=0
-11> 2022-04-12T16:09:02.767+0300 7f2a06882700 4 rgw rados thread: AWS: download begin: z=6fa70b8c-6324-4bee-8dbf-d323e8dc5f43 b=:testbucket[6fa70b8c-6324-4bee-8dbf-d323e8dc5f43.5105.1]) k=amazon_case.png size=1880884 mtime=2022-04-12T16:09:02.643682+0300 etag=8cd436573b0ee6e9cc4f6f2cb3ba0cf1 zone_short_id=2425654257 pg_ver=0
-10> 2022-04-12T16:09:02.767+0300 7f2a06882700 0 AWS: creating bucket rgw-cdn-e21904192857b447
-9> 2022-04-12T16:09:02.831+0300 7f28f6662700 10 monclient: tick
-8> 2022-04-12T16:09:02.831+0300 7f28f7e65700 10 monclient: _renew_subs
-7> 2022-04-12T16:09:02.831+0300 7f28f7e65700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-6> 2022-04-12T16:09:03.231+0300 7f2a0b08b700 10 monclient: tick
-5> 2022-04-12T16:09:03.663+0300 7f2a1f0b3700 10 monclient: tick
-4> 2022-04-12T16:09:03.831+0300 7f28f6662700 10 monclient: tick
-3> 2022-04-12T16:09:04.231+0300 7f2a0b08b700 10 monclient: tick
-2> 2022-04-12T16:09:04.663+0300 7f2a1f0b3700 10 monclient: tick
-1> 2022-04-12T16:09:04.831+0300 7f28f6662700 10 monclient: tick
0> 2022-04-12T16:09:04.839+0300 7f2a06882700 -1 *** Caught signal (Aborted) **
in thread 7f2a06882700 thread_name:data-sync

ceph version 16.2.7 (dd0603118f56ab514f133c8d2e3adfc983942503) pacific (stable)
1: /lib64/libpthread.so.0(+0x12c20) [0x7f2a2fa66c20]
2: gsignal()
3: abort()
4: /lib64/libstdc++.so.6(+0x9009b) [0x7f2a2ea5a09b]
5: /lib64/libstdc++.so.6(+0x9653c) [0x7f2a2ea6053c]
6: /lib64/libstdc++.so.6(+0x96597) [0x7f2a2ea60597]
7: /lib64/libstdc++.so.6(+0x967f8) [0x7f2a2ea607f8]
8: /lib64/libstdc++.so.6(+0x92025) [0x7f2a2ea5c025]
9: /lib64/libradosgw.so.2(+0x48b109) [0x7f2a3a5a1109]
10: (std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::_M_mutate(unsigned long, unsigned long, char const*, unsigned long)+0x6d) [0x7f2a3a5a471d]
11: (std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::_M_append(char const*, unsigned long)+0x6b) [0x7f2a3a5a489b]
12: (rgw_obj_key::get_oid[abi:cxx11]() const+0xc6) [0x7f2a3a7c9066]
13: (RGWRESTStreamS3PutObj::send_init(rgw::sal::RGWObject*)+0xb3) [0x7f2a3aa8f4b3]
14: (RGWRESTConn::put_obj_send_init(rgw::sal::RGWObject*, rgw_http_param_pair const*, RGWRESTStreamS3PutObj**)+0x5c1) [0x7f2a3aa99891]
15: (RGWAWSStreamPutCRF::init()+0xf1) [0x7f2a3a918ec1]
16: (RGWStreamSpliceCR::operate(DoutPrefixProvider const*)+0xa2f) [0x7f2a3a9b567f]
17: (RGWCoroutinesStack::operate(DoutPrefixProvider const*, RGWCoroutinesEnv*)+0x15c) [0x7f2a3a9a0cbc]
18: (RGWCoroutinesManager::run(DoutPrefixProvider const*, std::__cxx11::list<RGWCoroutinesStack*, std::allocator<RGWCoroutinesStack*> >&)+0x296) [0x7f2a3a9a1b06]
19: (RGWCoroutinesManager::run(DoutPrefixProvider const*, RGWCoroutine*)+0x91) [0x7f2a3a9a2dd1]
20: (RGWRemoteDataLog::run_sync(DoutPrefixProvider const*, int)+0x1b4) [0x7f2a3a8b84b4]
21: (RGWDataSyncProcessorThread::process(DoutPrefixProvider const*)+0x59) [0x7f2a3aa6fe09]
22: (RGWRadosThread::Worker::entry()+0x13a) [0x7f2a3aa3368a]
23: /lib64/libpthread.so.0(+0x817a) [0x7f2a2fa5c17a]
24: clone()
NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.

--- logging levels ---
0/ 5 none
0/ 1 lockdep
0/ 1 context
1/ 1 crush
1/ 5 mds
1/ 5 mds_balancer
1/ 5 mds_locker
1/ 5 mds_log
1/ 5 mds_log_expire
1/ 5 mds_migrator
0/ 1 buffer
0/ 1 timer
0/ 1 filer
0/ 1 striper
0/ 1 objecter
0/ 5 rados
0/ 5 rbd
0/ 5 rbd_mirror
0/ 5 rbd_replay
0/ 5 rbd_pwl
0/ 5 journaler
0/ 5 objectcacher
0/ 5 immutable_obj_cache
0/ 5 client
1/ 5 osd
0/ 5 optracker
0/ 5 objclass
1/ 3 filestore
1/ 3 journal
0/ 0 ms
1/ 5 mon
0/10 monc
1/ 5 paxos
0/ 5 tp
1/ 5 auth
1/ 5 crypto
1/ 1 finisher
1/ 1 reserver
1/ 5 heartbeatmap
1/ 5 perfcounter
5/ 5 rgw
5/ 5 rgw_sync
1/10 civetweb
1/ 5 javaclient
1/ 5 asok
1/ 1 throttle
0/ 0 refs
1/ 5 compressor
1/ 5 bluestore
1/ 5 bluefs
1/ 3 bdev
1/ 5 kstore
4/ 5 rocksdb
4/ 5 leveldb
4/ 5 memdb
1/ 5 fuse
2/ 5 mgr
1/ 5 mgrc
1/ 5 dpdk
1/ 5 eventtrace
1/ 5 prioritycache
0/ 5 test
0/ 5 cephfs_mirror
0/ 5 cephsqlite
-2/-2 (syslog threshold)
99/99 (stderr threshold)
--- pthread ID / name mapping for recent threads ---
139813909309184 / safe_timer
139813926094592 / ms_dispatch
139813934487296 / ceph_timer
139813951272704 / io_context_pool
139816662116096 / radosgw
139816729257728 / radosgw
139816846755584 / radosgw
139816964253440 / radosgw
139817123714816 / radosgw
139817425852160 / radosgw
139817501386496 / radosgw
139817937807104 / radosgw
139818063697664 / radosgw
139818105661184 / radosgw
139818147624704 / radosgw
139818181195520 / radosgw
139818197980928 / radosgw
139818231551744 / radosgw
139818256729856 / radosgw
139818315478784 / lifecycle_thr_2
139818349049600 / lifecycle_thr_1
139818382620416 / lifecycle_thr_0
139818474940160 / data-sync
139818491725568 / meta-sync
139818525296384 / rgw_obj_expirer
139818533689088 / rgw_gc
139818550474496 / safe_timer
139818567259904 / ms_dispatch
139818592438016 / io_context_pool
139818600830720 / rgw_dt_lg_renew
139818701543168 / rados_async
139818886182656 / safe_timer
139818902968064 / ms_dispatch
139818911360768 / ceph_timer
139818919753472 / io_context_pool
139818928146176 / io_context_pool
139818986895104 / admin_socket
139818995287808 / service
139819003680512 / msgr-worker-2
139819012073216 / msgr-worker-1
139819020465920 / msgr-worker-0
139819359798272 / radosgw
max_recent 10000
max_new 10000
log_file
--- end dump of recent events ---
--- begin dump of recent events ---
-1216> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command assert hook 0x559e54a2a760
-1215> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command abort hook 0x559e54a2a760
-1214> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command leak_some_memory hook 0x559e54a2a760
-1213> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command perfcounters_dump hook 0x559e54a2a760
-1212> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command 1 hook 0x559e54a2a760
-1211> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command perf dump hook 0x559e54a2a760
-1210> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command perfcounters_schema hook 0x559e54a2a760
-1209> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command perf histogram dump hook 0x559e54a2a760
-1208> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command 2 hook 0x559e54a2a760
-1207> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command perf schema hook 0x559e54a2a760
-1206> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command perf histogram schema hook 0x559e54a2a760
-1205> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command perf reset hook 0x559e54a2a760
-1204> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command config show hook 0x559e54a2a760
-1203> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command config help hook 0x559e54a2a760
-1202> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command config set hook 0x559e54a2a760
-1201> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command config unset hook 0x559e54a2a760
-1200> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command config get hook 0x559e54a2a760
-1199> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command config diff hook 0x559e54a2a760
-1198> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command config diff get hook 0x559e54a2a760
-1197> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command injectargs hook 0x559e54a2a760
-1196> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command log flush hook 0x559e54a2a760
-1195> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command log dump hook 0x559e54a2a760
-1194> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command log reopen hook 0x559e54a2a760
-1193> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command dump_mempools hook 0x559e54b02068
-1192> 2022-04-12T16:07:44.643+0300 7f2a3b460400 10 monclient: get_monmap_and_config
-1191> 2022-04-12T16:07:44.643+0300 7f2a3b460400 10 monclient: build_initial_monmap
-1190> 2022-04-12T16:07:44.643+0300 7f2a3b460400 10 monclient: monmap:
epoch 0
fsid 65ccd25a-39ce-4072-a5c6-a0e6654bda87
last_changed 2022-04-12T16:07:44.650033+0300
created 2022-04-12T16:07:44.650033+0300
min_mon_release 0 (unknown)
election_strategy: 1
0: [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] mon.noname-a
1: [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] mon.noname-b
2: [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] mon.noname-c

-1189> 2022-04-12T16:07:44.643+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding auth protocol: cephx
-1188> 2022-04-12T16:07:44.643+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding auth protocol: cephx
-1187> 2022-04-12T16:07:44.643+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding auth protocol: cephx
-1186> 2022-04-12T16:07:44.643+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding auth protocol: none
-1185> 2022-04-12T16:07:44.643+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: secure
-1184> 2022-04-12T16:07:44.643+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: crc
-1183> 2022-04-12T16:07:44.643+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: secure
-1182> 2022-04-12T16:07:44.643+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: crc
-1181> 2022-04-12T16:07:44.643+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: secure
-1180> 2022-04-12T16:07:44.643+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: crc
-1179> 2022-04-12T16:07:44.643+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: crc
-1178> 2022-04-12T16:07:44.643+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: secure
-1177> 2022-04-12T16:07:44.643+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: crc
-1176> 2022-04-12T16:07:44.643+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: secure
-1175> 2022-04-12T16:07:44.643+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: crc
-1174> 2022-04-12T16:07:44.643+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: secure
-1173> 2022-04-12T16:07:44.647+0300 7f2a3b460400 2 auth: KeyRing::load: loaded key file /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring
-1172> 2022-04-12T16:07:44.647+0300 7f2a3b460400 10 monclient: init
-1171> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d65c0) adding auth protocol: cephx
-1170> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d65c0) adding auth protocol: cephx
-1169> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d65c0) adding auth protocol: cephx
-1168> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d65c0) adding auth protocol: none
-1167> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d65c0) adding con mode: secure
-1166> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d65c0) adding con mode: crc
-1165> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d65c0) adding con mode: secure
-1164> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d65c0) adding con mode: crc
-1163> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d65c0) adding con mode: secure
-1162> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d65c0) adding con mode: crc
-1161> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d65c0) adding con mode: crc
-1160> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d65c0) adding con mode: secure
-1159> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d65c0) adding con mode: crc
-1158> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d65c0) adding con mode: secure
-1157> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d65c0) adding con mode: crc
-1156> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d65c0) adding con mode: secure
-1155> 2022-04-12T16:07:44.647+0300 7f2a3b460400 2 auth: KeyRing::load: loaded key file /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring
-1154> 2022-04-12T16:07:44.647+0300 7f2a3b460400 2 auth: KeyRing::load: loaded key file /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring
-1153> 2022-04-12T16:07:44.647+0300 7f2a3b460400 10 monclient: _reopen_session rank -1
-1152> 2022-04-12T16:07:44.647+0300 7f2a3b460400 10 monclient: _add_conns ranks=[1,2,0]
-1151> 2022-04-12T16:07:44.647+0300 7f2a3b460400 10 monclient(hunting): picked mon.noname-b con 0x559e54b38800 addr [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0]
-1150> 2022-04-12T16:07:44.647+0300 7f2a3b460400 10 monclient(hunting): picked mon.noname-c con 0x559e54b38c00 addr [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0]
-1149> 2022-04-12T16:07:44.647+0300 7f2a3b460400 10 monclient(hunting): picked mon.noname-a con 0x559e54b39000 addr [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0]
-1148> 2022-04-12T16:07:44.647+0300 7f2a3b460400 10 monclient(hunting): start opening mon connection
-1147> 2022-04-12T16:07:44.647+0300 7f2a3b460400 10 monclient(hunting): start opening mon connection
-1146> 2022-04-12T16:07:44.647+0300 7f2a3b460400 10 monclient(hunting): start opening mon connection
-1145> 2022-04-12T16:07:44.647+0300 7f2a3b460400 10 monclient(hunting): _renew_subs
-1144> 2022-04-12T16:07:44.647+0300 7f2a3b460400 10 monclient(hunting): authenticate will time out at 2022-04-12T16:12:44.651302+0300
-1143> 2022-04-12T16:07:44.647+0300 7f2a270c3700 10 monclient(hunting): get_auth_request con 0x559e54b39000 auth_method 0
-1142> 2022-04-12T16:07:44.647+0300 7f2a270c3700 10 monclient(hunting): get_auth_request method 2 preferred_modes [2,1]
-1141> 2022-04-12T16:07:44.647+0300 7f2a270c3700 10 monclient(hunting): _init_auth method 2
-1140> 2022-04-12T16:07:44.647+0300 7f2a270c3700 10 monclient(hunting): _init_auth creating new auth
-1139> 2022-04-12T16:07:44.647+0300 7f2a270c3700 10 monclient(hunting): handle_auth_reply_more payload 9
-1138> 2022-04-12T16:07:44.647+0300 7f2a270c3700 10 monclient(hunting): handle_auth_reply_more payload_len 9
-1137> 2022-04-12T16:07:44.647+0300 7f2a270c3700 10 monclient(hunting): handle_auth_reply_more responding with 36 bytes
-1136> 2022-04-12T16:07:44.647+0300 7f2a270c3700 10 monclient(hunting): handle_auth_done global_id 5715 payload 322
-1135> 2022-04-12T16:07:44.647+0300 7f2a270c3700 10 monclient: _finish_hunting 0
-1134> 2022-04-12T16:07:44.647+0300 7f2a270c3700 1 monclient: found mon.noname-a
-1133> 2022-04-12T16:07:44.647+0300 7f2a270c3700 10 monclient: _send_mon_message to mon.noname-a at v2:192.168.166.83:3300/0
-1132> 2022-04-12T16:07:44.647+0300 7f2a258c0700 10 monclient: handle_monmap mon_map magic: 0 v1
-1131> 2022-04-12T16:07:44.647+0300 7f2a258c0700 10 monclient: got monmap 1 from mon.noname-a (according to old e1)
-1130> 2022-04-12T16:07:44.647+0300 7f2a258c0700 10 monclient: dump:
epoch 1
fsid 65ccd25a-39ce-4072-a5c6-a0e6654bda87
last_changed 2022-04-12T15:48:27.663651+0300
created 2022-04-12T15:48:27.663651+0300
min_mon_release 16 (pacific)
election_strategy: 1
0: [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] mon.ceph-ss-test-oct-controller-av-01
1: [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] mon.ceph-ss-test-oct-controller-av-02
2: [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] mon.ceph-ss-test-oct-controller-av-03

-1129> 2022-04-12T16:07:44.647+0300 7f2a258c0700 10 monclient: _finish_auth 0
-1128> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 monclient: authenticate success, global_id 5715
-1127> 2022-04-12T16:07:44.647+0300 7f2a258c0700 10 monclient: handle_config config(0 keys) v1
-1126> 2022-04-12T16:07:44.647+0300 7f2a3b460400 10 monclient: get_monmap_and_config success
-1125> 2022-04-12T16:07:44.647+0300 7f2a3b460400 4 set_mon_vals no callback set
-1124> 2022-04-12T16:07:44.647+0300 7f2a3b460400 10 monclient: shutdown
-1123> 2022-04-12T16:07:44.647+0300 7f2a3b460400 0 deferred set uid:gid to 167:167 (ceph:ceph)
-1122> 2022-04-12T16:07:44.647+0300 7f2a3b460400 0 ceph version 16.2.7 (dd0603118f56ab514f133c8d2e3adfc983942503) pacific (stable), process radosgw, pid 54
-1121> 2022-04-12T16:07:44.647+0300 7f2a3b460400 0 framework: beast
-1120> 2022-04-12T16:07:44.647+0300 7f2a3b460400 0 framework conf key: endpoint, val: 192.168.166.83:8081
-1119> 2022-04-12T16:07:44.647+0300 7f2a3b460400 1 radosgw_Main not setting numa affinity
-1118> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 asok(0x559e54b34000) init /var/run/ceph/av-ceph-test-client.rgw.ceph-ss-test-oct-controller-av-01.rgw1.54.94138514240960.asok
-1117> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 asok(0x559e54b34000) bind_and_listen /var/run/ceph/av-ceph-test-client.rgw.ceph-ss-test-oct-controller-av-01.rgw1.54.94138514240960.asok
-1116> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command 0 hook 0x559e54a6c360
-1115> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command version hook 0x559e54a6c360
-1114> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command git_version hook 0x559e54a6c360
-1113> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command help hook 0x559e54a2a4b0
-1112> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command get_command_descriptions hook 0x559e54a2a4c0
-1111> 2022-04-12T16:07:44.651+0300 7f2a250bf700 5 asok(0x559e54b34000) entry start
-1110> 2022-04-12T16:07:44.651+0300 7f2a3b460400 10 monclient: get_monmap_and_config
-1109> 2022-04-12T16:07:44.651+0300 7f2a3b460400 10 monclient: build_initial_monmap
-1108> 2022-04-12T16:07:44.651+0300 7f2a3b460400 10 monclient: monmap:
epoch 0
fsid 00000000-0000-0000-0000-000000000000
last_changed 0.000000
created 0.000000
min_mon_release 0 (unknown)
election_strategy: 1
0: [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] mon.noname-a
1: [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] mon.noname-b
2: [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] mon.noname-c

-1107> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding auth protocol: cephx
-1106> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding auth protocol: cephx
-1105> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding auth protocol: cephx
-1104> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding auth protocol: none
-1103> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: secure
-1102> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: crc
-1101> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: secure
-1100> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: crc
-1099> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: secure
-1098> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: crc
-1097> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: crc
-1096> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: secure
-1095> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: crc
-1094> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: secure
-1093> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: crc
-1092> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: secure
-1091> 2022-04-12T16:07:44.651+0300 7f2a3b460400 2 auth: KeyRing::load: loaded key file /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring
-1090> 2022-04-12T16:07:44.651+0300 7f2a3b460400 10 monclient: init
-1089> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d6fa0) adding auth protocol: cephx
-1088> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d6fa0) adding auth protocol: cephx
-1087> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d6fa0) adding auth protocol: cephx
-1086> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d6fa0) adding auth protocol: none
-1085> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d6fa0) adding con mode: secure
-1084> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d6fa0) adding con mode: crc
-1083> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d6fa0) adding con mode: secure
-1082> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d6fa0) adding con mode: crc
-1081> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d6fa0) adding con mode: secure
-1080> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d6fa0) adding con mode: crc
-1079> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d6fa0) adding con mode: crc
-1078> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d6fa0) adding con mode: secure
-1077> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d6fa0) adding con mode: crc
-1076> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d6fa0) adding con mode: secure
-1075> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d6fa0) adding con mode: crc
-1074> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d6fa0) adding con mode: secure
-1073> 2022-04-12T16:07:44.651+0300 7f2a3b460400 2 auth: KeyRing::load: loaded key file /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring
-1072> 2022-04-12T16:07:44.651+0300 7f2a3b460400 2 auth: KeyRing::load: loaded key file /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring
-1071> 2022-04-12T16:07:44.651+0300 7f2a3b460400 10 monclient: _reopen_session rank -1
-1070> 2022-04-12T16:07:44.651+0300 7f2a3b460400 10 monclient: _add_conns ranks=[0,1,2]
-1069> 2022-04-12T16:07:44.651+0300 7f2a3b460400 10 monclient(hunting): picked mon.noname-a con 0x559e54b39000 addr [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0]
-1068> 2022-04-12T16:07:44.651+0300 7f2a3b460400 10 monclient(hunting): picked mon.noname-b con 0x559e54b38c00 addr [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0]
-1067> 2022-04-12T16:07:44.651+0300 7f2a3b460400 10 monclient(hunting): picked mon.noname-c con 0x559e54b38800 addr [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0]
-1066> 2022-04-12T16:07:44.651+0300 7f2a3b460400 10 monclient(hunting): start opening mon connection
-1065> 2022-04-12T16:07:44.651+0300 7f2a3b460400 10 monclient(hunting): start opening mon connection
-1064> 2022-04-12T16:07:44.651+0300 7f2a3b460400 10 monclient(hunting): start opening mon connection
-1063> 2022-04-12T16:07:44.651+0300 7f2a3b460400 10 monclient(hunting): _renew_subs
-1062> 2022-04-12T16:07:44.651+0300 7f2a3b460400 10 monclient(hunting): authenticate will time out at 2022-04-12T16:12:44.656631+0300
-1061> 2022-04-12T16:07:44.651+0300 7f2a268c2700 10 monclient(hunting): get_auth_request con 0x559e54b39000 auth_method 0
-1060> 2022-04-12T16:07:44.651+0300 7f2a268c2700 10 monclient(hunting): get_auth_request method 2 preferred_modes [2,1]
-1059> 2022-04-12T16:07:44.651+0300 7f2a268c2700 10 monclient(hunting): _init_auth method 2
-1058> 2022-04-12T16:07:44.651+0300 7f2a268c2700 10 monclient(hunting): _init_auth creating new auth
-1057> 2022-04-12T16:07:44.651+0300 7f2a268c2700 10 monclient(hunting): handle_auth_reply_more payload 9
-1056> 2022-04-12T16:07:44.651+0300 7f2a268c2700 10 monclient(hunting): handle_auth_reply_more payload_len 9
-1055> 2022-04-12T16:07:44.651+0300 7f2a268c2700 10 monclient(hunting): handle_auth_reply_more responding with 36 bytes
-1054> 2022-04-12T16:07:44.651+0300 7f2a268c2700 10 monclient(hunting): handle_auth_done global_id 5718 payload 322
-1053> 2022-04-12T16:07:44.651+0300 7f2a268c2700 10 monclient: _finish_hunting 0
-1052> 2022-04-12T16:07:44.651+0300 7f2a268c2700 1 monclient: found mon.noname-a
-1051> 2022-04-12T16:07:44.651+0300 7f2a268c2700 10 monclient: _send_mon_message to mon.noname-a at v2:192.168.166.83:3300/0
-1050> 2022-04-12T16:07:44.651+0300 7f2a218b8700 10 monclient: handle_monmap mon_map magic: 0 v1
-1049> 2022-04-12T16:07:44.651+0300 7f2a218b8700 10 monclient: got monmap 1 from mon.noname-a (according to old e1)
-1048> 2022-04-12T16:07:44.651+0300 7f2a218b8700 10 monclient: dump:
epoch 1
fsid 65ccd25a-39ce-4072-a5c6-a0e6654bda87
last_changed 2022-04-12T15:48:27.663651+0300
created 2022-04-12T15:48:27.663651+0300
min_mon_release 16 (pacific)
election_strategy: 1
0: [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] mon.ceph-ss-test-oct-controller-av-01
1: [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] mon.ceph-ss-test-oct-controller-av-02
2: [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] mon.ceph-ss-test-oct-controller-av-03

-1047> 2022-04-12T16:07:44.651+0300 7f2a218b8700 10 monclient: _finish_auth 0
-1046> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 monclient: authenticate success, global_id 5718
-1045> 2022-04-12T16:07:44.651+0300 7f2a218b8700 10 monclient: handle_config config(0 keys) v1
-1044> 2022-04-12T16:07:44.651+0300 7f2a3b460400 10 monclient: get_monmap_and_config success
-1043> 2022-04-12T16:07:44.651+0300 7f2a3b460400 4 set_mon_vals no callback set
-1042> 2022-04-12T16:07:44.651+0300 7f2a3b460400 10 monclient: shutdown
-1041> 2022-04-12T16:07:44.655+0300 7f2a3b460400 10 monclient: build_initial_monmap
-1040> 2022-04-12T16:07:44.655+0300 7f2a3b460400 10 monclient: monmap:
epoch 0
fsid 00000000-0000-0000-0000-000000000000
last_changed 0.000000
created 0.000000
min_mon_release 0 (unknown)
election_strategy: 1
0: [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] mon.noname-a
1: [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] mon.noname-b
2: [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] mon.noname-c

-1039> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding auth protocol: cephx
-1038> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding auth protocol: cephx
-1037> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding auth protocol: cephx
-1036> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding auth protocol: none
-1035> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: secure
-1034> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: crc
-1033> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: secure
-1032> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: crc
-1031> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: secure
-1030> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: crc
-1029> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: crc
-1028> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: secure
-1027> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: crc
-1026> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: secure
-1025> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: crc
-1024> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: secure
-1023> 2022-04-12T16:07:44.655+0300 7f2a3b460400 2 auth: KeyRing::load: loaded key file /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring
-1022> 2022-04-12T16:07:44.655+0300 7f2a3b460400 1 librados: starting msgr at
-1021> 2022-04-12T16:07:44.655+0300 7f2a3b460400 1 librados: starting objecter
-1020> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command objecter_requests hook 0x559e54a2a510
-1019> 2022-04-12T16:07:44.655+0300 7f2a3b460400 1 librados: setting wanted keys
-1018> 2022-04-12T16:07:44.655+0300 7f2a3b460400 1 librados: calling monclient init
-1017> 2022-04-12T16:07:44.655+0300 7f2a3b460400 10 monclient: init
-1016> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54a700d8) adding auth protocol: cephx
-1015> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54a700d8) adding auth protocol: cephx
-1014> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54a700d8) adding auth protocol: cephx
-1013> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54a700d8) adding auth protocol: none
-1012> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54a700d8) adding con mode: secure
-1011> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54a700d8) adding con mode: crc
-1010> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54a700d8) adding con mode: secure
-1009> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54a700d8) adding con mode: crc
-1008> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54a700d8) adding con mode: secure
-1007> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54a700d8) adding con mode: crc
-1006> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54a700d8) adding con mode: crc
-1005> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54a700d8) adding con mode: secure
-1004> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54a700d8) adding con mode: crc
-1003> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54a700d8) adding con mode: secure
-1002> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54a700d8) adding con mode: crc
-1001> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54a700d8) adding con mode: secure
-1000> 2022-04-12T16:07:44.655+0300 7f2a3b460400 2 auth: KeyRing::load: loaded key file /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring
-999> 2022-04-12T16:07:44.655+0300 7f2a3b460400 2 auth: KeyRing::load: loaded key file /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring
-998> 2022-04-12T16:07:44.655+0300 7f2a3b460400 10 monclient: _reopen_session rank -1
-997> 2022-04-12T16:07:44.655+0300 7f2a3b460400 10 monclient: _add_conns ranks=[0,2,1]
-996> 2022-04-12T16:07:44.655+0300 7f2a3b460400 10 monclient(hunting): picked mon.noname-a con 0x559e54b38800 addr [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0]
-995> 2022-04-12T16:07:44.655+0300 7f2a3b460400 10 monclient(hunting): picked mon.noname-c con 0x559e54b38c00 addr [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0]
-994> 2022-04-12T16:07:44.655+0300 7f2a3b460400 10 monclient(hunting): picked mon.noname-b con 0x559e54b39000 addr [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0]
-993> 2022-04-12T16:07:44.655+0300 7f2a3b460400 10 monclient(hunting): start opening mon connection
-992> 2022-04-12T16:07:44.655+0300 7f2a3b460400 10 monclient(hunting): start opening mon connection
-991> 2022-04-12T16:07:44.655+0300 7f2a3b460400 10 monclient(hunting): start opening mon connection
-990> 2022-04-12T16:07:44.655+0300 7f2a3b460400 10 monclient(hunting): _renew_subs
-989> 2022-04-12T16:07:44.655+0300 7f2a3b460400 10 monclient(hunting): authenticate will time out at 2022-04-12T16:12:44.659081+0300
-988> 2022-04-12T16:07:44.655+0300 7f2a268c2700 10 monclient(hunting): get_auth_request con 0x559e54b38800 auth_method 0
-987> 2022-04-12T16:07:44.655+0300 7f2a268c2700 10 monclient(hunting): get_auth_request method 2 preferred_modes [2,1]
-986> 2022-04-12T16:07:44.655+0300 7f2a268c2700 10 monclient(hunting): _init_auth method 2
-985> 2022-04-12T16:07:44.655+0300 7f2a268c2700 10 monclient(hunting): _init_auth creating new auth
-984> 2022-04-12T16:07:44.655+0300 7f2a268c2700 10 monclient(hunting): handle_auth_reply_more payload 9
-983> 2022-04-12T16:07:44.655+0300 7f2a268c2700 10 monclient(hunting): handle_auth_reply_more payload_len 9
-982> 2022-04-12T16:07:44.655+0300 7f2a268c2700 10 monclient(hunting): handle_auth_reply_more responding with 36 bytes
-981> 2022-04-12T16:07:44.655+0300 7f2a268c2700 10 monclient(hunting): handle_auth_done global_id 5721 payload 1000
-980> 2022-04-12T16:07:44.655+0300 7f2a268c2700 10 monclient: _finish_hunting 0
-979> 2022-04-12T16:07:44.655+0300 7f2a268c2700 1 monclient: found mon.noname-a
-978> 2022-04-12T16:07:44.655+0300 7f2a268c2700 10 monclient: _send_mon_message to mon.noname-a at v2:192.168.166.83:3300/0
-977> 2022-04-12T16:07:44.655+0300 7f2a200b5700 10 monclient: handle_monmap mon_map magic: 0 v1
-976> 2022-04-12T16:07:44.655+0300 7f2a200b5700 10 monclient: got monmap 1 from mon.noname-a (according to old e1)
-975> 2022-04-12T16:07:44.655+0300 7f2a200b5700 10 monclient: dump:
epoch 1
fsid 65ccd25a-39ce-4072-a5c6-a0e6654bda87
last_changed 2022-04-12T15:48:27.663651+0300
created 2022-04-12T15:48:27.663651+0300
min_mon_release 16 (pacific)
election_strategy: 1
0: [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] mon.ceph-ss-test-oct-controller-av-01
1: [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] mon.ceph-ss-test-oct-controller-av-02
2: [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] mon.ceph-ss-test-oct-controller-av-03

-974> 2022-04-12T16:07:44.655+0300 7f2a200b5700 10 monclient: _finish_auth 0
-973> 2022-04-12T16:07:44.655+0300 7f2a200b5700 10 monclient: handle_config config(0 keys) v1
-972> 2022-04-12T16:07:44.655+0300 7f2a200b5700 10 monclient: handle_monmap mon_map magic: 0 v1
-971> 2022-04-12T16:07:44.655+0300 7f2a218b8700 4 set_mon_vals no callback set
-970> 2022-04-12T16:07:44.655+0300 7f2a200b5700 10 monclient: got monmap 1 from mon.ceph-ss-test-oct-controller-av-01 (according to old e1)
-969> 2022-04-12T16:07:44.655+0300 7f2a200b5700 10 monclient: dump:
epoch 1
fsid 65ccd25a-39ce-4072-a5c6-a0e6654bda87
last_changed 2022-04-12T15:48:27.663651+0300
created 2022-04-12T15:48:27.663651+0300
min_mon_release 16 (pacific)
election_strategy: 1
0: [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] mon.ceph-ss-test-oct-controller-av-01
1: [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] mon.ceph-ss-test-oct-controller-av-02
2: [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] mon.ceph-ss-test-oct-controller-av-03

-968> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 monclient: authenticate success, global_id 5721
-967> 2022-04-12T16:07:44.655+0300 7f2a3b460400 10 monclient: _renew_subs
-966> 2022-04-12T16:07:44.655+0300 7f2a3b460400 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-965> 2022-04-12T16:07:44.655+0300 7f2a3b460400 10 monclient: _renew_subs
-964> 2022-04-12T16:07:44.655+0300 7f2a3b460400 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-963> 2022-04-12T16:07:44.655+0300 7f2a3b460400 1 librados: init done
-962> 2022-04-12T16:07:44.659+0300 7f2a200b5700 4 mgrc handle_mgr_map Got map version 29
-961> 2022-04-12T16:07:44.659+0300 7f2a200b5700 4 mgrc handle_mgr_map Active mgr is now [v2:192.168.166.84:6800/56,v1:192.168.166.84:6801/56]
-960> 2022-04-12T16:07:44.659+0300 7f2a200b5700 4 mgrc reconnect Starting new session with [v2:192.168.166.84:6800/56,v1:192.168.166.84:6801/56]
-959> 2022-04-12T16:07:44.659+0300 7f2a260c1700 10 monclient: get_auth_request con 0x559e54b39800 auth_method 0
-958> 2022-04-12T16:07:44.659+0300 7f2a270c3700 10 monclient: get_auth_request con 0x559e54b38400 auth_method 0
-957> 2022-04-12T16:07:44.659+0300 7f2a268c2700 10 monclient: get_auth_request con 0x559e55914000 auth_method 0
-956> 2022-04-12T16:07:44.663+0300 7f2a260c1700 10 monclient: get_auth_request con 0x559e55914c00 auth_method 0
-955> 2022-04-12T16:07:44.663+0300 7f2a270c3700 10 monclient: get_auth_request con 0x559e55915800 auth_method 0
-954> 2022-04-12T16:07:44.667+0300 7f2a268c2700 10 monclient: get_auth_request con 0x559e55914800 auth_method 0
-953> 2022-04-12T16:07:44.667+0300 7f2a260c1700 10 monclient: get_auth_request con 0x559e55914400 auth_method 0
-952> 2022-04-12T16:07:44.671+0300 7f2a270c3700 10 monclient: get_auth_request con 0x559e55915c00 auth_method 0
-951> 2022-04-12T16:07:44.671+0300 7f2a268c2700 10 monclient: get_auth_request con 0x559e5591c000 auth_method 0
-950> 2022-04-12T16:07:44.671+0300 7f2a260c1700 10 monclient: get_auth_request con 0x559e55915400 auth_method 0
-949> 2022-04-12T16:07:44.675+0300 7f2a270c3700 10 monclient: get_auth_request con 0x559e55915000 auth_method 0
-948> 2022-04-12T16:07:44.675+0300 7f2a268c2700 10 monclient: get_auth_request con 0x559e5591c400 auth_method 0
-947> 2022-04-12T16:07:44.675+0300 7f2a260c1700 10 monclient: get_auth_request con 0x559e5591c800 auth_method 0
-946> 2022-04-12T16:07:44.679+0300 7f2a270c3700 10 monclient: get_auth_request con 0x559e5591d000 auth_method 0
-945> 2022-04-12T16:07:44.679+0300 7f2a268c2700 10 monclient: get_auth_request con 0x559e5591d400 auth_method 0
-944> 2022-04-12T16:07:44.679+0300 7f2a260c1700 10 monclient: get_auth_request con 0x559e5591cc00 auth_method 0
-943> 2022-04-12T16:07:44.683+0300 7f2a270c3700 10 monclient: get_auth_request con 0x559e5591d800 auth_method 0
-942> 2022-04-12T16:07:44.687+0300 7f2a268c2700 10 monclient: get_auth_request con 0x559e55978000 auth_method 0
-941> 2022-04-12T16:07:44.691+0300 7f2a3b460400 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-940> 2022-04-12T16:07:44.691+0300 7f2a200b5700 10 monclient: handle_get_version_reply finishing 1 version 64
-939> 2022-04-12T16:07:44.691+0300 7f2a3b460400 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-938> 2022-04-12T16:07:45.539+0300 7f2a200b5700 10 monclient: _renew_subs
-937> 2022-04-12T16:07:45.539+0300 7f2a200b5700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-936> 2022-04-12T16:07:45.539+0300 7f2a3b460400 10 start_mon_command cmd=[{"prefix": "osd pool application enable","pool": "sync.rgw.log","app": "rgw"}]
-935> 2022-04-12T16:07:45.539+0300 7f2a3b460400 10 monclient: _send_command 1 [{"prefix": "osd pool application enable","pool": "sync.rgw.log","app": "rgw"}]
-934> 2022-04-12T16:07:45.539+0300 7f2a3b460400 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-933> 2022-04-12T16:07:46.559+0300 7f2a200b5700 10 monclient: handle_mon_command_ack 1 [{"prefix": "osd pool application enable","pool": "sync.rgw.log","app": "rgw"}]
-932> 2022-04-12T16:07:46.559+0300 7f2a200b5700 10 monclient: _finish_command 1 = system:0 enabled application 'rgw' on pool 'sync.rgw.log'
-931> 2022-04-12T16:07:46.559+0300 7f2a3b460400 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-930> 2022-04-12T16:07:46.559+0300 7f2a200b5700 10 monclient: handle_get_version_reply finishing 2 version 66
-929> 2022-04-12T16:07:46.559+0300 7f2a218b8700 10 monclient: _renew_subs
-928> 2022-04-12T16:07:46.559+0300 7f2a218b8700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-927> 2022-04-12T16:07:46.835+0300 7f2a3b460400 5 sync module config (parsed representation):
{
"config": {
"source_bucket": "",
"target_path": "rgw-${zonegroup}-${sid}/${bucket}",
"connection_id": "",
"acls_id": "",
"connection": {
"id": "",
"endpoint": "https://storage.googleapis.com",
"host_style": "path",
"key": {
"access_key": "<gcs_access_key>",
"secret": "******"
}
}
},
"connections": [],
"acl_profiles": [],
"profiles": [
{
"name": "",
"config": {
"source_bucket": "",
"target_path": "rgw-${zonegroup}-${sid}/${bucket}",
"connection_id": "",
"acls_id": "",
"connection": {
"id": "",
"endpoint": "https://storage.googleapis.com",
"host_style": "path",
"key": {
"access_key": "<gcs_access_key>",
"secret": "******"
}
}
}
}
]
}

-926> 2022-04-12T16:07:46.835+0300 7f2a3b460400 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-925> 2022-04-12T16:07:46.835+0300 7f2a200b5700 10 monclient: handle_get_version_reply finishing 3 version 66
-924> 2022-04-12T16:07:46.835+0300 7f2a3b460400 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-923> 2022-04-12T16:07:47.563+0300 7f2a200b5700 10 monclient: _renew_subs
-922> 2022-04-12T16:07:47.563+0300 7f2a200b5700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-921> 2022-04-12T16:07:47.567+0300 7f2a3b460400 10 start_mon_command cmd=[{"prefix": "osd pool application enable","pool": "sync.rgw.control","app": "rgw"}]
-920> 2022-04-12T16:07:47.567+0300 7f2a3b460400 10 monclient: _send_command 2 [{"prefix": "osd pool application enable","pool": "sync.rgw.control","app": "rgw"}]
-919> 2022-04-12T16:07:47.567+0300 7f2a3b460400 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-918> 2022-04-12T16:07:47.655+0300 7f2a1f0b3700 10 monclient: tick
-917> 2022-04-12T16:07:48.575+0300 7f2a200b5700 10 monclient: handle_mon_command_ack 2 [{"prefix": "osd pool application enable","pool": "sync.rgw.control","app": "rgw"}]
-916> 2022-04-12T16:07:48.575+0300 7f2a200b5700 10 monclient: _finish_command 2 = system:0 enabled application 'rgw' on pool 'sync.rgw.control'
-915> 2022-04-12T16:07:48.575+0300 7f2a3b460400 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-914> 2022-04-12T16:07:48.579+0300 7f2a200b5700 10 monclient: handle_get_version_reply finishing 4 version 68
-913> 2022-04-12T16:07:48.579+0300 7f2a210b7700 10 monclient: _renew_subs
-912> 2022-04-12T16:07:48.579+0300 7f2a210b7700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-911> 2022-04-12T16:07:48.655+0300 7f2a1f0b3700 10 monclient: tick
-910> 2022-04-12T16:07:49.163+0300 7f2a3b460400 2 all 8 watchers are set, enabling cache
-909> 2022-04-12T16:07:49.163+0300 7f2a3b460400 5 rgw main: tl::expected<std::pair<boost::container::flat_map<long unsigned int, logback_generation>, obj_version>, boost::system::error_code> logback_generations::read(const DoutPrefixProvider*, optional_yield):409: oid=data_loggenerations_metadata not found
-908> 2022-04-12T16:07:49.167+0300 7f2a260c1700 10 monclient: get_auth_request con 0x559e55978400 auth_method 0
-907> 2022-04-12T16:07:49.211+0300 7f2a0e091700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start
-906> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d78b0) adding auth protocol: cephx
-905> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d78b0) adding auth protocol: cephx
-904> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d78b0) adding auth protocol: cephx
-903> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d78b0) adding auth protocol: none
-902> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d78b0) adding con mode: secure
-901> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d78b0) adding con mode: crc
-900> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d78b0) adding con mode: secure
-899> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d78b0) adding con mode: crc
-898> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d78b0) adding con mode: secure
-897> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d78b0) adding con mode: crc
-896> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d78b0) adding con mode: crc
-895> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d78b0) adding con mode: secure
-894> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d78b0) adding con mode: crc
-893> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d78b0) adding con mode: secure
-892> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d78b0) adding con mode: crc
-891> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d78b0) adding con mode: secure
-890> 2022-04-12T16:07:49.219+0300 7f2a3b460400 2 auth: KeyRing::load: loaded key file /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring
-889> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command cache list hook 0x559e54a2aa10
-888> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command cache inspect hook 0x559e54a2aa10
-887> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command cache erase hook 0x559e54a2aa10
-886> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command cache zap hook 0x559e54a2aa10
-885> 2022-04-12T16:07:49.219+0300 7f2a3b460400 10 monclient: get_monmap_and_config
-884> 2022-04-12T16:07:49.219+0300 7f2a3b460400 10 monclient: build_initial_monmap
-883> 2022-04-12T16:07:49.219+0300 7f2a3b460400 10 monclient: monmap:
epoch 0
fsid 00000000-0000-0000-0000-000000000000
last_changed 0.000000
created 0.000000
min_mon_release 0 (unknown)
election_strategy: 1
0: [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] mon.noname-a
1: [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] mon.noname-b
2: [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] mon.noname-c

-882> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding auth protocol: cephx
-881> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding auth protocol: cephx
-880> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding auth protocol: cephx
-879> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding auth protocol: none
-878> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: secure
-877> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: crc
-876> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: secure
-875> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: crc
-874> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: secure
-873> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: crc
-872> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: crc
-871> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: secure
-870> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: crc
-869> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: secure
-868> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: crc
-867> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: secure
-866> 2022-04-12T16:07:49.219+0300 7f2a3b460400 2 auth: KeyRing::load: loaded key file /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring
-865> 2022-04-12T16:07:49.219+0300 7f2a3b460400 10 monclient: init
-864> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding auth protocol: cephx
-863> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding auth protocol: cephx
-862> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding auth protocol: cephx
-861> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding auth protocol: none
-860> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: secure
-859> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: crc
-858> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: secure
-857> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: crc
-856> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: secure
-855> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: crc
-854> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: crc
-853> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: secure
-852> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: crc
-851> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: secure
-850> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: crc
-849> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: secure
-848> 2022-04-12T16:07:49.219+0300 7f2a3b460400 2 auth: KeyRing::load: loaded key file /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring
-847> 2022-04-12T16:07:49.219+0300 7f2a3b460400 2 auth: KeyRing::load: loaded key file /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring
-846> 2022-04-12T16:07:49.219+0300 7f2a3b460400 10 monclient: _reopen_session rank -1
-845> 2022-04-12T16:07:49.219+0300 7f2a3b460400 10 monclient: _add_conns ranks=[1,0,2]
-844> 2022-04-12T16:07:49.219+0300 7f2a3b460400 10 monclient(hunting): picked mon.noname-b con 0x559e55990400 addr [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0]
-843> 2022-04-12T16:07:49.219+0300 7f2a3b460400 10 monclient(hunting): picked mon.noname-a con 0x559e55991000 addr [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0]
-842> 2022-04-12T16:07:49.219+0300 7f2a3b460400 10 monclient(hunting): picked mon.noname-c con 0x559e55990c00 addr [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0]
-841> 2022-04-12T16:07:49.219+0300 7f2a3b460400 10 monclient(hunting): start opening mon connection
-840> 2022-04-12T16:07:49.219+0300 7f2a3b460400 10 monclient(hunting): start opening mon connection
-839> 2022-04-12T16:07:49.219+0300 7f2a3b460400 10 monclient(hunting): start opening mon connection
-838> 2022-04-12T16:07:49.219+0300 7f2a3b460400 10 monclient(hunting): _renew_subs
-837> 2022-04-12T16:07:49.219+0300 7f2a3b460400 10 monclient(hunting): authenticate will time out at 2022-04-12T16:12:49.224469+0300
-836> 2022-04-12T16:07:49.219+0300 7f2a260c1700 10 monclient(hunting): get_auth_request con 0x559e55991000 auth_method 0
-835> 2022-04-12T16:07:49.219+0300 7f2a260c1700 10 monclient(hunting): get_auth_request method 2 preferred_modes [2,1]
-834> 2022-04-12T16:07:49.219+0300 7f2a260c1700 10 monclient(hunting): _init_auth method 2
-833> 2022-04-12T16:07:49.219+0300 7f2a260c1700 10 monclient(hunting): _init_auth creating new auth
-832> 2022-04-12T16:07:49.219+0300 7f2a260c1700 10 monclient(hunting): handle_auth_reply_more payload 9
-831> 2022-04-12T16:07:49.219+0300 7f2a260c1700 10 monclient(hunting): handle_auth_reply_more payload_len 9
-830> 2022-04-12T16:07:49.219+0300 7f2a260c1700 10 monclient(hunting): handle_auth_reply_more responding with 36 bytes
-829> 2022-04-12T16:07:49.219+0300 7f2a268c2700 10 monclient(hunting): get_auth_request con 0x559e55990400 auth_method 0
-828> 2022-04-12T16:07:49.219+0300 7f2a268c2700 10 monclient(hunting): get_auth_request method 2 preferred_modes [2,1]
-827> 2022-04-12T16:07:49.219+0300 7f2a268c2700 10 monclient(hunting): _init_auth method 2
-826> 2022-04-12T16:07:49.219+0300 7f2a268c2700 10 monclient(hunting): _init_auth creating new auth
-825> 2022-04-12T16:07:49.219+0300 7f2a260c1700 10 monclient(hunting): handle_auth_done global_id 5724 payload 322
-824> 2022-04-12T16:07:49.219+0300 7f2a260c1700 10 monclient: _finish_hunting 0
-823> 2022-04-12T16:07:49.219+0300 7f2a260c1700 1 monclient: found mon.noname-a
-822> 2022-04-12T16:07:49.219+0300 7f2a260c1700 10 monclient: _send_mon_message to mon.noname-a at v2:192.168.166.83:3300/0
-821> 2022-04-12T16:07:49.219+0300 7f2a0d890700 10 monclient: handle_monmap mon_map magic: 0 v1
-820> 2022-04-12T16:07:49.219+0300 7f2a0d890700 10 monclient: got monmap 1 from mon.noname-a (according to old e1)
-819> 2022-04-12T16:07:49.219+0300 7f2a0d890700 10 monclient: dump:
epoch 1
fsid 65ccd25a-39ce-4072-a5c6-a0e6654bda87
last_changed 2022-04-12T15:48:27.663651+0300
created 2022-04-12T15:48:27.663651+0300
min_mon_release 16 (pacific)
election_strategy: 1
0: [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] mon.ceph-ss-test-oct-controller-av-01
1: [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] mon.ceph-ss-test-oct-controller-av-02
2: [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] mon.ceph-ss-test-oct-controller-av-03

-818> 2022-04-12T16:07:49.219+0300 7f2a0d890700 10 monclient: _finish_auth 0
-817> 2022-04-12T16:07:49.219+0300 7f2a0d890700 10 monclient: handle_config config(0 keys) v1
-816> 2022-04-12T16:07:49.219+0300 7f2a0d890700 10 monclient: handle_monmap mon_map magic: 0 v1
-815> 2022-04-12T16:07:49.219+0300 7f2a0d890700 10 monclient: got monmap 1 from mon.ceph-ss-test-oct-controller-av-01 (according to old e1)
-814> 2022-04-12T16:07:49.219+0300 7f2a0d890700 10 monclient: dump:
epoch 1
fsid 65ccd25a-39ce-4072-a5c6-a0e6654bda87
last_changed 2022-04-12T15:48:27.663651+0300
created 2022-04-12T15:48:27.663651+0300
min_mon_release 16 (pacific)
election_strategy: 1
0: [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] mon.ceph-ss-test-oct-controller-av-01
1: [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] mon.ceph-ss-test-oct-controller-av-02
2: [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] mon.ceph-ss-test-oct-controller-av-03

-813> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 monclient: authenticate success, global_id 5724
-812> 2022-04-12T16:07:49.219+0300 7f2a3b460400 10 monclient: get_monmap_and_config success
-811> 2022-04-12T16:07:49.219+0300 7f2a3b460400 4 set_mon_vals no callback set
-810> 2022-04-12T16:07:49.219+0300 7f2a3b460400 10 monclient: shutdown
-809> 2022-04-12T16:07:49.223+0300 7f2a3b460400 10 monclient: build_initial_monmap
-808> 2022-04-12T16:07:49.223+0300 7f2a3b460400 10 monclient: monmap:
epoch 0
fsid 00000000-0000-0000-0000-000000000000
last_changed 0.000000
created 0.000000
min_mon_release 0 (unknown)
election_strategy: 1
0: [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] mon.noname-a
1: [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] mon.noname-b
2: [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] mon.noname-c

-807> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding auth protocol: cephx
-806> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding auth protocol: cephx
-805> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding auth protocol: cephx
-804> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding auth protocol: none
-803> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: secure
-802> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: crc
-801> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: secure
-800> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: crc
-799> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: secure
-798> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: crc
-797> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: crc
-796> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: secure
-795> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: crc
-794> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: secure
-793> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: crc
-792> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: secure
-791> 2022-04-12T16:07:49.223+0300 7f2a3b460400 2 auth: KeyRing::load: loaded key file /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring
-790> 2022-04-12T16:07:49.223+0300 7f2a3b460400 1 librados: starting msgr at
-789> 2022-04-12T16:07:49.223+0300 7f2a3b460400 1 librados: starting objecter
-788> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command objecter_requests cmddesc objecter_requests hook 0x559e54a2ba40 EEXIST
-787> 2022-04-12T16:07:49.223+0300 7f2a3b460400 1 librados: setting wanted keys
-786> 2022-04-12T16:07:49.223+0300 7f2a3b460400 1 librados: calling monclient init
-785> 2022-04-12T16:07:49.223+0300 7f2a3b460400 10 monclient: init
-784> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e559dd0d8) adding auth protocol: cephx
-783> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e559dd0d8) adding auth protocol: cephx
-782> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e559dd0d8) adding auth protocol: cephx
-781> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e559dd0d8) adding auth protocol: none
-780> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e559dd0d8) adding con mode: secure
-779> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e559dd0d8) adding con mode: crc
-778> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e559dd0d8) adding con mode: secure
-777> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e559dd0d8) adding con mode: crc
-776> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e559dd0d8) adding con mode: secure
-775> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e559dd0d8) adding con mode: crc
-774> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e559dd0d8) adding con mode: crc
-773> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e559dd0d8) adding con mode: secure
-772> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e559dd0d8) adding con mode: crc
-771> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e559dd0d8) adding con mode: secure
-770> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e559dd0d8) adding con mode: crc
-769> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e559dd0d8) adding con mode: secure
-768> 2022-04-12T16:07:49.223+0300 7f2a3b460400 2 auth: KeyRing::load: loaded key file /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring
-767> 2022-04-12T16:07:49.223+0300 7f2a3b460400 2 auth: KeyRing::load: loaded key file /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring
-766> 2022-04-12T16:07:49.223+0300 7f2a3b460400 10 monclient: _reopen_session rank -1
-765> 2022-04-12T16:07:49.223+0300 7f2a3b460400 10 monclient: _add_conns ranks=[0,2,1]
-764> 2022-04-12T16:07:49.223+0300 7f2a3b460400 10 monclient(hunting): picked mon.noname-a con 0x559e55990400 addr [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0]
-763> 2022-04-12T16:07:49.223+0300 7f2a3b460400 10 monclient(hunting): picked mon.noname-c con 0x559e55990c00 addr [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0]
-762> 2022-04-12T16:07:49.223+0300 7f2a3b460400 10 monclient(hunting): picked mon.noname-b con 0x559e55991000 addr [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0]
-761> 2022-04-12T16:07:49.223+0300 7f2a3b460400 10 monclient(hunting): start opening mon connection
-760> 2022-04-12T16:07:49.223+0300 7f2a3b460400 10 monclient(hunting): start opening mon connection
-759> 2022-04-12T16:07:49.223+0300 7f2a3b460400 10 monclient(hunting): start opening mon connection
-758> 2022-04-12T16:07:49.223+0300 7f2a3b460400 10 monclient(hunting): _renew_subs
-757> 2022-04-12T16:07:49.223+0300 7f2a3b460400 10 monclient(hunting): authenticate will time out at 2022-04-12T16:12:49.227127+0300
-756> 2022-04-12T16:07:49.223+0300 7f2a268c2700 10 monclient(hunting): get_auth_request con 0x559e55990400 auth_method 0
-755> 2022-04-12T16:07:49.223+0300 7f2a268c2700 10 monclient(hunting): get_auth_request method 2 preferred_modes [2,1]
-754> 2022-04-12T16:07:49.223+0300 7f2a268c2700 10 monclient(hunting): _init_auth method 2
-753> 2022-04-12T16:07:49.223+0300 7f2a268c2700 10 monclient(hunting): _init_auth creating new auth
-752> 2022-04-12T16:07:49.223+0300 7f2a268c2700 10 monclient(hunting): handle_auth_reply_more payload 9
-751> 2022-04-12T16:07:49.223+0300 7f2a268c2700 10 monclient(hunting): handle_auth_reply_more payload_len 9
-750> 2022-04-12T16:07:49.223+0300 7f2a268c2700 10 monclient(hunting): handle_auth_reply_more responding with 36 bytes
-749> 2022-04-12T16:07:49.223+0300 7f2a268c2700 10 monclient(hunting): handle_auth_done global_id 5727 payload 1000
-748> 2022-04-12T16:07:49.223+0300 7f2a268c2700 10 monclient: _finish_hunting 0
-747> 2022-04-12T16:07:49.223+0300 7f2a268c2700 1 monclient: found mon.noname-a
-746> 2022-04-12T16:07:49.223+0300 7f2a268c2700 10 monclient: _send_mon_message to mon.noname-a at v2:192.168.166.83:3300/0
-745> 2022-04-12T16:07:49.223+0300 7f2a0c08d700 10 monclient: handle_monmap mon_map magic: 0 v1
-744> 2022-04-12T16:07:49.223+0300 7f2a0c08d700 10 monclient: got monmap 1 from mon.noname-a (according to old e1)
-743> 2022-04-12T16:07:49.223+0300 7f2a0c08d700 10 monclient: dump:
epoch 1
fsid 65ccd25a-39ce-4072-a5c6-a0e6654bda87
last_changed 2022-04-12T15:48:27.663651+0300
created 2022-04-12T15:48:27.663651+0300
min_mon_release 16 (pacific)
election_strategy: 1
0: [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] mon.ceph-ss-test-oct-controller-av-01
1: [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] mon.ceph-ss-test-oct-controller-av-02
2: [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] mon.ceph-ss-test-oct-controller-av-03

-742> 2022-04-12T16:07:49.223+0300 7f2a0c08d700 10 monclient: _finish_auth 0
-741> 2022-04-12T16:07:49.223+0300 7f2a0c08d700 10 monclient: handle_config config(0 keys) v1
-740> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 monclient: authenticate success, global_id 5727
-739> 2022-04-12T16:07:49.223+0300 7f2a3b460400 10 monclient: _renew_subs
-738> 2022-04-12T16:07:49.223+0300 7f2a0d890700 4 set_mon_vals no callback set
-737> 2022-04-12T16:07:49.223+0300 7f2a3b460400 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-736> 2022-04-12T16:07:49.223+0300 7f2a0c08d700 10 monclient: handle_monmap mon_map magic: 0 v1
-735> 2022-04-12T16:07:49.223+0300 7f2a0c08d700 10 monclient: got monmap 1 from mon.ceph-ss-test-oct-controller-av-01 (according to old e1)
-734> 2022-04-12T16:07:49.223+0300 7f2a0c08d700 10 monclient: dump:
epoch 1
fsid 65ccd25a-39ce-4072-a5c6-a0e6654bda87
last_changed 2022-04-12T15:48:27.663651+0300
created 2022-04-12T15:48:27.663651+0300
min_mon_release 16 (pacific)
election_strategy: 1
0: [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] mon.ceph-ss-test-oct-controller-av-01
1: [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] mon.ceph-ss-test-oct-controller-av-02
2: [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] mon.ceph-ss-test-oct-controller-av-03

-733> 2022-04-12T16:07:49.223+0300 7f2a3b460400 10 monclient: _renew_subs
-732> 2022-04-12T16:07:49.223+0300 7f2a3b460400 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-731> 2022-04-12T16:07:49.223+0300 7f2a3b460400 1 librados: init done
-730> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command cr dump hook 0x559e558a04e8
-729> 2022-04-12T16:07:49.227+0300 7f2a0c08d700 4 mgrc handle_mgr_map Got map version 29
-728> 2022-04-12T16:07:49.227+0300 7f2a0c08d700 4 mgrc handle_mgr_map Active mgr is now [v2:192.168.166.84:6800/56,v1:192.168.166.84:6801/56]
-727> 2022-04-12T16:07:49.227+0300 7f2a0c08d700 4 mgrc reconnect Starting new session with [v2:192.168.166.84:6800/56,v1:192.168.166.84:6801/56]
-726> 2022-04-12T16:07:49.227+0300 7f2a3b460400 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-725> 2022-04-12T16:07:49.227+0300 7f2a0c08d700 10 monclient: handle_get_version_reply finishing 1 version 68
-724> 2022-04-12T16:07:49.227+0300 7f2a260c1700 10 monclient: get_auth_request con 0x559e559cbc00 auth_method 0
-723> 2022-04-12T16:07:49.227+0300 7f2a3b460400 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-722> 2022-04-12T16:07:49.603+0300 7f2a0c08d700 10 monclient: _renew_subs
-721> 2022-04-12T16:07:49.603+0300 7f2a0c08d700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-720> 2022-04-12T16:07:49.607+0300 7f2a3b460400 10 start_mon_command cmd=[{"prefix": "osd pool application enable","pool": "sync.rgw.meta","app": "rgw"}]
-719> 2022-04-12T16:07:49.607+0300 7f2a3b460400 10 monclient: _send_command 1 [{"prefix": "osd pool application enable","pool": "sync.rgw.meta","app": "rgw"}]
-718> 2022-04-12T16:07:49.607+0300 7f2a3b460400 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-717> 2022-04-12T16:07:49.655+0300 7f2a1f0b3700 10 monclient: tick
-716> 2022-04-12T16:07:49.655+0300 7f2a208b6700 10 monclient: _renew_subs
-715> 2022-04-12T16:07:49.655+0300 7f2a208b6700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-714> 2022-04-12T16:07:50.611+0300 7f2a0c08d700 10 monclient: handle_mon_command_ack 1 [{"prefix": "osd pool application enable","pool": "sync.rgw.meta","app": "rgw"}]
-713> 2022-04-12T16:07:50.611+0300 7f2a0c08d700 10 monclient: _finish_command 1 = system:0 enabled application 'rgw' on pool 'sync.rgw.meta'
-712> 2022-04-12T16:07:50.611+0300 7f2a3b460400 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-711> 2022-04-12T16:07:50.615+0300 7f2a0c08d700 10 monclient: handle_get_version_reply finishing 2 version 70
-710> 2022-04-12T16:07:50.615+0300 7f2a0d890700 10 monclient: _renew_subs
-709> 2022-04-12T16:07:50.615+0300 7f2a0d890700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-708> 2022-04-12T16:07:50.615+0300 7f2a3b460400 10 start_mon_command cmd=[{"prefix": "osd pool set", "pool": "sync.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]
-707> 2022-04-12T16:07:50.615+0300 7f2a3b460400 10 monclient: _send_command 2 [{"prefix": "osd pool set", "pool": "sync.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]
-706> 2022-04-12T16:07:50.615+0300 7f2a3b460400 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-705> 2022-04-12T16:07:50.655+0300 7f2a1f0b3700 10 monclient: tick
-704> 2022-04-12T16:07:51.615+0300 7f2a0c08d700 10 monclient: handle_mon_command_ack 2 [{"prefix": "osd pool set", "pool": "sync.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]
-703> 2022-04-12T16:07:51.615+0300 7f2a0c08d700 10 monclient: _finish_command 2 = system:0 set pool 11 pg_autoscale_bias to 4
-702> 2022-04-12T16:07:51.615+0300 7f2a3b460400 10 start_mon_command cmd=[{"prefix": "osd pool set", "pool": "sync.rgw.meta", "var": "pg_num_min", "val": "8"}]
-701> 2022-04-12T16:07:51.615+0300 7f2a3b460400 10 monclient: _send_command 3 [{"prefix": "osd pool set", "pool": "sync.rgw.meta", "var": "pg_num_min", "val": "8"}]
-700> 2022-04-12T16:07:51.615+0300 7f2a3b460400 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-699> 2022-04-12T16:07:51.655+0300 7f2a1f0b3700 10 monclient: tick
-698> 2022-04-12T16:07:52.223+0300 7f2a0b08b700 10 monclient: tick
-697> 2022-04-12T16:07:52.619+0300 7f2a0c08d700 10 monclient: handle_mon_command_ack 3 [{"prefix": "osd pool set", "pool": "sync.rgw.meta", "var": "pg_num_min", "val": "8"}]
-696> 2022-04-12T16:07:52.619+0300 7f2a0c08d700 10 monclient: _finish_command 3 = system:0 set pool 11 pg_num_min to 8
-695> 2022-04-12T16:07:52.619+0300 7f2a3b460400 10 start_mon_command cmd=[{"prefix": "osd pool set", "pool": "sync.rgw.meta", "var": "recovery_priority": "5"}]
-694> 2022-04-12T16:07:52.619+0300 7f2a3b460400 10 monclient: _send_command 4 [{"prefix": "osd pool set", "pool": "sync.rgw.meta", "var": "recovery_priority": "5"}]
-693> 2022-04-12T16:07:52.619+0300 7f2a3b460400 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-692> 2022-04-12T16:07:52.623+0300 7f2a0c08d700 10 monclient: handle_mon_command_ack 4 [{"prefix": "osd pool set", "pool": "sync.rgw.meta", "var": "recovery_priority": "5"}]
-691> 2022-04-12T16:07:52.623+0300 7f2a0c08d700 10 monclient: _finish_command 4 = mon:22 unparseable JSON {"prefix": "osd pool set", "pool": "sync.rgw.meta", "var": "recovery_priority": "5"}
-690> 2022-04-12T16:07:52.623+0300 7f2a270c3700 10 monclient: get_auth_request con 0x559e559f1400 auth_method 0
-689> 2022-04-12T16:07:52.627+0300 7f2a268c2700 10 monclient: get_auth_request con 0x559e559cac00 auth_method 0
-688> 2022-04-12T16:07:52.635+0300 7f2a260c1700 10 monclient: get_auth_request con 0x559e559ca000 auth_method 0
-687> 2022-04-12T16:07:52.639+0300 7f2a270c3700 10 monclient: get_auth_request con 0x559e55b21400 auth_method 0
-686> 2022-04-12T16:07:52.647+0300 7f2a268c2700 10 monclient: get_auth_request con 0x559e54b39c00 auth_method 0
-685> 2022-04-12T16:07:52.651+0300 7f2a260c1700 10 monclient: get_auth_request con 0x559e559cb000 auth_method 0
-684> 2022-04-12T16:07:52.659+0300 7f2a1f0b3700 10 monclient: tick
-683> 2022-04-12T16:07:52.659+0300 7f2a270c3700 10 monclient: get_auth_request con 0x559e55990000 auth_method 0
-682> 2022-04-12T16:07:52.671+0300 7f2a268c2700 10 monclient: get_auth_request con 0x559e559f1800 auth_method 0
-681> 2022-04-12T16:07:52.699+0300 7f2a260c1700 10 monclient: get_auth_request con 0x559e55b51800 auth_method 0
-680> 2022-04-12T16:07:52.711+0300 7f2a270c3700 10 monclient: get_auth_request con 0x559e55b50000 auth_method 0
-679> 2022-04-12T16:07:52.723+0300 7f2a268c2700 10 monclient: get_auth_request con 0x559e559f0400 auth_method 0
-678> 2022-04-12T16:07:52.731+0300 7f2a260c1700 10 monclient: get_auth_request con 0x559e559ca400 auth_method 0
-677> 2022-04-12T16:07:52.743+0300 7f2a270c3700 10 monclient: get_auth_request con 0x559e55979c00 auth_method 0
-676> 2022-04-12T16:07:52.747+0300 7f2a09888700 2 object expiration: start
-675> 2022-04-12T16:07:52.747+0300 7f2a0a089700 2 garbage collection: garbage collection: start
-674> 2022-04-12T16:07:52.747+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command sync trace show hook 0x559e5592a1c0
-673> 2022-04-12T16:07:52.747+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command sync trace history hook 0x559e5592a1c0
-672> 2022-04-12T16:07:52.747+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command sync trace active hook 0x559e5592a1c0
-671> 2022-04-12T16:07:52.747+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command sync trace active_short hook 0x559e5592a1c0
-670> 2022-04-12T16:07:52.751+0300 7f2a3b460400 4 rgw main: failed to read sync status info with (2) No such file or directory
-669> 2022-04-12T16:07:52.755+0300 7f2a07884700 4 rgw rados thread: failed to read sync status info with (2) No such file or directory
-668> 2022-04-12T16:07:52.755+0300 7f2a07884700 1 RGW-SYNC:meta: initialize sync (reason: period is empty)
-667> 2022-04-12T16:07:52.755+0300 7f2a07884700 1 rgw rados thread: epoch=0 in sync status comes before remote's oldest mdlog epoch=1, restarting sync
-666> 2022-04-12T16:07:52.759+0300 7f2a3b460400 5 rgw main: starting data sync thread for zone avrupa
-665> 2022-04-12T16:07:52.759+0300 7f29ff073700 5 lifecycle: schedule life cycle next start time: Tue Apr 12 21:00:00 2022
-664> 2022-04-12T16:07:52.759+0300 7f2a01077700 5 lifecycle: schedule life cycle next start time: Tue Apr 12 21:00:00 2022
-663> 2022-04-12T16:07:52.759+0300 7f29fd06f700 5 lifecycle: schedule life cycle next start time: Tue Apr 12 21:00:00 2022
-662> 2022-04-12T16:07:52.763+0300 7f2a3b460400 0 framework: beast
-661> 2022-04-12T16:07:52.763+0300 7f2a3b460400 0 framework conf key: ssl_certificate, val: config://rgw/cert/$realm/$zone.crt
-660> 2022-04-12T16:07:52.763+0300 7f2a3b460400 0 framework conf key: ssl_private_key, val: config://rgw/cert/$realm/$zone.key
-659> 2022-04-12T16:07:52.763+0300 7f2a3b460400 0 starting handler: beast
-658> 2022-04-12T16:07:52.763+0300 7f2a06882700 4 rgw rados thread: failed to read sync status info with (2) No such file or directory
-657> 2022-04-12T16:07:52.763+0300 7f2a3b460400 4 frontend listening on 192.168.166.83:8081
-656> 2022-04-12T16:07:52.763+0300 7f2a3b460400 0 set uid:gid to 167:167 (ceph:ceph)
-655> 2022-04-12T16:07:52.763+0300 7f2a3b460400 4 frontend spawning 512 threads
-654> 2022-04-12T16:07:52.799+0300 7f2a3b460400 1 mgrc service_daemon_register rgw.5727 metadata {arch=x86_64,ceph_release=pacific,ceph_version=ceph version 16.2.7 (dd0603118f56ab514f133c8d2e3adfc983942503) pacific (stable),ceph_version_short=16.2.7,container_image=gitlab.example.com:5050/openstack/registry/ceph/daemon:v6.0.6-stable-6.0-pacific-centos-8-x86_64-ceph-ansible-20220121,cpu=Intel(R) Xeon(R) Gold 6230R CPU @ 2.10GHz,distro=centos,distro_description=CentOS Linux 8,distro_version=8,frontend_config#0=beast endpoint=192.168.166.83:8081,frontend_type#0=beast,hostname=ceph-ss-test-oct-controller-av-01,id=ceph-ss-test-oct-controller-av-01.rgw1,kernel_description=#121-Ubuntu SMP Thu Mar 24 16:04:27 UTC 2022,kernel_version=5.4.0-107-generic,mem_cgroup_limit=4294967296,mem_swap_kb=4030460,mem_total_kb=8148192,num_handles=1,os=Linux,pid=54,realm_id=532f2481-54e3-4d51-95fd-19867ca818e9,realm_name=myrealm,zone_id=c81e6d95-54da-48d3-83a4-7bb7b286709f,zone_name=sync,zonegroup_id=784a05a5-2089-4564-ab1b-a55c43c5f401,zonegroup_name=cdn}
-653> 2022-04-12T16:07:52.807+0300 7f2a0c08d700 4 mgrc handle_mgr_configure stats_period=5
-652> 2022-04-12T16:07:52.807+0300 7f2a0c08d700 4 mgrc handle_mgr_configure updated stats threshold: 5
-651> 2022-04-12T16:07:52.807+0300 7f2a3b460400 10 monclient: get_monmap_and_config
-650> 2022-04-12T16:07:52.807+0300 7f2a3b460400 10 monclient: build_initial_monmap
-649> 2022-04-12T16:07:52.807+0300 7f2a3b460400 10 monclient: monmap:
epoch 0
fsid 00000000-0000-0000-0000-000000000000
last_changed 0.000000
created 0.000000
min_mon_release 0 (unknown)
election_strategy: 1
0: [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] mon.noname-a
1: [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] mon.noname-b
2: [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] mon.noname-c

-648> 2022-04-12T16:07:52.807+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding auth protocol: cephx
-647> 2022-04-12T16:07:52.807+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding auth protocol: cephx
-646> 2022-04-12T16:07:52.807+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding auth protocol: cephx
-645> 2022-04-12T16:07:52.807+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding auth protocol: none
-644> 2022-04-12T16:07:52.807+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: secure
-643> 2022-04-12T16:07:52.807+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: crc
-642> 2022-04-12T16:07:52.807+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: secure
-641> 2022-04-12T16:07:52.807+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: crc
-640> 2022-04-12T16:07:52.807+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: secure
-639> 2022-04-12T16:07:52.807+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: crc
-638> 2022-04-12T16:07:52.807+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: crc
-637> 2022-04-12T16:07:52.807+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: secure
-636> 2022-04-12T16:07:52.807+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: crc
-635> 2022-04-12T16:07:52.807+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: secure
-634> 2022-04-12T16:07:52.807+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: crc
-633> 2022-04-12T16:07:52.807+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: secure
-632> 2022-04-12T16:07:52.807+0300 7f2a3b460400 2 auth: KeyRing::load: loaded key file /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring
-631> 2022-04-12T16:07:52.819+0300 7f2a3b460400 10 monclient: init
-630> 2022-04-12T16:07:52.819+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding auth protocol: cephx
-629> 2022-04-12T16:07:52.819+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding auth protocol: cephx
-628> 2022-04-12T16:07:52.819+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding auth protocol: cephx
-627> 2022-04-12T16:07:52.819+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding auth protocol: none
-626> 2022-04-12T16:07:52.819+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: secure
-625> 2022-04-12T16:07:52.819+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: crc
-624> 2022-04-12T16:07:52.819+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: secure
-623> 2022-04-12T16:07:52.819+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: crc
-622> 2022-04-12T16:07:52.819+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: secure
-621> 2022-04-12T16:07:52.819+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: crc
-620> 2022-04-12T16:07:52.819+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: crc
-619> 2022-04-12T16:07:52.819+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: secure
-618> 2022-04-12T16:07:52.819+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: crc
-617> 2022-04-12T16:07:52.819+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: secure
-616> 2022-04-12T16:07:52.819+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: crc
-615> 2022-04-12T16:07:52.819+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: secure
-614> 2022-04-12T16:07:52.819+0300 7f2a3b460400 2 auth: KeyRing::load: loaded key file /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring
-613> 2022-04-12T16:07:52.819+0300 7f2a3b460400 2 auth: KeyRing::load: loaded key file /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring
-612> 2022-04-12T16:07:52.819+0300 7f2a3b460400 10 monclient: _reopen_session rank -1
-611> 2022-04-12T16:07:52.819+0300 7f2a3b460400 10 monclient: _add_conns ranks=[1,0,2]
-610> 2022-04-12T16:07:52.819+0300 7f2a3b460400 10 monclient(hunting): picked mon.noname-b con 0x559e55db1c00 addr [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0]
-609> 2022-04-12T16:07:52.819+0300 7f2a3b460400 10 monclient(hunting): picked mon.noname-a con 0x559e559f1000 addr [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0]
-608> 2022-04-12T16:07:52.819+0300 7f2a3b460400 10 monclient(hunting): picked mon.noname-c con 0x559e559f0800 addr [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0]
-607> 2022-04-12T16:07:52.819+0300 7f2a3b460400 10 monclient(hunting): start opening mon connection
-606> 2022-04-12T16:07:52.819+0300 7f2a3b460400 10 monclient(hunting): start opening mon connection
-605> 2022-04-12T16:07:52.819+0300 7f2a3b460400 10 monclient(hunting): start opening mon connection
-604> 2022-04-12T16:07:52.819+0300 7f2a3b460400 10 monclient(hunting): _renew_subs
-603> 2022-04-12T16:07:52.819+0300 7f2a3b460400 10 monclient(hunting): authenticate will time out at 2022-04-12T16:12:52.824021+0300
-602> 2022-04-12T16:07:52.819+0300 7f2a260c1700 10 monclient(hunting): get_auth_request con 0x559e55db1c00 auth_method 0
-601> 2022-04-12T16:07:52.819+0300 7f2a260c1700 10 monclient(hunting): get_auth_request method 2 preferred_modes [2,1]
-600> 2022-04-12T16:07:52.819+0300 7f2a260c1700 10 monclient(hunting): _init_auth method 2
-599> 2022-04-12T16:07:52.819+0300 7f2a260c1700 10 monclient(hunting): _init_auth creating new auth
-598> 2022-04-12T16:07:52.819+0300 7f2a268c2700 10 monclient(hunting): get_auth_request con 0x559e559f0800 auth_method 0
-597> 2022-04-12T16:07:52.819+0300 7f2a268c2700 10 monclient(hunting): get_auth_request method 2 preferred_modes [2,1]
-596> 2022-04-12T16:07:52.819+0300 7f2a268c2700 10 monclient(hunting): _init_auth method 2
-595> 2022-04-12T16:07:52.819+0300 7f2a268c2700 10 monclient(hunting): _init_auth creating new auth
-594> 2022-04-12T16:07:52.819+0300 7f2a260c1700 10 monclient(hunting): handle_auth_reply_more payload 9
-593> 2022-04-12T16:07:52.819+0300 7f2a260c1700 10 monclient(hunting): handle_auth_reply_more payload_len 9
-592> 2022-04-12T16:07:52.819+0300 7f2a260c1700 10 monclient(hunting): handle_auth_reply_more responding with 36 bytes
-591> 2022-04-12T16:07:52.819+0300 7f2a268c2700 10 monclient(hunting): handle_auth_reply_more payload 9
-590> 2022-04-12T16:07:52.819+0300 7f2a268c2700 10 monclient(hunting): handle_auth_reply_more payload_len 9
-589> 2022-04-12T16:07:52.819+0300 7f2a268c2700 10 monclient(hunting): handle_auth_reply_more responding with 36 bytes
-588> 2022-04-12T16:07:52.819+0300 7f2a260c1700 10 monclient(hunting): handle_auth_done global_id 5197 payload 322
-587> 2022-04-12T16:07:52.819+0300 7f2a260c1700 10 monclient: _finish_hunting 0
-586> 2022-04-12T16:07:52.819+0300 7f2a260c1700 1 monclient: found mon.noname-b
-585> 2022-04-12T16:07:52.819+0300 7f2a260c1700 10 monclient: _send_mon_message to mon.noname-b at v2:192.168.166.84:3300/0
-584> 2022-04-12T16:07:52.819+0300 7f28f8e67700 10 monclient: handle_monmap mon_map magic: 0 v1
-583> 2022-04-12T16:07:52.819+0300 7f28f8e67700 10 monclient: got monmap 1 from mon.noname-b (according to old e1)
-582> 2022-04-12T16:07:52.819+0300 7f28f8e67700 10 monclient: dump:
epoch 1
fsid 65ccd25a-39ce-4072-a5c6-a0e6654bda87
last_changed 2022-04-12T15:48:27.663651+0300
created 2022-04-12T15:48:27.663651+0300
min_mon_release 16 (pacific)
election_strategy: 1
0: [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] mon.ceph-ss-test-oct-controller-av-01
1: [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] mon.ceph-ss-test-oct-controller-av-02
2: [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] mon.ceph-ss-test-oct-controller-av-03

-581> 2022-04-12T16:07:52.819+0300 7f28f8e67700 10 monclient: _finish_auth 0
-580> 2022-04-12T16:07:52.819+0300 7f28f8e67700 10 monclient: handle_config config(0 keys) v1
-579> 2022-04-12T16:07:52.819+0300 7f28f8e67700 10 monclient: handle_monmap mon_map magic: 0 v1
-578> 2022-04-12T16:07:52.819+0300 7f28f8e67700 10 monclient: got monmap 1 from mon.ceph-ss-test-oct-controller-av-02 (according to old e1)
-577> 2022-04-12T16:07:52.819+0300 7f28f8e67700 10 monclient: dump:
epoch 1
fsid 65ccd25a-39ce-4072-a5c6-a0e6654bda87
last_changed 2022-04-12T15:48:27.663651+0300
created 2022-04-12T15:48:27.663651+0300
min_mon_release 16 (pacific)
election_strategy: 1
0: [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] mon.ceph-ss-test-oct-controller-av-01
1: [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] mon.ceph-ss-test-oct-controller-av-02
2: [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] mon.ceph-ss-test-oct-controller-av-03

-576> 2022-04-12T16:07:52.819+0300 7f2a3b460400 5 monclient: authenticate success, global_id 5197
-575> 2022-04-12T16:07:52.819+0300 7f2a3b460400 10 monclient: get_monmap_and_config success
-574> 2022-04-12T16:07:52.819+0300 7f2a3b460400 4 set_mon_vals no callback set
-573> 2022-04-12T16:07:52.823+0300 7f2a3b460400 10 monclient: shutdown
-572> 2022-04-12T16:07:52.823+0300 7f2a3b460400 10 monclient: build_initial_monmap
-571> 2022-04-12T16:07:52.823+0300 7f2a3b460400 10 monclient: monmap:
epoch 0
fsid 00000000-0000-0000-0000-000000000000
last_changed 0.000000
created 0.000000
min_mon_release 0 (unknown)
election_strategy: 1
0: [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] mon.noname-a
1: [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] mon.noname-b
2: [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] mon.noname-c

-570> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding auth protocol: cephx
-569> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding auth protocol: cephx
-568> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding auth protocol: cephx
-567> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding auth protocol: none
-566> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: secure
-565> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: crc
-564> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: secure
-563> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: crc
-562> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: secure
-561> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: crc
-560> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: crc
-559> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: secure
-558> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: crc
-557> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: secure
-556> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: crc
-555> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: secure
-554> 2022-04-12T16:07:52.823+0300 7f2a3b460400 2 auth: KeyRing::load: loaded key file /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring
-553> 2022-04-12T16:07:52.823+0300 7f2a3b460400 1 librados: starting msgr at
-552> 2022-04-12T16:07:52.823+0300 7f2a3b460400 1 librados: starting objecter
-551> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command objecter_requests cmddesc objecter_requests hook 0x559e58ce73c0 EEXIST
-550> 2022-04-12T16:07:52.823+0300 7f2a3b460400 1 librados: setting wanted keys
-549> 2022-04-12T16:07:52.823+0300 7f2a3b460400 1 librados: calling monclient init
-548> 2022-04-12T16:07:52.823+0300 7f2a3b460400 10 monclient: init
-547> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e594860d8) adding auth protocol: cephx
-546> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e594860d8) adding auth protocol: cephx
-545> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e594860d8) adding auth protocol: cephx
-544> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e594860d8) adding auth protocol: none
-543> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e594860d8) adding con mode: secure
-542> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e594860d8) adding con mode: crc
-541> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e594860d8) adding con mode: secure
-540> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e594860d8) adding con mode: crc
-539> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e594860d8) adding con mode: secure
-538> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e594860d8) adding con mode: crc
-537> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e594860d8) adding con mode: crc
-536> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e594860d8) adding con mode: secure
-535> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e594860d8) adding con mode: crc
-534> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e594860d8) adding con mode: secure
-533> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e594860d8) adding con mode: crc
-532> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e594860d8) adding con mode: secure
-531> 2022-04-12T16:07:52.823+0300 7f2a3b460400 2 auth: KeyRing::load: loaded key file /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring
-530> 2022-04-12T16:07:52.823+0300 7f2a3b460400 2 auth: KeyRing::load: loaded key file /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring
-529> 2022-04-12T16:07:52.823+0300 7f2a3b460400 10 monclient: _reopen_session rank -1
-528> 2022-04-12T16:07:52.823+0300 7f2a3b460400 10 monclient: _add_conns ranks=[0,2,1]
-527> 2022-04-12T16:07:52.823+0300 7f2a3b460400 10 monclient(hunting): picked mon.noname-a con 0x559e559f0800 addr [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0]
-526> 2022-04-12T16:07:52.823+0300 7f2a3b460400 10 monclient(hunting): picked mon.noname-c con 0x559e559f1000 addr [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0]
-525> 2022-04-12T16:07:52.823+0300 7f2a3b460400 10 monclient(hunting): picked mon.noname-b con 0x559e55db1c00 addr [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0]
-524> 2022-04-12T16:07:52.823+0300 7f2a3b460400 10 monclient(hunting): start opening mon connection
-523> 2022-04-12T16:07:52.823+0300 7f2a3b460400 10 monclient(hunting): start opening mon connection
-522> 2022-04-12T16:07:52.823+0300 7f2a3b460400 10 monclient(hunting): start opening mon connection
-521> 2022-04-12T16:07:52.823+0300 7f2a3b460400 10 monclient(hunting): _renew_subs
-520> 2022-04-12T16:07:52.823+0300 7f2a3b460400 10 monclient(hunting): authenticate will time out at 2022-04-12T16:12:52.827867+0300
-519> 2022-04-12T16:07:52.823+0300 7f2a260c1700 10 monclient(hunting): get_auth_request con 0x559e559f0800 auth_method 0
-518> 2022-04-12T16:07:52.823+0300 7f2a260c1700 10 monclient(hunting): get_auth_request method 2 preferred_modes [2,1]
-517> 2022-04-12T16:07:52.823+0300 7f2a260c1700 10 monclient(hunting): _init_auth method 2
-516> 2022-04-12T16:07:52.823+0300 7f2a260c1700 10 monclient(hunting): _init_auth creating new auth
-515> 2022-04-12T16:07:52.823+0300 7f2a260c1700 10 monclient(hunting): handle_auth_reply_more payload 9
-514> 2022-04-12T16:07:52.823+0300 7f2a260c1700 10 monclient(hunting): handle_auth_reply_more payload_len 9
-513> 2022-04-12T16:07:52.823+0300 7f2a260c1700 10 monclient(hunting): handle_auth_reply_more responding with 36 bytes
-512> 2022-04-12T16:07:52.823+0300 7f2a270c3700 10 monclient(hunting): get_auth_request con 0x559e559f1000 auth_method 0
-511> 2022-04-12T16:07:52.823+0300 7f2a270c3700 10 monclient(hunting): get_auth_request method 2 preferred_modes [2,1]
-510> 2022-04-12T16:07:52.823+0300 7f2a270c3700 10 monclient(hunting): _init_auth method 2
-509> 2022-04-12T16:07:52.823+0300 7f2a270c3700 10 monclient(hunting): _init_auth creating new auth
-508> 2022-04-12T16:07:52.823+0300 7f2a260c1700 10 monclient(hunting): handle_auth_done global_id 5730 payload 1000
-507> 2022-04-12T16:07:52.823+0300 7f2a260c1700 10 monclient: _finish_hunting 0
-506> 2022-04-12T16:07:52.823+0300 7f2a260c1700 1 monclient: found mon.noname-a
-505> 2022-04-12T16:07:52.823+0300 7f2a260c1700 10 monclient: _send_mon_message to mon.noname-a at v2:192.168.166.83:3300/0
-504> 2022-04-12T16:07:52.823+0300 7f28f7664700 10 monclient: handle_monmap mon_map magic: 0 v1
-503> 2022-04-12T16:07:52.827+0300 7f28f7664700 10 monclient: got monmap 1 from mon.noname-a (according to old e1)
-502> 2022-04-12T16:07:52.827+0300 7f28f7664700 10 monclient: dump:
epoch 1
fsid 65ccd25a-39ce-4072-a5c6-a0e6654bda87
last_changed 2022-04-12T15:48:27.663651+0300
created 2022-04-12T15:48:27.663651+0300
min_mon_release 16 (pacific)
election_strategy: 1
0: [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] mon.ceph-ss-test-oct-controller-av-01
1: [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] mon.ceph-ss-test-oct-controller-av-02
2: [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] mon.ceph-ss-test-oct-controller-av-03

-501> 2022-04-12T16:07:52.827+0300 7f28f7664700 10 monclient: _finish_auth 0
-500> 2022-04-12T16:07:52.827+0300 7f28f7664700 10 monclient: handle_config config(0 keys) v1
-499> 2022-04-12T16:07:52.827+0300 7f28f7664700 10 monclient: handle_monmap mon_map magic: 0 v1
-498> 2022-04-12T16:07:52.827+0300 7f28f7664700 10 monclient: got monmap 1 from mon.ceph-ss-test-oct-controller-av-01 (according to old e1)
-497> 2022-04-12T16:07:52.827+0300 7f28f7664700 10 monclient: dump:
epoch 1
fsid 65ccd25a-39ce-4072-a5c6-a0e6654bda87
last_changed 2022-04-12T15:48:27.663651+0300
created 2022-04-12T15:48:27.663651+0300
min_mon_release 16 (pacific)
election_strategy: 1
0: [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] mon.ceph-ss-test-oct-controller-av-01
1: [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] mon.ceph-ss-test-oct-controller-av-02
2: [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] mon.ceph-ss-test-oct-controller-av-03

-496> 2022-04-12T16:07:52.827+0300 7f28f8e67700 4 set_mon_vals no callback set
-495> 2022-04-12T16:07:52.827+0300 7f2a3b460400 5 monclient: authenticate success, global_id 5730
-494> 2022-04-12T16:07:52.827+0300 7f2a3b460400 10 monclient: _renew_subs
-493> 2022-04-12T16:07:52.827+0300 7f2a3b460400 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-492> 2022-04-12T16:07:52.827+0300 7f2a3b460400 10 monclient: _renew_subs
-491> 2022-04-12T16:07:52.827+0300 7f2a3b460400 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-490> 2022-04-12T16:07:52.827+0300 7f2a3b460400 1 librados: init done
-489> 2022-04-12T16:07:52.831+0300 7f28f7664700 4 mgrc handle_mgr_map Got map version 29
-488> 2022-04-12T16:07:52.831+0300 7f28f7664700 4 mgrc handle_mgr_map Active mgr is now [v2:192.168.166.84:6800/56,v1:192.168.166.84:6801/56]
-487> 2022-04-12T16:07:52.831+0300 7f28f7664700 4 mgrc reconnect Starting new session with [v2:192.168.166.84:6800/56,v1:192.168.166.84:6801/56]
-486> 2022-04-12T16:07:52.831+0300 7f2a268c2700 10 monclient: get_auth_request con 0x559e593a1000 auth_method 0
-485> 2022-04-12T16:07:52.835+0300 7f2a270c3700 10 monclient: get_auth_request con 0x559e55979800 auth_method 0
-484> 2022-04-12T16:07:52.855+0300 7f2a260c1700 10 monclient: get_auth_request con 0x559e562ab000 auth_method 0
-483> 2022-04-12T16:07:53.075+0300 7f2a270c3700 10 monclient: get_auth_request con 0x559e59cf6000 auth_method 0
-482> 2022-04-12T16:07:53.083+0300 7f2a0a089700 2 garbage collection: garbage collection: stop
-481> 2022-04-12T16:07:53.223+0300 7f2a0b08b700 10 monclient: tick
-480> 2022-04-12T16:07:53.643+0300 7f2a09888700 2 object expiration: stop
-479> 2022-04-12T16:07:53.655+0300 7f2a1f0b3700 10 monclient: tick
-478> 2022-04-12T16:07:53.951+0300 7f29f8065700 1 ====== starting new request req=0x7f2a3b415620 =====
-477> 2022-04-12T16:07:53.951+0300 7f29f8065700 2 req 6350979191525555842 0.000000000s initializing for trans_id = tx00000582335ecfec73e82-00625579a9-1659-sync
-476> 2022-04-12T16:07:53.951+0300 7f29f8065700 2 req 6350979191525555842 0.000000000s getting op 4
-475> 2022-04-12T16:07:53.951+0300 7f29f6061700 2 req 6350979191525555842 0.000000000s :post_period verifying requester
-474> 2022-04-12T16:07:53.951+0300 7f29f9868700 1 ====== starting new request req=0x7f2a3b313620 =====
-473> 2022-04-12T16:07:53.951+0300 7f29f305b700 1 ====== starting new request req=0x7f2a3b394620 =====
-472> 2022-04-12T16:07:53.951+0300 7f29f9868700 2 req 8268245483269231629 0.000000000s initializing for trans_id = tx0000072beb575c1fd940d-00625579a9-1659-sync
-471> 2022-04-12T16:07:53.951+0300 7f29f305b700 2 req 10560051255912296523 0.000000000s initializing for trans_id = tx00000928cd6aaee91f04b-00625579a9-1659-sync
-470> 2022-04-12T16:07:53.955+0300 7f29f9868700 2 req 8268245483269231629 0.004000003s getting op 4
-469> 2022-04-12T16:07:53.955+0300 7f29f9868700 2 req 8268245483269231629 0.004000003s :post_period verifying requester
-468> 2022-04-12T16:07:53.955+0300 7f29f9868700 5 req 8268245483269231629 0.004000003s :post_period error reading user info, uid=m7yH86NH1D44QiwaUUV9 can't authenticate
-467> 2022-04-12T16:07:53.955+0300 7f29f9868700 5 req 8268245483269231629 0.004000003s :post_period error reading user info, uid=m7yH86NH1D44QiwaUUV9 can't authenticate
-466> 2022-04-12T16:07:53.955+0300 7f29f9868700 5 req 8268245483269231629 0.004000003s :post_period Failed the auth strategy, reason=-2028
-465> 2022-04-12T16:07:53.955+0300 7f29f9868700 1 req 8268245483269231629 0.004000003s op->ERRORHANDLER: err_no=-2028 new_err_no=-2028
-464> 2022-04-12T16:07:53.955+0300 7f29f0856700 5 req 6350979191525555842 0.004000003s :post_period error reading user info, uid=m7yH86NH1D44QiwaUUV9 can't authenticate
-463> 2022-04-12T16:07:53.955+0300 7f29f0856700 5 req 6350979191525555842 0.004000003s :post_period error reading user info, uid=m7yH86NH1D44QiwaUUV9 can't authenticate
-462> 2022-04-12T16:07:53.955+0300 7f29f0856700 5 req 6350979191525555842 0.004000003s :post_period Failed the auth strategy, reason=-2028
-461> 2022-04-12T16:07:53.955+0300 7f29f0856700 1 req 6350979191525555842 0.004000003s op->ERRORHANDLER: err_no=-2028 new_err_no=-2028
-460> 2022-04-12T16:07:53.955+0300 7f29f305b700 2 req 10560051255912296523 0.004000003s getting op 4
-459> 2022-04-12T16:07:53.955+0300 7f29f305b700 2 req 10560051255912296523 0.004000003s :post_period verifying requester
-458> 2022-04-12T16:07:53.955+0300 7f29e6842700 2 req 8268245483269231629 0.004000003s :post_period op status=0
-457> 2022-04-12T16:07:53.955+0300 7f29e6842700 2 req 8268245483269231629 0.004000003s :post_period http status=403
-456> 2022-04-12T16:07:53.955+0300 7f29e6842700 1 ====== req done req=0x7f2a3b313620 op status=0 http_status=403 latency=0.004000003s ======
-455> 2022-04-12T16:07:53.955+0300 7f29e6842700 1 beast: 0x7f2a3b313620: 192.168.166.84 - - [12/Apr/2022:16:07:53.951 +0300] "POST /admin/realm/period?period=8b4c894f-cf43-4c56-9af0-1c2b54c74991&epoch=3&rgwx-zonegroup=784a05a5-2089-4564-ab1b-a55c43c5f401 HTTP/1.1" 403 113 - - - latency=0.004000003s
-454> 2022-04-12T16:07:53.955+0300 7f29f0856700 2 req 6350979191525555842 0.004000003s :post_period op status=0
-453> 2022-04-12T16:07:53.955+0300 7f29f0856700 2 req 6350979191525555842 0.004000003s :post_period http status=403
-452> 2022-04-12T16:07:53.955+0300 7f29f0856700 1 ====== req done req=0x7f2a3b415620 op status=0 http_status=403 latency=0.004000003s ======
-451> 2022-04-12T16:07:53.955+0300 7f29f305b700 5 req 10560051255912296523 0.004000003s :post_period error reading user info, uid=m7yH86NH1D44QiwaUUV9 can't authenticate
-450> 2022-04-12T16:07:53.955+0300 7f29f305b700 5 req 10560051255912296523 0.004000003s :post_period error reading user info, uid=m7yH86NH1D44QiwaUUV9 can't authenticate
-449> 2022-04-12T16:07:53.955+0300 7f29f305b700 5 req 10560051255912296523 0.004000003s :post_period Failed the auth strategy, reason=-2028
-448> 2022-04-12T16:07:53.955+0300 7f29f305b700 1 req 10560051255912296523 0.004000003s op->ERRORHANDLER: err_no=-2028 new_err_no=-2028
-447> 2022-04-12T16:07:53.955+0300 7f29f0856700 1 beast: 0x7f2a3b415620: 192.168.166.83 - - [12/Apr/2022:16:07:53.951 +0300] "POST /admin/realm/period?period=8b4c894f-cf43-4c56-9af0-1c2b54c74991&epoch=3&rgwx-zonegroup=784a05a5-2089-4564-ab1b-a55c43c5f401 HTTP/1.1" 403 113 - - - latency=0.004000003s
-446> 2022-04-12T16:07:53.955+0300 7f29ee051700 2 req 10560051255912296523 0.004000003s :post_period op status=0
-445> 2022-04-12T16:07:53.955+0300 7f29ee051700 2 req 10560051255912296523 0.004000003s :post_period http status=403
-444> 2022-04-12T16:07:53.955+0300 7f29ee051700 1 ====== req done req=0x7f2a3b394620 op status=0 http_status=403 latency=0.004000003s ======
-443> 2022-04-12T16:07:53.955+0300 7f29ee051700 1 beast: 0x7f2a3b394620: 192.168.166.85 - - [12/Apr/2022:16:07:53.951 +0300] "POST /admin/realm/period?period=8b4c894f-cf43-4c56-9af0-1c2b54c74991&epoch=3&rgwx-zonegroup=784a05a5-2089-4564-ab1b-a55c43c5f401 HTTP/1.1" 403 113 - - - latency=0.004000003s
-442> 2022-04-12T16:07:54.223+0300 7f2a0b08b700 10 monclient: tick
-441> 2022-04-12T16:07:54.655+0300 7f2a1f0b3700 10 monclient: tick
-440> 2022-04-12T16:07:54.655+0300 7f2a208b6700 10 monclient: _renew_subs
-439> 2022-04-12T16:07:54.655+0300 7f2a208b6700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-438> 2022-04-12T16:07:54.731+0300 7f2a07884700 1 RGW-SYNC:meta: start
-437> 2022-04-12T16:07:54.731+0300 7f2a07884700 1 RGW-SYNC:meta: realm epoch=2 period id=8b4c894f-cf43-4c56-9af0-1c2b54c74991
-436> 2022-04-12T16:07:54.735+0300 7f2a07884700 4 meta sync: cr:s=0x559e571e3180:op=0x559e59df4000:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-435> 2022-04-12T16:07:54.735+0300 7f2a07884700 4 meta sync: cr:s=0x559e59bc08c0:op=0x559e5578a800:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-434> 2022-04-12T16:07:54.735+0300 7f2a07884700 4 meta sync: cr:s=0x559e594b3540:op=0x559e59df4a00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-433> 2022-04-12T16:07:54.735+0300 7f2a07884700 4 meta sync: cr:s=0x559e5988b680:op=0x559e59df5400:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-432> 2022-04-12T16:07:54.735+0300 7f2a07884700 4 meta sync: cr:s=0x559e59bc1180:op=0x559e59d78a00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-431> 2022-04-12T16:07:54.735+0300 7f2a07884700 4 meta sync: cr:s=0x559e596883c0:op=0x559e59dfa800:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-430> 2022-04-12T16:07:54.735+0300 7f2a07884700 4 meta sync: cr:s=0x559e59689b80:op=0x559e59df7200:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-429> 2022-04-12T16:07:54.735+0300 7f2a07884700 4 meta sync: cr:s=0x559e55ddeb40:op=0x559e59df9400:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-428> 2022-04-12T16:07:54.735+0300 7f2a07884700 4 meta sync: cr:s=0x559e59bc17c0:op=0x559e59d79e00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-427> 2022-04-12T16:07:54.735+0300 7f2a07884700 4 meta sync: cr:s=0x559e5965aa00:op=0x559e59dfb200:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-426> 2022-04-12T16:07:54.735+0300 7f2a07884700 4 meta sync: cr:s=0x559e59bc0140:op=0x559e5578b200:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-425> 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e595b1e00:op=0x559e59dfca00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-424> 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e55dfd900:op=0x559e59df8a00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-423> 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e59bc12c0:op=0x559e59d78000:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-422> 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e59688a00:op=0x559e59e01400:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-421> 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e59689cc0:op=0x559e59e02800:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-420> 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e5988b900:op=0x559e59e09400:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-419> 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e55dfc8c0:op=0x559e59dfde00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-418> 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e571e3040:op=0x559e59e05400:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-417> 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e55dfcc80:op=0x559e59d7b200:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-416> 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e571e3540:op=0x559e59e00a00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-415> 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e55dfdb80:op=0x559e59dfe800:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-414> 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e596888c0:op=0x559e59e01e00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-413> 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e55dfd2c0:op=0x559e59df6800:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-412> 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e594b3b80:op=0x559e59e08a00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-411> 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e55dfc280:op=0x559e59dff200:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-410> 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e5965ac80:op=0x559e59e04000:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-409> 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e5988bcc0:op=0x559e59e0d400:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-408> 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e55dfc3c0:op=0x559e59dfd400:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-407> 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e594b37c0:op=0x559e59e09e00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-406> 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e5965ab40:op=0x559e59df9e00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-405> 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e571e2dc0:op=0x559e59e00000:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-404> 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e5988b540:op=0x559e59e08000:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-403> 2022-04-12T16:07:54.743+0300 7f2a07884700 4 meta sync: cr:s=0x559e55dfcdc0:op=0x559e59df5e00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-402> 2022-04-12T16:07:54.743+0300 7f2a07884700 4 meta sync: cr:s=0x559e59acc3c0:op=0x559e59d79400:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-401> 2022-04-12T16:07:54.743+0300 7f2a07884700 4 meta sync: cr:s=0x559e55dfc780:op=0x559e59d7a800:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-400> 2022-04-12T16:07:54.743+0300 7f2a07884700 4 meta sync: cr:s=0x559e57c2af00:op=0x559e59e17200:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-399> 2022-04-12T16:07:54.747+0300 7f2a07884700 4 meta sync: cr:s=0x559e571e2c80:op=0x559e59e06800:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-398> 2022-04-12T16:07:54.747+0300 7f2a07884700 4 meta sync: cr:s=0x559e59bc0dc0:op=0x559e59e11400:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-397> 2022-04-12T16:07:54.747+0300 7f2a07884700 4 meta sync: cr:s=0x559e59bcd540:op=0x559e59e14000:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-396> 2022-04-12T16:07:54.747+0300 7f2a07884700 4 meta sync: cr:s=0x559e57c2adc0:op=0x559e59e16800:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-395> 2022-04-12T16:07:54.747+0300 7f2a07884700 4 meta sync: cr:s=0x559e55a08140:op=0x559e59e14a00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-394> 2022-04-12T16:07:54.747+0300 7f2a07884700 4 meta sync: cr:s=0x559e564c1cc0:op=0x559e59e04a00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-393> 2022-04-12T16:07:54.747+0300 7f2a07884700 4 meta sync: cr:s=0x559e57c2b040:op=0x559e59e18000:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-392> 2022-04-12T16:07:54.747+0300 7f2a07884700 4 meta sync: cr:s=0x559e59bc1900:op=0x559e59e0e800:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-391> 2022-04-12T16:07:54.747+0300 7f2a07884700 4 meta sync: cr:s=0x559e59bc0640:op=0x559e59e0f200:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-390> 2022-04-12T16:07:54.751+0300 7f2a07884700 4 meta sync: cr:s=0x559e59689e00:op=0x559e59e03200:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-389> 2022-04-12T16:07:54.751+0300 7f2a07884700 4 meta sync: cr:s=0x559e55a083c0:op=0x559e59e0de00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-388> 2022-04-12T16:07:54.751+0300 7f2a07884700 4 meta sync: cr:s=0x559e571e3cc0:op=0x559e59e07200:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-387> 2022-04-12T16:07:54.751+0300 7f2a07884700 4 meta sync: cr:s=0x559e5988be00:op=0x559e59e0b200:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-386> 2022-04-12T16:07:54.751+0300 7f2a07884700 4 meta sync: cr:s=0x559e594b3680:op=0x559e59e0ca00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-385> 2022-04-12T16:07:54.751+0300 7f2a07884700 4 meta sync: cr:s=0x559e5988ba40:op=0x559e59df8000:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-384> 2022-04-12T16:07:54.751+0300 7f2a07884700 4 meta sync: cr:s=0x559e571e3900:op=0x559e59e05e00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-383> 2022-04-12T16:07:54.751+0300 7f2a07884700 4 meta sync: cr:s=0x559e5988bb80:op=0x559e59e0a800:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-382> 2022-04-12T16:07:54.751+0300 7f2a07884700 4 meta sync: cr:s=0x559e55a08780:op=0x559e59e15400:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-381> 2022-04-12T16:07:54.751+0300 7f2a07884700 4 meta sync: cr:s=0x559e5988b7c0:op=0x559e59e0c000:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-380> 2022-04-12T16:07:54.751+0300 7f2a07884700 4 meta sync: cr:s=0x559e595b1b80:op=0x559e59dfc000:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-379> 2022-04-12T16:07:54.751+0300 7f2a07884700 4 meta sync: cr:s=0x559e59bc1540:op=0x559e59e10a00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-378> 2022-04-12T16:07:54.751+0300 7f2a07884700 4 meta sync: cr:s=0x559e59bc0780:op=0x559e59e11e00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-377> 2022-04-12T16:07:54.751+0300 7f2a07884700 4 meta sync: cr:s=0x559e59bcc000:op=0x559e59e13200:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-376> 2022-04-12T16:07:54.755+0300 7f2a07884700 4 meta sync: cr:s=0x559e59bc1b80:op=0x559e59e10000:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-375> 2022-04-12T16:07:54.755+0300 7f2a07884700 4 meta sync: cr:s=0x559e55a08280:op=0x559e59e15e00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-374> 2022-04-12T16:07:54.755+0300 7f2a07884700 4 meta sync: cr:s=0x559e59bcc500:op=0x559e59e12800:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-373> 2022-04-12T16:07:54.803+0300 7f2a07884700 4 meta sync: cr:s=0x559e54a72640:op=0x559e55789e00:18RGWMetaSyncShardCR: adjusting marker pos=user:dashboard
-372> 2022-04-12T16:07:54.811+0300 7f2a07884700 4 meta sync: cr:s=0x559e54a72640:op=0x559e55789e00:18RGWMetaSyncShardCR: adjusting marker pos=user:testuser
-371> 2022-04-12T16:07:54.835+0300 7f2a07884700 4 meta sync: cr:s=0x559e54a72640:op=0x559e55789e00:18RGWMetaSyncShardCR: adjusting marker pos=user:testuser
-370> 2022-04-12T16:07:54.835+0300 7f2a07884700 4 meta sync: cr:s=0x559e54a72640:op=0x559e55789e00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2
-369> 2022-04-12T16:07:55.223+0300 7f2a0b08b700 10 monclient: tick
-368> 2022-04-12T16:07:55.655+0300 7f2a1f0b3700 10 monclient: tick
-367> 2022-04-12T16:07:55.823+0300 7f28f6662700 10 monclient: tick
-366> 2022-04-12T16:07:56.223+0300 7f2a0b08b700 10 monclient: tick
-365> 2022-04-12T16:07:56.659+0300 7f2a1f0b3700 10 monclient: tick
-364> 2022-04-12T16:07:56.827+0300 7f28f6662700 10 monclient: tick
-363> 2022-04-12T16:07:57.227+0300 7f2a0b08b700 10 monclient: tick
-362> 2022-04-12T16:07:57.655+0300 7f2a1f0b3700 10 monclient: tick
-361> 2022-04-12T16:07:57.823+0300 7f28f6662700 10 monclient: tick
-360> 2022-04-12T16:07:57.827+0300 7f28f7e65700 10 monclient: _renew_subs
-359> 2022-04-12T16:07:57.827+0300 7f28f7e65700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-358> 2022-04-12T16:07:58.223+0300 7f2a0b08b700 10 monclient: tick
-357> 2022-04-12T16:07:58.659+0300 7f2a1f0b3700 10 monclient: tick
-356> 2022-04-12T16:07:58.823+0300 7f28f6662700 10 monclient: tick
-355> 2022-04-12T16:07:59.223+0300 7f2a0b08b700 10 monclient: tick
-354> 2022-04-12T16:07:59.655+0300 7f2a208b6700 10 monclient: _renew_subs
-353> 2022-04-12T16:07:59.655+0300 7f2a208b6700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-352> 2022-04-12T16:07:59.655+0300 7f2a1f0b3700 10 monclient: tick
-351> 2022-04-12T16:07:59.823+0300 7f28f6662700 10 monclient: tick
-350> 2022-04-12T16:08:00.223+0300 7f2a0b08b700 10 monclient: tick
-349> 2022-04-12T16:08:00.659+0300 7f2a1f0b3700 10 monclient: tick
-348> 2022-04-12T16:08:00.823+0300 7f28f6662700 10 monclient: tick
-347> 2022-04-12T16:08:01.227+0300 7f2a0b08b700 10 monclient: tick
-346> 2022-04-12T16:08:01.655+0300 7f2a1f0b3700 10 monclient: tick
-345> 2022-04-12T16:08:01.827+0300 7f28f6662700 10 monclient: tick
-344> 2022-04-12T16:08:02.227+0300 7f2a0b08b700 10 monclient: tick
-343> 2022-04-12T16:08:02.659+0300 7f2a1f0b3700 10 monclient: tick
-342> 2022-04-12T16:08:02.827+0300 7f28f6662700 10 monclient: tick
-341> 2022-04-12T16:08:02.827+0300 7f28f7e65700 10 monclient: _renew_subs
-340> 2022-04-12T16:08:02.827+0300 7f28f7e65700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-339> 2022-04-12T16:08:03.223+0300 7f2a0b08b700 10 monclient: tick
-338> 2022-04-12T16:08:03.655+0300 7f2a1f0b3700 10 monclient: tick
-337> 2022-04-12T16:08:03.827+0300 7f28f6662700 10 monclient: tick
-336> 2022-04-12T16:08:04.227+0300 7f2a0b08b700 10 monclient: tick
-335> 2022-04-12T16:08:04.655+0300 7f2a208b6700 10 monclient: _renew_subs
-334> 2022-04-12T16:08:04.655+0300 7f2a208b6700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-333> 2022-04-12T16:08:04.655+0300 7f2a1f0b3700 10 monclient: tick
-332> 2022-04-12T16:08:04.823+0300 7f28f6662700 10 monclient: tick
-331> 2022-04-12T16:08:05.223+0300 7f2a0b08b700 10 monclient: tick
-330> 2022-04-12T16:08:05.659+0300 7f2a1f0b3700 10 monclient: tick
-329> 2022-04-12T16:08:05.823+0300 7f28f6662700 10 monclient: tick
-328> 2022-04-12T16:08:06.227+0300 7f2a0b08b700 10 monclient: tick
-327> 2022-04-12T16:08:06.659+0300 7f2a1f0b3700 10 monclient: tick
-326> 2022-04-12T16:08:06.823+0300 7f28f6662700 10 monclient: tick
-325> 2022-04-12T16:08:07.227+0300 7f2a0b08b700 10 monclient: tick
-324> 2022-04-12T16:08:07.655+0300 7f2a1f0b3700 10 monclient: tick
-323> 2022-04-12T16:08:07.827+0300 7f28f6662700 10 monclient: tick
-322> 2022-04-12T16:08:08.227+0300 7f2a0b08b700 10 monclient: tick
-321> 2022-04-12T16:08:08.659+0300 7f2a1f0b3700 10 monclient: tick
-320> 2022-04-12T16:08:08.823+0300 7f28f6662700 10 monclient: tick
-319> 2022-04-12T16:08:09.223+0300 7f2a0b08b700 10 monclient: tick
-318> 2022-04-12T16:08:09.659+0300 7f2a1f0b3700 10 monclient: tick
-317> 2022-04-12T16:08:09.823+0300 7f28f6662700 10 monclient: tick
-316> 2022-04-12T16:08:10.223+0300 7f2a0b08b700 10 monclient: tick
-315> 2022-04-12T16:08:10.655+0300 7f2a1f0b3700 10 monclient: tick
-314> 2022-04-12T16:08:10.827+0300 7f28f6662700 10 monclient: tick
-313> 2022-04-12T16:08:11.211+0300 7f2a0e091700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start
-312> 2022-04-12T16:08:11.223+0300 7f2a0b08b700 10 monclient: tick
-311> 2022-04-12T16:08:11.655+0300 7f2a1f0b3700 10 monclient: tick
-310> 2022-04-12T16:08:11.827+0300 7f28f6662700 10 monclient: tick
-309> 2022-04-12T16:08:12.223+0300 7f2a0b08b700 10 monclient: tick
-308> 2022-04-12T16:08:12.655+0300 7f2a1f0b3700 10 monclient: tick
-307> 2022-04-12T16:08:12.823+0300 7f28f6662700 10 monclient: tick
-306> 2022-04-12T16:08:13.223+0300 7f2a0b08b700 10 monclient: tick
-305> 2022-04-12T16:08:13.659+0300 7f2a1f0b3700 10 monclient: tick
-304> 2022-04-12T16:08:13.823+0300 7f28f6662700 10 monclient: tick
-303> 2022-04-12T16:08:14.227+0300 7f2a0b08b700 10 monclient: tick
-302> 2022-04-12T16:08:14.659+0300 7f2a1f0b3700 10 monclient: tick
-301> 2022-04-12T16:08:14.827+0300 7f28f6662700 10 monclient: tick
-300> 2022-04-12T16:08:15.227+0300 7f2a0b08b700 10 monclient: tick
-299> 2022-04-12T16:08:15.659+0300 7f2a1f0b3700 10 monclient: tick
-298> 2022-04-12T16:08:15.827+0300 7f28f6662700 10 monclient: tick
-297> 2022-04-12T16:08:16.227+0300 7f2a0b08b700 10 monclient: tick
-296> 2022-04-12T16:08:16.659+0300 7f2a1f0b3700 10 monclient: tick
-295> 2022-04-12T16:08:16.827+0300 7f28f6662700 10 monclient: tick
-294> 2022-04-12T16:08:17.227+0300 7f2a0b08b700 10 monclient: tick
-293> 2022-04-12T16:08:17.659+0300 7f2a1f0b3700 10 monclient: tick
-292> 2022-04-12T16:08:17.827+0300 7f28f6662700 10 monclient: tick
-291> 2022-04-12T16:08:18.227+0300 7f2a0b08b700 10 monclient: tick
-290> 2022-04-12T16:08:18.659+0300 7f2a1f0b3700 10 monclient: tick
-289> 2022-04-12T16:08:18.827+0300 7f28f6662700 10 monclient: tick
-288> 2022-04-12T16:08:19.227+0300 7f2a0b08b700 10 monclient: tick
-287> 2022-04-12T16:08:19.659+0300 7f2a1f0b3700 10 monclient: tick
-286> 2022-04-12T16:08:19.827+0300 7f28f6662700 10 monclient: tick
-285> 2022-04-12T16:08:20.227+0300 7f2a0b08b700 10 monclient: tick
-284> 2022-04-12T16:08:20.659+0300 7f2a1f0b3700 10 monclient: tick
-283> 2022-04-12T16:08:20.827+0300 7f28f6662700 10 monclient: tick
-282> 2022-04-12T16:08:21.227+0300 7f2a0b08b700 10 monclient: tick
-281> 2022-04-12T16:08:21.659+0300 7f2a1f0b3700 10 monclient: tick
-280> 2022-04-12T16:08:21.827+0300 7f28f6662700 10 monclient: tick
-279> 2022-04-12T16:08:22.227+0300 7f2a0b08b700 10 monclient: tick
-278> 2022-04-12T16:08:22.659+0300 7f2a1f0b3700 10 monclient: tick
-277> 2022-04-12T16:08:22.827+0300 7f28f6662700 10 monclient: tick
-276> 2022-04-12T16:08:23.227+0300 7f2a0b08b700 10 monclient: tick
-275> 2022-04-12T16:08:23.659+0300 7f2a1f0b3700 10 monclient: tick
-274> 2022-04-12T16:08:23.827+0300 7f28f6662700 10 monclient: tick
-273> 2022-04-12T16:08:23.959+0300 7f29f505f700 1 ====== starting new request req=0x7f2a3b313620 =====
-272> 2022-04-12T16:08:23.959+0300 7f29f505f700 2 req 8920709381089314344 0.000000000s initializing for trans_id = tx000007bccb9eb0c52c628-00625579c7-1659-sync
-271> 2022-04-12T16:08:23.959+0300 7f29f505f700 2 req 8920709381089314344 0.000000000s getting op 4
-270> 2022-04-12T16:08:23.959+0300 7f29f505f700 2 req 8920709381089314344 0.000000000s :post_period verifying requester
-269> 2022-04-12T16:08:23.959+0300 7f29f505f700 2 req 8920709381089314344 0.000000000s :post_period normalizing buckets and tenants
-268> 2022-04-12T16:08:23.959+0300 7f29f505f700 2 req 8920709381089314344 0.000000000s :post_period init permissions
-267> 2022-04-12T16:08:23.959+0300 7f29f505f700 2 req 8920709381089314344 0.000000000s :post_period recalculating target
-266> 2022-04-12T16:08:23.959+0300 7f29f505f700 2 req 8920709381089314344 0.000000000s :post_period reading permissions
-265> 2022-04-12T16:08:23.959+0300 7f29f505f700 2 req 8920709381089314344 0.000000000s :post_period init op
-264> 2022-04-12T16:08:23.959+0300 7f29f505f700 2 req 8920709381089314344 0.000000000s :post_period verifying op mask
-263> 2022-04-12T16:08:23.959+0300 7f29f505f700 2 req 8920709381089314344 0.000000000s :post_period verifying op permissions
-262> 2022-04-12T16:08:23.959+0300 7f29f505f700 2 overriding permissions due to system operation
-261> 2022-04-12T16:08:23.959+0300 7f29f505f700 2 req 8920709381089314344 0.000000000s :post_period verifying op params
-260> 2022-04-12T16:08:23.959+0300 7f29f505f700 2 req 8920709381089314344 0.000000000s :post_period pre-executing
-259> 2022-04-12T16:08:23.959+0300 7f29f505f700 2 req 8920709381089314344 0.000000000s :post_period executing
-258> 2022-04-12T16:08:23.959+0300 7f29cc80e700 1 ====== starting new request req=0x7f2a3b415620 =====
-257> 2022-04-12T16:08:23.959+0300 7f29cc80e700 2 req 8609644640538370824 0.000000000s initializing for trans_id = tx00000777b9a2f49a2a308-00625579c7-1659-sync
-256> 2022-04-12T16:08:23.959+0300 7f29cc80e700 2 req 8609644640538370824 0.000000000s getting op 4
-255> 2022-04-12T16:08:23.959+0300 7f29cc80e700 2 req 8609644640538370824 0.000000000s :post_period verifying requester
-254> 2022-04-12T16:08:23.959+0300 7f29cc80e700 2 req 8609644640538370824 0.000000000s :post_period normalizing buckets and tenants
-253> 2022-04-12T16:08:23.959+0300 7f29cc80e700 2 req 8609644640538370824 0.000000000s :post_period init permissions
-252> 2022-04-12T16:08:23.959+0300 7f29cc80e700 2 req 8609644640538370824 0.000000000s :post_period recalculating target
-251> 2022-04-12T16:08:23.959+0300 7f29cc80e700 2 req 8609644640538370824 0.000000000s :post_period reading permissions
-250> 2022-04-12T16:08:23.959+0300 7f29cc80e700 2 req 8609644640538370824 0.000000000s :post_period init op
-249> 2022-04-12T16:08:23.959+0300 7f29cc80e700 2 req 8609644640538370824 0.000000000s :post_period verifying op mask
-248> 2022-04-12T16:08:23.959+0300 7f29cc80e700 2 req 8609644640538370824 0.000000000s :post_period verifying op permissions
-247> 2022-04-12T16:08:23.959+0300 7f29cc80e700 2 overriding permissions due to system operation
-246> 2022-04-12T16:08:23.959+0300 7f29cc80e700 2 req 8609644640538370824 0.000000000s :post_period verifying op params
-245> 2022-04-12T16:08:23.959+0300 7f29cc80e700 2 req 8609644640538370824 0.000000000s :post_period pre-executing
-244> 2022-04-12T16:08:23.959+0300 7f29cc80e700 2 req 8609644640538370824 0.000000000s :post_period executing
-243> 2022-04-12T16:08:23.959+0300 7f29c8005700 1 ====== starting new request req=0x7f2a3b394620 =====
-242> 2022-04-12T16:08:23.959+0300 7f29c8005700 2 req 14231248929957790225 0.000000000s initializing for trans_id = tx00000c57f8d805c4ac211-00625579c7-1659-sync
-241> 2022-04-12T16:08:23.959+0300 7f29c8005700 2 req 14231248929957790225 0.000000000s getting op 4
-240> 2022-04-12T16:08:23.959+0300 7f29c8005700 2 req 14231248929957790225 0.000000000s :post_period verifying requester
-239> 2022-04-12T16:08:23.959+0300 7f29c8005700 2 req 14231248929957790225 0.000000000s :post_period normalizing buckets and tenants
-238> 2022-04-12T16:08:23.959+0300 7f29c8005700 2 req 14231248929957790225 0.000000000s :post_period init permissions
-237> 2022-04-12T16:08:23.959+0300 7f29c8005700 2 req 14231248929957790225 0.000000000s :post_period recalculating target
-236> 2022-04-12T16:08:23.959+0300 7f29c8005700 2 req 14231248929957790225 0.000000000s :post_period reading permissions
-235> 2022-04-12T16:08:23.959+0300 7f29c8005700 2 req 14231248929957790225 0.000000000s :post_period init op
-234> 2022-04-12T16:08:23.959+0300 7f29c8005700 2 req 14231248929957790225 0.000000000s :post_period verifying op mask
-233> 2022-04-12T16:08:23.959+0300 7f29c8005700 2 req 14231248929957790225 0.000000000s :post_period verifying op permissions
-232> 2022-04-12T16:08:23.959+0300 7f29c8005700 2 overriding permissions due to system operation
-231> 2022-04-12T16:08:23.959+0300 7f29c8005700 2 req 14231248929957790225 0.000000000s :post_period verifying op params
-230> 2022-04-12T16:08:23.959+0300 7f29c8005700 2 req 14231248929957790225 0.000000000s :post_period pre-executing
-229> 2022-04-12T16:08:23.959+0300 7f29c8005700 2 req 14231248929957790225 0.000000000s :post_period executing
-228> 2022-04-12T16:08:23.963+0300 7f29ac7ce700 4 req 8920709381089314344 0.004000003s :post_period already have epoch >= 3 for period 8b4c894f-cf43-4c56-9af0-1c2b54c74991
-227> 2022-04-12T16:08:23.963+0300 7f29ac7ce700 2 req 8920709381089314344 0.004000003s :post_period completing
-226> 2022-04-12T16:08:23.963+0300 7f29ac7ce700 2 req 8920709381089314344 0.004000003s :post_period op status=0
-225> 2022-04-12T16:08:23.963+0300 7f29ac7ce700 2 req 8920709381089314344 0.004000003s :post_period http status=200
-224> 2022-04-12T16:08:23.963+0300 7f29ac7ce700 1 ====== req done req=0x7f2a3b313620 op status=0 http_status=200 latency=0.004000003s ======
-223> 2022-04-12T16:08:23.963+0300 7f29ac7ce700 1 beast: 0x7f2a3b313620: 192.168.166.84 - rgw_zone_user [12/Apr/2022:16:08:23.959 +0300] "POST /admin/realm/period?period=8b4c894f-cf43-4c56-9af0-1c2b54c74991&epoch=3&rgwx-zonegroup=784a05a5-2089-4564-ab1b-a55c43c5f401 HTTP/1.1" 200 1868 - - - latency=0.004000003s
-222> 2022-04-12T16:08:23.963+0300 7f29a57c0700 4 req 14231248929957790225 0.004000003s :post_period already have epoch >= 3 for period 8b4c894f-cf43-4c56-9af0-1c2b54c74991
-221> 2022-04-12T16:08:23.963+0300 7f29a57c0700 2 req 14231248929957790225 0.004000003s :post_period completing
-220> 2022-04-12T16:08:23.963+0300 7f29a57c0700 2 req 14231248929957790225 0.004000003s :post_period op status=0
-219> 2022-04-12T16:08:23.963+0300 7f29a57c0700 2 req 14231248929957790225 0.004000003s :post_period http status=200
-218> 2022-04-12T16:08:23.963+0300 7f29a57c0700 1 ====== req done req=0x7f2a3b394620 op status=0 http_status=200 latency=0.004000003s ======
-217> 2022-04-12T16:08:23.963+0300 7f29a57c0700 1 beast: 0x7f2a3b394620: 192.168.166.85 - rgw_zone_user [12/Apr/2022:16:08:23.959 +0300] "POST /admin/realm/period?period=8b4c894f-cf43-4c56-9af0-1c2b54c74991&epoch=3&rgwx-zonegroup=784a05a5-2089-4564-ab1b-a55c43c5f401 HTTP/1.1" 200 1868 - - - latency=0.004000003s
-216> 2022-04-12T16:08:23.967+0300 7f299e7b2700 4 req 8609644640538370824 0.008000007s :post_period already have epoch >= 3 for period 8b4c894f-cf43-4c56-9af0-1c2b54c74991
-215> 2022-04-12T16:08:23.967+0300 7f299e7b2700 2 req 8609644640538370824 0.008000007s :post_period completing
-214> 2022-04-12T16:08:23.967+0300 7f299e7b2700 2 req 8609644640538370824 0.008000007s :post_period op status=0
-213> 2022-04-12T16:08:23.967+0300 7f299e7b2700 2 req 8609644640538370824 0.008000007s :post_period http status=200
-212> 2022-04-12T16:08:23.967+0300 7f299e7b2700 1 ====== req done req=0x7f2a3b415620 op status=0 http_status=200 latency=0.008000007s ======
-211> 2022-04-12T16:08:23.967+0300 7f299e7b2700 1 beast: 0x7f2a3b415620: 192.168.166.83 - rgw_zone_user [12/Apr/2022:16:08:23.959 +0300] "POST /admin/realm/period?period=8b4c894f-cf43-4c56-9af0-1c2b54c74991&epoch=3&rgwx-zonegroup=784a05a5-2089-4564-ab1b-a55c43c5f401 HTTP/1.1" 200 1868 - - - latency=0.008000007s
-210> 2022-04-12T16:08:24.227+0300 7f2a0b08b700 10 monclient: tick
-209> 2022-04-12T16:08:24.663+0300 7f2a1f0b3700 10 monclient: tick
-208> 2022-04-12T16:08:24.827+0300 7f28f6662700 10 monclient: tick
-207> 2022-04-12T16:08:25.227+0300 7f2a0b08b700 10 monclient: tick
-206> 2022-04-12T16:08:25.663+0300 7f2a1f0b3700 10 monclient: tick
-205> 2022-04-12T16:08:25.827+0300 7f28f6662700 10 monclient: tick
-204> 2022-04-12T16:08:26.231+0300 7f2a0b08b700 10 monclient: tick
-203> 2022-04-12T16:08:26.663+0300 7f2a1f0b3700 10 monclient: tick
-202> 2022-04-12T16:08:26.831+0300 7f28f6662700 10 monclient: tick
-201> 2022-04-12T16:08:27.227+0300 7f2a0b08b700 10 monclient: tick
-200> 2022-04-12T16:08:27.663+0300 7f2a1f0b3700 10 monclient: tick
-199> 2022-04-12T16:08:27.827+0300 7f28f6662700 10 monclient: tick
-198> 2022-04-12T16:08:28.227+0300 7f2a0b08b700 10 monclient: tick
-197> 2022-04-12T16:08:28.659+0300 7f2a1f0b3700 10 monclient: tick
-196> 2022-04-12T16:08:28.827+0300 7f28f6662700 10 monclient: tick
-195> 2022-04-12T16:08:29.227+0300 7f2a0b08b700 10 monclient: tick
-194> 2022-04-12T16:08:29.663+0300 7f2a1f0b3700 10 monclient: tick
-193> 2022-04-12T16:08:29.827+0300 7f28f6662700 10 monclient: tick
-192> 2022-04-12T16:08:30.227+0300 7f2a0b08b700 10 monclient: tick
-191> 2022-04-12T16:08:30.663+0300 7f2a1f0b3700 10 monclient: tick
-190> 2022-04-12T16:08:30.827+0300 7f28f6662700 10 monclient: tick
-189> 2022-04-12T16:08:31.227+0300 7f2a0b08b700 10 monclient: tick
-188> 2022-04-12T16:08:31.659+0300 7f2a1f0b3700 10 monclient: tick
-187> 2022-04-12T16:08:31.831+0300 7f28f6662700 10 monclient: tick
-186> 2022-04-12T16:08:32.227+0300 7f2a0b08b700 10 monclient: tick
-185> 2022-04-12T16:08:32.659+0300 7f2a1f0b3700 10 monclient: tick
-184> 2022-04-12T16:08:32.827+0300 7f28f6662700 10 monclient: tick
-183> 2022-04-12T16:08:33.211+0300 7f2a0e091700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start
-182> 2022-04-12T16:08:33.227+0300 7f2a0b08b700 10 monclient: tick
-181> 2022-04-12T16:08:33.659+0300 7f2a1f0b3700 10 monclient: tick
-180> 2022-04-12T16:08:33.831+0300 7f28f6662700 10 monclient: tick
-179> 2022-04-12T16:08:34.227+0300 7f2a0b08b700 10 monclient: tick
-178> 2022-04-12T16:08:34.659+0300 7f2a1f0b3700 10 monclient: tick
-177> 2022-04-12T16:08:34.827+0300 7f28f6662700 10 monclient: tick
-176> 2022-04-12T16:08:35.227+0300 7f2a0b08b700 10 monclient: tick
-175> 2022-04-12T16:08:35.659+0300 7f2a1f0b3700 10 monclient: tick
-174> 2022-04-12T16:08:35.827+0300 7f28f6662700 10 monclient: tick
-173> 2022-04-12T16:08:36.227+0300 7f2a0b08b700 10 monclient: tick
-172> 2022-04-12T16:08:36.659+0300 7f2a1f0b3700 10 monclient: tick
-171> 2022-04-12T16:08:36.827+0300 7f28f6662700 10 monclient: tick
-170> 2022-04-12T16:08:37.231+0300 7f2a0b08b700 10 monclient: tick
-169> 2022-04-12T16:08:37.659+0300 7f2a1f0b3700 10 monclient: tick
-168> 2022-04-12T16:08:37.831+0300 7f28f6662700 10 monclient: tick
-167> 2022-04-12T16:08:38.227+0300 7f2a0b08b700 10 monclient: tick
-166> 2022-04-12T16:08:38.663+0300 7f2a1f0b3700 10 monclient: tick
-165> 2022-04-12T16:08:38.831+0300 7f28f6662700 10 monclient: tick
-164> 2022-04-12T16:08:39.227+0300 7f2a0b08b700 10 monclient: tick
-163> 2022-04-12T16:08:39.659+0300 7f2a1f0b3700 10 monclient: tick
-162> 2022-04-12T16:08:39.827+0300 7f28f6662700 10 monclient: tick
-161> 2022-04-12T16:08:40.227+0300 7f2a0b08b700 10 monclient: tick
-160> 2022-04-12T16:08:40.659+0300 7f2a1f0b3700 10 monclient: tick
-159> 2022-04-12T16:08:40.827+0300 7f28f6662700 10 monclient: tick
-158> 2022-04-12T16:08:41.231+0300 7f2a0b08b700 10 monclient: tick
-157> 2022-04-12T16:08:41.659+0300 7f2a1f0b3700 10 monclient: tick
-156> 2022-04-12T16:08:41.831+0300 7f28f6662700 10 monclient: tick
-155> 2022-04-12T16:08:42.227+0300 7f2a0b08b700 10 monclient: tick
-154> 2022-04-12T16:08:42.659+0300 7f2a1f0b3700 10 monclient: tick
-153> 2022-04-12T16:08:42.827+0300 7f28f6662700 10 monclient: tick
-152> 2022-04-12T16:08:43.227+0300 7f2a0b08b700 10 monclient: tick
-151> 2022-04-12T16:08:43.659+0300 7f2a1f0b3700 10 monclient: tick
-150> 2022-04-12T16:08:43.831+0300 7f28f6662700 10 monclient: tick
-149> 2022-04-12T16:08:44.227+0300 7f2a0b08b700 10 monclient: tick
-148> 2022-04-12T16:08:44.659+0300 7f2a1f0b3700 10 monclient: tick
-147> 2022-04-12T16:08:44.831+0300 7f28f6662700 10 monclient: tick
-146> 2022-04-12T16:08:45.231+0300 7f2a0b08b700 10 monclient: tick
-145> 2022-04-12T16:08:45.663+0300 7f2a1f0b3700 10 monclient: tick
-144> 2022-04-12T16:08:45.831+0300 7f28f6662700 10 monclient: tick
-143> 2022-04-12T16:08:46.231+0300 7f2a0b08b700 10 monclient: tick
-142> 2022-04-12T16:08:46.663+0300 7f2a1f0b3700 10 monclient: tick
-141> 2022-04-12T16:08:46.827+0300 7f28f6662700 10 monclient: tick
-140> 2022-04-12T16:08:47.231+0300 7f2a0b08b700 10 monclient: tick
-139> 2022-04-12T16:08:47.663+0300 7f2a1f0b3700 10 monclient: tick
-138> 2022-04-12T16:08:47.831+0300 7f28f6662700 10 monclient: tick
-137> 2022-04-12T16:08:48.231+0300 7f2a0b08b700 10 monclient: tick
-136> 2022-04-12T16:08:48.663+0300 7f2a1f0b3700 10 monclient: tick
-135> 2022-04-12T16:08:48.831+0300 7f28f6662700 10 monclient: tick
-134> 2022-04-12T16:08:49.231+0300 7f2a0b08b700 10 monclient: tick
-133> 2022-04-12T16:08:49.663+0300 7f2a1f0b3700 10 monclient: tick
-132> 2022-04-12T16:08:49.667+0300 7f2a208b6700 10 monclient: _renew_subs
-131> 2022-04-12T16:08:49.667+0300 7f2a208b6700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-130> 2022-04-12T16:08:49.831+0300 7f28f6662700 10 monclient: tick
-129> 2022-04-12T16:08:50.231+0300 7f2a0b08b700 10 monclient: tick
-128> 2022-04-12T16:08:50.663+0300 7f2a1f0b3700 10 monclient: tick
-127> 2022-04-12T16:08:50.831+0300 7f28f6662700 10 monclient: tick
-126> 2022-04-12T16:08:51.231+0300 7f2a0b08b700 10 monclient: tick
-125> 2022-04-12T16:08:51.663+0300 7f2a1f0b3700 10 monclient: tick
-124> 2022-04-12T16:08:51.831+0300 7f28f6662700 10 monclient: tick
-123> 2022-04-12T16:08:52.231+0300 7f2a0b08b700 10 monclient: tick
-122> 2022-04-12T16:08:52.663+0300 7f2a1f0b3700 10 monclient: tick
-121> 2022-04-12T16:08:52.691+0300 7f29b5fe1700 1 ====== starting new request req=0x7f28f565f620 =====
-120> 2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s initializing for trans_id = tx00000ecd2ee05e4c9a4ca-00625579e4-1659-sync
-119> 2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s getting op 4
-118> 2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s :mdlog_notify verifying requester
-117> 2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s :mdlog_notify normalizing buckets and tenants
-116> 2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s :mdlog_notify init permissions
-115> 2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s :mdlog_notify recalculating target
-114> 2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s :mdlog_notify reading permissions
-113> 2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s :mdlog_notify init op
-112> 2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s :mdlog_notify verifying op mask
-111> 2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s :mdlog_notify verifying op permissions
-110> 2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 overriding permissions due to system operation
-109> 2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s :mdlog_notify verifying op params
-108> 2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s :mdlog_notify pre-executing
-107> 2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s :mdlog_notify executing
-106> 2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s :mdlog_notify completing
-105> 2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s :mdlog_notify op status=0
-104> 2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s :mdlog_notify http status=200
-103> 2022-04-12T16:08:52.691+0300 7f29b5fe1700 1 ====== req done req=0x7f28f565f620 op status=0 http_status=200 latency=0.000000000s ======
-102> 2022-04-12T16:08:52.691+0300 7f29b5fe1700 1 beast: 0x7f28f565f620: 192.168.166.85 - rgw_zone_user [12/Apr/2022:16:08:52.691 +0300] "POST /admin/log?type=metadata&notify&rgwx-zonegroup=784a05a5-2089-4564-ab1b-a55c43c5f401 HTTP/1.1" 200 0 - - - latency=0.000000000s
-101> 2022-04-12T16:08:52.699+0300 7f2a07884700 4 meta sync: cr:s=0x559e57c2af00:op=0x559e59e17200:18RGWMetaSyncShardCR: adjusting marker pos=
-100> 2022-04-12T16:08:52.759+0300 7f2a1409d700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-99> 2022-04-12T16:08:52.759+0300 7f2a200b5700 10 monclient: handle_get_version_reply finishing 5 version 76
-98> 2022-04-12T16:08:52.759+0300 7f2a1409d700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-97> 2022-04-12T16:08:52.831+0300 7f28f6662700 10 monclient: tick
-96> 2022-04-12T16:08:52.831+0300 7f28f7e65700 10 monclient: _renew_subs
-95> 2022-04-12T16:08:52.831+0300 7f28f7e65700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-94> 2022-04-12T16:08:53.231+0300 7f2a0b08b700 10 monclient: tick
-93> 2022-04-12T16:08:53.663+0300 7f2a1f0b3700 10 monclient: tick
-92> 2022-04-12T16:08:53.751+0300 7f2a1409d700 10 start_mon_command cmd=[{"prefix": "osd pool application enable","pool": "sync.rgw.buckets.index","app": "rgw"}]
-91> 2022-04-12T16:08:53.751+0300 7f2a1409d700 10 monclient: _send_command 3 [{"prefix": "osd pool application enable","pool": "sync.rgw.buckets.index","app": "rgw"}]
-90> 2022-04-12T16:08:53.751+0300 7f2a1409d700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-89> 2022-04-12T16:08:53.831+0300 7f28f6662700 10 monclient: tick
-88> 2022-04-12T16:08:54.231+0300 7f2a0b08b700 10 monclient: tick
-87> 2022-04-12T16:08:54.663+0300 7f2a1f0b3700 10 monclient: tick
-86> 2022-04-12T16:08:54.667+0300 7f2a208b6700 10 monclient: _renew_subs
-85> 2022-04-12T16:08:54.667+0300 7f2a208b6700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-84> 2022-04-12T16:08:54.759+0300 7f2a200b5700 10 monclient: handle_mon_command_ack 3 [{"prefix": "osd pool application enable","pool": "sync.rgw.buckets.index","app": "rgw"}]
-83> 2022-04-12T16:08:54.759+0300 7f2a200b5700 10 monclient: _finish_command 3 = system:0 enabled application 'rgw' on pool 'sync.rgw.buckets.index'
-82> 2022-04-12T16:08:54.759+0300 7f2a1409d700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-81> 2022-04-12T16:08:54.763+0300 7f2a200b5700 10 monclient: handle_get_version_reply finishing 6 version 78
-80> 2022-04-12T16:08:54.763+0300 7f2a1409d700 10 start_mon_command cmd=[{"prefix": "osd pool set", "pool": "sync.rgw.buckets.index", "var": "pg_autoscale_bias", "val": "4"}]
-79> 2022-04-12T16:08:54.763+0300 7f2a1409d700 10 monclient: _send_command 4 [{"prefix": "osd pool set", "pool": "sync.rgw.buckets.index", "var": "pg_autoscale_bias", "val": "4"}]
-78> 2022-04-12T16:08:54.763+0300 7f2a1409d700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-77> 2022-04-12T16:08:54.783+0300 7f2a07884700 4 meta sync: cr:s=0x559e57c2af00:op=0x559e59e17200:18RGWMetaSyncShardCR: adjusting marker pos=1_1649768932.547706_90.1
-76> 2022-04-12T16:08:54.783+0300 7f2a07884700 4 meta sync: cr:s=0x559e57c2af00:op=0x559e59e17200:18RGWMetaSyncShardCR: adjusting marker pos=1_1649768932.547706_90.1
-75> 2022-04-12T16:08:54.831+0300 7f28f6662700 10 monclient: tick
-74> 2022-04-12T16:08:55.215+0300 7f2a0e091700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start
-73> 2022-04-12T16:08:55.231+0300 7f2a0b08b700 10 monclient: tick
-72> 2022-04-12T16:08:55.663+0300 7f2a1f0b3700 10 monclient: tick
-71> 2022-04-12T16:08:55.767+0300 7f2a200b5700 10 monclient: handle_mon_command_ack 4 [{"prefix": "osd pool set", "pool": "sync.rgw.buckets.index", "var": "pg_autoscale_bias", "val": "4"}]
-70> 2022-04-12T16:08:55.767+0300 7f2a200b5700 10 monclient: _finish_command 4 = system:0 set pool 13 pg_autoscale_bias to 4
-69> 2022-04-12T16:08:55.767+0300 7f2a1409d700 10 start_mon_command cmd=[{"prefix": "osd pool set", "pool": "sync.rgw.buckets.index", "var": "pg_num_min", "val": "8"}]
-68> 2022-04-12T16:08:55.767+0300 7f2a1409d700 10 monclient: _send_command 5 [{"prefix": "osd pool set", "pool": "sync.rgw.buckets.index", "var": "pg_num_min", "val": "8"}]
-67> 2022-04-12T16:08:55.767+0300 7f2a1409d700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-66> 2022-04-12T16:08:55.831+0300 7f28f6662700 10 monclient: tick
-65> 2022-04-12T16:08:56.231+0300 7f2a0b08b700 10 monclient: tick
-64> 2022-04-12T16:08:56.663+0300 7f2a1f0b3700 10 monclient: tick
-63> 2022-04-12T16:08:56.775+0300 7f2a200b5700 10 monclient: handle_mon_command_ack 5 [{"prefix": "osd pool set", "pool": "sync.rgw.buckets.index", "var": "pg_num_min", "val": "8"}]
-62> 2022-04-12T16:08:56.775+0300 7f2a200b5700 10 monclient: _finish_command 5 = system:0 set pool 13 pg_num_min to 8
-61> 2022-04-12T16:08:56.775+0300 7f2a1409d700 10 start_mon_command cmd=[{"prefix": "osd pool set", "pool": "sync.rgw.buckets.index", "var": "recovery_priority": "5"}]
-60> 2022-04-12T16:08:56.775+0300 7f2a1409d700 10 monclient: _send_command 6 [{"prefix": "osd pool set", "pool": "sync.rgw.buckets.index", "var": "recovery_priority": "5"}]
-59> 2022-04-12T16:08:56.775+0300 7f2a1409d700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-58> 2022-04-12T16:08:56.775+0300 7f2a200b5700 10 monclient: handle_mon_command_ack 6 [{"prefix": "osd pool set", "pool": "sync.rgw.buckets.index", "var": "recovery_priority": "5"}]
-57> 2022-04-12T16:08:56.775+0300 7f2a200b5700 10 monclient: _finish_command 6 = mon:22 unparseable JSON {"prefix": "osd pool set", "pool": "sync.rgw.buckets.index", "var": "recovery_priority": "5"}
-56> 2022-04-12T16:08:56.831+0300 7f28f6662700 10 monclient: tick
-55> 2022-04-12T16:08:57.231+0300 7f2a0b08b700 10 monclient: tick
-54> 2022-04-12T16:08:57.663+0300 7f2a1f0b3700 10 monclient: tick
-53> 2022-04-12T16:08:57.831+0300 7f28f6662700 10 monclient: tick
-52> 2022-04-12T16:08:57.831+0300 7f28f7e65700 10 monclient: _renew_subs
-51> 2022-04-12T16:08:57.831+0300 7f28f7e65700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-50> 2022-04-12T16:08:58.231+0300 7f2a0b08b700 10 monclient: tick
-49> 2022-04-12T16:08:58.667+0300 7f2a1f0b3700 10 monclient: tick
-48> 2022-04-12T16:08:58.831+0300 7f28f6662700 10 monclient: tick
-47> 2022-04-12T16:08:59.231+0300 7f2a0b08b700 10 monclient: tick
-46> 2022-04-12T16:08:59.667+0300 7f2a1f0b3700 10 monclient: tick
-45> 2022-04-12T16:08:59.667+0300 7f2a208b6700 10 monclient: _renew_subs
-44> 2022-04-12T16:08:59.667+0300 7f2a208b6700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-43> 2022-04-12T16:08:59.831+0300 7f28f6662700 10 monclient: tick
-42> 2022-04-12T16:09:00.231+0300 7f2a0b08b700 10 monclient: tick
-41> 2022-04-12T16:09:00.663+0300 7f2a1f0b3700 10 monclient: tick
-40> 2022-04-12T16:09:00.831+0300 7f28f6662700 10 monclient: tick
-39> 2022-04-12T16:09:01.235+0300 7f2a0b08b700 10 monclient: tick
-38> 2022-04-12T16:09:01.663+0300 7f2a1f0b3700 10 monclient: tick
-37> 2022-04-12T16:09:01.835+0300 7f28f6662700 10 monclient: tick
-36> 2022-04-12T16:09:02.231+0300 7f2a0b08b700 10 monclient: tick
-35> 2022-04-12T16:09:02.663+0300 7f2a1f0b3700 10 monclient: tick
-34> 2022-04-12T16:09:02.747+0300 7f299a7aa700 1 ====== starting new request req=0x7f28f55de620 =====
-33> 2022-04-12T16:09:02.747+0300 7f299a7aa700 2 req 6236571265947506715 0.000000000s initializing for trans_id = tx00000568cc0860b35981b-00625579ee-1659-sync
-32> 2022-04-12T16:09:02.747+0300 7f299a7aa700 2 req 6236571265947506715 0.000000000s getting op 4
-31> 2022-04-12T16:09:02.747+0300 7f299a7aa700 2 req 6236571265947506715 0.000000000s :datalog_notify verifying requester
-30> 2022-04-12T16:09:02.751+0300 7f299a7aa700 2 req 6236571265947506715 0.004000004s :datalog_notify normalizing buckets and tenants
-29> 2022-04-12T16:09:02.751+0300 7f299a7aa700 2 req 6236571265947506715 0.004000004s :datalog_notify init permissions
-28> 2022-04-12T16:09:02.751+0300 7f299a7aa700 2 req 6236571265947506715 0.004000004s :datalog_notify recalculating target
-27> 2022-04-12T16:09:02.751+0300 7f299a7aa700 2 req 6236571265947506715 0.004000004s :datalog_notify reading permissions
-26> 2022-04-12T16:09:02.751+0300 7f299a7aa700 2 req 6236571265947506715 0.004000004s :datalog_notify init op
-25> 2022-04-12T16:09:02.751+0300 7f299a7aa700 2 req 6236571265947506715 0.004000004s :datalog_notify verifying op mask
-24> 2022-04-12T16:09:02.751+0300 7f299a7aa700 2 req 6236571265947506715 0.004000004s :datalog_notify verifying op permissions
-23> 2022-04-12T16:09:02.751+0300 7f299a7aa700 2 overriding permissions due to system operation
-22> 2022-04-12T16:09:02.751+0300 7f299a7aa700 2 req 6236571265947506715 0.004000004s :datalog_notify verifying op params
-21> 2022-04-12T16:09:02.751+0300 7f299a7aa700 2 req 6236571265947506715 0.004000004s :datalog_notify pre-executing
-20> 2022-04-12T16:09:02.751+0300 7f299a7aa700 2 req 6236571265947506715 0.004000004s :datalog_notify executing
-19> 2022-04-12T16:09:02.751+0300 7f299a7aa700 2 req 6236571265947506715 0.004000004s :datalog_notify completing
-18> 2022-04-12T16:09:02.751+0300 7f299a7aa700 2 req 6236571265947506715 0.004000004s :datalog_notify op status=0
-17> 2022-04-12T16:09:02.751+0300 7f299a7aa700 2 req 6236571265947506715 0.004000004s :datalog_notify http status=200
-16> 2022-04-12T16:09:02.751+0300 7f299a7aa700 1 ====== req done req=0x7f28f55de620 op status=0 http_status=200 latency=0.004000004s ======
-15> 2022-04-12T16:09:02.751+0300 7f299a7aa700 1 beast: 0x7f28f55de620: 192.168.166.83 - rgw_zone_user [12/Apr/2022:16:09:02.747 +0300] "POST /admin/log?type=data&notify&source-zone=6fa70b8c-6324-4bee-8dbf-d323e8dc5f43&rgwx-zonegroup=784a05a5-2089-4564-ab1b-a55c43c5f401 HTTP/1.1" 200 0 - - - latency=0.004000004s
-14> 2022-04-12T16:09:02.751+0300 7f2a06882700 4 data sync: starting sync on testbucket:6fa70b8c-6324-4bee-8dbf-d323e8dc5f43.5105.1:3 key=testbucket:6fa70b8c-6324-4bee-8dbf-d323e8dc5f43.5105.1:3
-13> 2022-04-12T16:09:02.759+0300 7f2a06882700 5 RGW-SYNC:data:sync:shard[30]:entry:bucket_sync_sources[target=:[]):source_bucket=:[]):source_zone=6fa70b8c-6324-4bee-8dbf-d323e8dc5f43]:bucket[testbucket:6fa70b8c-6324-4bee-8dbf-d323e8dc5f43.5105.1:3<-testbucket:6fa70b8c-6324-4bee-8dbf-d323e8dc5f43.5105.1:3]:full_sync[testbucket:6fa70b8c-6324-4bee-8dbf-d323e8dc5f43.5105.1:3]:entry[amazon_case.png]: bucket sync: sync obj: 6fa70b8c-6324-4bee-8dbf-d323e8dc5f43/:testbucket[6fa70b8c-6324-4bee-8dbf-d323e8dc5f43.5105.1])/amazon_case.png[0]
-12> 2022-04-12T16:09:02.759+0300 7f2a06882700 0 e21904192857b447: sync_object: b=:testbucket[6fa70b8c-6324-4bee-8dbf-d323e8dc5f43.5105.1]) k=amazon_case.png versioned_epoch=0
-11> 2022-04-12T16:09:02.767+0300 7f2a06882700 4 rgw rados thread: AWS: download begin: z=6fa70b8c-6324-4bee-8dbf-d323e8dc5f43 b=:testbucket[6fa70b8c-6324-4bee-8dbf-d323e8dc5f43.5105.1]) k=amazon_case.png size=1880884 mtime=2022-04-12T16:09:02.643682+0300 etag=8cd436573b0ee6e9cc4f6f2cb3ba0cf1 zone_short_id=2425654257 pg_ver=0
-10> 2022-04-12T16:09:02.767+0300 7f2a06882700 0 AWS: creating bucket rgw-cdn-e21904192857b447
-9> 2022-04-12T16:09:02.831+0300 7f28f6662700 10 monclient: tick
-8> 2022-04-12T16:09:02.831+0300 7f28f7e65700 10 monclient: _renew_subs
-7> 2022-04-12T16:09:02.831+0300 7f28f7e65700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0
-6> 2022-04-12T16:09:03.231+0300 7f2a0b08b700 10 monclient: tick
-5> 2022-04-12T16:09:03.663+0300 7f2a1f0b3700 10 monclient: tick
-4> 2022-04-12T16:09:03.831+0300 7f28f6662700 10 monclient: tick
-3> 2022-04-12T16:09:04.231+0300 7f2a0b08b700 10 monclient: tick
-2> 2022-04-12T16:09:04.663+0300 7f2a1f0b3700 10 monclient: tick
-1> 2022-04-12T16:09:04.831+0300 7f28f6662700 10 monclient: tick
0> 2022-04-12T16:09:04.839+0300 7f2a06882700 -1 *** Caught signal (Aborted) **
in thread 7f2a06882700 thread_name:data-sync

ceph version 16.2.7 (dd0603118f56ab514f133c8d2e3adfc983942503) pacific (stable)
1: /lib64/libpthread.so.0(+0x12c20) [0x7f2a2fa66c20]
2: gsignal()
3: abort()
4: /lib64/libstdc++.so.6(+0x9009b) [0x7f2a2ea5a09b]
5: /lib64/libstdc++.so.6(+0x9653c) [0x7f2a2ea6053c]
6: /lib64/libstdc++.so.6(+0x96597) [0x7f2a2ea60597]
7: /lib64/libstdc++.so.6(+0x967f8) [0x7f2a2ea607f8]
8: /lib64/libstdc++.so.6(+0x92025) [0x7f2a2ea5c025]
9: /lib64/libradosgw.so.2(+0x48b109) [0x7f2a3a5a1109]
10: (std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::_M_mutate(unsigned long, unsigned long, char const*, unsigned long)+0x6d) [0x7f2a3a5a471d]
11: (std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::_M_append(char const*, unsigned long)+0x6b) [0x7f2a3a5a489b]
12: (rgw_obj_key::get_oid[abi:cxx11]() const+0xc6) [0x7f2a3a7c9066]
13: (RGWRESTStreamS3PutObj::send_init(rgw::sal::RGWObject*)+0xb3) [0x7f2a3aa8f4b3]
14: (RGWRESTConn::put_obj_send_init(rgw::sal::RGWObject*, rgw_http_param_pair const*, RGWRESTStreamS3PutObj**)+0x5c1) [0x7f2a3aa99891]
15: (RGWAWSStreamPutCRF::init()+0xf1) [0x7f2a3a918ec1]
16: (RGWStreamSpliceCR::operate(DoutPrefixProvider const*)+0xa2f) [0x7f2a3a9b567f]
17: (RGWCoroutinesStack::operate(DoutPrefixProvider const*, RGWCoroutinesEnv*)+0x15c) [0x7f2a3a9a0cbc]
18: (RGWCoroutinesManager::run(DoutPrefixProvider const*, std::__cxx11::list<RGWCoroutinesStack*, std::allocator<RGWCoroutinesStack*> >&)+0x296) [0x7f2a3a9a1b06]
19: (RGWCoroutinesManager::run(DoutPrefixProvider const*, RGWCoroutine*)+0x91) [0x7f2a3a9a2dd1]
20: (RGWRemoteDataLog::run_sync(DoutPrefixProvider const*, int)+0x1b4) [0x7f2a3a8b84b4]
21: (RGWDataSyncProcessorThread::process(DoutPrefixProvider const*)+0x59) [0x7f2a3aa6fe09]
22: (RGWRadosThread::Worker::entry()+0x13a) [0x7f2a3aa3368a]
23: /lib64/libpthread.so.0(+0x817a) [0x7f2a2fa5c17a]
24: clone()
NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.

--- logging levels ---
0/ 5 none
0/ 1 lockdep
0/ 1 context
1/ 1 crush
1/ 5 mds
1/ 5 mds_balancer
1/ 5 mds_locker
1/ 5 mds_log
1/ 5 mds_log_expire
1/ 5 mds_migrator
0/ 1 buffer
0/ 1 timer
0/ 1 filer
0/ 1 striper
0/ 1 objecter
0/ 5 rados
0/ 5 rbd
0/ 5 rbd_mirror
0/ 5 rbd_replay
0/ 5 rbd_pwl
0/ 5 journaler
0/ 5 objectcacher
0/ 5 immutable_obj_cache
0/ 5 client
1/ 5 osd
0/ 5 optracker
0/ 5 objclass
1/ 3 filestore
1/ 3 journal
0/ 0 ms
1/ 5 mon
0/10 monc
1/ 5 paxos
0/ 5 tp
1/ 5 auth
1/ 5 crypto
1/ 1 finisher
1/ 1 reserver
1/ 5 heartbeatmap
1/ 5 perfcounter
5/ 5 rgw
5/ 5 rgw_sync
1/10 civetweb
1/ 5 javaclient
1/ 5 asok
1/ 1 throttle
0/ 0 refs
1/ 5 compressor
1/ 5 bluestore
1/ 5 bluefs
1/ 3 bdev
1/ 5 kstore
4/ 5 rocksdb
4/ 5 leveldb
4/ 5 memdb
1/ 5 fuse
2/ 5 mgr
1/ 5 mgrc
1/ 5 dpdk
1/ 5 eventtrace
1/ 5 prioritycache
0/ 5 test
0/ 5 cephfs_mirror
0/ 5 cephsqlite
-2/-2 (syslog threshold)
99/99 (stderr threshold)
--- pthread ID / name mapping for recent threads ---
139813909309184 / safe_timer
139813926094592 / ms_dispatch
139813934487296 / ceph_timer
139813951272704 / io_context_pool
139816662116096 / radosgw
139816729257728 / radosgw
139816846755584 / radosgw
139816964253440 / radosgw
139817123714816 / radosgw
139817425852160 / radosgw
139817501386496 / radosgw
139817937807104 / radosgw
139818063697664 / radosgw
139818105661184 / radosgw
139818147624704 / radosgw
139818181195520 / radosgw
139818197980928 / radosgw
139818231551744 / radosgw
139818256729856 / radosgw
139818315478784 / lifecycle_thr_2
139818349049600 / lifecycle_thr_1
139818382620416 / lifecycle_thr_0
139818474940160 / data-sync
139818491725568 / meta-sync
139818525296384 / rgw_obj_expirer
139818533689088 / rgw_gc
139818550474496 / safe_timer
139818567259904 / ms_dispatch
139818592438016 / io_context_pool
139818600830720 / rgw_dt_lg_renew
139818701543168 / rados_async
139818886182656 / safe_timer
139818902968064 / ms_dispatch
139818911360768 / ceph_timer
139818919753472 / io_context_pool
139818928146176 / io_context_pool
139818986895104 / admin_socket
139818995287808 / service
139819003680512 / msgr-worker-2
139819012073216 / msgr-worker-1
139819020465920 / msgr-worker-0
139819359798272 / radosgw
max_recent 10000
max_new 10000
log_file /var/lib/ceph/crash/2022-04-12T13:09:04.842414Z_3dfe2d76-d561-4c73-ad91-b3b67a78e9a9/log
--- end dump of recent events ---
teardown: managing teardown after SIGCHLD
teardown: Waiting PID 54 to terminate
teardown: Process 54 is terminated
teardown: Bye Bye, container will die with return code 0
    (1-1/1)