root@ceph-ss-test-oct-controller-av-01:~# docker logs -f ceph-rgw-ceph-ss-test-oct-controller-av-01-rgw1 2022-04-12 16:07:44 /opt/ceph-container/bin/entrypoint.sh: static: does not generate config 2022-04-12 16:07:44 /opt/ceph-container/bin/entrypoint.sh: SUCCESS exec: PID 54: spawning /usr/bin/radosgw --cluster av-ceph-test --setuser ceph --setgroup ceph --default-log-to-stderr=true --err-to-stderr=true --default-log-to-file=false --foreground -n client.rgw.ceph-ss-test-oct-controller-av-01.rgw1 -k /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring exec: Waiting 54 to quit 2022-04-12T16:07:44.647+0300 7f2a3b460400 0 deferred set uid:gid to 167:167 (ceph:ceph) 2022-04-12T16:07:44.647+0300 7f2a3b460400 0 ceph version 16.2.7 (dd0603118f56ab514f133c8d2e3adfc983942503) pacific (stable), process radosgw, pid 54 2022-04-12T16:07:44.647+0300 7f2a3b460400 0 framework: beast 2022-04-12T16:07:44.647+0300 7f2a3b460400 0 framework conf key: endpoint, val: 192.168.166.83:8081 2022-04-12T16:07:44.647+0300 7f2a3b460400 1 radosgw_Main not setting numa affinity 2022-04-12T16:07:46.835+0300 7f2a3b460400 5 sync module config (parsed representation): { "config": { "source_bucket": "", "target_path": "rgw-${zonegroup}-${sid}/${bucket}", "connection_id": "", "acls_id": "", "connection": { "id": "", "endpoint": "https://storage.googleapis.com", "host_style": "path", "key": { "access_key": "", "secret": "******" } } }, "connections": [], "acl_profiles": [], "profiles": [ { "name": "", "config": { "source_bucket": "", "target_path": "rgw-${zonegroup}-${sid}/${bucket}", "connection_id": "", "acls_id": "", "connection": { "id": "", "endpoint": "https://storage.googleapis.com", "host_style": "path", "key": { "access_key": "", "secret": "******" } } } } ] } 2022-04-12T16:07:49.163+0300 7f2a3b460400 2 all 8 watchers are set, enabling cache 2022-04-12T16:07:49.163+0300 7f2a3b460400 5 rgw main: tl::expected, obj_version>, boost::system::error_code> logback_generations::read(const DoutPrefixProvider*, optional_yield):409: oid=data_loggenerations_metadata not found 2022-04-12T16:07:49.211+0300 7f2a0e091700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2022-04-12T16:07:52.747+0300 7f2a09888700 2 object expiration: start 2022-04-12T16:07:52.747+0300 7f2a0a089700 2 garbage collection: garbage collection: start 2022-04-12T16:07:52.751+0300 7f2a3b460400 4 rgw main: failed to read sync status info with (2) No such file or directory 2022-04-12T16:07:52.755+0300 7f2a07884700 4 rgw rados thread: failed to read sync status info with (2) No such file or directory 2022-04-12T16:07:52.755+0300 7f2a07884700 1 RGW-SYNC:meta: initialize sync (reason: period is empty) 2022-04-12T16:07:52.755+0300 7f2a07884700 1 rgw rados thread: epoch=0 in sync status comes before remote's oldest mdlog epoch=1, restarting sync 2022-04-12T16:07:52.759+0300 7f2a3b460400 5 rgw main: starting data sync thread for zone avrupa 2022-04-12T16:07:52.759+0300 7f29ff073700 5 lifecycle: schedule life cycle next start time: Tue Apr 12 21:00:00 2022 2022-04-12T16:07:52.759+0300 7f2a01077700 5 lifecycle: schedule life cycle next start time: Tue Apr 12 21:00:00 2022 2022-04-12T16:07:52.759+0300 7f29fd06f700 5 lifecycle: schedule life cycle next start time: Tue Apr 12 21:00:00 2022 2022-04-12T16:07:52.763+0300 7f2a3b460400 0 framework: beast 2022-04-12T16:07:52.763+0300 7f2a3b460400 0 framework conf key: ssl_certificate, val: config://rgw/cert/$realm/$zone.crt 2022-04-12T16:07:52.763+0300 7f2a3b460400 0 framework conf key: ssl_private_key, val: config://rgw/cert/$realm/$zone.key 2022-04-12T16:07:52.763+0300 7f2a3b460400 0 starting handler: beast 2022-04-12T16:07:52.763+0300 7f2a06882700 4 rgw rados thread: failed to read sync status info with (2) No such file or directory 2022-04-12T16:07:52.763+0300 7f2a3b460400 4 frontend listening on 192.168.166.83:8081 2022-04-12T16:07:52.763+0300 7f2a3b460400 0 set uid:gid to 167:167 (ceph:ceph) 2022-04-12T16:07:52.763+0300 7f2a3b460400 4 frontend spawning 512 threads 2022-04-12T16:07:52.799+0300 7f2a3b460400 1 mgrc service_daemon_register rgw.5727 metadata {arch=x86_64,ceph_release=pacific,ceph_version=ceph version 16.2.7 (dd0603118f56ab514f133c8d2e3adfc983942503) pacific (stable),ceph_version_short=16.2.7,container_image=gitlab.example.com:5050/openstack/registry/ceph/daemon:v6.0.6-stable-6.0-pacific-centos-8-x86_64-ceph-ansible-20220121,cpu=Intel(R) Xeon(R) Gold 6230R CPU @ 2.10GHz,distro=centos,distro_description=CentOS Linux 8,distro_version=8,frontend_config#0=beast endpoint=192.168.166.83:8081,frontend_type#0=beast,hostname=ceph-ss-test-oct-controller-av-01,id=ceph-ss-test-oct-controller-av-01.rgw1,kernel_description=#121-Ubuntu SMP Thu Mar 24 16:04:27 UTC 2022,kernel_version=5.4.0-107-generic,mem_cgroup_limit=4294967296,mem_swap_kb=4030460,mem_total_kb=8148192,num_handles=1,os=Linux,pid=54,realm_id=532f2481-54e3-4d51-95fd-19867ca818e9,realm_name=myrealm,zone_id=c81e6d95-54da-48d3-83a4-7bb7b286709f,zone_name=sync,zonegroup_id=784a05a5-2089-4564-ab1b-a55c43c5f401,zonegroup_name=cdn} 2022-04-12T16:07:53.083+0300 7f2a0a089700 2 garbage collection: garbage collection: stop 2022-04-12T16:07:53.643+0300 7f2a09888700 2 object expiration: stop 2022-04-12T16:07:53.951+0300 7f29f8065700 1 ====== starting new request req=0x7f2a3b415620 ===== 2022-04-12T16:07:53.951+0300 7f29f8065700 2 req 6350979191525555842 0.000000000s initializing for trans_id = tx00000582335ecfec73e82-00625579a9-1659-sync 2022-04-12T16:07:53.951+0300 7f29f8065700 2 req 6350979191525555842 0.000000000s getting op 4 2022-04-12T16:07:53.951+0300 7f29f6061700 2 req 6350979191525555842 0.000000000s :post_period verifying requester 2022-04-12T16:07:53.951+0300 7f29f9868700 1 ====== starting new request req=0x7f2a3b313620 ===== 2022-04-12T16:07:53.951+0300 7f29f305b700 1 ====== starting new request req=0x7f2a3b394620 ===== 2022-04-12T16:07:53.951+0300 7f29f9868700 2 req 8268245483269231629 0.000000000s initializing for trans_id = tx0000072beb575c1fd940d-00625579a9-1659-sync 2022-04-12T16:07:53.951+0300 7f29f305b700 2 req 10560051255912296523 0.000000000s initializing for trans_id = tx00000928cd6aaee91f04b-00625579a9-1659-sync 2022-04-12T16:07:53.955+0300 7f29f9868700 2 req 8268245483269231629 0.004000003s getting op 4 2022-04-12T16:07:53.955+0300 7f29f9868700 2 req 8268245483269231629 0.004000003s :post_period verifying requester 2022-04-12T16:07:53.955+0300 7f29f9868700 5 req 8268245483269231629 0.004000003s :post_period error reading user info, uid=m7yH86NH1D44QiwaUUV9 can't authenticate 2022-04-12T16:07:53.955+0300 7f29f9868700 5 req 8268245483269231629 0.004000003s :post_period error reading user info, uid=m7yH86NH1D44QiwaUUV9 can't authenticate 2022-04-12T16:07:53.955+0300 7f29f9868700 5 req 8268245483269231629 0.004000003s :post_period Failed the auth strategy, reason=-2028 2022-04-12T16:07:53.955+0300 7f29f9868700 1 req 8268245483269231629 0.004000003s op->ERRORHANDLER: err_no=-2028 new_err_no=-2028 2022-04-12T16:07:53.955+0300 7f29f0856700 5 req 6350979191525555842 0.004000003s :post_period error reading user info, uid=m7yH86NH1D44QiwaUUV9 can't authenticate 2022-04-12T16:07:53.955+0300 7f29f0856700 5 req 6350979191525555842 0.004000003s :post_period error reading user info, uid=m7yH86NH1D44QiwaUUV9 can't authenticate 2022-04-12T16:07:53.955+0300 7f29f0856700 5 req 6350979191525555842 0.004000003s :post_period Failed the auth strategy, reason=-2028 2022-04-12T16:07:53.955+0300 7f29f0856700 1 req 6350979191525555842 0.004000003s op->ERRORHANDLER: err_no=-2028 new_err_no=-2028 2022-04-12T16:07:53.955+0300 7f29f305b700 2 req 10560051255912296523 0.004000003s getting op 4 2022-04-12T16:07:53.955+0300 7f29f305b700 2 req 10560051255912296523 0.004000003s :post_period verifying requester 2022-04-12T16:07:53.955+0300 7f29e6842700 2 req 8268245483269231629 0.004000003s :post_period op status=0 2022-04-12T16:07:53.955+0300 7f29e6842700 2 req 8268245483269231629 0.004000003s :post_period http status=403 2022-04-12T16:07:53.955+0300 7f29e6842700 1 ====== req done req=0x7f2a3b313620 op status=0 http_status=403 latency=0.004000003s ====== 2022-04-12T16:07:53.955+0300 7f29e6842700 1 beast: 0x7f2a3b313620: 192.168.166.84 - - [12/Apr/2022:16:07:53.951 +0300] "POST /admin/realm/period?period=8b4c894f-cf43-4c56-9af0-1c2b54c74991&epoch=3&rgwx-zonegroup=784a05a5-2089-4564-ab1b-a55c43c5f401 HTTP/1.1" 403 113 - - - latency=0.004000003s 2022-04-12T16:07:53.955+0300 7f29f0856700 2 req 6350979191525555842 0.004000003s :post_period op status=0 2022-04-12T16:07:53.955+0300 7f29f0856700 2 req 6350979191525555842 0.004000003s :post_period http status=403 2022-04-12T16:07:53.955+0300 7f29f0856700 1 ====== req done req=0x7f2a3b415620 op status=0 http_status=403 latency=0.004000003s ====== 2022-04-12T16:07:53.955+0300 7f29f305b700 5 req 10560051255912296523 0.004000003s :post_period error reading user info, uid=m7yH86NH1D44QiwaUUV9 can't authenticate 2022-04-12T16:07:53.955+0300 7f29f305b700 5 req 10560051255912296523 0.004000003s :post_period error reading user info, uid=m7yH86NH1D44QiwaUUV9 can't authenticate 2022-04-12T16:07:53.955+0300 7f29f305b700 5 req 10560051255912296523 0.004000003s :post_period Failed the auth strategy, reason=-2028 2022-04-12T16:07:53.955+0300 7f29f305b700 1 req 10560051255912296523 0.004000003s op->ERRORHANDLER: err_no=-2028 new_err_no=-2028 2022-04-12T16:07:53.955+0300 7f29f0856700 1 beast: 0x7f2a3b415620: 192.168.166.83 - - [12/Apr/2022:16:07:53.951 +0300] "POST /admin/realm/period?period=8b4c894f-cf43-4c56-9af0-1c2b54c74991&epoch=3&rgwx-zonegroup=784a05a5-2089-4564-ab1b-a55c43c5f401 HTTP/1.1" 403 113 - - - latency=0.004000003s 2022-04-12T16:07:53.955+0300 7f29ee051700 2 req 10560051255912296523 0.004000003s :post_period op status=0 2022-04-12T16:07:53.955+0300 7f29ee051700 2 req 10560051255912296523 0.004000003s :post_period http status=403 2022-04-12T16:07:53.955+0300 7f29ee051700 1 ====== req done req=0x7f2a3b394620 op status=0 http_status=403 latency=0.004000003s ====== 2022-04-12T16:07:53.955+0300 7f29ee051700 1 beast: 0x7f2a3b394620: 192.168.166.85 - - [12/Apr/2022:16:07:53.951 +0300] "POST /admin/realm/period?period=8b4c894f-cf43-4c56-9af0-1c2b54c74991&epoch=3&rgwx-zonegroup=784a05a5-2089-4564-ab1b-a55c43c5f401 HTTP/1.1" 403 113 - - - latency=0.004000003s 2022-04-12T16:07:54.731+0300 7f2a07884700 1 RGW-SYNC:meta: start 2022-04-12T16:07:54.731+0300 7f2a07884700 1 RGW-SYNC:meta: realm epoch=2 period id=8b4c894f-cf43-4c56-9af0-1c2b54c74991 2022-04-12T16:07:54.735+0300 7f2a07884700 4 meta sync: cr:s=0x559e571e3180:op=0x559e59df4000:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 2022-04-12T16:07:54.735+0300 7f2a07884700 4 meta sync: cr:s=0x559e59bc08c0:op=0x559e5578a800:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 2022-04-12T16:07:54.735+0300 7f2a07884700 4 meta sync: cr:s=0x559e594b3540:op=0x559e59df4a00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 2022-04-12T16:07:54.735+0300 7f2a07884700 4 meta sync: cr:s=0x559e5988b680:op=0x559e59df5400:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 2022-04-12T16:07:54.735+0300 7f2a07884700 4 meta sync: cr:s=0x559e59bc1180:op=0x559e59d78a00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 2022-04-12T16:07:54.735+0300 7f2a07884700 4 meta sync: cr:s=0x559e596883c0:op=0x559e59dfa800:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 2022-04-12T16:07:54.735+0300 7f2a07884700 4 meta sync: cr:s=0x559e59689b80:op=0x559e59df7200:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 2022-04-12T16:07:54.735+0300 7f2a07884700 4 meta sync: cr:s=0x559e55ddeb40:op=0x559e59df9400:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 2022-04-12T16:07:54.735+0300 7f2a07884700 4 meta sync: cr:s=0x559e59bc17c0:op=0x559e59d79e00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 2022-04-12T16:07:54.735+0300 7f2a07884700 4 meta sync: cr:s=0x559e5965aa00:op=0x559e59dfb200:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 2022-04-12T16:07:54.735+0300 7f2a07884700 4 meta sync: cr:s=0x559e59bc0140:op=0x559e5578b200:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e595b1e00:op=0x559e59dfca00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e55dfd900:op=0x559e59df8a00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e59bc12c0:op=0x559e59d78000:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e59688a00:op=0x559e59e01400:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e59689cc0:op=0x559e59e02800:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e5988b900:op=0x559e59e09400:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e55dfc8c0:op=0x559e59dfde00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e571e3040:op=0x559e59e05400:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e55dfcc80:op=0x559e59d7b200:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e571e3540:op=0x559e59e00a00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e55dfdb80:op=0x559e59dfe800:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e596888c0:op=0x559e59e01e00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e55dfd2c0:op=0x559e59df6800:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e594b3b80:op=0x559e59e08a00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e55dfc280:op=0x559e59dff200:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e5965ac80:op=0x559e59e04000:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e5988bcc0:op=0x559e59e0d400:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e55dfc3c0:op=0x559e59dfd400:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e594b37c0:op=0x559e59e09e00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e5965ab40:op=0x559e59df9e00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e571e2dc0:op=0x559e59e00000:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e5988b540:op=0x559e59e08000:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 2022-04-12T16:07:54.743+0300 7f2a07884700 4 meta sync: cr:s=0x559e55dfcdc0:op=0x559e59df5e00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 2022-04-12T16:07:54.743+0300 7f2a07884700 4 meta sync: cr:s=0x559e59acc3c0:op=0x559e59d79400:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 2022-04-12T16:07:54.743+0300 7f2a07884700 4 meta sync: cr:s=0x559e55dfc780:op=0x559e59d7a800:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 2022-04-12T16:07:54.743+0300 7f2a07884700 4 meta sync: cr:s=0x559e57c2af00:op=0x559e59e17200:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 2022-04-12T16:07:54.747+0300 7f2a07884700 4 meta sync: cr:s=0x559e571e2c80:op=0x559e59e06800:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 2022-04-12T16:07:54.747+0300 7f2a07884700 4 meta sync: cr:s=0x559e59bc0dc0:op=0x559e59e11400:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 2022-04-12T16:07:54.747+0300 7f2a07884700 4 meta sync: cr:s=0x559e59bcd540:op=0x559e59e14000:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 2022-04-12T16:07:54.747+0300 7f2a07884700 4 meta sync: cr:s=0x559e57c2adc0:op=0x559e59e16800:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 2022-04-12T16:07:54.747+0300 7f2a07884700 4 meta sync: cr:s=0x559e55a08140:op=0x559e59e14a00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 2022-04-12T16:07:54.747+0300 7f2a07884700 4 meta sync: cr:s=0x559e564c1cc0:op=0x559e59e04a00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 2022-04-12T16:07:54.747+0300 7f2a07884700 4 meta sync: cr:s=0x559e57c2b040:op=0x559e59e18000:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 2022-04-12T16:07:54.747+0300 7f2a07884700 4 meta sync: cr:s=0x559e59bc1900:op=0x559e59e0e800:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 2022-04-12T16:07:54.747+0300 7f2a07884700 4 meta sync: cr:s=0x559e59bc0640:op=0x559e59e0f200:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 2022-04-12T16:07:54.751+0300 7f2a07884700 4 meta sync: cr:s=0x559e59689e00:op=0x559e59e03200:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 2022-04-12T16:07:54.751+0300 7f2a07884700 4 meta sync: cr:s=0x559e55a083c0:op=0x559e59e0de00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 2022-04-12T16:07:54.751+0300 7f2a07884700 4 meta sync: cr:s=0x559e571e3cc0:op=0x559e59e07200:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 2022-04-12T16:07:54.751+0300 7f2a07884700 4 meta sync: cr:s=0x559e5988be00:op=0x559e59e0b200:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 2022-04-12T16:07:54.751+0300 7f2a07884700 4 meta sync: cr:s=0x559e594b3680:op=0x559e59e0ca00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 2022-04-12T16:07:54.751+0300 7f2a07884700 4 meta sync: cr:s=0x559e5988ba40:op=0x559e59df8000:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 2022-04-12T16:07:54.751+0300 7f2a07884700 4 meta sync: cr:s=0x559e571e3900:op=0x559e59e05e00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 2022-04-12T16:07:54.751+0300 7f2a07884700 4 meta sync: cr:s=0x559e5988bb80:op=0x559e59e0a800:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 2022-04-12T16:07:54.751+0300 7f2a07884700 4 meta sync: cr:s=0x559e55a08780:op=0x559e59e15400:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 2022-04-12T16:07:54.751+0300 7f2a07884700 4 meta sync: cr:s=0x559e5988b7c0:op=0x559e59e0c000:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 2022-04-12T16:07:54.751+0300 7f2a07884700 4 meta sync: cr:s=0x559e595b1b80:op=0x559e59dfc000:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 2022-04-12T16:07:54.751+0300 7f2a07884700 4 meta sync: cr:s=0x559e59bc1540:op=0x559e59e10a00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 2022-04-12T16:07:54.751+0300 7f2a07884700 4 meta sync: cr:s=0x559e59bc0780:op=0x559e59e11e00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 2022-04-12T16:07:54.751+0300 7f2a07884700 4 meta sync: cr:s=0x559e59bcc000:op=0x559e59e13200:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 2022-04-12T16:07:54.755+0300 7f2a07884700 4 meta sync: cr:s=0x559e59bc1b80:op=0x559e59e10000:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 2022-04-12T16:07:54.755+0300 7f2a07884700 4 meta sync: cr:s=0x559e55a08280:op=0x559e59e15e00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 2022-04-12T16:07:54.755+0300 7f2a07884700 4 meta sync: cr:s=0x559e59bcc500:op=0x559e59e12800:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 2022-04-12T16:07:54.803+0300 7f2a07884700 4 meta sync: cr:s=0x559e54a72640:op=0x559e55789e00:18RGWMetaSyncShardCR: adjusting marker pos=user:dashboard 2022-04-12T16:07:54.811+0300 7f2a07884700 4 meta sync: cr:s=0x559e54a72640:op=0x559e55789e00:18RGWMetaSyncShardCR: adjusting marker pos=user:testuser 2022-04-12T16:07:54.835+0300 7f2a07884700 4 meta sync: cr:s=0x559e54a72640:op=0x559e55789e00:18RGWMetaSyncShardCR: adjusting marker pos=user:testuser 2022-04-12T16:07:54.835+0300 7f2a07884700 4 meta sync: cr:s=0x559e54a72640:op=0x559e55789e00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 2022-04-12T16:08:11.211+0300 7f2a0e091700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2022-04-12T16:08:23.959+0300 7f29f505f700 1 ====== starting new request req=0x7f2a3b313620 ===== 2022-04-12T16:08:23.959+0300 7f29f505f700 2 req 8920709381089314344 0.000000000s initializing for trans_id = tx000007bccb9eb0c52c628-00625579c7-1659-sync 2022-04-12T16:08:23.959+0300 7f29f505f700 2 req 8920709381089314344 0.000000000s getting op 4 2022-04-12T16:08:23.959+0300 7f29f505f700 2 req 8920709381089314344 0.000000000s :post_period verifying requester 2022-04-12T16:08:23.959+0300 7f29f505f700 2 req 8920709381089314344 0.000000000s :post_period normalizing buckets and tenants 2022-04-12T16:08:23.959+0300 7f29f505f700 2 req 8920709381089314344 0.000000000s :post_period init permissions 2022-04-12T16:08:23.959+0300 7f29f505f700 2 req 8920709381089314344 0.000000000s :post_period recalculating target 2022-04-12T16:08:23.959+0300 7f29f505f700 2 req 8920709381089314344 0.000000000s :post_period reading permissions 2022-04-12T16:08:23.959+0300 7f29f505f700 2 req 8920709381089314344 0.000000000s :post_period init op 2022-04-12T16:08:23.959+0300 7f29f505f700 2 req 8920709381089314344 0.000000000s :post_period verifying op mask 2022-04-12T16:08:23.959+0300 7f29f505f700 2 req 8920709381089314344 0.000000000s :post_period verifying op permissions 2022-04-12T16:08:23.959+0300 7f29f505f700 2 overriding permissions due to system operation 2022-04-12T16:08:23.959+0300 7f29f505f700 2 req 8920709381089314344 0.000000000s :post_period verifying op params 2022-04-12T16:08:23.959+0300 7f29f505f700 2 req 8920709381089314344 0.000000000s :post_period pre-executing 2022-04-12T16:08:23.959+0300 7f29f505f700 2 req 8920709381089314344 0.000000000s :post_period executing 2022-04-12T16:08:23.959+0300 7f29cc80e700 1 ====== starting new request req=0x7f2a3b415620 ===== 2022-04-12T16:08:23.959+0300 7f29cc80e700 2 req 8609644640538370824 0.000000000s initializing for trans_id = tx00000777b9a2f49a2a308-00625579c7-1659-sync 2022-04-12T16:08:23.959+0300 7f29cc80e700 2 req 8609644640538370824 0.000000000s getting op 4 2022-04-12T16:08:23.959+0300 7f29cc80e700 2 req 8609644640538370824 0.000000000s :post_period verifying requester 2022-04-12T16:08:23.959+0300 7f29cc80e700 2 req 8609644640538370824 0.000000000s :post_period normalizing buckets and tenants 2022-04-12T16:08:23.959+0300 7f29cc80e700 2 req 8609644640538370824 0.000000000s :post_period init permissions 2022-04-12T16:08:23.959+0300 7f29cc80e700 2 req 8609644640538370824 0.000000000s :post_period recalculating target 2022-04-12T16:08:23.959+0300 7f29cc80e700 2 req 8609644640538370824 0.000000000s :post_period reading permissions 2022-04-12T16:08:23.959+0300 7f29cc80e700 2 req 8609644640538370824 0.000000000s :post_period init op 2022-04-12T16:08:23.959+0300 7f29cc80e700 2 req 8609644640538370824 0.000000000s :post_period verifying op mask 2022-04-12T16:08:23.959+0300 7f29cc80e700 2 req 8609644640538370824 0.000000000s :post_period verifying op permissions 2022-04-12T16:08:23.959+0300 7f29cc80e700 2 overriding permissions due to system operation 2022-04-12T16:08:23.959+0300 7f29cc80e700 2 req 8609644640538370824 0.000000000s :post_period verifying op params 2022-04-12T16:08:23.959+0300 7f29cc80e700 2 req 8609644640538370824 0.000000000s :post_period pre-executing 2022-04-12T16:08:23.959+0300 7f29cc80e700 2 req 8609644640538370824 0.000000000s :post_period executing 2022-04-12T16:08:23.959+0300 7f29c8005700 1 ====== starting new request req=0x7f2a3b394620 ===== 2022-04-12T16:08:23.959+0300 7f29c8005700 2 req 14231248929957790225 0.000000000s initializing for trans_id = tx00000c57f8d805c4ac211-00625579c7-1659-sync 2022-04-12T16:08:23.959+0300 7f29c8005700 2 req 14231248929957790225 0.000000000s getting op 4 2022-04-12T16:08:23.959+0300 7f29c8005700 2 req 14231248929957790225 0.000000000s :post_period verifying requester 2022-04-12T16:08:23.959+0300 7f29c8005700 2 req 14231248929957790225 0.000000000s :post_period normalizing buckets and tenants 2022-04-12T16:08:23.959+0300 7f29c8005700 2 req 14231248929957790225 0.000000000s :post_period init permissions 2022-04-12T16:08:23.959+0300 7f29c8005700 2 req 14231248929957790225 0.000000000s :post_period recalculating target 2022-04-12T16:08:23.959+0300 7f29c8005700 2 req 14231248929957790225 0.000000000s :post_period reading permissions 2022-04-12T16:08:23.959+0300 7f29c8005700 2 req 14231248929957790225 0.000000000s :post_period init op 2022-04-12T16:08:23.959+0300 7f29c8005700 2 req 14231248929957790225 0.000000000s :post_period verifying op mask 2022-04-12T16:08:23.959+0300 7f29c8005700 2 req 14231248929957790225 0.000000000s :post_period verifying op permissions 2022-04-12T16:08:23.959+0300 7f29c8005700 2 overriding permissions due to system operation 2022-04-12T16:08:23.959+0300 7f29c8005700 2 req 14231248929957790225 0.000000000s :post_period verifying op params 2022-04-12T16:08:23.959+0300 7f29c8005700 2 req 14231248929957790225 0.000000000s :post_period pre-executing 2022-04-12T16:08:23.959+0300 7f29c8005700 2 req 14231248929957790225 0.000000000s :post_period executing 2022-04-12T16:08:23.963+0300 7f29ac7ce700 4 req 8920709381089314344 0.004000003s :post_period already have epoch >= 3 for period 8b4c894f-cf43-4c56-9af0-1c2b54c74991 2022-04-12T16:08:23.963+0300 7f29ac7ce700 2 req 8920709381089314344 0.004000003s :post_period completing 2022-04-12T16:08:23.963+0300 7f29ac7ce700 2 req 8920709381089314344 0.004000003s :post_period op status=0 2022-04-12T16:08:23.963+0300 7f29ac7ce700 2 req 8920709381089314344 0.004000003s :post_period http status=200 2022-04-12T16:08:23.963+0300 7f29ac7ce700 1 ====== req done req=0x7f2a3b313620 op status=0 http_status=200 latency=0.004000003s ====== 2022-04-12T16:08:23.963+0300 7f29ac7ce700 1 beast: 0x7f2a3b313620: 192.168.166.84 - rgw_zone_user [12/Apr/2022:16:08:23.959 +0300] "POST /admin/realm/period?period=8b4c894f-cf43-4c56-9af0-1c2b54c74991&epoch=3&rgwx-zonegroup=784a05a5-2089-4564-ab1b-a55c43c5f401 HTTP/1.1" 200 1868 - - - latency=0.004000003s 2022-04-12T16:08:23.963+0300 7f29a57c0700 4 req 14231248929957790225 0.004000003s :post_period already have epoch >= 3 for period 8b4c894f-cf43-4c56-9af0-1c2b54c74991 2022-04-12T16:08:23.963+0300 7f29a57c0700 2 req 14231248929957790225 0.004000003s :post_period completing 2022-04-12T16:08:23.963+0300 7f29a57c0700 2 req 14231248929957790225 0.004000003s :post_period op status=0 2022-04-12T16:08:23.963+0300 7f29a57c0700 2 req 14231248929957790225 0.004000003s :post_period http status=200 2022-04-12T16:08:23.963+0300 7f29a57c0700 1 ====== req done req=0x7f2a3b394620 op status=0 http_status=200 latency=0.004000003s ====== 2022-04-12T16:08:23.963+0300 7f29a57c0700 1 beast: 0x7f2a3b394620: 192.168.166.85 - rgw_zone_user [12/Apr/2022:16:08:23.959 +0300] "POST /admin/realm/period?period=8b4c894f-cf43-4c56-9af0-1c2b54c74991&epoch=3&rgwx-zonegroup=784a05a5-2089-4564-ab1b-a55c43c5f401 HTTP/1.1" 200 1868 - - - latency=0.004000003s 2022-04-12T16:08:23.967+0300 7f299e7b2700 4 req 8609644640538370824 0.008000007s :post_period already have epoch >= 3 for period 8b4c894f-cf43-4c56-9af0-1c2b54c74991 2022-04-12T16:08:23.967+0300 7f299e7b2700 2 req 8609644640538370824 0.008000007s :post_period completing 2022-04-12T16:08:23.967+0300 7f299e7b2700 2 req 8609644640538370824 0.008000007s :post_period op status=0 2022-04-12T16:08:23.967+0300 7f299e7b2700 2 req 8609644640538370824 0.008000007s :post_period http status=200 2022-04-12T16:08:23.967+0300 7f299e7b2700 1 ====== req done req=0x7f2a3b415620 op status=0 http_status=200 latency=0.008000007s ====== 2022-04-12T16:08:23.967+0300 7f299e7b2700 1 beast: 0x7f2a3b415620: 192.168.166.83 - rgw_zone_user [12/Apr/2022:16:08:23.959 +0300] "POST /admin/realm/period?period=8b4c894f-cf43-4c56-9af0-1c2b54c74991&epoch=3&rgwx-zonegroup=784a05a5-2089-4564-ab1b-a55c43c5f401 HTTP/1.1" 200 1868 - - - latency=0.008000007s 2022-04-12T16:08:33.211+0300 7f2a0e091700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2022-04-12T16:08:52.691+0300 7f29b5fe1700 1 ====== starting new request req=0x7f28f565f620 ===== 2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s initializing for trans_id = tx00000ecd2ee05e4c9a4ca-00625579e4-1659-sync 2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s getting op 4 2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s :mdlog_notify verifying requester 2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s :mdlog_notify normalizing buckets and tenants 2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s :mdlog_notify init permissions 2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s :mdlog_notify recalculating target 2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s :mdlog_notify reading permissions 2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s :mdlog_notify init op 2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s :mdlog_notify verifying op mask 2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s :mdlog_notify verifying op permissions 2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 overriding permissions due to system operation 2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s :mdlog_notify verifying op params 2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s :mdlog_notify pre-executing 2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s :mdlog_notify executing 2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s :mdlog_notify completing 2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s :mdlog_notify op status=0 2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s :mdlog_notify http status=200 2022-04-12T16:08:52.691+0300 7f29b5fe1700 1 ====== req done req=0x7f28f565f620 op status=0 http_status=200 latency=0.000000000s ====== 2022-04-12T16:08:52.691+0300 7f29b5fe1700 1 beast: 0x7f28f565f620: 192.168.166.85 - rgw_zone_user [12/Apr/2022:16:08:52.691 +0300] "POST /admin/log?type=metadata¬ify&rgwx-zonegroup=784a05a5-2089-4564-ab1b-a55c43c5f401 HTTP/1.1" 200 0 - - - latency=0.000000000s 2022-04-12T16:08:52.699+0300 7f2a07884700 4 meta sync: cr:s=0x559e57c2af00:op=0x559e59e17200:18RGWMetaSyncShardCR: adjusting marker pos= 2022-04-12T16:08:54.783+0300 7f2a07884700 4 meta sync: cr:s=0x559e57c2af00:op=0x559e59e17200:18RGWMetaSyncShardCR: adjusting marker pos=1_1649768932.547706_90.1 2022-04-12T16:08:54.783+0300 7f2a07884700 4 meta sync: cr:s=0x559e57c2af00:op=0x559e59e17200:18RGWMetaSyncShardCR: adjusting marker pos=1_1649768932.547706_90.1 2022-04-12T16:08:55.215+0300 7f2a0e091700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start 2022-04-12T16:09:02.747+0300 7f299a7aa700 1 ====== starting new request req=0x7f28f55de620 ===== 2022-04-12T16:09:02.747+0300 7f299a7aa700 2 req 6236571265947506715 0.000000000s initializing for trans_id = tx00000568cc0860b35981b-00625579ee-1659-sync 2022-04-12T16:09:02.747+0300 7f299a7aa700 2 req 6236571265947506715 0.000000000s getting op 4 2022-04-12T16:09:02.747+0300 7f299a7aa700 2 req 6236571265947506715 0.000000000s :datalog_notify verifying requester 2022-04-12T16:09:02.751+0300 7f299a7aa700 2 req 6236571265947506715 0.004000004s :datalog_notify normalizing buckets and tenants 2022-04-12T16:09:02.751+0300 7f299a7aa700 2 req 6236571265947506715 0.004000004s :datalog_notify init permissions 2022-04-12T16:09:02.751+0300 7f299a7aa700 2 req 6236571265947506715 0.004000004s :datalog_notify recalculating target 2022-04-12T16:09:02.751+0300 7f299a7aa700 2 req 6236571265947506715 0.004000004s :datalog_notify reading permissions 2022-04-12T16:09:02.751+0300 7f299a7aa700 2 req 6236571265947506715 0.004000004s :datalog_notify init op 2022-04-12T16:09:02.751+0300 7f299a7aa700 2 req 6236571265947506715 0.004000004s :datalog_notify verifying op mask 2022-04-12T16:09:02.751+0300 7f299a7aa700 2 req 6236571265947506715 0.004000004s :datalog_notify verifying op permissions 2022-04-12T16:09:02.751+0300 7f299a7aa700 2 overriding permissions due to system operation 2022-04-12T16:09:02.751+0300 7f299a7aa700 2 req 6236571265947506715 0.004000004s :datalog_notify verifying op params 2022-04-12T16:09:02.751+0300 7f299a7aa700 2 req 6236571265947506715 0.004000004s :datalog_notify pre-executing 2022-04-12T16:09:02.751+0300 7f299a7aa700 2 req 6236571265947506715 0.004000004s :datalog_notify executing 2022-04-12T16:09:02.751+0300 7f299a7aa700 2 req 6236571265947506715 0.004000004s :datalog_notify completing 2022-04-12T16:09:02.751+0300 7f299a7aa700 2 req 6236571265947506715 0.004000004s :datalog_notify op status=0 2022-04-12T16:09:02.751+0300 7f299a7aa700 2 req 6236571265947506715 0.004000004s :datalog_notify http status=200 2022-04-12T16:09:02.751+0300 7f299a7aa700 1 ====== req done req=0x7f28f55de620 op status=0 http_status=200 latency=0.004000004s ====== 2022-04-12T16:09:02.751+0300 7f299a7aa700 1 beast: 0x7f28f55de620: 192.168.166.83 - rgw_zone_user [12/Apr/2022:16:09:02.747 +0300] "POST /admin/log?type=data¬ify&source-zone=6fa70b8c-6324-4bee-8dbf-d323e8dc5f43&rgwx-zonegroup=784a05a5-2089-4564-ab1b-a55c43c5f401 HTTP/1.1" 200 0 - - - latency=0.004000004s 2022-04-12T16:09:02.751+0300 7f2a06882700 4 data sync: starting sync on testbucket:6fa70b8c-6324-4bee-8dbf-d323e8dc5f43.5105.1:3 key=testbucket:6fa70b8c-6324-4bee-8dbf-d323e8dc5f43.5105.1:3 2022-04-12T16:09:02.759+0300 7f2a06882700 5 RGW-SYNC:data:sync:shard[30]:entry:bucket_sync_sources[target=:[]):source_bucket=:[]):source_zone=6fa70b8c-6324-4bee-8dbf-d323e8dc5f43]:bucket[testbucket:6fa70b8c-6324-4bee-8dbf-d323e8dc5f43.5105.1:3<-testbucket:6fa70b8c-6324-4bee-8dbf-d323e8dc5f43.5105.1:3]:full_sync[testbucket:6fa70b8c-6324-4bee-8dbf-d323e8dc5f43.5105.1:3]:entry[amazon_case.png]: bucket sync: sync obj: 6fa70b8c-6324-4bee-8dbf-d323e8dc5f43/:testbucket[6fa70b8c-6324-4bee-8dbf-d323e8dc5f43.5105.1])/amazon_case.png[0] 2022-04-12T16:09:02.759+0300 7f2a06882700 0 e21904192857b447: sync_object: b=:testbucket[6fa70b8c-6324-4bee-8dbf-d323e8dc5f43.5105.1]) k=amazon_case.png versioned_epoch=0 2022-04-12T16:09:02.767+0300 7f2a06882700 4 rgw rados thread: AWS: download begin: z=6fa70b8c-6324-4bee-8dbf-d323e8dc5f43 b=:testbucket[6fa70b8c-6324-4bee-8dbf-d323e8dc5f43.5105.1]) k=amazon_case.png size=1880884 mtime=2022-04-12T16:09:02.643682+0300 etag=8cd436573b0ee6e9cc4f6f2cb3ba0cf1 zone_short_id=2425654257 pg_ver=0 2022-04-12T16:09:02.767+0300 7f2a06882700 0 AWS: creating bucket rgw-cdn-e21904192857b447 terminate called after throwing an instance of 'std::length_error' what(): basic_string::_M_create *** Caught signal (Aborted) ** in thread 7f2a06882700 thread_name:data-sync ceph version 16.2.7 (dd0603118f56ab514f133c8d2e3adfc983942503) pacific (stable) 1: /lib64/libpthread.so.0(+0x12c20) [0x7f2a2fa66c20] 2: gsignal() 3: abort() 4: /lib64/libstdc++.so.6(+0x9009b) [0x7f2a2ea5a09b] 5: /lib64/libstdc++.so.6(+0x9653c) [0x7f2a2ea6053c] 6: /lib64/libstdc++.so.6(+0x96597) [0x7f2a2ea60597] 7: /lib64/libstdc++.so.6(+0x967f8) [0x7f2a2ea607f8] 8: /lib64/libstdc++.so.6(+0x92025) [0x7f2a2ea5c025] 9: /lib64/libradosgw.so.2(+0x48b109) [0x7f2a3a5a1109] 10: (std::__cxx11::basic_string, std::allocator >::_M_mutate(unsigned long, unsigned long, char const*, unsigned long)+0x6d) [0x7f2a3a5a471d] 11: (std::__cxx11::basic_string, std::allocator >::_M_append(char const*, unsigned long)+0x6b) [0x7f2a3a5a489b] 12: (rgw_obj_key::get_oid[abi:cxx11]() const+0xc6) [0x7f2a3a7c9066] 13: (RGWRESTStreamS3PutObj::send_init(rgw::sal::RGWObject*)+0xb3) [0x7f2a3aa8f4b3] 14: (RGWRESTConn::put_obj_send_init(rgw::sal::RGWObject*, rgw_http_param_pair const*, RGWRESTStreamS3PutObj**)+0x5c1) [0x7f2a3aa99891] 15: (RGWAWSStreamPutCRF::init()+0xf1) [0x7f2a3a918ec1] 16: (RGWStreamSpliceCR::operate(DoutPrefixProvider const*)+0xa2f) [0x7f2a3a9b567f] 17: (RGWCoroutinesStack::operate(DoutPrefixProvider const*, RGWCoroutinesEnv*)+0x15c) [0x7f2a3a9a0cbc] 18: (RGWCoroutinesManager::run(DoutPrefixProvider const*, std::__cxx11::list >&)+0x296) [0x7f2a3a9a1b06] 19: (RGWCoroutinesManager::run(DoutPrefixProvider const*, RGWCoroutine*)+0x91) [0x7f2a3a9a2dd1] 20: (RGWRemoteDataLog::run_sync(DoutPrefixProvider const*, int)+0x1b4) [0x7f2a3a8b84b4] 21: (RGWDataSyncProcessorThread::process(DoutPrefixProvider const*)+0x59) [0x7f2a3aa6fe09] 22: (RGWRadosThread::Worker::entry()+0x13a) [0x7f2a3aa3368a] 23: /lib64/libpthread.so.0(+0x817a) [0x7f2a2fa5c17a] 24: clone() 2022-04-12T16:09:04.839+0300 7f2a06882700 -1 *** Caught signal (Aborted) ** in thread 7f2a06882700 thread_name:data-sync ceph version 16.2.7 (dd0603118f56ab514f133c8d2e3adfc983942503) pacific (stable) 1: /lib64/libpthread.so.0(+0x12c20) [0x7f2a2fa66c20] 2: gsignal() 3: abort() 4: /lib64/libstdc++.so.6(+0x9009b) [0x7f2a2ea5a09b] 5: /lib64/libstdc++.so.6(+0x9653c) [0x7f2a2ea6053c] 6: /lib64/libstdc++.so.6(+0x96597) [0x7f2a2ea60597] 7: /lib64/libstdc++.so.6(+0x967f8) [0x7f2a2ea607f8] 8: /lib64/libstdc++.so.6(+0x92025) [0x7f2a2ea5c025] 9: /lib64/libradosgw.so.2(+0x48b109) [0x7f2a3a5a1109] 10: (std::__cxx11::basic_string, std::allocator >::_M_mutate(unsigned long, unsigned long, char const*, unsigned long)+0x6d) [0x7f2a3a5a471d] 11: (std::__cxx11::basic_string, std::allocator >::_M_append(char const*, unsigned long)+0x6b) [0x7f2a3a5a489b] 12: (rgw_obj_key::get_oid[abi:cxx11]() const+0xc6) [0x7f2a3a7c9066] 13: (RGWRESTStreamS3PutObj::send_init(rgw::sal::RGWObject*)+0xb3) [0x7f2a3aa8f4b3] 14: (RGWRESTConn::put_obj_send_init(rgw::sal::RGWObject*, rgw_http_param_pair const*, RGWRESTStreamS3PutObj**)+0x5c1) [0x7f2a3aa99891] 15: (RGWAWSStreamPutCRF::init()+0xf1) [0x7f2a3a918ec1] 16: (RGWStreamSpliceCR::operate(DoutPrefixProvider const*)+0xa2f) [0x7f2a3a9b567f] 17: (RGWCoroutinesStack::operate(DoutPrefixProvider const*, RGWCoroutinesEnv*)+0x15c) [0x7f2a3a9a0cbc] 18: (RGWCoroutinesManager::run(DoutPrefixProvider const*, std::__cxx11::list >&)+0x296) [0x7f2a3a9a1b06] 19: (RGWCoroutinesManager::run(DoutPrefixProvider const*, RGWCoroutine*)+0x91) [0x7f2a3a9a2dd1] 20: (RGWRemoteDataLog::run_sync(DoutPrefixProvider const*, int)+0x1b4) [0x7f2a3a8b84b4] 21: (RGWDataSyncProcessorThread::process(DoutPrefixProvider const*)+0x59) [0x7f2a3aa6fe09] 22: (RGWRadosThread::Worker::entry()+0x13a) [0x7f2a3aa3368a] 23: /lib64/libpthread.so.0(+0x817a) [0x7f2a2fa5c17a] 24: clone() NOTE: a copy of the executable, or `objdump -rdS ` is needed to interpret this. --- begin dump of recent events --- -1216> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command assert hook 0x559e54a2a760 -1215> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command abort hook 0x559e54a2a760 -1214> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command leak_some_memory hook 0x559e54a2a760 -1213> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command perfcounters_dump hook 0x559e54a2a760 -1212> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command 1 hook 0x559e54a2a760 -1211> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command perf dump hook 0x559e54a2a760 -1210> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command perfcounters_schema hook 0x559e54a2a760 -1209> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command perf histogram dump hook 0x559e54a2a760 -1208> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command 2 hook 0x559e54a2a760 -1207> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command perf schema hook 0x559e54a2a760 -1206> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command perf histogram schema hook 0x559e54a2a760 -1205> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command perf reset hook 0x559e54a2a760 -1204> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command config show hook 0x559e54a2a760 -1203> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command config help hook 0x559e54a2a760 -1202> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command config set hook 0x559e54a2a760 -1201> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command config unset hook 0x559e54a2a760 -1200> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command config get hook 0x559e54a2a760 -1199> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command config diff hook 0x559e54a2a760 -1198> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command config diff get hook 0x559e54a2a760 -1197> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command injectargs hook 0x559e54a2a760 -1196> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command log flush hook 0x559e54a2a760 -1195> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command log dump hook 0x559e54a2a760 -1194> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command log reopen hook 0x559e54a2a760 -1193> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command dump_mempools hook 0x559e54b02068 -1192> 2022-04-12T16:07:44.643+0300 7f2a3b460400 10 monclient: get_monmap_and_config -1191> 2022-04-12T16:07:44.643+0300 7f2a3b460400 10 monclient: build_initial_monmap -1190> 2022-04-12T16:07:44.643+0300 7f2a3b460400 10 monclient: monmap: epoch 0 fsid 65ccd25a-39ce-4072-a5c6-a0e6654bda87 last_changed 2022-04-12T16:07:44.650033+0300 created 2022-04-12T16:07:44.650033+0300 min_mon_release 0 (unknown) election_strategy: 1 0: [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] mon.noname-a 1: [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] mon.noname-b 2: [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] mon.noname-c -1189> 2022-04-12T16:07:44.643+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding auth protocol: cephx -1188> 2022-04-12T16:07:44.643+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding auth protocol: cephx -1187> 2022-04-12T16:07:44.643+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding auth protocol: cephx -1186> 2022-04-12T16:07:44.643+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding auth protocol: none -1185> 2022-04-12T16:07:44.643+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: secure -1184> 2022-04-12T16:07:44.643+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: crc -1183> 2022-04-12T16:07:44.643+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: secure -1182> 2022-04-12T16:07:44.643+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: crc -1181> 2022-04-12T16:07:44.643+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: secure -1180> 2022-04-12T16:07:44.643+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: crc -1179> 2022-04-12T16:07:44.643+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: crc -1178> 2022-04-12T16:07:44.643+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: secure -1177> 2022-04-12T16:07:44.643+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: crc -1176> 2022-04-12T16:07:44.643+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: secure -1175> 2022-04-12T16:07:44.643+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: crc -1174> 2022-04-12T16:07:44.643+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: secure -1173> 2022-04-12T16:07:44.647+0300 7f2a3b460400 2 auth: KeyRing::load: loaded key file /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring -1172> 2022-04-12T16:07:44.647+0300 7f2a3b460400 10 monclient: init -1171> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d65c0) adding auth protocol: cephx -1170> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d65c0) adding auth protocol: cephx -1169> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d65c0) adding auth protocol: cephx -1168> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d65c0) adding auth protocol: none -1167> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d65c0) adding con mode: secure -1166> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d65c0) adding con mode: crc -1165> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d65c0) adding con mode: secure -1164> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d65c0) adding con mode: crc -1163> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d65c0) adding con mode: secure -1162> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d65c0) adding con mode: crc -1161> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d65c0) adding con mode: crc -1160> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d65c0) adding con mode: secure -1159> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d65c0) adding con mode: crc -1158> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d65c0) adding con mode: secure -1157> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d65c0) adding con mode: crc -1156> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d65c0) adding con mode: secure -1155> 2022-04-12T16:07:44.647+0300 7f2a3b460400 2 auth: KeyRing::load: loaded key file /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring -1154> 2022-04-12T16:07:44.647+0300 7f2a3b460400 2 auth: KeyRing::load: loaded key file /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring -1153> 2022-04-12T16:07:44.647+0300 7f2a3b460400 10 monclient: _reopen_session rank -1 -1152> 2022-04-12T16:07:44.647+0300 7f2a3b460400 10 monclient: _add_conns ranks=[1,2,0] -1151> 2022-04-12T16:07:44.647+0300 7f2a3b460400 10 monclient(hunting): picked mon.noname-b con 0x559e54b38800 addr [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] -1150> 2022-04-12T16:07:44.647+0300 7f2a3b460400 10 monclient(hunting): picked mon.noname-c con 0x559e54b38c00 addr [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] -1149> 2022-04-12T16:07:44.647+0300 7f2a3b460400 10 monclient(hunting): picked mon.noname-a con 0x559e54b39000 addr [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] -1148> 2022-04-12T16:07:44.647+0300 7f2a3b460400 10 monclient(hunting): start opening mon connection -1147> 2022-04-12T16:07:44.647+0300 7f2a3b460400 10 monclient(hunting): start opening mon connection -1146> 2022-04-12T16:07:44.647+0300 7f2a3b460400 10 monclient(hunting): start opening mon connection -1145> 2022-04-12T16:07:44.647+0300 7f2a3b460400 10 monclient(hunting): _renew_subs -1144> 2022-04-12T16:07:44.647+0300 7f2a3b460400 10 monclient(hunting): authenticate will time out at 2022-04-12T16:12:44.651302+0300 -1143> 2022-04-12T16:07:44.647+0300 7f2a270c3700 10 monclient(hunting): get_auth_request con 0x559e54b39000 auth_method 0 -1142> 2022-04-12T16:07:44.647+0300 7f2a270c3700 10 monclient(hunting): get_auth_request method 2 preferred_modes [2,1] -1141> 2022-04-12T16:07:44.647+0300 7f2a270c3700 10 monclient(hunting): _init_auth method 2 -1140> 2022-04-12T16:07:44.647+0300 7f2a270c3700 10 monclient(hunting): _init_auth creating new auth -1139> 2022-04-12T16:07:44.647+0300 7f2a270c3700 10 monclient(hunting): handle_auth_reply_more payload 9 -1138> 2022-04-12T16:07:44.647+0300 7f2a270c3700 10 monclient(hunting): handle_auth_reply_more payload_len 9 -1137> 2022-04-12T16:07:44.647+0300 7f2a270c3700 10 monclient(hunting): handle_auth_reply_more responding with 36 bytes -1136> 2022-04-12T16:07:44.647+0300 7f2a270c3700 10 monclient(hunting): handle_auth_done global_id 5715 payload 322 -1135> 2022-04-12T16:07:44.647+0300 7f2a270c3700 10 monclient: _finish_hunting 0 -1134> 2022-04-12T16:07:44.647+0300 7f2a270c3700 1 monclient: found mon.noname-a -1133> 2022-04-12T16:07:44.647+0300 7f2a270c3700 10 monclient: _send_mon_message to mon.noname-a at v2:192.168.166.83:3300/0 -1132> 2022-04-12T16:07:44.647+0300 7f2a258c0700 10 monclient: handle_monmap mon_map magic: 0 v1 -1131> 2022-04-12T16:07:44.647+0300 7f2a258c0700 10 monclient: got monmap 1 from mon.noname-a (according to old e1) -1130> 2022-04-12T16:07:44.647+0300 7f2a258c0700 10 monclient: dump: epoch 1 fsid 65ccd25a-39ce-4072-a5c6-a0e6654bda87 last_changed 2022-04-12T15:48:27.663651+0300 created 2022-04-12T15:48:27.663651+0300 min_mon_release 16 (pacific) election_strategy: 1 0: [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] mon.ceph-ss-test-oct-controller-av-01 1: [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] mon.ceph-ss-test-oct-controller-av-02 2: [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] mon.ceph-ss-test-oct-controller-av-03 -1129> 2022-04-12T16:07:44.647+0300 7f2a258c0700 10 monclient: _finish_auth 0 -1128> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 monclient: authenticate success, global_id 5715 -1127> 2022-04-12T16:07:44.647+0300 7f2a258c0700 10 monclient: handle_config config(0 keys) v1 -1126> 2022-04-12T16:07:44.647+0300 7f2a3b460400 10 monclient: get_monmap_and_config success -1125> 2022-04-12T16:07:44.647+0300 7f2a3b460400 4 set_mon_vals no callback set -1124> 2022-04-12T16:07:44.647+0300 7f2a3b460400 10 monclient: shutdown -1123> 2022-04-12T16:07:44.647+0300 7f2a3b460400 0 deferred set uid:gid to 167:167 (ceph:ceph) -1122> 2022-04-12T16:07:44.647+0300 7f2a3b460400 0 ceph version 16.2.7 (dd0603118f56ab514f133c8d2e3adfc983942503) pacific (stable), process radosgw, pid 54 -1121> 2022-04-12T16:07:44.647+0300 7f2a3b460400 0 framework: beast -1120> 2022-04-12T16:07:44.647+0300 7f2a3b460400 0 framework conf key: endpoint, val: 192.168.166.83:8081 -1119> 2022-04-12T16:07:44.647+0300 7f2a3b460400 1 radosgw_Main not setting numa affinity -1118> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 asok(0x559e54b34000) init /var/run/ceph/av-ceph-test-client.rgw.ceph-ss-test-oct-controller-av-01.rgw1.54.94138514240960.asok -1117> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 asok(0x559e54b34000) bind_and_listen /var/run/ceph/av-ceph-test-client.rgw.ceph-ss-test-oct-controller-av-01.rgw1.54.94138514240960.asok -1116> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command 0 hook 0x559e54a6c360 -1115> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command version hook 0x559e54a6c360 -1114> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command git_version hook 0x559e54a6c360 -1113> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command help hook 0x559e54a2a4b0 -1112> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command get_command_descriptions hook 0x559e54a2a4c0 -1111> 2022-04-12T16:07:44.651+0300 7f2a250bf700 5 asok(0x559e54b34000) entry start -1110> 2022-04-12T16:07:44.651+0300 7f2a3b460400 10 monclient: get_monmap_and_config -1109> 2022-04-12T16:07:44.651+0300 7f2a3b460400 10 monclient: build_initial_monmap -1108> 2022-04-12T16:07:44.651+0300 7f2a3b460400 10 monclient: monmap: epoch 0 fsid 00000000-0000-0000-0000-000000000000 last_changed 0.000000 created 0.000000 min_mon_release 0 (unknown) election_strategy: 1 0: [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] mon.noname-a 1: [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] mon.noname-b 2: [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] mon.noname-c -1107> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding auth protocol: cephx -1106> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding auth protocol: cephx -1105> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding auth protocol: cephx -1104> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding auth protocol: none -1103> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: secure -1102> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: crc -1101> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: secure -1100> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: crc -1099> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: secure -1098> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: crc -1097> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: crc -1096> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: secure -1095> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: crc -1094> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: secure -1093> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: crc -1092> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: secure -1091> 2022-04-12T16:07:44.651+0300 7f2a3b460400 2 auth: KeyRing::load: loaded key file /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring -1090> 2022-04-12T16:07:44.651+0300 7f2a3b460400 10 monclient: init -1089> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d6fa0) adding auth protocol: cephx -1088> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d6fa0) adding auth protocol: cephx -1087> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d6fa0) adding auth protocol: cephx -1086> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d6fa0) adding auth protocol: none -1085> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d6fa0) adding con mode: secure -1084> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d6fa0) adding con mode: crc -1083> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d6fa0) adding con mode: secure -1082> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d6fa0) adding con mode: crc -1081> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d6fa0) adding con mode: secure -1080> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d6fa0) adding con mode: crc -1079> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d6fa0) adding con mode: crc -1078> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d6fa0) adding con mode: secure -1077> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d6fa0) adding con mode: crc -1076> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d6fa0) adding con mode: secure -1075> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d6fa0) adding con mode: crc -1074> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d6fa0) adding con mode: secure -1073> 2022-04-12T16:07:44.651+0300 7f2a3b460400 2 auth: KeyRing::load: loaded key file /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring -1072> 2022-04-12T16:07:44.651+0300 7f2a3b460400 2 auth: KeyRing::load: loaded key file /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring -1071> 2022-04-12T16:07:44.651+0300 7f2a3b460400 10 monclient: _reopen_session rank -1 -1070> 2022-04-12T16:07:44.651+0300 7f2a3b460400 10 monclient: _add_conns ranks=[0,1,2] -1069> 2022-04-12T16:07:44.651+0300 7f2a3b460400 10 monclient(hunting): picked mon.noname-a con 0x559e54b39000 addr [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] -1068> 2022-04-12T16:07:44.651+0300 7f2a3b460400 10 monclient(hunting): picked mon.noname-b con 0x559e54b38c00 addr [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] -1067> 2022-04-12T16:07:44.651+0300 7f2a3b460400 10 monclient(hunting): picked mon.noname-c con 0x559e54b38800 addr [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] -1066> 2022-04-12T16:07:44.651+0300 7f2a3b460400 10 monclient(hunting): start opening mon connection -1065> 2022-04-12T16:07:44.651+0300 7f2a3b460400 10 monclient(hunting): start opening mon connection -1064> 2022-04-12T16:07:44.651+0300 7f2a3b460400 10 monclient(hunting): start opening mon connection -1063> 2022-04-12T16:07:44.651+0300 7f2a3b460400 10 monclient(hunting): _renew_subs -1062> 2022-04-12T16:07:44.651+0300 7f2a3b460400 10 monclient(hunting): authenticate will time out at 2022-04-12T16:12:44.656631+0300 -1061> 2022-04-12T16:07:44.651+0300 7f2a268c2700 10 monclient(hunting): get_auth_request con 0x559e54b39000 auth_method 0 -1060> 2022-04-12T16:07:44.651+0300 7f2a268c2700 10 monclient(hunting): get_auth_request method 2 preferred_modes [2,1] -1059> 2022-04-12T16:07:44.651+0300 7f2a268c2700 10 monclient(hunting): _init_auth method 2 -1058> 2022-04-12T16:07:44.651+0300 7f2a268c2700 10 monclient(hunting): _init_auth creating new auth -1057> 2022-04-12T16:07:44.651+0300 7f2a268c2700 10 monclient(hunting): handle_auth_reply_more payload 9 -1056> 2022-04-12T16:07:44.651+0300 7f2a268c2700 10 monclient(hunting): handle_auth_reply_more payload_len 9 -1055> 2022-04-12T16:07:44.651+0300 7f2a268c2700 10 monclient(hunting): handle_auth_reply_more responding with 36 bytes -1054> 2022-04-12T16:07:44.651+0300 7f2a268c2700 10 monclient(hunting): handle_auth_done global_id 5718 payload 322 -1053> 2022-04-12T16:07:44.651+0300 7f2a268c2700 10 monclient: _finish_hunting 0 -1052> 2022-04-12T16:07:44.651+0300 7f2a268c2700 1 monclient: found mon.noname-a -1051> 2022-04-12T16:07:44.651+0300 7f2a268c2700 10 monclient: _send_mon_message to mon.noname-a at v2:192.168.166.83:3300/0 -1050> 2022-04-12T16:07:44.651+0300 7f2a218b8700 10 monclient: handle_monmap mon_map magic: 0 v1 -1049> 2022-04-12T16:07:44.651+0300 7f2a218b8700 10 monclient: got monmap 1 from mon.noname-a (according to old e1) -1048> 2022-04-12T16:07:44.651+0300 7f2a218b8700 10 monclient: dump: epoch 1 fsid 65ccd25a-39ce-4072-a5c6-a0e6654bda87 last_changed 2022-04-12T15:48:27.663651+0300 created 2022-04-12T15:48:27.663651+0300 min_mon_release 16 (pacific) election_strategy: 1 0: [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] mon.ceph-ss-test-oct-controller-av-01 1: [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] mon.ceph-ss-test-oct-controller-av-02 2: [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] mon.ceph-ss-test-oct-controller-av-03 -1047> 2022-04-12T16:07:44.651+0300 7f2a218b8700 10 monclient: _finish_auth 0 -1046> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 monclient: authenticate success, global_id 5718 -1045> 2022-04-12T16:07:44.651+0300 7f2a218b8700 10 monclient: handle_config config(0 keys) v1 -1044> 2022-04-12T16:07:44.651+0300 7f2a3b460400 10 monclient: get_monmap_and_config success -1043> 2022-04-12T16:07:44.651+0300 7f2a3b460400 4 set_mon_vals no callback set -1042> 2022-04-12T16:07:44.651+0300 7f2a3b460400 10 monclient: shutdown -1041> 2022-04-12T16:07:44.655+0300 7f2a3b460400 10 monclient: build_initial_monmap -1040> 2022-04-12T16:07:44.655+0300 7f2a3b460400 10 monclient: monmap: epoch 0 fsid 00000000-0000-0000-0000-000000000000 last_changed 0.000000 created 0.000000 min_mon_release 0 (unknown) election_strategy: 1 0: [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] mon.noname-a 1: [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] mon.noname-b 2: [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] mon.noname-c -1039> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding auth protocol: cephx -1038> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding auth protocol: cephx -1037> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding auth protocol: cephx -1036> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding auth protocol: none -1035> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: secure -1034> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: crc -1033> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: secure -1032> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: crc -1031> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: secure -1030> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: crc -1029> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: crc -1028> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: secure -1027> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: crc -1026> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: secure -1025> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: crc -1024> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: secure -1023> 2022-04-12T16:07:44.655+0300 7f2a3b460400 2 auth: KeyRing::load: loaded key file /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring -1022> 2022-04-12T16:07:44.655+0300 7f2a3b460400 1 librados: starting msgr at -1021> 2022-04-12T16:07:44.655+0300 7f2a3b460400 1 librados: starting objecter -1020> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command objecter_requests hook 0x559e54a2a510 -1019> 2022-04-12T16:07:44.655+0300 7f2a3b460400 1 librados: setting wanted keys -1018> 2022-04-12T16:07:44.655+0300 7f2a3b460400 1 librados: calling monclient init -1017> 2022-04-12T16:07:44.655+0300 7f2a3b460400 10 monclient: init -1016> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54a700d8) adding auth protocol: cephx -1015> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54a700d8) adding auth protocol: cephx -1014> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54a700d8) adding auth protocol: cephx -1013> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54a700d8) adding auth protocol: none -1012> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54a700d8) adding con mode: secure -1011> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54a700d8) adding con mode: crc -1010> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54a700d8) adding con mode: secure -1009> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54a700d8) adding con mode: crc -1008> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54a700d8) adding con mode: secure -1007> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54a700d8) adding con mode: crc -1006> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54a700d8) adding con mode: crc -1005> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54a700d8) adding con mode: secure -1004> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54a700d8) adding con mode: crc -1003> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54a700d8) adding con mode: secure -1002> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54a700d8) adding con mode: crc -1001> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54a700d8) adding con mode: secure -1000> 2022-04-12T16:07:44.655+0300 7f2a3b460400 2 auth: KeyRing::load: loaded key file /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring -999> 2022-04-12T16:07:44.655+0300 7f2a3b460400 2 auth: KeyRing::load: loaded key file /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring -998> 2022-04-12T16:07:44.655+0300 7f2a3b460400 10 monclient: _reopen_session rank -1 -997> 2022-04-12T16:07:44.655+0300 7f2a3b460400 10 monclient: _add_conns ranks=[0,2,1] -996> 2022-04-12T16:07:44.655+0300 7f2a3b460400 10 monclient(hunting): picked mon.noname-a con 0x559e54b38800 addr [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] -995> 2022-04-12T16:07:44.655+0300 7f2a3b460400 10 monclient(hunting): picked mon.noname-c con 0x559e54b38c00 addr [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] -994> 2022-04-12T16:07:44.655+0300 7f2a3b460400 10 monclient(hunting): picked mon.noname-b con 0x559e54b39000 addr [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] -993> 2022-04-12T16:07:44.655+0300 7f2a3b460400 10 monclient(hunting): start opening mon connection -992> 2022-04-12T16:07:44.655+0300 7f2a3b460400 10 monclient(hunting): start opening mon connection -991> 2022-04-12T16:07:44.655+0300 7f2a3b460400 10 monclient(hunting): start opening mon connection -990> 2022-04-12T16:07:44.655+0300 7f2a3b460400 10 monclient(hunting): _renew_subs -989> 2022-04-12T16:07:44.655+0300 7f2a3b460400 10 monclient(hunting): authenticate will time out at 2022-04-12T16:12:44.659081+0300 -988> 2022-04-12T16:07:44.655+0300 7f2a268c2700 10 monclient(hunting): get_auth_request con 0x559e54b38800 auth_method 0 -987> 2022-04-12T16:07:44.655+0300 7f2a268c2700 10 monclient(hunting): get_auth_request method 2 preferred_modes [2,1] -986> 2022-04-12T16:07:44.655+0300 7f2a268c2700 10 monclient(hunting): _init_auth method 2 -985> 2022-04-12T16:07:44.655+0300 7f2a268c2700 10 monclient(hunting): _init_auth creating new auth -984> 2022-04-12T16:07:44.655+0300 7f2a268c2700 10 monclient(hunting): handle_auth_reply_more payload 9 -983> 2022-04-12T16:07:44.655+0300 7f2a268c2700 10 monclient(hunting): handle_auth_reply_more payload_len 9 -982> 2022-04-12T16:07:44.655+0300 7f2a268c2700 10 monclient(hunting): handle_auth_reply_more responding with 36 bytes -981> 2022-04-12T16:07:44.655+0300 7f2a268c2700 10 monclient(hunting): handle_auth_done global_id 5721 payload 1000 -980> 2022-04-12T16:07:44.655+0300 7f2a268c2700 10 monclient: _finish_hunting 0 -979> 2022-04-12T16:07:44.655+0300 7f2a268c2700 1 monclient: found mon.noname-a -978> 2022-04-12T16:07:44.655+0300 7f2a268c2700 10 monclient: _send_mon_message to mon.noname-a at v2:192.168.166.83:3300/0 -977> 2022-04-12T16:07:44.655+0300 7f2a200b5700 10 monclient: handle_monmap mon_map magic: 0 v1 -976> 2022-04-12T16:07:44.655+0300 7f2a200b5700 10 monclient: got monmap 1 from mon.noname-a (according to old e1) -975> 2022-04-12T16:07:44.655+0300 7f2a200b5700 10 monclient: dump: epoch 1 fsid 65ccd25a-39ce-4072-a5c6-a0e6654bda87 last_changed 2022-04-12T15:48:27.663651+0300 created 2022-04-12T15:48:27.663651+0300 min_mon_release 16 (pacific) election_strategy: 1 0: [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] mon.ceph-ss-test-oct-controller-av-01 1: [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] mon.ceph-ss-test-oct-controller-av-02 2: [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] mon.ceph-ss-test-oct-controller-av-03 -974> 2022-04-12T16:07:44.655+0300 7f2a200b5700 10 monclient: _finish_auth 0 -973> 2022-04-12T16:07:44.655+0300 7f2a200b5700 10 monclient: handle_config config(0 keys) v1 -972> 2022-04-12T16:07:44.655+0300 7f2a200b5700 10 monclient: handle_monmap mon_map magic: 0 v1 -971> 2022-04-12T16:07:44.655+0300 7f2a218b8700 4 set_mon_vals no callback set -970> 2022-04-12T16:07:44.655+0300 7f2a200b5700 10 monclient: got monmap 1 from mon.ceph-ss-test-oct-controller-av-01 (according to old e1) -969> 2022-04-12T16:07:44.655+0300 7f2a200b5700 10 monclient: dump: epoch 1 fsid 65ccd25a-39ce-4072-a5c6-a0e6654bda87 last_changed 2022-04-12T15:48:27.663651+0300 created 2022-04-12T15:48:27.663651+0300 min_mon_release 16 (pacific) election_strategy: 1 0: [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] mon.ceph-ss-test-oct-controller-av-01 1: [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] mon.ceph-ss-test-oct-controller-av-02 2: [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] mon.ceph-ss-test-oct-controller-av-03 -968> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 monclient: authenticate success, global_id 5721 -967> 2022-04-12T16:07:44.655+0300 7f2a3b460400 10 monclient: _renew_subs -966> 2022-04-12T16:07:44.655+0300 7f2a3b460400 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -965> 2022-04-12T16:07:44.655+0300 7f2a3b460400 10 monclient: _renew_subs -964> 2022-04-12T16:07:44.655+0300 7f2a3b460400 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -963> 2022-04-12T16:07:44.655+0300 7f2a3b460400 1 librados: init done -962> 2022-04-12T16:07:44.659+0300 7f2a200b5700 4 mgrc handle_mgr_map Got map version 29 -961> 2022-04-12T16:07:44.659+0300 7f2a200b5700 4 mgrc handle_mgr_map Active mgr is now [v2:192.168.166.84:6800/56,v1:192.168.166.84:6801/56] -960> 2022-04-12T16:07:44.659+0300 7f2a200b5700 4 mgrc reconnect Starting new session with [v2:192.168.166.84:6800/56,v1:192.168.166.84:6801/56] -959> 2022-04-12T16:07:44.659+0300 7f2a260c1700 10 monclient: get_auth_request con 0x559e54b39800 auth_method 0 -958> 2022-04-12T16:07:44.659+0300 7f2a270c3700 10 monclient: get_auth_request con 0x559e54b38400 auth_method 0 -957> 2022-04-12T16:07:44.659+0300 7f2a268c2700 10 monclient: get_auth_request con 0x559e55914000 auth_method 0 -956> 2022-04-12T16:07:44.663+0300 7f2a260c1700 10 monclient: get_auth_request con 0x559e55914c00 auth_method 0 -955> 2022-04-12T16:07:44.663+0300 7f2a270c3700 10 monclient: get_auth_request con 0x559e55915800 auth_method 0 -954> 2022-04-12T16:07:44.667+0300 7f2a268c2700 10 monclient: get_auth_request con 0x559e55914800 auth_method 0 -953> 2022-04-12T16:07:44.667+0300 7f2a260c1700 10 monclient: get_auth_request con 0x559e55914400 auth_method 0 -952> 2022-04-12T16:07:44.671+0300 7f2a270c3700 10 monclient: get_auth_request con 0x559e55915c00 auth_method 0 -951> 2022-04-12T16:07:44.671+0300 7f2a268c2700 10 monclient: get_auth_request con 0x559e5591c000 auth_method 0 -950> 2022-04-12T16:07:44.671+0300 7f2a260c1700 10 monclient: get_auth_request con 0x559e55915400 auth_method 0 -949> 2022-04-12T16:07:44.675+0300 7f2a270c3700 10 monclient: get_auth_request con 0x559e55915000 auth_method 0 -948> 2022-04-12T16:07:44.675+0300 7f2a268c2700 10 monclient: get_auth_request con 0x559e5591c400 auth_method 0 -947> 2022-04-12T16:07:44.675+0300 7f2a260c1700 10 monclient: get_auth_request con 0x559e5591c800 auth_method 0 -946> 2022-04-12T16:07:44.679+0300 7f2a270c3700 10 monclient: get_auth_request con 0x559e5591d000 auth_method 0 -945> 2022-04-12T16:07:44.679+0300 7f2a268c2700 10 monclient: get_auth_request con 0x559e5591d400 auth_method 0 -944> 2022-04-12T16:07:44.679+0300 7f2a260c1700 10 monclient: get_auth_request con 0x559e5591cc00 auth_method 0 -943> 2022-04-12T16:07:44.683+0300 7f2a270c3700 10 monclient: get_auth_request con 0x559e5591d800 auth_method 0 -942> 2022-04-12T16:07:44.687+0300 7f2a268c2700 10 monclient: get_auth_request con 0x559e55978000 auth_method 0 -941> 2022-04-12T16:07:44.691+0300 7f2a3b460400 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -940> 2022-04-12T16:07:44.691+0300 7f2a200b5700 10 monclient: handle_get_version_reply finishing 1 version 64 -939> 2022-04-12T16:07:44.691+0300 7f2a3b460400 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -938> 2022-04-12T16:07:45.539+0300 7f2a200b5700 10 monclient: _renew_subs -937> 2022-04-12T16:07:45.539+0300 7f2a200b5700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -936> 2022-04-12T16:07:45.539+0300 7f2a3b460400 10 start_mon_command cmd=[{"prefix": "osd pool application enable","pool": "sync.rgw.log","app": "rgw"}] -935> 2022-04-12T16:07:45.539+0300 7f2a3b460400 10 monclient: _send_command 1 [{"prefix": "osd pool application enable","pool": "sync.rgw.log","app": "rgw"}] -934> 2022-04-12T16:07:45.539+0300 7f2a3b460400 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -933> 2022-04-12T16:07:46.559+0300 7f2a200b5700 10 monclient: handle_mon_command_ack 1 [{"prefix": "osd pool application enable","pool": "sync.rgw.log","app": "rgw"}] -932> 2022-04-12T16:07:46.559+0300 7f2a200b5700 10 monclient: _finish_command 1 = system:0 enabled application 'rgw' on pool 'sync.rgw.log' -931> 2022-04-12T16:07:46.559+0300 7f2a3b460400 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -930> 2022-04-12T16:07:46.559+0300 7f2a200b5700 10 monclient: handle_get_version_reply finishing 2 version 66 -929> 2022-04-12T16:07:46.559+0300 7f2a218b8700 10 monclient: _renew_subs -928> 2022-04-12T16:07:46.559+0300 7f2a218b8700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -927> 2022-04-12T16:07:46.835+0300 7f2a3b460400 5 sync module config (parsed representation): { "config": { "source_bucket": "", "target_path": "rgw-${zonegroup}-${sid}/${bucket}", "connection_id": "", "acls_id": "", "connection": { "id": "", "endpoint": "https://storage.googleapis.com", "host_style": "path", "key": { "access_key": "", "secret": "******" } } }, "connections": [], "acl_profiles": [], "profiles": [ { "name": "", "config": { "source_bucket": "", "target_path": "rgw-${zonegroup}-${sid}/${bucket}", "connection_id": "", "acls_id": "", "connection": { "id": "", "endpoint": "https://storage.googleapis.com", "host_style": "path", "key": { "access_key": "", "secret": "******" } } } } ] } -926> 2022-04-12T16:07:46.835+0300 7f2a3b460400 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -925> 2022-04-12T16:07:46.835+0300 7f2a200b5700 10 monclient: handle_get_version_reply finishing 3 version 66 -924> 2022-04-12T16:07:46.835+0300 7f2a3b460400 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -923> 2022-04-12T16:07:47.563+0300 7f2a200b5700 10 monclient: _renew_subs -922> 2022-04-12T16:07:47.563+0300 7f2a200b5700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -921> 2022-04-12T16:07:47.567+0300 7f2a3b460400 10 start_mon_command cmd=[{"prefix": "osd pool application enable","pool": "sync.rgw.control","app": "rgw"}] -920> 2022-04-12T16:07:47.567+0300 7f2a3b460400 10 monclient: _send_command 2 [{"prefix": "osd pool application enable","pool": "sync.rgw.control","app": "rgw"}] -919> 2022-04-12T16:07:47.567+0300 7f2a3b460400 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -918> 2022-04-12T16:07:47.655+0300 7f2a1f0b3700 10 monclient: tick -917> 2022-04-12T16:07:48.575+0300 7f2a200b5700 10 monclient: handle_mon_command_ack 2 [{"prefix": "osd pool application enable","pool": "sync.rgw.control","app": "rgw"}] -916> 2022-04-12T16:07:48.575+0300 7f2a200b5700 10 monclient: _finish_command 2 = system:0 enabled application 'rgw' on pool 'sync.rgw.control' -915> 2022-04-12T16:07:48.575+0300 7f2a3b460400 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -914> 2022-04-12T16:07:48.579+0300 7f2a200b5700 10 monclient: handle_get_version_reply finishing 4 version 68 -913> 2022-04-12T16:07:48.579+0300 7f2a210b7700 10 monclient: _renew_subs -912> 2022-04-12T16:07:48.579+0300 7f2a210b7700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -911> 2022-04-12T16:07:48.655+0300 7f2a1f0b3700 10 monclient: tick -910> 2022-04-12T16:07:49.163+0300 7f2a3b460400 2 all 8 watchers are set, enabling cache -909> 2022-04-12T16:07:49.163+0300 7f2a3b460400 5 rgw main: tl::expected, obj_version>, boost::system::error_code> logback_generations::read(const DoutPrefixProvider*, optional_yield):409: oid=data_loggenerations_metadata not found -908> 2022-04-12T16:07:49.167+0300 7f2a260c1700 10 monclient: get_auth_request con 0x559e55978400 auth_method 0 -907> 2022-04-12T16:07:49.211+0300 7f2a0e091700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start -906> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d78b0) adding auth protocol: cephx -905> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d78b0) adding auth protocol: cephx -904> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d78b0) adding auth protocol: cephx -903> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d78b0) adding auth protocol: none -902> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d78b0) adding con mode: secure -901> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d78b0) adding con mode: crc -900> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d78b0) adding con mode: secure -899> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d78b0) adding con mode: crc -898> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d78b0) adding con mode: secure -897> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d78b0) adding con mode: crc -896> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d78b0) adding con mode: crc -895> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d78b0) adding con mode: secure -894> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d78b0) adding con mode: crc -893> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d78b0) adding con mode: secure -892> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d78b0) adding con mode: crc -891> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d78b0) adding con mode: secure -890> 2022-04-12T16:07:49.219+0300 7f2a3b460400 2 auth: KeyRing::load: loaded key file /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring -889> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command cache list hook 0x559e54a2aa10 -888> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command cache inspect hook 0x559e54a2aa10 -887> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command cache erase hook 0x559e54a2aa10 -886> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command cache zap hook 0x559e54a2aa10 -885> 2022-04-12T16:07:49.219+0300 7f2a3b460400 10 monclient: get_monmap_and_config -884> 2022-04-12T16:07:49.219+0300 7f2a3b460400 10 monclient: build_initial_monmap -883> 2022-04-12T16:07:49.219+0300 7f2a3b460400 10 monclient: monmap: epoch 0 fsid 00000000-0000-0000-0000-000000000000 last_changed 0.000000 created 0.000000 min_mon_release 0 (unknown) election_strategy: 1 0: [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] mon.noname-a 1: [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] mon.noname-b 2: [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] mon.noname-c -882> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding auth protocol: cephx -881> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding auth protocol: cephx -880> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding auth protocol: cephx -879> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding auth protocol: none -878> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: secure -877> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: crc -876> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: secure -875> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: crc -874> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: secure -873> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: crc -872> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: crc -871> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: secure -870> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: crc -869> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: secure -868> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: crc -867> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: secure -866> 2022-04-12T16:07:49.219+0300 7f2a3b460400 2 auth: KeyRing::load: loaded key file /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring -865> 2022-04-12T16:07:49.219+0300 7f2a3b460400 10 monclient: init -864> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding auth protocol: cephx -863> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding auth protocol: cephx -862> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding auth protocol: cephx -861> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding auth protocol: none -860> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: secure -859> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: crc -858> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: secure -857> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: crc -856> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: secure -855> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: crc -854> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: crc -853> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: secure -852> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: crc -851> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: secure -850> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: crc -849> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: secure -848> 2022-04-12T16:07:49.219+0300 7f2a3b460400 2 auth: KeyRing::load: loaded key file /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring -847> 2022-04-12T16:07:49.219+0300 7f2a3b460400 2 auth: KeyRing::load: loaded key file /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring -846> 2022-04-12T16:07:49.219+0300 7f2a3b460400 10 monclient: _reopen_session rank -1 -845> 2022-04-12T16:07:49.219+0300 7f2a3b460400 10 monclient: _add_conns ranks=[1,0,2] -844> 2022-04-12T16:07:49.219+0300 7f2a3b460400 10 monclient(hunting): picked mon.noname-b con 0x559e55990400 addr [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] -843> 2022-04-12T16:07:49.219+0300 7f2a3b460400 10 monclient(hunting): picked mon.noname-a con 0x559e55991000 addr [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] -842> 2022-04-12T16:07:49.219+0300 7f2a3b460400 10 monclient(hunting): picked mon.noname-c con 0x559e55990c00 addr [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] -841> 2022-04-12T16:07:49.219+0300 7f2a3b460400 10 monclient(hunting): start opening mon connection -840> 2022-04-12T16:07:49.219+0300 7f2a3b460400 10 monclient(hunting): start opening mon connection -839> 2022-04-12T16:07:49.219+0300 7f2a3b460400 10 monclient(hunting): start opening mon connection -838> 2022-04-12T16:07:49.219+0300 7f2a3b460400 10 monclient(hunting): _renew_subs -837> 2022-04-12T16:07:49.219+0300 7f2a3b460400 10 monclient(hunting): authenticate will time out at 2022-04-12T16:12:49.224469+0300 -836> 2022-04-12T16:07:49.219+0300 7f2a260c1700 10 monclient(hunting): get_auth_request con 0x559e55991000 auth_method 0 -835> 2022-04-12T16:07:49.219+0300 7f2a260c1700 10 monclient(hunting): get_auth_request method 2 preferred_modes [2,1] -834> 2022-04-12T16:07:49.219+0300 7f2a260c1700 10 monclient(hunting): _init_auth method 2 -833> 2022-04-12T16:07:49.219+0300 7f2a260c1700 10 monclient(hunting): _init_auth creating new auth -832> 2022-04-12T16:07:49.219+0300 7f2a260c1700 10 monclient(hunting): handle_auth_reply_more payload 9 -831> 2022-04-12T16:07:49.219+0300 7f2a260c1700 10 monclient(hunting): handle_auth_reply_more payload_len 9 -830> 2022-04-12T16:07:49.219+0300 7f2a260c1700 10 monclient(hunting): handle_auth_reply_more responding with 36 bytes -829> 2022-04-12T16:07:49.219+0300 7f2a268c2700 10 monclient(hunting): get_auth_request con 0x559e55990400 auth_method 0 -828> 2022-04-12T16:07:49.219+0300 7f2a268c2700 10 monclient(hunting): get_auth_request method 2 preferred_modes [2,1] -827> 2022-04-12T16:07:49.219+0300 7f2a268c2700 10 monclient(hunting): _init_auth method 2 -826> 2022-04-12T16:07:49.219+0300 7f2a268c2700 10 monclient(hunting): _init_auth creating new auth -825> 2022-04-12T16:07:49.219+0300 7f2a260c1700 10 monclient(hunting): handle_auth_done global_id 5724 payload 322 -824> 2022-04-12T16:07:49.219+0300 7f2a260c1700 10 monclient: _finish_hunting 0 -823> 2022-04-12T16:07:49.219+0300 7f2a260c1700 1 monclient: found mon.noname-a -822> 2022-04-12T16:07:49.219+0300 7f2a260c1700 10 monclient: _send_mon_message to mon.noname-a at v2:192.168.166.83:3300/0 -821> 2022-04-12T16:07:49.219+0300 7f2a0d890700 10 monclient: handle_monmap mon_map magic: 0 v1 -820> 2022-04-12T16:07:49.219+0300 7f2a0d890700 10 monclient: got monmap 1 from mon.noname-a (according to old e1) -819> 2022-04-12T16:07:49.219+0300 7f2a0d890700 10 monclient: dump: epoch 1 fsid 65ccd25a-39ce-4072-a5c6-a0e6654bda87 last_changed 2022-04-12T15:48:27.663651+0300 created 2022-04-12T15:48:27.663651+0300 min_mon_release 16 (pacific) election_strategy: 1 0: [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] mon.ceph-ss-test-oct-controller-av-01 1: [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] mon.ceph-ss-test-oct-controller-av-02 2: [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] mon.ceph-ss-test-oct-controller-av-03 -818> 2022-04-12T16:07:49.219+0300 7f2a0d890700 10 monclient: _finish_auth 0 -817> 2022-04-12T16:07:49.219+0300 7f2a0d890700 10 monclient: handle_config config(0 keys) v1 -816> 2022-04-12T16:07:49.219+0300 7f2a0d890700 10 monclient: handle_monmap mon_map magic: 0 v1 -815> 2022-04-12T16:07:49.219+0300 7f2a0d890700 10 monclient: got monmap 1 from mon.ceph-ss-test-oct-controller-av-01 (according to old e1) -814> 2022-04-12T16:07:49.219+0300 7f2a0d890700 10 monclient: dump: epoch 1 fsid 65ccd25a-39ce-4072-a5c6-a0e6654bda87 last_changed 2022-04-12T15:48:27.663651+0300 created 2022-04-12T15:48:27.663651+0300 min_mon_release 16 (pacific) election_strategy: 1 0: [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] mon.ceph-ss-test-oct-controller-av-01 1: [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] mon.ceph-ss-test-oct-controller-av-02 2: [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] mon.ceph-ss-test-oct-controller-av-03 -813> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 monclient: authenticate success, global_id 5724 -812> 2022-04-12T16:07:49.219+0300 7f2a3b460400 10 monclient: get_monmap_and_config success -811> 2022-04-12T16:07:49.219+0300 7f2a3b460400 4 set_mon_vals no callback set -810> 2022-04-12T16:07:49.219+0300 7f2a3b460400 10 monclient: shutdown -809> 2022-04-12T16:07:49.223+0300 7f2a3b460400 10 monclient: build_initial_monmap -808> 2022-04-12T16:07:49.223+0300 7f2a3b460400 10 monclient: monmap: epoch 0 fsid 00000000-0000-0000-0000-000000000000 last_changed 0.000000 created 0.000000 min_mon_release 0 (unknown) election_strategy: 1 0: [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] mon.noname-a 1: [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] mon.noname-b 2: [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] mon.noname-c -807> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding auth protocol: cephx -806> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding auth protocol: cephx -805> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding auth protocol: cephx -804> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding auth protocol: none -803> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: secure -802> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: crc -801> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: secure -800> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: crc -799> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: secure -798> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: crc -797> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: crc -796> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: secure -795> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: crc -794> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: secure -793> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: crc -792> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: secure -791> 2022-04-12T16:07:49.223+0300 7f2a3b460400 2 auth: KeyRing::load: loaded key file /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring -790> 2022-04-12T16:07:49.223+0300 7f2a3b460400 1 librados: starting msgr at -789> 2022-04-12T16:07:49.223+0300 7f2a3b460400 1 librados: starting objecter -788> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command objecter_requests cmddesc objecter_requests hook 0x559e54a2ba40 EEXIST -787> 2022-04-12T16:07:49.223+0300 7f2a3b460400 1 librados: setting wanted keys -786> 2022-04-12T16:07:49.223+0300 7f2a3b460400 1 librados: calling monclient init -785> 2022-04-12T16:07:49.223+0300 7f2a3b460400 10 monclient: init -784> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e559dd0d8) adding auth protocol: cephx -783> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e559dd0d8) adding auth protocol: cephx -782> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e559dd0d8) adding auth protocol: cephx -781> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e559dd0d8) adding auth protocol: none -780> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e559dd0d8) adding con mode: secure -779> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e559dd0d8) adding con mode: crc -778> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e559dd0d8) adding con mode: secure -777> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e559dd0d8) adding con mode: crc -776> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e559dd0d8) adding con mode: secure -775> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e559dd0d8) adding con mode: crc -774> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e559dd0d8) adding con mode: crc -773> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e559dd0d8) adding con mode: secure -772> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e559dd0d8) adding con mode: crc -771> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e559dd0d8) adding con mode: secure -770> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e559dd0d8) adding con mode: crc -769> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e559dd0d8) adding con mode: secure -768> 2022-04-12T16:07:49.223+0300 7f2a3b460400 2 auth: KeyRing::load: loaded key file /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring -767> 2022-04-12T16:07:49.223+0300 7f2a3b460400 2 auth: KeyRing::load: loaded key file /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring -766> 2022-04-12T16:07:49.223+0300 7f2a3b460400 10 monclient: _reopen_session rank -1 -765> 2022-04-12T16:07:49.223+0300 7f2a3b460400 10 monclient: _add_conns ranks=[0,2,1] -764> 2022-04-12T16:07:49.223+0300 7f2a3b460400 10 monclient(hunting): picked mon.noname-a con 0x559e55990400 addr [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] -763> 2022-04-12T16:07:49.223+0300 7f2a3b460400 10 monclient(hunting): picked mon.noname-c con 0x559e55990c00 addr [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] -762> 2022-04-12T16:07:49.223+0300 7f2a3b460400 10 monclient(hunting): picked mon.noname-b con 0x559e55991000 addr [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] -761> 2022-04-12T16:07:49.223+0300 7f2a3b460400 10 monclient(hunting): start opening mon connection -760> 2022-04-12T16:07:49.223+0300 7f2a3b460400 10 monclient(hunting): start opening mon connection -759> 2022-04-12T16:07:49.223+0300 7f2a3b460400 10 monclient(hunting): start opening mon connection -758> 2022-04-12T16:07:49.223+0300 7f2a3b460400 10 monclient(hunting): _renew_subs -757> 2022-04-12T16:07:49.223+0300 7f2a3b460400 10 monclient(hunting): authenticate will time out at 2022-04-12T16:12:49.227127+0300 -756> 2022-04-12T16:07:49.223+0300 7f2a268c2700 10 monclient(hunting): get_auth_request con 0x559e55990400 auth_method 0 -755> 2022-04-12T16:07:49.223+0300 7f2a268c2700 10 monclient(hunting): get_auth_request method 2 preferred_modes [2,1] -754> 2022-04-12T16:07:49.223+0300 7f2a268c2700 10 monclient(hunting): _init_auth method 2 -753> 2022-04-12T16:07:49.223+0300 7f2a268c2700 10 monclient(hunting): _init_auth creating new auth -752> 2022-04-12T16:07:49.223+0300 7f2a268c2700 10 monclient(hunting): handle_auth_reply_more payload 9 -751> 2022-04-12T16:07:49.223+0300 7f2a268c2700 10 monclient(hunting): handle_auth_reply_more payload_len 9 -750> 2022-04-12T16:07:49.223+0300 7f2a268c2700 10 monclient(hunting): handle_auth_reply_more responding with 36 bytes -749> 2022-04-12T16:07:49.223+0300 7f2a268c2700 10 monclient(hunting): handle_auth_done global_id 5727 payload 1000 -748> 2022-04-12T16:07:49.223+0300 7f2a268c2700 10 monclient: _finish_hunting 0 -747> 2022-04-12T16:07:49.223+0300 7f2a268c2700 1 monclient: found mon.noname-a -746> 2022-04-12T16:07:49.223+0300 7f2a268c2700 10 monclient: _send_mon_message to mon.noname-a at v2:192.168.166.83:3300/0 -745> 2022-04-12T16:07:49.223+0300 7f2a0c08d700 10 monclient: handle_monmap mon_map magic: 0 v1 -744> 2022-04-12T16:07:49.223+0300 7f2a0c08d700 10 monclient: got monmap 1 from mon.noname-a (according to old e1) -743> 2022-04-12T16:07:49.223+0300 7f2a0c08d700 10 monclient: dump: epoch 1 fsid 65ccd25a-39ce-4072-a5c6-a0e6654bda87 last_changed 2022-04-12T15:48:27.663651+0300 created 2022-04-12T15:48:27.663651+0300 min_mon_release 16 (pacific) election_strategy: 1 0: [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] mon.ceph-ss-test-oct-controller-av-01 1: [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] mon.ceph-ss-test-oct-controller-av-02 2: [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] mon.ceph-ss-test-oct-controller-av-03 -742> 2022-04-12T16:07:49.223+0300 7f2a0c08d700 10 monclient: _finish_auth 0 -741> 2022-04-12T16:07:49.223+0300 7f2a0c08d700 10 monclient: handle_config config(0 keys) v1 -740> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 monclient: authenticate success, global_id 5727 -739> 2022-04-12T16:07:49.223+0300 7f2a3b460400 10 monclient: _renew_subs -738> 2022-04-12T16:07:49.223+0300 7f2a0d890700 4 set_mon_vals no callback set -737> 2022-04-12T16:07:49.223+0300 7f2a3b460400 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -736> 2022-04-12T16:07:49.223+0300 7f2a0c08d700 10 monclient: handle_monmap mon_map magic: 0 v1 -735> 2022-04-12T16:07:49.223+0300 7f2a0c08d700 10 monclient: got monmap 1 from mon.ceph-ss-test-oct-controller-av-01 (according to old e1) -734> 2022-04-12T16:07:49.223+0300 7f2a0c08d700 10 monclient: dump: epoch 1 fsid 65ccd25a-39ce-4072-a5c6-a0e6654bda87 last_changed 2022-04-12T15:48:27.663651+0300 created 2022-04-12T15:48:27.663651+0300 min_mon_release 16 (pacific) election_strategy: 1 0: [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] mon.ceph-ss-test-oct-controller-av-01 1: [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] mon.ceph-ss-test-oct-controller-av-02 2: [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] mon.ceph-ss-test-oct-controller-av-03 -733> 2022-04-12T16:07:49.223+0300 7f2a3b460400 10 monclient: _renew_subs -732> 2022-04-12T16:07:49.223+0300 7f2a3b460400 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -731> 2022-04-12T16:07:49.223+0300 7f2a3b460400 1 librados: init done -730> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command cr dump hook 0x559e558a04e8 -729> 2022-04-12T16:07:49.227+0300 7f2a0c08d700 4 mgrc handle_mgr_map Got map version 29 -728> 2022-04-12T16:07:49.227+0300 7f2a0c08d700 4 mgrc handle_mgr_map Active mgr is now [v2:192.168.166.84:6800/56,v1:192.168.166.84:6801/56] -727> 2022-04-12T16:07:49.227+0300 7f2a0c08d700 4 mgrc reconnect Starting new session with [v2:192.168.166.84:6800/56,v1:192.168.166.84:6801/56] -726> 2022-04-12T16:07:49.227+0300 7f2a3b460400 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -725> 2022-04-12T16:07:49.227+0300 7f2a0c08d700 10 monclient: handle_get_version_reply finishing 1 version 68 -724> 2022-04-12T16:07:49.227+0300 7f2a260c1700 10 monclient: get_auth_request con 0x559e559cbc00 auth_method 0 -723> 2022-04-12T16:07:49.227+0300 7f2a3b460400 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -722> 2022-04-12T16:07:49.603+0300 7f2a0c08d700 10 monclient: _renew_subs -721> 2022-04-12T16:07:49.603+0300 7f2a0c08d700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -720> 2022-04-12T16:07:49.607+0300 7f2a3b460400 10 start_mon_command cmd=[{"prefix": "osd pool application enable","pool": "sync.rgw.meta","app": "rgw"}] -719> 2022-04-12T16:07:49.607+0300 7f2a3b460400 10 monclient: _send_command 1 [{"prefix": "osd pool application enable","pool": "sync.rgw.meta","app": "rgw"}] -718> 2022-04-12T16:07:49.607+0300 7f2a3b460400 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -717> 2022-04-12T16:07:49.655+0300 7f2a1f0b3700 10 monclient: tick -716> 2022-04-12T16:07:49.655+0300 7f2a208b6700 10 monclient: _renew_subs -715> 2022-04-12T16:07:49.655+0300 7f2a208b6700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -714> 2022-04-12T16:07:50.611+0300 7f2a0c08d700 10 monclient: handle_mon_command_ack 1 [{"prefix": "osd pool application enable","pool": "sync.rgw.meta","app": "rgw"}] -713> 2022-04-12T16:07:50.611+0300 7f2a0c08d700 10 monclient: _finish_command 1 = system:0 enabled application 'rgw' on pool 'sync.rgw.meta' -712> 2022-04-12T16:07:50.611+0300 7f2a3b460400 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -711> 2022-04-12T16:07:50.615+0300 7f2a0c08d700 10 monclient: handle_get_version_reply finishing 2 version 70 -710> 2022-04-12T16:07:50.615+0300 7f2a0d890700 10 monclient: _renew_subs -709> 2022-04-12T16:07:50.615+0300 7f2a0d890700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -708> 2022-04-12T16:07:50.615+0300 7f2a3b460400 10 start_mon_command cmd=[{"prefix": "osd pool set", "pool": "sync.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}] -707> 2022-04-12T16:07:50.615+0300 7f2a3b460400 10 monclient: _send_command 2 [{"prefix": "osd pool set", "pool": "sync.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}] -706> 2022-04-12T16:07:50.615+0300 7f2a3b460400 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -705> 2022-04-12T16:07:50.655+0300 7f2a1f0b3700 10 monclient: tick -704> 2022-04-12T16:07:51.615+0300 7f2a0c08d700 10 monclient: handle_mon_command_ack 2 [{"prefix": "osd pool set", "pool": "sync.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}] -703> 2022-04-12T16:07:51.615+0300 7f2a0c08d700 10 monclient: _finish_command 2 = system:0 set pool 11 pg_autoscale_bias to 4 -702> 2022-04-12T16:07:51.615+0300 7f2a3b460400 10 start_mon_command cmd=[{"prefix": "osd pool set", "pool": "sync.rgw.meta", "var": "pg_num_min", "val": "8"}] -701> 2022-04-12T16:07:51.615+0300 7f2a3b460400 10 monclient: _send_command 3 [{"prefix": "osd pool set", "pool": "sync.rgw.meta", "var": "pg_num_min", "val": "8"}] -700> 2022-04-12T16:07:51.615+0300 7f2a3b460400 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -699> 2022-04-12T16:07:51.655+0300 7f2a1f0b3700 10 monclient: tick -698> 2022-04-12T16:07:52.223+0300 7f2a0b08b700 10 monclient: tick -697> 2022-04-12T16:07:52.619+0300 7f2a0c08d700 10 monclient: handle_mon_command_ack 3 [{"prefix": "osd pool set", "pool": "sync.rgw.meta", "var": "pg_num_min", "val": "8"}] -696> 2022-04-12T16:07:52.619+0300 7f2a0c08d700 10 monclient: _finish_command 3 = system:0 set pool 11 pg_num_min to 8 -695> 2022-04-12T16:07:52.619+0300 7f2a3b460400 10 start_mon_command cmd=[{"prefix": "osd pool set", "pool": "sync.rgw.meta", "var": "recovery_priority": "5"}] -694> 2022-04-12T16:07:52.619+0300 7f2a3b460400 10 monclient: _send_command 4 [{"prefix": "osd pool set", "pool": "sync.rgw.meta", "var": "recovery_priority": "5"}] -693> 2022-04-12T16:07:52.619+0300 7f2a3b460400 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -692> 2022-04-12T16:07:52.623+0300 7f2a0c08d700 10 monclient: handle_mon_command_ack 4 [{"prefix": "osd pool set", "pool": "sync.rgw.meta", "var": "recovery_priority": "5"}] -691> 2022-04-12T16:07:52.623+0300 7f2a0c08d700 10 monclient: _finish_command 4 = mon:22 unparseable JSON {"prefix": "osd pool set", "pool": "sync.rgw.meta", "var": "recovery_priority": "5"} -690> 2022-04-12T16:07:52.623+0300 7f2a270c3700 10 monclient: get_auth_request con 0x559e559f1400 auth_method 0 -689> 2022-04-12T16:07:52.627+0300 7f2a268c2700 10 monclient: get_auth_request con 0x559e559cac00 auth_method 0 -688> 2022-04-12T16:07:52.635+0300 7f2a260c1700 10 monclient: get_auth_request con 0x559e559ca000 auth_method 0 -687> 2022-04-12T16:07:52.639+0300 7f2a270c3700 10 monclient: get_auth_request con 0x559e55b21400 auth_method 0 -686> 2022-04-12T16:07:52.647+0300 7f2a268c2700 10 monclient: get_auth_request con 0x559e54b39c00 auth_method 0 -685> 2022-04-12T16:07:52.651+0300 7f2a260c1700 10 monclient: get_auth_request con 0x559e559cb000 auth_method 0 -684> 2022-04-12T16:07:52.659+0300 7f2a1f0b3700 10 monclient: tick -683> 2022-04-12T16:07:52.659+0300 7f2a270c3700 10 monclient: get_auth_request con 0x559e55990000 auth_method 0 -682> 2022-04-12T16:07:52.671+0300 7f2a268c2700 10 monclient: get_auth_request con 0x559e559f1800 auth_method 0 -681> 2022-04-12T16:07:52.699+0300 7f2a260c1700 10 monclient: get_auth_request con 0x559e55b51800 auth_method 0 -680> 2022-04-12T16:07:52.711+0300 7f2a270c3700 10 monclient: get_auth_request con 0x559e55b50000 auth_method 0 -679> 2022-04-12T16:07:52.723+0300 7f2a268c2700 10 monclient: get_auth_request con 0x559e559f0400 auth_method 0 -678> 2022-04-12T16:07:52.731+0300 7f2a260c1700 10 monclient: get_auth_request con 0x559e559ca400 auth_method 0 -677> 2022-04-12T16:07:52.743+0300 7f2a270c3700 10 monclient: get_auth_request con 0x559e55979c00 auth_method 0 -676> 2022-04-12T16:07:52.747+0300 7f2a09888700 2 object expiration: start -675> 2022-04-12T16:07:52.747+0300 7f2a0a089700 2 garbage collection: garbage collection: start -674> 2022-04-12T16:07:52.747+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command sync trace show hook 0x559e5592a1c0 -673> 2022-04-12T16:07:52.747+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command sync trace history hook 0x559e5592a1c0 -672> 2022-04-12T16:07:52.747+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command sync trace active hook 0x559e5592a1c0 -671> 2022-04-12T16:07:52.747+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command sync trace active_short hook 0x559e5592a1c0 -670> 2022-04-12T16:07:52.751+0300 7f2a3b460400 4 rgw main: failed to read sync status info with (2) No such file or directory -669> 2022-04-12T16:07:52.755+0300 7f2a07884700 4 rgw rados thread: failed to read sync status info with (2) No such file or directory -668> 2022-04-12T16:07:52.755+0300 7f2a07884700 1 RGW-SYNC:meta: initialize sync (reason: period is empty) -667> 2022-04-12T16:07:52.755+0300 7f2a07884700 1 rgw rados thread: epoch=0 in sync status comes before remote's oldest mdlog epoch=1, restarting sync -666> 2022-04-12T16:07:52.759+0300 7f2a3b460400 5 rgw main: starting data sync thread for zone avrupa -665> 2022-04-12T16:07:52.759+0300 7f29ff073700 5 lifecycle: schedule life cycle next start time: Tue Apr 12 21:00:00 2022 -664> 2022-04-12T16:07:52.759+0300 7f2a01077700 5 lifecycle: schedule life cycle next start time: Tue Apr 12 21:00:00 2022 -663> 2022-04-12T16:07:52.759+0300 7f29fd06f700 5 lifecycle: schedule life cycle next start time: Tue Apr 12 21:00:00 2022 -662> 2022-04-12T16:07:52.763+0300 7f2a3b460400 0 framework: beast -661> 2022-04-12T16:07:52.763+0300 7f2a3b460400 0 framework conf key: ssl_certificate, val: config://rgw/cert/$realm/$zone.crt -660> 2022-04-12T16:07:52.763+0300 7f2a3b460400 0 framework conf key: ssl_private_key, val: config://rgw/cert/$realm/$zone.key -659> 2022-04-12T16:07:52.763+0300 7f2a3b460400 0 starting handler: beast -658> 2022-04-12T16:07:52.763+0300 7f2a06882700 4 rgw rados thread: failed to read sync status info with (2) No such file or directory -657> 2022-04-12T16:07:52.763+0300 7f2a3b460400 4 frontend listening on 192.168.166.83:8081 -656> 2022-04-12T16:07:52.763+0300 7f2a3b460400 0 set uid:gid to 167:167 (ceph:ceph) -655> 2022-04-12T16:07:52.763+0300 7f2a3b460400 4 frontend spawning 512 threads -654> 2022-04-12T16:07:52.799+0300 7f2a3b460400 1 mgrc service_daemon_register rgw.5727 metadata {arch=x86_64,ceph_release=pacific,ceph_version=ceph version 16.2.7 (dd0603118f56ab514f133c8d2e3adfc983942503) pacific (stable),ceph_version_short=16.2.7,container_image=gitlab.example.com:5050/openstack/registry/ceph/daemon:v6.0.6-stable-6.0-pacific-centos-8-x86_64-ceph-ansible-20220121,cpu=Intel(R) Xeon(R) Gold 6230R CPU @ 2.10GHz,distro=centos,distro_description=CentOS Linux 8,distro_version=8,frontend_config#0=beast endpoint=192.168.166.83:8081,frontend_type#0=beast,hostname=ceph-ss-test-oct-controller-av-01,id=ceph-ss-test-oct-controller-av-01.rgw1,kernel_description=#121-Ubuntu SMP Thu Mar 24 16:04:27 UTC 2022,kernel_version=5.4.0-107-generic,mem_cgroup_limit=4294967296,mem_swap_kb=4030460,mem_total_kb=8148192,num_handles=1,os=Linux,pid=54,realm_id=532f2481-54e3-4d51-95fd-19867ca818e9,realm_name=myrealm,zone_id=c81e6d95-54da-48d3-83a4-7bb7b286709f,zone_name=sync,zonegroup_id=784a05a5-2089-4564-ab1b-a55c43c5f401,zonegroup_name=cdn} -653> 2022-04-12T16:07:52.807+0300 7f2a0c08d700 4 mgrc handle_mgr_configure stats_period=5 -652> 2022-04-12T16:07:52.807+0300 7f2a0c08d700 4 mgrc handle_mgr_configure updated stats threshold: 5 -651> 2022-04-12T16:07:52.807+0300 7f2a3b460400 10 monclient: get_monmap_and_config -650> 2022-04-12T16:07:52.807+0300 7f2a3b460400 10 monclient: build_initial_monmap -649> 2022-04-12T16:07:52.807+0300 7f2a3b460400 10 monclient: monmap: epoch 0 fsid 00000000-0000-0000-0000-000000000000 last_changed 0.000000 created 0.000000 min_mon_release 0 (unknown) election_strategy: 1 0: [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] mon.noname-a 1: [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] mon.noname-b 2: [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] mon.noname-c -648> 2022-04-12T16:07:52.807+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding auth protocol: cephx -647> 2022-04-12T16:07:52.807+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding auth protocol: cephx -646> 2022-04-12T16:07:52.807+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding auth protocol: cephx -645> 2022-04-12T16:07:52.807+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding auth protocol: none -644> 2022-04-12T16:07:52.807+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: secure -643> 2022-04-12T16:07:52.807+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: crc -642> 2022-04-12T16:07:52.807+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: secure -641> 2022-04-12T16:07:52.807+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: crc -640> 2022-04-12T16:07:52.807+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: secure -639> 2022-04-12T16:07:52.807+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: crc -638> 2022-04-12T16:07:52.807+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: crc -637> 2022-04-12T16:07:52.807+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: secure -636> 2022-04-12T16:07:52.807+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: crc -635> 2022-04-12T16:07:52.807+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: secure -634> 2022-04-12T16:07:52.807+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: crc -633> 2022-04-12T16:07:52.807+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: secure -632> 2022-04-12T16:07:52.807+0300 7f2a3b460400 2 auth: KeyRing::load: loaded key file /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring -631> 2022-04-12T16:07:52.819+0300 7f2a3b460400 10 monclient: init -630> 2022-04-12T16:07:52.819+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding auth protocol: cephx -629> 2022-04-12T16:07:52.819+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding auth protocol: cephx -628> 2022-04-12T16:07:52.819+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding auth protocol: cephx -627> 2022-04-12T16:07:52.819+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding auth protocol: none -626> 2022-04-12T16:07:52.819+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: secure -625> 2022-04-12T16:07:52.819+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: crc -624> 2022-04-12T16:07:52.819+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: secure -623> 2022-04-12T16:07:52.819+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: crc -622> 2022-04-12T16:07:52.819+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: secure -621> 2022-04-12T16:07:52.819+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: crc -620> 2022-04-12T16:07:52.819+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: crc -619> 2022-04-12T16:07:52.819+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: secure -618> 2022-04-12T16:07:52.819+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: crc -617> 2022-04-12T16:07:52.819+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: secure -616> 2022-04-12T16:07:52.819+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: crc -615> 2022-04-12T16:07:52.819+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: secure -614> 2022-04-12T16:07:52.819+0300 7f2a3b460400 2 auth: KeyRing::load: loaded key file /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring -613> 2022-04-12T16:07:52.819+0300 7f2a3b460400 2 auth: KeyRing::load: loaded key file /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring -612> 2022-04-12T16:07:52.819+0300 7f2a3b460400 10 monclient: _reopen_session rank -1 -611> 2022-04-12T16:07:52.819+0300 7f2a3b460400 10 monclient: _add_conns ranks=[1,0,2] -610> 2022-04-12T16:07:52.819+0300 7f2a3b460400 10 monclient(hunting): picked mon.noname-b con 0x559e55db1c00 addr [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] -609> 2022-04-12T16:07:52.819+0300 7f2a3b460400 10 monclient(hunting): picked mon.noname-a con 0x559e559f1000 addr [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] -608> 2022-04-12T16:07:52.819+0300 7f2a3b460400 10 monclient(hunting): picked mon.noname-c con 0x559e559f0800 addr [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] -607> 2022-04-12T16:07:52.819+0300 7f2a3b460400 10 monclient(hunting): start opening mon connection -606> 2022-04-12T16:07:52.819+0300 7f2a3b460400 10 monclient(hunting): start opening mon connection -605> 2022-04-12T16:07:52.819+0300 7f2a3b460400 10 monclient(hunting): start opening mon connection -604> 2022-04-12T16:07:52.819+0300 7f2a3b460400 10 monclient(hunting): _renew_subs -603> 2022-04-12T16:07:52.819+0300 7f2a3b460400 10 monclient(hunting): authenticate will time out at 2022-04-12T16:12:52.824021+0300 -602> 2022-04-12T16:07:52.819+0300 7f2a260c1700 10 monclient(hunting): get_auth_request con 0x559e55db1c00 auth_method 0 -601> 2022-04-12T16:07:52.819+0300 7f2a260c1700 10 monclient(hunting): get_auth_request method 2 preferred_modes [2,1] -600> 2022-04-12T16:07:52.819+0300 7f2a260c1700 10 monclient(hunting): _init_auth method 2 -599> 2022-04-12T16:07:52.819+0300 7f2a260c1700 10 monclient(hunting): _init_auth creating new auth -598> 2022-04-12T16:07:52.819+0300 7f2a268c2700 10 monclient(hunting): get_auth_request con 0x559e559f0800 auth_method 0 -597> 2022-04-12T16:07:52.819+0300 7f2a268c2700 10 monclient(hunting): get_auth_request method 2 preferred_modes [2,1] -596> 2022-04-12T16:07:52.819+0300 7f2a268c2700 10 monclient(hunting): _init_auth method 2 -595> 2022-04-12T16:07:52.819+0300 7f2a268c2700 10 monclient(hunting): _init_auth creating new auth -594> 2022-04-12T16:07:52.819+0300 7f2a260c1700 10 monclient(hunting): handle_auth_reply_more payload 9 -593> 2022-04-12T16:07:52.819+0300 7f2a260c1700 10 monclient(hunting): handle_auth_reply_more payload_len 9 -592> 2022-04-12T16:07:52.819+0300 7f2a260c1700 10 monclient(hunting): handle_auth_reply_more responding with 36 bytes -591> 2022-04-12T16:07:52.819+0300 7f2a268c2700 10 monclient(hunting): handle_auth_reply_more payload 9 -590> 2022-04-12T16:07:52.819+0300 7f2a268c2700 10 monclient(hunting): handle_auth_reply_more payload_len 9 -589> 2022-04-12T16:07:52.819+0300 7f2a268c2700 10 monclient(hunting): handle_auth_reply_more responding with 36 bytes -588> 2022-04-12T16:07:52.819+0300 7f2a260c1700 10 monclient(hunting): handle_auth_done global_id 5197 payload 322 -587> 2022-04-12T16:07:52.819+0300 7f2a260c1700 10 monclient: _finish_hunting 0 -586> 2022-04-12T16:07:52.819+0300 7f2a260c1700 1 monclient: found mon.noname-b -585> 2022-04-12T16:07:52.819+0300 7f2a260c1700 10 monclient: _send_mon_message to mon.noname-b at v2:192.168.166.84:3300/0 -584> 2022-04-12T16:07:52.819+0300 7f28f8e67700 10 monclient: handle_monmap mon_map magic: 0 v1 -583> 2022-04-12T16:07:52.819+0300 7f28f8e67700 10 monclient: got monmap 1 from mon.noname-b (according to old e1) -582> 2022-04-12T16:07:52.819+0300 7f28f8e67700 10 monclient: dump: epoch 1 fsid 65ccd25a-39ce-4072-a5c6-a0e6654bda87 last_changed 2022-04-12T15:48:27.663651+0300 created 2022-04-12T15:48:27.663651+0300 min_mon_release 16 (pacific) election_strategy: 1 0: [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] mon.ceph-ss-test-oct-controller-av-01 1: [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] mon.ceph-ss-test-oct-controller-av-02 2: [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] mon.ceph-ss-test-oct-controller-av-03 -581> 2022-04-12T16:07:52.819+0300 7f28f8e67700 10 monclient: _finish_auth 0 -580> 2022-04-12T16:07:52.819+0300 7f28f8e67700 10 monclient: handle_config config(0 keys) v1 -579> 2022-04-12T16:07:52.819+0300 7f28f8e67700 10 monclient: handle_monmap mon_map magic: 0 v1 -578> 2022-04-12T16:07:52.819+0300 7f28f8e67700 10 monclient: got monmap 1 from mon.ceph-ss-test-oct-controller-av-02 (according to old e1) -577> 2022-04-12T16:07:52.819+0300 7f28f8e67700 10 monclient: dump: epoch 1 fsid 65ccd25a-39ce-4072-a5c6-a0e6654bda87 last_changed 2022-04-12T15:48:27.663651+0300 created 2022-04-12T15:48:27.663651+0300 min_mon_release 16 (pacific) election_strategy: 1 0: [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] mon.ceph-ss-test-oct-controller-av-01 1: [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] mon.ceph-ss-test-oct-controller-av-02 2: [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] mon.ceph-ss-test-oct-controller-av-03 -576> 2022-04-12T16:07:52.819+0300 7f2a3b460400 5 monclient: authenticate success, global_id 5197 -575> 2022-04-12T16:07:52.819+0300 7f2a3b460400 10 monclient: get_monmap_and_config success -574> 2022-04-12T16:07:52.819+0300 7f2a3b460400 4 set_mon_vals no callback set -573> 2022-04-12T16:07:52.823+0300 7f2a3b460400 10 monclient: shutdown -572> 2022-04-12T16:07:52.823+0300 7f2a3b460400 10 monclient: build_initial_monmap -571> 2022-04-12T16:07:52.823+0300 7f2a3b460400 10 monclient: monmap: epoch 0 fsid 00000000-0000-0000-0000-000000000000 last_changed 0.000000 created 0.000000 min_mon_release 0 (unknown) election_strategy: 1 0: [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] mon.noname-a 1: [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] mon.noname-b 2: [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] mon.noname-c -570> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding auth protocol: cephx -569> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding auth protocol: cephx -568> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding auth protocol: cephx -567> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding auth protocol: none -566> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: secure -565> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: crc -564> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: secure -563> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: crc -562> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: secure -561> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: crc -560> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: crc -559> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: secure -558> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: crc -557> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: secure -556> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: crc -555> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: secure -554> 2022-04-12T16:07:52.823+0300 7f2a3b460400 2 auth: KeyRing::load: loaded key file /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring -553> 2022-04-12T16:07:52.823+0300 7f2a3b460400 1 librados: starting msgr at -552> 2022-04-12T16:07:52.823+0300 7f2a3b460400 1 librados: starting objecter -551> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command objecter_requests cmddesc objecter_requests hook 0x559e58ce73c0 EEXIST -550> 2022-04-12T16:07:52.823+0300 7f2a3b460400 1 librados: setting wanted keys -549> 2022-04-12T16:07:52.823+0300 7f2a3b460400 1 librados: calling monclient init -548> 2022-04-12T16:07:52.823+0300 7f2a3b460400 10 monclient: init -547> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e594860d8) adding auth protocol: cephx -546> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e594860d8) adding auth protocol: cephx -545> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e594860d8) adding auth protocol: cephx -544> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e594860d8) adding auth protocol: none -543> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e594860d8) adding con mode: secure -542> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e594860d8) adding con mode: crc -541> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e594860d8) adding con mode: secure -540> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e594860d8) adding con mode: crc -539> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e594860d8) adding con mode: secure -538> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e594860d8) adding con mode: crc -537> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e594860d8) adding con mode: crc -536> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e594860d8) adding con mode: secure -535> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e594860d8) adding con mode: crc -534> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e594860d8) adding con mode: secure -533> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e594860d8) adding con mode: crc -532> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e594860d8) adding con mode: secure -531> 2022-04-12T16:07:52.823+0300 7f2a3b460400 2 auth: KeyRing::load: loaded key file /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring -530> 2022-04-12T16:07:52.823+0300 7f2a3b460400 2 auth: KeyRing::load: loaded key file /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring -529> 2022-04-12T16:07:52.823+0300 7f2a3b460400 10 monclient: _reopen_session rank -1 -528> 2022-04-12T16:07:52.823+0300 7f2a3b460400 10 monclient: _add_conns ranks=[0,2,1] -527> 2022-04-12T16:07:52.823+0300 7f2a3b460400 10 monclient(hunting): picked mon.noname-a con 0x559e559f0800 addr [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] -526> 2022-04-12T16:07:52.823+0300 7f2a3b460400 10 monclient(hunting): picked mon.noname-c con 0x559e559f1000 addr [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] -525> 2022-04-12T16:07:52.823+0300 7f2a3b460400 10 monclient(hunting): picked mon.noname-b con 0x559e55db1c00 addr [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] -524> 2022-04-12T16:07:52.823+0300 7f2a3b460400 10 monclient(hunting): start opening mon connection -523> 2022-04-12T16:07:52.823+0300 7f2a3b460400 10 monclient(hunting): start opening mon connection -522> 2022-04-12T16:07:52.823+0300 7f2a3b460400 10 monclient(hunting): start opening mon connection -521> 2022-04-12T16:07:52.823+0300 7f2a3b460400 10 monclient(hunting): _renew_subs -520> 2022-04-12T16:07:52.823+0300 7f2a3b460400 10 monclient(hunting): authenticate will time out at 2022-04-12T16:12:52.827867+0300 -519> 2022-04-12T16:07:52.823+0300 7f2a260c1700 10 monclient(hunting): get_auth_request con 0x559e559f0800 auth_method 0 -518> 2022-04-12T16:07:52.823+0300 7f2a260c1700 10 monclient(hunting): get_auth_request method 2 preferred_modes [2,1] -517> 2022-04-12T16:07:52.823+0300 7f2a260c1700 10 monclient(hunting): _init_auth method 2 -516> 2022-04-12T16:07:52.823+0300 7f2a260c1700 10 monclient(hunting): _init_auth creating new auth -515> 2022-04-12T16:07:52.823+0300 7f2a260c1700 10 monclient(hunting): handle_auth_reply_more payload 9 -514> 2022-04-12T16:07:52.823+0300 7f2a260c1700 10 monclient(hunting): handle_auth_reply_more payload_len 9 -513> 2022-04-12T16:07:52.823+0300 7f2a260c1700 10 monclient(hunting): handle_auth_reply_more responding with 36 bytes -512> 2022-04-12T16:07:52.823+0300 7f2a270c3700 10 monclient(hunting): get_auth_request con 0x559e559f1000 auth_method 0 -511> 2022-04-12T16:07:52.823+0300 7f2a270c3700 10 monclient(hunting): get_auth_request method 2 preferred_modes [2,1] -510> 2022-04-12T16:07:52.823+0300 7f2a270c3700 10 monclient(hunting): _init_auth method 2 -509> 2022-04-12T16:07:52.823+0300 7f2a270c3700 10 monclient(hunting): _init_auth creating new auth -508> 2022-04-12T16:07:52.823+0300 7f2a260c1700 10 monclient(hunting): handle_auth_done global_id 5730 payload 1000 -507> 2022-04-12T16:07:52.823+0300 7f2a260c1700 10 monclient: _finish_hunting 0 -506> 2022-04-12T16:07:52.823+0300 7f2a260c1700 1 monclient: found mon.noname-a -505> 2022-04-12T16:07:52.823+0300 7f2a260c1700 10 monclient: _send_mon_message to mon.noname-a at v2:192.168.166.83:3300/0 -504> 2022-04-12T16:07:52.823+0300 7f28f7664700 10 monclient: handle_monmap mon_map magic: 0 v1 -503> 2022-04-12T16:07:52.827+0300 7f28f7664700 10 monclient: got monmap 1 from mon.noname-a (according to old e1) -502> 2022-04-12T16:07:52.827+0300 7f28f7664700 10 monclient: dump: epoch 1 fsid 65ccd25a-39ce-4072-a5c6-a0e6654bda87 last_changed 2022-04-12T15:48:27.663651+0300 created 2022-04-12T15:48:27.663651+0300 min_mon_release 16 (pacific) election_strategy: 1 0: [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] mon.ceph-ss-test-oct-controller-av-01 1: [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] mon.ceph-ss-test-oct-controller-av-02 2: [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] mon.ceph-ss-test-oct-controller-av-03 -501> 2022-04-12T16:07:52.827+0300 7f28f7664700 10 monclient: _finish_auth 0 -500> 2022-04-12T16:07:52.827+0300 7f28f7664700 10 monclient: handle_config config(0 keys) v1 -499> 2022-04-12T16:07:52.827+0300 7f28f7664700 10 monclient: handle_monmap mon_map magic: 0 v1 -498> 2022-04-12T16:07:52.827+0300 7f28f7664700 10 monclient: got monmap 1 from mon.ceph-ss-test-oct-controller-av-01 (according to old e1) -497> 2022-04-12T16:07:52.827+0300 7f28f7664700 10 monclient: dump: epoch 1 fsid 65ccd25a-39ce-4072-a5c6-a0e6654bda87 last_changed 2022-04-12T15:48:27.663651+0300 created 2022-04-12T15:48:27.663651+0300 min_mon_release 16 (pacific) election_strategy: 1 0: [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] mon.ceph-ss-test-oct-controller-av-01 1: [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] mon.ceph-ss-test-oct-controller-av-02 2: [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] mon.ceph-ss-test-oct-controller-av-03 -496> 2022-04-12T16:07:52.827+0300 7f28f8e67700 4 set_mon_vals no callback set -495> 2022-04-12T16:07:52.827+0300 7f2a3b460400 5 monclient: authenticate success, global_id 5730 -494> 2022-04-12T16:07:52.827+0300 7f2a3b460400 10 monclient: _renew_subs -493> 2022-04-12T16:07:52.827+0300 7f2a3b460400 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -492> 2022-04-12T16:07:52.827+0300 7f2a3b460400 10 monclient: _renew_subs -491> 2022-04-12T16:07:52.827+0300 7f2a3b460400 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -490> 2022-04-12T16:07:52.827+0300 7f2a3b460400 1 librados: init done -489> 2022-04-12T16:07:52.831+0300 7f28f7664700 4 mgrc handle_mgr_map Got map version 29 -488> 2022-04-12T16:07:52.831+0300 7f28f7664700 4 mgrc handle_mgr_map Active mgr is now [v2:192.168.166.84:6800/56,v1:192.168.166.84:6801/56] -487> 2022-04-12T16:07:52.831+0300 7f28f7664700 4 mgrc reconnect Starting new session with [v2:192.168.166.84:6800/56,v1:192.168.166.84:6801/56] -486> 2022-04-12T16:07:52.831+0300 7f2a268c2700 10 monclient: get_auth_request con 0x559e593a1000 auth_method 0 -485> 2022-04-12T16:07:52.835+0300 7f2a270c3700 10 monclient: get_auth_request con 0x559e55979800 auth_method 0 -484> 2022-04-12T16:07:52.855+0300 7f2a260c1700 10 monclient: get_auth_request con 0x559e562ab000 auth_method 0 -483> 2022-04-12T16:07:53.075+0300 7f2a270c3700 10 monclient: get_auth_request con 0x559e59cf6000 auth_method 0 -482> 2022-04-12T16:07:53.083+0300 7f2a0a089700 2 garbage collection: garbage collection: stop -481> 2022-04-12T16:07:53.223+0300 7f2a0b08b700 10 monclient: tick -480> 2022-04-12T16:07:53.643+0300 7f2a09888700 2 object expiration: stop -479> 2022-04-12T16:07:53.655+0300 7f2a1f0b3700 10 monclient: tick -478> 2022-04-12T16:07:53.951+0300 7f29f8065700 1 ====== starting new request req=0x7f2a3b415620 ===== -477> 2022-04-12T16:07:53.951+0300 7f29f8065700 2 req 6350979191525555842 0.000000000s initializing for trans_id = tx00000582335ecfec73e82-00625579a9-1659-sync -476> 2022-04-12T16:07:53.951+0300 7f29f8065700 2 req 6350979191525555842 0.000000000s getting op 4 -475> 2022-04-12T16:07:53.951+0300 7f29f6061700 2 req 6350979191525555842 0.000000000s :post_period verifying requester -474> 2022-04-12T16:07:53.951+0300 7f29f9868700 1 ====== starting new request req=0x7f2a3b313620 ===== -473> 2022-04-12T16:07:53.951+0300 7f29f305b700 1 ====== starting new request req=0x7f2a3b394620 ===== -472> 2022-04-12T16:07:53.951+0300 7f29f9868700 2 req 8268245483269231629 0.000000000s initializing for trans_id = tx0000072beb575c1fd940d-00625579a9-1659-sync -471> 2022-04-12T16:07:53.951+0300 7f29f305b700 2 req 10560051255912296523 0.000000000s initializing for trans_id = tx00000928cd6aaee91f04b-00625579a9-1659-sync -470> 2022-04-12T16:07:53.955+0300 7f29f9868700 2 req 8268245483269231629 0.004000003s getting op 4 -469> 2022-04-12T16:07:53.955+0300 7f29f9868700 2 req 8268245483269231629 0.004000003s :post_period verifying requester -468> 2022-04-12T16:07:53.955+0300 7f29f9868700 5 req 8268245483269231629 0.004000003s :post_period error reading user info, uid=m7yH86NH1D44QiwaUUV9 can't authenticate -467> 2022-04-12T16:07:53.955+0300 7f29f9868700 5 req 8268245483269231629 0.004000003s :post_period error reading user info, uid=m7yH86NH1D44QiwaUUV9 can't authenticate -466> 2022-04-12T16:07:53.955+0300 7f29f9868700 5 req 8268245483269231629 0.004000003s :post_period Failed the auth strategy, reason=-2028 -465> 2022-04-12T16:07:53.955+0300 7f29f9868700 1 req 8268245483269231629 0.004000003s op->ERRORHANDLER: err_no=-2028 new_err_no=-2028 -464> 2022-04-12T16:07:53.955+0300 7f29f0856700 5 req 6350979191525555842 0.004000003s :post_period error reading user info, uid=m7yH86NH1D44QiwaUUV9 can't authenticate -463> 2022-04-12T16:07:53.955+0300 7f29f0856700 5 req 6350979191525555842 0.004000003s :post_period error reading user info, uid=m7yH86NH1D44QiwaUUV9 can't authenticate -462> 2022-04-12T16:07:53.955+0300 7f29f0856700 5 req 6350979191525555842 0.004000003s :post_period Failed the auth strategy, reason=-2028 -461> 2022-04-12T16:07:53.955+0300 7f29f0856700 1 req 6350979191525555842 0.004000003s op->ERRORHANDLER: err_no=-2028 new_err_no=-2028 -460> 2022-04-12T16:07:53.955+0300 7f29f305b700 2 req 10560051255912296523 0.004000003s getting op 4 -459> 2022-04-12T16:07:53.955+0300 7f29f305b700 2 req 10560051255912296523 0.004000003s :post_period verifying requester -458> 2022-04-12T16:07:53.955+0300 7f29e6842700 2 req 8268245483269231629 0.004000003s :post_period op status=0 -457> 2022-04-12T16:07:53.955+0300 7f29e6842700 2 req 8268245483269231629 0.004000003s :post_period http status=403 -456> 2022-04-12T16:07:53.955+0300 7f29e6842700 1 ====== req done req=0x7f2a3b313620 op status=0 http_status=403 latency=0.004000003s ====== -455> 2022-04-12T16:07:53.955+0300 7f29e6842700 1 beast: 0x7f2a3b313620: 192.168.166.84 - - [12/Apr/2022:16:07:53.951 +0300] "POST /admin/realm/period?period=8b4c894f-cf43-4c56-9af0-1c2b54c74991&epoch=3&rgwx-zonegroup=784a05a5-2089-4564-ab1b-a55c43c5f401 HTTP/1.1" 403 113 - - - latency=0.004000003s -454> 2022-04-12T16:07:53.955+0300 7f29f0856700 2 req 6350979191525555842 0.004000003s :post_period op status=0 -453> 2022-04-12T16:07:53.955+0300 7f29f0856700 2 req 6350979191525555842 0.004000003s :post_period http status=403 -452> 2022-04-12T16:07:53.955+0300 7f29f0856700 1 ====== req done req=0x7f2a3b415620 op status=0 http_status=403 latency=0.004000003s ====== -451> 2022-04-12T16:07:53.955+0300 7f29f305b700 5 req 10560051255912296523 0.004000003s :post_period error reading user info, uid=m7yH86NH1D44QiwaUUV9 can't authenticate -450> 2022-04-12T16:07:53.955+0300 7f29f305b700 5 req 10560051255912296523 0.004000003s :post_period error reading user info, uid=m7yH86NH1D44QiwaUUV9 can't authenticate -449> 2022-04-12T16:07:53.955+0300 7f29f305b700 5 req 10560051255912296523 0.004000003s :post_period Failed the auth strategy, reason=-2028 -448> 2022-04-12T16:07:53.955+0300 7f29f305b700 1 req 10560051255912296523 0.004000003s op->ERRORHANDLER: err_no=-2028 new_err_no=-2028 -447> 2022-04-12T16:07:53.955+0300 7f29f0856700 1 beast: 0x7f2a3b415620: 192.168.166.83 - - [12/Apr/2022:16:07:53.951 +0300] "POST /admin/realm/period?period=8b4c894f-cf43-4c56-9af0-1c2b54c74991&epoch=3&rgwx-zonegroup=784a05a5-2089-4564-ab1b-a55c43c5f401 HTTP/1.1" 403 113 - - - latency=0.004000003s -446> 2022-04-12T16:07:53.955+0300 7f29ee051700 2 req 10560051255912296523 0.004000003s :post_period op status=0 -445> 2022-04-12T16:07:53.955+0300 7f29ee051700 2 req 10560051255912296523 0.004000003s :post_period http status=403 -444> 2022-04-12T16:07:53.955+0300 7f29ee051700 1 ====== req done req=0x7f2a3b394620 op status=0 http_status=403 latency=0.004000003s ====== -443> 2022-04-12T16:07:53.955+0300 7f29ee051700 1 beast: 0x7f2a3b394620: 192.168.166.85 - - [12/Apr/2022:16:07:53.951 +0300] "POST /admin/realm/period?period=8b4c894f-cf43-4c56-9af0-1c2b54c74991&epoch=3&rgwx-zonegroup=784a05a5-2089-4564-ab1b-a55c43c5f401 HTTP/1.1" 403 113 - - - latency=0.004000003s -442> 2022-04-12T16:07:54.223+0300 7f2a0b08b700 10 monclient: tick -441> 2022-04-12T16:07:54.655+0300 7f2a1f0b3700 10 monclient: tick -440> 2022-04-12T16:07:54.655+0300 7f2a208b6700 10 monclient: _renew_subs -439> 2022-04-12T16:07:54.655+0300 7f2a208b6700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -438> 2022-04-12T16:07:54.731+0300 7f2a07884700 1 RGW-SYNC:meta: start -437> 2022-04-12T16:07:54.731+0300 7f2a07884700 1 RGW-SYNC:meta: realm epoch=2 period id=8b4c894f-cf43-4c56-9af0-1c2b54c74991 -436> 2022-04-12T16:07:54.735+0300 7f2a07884700 4 meta sync: cr:s=0x559e571e3180:op=0x559e59df4000:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -435> 2022-04-12T16:07:54.735+0300 7f2a07884700 4 meta sync: cr:s=0x559e59bc08c0:op=0x559e5578a800:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -434> 2022-04-12T16:07:54.735+0300 7f2a07884700 4 meta sync: cr:s=0x559e594b3540:op=0x559e59df4a00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -433> 2022-04-12T16:07:54.735+0300 7f2a07884700 4 meta sync: cr:s=0x559e5988b680:op=0x559e59df5400:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -432> 2022-04-12T16:07:54.735+0300 7f2a07884700 4 meta sync: cr:s=0x559e59bc1180:op=0x559e59d78a00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -431> 2022-04-12T16:07:54.735+0300 7f2a07884700 4 meta sync: cr:s=0x559e596883c0:op=0x559e59dfa800:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -430> 2022-04-12T16:07:54.735+0300 7f2a07884700 4 meta sync: cr:s=0x559e59689b80:op=0x559e59df7200:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -429> 2022-04-12T16:07:54.735+0300 7f2a07884700 4 meta sync: cr:s=0x559e55ddeb40:op=0x559e59df9400:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -428> 2022-04-12T16:07:54.735+0300 7f2a07884700 4 meta sync: cr:s=0x559e59bc17c0:op=0x559e59d79e00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -427> 2022-04-12T16:07:54.735+0300 7f2a07884700 4 meta sync: cr:s=0x559e5965aa00:op=0x559e59dfb200:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -426> 2022-04-12T16:07:54.735+0300 7f2a07884700 4 meta sync: cr:s=0x559e59bc0140:op=0x559e5578b200:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -425> 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e595b1e00:op=0x559e59dfca00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -424> 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e55dfd900:op=0x559e59df8a00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -423> 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e59bc12c0:op=0x559e59d78000:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -422> 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e59688a00:op=0x559e59e01400:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -421> 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e59689cc0:op=0x559e59e02800:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -420> 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e5988b900:op=0x559e59e09400:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -419> 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e55dfc8c0:op=0x559e59dfde00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -418> 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e571e3040:op=0x559e59e05400:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -417> 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e55dfcc80:op=0x559e59d7b200:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -416> 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e571e3540:op=0x559e59e00a00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -415> 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e55dfdb80:op=0x559e59dfe800:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -414> 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e596888c0:op=0x559e59e01e00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -413> 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e55dfd2c0:op=0x559e59df6800:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -412> 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e594b3b80:op=0x559e59e08a00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -411> 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e55dfc280:op=0x559e59dff200:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -410> 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e5965ac80:op=0x559e59e04000:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -409> 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e5988bcc0:op=0x559e59e0d400:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -408> 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e55dfc3c0:op=0x559e59dfd400:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -407> 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e594b37c0:op=0x559e59e09e00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -406> 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e5965ab40:op=0x559e59df9e00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -405> 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e571e2dc0:op=0x559e59e00000:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -404> 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e5988b540:op=0x559e59e08000:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -403> 2022-04-12T16:07:54.743+0300 7f2a07884700 4 meta sync: cr:s=0x559e55dfcdc0:op=0x559e59df5e00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -402> 2022-04-12T16:07:54.743+0300 7f2a07884700 4 meta sync: cr:s=0x559e59acc3c0:op=0x559e59d79400:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -401> 2022-04-12T16:07:54.743+0300 7f2a07884700 4 meta sync: cr:s=0x559e55dfc780:op=0x559e59d7a800:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -400> 2022-04-12T16:07:54.743+0300 7f2a07884700 4 meta sync: cr:s=0x559e57c2af00:op=0x559e59e17200:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -399> 2022-04-12T16:07:54.747+0300 7f2a07884700 4 meta sync: cr:s=0x559e571e2c80:op=0x559e59e06800:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -398> 2022-04-12T16:07:54.747+0300 7f2a07884700 4 meta sync: cr:s=0x559e59bc0dc0:op=0x559e59e11400:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -397> 2022-04-12T16:07:54.747+0300 7f2a07884700 4 meta sync: cr:s=0x559e59bcd540:op=0x559e59e14000:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -396> 2022-04-12T16:07:54.747+0300 7f2a07884700 4 meta sync: cr:s=0x559e57c2adc0:op=0x559e59e16800:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -395> 2022-04-12T16:07:54.747+0300 7f2a07884700 4 meta sync: cr:s=0x559e55a08140:op=0x559e59e14a00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -394> 2022-04-12T16:07:54.747+0300 7f2a07884700 4 meta sync: cr:s=0x559e564c1cc0:op=0x559e59e04a00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -393> 2022-04-12T16:07:54.747+0300 7f2a07884700 4 meta sync: cr:s=0x559e57c2b040:op=0x559e59e18000:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -392> 2022-04-12T16:07:54.747+0300 7f2a07884700 4 meta sync: cr:s=0x559e59bc1900:op=0x559e59e0e800:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -391> 2022-04-12T16:07:54.747+0300 7f2a07884700 4 meta sync: cr:s=0x559e59bc0640:op=0x559e59e0f200:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -390> 2022-04-12T16:07:54.751+0300 7f2a07884700 4 meta sync: cr:s=0x559e59689e00:op=0x559e59e03200:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -389> 2022-04-12T16:07:54.751+0300 7f2a07884700 4 meta sync: cr:s=0x559e55a083c0:op=0x559e59e0de00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -388> 2022-04-12T16:07:54.751+0300 7f2a07884700 4 meta sync: cr:s=0x559e571e3cc0:op=0x559e59e07200:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -387> 2022-04-12T16:07:54.751+0300 7f2a07884700 4 meta sync: cr:s=0x559e5988be00:op=0x559e59e0b200:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -386> 2022-04-12T16:07:54.751+0300 7f2a07884700 4 meta sync: cr:s=0x559e594b3680:op=0x559e59e0ca00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -385> 2022-04-12T16:07:54.751+0300 7f2a07884700 4 meta sync: cr:s=0x559e5988ba40:op=0x559e59df8000:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -384> 2022-04-12T16:07:54.751+0300 7f2a07884700 4 meta sync: cr:s=0x559e571e3900:op=0x559e59e05e00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -383> 2022-04-12T16:07:54.751+0300 7f2a07884700 4 meta sync: cr:s=0x559e5988bb80:op=0x559e59e0a800:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -382> 2022-04-12T16:07:54.751+0300 7f2a07884700 4 meta sync: cr:s=0x559e55a08780:op=0x559e59e15400:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -381> 2022-04-12T16:07:54.751+0300 7f2a07884700 4 meta sync: cr:s=0x559e5988b7c0:op=0x559e59e0c000:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -380> 2022-04-12T16:07:54.751+0300 7f2a07884700 4 meta sync: cr:s=0x559e595b1b80:op=0x559e59dfc000:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -379> 2022-04-12T16:07:54.751+0300 7f2a07884700 4 meta sync: cr:s=0x559e59bc1540:op=0x559e59e10a00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -378> 2022-04-12T16:07:54.751+0300 7f2a07884700 4 meta sync: cr:s=0x559e59bc0780:op=0x559e59e11e00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -377> 2022-04-12T16:07:54.751+0300 7f2a07884700 4 meta sync: cr:s=0x559e59bcc000:op=0x559e59e13200:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -376> 2022-04-12T16:07:54.755+0300 7f2a07884700 4 meta sync: cr:s=0x559e59bc1b80:op=0x559e59e10000:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -375> 2022-04-12T16:07:54.755+0300 7f2a07884700 4 meta sync: cr:s=0x559e55a08280:op=0x559e59e15e00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -374> 2022-04-12T16:07:54.755+0300 7f2a07884700 4 meta sync: cr:s=0x559e59bcc500:op=0x559e59e12800:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -373> 2022-04-12T16:07:54.803+0300 7f2a07884700 4 meta sync: cr:s=0x559e54a72640:op=0x559e55789e00:18RGWMetaSyncShardCR: adjusting marker pos=user:dashboard -372> 2022-04-12T16:07:54.811+0300 7f2a07884700 4 meta sync: cr:s=0x559e54a72640:op=0x559e55789e00:18RGWMetaSyncShardCR: adjusting marker pos=user:testuser -371> 2022-04-12T16:07:54.835+0300 7f2a07884700 4 meta sync: cr:s=0x559e54a72640:op=0x559e55789e00:18RGWMetaSyncShardCR: adjusting marker pos=user:testuser -370> 2022-04-12T16:07:54.835+0300 7f2a07884700 4 meta sync: cr:s=0x559e54a72640:op=0x559e55789e00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -369> 2022-04-12T16:07:55.223+0300 7f2a0b08b700 10 monclient: tick -368> 2022-04-12T16:07:55.655+0300 7f2a1f0b3700 10 monclient: tick -367> 2022-04-12T16:07:55.823+0300 7f28f6662700 10 monclient: tick -366> 2022-04-12T16:07:56.223+0300 7f2a0b08b700 10 monclient: tick -365> 2022-04-12T16:07:56.659+0300 7f2a1f0b3700 10 monclient: tick -364> 2022-04-12T16:07:56.827+0300 7f28f6662700 10 monclient: tick -363> 2022-04-12T16:07:57.227+0300 7f2a0b08b700 10 monclient: tick -362> 2022-04-12T16:07:57.655+0300 7f2a1f0b3700 10 monclient: tick -361> 2022-04-12T16:07:57.823+0300 7f28f6662700 10 monclient: tick -360> 2022-04-12T16:07:57.827+0300 7f28f7e65700 10 monclient: _renew_subs -359> 2022-04-12T16:07:57.827+0300 7f28f7e65700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -358> 2022-04-12T16:07:58.223+0300 7f2a0b08b700 10 monclient: tick -357> 2022-04-12T16:07:58.659+0300 7f2a1f0b3700 10 monclient: tick -356> 2022-04-12T16:07:58.823+0300 7f28f6662700 10 monclient: tick -355> 2022-04-12T16:07:59.223+0300 7f2a0b08b700 10 monclient: tick -354> 2022-04-12T16:07:59.655+0300 7f2a208b6700 10 monclient: _renew_subs -353> 2022-04-12T16:07:59.655+0300 7f2a208b6700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -352> 2022-04-12T16:07:59.655+0300 7f2a1f0b3700 10 monclient: tick -351> 2022-04-12T16:07:59.823+0300 7f28f6662700 10 monclient: tick -350> 2022-04-12T16:08:00.223+0300 7f2a0b08b700 10 monclient: tick -349> 2022-04-12T16:08:00.659+0300 7f2a1f0b3700 10 monclient: tick -348> 2022-04-12T16:08:00.823+0300 7f28f6662700 10 monclient: tick -347> 2022-04-12T16:08:01.227+0300 7f2a0b08b700 10 monclient: tick -346> 2022-04-12T16:08:01.655+0300 7f2a1f0b3700 10 monclient: tick -345> 2022-04-12T16:08:01.827+0300 7f28f6662700 10 monclient: tick -344> 2022-04-12T16:08:02.227+0300 7f2a0b08b700 10 monclient: tick -343> 2022-04-12T16:08:02.659+0300 7f2a1f0b3700 10 monclient: tick -342> 2022-04-12T16:08:02.827+0300 7f28f6662700 10 monclient: tick -341> 2022-04-12T16:08:02.827+0300 7f28f7e65700 10 monclient: _renew_subs -340> 2022-04-12T16:08:02.827+0300 7f28f7e65700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -339> 2022-04-12T16:08:03.223+0300 7f2a0b08b700 10 monclient: tick -338> 2022-04-12T16:08:03.655+0300 7f2a1f0b3700 10 monclient: tick -337> 2022-04-12T16:08:03.827+0300 7f28f6662700 10 monclient: tick -336> 2022-04-12T16:08:04.227+0300 7f2a0b08b700 10 monclient: tick -335> 2022-04-12T16:08:04.655+0300 7f2a208b6700 10 monclient: _renew_subs -334> 2022-04-12T16:08:04.655+0300 7f2a208b6700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -333> 2022-04-12T16:08:04.655+0300 7f2a1f0b3700 10 monclient: tick -332> 2022-04-12T16:08:04.823+0300 7f28f6662700 10 monclient: tick -331> 2022-04-12T16:08:05.223+0300 7f2a0b08b700 10 monclient: tick -330> 2022-04-12T16:08:05.659+0300 7f2a1f0b3700 10 monclient: tick -329> 2022-04-12T16:08:05.823+0300 7f28f6662700 10 monclient: tick -328> 2022-04-12T16:08:06.227+0300 7f2a0b08b700 10 monclient: tick -327> 2022-04-12T16:08:06.659+0300 7f2a1f0b3700 10 monclient: tick -326> 2022-04-12T16:08:06.823+0300 7f28f6662700 10 monclient: tick -325> 2022-04-12T16:08:07.227+0300 7f2a0b08b700 10 monclient: tick -324> 2022-04-12T16:08:07.655+0300 7f2a1f0b3700 10 monclient: tick -323> 2022-04-12T16:08:07.827+0300 7f28f6662700 10 monclient: tick -322> 2022-04-12T16:08:08.227+0300 7f2a0b08b700 10 monclient: tick -321> 2022-04-12T16:08:08.659+0300 7f2a1f0b3700 10 monclient: tick -320> 2022-04-12T16:08:08.823+0300 7f28f6662700 10 monclient: tick -319> 2022-04-12T16:08:09.223+0300 7f2a0b08b700 10 monclient: tick -318> 2022-04-12T16:08:09.659+0300 7f2a1f0b3700 10 monclient: tick -317> 2022-04-12T16:08:09.823+0300 7f28f6662700 10 monclient: tick -316> 2022-04-12T16:08:10.223+0300 7f2a0b08b700 10 monclient: tick -315> 2022-04-12T16:08:10.655+0300 7f2a1f0b3700 10 monclient: tick -314> 2022-04-12T16:08:10.827+0300 7f28f6662700 10 monclient: tick -313> 2022-04-12T16:08:11.211+0300 7f2a0e091700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start -312> 2022-04-12T16:08:11.223+0300 7f2a0b08b700 10 monclient: tick -311> 2022-04-12T16:08:11.655+0300 7f2a1f0b3700 10 monclient: tick -310> 2022-04-12T16:08:11.827+0300 7f28f6662700 10 monclient: tick -309> 2022-04-12T16:08:12.223+0300 7f2a0b08b700 10 monclient: tick -308> 2022-04-12T16:08:12.655+0300 7f2a1f0b3700 10 monclient: tick -307> 2022-04-12T16:08:12.823+0300 7f28f6662700 10 monclient: tick -306> 2022-04-12T16:08:13.223+0300 7f2a0b08b700 10 monclient: tick -305> 2022-04-12T16:08:13.659+0300 7f2a1f0b3700 10 monclient: tick -304> 2022-04-12T16:08:13.823+0300 7f28f6662700 10 monclient: tick -303> 2022-04-12T16:08:14.227+0300 7f2a0b08b700 10 monclient: tick -302> 2022-04-12T16:08:14.659+0300 7f2a1f0b3700 10 monclient: tick -301> 2022-04-12T16:08:14.827+0300 7f28f6662700 10 monclient: tick -300> 2022-04-12T16:08:15.227+0300 7f2a0b08b700 10 monclient: tick -299> 2022-04-12T16:08:15.659+0300 7f2a1f0b3700 10 monclient: tick -298> 2022-04-12T16:08:15.827+0300 7f28f6662700 10 monclient: tick -297> 2022-04-12T16:08:16.227+0300 7f2a0b08b700 10 monclient: tick -296> 2022-04-12T16:08:16.659+0300 7f2a1f0b3700 10 monclient: tick -295> 2022-04-12T16:08:16.827+0300 7f28f6662700 10 monclient: tick -294> 2022-04-12T16:08:17.227+0300 7f2a0b08b700 10 monclient: tick -293> 2022-04-12T16:08:17.659+0300 7f2a1f0b3700 10 monclient: tick -292> 2022-04-12T16:08:17.827+0300 7f28f6662700 10 monclient: tick -291> 2022-04-12T16:08:18.227+0300 7f2a0b08b700 10 monclient: tick -290> 2022-04-12T16:08:18.659+0300 7f2a1f0b3700 10 monclient: tick -289> 2022-04-12T16:08:18.827+0300 7f28f6662700 10 monclient: tick -288> 2022-04-12T16:08:19.227+0300 7f2a0b08b700 10 monclient: tick -287> 2022-04-12T16:08:19.659+0300 7f2a1f0b3700 10 monclient: tick -286> 2022-04-12T16:08:19.827+0300 7f28f6662700 10 monclient: tick -285> 2022-04-12T16:08:20.227+0300 7f2a0b08b700 10 monclient: tick -284> 2022-04-12T16:08:20.659+0300 7f2a1f0b3700 10 monclient: tick -283> 2022-04-12T16:08:20.827+0300 7f28f6662700 10 monclient: tick -282> 2022-04-12T16:08:21.227+0300 7f2a0b08b700 10 monclient: tick -281> 2022-04-12T16:08:21.659+0300 7f2a1f0b3700 10 monclient: tick -280> 2022-04-12T16:08:21.827+0300 7f28f6662700 10 monclient: tick -279> 2022-04-12T16:08:22.227+0300 7f2a0b08b700 10 monclient: tick -278> 2022-04-12T16:08:22.659+0300 7f2a1f0b3700 10 monclient: tick -277> 2022-04-12T16:08:22.827+0300 7f28f6662700 10 monclient: tick -276> 2022-04-12T16:08:23.227+0300 7f2a0b08b700 10 monclient: tick -275> 2022-04-12T16:08:23.659+0300 7f2a1f0b3700 10 monclient: tick -274> 2022-04-12T16:08:23.827+0300 7f28f6662700 10 monclient: tick -273> 2022-04-12T16:08:23.959+0300 7f29f505f700 1 ====== starting new request req=0x7f2a3b313620 ===== -272> 2022-04-12T16:08:23.959+0300 7f29f505f700 2 req 8920709381089314344 0.000000000s initializing for trans_id = tx000007bccb9eb0c52c628-00625579c7-1659-sync -271> 2022-04-12T16:08:23.959+0300 7f29f505f700 2 req 8920709381089314344 0.000000000s getting op 4 -270> 2022-04-12T16:08:23.959+0300 7f29f505f700 2 req 8920709381089314344 0.000000000s :post_period verifying requester -269> 2022-04-12T16:08:23.959+0300 7f29f505f700 2 req 8920709381089314344 0.000000000s :post_period normalizing buckets and tenants -268> 2022-04-12T16:08:23.959+0300 7f29f505f700 2 req 8920709381089314344 0.000000000s :post_period init permissions -267> 2022-04-12T16:08:23.959+0300 7f29f505f700 2 req 8920709381089314344 0.000000000s :post_period recalculating target -266> 2022-04-12T16:08:23.959+0300 7f29f505f700 2 req 8920709381089314344 0.000000000s :post_period reading permissions -265> 2022-04-12T16:08:23.959+0300 7f29f505f700 2 req 8920709381089314344 0.000000000s :post_period init op -264> 2022-04-12T16:08:23.959+0300 7f29f505f700 2 req 8920709381089314344 0.000000000s :post_period verifying op mask -263> 2022-04-12T16:08:23.959+0300 7f29f505f700 2 req 8920709381089314344 0.000000000s :post_period verifying op permissions -262> 2022-04-12T16:08:23.959+0300 7f29f505f700 2 overriding permissions due to system operation -261> 2022-04-12T16:08:23.959+0300 7f29f505f700 2 req 8920709381089314344 0.000000000s :post_period verifying op params -260> 2022-04-12T16:08:23.959+0300 7f29f505f700 2 req 8920709381089314344 0.000000000s :post_period pre-executing -259> 2022-04-12T16:08:23.959+0300 7f29f505f700 2 req 8920709381089314344 0.000000000s :post_period executing -258> 2022-04-12T16:08:23.959+0300 7f29cc80e700 1 ====== starting new request req=0x7f2a3b415620 ===== -257> 2022-04-12T16:08:23.959+0300 7f29cc80e700 2 req 8609644640538370824 0.000000000s initializing for trans_id = tx00000777b9a2f49a2a308-00625579c7-1659-sync -256> 2022-04-12T16:08:23.959+0300 7f29cc80e700 2 req 8609644640538370824 0.000000000s getting op 4 -255> 2022-04-12T16:08:23.959+0300 7f29cc80e700 2 req 8609644640538370824 0.000000000s :post_period verifying requester -254> 2022-04-12T16:08:23.959+0300 7f29cc80e700 2 req 8609644640538370824 0.000000000s :post_period normalizing buckets and tenants -253> 2022-04-12T16:08:23.959+0300 7f29cc80e700 2 req 8609644640538370824 0.000000000s :post_period init permissions -252> 2022-04-12T16:08:23.959+0300 7f29cc80e700 2 req 8609644640538370824 0.000000000s :post_period recalculating target -251> 2022-04-12T16:08:23.959+0300 7f29cc80e700 2 req 8609644640538370824 0.000000000s :post_period reading permissions -250> 2022-04-12T16:08:23.959+0300 7f29cc80e700 2 req 8609644640538370824 0.000000000s :post_period init op -249> 2022-04-12T16:08:23.959+0300 7f29cc80e700 2 req 8609644640538370824 0.000000000s :post_period verifying op mask -248> 2022-04-12T16:08:23.959+0300 7f29cc80e700 2 req 8609644640538370824 0.000000000s :post_period verifying op permissions -247> 2022-04-12T16:08:23.959+0300 7f29cc80e700 2 overriding permissions due to system operation -246> 2022-04-12T16:08:23.959+0300 7f29cc80e700 2 req 8609644640538370824 0.000000000s :post_period verifying op params -245> 2022-04-12T16:08:23.959+0300 7f29cc80e700 2 req 8609644640538370824 0.000000000s :post_period pre-executing -244> 2022-04-12T16:08:23.959+0300 7f29cc80e700 2 req 8609644640538370824 0.000000000s :post_period executing -243> 2022-04-12T16:08:23.959+0300 7f29c8005700 1 ====== starting new request req=0x7f2a3b394620 ===== -242> 2022-04-12T16:08:23.959+0300 7f29c8005700 2 req 14231248929957790225 0.000000000s initializing for trans_id = tx00000c57f8d805c4ac211-00625579c7-1659-sync -241> 2022-04-12T16:08:23.959+0300 7f29c8005700 2 req 14231248929957790225 0.000000000s getting op 4 -240> 2022-04-12T16:08:23.959+0300 7f29c8005700 2 req 14231248929957790225 0.000000000s :post_period verifying requester -239> 2022-04-12T16:08:23.959+0300 7f29c8005700 2 req 14231248929957790225 0.000000000s :post_period normalizing buckets and tenants -238> 2022-04-12T16:08:23.959+0300 7f29c8005700 2 req 14231248929957790225 0.000000000s :post_period init permissions -237> 2022-04-12T16:08:23.959+0300 7f29c8005700 2 req 14231248929957790225 0.000000000s :post_period recalculating target -236> 2022-04-12T16:08:23.959+0300 7f29c8005700 2 req 14231248929957790225 0.000000000s :post_period reading permissions -235> 2022-04-12T16:08:23.959+0300 7f29c8005700 2 req 14231248929957790225 0.000000000s :post_period init op -234> 2022-04-12T16:08:23.959+0300 7f29c8005700 2 req 14231248929957790225 0.000000000s :post_period verifying op mask -233> 2022-04-12T16:08:23.959+0300 7f29c8005700 2 req 14231248929957790225 0.000000000s :post_period verifying op permissions -232> 2022-04-12T16:08:23.959+0300 7f29c8005700 2 overriding permissions due to system operation -231> 2022-04-12T16:08:23.959+0300 7f29c8005700 2 req 14231248929957790225 0.000000000s :post_period verifying op params -230> 2022-04-12T16:08:23.959+0300 7f29c8005700 2 req 14231248929957790225 0.000000000s :post_period pre-executing -229> 2022-04-12T16:08:23.959+0300 7f29c8005700 2 req 14231248929957790225 0.000000000s :post_period executing -228> 2022-04-12T16:08:23.963+0300 7f29ac7ce700 4 req 8920709381089314344 0.004000003s :post_period already have epoch >= 3 for period 8b4c894f-cf43-4c56-9af0-1c2b54c74991 -227> 2022-04-12T16:08:23.963+0300 7f29ac7ce700 2 req 8920709381089314344 0.004000003s :post_period completing -226> 2022-04-12T16:08:23.963+0300 7f29ac7ce700 2 req 8920709381089314344 0.004000003s :post_period op status=0 -225> 2022-04-12T16:08:23.963+0300 7f29ac7ce700 2 req 8920709381089314344 0.004000003s :post_period http status=200 -224> 2022-04-12T16:08:23.963+0300 7f29ac7ce700 1 ====== req done req=0x7f2a3b313620 op status=0 http_status=200 latency=0.004000003s ====== -223> 2022-04-12T16:08:23.963+0300 7f29ac7ce700 1 beast: 0x7f2a3b313620: 192.168.166.84 - rgw_zone_user [12/Apr/2022:16:08:23.959 +0300] "POST /admin/realm/period?period=8b4c894f-cf43-4c56-9af0-1c2b54c74991&epoch=3&rgwx-zonegroup=784a05a5-2089-4564-ab1b-a55c43c5f401 HTTP/1.1" 200 1868 - - - latency=0.004000003s -222> 2022-04-12T16:08:23.963+0300 7f29a57c0700 4 req 14231248929957790225 0.004000003s :post_period already have epoch >= 3 for period 8b4c894f-cf43-4c56-9af0-1c2b54c74991 -221> 2022-04-12T16:08:23.963+0300 7f29a57c0700 2 req 14231248929957790225 0.004000003s :post_period completing -220> 2022-04-12T16:08:23.963+0300 7f29a57c0700 2 req 14231248929957790225 0.004000003s :post_period op status=0 -219> 2022-04-12T16:08:23.963+0300 7f29a57c0700 2 req 14231248929957790225 0.004000003s :post_period http status=200 -218> 2022-04-12T16:08:23.963+0300 7f29a57c0700 1 ====== req done req=0x7f2a3b394620 op status=0 http_status=200 latency=0.004000003s ====== -217> 2022-04-12T16:08:23.963+0300 7f29a57c0700 1 beast: 0x7f2a3b394620: 192.168.166.85 - rgw_zone_user [12/Apr/2022:16:08:23.959 +0300] "POST /admin/realm/period?period=8b4c894f-cf43-4c56-9af0-1c2b54c74991&epoch=3&rgwx-zonegroup=784a05a5-2089-4564-ab1b-a55c43c5f401 HTTP/1.1" 200 1868 - - - latency=0.004000003s -216> 2022-04-12T16:08:23.967+0300 7f299e7b2700 4 req 8609644640538370824 0.008000007s :post_period already have epoch >= 3 for period 8b4c894f-cf43-4c56-9af0-1c2b54c74991 -215> 2022-04-12T16:08:23.967+0300 7f299e7b2700 2 req 8609644640538370824 0.008000007s :post_period completing -214> 2022-04-12T16:08:23.967+0300 7f299e7b2700 2 req 8609644640538370824 0.008000007s :post_period op status=0 -213> 2022-04-12T16:08:23.967+0300 7f299e7b2700 2 req 8609644640538370824 0.008000007s :post_period http status=200 -212> 2022-04-12T16:08:23.967+0300 7f299e7b2700 1 ====== req done req=0x7f2a3b415620 op status=0 http_status=200 latency=0.008000007s ====== -211> 2022-04-12T16:08:23.967+0300 7f299e7b2700 1 beast: 0x7f2a3b415620: 192.168.166.83 - rgw_zone_user [12/Apr/2022:16:08:23.959 +0300] "POST /admin/realm/period?period=8b4c894f-cf43-4c56-9af0-1c2b54c74991&epoch=3&rgwx-zonegroup=784a05a5-2089-4564-ab1b-a55c43c5f401 HTTP/1.1" 200 1868 - - - latency=0.008000007s -210> 2022-04-12T16:08:24.227+0300 7f2a0b08b700 10 monclient: tick -209> 2022-04-12T16:08:24.663+0300 7f2a1f0b3700 10 monclient: tick -208> 2022-04-12T16:08:24.827+0300 7f28f6662700 10 monclient: tick -207> 2022-04-12T16:08:25.227+0300 7f2a0b08b700 10 monclient: tick -206> 2022-04-12T16:08:25.663+0300 7f2a1f0b3700 10 monclient: tick -205> 2022-04-12T16:08:25.827+0300 7f28f6662700 10 monclient: tick -204> 2022-04-12T16:08:26.231+0300 7f2a0b08b700 10 monclient: tick -203> 2022-04-12T16:08:26.663+0300 7f2a1f0b3700 10 monclient: tick -202> 2022-04-12T16:08:26.831+0300 7f28f6662700 10 monclient: tick -201> 2022-04-12T16:08:27.227+0300 7f2a0b08b700 10 monclient: tick -200> 2022-04-12T16:08:27.663+0300 7f2a1f0b3700 10 monclient: tick -199> 2022-04-12T16:08:27.827+0300 7f28f6662700 10 monclient: tick -198> 2022-04-12T16:08:28.227+0300 7f2a0b08b700 10 monclient: tick -197> 2022-04-12T16:08:28.659+0300 7f2a1f0b3700 10 monclient: tick -196> 2022-04-12T16:08:28.827+0300 7f28f6662700 10 monclient: tick -195> 2022-04-12T16:08:29.227+0300 7f2a0b08b700 10 monclient: tick -194> 2022-04-12T16:08:29.663+0300 7f2a1f0b3700 10 monclient: tick -193> 2022-04-12T16:08:29.827+0300 7f28f6662700 10 monclient: tick -192> 2022-04-12T16:08:30.227+0300 7f2a0b08b700 10 monclient: tick -191> 2022-04-12T16:08:30.663+0300 7f2a1f0b3700 10 monclient: tick -190> 2022-04-12T16:08:30.827+0300 7f28f6662700 10 monclient: tick -189> 2022-04-12T16:08:31.227+0300 7f2a0b08b700 10 monclient: tick -188> 2022-04-12T16:08:31.659+0300 7f2a1f0b3700 10 monclient: tick -187> 2022-04-12T16:08:31.831+0300 7f28f6662700 10 monclient: tick -186> 2022-04-12T16:08:32.227+0300 7f2a0b08b700 10 monclient: tick -185> 2022-04-12T16:08:32.659+0300 7f2a1f0b3700 10 monclient: tick -184> 2022-04-12T16:08:32.827+0300 7f28f6662700 10 monclient: tick -183> 2022-04-12T16:08:33.211+0300 7f2a0e091700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start -182> 2022-04-12T16:08:33.227+0300 7f2a0b08b700 10 monclient: tick -181> 2022-04-12T16:08:33.659+0300 7f2a1f0b3700 10 monclient: tick -180> 2022-04-12T16:08:33.831+0300 7f28f6662700 10 monclient: tick -179> 2022-04-12T16:08:34.227+0300 7f2a0b08b700 10 monclient: tick -178> 2022-04-12T16:08:34.659+0300 7f2a1f0b3700 10 monclient: tick -177> 2022-04-12T16:08:34.827+0300 7f28f6662700 10 monclient: tick -176> 2022-04-12T16:08:35.227+0300 7f2a0b08b700 10 monclient: tick -175> 2022-04-12T16:08:35.659+0300 7f2a1f0b3700 10 monclient: tick -174> 2022-04-12T16:08:35.827+0300 7f28f6662700 10 monclient: tick -173> 2022-04-12T16:08:36.227+0300 7f2a0b08b700 10 monclient: tick -172> 2022-04-12T16:08:36.659+0300 7f2a1f0b3700 10 monclient: tick -171> 2022-04-12T16:08:36.827+0300 7f28f6662700 10 monclient: tick -170> 2022-04-12T16:08:37.231+0300 7f2a0b08b700 10 monclient: tick -169> 2022-04-12T16:08:37.659+0300 7f2a1f0b3700 10 monclient: tick -168> 2022-04-12T16:08:37.831+0300 7f28f6662700 10 monclient: tick -167> 2022-04-12T16:08:38.227+0300 7f2a0b08b700 10 monclient: tick -166> 2022-04-12T16:08:38.663+0300 7f2a1f0b3700 10 monclient: tick -165> 2022-04-12T16:08:38.831+0300 7f28f6662700 10 monclient: tick -164> 2022-04-12T16:08:39.227+0300 7f2a0b08b700 10 monclient: tick -163> 2022-04-12T16:08:39.659+0300 7f2a1f0b3700 10 monclient: tick -162> 2022-04-12T16:08:39.827+0300 7f28f6662700 10 monclient: tick -161> 2022-04-12T16:08:40.227+0300 7f2a0b08b700 10 monclient: tick -160> 2022-04-12T16:08:40.659+0300 7f2a1f0b3700 10 monclient: tick -159> 2022-04-12T16:08:40.827+0300 7f28f6662700 10 monclient: tick -158> 2022-04-12T16:08:41.231+0300 7f2a0b08b700 10 monclient: tick -157> 2022-04-12T16:08:41.659+0300 7f2a1f0b3700 10 monclient: tick -156> 2022-04-12T16:08:41.831+0300 7f28f6662700 10 monclient: tick -155> 2022-04-12T16:08:42.227+0300 7f2a0b08b700 10 monclient: tick -154> 2022-04-12T16:08:42.659+0300 7f2a1f0b3700 10 monclient: tick -153> 2022-04-12T16:08:42.827+0300 7f28f6662700 10 monclient: tick -152> 2022-04-12T16:08:43.227+0300 7f2a0b08b700 10 monclient: tick -151> 2022-04-12T16:08:43.659+0300 7f2a1f0b3700 10 monclient: tick -150> 2022-04-12T16:08:43.831+0300 7f28f6662700 10 monclient: tick -149> 2022-04-12T16:08:44.227+0300 7f2a0b08b700 10 monclient: tick -148> 2022-04-12T16:08:44.659+0300 7f2a1f0b3700 10 monclient: tick -147> 2022-04-12T16:08:44.831+0300 7f28f6662700 10 monclient: tick -146> 2022-04-12T16:08:45.231+0300 7f2a0b08b700 10 monclient: tick -145> 2022-04-12T16:08:45.663+0300 7f2a1f0b3700 10 monclient: tick -144> 2022-04-12T16:08:45.831+0300 7f28f6662700 10 monclient: tick -143> 2022-04-12T16:08:46.231+0300 7f2a0b08b700 10 monclient: tick -142> 2022-04-12T16:08:46.663+0300 7f2a1f0b3700 10 monclient: tick -141> 2022-04-12T16:08:46.827+0300 7f28f6662700 10 monclient: tick -140> 2022-04-12T16:08:47.231+0300 7f2a0b08b700 10 monclient: tick -139> 2022-04-12T16:08:47.663+0300 7f2a1f0b3700 10 monclient: tick -138> 2022-04-12T16:08:47.831+0300 7f28f6662700 10 monclient: tick -137> 2022-04-12T16:08:48.231+0300 7f2a0b08b700 10 monclient: tick -136> 2022-04-12T16:08:48.663+0300 7f2a1f0b3700 10 monclient: tick -135> 2022-04-12T16:08:48.831+0300 7f28f6662700 10 monclient: tick -134> 2022-04-12T16:08:49.231+0300 7f2a0b08b700 10 monclient: tick -133> 2022-04-12T16:08:49.663+0300 7f2a1f0b3700 10 monclient: tick -132> 2022-04-12T16:08:49.667+0300 7f2a208b6700 10 monclient: _renew_subs -131> 2022-04-12T16:08:49.667+0300 7f2a208b6700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -130> 2022-04-12T16:08:49.831+0300 7f28f6662700 10 monclient: tick -129> 2022-04-12T16:08:50.231+0300 7f2a0b08b700 10 monclient: tick -128> 2022-04-12T16:08:50.663+0300 7f2a1f0b3700 10 monclient: tick -127> 2022-04-12T16:08:50.831+0300 7f28f6662700 10 monclient: tick -126> 2022-04-12T16:08:51.231+0300 7f2a0b08b700 10 monclient: tick -125> 2022-04-12T16:08:51.663+0300 7f2a1f0b3700 10 monclient: tick -124> 2022-04-12T16:08:51.831+0300 7f28f6662700 10 monclient: tick -123> 2022-04-12T16:08:52.231+0300 7f2a0b08b700 10 monclient: tick -122> 2022-04-12T16:08:52.663+0300 7f2a1f0b3700 10 monclient: tick -121> 2022-04-12T16:08:52.691+0300 7f29b5fe1700 1 ====== starting new request req=0x7f28f565f620 ===== -120> 2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s initializing for trans_id = tx00000ecd2ee05e4c9a4ca-00625579e4-1659-sync -119> 2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s getting op 4 -118> 2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s :mdlog_notify verifying requester -117> 2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s :mdlog_notify normalizing buckets and tenants -116> 2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s :mdlog_notify init permissions -115> 2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s :mdlog_notify recalculating target -114> 2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s :mdlog_notify reading permissions -113> 2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s :mdlog_notify init op -112> 2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s :mdlog_notify verifying op mask -111> 2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s :mdlog_notify verifying op permissions -110> 2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 overriding permissions due to system operation -109> 2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s :mdlog_notify verifying op params -108> 2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s :mdlog_notify pre-executing -107> 2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s :mdlog_notify executing -106> 2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s :mdlog_notify completing -105> 2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s :mdlog_notify op status=0 -104> 2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s :mdlog_notify http status=200 -103> 2022-04-12T16:08:52.691+0300 7f29b5fe1700 1 ====== req done req=0x7f28f565f620 op status=0 http_status=200 latency=0.000000000s ====== -102> 2022-04-12T16:08:52.691+0300 7f29b5fe1700 1 beast: 0x7f28f565f620: 192.168.166.85 - rgw_zone_user [12/Apr/2022:16:08:52.691 +0300] "POST /admin/log?type=metadata¬ify&rgwx-zonegroup=784a05a5-2089-4564-ab1b-a55c43c5f401 HTTP/1.1" 200 0 - - - latency=0.000000000s -101> 2022-04-12T16:08:52.699+0300 7f2a07884700 4 meta sync: cr:s=0x559e57c2af00:op=0x559e59e17200:18RGWMetaSyncShardCR: adjusting marker pos= -100> 2022-04-12T16:08:52.759+0300 7f2a1409d700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -99> 2022-04-12T16:08:52.759+0300 7f2a200b5700 10 monclient: handle_get_version_reply finishing 5 version 76 -98> 2022-04-12T16:08:52.759+0300 7f2a1409d700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -97> 2022-04-12T16:08:52.831+0300 7f28f6662700 10 monclient: tick -96> 2022-04-12T16:08:52.831+0300 7f28f7e65700 10 monclient: _renew_subs -95> 2022-04-12T16:08:52.831+0300 7f28f7e65700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -94> 2022-04-12T16:08:53.231+0300 7f2a0b08b700 10 monclient: tick -93> 2022-04-12T16:08:53.663+0300 7f2a1f0b3700 10 monclient: tick -92> 2022-04-12T16:08:53.751+0300 7f2a1409d700 10 start_mon_command cmd=[{"prefix": "osd pool application enable","pool": "sync.rgw.buckets.index","app": "rgw"}] -91> 2022-04-12T16:08:53.751+0300 7f2a1409d700 10 monclient: _send_command 3 [{"prefix": "osd pool application enable","pool": "sync.rgw.buckets.index","app": "rgw"}] -90> 2022-04-12T16:08:53.751+0300 7f2a1409d700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -89> 2022-04-12T16:08:53.831+0300 7f28f6662700 10 monclient: tick -88> 2022-04-12T16:08:54.231+0300 7f2a0b08b700 10 monclient: tick -87> 2022-04-12T16:08:54.663+0300 7f2a1f0b3700 10 monclient: tick -86> 2022-04-12T16:08:54.667+0300 7f2a208b6700 10 monclient: _renew_subs -85> 2022-04-12T16:08:54.667+0300 7f2a208b6700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -84> 2022-04-12T16:08:54.759+0300 7f2a200b5700 10 monclient: handle_mon_command_ack 3 [{"prefix": "osd pool application enable","pool": "sync.rgw.buckets.index","app": "rgw"}] -83> 2022-04-12T16:08:54.759+0300 7f2a200b5700 10 monclient: _finish_command 3 = system:0 enabled application 'rgw' on pool 'sync.rgw.buckets.index' -82> 2022-04-12T16:08:54.759+0300 7f2a1409d700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -81> 2022-04-12T16:08:54.763+0300 7f2a200b5700 10 monclient: handle_get_version_reply finishing 6 version 78 -80> 2022-04-12T16:08:54.763+0300 7f2a1409d700 10 start_mon_command cmd=[{"prefix": "osd pool set", "pool": "sync.rgw.buckets.index", "var": "pg_autoscale_bias", "val": "4"}] -79> 2022-04-12T16:08:54.763+0300 7f2a1409d700 10 monclient: _send_command 4 [{"prefix": "osd pool set", "pool": "sync.rgw.buckets.index", "var": "pg_autoscale_bias", "val": "4"}] -78> 2022-04-12T16:08:54.763+0300 7f2a1409d700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -77> 2022-04-12T16:08:54.783+0300 7f2a07884700 4 meta sync: cr:s=0x559e57c2af00:op=0x559e59e17200:18RGWMetaSyncShardCR: adjusting marker pos=1_1649768932.547706_90.1 -76> 2022-04-12T16:08:54.783+0300 7f2a07884700 4 meta sync: cr:s=0x559e57c2af00:op=0x559e59e17200:18RGWMetaSyncShardCR: adjusting marker pos=1_1649768932.547706_90.1 -75> 2022-04-12T16:08:54.831+0300 7f28f6662700 10 monclient: tick -74> 2022-04-12T16:08:55.215+0300 7f2a0e091700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start -73> 2022-04-12T16:08:55.231+0300 7f2a0b08b700 10 monclient: tick -72> 2022-04-12T16:08:55.663+0300 7f2a1f0b3700 10 monclient: tick -71> 2022-04-12T16:08:55.767+0300 7f2a200b5700 10 monclient: handle_mon_command_ack 4 [{"prefix": "osd pool set", "pool": "sync.rgw.buckets.index", "var": "pg_autoscale_bias", "val": "4"}] -70> 2022-04-12T16:08:55.767+0300 7f2a200b5700 10 monclient: _finish_command 4 = system:0 set pool 13 pg_autoscale_bias to 4 -69> 2022-04-12T16:08:55.767+0300 7f2a1409d700 10 start_mon_command cmd=[{"prefix": "osd pool set", "pool": "sync.rgw.buckets.index", "var": "pg_num_min", "val": "8"}] -68> 2022-04-12T16:08:55.767+0300 7f2a1409d700 10 monclient: _send_command 5 [{"prefix": "osd pool set", "pool": "sync.rgw.buckets.index", "var": "pg_num_min", "val": "8"}] -67> 2022-04-12T16:08:55.767+0300 7f2a1409d700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -66> 2022-04-12T16:08:55.831+0300 7f28f6662700 10 monclient: tick -65> 2022-04-12T16:08:56.231+0300 7f2a0b08b700 10 monclient: tick -64> 2022-04-12T16:08:56.663+0300 7f2a1f0b3700 10 monclient: tick -63> 2022-04-12T16:08:56.775+0300 7f2a200b5700 10 monclient: handle_mon_command_ack 5 [{"prefix": "osd pool set", "pool": "sync.rgw.buckets.index", "var": "pg_num_min", "val": "8"}] -62> 2022-04-12T16:08:56.775+0300 7f2a200b5700 10 monclient: _finish_command 5 = system:0 set pool 13 pg_num_min to 8 -61> 2022-04-12T16:08:56.775+0300 7f2a1409d700 10 start_mon_command cmd=[{"prefix": "osd pool set", "pool": "sync.rgw.buckets.index", "var": "recovery_priority": "5"}] -60> 2022-04-12T16:08:56.775+0300 7f2a1409d700 10 monclient: _send_command 6 [{"prefix": "osd pool set", "pool": "sync.rgw.buckets.index", "var": "recovery_priority": "5"}] -59> 2022-04-12T16:08:56.775+0300 7f2a1409d700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -58> 2022-04-12T16:08:56.775+0300 7f2a200b5700 10 monclient: handle_mon_command_ack 6 [{"prefix": "osd pool set", "pool": "sync.rgw.buckets.index", "var": "recovery_priority": "5"}] -57> 2022-04-12T16:08:56.775+0300 7f2a200b5700 10 monclient: _finish_command 6 = mon:22 unparseable JSON {"prefix": "osd pool set", "pool": "sync.rgw.buckets.index", "var": "recovery_priority": "5"} -56> 2022-04-12T16:08:56.831+0300 7f28f6662700 10 monclient: tick -55> 2022-04-12T16:08:57.231+0300 7f2a0b08b700 10 monclient: tick -54> 2022-04-12T16:08:57.663+0300 7f2a1f0b3700 10 monclient: tick -53> 2022-04-12T16:08:57.831+0300 7f28f6662700 10 monclient: tick -52> 2022-04-12T16:08:57.831+0300 7f28f7e65700 10 monclient: _renew_subs -51> 2022-04-12T16:08:57.831+0300 7f28f7e65700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -50> 2022-04-12T16:08:58.231+0300 7f2a0b08b700 10 monclient: tick -49> 2022-04-12T16:08:58.667+0300 7f2a1f0b3700 10 monclient: tick -48> 2022-04-12T16:08:58.831+0300 7f28f6662700 10 monclient: tick -47> 2022-04-12T16:08:59.231+0300 7f2a0b08b700 10 monclient: tick -46> 2022-04-12T16:08:59.667+0300 7f2a1f0b3700 10 monclient: tick -45> 2022-04-12T16:08:59.667+0300 7f2a208b6700 10 monclient: _renew_subs -44> 2022-04-12T16:08:59.667+0300 7f2a208b6700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -43> 2022-04-12T16:08:59.831+0300 7f28f6662700 10 monclient: tick -42> 2022-04-12T16:09:00.231+0300 7f2a0b08b700 10 monclient: tick -41> 2022-04-12T16:09:00.663+0300 7f2a1f0b3700 10 monclient: tick -40> 2022-04-12T16:09:00.831+0300 7f28f6662700 10 monclient: tick -39> 2022-04-12T16:09:01.235+0300 7f2a0b08b700 10 monclient: tick -38> 2022-04-12T16:09:01.663+0300 7f2a1f0b3700 10 monclient: tick -37> 2022-04-12T16:09:01.835+0300 7f28f6662700 10 monclient: tick -36> 2022-04-12T16:09:02.231+0300 7f2a0b08b700 10 monclient: tick -35> 2022-04-12T16:09:02.663+0300 7f2a1f0b3700 10 monclient: tick -34> 2022-04-12T16:09:02.747+0300 7f299a7aa700 1 ====== starting new request req=0x7f28f55de620 ===== -33> 2022-04-12T16:09:02.747+0300 7f299a7aa700 2 req 6236571265947506715 0.000000000s initializing for trans_id = tx00000568cc0860b35981b-00625579ee-1659-sync -32> 2022-04-12T16:09:02.747+0300 7f299a7aa700 2 req 6236571265947506715 0.000000000s getting op 4 -31> 2022-04-12T16:09:02.747+0300 7f299a7aa700 2 req 6236571265947506715 0.000000000s :datalog_notify verifying requester -30> 2022-04-12T16:09:02.751+0300 7f299a7aa700 2 req 6236571265947506715 0.004000004s :datalog_notify normalizing buckets and tenants -29> 2022-04-12T16:09:02.751+0300 7f299a7aa700 2 req 6236571265947506715 0.004000004s :datalog_notify init permissions -28> 2022-04-12T16:09:02.751+0300 7f299a7aa700 2 req 6236571265947506715 0.004000004s :datalog_notify recalculating target -27> 2022-04-12T16:09:02.751+0300 7f299a7aa700 2 req 6236571265947506715 0.004000004s :datalog_notify reading permissions -26> 2022-04-12T16:09:02.751+0300 7f299a7aa700 2 req 6236571265947506715 0.004000004s :datalog_notify init op -25> 2022-04-12T16:09:02.751+0300 7f299a7aa700 2 req 6236571265947506715 0.004000004s :datalog_notify verifying op mask -24> 2022-04-12T16:09:02.751+0300 7f299a7aa700 2 req 6236571265947506715 0.004000004s :datalog_notify verifying op permissions -23> 2022-04-12T16:09:02.751+0300 7f299a7aa700 2 overriding permissions due to system operation -22> 2022-04-12T16:09:02.751+0300 7f299a7aa700 2 req 6236571265947506715 0.004000004s :datalog_notify verifying op params -21> 2022-04-12T16:09:02.751+0300 7f299a7aa700 2 req 6236571265947506715 0.004000004s :datalog_notify pre-executing -20> 2022-04-12T16:09:02.751+0300 7f299a7aa700 2 req 6236571265947506715 0.004000004s :datalog_notify executing -19> 2022-04-12T16:09:02.751+0300 7f299a7aa700 2 req 6236571265947506715 0.004000004s :datalog_notify completing -18> 2022-04-12T16:09:02.751+0300 7f299a7aa700 2 req 6236571265947506715 0.004000004s :datalog_notify op status=0 -17> 2022-04-12T16:09:02.751+0300 7f299a7aa700 2 req 6236571265947506715 0.004000004s :datalog_notify http status=200 -16> 2022-04-12T16:09:02.751+0300 7f299a7aa700 1 ====== req done req=0x7f28f55de620 op status=0 http_status=200 latency=0.004000004s ====== -15> 2022-04-12T16:09:02.751+0300 7f299a7aa700 1 beast: 0x7f28f55de620: 192.168.166.83 - rgw_zone_user [12/Apr/2022:16:09:02.747 +0300] "POST /admin/log?type=data¬ify&source-zone=6fa70b8c-6324-4bee-8dbf-d323e8dc5f43&rgwx-zonegroup=784a05a5-2089-4564-ab1b-a55c43c5f401 HTTP/1.1" 200 0 - - - latency=0.004000004s -14> 2022-04-12T16:09:02.751+0300 7f2a06882700 4 data sync: starting sync on testbucket:6fa70b8c-6324-4bee-8dbf-d323e8dc5f43.5105.1:3 key=testbucket:6fa70b8c-6324-4bee-8dbf-d323e8dc5f43.5105.1:3 -13> 2022-04-12T16:09:02.759+0300 7f2a06882700 5 RGW-SYNC:data:sync:shard[30]:entry:bucket_sync_sources[target=:[]):source_bucket=:[]):source_zone=6fa70b8c-6324-4bee-8dbf-d323e8dc5f43]:bucket[testbucket:6fa70b8c-6324-4bee-8dbf-d323e8dc5f43.5105.1:3<-testbucket:6fa70b8c-6324-4bee-8dbf-d323e8dc5f43.5105.1:3]:full_sync[testbucket:6fa70b8c-6324-4bee-8dbf-d323e8dc5f43.5105.1:3]:entry[amazon_case.png]: bucket sync: sync obj: 6fa70b8c-6324-4bee-8dbf-d323e8dc5f43/:testbucket[6fa70b8c-6324-4bee-8dbf-d323e8dc5f43.5105.1])/amazon_case.png[0] -12> 2022-04-12T16:09:02.759+0300 7f2a06882700 0 e21904192857b447: sync_object: b=:testbucket[6fa70b8c-6324-4bee-8dbf-d323e8dc5f43.5105.1]) k=amazon_case.png versioned_epoch=0 -11> 2022-04-12T16:09:02.767+0300 7f2a06882700 4 rgw rados thread: AWS: download begin: z=6fa70b8c-6324-4bee-8dbf-d323e8dc5f43 b=:testbucket[6fa70b8c-6324-4bee-8dbf-d323e8dc5f43.5105.1]) k=amazon_case.png size=1880884 mtime=2022-04-12T16:09:02.643682+0300 etag=8cd436573b0ee6e9cc4f6f2cb3ba0cf1 zone_short_id=2425654257 pg_ver=0 -10> 2022-04-12T16:09:02.767+0300 7f2a06882700 0 AWS: creating bucket rgw-cdn-e21904192857b447 -9> 2022-04-12T16:09:02.831+0300 7f28f6662700 10 monclient: tick -8> 2022-04-12T16:09:02.831+0300 7f28f7e65700 10 monclient: _renew_subs -7> 2022-04-12T16:09:02.831+0300 7f28f7e65700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -6> 2022-04-12T16:09:03.231+0300 7f2a0b08b700 10 monclient: tick -5> 2022-04-12T16:09:03.663+0300 7f2a1f0b3700 10 monclient: tick -4> 2022-04-12T16:09:03.831+0300 7f28f6662700 10 monclient: tick -3> 2022-04-12T16:09:04.231+0300 7f2a0b08b700 10 monclient: tick -2> 2022-04-12T16:09:04.663+0300 7f2a1f0b3700 10 monclient: tick -1> 2022-04-12T16:09:04.831+0300 7f28f6662700 10 monclient: tick 0> 2022-04-12T16:09:04.839+0300 7f2a06882700 -1 *** Caught signal (Aborted) ** in thread 7f2a06882700 thread_name:data-sync ceph version 16.2.7 (dd0603118f56ab514f133c8d2e3adfc983942503) pacific (stable) 1: /lib64/libpthread.so.0(+0x12c20) [0x7f2a2fa66c20] 2: gsignal() 3: abort() 4: /lib64/libstdc++.so.6(+0x9009b) [0x7f2a2ea5a09b] 5: /lib64/libstdc++.so.6(+0x9653c) [0x7f2a2ea6053c] 6: /lib64/libstdc++.so.6(+0x96597) [0x7f2a2ea60597] 7: /lib64/libstdc++.so.6(+0x967f8) [0x7f2a2ea607f8] 8: /lib64/libstdc++.so.6(+0x92025) [0x7f2a2ea5c025] 9: /lib64/libradosgw.so.2(+0x48b109) [0x7f2a3a5a1109] 10: (std::__cxx11::basic_string, std::allocator >::_M_mutate(unsigned long, unsigned long, char const*, unsigned long)+0x6d) [0x7f2a3a5a471d] 11: (std::__cxx11::basic_string, std::allocator >::_M_append(char const*, unsigned long)+0x6b) [0x7f2a3a5a489b] 12: (rgw_obj_key::get_oid[abi:cxx11]() const+0xc6) [0x7f2a3a7c9066] 13: (RGWRESTStreamS3PutObj::send_init(rgw::sal::RGWObject*)+0xb3) [0x7f2a3aa8f4b3] 14: (RGWRESTConn::put_obj_send_init(rgw::sal::RGWObject*, rgw_http_param_pair const*, RGWRESTStreamS3PutObj**)+0x5c1) [0x7f2a3aa99891] 15: (RGWAWSStreamPutCRF::init()+0xf1) [0x7f2a3a918ec1] 16: (RGWStreamSpliceCR::operate(DoutPrefixProvider const*)+0xa2f) [0x7f2a3a9b567f] 17: (RGWCoroutinesStack::operate(DoutPrefixProvider const*, RGWCoroutinesEnv*)+0x15c) [0x7f2a3a9a0cbc] 18: (RGWCoroutinesManager::run(DoutPrefixProvider const*, std::__cxx11::list >&)+0x296) [0x7f2a3a9a1b06] 19: (RGWCoroutinesManager::run(DoutPrefixProvider const*, RGWCoroutine*)+0x91) [0x7f2a3a9a2dd1] 20: (RGWRemoteDataLog::run_sync(DoutPrefixProvider const*, int)+0x1b4) [0x7f2a3a8b84b4] 21: (RGWDataSyncProcessorThread::process(DoutPrefixProvider const*)+0x59) [0x7f2a3aa6fe09] 22: (RGWRadosThread::Worker::entry()+0x13a) [0x7f2a3aa3368a] 23: /lib64/libpthread.so.0(+0x817a) [0x7f2a2fa5c17a] 24: clone() NOTE: a copy of the executable, or `objdump -rdS ` is needed to interpret this. --- logging levels --- 0/ 5 none 0/ 1 lockdep 0/ 1 context 1/ 1 crush 1/ 5 mds 1/ 5 mds_balancer 1/ 5 mds_locker 1/ 5 mds_log 1/ 5 mds_log_expire 1/ 5 mds_migrator 0/ 1 buffer 0/ 1 timer 0/ 1 filer 0/ 1 striper 0/ 1 objecter 0/ 5 rados 0/ 5 rbd 0/ 5 rbd_mirror 0/ 5 rbd_replay 0/ 5 rbd_pwl 0/ 5 journaler 0/ 5 objectcacher 0/ 5 immutable_obj_cache 0/ 5 client 1/ 5 osd 0/ 5 optracker 0/ 5 objclass 1/ 3 filestore 1/ 3 journal 0/ 0 ms 1/ 5 mon 0/10 monc 1/ 5 paxos 0/ 5 tp 1/ 5 auth 1/ 5 crypto 1/ 1 finisher 1/ 1 reserver 1/ 5 heartbeatmap 1/ 5 perfcounter 5/ 5 rgw 5/ 5 rgw_sync 1/10 civetweb 1/ 5 javaclient 1/ 5 asok 1/ 1 throttle 0/ 0 refs 1/ 5 compressor 1/ 5 bluestore 1/ 5 bluefs 1/ 3 bdev 1/ 5 kstore 4/ 5 rocksdb 4/ 5 leveldb 4/ 5 memdb 1/ 5 fuse 2/ 5 mgr 1/ 5 mgrc 1/ 5 dpdk 1/ 5 eventtrace 1/ 5 prioritycache 0/ 5 test 0/ 5 cephfs_mirror 0/ 5 cephsqlite -2/-2 (syslog threshold) 99/99 (stderr threshold) --- pthread ID / name mapping for recent threads --- 139813909309184 / safe_timer 139813926094592 / ms_dispatch 139813934487296 / ceph_timer 139813951272704 / io_context_pool 139816662116096 / radosgw 139816729257728 / radosgw 139816846755584 / radosgw 139816964253440 / radosgw 139817123714816 / radosgw 139817425852160 / radosgw 139817501386496 / radosgw 139817937807104 / radosgw 139818063697664 / radosgw 139818105661184 / radosgw 139818147624704 / radosgw 139818181195520 / radosgw 139818197980928 / radosgw 139818231551744 / radosgw 139818256729856 / radosgw 139818315478784 / lifecycle_thr_2 139818349049600 / lifecycle_thr_1 139818382620416 / lifecycle_thr_0 139818474940160 / data-sync 139818491725568 / meta-sync 139818525296384 / rgw_obj_expirer 139818533689088 / rgw_gc 139818550474496 / safe_timer 139818567259904 / ms_dispatch 139818592438016 / io_context_pool 139818600830720 / rgw_dt_lg_renew 139818701543168 / rados_async 139818886182656 / safe_timer 139818902968064 / ms_dispatch 139818911360768 / ceph_timer 139818919753472 / io_context_pool 139818928146176 / io_context_pool 139818986895104 / admin_socket 139818995287808 / service 139819003680512 / msgr-worker-2 139819012073216 / msgr-worker-1 139819020465920 / msgr-worker-0 139819359798272 / radosgw max_recent 10000 max_new 10000 log_file --- end dump of recent events --- --- begin dump of recent events --- -1216> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command assert hook 0x559e54a2a760 -1215> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command abort hook 0x559e54a2a760 -1214> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command leak_some_memory hook 0x559e54a2a760 -1213> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command perfcounters_dump hook 0x559e54a2a760 -1212> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command 1 hook 0x559e54a2a760 -1211> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command perf dump hook 0x559e54a2a760 -1210> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command perfcounters_schema hook 0x559e54a2a760 -1209> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command perf histogram dump hook 0x559e54a2a760 -1208> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command 2 hook 0x559e54a2a760 -1207> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command perf schema hook 0x559e54a2a760 -1206> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command perf histogram schema hook 0x559e54a2a760 -1205> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command perf reset hook 0x559e54a2a760 -1204> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command config show hook 0x559e54a2a760 -1203> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command config help hook 0x559e54a2a760 -1202> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command config set hook 0x559e54a2a760 -1201> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command config unset hook 0x559e54a2a760 -1200> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command config get hook 0x559e54a2a760 -1199> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command config diff hook 0x559e54a2a760 -1198> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command config diff get hook 0x559e54a2a760 -1197> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command injectargs hook 0x559e54a2a760 -1196> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command log flush hook 0x559e54a2a760 -1195> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command log dump hook 0x559e54a2a760 -1194> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command log reopen hook 0x559e54a2a760 -1193> 2022-04-12T16:07:44.639+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command dump_mempools hook 0x559e54b02068 -1192> 2022-04-12T16:07:44.643+0300 7f2a3b460400 10 monclient: get_monmap_and_config -1191> 2022-04-12T16:07:44.643+0300 7f2a3b460400 10 monclient: build_initial_monmap -1190> 2022-04-12T16:07:44.643+0300 7f2a3b460400 10 monclient: monmap: epoch 0 fsid 65ccd25a-39ce-4072-a5c6-a0e6654bda87 last_changed 2022-04-12T16:07:44.650033+0300 created 2022-04-12T16:07:44.650033+0300 min_mon_release 0 (unknown) election_strategy: 1 0: [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] mon.noname-a 1: [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] mon.noname-b 2: [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] mon.noname-c -1189> 2022-04-12T16:07:44.643+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding auth protocol: cephx -1188> 2022-04-12T16:07:44.643+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding auth protocol: cephx -1187> 2022-04-12T16:07:44.643+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding auth protocol: cephx -1186> 2022-04-12T16:07:44.643+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding auth protocol: none -1185> 2022-04-12T16:07:44.643+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: secure -1184> 2022-04-12T16:07:44.643+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: crc -1183> 2022-04-12T16:07:44.643+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: secure -1182> 2022-04-12T16:07:44.643+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: crc -1181> 2022-04-12T16:07:44.643+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: secure -1180> 2022-04-12T16:07:44.643+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: crc -1179> 2022-04-12T16:07:44.643+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: crc -1178> 2022-04-12T16:07:44.643+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: secure -1177> 2022-04-12T16:07:44.643+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: crc -1176> 2022-04-12T16:07:44.643+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: secure -1175> 2022-04-12T16:07:44.643+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: crc -1174> 2022-04-12T16:07:44.643+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: secure -1173> 2022-04-12T16:07:44.647+0300 7f2a3b460400 2 auth: KeyRing::load: loaded key file /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring -1172> 2022-04-12T16:07:44.647+0300 7f2a3b460400 10 monclient: init -1171> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d65c0) adding auth protocol: cephx -1170> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d65c0) adding auth protocol: cephx -1169> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d65c0) adding auth protocol: cephx -1168> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d65c0) adding auth protocol: none -1167> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d65c0) adding con mode: secure -1166> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d65c0) adding con mode: crc -1165> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d65c0) adding con mode: secure -1164> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d65c0) adding con mode: crc -1163> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d65c0) adding con mode: secure -1162> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d65c0) adding con mode: crc -1161> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d65c0) adding con mode: crc -1160> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d65c0) adding con mode: secure -1159> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d65c0) adding con mode: crc -1158> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d65c0) adding con mode: secure -1157> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d65c0) adding con mode: crc -1156> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d65c0) adding con mode: secure -1155> 2022-04-12T16:07:44.647+0300 7f2a3b460400 2 auth: KeyRing::load: loaded key file /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring -1154> 2022-04-12T16:07:44.647+0300 7f2a3b460400 2 auth: KeyRing::load: loaded key file /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring -1153> 2022-04-12T16:07:44.647+0300 7f2a3b460400 10 monclient: _reopen_session rank -1 -1152> 2022-04-12T16:07:44.647+0300 7f2a3b460400 10 monclient: _add_conns ranks=[1,2,0] -1151> 2022-04-12T16:07:44.647+0300 7f2a3b460400 10 monclient(hunting): picked mon.noname-b con 0x559e54b38800 addr [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] -1150> 2022-04-12T16:07:44.647+0300 7f2a3b460400 10 monclient(hunting): picked mon.noname-c con 0x559e54b38c00 addr [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] -1149> 2022-04-12T16:07:44.647+0300 7f2a3b460400 10 monclient(hunting): picked mon.noname-a con 0x559e54b39000 addr [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] -1148> 2022-04-12T16:07:44.647+0300 7f2a3b460400 10 monclient(hunting): start opening mon connection -1147> 2022-04-12T16:07:44.647+0300 7f2a3b460400 10 monclient(hunting): start opening mon connection -1146> 2022-04-12T16:07:44.647+0300 7f2a3b460400 10 monclient(hunting): start opening mon connection -1145> 2022-04-12T16:07:44.647+0300 7f2a3b460400 10 monclient(hunting): _renew_subs -1144> 2022-04-12T16:07:44.647+0300 7f2a3b460400 10 monclient(hunting): authenticate will time out at 2022-04-12T16:12:44.651302+0300 -1143> 2022-04-12T16:07:44.647+0300 7f2a270c3700 10 monclient(hunting): get_auth_request con 0x559e54b39000 auth_method 0 -1142> 2022-04-12T16:07:44.647+0300 7f2a270c3700 10 monclient(hunting): get_auth_request method 2 preferred_modes [2,1] -1141> 2022-04-12T16:07:44.647+0300 7f2a270c3700 10 monclient(hunting): _init_auth method 2 -1140> 2022-04-12T16:07:44.647+0300 7f2a270c3700 10 monclient(hunting): _init_auth creating new auth -1139> 2022-04-12T16:07:44.647+0300 7f2a270c3700 10 monclient(hunting): handle_auth_reply_more payload 9 -1138> 2022-04-12T16:07:44.647+0300 7f2a270c3700 10 monclient(hunting): handle_auth_reply_more payload_len 9 -1137> 2022-04-12T16:07:44.647+0300 7f2a270c3700 10 monclient(hunting): handle_auth_reply_more responding with 36 bytes -1136> 2022-04-12T16:07:44.647+0300 7f2a270c3700 10 monclient(hunting): handle_auth_done global_id 5715 payload 322 -1135> 2022-04-12T16:07:44.647+0300 7f2a270c3700 10 monclient: _finish_hunting 0 -1134> 2022-04-12T16:07:44.647+0300 7f2a270c3700 1 monclient: found mon.noname-a -1133> 2022-04-12T16:07:44.647+0300 7f2a270c3700 10 monclient: _send_mon_message to mon.noname-a at v2:192.168.166.83:3300/0 -1132> 2022-04-12T16:07:44.647+0300 7f2a258c0700 10 monclient: handle_monmap mon_map magic: 0 v1 -1131> 2022-04-12T16:07:44.647+0300 7f2a258c0700 10 monclient: got monmap 1 from mon.noname-a (according to old e1) -1130> 2022-04-12T16:07:44.647+0300 7f2a258c0700 10 monclient: dump: epoch 1 fsid 65ccd25a-39ce-4072-a5c6-a0e6654bda87 last_changed 2022-04-12T15:48:27.663651+0300 created 2022-04-12T15:48:27.663651+0300 min_mon_release 16 (pacific) election_strategy: 1 0: [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] mon.ceph-ss-test-oct-controller-av-01 1: [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] mon.ceph-ss-test-oct-controller-av-02 2: [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] mon.ceph-ss-test-oct-controller-av-03 -1129> 2022-04-12T16:07:44.647+0300 7f2a258c0700 10 monclient: _finish_auth 0 -1128> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 monclient: authenticate success, global_id 5715 -1127> 2022-04-12T16:07:44.647+0300 7f2a258c0700 10 monclient: handle_config config(0 keys) v1 -1126> 2022-04-12T16:07:44.647+0300 7f2a3b460400 10 monclient: get_monmap_and_config success -1125> 2022-04-12T16:07:44.647+0300 7f2a3b460400 4 set_mon_vals no callback set -1124> 2022-04-12T16:07:44.647+0300 7f2a3b460400 10 monclient: shutdown -1123> 2022-04-12T16:07:44.647+0300 7f2a3b460400 0 deferred set uid:gid to 167:167 (ceph:ceph) -1122> 2022-04-12T16:07:44.647+0300 7f2a3b460400 0 ceph version 16.2.7 (dd0603118f56ab514f133c8d2e3adfc983942503) pacific (stable), process radosgw, pid 54 -1121> 2022-04-12T16:07:44.647+0300 7f2a3b460400 0 framework: beast -1120> 2022-04-12T16:07:44.647+0300 7f2a3b460400 0 framework conf key: endpoint, val: 192.168.166.83:8081 -1119> 2022-04-12T16:07:44.647+0300 7f2a3b460400 1 radosgw_Main not setting numa affinity -1118> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 asok(0x559e54b34000) init /var/run/ceph/av-ceph-test-client.rgw.ceph-ss-test-oct-controller-av-01.rgw1.54.94138514240960.asok -1117> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 asok(0x559e54b34000) bind_and_listen /var/run/ceph/av-ceph-test-client.rgw.ceph-ss-test-oct-controller-av-01.rgw1.54.94138514240960.asok -1116> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command 0 hook 0x559e54a6c360 -1115> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command version hook 0x559e54a6c360 -1114> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command git_version hook 0x559e54a6c360 -1113> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command help hook 0x559e54a2a4b0 -1112> 2022-04-12T16:07:44.647+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command get_command_descriptions hook 0x559e54a2a4c0 -1111> 2022-04-12T16:07:44.651+0300 7f2a250bf700 5 asok(0x559e54b34000) entry start -1110> 2022-04-12T16:07:44.651+0300 7f2a3b460400 10 monclient: get_monmap_and_config -1109> 2022-04-12T16:07:44.651+0300 7f2a3b460400 10 monclient: build_initial_monmap -1108> 2022-04-12T16:07:44.651+0300 7f2a3b460400 10 monclient: monmap: epoch 0 fsid 00000000-0000-0000-0000-000000000000 last_changed 0.000000 created 0.000000 min_mon_release 0 (unknown) election_strategy: 1 0: [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] mon.noname-a 1: [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] mon.noname-b 2: [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] mon.noname-c -1107> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding auth protocol: cephx -1106> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding auth protocol: cephx -1105> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding auth protocol: cephx -1104> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding auth protocol: none -1103> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: secure -1102> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: crc -1101> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: secure -1100> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: crc -1099> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: secure -1098> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: crc -1097> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: crc -1096> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: secure -1095> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: crc -1094> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: secure -1093> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: crc -1092> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: secure -1091> 2022-04-12T16:07:44.651+0300 7f2a3b460400 2 auth: KeyRing::load: loaded key file /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring -1090> 2022-04-12T16:07:44.651+0300 7f2a3b460400 10 monclient: init -1089> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d6fa0) adding auth protocol: cephx -1088> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d6fa0) adding auth protocol: cephx -1087> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d6fa0) adding auth protocol: cephx -1086> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d6fa0) adding auth protocol: none -1085> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d6fa0) adding con mode: secure -1084> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d6fa0) adding con mode: crc -1083> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d6fa0) adding con mode: secure -1082> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d6fa0) adding con mode: crc -1081> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d6fa0) adding con mode: secure -1080> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d6fa0) adding con mode: crc -1079> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d6fa0) adding con mode: crc -1078> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d6fa0) adding con mode: secure -1077> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d6fa0) adding con mode: crc -1076> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d6fa0) adding con mode: secure -1075> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d6fa0) adding con mode: crc -1074> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d6fa0) adding con mode: secure -1073> 2022-04-12T16:07:44.651+0300 7f2a3b460400 2 auth: KeyRing::load: loaded key file /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring -1072> 2022-04-12T16:07:44.651+0300 7f2a3b460400 2 auth: KeyRing::load: loaded key file /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring -1071> 2022-04-12T16:07:44.651+0300 7f2a3b460400 10 monclient: _reopen_session rank -1 -1070> 2022-04-12T16:07:44.651+0300 7f2a3b460400 10 monclient: _add_conns ranks=[0,1,2] -1069> 2022-04-12T16:07:44.651+0300 7f2a3b460400 10 monclient(hunting): picked mon.noname-a con 0x559e54b39000 addr [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] -1068> 2022-04-12T16:07:44.651+0300 7f2a3b460400 10 monclient(hunting): picked mon.noname-b con 0x559e54b38c00 addr [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] -1067> 2022-04-12T16:07:44.651+0300 7f2a3b460400 10 monclient(hunting): picked mon.noname-c con 0x559e54b38800 addr [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] -1066> 2022-04-12T16:07:44.651+0300 7f2a3b460400 10 monclient(hunting): start opening mon connection -1065> 2022-04-12T16:07:44.651+0300 7f2a3b460400 10 monclient(hunting): start opening mon connection -1064> 2022-04-12T16:07:44.651+0300 7f2a3b460400 10 monclient(hunting): start opening mon connection -1063> 2022-04-12T16:07:44.651+0300 7f2a3b460400 10 monclient(hunting): _renew_subs -1062> 2022-04-12T16:07:44.651+0300 7f2a3b460400 10 monclient(hunting): authenticate will time out at 2022-04-12T16:12:44.656631+0300 -1061> 2022-04-12T16:07:44.651+0300 7f2a268c2700 10 monclient(hunting): get_auth_request con 0x559e54b39000 auth_method 0 -1060> 2022-04-12T16:07:44.651+0300 7f2a268c2700 10 monclient(hunting): get_auth_request method 2 preferred_modes [2,1] -1059> 2022-04-12T16:07:44.651+0300 7f2a268c2700 10 monclient(hunting): _init_auth method 2 -1058> 2022-04-12T16:07:44.651+0300 7f2a268c2700 10 monclient(hunting): _init_auth creating new auth -1057> 2022-04-12T16:07:44.651+0300 7f2a268c2700 10 monclient(hunting): handle_auth_reply_more payload 9 -1056> 2022-04-12T16:07:44.651+0300 7f2a268c2700 10 monclient(hunting): handle_auth_reply_more payload_len 9 -1055> 2022-04-12T16:07:44.651+0300 7f2a268c2700 10 monclient(hunting): handle_auth_reply_more responding with 36 bytes -1054> 2022-04-12T16:07:44.651+0300 7f2a268c2700 10 monclient(hunting): handle_auth_done global_id 5718 payload 322 -1053> 2022-04-12T16:07:44.651+0300 7f2a268c2700 10 monclient: _finish_hunting 0 -1052> 2022-04-12T16:07:44.651+0300 7f2a268c2700 1 monclient: found mon.noname-a -1051> 2022-04-12T16:07:44.651+0300 7f2a268c2700 10 monclient: _send_mon_message to mon.noname-a at v2:192.168.166.83:3300/0 -1050> 2022-04-12T16:07:44.651+0300 7f2a218b8700 10 monclient: handle_monmap mon_map magic: 0 v1 -1049> 2022-04-12T16:07:44.651+0300 7f2a218b8700 10 monclient: got monmap 1 from mon.noname-a (according to old e1) -1048> 2022-04-12T16:07:44.651+0300 7f2a218b8700 10 monclient: dump: epoch 1 fsid 65ccd25a-39ce-4072-a5c6-a0e6654bda87 last_changed 2022-04-12T15:48:27.663651+0300 created 2022-04-12T15:48:27.663651+0300 min_mon_release 16 (pacific) election_strategy: 1 0: [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] mon.ceph-ss-test-oct-controller-av-01 1: [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] mon.ceph-ss-test-oct-controller-av-02 2: [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] mon.ceph-ss-test-oct-controller-av-03 -1047> 2022-04-12T16:07:44.651+0300 7f2a218b8700 10 monclient: _finish_auth 0 -1046> 2022-04-12T16:07:44.651+0300 7f2a3b460400 5 monclient: authenticate success, global_id 5718 -1045> 2022-04-12T16:07:44.651+0300 7f2a218b8700 10 monclient: handle_config config(0 keys) v1 -1044> 2022-04-12T16:07:44.651+0300 7f2a3b460400 10 monclient: get_monmap_and_config success -1043> 2022-04-12T16:07:44.651+0300 7f2a3b460400 4 set_mon_vals no callback set -1042> 2022-04-12T16:07:44.651+0300 7f2a3b460400 10 monclient: shutdown -1041> 2022-04-12T16:07:44.655+0300 7f2a3b460400 10 monclient: build_initial_monmap -1040> 2022-04-12T16:07:44.655+0300 7f2a3b460400 10 monclient: monmap: epoch 0 fsid 00000000-0000-0000-0000-000000000000 last_changed 0.000000 created 0.000000 min_mon_release 0 (unknown) election_strategy: 1 0: [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] mon.noname-a 1: [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] mon.noname-b 2: [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] mon.noname-c -1039> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding auth protocol: cephx -1038> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding auth protocol: cephx -1037> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding auth protocol: cephx -1036> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding auth protocol: none -1035> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: secure -1034> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: crc -1033> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: secure -1032> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: crc -1031> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: secure -1030> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: crc -1029> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: crc -1028> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: secure -1027> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: crc -1026> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: secure -1025> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: crc -1024> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae0a40) adding con mode: secure -1023> 2022-04-12T16:07:44.655+0300 7f2a3b460400 2 auth: KeyRing::load: loaded key file /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring -1022> 2022-04-12T16:07:44.655+0300 7f2a3b460400 1 librados: starting msgr at -1021> 2022-04-12T16:07:44.655+0300 7f2a3b460400 1 librados: starting objecter -1020> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command objecter_requests hook 0x559e54a2a510 -1019> 2022-04-12T16:07:44.655+0300 7f2a3b460400 1 librados: setting wanted keys -1018> 2022-04-12T16:07:44.655+0300 7f2a3b460400 1 librados: calling monclient init -1017> 2022-04-12T16:07:44.655+0300 7f2a3b460400 10 monclient: init -1016> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54a700d8) adding auth protocol: cephx -1015> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54a700d8) adding auth protocol: cephx -1014> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54a700d8) adding auth protocol: cephx -1013> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54a700d8) adding auth protocol: none -1012> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54a700d8) adding con mode: secure -1011> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54a700d8) adding con mode: crc -1010> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54a700d8) adding con mode: secure -1009> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54a700d8) adding con mode: crc -1008> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54a700d8) adding con mode: secure -1007> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54a700d8) adding con mode: crc -1006> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54a700d8) adding con mode: crc -1005> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54a700d8) adding con mode: secure -1004> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54a700d8) adding con mode: crc -1003> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54a700d8) adding con mode: secure -1002> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54a700d8) adding con mode: crc -1001> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 AuthRegistry(0x559e54a700d8) adding con mode: secure -1000> 2022-04-12T16:07:44.655+0300 7f2a3b460400 2 auth: KeyRing::load: loaded key file /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring -999> 2022-04-12T16:07:44.655+0300 7f2a3b460400 2 auth: KeyRing::load: loaded key file /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring -998> 2022-04-12T16:07:44.655+0300 7f2a3b460400 10 monclient: _reopen_session rank -1 -997> 2022-04-12T16:07:44.655+0300 7f2a3b460400 10 monclient: _add_conns ranks=[0,2,1] -996> 2022-04-12T16:07:44.655+0300 7f2a3b460400 10 monclient(hunting): picked mon.noname-a con 0x559e54b38800 addr [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] -995> 2022-04-12T16:07:44.655+0300 7f2a3b460400 10 monclient(hunting): picked mon.noname-c con 0x559e54b38c00 addr [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] -994> 2022-04-12T16:07:44.655+0300 7f2a3b460400 10 monclient(hunting): picked mon.noname-b con 0x559e54b39000 addr [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] -993> 2022-04-12T16:07:44.655+0300 7f2a3b460400 10 monclient(hunting): start opening mon connection -992> 2022-04-12T16:07:44.655+0300 7f2a3b460400 10 monclient(hunting): start opening mon connection -991> 2022-04-12T16:07:44.655+0300 7f2a3b460400 10 monclient(hunting): start opening mon connection -990> 2022-04-12T16:07:44.655+0300 7f2a3b460400 10 monclient(hunting): _renew_subs -989> 2022-04-12T16:07:44.655+0300 7f2a3b460400 10 monclient(hunting): authenticate will time out at 2022-04-12T16:12:44.659081+0300 -988> 2022-04-12T16:07:44.655+0300 7f2a268c2700 10 monclient(hunting): get_auth_request con 0x559e54b38800 auth_method 0 -987> 2022-04-12T16:07:44.655+0300 7f2a268c2700 10 monclient(hunting): get_auth_request method 2 preferred_modes [2,1] -986> 2022-04-12T16:07:44.655+0300 7f2a268c2700 10 monclient(hunting): _init_auth method 2 -985> 2022-04-12T16:07:44.655+0300 7f2a268c2700 10 monclient(hunting): _init_auth creating new auth -984> 2022-04-12T16:07:44.655+0300 7f2a268c2700 10 monclient(hunting): handle_auth_reply_more payload 9 -983> 2022-04-12T16:07:44.655+0300 7f2a268c2700 10 monclient(hunting): handle_auth_reply_more payload_len 9 -982> 2022-04-12T16:07:44.655+0300 7f2a268c2700 10 monclient(hunting): handle_auth_reply_more responding with 36 bytes -981> 2022-04-12T16:07:44.655+0300 7f2a268c2700 10 monclient(hunting): handle_auth_done global_id 5721 payload 1000 -980> 2022-04-12T16:07:44.655+0300 7f2a268c2700 10 monclient: _finish_hunting 0 -979> 2022-04-12T16:07:44.655+0300 7f2a268c2700 1 monclient: found mon.noname-a -978> 2022-04-12T16:07:44.655+0300 7f2a268c2700 10 monclient: _send_mon_message to mon.noname-a at v2:192.168.166.83:3300/0 -977> 2022-04-12T16:07:44.655+0300 7f2a200b5700 10 monclient: handle_monmap mon_map magic: 0 v1 -976> 2022-04-12T16:07:44.655+0300 7f2a200b5700 10 monclient: got monmap 1 from mon.noname-a (according to old e1) -975> 2022-04-12T16:07:44.655+0300 7f2a200b5700 10 monclient: dump: epoch 1 fsid 65ccd25a-39ce-4072-a5c6-a0e6654bda87 last_changed 2022-04-12T15:48:27.663651+0300 created 2022-04-12T15:48:27.663651+0300 min_mon_release 16 (pacific) election_strategy: 1 0: [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] mon.ceph-ss-test-oct-controller-av-01 1: [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] mon.ceph-ss-test-oct-controller-av-02 2: [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] mon.ceph-ss-test-oct-controller-av-03 -974> 2022-04-12T16:07:44.655+0300 7f2a200b5700 10 monclient: _finish_auth 0 -973> 2022-04-12T16:07:44.655+0300 7f2a200b5700 10 monclient: handle_config config(0 keys) v1 -972> 2022-04-12T16:07:44.655+0300 7f2a200b5700 10 monclient: handle_monmap mon_map magic: 0 v1 -971> 2022-04-12T16:07:44.655+0300 7f2a218b8700 4 set_mon_vals no callback set -970> 2022-04-12T16:07:44.655+0300 7f2a200b5700 10 monclient: got monmap 1 from mon.ceph-ss-test-oct-controller-av-01 (according to old e1) -969> 2022-04-12T16:07:44.655+0300 7f2a200b5700 10 monclient: dump: epoch 1 fsid 65ccd25a-39ce-4072-a5c6-a0e6654bda87 last_changed 2022-04-12T15:48:27.663651+0300 created 2022-04-12T15:48:27.663651+0300 min_mon_release 16 (pacific) election_strategy: 1 0: [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] mon.ceph-ss-test-oct-controller-av-01 1: [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] mon.ceph-ss-test-oct-controller-av-02 2: [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] mon.ceph-ss-test-oct-controller-av-03 -968> 2022-04-12T16:07:44.655+0300 7f2a3b460400 5 monclient: authenticate success, global_id 5721 -967> 2022-04-12T16:07:44.655+0300 7f2a3b460400 10 monclient: _renew_subs -966> 2022-04-12T16:07:44.655+0300 7f2a3b460400 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -965> 2022-04-12T16:07:44.655+0300 7f2a3b460400 10 monclient: _renew_subs -964> 2022-04-12T16:07:44.655+0300 7f2a3b460400 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -963> 2022-04-12T16:07:44.655+0300 7f2a3b460400 1 librados: init done -962> 2022-04-12T16:07:44.659+0300 7f2a200b5700 4 mgrc handle_mgr_map Got map version 29 -961> 2022-04-12T16:07:44.659+0300 7f2a200b5700 4 mgrc handle_mgr_map Active mgr is now [v2:192.168.166.84:6800/56,v1:192.168.166.84:6801/56] -960> 2022-04-12T16:07:44.659+0300 7f2a200b5700 4 mgrc reconnect Starting new session with [v2:192.168.166.84:6800/56,v1:192.168.166.84:6801/56] -959> 2022-04-12T16:07:44.659+0300 7f2a260c1700 10 monclient: get_auth_request con 0x559e54b39800 auth_method 0 -958> 2022-04-12T16:07:44.659+0300 7f2a270c3700 10 monclient: get_auth_request con 0x559e54b38400 auth_method 0 -957> 2022-04-12T16:07:44.659+0300 7f2a268c2700 10 monclient: get_auth_request con 0x559e55914000 auth_method 0 -956> 2022-04-12T16:07:44.663+0300 7f2a260c1700 10 monclient: get_auth_request con 0x559e55914c00 auth_method 0 -955> 2022-04-12T16:07:44.663+0300 7f2a270c3700 10 monclient: get_auth_request con 0x559e55915800 auth_method 0 -954> 2022-04-12T16:07:44.667+0300 7f2a268c2700 10 monclient: get_auth_request con 0x559e55914800 auth_method 0 -953> 2022-04-12T16:07:44.667+0300 7f2a260c1700 10 monclient: get_auth_request con 0x559e55914400 auth_method 0 -952> 2022-04-12T16:07:44.671+0300 7f2a270c3700 10 monclient: get_auth_request con 0x559e55915c00 auth_method 0 -951> 2022-04-12T16:07:44.671+0300 7f2a268c2700 10 monclient: get_auth_request con 0x559e5591c000 auth_method 0 -950> 2022-04-12T16:07:44.671+0300 7f2a260c1700 10 monclient: get_auth_request con 0x559e55915400 auth_method 0 -949> 2022-04-12T16:07:44.675+0300 7f2a270c3700 10 monclient: get_auth_request con 0x559e55915000 auth_method 0 -948> 2022-04-12T16:07:44.675+0300 7f2a268c2700 10 monclient: get_auth_request con 0x559e5591c400 auth_method 0 -947> 2022-04-12T16:07:44.675+0300 7f2a260c1700 10 monclient: get_auth_request con 0x559e5591c800 auth_method 0 -946> 2022-04-12T16:07:44.679+0300 7f2a270c3700 10 monclient: get_auth_request con 0x559e5591d000 auth_method 0 -945> 2022-04-12T16:07:44.679+0300 7f2a268c2700 10 monclient: get_auth_request con 0x559e5591d400 auth_method 0 -944> 2022-04-12T16:07:44.679+0300 7f2a260c1700 10 monclient: get_auth_request con 0x559e5591cc00 auth_method 0 -943> 2022-04-12T16:07:44.683+0300 7f2a270c3700 10 monclient: get_auth_request con 0x559e5591d800 auth_method 0 -942> 2022-04-12T16:07:44.687+0300 7f2a268c2700 10 monclient: get_auth_request con 0x559e55978000 auth_method 0 -941> 2022-04-12T16:07:44.691+0300 7f2a3b460400 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -940> 2022-04-12T16:07:44.691+0300 7f2a200b5700 10 monclient: handle_get_version_reply finishing 1 version 64 -939> 2022-04-12T16:07:44.691+0300 7f2a3b460400 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -938> 2022-04-12T16:07:45.539+0300 7f2a200b5700 10 monclient: _renew_subs -937> 2022-04-12T16:07:45.539+0300 7f2a200b5700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -936> 2022-04-12T16:07:45.539+0300 7f2a3b460400 10 start_mon_command cmd=[{"prefix": "osd pool application enable","pool": "sync.rgw.log","app": "rgw"}] -935> 2022-04-12T16:07:45.539+0300 7f2a3b460400 10 monclient: _send_command 1 [{"prefix": "osd pool application enable","pool": "sync.rgw.log","app": "rgw"}] -934> 2022-04-12T16:07:45.539+0300 7f2a3b460400 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -933> 2022-04-12T16:07:46.559+0300 7f2a200b5700 10 monclient: handle_mon_command_ack 1 [{"prefix": "osd pool application enable","pool": "sync.rgw.log","app": "rgw"}] -932> 2022-04-12T16:07:46.559+0300 7f2a200b5700 10 monclient: _finish_command 1 = system:0 enabled application 'rgw' on pool 'sync.rgw.log' -931> 2022-04-12T16:07:46.559+0300 7f2a3b460400 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -930> 2022-04-12T16:07:46.559+0300 7f2a200b5700 10 monclient: handle_get_version_reply finishing 2 version 66 -929> 2022-04-12T16:07:46.559+0300 7f2a218b8700 10 monclient: _renew_subs -928> 2022-04-12T16:07:46.559+0300 7f2a218b8700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -927> 2022-04-12T16:07:46.835+0300 7f2a3b460400 5 sync module config (parsed representation): { "config": { "source_bucket": "", "target_path": "rgw-${zonegroup}-${sid}/${bucket}", "connection_id": "", "acls_id": "", "connection": { "id": "", "endpoint": "https://storage.googleapis.com", "host_style": "path", "key": { "access_key": "", "secret": "******" } } }, "connections": [], "acl_profiles": [], "profiles": [ { "name": "", "config": { "source_bucket": "", "target_path": "rgw-${zonegroup}-${sid}/${bucket}", "connection_id": "", "acls_id": "", "connection": { "id": "", "endpoint": "https://storage.googleapis.com", "host_style": "path", "key": { "access_key": "", "secret": "******" } } } } ] } -926> 2022-04-12T16:07:46.835+0300 7f2a3b460400 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -925> 2022-04-12T16:07:46.835+0300 7f2a200b5700 10 monclient: handle_get_version_reply finishing 3 version 66 -924> 2022-04-12T16:07:46.835+0300 7f2a3b460400 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -923> 2022-04-12T16:07:47.563+0300 7f2a200b5700 10 monclient: _renew_subs -922> 2022-04-12T16:07:47.563+0300 7f2a200b5700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -921> 2022-04-12T16:07:47.567+0300 7f2a3b460400 10 start_mon_command cmd=[{"prefix": "osd pool application enable","pool": "sync.rgw.control","app": "rgw"}] -920> 2022-04-12T16:07:47.567+0300 7f2a3b460400 10 monclient: _send_command 2 [{"prefix": "osd pool application enable","pool": "sync.rgw.control","app": "rgw"}] -919> 2022-04-12T16:07:47.567+0300 7f2a3b460400 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -918> 2022-04-12T16:07:47.655+0300 7f2a1f0b3700 10 monclient: tick -917> 2022-04-12T16:07:48.575+0300 7f2a200b5700 10 monclient: handle_mon_command_ack 2 [{"prefix": "osd pool application enable","pool": "sync.rgw.control","app": "rgw"}] -916> 2022-04-12T16:07:48.575+0300 7f2a200b5700 10 monclient: _finish_command 2 = system:0 enabled application 'rgw' on pool 'sync.rgw.control' -915> 2022-04-12T16:07:48.575+0300 7f2a3b460400 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -914> 2022-04-12T16:07:48.579+0300 7f2a200b5700 10 monclient: handle_get_version_reply finishing 4 version 68 -913> 2022-04-12T16:07:48.579+0300 7f2a210b7700 10 monclient: _renew_subs -912> 2022-04-12T16:07:48.579+0300 7f2a210b7700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -911> 2022-04-12T16:07:48.655+0300 7f2a1f0b3700 10 monclient: tick -910> 2022-04-12T16:07:49.163+0300 7f2a3b460400 2 all 8 watchers are set, enabling cache -909> 2022-04-12T16:07:49.163+0300 7f2a3b460400 5 rgw main: tl::expected, obj_version>, boost::system::error_code> logback_generations::read(const DoutPrefixProvider*, optional_yield):409: oid=data_loggenerations_metadata not found -908> 2022-04-12T16:07:49.167+0300 7f2a260c1700 10 monclient: get_auth_request con 0x559e55978400 auth_method 0 -907> 2022-04-12T16:07:49.211+0300 7f2a0e091700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start -906> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d78b0) adding auth protocol: cephx -905> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d78b0) adding auth protocol: cephx -904> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d78b0) adding auth protocol: cephx -903> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d78b0) adding auth protocol: none -902> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d78b0) adding con mode: secure -901> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d78b0) adding con mode: crc -900> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d78b0) adding con mode: secure -899> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d78b0) adding con mode: crc -898> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d78b0) adding con mode: secure -897> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d78b0) adding con mode: crc -896> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d78b0) adding con mode: crc -895> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d78b0) adding con mode: secure -894> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d78b0) adding con mode: crc -893> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d78b0) adding con mode: secure -892> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d78b0) adding con mode: crc -891> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d78b0) adding con mode: secure -890> 2022-04-12T16:07:49.219+0300 7f2a3b460400 2 auth: KeyRing::load: loaded key file /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring -889> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command cache list hook 0x559e54a2aa10 -888> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command cache inspect hook 0x559e54a2aa10 -887> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command cache erase hook 0x559e54a2aa10 -886> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command cache zap hook 0x559e54a2aa10 -885> 2022-04-12T16:07:49.219+0300 7f2a3b460400 10 monclient: get_monmap_and_config -884> 2022-04-12T16:07:49.219+0300 7f2a3b460400 10 monclient: build_initial_monmap -883> 2022-04-12T16:07:49.219+0300 7f2a3b460400 10 monclient: monmap: epoch 0 fsid 00000000-0000-0000-0000-000000000000 last_changed 0.000000 created 0.000000 min_mon_release 0 (unknown) election_strategy: 1 0: [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] mon.noname-a 1: [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] mon.noname-b 2: [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] mon.noname-c -882> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding auth protocol: cephx -881> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding auth protocol: cephx -880> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding auth protocol: cephx -879> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding auth protocol: none -878> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: secure -877> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: crc -876> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: secure -875> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: crc -874> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: secure -873> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: crc -872> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: crc -871> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: secure -870> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: crc -869> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: secure -868> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: crc -867> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: secure -866> 2022-04-12T16:07:49.219+0300 7f2a3b460400 2 auth: KeyRing::load: loaded key file /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring -865> 2022-04-12T16:07:49.219+0300 7f2a3b460400 10 monclient: init -864> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding auth protocol: cephx -863> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding auth protocol: cephx -862> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding auth protocol: cephx -861> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding auth protocol: none -860> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: secure -859> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: crc -858> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: secure -857> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: crc -856> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: secure -855> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: crc -854> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: crc -853> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: secure -852> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: crc -851> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: secure -850> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: crc -849> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: secure -848> 2022-04-12T16:07:49.219+0300 7f2a3b460400 2 auth: KeyRing::load: loaded key file /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring -847> 2022-04-12T16:07:49.219+0300 7f2a3b460400 2 auth: KeyRing::load: loaded key file /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring -846> 2022-04-12T16:07:49.219+0300 7f2a3b460400 10 monclient: _reopen_session rank -1 -845> 2022-04-12T16:07:49.219+0300 7f2a3b460400 10 monclient: _add_conns ranks=[1,0,2] -844> 2022-04-12T16:07:49.219+0300 7f2a3b460400 10 monclient(hunting): picked mon.noname-b con 0x559e55990400 addr [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] -843> 2022-04-12T16:07:49.219+0300 7f2a3b460400 10 monclient(hunting): picked mon.noname-a con 0x559e55991000 addr [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] -842> 2022-04-12T16:07:49.219+0300 7f2a3b460400 10 monclient(hunting): picked mon.noname-c con 0x559e55990c00 addr [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] -841> 2022-04-12T16:07:49.219+0300 7f2a3b460400 10 monclient(hunting): start opening mon connection -840> 2022-04-12T16:07:49.219+0300 7f2a3b460400 10 monclient(hunting): start opening mon connection -839> 2022-04-12T16:07:49.219+0300 7f2a3b460400 10 monclient(hunting): start opening mon connection -838> 2022-04-12T16:07:49.219+0300 7f2a3b460400 10 monclient(hunting): _renew_subs -837> 2022-04-12T16:07:49.219+0300 7f2a3b460400 10 monclient(hunting): authenticate will time out at 2022-04-12T16:12:49.224469+0300 -836> 2022-04-12T16:07:49.219+0300 7f2a260c1700 10 monclient(hunting): get_auth_request con 0x559e55991000 auth_method 0 -835> 2022-04-12T16:07:49.219+0300 7f2a260c1700 10 monclient(hunting): get_auth_request method 2 preferred_modes [2,1] -834> 2022-04-12T16:07:49.219+0300 7f2a260c1700 10 monclient(hunting): _init_auth method 2 -833> 2022-04-12T16:07:49.219+0300 7f2a260c1700 10 monclient(hunting): _init_auth creating new auth -832> 2022-04-12T16:07:49.219+0300 7f2a260c1700 10 monclient(hunting): handle_auth_reply_more payload 9 -831> 2022-04-12T16:07:49.219+0300 7f2a260c1700 10 monclient(hunting): handle_auth_reply_more payload_len 9 -830> 2022-04-12T16:07:49.219+0300 7f2a260c1700 10 monclient(hunting): handle_auth_reply_more responding with 36 bytes -829> 2022-04-12T16:07:49.219+0300 7f2a268c2700 10 monclient(hunting): get_auth_request con 0x559e55990400 auth_method 0 -828> 2022-04-12T16:07:49.219+0300 7f2a268c2700 10 monclient(hunting): get_auth_request method 2 preferred_modes [2,1] -827> 2022-04-12T16:07:49.219+0300 7f2a268c2700 10 monclient(hunting): _init_auth method 2 -826> 2022-04-12T16:07:49.219+0300 7f2a268c2700 10 monclient(hunting): _init_auth creating new auth -825> 2022-04-12T16:07:49.219+0300 7f2a260c1700 10 monclient(hunting): handle_auth_done global_id 5724 payload 322 -824> 2022-04-12T16:07:49.219+0300 7f2a260c1700 10 monclient: _finish_hunting 0 -823> 2022-04-12T16:07:49.219+0300 7f2a260c1700 1 monclient: found mon.noname-a -822> 2022-04-12T16:07:49.219+0300 7f2a260c1700 10 monclient: _send_mon_message to mon.noname-a at v2:192.168.166.83:3300/0 -821> 2022-04-12T16:07:49.219+0300 7f2a0d890700 10 monclient: handle_monmap mon_map magic: 0 v1 -820> 2022-04-12T16:07:49.219+0300 7f2a0d890700 10 monclient: got monmap 1 from mon.noname-a (according to old e1) -819> 2022-04-12T16:07:49.219+0300 7f2a0d890700 10 monclient: dump: epoch 1 fsid 65ccd25a-39ce-4072-a5c6-a0e6654bda87 last_changed 2022-04-12T15:48:27.663651+0300 created 2022-04-12T15:48:27.663651+0300 min_mon_release 16 (pacific) election_strategy: 1 0: [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] mon.ceph-ss-test-oct-controller-av-01 1: [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] mon.ceph-ss-test-oct-controller-av-02 2: [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] mon.ceph-ss-test-oct-controller-av-03 -818> 2022-04-12T16:07:49.219+0300 7f2a0d890700 10 monclient: _finish_auth 0 -817> 2022-04-12T16:07:49.219+0300 7f2a0d890700 10 monclient: handle_config config(0 keys) v1 -816> 2022-04-12T16:07:49.219+0300 7f2a0d890700 10 monclient: handle_monmap mon_map magic: 0 v1 -815> 2022-04-12T16:07:49.219+0300 7f2a0d890700 10 monclient: got monmap 1 from mon.ceph-ss-test-oct-controller-av-01 (according to old e1) -814> 2022-04-12T16:07:49.219+0300 7f2a0d890700 10 monclient: dump: epoch 1 fsid 65ccd25a-39ce-4072-a5c6-a0e6654bda87 last_changed 2022-04-12T15:48:27.663651+0300 created 2022-04-12T15:48:27.663651+0300 min_mon_release 16 (pacific) election_strategy: 1 0: [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] mon.ceph-ss-test-oct-controller-av-01 1: [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] mon.ceph-ss-test-oct-controller-av-02 2: [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] mon.ceph-ss-test-oct-controller-av-03 -813> 2022-04-12T16:07:49.219+0300 7f2a3b460400 5 monclient: authenticate success, global_id 5724 -812> 2022-04-12T16:07:49.219+0300 7f2a3b460400 10 monclient: get_monmap_and_config success -811> 2022-04-12T16:07:49.219+0300 7f2a3b460400 4 set_mon_vals no callback set -810> 2022-04-12T16:07:49.219+0300 7f2a3b460400 10 monclient: shutdown -809> 2022-04-12T16:07:49.223+0300 7f2a3b460400 10 monclient: build_initial_monmap -808> 2022-04-12T16:07:49.223+0300 7f2a3b460400 10 monclient: monmap: epoch 0 fsid 00000000-0000-0000-0000-000000000000 last_changed 0.000000 created 0.000000 min_mon_release 0 (unknown) election_strategy: 1 0: [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] mon.noname-a 1: [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] mon.noname-b 2: [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] mon.noname-c -807> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding auth protocol: cephx -806> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding auth protocol: cephx -805> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding auth protocol: cephx -804> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding auth protocol: none -803> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: secure -802> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: crc -801> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: secure -800> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: crc -799> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: secure -798> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: crc -797> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: crc -796> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: secure -795> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: crc -794> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: secure -793> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: crc -792> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e54ae1340) adding con mode: secure -791> 2022-04-12T16:07:49.223+0300 7f2a3b460400 2 auth: KeyRing::load: loaded key file /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring -790> 2022-04-12T16:07:49.223+0300 7f2a3b460400 1 librados: starting msgr at -789> 2022-04-12T16:07:49.223+0300 7f2a3b460400 1 librados: starting objecter -788> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command objecter_requests cmddesc objecter_requests hook 0x559e54a2ba40 EEXIST -787> 2022-04-12T16:07:49.223+0300 7f2a3b460400 1 librados: setting wanted keys -786> 2022-04-12T16:07:49.223+0300 7f2a3b460400 1 librados: calling monclient init -785> 2022-04-12T16:07:49.223+0300 7f2a3b460400 10 monclient: init -784> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e559dd0d8) adding auth protocol: cephx -783> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e559dd0d8) adding auth protocol: cephx -782> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e559dd0d8) adding auth protocol: cephx -781> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e559dd0d8) adding auth protocol: none -780> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e559dd0d8) adding con mode: secure -779> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e559dd0d8) adding con mode: crc -778> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e559dd0d8) adding con mode: secure -777> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e559dd0d8) adding con mode: crc -776> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e559dd0d8) adding con mode: secure -775> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e559dd0d8) adding con mode: crc -774> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e559dd0d8) adding con mode: crc -773> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e559dd0d8) adding con mode: secure -772> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e559dd0d8) adding con mode: crc -771> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e559dd0d8) adding con mode: secure -770> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e559dd0d8) adding con mode: crc -769> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 AuthRegistry(0x559e559dd0d8) adding con mode: secure -768> 2022-04-12T16:07:49.223+0300 7f2a3b460400 2 auth: KeyRing::load: loaded key file /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring -767> 2022-04-12T16:07:49.223+0300 7f2a3b460400 2 auth: KeyRing::load: loaded key file /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring -766> 2022-04-12T16:07:49.223+0300 7f2a3b460400 10 monclient: _reopen_session rank -1 -765> 2022-04-12T16:07:49.223+0300 7f2a3b460400 10 monclient: _add_conns ranks=[0,2,1] -764> 2022-04-12T16:07:49.223+0300 7f2a3b460400 10 monclient(hunting): picked mon.noname-a con 0x559e55990400 addr [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] -763> 2022-04-12T16:07:49.223+0300 7f2a3b460400 10 monclient(hunting): picked mon.noname-c con 0x559e55990c00 addr [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] -762> 2022-04-12T16:07:49.223+0300 7f2a3b460400 10 monclient(hunting): picked mon.noname-b con 0x559e55991000 addr [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] -761> 2022-04-12T16:07:49.223+0300 7f2a3b460400 10 monclient(hunting): start opening mon connection -760> 2022-04-12T16:07:49.223+0300 7f2a3b460400 10 monclient(hunting): start opening mon connection -759> 2022-04-12T16:07:49.223+0300 7f2a3b460400 10 monclient(hunting): start opening mon connection -758> 2022-04-12T16:07:49.223+0300 7f2a3b460400 10 monclient(hunting): _renew_subs -757> 2022-04-12T16:07:49.223+0300 7f2a3b460400 10 monclient(hunting): authenticate will time out at 2022-04-12T16:12:49.227127+0300 -756> 2022-04-12T16:07:49.223+0300 7f2a268c2700 10 monclient(hunting): get_auth_request con 0x559e55990400 auth_method 0 -755> 2022-04-12T16:07:49.223+0300 7f2a268c2700 10 monclient(hunting): get_auth_request method 2 preferred_modes [2,1] -754> 2022-04-12T16:07:49.223+0300 7f2a268c2700 10 monclient(hunting): _init_auth method 2 -753> 2022-04-12T16:07:49.223+0300 7f2a268c2700 10 monclient(hunting): _init_auth creating new auth -752> 2022-04-12T16:07:49.223+0300 7f2a268c2700 10 monclient(hunting): handle_auth_reply_more payload 9 -751> 2022-04-12T16:07:49.223+0300 7f2a268c2700 10 monclient(hunting): handle_auth_reply_more payload_len 9 -750> 2022-04-12T16:07:49.223+0300 7f2a268c2700 10 monclient(hunting): handle_auth_reply_more responding with 36 bytes -749> 2022-04-12T16:07:49.223+0300 7f2a268c2700 10 monclient(hunting): handle_auth_done global_id 5727 payload 1000 -748> 2022-04-12T16:07:49.223+0300 7f2a268c2700 10 monclient: _finish_hunting 0 -747> 2022-04-12T16:07:49.223+0300 7f2a268c2700 1 monclient: found mon.noname-a -746> 2022-04-12T16:07:49.223+0300 7f2a268c2700 10 monclient: _send_mon_message to mon.noname-a at v2:192.168.166.83:3300/0 -745> 2022-04-12T16:07:49.223+0300 7f2a0c08d700 10 monclient: handle_monmap mon_map magic: 0 v1 -744> 2022-04-12T16:07:49.223+0300 7f2a0c08d700 10 monclient: got monmap 1 from mon.noname-a (according to old e1) -743> 2022-04-12T16:07:49.223+0300 7f2a0c08d700 10 monclient: dump: epoch 1 fsid 65ccd25a-39ce-4072-a5c6-a0e6654bda87 last_changed 2022-04-12T15:48:27.663651+0300 created 2022-04-12T15:48:27.663651+0300 min_mon_release 16 (pacific) election_strategy: 1 0: [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] mon.ceph-ss-test-oct-controller-av-01 1: [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] mon.ceph-ss-test-oct-controller-av-02 2: [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] mon.ceph-ss-test-oct-controller-av-03 -742> 2022-04-12T16:07:49.223+0300 7f2a0c08d700 10 monclient: _finish_auth 0 -741> 2022-04-12T16:07:49.223+0300 7f2a0c08d700 10 monclient: handle_config config(0 keys) v1 -740> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 monclient: authenticate success, global_id 5727 -739> 2022-04-12T16:07:49.223+0300 7f2a3b460400 10 monclient: _renew_subs -738> 2022-04-12T16:07:49.223+0300 7f2a0d890700 4 set_mon_vals no callback set -737> 2022-04-12T16:07:49.223+0300 7f2a3b460400 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -736> 2022-04-12T16:07:49.223+0300 7f2a0c08d700 10 monclient: handle_monmap mon_map magic: 0 v1 -735> 2022-04-12T16:07:49.223+0300 7f2a0c08d700 10 monclient: got monmap 1 from mon.ceph-ss-test-oct-controller-av-01 (according to old e1) -734> 2022-04-12T16:07:49.223+0300 7f2a0c08d700 10 monclient: dump: epoch 1 fsid 65ccd25a-39ce-4072-a5c6-a0e6654bda87 last_changed 2022-04-12T15:48:27.663651+0300 created 2022-04-12T15:48:27.663651+0300 min_mon_release 16 (pacific) election_strategy: 1 0: [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] mon.ceph-ss-test-oct-controller-av-01 1: [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] mon.ceph-ss-test-oct-controller-av-02 2: [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] mon.ceph-ss-test-oct-controller-av-03 -733> 2022-04-12T16:07:49.223+0300 7f2a3b460400 10 monclient: _renew_subs -732> 2022-04-12T16:07:49.223+0300 7f2a3b460400 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -731> 2022-04-12T16:07:49.223+0300 7f2a3b460400 1 librados: init done -730> 2022-04-12T16:07:49.223+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command cr dump hook 0x559e558a04e8 -729> 2022-04-12T16:07:49.227+0300 7f2a0c08d700 4 mgrc handle_mgr_map Got map version 29 -728> 2022-04-12T16:07:49.227+0300 7f2a0c08d700 4 mgrc handle_mgr_map Active mgr is now [v2:192.168.166.84:6800/56,v1:192.168.166.84:6801/56] -727> 2022-04-12T16:07:49.227+0300 7f2a0c08d700 4 mgrc reconnect Starting new session with [v2:192.168.166.84:6800/56,v1:192.168.166.84:6801/56] -726> 2022-04-12T16:07:49.227+0300 7f2a3b460400 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -725> 2022-04-12T16:07:49.227+0300 7f2a0c08d700 10 monclient: handle_get_version_reply finishing 1 version 68 -724> 2022-04-12T16:07:49.227+0300 7f2a260c1700 10 monclient: get_auth_request con 0x559e559cbc00 auth_method 0 -723> 2022-04-12T16:07:49.227+0300 7f2a3b460400 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -722> 2022-04-12T16:07:49.603+0300 7f2a0c08d700 10 monclient: _renew_subs -721> 2022-04-12T16:07:49.603+0300 7f2a0c08d700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -720> 2022-04-12T16:07:49.607+0300 7f2a3b460400 10 start_mon_command cmd=[{"prefix": "osd pool application enable","pool": "sync.rgw.meta","app": "rgw"}] -719> 2022-04-12T16:07:49.607+0300 7f2a3b460400 10 monclient: _send_command 1 [{"prefix": "osd pool application enable","pool": "sync.rgw.meta","app": "rgw"}] -718> 2022-04-12T16:07:49.607+0300 7f2a3b460400 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -717> 2022-04-12T16:07:49.655+0300 7f2a1f0b3700 10 monclient: tick -716> 2022-04-12T16:07:49.655+0300 7f2a208b6700 10 monclient: _renew_subs -715> 2022-04-12T16:07:49.655+0300 7f2a208b6700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -714> 2022-04-12T16:07:50.611+0300 7f2a0c08d700 10 monclient: handle_mon_command_ack 1 [{"prefix": "osd pool application enable","pool": "sync.rgw.meta","app": "rgw"}] -713> 2022-04-12T16:07:50.611+0300 7f2a0c08d700 10 monclient: _finish_command 1 = system:0 enabled application 'rgw' on pool 'sync.rgw.meta' -712> 2022-04-12T16:07:50.611+0300 7f2a3b460400 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -711> 2022-04-12T16:07:50.615+0300 7f2a0c08d700 10 monclient: handle_get_version_reply finishing 2 version 70 -710> 2022-04-12T16:07:50.615+0300 7f2a0d890700 10 monclient: _renew_subs -709> 2022-04-12T16:07:50.615+0300 7f2a0d890700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -708> 2022-04-12T16:07:50.615+0300 7f2a3b460400 10 start_mon_command cmd=[{"prefix": "osd pool set", "pool": "sync.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}] -707> 2022-04-12T16:07:50.615+0300 7f2a3b460400 10 monclient: _send_command 2 [{"prefix": "osd pool set", "pool": "sync.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}] -706> 2022-04-12T16:07:50.615+0300 7f2a3b460400 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -705> 2022-04-12T16:07:50.655+0300 7f2a1f0b3700 10 monclient: tick -704> 2022-04-12T16:07:51.615+0300 7f2a0c08d700 10 monclient: handle_mon_command_ack 2 [{"prefix": "osd pool set", "pool": "sync.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}] -703> 2022-04-12T16:07:51.615+0300 7f2a0c08d700 10 monclient: _finish_command 2 = system:0 set pool 11 pg_autoscale_bias to 4 -702> 2022-04-12T16:07:51.615+0300 7f2a3b460400 10 start_mon_command cmd=[{"prefix": "osd pool set", "pool": "sync.rgw.meta", "var": "pg_num_min", "val": "8"}] -701> 2022-04-12T16:07:51.615+0300 7f2a3b460400 10 monclient: _send_command 3 [{"prefix": "osd pool set", "pool": "sync.rgw.meta", "var": "pg_num_min", "val": "8"}] -700> 2022-04-12T16:07:51.615+0300 7f2a3b460400 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -699> 2022-04-12T16:07:51.655+0300 7f2a1f0b3700 10 monclient: tick -698> 2022-04-12T16:07:52.223+0300 7f2a0b08b700 10 monclient: tick -697> 2022-04-12T16:07:52.619+0300 7f2a0c08d700 10 monclient: handle_mon_command_ack 3 [{"prefix": "osd pool set", "pool": "sync.rgw.meta", "var": "pg_num_min", "val": "8"}] -696> 2022-04-12T16:07:52.619+0300 7f2a0c08d700 10 monclient: _finish_command 3 = system:0 set pool 11 pg_num_min to 8 -695> 2022-04-12T16:07:52.619+0300 7f2a3b460400 10 start_mon_command cmd=[{"prefix": "osd pool set", "pool": "sync.rgw.meta", "var": "recovery_priority": "5"}] -694> 2022-04-12T16:07:52.619+0300 7f2a3b460400 10 monclient: _send_command 4 [{"prefix": "osd pool set", "pool": "sync.rgw.meta", "var": "recovery_priority": "5"}] -693> 2022-04-12T16:07:52.619+0300 7f2a3b460400 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -692> 2022-04-12T16:07:52.623+0300 7f2a0c08d700 10 monclient: handle_mon_command_ack 4 [{"prefix": "osd pool set", "pool": "sync.rgw.meta", "var": "recovery_priority": "5"}] -691> 2022-04-12T16:07:52.623+0300 7f2a0c08d700 10 monclient: _finish_command 4 = mon:22 unparseable JSON {"prefix": "osd pool set", "pool": "sync.rgw.meta", "var": "recovery_priority": "5"} -690> 2022-04-12T16:07:52.623+0300 7f2a270c3700 10 monclient: get_auth_request con 0x559e559f1400 auth_method 0 -689> 2022-04-12T16:07:52.627+0300 7f2a268c2700 10 monclient: get_auth_request con 0x559e559cac00 auth_method 0 -688> 2022-04-12T16:07:52.635+0300 7f2a260c1700 10 monclient: get_auth_request con 0x559e559ca000 auth_method 0 -687> 2022-04-12T16:07:52.639+0300 7f2a270c3700 10 monclient: get_auth_request con 0x559e55b21400 auth_method 0 -686> 2022-04-12T16:07:52.647+0300 7f2a268c2700 10 monclient: get_auth_request con 0x559e54b39c00 auth_method 0 -685> 2022-04-12T16:07:52.651+0300 7f2a260c1700 10 monclient: get_auth_request con 0x559e559cb000 auth_method 0 -684> 2022-04-12T16:07:52.659+0300 7f2a1f0b3700 10 monclient: tick -683> 2022-04-12T16:07:52.659+0300 7f2a270c3700 10 monclient: get_auth_request con 0x559e55990000 auth_method 0 -682> 2022-04-12T16:07:52.671+0300 7f2a268c2700 10 monclient: get_auth_request con 0x559e559f1800 auth_method 0 -681> 2022-04-12T16:07:52.699+0300 7f2a260c1700 10 monclient: get_auth_request con 0x559e55b51800 auth_method 0 -680> 2022-04-12T16:07:52.711+0300 7f2a270c3700 10 monclient: get_auth_request con 0x559e55b50000 auth_method 0 -679> 2022-04-12T16:07:52.723+0300 7f2a268c2700 10 monclient: get_auth_request con 0x559e559f0400 auth_method 0 -678> 2022-04-12T16:07:52.731+0300 7f2a260c1700 10 monclient: get_auth_request con 0x559e559ca400 auth_method 0 -677> 2022-04-12T16:07:52.743+0300 7f2a270c3700 10 monclient: get_auth_request con 0x559e55979c00 auth_method 0 -676> 2022-04-12T16:07:52.747+0300 7f2a09888700 2 object expiration: start -675> 2022-04-12T16:07:52.747+0300 7f2a0a089700 2 garbage collection: garbage collection: start -674> 2022-04-12T16:07:52.747+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command sync trace show hook 0x559e5592a1c0 -673> 2022-04-12T16:07:52.747+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command sync trace history hook 0x559e5592a1c0 -672> 2022-04-12T16:07:52.747+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command sync trace active hook 0x559e5592a1c0 -671> 2022-04-12T16:07:52.747+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command sync trace active_short hook 0x559e5592a1c0 -670> 2022-04-12T16:07:52.751+0300 7f2a3b460400 4 rgw main: failed to read sync status info with (2) No such file or directory -669> 2022-04-12T16:07:52.755+0300 7f2a07884700 4 rgw rados thread: failed to read sync status info with (2) No such file or directory -668> 2022-04-12T16:07:52.755+0300 7f2a07884700 1 RGW-SYNC:meta: initialize sync (reason: period is empty) -667> 2022-04-12T16:07:52.755+0300 7f2a07884700 1 rgw rados thread: epoch=0 in sync status comes before remote's oldest mdlog epoch=1, restarting sync -666> 2022-04-12T16:07:52.759+0300 7f2a3b460400 5 rgw main: starting data sync thread for zone avrupa -665> 2022-04-12T16:07:52.759+0300 7f29ff073700 5 lifecycle: schedule life cycle next start time: Tue Apr 12 21:00:00 2022 -664> 2022-04-12T16:07:52.759+0300 7f2a01077700 5 lifecycle: schedule life cycle next start time: Tue Apr 12 21:00:00 2022 -663> 2022-04-12T16:07:52.759+0300 7f29fd06f700 5 lifecycle: schedule life cycle next start time: Tue Apr 12 21:00:00 2022 -662> 2022-04-12T16:07:52.763+0300 7f2a3b460400 0 framework: beast -661> 2022-04-12T16:07:52.763+0300 7f2a3b460400 0 framework conf key: ssl_certificate, val: config://rgw/cert/$realm/$zone.crt -660> 2022-04-12T16:07:52.763+0300 7f2a3b460400 0 framework conf key: ssl_private_key, val: config://rgw/cert/$realm/$zone.key -659> 2022-04-12T16:07:52.763+0300 7f2a3b460400 0 starting handler: beast -658> 2022-04-12T16:07:52.763+0300 7f2a06882700 4 rgw rados thread: failed to read sync status info with (2) No such file or directory -657> 2022-04-12T16:07:52.763+0300 7f2a3b460400 4 frontend listening on 192.168.166.83:8081 -656> 2022-04-12T16:07:52.763+0300 7f2a3b460400 0 set uid:gid to 167:167 (ceph:ceph) -655> 2022-04-12T16:07:52.763+0300 7f2a3b460400 4 frontend spawning 512 threads -654> 2022-04-12T16:07:52.799+0300 7f2a3b460400 1 mgrc service_daemon_register rgw.5727 metadata {arch=x86_64,ceph_release=pacific,ceph_version=ceph version 16.2.7 (dd0603118f56ab514f133c8d2e3adfc983942503) pacific (stable),ceph_version_short=16.2.7,container_image=gitlab.example.com:5050/openstack/registry/ceph/daemon:v6.0.6-stable-6.0-pacific-centos-8-x86_64-ceph-ansible-20220121,cpu=Intel(R) Xeon(R) Gold 6230R CPU @ 2.10GHz,distro=centos,distro_description=CentOS Linux 8,distro_version=8,frontend_config#0=beast endpoint=192.168.166.83:8081,frontend_type#0=beast,hostname=ceph-ss-test-oct-controller-av-01,id=ceph-ss-test-oct-controller-av-01.rgw1,kernel_description=#121-Ubuntu SMP Thu Mar 24 16:04:27 UTC 2022,kernel_version=5.4.0-107-generic,mem_cgroup_limit=4294967296,mem_swap_kb=4030460,mem_total_kb=8148192,num_handles=1,os=Linux,pid=54,realm_id=532f2481-54e3-4d51-95fd-19867ca818e9,realm_name=myrealm,zone_id=c81e6d95-54da-48d3-83a4-7bb7b286709f,zone_name=sync,zonegroup_id=784a05a5-2089-4564-ab1b-a55c43c5f401,zonegroup_name=cdn} -653> 2022-04-12T16:07:52.807+0300 7f2a0c08d700 4 mgrc handle_mgr_configure stats_period=5 -652> 2022-04-12T16:07:52.807+0300 7f2a0c08d700 4 mgrc handle_mgr_configure updated stats threshold: 5 -651> 2022-04-12T16:07:52.807+0300 7f2a3b460400 10 monclient: get_monmap_and_config -650> 2022-04-12T16:07:52.807+0300 7f2a3b460400 10 monclient: build_initial_monmap -649> 2022-04-12T16:07:52.807+0300 7f2a3b460400 10 monclient: monmap: epoch 0 fsid 00000000-0000-0000-0000-000000000000 last_changed 0.000000 created 0.000000 min_mon_release 0 (unknown) election_strategy: 1 0: [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] mon.noname-a 1: [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] mon.noname-b 2: [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] mon.noname-c -648> 2022-04-12T16:07:52.807+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding auth protocol: cephx -647> 2022-04-12T16:07:52.807+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding auth protocol: cephx -646> 2022-04-12T16:07:52.807+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding auth protocol: cephx -645> 2022-04-12T16:07:52.807+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding auth protocol: none -644> 2022-04-12T16:07:52.807+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: secure -643> 2022-04-12T16:07:52.807+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: crc -642> 2022-04-12T16:07:52.807+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: secure -641> 2022-04-12T16:07:52.807+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: crc -640> 2022-04-12T16:07:52.807+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: secure -639> 2022-04-12T16:07:52.807+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: crc -638> 2022-04-12T16:07:52.807+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: crc -637> 2022-04-12T16:07:52.807+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: secure -636> 2022-04-12T16:07:52.807+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: crc -635> 2022-04-12T16:07:52.807+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: secure -634> 2022-04-12T16:07:52.807+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: crc -633> 2022-04-12T16:07:52.807+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: secure -632> 2022-04-12T16:07:52.807+0300 7f2a3b460400 2 auth: KeyRing::load: loaded key file /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring -631> 2022-04-12T16:07:52.819+0300 7f2a3b460400 10 monclient: init -630> 2022-04-12T16:07:52.819+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding auth protocol: cephx -629> 2022-04-12T16:07:52.819+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding auth protocol: cephx -628> 2022-04-12T16:07:52.819+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding auth protocol: cephx -627> 2022-04-12T16:07:52.819+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding auth protocol: none -626> 2022-04-12T16:07:52.819+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: secure -625> 2022-04-12T16:07:52.819+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: crc -624> 2022-04-12T16:07:52.819+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: secure -623> 2022-04-12T16:07:52.819+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: crc -622> 2022-04-12T16:07:52.819+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: secure -621> 2022-04-12T16:07:52.819+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: crc -620> 2022-04-12T16:07:52.819+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: crc -619> 2022-04-12T16:07:52.819+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: secure -618> 2022-04-12T16:07:52.819+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: crc -617> 2022-04-12T16:07:52.819+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: secure -616> 2022-04-12T16:07:52.819+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: crc -615> 2022-04-12T16:07:52.819+0300 7f2a3b460400 5 AuthRegistry(0x7ffc391d7420) adding con mode: secure -614> 2022-04-12T16:07:52.819+0300 7f2a3b460400 2 auth: KeyRing::load: loaded key file /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring -613> 2022-04-12T16:07:52.819+0300 7f2a3b460400 2 auth: KeyRing::load: loaded key file /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring -612> 2022-04-12T16:07:52.819+0300 7f2a3b460400 10 monclient: _reopen_session rank -1 -611> 2022-04-12T16:07:52.819+0300 7f2a3b460400 10 monclient: _add_conns ranks=[1,0,2] -610> 2022-04-12T16:07:52.819+0300 7f2a3b460400 10 monclient(hunting): picked mon.noname-b con 0x559e55db1c00 addr [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] -609> 2022-04-12T16:07:52.819+0300 7f2a3b460400 10 monclient(hunting): picked mon.noname-a con 0x559e559f1000 addr [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] -608> 2022-04-12T16:07:52.819+0300 7f2a3b460400 10 monclient(hunting): picked mon.noname-c con 0x559e559f0800 addr [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] -607> 2022-04-12T16:07:52.819+0300 7f2a3b460400 10 monclient(hunting): start opening mon connection -606> 2022-04-12T16:07:52.819+0300 7f2a3b460400 10 monclient(hunting): start opening mon connection -605> 2022-04-12T16:07:52.819+0300 7f2a3b460400 10 monclient(hunting): start opening mon connection -604> 2022-04-12T16:07:52.819+0300 7f2a3b460400 10 monclient(hunting): _renew_subs -603> 2022-04-12T16:07:52.819+0300 7f2a3b460400 10 monclient(hunting): authenticate will time out at 2022-04-12T16:12:52.824021+0300 -602> 2022-04-12T16:07:52.819+0300 7f2a260c1700 10 monclient(hunting): get_auth_request con 0x559e55db1c00 auth_method 0 -601> 2022-04-12T16:07:52.819+0300 7f2a260c1700 10 monclient(hunting): get_auth_request method 2 preferred_modes [2,1] -600> 2022-04-12T16:07:52.819+0300 7f2a260c1700 10 monclient(hunting): _init_auth method 2 -599> 2022-04-12T16:07:52.819+0300 7f2a260c1700 10 monclient(hunting): _init_auth creating new auth -598> 2022-04-12T16:07:52.819+0300 7f2a268c2700 10 monclient(hunting): get_auth_request con 0x559e559f0800 auth_method 0 -597> 2022-04-12T16:07:52.819+0300 7f2a268c2700 10 monclient(hunting): get_auth_request method 2 preferred_modes [2,1] -596> 2022-04-12T16:07:52.819+0300 7f2a268c2700 10 monclient(hunting): _init_auth method 2 -595> 2022-04-12T16:07:52.819+0300 7f2a268c2700 10 monclient(hunting): _init_auth creating new auth -594> 2022-04-12T16:07:52.819+0300 7f2a260c1700 10 monclient(hunting): handle_auth_reply_more payload 9 -593> 2022-04-12T16:07:52.819+0300 7f2a260c1700 10 monclient(hunting): handle_auth_reply_more payload_len 9 -592> 2022-04-12T16:07:52.819+0300 7f2a260c1700 10 monclient(hunting): handle_auth_reply_more responding with 36 bytes -591> 2022-04-12T16:07:52.819+0300 7f2a268c2700 10 monclient(hunting): handle_auth_reply_more payload 9 -590> 2022-04-12T16:07:52.819+0300 7f2a268c2700 10 monclient(hunting): handle_auth_reply_more payload_len 9 -589> 2022-04-12T16:07:52.819+0300 7f2a268c2700 10 monclient(hunting): handle_auth_reply_more responding with 36 bytes -588> 2022-04-12T16:07:52.819+0300 7f2a260c1700 10 monclient(hunting): handle_auth_done global_id 5197 payload 322 -587> 2022-04-12T16:07:52.819+0300 7f2a260c1700 10 monclient: _finish_hunting 0 -586> 2022-04-12T16:07:52.819+0300 7f2a260c1700 1 monclient: found mon.noname-b -585> 2022-04-12T16:07:52.819+0300 7f2a260c1700 10 monclient: _send_mon_message to mon.noname-b at v2:192.168.166.84:3300/0 -584> 2022-04-12T16:07:52.819+0300 7f28f8e67700 10 monclient: handle_monmap mon_map magic: 0 v1 -583> 2022-04-12T16:07:52.819+0300 7f28f8e67700 10 monclient: got monmap 1 from mon.noname-b (according to old e1) -582> 2022-04-12T16:07:52.819+0300 7f28f8e67700 10 monclient: dump: epoch 1 fsid 65ccd25a-39ce-4072-a5c6-a0e6654bda87 last_changed 2022-04-12T15:48:27.663651+0300 created 2022-04-12T15:48:27.663651+0300 min_mon_release 16 (pacific) election_strategy: 1 0: [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] mon.ceph-ss-test-oct-controller-av-01 1: [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] mon.ceph-ss-test-oct-controller-av-02 2: [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] mon.ceph-ss-test-oct-controller-av-03 -581> 2022-04-12T16:07:52.819+0300 7f28f8e67700 10 monclient: _finish_auth 0 -580> 2022-04-12T16:07:52.819+0300 7f28f8e67700 10 monclient: handle_config config(0 keys) v1 -579> 2022-04-12T16:07:52.819+0300 7f28f8e67700 10 monclient: handle_monmap mon_map magic: 0 v1 -578> 2022-04-12T16:07:52.819+0300 7f28f8e67700 10 monclient: got monmap 1 from mon.ceph-ss-test-oct-controller-av-02 (according to old e1) -577> 2022-04-12T16:07:52.819+0300 7f28f8e67700 10 monclient: dump: epoch 1 fsid 65ccd25a-39ce-4072-a5c6-a0e6654bda87 last_changed 2022-04-12T15:48:27.663651+0300 created 2022-04-12T15:48:27.663651+0300 min_mon_release 16 (pacific) election_strategy: 1 0: [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] mon.ceph-ss-test-oct-controller-av-01 1: [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] mon.ceph-ss-test-oct-controller-av-02 2: [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] mon.ceph-ss-test-oct-controller-av-03 -576> 2022-04-12T16:07:52.819+0300 7f2a3b460400 5 monclient: authenticate success, global_id 5197 -575> 2022-04-12T16:07:52.819+0300 7f2a3b460400 10 monclient: get_monmap_and_config success -574> 2022-04-12T16:07:52.819+0300 7f2a3b460400 4 set_mon_vals no callback set -573> 2022-04-12T16:07:52.823+0300 7f2a3b460400 10 monclient: shutdown -572> 2022-04-12T16:07:52.823+0300 7f2a3b460400 10 monclient: build_initial_monmap -571> 2022-04-12T16:07:52.823+0300 7f2a3b460400 10 monclient: monmap: epoch 0 fsid 00000000-0000-0000-0000-000000000000 last_changed 0.000000 created 0.000000 min_mon_release 0 (unknown) election_strategy: 1 0: [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] mon.noname-a 1: [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] mon.noname-b 2: [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] mon.noname-c -570> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding auth protocol: cephx -569> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding auth protocol: cephx -568> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding auth protocol: cephx -567> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding auth protocol: none -566> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: secure -565> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: crc -564> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: secure -563> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: crc -562> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: secure -561> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: crc -560> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: crc -559> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: secure -558> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: crc -557> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: secure -556> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: crc -555> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e55987740) adding con mode: secure -554> 2022-04-12T16:07:52.823+0300 7f2a3b460400 2 auth: KeyRing::load: loaded key file /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring -553> 2022-04-12T16:07:52.823+0300 7f2a3b460400 1 librados: starting msgr at -552> 2022-04-12T16:07:52.823+0300 7f2a3b460400 1 librados: starting objecter -551> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 asok(0x559e54b34000) register_command objecter_requests cmddesc objecter_requests hook 0x559e58ce73c0 EEXIST -550> 2022-04-12T16:07:52.823+0300 7f2a3b460400 1 librados: setting wanted keys -549> 2022-04-12T16:07:52.823+0300 7f2a3b460400 1 librados: calling monclient init -548> 2022-04-12T16:07:52.823+0300 7f2a3b460400 10 monclient: init -547> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e594860d8) adding auth protocol: cephx -546> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e594860d8) adding auth protocol: cephx -545> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e594860d8) adding auth protocol: cephx -544> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e594860d8) adding auth protocol: none -543> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e594860d8) adding con mode: secure -542> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e594860d8) adding con mode: crc -541> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e594860d8) adding con mode: secure -540> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e594860d8) adding con mode: crc -539> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e594860d8) adding con mode: secure -538> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e594860d8) adding con mode: crc -537> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e594860d8) adding con mode: crc -536> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e594860d8) adding con mode: secure -535> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e594860d8) adding con mode: crc -534> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e594860d8) adding con mode: secure -533> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e594860d8) adding con mode: crc -532> 2022-04-12T16:07:52.823+0300 7f2a3b460400 5 AuthRegistry(0x559e594860d8) adding con mode: secure -531> 2022-04-12T16:07:52.823+0300 7f2a3b460400 2 auth: KeyRing::load: loaded key file /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring -530> 2022-04-12T16:07:52.823+0300 7f2a3b460400 2 auth: KeyRing::load: loaded key file /var/lib/ceph/radosgw/av-ceph-test-rgw.ceph-ss-test-oct-controller-av-01.rgw1/keyring -529> 2022-04-12T16:07:52.823+0300 7f2a3b460400 10 monclient: _reopen_session rank -1 -528> 2022-04-12T16:07:52.823+0300 7f2a3b460400 10 monclient: _add_conns ranks=[0,2,1] -527> 2022-04-12T16:07:52.823+0300 7f2a3b460400 10 monclient(hunting): picked mon.noname-a con 0x559e559f0800 addr [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] -526> 2022-04-12T16:07:52.823+0300 7f2a3b460400 10 monclient(hunting): picked mon.noname-c con 0x559e559f1000 addr [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] -525> 2022-04-12T16:07:52.823+0300 7f2a3b460400 10 monclient(hunting): picked mon.noname-b con 0x559e55db1c00 addr [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] -524> 2022-04-12T16:07:52.823+0300 7f2a3b460400 10 monclient(hunting): start opening mon connection -523> 2022-04-12T16:07:52.823+0300 7f2a3b460400 10 monclient(hunting): start opening mon connection -522> 2022-04-12T16:07:52.823+0300 7f2a3b460400 10 monclient(hunting): start opening mon connection -521> 2022-04-12T16:07:52.823+0300 7f2a3b460400 10 monclient(hunting): _renew_subs -520> 2022-04-12T16:07:52.823+0300 7f2a3b460400 10 monclient(hunting): authenticate will time out at 2022-04-12T16:12:52.827867+0300 -519> 2022-04-12T16:07:52.823+0300 7f2a260c1700 10 monclient(hunting): get_auth_request con 0x559e559f0800 auth_method 0 -518> 2022-04-12T16:07:52.823+0300 7f2a260c1700 10 monclient(hunting): get_auth_request method 2 preferred_modes [2,1] -517> 2022-04-12T16:07:52.823+0300 7f2a260c1700 10 monclient(hunting): _init_auth method 2 -516> 2022-04-12T16:07:52.823+0300 7f2a260c1700 10 monclient(hunting): _init_auth creating new auth -515> 2022-04-12T16:07:52.823+0300 7f2a260c1700 10 monclient(hunting): handle_auth_reply_more payload 9 -514> 2022-04-12T16:07:52.823+0300 7f2a260c1700 10 monclient(hunting): handle_auth_reply_more payload_len 9 -513> 2022-04-12T16:07:52.823+0300 7f2a260c1700 10 monclient(hunting): handle_auth_reply_more responding with 36 bytes -512> 2022-04-12T16:07:52.823+0300 7f2a270c3700 10 monclient(hunting): get_auth_request con 0x559e559f1000 auth_method 0 -511> 2022-04-12T16:07:52.823+0300 7f2a270c3700 10 monclient(hunting): get_auth_request method 2 preferred_modes [2,1] -510> 2022-04-12T16:07:52.823+0300 7f2a270c3700 10 monclient(hunting): _init_auth method 2 -509> 2022-04-12T16:07:52.823+0300 7f2a270c3700 10 monclient(hunting): _init_auth creating new auth -508> 2022-04-12T16:07:52.823+0300 7f2a260c1700 10 monclient(hunting): handle_auth_done global_id 5730 payload 1000 -507> 2022-04-12T16:07:52.823+0300 7f2a260c1700 10 monclient: _finish_hunting 0 -506> 2022-04-12T16:07:52.823+0300 7f2a260c1700 1 monclient: found mon.noname-a -505> 2022-04-12T16:07:52.823+0300 7f2a260c1700 10 monclient: _send_mon_message to mon.noname-a at v2:192.168.166.83:3300/0 -504> 2022-04-12T16:07:52.823+0300 7f28f7664700 10 monclient: handle_monmap mon_map magic: 0 v1 -503> 2022-04-12T16:07:52.827+0300 7f28f7664700 10 monclient: got monmap 1 from mon.noname-a (according to old e1) -502> 2022-04-12T16:07:52.827+0300 7f28f7664700 10 monclient: dump: epoch 1 fsid 65ccd25a-39ce-4072-a5c6-a0e6654bda87 last_changed 2022-04-12T15:48:27.663651+0300 created 2022-04-12T15:48:27.663651+0300 min_mon_release 16 (pacific) election_strategy: 1 0: [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] mon.ceph-ss-test-oct-controller-av-01 1: [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] mon.ceph-ss-test-oct-controller-av-02 2: [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] mon.ceph-ss-test-oct-controller-av-03 -501> 2022-04-12T16:07:52.827+0300 7f28f7664700 10 monclient: _finish_auth 0 -500> 2022-04-12T16:07:52.827+0300 7f28f7664700 10 monclient: handle_config config(0 keys) v1 -499> 2022-04-12T16:07:52.827+0300 7f28f7664700 10 monclient: handle_monmap mon_map magic: 0 v1 -498> 2022-04-12T16:07:52.827+0300 7f28f7664700 10 monclient: got monmap 1 from mon.ceph-ss-test-oct-controller-av-01 (according to old e1) -497> 2022-04-12T16:07:52.827+0300 7f28f7664700 10 monclient: dump: epoch 1 fsid 65ccd25a-39ce-4072-a5c6-a0e6654bda87 last_changed 2022-04-12T15:48:27.663651+0300 created 2022-04-12T15:48:27.663651+0300 min_mon_release 16 (pacific) election_strategy: 1 0: [v2:192.168.166.83:3300/0,v1:192.168.166.83:6789/0] mon.ceph-ss-test-oct-controller-av-01 1: [v2:192.168.166.84:3300/0,v1:192.168.166.84:6789/0] mon.ceph-ss-test-oct-controller-av-02 2: [v2:192.168.166.85:3300/0,v1:192.168.166.85:6789/0] mon.ceph-ss-test-oct-controller-av-03 -496> 2022-04-12T16:07:52.827+0300 7f28f8e67700 4 set_mon_vals no callback set -495> 2022-04-12T16:07:52.827+0300 7f2a3b460400 5 monclient: authenticate success, global_id 5730 -494> 2022-04-12T16:07:52.827+0300 7f2a3b460400 10 monclient: _renew_subs -493> 2022-04-12T16:07:52.827+0300 7f2a3b460400 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -492> 2022-04-12T16:07:52.827+0300 7f2a3b460400 10 monclient: _renew_subs -491> 2022-04-12T16:07:52.827+0300 7f2a3b460400 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -490> 2022-04-12T16:07:52.827+0300 7f2a3b460400 1 librados: init done -489> 2022-04-12T16:07:52.831+0300 7f28f7664700 4 mgrc handle_mgr_map Got map version 29 -488> 2022-04-12T16:07:52.831+0300 7f28f7664700 4 mgrc handle_mgr_map Active mgr is now [v2:192.168.166.84:6800/56,v1:192.168.166.84:6801/56] -487> 2022-04-12T16:07:52.831+0300 7f28f7664700 4 mgrc reconnect Starting new session with [v2:192.168.166.84:6800/56,v1:192.168.166.84:6801/56] -486> 2022-04-12T16:07:52.831+0300 7f2a268c2700 10 monclient: get_auth_request con 0x559e593a1000 auth_method 0 -485> 2022-04-12T16:07:52.835+0300 7f2a270c3700 10 monclient: get_auth_request con 0x559e55979800 auth_method 0 -484> 2022-04-12T16:07:52.855+0300 7f2a260c1700 10 monclient: get_auth_request con 0x559e562ab000 auth_method 0 -483> 2022-04-12T16:07:53.075+0300 7f2a270c3700 10 monclient: get_auth_request con 0x559e59cf6000 auth_method 0 -482> 2022-04-12T16:07:53.083+0300 7f2a0a089700 2 garbage collection: garbage collection: stop -481> 2022-04-12T16:07:53.223+0300 7f2a0b08b700 10 monclient: tick -480> 2022-04-12T16:07:53.643+0300 7f2a09888700 2 object expiration: stop -479> 2022-04-12T16:07:53.655+0300 7f2a1f0b3700 10 monclient: tick -478> 2022-04-12T16:07:53.951+0300 7f29f8065700 1 ====== starting new request req=0x7f2a3b415620 ===== -477> 2022-04-12T16:07:53.951+0300 7f29f8065700 2 req 6350979191525555842 0.000000000s initializing for trans_id = tx00000582335ecfec73e82-00625579a9-1659-sync -476> 2022-04-12T16:07:53.951+0300 7f29f8065700 2 req 6350979191525555842 0.000000000s getting op 4 -475> 2022-04-12T16:07:53.951+0300 7f29f6061700 2 req 6350979191525555842 0.000000000s :post_period verifying requester -474> 2022-04-12T16:07:53.951+0300 7f29f9868700 1 ====== starting new request req=0x7f2a3b313620 ===== -473> 2022-04-12T16:07:53.951+0300 7f29f305b700 1 ====== starting new request req=0x7f2a3b394620 ===== -472> 2022-04-12T16:07:53.951+0300 7f29f9868700 2 req 8268245483269231629 0.000000000s initializing for trans_id = tx0000072beb575c1fd940d-00625579a9-1659-sync -471> 2022-04-12T16:07:53.951+0300 7f29f305b700 2 req 10560051255912296523 0.000000000s initializing for trans_id = tx00000928cd6aaee91f04b-00625579a9-1659-sync -470> 2022-04-12T16:07:53.955+0300 7f29f9868700 2 req 8268245483269231629 0.004000003s getting op 4 -469> 2022-04-12T16:07:53.955+0300 7f29f9868700 2 req 8268245483269231629 0.004000003s :post_period verifying requester -468> 2022-04-12T16:07:53.955+0300 7f29f9868700 5 req 8268245483269231629 0.004000003s :post_period error reading user info, uid=m7yH86NH1D44QiwaUUV9 can't authenticate -467> 2022-04-12T16:07:53.955+0300 7f29f9868700 5 req 8268245483269231629 0.004000003s :post_period error reading user info, uid=m7yH86NH1D44QiwaUUV9 can't authenticate -466> 2022-04-12T16:07:53.955+0300 7f29f9868700 5 req 8268245483269231629 0.004000003s :post_period Failed the auth strategy, reason=-2028 -465> 2022-04-12T16:07:53.955+0300 7f29f9868700 1 req 8268245483269231629 0.004000003s op->ERRORHANDLER: err_no=-2028 new_err_no=-2028 -464> 2022-04-12T16:07:53.955+0300 7f29f0856700 5 req 6350979191525555842 0.004000003s :post_period error reading user info, uid=m7yH86NH1D44QiwaUUV9 can't authenticate -463> 2022-04-12T16:07:53.955+0300 7f29f0856700 5 req 6350979191525555842 0.004000003s :post_period error reading user info, uid=m7yH86NH1D44QiwaUUV9 can't authenticate -462> 2022-04-12T16:07:53.955+0300 7f29f0856700 5 req 6350979191525555842 0.004000003s :post_period Failed the auth strategy, reason=-2028 -461> 2022-04-12T16:07:53.955+0300 7f29f0856700 1 req 6350979191525555842 0.004000003s op->ERRORHANDLER: err_no=-2028 new_err_no=-2028 -460> 2022-04-12T16:07:53.955+0300 7f29f305b700 2 req 10560051255912296523 0.004000003s getting op 4 -459> 2022-04-12T16:07:53.955+0300 7f29f305b700 2 req 10560051255912296523 0.004000003s :post_period verifying requester -458> 2022-04-12T16:07:53.955+0300 7f29e6842700 2 req 8268245483269231629 0.004000003s :post_period op status=0 -457> 2022-04-12T16:07:53.955+0300 7f29e6842700 2 req 8268245483269231629 0.004000003s :post_period http status=403 -456> 2022-04-12T16:07:53.955+0300 7f29e6842700 1 ====== req done req=0x7f2a3b313620 op status=0 http_status=403 latency=0.004000003s ====== -455> 2022-04-12T16:07:53.955+0300 7f29e6842700 1 beast: 0x7f2a3b313620: 192.168.166.84 - - [12/Apr/2022:16:07:53.951 +0300] "POST /admin/realm/period?period=8b4c894f-cf43-4c56-9af0-1c2b54c74991&epoch=3&rgwx-zonegroup=784a05a5-2089-4564-ab1b-a55c43c5f401 HTTP/1.1" 403 113 - - - latency=0.004000003s -454> 2022-04-12T16:07:53.955+0300 7f29f0856700 2 req 6350979191525555842 0.004000003s :post_period op status=0 -453> 2022-04-12T16:07:53.955+0300 7f29f0856700 2 req 6350979191525555842 0.004000003s :post_period http status=403 -452> 2022-04-12T16:07:53.955+0300 7f29f0856700 1 ====== req done req=0x7f2a3b415620 op status=0 http_status=403 latency=0.004000003s ====== -451> 2022-04-12T16:07:53.955+0300 7f29f305b700 5 req 10560051255912296523 0.004000003s :post_period error reading user info, uid=m7yH86NH1D44QiwaUUV9 can't authenticate -450> 2022-04-12T16:07:53.955+0300 7f29f305b700 5 req 10560051255912296523 0.004000003s :post_period error reading user info, uid=m7yH86NH1D44QiwaUUV9 can't authenticate -449> 2022-04-12T16:07:53.955+0300 7f29f305b700 5 req 10560051255912296523 0.004000003s :post_period Failed the auth strategy, reason=-2028 -448> 2022-04-12T16:07:53.955+0300 7f29f305b700 1 req 10560051255912296523 0.004000003s op->ERRORHANDLER: err_no=-2028 new_err_no=-2028 -447> 2022-04-12T16:07:53.955+0300 7f29f0856700 1 beast: 0x7f2a3b415620: 192.168.166.83 - - [12/Apr/2022:16:07:53.951 +0300] "POST /admin/realm/period?period=8b4c894f-cf43-4c56-9af0-1c2b54c74991&epoch=3&rgwx-zonegroup=784a05a5-2089-4564-ab1b-a55c43c5f401 HTTP/1.1" 403 113 - - - latency=0.004000003s -446> 2022-04-12T16:07:53.955+0300 7f29ee051700 2 req 10560051255912296523 0.004000003s :post_period op status=0 -445> 2022-04-12T16:07:53.955+0300 7f29ee051700 2 req 10560051255912296523 0.004000003s :post_period http status=403 -444> 2022-04-12T16:07:53.955+0300 7f29ee051700 1 ====== req done req=0x7f2a3b394620 op status=0 http_status=403 latency=0.004000003s ====== -443> 2022-04-12T16:07:53.955+0300 7f29ee051700 1 beast: 0x7f2a3b394620: 192.168.166.85 - - [12/Apr/2022:16:07:53.951 +0300] "POST /admin/realm/period?period=8b4c894f-cf43-4c56-9af0-1c2b54c74991&epoch=3&rgwx-zonegroup=784a05a5-2089-4564-ab1b-a55c43c5f401 HTTP/1.1" 403 113 - - - latency=0.004000003s -442> 2022-04-12T16:07:54.223+0300 7f2a0b08b700 10 monclient: tick -441> 2022-04-12T16:07:54.655+0300 7f2a1f0b3700 10 monclient: tick -440> 2022-04-12T16:07:54.655+0300 7f2a208b6700 10 monclient: _renew_subs -439> 2022-04-12T16:07:54.655+0300 7f2a208b6700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -438> 2022-04-12T16:07:54.731+0300 7f2a07884700 1 RGW-SYNC:meta: start -437> 2022-04-12T16:07:54.731+0300 7f2a07884700 1 RGW-SYNC:meta: realm epoch=2 period id=8b4c894f-cf43-4c56-9af0-1c2b54c74991 -436> 2022-04-12T16:07:54.735+0300 7f2a07884700 4 meta sync: cr:s=0x559e571e3180:op=0x559e59df4000:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -435> 2022-04-12T16:07:54.735+0300 7f2a07884700 4 meta sync: cr:s=0x559e59bc08c0:op=0x559e5578a800:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -434> 2022-04-12T16:07:54.735+0300 7f2a07884700 4 meta sync: cr:s=0x559e594b3540:op=0x559e59df4a00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -433> 2022-04-12T16:07:54.735+0300 7f2a07884700 4 meta sync: cr:s=0x559e5988b680:op=0x559e59df5400:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -432> 2022-04-12T16:07:54.735+0300 7f2a07884700 4 meta sync: cr:s=0x559e59bc1180:op=0x559e59d78a00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -431> 2022-04-12T16:07:54.735+0300 7f2a07884700 4 meta sync: cr:s=0x559e596883c0:op=0x559e59dfa800:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -430> 2022-04-12T16:07:54.735+0300 7f2a07884700 4 meta sync: cr:s=0x559e59689b80:op=0x559e59df7200:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -429> 2022-04-12T16:07:54.735+0300 7f2a07884700 4 meta sync: cr:s=0x559e55ddeb40:op=0x559e59df9400:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -428> 2022-04-12T16:07:54.735+0300 7f2a07884700 4 meta sync: cr:s=0x559e59bc17c0:op=0x559e59d79e00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -427> 2022-04-12T16:07:54.735+0300 7f2a07884700 4 meta sync: cr:s=0x559e5965aa00:op=0x559e59dfb200:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -426> 2022-04-12T16:07:54.735+0300 7f2a07884700 4 meta sync: cr:s=0x559e59bc0140:op=0x559e5578b200:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -425> 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e595b1e00:op=0x559e59dfca00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -424> 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e55dfd900:op=0x559e59df8a00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -423> 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e59bc12c0:op=0x559e59d78000:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -422> 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e59688a00:op=0x559e59e01400:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -421> 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e59689cc0:op=0x559e59e02800:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -420> 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e5988b900:op=0x559e59e09400:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -419> 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e55dfc8c0:op=0x559e59dfde00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -418> 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e571e3040:op=0x559e59e05400:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -417> 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e55dfcc80:op=0x559e59d7b200:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -416> 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e571e3540:op=0x559e59e00a00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -415> 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e55dfdb80:op=0x559e59dfe800:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -414> 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e596888c0:op=0x559e59e01e00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -413> 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e55dfd2c0:op=0x559e59df6800:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -412> 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e594b3b80:op=0x559e59e08a00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -411> 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e55dfc280:op=0x559e59dff200:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -410> 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e5965ac80:op=0x559e59e04000:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -409> 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e5988bcc0:op=0x559e59e0d400:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -408> 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e55dfc3c0:op=0x559e59dfd400:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -407> 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e594b37c0:op=0x559e59e09e00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -406> 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e5965ab40:op=0x559e59df9e00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -405> 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e571e2dc0:op=0x559e59e00000:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -404> 2022-04-12T16:07:54.739+0300 7f2a07884700 4 meta sync: cr:s=0x559e5988b540:op=0x559e59e08000:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -403> 2022-04-12T16:07:54.743+0300 7f2a07884700 4 meta sync: cr:s=0x559e55dfcdc0:op=0x559e59df5e00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -402> 2022-04-12T16:07:54.743+0300 7f2a07884700 4 meta sync: cr:s=0x559e59acc3c0:op=0x559e59d79400:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -401> 2022-04-12T16:07:54.743+0300 7f2a07884700 4 meta sync: cr:s=0x559e55dfc780:op=0x559e59d7a800:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -400> 2022-04-12T16:07:54.743+0300 7f2a07884700 4 meta sync: cr:s=0x559e57c2af00:op=0x559e59e17200:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -399> 2022-04-12T16:07:54.747+0300 7f2a07884700 4 meta sync: cr:s=0x559e571e2c80:op=0x559e59e06800:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -398> 2022-04-12T16:07:54.747+0300 7f2a07884700 4 meta sync: cr:s=0x559e59bc0dc0:op=0x559e59e11400:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -397> 2022-04-12T16:07:54.747+0300 7f2a07884700 4 meta sync: cr:s=0x559e59bcd540:op=0x559e59e14000:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -396> 2022-04-12T16:07:54.747+0300 7f2a07884700 4 meta sync: cr:s=0x559e57c2adc0:op=0x559e59e16800:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -395> 2022-04-12T16:07:54.747+0300 7f2a07884700 4 meta sync: cr:s=0x559e55a08140:op=0x559e59e14a00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -394> 2022-04-12T16:07:54.747+0300 7f2a07884700 4 meta sync: cr:s=0x559e564c1cc0:op=0x559e59e04a00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -393> 2022-04-12T16:07:54.747+0300 7f2a07884700 4 meta sync: cr:s=0x559e57c2b040:op=0x559e59e18000:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -392> 2022-04-12T16:07:54.747+0300 7f2a07884700 4 meta sync: cr:s=0x559e59bc1900:op=0x559e59e0e800:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -391> 2022-04-12T16:07:54.747+0300 7f2a07884700 4 meta sync: cr:s=0x559e59bc0640:op=0x559e59e0f200:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -390> 2022-04-12T16:07:54.751+0300 7f2a07884700 4 meta sync: cr:s=0x559e59689e00:op=0x559e59e03200:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -389> 2022-04-12T16:07:54.751+0300 7f2a07884700 4 meta sync: cr:s=0x559e55a083c0:op=0x559e59e0de00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -388> 2022-04-12T16:07:54.751+0300 7f2a07884700 4 meta sync: cr:s=0x559e571e3cc0:op=0x559e59e07200:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -387> 2022-04-12T16:07:54.751+0300 7f2a07884700 4 meta sync: cr:s=0x559e5988be00:op=0x559e59e0b200:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -386> 2022-04-12T16:07:54.751+0300 7f2a07884700 4 meta sync: cr:s=0x559e594b3680:op=0x559e59e0ca00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -385> 2022-04-12T16:07:54.751+0300 7f2a07884700 4 meta sync: cr:s=0x559e5988ba40:op=0x559e59df8000:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -384> 2022-04-12T16:07:54.751+0300 7f2a07884700 4 meta sync: cr:s=0x559e571e3900:op=0x559e59e05e00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -383> 2022-04-12T16:07:54.751+0300 7f2a07884700 4 meta sync: cr:s=0x559e5988bb80:op=0x559e59e0a800:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -382> 2022-04-12T16:07:54.751+0300 7f2a07884700 4 meta sync: cr:s=0x559e55a08780:op=0x559e59e15400:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -381> 2022-04-12T16:07:54.751+0300 7f2a07884700 4 meta sync: cr:s=0x559e5988b7c0:op=0x559e59e0c000:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -380> 2022-04-12T16:07:54.751+0300 7f2a07884700 4 meta sync: cr:s=0x559e595b1b80:op=0x559e59dfc000:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -379> 2022-04-12T16:07:54.751+0300 7f2a07884700 4 meta sync: cr:s=0x559e59bc1540:op=0x559e59e10a00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -378> 2022-04-12T16:07:54.751+0300 7f2a07884700 4 meta sync: cr:s=0x559e59bc0780:op=0x559e59e11e00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -377> 2022-04-12T16:07:54.751+0300 7f2a07884700 4 meta sync: cr:s=0x559e59bcc000:op=0x559e59e13200:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -376> 2022-04-12T16:07:54.755+0300 7f2a07884700 4 meta sync: cr:s=0x559e59bc1b80:op=0x559e59e10000:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -375> 2022-04-12T16:07:54.755+0300 7f2a07884700 4 meta sync: cr:s=0x559e55a08280:op=0x559e59e15e00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -374> 2022-04-12T16:07:54.755+0300 7f2a07884700 4 meta sync: cr:s=0x559e59bcc500:op=0x559e59e12800:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -373> 2022-04-12T16:07:54.803+0300 7f2a07884700 4 meta sync: cr:s=0x559e54a72640:op=0x559e55789e00:18RGWMetaSyncShardCR: adjusting marker pos=user:dashboard -372> 2022-04-12T16:07:54.811+0300 7f2a07884700 4 meta sync: cr:s=0x559e54a72640:op=0x559e55789e00:18RGWMetaSyncShardCR: adjusting marker pos=user:testuser -371> 2022-04-12T16:07:54.835+0300 7f2a07884700 4 meta sync: cr:s=0x559e54a72640:op=0x559e55789e00:18RGWMetaSyncShardCR: adjusting marker pos=user:testuser -370> 2022-04-12T16:07:54.835+0300 7f2a07884700 4 meta sync: cr:s=0x559e54a72640:op=0x559e55789e00:18RGWMetaSyncShardCR: saving marker pos= realm_epoch=2 -369> 2022-04-12T16:07:55.223+0300 7f2a0b08b700 10 monclient: tick -368> 2022-04-12T16:07:55.655+0300 7f2a1f0b3700 10 monclient: tick -367> 2022-04-12T16:07:55.823+0300 7f28f6662700 10 monclient: tick -366> 2022-04-12T16:07:56.223+0300 7f2a0b08b700 10 monclient: tick -365> 2022-04-12T16:07:56.659+0300 7f2a1f0b3700 10 monclient: tick -364> 2022-04-12T16:07:56.827+0300 7f28f6662700 10 monclient: tick -363> 2022-04-12T16:07:57.227+0300 7f2a0b08b700 10 monclient: tick -362> 2022-04-12T16:07:57.655+0300 7f2a1f0b3700 10 monclient: tick -361> 2022-04-12T16:07:57.823+0300 7f28f6662700 10 monclient: tick -360> 2022-04-12T16:07:57.827+0300 7f28f7e65700 10 monclient: _renew_subs -359> 2022-04-12T16:07:57.827+0300 7f28f7e65700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -358> 2022-04-12T16:07:58.223+0300 7f2a0b08b700 10 monclient: tick -357> 2022-04-12T16:07:58.659+0300 7f2a1f0b3700 10 monclient: tick -356> 2022-04-12T16:07:58.823+0300 7f28f6662700 10 monclient: tick -355> 2022-04-12T16:07:59.223+0300 7f2a0b08b700 10 monclient: tick -354> 2022-04-12T16:07:59.655+0300 7f2a208b6700 10 monclient: _renew_subs -353> 2022-04-12T16:07:59.655+0300 7f2a208b6700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -352> 2022-04-12T16:07:59.655+0300 7f2a1f0b3700 10 monclient: tick -351> 2022-04-12T16:07:59.823+0300 7f28f6662700 10 monclient: tick -350> 2022-04-12T16:08:00.223+0300 7f2a0b08b700 10 monclient: tick -349> 2022-04-12T16:08:00.659+0300 7f2a1f0b3700 10 monclient: tick -348> 2022-04-12T16:08:00.823+0300 7f28f6662700 10 monclient: tick -347> 2022-04-12T16:08:01.227+0300 7f2a0b08b700 10 monclient: tick -346> 2022-04-12T16:08:01.655+0300 7f2a1f0b3700 10 monclient: tick -345> 2022-04-12T16:08:01.827+0300 7f28f6662700 10 monclient: tick -344> 2022-04-12T16:08:02.227+0300 7f2a0b08b700 10 monclient: tick -343> 2022-04-12T16:08:02.659+0300 7f2a1f0b3700 10 monclient: tick -342> 2022-04-12T16:08:02.827+0300 7f28f6662700 10 monclient: tick -341> 2022-04-12T16:08:02.827+0300 7f28f7e65700 10 monclient: _renew_subs -340> 2022-04-12T16:08:02.827+0300 7f28f7e65700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -339> 2022-04-12T16:08:03.223+0300 7f2a0b08b700 10 monclient: tick -338> 2022-04-12T16:08:03.655+0300 7f2a1f0b3700 10 monclient: tick -337> 2022-04-12T16:08:03.827+0300 7f28f6662700 10 monclient: tick -336> 2022-04-12T16:08:04.227+0300 7f2a0b08b700 10 monclient: tick -335> 2022-04-12T16:08:04.655+0300 7f2a208b6700 10 monclient: _renew_subs -334> 2022-04-12T16:08:04.655+0300 7f2a208b6700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -333> 2022-04-12T16:08:04.655+0300 7f2a1f0b3700 10 monclient: tick -332> 2022-04-12T16:08:04.823+0300 7f28f6662700 10 monclient: tick -331> 2022-04-12T16:08:05.223+0300 7f2a0b08b700 10 monclient: tick -330> 2022-04-12T16:08:05.659+0300 7f2a1f0b3700 10 monclient: tick -329> 2022-04-12T16:08:05.823+0300 7f28f6662700 10 monclient: tick -328> 2022-04-12T16:08:06.227+0300 7f2a0b08b700 10 monclient: tick -327> 2022-04-12T16:08:06.659+0300 7f2a1f0b3700 10 monclient: tick -326> 2022-04-12T16:08:06.823+0300 7f28f6662700 10 monclient: tick -325> 2022-04-12T16:08:07.227+0300 7f2a0b08b700 10 monclient: tick -324> 2022-04-12T16:08:07.655+0300 7f2a1f0b3700 10 monclient: tick -323> 2022-04-12T16:08:07.827+0300 7f28f6662700 10 monclient: tick -322> 2022-04-12T16:08:08.227+0300 7f2a0b08b700 10 monclient: tick -321> 2022-04-12T16:08:08.659+0300 7f2a1f0b3700 10 monclient: tick -320> 2022-04-12T16:08:08.823+0300 7f28f6662700 10 monclient: tick -319> 2022-04-12T16:08:09.223+0300 7f2a0b08b700 10 monclient: tick -318> 2022-04-12T16:08:09.659+0300 7f2a1f0b3700 10 monclient: tick -317> 2022-04-12T16:08:09.823+0300 7f28f6662700 10 monclient: tick -316> 2022-04-12T16:08:10.223+0300 7f2a0b08b700 10 monclient: tick -315> 2022-04-12T16:08:10.655+0300 7f2a1f0b3700 10 monclient: tick -314> 2022-04-12T16:08:10.827+0300 7f28f6662700 10 monclient: tick -313> 2022-04-12T16:08:11.211+0300 7f2a0e091700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start -312> 2022-04-12T16:08:11.223+0300 7f2a0b08b700 10 monclient: tick -311> 2022-04-12T16:08:11.655+0300 7f2a1f0b3700 10 monclient: tick -310> 2022-04-12T16:08:11.827+0300 7f28f6662700 10 monclient: tick -309> 2022-04-12T16:08:12.223+0300 7f2a0b08b700 10 monclient: tick -308> 2022-04-12T16:08:12.655+0300 7f2a1f0b3700 10 monclient: tick -307> 2022-04-12T16:08:12.823+0300 7f28f6662700 10 monclient: tick -306> 2022-04-12T16:08:13.223+0300 7f2a0b08b700 10 monclient: tick -305> 2022-04-12T16:08:13.659+0300 7f2a1f0b3700 10 monclient: tick -304> 2022-04-12T16:08:13.823+0300 7f28f6662700 10 monclient: tick -303> 2022-04-12T16:08:14.227+0300 7f2a0b08b700 10 monclient: tick -302> 2022-04-12T16:08:14.659+0300 7f2a1f0b3700 10 monclient: tick -301> 2022-04-12T16:08:14.827+0300 7f28f6662700 10 monclient: tick -300> 2022-04-12T16:08:15.227+0300 7f2a0b08b700 10 monclient: tick -299> 2022-04-12T16:08:15.659+0300 7f2a1f0b3700 10 monclient: tick -298> 2022-04-12T16:08:15.827+0300 7f28f6662700 10 monclient: tick -297> 2022-04-12T16:08:16.227+0300 7f2a0b08b700 10 monclient: tick -296> 2022-04-12T16:08:16.659+0300 7f2a1f0b3700 10 monclient: tick -295> 2022-04-12T16:08:16.827+0300 7f28f6662700 10 monclient: tick -294> 2022-04-12T16:08:17.227+0300 7f2a0b08b700 10 monclient: tick -293> 2022-04-12T16:08:17.659+0300 7f2a1f0b3700 10 monclient: tick -292> 2022-04-12T16:08:17.827+0300 7f28f6662700 10 monclient: tick -291> 2022-04-12T16:08:18.227+0300 7f2a0b08b700 10 monclient: tick -290> 2022-04-12T16:08:18.659+0300 7f2a1f0b3700 10 monclient: tick -289> 2022-04-12T16:08:18.827+0300 7f28f6662700 10 monclient: tick -288> 2022-04-12T16:08:19.227+0300 7f2a0b08b700 10 monclient: tick -287> 2022-04-12T16:08:19.659+0300 7f2a1f0b3700 10 monclient: tick -286> 2022-04-12T16:08:19.827+0300 7f28f6662700 10 monclient: tick -285> 2022-04-12T16:08:20.227+0300 7f2a0b08b700 10 monclient: tick -284> 2022-04-12T16:08:20.659+0300 7f2a1f0b3700 10 monclient: tick -283> 2022-04-12T16:08:20.827+0300 7f28f6662700 10 monclient: tick -282> 2022-04-12T16:08:21.227+0300 7f2a0b08b700 10 monclient: tick -281> 2022-04-12T16:08:21.659+0300 7f2a1f0b3700 10 monclient: tick -280> 2022-04-12T16:08:21.827+0300 7f28f6662700 10 monclient: tick -279> 2022-04-12T16:08:22.227+0300 7f2a0b08b700 10 monclient: tick -278> 2022-04-12T16:08:22.659+0300 7f2a1f0b3700 10 monclient: tick -277> 2022-04-12T16:08:22.827+0300 7f28f6662700 10 monclient: tick -276> 2022-04-12T16:08:23.227+0300 7f2a0b08b700 10 monclient: tick -275> 2022-04-12T16:08:23.659+0300 7f2a1f0b3700 10 monclient: tick -274> 2022-04-12T16:08:23.827+0300 7f28f6662700 10 monclient: tick -273> 2022-04-12T16:08:23.959+0300 7f29f505f700 1 ====== starting new request req=0x7f2a3b313620 ===== -272> 2022-04-12T16:08:23.959+0300 7f29f505f700 2 req 8920709381089314344 0.000000000s initializing for trans_id = tx000007bccb9eb0c52c628-00625579c7-1659-sync -271> 2022-04-12T16:08:23.959+0300 7f29f505f700 2 req 8920709381089314344 0.000000000s getting op 4 -270> 2022-04-12T16:08:23.959+0300 7f29f505f700 2 req 8920709381089314344 0.000000000s :post_period verifying requester -269> 2022-04-12T16:08:23.959+0300 7f29f505f700 2 req 8920709381089314344 0.000000000s :post_period normalizing buckets and tenants -268> 2022-04-12T16:08:23.959+0300 7f29f505f700 2 req 8920709381089314344 0.000000000s :post_period init permissions -267> 2022-04-12T16:08:23.959+0300 7f29f505f700 2 req 8920709381089314344 0.000000000s :post_period recalculating target -266> 2022-04-12T16:08:23.959+0300 7f29f505f700 2 req 8920709381089314344 0.000000000s :post_period reading permissions -265> 2022-04-12T16:08:23.959+0300 7f29f505f700 2 req 8920709381089314344 0.000000000s :post_period init op -264> 2022-04-12T16:08:23.959+0300 7f29f505f700 2 req 8920709381089314344 0.000000000s :post_period verifying op mask -263> 2022-04-12T16:08:23.959+0300 7f29f505f700 2 req 8920709381089314344 0.000000000s :post_period verifying op permissions -262> 2022-04-12T16:08:23.959+0300 7f29f505f700 2 overriding permissions due to system operation -261> 2022-04-12T16:08:23.959+0300 7f29f505f700 2 req 8920709381089314344 0.000000000s :post_period verifying op params -260> 2022-04-12T16:08:23.959+0300 7f29f505f700 2 req 8920709381089314344 0.000000000s :post_period pre-executing -259> 2022-04-12T16:08:23.959+0300 7f29f505f700 2 req 8920709381089314344 0.000000000s :post_period executing -258> 2022-04-12T16:08:23.959+0300 7f29cc80e700 1 ====== starting new request req=0x7f2a3b415620 ===== -257> 2022-04-12T16:08:23.959+0300 7f29cc80e700 2 req 8609644640538370824 0.000000000s initializing for trans_id = tx00000777b9a2f49a2a308-00625579c7-1659-sync -256> 2022-04-12T16:08:23.959+0300 7f29cc80e700 2 req 8609644640538370824 0.000000000s getting op 4 -255> 2022-04-12T16:08:23.959+0300 7f29cc80e700 2 req 8609644640538370824 0.000000000s :post_period verifying requester -254> 2022-04-12T16:08:23.959+0300 7f29cc80e700 2 req 8609644640538370824 0.000000000s :post_period normalizing buckets and tenants -253> 2022-04-12T16:08:23.959+0300 7f29cc80e700 2 req 8609644640538370824 0.000000000s :post_period init permissions -252> 2022-04-12T16:08:23.959+0300 7f29cc80e700 2 req 8609644640538370824 0.000000000s :post_period recalculating target -251> 2022-04-12T16:08:23.959+0300 7f29cc80e700 2 req 8609644640538370824 0.000000000s :post_period reading permissions -250> 2022-04-12T16:08:23.959+0300 7f29cc80e700 2 req 8609644640538370824 0.000000000s :post_period init op -249> 2022-04-12T16:08:23.959+0300 7f29cc80e700 2 req 8609644640538370824 0.000000000s :post_period verifying op mask -248> 2022-04-12T16:08:23.959+0300 7f29cc80e700 2 req 8609644640538370824 0.000000000s :post_period verifying op permissions -247> 2022-04-12T16:08:23.959+0300 7f29cc80e700 2 overriding permissions due to system operation -246> 2022-04-12T16:08:23.959+0300 7f29cc80e700 2 req 8609644640538370824 0.000000000s :post_period verifying op params -245> 2022-04-12T16:08:23.959+0300 7f29cc80e700 2 req 8609644640538370824 0.000000000s :post_period pre-executing -244> 2022-04-12T16:08:23.959+0300 7f29cc80e700 2 req 8609644640538370824 0.000000000s :post_period executing -243> 2022-04-12T16:08:23.959+0300 7f29c8005700 1 ====== starting new request req=0x7f2a3b394620 ===== -242> 2022-04-12T16:08:23.959+0300 7f29c8005700 2 req 14231248929957790225 0.000000000s initializing for trans_id = tx00000c57f8d805c4ac211-00625579c7-1659-sync -241> 2022-04-12T16:08:23.959+0300 7f29c8005700 2 req 14231248929957790225 0.000000000s getting op 4 -240> 2022-04-12T16:08:23.959+0300 7f29c8005700 2 req 14231248929957790225 0.000000000s :post_period verifying requester -239> 2022-04-12T16:08:23.959+0300 7f29c8005700 2 req 14231248929957790225 0.000000000s :post_period normalizing buckets and tenants -238> 2022-04-12T16:08:23.959+0300 7f29c8005700 2 req 14231248929957790225 0.000000000s :post_period init permissions -237> 2022-04-12T16:08:23.959+0300 7f29c8005700 2 req 14231248929957790225 0.000000000s :post_period recalculating target -236> 2022-04-12T16:08:23.959+0300 7f29c8005700 2 req 14231248929957790225 0.000000000s :post_period reading permissions -235> 2022-04-12T16:08:23.959+0300 7f29c8005700 2 req 14231248929957790225 0.000000000s :post_period init op -234> 2022-04-12T16:08:23.959+0300 7f29c8005700 2 req 14231248929957790225 0.000000000s :post_period verifying op mask -233> 2022-04-12T16:08:23.959+0300 7f29c8005700 2 req 14231248929957790225 0.000000000s :post_period verifying op permissions -232> 2022-04-12T16:08:23.959+0300 7f29c8005700 2 overriding permissions due to system operation -231> 2022-04-12T16:08:23.959+0300 7f29c8005700 2 req 14231248929957790225 0.000000000s :post_period verifying op params -230> 2022-04-12T16:08:23.959+0300 7f29c8005700 2 req 14231248929957790225 0.000000000s :post_period pre-executing -229> 2022-04-12T16:08:23.959+0300 7f29c8005700 2 req 14231248929957790225 0.000000000s :post_period executing -228> 2022-04-12T16:08:23.963+0300 7f29ac7ce700 4 req 8920709381089314344 0.004000003s :post_period already have epoch >= 3 for period 8b4c894f-cf43-4c56-9af0-1c2b54c74991 -227> 2022-04-12T16:08:23.963+0300 7f29ac7ce700 2 req 8920709381089314344 0.004000003s :post_period completing -226> 2022-04-12T16:08:23.963+0300 7f29ac7ce700 2 req 8920709381089314344 0.004000003s :post_period op status=0 -225> 2022-04-12T16:08:23.963+0300 7f29ac7ce700 2 req 8920709381089314344 0.004000003s :post_period http status=200 -224> 2022-04-12T16:08:23.963+0300 7f29ac7ce700 1 ====== req done req=0x7f2a3b313620 op status=0 http_status=200 latency=0.004000003s ====== -223> 2022-04-12T16:08:23.963+0300 7f29ac7ce700 1 beast: 0x7f2a3b313620: 192.168.166.84 - rgw_zone_user [12/Apr/2022:16:08:23.959 +0300] "POST /admin/realm/period?period=8b4c894f-cf43-4c56-9af0-1c2b54c74991&epoch=3&rgwx-zonegroup=784a05a5-2089-4564-ab1b-a55c43c5f401 HTTP/1.1" 200 1868 - - - latency=0.004000003s -222> 2022-04-12T16:08:23.963+0300 7f29a57c0700 4 req 14231248929957790225 0.004000003s :post_period already have epoch >= 3 for period 8b4c894f-cf43-4c56-9af0-1c2b54c74991 -221> 2022-04-12T16:08:23.963+0300 7f29a57c0700 2 req 14231248929957790225 0.004000003s :post_period completing -220> 2022-04-12T16:08:23.963+0300 7f29a57c0700 2 req 14231248929957790225 0.004000003s :post_period op status=0 -219> 2022-04-12T16:08:23.963+0300 7f29a57c0700 2 req 14231248929957790225 0.004000003s :post_period http status=200 -218> 2022-04-12T16:08:23.963+0300 7f29a57c0700 1 ====== req done req=0x7f2a3b394620 op status=0 http_status=200 latency=0.004000003s ====== -217> 2022-04-12T16:08:23.963+0300 7f29a57c0700 1 beast: 0x7f2a3b394620: 192.168.166.85 - rgw_zone_user [12/Apr/2022:16:08:23.959 +0300] "POST /admin/realm/period?period=8b4c894f-cf43-4c56-9af0-1c2b54c74991&epoch=3&rgwx-zonegroup=784a05a5-2089-4564-ab1b-a55c43c5f401 HTTP/1.1" 200 1868 - - - latency=0.004000003s -216> 2022-04-12T16:08:23.967+0300 7f299e7b2700 4 req 8609644640538370824 0.008000007s :post_period already have epoch >= 3 for period 8b4c894f-cf43-4c56-9af0-1c2b54c74991 -215> 2022-04-12T16:08:23.967+0300 7f299e7b2700 2 req 8609644640538370824 0.008000007s :post_period completing -214> 2022-04-12T16:08:23.967+0300 7f299e7b2700 2 req 8609644640538370824 0.008000007s :post_period op status=0 -213> 2022-04-12T16:08:23.967+0300 7f299e7b2700 2 req 8609644640538370824 0.008000007s :post_period http status=200 -212> 2022-04-12T16:08:23.967+0300 7f299e7b2700 1 ====== req done req=0x7f2a3b415620 op status=0 http_status=200 latency=0.008000007s ====== -211> 2022-04-12T16:08:23.967+0300 7f299e7b2700 1 beast: 0x7f2a3b415620: 192.168.166.83 - rgw_zone_user [12/Apr/2022:16:08:23.959 +0300] "POST /admin/realm/period?period=8b4c894f-cf43-4c56-9af0-1c2b54c74991&epoch=3&rgwx-zonegroup=784a05a5-2089-4564-ab1b-a55c43c5f401 HTTP/1.1" 200 1868 - - - latency=0.008000007s -210> 2022-04-12T16:08:24.227+0300 7f2a0b08b700 10 monclient: tick -209> 2022-04-12T16:08:24.663+0300 7f2a1f0b3700 10 monclient: tick -208> 2022-04-12T16:08:24.827+0300 7f28f6662700 10 monclient: tick -207> 2022-04-12T16:08:25.227+0300 7f2a0b08b700 10 monclient: tick -206> 2022-04-12T16:08:25.663+0300 7f2a1f0b3700 10 monclient: tick -205> 2022-04-12T16:08:25.827+0300 7f28f6662700 10 monclient: tick -204> 2022-04-12T16:08:26.231+0300 7f2a0b08b700 10 monclient: tick -203> 2022-04-12T16:08:26.663+0300 7f2a1f0b3700 10 monclient: tick -202> 2022-04-12T16:08:26.831+0300 7f28f6662700 10 monclient: tick -201> 2022-04-12T16:08:27.227+0300 7f2a0b08b700 10 monclient: tick -200> 2022-04-12T16:08:27.663+0300 7f2a1f0b3700 10 monclient: tick -199> 2022-04-12T16:08:27.827+0300 7f28f6662700 10 monclient: tick -198> 2022-04-12T16:08:28.227+0300 7f2a0b08b700 10 monclient: tick -197> 2022-04-12T16:08:28.659+0300 7f2a1f0b3700 10 monclient: tick -196> 2022-04-12T16:08:28.827+0300 7f28f6662700 10 monclient: tick -195> 2022-04-12T16:08:29.227+0300 7f2a0b08b700 10 monclient: tick -194> 2022-04-12T16:08:29.663+0300 7f2a1f0b3700 10 monclient: tick -193> 2022-04-12T16:08:29.827+0300 7f28f6662700 10 monclient: tick -192> 2022-04-12T16:08:30.227+0300 7f2a0b08b700 10 monclient: tick -191> 2022-04-12T16:08:30.663+0300 7f2a1f0b3700 10 monclient: tick -190> 2022-04-12T16:08:30.827+0300 7f28f6662700 10 monclient: tick -189> 2022-04-12T16:08:31.227+0300 7f2a0b08b700 10 monclient: tick -188> 2022-04-12T16:08:31.659+0300 7f2a1f0b3700 10 monclient: tick -187> 2022-04-12T16:08:31.831+0300 7f28f6662700 10 monclient: tick -186> 2022-04-12T16:08:32.227+0300 7f2a0b08b700 10 monclient: tick -185> 2022-04-12T16:08:32.659+0300 7f2a1f0b3700 10 monclient: tick -184> 2022-04-12T16:08:32.827+0300 7f28f6662700 10 monclient: tick -183> 2022-04-12T16:08:33.211+0300 7f2a0e091700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start -182> 2022-04-12T16:08:33.227+0300 7f2a0b08b700 10 monclient: tick -181> 2022-04-12T16:08:33.659+0300 7f2a1f0b3700 10 monclient: tick -180> 2022-04-12T16:08:33.831+0300 7f28f6662700 10 monclient: tick -179> 2022-04-12T16:08:34.227+0300 7f2a0b08b700 10 monclient: tick -178> 2022-04-12T16:08:34.659+0300 7f2a1f0b3700 10 monclient: tick -177> 2022-04-12T16:08:34.827+0300 7f28f6662700 10 monclient: tick -176> 2022-04-12T16:08:35.227+0300 7f2a0b08b700 10 monclient: tick -175> 2022-04-12T16:08:35.659+0300 7f2a1f0b3700 10 monclient: tick -174> 2022-04-12T16:08:35.827+0300 7f28f6662700 10 monclient: tick -173> 2022-04-12T16:08:36.227+0300 7f2a0b08b700 10 monclient: tick -172> 2022-04-12T16:08:36.659+0300 7f2a1f0b3700 10 monclient: tick -171> 2022-04-12T16:08:36.827+0300 7f28f6662700 10 monclient: tick -170> 2022-04-12T16:08:37.231+0300 7f2a0b08b700 10 monclient: tick -169> 2022-04-12T16:08:37.659+0300 7f2a1f0b3700 10 monclient: tick -168> 2022-04-12T16:08:37.831+0300 7f28f6662700 10 monclient: tick -167> 2022-04-12T16:08:38.227+0300 7f2a0b08b700 10 monclient: tick -166> 2022-04-12T16:08:38.663+0300 7f2a1f0b3700 10 monclient: tick -165> 2022-04-12T16:08:38.831+0300 7f28f6662700 10 monclient: tick -164> 2022-04-12T16:08:39.227+0300 7f2a0b08b700 10 monclient: tick -163> 2022-04-12T16:08:39.659+0300 7f2a1f0b3700 10 monclient: tick -162> 2022-04-12T16:08:39.827+0300 7f28f6662700 10 monclient: tick -161> 2022-04-12T16:08:40.227+0300 7f2a0b08b700 10 monclient: tick -160> 2022-04-12T16:08:40.659+0300 7f2a1f0b3700 10 monclient: tick -159> 2022-04-12T16:08:40.827+0300 7f28f6662700 10 monclient: tick -158> 2022-04-12T16:08:41.231+0300 7f2a0b08b700 10 monclient: tick -157> 2022-04-12T16:08:41.659+0300 7f2a1f0b3700 10 monclient: tick -156> 2022-04-12T16:08:41.831+0300 7f28f6662700 10 monclient: tick -155> 2022-04-12T16:08:42.227+0300 7f2a0b08b700 10 monclient: tick -154> 2022-04-12T16:08:42.659+0300 7f2a1f0b3700 10 monclient: tick -153> 2022-04-12T16:08:42.827+0300 7f28f6662700 10 monclient: tick -152> 2022-04-12T16:08:43.227+0300 7f2a0b08b700 10 monclient: tick -151> 2022-04-12T16:08:43.659+0300 7f2a1f0b3700 10 monclient: tick -150> 2022-04-12T16:08:43.831+0300 7f28f6662700 10 monclient: tick -149> 2022-04-12T16:08:44.227+0300 7f2a0b08b700 10 monclient: tick -148> 2022-04-12T16:08:44.659+0300 7f2a1f0b3700 10 monclient: tick -147> 2022-04-12T16:08:44.831+0300 7f28f6662700 10 monclient: tick -146> 2022-04-12T16:08:45.231+0300 7f2a0b08b700 10 monclient: tick -145> 2022-04-12T16:08:45.663+0300 7f2a1f0b3700 10 monclient: tick -144> 2022-04-12T16:08:45.831+0300 7f28f6662700 10 monclient: tick -143> 2022-04-12T16:08:46.231+0300 7f2a0b08b700 10 monclient: tick -142> 2022-04-12T16:08:46.663+0300 7f2a1f0b3700 10 monclient: tick -141> 2022-04-12T16:08:46.827+0300 7f28f6662700 10 monclient: tick -140> 2022-04-12T16:08:47.231+0300 7f2a0b08b700 10 monclient: tick -139> 2022-04-12T16:08:47.663+0300 7f2a1f0b3700 10 monclient: tick -138> 2022-04-12T16:08:47.831+0300 7f28f6662700 10 monclient: tick -137> 2022-04-12T16:08:48.231+0300 7f2a0b08b700 10 monclient: tick -136> 2022-04-12T16:08:48.663+0300 7f2a1f0b3700 10 monclient: tick -135> 2022-04-12T16:08:48.831+0300 7f28f6662700 10 monclient: tick -134> 2022-04-12T16:08:49.231+0300 7f2a0b08b700 10 monclient: tick -133> 2022-04-12T16:08:49.663+0300 7f2a1f0b3700 10 monclient: tick -132> 2022-04-12T16:08:49.667+0300 7f2a208b6700 10 monclient: _renew_subs -131> 2022-04-12T16:08:49.667+0300 7f2a208b6700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -130> 2022-04-12T16:08:49.831+0300 7f28f6662700 10 monclient: tick -129> 2022-04-12T16:08:50.231+0300 7f2a0b08b700 10 monclient: tick -128> 2022-04-12T16:08:50.663+0300 7f2a1f0b3700 10 monclient: tick -127> 2022-04-12T16:08:50.831+0300 7f28f6662700 10 monclient: tick -126> 2022-04-12T16:08:51.231+0300 7f2a0b08b700 10 monclient: tick -125> 2022-04-12T16:08:51.663+0300 7f2a1f0b3700 10 monclient: tick -124> 2022-04-12T16:08:51.831+0300 7f28f6662700 10 monclient: tick -123> 2022-04-12T16:08:52.231+0300 7f2a0b08b700 10 monclient: tick -122> 2022-04-12T16:08:52.663+0300 7f2a1f0b3700 10 monclient: tick -121> 2022-04-12T16:08:52.691+0300 7f29b5fe1700 1 ====== starting new request req=0x7f28f565f620 ===== -120> 2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s initializing for trans_id = tx00000ecd2ee05e4c9a4ca-00625579e4-1659-sync -119> 2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s getting op 4 -118> 2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s :mdlog_notify verifying requester -117> 2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s :mdlog_notify normalizing buckets and tenants -116> 2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s :mdlog_notify init permissions -115> 2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s :mdlog_notify recalculating target -114> 2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s :mdlog_notify reading permissions -113> 2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s :mdlog_notify init op -112> 2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s :mdlog_notify verifying op mask -111> 2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s :mdlog_notify verifying op permissions -110> 2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 overriding permissions due to system operation -109> 2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s :mdlog_notify verifying op params -108> 2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s :mdlog_notify pre-executing -107> 2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s :mdlog_notify executing -106> 2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s :mdlog_notify completing -105> 2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s :mdlog_notify op status=0 -104> 2022-04-12T16:08:52.691+0300 7f29b5fe1700 2 req 17064963647140897994 0.000000000s :mdlog_notify http status=200 -103> 2022-04-12T16:08:52.691+0300 7f29b5fe1700 1 ====== req done req=0x7f28f565f620 op status=0 http_status=200 latency=0.000000000s ====== -102> 2022-04-12T16:08:52.691+0300 7f29b5fe1700 1 beast: 0x7f28f565f620: 192.168.166.85 - rgw_zone_user [12/Apr/2022:16:08:52.691 +0300] "POST /admin/log?type=metadata¬ify&rgwx-zonegroup=784a05a5-2089-4564-ab1b-a55c43c5f401 HTTP/1.1" 200 0 - - - latency=0.000000000s -101> 2022-04-12T16:08:52.699+0300 7f2a07884700 4 meta sync: cr:s=0x559e57c2af00:op=0x559e59e17200:18RGWMetaSyncShardCR: adjusting marker pos= -100> 2022-04-12T16:08:52.759+0300 7f2a1409d700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -99> 2022-04-12T16:08:52.759+0300 7f2a200b5700 10 monclient: handle_get_version_reply finishing 5 version 76 -98> 2022-04-12T16:08:52.759+0300 7f2a1409d700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -97> 2022-04-12T16:08:52.831+0300 7f28f6662700 10 monclient: tick -96> 2022-04-12T16:08:52.831+0300 7f28f7e65700 10 monclient: _renew_subs -95> 2022-04-12T16:08:52.831+0300 7f28f7e65700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -94> 2022-04-12T16:08:53.231+0300 7f2a0b08b700 10 monclient: tick -93> 2022-04-12T16:08:53.663+0300 7f2a1f0b3700 10 monclient: tick -92> 2022-04-12T16:08:53.751+0300 7f2a1409d700 10 start_mon_command cmd=[{"prefix": "osd pool application enable","pool": "sync.rgw.buckets.index","app": "rgw"}] -91> 2022-04-12T16:08:53.751+0300 7f2a1409d700 10 monclient: _send_command 3 [{"prefix": "osd pool application enable","pool": "sync.rgw.buckets.index","app": "rgw"}] -90> 2022-04-12T16:08:53.751+0300 7f2a1409d700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -89> 2022-04-12T16:08:53.831+0300 7f28f6662700 10 monclient: tick -88> 2022-04-12T16:08:54.231+0300 7f2a0b08b700 10 monclient: tick -87> 2022-04-12T16:08:54.663+0300 7f2a1f0b3700 10 monclient: tick -86> 2022-04-12T16:08:54.667+0300 7f2a208b6700 10 monclient: _renew_subs -85> 2022-04-12T16:08:54.667+0300 7f2a208b6700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -84> 2022-04-12T16:08:54.759+0300 7f2a200b5700 10 monclient: handle_mon_command_ack 3 [{"prefix": "osd pool application enable","pool": "sync.rgw.buckets.index","app": "rgw"}] -83> 2022-04-12T16:08:54.759+0300 7f2a200b5700 10 monclient: _finish_command 3 = system:0 enabled application 'rgw' on pool 'sync.rgw.buckets.index' -82> 2022-04-12T16:08:54.759+0300 7f2a1409d700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -81> 2022-04-12T16:08:54.763+0300 7f2a200b5700 10 monclient: handle_get_version_reply finishing 6 version 78 -80> 2022-04-12T16:08:54.763+0300 7f2a1409d700 10 start_mon_command cmd=[{"prefix": "osd pool set", "pool": "sync.rgw.buckets.index", "var": "pg_autoscale_bias", "val": "4"}] -79> 2022-04-12T16:08:54.763+0300 7f2a1409d700 10 monclient: _send_command 4 [{"prefix": "osd pool set", "pool": "sync.rgw.buckets.index", "var": "pg_autoscale_bias", "val": "4"}] -78> 2022-04-12T16:08:54.763+0300 7f2a1409d700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -77> 2022-04-12T16:08:54.783+0300 7f2a07884700 4 meta sync: cr:s=0x559e57c2af00:op=0x559e59e17200:18RGWMetaSyncShardCR: adjusting marker pos=1_1649768932.547706_90.1 -76> 2022-04-12T16:08:54.783+0300 7f2a07884700 4 meta sync: cr:s=0x559e57c2af00:op=0x559e59e17200:18RGWMetaSyncShardCR: adjusting marker pos=1_1649768932.547706_90.1 -75> 2022-04-12T16:08:54.831+0300 7f28f6662700 10 monclient: tick -74> 2022-04-12T16:08:55.215+0300 7f2a0e091700 2 rgw data changes log: RGWDataChangesLog::ChangesRenewThread: start -73> 2022-04-12T16:08:55.231+0300 7f2a0b08b700 10 monclient: tick -72> 2022-04-12T16:08:55.663+0300 7f2a1f0b3700 10 monclient: tick -71> 2022-04-12T16:08:55.767+0300 7f2a200b5700 10 monclient: handle_mon_command_ack 4 [{"prefix": "osd pool set", "pool": "sync.rgw.buckets.index", "var": "pg_autoscale_bias", "val": "4"}] -70> 2022-04-12T16:08:55.767+0300 7f2a200b5700 10 monclient: _finish_command 4 = system:0 set pool 13 pg_autoscale_bias to 4 -69> 2022-04-12T16:08:55.767+0300 7f2a1409d700 10 start_mon_command cmd=[{"prefix": "osd pool set", "pool": "sync.rgw.buckets.index", "var": "pg_num_min", "val": "8"}] -68> 2022-04-12T16:08:55.767+0300 7f2a1409d700 10 monclient: _send_command 5 [{"prefix": "osd pool set", "pool": "sync.rgw.buckets.index", "var": "pg_num_min", "val": "8"}] -67> 2022-04-12T16:08:55.767+0300 7f2a1409d700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -66> 2022-04-12T16:08:55.831+0300 7f28f6662700 10 monclient: tick -65> 2022-04-12T16:08:56.231+0300 7f2a0b08b700 10 monclient: tick -64> 2022-04-12T16:08:56.663+0300 7f2a1f0b3700 10 monclient: tick -63> 2022-04-12T16:08:56.775+0300 7f2a200b5700 10 monclient: handle_mon_command_ack 5 [{"prefix": "osd pool set", "pool": "sync.rgw.buckets.index", "var": "pg_num_min", "val": "8"}] -62> 2022-04-12T16:08:56.775+0300 7f2a200b5700 10 monclient: _finish_command 5 = system:0 set pool 13 pg_num_min to 8 -61> 2022-04-12T16:08:56.775+0300 7f2a1409d700 10 start_mon_command cmd=[{"prefix": "osd pool set", "pool": "sync.rgw.buckets.index", "var": "recovery_priority": "5"}] -60> 2022-04-12T16:08:56.775+0300 7f2a1409d700 10 monclient: _send_command 6 [{"prefix": "osd pool set", "pool": "sync.rgw.buckets.index", "var": "recovery_priority": "5"}] -59> 2022-04-12T16:08:56.775+0300 7f2a1409d700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -58> 2022-04-12T16:08:56.775+0300 7f2a200b5700 10 monclient: handle_mon_command_ack 6 [{"prefix": "osd pool set", "pool": "sync.rgw.buckets.index", "var": "recovery_priority": "5"}] -57> 2022-04-12T16:08:56.775+0300 7f2a200b5700 10 monclient: _finish_command 6 = mon:22 unparseable JSON {"prefix": "osd pool set", "pool": "sync.rgw.buckets.index", "var": "recovery_priority": "5"} -56> 2022-04-12T16:08:56.831+0300 7f28f6662700 10 monclient: tick -55> 2022-04-12T16:08:57.231+0300 7f2a0b08b700 10 monclient: tick -54> 2022-04-12T16:08:57.663+0300 7f2a1f0b3700 10 monclient: tick -53> 2022-04-12T16:08:57.831+0300 7f28f6662700 10 monclient: tick -52> 2022-04-12T16:08:57.831+0300 7f28f7e65700 10 monclient: _renew_subs -51> 2022-04-12T16:08:57.831+0300 7f28f7e65700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -50> 2022-04-12T16:08:58.231+0300 7f2a0b08b700 10 monclient: tick -49> 2022-04-12T16:08:58.667+0300 7f2a1f0b3700 10 monclient: tick -48> 2022-04-12T16:08:58.831+0300 7f28f6662700 10 monclient: tick -47> 2022-04-12T16:08:59.231+0300 7f2a0b08b700 10 monclient: tick -46> 2022-04-12T16:08:59.667+0300 7f2a1f0b3700 10 monclient: tick -45> 2022-04-12T16:08:59.667+0300 7f2a208b6700 10 monclient: _renew_subs -44> 2022-04-12T16:08:59.667+0300 7f2a208b6700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -43> 2022-04-12T16:08:59.831+0300 7f28f6662700 10 monclient: tick -42> 2022-04-12T16:09:00.231+0300 7f2a0b08b700 10 monclient: tick -41> 2022-04-12T16:09:00.663+0300 7f2a1f0b3700 10 monclient: tick -40> 2022-04-12T16:09:00.831+0300 7f28f6662700 10 monclient: tick -39> 2022-04-12T16:09:01.235+0300 7f2a0b08b700 10 monclient: tick -38> 2022-04-12T16:09:01.663+0300 7f2a1f0b3700 10 monclient: tick -37> 2022-04-12T16:09:01.835+0300 7f28f6662700 10 monclient: tick -36> 2022-04-12T16:09:02.231+0300 7f2a0b08b700 10 monclient: tick -35> 2022-04-12T16:09:02.663+0300 7f2a1f0b3700 10 monclient: tick -34> 2022-04-12T16:09:02.747+0300 7f299a7aa700 1 ====== starting new request req=0x7f28f55de620 ===== -33> 2022-04-12T16:09:02.747+0300 7f299a7aa700 2 req 6236571265947506715 0.000000000s initializing for trans_id = tx00000568cc0860b35981b-00625579ee-1659-sync -32> 2022-04-12T16:09:02.747+0300 7f299a7aa700 2 req 6236571265947506715 0.000000000s getting op 4 -31> 2022-04-12T16:09:02.747+0300 7f299a7aa700 2 req 6236571265947506715 0.000000000s :datalog_notify verifying requester -30> 2022-04-12T16:09:02.751+0300 7f299a7aa700 2 req 6236571265947506715 0.004000004s :datalog_notify normalizing buckets and tenants -29> 2022-04-12T16:09:02.751+0300 7f299a7aa700 2 req 6236571265947506715 0.004000004s :datalog_notify init permissions -28> 2022-04-12T16:09:02.751+0300 7f299a7aa700 2 req 6236571265947506715 0.004000004s :datalog_notify recalculating target -27> 2022-04-12T16:09:02.751+0300 7f299a7aa700 2 req 6236571265947506715 0.004000004s :datalog_notify reading permissions -26> 2022-04-12T16:09:02.751+0300 7f299a7aa700 2 req 6236571265947506715 0.004000004s :datalog_notify init op -25> 2022-04-12T16:09:02.751+0300 7f299a7aa700 2 req 6236571265947506715 0.004000004s :datalog_notify verifying op mask -24> 2022-04-12T16:09:02.751+0300 7f299a7aa700 2 req 6236571265947506715 0.004000004s :datalog_notify verifying op permissions -23> 2022-04-12T16:09:02.751+0300 7f299a7aa700 2 overriding permissions due to system operation -22> 2022-04-12T16:09:02.751+0300 7f299a7aa700 2 req 6236571265947506715 0.004000004s :datalog_notify verifying op params -21> 2022-04-12T16:09:02.751+0300 7f299a7aa700 2 req 6236571265947506715 0.004000004s :datalog_notify pre-executing -20> 2022-04-12T16:09:02.751+0300 7f299a7aa700 2 req 6236571265947506715 0.004000004s :datalog_notify executing -19> 2022-04-12T16:09:02.751+0300 7f299a7aa700 2 req 6236571265947506715 0.004000004s :datalog_notify completing -18> 2022-04-12T16:09:02.751+0300 7f299a7aa700 2 req 6236571265947506715 0.004000004s :datalog_notify op status=0 -17> 2022-04-12T16:09:02.751+0300 7f299a7aa700 2 req 6236571265947506715 0.004000004s :datalog_notify http status=200 -16> 2022-04-12T16:09:02.751+0300 7f299a7aa700 1 ====== req done req=0x7f28f55de620 op status=0 http_status=200 latency=0.004000004s ====== -15> 2022-04-12T16:09:02.751+0300 7f299a7aa700 1 beast: 0x7f28f55de620: 192.168.166.83 - rgw_zone_user [12/Apr/2022:16:09:02.747 +0300] "POST /admin/log?type=data¬ify&source-zone=6fa70b8c-6324-4bee-8dbf-d323e8dc5f43&rgwx-zonegroup=784a05a5-2089-4564-ab1b-a55c43c5f401 HTTP/1.1" 200 0 - - - latency=0.004000004s -14> 2022-04-12T16:09:02.751+0300 7f2a06882700 4 data sync: starting sync on testbucket:6fa70b8c-6324-4bee-8dbf-d323e8dc5f43.5105.1:3 key=testbucket:6fa70b8c-6324-4bee-8dbf-d323e8dc5f43.5105.1:3 -13> 2022-04-12T16:09:02.759+0300 7f2a06882700 5 RGW-SYNC:data:sync:shard[30]:entry:bucket_sync_sources[target=:[]):source_bucket=:[]):source_zone=6fa70b8c-6324-4bee-8dbf-d323e8dc5f43]:bucket[testbucket:6fa70b8c-6324-4bee-8dbf-d323e8dc5f43.5105.1:3<-testbucket:6fa70b8c-6324-4bee-8dbf-d323e8dc5f43.5105.1:3]:full_sync[testbucket:6fa70b8c-6324-4bee-8dbf-d323e8dc5f43.5105.1:3]:entry[amazon_case.png]: bucket sync: sync obj: 6fa70b8c-6324-4bee-8dbf-d323e8dc5f43/:testbucket[6fa70b8c-6324-4bee-8dbf-d323e8dc5f43.5105.1])/amazon_case.png[0] -12> 2022-04-12T16:09:02.759+0300 7f2a06882700 0 e21904192857b447: sync_object: b=:testbucket[6fa70b8c-6324-4bee-8dbf-d323e8dc5f43.5105.1]) k=amazon_case.png versioned_epoch=0 -11> 2022-04-12T16:09:02.767+0300 7f2a06882700 4 rgw rados thread: AWS: download begin: z=6fa70b8c-6324-4bee-8dbf-d323e8dc5f43 b=:testbucket[6fa70b8c-6324-4bee-8dbf-d323e8dc5f43.5105.1]) k=amazon_case.png size=1880884 mtime=2022-04-12T16:09:02.643682+0300 etag=8cd436573b0ee6e9cc4f6f2cb3ba0cf1 zone_short_id=2425654257 pg_ver=0 -10> 2022-04-12T16:09:02.767+0300 7f2a06882700 0 AWS: creating bucket rgw-cdn-e21904192857b447 -9> 2022-04-12T16:09:02.831+0300 7f28f6662700 10 monclient: tick -8> 2022-04-12T16:09:02.831+0300 7f28f7e65700 10 monclient: _renew_subs -7> 2022-04-12T16:09:02.831+0300 7f28f7e65700 10 monclient: _send_mon_message to mon.ceph-ss-test-oct-controller-av-01 at v2:192.168.166.83:3300/0 -6> 2022-04-12T16:09:03.231+0300 7f2a0b08b700 10 monclient: tick -5> 2022-04-12T16:09:03.663+0300 7f2a1f0b3700 10 monclient: tick -4> 2022-04-12T16:09:03.831+0300 7f28f6662700 10 monclient: tick -3> 2022-04-12T16:09:04.231+0300 7f2a0b08b700 10 monclient: tick -2> 2022-04-12T16:09:04.663+0300 7f2a1f0b3700 10 monclient: tick -1> 2022-04-12T16:09:04.831+0300 7f28f6662700 10 monclient: tick 0> 2022-04-12T16:09:04.839+0300 7f2a06882700 -1 *** Caught signal (Aborted) ** in thread 7f2a06882700 thread_name:data-sync ceph version 16.2.7 (dd0603118f56ab514f133c8d2e3adfc983942503) pacific (stable) 1: /lib64/libpthread.so.0(+0x12c20) [0x7f2a2fa66c20] 2: gsignal() 3: abort() 4: /lib64/libstdc++.so.6(+0x9009b) [0x7f2a2ea5a09b] 5: /lib64/libstdc++.so.6(+0x9653c) [0x7f2a2ea6053c] 6: /lib64/libstdc++.so.6(+0x96597) [0x7f2a2ea60597] 7: /lib64/libstdc++.so.6(+0x967f8) [0x7f2a2ea607f8] 8: /lib64/libstdc++.so.6(+0x92025) [0x7f2a2ea5c025] 9: /lib64/libradosgw.so.2(+0x48b109) [0x7f2a3a5a1109] 10: (std::__cxx11::basic_string, std::allocator >::_M_mutate(unsigned long, unsigned long, char const*, unsigned long)+0x6d) [0x7f2a3a5a471d] 11: (std::__cxx11::basic_string, std::allocator >::_M_append(char const*, unsigned long)+0x6b) [0x7f2a3a5a489b] 12: (rgw_obj_key::get_oid[abi:cxx11]() const+0xc6) [0x7f2a3a7c9066] 13: (RGWRESTStreamS3PutObj::send_init(rgw::sal::RGWObject*)+0xb3) [0x7f2a3aa8f4b3] 14: (RGWRESTConn::put_obj_send_init(rgw::sal::RGWObject*, rgw_http_param_pair const*, RGWRESTStreamS3PutObj**)+0x5c1) [0x7f2a3aa99891] 15: (RGWAWSStreamPutCRF::init()+0xf1) [0x7f2a3a918ec1] 16: (RGWStreamSpliceCR::operate(DoutPrefixProvider const*)+0xa2f) [0x7f2a3a9b567f] 17: (RGWCoroutinesStack::operate(DoutPrefixProvider const*, RGWCoroutinesEnv*)+0x15c) [0x7f2a3a9a0cbc] 18: (RGWCoroutinesManager::run(DoutPrefixProvider const*, std::__cxx11::list >&)+0x296) [0x7f2a3a9a1b06] 19: (RGWCoroutinesManager::run(DoutPrefixProvider const*, RGWCoroutine*)+0x91) [0x7f2a3a9a2dd1] 20: (RGWRemoteDataLog::run_sync(DoutPrefixProvider const*, int)+0x1b4) [0x7f2a3a8b84b4] 21: (RGWDataSyncProcessorThread::process(DoutPrefixProvider const*)+0x59) [0x7f2a3aa6fe09] 22: (RGWRadosThread::Worker::entry()+0x13a) [0x7f2a3aa3368a] 23: /lib64/libpthread.so.0(+0x817a) [0x7f2a2fa5c17a] 24: clone() NOTE: a copy of the executable, or `objdump -rdS ` is needed to interpret this. --- logging levels --- 0/ 5 none 0/ 1 lockdep 0/ 1 context 1/ 1 crush 1/ 5 mds 1/ 5 mds_balancer 1/ 5 mds_locker 1/ 5 mds_log 1/ 5 mds_log_expire 1/ 5 mds_migrator 0/ 1 buffer 0/ 1 timer 0/ 1 filer 0/ 1 striper 0/ 1 objecter 0/ 5 rados 0/ 5 rbd 0/ 5 rbd_mirror 0/ 5 rbd_replay 0/ 5 rbd_pwl 0/ 5 journaler 0/ 5 objectcacher 0/ 5 immutable_obj_cache 0/ 5 client 1/ 5 osd 0/ 5 optracker 0/ 5 objclass 1/ 3 filestore 1/ 3 journal 0/ 0 ms 1/ 5 mon 0/10 monc 1/ 5 paxos 0/ 5 tp 1/ 5 auth 1/ 5 crypto 1/ 1 finisher 1/ 1 reserver 1/ 5 heartbeatmap 1/ 5 perfcounter 5/ 5 rgw 5/ 5 rgw_sync 1/10 civetweb 1/ 5 javaclient 1/ 5 asok 1/ 1 throttle 0/ 0 refs 1/ 5 compressor 1/ 5 bluestore 1/ 5 bluefs 1/ 3 bdev 1/ 5 kstore 4/ 5 rocksdb 4/ 5 leveldb 4/ 5 memdb 1/ 5 fuse 2/ 5 mgr 1/ 5 mgrc 1/ 5 dpdk 1/ 5 eventtrace 1/ 5 prioritycache 0/ 5 test 0/ 5 cephfs_mirror 0/ 5 cephsqlite -2/-2 (syslog threshold) 99/99 (stderr threshold) --- pthread ID / name mapping for recent threads --- 139813909309184 / safe_timer 139813926094592 / ms_dispatch 139813934487296 / ceph_timer 139813951272704 / io_context_pool 139816662116096 / radosgw 139816729257728 / radosgw 139816846755584 / radosgw 139816964253440 / radosgw 139817123714816 / radosgw 139817425852160 / radosgw 139817501386496 / radosgw 139817937807104 / radosgw 139818063697664 / radosgw 139818105661184 / radosgw 139818147624704 / radosgw 139818181195520 / radosgw 139818197980928 / radosgw 139818231551744 / radosgw 139818256729856 / radosgw 139818315478784 / lifecycle_thr_2 139818349049600 / lifecycle_thr_1 139818382620416 / lifecycle_thr_0 139818474940160 / data-sync 139818491725568 / meta-sync 139818525296384 / rgw_obj_expirer 139818533689088 / rgw_gc 139818550474496 / safe_timer 139818567259904 / ms_dispatch 139818592438016 / io_context_pool 139818600830720 / rgw_dt_lg_renew 139818701543168 / rados_async 139818886182656 / safe_timer 139818902968064 / ms_dispatch 139818911360768 / ceph_timer 139818919753472 / io_context_pool 139818928146176 / io_context_pool 139818986895104 / admin_socket 139818995287808 / service 139819003680512 / msgr-worker-2 139819012073216 / msgr-worker-1 139819020465920 / msgr-worker-0 139819359798272 / radosgw max_recent 10000 max_new 10000 log_file /var/lib/ceph/crash/2022-04-12T13:09:04.842414Z_3dfe2d76-d561-4c73-ad91-b3b67a78e9a9/log --- end dump of recent events --- teardown: managing teardown after SIGCHLD teardown: Waiting PID 54 to terminate teardown: Process 54 is terminated teardown: Bye Bye, container will die with return code 0