Project

General

Profile

Bug #58375 » topic_info.txt

xiaobao wen, 01/13/2023 04:02 AM

 
[root@node01 deeproute]# kubectl get pods rook-ceph-rgw-os-dsglczutvqsgowpz-a-5454c7bdf-k7jhk -n rook-ceph -o yaml | grep image
image: reg.deeproute.ai/deeproute-public/ceph/ceph:v16.2.10
imagePullPolicy: IfNotPresent

root@node04:/# radosgw-admin topic list
{
"topics": [
{
"topic": {
"user": "",
"name": "MLP_ALL_BUCKET_NOTIFICATION_c-dev-prod-bucket-notification",
"dest": {
"bucket_name": "",
"oid_prefix": "",
"push_endpoint": "kafka://prod-kafka.deeproute.cn:9092",
"push_endpoint_args": "Attributes.entry.1.key=OpaqueData&Attributes.entry.1.value=liangsu@deeproute.ai&Attributes.entry.2.key=kafka-ack-level&Attributes.entry.2.value=broker&Attributes.entry.3.key=persistent&Attributes.entry.3.value=true&Attributes.entry.4.key=push-endpoint&Attributes.entry.4.value=kafka://prod-kafka.deeproute.cn:9092&Attributes.entry.5.key=use-ssl&Attributes.entry.5.value=false&Attributes.entry.6.key=verify-ssl&Attributes.entry.6.value=false&OpaqueData=liangsu@deeproute.ai&Version=2010-03-31&kafka-ack-level=broker&persistent=true&push-endpoint=kafka://prod-kafka.deeproute.cn:9092&use-ssl=false&verify-ssl=false",
"push_endpoint_topic": "c-dev-prod-bucket-notification",
"stored_secret": "false",
"persistent": "true"
},
"arn": "arn:aws:sns:os-dsglczutvqsgowpz::c-dev-prod-bucket-notification",
"opaqueData": "liangsu@deeproute.ai"
},
"subs": []
},
{
"topic": {
"user": "",
"name": "bucket-test-010-54e451a37af49212c720d374b5f9cac0_c-dev-dev-bucket-notification",
"dest": {
"bucket_name": "",
"oid_prefix": "",
"push_endpoint": "kafka://dev-kafka.deeproute.cn:9092",
"push_endpoint_args": "Attributes.entry.1.key=OpaqueData&Attributes.entry.1.value=liangsu@deeproute.ai&Attributes.entry.2.key=kafka-ack-level&Attributes.entry.2.value=broker&Attributes.entry.3.key=persistent&Attributes.entry.3.value=true&Attributes.entry.4.key=push-endpoint&Attributes.entry.4.value=kafka://dev-kafka.deeproute.cn:9092&Attributes.entry.5.key=use-ssl&Attributes.entry.5.value=false&Attributes.entry.6.key=verify-ssl&Attributes.entry.6.value=false&OpaqueData=liangsu@deeproute.ai&Version=2010-03-31&kafka-ack-level=broker&persistent=true&push-endpoint=kafka://dev-kafka.deeproute.cn:9092&use-ssl=false&verify-ssl=false",
"push_endpoint_topic": "c-dev-dev-bucket-notification",
"stored_secret": "false",
"persistent": "true"
},
"arn": "arn:aws:sns:os-dsglczutvqsgowpz::c-dev-dev-bucket-notification",
"opaqueData": "liangsu@deeproute.ai"
},
"subs": []
},
{
"topic": {
"user": "",
"name": "bucket-test-012-78f88807ad4e5baf97ff1ce77cc2e6b9_c-dev-stg-bucket-notification",
"dest": {
"bucket_name": "",
"oid_prefix": "",
"push_endpoint": "kafka://stg-kafka.deeproute.cn:9092",
"push_endpoint_args": "Attributes.entry.1.key=OpaqueData&Attributes.entry.1.value=liangsu@deeproute.ai&Attributes.entry.2.key=kafka-ack-level&Attributes.entry.2.value=broker&Attributes.entry.3.key=persistent&Attributes.entry.3.value=true&Attributes.entry.4.key=push-endpoint&Attributes.entry.4.value=kafka://stg-kafka.deeproute.cn:9092&Attributes.entry.5.key=use-ssl&Attributes.entry.5.value=false&Attributes.entry.6.key=verify-ssl&Attributes.entry.6.value=false&OpaqueData=liangsu@deeproute.ai&Version=2010-03-31&kafka-ack-level=broker&persistent=true&push-endpoint=kafka://stg-kafka.deeproute.cn:9092&use-ssl=false&verify-ssl=false",
"push_endpoint_topic": "c-dev-stg-bucket-notification",
"stored_secret": "false",
"persistent": "true"
},
"arn": "arn:aws:sns:os-dsglczutvqsgowpz::c-dev-stg-bucket-notification",
"opaqueData": "liangsu@deeproute.ai"
},
"subs": []
},
{
"topic": {
"user": "",
"name": "bucket-test-016-78f88807ad4e5baf97ff1ce77cc2e6b9_c-dev-stg-bucket-notification",
"dest": {
"bucket_name": "",
"oid_prefix": "",
"push_endpoint": "kafka://stg-kafka.deeproute.cn:9092",
"push_endpoint_args": "Attributes.entry.1.key=OpaqueData&Attributes.entry.1.value=liangsu@deeproute.ai&Attributes.entry.2.key=kafka-ack-level&Attributes.entry.2.value=broker&Attributes.entry.3.key=persistent&Attributes.entry.3.value=true&Attributes.entry.4.key=push-endpoint&Attributes.entry.4.value=kafka://stg-kafka.deeproute.cn:9092&Attributes.entry.5.key=use-ssl&Attributes.entry.5.value=false&Attributes.entry.6.key=verify-ssl&Attributes.entry.6.value=false&OpaqueData=liangsu@deeproute.ai&Version=2010-03-31&kafka-ack-level=broker&persistent=true&push-endpoint=kafka://stg-kafka.deeproute.cn:9092&use-ssl=false&verify-ssl=false",
"push_endpoint_topic": "c-dev-stg-bucket-notification",
"stored_secret": "false",
"persistent": "true"
},
"arn": "arn:aws:sns:os-dsglczutvqsgowpz::c-dev-stg-bucket-notification",
"opaqueData": "liangsu@deeproute.ai"
},
"subs": []
},
{
"topic": {
"user": "",
"name": "c-dev-dev-bucket-notification",
"dest": {
"bucket_name": "",
"oid_prefix": "",
"push_endpoint": "kafka://dev-kafka.deeproute.cn:9092",
"push_endpoint_args": "Attributes.entry.1.key=OpaqueData&Attributes.entry.1.value=liangsu@deeproute.ai&Attributes.entry.2.key=kafka-ack-level&Attributes.entry.2.value=broker&Attributes.entry.3.key=persistent&Attributes.entry.3.value=true&Attributes.entry.4.key=push-endpoint&Attributes.entry.4.value=kafka://dev-kafka.deeproute.cn:9092&Attributes.entry.5.key=use-ssl&Attributes.entry.5.value=false&Attributes.entry.6.key=verify-ssl&Attributes.entry.6.value=false&OpaqueData=liangsu@deeproute.ai&Version=2010-03-31&kafka-ack-level=broker&persistent=true&push-endpoint=kafka://dev-kafka.deeproute.cn:9092&use-ssl=false&verify-ssl=false",
"push_endpoint_topic": "c-dev-dev-bucket-notification",
"stored_secret": "false",
"persistent": "true"
},
"arn": "arn:aws:sns:os-dsglczutvqsgowpz::c-dev-dev-bucket-notification",
"opaqueData": "liangsu@deeproute.ai"
},
"subs": []
},
{
"topic": {
"user": "",
"name": "c-dev-prod-bucket-notification",
"dest": {
"bucket_name": "",
"oid_prefix": "",
"push_endpoint": "kafka://prod-kafka.deeproute.cn:9092",
"push_endpoint_args": "Attributes.entry.1.key=OpaqueData&Attributes.entry.1.value=liangsu@deeproute.ai&Attributes.entry.2.key=kafka-ack-level&Attributes.entry.2.value=broker&Attributes.entry.3.key=persistent&Attributes.entry.3.value=true&Attributes.entry.4.key=push-endpoint&Attributes.entry.4.value=kafka://prod-kafka.deeproute.cn:9092&Attributes.entry.5.key=use-ssl&Attributes.entry.5.value=false&Attributes.entry.6.key=verify-ssl&Attributes.entry.6.value=false&OpaqueData=liangsu@deeproute.ai&Version=2010-03-31&kafka-ack-level=broker&persistent=true&push-endpoint=kafka://prod-kafka.deeproute.cn:9092&use-ssl=false&verify-ssl=false",
"push_endpoint_topic": "c-dev-prod-bucket-notification",
"stored_secret": "false",
"persistent": "true"
},
"arn": "arn:aws:sns:os-dsglczutvqsgowpz::c-dev-prod-bucket-notification",
"opaqueData": "liangsu@deeproute.ai"
},
"subs": []
},
{
"topic": {
"user": "",
"name": "c-dev-stg-bucket-notification",
"dest": {
"bucket_name": "",
"oid_prefix": "",
"push_endpoint": "kafka://stg-kafka.deeproute.cn:9092",
"push_endpoint_args": "Attributes.entry.1.key=OpaqueData&Attributes.entry.1.value=liangsu@deeproute.ai&Attributes.entry.2.key=kafka-ack-level&Attributes.entry.2.value=broker&Attributes.entry.3.key=persistent&Attributes.entry.3.value=true&Attributes.entry.4.key=push-endpoint&Attributes.entry.4.value=kafka://stg-kafka.deeproute.cn:9092&Attributes.entry.5.key=use-ssl&Attributes.entry.5.value=false&Attributes.entry.6.key=verify-ssl&Attributes.entry.6.value=false&OpaqueData=liangsu@deeproute.ai&Version=2010-03-31&kafka-ack-level=broker&persistent=true&push-endpoint=kafka://stg-kafka.deeproute.cn:9092&use-ssl=false&verify-ssl=false",
"push_endpoint_topic": "c-dev-stg-bucket-notification",
"stored_secret": "false",
"persistent": "true"
},
"arn": "arn:aws:sns:os-dsglczutvqsgowpz::c-dev-stg-bucket-notification",
"opaqueData": "liangsu@deeproute.ai"
},
"subs": []
}
]
}

root@node04:/# radosgw-admin bucket stats --bucket mlp-data-warehouse
{
"bucket": "mlp-data-warehouse",
"num_shards": 397,
"tenant": "",
"zonegroup": "47efd394-5cac-401e-bd92-debb9c55254b",
"placement_rule": "default-placement",
"explicit_placement": {
"data_pool": "",
"data_extra_pool": "",
"index_pool": ""
},
"id": "1db8bc57-cb5c-4fcf-a389-754c16ec70c2.28703777.6",
"marker": "1db8bc57-cb5c-4fcf-a389-754c16ec70c2.28703777.3",
"index_type": "Normal",
"owner": "mlp",
"ver": "0#39801,1#39896,2#39677,3#40182,4#40044,5#39612,6#39663,7#39734,8#39772,9#40057,10#39933,11#39727,12#39822,13#39818,14#40080,15#39934,16#40014,17#39928,18#40002,19#39822,20#40005,21#39600,22#39752,23#39893,24#40256,25#40088,26#40288,27#39880,28#39836,29#39976,30#40199,31#40163,32#40007,33#39867,34#40048,35#39851,36#40466,37#39943,38#39823,39#39976,40#40183,41#39845,42#40103,43#40274,44#39891,45#40124,46#39907,47#39669,48#39795,49#39801,50#39888,51#40177,52#39846,53#39832,54#39520,55#39999,56#39732,57#39892,58#40115,59#39894,60#39780,61#39644,62#39946,63#39874,64#39596,65#39891,66#39915,67#39791,68#39919,69#39884,70#39862,71#39726,72#39963,73#40166,74#40118,75#40057,76#39997,77#39819,78#39965,79#40197,80#39854,81#39836,82#40213,83#39763,84#39843,85#39690,86#40176,87#39950,88#39495,89#39363,90#39651,91#39776,92#40038,93#40244,94#39862,95#40196,96#39915,97#39895,98#40069,99#39731,100#40103,101#39555,102#39802,103#39933,104#40244,105#39798,106#40047,107#39744,108#39750,109#39744,110#39669,111#39608,112#39795,113#40074,114#39761,115#39769,116#39732,117#39903,118#39824,119#40103,120#40081,121#39799,122#40095,123#40149,124#40012,125#39998,126#39804,127#40179,128#39568,129#40182,130#39653,131#40080,132#39562,133#40122,134#40379,135#40130,136#40253,137#40080,138#39840,139#40047,140#39888,141#39738,142#40132,143#39716,144#40159,145#40034,146#40182,147#39951,148#40023,149#39784,150#40191,151#39697,152#39872,153#39670,154#39921,155#40006,156#39989,157#40027,158#39832,159#40114,160#39797,161#39790,162#39848,163#40103,164#40053,165#39927,166#39882,167#39804,168#39552,169#39789,170#39977,171#39798,172#39923,173#39664,174#39941,175#40050,176#39749,177#39617,178#39505,179#39989,180#39792,181#39932,182#39702,183#40019,184#39828,185#39625,186#40061,187#39654,188#39624,189#39838,190#39600,191#39632,192#39904,193#39651,194#40154,195#39898,196#40334,197#39984,198#40171,199#39885,200#39779,201#39741,202#40012,203#40341,204#40016,205#40100,206#39870,207#39923,208#40035,209#39934,210#40045,211#39792,212#40167,213#40084,214#40099,215#39674,216#39964,217#39849,218#39927,219#39851,220#39878,221#40098,222#39955,223#39815,224#39702,225#40127,226#39539,227#39912,228#39743,229#39728,230#39825,231#40116,232#39585,233#39732,234#40154,235#39710,236#39912,237#40186,238#40209,239#40106,240#39994,241#39663,242#40132,243#40124,244#40289,245#39989,246#39965,247#40077,248#39942,249#40035,250#39581,251#40067,252#40105,253#39838,254#40053,255#39840,256#39817,257#39591,258#39864,259#39804,260#40151,261#40336,262#39648,263#39651,264#40189,265#40148,266#39536,267#40144,268#39942,269#39901,270#39914,271#40142,272#40136,273#40005,274#39786,275#39802,276#40308,277#40189,278#39978,279#39677,280#40183,281#40285,282#40060,283#39933,284#40214,285#40176,286#40065,287#40077,288#39834,289#39801,290#39839,291#39947,292#40021,293#39721,294#39866,295#39873,296#39898,297#40134,298#39992,299#39656,300#39722,301#40084,302#39964,303#40101,304#39769,305#39709,306#39806,307#40006,308#40066,309#39999,310#40174,311#40110,312#40186,313#40241,314#39902,315#40073,316#39891,317#40057,318#40086,319#40551,320#39716,321#40066,322#39939,323#39768,324#40088,325#39784,326#40048,327#40307,328#39690,329#39720,330#39795,331#39584,332#39621,333#39917,334#38098,335#37828,336#37948,337#37679,338#37789,339#37872,340#37976,341#37725,342#37963,343#38120,344#37894,345#37855,346#37676,347#37493,348#37799,349#38088,350#38221,351#38015,352#37966,353#38024,354#38167,355#38134,356#38080,357#37637,358#38227,359#38112,360#37990,361#38069,362#37876,363#37949,364#37922,365#38084,366#37891,367#38122,368#37995,369#38113,370#38054,371#37767,372#37575,373#37764,374#37780,375#37512,376#38211,377#37623,378#37835,379#37638,380#37972,381#38191,382#37996,383#37708,384#37841,385#38132,386#37988,387#37882,388#37945,389#38079,390#38328,391#38392,392#38268,393#37851,394#37887,395#38079,396#38060",
"master_ver": "0#0,1#0,2#0,3#0,4#0,5#0,6#0,7#0,8#0,9#0,10#0,11#0,12#0,13#0,14#0,15#0,16#0,17#0,18#0,19#0,20#0,21#0,22#0,23#0,24#0,25#0,26#0,27#0,28#0,29#0,30#0,31#0,32#0,33#0,34#0,35#0,36#0,37#0,38#0,39#0,40#0,41#0,42#0,43#0,44#0,45#0,46#0,47#0,48#0,49#0,50#0,51#0,52#0,53#0,54#0,55#0,56#0,57#0,58#0,59#0,60#0,61#0,62#0,63#0,64#0,65#0,66#0,67#0,68#0,69#0,70#0,71#0,72#0,73#0,74#0,75#0,76#0,77#0,78#0,79#0,80#0,81#0,82#0,83#0,84#0,85#0,86#0,87#0,88#0,89#0,90#0,91#0,92#0,93#0,94#0,95#0,96#0,97#0,98#0,99#0,100#0,101#0,102#0,103#0,104#0,105#0,106#0,107#0,108#0,109#0,110#0,111#0,112#0,113#0,114#0,115#0,116#0,117#0,118#0,119#0,120#0,121#0,122#0,123#0,124#0,125#0,126#0,127#0,128#0,129#0,130#0,131#0,132#0,133#0,134#0,135#0,136#0,137#0,138#0,139#0,140#0,141#0,142#0,143#0,144#0,145#0,146#0,147#0,148#0,149#0,150#0,151#0,152#0,153#0,154#0,155#0,156#0,157#0,158#0,159#0,160#0,161#0,162#0,163#0,164#0,165#0,166#0,167#0,168#0,169#0,170#0,171#0,172#0,173#0,174#0,175#0,176#0,177#0,178#0,179#0,180#0,181#0,182#0,183#0,184#0,185#0,186#0,187#0,188#0,189#0,190#0,191#0,192#0,193#0,194#0,195#0,196#0,197#0,198#0,199#0,200#0,201#0,202#0,203#0,204#0,205#0,206#0,207#0,208#0,209#0,210#0,211#0,212#0,213#0,214#0,215#0,216#0,217#0,218#0,219#0,220#0,221#0,222#0,223#0,224#0,225#0,226#0,227#0,228#0,229#0,230#0,231#0,232#0,233#0,234#0,235#0,236#0,237#0,238#0,239#0,240#0,241#0,242#0,243#0,244#0,245#0,246#0,247#0,248#0,249#0,250#0,251#0,252#0,253#0,254#0,255#0,256#0,257#0,258#0,259#0,260#0,261#0,262#0,263#0,264#0,265#0,266#0,267#0,268#0,269#0,270#0,271#0,272#0,273#0,274#0,275#0,276#0,277#0,278#0,279#0,280#0,281#0,282#0,283#0,284#0,285#0,286#0,287#0,288#0,289#0,290#0,291#0,292#0,293#0,294#0,295#0,296#0,297#0,298#0,299#0,300#0,301#0,302#0,303#0,304#0,305#0,306#0,307#0,308#0,309#0,310#0,311#0,312#0,313#0,314#0,315#0,316#0,317#0,318#0,319#0,320#0,321#0,322#0,323#0,324#0,325#0,326#0,327#0,328#0,329#0,330#0,331#0,332#0,333#0,334#0,335#0,336#0,337#0,338#0,339#0,340#0,341#0,342#0,343#0,344#0,345#0,346#0,347#0,348#0,349#0,350#0,351#0,352#0,353#0,354#0,355#0,356#0,357#0,358#0,359#0,360#0,361#0,362#0,363#0,364#0,365#0,366#0,367#0,368#0,369#0,370#0,371#0,372#0,373#0,374#0,375#0,376#0,377#0,378#0,379#0,380#0,381#0,382#0,383#0,384#0,385#0,386#0,387#0,388#0,389#0,390#0,391#0,392#0,393#0,394#0,395#0,396#0",
"mtime": "0.000000",
"creation_time": "2022-11-29T10:05:40.490210Z",
"max_marker": "0#,1#,2#,3#,4#,5#,6#,7#,8#,9#,10#,11#,12#,13#,14#,15#,16#,17#,18#,19#,20#,21#,22#,23#,24#,25#,26#,27#,28#,29#,30#,31#,32#,33#,34#,35#,36#,37#,38#,39#,40#,41#,42#,43#,44#,45#,46#,47#,48#,49#,50#,51#,52#,53#,54#,55#,56#,57#,58#,59#,60#,61#,62#,63#,64#,65#,66#,67#,68#,69#,70#,71#,72#,73#,74#,75#,76#,77#,78#,79#,80#,81#,82#,83#,84#,85#,86#,87#,88#,89#,90#,91#,92#,93#,94#,95#,96#,97#,98#,99#,100#,101#,102#,103#,104#,105#,106#,107#,108#,109#,110#,111#,112#,113#,114#,115#,116#,117#,118#,119#,120#,121#,122#,123#,124#,125#,126#,127#,128#,129#,130#,131#,132#,133#,134#,135#,136#,137#,138#,139#,140#,141#,142#,143#,144#,145#,146#,147#,148#,149#,150#,151#,152#,153#,154#,155#,156#,157#,158#,159#,160#,161#,162#,163#,164#,165#,166#,167#,168#,169#,170#,171#,172#,173#,174#,175#,176#,177#,178#,179#,180#,181#,182#,183#,184#,185#,186#,187#,188#,189#,190#,191#,192#,193#,194#,195#,196#,197#,198#,199#,200#,201#,202#,203#,204#,205#,206#,207#,208#,209#,210#,211#,212#,213#,214#,215#,216#,217#,218#,219#,220#,221#,222#,223#,224#,225#,226#,227#,228#,229#,230#,231#,232#,233#,234#,235#,236#,237#,238#,239#,240#,241#,242#,243#,244#,245#,246#,247#,248#,249#,250#,251#,252#,253#,254#,255#,256#,257#,258#,259#,260#,261#,262#,263#,264#,265#,266#,267#,268#,269#,270#,271#,272#,273#,274#,275#,276#,277#,278#,279#,280#,281#,282#,283#,284#,285#,286#,287#,288#,289#,290#,291#,292#,293#,294#,295#,296#,297#,298#,299#,300#,301#,302#,303#,304#,305#,306#,307#,308#,309#,310#,311#,312#,313#,314#,315#,316#,317#,318#,319#,320#,321#,322#,323#,324#,325#,326#,327#,328#,329#,330#,331#,332#,333#,334#,335#,336#,337#,338#,339#,340#,341#,342#,343#,344#,345#,346#,347#,348#,349#,350#,351#,352#,353#,354#,355#,356#,357#,358#,359#,360#,361#,362#,363#,364#,365#,366#,367#,368#,369#,370#,371#,372#,373#,374#,375#,376#,377#,378#,379#,380#,381#,382#,383#,384#,385#,386#,387#,388#,389#,390#,391#,392#,393#,394#,395#,396#",
"usage": {
"rgw.main": {
"size": 25614247928662,
"size_actual": 25686133301248,
"size_utilized": 25614247928662,
"size_kb": 25013913993,
"size_kb_actual": 25084114552,
"size_kb_utilized": 25013913993,
"num_objects": 34667961
},
"rgw.multimeta": {
"size": 0,
"size_actual": 0,
"size_utilized": 0,
"size_kb": 0,
"size_kb_actual": 0,
"size_kb_utilized": 0,
"num_objects": 0
}
},
"bucket_quota": {
"enabled": false,
"check_on_raw": false,
"max_size": -1,
"max_size_kb": 0,
"max_objects": -1
}
}


# get bucket notification configuration, now we meet another problem that configuration delete failed, so we modify Event to useless DeleteMarkerCreated
<NotificationConfiguration>
<TopicConfiguration>
<Id>dataocean-prod-kafka-topic</Id>
<Topic>arn:aws:sns:os-dsglczutvqsgowpz::c-dev-prod-bucket-notification</Topic>
<Event>s3:ObjectRemoved:DeleteMarkerCreated</Event>
</TopicConfiguration>
</NotificationConfiguration>
(2-2/2)