#网络需求：
上海集群:配置一个vip和域名s3sh.imu.cn -->172.18.52.225
172.18.216.129  
172.18.216.113  
172.18.216.97   

广州集群:配置一个vip和域名s3gz.imu.cn  -->172.26.99.2
172.26.217.48 
172.26.217.49 
172.26.217.50 

上海广州vip互通。

172.26.217.16、17、18 配置一个vip，绑定域名s3sxtest.imu.cn

三个vip下节点能访问任意域名 
 
#master nede：
 ceph-deploy new sx-3f3r-ceph-s3-c1-01  sx-3f3r-ceph-s3-c1-02 sx-3f3r-ceph-s3-c1-03
 ceph-deploy mon   create-initial
  
  ceph-deploy --overwrite-conf admin  sx-3f3r-ceph-s3-c1-01  sx-3f3r-ceph-s3-c1-02 sx-3f3r-ceph-s3-c1-03
  
  #ceph-deploy mgr create sx-3f3r-ceph-s3-c1-03   # L版本才支持mgr
  #ceph mgr module enable dashboard
  
  #
  #An address where the dashboard will listen on needs to be configured as well, set this to ``::`` to listen on all
  #IPv4 and IPv6 addresses
  #ceph config-key set mgr/dashboard/server_addr ::

    for i in {a..n};do ceph-deploy disk zap sx-3f3r-ceph-s3-c1-01:sd$i;done 
    for i in {a..n};do ceph-deploy disk zap sx-3f3r-ceph-s3-c1-02:sd$i;done 
    for i in {a..n};do ceph-deploy disk zap sx-3f3r-ceph-s3-c1-03:sd$i;done 
      

    for i in {a..n};do ceph-deploy --overwrite-conf osd create sx-3f3r-ceph-s3-c1-01:sd$i;done   
    for i in {a..n};do ceph-deploy --overwrite-conf osd create sx-3f3r-ceph-s3-c1-02:sd$i;done 
    for i in {a..n};do ceph-deploy --overwrite-conf osd create sx-3f3r-ceph-s3-c1-03:sd$i;done 


#slave node ：
	
  ceph-deploy new xxy-2f201r-ceph-s3-c1-01  xxy-2f201r-ceph-s3-c1-02 xxy-2f201r-ceph-s3-c1-03
  ceph-deploy mon   create-initial
  
  
  #for i in {b..m};do sh -c "dd if=/dev/zero of=/dev/sd$i bs=10M count=400&";done
  
  for i in {b..m};do ceph-deploy  disk zap xxy-2f201r-ceph-s3-c1-01:sd$i;done
  for i in {b..m};do ceph-deploy  osd create xxy-2f201r-ceph-s3-c1-01:sd$i;done
      
  for i in {b..m};do ceph-deploy  disk zap xxy-2f201r-ceph-s3-c1-02:sd$i;done
  for i in {b..m};do ceph-deploy  osd create xxy-2f201r-ceph-s3-c1-02:sd$i;done
  
  for i in {b..m};do ceph-deploy  disk zap xxy-2f201r-ceph-s3-c1-03:sd$i;done
  for i in {b..m};do ceph-deploy  osd create xxy-2f201r-ceph-s3-c1-03:sd$i;done

#####
#安装rgw
ceph-deploy install --rgw  sx-3f3r-ceph-s3-c1-01  sx-3f3r-ceph-s3-c1-02 sx-3f3r-ceph-s3-c1-03
#建议 yum -y install ceph-radosgw

 
ceph-deploy rgw create   sx-3f3r-ceph-s3-c1-01  sx-3f3r-ceph-s3-c1-02 sx-3f3r-ceph-s3-c1-03
#创建主领域realm

radosgw-admin realm create --rgw-realm=imusic --default

#创建组zonegroup
radosgw-admin zonegroup create --rgw-zonegroup=imu --endpoints=http://s3gz.imu.cn:80 --rgw-realm=imusic --master --default
{
    "id": "41e5334a-0b66-4e39-bdad-e210c8ca8730",
    "name": "imusic",
    "current_period": "d220c25c-dbf7-44b1-b410-f5cc30c52eb1",
    "epoch": 1
}

[root@sx-3f3r-ceph-s3-c1-03 my-cluster]# radosgw-admin zonegroup create --rgw-zonegroup=imu --endpoints=http://s3gz.imu.cn:80 --rgw-realm=imusic --master --default
{
    "id": "1825aa39-e1d3-42ca-8cb6-3920335f2434",
    "name": "imu",
    "api_name": "imu",
    "is_master": "true",
    "endpoints": [
        "http:\/\/s3gz.imu.cn:80"
    ],
    "hostnames": [],
    "hostnames_s3website": [],
    "master_zone": "",
    "zones": [],
    "placement_targets": [],
    "default_placement": "",
    "realm_id": "41e5334a-0b66-4e39-bdad-e210c8ca8730"
}

#创建zone
  
radosgw-admin zone create --rgw-zonegroup=imu --rgw-zone=imu-gzsx \
                            --master --default \
                            --endpoints=http://s3gz.imu.cn:80
							
{
    "id": "f608ce2b-5584-45af-b0c5-f4896995bd22",
    "name": "imu-gzsx",
    "domain_root": "imu-gzsx.rgw.data.root",
    "control_pool": "imu-gzsx.rgw.control",
    "gc_pool": "imu-gzsx.rgw.gc",
    "log_pool": "imu-gzsx.rgw.log",
    "intent_log_pool": "imu-gzsx.rgw.intent-log",
    "usage_log_pool": "imu-gzsx.rgw.usage",
    "user_keys_pool": "imu-gzsx.rgw.users.keys",
    "user_email_pool": "imu-gzsx.rgw.users.email",
    "user_swift_pool": "imu-gzsx.rgw.users.swift",
    "user_uid_pool": "imu-gzsx.rgw.users.uid",
    "system_key": {
        "access_key": "",
        "secret_key": ""
    },
    "placement_pools": [
        {
            "key": "default-placement",
            "val": {
                "index_pool": "imu-gzsx.rgw.buckets.index",
                "data_pool": "imu-gzsx.rgw.buckets.data",
                "data_extra_pool": "imu-gzsx.rgw.buckets.non-ec",
                "index_type": 0
            }
        }
    ],
    "metadata_heap": "",
    "realm_id": "41e5334a-0b66-4e39-bdad-e210c8ca8730"
}
							

#创建同步用户	
radosgw-admin user create --uid="synchronization-user" --display-name="sync-user" --system   --access-key=K9237GSUFH0SZRQHBBP2 --secret=bKob0LirCx48EHrTAwDlB563rXa9dDziuJBHOQDS 


 
  
#删除默认
 radosgw-admin zonegroup remove --rgw-zonegroup=default --rgw-zone=default
 radosgw-admin period update --commit
 radosgw-admin zone delete --rgw-zone=default
 radosgw-admin period update --commit
 radosgw-admin zonegroup delete --rgw-zonegroup=default
 radosgw-admin period update --commit

 rados rmpool default.rgw.control default.rgw.control --yes-i-really-really-mean-it
 rados rmpool default.rgw.data.root default.rgw.data.root --yes-i-really-really-mean-it
 rados rmpool default.rgw.gc default.rgw.gc --yes-i-really-really-mean-it
 rados rmpool default.rgw.log default.rgw.log --yes-i-really-really-mean-it
	
	
#给zone添加用户
radosgw-admin zone modify --rgw-zone=imu-gzsx  --access-key=K9237GSUFH0SZRQHBBP2 --secret=bKob0LirCx48EHrTAwDlB563rXa9dDziuJBHOQDS 

 
#使配置永久生效
radosgw-admin period update --commit




#修改配置

[client.rgw.sx-3f3r-ceph-s3-c1-01]
host = sx-3f3r-ceph-s3-c1-01
rgw frontends = "civetweb port=80"
rgw_zone=imu-gzsx # add 
rgw_dns_name = s3gz.imu.cn #add
rgw_content_length_compat = true

[client.rgw.sx-3f3r-ceph-s3-c1-02]
host = sx-3f3r-ceph-s3-c1-02
rgw frontends = "civetweb port=80"
rgw_zone=imu-gzsx
rgw_dns_name = s3gz.imu.cn
rgw_content_length_compat = true

[client.rgw.sx-3f3r-ceph-s3-c1-03]
host = sx-3f3r-ceph-s3-c1-03
rgw frontends = "civetweb port=80"
rgw_zone=imu-gzsx
rgw_dns_name = s3gz.imu.cn
rgw_content_length_compat = true


ceph-deploy --overwrite-conf config push sx-3f3r-ceph-s3-c1-0{1..3}



#重启gw

#
systemctl enable ceph-radosgw@rgw.`hostname -s`
systemctl restart ceph-radosgw@rgw.`hostname -s`	
systemctl status  ceph-radosgw@rgw.`hostname -s`




#第二个集群：slave zone

#安装gw
ceph-deploy install --rgw xxy-2f201r-ceph-s3-c1-01 xxy-2f201r-ceph-s3-c1-02 xxy-2f201r-ceph-s3-c1-03
#建议每个节点执行 yum -y install ceph-radosgw

ceph-deploy --overwrite-conf rgw create xxy-2f201r-ceph-s3-c1-01 xxy-2f201r-ceph-s3-c1-02 xxy-2f201r-ceph-s3-c1-03

#增加rgw配置：
[client.rgw.xxy-2f201r-ceph-s3-c1-01]
host = xxy-2f201r-ceph-s3-c1-01
rgw_frontends = "civetweb port=80"

[client.rgw.xxy-2f201r-ceph-s3-c1-02]
host = xxy-2f201r-ceph-s3-c1-02
rgw_frontends = "civetweb port=80"

[client.rgw.xxy-2f201r-ceph-s3-c1-03]
host = xxy-2f201r-ceph-s3-c1-03
rgw_frontends = "civetweb port=80"

ceph-deploy --overwrite-conf config push xxy-2f201r-ceph-s3-c1-0{1..3}

systemctl restart ceph-radosgw@rgw.`hostname -s`

#拉区域
radosgw-admin realm pull --url=s3gz.imu.cn:80 --access-key=K9237GSUFH0SZRQHBBP2 --secret=bKob0LirCx48EHrTAwDlB563rXa9dDziuJBHOQDS


2018-01-03 15:43:53.817285 7f928aabc9c0  1 error read_lastest_epoch .rgw.root:periods.0f9aa0e6-2a6e-4689-abee-d64f43f0ccac.latest_epoch
2018-01-03 15:43:53.852182 7f928aabc9c0  1 Set the period's master zonegroup 1825aa39-e1d3-42ca-8cb6-3920335f2434 as the default
{
    "id": "41e5334a-0b66-4e39-bdad-e210c8ca8730",
    "name": "imusic",
    "current_period": "0f9aa0e6-2a6e-4689-abee-d64f43f0ccac",
    "epoch": 2
}

#拉 period
radosgw-admin period pull --url=s3gz.imu.cn:80 --access-key=K9237GSUFH0SZRQHBBP2 --secret=bKob0LirCx48EHrTAwDlB563rXa9dDziuJBHOQDS

2018-01-03 15:44:55.503030 7fcb113879c0  1 found existing latest_epoch 4 >= given epoch 4, returning r=-17
{
    "id": "0f9aa0e6-2a6e-4689-abee-d64f43f0ccac",
    "epoch": 4,
    "predecessor_uuid": "d220c25c-dbf7-44b1-b410-f5cc30c52eb1",
    "sync_status": [],
    "period_map": {
        "id": "0f9aa0e6-2a6e-4689-abee-d64f43f0ccac",
        "zonegroups": [
            {
                "id": "1825aa39-e1d3-42ca-8cb6-3920335f2434",
                "name": "imu",
                "api_name": "imu",
                "is_master": "true",
                "endpoints": [
                    "http:\/\/s3gz.imu.cn:80"
                ],
                "hostnames": [],
                "hostnames_s3website": [],
                "master_zone": "f608ce2b-5584-45af-b0c5-f4896995bd22",
                "zones": [
                    {
                        "id": "f608ce2b-5584-45af-b0c5-f4896995bd22",
                        "name": "imu-gzsx",
                        "endpoints": [
                            "http:\/\/s3gz.imu.cn:80"
                        ],
                        "log_meta": "false",
                        "log_data": "false",
                        "bucket_index_max_shards": 0,
                        "read_only": "false"
                    }
                ],
                "placement_targets": [
                    {
                        "name": "default-placement",
                        "tags": []
                    }
                ],
                "default_placement": "default-placement",
                "realm_id": "41e5334a-0b66-4e39-bdad-e210c8ca8730"
            }
        ],
        "short_zone_ids": [
            {
                "key": "f608ce2b-5584-45af-b0c5-f4896995bd22",
                "val": 1331659605
            }
        ]
    },
    "master_zonegroup": "1825aa39-e1d3-42ca-8cb6-3920335f2434",
    "master_zone": "f608ce2b-5584-45af-b0c5-f4896995bd22",
    "period_config": {
        "bucket_quota": {
            "enabled": false,
            "max_size_kb": -1,
            "max_objects": -1
        },
        "user_quota": {
            "enabled": false,
            "max_size_kb": -1,
            "max_objects": -1
        }
    },
    "realm_id": "41e5334a-0b66-4e39-bdad-e210c8ca8730",
    "realm_name": "imusic",
    "realm_epoch": 2
}

radosgw-admin realm default --rgw-realm=imusic
radosgw-admin zonegroup default --rgw-zonegroup=imu

#创建slave领域

radosgw-admin zone create  --rgw-zonegroup=imu --rgw-zone=imu-shxxy --access-key=K9237GSUFH0SZRQHBBP2 --secret=bKob0LirCx48EHrTAwDlB563rXa9dDziuJBHOQDS --endpoints=http://s3sh.imu.cn:80 --default

2018-01-03 15:48:15.182574 7fd289e969c0  0 failed reading obj info from .rgw.root:zone_info.f608ce2b-5584-45af-b0c5-f4896995bd22: (2) No such file or directory
2018-01-03 15:48:15.182614 7fd289e969c0  0 WARNING: could not read zone params for zone id=f608ce2b-5584-45af-b0c5-f4896995bd22 name=imu-gzsx
{
    "id": "01e0adba-e978-4c2f-be85-f42864464e87",
    "name": "imu-shxxy",
    "domain_root": "imu-shxxy.rgw.data.root",
    "control_pool": "imu-shxxy.rgw.control",
    "gc_pool": "imu-shxxy.rgw.gc",
    "log_pool": "imu-shxxy.rgw.log",
    "intent_log_pool": "imu-shxxy.rgw.intent-log",
    "usage_log_pool": "imu-shxxy.rgw.usage",
    "user_keys_pool": "imu-shxxy.rgw.users.keys",
    "user_email_pool": "imu-shxxy.rgw.users.email",
    "user_swift_pool": "imu-shxxy.rgw.users.swift",
    "user_uid_pool": "imu-shxxy.rgw.users.uid",
    "system_key": {
        "access_key": "K9237GSUFH0SZRQHBBP2",
        "secret_key": "bKob0LirCx48EHrTAwDlB563rXa9dDziuJBHOQDS"
    },
    "placement_pools": [
        {
            "key": "default-placement",
            "val": {
                "index_pool": "imu-shxxy.rgw.buckets.index",
                "data_pool": "imu-shxxy.rgw.buckets.data",
                "data_extra_pool": "imu-shxxy.rgw.buckets.non-ec",
                "index_type": 0
            }
        }
    ],
    "metadata_heap": "",
    "realm_id": "41e5334a-0b66-4e39-bdad-e210c8ca8730"
}

#update 

radosgw-admin period update --commit --rgw-zone=imu-shxxy

2018-01-03 15:48:38.326358 7f8465b9a9c0  1 Cannot find zone id=01e0adba-e978-4c2f-be85-f42864464e87 (name=imu-shxxy), switching to local zonegroup configuration
Sending period to new master zone f608ce2b-5584-45af-b0c5-f4896995bd22
{
    "id": "0f9aa0e6-2a6e-4689-abee-d64f43f0ccac",
    "epoch": 5,
    "predecessor_uuid": "d220c25c-dbf7-44b1-b410-f5cc30c52eb1",
    "sync_status": [],
    "period_map": {
        "id": "0f9aa0e6-2a6e-4689-abee-d64f43f0ccac",
        "zonegroups": [
            {
                "id": "1825aa39-e1d3-42ca-8cb6-3920335f2434",
                "name": "imu",
                "api_name": "imu",
                "is_master": "true",
                "endpoints": [
                    "http:\/\/s3gz.imu.cn:80"
                ],
                "hostnames": [],
                "hostnames_s3website": [],
                "master_zone": "f608ce2b-5584-45af-b0c5-f4896995bd22",
                "zones": [
                    {
                        "id": "01e0adba-e978-4c2f-be85-f42864464e87",
                        "name": "imu-shxxy",
                        "endpoints": [
                            "http:\/\/s3sh.imu.cn:80"
                        ],
                        "log_meta": "false",
                        "log_data": "true",
                        "bucket_index_max_shards": 0,
                        "read_only": "false"
                    },
                    {
                        "id": "f608ce2b-5584-45af-b0c5-f4896995bd22",
                        "name": "imu-gzsx",
                        "endpoints": [
                            "http:\/\/s3gz.imu.cn:80"
                        ],
                        "log_meta": "false",
                        "log_data": "true",
                        "bucket_index_max_shards": 0,
                        "read_only": "false"
                    }
                ],
                "placement_targets": [
                    {
                        "name": "default-placement",
                        "tags": []
                    }
                ],
                "default_placement": "default-placement",
                "realm_id": "41e5334a-0b66-4e39-bdad-e210c8ca8730"
            }
        ],
        "short_zone_ids": [
            {
                "key": "01e0adba-e978-4c2f-be85-f42864464e87",
                "val": 3509215161
            },
            {
                "key": "f608ce2b-5584-45af-b0c5-f4896995bd22",
                "val": 1331659605
            }
        ]
    },
    "master_zonegroup": "1825aa39-e1d3-42ca-8cb6-3920335f2434",
    "master_zone": "f608ce2b-5584-45af-b0c5-f4896995bd22",
    "period_config": {
        "bucket_quota": {
            "enabled": false,
            "max_size_kb": -1,
            "max_objects": -1
        },
        "user_quota": {
            "enabled": false,
            "max_size_kb": -1,
            "max_objects": -1
        }
    },
    "realm_id": "41e5334a-0b66-4e39-bdad-e210c8ca8730",
    "realm_name": "imusic",
    "realm_epoch": 2
}

#删除默认
 radosgw-admin zonegroup remove --rgw-zonegroup=default --rgw-zone=default
 radosgw-admin period update --commit
 radosgw-admin zone delete --rgw-zone=default
 radosgw-admin period update --commit
 radosgw-admin zonegroup delete --rgw-zonegroup=default
 radosgw-admin period update --commit

 rados rmpool default.rgw.control default.rgw.control --yes-i-really-really-mean-it
 rados rmpool default.rgw.meta default.rgw.meta --yes-i-really-really-mean-it
 rados rmpool default.rgw.log default.rgw.log --yes-i-really-really-mean-it
 
 rados rmpool default.rgw.data.root default.rgw.data.root --yes-i-really-really-mean-it
 rados rmpool default.rgw.gc default.rgw.gc --yes-i-really-really-mean-it
 rados rmpool default.rgw.users.keys default.rgw.users.keys --yes-i-really-really-mean-it


#修改配置

[client.rgw.xxy-2f201r-ceph-s3-c1-01]
host = xxy-2f201r-ceph-s3-c1-01
rgw_frontends = "civetweb port=80"
rgw_zone=imu-shxxy
rgw_dns_name = s3sh.imu.cn
rgw_content_length_compat = true

ceph-deploy --overwrite-conf config push xxy-2f201r-ceph-s3-c1-0{1..3}

systemctl restart ceph-radosgw@rgw.`hostname -s`
#

ERROR:

2018-01-03 15:53:54.081272 7ffae4959700  0 -- 172.18.216.129:0/3441061439 submit_message mon_subscribe({osdmap=202}) v2 remote, 172.18.216.113:6789/0, failed lossy con, dropping message 0x7ffacc016320
2018-01-03 15:53:54.081304 7ffacbfff700  0 -- 172.18.216.129:0/1830369741 submit_message mon_subscribe({osdmap=202}) v2 remote, 172.18.216.113:6789/0, failed lossy con, dropping message 0x7ffabc0130f0
2018-01-03 15:53:54.084664 7ffacbfff700  0 monclient: hunting for new mon
2018-01-03 15:53:54.098603 7ffae4959700  0 monclient: hunting for new mon
2018-01-03 15:53:54.157355 7ff7b2e79700  1 rgw meta sync: epoch=0 in sync status comes before remote's oldest mdlog epoch=1, restarting sync
2018-01-03 15:53:54.167100 7ff7b1c75700  0 ERROR: failed to take a lock on datalog.sync-status.f608ce2b-5584-45af-b0c5-f4896995bd22
2018-01-03 15:53:54.167116 7ff7b1c75700  0 ERROR: failed to init sync, retcode=-16
2018-01-03 15:53:55.257017 7ff7b1c75700  0 ERROR: failed to take a lock on datalog.sync-status.f608ce2b-5584-45af-b0c5-f4896995bd22
2018-01-03 15:53:55.257032 7ff7b1c75700  0 ERROR: failed to init sync, retcode=-16
2018-01-03 15:53:57.336909 7ff7b1c75700  0 ERROR: failed to take a lock on datalog.sync-status.f608ce2b-5584-45af-b0c5-f4896995bd22
2018-01-03 15:53:57.336925 7ff7b1c75700  0 ERROR: failed to init sync, retcode=-16
2018-01-03 15:54:01.404930 7ff7b1c75700  0 ERROR: failed to take a lock on datalog.sync-status.f608ce2b-5584-45af-b0c5-f4896995bd22
2018-01-03 15:54:01.404946 7ff7b1c75700  0 ERROR: failed to init sync, retcode=-16
2018-01-03 15:54:02.123630 7ff7aba67700  1 ====== starting new request req=0x7ff7aba61710 =====
2018-01-03 15:54:03.285785 7ff7ab266700  1 ====== starting new request req=0x7ff7ab260710 =====
2018-01-03 15:54:03.786137 7ff7ab266700  1 ====== req done req=0x7ff7ab260710 op status=0 http_status=403 ======
2018-01-03 15:54:03.786178 7ff7aba67700  1 ====== req done req=0x7ff7aba61710 op status=0 http_status=403 ======
2018-01-03 15:54:03.786195 7ff7ab266700  1 civetweb: 0x7ffa3000b370: 172.18.52.241 - - [03/Jan/2018:15:54:03 +0800] "GET /admin/log HTTP/1.1" 403 0 - -
2018-01-03 15:54:03.786272 7ff7aba67700  1 civetweb: 0x7ffaac01de80: 172.18.52.241 - - [03/Jan/2018:15:54:02 +0800] "GET /admin/log HTTP/1.1" 403 0 - -

2018-01-03 15:54:09.456163 7ff7b1c75700  0 ERROR: failed to take a lock on datalog.sync-status.f608ce2b-5584-45af-b0c5-f4896995bd22
2018-01-03 15:54:09.456177 7ff7b1c75700  0 ERROR: failed to init sync, retcode=-16
2018-01-03 15:54:22.011654 7ff7a725e700  1 ====== starting new request req=0x7ff7a7258710 =====
2018-01-03 15:54:22.015628 7ff7a725e700  1 ====== req done req=0x7ff7a7258710 op status=0 http_status=403 ======
2018-01-03 15:54:22.015697 7ff7a725e700  1 civetweb: 0x7ffaac0223c0: 172.18.52.242 - - [03/Jan/2018:15:54:22 +0800] "GET /admin/log HTTP/1.1" 403 0 - -
2018-01-03 15:54:25.508925 7ff7b1c75700  0 ERROR: failed to take a lock on datalog.sync-status.f608ce2b-5584-45af-b0c5-f4896995bd22
2018-01-03 15:54:25.508941 7ff7b1c75700  0 ERROR: failed to init sync, retcode=-16

2018-01-03 15:54:41.801518 7ff7a2a55700  1 ====== starting new request req=0x7ff7a2a4f710 =====
2018-01-03 15:54:41.805426 7ff7a2a55700  1 ====== req done req=0x7ff7a2a4f710 op status=0 http_status=403 ======
2018-01-03 15:54:41.806707 7ff7a2254700  1 ====== starting new request req=0x7ff7a224e710 =====
2018-01-03 15:54:41.810124 7ff7a2254700  1 ====== req done req=0x7ff7a224e710 op status=0 http_status=403 ======
2018-01-03 15:54:41.834157 7ff7a2a55700  1 civetweb: 0x7ffaa40219c0: 172.18.52.242 - - [03/Jan/2018:15:54:41 +0800] "POST /admin/realm/period HTTP/1.1" 403 0 - -
2018-01-03 15:54:41.838756 7ff7a2254700  1 civetweb: 0x7ffaac0358e0: 172.18.52.241 - - [03/Jan/2018:15:54:41 +0800] "POST /admin/realm/period HTTP/1.1" 403 0 - -
2018-01-03 15:54:55.562206 7ff7b1c75700  0 ERROR: failed to take a lock on datalog.sync-status.f608ce2b-5584-45af-b0c5-f4896995bd22
2018-01-03 15:54:55.562224 7ff7b1c75700  0 ERROR: failed to init sync, retcode=-16
2018-01-03 15:55:02.011254 7ff79da4b700  1 ====== starting new request req=0x7ff79da45710 =====
2018-01-03 15:55:02.015640 7ff79da4b700  1 ====== req done req=0x7ff79da45710 op status=0 http_status=403 ======
2018-01-03 15:55:02.015709 7ff79da4b700  1 civetweb: 0x7ffaac048d90: 172.18.52.242 - - [03/Jan/2018:15:55:02 +0800] "GET /admin/log HTTP/1.1" 403 0 - -

2018-01-03 15:55:11.864108 7ff79aa45700  1 ====== starting new request req=0x7ff79aa3f710 =====
2018-01-03 15:55:11.868045 7ff79aa45700  1 ====== req done req=0x7ff79aa3f710 op status=0 http_status=403 ======
2018-01-03 15:55:11.868798 7ff79a244700  1 ====== starting new request req=0x7ff79a23e710 =====
2018-01-03 15:55:11.872069 7ff79a244700  1 ====== req done req=0x7ff79a23e710 op status=0 http_status=403 ======
2018-01-03 15:55:11.897109 7ff79aa45700  1 civetweb: 0x7ffaa404dfd0: 172.18.52.241 - - [03/Jan/2018:15:55:11 +0800] "POST /admin/realm/period HTTP/1.1" 403 0 - -
2018-01-03 15:55:11.900796 7ff79a244700  1 civetweb: 0x7ffaac053830: 172.18.52.241 - - [03/Jan/2018:15:55:11 +0800] "POST /admin/realm/period HTTP/1.1" 403 0 - -
2018-01-03 15:55:25.606730 7ff7b1c75700  0 ERROR: failed to take a lock on datalog.sync-status.f608ce2b-5584-45af-b0c5-f4896995bd22
2018-01-03 15:55:25.606744 7ff7b1c75700  0 ERROR: failed to init sync, retcode=-16



#附录命令：

systemctl enable ceph-radosgw@rgw.`hostname -s`
systemctl restart ceph-radosgw@rgw.`hostname -s`
systemctl status ceph-radosgw@rgw.`hostname -s`





##

#radosgw-admin realm  delete --rgw-realm=imusic
#radosgw-admin zone delete --rgw-zone=imu-gzsx
					


#创建自定义pool映射bucket

@gzsx zone
ceph osd pool create imu-gzsx.rgw.buckets.video.data 512 512
ceph osd pool create imu-gzsx.rgw.buckets.video.index 64 64
ceph osd pool create imu-gzsx.rgw.buckets.video.extra 64 64

@shxxy zone
ceph osd pool create imu-shxxy.rgw.buckets.video.data 512 512
ceph osd pool create imu-shxxy.rgw.buckets.video.index 64 64
ceph osd pool create imu-shxxy.rgw.buckets.video.extra 64 64


#分别在2个zone导出 zonegroup 和zone ：
#下面仅举例一个zone修改
radosgw-admin zonegroup  get >zonegroup.conf.json
#修改
 "placement_targets": [
        {
            "name": "default-placement",
            "tags": []
        },
        {
            "name": "video-placement",#增加local bucket
            "tags": []
        }
    ],
	
radosgw-admin zone   get >zone.conf.json

#修改
 "placement_pools": [
        {
            "key": "default-placement",
            "val": {
                "index_pool": "imu-gzsx.rgw.buckets.index",
                "data_pool": "imu-gzsx.rgw.buckets.data",
                "data_extra_pool": "imu-gzsx.rgw.buckets.non-ec",
                "index_type": 0
            }
        },
        {
            "key": "My-bucket",
            "val": {
                "index_pool": "imu-gzsx.rgw.buckets.video.index",
                "data_pool": "imu-gzsx.rgw.buckets.video.data",
                "data_extra_pool": "imu-gzsx.rgw.buckets.video.extra",
                "index_type": 0
            }
        }
    ],


#更新
radosgw-admin zonegroup set  < zonegroup.conf.json
radosgw-admin zone set  < zone.conf.json

#刷新配置
radosgw-admin period update --commit
#重启所有节点：
systemctl restart ceph-radosgw@rgw.`hostname -f`
systemctl status ceph-radosgw@rgw.`hostname -f`

#创建bucket
 s3cmd  -c Ymliu.cfg  mb s3://video  --bucket-location=:video-placement
 
#查看

radosgw-admin bucket stats --bucket=video

上传测试22MB 成功
查看rados 成功