|
2015-07-01 10:25:35,489 13112 [radosgw_agent][INFO ] __ __ __ ___ ___
|
|
2015-07-01 10:25:35,490 13112 [radosgw_agent][INFO ] /__` \ / |\ | / ` /\ / _` |__ |\ | |
|
|
2015-07-01 10:25:35,491 13112 [radosgw_agent][INFO ] .__/ | | \| \__, /~~\ \__> |___ | \| |
|
|
2015-07-01 10:25:35,491 13112 [radosgw_agent][INFO ] v1.2.2
|
|
2015-07-01 10:25:35,492 13112 [radosgw_agent][INFO ] agent options:
|
|
2015-07-01 10:25:35,492 13112 [radosgw_agent][INFO ] args:
|
|
2015-07-01 10:25:35,493 13112 [radosgw_agent][INFO ] conf : None
|
|
2015-07-01 10:25:35,494 13112 [radosgw_agent][INFO ] dest_access_key : ****************
|
|
2015-07-01 10:25:35,495 13112 [radosgw_agent][INFO ] dest_secret_key : ****************
|
|
2015-07-01 10:25:35,495 13112 [radosgw_agent][INFO ] destination : http://10.35.64.71:80
|
|
2015-07-01 10:25:35,496 13112 [radosgw_agent][INFO ] incremental_sync_delay : 30
|
|
2015-07-01 10:25:35,496 13112 [radosgw_agent][INFO ] lock_timeout : 60
|
|
2015-07-01 10:25:35,497 13112 [radosgw_agent][INFO ] log_file : /var/log/radosgw/radosgw-sync-tw-east-west.log
|
|
2015-07-01 10:25:35,497 13112 [radosgw_agent][INFO ] log_lock_time : 20
|
|
2015-07-01 10:25:35,498 13112 [radosgw_agent][INFO ] max_entries : 1000
|
|
2015-07-01 10:25:35,498 13112 [radosgw_agent][INFO ] metadata_only : False
|
|
2015-07-01 10:25:35,499 13112 [radosgw_agent][INFO ] num_workers : 1
|
|
2015-07-01 10:25:35,500 13112 [radosgw_agent][INFO ] object_sync_timeout : 216000
|
|
2015-07-01 10:25:35,500 13112 [radosgw_agent][INFO ] prepare_error_delay : 10
|
|
2015-07-01 10:25:35,501 13112 [radosgw_agent][INFO ] quiet : False
|
|
2015-07-01 10:25:35,501 13112 [radosgw_agent][INFO ] rgw_data_log_window : 30
|
|
2015-07-01 10:25:35,502 13112 [radosgw_agent][INFO ] source : http://10.35.64.81:80
|
|
2015-07-01 10:25:35,502 13112 [radosgw_agent][INFO ] src_access_key : ****************
|
|
2015-07-01 10:25:35,503 13112 [radosgw_agent][INFO ] src_secret_key : ****************
|
|
2015-07-01 10:25:35,503 13112 [radosgw_agent][INFO ] src_zone : None
|
|
2015-07-01 10:25:35,504 13112 [radosgw_agent][INFO ] sync_scope : incremental
|
|
2015-07-01 10:25:35,504 13112 [radosgw_agent][INFO ] test_server_host : None
|
|
2015-07-01 10:25:35,505 13112 [radosgw_agent][INFO ] test_server_port : 8080
|
|
2015-07-01 10:25:35,505 13112 [radosgw_agent][INFO ] verbose : False
|
|
2015-07-01 10:25:35,506 13112 [radosgw_agent][INFO ] versioned : False
|
|
2015-07-01 10:25:35,506 13112 [radosgw_agent.client][INFO ] creating connection to endpoint: http://10.35.64.71:80
|
|
2015-07-01 10:25:35,507 13112 [boto][DEBUG ] Using access key provided by client.
|
|
2015-07-01 10:25:35,507 13112 [boto][DEBUG ] Using secret key provided by client.
|
|
2015-07-01 10:25:35,507 13112 [boto][DEBUG ] StringToSign:
|
|
GET
|
|
|
|
|
|
|
|
Wed, 01 Jul 2015 02:34:38 GMT
|
|
/admin/log
|
|
2015-07-01 10:34:38,643 13396 [boto][DEBUG ] Signature:
|
|
AWS tw_access_key:FVxX1NG3HVthpPE2ibJdBng2AlQ=
|
|
2015-07-01 10:34:38,643 13396 [boto][DEBUG ] Final headers: {'Date': 'Wed, 01 Jul 2015 02:34:38 GMT', 'Content-Length': '0', 'Authorization': u'AWS tw_access_key:FVxX1NG3HVthpPE2ibJdBng2AlQ=', 'User-Agent': 'Boto/2.38.0 Python/2.7.6 Linux/3.13.0-32-generic'}
|
|
2015-07-01 10:34:38,647 13396 [boto][DEBUG ] Response headers: [('date', 'Wed, 01 Jul 2015 02:34:38 GMT'), ('transfer-encoding', 'chunked'), ('content-type', 'application/json'), ('server', 'Apache/2.4.7 (Ubuntu)')]
|
|
2015-07-01 10:34:38,648 13396 [radosgw_agent.worker][DEBUG ] bucket instance "BASKET1:tw-east.6930.1" has 2 entries after " "
|
|
2015-07-01 10:34:38,648 13396 [radosgw_agent.worker][INFO ] ********************************************************************************
|
|
2015-07-01 10:34:38,648 13396 [radosgw_agent.worker][INFO ] syncing bucket "BASKET1"
|
|
2015-07-01 10:34:38,649 13396 [radosgw_agent.worker][DEBUG ] syncing object BASKET1/tw.json
|
|
2015-07-01 10:34:38,649 13396 [boto][DEBUG ] StringToSign:
|
|
PUT
|
|
|
|
application/json; charset=UTF-8
|
|
Wed, 01 Jul 2015 02:34:38 GMT
|
|
x-amz-copy-source:BASKET1/tw.json
|
|
/BASKET1/tw.json
|
|
2015-07-01 10:34:38,649 13396 [boto][DEBUG ] Signature:
|
|
AWS tw_access_key:UjJpeisLGfca9SE0oNsLeT0Igx0=
|
|
2015-07-01 10:34:38,650 13396 [boto][DEBUG ] url = 'http://10.35.64.71/BASKET1/tw.json'
|
|
params={'rgwx-op-id': 'node1:13112:1', 'rgwx-source-zone': u'tw-east', 'rgwx-client-id': 'radosgw-agent'}
|
|
headers={'Content-Length': '0', 'User-Agent': 'Boto/2.38.0 Python/2.7.6 Linux/3.13.0-32-generic', 'x-amz-copy-source': 'BASKET1/tw.json', 'Date': 'Wed, 01 Jul 2015 02:34:38 GMT', 'Content-Type': 'application/json; charset=UTF-8', 'Authorization': u'AWS tw_access_key:UjJpeisLGfca9SE0oNsLeT0Igx0='}
|
|
data=None
|
|
2015-07-01 10:34:38,650 13396 [boto][DEBUG ] Method: PUT
|
|
2015-07-01 10:34:38,650 13396 [boto][DEBUG ] Path: /BASKET1/tw.json?rgwx-op-id=node1%3A13112%3A1&rgwx-source-zone=tw-east&rgwx-client-id=radosgw-agent
|
|
2015-07-01 10:34:38,650 13396 [boto][DEBUG ] Data:
|
|
2015-07-01 10:34:38,650 13396 [boto][DEBUG ] Headers: {'Content-Type': 'application/json; charset=UTF-8', 'x-amz-copy-source': 'BASKET1/tw.json'}
|
|
2015-07-01 10:34:38,650 13396 [boto][DEBUG ] Host: 10.35.64.71
|
|
2015-07-01 10:34:38,650 13396 [boto][DEBUG ] Port: 80
|
|
2015-07-01 10:34:38,650 13396 [boto][DEBUG ] Params: {'rgwx-op-id': 'node1%3A13112%3A1', 'rgwx-source-zone': 'tw-east', 'rgwx-client-id': 'radosgw-agent'}
|
|
2015-07-01 10:34:38,651 13396 [boto][DEBUG ] establishing HTTP connection: kwargs={'port': 80, 'timeout': 70}
|
|
2015-07-01 10:34:38,652 13396 [boto][DEBUG ] Token: None
|
|
2015-07-01 10:34:38,652 13396 [boto][DEBUG ] StringToSign:
|
|
PUT
|
|
|
|
application/json; charset=UTF-8
|
|
Wed, 01 Jul 2015 02:34:38 GMT
|
|
x-amz-copy-source:BASKET1/tw.json
|
|
/BASKET1/tw.json
|
|
2015-07-01 10:34:38,652 13396 [boto][DEBUG ] Signature:
|
|
AWS tw_access_key:UjJpeisLGfca9SE0oNsLeT0Igx0=
|
|
2015-07-01 10:34:38,652 13396 [boto][DEBUG ] Final headers: {'Content-Length': '0', 'User-Agent': 'Boto/2.38.0 Python/2.7.6 Linux/3.13.0-32-generic', 'x-amz-copy-source': 'BASKET1/tw.json', 'Date': 'Wed, 01 Jul 2015 02:34:38 GMT', 'Content-Type': 'application/json; charset=UTF-8', 'Authorization': u'AWS tw_access_key:UjJpeisLGfca9SE0oNsLeT0Igx0='}
|
|
2015-07-01 10:34:39,734 13396 [boto][DEBUG ] Response headers: [('content-length', '78'), ('accept-ranges', 'bytes'), ('server', 'Apache/2.4.7 (Ubuntu)'), ('connection', 'close'), ('date', 'Wed, 01 Jul 2015 02:34:38 GMT'), ('content-type', 'application/xml')]
|
|
2015-07-01 10:34:39,734 13396 [boto][DEBUG ] Received 500 response. Retrying in 0.6 seconds
|
|
2015-07-01 10:34:40,287 13396 [boto][DEBUG ] Token: None
|
|
2015-07-01 10:34:40,287 13396 [boto][DEBUG ] StringToSign:
|
|
PUT
|
|
|
|
application/json; charset=UTF-8
|
|
Wed, 01 Jul 2015 02:34:38 GMT
|
|
x-amz-copy-source:BASKET1/tw.json
|
|
/BASKET1/tw.json
|
|
2015-07-01 10:34:40,288 13396 [boto][DEBUG ] Signature:
|
|
AWS tw_access_key:UjJpeisLGfca9SE0oNsLeT0Igx0=
|
|
2015-07-01 10:34:40,288 13396 [boto][DEBUG ] Final headers: {'Content-Length': '0', 'User-Agent': 'Boto/2.38.0 Python/2.7.6 Linux/3.13.0-32-generic', 'x-amz-copy-source': 'BASKET1/tw.json', 'Date': 'Wed, 01 Jul 2015 02:34:38 GMT', 'Content-Type': 'application/json; charset=UTF-8', 'Authorization': u'AWS tw_access_key:UjJpeisLGfca9SE0oNsLeT0Igx0='}
|
|
2015-07-01 10:34:40,319 13396 [boto][DEBUG ] Response headers: [('content-length', '78'), ('accept-ranges', 'bytes'), ('server', 'Apache/2.4.7 (Ubuntu)'), ('connection', 'close'), ('date', 'Wed, 01 Jul 2015 02:34:40 GMT'), ('content-type', 'application/xml')]
|
|
2015-07-01 10:34:40,319 13396 [boto][DEBUG ] Received 500 response. Retrying in 1.6 seconds
|
|
2015-07-01 10:34:41,897 13396 [boto][DEBUG ] Token: None
|
|
2015-07-01 10:34:41,898 13396 [boto][DEBUG ] StringToSign:
|
|
PUT
|
|
|
|
application/json; charset=UTF-8
|
|
Wed, 01 Jul 2015 02:34:38 GMT
|
|
x-amz-copy-source:BASKET1/tw.json
|
|
/BASKET1/tw.json
|
|
2015-07-01 10:34:41,898 13396 [boto][DEBUG ] Signature:
|
|
AWS tw_access_key:UjJpeisLGfca9SE0oNsLeT0Igx0=
|
|
2015-07-01 10:34:41,899 13396 [boto][DEBUG ] Final headers: {'Content-Length': '0', 'User-Agent': 'Boto/2.38.0 Python/2.7.6 Linux/3.13.0-32-generic', 'x-amz-copy-source': 'BASKET1/tw.json', 'Date': 'Wed, 01 Jul 2015 02:34:38 GMT', 'Content-Type': 'application/json; charset=UTF-8', 'Authorization': u'AWS tw_access_key:UjJpeisLGfca9SE0oNsLeT0Igx0='}
|
|
2015-07-01 10:34:41,932 13396 [boto][DEBUG ] Response headers: [('content-length', '78'), ('accept-ranges', 'bytes'), ('server', 'Apache/2.4.7 (Ubuntu)'), ('connection', 'close'), ('date', 'Wed, 01 Jul 2015 02:34:41 GMT'), ('content-type', 'application/xml')]
|
|
2015-07-01 10:34:41,932 13396 [boto][DEBUG ] Received 500 response. Retrying in 2.5 seconds
|
|
2015-07-01 10:34:44,480 13396 [boto][DEBUG ] Token: None
|
|
2015-07-01 10:34:44,481 13396 [boto][DEBUG ] StringToSign:
|
|
PUT
|
|
|
|
application/json; charset=UTF-8
|
|
Wed, 01 Jul 2015 02:34:38 GMT
|
|
x-amz-copy-source:BASKET1/tw.json
|
|
/BASKET1/tw.json
|
|
2015-07-01 10:34:44,481 13396 [boto][DEBUG ] Signature:
|
|
AWS tw_access_key:UjJpeisLGfca9SE0oNsLeT0Igx0=
|
|
2015-07-01 10:34:44,482 13396 [boto][DEBUG ] Final headers: {'Content-Length': '0', 'User-Agent': 'Boto/2.38.0 Python/2.7.6 Linux/3.13.0-32-generic', 'x-amz-copy-source': 'BASKET1/tw.json', 'Date': 'Wed, 01 Jul 2015 02:34:38 GMT', 'Content-Type': 'application/json; charset=UTF-8', 'Authorization': u'AWS tw_access_key:UjJpeisLGfca9SE0oNsLeT0Igx0='}
|
|
2015-07-01 10:34:44,551 13396 [boto][DEBUG ] Response headers: [('content-length', '78'), ('accept-ranges', 'bytes'), ('server', 'Apache/2.4.7 (Ubuntu)'), ('connection', 'close'), ('date', 'Wed, 01 Jul 2015 02:34:44 GMT'), ('content-type', 'application/xml')]
|
|
2015-07-01 10:34:44,551 13396 [boto][DEBUG ] Received 500 response. Retrying in 1.8 seconds
|
|
2015-07-01 10:34:46,385 13396 [dev.radosgw_agent.worker][WARNING] encountered an error during sync
|
|
Traceback (most recent call last):
|
|
File "build/bdist.linux-x86_64/egg/radosgw_agent/worker.py", line 232, in sync_object
|
|
local_op_id)
|
|
File "build/bdist.linux-x86_64/egg/radosgw_agent/client.py", line 346, in sync_object_intra_region
|
|
expect_json=False)
|
|
File "build/bdist.linux-x86_64/egg/radosgw_agent/client.py", line 145, in request
|
|
check_result_status(error)
|
|
File "build/bdist.linux-x86_64/egg/radosgw_agent/client.py", line 89, in check_result_status
|
|
exc.HttpError)(result.status, result.reason)
|
|
HttpError: Http error code 500 content Internal Server Error
|
|
2015-07-01 10:34:46,386 13396 [radosgw_agent.worker][WARNING] encountered an error during sync: Http error code 500 content Internal Server Error
|
|
2015-07-01 10:34:46,387 13396 [boto][DEBUG ] StringToSign:
|
|
GET
|
|
|
|
|
|
Wed, 01 Jul 2015 02:34:46 GMT
|
|
/admin/opstate
|
|
2015-07-01 10:34:46,387 13396 [boto][DEBUG ] Signature:
|
|
AWS tw_access_key:EEyQmZ/LFXkDQO0VjNGMKC/CSi8=
|
|
2015-07-01 10:34:46,387 13396 [boto][DEBUG ] url = 'http://10.35.64.71/admin/opstate'
|
|
params={'client-id': 'radosgw-agent', 'object': u'BASKET1/tw.json', 'op-id': 'node1:13112:1'}
|
|
headers={'Date': 'Wed, 01 Jul 2015 02:34:46 GMT', 'Content-Length': '0', 'Authorization': u'AWS tw_access_key:EEyQmZ/LFXkDQO0VjNGMKC/CSi8=', 'User-Agent': 'Boto/2.38.0 Python/2.7.6 Linux/3.13.0-32-generic'}
|
|
data=None
|
|
2015-07-01 10:34:46,388 13396 [boto][DEBUG ] Method: GET
|
|
2015-07-01 10:34:46,388 13396 [boto][DEBUG ] Path: /admin/opstate?client-id=radosgw-agent&object=BASKET1/tw.json&op-id=node1%3A13112%3A1
|
|
2015-07-01 10:34:46,388 13396 [boto][DEBUG ] Data:
|
|
2015-07-01 10:34:46,388 13396 [boto][DEBUG ] Headers: {}
|
|
2015-07-01 10:34:46,388 13396 [boto][DEBUG ] Host: 10.35.64.71
|
|
2015-07-01 10:34:46,388 13396 [boto][DEBUG ] Port: 80
|
|
2015-07-01 10:34:46,388 13396 [boto][DEBUG ] Params: {'client-id': 'radosgw-agent', 'object': 'BASKET1/tw.json', 'op-id': 'node1%3A13112%3A1'}
|
|
2015-07-01 10:34:46,389 13396 [boto][DEBUG ] establishing HTTP connection: kwargs={'port': 80, 'timeout': 70}
|
|
2015-07-01 10:34:46,389 13396 [boto][DEBUG ] Token: None
|
|
2015-07-01 10:34:46,389 13396 [boto][DEBUG ] StringToSign:
|
|
GET
|
|
|
|
|
|
Wed, 01 Jul 2015 02:34:46 GMT
|
|
/admin/opstate
|
|
2015-07-01 10:34:46,389 13396 [boto][DEBUG ] Signature:
|
|
AWS tw_access_key:EEyQmZ/LFXkDQO0VjNGMKC/CSi8=
|
|
2015-07-01 10:34:46,389 13396 [boto][DEBUG ] Final headers: {'Date': 'Wed, 01 Jul 2015 02:34:46 GMT', 'Content-Length': '0', 'Authorization': u'AWS tw_access_key:EEyQmZ/LFXkDQO0VjNGMKC/CSi8=', 'User-Agent': 'Boto/2.38.0 Python/2.7.6 Linux/3.13.0-32-generic'}
|
|
2015-07-01 10:34:46,398 13396 [boto][DEBUG ] Response headers: [('date', 'Wed, 01 Jul 2015 02:34:46 GMT'), ('transfer-encoding', 'chunked'), ('content-type', 'application/json'), ('server', 'Apache/2.4.7 (Ubuntu)')]
|
|
2015-07-01 10:34:46,398 13396 [radosgw_agent.worker][DEBUG ] op state is [{u'timestamp': u'2015-07-01 02:34:44.512157Z', u'op_id': u'node1:13112:1', u'object': u'BASKET1/tw.json', u'state': u'error', u'client_id': u'radosgw-agent'}]
|
|
2015-07-01 10:34:46,398 13396 [radosgw_agent.worker][ERROR ] failed to sync object BASKET1/tw.json: state is error
|
|
2015-07-01 10:34:46,399 13396 [radosgw_agent.worker][WARNING] will retry sync of failed object at next incremental sync
|
|
2015-07-01 10:34:46,400 13396 [radosgw_agent.worker][DEBUG ] syncing object BASKET1/tw.json
|
|
2015-07-01 10:34:46,400 13396 [boto][DEBUG ] StringToSign:
|
|
PUT
|
|
|
|
application/json; charset=UTF-8
|
|
Wed, 01 Jul 2015 02:34:46 GMT
|
|
x-amz-copy-source:BASKET1/tw.json
|
|
/BASKET1/tw.json
|
|
2015-07-01 10:34:46,400 13396 [boto][DEBUG ] Signature:
|
|
AWS tw_access_key:7Rtkf63skVqApbisb4TE630wbZ4=
|
|
2015-07-01 10:34:46,401 13396 [boto][DEBUG ] url = 'http://10.35.64.71/BASKET1/tw.json'
|
|
params={'rgwx-op-id': 'node1:13112:2', 'rgwx-source-zone': u'tw-east', 'rgwx-client-id': 'radosgw-agent'}
|
|
headers={'Content-Length': '0', 'User-Agent': 'Boto/2.38.0 Python/2.7.6 Linux/3.13.0-32-generic', 'x-amz-copy-source': 'BASKET1/tw.json', 'Date': 'Wed, 01 Jul 2015 02:34:46 GMT', 'Content-Type': 'application/json; charset=UTF-8', 'Authorization': u'AWS tw_access_key:7Rtkf63skVqApbisb4TE630wbZ4='}
|
|
data=None
|
|
2015-07-01 10:34:46,401 13396 [boto][DEBUG ] Method: PUT
|
|
2015-07-01 10:34:46,401 13396 [boto][DEBUG ] Path: /BASKET1/tw.json?rgwx-op-id=node1%3A13112%3A2&rgwx-source-zone=tw-east&rgwx-client-id=radosgw-agent
|
|
2015-07-01 10:34:46,401 13396 [boto][DEBUG ] Data:
|
|
2015-07-01 10:34:46,401 13396 [boto][DEBUG ] Headers: {'Content-Type': 'application/json; charset=UTF-8', 'x-amz-copy-source': 'BASKET1/tw.json'}
|
|
2015-07-01 10:34:46,401 13396 [boto][DEBUG ] Host: 10.35.64.71
|
|
2015-07-01 10:34:46,401 13396 [boto][DEBUG ] Port: 80
|
|
2015-07-01 10:34:46,401 13396 [boto][DEBUG ] Params: {'rgwx-op-id': 'node1%3A13112%3A2', 'rgwx-source-zone': 'tw-east', 'rgwx-client-id': 'radosgw-agent'}
|
|
2015-07-01 10:34:46,402 13396 [boto][DEBUG ] Token: None
|
|
2015-07-01 10:34:46,402 13396 [boto][DEBUG ] StringToSign:
|
|
PUT
|
|
|
|
application/json; charset=UTF-8
|
|
Wed, 01 Jul 2015 02:34:46 GMT
|
|
x-amz-copy-source:BASKET1/tw.json
|
|
/BASKET1/tw.json
|
|
2015-07-01 10:34:46,402 13396 [boto][DEBUG ] Signature:
|
|
AWS tw_access_key:7Rtkf63skVqApbisb4TE630wbZ4=
|
|
2015-07-01 10:34:46,402 13396 [boto][DEBUG ] Final headers: {'Content-Length': '0', 'User-Agent': 'Boto/2.38.0 Python/2.7.6 Linux/3.13.0-32-generic', 'x-amz-copy-source': 'BASKET1/tw.json', 'Date': 'Wed, 01 Jul 2015 02:34:46 GMT', 'Content-Type': 'application/json; charset=UTF-8', 'Authorization': u'AWS tw_access_key:7Rtkf63skVqApbisb4TE630wbZ4='}
|
|
2015-07-01 10:34:46,553 13396 [boto][DEBUG ] Response headers: [('content-length', '78'), ('accept-ranges', 'bytes'), ('server', 'Apache/2.4.7 (Ubuntu)'), ('connection', 'close'), ('date', 'Wed, 01 Jul 2015 02:34:46 GMT'), ('content-type', 'application/xml')]
|
|
2015-07-01 10:34:46,553 13396 [boto][DEBUG ] Received 500 response. Retrying in 1.0 seconds
|
|
2015-07-01 10:34:47,544 13396 [boto][DEBUG ] Token: None
|
|
2015-07-01 10:34:47,545 13396 [boto][DEBUG ] StringToSign:
|
|
PUT
|
|
|
|
application/json; charset=UTF-8
|
|
Wed, 01 Jul 2015 02:34:46 GMT
|
|
x-amz-copy-source:BASKET1/tw.json
|
|
/BASKET1/tw.json
|
|
2015-07-01 10:34:47,545 13396 [boto][DEBUG ] Signature:
|
|
AWS tw_access_key:7Rtkf63skVqApbisb4TE630wbZ4=
|
|
2015-07-01 10:34:47,546 13396 [boto][DEBUG ] Final headers: {'Content-Length': '0', 'User-Agent': 'Boto/2.38.0 Python/2.7.6 Linux/3.13.0-32-generic', 'x-amz-copy-source': 'BASKET1/tw.json', 'Date': 'Wed, 01 Jul 2015 02:34:46 GMT', 'Content-Type': 'application/json; charset=UTF-8', 'Authorization': u'AWS tw_access_key:7Rtkf63skVqApbisb4TE630wbZ4='}
|
|
2015-07-01 10:34:47,567 13396 [boto][DEBUG ] Response headers: [('content-length', '78'), ('accept-ranges', 'bytes'), ('server', 'Apache/2.4.7 (Ubuntu)'), ('connection', 'close'), ('date', 'Wed, 01 Jul 2015 02:34:47 GMT'), ('content-type', 'application/xml')]
|
|
2015-07-01 10:34:47,568 13396 [boto][DEBUG ] Received 500 response. Retrying in 1.4 seconds
|
|
2015-07-01 10:34:48,926 13396 [boto][DEBUG ] Token: None
|
|
2015-07-01 10:34:48,927 13396 [boto][DEBUG ] StringToSign:
|
|
PUT
|
|
|
|
application/json; charset=UTF-8
|
|
Wed, 01 Jul 2015 02:34:46 GMT
|
|
x-amz-copy-source:BASKET1/tw.json
|
|
/BASKET1/tw.json
|
|
2015-07-01 10:34:48,927 13396 [boto][DEBUG ] Signature:
|
|
AWS tw_access_key:7Rtkf63skVqApbisb4TE630wbZ4=
|
|
2015-07-01 10:34:48,928 13396 [boto][DEBUG ] Final headers: {'Content-Length': '0', 'User-Agent': 'Boto/2.38.0 Python/2.7.6 Linux/3.13.0-32-generic', 'x-amz-copy-source': 'BASKET1/tw.json', 'Date': 'Wed, 01 Jul 2015 02:34:46 GMT', 'Content-Type': 'application/json; charset=UTF-8', 'Authorization': u'AWS tw_access_key:7Rtkf63skVqApbisb4TE630wbZ4='}
|
|
2015-07-01 10:34:48,958 13396 [boto][DEBUG ] Response headers: [('content-length', '78'), ('accept-ranges', 'bytes'), ('server', 'Apache/2.4.7 (Ubuntu)'), ('connection', 'close'), ('date', 'Wed, 01 Jul 2015 02:34:48 GMT'), ('content-type', 'application/xml')]
|
|
2015-07-01 10:34:48,959 13396 [boto][DEBUG ] Received 500 response. Retrying in 2.9 seconds
|
|
2015-07-01 10:34:51,899 13396 [boto][DEBUG ] Token: None
|
|
2015-07-01 10:34:51,900 13396 [boto][DEBUG ] StringToSign:
|
|
PUT
|
|
|
|
application/json; charset=UTF-8
|
|
Wed, 01 Jul 2015 02:34:46 GMT
|
|
x-amz-copy-source:BASKET1/tw.json
|
|
/BASKET1/tw.json
|
|
2015-07-01 10:34:51,900 13396 [boto][DEBUG ] Signature:
|
|
AWS tw_access_key:7Rtkf63skVqApbisb4TE630wbZ4=
|
|
2015-07-01 10:34:51,901 13396 [boto][DEBUG ] Final headers: {'Content-Length': '0', 'User-Agent': 'Boto/2.38.0 Python/2.7.6 Linux/3.13.0-32-generic', 'x-amz-copy-source': 'BASKET1/tw.json', 'Date': 'Wed, 01 Jul 2015 02:34:46 GMT', 'Content-Type': 'application/json; charset=UTF-8', 'Authorization': u'AWS tw_access_key:7Rtkf63skVqApbisb4TE630wbZ4='}
|
|
2015-07-01 10:34:51,934 13396 [boto][DEBUG ] Response headers: [('content-length', '78'), ('accept-ranges', 'bytes'), ('server', 'Apache/2.4.7 (Ubuntu)'), ('connection', 'close'), ('date', 'Wed, 01 Jul 2015 02:34:51 GMT'), ('content-type', 'application/xml')]
|
|
2015-07-01 10:34:51,934 13396 [boto][DEBUG ] Received 500 response. Retrying in 2.1 seconds
|
|
2015-07-01 10:34:54,084 13396 [dev.radosgw_agent.worker][WARNING] encountered an error during sync
|
|
Traceback (most recent call last):
|
|
File "build/bdist.linux-x86_64/egg/radosgw_agent/worker.py", line 232, in sync_object
|
|
local_op_id)
|
|
File "build/bdist.linux-x86_64/egg/radosgw_agent/client.py", line 346, in sync_object_intra_region
|
|
expect_json=False)
|
|
File "build/bdist.linux-x86_64/egg/radosgw_agent/client.py", line 145, in request
|
|
check_result_status(error)
|
|
File "build/bdist.linux-x86_64/egg/radosgw_agent/client.py", line 89, in check_result_status
|
|
exc.HttpError)(result.status, result.reason)
|
|
HttpError: Http error code 500 content Internal Server Error
|
|
2015-07-01 10:34:54,085 13396 [radosgw_agent.worker][WARNING] encountered an error during sync: Http error code 500 content Internal Server Error
|
|
2015-07-01 10:34:54,086 13396 [boto][DEBUG ] StringToSign:
|
|
GET
|
|
|
|
|
|
Wed, 01 Jul 2015 02:34:54 GMT
|
|
/admin/opstate
|
|
2015-07-01 10:34:54,087 13396 [boto][DEBUG ] Signature:
|
|
AWS tw_access_key:x/e1b4icIs/m9OE2vXBF3ImWIAY=
|
|
2015-07-01 10:34:54,087 13396 [boto][DEBUG ] url = 'http://10.35.64.71/admin/opstate'
|
|
params={'client-id': 'radosgw-agent', 'object': u'BASKET1/tw.json', 'op-id': 'node1:13112:2'}
|
|
headers={'Date': 'Wed, 01 Jul 2015 02:34:54 GMT', 'Content-Length': '0', 'Authorization': u'AWS tw_access_key:x/e1b4icIs/m9OE2vXBF3ImWIAY=', 'User-Agent': 'Boto/2.38.0 Python/2.7.6 Linux/3.13.0-32-generic'}
|
|
data=None
|
|
2015-07-01 10:34:54,088 13396 [boto][DEBUG ] Method: GET
|
|
2015-07-01 10:34:54,088 13396 [boto][DEBUG ] Path: /admin/opstate?client-id=radosgw-agent&object=BASKET1/tw.json&op-id=node1%3A13112%3A2
|
|
2015-07-01 10:34:54,088 13396 [boto][DEBUG ] Data:
|
|
2015-07-01 10:34:54,088 13396 [boto][DEBUG ] Headers: {}
|
|
2015-07-01 10:34:54,088 13396 [boto][DEBUG ] Host: 10.35.64.71
|
|
2015-07-01 10:34:54,089 13396 [boto][DEBUG ] Port: 80
|
|
2015-07-01 10:34:54,090 13396 [boto][DEBUG ] Params: {'client-id': 'radosgw-agent', 'object': 'BASKET1/tw.json', 'op-id': 'node1%3A13112%3A2'}
|
|
2015-07-01 10:34:54,091 13396 [boto][DEBUG ] establishing HTTP connection: kwargs={'port': 80, 'timeout': 70}
|
|
2015-07-01 10:34:54,091 13396 [boto][DEBUG ] Token: None
|
|
2015-07-01 10:34:54,091 13396 [boto][DEBUG ] StringToSign:
|
|
GET
|
|
|
|
|
|
Wed, 01 Jul 2015 02:34:54 GMT
|
|
/admin/opstate
|
|
2015-07-01 10:34:54,091 13396 [boto][DEBUG ] Signature:
|
|
AWS tw_access_key:x/e1b4icIs/m9OE2vXBF3ImWIAY=
|
|
2015-07-01 10:34:54,091 13396 [boto][DEBUG ] Final headers: {'Date': 'Wed, 01 Jul 2015 02:34:54 GMT', 'Content-Length': '0', 'Authorization': u'AWS tw_access_key:x/e1b4icIs/m9OE2vXBF3ImWIAY=', 'User-Agent': 'Boto/2.38.0 Python/2.7.6 Linux/3.13.0-32-generic'}
|
|
2015-07-01 10:34:54,100 13396 [boto][DEBUG ] Response headers: [('date', 'Wed, 01 Jul 2015 02:34:54 GMT'), ('transfer-encoding', 'chunked'), ('content-type', 'application/json'), ('server', 'Apache/2.4.7 (Ubuntu)')]
|
|
2015-07-01 10:34:54,101 13396 [radosgw_agent.worker][DEBUG ] op state is [{u'timestamp': u'2015-07-01 02:34:51.925204Z', u'op_id': u'node1:13112:2', u'object': u'BASKET1/tw.json', u'state': u'error', u'client_id': u'radosgw-agent'}]
|
|
2015-07-01 10:34:54,101 13396 [radosgw_agent.worker][ERROR ] failed to sync object BASKET1/tw.json: state is error
|
|
2015-07-01 10:34:54,101 13396 [radosgw_agent.worker][WARNING] will retry sync of failed object at next incremental sync
|
|
2015-07-01 10:34:54,102 13396 [radosgw_agent.worker][INFO ] synced 0 objects
|
|
2015-07-01 10:34:54,102 13396 [radosgw_agent.worker][INFO ] completed syncing bucket "BASKET1"
|
|
2015-07-01 10:34:54,103 13396 [radosgw_agent.worker][INFO ] ********************************************************************************
|
|
2015-07-01 10:34:54,105 13396 [boto][DEBUG ] set_worker_bound: data = [{'ver': BucketVer(epoch=0, pool=-1), 'name': u'tw.json', 'versioned': False, 'timestamp': u'0.000000', 'object': u'tw.json', 'version_id': 'null', 'VersionedEpoch': 0, 'time': '1970-01-01 00:00:00', 'marker': u'00000000001.1.2', 'op': u'write'}, {'ver': BucketVer(epoch=1, pool=79), 'name': u'tw.json', 'versioned': False, 'timestamp': u'2015-07-01 02:33:24.000000Z', 'object': u'tw.json', 'version_id': 'null', 'VersionedEpoch': 1, 'time': '1970-01-01 00:00:00', 'marker': u'00000000002.2.3', 'op': u'write'}]
|
|
2015-07-01 10:34:54,105 13396 [boto][DEBUG ] path=/admin/replica_log?work_bound
|
|
2015-07-01 10:34:54,105 13396 [boto][DEBUG ] auth_path=/admin/replica_log?work_bound
|
|
2015-07-01 10:34:54,106 13396 [boto][DEBUG ] StringToSign:
|
|
POST
|
|
|