|
1
|
2015-07-01 10:25:35,489 13112 [radosgw_agent][INFO ] __ __ __ ___ ___
|
|
2
|
2015-07-01 10:25:35,490 13112 [radosgw_agent][INFO ] /__` \ / |\ | / ` /\ / _` |__ |\ | |
|
|
3
|
2015-07-01 10:25:35,491 13112 [radosgw_agent][INFO ] .__/ | | \| \__, /~~\ \__> |___ | \| |
|
|
4
|
2015-07-01 10:25:35,491 13112 [radosgw_agent][INFO ] v1.2.2
|
|
5
|
2015-07-01 10:25:35,492 13112 [radosgw_agent][INFO ] agent options:
|
|
6
|
2015-07-01 10:25:35,492 13112 [radosgw_agent][INFO ] args:
|
|
7
|
2015-07-01 10:25:35,493 13112 [radosgw_agent][INFO ] conf : None
|
|
8
|
2015-07-01 10:25:35,494 13112 [radosgw_agent][INFO ] dest_access_key : ****************
|
|
9
|
2015-07-01 10:25:35,495 13112 [radosgw_agent][INFO ] dest_secret_key : ****************
|
|
10
|
2015-07-01 10:25:35,495 13112 [radosgw_agent][INFO ] destination : http://10.35.64.71:80
|
|
11
|
2015-07-01 10:25:35,496 13112 [radosgw_agent][INFO ] incremental_sync_delay : 30
|
|
12
|
2015-07-01 10:25:35,496 13112 [radosgw_agent][INFO ] lock_timeout : 60
|
|
13
|
2015-07-01 10:25:35,497 13112 [radosgw_agent][INFO ] log_file : /var/log/radosgw/radosgw-sync-tw-east-west.log
|
|
14
|
2015-07-01 10:25:35,497 13112 [radosgw_agent][INFO ] log_lock_time : 20
|
|
15
|
2015-07-01 10:25:35,498 13112 [radosgw_agent][INFO ] max_entries : 1000
|
|
16
|
2015-07-01 10:25:35,498 13112 [radosgw_agent][INFO ] metadata_only : False
|
|
17
|
2015-07-01 10:25:35,499 13112 [radosgw_agent][INFO ] num_workers : 1
|
|
18
|
2015-07-01 10:25:35,500 13112 [radosgw_agent][INFO ] object_sync_timeout : 216000
|
|
19
|
2015-07-01 10:25:35,500 13112 [radosgw_agent][INFO ] prepare_error_delay : 10
|
|
20
|
2015-07-01 10:25:35,501 13112 [radosgw_agent][INFO ] quiet : False
|
|
21
|
2015-07-01 10:25:35,501 13112 [radosgw_agent][INFO ] rgw_data_log_window : 30
|
|
22
|
2015-07-01 10:25:35,502 13112 [radosgw_agent][INFO ] source : http://10.35.64.81:80
|
|
23
|
2015-07-01 10:25:35,502 13112 [radosgw_agent][INFO ] src_access_key : ****************
|
|
24
|
2015-07-01 10:25:35,503 13112 [radosgw_agent][INFO ] src_secret_key : ****************
|
|
25
|
2015-07-01 10:25:35,503 13112 [radosgw_agent][INFO ] src_zone : None
|
|
26
|
2015-07-01 10:25:35,504 13112 [radosgw_agent][INFO ] sync_scope : incremental
|
|
27
|
2015-07-01 10:25:35,504 13112 [radosgw_agent][INFO ] test_server_host : None
|
|
28
|
2015-07-01 10:25:35,505 13112 [radosgw_agent][INFO ] test_server_port : 8080
|
|
29
|
2015-07-01 10:25:35,505 13112 [radosgw_agent][INFO ] verbose : False
|
|
30
|
2015-07-01 10:25:35,506 13112 [radosgw_agent][INFO ] versioned : False
|
|
31
|
2015-07-01 10:25:35,506 13112 [radosgw_agent.client][INFO ] creating connection to endpoint: http://10.35.64.71:80
|
|
32
|
2015-07-01 10:25:35,507 13112 [boto][DEBUG ] Using access key provided by client.
|
|
33
|
2015-07-01 10:25:35,507 13112 [boto][DEBUG ] Using secret key provided by client.
|
|
34
|
2015-07-01 10:25:35,507 13112 [boto][DEBUG ] StringToSign:
|
|
35
|
GET
|
|
36
|
|
|
37
|
|
|
38
|
|
|
39
|
Wed, 01 Jul 2015 02:34:38 GMT
|
|
40
|
/admin/log
|
|
41
|
2015-07-01 10:34:38,643 13396 [boto][DEBUG ] Signature:
|
|
42
|
AWS tw_access_key:FVxX1NG3HVthpPE2ibJdBng2AlQ=
|
|
43
|
2015-07-01 10:34:38,643 13396 [boto][DEBUG ] Final headers: {'Date': 'Wed, 01 Jul 2015 02:34:38 GMT', 'Content-Length': '0', 'Authorization': u'AWS tw_access_key:FVxX1NG3HVthpPE2ibJdBng2AlQ=', 'User-Agent': 'Boto/2.38.0 Python/2.7.6 Linux/3.13.0-32-generic'}
|
|
44
|
2015-07-01 10:34:38,647 13396 [boto][DEBUG ] Response headers: [('date', 'Wed, 01 Jul 2015 02:34:38 GMT'), ('transfer-encoding', 'chunked'), ('content-type', 'application/json'), ('server', 'Apache/2.4.7 (Ubuntu)')]
|
|
45
|
2015-07-01 10:34:38,648 13396 [radosgw_agent.worker][DEBUG ] bucket instance "BASKET1:tw-east.6930.1" has 2 entries after " "
|
|
46
|
2015-07-01 10:34:38,648 13396 [radosgw_agent.worker][INFO ] ********************************************************************************
|
|
47
|
2015-07-01 10:34:38,648 13396 [radosgw_agent.worker][INFO ] syncing bucket "BASKET1"
|
|
48
|
2015-07-01 10:34:38,649 13396 [radosgw_agent.worker][DEBUG ] syncing object BASKET1/tw.json
|
|
49
|
2015-07-01 10:34:38,649 13396 [boto][DEBUG ] StringToSign:
|
|
50
|
PUT
|
|
51
|
|
|
52
|
application/json; charset=UTF-8
|
|
53
|
Wed, 01 Jul 2015 02:34:38 GMT
|
|
54
|
x-amz-copy-source:BASKET1/tw.json
|
|
55
|
/BASKET1/tw.json
|
|
56
|
2015-07-01 10:34:38,649 13396 [boto][DEBUG ] Signature:
|
|
57
|
AWS tw_access_key:UjJpeisLGfca9SE0oNsLeT0Igx0=
|
|
58
|
2015-07-01 10:34:38,650 13396 [boto][DEBUG ] url = 'http://10.35.64.71/BASKET1/tw.json'
|
|
59
|
params={'rgwx-op-id': 'node1:13112:1', 'rgwx-source-zone': u'tw-east', 'rgwx-client-id': 'radosgw-agent'}
|
|
60
|
headers={'Content-Length': '0', 'User-Agent': 'Boto/2.38.0 Python/2.7.6 Linux/3.13.0-32-generic', 'x-amz-copy-source': 'BASKET1/tw.json', 'Date': 'Wed, 01 Jul 2015 02:34:38 GMT', 'Content-Type': 'application/json; charset=UTF-8', 'Authorization': u'AWS tw_access_key:UjJpeisLGfca9SE0oNsLeT0Igx0='}
|
|
61
|
data=None
|
|
62
|
2015-07-01 10:34:38,650 13396 [boto][DEBUG ] Method: PUT
|
|
63
|
2015-07-01 10:34:38,650 13396 [boto][DEBUG ] Path: /BASKET1/tw.json?rgwx-op-id=node1%3A13112%3A1&rgwx-source-zone=tw-east&rgwx-client-id=radosgw-agent
|
|
64
|
2015-07-01 10:34:38,650 13396 [boto][DEBUG ] Data:
|
|
65
|
2015-07-01 10:34:38,650 13396 [boto][DEBUG ] Headers: {'Content-Type': 'application/json; charset=UTF-8', 'x-amz-copy-source': 'BASKET1/tw.json'}
|
|
66
|
2015-07-01 10:34:38,650 13396 [boto][DEBUG ] Host: 10.35.64.71
|
|
67
|
2015-07-01 10:34:38,650 13396 [boto][DEBUG ] Port: 80
|
|
68
|
2015-07-01 10:34:38,650 13396 [boto][DEBUG ] Params: {'rgwx-op-id': 'node1%3A13112%3A1', 'rgwx-source-zone': 'tw-east', 'rgwx-client-id': 'radosgw-agent'}
|
|
69
|
2015-07-01 10:34:38,651 13396 [boto][DEBUG ] establishing HTTP connection: kwargs={'port': 80, 'timeout': 70}
|
|
70
|
2015-07-01 10:34:38,652 13396 [boto][DEBUG ] Token: None
|
|
71
|
2015-07-01 10:34:38,652 13396 [boto][DEBUG ] StringToSign:
|
|
72
|
PUT
|
|
73
|
|
|
74
|
application/json; charset=UTF-8
|
|
75
|
Wed, 01 Jul 2015 02:34:38 GMT
|
|
76
|
x-amz-copy-source:BASKET1/tw.json
|
|
77
|
/BASKET1/tw.json
|
|
78
|
2015-07-01 10:34:38,652 13396 [boto][DEBUG ] Signature:
|
|
79
|
AWS tw_access_key:UjJpeisLGfca9SE0oNsLeT0Igx0=
|
|
80
|
2015-07-01 10:34:38,652 13396 [boto][DEBUG ] Final headers: {'Content-Length': '0', 'User-Agent': 'Boto/2.38.0 Python/2.7.6 Linux/3.13.0-32-generic', 'x-amz-copy-source': 'BASKET1/tw.json', 'Date': 'Wed, 01 Jul 2015 02:34:38 GMT', 'Content-Type': 'application/json; charset=UTF-8', 'Authorization': u'AWS tw_access_key:UjJpeisLGfca9SE0oNsLeT0Igx0='}
|
|
81
|
2015-07-01 10:34:39,734 13396 [boto][DEBUG ] Response headers: [('content-length', '78'), ('accept-ranges', 'bytes'), ('server', 'Apache/2.4.7 (Ubuntu)'), ('connection', 'close'), ('date', 'Wed, 01 Jul 2015 02:34:38 GMT'), ('content-type', 'application/xml')]
|
|
82
|
2015-07-01 10:34:39,734 13396 [boto][DEBUG ] Received 500 response. Retrying in 0.6 seconds
|
|
83
|
2015-07-01 10:34:40,287 13396 [boto][DEBUG ] Token: None
|
|
84
|
2015-07-01 10:34:40,287 13396 [boto][DEBUG ] StringToSign:
|
|
85
|
PUT
|
|
86
|
|
|
87
|
application/json; charset=UTF-8
|
|
88
|
Wed, 01 Jul 2015 02:34:38 GMT
|
|
89
|
x-amz-copy-source:BASKET1/tw.json
|
|
90
|
/BASKET1/tw.json
|
|
91
|
2015-07-01 10:34:40,288 13396 [boto][DEBUG ] Signature:
|
|
92
|
AWS tw_access_key:UjJpeisLGfca9SE0oNsLeT0Igx0=
|
|
93
|
2015-07-01 10:34:40,288 13396 [boto][DEBUG ] Final headers: {'Content-Length': '0', 'User-Agent': 'Boto/2.38.0 Python/2.7.6 Linux/3.13.0-32-generic', 'x-amz-copy-source': 'BASKET1/tw.json', 'Date': 'Wed, 01 Jul 2015 02:34:38 GMT', 'Content-Type': 'application/json; charset=UTF-8', 'Authorization': u'AWS tw_access_key:UjJpeisLGfca9SE0oNsLeT0Igx0='}
|
|
94
|
2015-07-01 10:34:40,319 13396 [boto][DEBUG ] Response headers: [('content-length', '78'), ('accept-ranges', 'bytes'), ('server', 'Apache/2.4.7 (Ubuntu)'), ('connection', 'close'), ('date', 'Wed, 01 Jul 2015 02:34:40 GMT'), ('content-type', 'application/xml')]
|
|
95
|
2015-07-01 10:34:40,319 13396 [boto][DEBUG ] Received 500 response. Retrying in 1.6 seconds
|
|
96
|
2015-07-01 10:34:41,897 13396 [boto][DEBUG ] Token: None
|
|
97
|
2015-07-01 10:34:41,898 13396 [boto][DEBUG ] StringToSign:
|
|
98
|
PUT
|
|
99
|
|
|
100
|
application/json; charset=UTF-8
|
|
101
|
Wed, 01 Jul 2015 02:34:38 GMT
|
|
102
|
x-amz-copy-source:BASKET1/tw.json
|
|
103
|
/BASKET1/tw.json
|
|
104
|
2015-07-01 10:34:41,898 13396 [boto][DEBUG ] Signature:
|
|
105
|
AWS tw_access_key:UjJpeisLGfca9SE0oNsLeT0Igx0=
|
|
106
|
2015-07-01 10:34:41,899 13396 [boto][DEBUG ] Final headers: {'Content-Length': '0', 'User-Agent': 'Boto/2.38.0 Python/2.7.6 Linux/3.13.0-32-generic', 'x-amz-copy-source': 'BASKET1/tw.json', 'Date': 'Wed, 01 Jul 2015 02:34:38 GMT', 'Content-Type': 'application/json; charset=UTF-8', 'Authorization': u'AWS tw_access_key:UjJpeisLGfca9SE0oNsLeT0Igx0='}
|
|
107
|
2015-07-01 10:34:41,932 13396 [boto][DEBUG ] Response headers: [('content-length', '78'), ('accept-ranges', 'bytes'), ('server', 'Apache/2.4.7 (Ubuntu)'), ('connection', 'close'), ('date', 'Wed, 01 Jul 2015 02:34:41 GMT'), ('content-type', 'application/xml')]
|
|
108
|
2015-07-01 10:34:41,932 13396 [boto][DEBUG ] Received 500 response. Retrying in 2.5 seconds
|
|
109
|
2015-07-01 10:34:44,480 13396 [boto][DEBUG ] Token: None
|
|
110
|
2015-07-01 10:34:44,481 13396 [boto][DEBUG ] StringToSign:
|
|
111
|
PUT
|
|
112
|
|
|
113
|
application/json; charset=UTF-8
|
|
114
|
Wed, 01 Jul 2015 02:34:38 GMT
|
|
115
|
x-amz-copy-source:BASKET1/tw.json
|
|
116
|
/BASKET1/tw.json
|
|
117
|
2015-07-01 10:34:44,481 13396 [boto][DEBUG ] Signature:
|
|
118
|
AWS tw_access_key:UjJpeisLGfca9SE0oNsLeT0Igx0=
|
|
119
|
2015-07-01 10:34:44,482 13396 [boto][DEBUG ] Final headers: {'Content-Length': '0', 'User-Agent': 'Boto/2.38.0 Python/2.7.6 Linux/3.13.0-32-generic', 'x-amz-copy-source': 'BASKET1/tw.json', 'Date': 'Wed, 01 Jul 2015 02:34:38 GMT', 'Content-Type': 'application/json; charset=UTF-8', 'Authorization': u'AWS tw_access_key:UjJpeisLGfca9SE0oNsLeT0Igx0='}
|
|
120
|
2015-07-01 10:34:44,551 13396 [boto][DEBUG ] Response headers: [('content-length', '78'), ('accept-ranges', 'bytes'), ('server', 'Apache/2.4.7 (Ubuntu)'), ('connection', 'close'), ('date', 'Wed, 01 Jul 2015 02:34:44 GMT'), ('content-type', 'application/xml')]
|
|
121
|
2015-07-01 10:34:44,551 13396 [boto][DEBUG ] Received 500 response. Retrying in 1.8 seconds
|
|
122
|
2015-07-01 10:34:46,385 13396 [dev.radosgw_agent.worker][WARNING] encountered an error during sync
|
|
123
|
Traceback (most recent call last):
|
|
124
|
File "build/bdist.linux-x86_64/egg/radosgw_agent/worker.py", line 232, in sync_object
|
|
125
|
local_op_id)
|
|
126
|
File "build/bdist.linux-x86_64/egg/radosgw_agent/client.py", line 346, in sync_object_intra_region
|
|
127
|
expect_json=False)
|
|
128
|
File "build/bdist.linux-x86_64/egg/radosgw_agent/client.py", line 145, in request
|
|
129
|
check_result_status(error)
|
|
130
|
File "build/bdist.linux-x86_64/egg/radosgw_agent/client.py", line 89, in check_result_status
|
|
131
|
exc.HttpError)(result.status, result.reason)
|
|
132
|
HttpError: Http error code 500 content Internal Server Error
|
|
133
|
2015-07-01 10:34:46,386 13396 [radosgw_agent.worker][WARNING] encountered an error during sync: Http error code 500 content Internal Server Error
|
|
134
|
2015-07-01 10:34:46,387 13396 [boto][DEBUG ] StringToSign:
|
|
135
|
GET
|
|
136
|
|
|
137
|
|
|
138
|
Wed, 01 Jul 2015 02:34:46 GMT
|
|
139
|
/admin/opstate
|
|
140
|
2015-07-01 10:34:46,387 13396 [boto][DEBUG ] Signature:
|
|
141
|
AWS tw_access_key:EEyQmZ/LFXkDQO0VjNGMKC/CSi8=
|
|
142
|
2015-07-01 10:34:46,387 13396 [boto][DEBUG ] url = 'http://10.35.64.71/admin/opstate'
|
|
143
|
params={'client-id': 'radosgw-agent', 'object': u'BASKET1/tw.json', 'op-id': 'node1:13112:1'}
|
|
144
|
headers={'Date': 'Wed, 01 Jul 2015 02:34:46 GMT', 'Content-Length': '0', 'Authorization': u'AWS tw_access_key:EEyQmZ/LFXkDQO0VjNGMKC/CSi8=', 'User-Agent': 'Boto/2.38.0 Python/2.7.6 Linux/3.13.0-32-generic'}
|
|
145
|
data=None
|
|
146
|
2015-07-01 10:34:46,388 13396 [boto][DEBUG ] Method: GET
|
|
147
|
2015-07-01 10:34:46,388 13396 [boto][DEBUG ] Path: /admin/opstate?client-id=radosgw-agent&object=BASKET1/tw.json&op-id=node1%3A13112%3A1
|
|
148
|
2015-07-01 10:34:46,388 13396 [boto][DEBUG ] Data:
|
|
149
|
2015-07-01 10:34:46,388 13396 [boto][DEBUG ] Headers: {}
|
|
150
|
2015-07-01 10:34:46,388 13396 [boto][DEBUG ] Host: 10.35.64.71
|
|
151
|
2015-07-01 10:34:46,388 13396 [boto][DEBUG ] Port: 80
|
|
152
|
2015-07-01 10:34:46,388 13396 [boto][DEBUG ] Params: {'client-id': 'radosgw-agent', 'object': 'BASKET1/tw.json', 'op-id': 'node1%3A13112%3A1'}
|
|
153
|
2015-07-01 10:34:46,389 13396 [boto][DEBUG ] establishing HTTP connection: kwargs={'port': 80, 'timeout': 70}
|
|
154
|
2015-07-01 10:34:46,389 13396 [boto][DEBUG ] Token: None
|
|
155
|
2015-07-01 10:34:46,389 13396 [boto][DEBUG ] StringToSign:
|
|
156
|
GET
|
|
157
|
|
|
158
|
|
|
159
|
Wed, 01 Jul 2015 02:34:46 GMT
|
|
160
|
/admin/opstate
|
|
161
|
2015-07-01 10:34:46,389 13396 [boto][DEBUG ] Signature:
|
|
162
|
AWS tw_access_key:EEyQmZ/LFXkDQO0VjNGMKC/CSi8=
|
|
163
|
2015-07-01 10:34:46,389 13396 [boto][DEBUG ] Final headers: {'Date': 'Wed, 01 Jul 2015 02:34:46 GMT', 'Content-Length': '0', 'Authorization': u'AWS tw_access_key:EEyQmZ/LFXkDQO0VjNGMKC/CSi8=', 'User-Agent': 'Boto/2.38.0 Python/2.7.6 Linux/3.13.0-32-generic'}
|
|
164
|
2015-07-01 10:34:46,398 13396 [boto][DEBUG ] Response headers: [('date', 'Wed, 01 Jul 2015 02:34:46 GMT'), ('transfer-encoding', 'chunked'), ('content-type', 'application/json'), ('server', 'Apache/2.4.7 (Ubuntu)')]
|
|
165
|
2015-07-01 10:34:46,398 13396 [radosgw_agent.worker][DEBUG ] op state is [{u'timestamp': u'2015-07-01 02:34:44.512157Z', u'op_id': u'node1:13112:1', u'object': u'BASKET1/tw.json', u'state': u'error', u'client_id': u'radosgw-agent'}]
|
|
166
|
2015-07-01 10:34:46,398 13396 [radosgw_agent.worker][ERROR ] failed to sync object BASKET1/tw.json: state is error
|
|
167
|
2015-07-01 10:34:46,399 13396 [radosgw_agent.worker][WARNING] will retry sync of failed object at next incremental sync
|
|
168
|
2015-07-01 10:34:46,400 13396 [radosgw_agent.worker][DEBUG ] syncing object BASKET1/tw.json
|
|
169
|
2015-07-01 10:34:46,400 13396 [boto][DEBUG ] StringToSign:
|
|
170
|
PUT
|
|
171
|
|
|
172
|
application/json; charset=UTF-8
|
|
173
|
Wed, 01 Jul 2015 02:34:46 GMT
|
|
174
|
x-amz-copy-source:BASKET1/tw.json
|
|
175
|
/BASKET1/tw.json
|
|
176
|
2015-07-01 10:34:46,400 13396 [boto][DEBUG ] Signature:
|
|
177
|
AWS tw_access_key:7Rtkf63skVqApbisb4TE630wbZ4=
|
|
178
|
2015-07-01 10:34:46,401 13396 [boto][DEBUG ] url = 'http://10.35.64.71/BASKET1/tw.json'
|
|
179
|
params={'rgwx-op-id': 'node1:13112:2', 'rgwx-source-zone': u'tw-east', 'rgwx-client-id': 'radosgw-agent'}
|
|
180
|
headers={'Content-Length': '0', 'User-Agent': 'Boto/2.38.0 Python/2.7.6 Linux/3.13.0-32-generic', 'x-amz-copy-source': 'BASKET1/tw.json', 'Date': 'Wed, 01 Jul 2015 02:34:46 GMT', 'Content-Type': 'application/json; charset=UTF-8', 'Authorization': u'AWS tw_access_key:7Rtkf63skVqApbisb4TE630wbZ4='}
|
|
181
|
data=None
|
|
182
|
2015-07-01 10:34:46,401 13396 [boto][DEBUG ] Method: PUT
|
|
183
|
2015-07-01 10:34:46,401 13396 [boto][DEBUG ] Path: /BASKET1/tw.json?rgwx-op-id=node1%3A13112%3A2&rgwx-source-zone=tw-east&rgwx-client-id=radosgw-agent
|
|
184
|
2015-07-01 10:34:46,401 13396 [boto][DEBUG ] Data:
|
|
185
|
2015-07-01 10:34:46,401 13396 [boto][DEBUG ] Headers: {'Content-Type': 'application/json; charset=UTF-8', 'x-amz-copy-source': 'BASKET1/tw.json'}
|
|
186
|
2015-07-01 10:34:46,401 13396 [boto][DEBUG ] Host: 10.35.64.71
|
|
187
|
2015-07-01 10:34:46,401 13396 [boto][DEBUG ] Port: 80
|
|
188
|
2015-07-01 10:34:46,401 13396 [boto][DEBUG ] Params: {'rgwx-op-id': 'node1%3A13112%3A2', 'rgwx-source-zone': 'tw-east', 'rgwx-client-id': 'radosgw-agent'}
|
|
189
|
2015-07-01 10:34:46,402 13396 [boto][DEBUG ] Token: None
|
|
190
|
2015-07-01 10:34:46,402 13396 [boto][DEBUG ] StringToSign:
|
|
191
|
PUT
|
|
192
|
|
|
193
|
application/json; charset=UTF-8
|
|
194
|
Wed, 01 Jul 2015 02:34:46 GMT
|
|
195
|
x-amz-copy-source:BASKET1/tw.json
|
|
196
|
/BASKET1/tw.json
|
|
197
|
2015-07-01 10:34:46,402 13396 [boto][DEBUG ] Signature:
|
|
198
|
AWS tw_access_key:7Rtkf63skVqApbisb4TE630wbZ4=
|
|
199
|
2015-07-01 10:34:46,402 13396 [boto][DEBUG ] Final headers: {'Content-Length': '0', 'User-Agent': 'Boto/2.38.0 Python/2.7.6 Linux/3.13.0-32-generic', 'x-amz-copy-source': 'BASKET1/tw.json', 'Date': 'Wed, 01 Jul 2015 02:34:46 GMT', 'Content-Type': 'application/json; charset=UTF-8', 'Authorization': u'AWS tw_access_key:7Rtkf63skVqApbisb4TE630wbZ4='}
|
|
200
|
2015-07-01 10:34:46,553 13396 [boto][DEBUG ] Response headers: [('content-length', '78'), ('accept-ranges', 'bytes'), ('server', 'Apache/2.4.7 (Ubuntu)'), ('connection', 'close'), ('date', 'Wed, 01 Jul 2015 02:34:46 GMT'), ('content-type', 'application/xml')]
|
|
201
|
2015-07-01 10:34:46,553 13396 [boto][DEBUG ] Received 500 response. Retrying in 1.0 seconds
|
|
202
|
2015-07-01 10:34:47,544 13396 [boto][DEBUG ] Token: None
|
|
203
|
2015-07-01 10:34:47,545 13396 [boto][DEBUG ] StringToSign:
|
|
204
|
PUT
|
|
205
|
|
|
206
|
application/json; charset=UTF-8
|
|
207
|
Wed, 01 Jul 2015 02:34:46 GMT
|
|
208
|
x-amz-copy-source:BASKET1/tw.json
|
|
209
|
/BASKET1/tw.json
|
|
210
|
2015-07-01 10:34:47,545 13396 [boto][DEBUG ] Signature:
|
|
211
|
AWS tw_access_key:7Rtkf63skVqApbisb4TE630wbZ4=
|
|
212
|
2015-07-01 10:34:47,546 13396 [boto][DEBUG ] Final headers: {'Content-Length': '0', 'User-Agent': 'Boto/2.38.0 Python/2.7.6 Linux/3.13.0-32-generic', 'x-amz-copy-source': 'BASKET1/tw.json', 'Date': 'Wed, 01 Jul 2015 02:34:46 GMT', 'Content-Type': 'application/json; charset=UTF-8', 'Authorization': u'AWS tw_access_key:7Rtkf63skVqApbisb4TE630wbZ4='}
|
|
213
|
2015-07-01 10:34:47,567 13396 [boto][DEBUG ] Response headers: [('content-length', '78'), ('accept-ranges', 'bytes'), ('server', 'Apache/2.4.7 (Ubuntu)'), ('connection', 'close'), ('date', 'Wed, 01 Jul 2015 02:34:47 GMT'), ('content-type', 'application/xml')]
|
|
214
|
2015-07-01 10:34:47,568 13396 [boto][DEBUG ] Received 500 response. Retrying in 1.4 seconds
|
|
215
|
2015-07-01 10:34:48,926 13396 [boto][DEBUG ] Token: None
|
|
216
|
2015-07-01 10:34:48,927 13396 [boto][DEBUG ] StringToSign:
|
|
217
|
PUT
|
|
218
|
|
|
219
|
application/json; charset=UTF-8
|
|
220
|
Wed, 01 Jul 2015 02:34:46 GMT
|
|
221
|
x-amz-copy-source:BASKET1/tw.json
|
|
222
|
/BASKET1/tw.json
|
|
223
|
2015-07-01 10:34:48,927 13396 [boto][DEBUG ] Signature:
|
|
224
|
AWS tw_access_key:7Rtkf63skVqApbisb4TE630wbZ4=
|
|
225
|
2015-07-01 10:34:48,928 13396 [boto][DEBUG ] Final headers: {'Content-Length': '0', 'User-Agent': 'Boto/2.38.0 Python/2.7.6 Linux/3.13.0-32-generic', 'x-amz-copy-source': 'BASKET1/tw.json', 'Date': 'Wed, 01 Jul 2015 02:34:46 GMT', 'Content-Type': 'application/json; charset=UTF-8', 'Authorization': u'AWS tw_access_key:7Rtkf63skVqApbisb4TE630wbZ4='}
|
|
226
|
2015-07-01 10:34:48,958 13396 [boto][DEBUG ] Response headers: [('content-length', '78'), ('accept-ranges', 'bytes'), ('server', 'Apache/2.4.7 (Ubuntu)'), ('connection', 'close'), ('date', 'Wed, 01 Jul 2015 02:34:48 GMT'), ('content-type', 'application/xml')]
|
|
227
|
2015-07-01 10:34:48,959 13396 [boto][DEBUG ] Received 500 response. Retrying in 2.9 seconds
|
|
228
|
2015-07-01 10:34:51,899 13396 [boto][DEBUG ] Token: None
|
|
229
|
2015-07-01 10:34:51,900 13396 [boto][DEBUG ] StringToSign:
|
|
230
|
PUT
|
|
231
|
|
|
232
|
application/json; charset=UTF-8
|
|
233
|
Wed, 01 Jul 2015 02:34:46 GMT
|
|
234
|
x-amz-copy-source:BASKET1/tw.json
|
|
235
|
/BASKET1/tw.json
|
|
236
|
2015-07-01 10:34:51,900 13396 [boto][DEBUG ] Signature:
|
|
237
|
AWS tw_access_key:7Rtkf63skVqApbisb4TE630wbZ4=
|
|
238
|
2015-07-01 10:34:51,901 13396 [boto][DEBUG ] Final headers: {'Content-Length': '0', 'User-Agent': 'Boto/2.38.0 Python/2.7.6 Linux/3.13.0-32-generic', 'x-amz-copy-source': 'BASKET1/tw.json', 'Date': 'Wed, 01 Jul 2015 02:34:46 GMT', 'Content-Type': 'application/json; charset=UTF-8', 'Authorization': u'AWS tw_access_key:7Rtkf63skVqApbisb4TE630wbZ4='}
|
|
239
|
2015-07-01 10:34:51,934 13396 [boto][DEBUG ] Response headers: [('content-length', '78'), ('accept-ranges', 'bytes'), ('server', 'Apache/2.4.7 (Ubuntu)'), ('connection', 'close'), ('date', 'Wed, 01 Jul 2015 02:34:51 GMT'), ('content-type', 'application/xml')]
|
|
240
|
2015-07-01 10:34:51,934 13396 [boto][DEBUG ] Received 500 response. Retrying in 2.1 seconds
|
|
241
|
2015-07-01 10:34:54,084 13396 [dev.radosgw_agent.worker][WARNING] encountered an error during sync
|
|
242
|
Traceback (most recent call last):
|
|
243
|
File "build/bdist.linux-x86_64/egg/radosgw_agent/worker.py", line 232, in sync_object
|
|
244
|
local_op_id)
|
|
245
|
File "build/bdist.linux-x86_64/egg/radosgw_agent/client.py", line 346, in sync_object_intra_region
|
|
246
|
expect_json=False)
|
|
247
|
File "build/bdist.linux-x86_64/egg/radosgw_agent/client.py", line 145, in request
|
|
248
|
check_result_status(error)
|
|
249
|
File "build/bdist.linux-x86_64/egg/radosgw_agent/client.py", line 89, in check_result_status
|
|
250
|
exc.HttpError)(result.status, result.reason)
|
|
251
|
HttpError: Http error code 500 content Internal Server Error
|
|
252
|
2015-07-01 10:34:54,085 13396 [radosgw_agent.worker][WARNING] encountered an error during sync: Http error code 500 content Internal Server Error
|
|
253
|
2015-07-01 10:34:54,086 13396 [boto][DEBUG ] StringToSign:
|
|
254
|
GET
|
|
255
|
|
|
256
|
|
|
257
|
Wed, 01 Jul 2015 02:34:54 GMT
|
|
258
|
/admin/opstate
|
|
259
|
2015-07-01 10:34:54,087 13396 [boto][DEBUG ] Signature:
|
|
260
|
AWS tw_access_key:x/e1b4icIs/m9OE2vXBF3ImWIAY=
|
|
261
|
2015-07-01 10:34:54,087 13396 [boto][DEBUG ] url = 'http://10.35.64.71/admin/opstate'
|
|
262
|
params={'client-id': 'radosgw-agent', 'object': u'BASKET1/tw.json', 'op-id': 'node1:13112:2'}
|
|
263
|
headers={'Date': 'Wed, 01 Jul 2015 02:34:54 GMT', 'Content-Length': '0', 'Authorization': u'AWS tw_access_key:x/e1b4icIs/m9OE2vXBF3ImWIAY=', 'User-Agent': 'Boto/2.38.0 Python/2.7.6 Linux/3.13.0-32-generic'}
|
|
264
|
data=None
|
|
265
|
2015-07-01 10:34:54,088 13396 [boto][DEBUG ] Method: GET
|
|
266
|
2015-07-01 10:34:54,088 13396 [boto][DEBUG ] Path: /admin/opstate?client-id=radosgw-agent&object=BASKET1/tw.json&op-id=node1%3A13112%3A2
|
|
267
|
2015-07-01 10:34:54,088 13396 [boto][DEBUG ] Data:
|
|
268
|
2015-07-01 10:34:54,088 13396 [boto][DEBUG ] Headers: {}
|
|
269
|
2015-07-01 10:34:54,088 13396 [boto][DEBUG ] Host: 10.35.64.71
|
|
270
|
2015-07-01 10:34:54,089 13396 [boto][DEBUG ] Port: 80
|
|
271
|
2015-07-01 10:34:54,090 13396 [boto][DEBUG ] Params: {'client-id': 'radosgw-agent', 'object': 'BASKET1/tw.json', 'op-id': 'node1%3A13112%3A2'}
|
|
272
|
2015-07-01 10:34:54,091 13396 [boto][DEBUG ] establishing HTTP connection: kwargs={'port': 80, 'timeout': 70}
|
|
273
|
2015-07-01 10:34:54,091 13396 [boto][DEBUG ] Token: None
|
|
274
|
2015-07-01 10:34:54,091 13396 [boto][DEBUG ] StringToSign:
|
|
275
|
GET
|
|
276
|
|
|
277
|
|
|
278
|
Wed, 01 Jul 2015 02:34:54 GMT
|
|
279
|
/admin/opstate
|
|
280
|
2015-07-01 10:34:54,091 13396 [boto][DEBUG ] Signature:
|
|
281
|
AWS tw_access_key:x/e1b4icIs/m9OE2vXBF3ImWIAY=
|
|
282
|
2015-07-01 10:34:54,091 13396 [boto][DEBUG ] Final headers: {'Date': 'Wed, 01 Jul 2015 02:34:54 GMT', 'Content-Length': '0', 'Authorization': u'AWS tw_access_key:x/e1b4icIs/m9OE2vXBF3ImWIAY=', 'User-Agent': 'Boto/2.38.0 Python/2.7.6 Linux/3.13.0-32-generic'}
|
|
283
|
2015-07-01 10:34:54,100 13396 [boto][DEBUG ] Response headers: [('date', 'Wed, 01 Jul 2015 02:34:54 GMT'), ('transfer-encoding', 'chunked'), ('content-type', 'application/json'), ('server', 'Apache/2.4.7 (Ubuntu)')]
|
|
284
|
2015-07-01 10:34:54,101 13396 [radosgw_agent.worker][DEBUG ] op state is [{u'timestamp': u'2015-07-01 02:34:51.925204Z', u'op_id': u'node1:13112:2', u'object': u'BASKET1/tw.json', u'state': u'error', u'client_id': u'radosgw-agent'}]
|
|
285
|
2015-07-01 10:34:54,101 13396 [radosgw_agent.worker][ERROR ] failed to sync object BASKET1/tw.json: state is error
|
|
286
|
2015-07-01 10:34:54,101 13396 [radosgw_agent.worker][WARNING] will retry sync of failed object at next incremental sync
|
|
287
|
2015-07-01 10:34:54,102 13396 [radosgw_agent.worker][INFO ] synced 0 objects
|
|
288
|
2015-07-01 10:34:54,102 13396 [radosgw_agent.worker][INFO ] completed syncing bucket "BASKET1"
|
|
289
|
2015-07-01 10:34:54,103 13396 [radosgw_agent.worker][INFO ] ********************************************************************************
|
|
290
|
2015-07-01 10:34:54,105 13396 [boto][DEBUG ] set_worker_bound: data = [{'ver': BucketVer(epoch=0, pool=-1), 'name': u'tw.json', 'versioned': False, 'timestamp': u'0.000000', 'object': u'tw.json', 'version_id': 'null', 'VersionedEpoch': 0, 'time': '1970-01-01 00:00:00', 'marker': u'00000000001.1.2', 'op': u'write'}, {'ver': BucketVer(epoch=1, pool=79), 'name': u'tw.json', 'versioned': False, 'timestamp': u'2015-07-01 02:33:24.000000Z', 'object': u'tw.json', 'version_id': 'null', 'VersionedEpoch': 1, 'time': '1970-01-01 00:00:00', 'marker': u'00000000002.2.3', 'op': u'write'}]
|
|
291
|
2015-07-01 10:34:54,105 13396 [boto][DEBUG ] path=/admin/replica_log?work_bound
|
|
292
|
2015-07-01 10:34:54,105 13396 [boto][DEBUG ] auth_path=/admin/replica_log?work_bound
|
|
293
|
2015-07-01 10:34:54,106 13396 [boto][DEBUG ] StringToSign:
|
|
294
|
POST
|
|
295
|
|