Project

General

Profile

Bug #12574 » bucket_stats.txt

Bucket_stats - sean redmond, 08/03/2015 03:40 PM

 
root@ceph-2-radosgw-c:~# radosgw-admin --id radosgw.gateway bucket stats --bucket=seantest
2015-08-03 16:20:08.253815 7f20bbffa780 1 -- :/0 messenger.start
2015-08-03 16:20:08.254628 7f20bbffa780 1 -- :/1006108 --> 172.28.196.13:6789/0 -- auth(proto 0 40 bytes epoch 0) v1 -- ?+0 0x2afe270 con 0x2af8190
2015-08-03 16:20:08.255456 7f20bbff6700 1 -- 172.28.196.123:0/1006108 learned my addr 172.28.196.123:0/1006108
2015-08-03 16:20:08.256787 7f20b10ff700 1 -- 172.28.196.123:0/1006108 <== mon.2 172.28.196.13:6789/0 1 ==== mon_map magic: 0 v1 ==== 500+0+0 (170246281 0 0) 0x7f20a4000be0 con 0x2af8190
2015-08-03 16:20:08.257312 7f20b10ff700 1 -- 172.28.196.123:0/1006108 <== mon.2 172.28.196.13:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (2442968086 0 0) 0x7f20a40010f0 con 0x2af8190
2015-08-03 16:20:08.258125 7f20b10ff700 1 -- 172.28.196.123:0/1006108 --> 172.28.196.13:6789/0 -- auth(proto 2 32 bytes epoch 0) v1 -- ?+0 0x7f20980015c0 con 0x2af8190
2015-08-03 16:20:08.259546 7f20b10ff700 1 -- 172.28.196.123:0/1006108 <== mon.2 172.28.196.13:6789/0 3 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 222+0+0 (1304762168 0 0) 0x7f20a40010f0 con 0x2af8190
2015-08-03 16:20:08.260165 7f20b10ff700 1 -- 172.28.196.123:0/1006108 --> 172.28.196.13:6789/0 -- auth(proto 2 181 bytes epoch 0) v1 -- ?+0 0x7f2098003750 con 0x2af8190
2015-08-03 16:20:08.261558 7f20b10ff700 1 -- 172.28.196.123:0/1006108 <== mon.2 172.28.196.13:6789/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 425+0+0 (1363949945 0 0) 0x7f20a40008c0 con 0x2af8190
2015-08-03 16:20:08.262108 7f20b10ff700 1 -- 172.28.196.123:0/1006108 --> 172.28.196.13:6789/0 -- mon_subscribe({monmap=0+}) v2 -- ?+0 0x2afe690 con 0x2af8190
2015-08-03 16:20:08.262579 7f20bbffa780 1 -- 172.28.196.123:0/1006108 --> 172.28.196.13:6789/0 -- mon_subscribe({monmap=2+,osdmap=0}) v2 -- ?+0 0x2af35d0 con 0x2af8190
2015-08-03 16:20:08.262650 7f20bbffa780 1 -- 172.28.196.123:0/1006108 --> 172.28.196.13:6789/0 -- mon_subscribe({monmap=2+,osdmap=0}) v2 -- ?+0 0x2afec20 con 0x2af8190
2015-08-03 16:20:08.262891 7f20ad7f7700 2 RGWDataChangesLog::ChangesRenewThread: start
2015-08-03 16:20:08.262962 7f20bbffa780 20 get_obj_state: rctx=0x7fff741962f0 obj=.rgw.root:default.region state=0x2b00b60 s->prefetch_data=0
2015-08-03 16:20:08.263862 7f20b10ff700 1 -- 172.28.196.123:0/1006108 <== mon.2 172.28.196.13:6789/0 5 ==== mon_map magic: 0 v1 ==== 500+0+0 (170246281 0 0) 0x7f20a4001350 con 0x2af8190
2015-08-03 16:20:08.264381 7f20b10ff700 1 -- 172.28.196.123:0/1006108 <== mon.2 172.28.196.13:6789/0 6 ==== mon_subscribe_ack(300s) v1 ==== 20+0+0 (988214889 0 0) 0x7f20a4001610 con 0x2af8190
2015-08-03 16:20:08.266303 7f20b10ff700 1 -- 172.28.196.123:0/1006108 <== mon.2 172.28.196.13:6789/0 7 ==== osd_map(161701..161701 src has 160998..161701) v3 ==== 149988+0+0 (983368496 0 0) 0x7f20a40008c0 con 0x2af8190
2015-08-03 16:20:08.268220 7f20b10ff700 1 -- 172.28.196.123:0/1006108 <== mon.2 172.28.196.13:6789/0 8 ==== mon_subscribe_ack(300s) v1 ==== 20+0+0 (988214889 0 0) 0x7f20a4001400 con 0x2af8190
2015-08-03 16:20:08.268388 7f20bbffa780 1 -- 172.28.196.123:0/1006108 --> 172.28.196.22:6831/157376 -- osd_op(client.15851354.0:1 default.region [getxattrs,stat] 6.9a566808 ack+read+known_if_redirected e161701) v5 -- ?+0 0x2b07e50 con 0x2b06ae0
2015-08-03 16:20:08.268953 7f20b10ff700 1 -- 172.28.196.123:0/1006108 <== mon.2 172.28.196.13:6789/0 9 ==== osd_map(161701..161701 src has 160998..161701) v3 ==== 149988+0+0 (983368496 0 0) 0x7f20a40008c0 con 0x2af8190
2015-08-03 16:20:08.269510 7f20b10ff700 1 -- 172.28.196.123:0/1006108 <== mon.2 172.28.196.13:6789/0 10 ==== mon_subscribe_ack(300s) v1 ==== 20+0+0 (988214889 0 0) 0x7f20a4001740 con 0x2af8190
2015-08-03 16:20:08.271153 7f20acef5700 1 -- 172.28.196.123:0/1006108 <== osd.179 172.28.196.22:6831/157376 1 ==== osd_op_reply(1 default.region [getxattrs,stat] v0'0 uv2 ondisk = 0) v6 ==== 223+0+20 (755999786 0 1880749811) 0x7f2094000c40 con 0x2b06ae0
2015-08-03 16:20:08.271306 7f20bbffa780 20 get_obj_state: s->obj_tag was set empty
2015-08-03 16:20:08.271322 7f20bbffa780 20 get_obj_state: rctx=0x7fff741962f0 obj=.rgw.root:default.region state=0x2b00b60 s->prefetch_data=0
2015-08-03 16:20:08.271328 7f20bbffa780 20 rados->read ofs=0 len=524288
2015-08-03 16:20:08.271377 7f20bbffa780 1 -- 172.28.196.123:0/1006108 --> 172.28.196.22:6831/157376 -- osd_op(client.15851354.0:2 default.region [read 0~524288] 6.9a566808 ack+read+known_if_redirected e161701) v5 -- ?+0 0x2b08550 con 0x2b06ae0
2015-08-03 16:20:08.272074 7f20acef5700 1 -- 172.28.196.123:0/1006108 <== osd.179 172.28.196.22:6831/157376 2 ==== osd_op_reply(2 default.region [read 0~17] v0'0 uv2 ondisk = 0) v6 ==== 181+0+17 (1483739437 0 2082767461) 0x7f2094000c40 con 0x2b06ae0
2015-08-03 16:20:08.272128 7f20bbffa780 20 rados->read r=0 bl.length=17
2015-08-03 16:20:08.272207 7f20bbffa780 20 get_obj_state: rctx=0x7fff741962f0 obj=.rgw.root:region_info.default state=0x2b00b30 s->prefetch_data=0
2015-08-03 16:20:08.272239 7f20bbffa780 1 -- 172.28.196.123:0/1006108 --> 172.28.196.22:6831/157376 -- osd_op(client.15851354.0:3 region_info.default [getxattrs,stat] 6.43611588 ack+read+known_if_redirected e161701) v5 -- ?+0 0x2b08960 con 0x2b06ae0
2015-08-03 16:20:08.272894 7f20acef5700 1 -- 172.28.196.123:0/1006108 <== osd.179 172.28.196.22:6831/157376 3 ==== osd_op_reply(3 region_info.default [getxattrs,stat] v0'0 uv1 ondisk = 0) v6 ==== 228+0+20 (3048854884 0 3240669963) 0x7f2094000c40 con 0x2b06ae0
2015-08-03 16:20:08.272934 7f20bbffa780 20 get_obj_state: s->obj_tag was set empty
2015-08-03 16:20:08.272940 7f20bbffa780 20 get_obj_state: rctx=0x7fff741962f0 obj=.rgw.root:region_info.default state=0x2b00b30 s->prefetch_data=0
2015-08-03 16:20:08.272943 7f20bbffa780 20 rados->read ofs=0 len=524288
2015-08-03 16:20:08.272975 7f20bbffa780 1 -- 172.28.196.123:0/1006108 --> 172.28.196.22:6831/157376 -- osd_op(client.15851354.0:4 region_info.default [read 0~524288] 6.43611588 ack+read+known_if_redirected e161701) v5 -- ?+0 0x2b08e70 con 0x2b06ae0
2015-08-03 16:20:08.273532 7f20acef5700 1 -- 172.28.196.123:0/1006108 <== osd.179 172.28.196.22:6831/157376 4 ==== osd_op_reply(4 region_info.default [read 0~145] v0'0 uv1 ondisk = 0) v6 ==== 186+0+145 (3886882152 0 4143625220) 0x7f2094000c40 con 0x2b06ae0
2015-08-03 16:20:08.273566 7f20bbffa780 20 rados->read r=0 bl.length=145
2015-08-03 16:20:08.273603 7f20bbffa780 20 get_obj_state: rctx=0x7fff74196460 obj=.rgw.root:zone_info.default state=0x2b00b30 s->prefetch_data=0
2015-08-03 16:20:08.273700 7f20bbffa780 1 -- 172.28.196.123:0/1006108 --> 172.28.196.24:6835/183641 -- osd_op(client.15851354.0:5 zone_info.default [getxattrs,stat] 6.e01f9ae ack+read+known_if_redirected e161701) v5 -- ?+0 0x2b0f660 con 0x2b0e2c0
2015-08-03 16:20:08.277388 7f20accf3700 1 -- 172.28.196.123:0/1006108 <== osd.70 172.28.196.24:6835/183641 1 ==== osd_op_reply(5 zone_info.default [getxattrs,stat] v0'0 uv1 ondisk = 0) v6 ==== 226+0+20 (234749789 0 1548152849) 0x7f208c000c40 con 0x2b0e2c0
2015-08-03 16:20:08.277504 7f20bbffa780 20 get_obj_state: s->obj_tag was set empty
2015-08-03 16:20:08.277512 7f20bbffa780 20 get_obj_state: rctx=0x7fff74196460 obj=.rgw.root:zone_info.default state=0x2b00b30 s->prefetch_data=0
2015-08-03 16:20:08.277516 7f20bbffa780 20 rados->read ofs=0 len=524288
2015-08-03 16:20:08.277540 7f20bbffa780 1 -- 172.28.196.123:0/1006108 --> 172.28.196.24:6835/183641 -- osd_op(client.15851354.0:6 zone_info.default [read 0~524288] 6.e01f9ae ack+read+known_if_redirected e161701) v5 -- ?+0 0x2b0fd60 con 0x2b0e2c0
2015-08-03 16:20:08.278907 7f20accf3700 1 -- 172.28.196.123:0/1006108 <== osd.70 172.28.196.24:6835/183641 2 ==== osd_op_reply(6 zone_info.default [read 0~616] v0'0 uv1 ondisk = 0) v6 ==== 184+0+616 (2611419515 0 3050520442) 0x7f208c0017b0 con 0x2b0e2c0
2015-08-03 16:20:08.279019 7f20bbffa780 20 rados->read r=0 bl.length=616
2015-08-03 16:20:08.279044 7f20bbffa780 2 zone default is master
2015-08-03 16:20:08.279058 7f20bbffa780 20 get_obj_state: rctx=0x7fff74196470 obj=.rgw.root:region_map state=0x2b00b30 s->prefetch_data=0
2015-08-03 16:20:08.279144 7f20bbffa780 1 -- 172.28.196.123:0/1006108 --> 172.28.196.23:6805/115706 -- osd_op(client.15851354.0:7 region_map [getxattrs,stat] 6.bd31b503 ack+read+known_if_redirected e161701) v5 -- ?+0 0x2b16ae0 con 0x2b15770
2015-08-03 16:20:08.281463 7f20acaf1700 1 -- 172.28.196.123:0/1006108 <== osd.13 172.28.196.23:6805/115706 1 ==== osd_op_reply(7 region_map [getxattrs,stat] v0'0 uv0 ack = -2 ((2) No such file or directory)) v6 ==== 219+0+0 (816786970 0 0) 0x7f2084000ad0 con 0x2b15770
2015-08-03 16:20:08.281641 7f20bbffa780 20 init_complete bucket index max shards: 0
2015-08-03 16:20:08.281684 7f20bbffa780 20 get_obj_state: rctx=0x7fff741963c0 obj=.rgw:seantest state=0x2b176e0 s->prefetch_data=0
2015-08-03 16:20:08.281785 7f20bbffa780 1 -- 172.28.196.123:0/1006108 --> 172.28.196.26:6870/244584 -- osd_op(client.15851354.0:8 seantest [call version.read,getxattrs,stat] 8.6ea27fc3 ack+read+known_if_redirected e161701) v5 -- ?+0 0x2b1fc30 con 0x2b1e850
2015-08-03 16:20:08.285039 7f207ffff700 1 -- 172.28.196.123:0/1006108 <== osd.112 172.28.196.26:6870/244584 1 ==== osd_op_reply(8 seantest [call,getxattrs,stat] v0'0 uv20 ondisk = 0) v6 ==== 259+0+361 (1151157477 0 3706883696) 0x7f2074000dc0 con 0x2b1e850
2015-08-03 16:20:08.285173 7f20bbffa780 20 get_obj_state: s->obj_tag was set empty
2015-08-03 16:20:08.285182 7f20bbffa780 20 get_obj_state: rctx=0x7fff741963c0 obj=.rgw:seantest state=0x2b176e0 s->prefetch_data=0
2015-08-03 16:20:08.285224 7f20bbffa780 20 rados->read ofs=0 len=524288
2015-08-03 16:20:08.285251 7f20bbffa780 1 -- 172.28.196.123:0/1006108 --> 172.28.196.26:6870/244584 -- osd_op(client.15851354.0:9 seantest [call version.check_conds,call version.read,read 0~524288] 8.6ea27fc3 ack+read+known_if_redirected e161701) v5 -- ?+0 0x2b20660 con 0x2b1e850
2015-08-03 16:20:08.286366 7f207ffff700 1 -- 172.28.196.123:0/1006108 <== osd.112 172.28.196.26:6870/244584 2 ==== osd_op_reply(9 seantest [call,call,read 0~127] v0'0 uv20 ondisk = 0) v6 ==== 259+0+175 (225997321 0 4159825245) 0x7f2074000dc0 con 0x2b1e850
2015-08-03 16:20:08.286476 7f20bbffa780 20 rados->read r=0 bl.length=127
2015-08-03 16:20:08.286495 7f20bbffa780 20 rgw_get_bucket_info: bucket instance: seantest(@{i=.rgw.buckets.index}.rgw.buckets[default.15358081.4])
2015-08-03 16:20:08.286508 7f20bbffa780 20 reading from .rgw:.bucket.meta.seantest:default.15358081.4
2015-08-03 16:20:08.286518 7f20bbffa780 20 get_obj_state: rctx=0x7fff741963c0 obj=.rgw:.bucket.meta.seantest:default.15358081.4 state=0x2b204d0 s->prefetch_data=0
2015-08-03 16:20:08.286607 7f20bbffa780 1 -- 172.28.196.123:0/1006108 --> 172.28.196.25:6800/219001 -- osd_op(client.15851354.0:10 .bucket.meta.seantest:default.15358081.4 [call version.read,getxattrs,stat] 8.afcb7682 ack+read+known_if_redirected e161701) v5 -- ?+0 0x2b23380 con 0x2b22010
2015-08-03 16:20:08.289087 7f207fdfd700 1 -- 172.28.196.123:0/1006108 <== osd.81 172.28.196.25:6800/219001 1 ==== osd_op_reply(10 .bucket.meta.seantest:default.15358081.4 [call,getxattrs,stat] v0'0 uv48 ondisk = 0) v6 ==== 291+0+286 (3814631589 0 4215094535) 0x7f2070000d90 con 0x2b22010
2015-08-03 16:20:08.289211 7f20bbffa780 20 get_obj_state: s->obj_tag was set empty
2015-08-03 16:20:08.289222 7f20bbffa780 20 Read xattr: user.rgw.acl
2015-08-03 16:20:08.289224 7f20bbffa780 20 Read xattr: user.rgw.idtag
2015-08-03 16:20:08.289225 7f20bbffa780 20 Read xattr: user.rgw.manifest
2015-08-03 16:20:08.289228 7f20bbffa780 20 get_obj_state: rctx=0x7fff741963c0 obj=.rgw:.bucket.meta.seantest:default.15358081.4 state=0x2b204d0 s->prefetch_data=0
2015-08-03 16:20:08.289253 7f20bbffa780 20 rados->read ofs=0 len=524288
2015-08-03 16:20:08.289284 7f20bbffa780 1 -- 172.28.196.123:0/1006108 --> 172.28.196.25:6800/219001 -- osd_op(client.15851354.0:11 .bucket.meta.seantest:default.15358081.4 [call version.check_conds,call version.read,read 0~524288] 8.afcb7682 ack+read+known_if_redirected e161701) v5 -- ?+0 0x2b2a510 con 0x2b22010
2015-08-03 16:20:08.290142 7f207fdfd700 1 -- 172.28.196.123:0/1006108 <== osd.81 172.28.196.25:6800/219001 2 ==== osd_op_reply(11 .bucket.meta.seantest:default.15358081.4 [call,call,read 0~186] v0'0 uv48 ondisk = 0) v6 ==== 291+0+234 (2597786973 0 3711520193) 0x7f2070001090 con 0x2b22010
2015-08-03 16:20:08.290188 7f20bbffa780 20 rados->read r=0 bl.length=186
2015-08-03 16:20:08.290201 7f20bbffa780 20 couldn't put binfo cache entry, might have raced with data changes
2015-08-03 16:20:08.290227 7f20bbffa780 20 get_obj_state: rctx=0x7fff74196350 obj=.rgw:seantest state=0x2b176e0 s->prefetch_data=0
2015-08-03 16:20:08.290260 7f20bbffa780 1 -- 172.28.196.123:0/1006108 --> 172.28.196.26:6870/244584 -- osd_op(client.15851354.0:12 seantest [call version.read,getxattrs,stat] 8.6ea27fc3 ack+read+known_if_redirected e161701) v5 -- ?+0 0x2b23380 con 0x2b1e850
2015-08-03 16:20:08.291375 7f207ffff700 1 -- 172.28.196.123:0/1006108 <== osd.112 172.28.196.26:6870/244584 3 ==== osd_op_reply(12 seantest [call,getxattrs,stat] v0'0 uv20 ondisk = 0) v6 ==== 259+0+361 (1151157477 0 3706883696) 0x7f2074000dc0 con 0x2b1e850
2015-08-03 16:20:08.291546 7f20bbffa780 20 get_obj_state: s->obj_tag was set empty
2015-08-03 16:20:08.291554 7f20bbffa780 20 get_obj_state: rctx=0x7fff74196350 obj=.rgw:seantest state=0x2b176e0 s->prefetch_data=0
2015-08-03 16:20:08.291579 7f20bbffa780 20 rados->read ofs=0 len=524288
2015-08-03 16:20:08.291604 7f20bbffa780 1 -- 172.28.196.123:0/1006108 --> 172.28.196.26:6870/244584 -- osd_op(client.15851354.0:13 seantest [call version.check_conds,call version.read,read 0~524288] 8.6ea27fc3 ack+read+known_if_redirected e161701) v5 -- ?+0 0x2b2a2f0 con 0x2b1e850
2015-08-03 16:20:08.292422 7f207ffff700 1 -- 172.28.196.123:0/1006108 <== osd.112 172.28.196.26:6870/244584 4 ==== osd_op_reply(13 seantest [call,call,read 0~127] v0'0 uv20 ondisk = 0) v6 ==== 259+0+175 (225997321 0 4159825245) 0x7f2074000dc0 con 0x2b1e850
2015-08-03 16:20:08.292513 7f20bbffa780 20 rados->read r=0 bl.length=127
2015-08-03 16:20:08.292530 7f20bbffa780 20 rgw_get_bucket_info: bucket instance: seantest(@{i=.rgw.buckets.index}.rgw.buckets[default.15358081.4])
2015-08-03 16:20:08.292536 7f20bbffa780 20 reading from .rgw:.bucket.meta.seantest:default.15358081.4
2015-08-03 16:20:08.292547 7f20bbffa780 20 get_obj_state: rctx=0x7fff74196350 obj=.rgw:.bucket.meta.seantest:default.15358081.4 state=0x2b21090 s->prefetch_data=0
2015-08-03 16:20:08.292581 7f20bbffa780 1 -- 172.28.196.123:0/1006108 --> 172.28.196.25:6800/219001 -- osd_op(client.15851354.0:14 .bucket.meta.seantest:default.15358081.4 [call version.read,getxattrs,stat] 8.afcb7682 ack+read+known_if_redirected e161701) v5 -- ?+0 0x2b21550 con 0x2b22010
2015-08-03 16:20:08.293420 7f207fdfd700 1 -- 172.28.196.123:0/1006108 <== osd.81 172.28.196.25:6800/219001 3 ==== osd_op_reply(14 .bucket.meta.seantest:default.15358081.4 [call,getxattrs,stat] v0'0 uv48 ondisk = 0) v6 ==== 291+0+286 (3814631589 0 4215094535) 0x7f2070001090 con 0x2b22010
2015-08-03 16:20:08.293530 7f20bbffa780 20 get_obj_state: s->obj_tag was set empty
2015-08-03 16:20:08.293536 7f20bbffa780 20 Read xattr: user.rgw.acl
2015-08-03 16:20:08.293537 7f20bbffa780 20 Read xattr: user.rgw.idtag
2015-08-03 16:20:08.293538 7f20bbffa780 20 Read xattr: user.rgw.manifest
2015-08-03 16:20:08.293541 7f20bbffa780 20 get_obj_state: rctx=0x7fff74196350 obj=.rgw:.bucket.meta.seantest:default.15358081.4 state=0x2b21090 s->prefetch_data=0
2015-08-03 16:20:08.293563 7f20bbffa780 20 rados->read ofs=0 len=524288
2015-08-03 16:20:08.293590 7f20bbffa780 1 -- 172.28.196.123:0/1006108 --> 172.28.196.25:6800/219001 -- osd_op(client.15851354.0:15 .bucket.meta.seantest:default.15358081.4 [call version.check_conds,call version.read,read 0~524288] 8.afcb7682 ack+read+known_if_redirected e161701) v5 -- ?+0 0x2b302b0 con 0x2b22010
2015-08-03 16:20:08.294414 7f207fdfd700 1 -- 172.28.196.123:0/1006108 <== osd.81 172.28.196.25:6800/219001 4 ==== osd_op_reply(15 .bucket.meta.seantest:default.15358081.4 [call,call,read 0~186] v0'0 uv48 ondisk = 0) v6 ==== 291+0+234 (2597786973 0 3711520193) 0x7f2070000f80 con 0x2b22010
2015-08-03 16:20:08.294464 7f20bbffa780 20 rados->read r=0 bl.length=186
2015-08-03 16:20:08.294477 7f20bbffa780 20 couldn't put binfo cache entry, might have raced with data changes
2015-08-03 16:20:08.294497 7f20bbffa780 20 reading from .rgw:.bucket.meta.seantest:default.15358081.4
2015-08-03 16:20:08.294508 7f20bbffa780 20 get_obj_state: rctx=0x7fff74195a80 obj=.rgw:.bucket.meta.seantest:default.15358081.4 state=0x2b30120 s->prefetch_data=0
2015-08-03 16:20:08.294540 7f20bbffa780 1 -- 172.28.196.123:0/1006108 --> 172.28.196.25:6800/219001 -- osd_op(client.15851354.0:16 .bucket.meta.seantest:default.15358081.4 [call version.read,getxattrs,stat] 8.afcb7682 ack+read+known_if_redirected e161701) v5 -- ?+0 0x2b21770 con 0x2b22010
2015-08-03 16:20:08.295279 7f207fdfd700 1 -- 172.28.196.123:0/1006108 <== osd.81 172.28.196.25:6800/219001 5 ==== osd_op_reply(16 .bucket.meta.seantest:default.15358081.4 [call,getxattrs,stat] v0'0 uv48 ondisk = 0) v6 ==== 291+0+286 (3814631589 0 4215094535) 0x7f2070000f80 con 0x2b22010
2015-08-03 16:20:08.295337 7f20bbffa780 20 get_obj_state: s->obj_tag was set empty
2015-08-03 16:20:08.295343 7f20bbffa780 20 get_obj_state: rctx=0x7fff74195a80 obj=.rgw:.bucket.meta.seantest:default.15358081.4 state=0x2b30120 s->prefetch_data=0
2015-08-03 16:20:08.295366 7f20bbffa780 20 rados->read ofs=0 len=524288
2015-08-03 16:20:08.295392 7f20bbffa780 1 -- 172.28.196.123:0/1006108 --> 172.28.196.25:6800/219001 -- osd_op(client.15851354.0:17 .bucket.meta.seantest:default.15358081.4 [call version.check_conds,call version.read,read 0~524288] 8.afcb7682 ack+read+known_if_redirected e161701) v5 -- ?+0 0x2b32010 con 0x2b22010
2015-08-03 16:20:08.296034 7f207fdfd700 1 -- 172.28.196.123:0/1006108 <== osd.81 172.28.196.25:6800/219001 6 ==== osd_op_reply(17 .bucket.meta.seantest:default.15358081.4 [call,call,read 0~186] v0'0 uv48 ondisk = 0) v6 ==== 291+0+234 (2597786973 0 3711520193) 0x7f2070001b90 con 0x2b22010
2015-08-03 16:20:08.296092 7f20bbffa780 20 rados->read r=0 bl.length=186
2015-08-03 16:20:08.296258 7f20bbffa780 1 -- 172.28.196.123:0/1006108 --> 172.28.196.26:6809/223758 -- osd_op(client.15851354.0:18 .dir.default.15358081.4 [call rgw.bucket_list] 15.127db47d ack+read+known_if_redirected e161701) v5 -- ?+0 0x2b32ba0 con 0x2b25dd0
2015-08-03 16:20:08.298837 7f207fbfb700 1 error getting bucket stats ret=-22

-- 172.28.196.123:0/1006108 <== osd.129 172.28.196.26:6809/223758 1 ==== osd_op_reply(18 .dir.default.15358081.4 [call] v0'0 uv0 ondisk = -22 ((22) Invalid argument)) v6 ==== 190+0+0 (752195456 0 0) 0x7f2068000ab0 con 0x2b25dd0
2015-08-03 16:20:08.299465 7f20bbffa780 1 -- 172.28.196.123:0/1006108 mark_down 0x2b15770 -- 0x2b114d0
2015-08-03 16:20:08.299550 7f20bbffa780 1 -- 172.28.196.123:0/1006108 mark_down 0x2b0e2c0 -- 0x2b0a020
2015-08-03 16:20:08.299579 7f20bbffa780 1 -- 172.28.196.123:0/1006108 mark_down 0x2b22010 -- 0x2b26050
2015-08-03 16:20:08.299665 7f20bbffa780 1 -- 172.28.196.123:0/1006108 mark_down 0x2b1e850 -- 0x2b1a5b0
2015-08-03 16:20:08.299707 7f20bbffa780 1 -- 172.28.196.123:0/1006108 mark_down 0x2b25dd0 -- 0x2b34010
2015-08-03 16:20:08.299741 7f20bbffa780 1 -- 172.28.196.123:0/1006108 mark_down 0x2b06ae0 -- 0x2b02840
2015-08-03 16:20:08.300158 7f20bbffa780 1 -- 172.28.196.123:0/1006108 mark_down 0x2af8190 -- 0x2af8fc0
2015-08-03 16:20:08.300213 7f20bbffa780 1 -- 172.28.196.123:0/1006108 mark_down_all
2015-08-03 16:20:08.300475 7f20bbffa780 1 -- 172.28.196.123:0/1006108 shutdown complete.
(1-1/2)