Project

General

Profile

Bug #22617 » test.txt

fpq f, 01/16/2018 01:02 PM

 
[root@ceph-rgw ~]# radosgw -d --debug-rgw=18 --debug-ms=1 --name client.radosgw.gateway
2018-01-16 21:00:33.481282 7fbd034e09c0 0 ceph version 10.2.2 (45107e21c568dd033c2f0a3107dec8f0b0e58374), process radosgw, pid 9406
2018-01-16 21:00:33.485362 7fbd034e09c0 -1 asok(0x55aa4d6b9e40) AdminSocketConfigObs::init: failed: AdminSocket::bind_and_listen: failed to bind the UNIX domain socket to '/var/run/ceph/ceph-client.radosgw.gateway.asok': (17) File exists
2018-01-16 21:00:33.487024 7fbd034e09c0 1 -- :/0 messenger.start
2018-01-16 21:00:33.487452 7fbd034e09c0 1 -- :/3412370064 --> 10.134.238.159:6789/0 -- auth(proto 0 40 bytes epoch 0) v1 -- ?+0 0x55aa4d71b970 con 0x55aa4d71b390
2018-01-16 21:00:33.489297 7fbd034d6700 1 -- 10.134.238.152:0/3412370064 learned my addr 10.134.238.152:0/3412370064
2018-01-16 21:00:33.490520 7fbce2887700 1 -- 10.134.238.152:0/3412370064 <== mon.2 10.134.238.159:6789/0 1 ==== mon_map magic: 0 v1 ==== 500+0+0 (3223744036 0 0) 0x7fbcd8000b90 con 0x55aa4d71b390
2018-01-16 21:00:33.490614 7fbce2887700 1 -- 10.134.238.152:0/3412370064 <== mon.2 10.134.238.159:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (2969333790 0 0) 0x7fbcd80008c0 con 0x55aa4d71b390
2018-01-16 21:00:33.490733 7fbce2887700 1 -- 10.134.238.152:0/3412370064 --> 10.134.238.159:6789/0 -- auth(proto 2 32 bytes epoch 0) v1 -- ?+0 0x7fbcd4001880 con 0x55aa4d71b390
2018-01-16 21:00:33.491562 7fbce2887700 1 -- 10.134.238.152:0/3412370064 <== mon.2 10.134.238.159:6789/0 3 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 222+0+0 (4237805047 0 0) 0x7fbcd8000a50 con 0x55aa4d71b390
2018-01-16 21:00:33.491667 7fbce2887700 1 -- 10.134.238.152:0/3412370064 --> 10.134.238.159:6789/0 -- auth(proto 2 181 bytes epoch 0) v1 -- ?+0 0x7fbcd4003390 con 0x55aa4d71b390
2018-01-16 21:00:33.492756 7fbce2887700 1 -- 10.134.238.152:0/3412370064 <== mon.2 10.134.238.159:6789/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 425+0+0 (217033238 0 0) 0x7fbcd8000e90 con 0x55aa4d71b390
2018-01-16 21:00:33.492839 7fbce2887700 1 -- 10.134.238.152:0/3412370064 --> 10.134.238.159:6789/0 -- mon_subscribe({monmap=0+}) v2 -- ?+0 0x55aa4d755800 con 0x55aa4d71b390
2018-01-16 21:00:33.492912 7fbd034e09c0 1 -- 10.134.238.152:0/3412370064 --> 10.134.238.159:6789/0 -- mon_subscribe({osdmap=0}) v2 -- ?+0 0x55aa4d754700 con 0x55aa4d71b390
2018-01-16 21:00:33.493380 7fbce2887700 1 -- 10.134.238.152:0/3412370064 <== mon.2 10.134.238.159:6789/0 5 ==== mon_map magic: 0 v1 ==== 500+0+0 (3223744036 0 0) 0x7fbcd8000e90 con 0x55aa4d71b390
2018-01-16 21:00:33.493464 7fbcd2ffd700 2 RGWDataChangesLog::ChangesRenewThread: start
2018-01-16 21:00:33.494284 7fbce2887700 1 -- 10.134.238.152:0/3412370064 <== mon.2 10.134.238.159:6789/0 6 ==== osd_map(265..265 src has 1..265) v3 ==== 8305+0+0 (896830133 0 0) 0x7fbcd8002f80 con 0x55aa4d71b390
2018-01-16 21:00:33.494566 7fbd034e09c0 1 -- 10.134.238.152:0/3412370064 --> 10.134.238.159:6800/11592 -- osd_op(client.74204.0:1 3.85fca992 default.realm [getxattrs,stat] snapc 0=[] ack+read+known_if_redirected e265) v7 -- ?+0 0x55aa4d75b150 con 0x55aa4d759c30
2018-01-16 21:00:33.497039 7fbcd26fb700 1 -- 10.134.238.152:0/3412370064 <== osd.5 10.134.238.159:6800/11592 1 ==== osd_op_reply(1 default.realm [getxattrs,stat] v0'0 uv0 ack = -2 ((2) No such file or directory)) v7 ==== 175+0+0 (2283260449 0 0) 0x7fbcc8000a50 con 0x55aa4d759c30
2018-01-16 21:00:33.497182 7fbd034e09c0 1 -- 10.134.238.152:0/3412370064 --> 10.134.238.159:6800/11592 -- osd_op(client.74204.0:2 3.85fca992 default.realm [getxattrs,stat] snapc 0=[] ack+read+known_if_redirected e265) v7 -- ?+0 0x55aa4d75b870 con 0x55aa4d759c30
2018-01-16 21:00:33.498337 7fbcd26fb700 1 -- 10.134.238.152:0/3412370064 <== osd.5 10.134.238.159:6800/11592 2 ==== osd_op_reply(2 default.realm [getxattrs,stat] v0'0 uv0 ack = -2 ((2) No such file or directory)) v7 ==== 175+0+0 (2283260449 0 0) 0x7fbcc8000a50 con 0x55aa4d759c30
2018-01-16 21:00:33.498407 7fbd034e09c0 10 could not read realm id: (2) No such file or directory
2018-01-16 21:00:33.498513 7fbd034e09c0 1 -- 10.134.238.152:0/3412370064 --> 10.134.238.158:6804/3586 -- osd_op(client.74204.0:3 3.0 [pgls start_epoch 0] snapc 0=[] ack+read+known_if_redirected e265) v7 -- ?+0 0x55aa4d75f490 con 0x55aa4d75e050
2018-01-16 21:00:33.501468 7fbcd24f9700 1 -- 10.134.238.152:0/3412370064 <== osd.4 10.134.238.158:6804/3586 1 ==== osd_op_reply(3 [pgls start_epoch 0] v0'0 uv0 ondisk = 1) v7 ==== 120+0+44 (73087745 0 3248547820) 0x7fbcc0000b50 con 0x55aa4d75e050
2018-01-16 21:00:33.501513 7fbcd24f9700 1 -- 10.134.238.152:0/3412370064 --> 10.134.238.159:6800/11592 -- osd_op(client.74204.0:4 3.1 [pgls start_epoch 0] snapc 0=[] ack+read+known_if_redirected e265) v7 -- ?+0 0x7fbcc0002580 con 0x55aa4d759c30
2018-01-16 21:00:33.502596 7fbcd26fb700 1 -- 10.134.238.152:0/3412370064 <== osd.5 10.134.238.159:6800/11592 3 ==== osd_op_reply(4 [pgls start_epoch 0] v0'0 uv0 ondisk = 1) v7 ==== 120+0+44 (2239183445 0 3248547820) 0x7fbcc8000a50 con 0x55aa4d759c30
2018-01-16 21:00:33.502634 7fbcd26fb700 1 -- 10.134.238.152:0/3412370064 --> 10.134.238.159:6800/11592 -- osd_op(client.74204.0:5 3.2 [pgls start_epoch 0] snapc 0=[] ack+read+known_if_redirected e265) v7 -- ?+0 0x7fbcc80029e0 con 0x55aa4d759c30
2018-01-16 21:00:33.503629 7fbcd26fb700 1 -- 10.134.238.152:0/3412370064 <== osd.5 10.134.238.159:6800/11592 4 ==== osd_op_reply(5 [pgls start_epoch 0] v0'0 uv0 ondisk = 1) v7 ==== 120+0+44 (66015064 0 3248547820) 0x7fbcc8000a50 con 0x55aa4d759c30
2018-01-16 21:00:33.503712 7fbcd26fb700 1 -- 10.134.238.152:0/3412370064 --> 10.134.238.157:6804/17168 -- osd_op(client.74204.0:6 3.3 [pgls start_epoch 0] snapc 0=[] ack+read+known_if_redirected e265) v7 -- ?+0 0x7fbcc8005d70 con 0x7fbcc8004930
2018-01-16 21:00:33.505142 7fbcd22f7700 1 -- 10.134.238.152:0/3412370064 <== osd.3 10.134.238.157:6804/17168 1 ==== osd_op_reply(6 [pgls start_epoch 0] v75'1 uv1 ondisk = 1) v7 ==== 120+0+103 (2535676660 0 4170302097) 0x7fbcb8000b80 con 0x7fbcc8004930
2018-01-16 21:00:33.505231 7fbd034e09c0 1 -- 10.134.238.152:0/3412370064 --> 10.134.238.157:6804/17168 -- osd_op(client.74204.0:7 3.4 [pgls start_epoch 0] snapc 0=[] ack+read+known_if_redirected e265) v7 -- ?+0 0x55aa4d75f8d0 con 0x7fbcc8004930
2018-01-16 21:00:33.505701 7fbcd22f7700 1 -- 10.134.238.152:0/3412370064 <== osd.3 10.134.238.157:6804/17168 2 ==== osd_op_reply(7 [pgls start_epoch 0] v0'0 uv0 ondisk = 1) v7 ==== 120+0+44 (187945907 0 3248547820) 0x7fbcb8000b80 con 0x7fbcc8004930
2018-01-16 21:00:33.505740 7fbcd22f7700 1 -- 10.134.238.152:0/3412370064 --> 10.134.238.159:6800/11592 -- osd_op(client.74204.0:8 3.5 [pgls start_epoch 0] snapc 0=[] ack+read+known_if_redirected e265) v7 -- ?+0 0x7fbcb80027f0 con 0x55aa4d759c30
2018-01-16 21:00:33.507468 7fbcd26fb700 1 -- 10.134.238.152:0/3412370064 <== osd.5 10.134.238.159:6800/11592 5 ==== osd_op_reply(8 [pgls start_epoch 0] v0'0 uv0 ondisk = 1) v7 ==== 120+0+44 (2317341415 0 3248547820) 0x7fbcc8000a50 con 0x55aa4d759c30
2018-01-16 21:00:33.507507 7fbcd26fb700 1 -- 10.134.238.152:0/3412370064 --> 10.134.238.159:6800/11592 -- osd_op(client.74204.0:9 3.6 [pgls start_epoch 0] snapc 0=[] ack+read+known_if_redirected e265) v7 -- ?+0 0x7fbcc80062a0 con 0x55aa4d759c30
2018-01-16 21:00:33.508924 7fbcd26fb700 1 -- 10.134.238.152:0/3412370064 <== osd.5 10.134.238.159:6800/11592 6 ==== osd_op_reply(9 [pgls start_epoch 0] v75'1 uv1 ondisk = 1) v7 ==== 120+0+98 (777405040 0 661761295) 0x7fbcc80062a0 con 0x55aa4d759c30
2018-01-16 21:00:33.508994 7fbd034e09c0 1 -- 10.134.238.152:0/3412370064 --> 10.134.238.159:6800/11592 -- osd_op(client.74204.0:10 3.7 [pgls start_epoch 0] snapc 0=[] ack+read+known_if_redirected e265) v7 -- ?+0 0x55aa4d75f8d0 con 0x55aa4d759c30
2018-01-16 21:00:33.510002 7fbcd26fb700 1 -- 10.134.238.152:0/3412370064 <== osd.5 10.134.238.159:6800/11592 7 ==== osd_op_reply(10 [pgls start_epoch 0] v75'2 uv2 ondisk = 1) v7 ==== 120+0+102 (3625854734 0 37397930) 0x7fbcc80062a0 con 0x55aa4d759c30
2018-01-16 21:00:33.510108 7fbd034e09c0 1 -- 10.134.238.152:0/3412370064 --> 10.134.238.157:6804/17168 -- osd_op(client.74204.0:11 3.bd31b503 region_map [getxattrs,stat] snapc 0=[] ack+read+known_if_redirected e265) v7 -- ?+0 0x55aa4d75b870 con 0x7fbcc8004930
2018-01-16 21:00:33.510528 7fbcd22f7700 1 -- 10.134.238.152:0/3412370064 <== osd.3 10.134.238.157:6804/17168 3 ==== osd_op_reply(11 region_map [getxattrs,stat] v0'0 uv0 ack = -2 ((2) No such file or directory)) v7 ==== 172+0+0 (832903909 0 0) 0x7fbcb80015a0 con 0x7fbcc8004930
2018-01-16 21:00:33.510597 7fbd034e09c0 10 cannot find current period zonegroup using local zonegroup
2018-01-16 21:00:33.510637 7fbd034e09c0 1 -- 10.134.238.152:0/3412370064 --> 10.134.238.159:6800/11592 -- osd_op(client.74204.0:12 3.85fca992 default.realm [getxattrs,stat] snapc 0=[] ack+read+known_if_redirected e265) v7 -- ?+0 0x55aa4d75b870 con 0x55aa4d759c30
2018-01-16 21:00:33.511600 7fbcd26fb700 1 -- 10.134.238.152:0/3412370064 <== osd.5 10.134.238.159:6800/11592 8 ==== osd_op_reply(12 default.realm [getxattrs,stat] v0'0 uv0 ack = -2 ((2) No such file or directory)) v7 ==== 175+0+0 (2283260449 0 0) 0x7fbcc80016a0 con 0x55aa4d759c30
2018-01-16 21:00:33.511664 7fbd034e09c0 10 could not read realm id: (2) No such file or directory
2018-01-16 21:00:33.511673 7fbd034e09c0 10 Creating default zonegroup
2018-01-16 21:00:33.511723 7fbd034e09c0 1 -- 10.134.238.152:0/3412370064 --> 10.134.238.159:6789/0 -- mon_get_version(what=osdmap handle=1) v1 -- ?+0 0x55aa4d75f8d0 con 0x55aa4d71b390
2018-01-16 21:00:33.512503 7fbce2887700 1 -- 10.134.238.152:0/3412370064 <== mon.2 10.134.238.159:6789/0 7 ==== mon_get_version_reply(handle=1 version=265) v2 ==== 24+0+0 (4011090033 0 0) 0x7fbcd8000f00 con 0x55aa4d71b390
2018-01-16 21:00:33.512596 7fbd034e09c0 10 couldn't find old data placement pools config, setting up new ones for the zone
2018-01-16 21:00:33.512645 7fbd034e09c0 1 -- 10.134.238.152:0/3412370064 --> 10.134.238.158:6804/3586 -- osd_op(client.74204.0:13 3.0 [pgls start_epoch 0] snapc 0=[] ack+read+known_if_redirected e265) v7 -- ?+0 0x55aa4d75c040 con 0x55aa4d75e050
2018-01-16 21:00:33.513657 7fbcd24f9700 1 -- 10.134.238.152:0/3412370064 <== osd.4 10.134.238.158:6804/3586 2 ==== osd_op_reply(13 [pgls start_epoch 0] v0'0 uv0 ondisk = 1) v7 ==== 120+0+44 (73087745 0 3248547820) 0x7fbcc0000b50 con 0x55aa4d75e050
2018-01-16 21:00:33.513693 7fbcd24f9700 1 -- 10.134.238.152:0/3412370064 --> 10.134.238.159:6800/11592 -- osd_op(client.74204.0:14 3.1 [pgls start_epoch 0] snapc 0=[] ack+read+known_if_redirected e265) v7 -- ?+0 0x7fbcc0002b90 con 0x55aa4d759c30
2018-01-16 21:00:33.514807 7fbcd26fb700 1 -- 10.134.238.152:0/3412370064 <== osd.5 10.134.238.159:6800/11592 9 ==== osd_op_reply(14 [pgls start_epoch 0] v0'0 uv0 ondisk = 1) v7 ==== 120+0+44 (2239183445 0 3248547820) 0x7fbcc80016a0 con 0x55aa4d759c30
2018-01-16 21:00:33.514844 7fbcd26fb700 1 -- 10.134.238.152:0/3412370064 --> 10.134.238.159:6800/11592 -- osd_op(client.74204.0:15 3.2 [pgls start_epoch 0] snapc 0=[] ack+read+known_if_redirected e265) v7 -- ?+0 0x7fbcc8001e70 con 0x55aa4d759c30
2018-01-16 21:00:33.515923 7fbcd26fb700 1 -- 10.134.238.152:0/3412370064 <== osd.5 10.134.238.159:6800/11592 10 ==== osd_op_reply(15 [pgls start_epoch 0] v0'0 uv0 ondisk = 1) v7 ==== 120+0+44 (66015064 0 3248547820) 0x7fbcc80016a0 con 0x55aa4d759c30
2018-01-16 21:00:33.515955 7fbcd26fb700 1 -- 10.134.238.152:0/3412370064 --> 10.134.238.157:6804/17168 -- osd_op(client.74204.0:16 3.3 [pgls start_epoch 0] snapc 0=[] ack+read+known_if_redirected e265) v7 -- ?+0 0x7fbcc80023f0 con 0x7fbcc8004930
2018-01-16 21:00:33.516494 7fbcd22f7700 1 -- 10.134.238.152:0/3412370064 <== osd.3 10.134.238.157:6804/17168 4 ==== osd_op_reply(16 [pgls start_epoch 0] v75'1 uv1 ondisk = 1) v7 ==== 120+0+103 (2535676660 0 4170302097) 0x7fbcb80015a0 con 0x7fbcc8004930
2018-01-16 21:00:33.516562 7fbd034e09c0 1 -- 10.134.238.152:0/3412370064 --> 10.134.238.157:6804/17168 -- osd_op(client.74204.0:17 3.4 [pgls start_epoch 0] snapc 0=[] ack+read+known_if_redirected e265) v7 -- ?+0 0x55aa4d75c040 con 0x7fbcc8004930
2018-01-16 21:00:33.517065 7fbcd22f7700 1 -- 10.134.238.152:0/3412370064 <== osd.3 10.134.238.157:6804/17168 5 ==== osd_op_reply(17 [pgls start_epoch 0] v0'0 uv0 ondisk = 1) v7 ==== 120+0+44 (187945907 0 3248547820) 0x7fbcb80015a0 con 0x7fbcc8004930
2018-01-16 21:00:33.517104 7fbcd22f7700 1 -- 10.134.238.152:0/3412370064 --> 10.134.238.159:6800/11592 -- osd_op(client.74204.0:18 3.5 [pgls start_epoch 0] snapc 0=[] ack+read+known_if_redirected e265) v7 -- ?+0 0x7fbcb8002040 con 0x55aa4d759c30
2018-01-16 21:00:33.518088 7fbcd26fb700 1 -- 10.134.238.152:0/3412370064 <== osd.5 10.134.238.159:6800/11592 11 ==== osd_op_reply(18 [pgls start_epoch 0] v0'0 uv0 ondisk = 1) v7 ==== 120+0+44 (2317341415 0 3248547820) 0x7fbcc80016a0 con 0x55aa4d759c30
2018-01-16 21:00:33.518124 7fbcd26fb700 1 -- 10.134.238.152:0/3412370064 --> 10.134.238.159:6800/11592 -- osd_op(client.74204.0:19 3.6 [pgls start_epoch 0] snapc 0=[] ack+read+known_if_redirected e265) v7 -- ?+0 0x7fbcc80023c0 con 0x55aa4d759c30
2018-01-16 21:00:33.519182 7fbcd26fb700 1 -- 10.134.238.152:0/3412370064 <== osd.5 10.134.238.159:6800/11592 12 ==== osd_op_reply(19 [pgls start_epoch 0] v75'1 uv1 ondisk = 1) v7 ==== 120+0+98 (777405040 0 661761295) 0x7fbcc80016a0 con 0x55aa4d759c30
2018-01-16 21:00:33.519250 7fbd034e09c0 1 -- 10.134.238.152:0/3412370064 --> 10.134.238.159:6800/11592 -- osd_op(client.74204.0:20 3.7 [pgls start_epoch 0] snapc 0=[] ack+read+known_if_redirected e265) v7 -- ?+0 0x55aa4d75c040 con 0x55aa4d759c30
2018-01-16 21:00:33.520264 7fbcd26fb700 1 -- 10.134.238.152:0/3412370064 <== osd.5 10.134.238.159:6800/11592 13 ==== osd_op_reply(20 [pgls start_epoch 0] v75'2 uv2 ondisk = 1) v7 ==== 120+0+102 (3625854734 0 37397930) 0x7fbcc80016a0 con 0x55aa4d759c30
2018-01-16 21:00:33.520366 7fbd034e09c0 1 -- 10.134.238.152:0/3412370064 --> 10.134.238.159:6800/11592 -- osd_op(client.74204.0:21 3.7376dc2f zone_names.default [getxattrs,stat] snapc 0=[] ack+read+known_if_redirected e265) v7 -- ?+0 0x55aa4d75c040 con 0x55aa4d759c30
2018-01-16 21:00:33.521343 7fbcd26fb700 1 -- 10.134.238.152:0/3412370064 <== osd.5 10.134.238.159:6800/11592 14 ==== osd_op_reply(21 zone_names.default [getxattrs,stat] v0'0 uv1 ondisk = 0) v7 ==== 180+0+20 (2828502694 0 2350394807) 0x7fbcc80016a0 con 0x55aa4d759c30
2018-01-16 21:00:33.521433 7fbd034e09c0 1 -- 10.134.238.152:0/3412370064 --> 10.134.238.159:6800/11592 -- osd_op(client.74204.0:22 3.7376dc2f zone_names.default [read 0~524288] snapc 0=[] ack+read+known_if_redirected e265) v7 -- ?+0 0x55aa4d7578f0 con 0x55aa4d759c30
2018-01-16 21:00:33.522479 7fbcd26fb700 1 -- 10.134.238.152:0/3412370064 <== osd.5 10.134.238.159:6800/11592 15 ==== osd_op_reply(22 zone_names.default [read 0~46] v0'0 uv1 ondisk = 0) v7 ==== 138+0+46 (2493670163 0 3868786191) 0x7fbcc80016a0 con 0x55aa4d759c30
2018-01-16 21:00:33.522576 7fbd034e09c0 1 -- 10.134.238.152:0/3412370064 --> 10.134.238.159:6800/11592 -- osd_op(client.74204.0:23 3.b3a6a33e zone_info.93541ddd-3a56-49fe-a109-83e09bcd7483 [getxattrs,stat] snapc 0=[] ack+read+known_if_redirected e265) v7 -- ?+0 0x55aa4d757890 con 0x55aa4d759c30
2018-01-16 21:00:33.523596 7fbcd26fb700 1 -- 10.134.238.152:0/3412370064 <== osd.5 10.134.238.159:6800/11592 16 ==== osd_op_reply(23 zone_info.93541ddd-3a56-49fe-a109-83e09bcd7483 [getxattrs,stat] v0'0 uv1 ondisk = 0) v7 ==== 208+0+20 (1890529474 0 2877186962) 0x7fbcc80016a0 con 0x55aa4d759c30
2018-01-16 21:00:33.523671 7fbd034e09c0 1 -- 10.134.238.152:0/3412370064 --> 10.134.238.159:6800/11592 -- osd_op(client.74204.0:24 3.b3a6a33e zone_info.93541ddd-3a56-49fe-a109-83e09bcd7483 [read 0~524288] snapc 0=[] ack+read+known_if_redirected e265) v7 -- ?+0 0x55aa4d7601d0 con 0x55aa4d759c30
2018-01-16 21:00:33.524642 7fbcd26fb700 1 -- 10.134.238.152:0/3412370064 <== osd.5 10.134.238.159:6800/11592 17 ==== osd_op_reply(24 zone_info.93541ddd-3a56-49fe-a109-83e09bcd7483 [read 0~1220] v0'0 uv1 ondisk = 0) v7 ==== 166+0+1220 (3743396430 0 3852199067) 0x7fbcc80016a0 con 0x55aa4d759c30
2018-01-16 21:00:33.524803 7fbd034e09c0 1 -- 10.134.238.152:0/3412370064 --> 10.134.238.159:6800/11592 -- osd_op(client.74204.0:25 3.7376dc2f zone_names.default [getxattrs,stat] snapc 0=[] ack+read+known_if_redirected e265) v7 -- ?+0 0x55aa4d760e90 con 0x55aa4d759c30
2018-01-16 21:00:33.525854 7fbcd26fb700 1 -- 10.134.238.152:0/3412370064 <== osd.5 10.134.238.159:6800/11592 18 ==== osd_op_reply(25 zone_names.default [getxattrs,stat] v0'0 uv1 ondisk = 0) v7 ==== 180+0+20 (2828502694 0 2350394807) 0x7fbcc80016a0 con 0x55aa4d759c30
2018-01-16 21:00:33.525926 7fbd034e09c0 1 -- 10.134.238.152:0/3412370064 --> 10.134.238.159:6800/11592 -- osd_op(client.74204.0:26 3.7376dc2f zone_names.default [read 0~524288] snapc 0=[] ack+read+known_if_redirected e265) v7 -- ?+0 0x55aa4d760e90 con 0x55aa4d759c30
2018-01-16 21:00:33.526977 7fbcd26fb700 1 -- 10.134.238.152:0/3412370064 <== osd.5 10.134.238.159:6800/11592 19 ==== osd_op_reply(26 zone_names.default [read 0~46] v0'0 uv1 ondisk = 0) v7 ==== 138+0+46 (2493670163 0 3868786191) 0x7fbcc80016a0 con 0x55aa4d759c30
2018-01-16 21:00:33.527038 7fbd034e09c0 10 ERROR: name default already in use for obj id 93541ddd-3a56-49fe-a109-83e09bcd7483
2018-01-16 21:00:33.527044 7fbd034e09c0 0 RGWZoneParams::create(): error creating default zone params: (17) File exists
2018-01-16 21:00:33.527050 7fbd034e09c0 10 zone_params::create_default() returned -EEXIST, we raced with another default zone_params creation
2018-01-16 21:00:33.527076 7fbd034e09c0 1 -- 10.134.238.152:0/3412370064 --> 10.134.238.159:6800/11592 -- osd_op(client.74204.0:27 3.7376dc2f zone_names.default [getxattrs,stat] snapc 0=[] ack+read+known_if_redirected e265) v7 -- ?+0 0x55aa4d7612f0 con 0x55aa4d759c30
2018-01-16 21:00:33.528118 7fbcd26fb700 1 -- 10.134.238.152:0/3412370064 <== osd.5 10.134.238.159:6800/11592 20 ==== osd_op_reply(27 zone_names.default [getxattrs,stat] v0'0 uv1 ondisk = 0) v7 ==== 180+0+20 (2828502694 0 2350394807) 0x7fbcc80016a0 con 0x55aa4d759c30
2018-01-16 21:00:33.528184 7fbd034e09c0 1 -- 10.134.238.152:0/3412370064 --> 10.134.238.159:6800/11592 -- osd_op(client.74204.0:28 3.7376dc2f zone_names.default [read 0~524288] snapc 0=[] ack+read+known_if_redirected e265) v7 -- ?+0 0x55aa4d760e90 con 0x55aa4d759c30
2018-01-16 21:00:33.529167 7fbcd26fb700 1 -- 10.134.238.152:0/3412370064 <== osd.5 10.134.238.159:6800/11592 21 ==== osd_op_reply(28 zone_names.default [read 0~46] v0'0 uv1 ondisk = 0) v7 ==== 138+0+46 (2493670163 0 3868786191) 0x7fbcc80016a0 con 0x55aa4d759c30
2018-01-16 21:00:33.529248 7fbd034e09c0 1 -- 10.134.238.152:0/3412370064 --> 10.134.238.159:6800/11592 -- osd_op(client.74204.0:29 3.b3a6a33e zone_info.93541ddd-3a56-49fe-a109-83e09bcd7483 [getxattrs,stat] snapc 0=[] ack+read+known_if_redirected e265) v7 -- ?+0 0x55aa4d760e90 con 0x55aa4d759c30
2018-01-16 21:00:33.530317 7fbcd26fb700 1 -- 10.134.238.152:0/3412370064 <== osd.5 10.134.238.159:6800/11592 22 ==== osd_op_reply(29 zone_info.93541ddd-3a56-49fe-a109-83e09bcd7483 [getxattrs,stat] v0'0 uv1 ondisk = 0) v7 ==== 208+0+20 (1890529474 0 2877186962) 0x7fbcc80016a0 con 0x55aa4d759c30
2018-01-16 21:00:33.530387 7fbd034e09c0 1 -- 10.134.238.152:0/3412370064 --> 10.134.238.159:6800/11592 -- osd_op(client.74204.0:30 3.b3a6a33e zone_info.93541ddd-3a56-49fe-a109-83e09bcd7483 [read 0~524288] snapc 0=[] ack+read+known_if_redirected e265) v7 -- ?+0 0x55aa4d760e90 con 0x55aa4d759c30
2018-01-16 21:00:33.531389 7fbcd26fb700 1 -- 10.134.238.152:0/3412370064 <== osd.5 10.134.238.159:6800/11592 23 ==== osd_op_reply(30 zone_info.93541ddd-3a56-49fe-a109-83e09bcd7483 [read 0~1220] v0'0 uv1 ondisk = 0) v7 ==== 166+0+1220 (3743396430 0 3852199067) 0x7fbcc80016a0 con 0x55aa4d759c30
2018-01-16 21:00:33.531496 7fbd034e09c0 1 -- 10.134.238.152:0/3412370064 --> 10.134.238.159:6800/11592 -- osd_op(client.74204.0:31 3.1eaca32f zonegroups_names.default [getxattrs,stat] snapc 0=[] ack+read+known_if_redirected e265) v7 -- ?+0 0x55aa4d761c90 con 0x55aa4d759c30
2018-01-16 21:00:33.532620 7fbcd26fb700 1 -- 10.134.238.152:0/3412370064 <== osd.5 10.134.238.159:6800/11592 24 ==== osd_op_reply(31 zonegroups_names.default [getxattrs,stat] v0'0 uv2 ondisk = 0) v7 ==== 186+0+20 (3398099016 0 769767392) 0x7fbcc80016a0 con 0x55aa4d759c30
2018-01-16 21:00:33.532726 7fbd034e09c0 1 -- 10.134.238.152:0/3412370064 --> 10.134.238.159:6800/11592 -- osd_op(client.74204.0:32 3.1eaca32f zonegroups_names.default [read 0~524288] snapc 0=[] ack+read+known_if_redirected e265) v7 -- ?+0 0x55aa4d761c90 con 0x55aa4d759c30
2018-01-16 21:00:33.533679 7fbcd26fb700 1 -- 10.134.238.152:0/3412370064 <== osd.5 10.134.238.159:6800/11592 25 ==== osd_op_reply(32 zonegroups_names.default [read 0~46] v0'0 uv2 ondisk = 0) v7 ==== 144+0+46 (520762147 0 994641691) 0x7fbcc80016a0 con 0x55aa4d759c30
2018-01-16 21:00:33.533760 7fbd034e09c0 10 ERROR: name default already in use for obj id 0356828d-f71f-4a85-8c71-8fb11aeb5683
2018-01-16 21:00:33.533765 7fbd034e09c0 10 create_default() returned -EEXIST, we raced with another zonegroup creation
2018-01-16 21:00:33.533791 7fbd034e09c0 1 -- 10.134.238.152:0/3412370064 --> 10.134.238.159:6800/11592 -- osd_op(client.74204.0:33 3.1eaca32f zonegroups_names.default [getxattrs,stat] snapc 0=[] ack+read+known_if_redirected e265) v7 -- ?+0 0x55aa4d762460 con 0x55aa4d759c30
2018-01-16 21:00:33.534684 7fbcd26fb700 1 -- 10.134.238.152:0/3412370064 <== osd.5 10.134.238.159:6800/11592 26 ==== osd_op_reply(33 zonegroups_names.default [getxattrs,stat] v0'0 uv2 ondisk = 0) v7 ==== 186+0+20 (3398099016 0 769767392) 0x7fbcc80016a0 con 0x55aa4d759c30
2018-01-16 21:00:33.534766 7fbd034e09c0 1 -- 10.134.238.152:0/3412370064 --> 10.134.238.159:6800/11592 -- osd_op(client.74204.0:34 3.1eaca32f zonegroups_names.default [read 0~524288] snapc 0=[] ack+read+known_if_redirected e265) v7 -- ?+0 0x55aa4d761c90 con 0x55aa4d759c30
2018-01-16 21:00:33.535829 7fbcd26fb700 1 -- 10.134.238.152:0/3412370064 <== osd.5 10.134.238.159:6800/11592 27 ==== osd_op_reply(34 zonegroups_names.default [read 0~46] v0'0 uv2 ondisk = 0) v7 ==== 144+0+46 (520762147 0 994641691) 0x7fbcc80016a0 con 0x55aa4d759c30
2018-01-16 21:00:33.535931 7fbd034e09c0 1 -- 10.134.238.152:0/3412370064 --> 10.134.238.157:6804/17168 -- osd_op(client.74204.0:35 3.e7017213 zonegroup_info.0356828d-f71f-4a85-8c71-8fb11aeb5683 [getxattrs,stat] snapc 0=[] ack+read+known_if_redirected e265) v7 -- ?+0 0x55aa4d762a60 con 0x7fbcc8004930
2018-01-16 21:00:33.536753 7fbcd22f7700 1 -- 10.134.238.152:0/3412370064 <== osd.3 10.134.238.157:6804/17168 6 ==== osd_op_reply(35 zonegroup_info.0356828d-f71f-4a85-8c71-8fb11aeb5683 [getxattrs,stat] v0'0 uv1 ondisk = 0) v7 ==== 213+0+20 (4083675718 0 178235766) 0x7fbcb80015a0 con 0x7fbcc8004930
2018-01-16 21:00:33.536850 7fbd034e09c0 1 -- 10.134.238.152:0/3412370064 --> 10.134.238.157:6804/17168 -- osd_op(client.74204.0:36 3.e7017213 zonegroup_info.0356828d-f71f-4a85-8c71-8fb11aeb5683 [read 0~524288] snapc 0=[] ack+read+known_if_redirected e265) v7 -- ?+0 0x55aa4d762460 con 0x7fbcc8004930
2018-01-16 21:00:33.537526 7fbcd22f7700 1 -- 10.134.238.152:0/3412370064 <== osd.3 10.134.238.157:6804/17168 7 ==== osd_op_reply(36 zonegroup_info.0356828d-f71f-4a85-8c71-8fb11aeb5683 [read 0~324] v0'0 uv1 ondisk = 0) v7 ==== 171+0+324 (2897459508 0 2519818751) 0x7fbcb80015a0 con 0x7fbcc8004930
2018-01-16 21:00:33.537646 7fbd034e09c0 1 -- 10.134.238.152:0/3412370064 --> 10.134.238.159:6800/11592 -- osd_op(client.74204.0:37 3.b3a6a33e zone_info.93541ddd-3a56-49fe-a109-83e09bcd7483 [getxattrs,stat] snapc 0=[] ack+read+known_if_redirected e265) v7 -- ?+0 0x55aa4d761e40 con 0x55aa4d759c30
2018-01-16 21:00:33.538784 7fbcd26fb700 1 -- 10.134.238.152:0/3412370064 <== osd.5 10.134.238.159:6800/11592 28 ==== osd_op_reply(37 zone_info.93541ddd-3a56-49fe-a109-83e09bcd7483 [getxattrs,stat] v0'0 uv1 ondisk = 0) v7 ==== 208+0+20 (1890529474 0 2877186962) 0x7fbcc80016a0 con 0x55aa4d759c30
2018-01-16 21:00:33.538884 7fbd034e09c0 1 -- 10.134.238.152:0/3412370064 --> 10.134.238.159:6800/11592 -- osd_op(client.74204.0:38 3.b3a6a33e zone_info.93541ddd-3a56-49fe-a109-83e09bcd7483 [read 0~524288] snapc 0=[] ack+read+known_if_redirected e265) v7 -- ?+0 0x55aa4d761e40 con 0x55aa4d759c30
2018-01-16 21:00:33.539991 7fbcd26fb700 1 -- 10.134.238.152:0/3412370064 <== osd.5 10.134.238.159:6800/11592 29 ==== osd_op_reply(38 zone_info.93541ddd-3a56-49fe-a109-83e09bcd7483 [read 0~1220] v0'0 uv1 ondisk = 0) v7 ==== 166+0+1220 (3743396430 0 3852199067) 0x7fbcc80016a0 con 0x55aa4d759c30
2018-01-16 21:00:33.540137 7fbd034e09c0 1 -- 10.134.238.152:0/3412370064 --> 10.134.238.157:6804/17168 -- osd_op(client.74204.0:39 3.e7017213 zonegroup_info.0356828d-f71f-4a85-8c71-8fb11aeb5683 [getxattrs,stat] snapc 0=[] ack+read+known_if_redirected e265) v7 -- ?+0 0x55aa4d761e40 con 0x7fbcc8004930
2018-01-16 21:00:33.540907 7fbcd22f7700 1 -- 10.134.238.152:0/3412370064 <== osd.3 10.134.238.157:6804/17168 8 ==== osd_op_reply(39 zonegroup_info.0356828d-f71f-4a85-8c71-8fb11aeb5683 [getxattrs,stat] v0'0 uv1 ondisk = 0) v7 ==== 213+0+20 (4083675718 0 178235766) 0x7fbcb80015a0 con 0x7fbcc8004930
2018-01-16 21:00:33.541022 7fbd034e09c0 1 -- 10.134.238.152:0/3412370064 --> 10.134.238.157:6804/17168 -- osd_op(client.74204.0:40 3.e7017213 zonegroup_info.0356828d-f71f-4a85-8c71-8fb11aeb5683 [read 0~524288] snapc 0=[] ack+read+known_if_redirected e265) v7 -- ?+0 0x55aa4d761e40 con 0x7fbcc8004930
2018-01-16 21:00:33.541673 7fbcd22f7700 1 -- 10.134.238.152:0/3412370064 <== osd.3 10.134.238.157:6804/17168 9 ==== osd_op_reply(40 zonegroup_info.0356828d-f71f-4a85-8c71-8fb11aeb5683 [read 0~324] v0'0 uv1 ondisk = 0) v7 ==== 171+0+324 (2897459508 0 2519818751) 0x7fbcb80015a0 con 0x7fbcc8004930
2018-01-16 21:00:33.541773 7fbd034e09c0 10 Cannot find current period zone using local zone
2018-01-16 21:00:33.541779 7fbd034e09c0 10 Using default name default
2018-01-16 21:00:33.541804 7fbd034e09c0 1 -- 10.134.238.152:0/3412370064 --> 10.134.238.159:6800/11592 -- osd_op(client.74204.0:41 3.7376dc2f zone_names.default [getxattrs,stat] snapc 0=[] ack+read+known_if_redirected e265) v7 -- ?+0 0x55aa4d761e40 con 0x55aa4d759c30
2018-01-16 21:00:33.542801 7fbcd26fb700 1 -- 10.134.238.152:0/3412370064 <== osd.5 10.134.238.159:6800/11592 30 ==== osd_op_reply(41 zone_names.default [getxattrs,stat] v0'0 uv1 ondisk = 0) v7 ==== 180+0+20 (2828502694 0 2350394807) 0x7fbcc80016a0 con 0x55aa4d759c30
2018-01-16 21:00:33.542899 7fbd034e09c0 1 -- 10.134.238.152:0/3412370064 --> 10.134.238.159:6800/11592 -- osd_op(client.74204.0:42 3.7376dc2f zone_names.default [read 0~524288] snapc 0=[] ack+read+known_if_redirected e265) v7 -- ?+0 0x55aa4d761e40 con 0x55aa4d759c30
2018-01-16 21:00:33.543896 7fbcd26fb700 1 -- 10.134.238.152:0/3412370064 <== osd.5 10.134.238.159:6800/11592 31 ==== osd_op_reply(42 zone_names.default [read 0~46] v0'0 uv1 ondisk = 0) v7 ==== 138+0+46 (2493670163 0 3868786191) 0x7fbcc80016a0 con 0x55aa4d759c30
2018-01-16 21:00:33.544001 7fbd034e09c0 1 -- 10.134.238.152:0/3412370064 --> 10.134.238.159:6800/11592 -- osd_op(client.74204.0:43 3.b3a6a33e zone_info.93541ddd-3a56-49fe-a109-83e09bcd7483 [getxattrs,stat] snapc 0=[] ack+read+known_if_redirected e265) v7 -- ?+0 0x55aa4d761e40 con 0x55aa4d759c30
2018-01-16 21:00:33.544994 7fbcd26fb700 1 -- 10.134.238.152:0/3412370064 <== osd.5 10.134.238.159:6800/11592 32 ==== osd_op_reply(43 zone_info.93541ddd-3a56-49fe-a109-83e09bcd7483 [getxattrs,stat] v0'0 uv1 ondisk = 0) v7 ==== 208+0+20 (1890529474 0 2877186962) 0x7fbcc80016a0 con 0x55aa4d759c30
2018-01-16 21:00:33.545102 7fbd034e09c0 1 -- 10.134.238.152:0/3412370064 --> 10.134.238.159:6800/11592 -- osd_op(client.74204.0:44 3.b3a6a33e zone_info.93541ddd-3a56-49fe-a109-83e09bcd7483 [read 0~524288] snapc 0=[] ack+read+known_if_redirected e265) v7 -- ?+0 0x55aa4d761e40 con 0x55aa4d759c30
2018-01-16 21:00:33.546142 7fbcd26fb700 1 -- 10.134.238.152:0/3412370064 <== osd.5 10.134.238.159:6800/11592 33 ==== osd_op_reply(44 zone_info.93541ddd-3a56-49fe-a109-83e09bcd7483 [read 0~1220] v0'0 uv1 ondisk = 0) v7 ==== 166+0+1220 (3743396430 0 3852199067) 0x7fbcc80016a0 con 0x55aa4d759c30
2018-01-16 21:00:33.546341 7fbd034e09c0 1 -- 10.134.238.152:0/3412370064 --> 10.134.238.158:6804/3586 -- osd_op(client.74204.0:45 4.4322fa9f notify.0 [create 0~0] snapc 0=[] ondisk+write+known_if_redirected e265) v7 -- ?+0 0x55aa4d763930 con 0x55aa4d75e050
2018-01-16 21:00:33.547508 7fbcd24f9700 1 -- 10.134.238.152:0/3412370064 <== osd.4 10.134.238.158:6804/3586 3 ==== osd_op_reply(45 notify.0 [create 0~0] v0'0 uv1 ondisk = 0) v7 ==== 128+0+0 (2679103017 0 0) 0x7fbcc0002b90 con 0x55aa4d75e050
2018-01-16 21:00:33.547645 7fbd034e09c0 1 -- 10.134.238.152:0/3412370064 --> 10.134.238.158:6804/3586 -- osd_op(client.74204.0:46 4.4322fa9f notify.0 [watch watch cookie 94189932382016] snapc 0=[] ondisk+write+known_if_redirected e265) v7 -- ?+0 0x55aa4d763fd0 con 0x55aa4d75e050
2018-01-16 21:00:33.577397 7fbcd24f9700 1 -- 10.134.238.152:0/3412370064 <== osd.4 10.134.238.158:6804/3586 4 ==== osd_op_reply(46 notify.0 [watch watch cookie 94189932382016] v265'43 uv1 ondisk = 0) v7 ==== 128+0+0 (3703752419 0 0) 0x7fbcc0002b90 con 0x55aa4d75e050
2018-01-16 21:00:33.577506 7fbd034e09c0 1 -- 10.134.238.152:0/3412370064 --> 10.134.238.157:6804/17168 -- osd_op(client.74204.0:47 4.16dafda0 notify.1 [create 0~0] snapc 0=[] ondisk+write+known_if_redirected e265) v7 -- ?+0 0x55aa4d763fd0 con 0x7fbcc8004930
2018-01-16 21:00:33.578014 7fbcd22f7700 1 -- 10.134.238.152:0/3412370064 <== osd.3 10.134.238.157:6804/17168 10 ==== osd_op_reply(47 notify.1 [create 0~0] v0'0 uv1 ondisk = 0) v7 ==== 128+0+0 (2954800741 0 0) 0x7fbcb80015a0 con 0x7fbcc8004930
2018-01-16 21:00:33.578069 7fbd034e09c0 1 -- 10.134.238.152:0/3412370064 --> 10.134.238.157:6804/17168 -- osd_op(client.74204.0:48 4.16dafda0 notify.1 [watch watch cookie 94189932395920] snapc 0=[] ondisk+write+known_if_redirected e265) v7 -- ?+0 0x55aa4d764900 con 0x7fbcc8004930
2018-01-16 21:00:33.596286 7fbcd22f7700 1 -- 10.134.238.152:0/3412370064 <== osd.3 10.134.238.157:6804/17168 11 ==== osd_op_reply(48 notify.1 [watch watch cookie 94189932395920] v265'43 uv1 ondisk = 0) v7 ==== 128+0+0 (2922030704 0 0) 0x7fbcb80015a0 con 0x7fbcc8004930
2018-01-16 21:00:33.596416 7fbd034e09c0 1 -- 10.134.238.152:0/3412370064 --> 10.134.238.159:6800/11592 -- osd_op(client.74204.0:49 4.88aa5c95 notify.2 [create 0~0] snapc 0=[] ondisk+write+known_if_redirected e265) v7 -- ?+0 0x55aa4d764900 con 0x55aa4d759c30
2018-01-16 21:00:33.597506 7fbcd26fb700 1 -- 10.134.238.152:0/3412370064 <== osd.5 10.134.238.159:6800/11592 34 ==== osd_op_reply(49 notify.2 [create 0~0] v0'0 uv1 ondisk = 0) v7 ==== 128+0+0 (3617160630 0 0) 0x7fbcc80016a0 con 0x55aa4d759c30
2018-01-16 21:00:33.597600 7fbd034e09c0 1 -- 10.134.238.152:0/3412370064 --> 10.134.238.159:6800/11592 -- osd_op(client.74204.0:50 4.88aa5c95 notify.2 [watch watch cookie 94189932398272] snapc 0=[] ondisk+write+known_if_redirected e265) v7 -- ?+0 0x55aa4d765140 con 0x55aa4d759c30
2018-01-16 21:00:33.613392 7fbcd26fb700 1 -- 10.134.238.152:0/3412370064 <== osd.5 10.134.238.159:6800/11592 35 ==== osd_op_reply(50 notify.2 [watch watch cookie 94189932398272] v265'85 uv1 ondisk = 0) v7 ==== 128+0+0 (716529099 0 0) 0x7fbcc80016a0 con 0x55aa4d759c30
2018-01-16 21:00:33.613474 7fbd034e09c0 1 -- 10.134.238.152:0/3412370064 --> 10.134.238.158:6804/3586 -- osd_op(client.74204.0:51 4.f8c99aee notify.3 [create 0~0] snapc 0=[] ondisk+write+known_if_redirected e265) v7 -- ?+0 0x55aa4d765140 con 0x55aa4d75e050
2018-01-16 21:00:33.614516 7fbcd24f9700 1 -- 10.134.238.152:0/3412370064 <== osd.4 10.134.238.158:6804/3586 5 ==== osd_op_reply(51 notify.3 [create 0~0] v0'0 uv1 ondisk = 0) v7 ==== 128+0+0 (3896087742 0 0) 0x7fbcc0002b90 con 0x55aa4d75e050
2018-01-16 21:00:33.614595 7fbd034e09c0 1 -- 10.134.238.152:0/3412370064 --> 10.134.238.158:6804/3586 -- osd_op(client.74204.0:52 4.f8c99aee notify.3 [watch watch cookie 94189932400384] snapc 0=[] ondisk+write+known_if_redirected e265) v7 -- ?+0 0x55aa4d7659f0 con 0x55aa4d75e050
2018-01-16 21:00:33.628666 7fbcd24f9700 1 -- 10.134.238.152:0/3412370064 <== osd.4 10.134.238.158:6804/3586 6 ==== osd_op_reply(52 notify.3 [watch watch cookie 94189932400384] v265'43 uv1 ondisk = 0) v7 ==== 128+0+0 (3679361280 0 0) 0x7fbcc0002b90 con 0x55aa4d75e050
2018-01-16 21:00:33.628777 7fbd034e09c0 1 -- 10.134.238.152:0/3412370064 --> 10.134.238.159:6800/11592 -- osd_op(client.74204.0:53 4.a204812d notify.4 [create 0~0] snapc 0=[] ondisk+write+known_if_redirected e265) v7 -- ?+0 0x55aa4d7659f0 con 0x55aa4d759c30
2018-01-16 21:00:33.629405 7fbcd26fb700 1 -- 10.134.238.152:0/3412370064 <== osd.5 10.134.238.159:6800/11592 36 ==== osd_op_reply(53 notify.4 [create 0~0] v0'0 uv3 ondisk = 0) v7 ==== 128+0+0 (838607157 0 0) 0x7fbcc80016a0 con 0x55aa4d759c30
2018-01-16 21:00:33.629492 7fbd034e09c0 1 -- 10.134.238.152:0/3412370064 --> 10.134.238.159:6800/11592 -- osd_op(client.74204.0:54 4.a204812d notify.4 [watch watch cookie 94189932402608] snapc 0=[] ondisk+write+known_if_redirected e265) v7 -- ?+0 0x55aa4d766320 con 0x55aa4d759c30
2018-01-16 21:00:33.642070 7fbcd26fb700 1 -- 10.134.238.152:0/3412370064 <== osd.5 10.134.238.159:6800/11592 37 ==== osd_op_reply(54 notify.4 [watch watch cookie 94189932402608] v265'86 uv3 ondisk = 0) v7 ==== 128+0+0 (3902195444 0 0) 0x7fbcc80016a0 con 0x55aa4d759c30
2018-01-16 21:00:33.642155 7fbd034e09c0 1 -- 10.134.238.152:0/3412370064 --> 10.134.238.159:6800/11592 -- osd_op(client.74204.0:55 4.31099063 notify.5 [create 0~0] snapc 0=[] ondisk+write+known_if_redirected e265) v7 -- ?+0 0x55aa4d766320 con 0x55aa4d759c30
2018-01-16 21:00:33.643219 7fbcd26fb700 1 -- 10.134.238.152:0/3412370064 <== osd.5 10.134.238.159:6800/11592 38 ==== osd_op_reply(55 notify.5 [create 0~0] v0'0 uv1 ondisk = 0) v7 ==== 128+0+0 (105356080 0 0) 0x7fbcc80016a0 con 0x55aa4d759c30
2018-01-16 21:00:33.643297 7fbd034e09c0 1 -- 10.134.238.152:0/3412370064 --> 10.134.238.159:6800/11592 -- osd_op(client.74204.0:56 4.31099063 notify.5 [watch watch cookie 94189932404960] snapc 0=[] ondisk+write+known_if_redirected e265) v7 -- ?+0 0x55aa4d766c50 con 0x55aa4d759c30
2018-01-16 21:00:33.659017 7fbcd26fb700 1 -- 10.134.238.152:0/3412370064 <== osd.5 10.134.238.159:6800/11592 39 ==== osd_op_reply(56 notify.5 [watch watch cookie 94189932404960] v265'44 uv1 ondisk = 0) v7 ==== 128+0+0 (1978113558 0 0) 0x7fbcc80016a0 con 0x55aa4d759c30
2018-01-16 21:00:33.659106 7fbd034e09c0 1 -- 10.134.238.152:0/3412370064 --> 10.134.238.159:6800/11592 -- osd_op(client.74204.0:57 4.97c520d4 notify.6 [create 0~0] snapc 0=[] ondisk+write+known_if_redirected e265) v7 -- ?+0 0x55aa4d766c50 con 0x55aa4d759c30
2018-01-16 21:00:33.660148 7fbcd26fb700 1 -- 10.134.238.152:0/3412370064 <== osd.5 10.134.238.159:6800/11592 40 ==== osd_op_reply(57 notify.6 [create 0~0] v0'0 uv1 ondisk = 0) v7 ==== 128+0+0 (3036545333 0 0) 0x7fbcc80016a0 con 0x55aa4d759c30
2018-01-16 21:00:33.660227 7fbd034e09c0 1 -- 10.134.238.152:0/3412370064 --> 10.134.238.159:6800/11592 -- osd_op(client.74204.0:58 4.97c520d4 notify.6 [watch watch cookie 94189932407312] snapc 0=[] ondisk+write+known_if_redirected e265) v7 -- ?+0 0x55aa4d767570 con 0x55aa4d759c30
2018-01-16 21:00:33.687585 7fbcd26fb700 1 -- 10.134.238.152:0/3412370064 <== osd.5 10.134.238.159:6800/11592 41 ==== osd_op_reply(58 notify.6 [watch watch cookie 94189932407312] v265'43 uv1 ondisk = 0) v7 ==== 128+0+0 (268471567 0 0) 0x7fbcc80016a0 con 0x55aa4d759c30
2018-01-16 21:00:33.687665 7fbd034e09c0 1 -- 10.134.238.152:0/3412370064 --> 10.134.238.157:6804/17168 -- osd_op(client.74204.0:59 4.84ada7c9 notify.7 [create 0~0] snapc 0=[] ondisk+write+known_if_redirected e265) v7 -- ?+0 0x55aa4d767570 con 0x7fbcc8004930
2018-01-16 21:00:33.688499 7fbcd22f7700 1 -- 10.134.238.152:0/3412370064 <== osd.3 10.134.238.157:6804/17168 12 ==== osd_op_reply(59 notify.7 [create 0~0] v0'0 uv1 ondisk = 0) v7 ==== 128+0+0 (3944968614 0 0) 0x7fbcb80015a0 con 0x7fbcc8004930
2018-01-16 21:00:33.688590 7fbd034e09c0 1 -- 10.134.238.152:0/3412370064 --> 10.134.238.157:6804/17168 -- osd_op(client.74204.0:60 4.84ada7c9 notify.7 [watch watch cookie 94189932409648] snapc 0=[] ondisk+write+known_if_redirected e265) v7 -- ?+0 0x55aa4d767e80 con 0x7fbcc8004930
2018-01-16 21:00:33.704814 7fbcd22f7700 1 -- 10.134.238.152:0/3412370064 <== osd.3 10.134.238.157:6804/17168 13 ==== osd_op_reply(60 notify.7 [watch watch cookie 94189932409648] v265'43 uv1 ondisk = 0) v7 ==== 128+0+0 (3032547970 0 0) 0x7fbcb80015a0 con 0x7fbcc8004930
2018-01-16 21:00:33.704887 7fbd034e09c0 2 all 8 watchers are set, enabling cache
2018-01-16 21:00:33.705023 7fbcd19f5700 2 garbage collection: start
2018-01-16 21:00:33.705435 7fbcd19f5700 1 -- 10.134.238.152:0/3412370064 --> 10.134.238.157:6804/17168 -- osd_op(client.74204.0:61 6.c2ff39f3 gc.16 [call lock.lock] snapc 0=[] ondisk+write+known_if_redirected e265) v7 -- ?+0 0x7fbcb0002fb0 con 0x7fbcc8004930
2018-01-16 21:00:33.705514 7fbcd11f4700 2 object expiration: start
2018-01-16 21:00:33.705664 7fbcd11f4700 1 -- 10.134.238.152:0/3412370064 --> 10.134.238.159:6800/11592 -- osd_op(client.74204.0:62 7.9a32ce58 obj_delete_at_hint.0000000000 [call lock.lock] snapc 0=[] ondisk+write+known_if_redirected e265) v7 -- ?+0 0x7fbca4003000 con 0x55aa4d759c30
2018-01-16 21:00:33.706537 7fbd034e09c0 10 find_oldest_log_period returning empty cursor
2018-01-16 21:00:33.706937 7fbcd26fb700 1 -- 10.134.238.152:0/3412370064 <== osd.5 10.134.238.159:6800/11592 42 ==== osd_op_reply(62 obj_delete_at_hint.0000000000 [call] v0'0 uv0 ondisk = -16 ((16) Device or resource busy)) v7 ==== 149+0+0 (2015788343 0 0) 0x7fbcc80016a0 con 0x55aa4d759c30
2018-01-16 21:00:33.707007 7fbcd11f4700 5 process_single_shard(): failed to acquire lock on obj_delete_at_hint.0000000000
2018-01-16 21:00:33.707034 7fbcd11f4700 1 -- 10.134.238.152:0/3412370064 --> 10.134.238.158:6804/3586 -- osd_op(client.74204.0:63 7.5dc198f obj_delete_at_hint.0000000001 [call lock.lock] snapc 0=[] ondisk+write+known_if_redirected e265) v7 -- ?+0 0x7fbca40035e0 con 0x55aa4d75e050
2018-01-16 21:00:33.707301 7fbc8ffff700 10 ERROR: can't get key: ret=-2
2018-01-16 21:00:33.707310 7fbc8ffff700 5 ERROR: sync_all_users() returned ret=-2
2018-01-16 21:00:33.708042 7fbd034e09c0 0 starting handler: civetweb
2018-01-16 21:00:33.708443 7fbcd24f9700 1 -- 10.134.238.152:0/3412370064 <== osd.4 10.134.238.158:6804/3586 7 ==== osd_op_reply(63 obj_delete_at_hint.0000000001 [call] v0'0 uv0 ondisk = -16 ((16) Device or resource busy)) v7 ==== 149+0+0 (3998785031 0 0) 0x7fbcc0001680 con 0x55aa4d75e050
2018-01-16 21:00:33.708483 7fbcd11f4700 5 process_single_shard(): failed to acquire lock on obj_delete_at_hint.0000000001
2018-01-16 21:00:33.708499 7fbd034e09c0 0 civetweb: 0x55aa4d2dcbe0: set_ports_option: cannot bind to 7480: 98 (Address already in use)
2018-01-16 21:00:33.708517 7fbd034e09c0 -1 ERROR: failed run
2018-01-16 21:00:33.708509 7fbcd11f4700 1 -- 10.134.238.152:0/3412370064 --> 10.134.238.157:6804/17168 -- osd_op(client.74204.0:64 7.68795c0e obj_delete_at_hint.0000000002 [call lock.lock] snapc 0=[] ondisk+write+known_if_redirected e265) v7 -- ?+0 0x7fbca40035e0 con 0x7fbcc8004930
[root@ceph-rgw ~]#
(2-2/2)