Ceph » rgw

We use S3FS and nfs-ganesha to upload file to s3 and sync metadata to elasticsearch by es sync module. When uploading file in nfs-ganesha, rgw sync status show "1 shards are recovering" and never change.

[vagrant@client-1 build]$ radosgw-admin sync status --rgw-zone us-east-es
2018-09-27 03:00:36.936575 7f12c16c6dc0 -1 WARNING: all dangerous and experimental features are enabled.
2018-09-27 03:00:36.936821 7f12c16c6dc0 -1 WARNING: all dangerous and experimental features are enabled.
2018-09-27 03:00:36.956403 7f12c16c6dc0 -1 WARNING: all dangerous and experimental features are enabled.
          realm 64a495cc-e622-44ae-b0d2-c48edfb6e395 (gold)
      zonegroup 84e47fe5-6153-4046-a10a-17d000a15d84 (us)
           zone 5313469f-4381-4a26-9ef4-f570d188d2fb (us-east-es)
  metadata sync syncing
                full sync: 0/64 shards
                incremental sync: 64/64 shards
                metadata is caught up with master
      data sync source: b4cda113-46e3-4662-b22e-ec427f11458c (us-east-1)
                        syncing
                        full sync: 0/128 shards
                        incremental sync: 128/128 shards
                        1 shards are recovering
                        recovering shards: [69]

radosgw us-east-es log show failed to sync object:

2018-09-27 02:53:47.500655 7efe374ce700 20 stat of remote obj: z=b4cda113-46e3-4662-b22e-ec427f11458c b=test[b4cda113-46e3-4662-b22e-ec427f11458c.4205.1] k=test_from_nfs10 size=1048576 mtime=2018-09-27 02:53:46.0.495825s attrs={user.rgw.acl=buffer::list(len=135,
    buffer::ptr(0~135 0x557dce0e50e0 in raw 0x557dce0e50e0 len 139 nref 1)
),user.rgw.etag=buffer::list(len=33,
    buffer::ptr(0~33 0x557dce187100 in raw 0x557dce187100 len 37 nref 1)
),user.rgw.idtag=buffer::list(len=46,
    buffer::ptr(0~46 0x557dce782240 in raw 0x557dce782240 len 52 nref 1)
),user.rgw.pg_ver=buffer::list(len=8,
    buffer::ptr(0~8 0x557dce69d880 in raw 0x557dce69d880 len 13 nref 1)
),user.rgw.source_zone=buffer::list(len=4,
    buffer::ptr(0~4 0x557dce8fb420 in raw 0x557dce8fb420 len 10 nref 1)
),user.rgw.tail_tag=buffer::list(len=46,
    buffer::ptr(0~46 0x557dce7822d0 in raw 0x557dce7822d0 len 52 nref 1)
),user.rgw.unix-key1=buffer::list(len=26,
    buffer::ptr(0~26 0x557dce187d00 in raw 0x557dce187d00 len 31 nref 1)
),user.rgw.unix1=buffer::list(len=74,
    buffer::ptr(0~74 0x557dcd9c11e0 in raw 0x557dcd9c11e0 len 79 nref 1)
)}
2018-09-27 02:53:47.500702 7efe374ce700 20 cr:s=0x557dcde8c540:op=0x557dce72b800:29RGWElasticHandleRemoteObjCBCR: operate()
2018-09-27 02:53:47.500709 7efe374ce700 10 : stat of remote obj: z=b4cda113-46e3-4662-b22e-ec427f11458c b=test[b4cda113-46e3-4662-b22e-ec427f11458c.4205.1] k=test_from_nfs10 size=1048576 mtime=2018-09-27 02:53:46.0.495825s attrs={user.rgw.acl=buffer::list(len=135,
    buffer::ptr(0~135 0x557dce0e50e0 in raw 0x557dce0e50e0 len 139 nref 1)
),user.rgw.etag=buffer::list(len=33,
    buffer::ptr(0~33 0x557dce187100 in raw 0x557dce187100 len 37 nref 1)
),user.rgw.idtag=buffer::list(len=46,
    buffer::ptr(0~46 0x557dce782240 in raw 0x557dce782240 len 52 nref 1)
),user.rgw.pg_ver=buffer::list(len=8,
    buffer::ptr(0~8 0x557dce69d880 in raw 0x557dce69d880 len 13 nref 1)
),user.rgw.source_zone=buffer::list(len=4,
    buffer::ptr(0~4 0x557dce8fb420 in raw 0x557dce8fb420 len 10 nref 1)
),user.rgw.tail_tag=buffer::list(len=46,
    buffer::ptr(0~46 0x557dce7822d0 in raw 0x557dce7822d0 len 52 nref 1)
),user.rgw.unix-key1=buffer::list(len=26,
    buffer::ptr(0~26 0x557dce187d00 in raw 0x557dce187d00 len 31 nref 1)
),user.rgw.unix1=buffer::list(len=74,
    buffer::ptr(0~74 0x557dcd9c11e0 in raw 0x557dcd9c11e0 len 79 nref 1)
)}
2018-09-27 02:53:47.500752 7efe374ce700 20 cr:s=0x557dcde8c540:op=0x557dce71d800:20RGWPutRESTResourceCRI15es_obj_metadataiE: operate()
2018-09-27 02:53:47.500755 7efe374ce700 20 cr:s=0x557dcde8c540:op=0x557dce71d800:20RGWPutRESTResourceCRI15es_obj_metadataiE: operate()
2018-09-27 02:53:47.500865 7efe374ce700 20 sending request to http://192.168.15.11:9200/rgw-gold-6dcfa3f5/object/b4cda113-46e3-4662-b22e-ec427f11458c.4205.1%3Atest_from_nfs10%3Anull
2018-09-27 02:53:47.500876 7efe374ce700 20 register_request mgr=0x557dcd8f39e0 req_data->id=1156, easy_handle=0x557dcea76000
2018-09-27 02:53:47.500884 7efe374ce700 20 run: stack=0x557dcde8c540 is io blocked
2018-09-27 02:53:47.500894 7efe324c4700 20 link_request req_data=0x557dce415c00 req_data->id=1156, easy_handle=0x557dcea76000
2018-09-27 02:53:47.502798 7efe374ce700 20 cr:s=0x557dcdbac540:op=0x557dce9db800:27RGWReadRemoteDataLogShardCR: operate()
2018-09-27 02:53:47.502921 7efe374ce700 20 cr:s=0x557dcdbac540:op=0x557dce33b600:18RGWDataSyncShardCR: operate()
2018-09-27 02:53:47.502927 7efe374ce700 20 data sync: incremental_sync:1396: shard_id=69 log_entry: 1_1538016826.301943_43113.1:2018-09-27 02:53:46.0.301943s:test:b4cda113-46e3-4662-b22e-ec427f11458c.4205.1
2018-09-27 02:53:47.502957 7efe374ce700 20 data sync: incremental_sync:1432: shard_id=69 datalog_marker=1_1538016826.301943_43113.1 sync_marker.marker=1_1538016826.301943_43113.1
2018-09-27 02:53:47.502965 7efe374ce700 20 run: stack=0x557dcdbac540 is io blocked
2018-09-27 02:53:47.502968 7efe374ce700 20 cr:s=0x557dcdcb55e0:op=0x557dce3e4300:24RGWDataSyncSingleEntryCR: operate()
2018-09-27 02:53:47.502977 7efe374ce700  5 data sync: Sync:b4cda113:data:Bucket:test:b4cda113-46e3-4662-b22e-ec427f11458c.4205.1:start
2018-09-27 02:53:47.502980 7efe374ce700 20 cr:s=0x557dcdcb55e0:op=0x557dce71d000:25RGWRunBucketSyncCoroutine: operate()
2018-09-27 02:53:47.503002 7efe374ce700 20 cr:s=0x557dce1af960:op=0x557dce3e3c00:20RGWContinuousLeaseCR: operate()
2018-09-27 02:53:47.503009 7efe374ce700 20 cr:s=0x557dcdcb55e0:op=0x557dce71d000:25RGWRunBucketSyncCoroutine: operate()
2018-09-27 02:53:47.503012 7efe374ce700 20 run: stack=0x557dcdcb55e0 is_blocked_by_stack()=0 is_sleeping=1 waiting_for_child()=0
2018-09-27 02:53:47.503013 7efe374ce700 20 cr:s=0x557dce1af960:op=0x557dce9db800:20RGWSimpleRadosLockCR: operate()
2018-09-27 02:53:47.503014 7efe374ce700 20 cr:s=0x557dce1af960:op=0x557dce9db800:20RGWSimpleRadosLockCR: operate()
2018-09-27 02:53:47.503030 7efe374ce700 20 enqueued request req=0x557dce7dfc70
2018-09-27 02:53:47.503032 7efe374ce700 20 RGWWQ:
2018-09-27 02:53:47.503033 7efe374ce700 20 req: 0x557dce7dfc70
2018-09-27 02:53:47.503036 7efe374ce700 20 run: stack=0x557dce1af960 is io blocked
2018-09-27 02:53:47.503043 7efe3b4d6700 20 dequeued request req=0x557dce7dfc70
2018-09-27 02:53:47.503046 7efe3b4d6700 20 RGWWQ: empty
2018-09-27 02:53:47.503356 7efe324c4700 10 receive_http_header
2018-09-27 02:53:47.503362 7efe324c4700 10 received header:HTTP/1.1 100 Continue
2018-09-27 02:53:47.503363 7efe324c4700 10 received header:HTTP/1.1
2018-09-27 02:53:47.503364 7efe324c4700 10 receive_http_header
2018-09-27 02:53:47.503365 7efe324c4700 10 received header:
2018-09-27 02:53:47.524418 7efe374ce700 20 cr:s=0x557dce1af960:op=0x557dce9db800:20RGWSimpleRadosLockCR: operate()
2018-09-27 02:53:47.524446 7efe374ce700 20 cr:s=0x557dce1af960:op=0x557dce9db800:20RGWSimpleRadosLockCR: operate() returned r=-16
2018-09-27 02:53:47.524454 7efe374ce700 20 cr:s=0x557dce1af960:op=0x557dce3e3c00:20RGWContinuousLeaseCR: operate()
2018-09-27 02:53:47.524456 7efe374ce700 20 cr:s=0x557dce1af960:op=0x557dce3e3c00:20RGWContinuousLeaseCR: couldn't lock us-east-es.rgw.log:bucket.sync-status.b4cda113-46e3-4662-b22e-ec427f11458c:test:b4cda113-46e3-4662-b22e-ec427f11458c.4205.1:sync_lock: retcode=-16
2018-09-27 02:53:47.524460 7efe374ce700 20 cr:s=0x557dce1af960:op=0x557dce3e3c00:20RGWContinuousLeaseCR: operate() returned r=-16
2018-09-27 02:53:47.524461 7efe374ce700 20 stack->operate() returned ret=-16
2018-09-27 02:53:47.524461 7efe374ce700 20 run: stack=0x557dce1af960 is done
2018-09-27 02:53:47.524465 7efe374ce700 20 cr:s=0x557dcdcb55e0:op=0x557dce71d000:25RGWRunBucketSyncCoroutine: operate()
2018-09-27 02:53:47.524467 7efe374ce700  5 data sync: lease cr failed, done early
2018-09-27 02:53:47.524470 7efe374ce700 20 cr:s=0x557dcdcb55e0:op=0x557dce71d000:25RGWRunBucketSyncCoroutine: operate() returned r=-16
2018-09-27 02:53:47.524474 7efe374ce700  5 data sync: Sync:b4cda113:data:Bucket:test:b4cda113-46e3-4662-b22e-ec427f11458c.4205.1:finish
2018-09-27 02:53:47.524478 7efe374ce700 20 cr:s=0x557dcdcb55e0:op=0x557dce3e4300:24RGWDataSyncSingleEntryCR: operate()
2018-09-27 02:53:47.524494 7efe374ce700 20 data sync: store_marker(): updating marker marker_oid=datalog.sync-status.shard.b4cda113-46e3-4662-b22e-ec427f11458c.69 marker=1_1538016826.301943_43113.1
2018-09-27 02:53:47.524510 7efe374ce700 20 cr:s=0x557dce6135e0:op=0x557dce59ec00:13RGWOmapAppend: operate()
2018-09-27 02:53:47.524522 7efe374ce700 20 cr:s=0x557dcdcb55e0:op=0x557dce9db800:21RGWSimpleRadosWriteCRI20rgw_data_sync_markerE: operate()
2018-09-27 02:53:47.524525 7efe374ce700 20 cr:s=0x557dce6135e0:op=0x557dce838600:21RGWRadosSetOmapKeysCR: operate()
2018-09-27 02:53:47.524529 7efe374ce700 20 cr:s=0x557dcdcb55e0:op=0x557dce9db800:21RGWSimpleRadosWriteCRI20rgw_data_sync_markerE: operate()
2018-09-27 02:53:47.524544 7efe374ce700 20 enqueued request req=0x557dcdfeb440
2018-09-27 02:53:47.524547 7efe374ce700 20 RGWWQ:
2018-09-27 02:53:47.524548 7efe374ce700 20 req: 0x557dcdfeb440
2018-09-27 02:53:47.524551 7efe374ce700 20 run: stack=0x557dcdcb55e0 is io blocked
2018-09-27 02:53:47.524552 7efe374ce700 20 cr:s=0x557dce6135e0:op=0x557dce838600:21RGWRadosSetOmapKeysCR: operate()
2018-09-27 02:53:47.524571 7efe414e2700 20 dequeued request req=0x557dcdfeb440
2018-09-27 02:53:47.524576 7efe414e2700 20 RGWWQ: empty
2018-09-27 02:53:47.524634 7efe374ce700 20 run: stack=0x557dce6135e0 is io blocked
2018-09-27 02:53:47.536832 7efe374ce700 20 cr:s=0x557dcdc23dc0:op=0x557dce42aa00:21RGWReadRESTResourceCRISt4listI16rgw_bi_log_entrySaIS1_EEE: operate()
2018-09-27 02:53:47.536898 7efe374ce700 20 cr:s=0x557dcdc23dc0:op=0x557dce42aa00:21RGWReadRESTResourceCRISt4listI16rgw_bi_log_entrySaIS1_EEE: operate()
2018-09-27 02:53:47.536907 7efe374ce700 20 cr:s=0x557dcdc23dc0:op=0x557dce42aa00:21RGWReadRESTResourceCRISt4listI16rgw_bi_log_entrySaIS1_EEE: operate()
2018-09-27 02:53:47.536908 7efe374ce700 20 cr:s=0x557dcdc23dc0:op=0x557dce42aa00:21RGWReadRESTResourceCRISt4listI16rgw_bi_log_entrySaIS1_EEE: operate()
2018-09-27 02:53:47.536911 7efe374ce700 20 cr:s=0x557dcdc23dc0:op=0x557dcde64c00:23RGWListBucketIndexLogCR: operate()
2018-09-27 02:53:47.536913 7efe374ce700 20 cr:s=0x557dcdc23dc0:op=0x557dce7a0800:31RGWBucketShardIncrementalSyncCR: operate()
2018-09-27 02:53:47.536915 7efe374ce700 20 run: stack=0x557dcdc23dc0 is_blocked_by_stack()=0 is_sleeping=0 waiting_for_child()=1
2018-09-27 02:53:47.549960 7efe324c4700 10 receive_http_header
2018-09-27 02:53:47.549973 7efe324c4700 10 received header:HTTP/1.1 400 Bad Request
2018-09-27 02:53:47.549986 7efe324c4700 10 receive_http_header
2018-09-27 02:53:47.549987 7efe324c4700 10 received header:Warning: 299 Elasticsearch-5.6.12-cfe3d9f "Content type detection for rest requests is deprecated. Specify the content type using the [Content-Type] header." "Thu, 27 Sep 2018 02:53:47 GMT" 
2018-09-27 02:53:47.549995 7efe324c4700 10 receive_http_header
2018-09-27 02:53:47.549995 7efe324c4700 10 received header:content-type: application/json; charset=UTF-8
2018-09-27 02:53:47.549998 7efe324c4700 10 receive_http_header
2018-09-27 02:53:47.549998 7efe324c4700 10 received header:content-length: 373
2018-09-27 02:53:47.550000 7efe324c4700 10 receive_http_header
2018-09-27 02:53:47.550001 7efe324c4700 10 received header:
2018-09-27 02:53:47.550038 7efe374ce700 20 cr:s=0x557dcde8c540:op=0x557dce71d800:20RGWPutRESTResourceCRI15es_obj_metadataiE: operate()
2018-09-27 02:53:47.550063 7efe374ce700  5 failed to wait for op, ret=-22: PUT http://192.168.15.11:9200/rgw-gold-6dcfa3f5/object/b4cda113-46e3-4662-b22e-ec427f11458c.4205.1%3Atest_from_nfs10%3Anull
2018-09-27 02:53:47.550141 7efe374ce700 20 cr:s=0x557dcde8c540:op=0x557dce71d800:20RGWPutRESTResourceCRI15es_obj_metadataiE: operate() returned r=-22
2018-09-27 02:53:47.550164 7efe374ce700 20 cr:s=0x557dcde8c540:op=0x557dce72b800:29RGWElasticHandleRemoteObjCBCR: operate()
2018-09-27 02:53:47.550165 7efe374ce700 20 cr:s=0x557dcde8c540:op=0x557dce72b800:29RGWElasticHandleRemoteObjCBCR: operate() returned r=-22
2018-09-27 02:53:47.550167 7efe374ce700 20 cr:s=0x557dcde8c540:op=0x557dce72b000:27RGWElasticHandleRemoteObjCR: operate()
2018-09-27 02:53:47.550168 7efe374ce700  0 RGWStatRemoteObjCR() callback returned -22
2018-09-27 02:53:47.550169 7efe374ce700 20 cr:s=0x557dcde8c540:op=0x557dce72b000:27RGWElasticHandleRemoteObjCR: operate() returned r=-22
2018-09-27 02:53:47.550171 7efe374ce700 20 cr:s=0x557dcde8c540:op=0x557dce7a1000:26RGWBucketSyncSingleEntryCRISs11rgw_obj_keyE: operate()
2018-09-27 02:53:47.550174 7efe374ce700  5 data sync: Sync:b4cda113:data:Object:test:b4cda113-46e3-4662-b22e-ec427f11458c.4205.1/test_from_nfs10[0]:done, retcode=-22
2018-09-27 02:53:47.550184 7efe374ce700  0 data sync: ERROR: failed to sync object: test:b4cda113-46e3-4662-b22e-ec427f11458c.4205.1/test_from_nfs10

elasticsearch log show filed to parse

[2018-09-27T02:53:47,546][DEBUG][o.e.a.b.TransportShardBulkAction] [Q8LOYBG] [rgw-gold-6dcfa3f5][9] failed to execute bulk item (index) BulkShardRequest [[rgw-gold-6dcfa3f5][9]] containing [index {[rgw-gold-6dcfa3f5][object][b4cda113-46e3-4662-b22e-ec427f11458c.4205.1:test_from_nfs10:null], source[{"bucket":"test","name":"test_from_nfs10","instance":"","versioned_epoch":0,"owner":{"id":"admin","display_name":"admin"},"permissions":["admin"],"meta":{"size":1048576,"mtime":"2018-09-27T02:53:46.495Z","etag":"68a1b9ffc7c1aab5f5cd155a0e251dbd","tail_tag":"b4cda113-46e3-4662-b22e-ec427f11458c.134118.0","unix-key1":"\u0002\u0001\u0014\u0000\u0000\u0000\u0019\u001d�?�����4�s�\u0004\u0014�\u0002\u0000\u0000","unix1":"\u0002\u0001D\u0000\u0000\u0000\u0001\u0000\u0000\u0000�I�F�n��\u0000\u0000\u0010\u0000\u0000\u0000\u0000\u0000\u0001\u0000\u0000\u0000\u0000\u0000\u0000\u0000�\u0003\u0000\u0000�\u0003\u0000\u0000��\u0000\u0000:F�[Eӌ\u001d:F�[Eӌ\u001d:F�[\u0018��\u0016\u0002\u0000\u0000"}}]}]
org.elasticsearch.index.mapper.MapperParsingException: failed to parse
    at org.elasticsearch.index.mapper.DocumentParser.wrapInMapperParsingException(DocumentParser.java:176) ~[elasticsearch-5.6.12.jar:5.6.12]
    at org.elasticsearch.index.mapper.DocumentParser.parseDocument(DocumentParser.java:69) ~[elasticsearch-5.6.12.jar:5.6.12]
    at org.elasticsearch.index.mapper.DocumentMapper.parse(DocumentMapper.java:277) ~[elasticsearch-5.6.12.jar:5.6.12]
    at org.elasticsearch.index.shard.IndexShard.prepareIndex(IndexShard.java:530) ~[elasticsearch-5.6.12.jar:5.6.12]
    at org.elasticsearch.index.shard.IndexShard.prepareIndexOnPrimary(IndexShard.java:507) ~[elasticsearch-5.6.12.jar:5.6.12]
    at org.elasticsearch.action.bulk.TransportShardBulkAction.prepareIndexOperationOnPrimary(TransportShardBulkAction.java:458) ~[elasticsearch-5.6.12.jar:5.6.12]
    at org.elasticsearch.action.bulk.TransportShardBulkAction.executeIndexRequestOnPrimary(TransportShardBulkAction.java:466) ~[elasticsearch-5.6.12.jar:5.6.12]
    at org.elasticsearch.action.bulk.TransportShardBulkAction.executeBulkItemRequest(TransportShardBulkAction.java:145) [elasticsearch-5.6.12.jar:5.6.12]
    at org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:114) [elasticsearch-5.6.12.jar:5.6.12]
    at org.elasticsearch.action.bulk.TransportShardBulkAction.shardOperationOnPrimary(TransportShardBulkAction.java:69) [elasticsearch-5.6.12.jar:5.6.12]
    at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryShardReference.perform(TransportReplicationAction.java:975) [elasticsearch-5.6.12.jar:5.6.12]
    at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryShardReference.perform(TransportReplicationAction.java:944) [elasticsearch-5.6.12.jar:5.6.12]
    at org.elasticsearch.action.support.replication.ReplicationOperation.execute(ReplicationOperation.java:113) [elasticsearch-5.6.12.jar:5.6.12]
    at org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncPrimaryAction.onResponse(TransportReplicationAction.java:345) [elasticsearch-5.6.12.jar:5.6.12]
    at org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncPrimaryAction.onResponse(TransportReplicationAction.java:270) [elasticsearch-5.6.12.jar:5.6.12]
    at org.elasticsearch.action.support.replication.TransportReplicationAction$1.onResponse(TransportReplicationAction.java:924) [elasticsearch-5.6.12.jar:5.6.12]
    at org.elasticsearch.action.support.replication.TransportReplicationAction$1.onResponse(TransportReplicationAction.java:921) [elasticsearch-5.6.12.jar:5.6.12]
    at org.elasticsearch.index.shard.IndexShardOperationsLock.acquire(IndexShardOperationsLock.java:151) [elasticsearch-5.6.12.jar:5.6.12]
    at org.elasticsearch.index.shard.IndexShard.acquirePrimaryOperationLock(IndexShard.java:1659) [elasticsearch-5.6.12.jar:5.6.12]
    at org.elasticsearch.action.support.replication.TransportReplicationAction.acquirePrimaryShardReference(TransportReplicationAction.java:933) [elasticsearch-5.6.12.jar:5.6.12]
    at org.elasticsearch.action.support.replication.TransportReplicationAction.access$500(TransportReplicationAction.java:92) [elasticsearch-5.6.12.jar:5.6.12]
    at org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncPrimaryAction.doRun(TransportReplicationAction.java:291) [elasticsearch-5.6.12.jar:5.6.12]
    at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-5.6.12.jar:5.6.12]
    at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:266) [elasticsearch-5.6.12.jar:5.6.12]
    at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:248) [elasticsearch-5.6.12.jar:5.6.12]
    at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:69) [elasticsearch-5.6.12.jar:5.6.12]
    at org.elasticsearch.transport.TransportService$7.doRun(TransportService.java:662) [elasticsearch-5.6.12.jar:5.6.12]
    at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:675) [elasticsearch-5.6.12.jar:5.6.12]
    at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-5.6.12.jar:5.6.12]
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_181]
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_181]
    at java.lang.Thread.run(Thread.java:748) [?:1.8.0_181]
Caused by: com.fasterxml.jackson.core.JsonParseException: Invalid UTF-8 start byte 0x87
 at [Source: org.elasticsearch.common.bytes.BytesReference$MarkSupportingStreamInputWrapper@1839db50; line: 1, column: 368]
    at com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:1702) ~[jackson-core-2.8.6.jar:2.8.6]
    at com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:558) ~[jackson-core-2.8.6.jar:2.8.6]
    at com.fasterxml.jackson.core.json.UTF8StreamJsonParser._reportInvalidInitial(UTF8StreamJsonParser.java:3544) ~[jackson-core-2.8.6.jar:2.8.6]
    at com.fasterxml.jackson.core.json.UTF8StreamJsonParser._reportInvalidChar(UTF8StreamJsonParser.java:3538) ~[jackson-core-2.8.6.jar:2.8.6]
    at com.fasterxml.jackson.core.json.UTF8StreamJsonParser._finishString2(UTF8StreamJsonParser.java:2543) ~[jackson-core-2.8.6.jar:2.8.6]
    at com.fasterxml.jackson.core.json.UTF8StreamJsonParser._finishAndReturnString(UTF8StreamJsonParser.java:2469) ~[jackson-core-2.8.6.jar:2.8.6]
    at com.fasterxml.jackson.core.json.UTF8StreamJsonParser.getText(UTF8StreamJsonParser.java:315) ~[jackson-core-2.8.6.jar:2.8.6]
    at org.elasticsearch.common.xcontent.json.JsonXContentParser.text(JsonXContentParser.java:86) ~[elasticsearch-5.6.12.jar:5.6.12]
    at org.elasticsearch.index.mapper.DocumentParser.createBuilderFromDynamicValue(DocumentParser.java:699) ~[elasticsearch-5.6.12.jar:5.6.12]
    at org.elasticsearch.index.mapper.DocumentParser.parseDynamicValue(DocumentParser.java:807) ~[elasticsearch-5.6.12.jar:5.6.12]
    at org.elasticsearch.index.mapper.DocumentParser.parseValue(DocumentParser.java:598) ~[elasticsearch-5.6.12.jar:5.6.12]
    at org.elasticsearch.index.mapper.DocumentParser.innerParseObject(DocumentParser.java:396) ~[elasticsearch-5.6.12.jar:5.6.12]
    at org.elasticsearch.index.mapper.DocumentParser.parseObjectOrNested(DocumentParser.java:373) ~[elasticsearch-5.6.12.jar:5.6.12]
    at org.elasticsearch.index.mapper.DocumentParser.parseObjectOrField(DocumentParser.java:465) ~[elasticsearch-5.6.12.jar:5.6.12]
    at org.elasticsearch.index.mapper.DocumentParser.parseObject(DocumentParser.java:484) ~[elasticsearch-5.6.12.jar:5.6.12]
    at org.elasticsearch.index.mapper.DocumentParser.innerParseObject(DocumentParser.java:383) ~[elasticsearch-5.6.12.jar:5.6.12]
    at org.elasticsearch.index.mapper.DocumentParser.parseObjectOrNested(DocumentParser.java:373) ~[elasticsearch-5.6.12.jar:5.6.12]
    at org.elasticsearch.index.mapper.DocumentParser.internalParseDocument(DocumentParser.java:93) ~[elasticsearch-5.6.12.jar:5.6.12]
    at org.elasticsearch.index.mapper.DocumentParser.parseDocument(DocumentParser.java:66) ~[elasticsearch-5.6.12.jar:5.6.12]
    ... 30 more