[ES] ES FAQ

FAQ for ElasticSearch

Q: 写入失败，字段数超出1000个的限制

问题描述

...
2024-08-26 10:37:06,775 WARN  org.apache.flink.runtime.taskmanager.Task                    [] - Sink: 设备历史状态写入ES (2/2)#5 (0c9d2bb8575b51dced4ba167a09ec08a) switched from RUNNING to FAILED.
org.elasticsearch.ElasticsearchStatusException: Elasticsearch exception [type=illegal_argument_exception, reason=Limit of total fields [1000] in index [device_status_snapshot_history_202408] has been exceeded]
	at org.elasticsearch.rest.BytesRestResponse.errorFromXContent(BytesRestResponse.java:187) ~[blob_p-ba3774ed58ac44003b7e38716877c41d2a9a31f5-2f4e02b4169d9934c84e916e8cbb11b7:?]
	at org.elasticsearch.client.RestHighLevelClient.parseEntity(RestHighLevelClient.java:1907) ~[blob_p-ba3774ed58ac44003b7e38716877c41d2a9a31f5-2f4e02b4169d9934c84e916e8cbb11b7:?]
	at org.elasticsearch.client.RestHighLevelClient.parseResponseException(RestHighLevelClient.java:1884) ~[blob_p-ba3774ed58ac44003b7e38716877c41d2a9a31f5-2f4e02b4169d9934c84e916e8cbb11b7:?]
	at org.elasticsearch.client.RestHighLevelClient.internalPerformRequest(RestHighLevelClient.java:1641) ~[blob_p-ba3774ed58ac44003b7e38716877c41d2a9a31f5-2f4e02b4169d9934c84e916e8cbb11b7:?]
	at org.elasticsearch.client.RestHighLevelClient.performRequest(RestHighLevelClient.java:1598) ~[blob_p-ba3774ed58ac44003b7e38716877c41d2a9a31f5-2f4e02b4169d9934c84e916e8cbb11b7:?]
	at org.elasticsearch.client.RestHighLevelClient.performRequestAndParseEntity(RestHighLevelClient.java:1568) ~[blob_p-ba3774ed58ac44003b7e38716877c41d2a9a31f5-2f4e02b4169d9934c84e916e8cbb11b7:?]
	at org.elasticsearch.client.RestHighLevelClient.index(RestHighLevelClient.java:985) ~[blob_p-ba3774ed58ac44003b7e38716877c41d2a9a31f5-2f4e02b4169d9934c84e916e8cbb11b7:?]
	at com.xxx.sink.DeviceStatusEsHistorySinkFunction.invoke(DeviceStatusEsHistorySinkFunction.java:87) ~[blob_p-ba3774ed58ac44003b7e38716877c41d2a9a31f5-2f4e02b4169d9934c84e916e8cbb11b7:?]
	at com.xxx.sink.DeviceStatusEsHistorySinkFunction.invoke(DeviceStatusEsHistorySinkFunction.java:25) ~[blob_p-ba3774ed58ac44003b7e38716877c41d2a9a31f5-2f4e02b4169d9934c84e916e8cbb11b7:?]
	at org.apache.flink.streaming.api.operators.StreamSink.processElement(StreamSink.java:57) ~[flink-dist_2.11-1.12.2-h0.cbu.dli.233.r28.jar:1.12.2-h0.cbu.dli.233.r28]
	at org.apache.flink.streaming.runtime.tasks.OneInputStreamTask$StreamTaskNetworkOutput.emitRecord(OneInputStreamTask.java:191) ~[flink-dist_2.11-1.12.2-h0.cbu.dli.233.r28.jar:1.12.2-h0.cbu.dli.233.r28]
	at org.apache.flink.streaming.runtime.io.StreamTaskNetworkInput.processElement(StreamTaskNetworkInput.java:204) ~[flink-dist_2.11-1.12.2-h0.cbu.dli.233.r28.jar:1.12.2-h0.cbu.dli.233.r28]
	at org.apache.flink.streaming.runtime.io.StreamTaskNetworkInput.emitNext(StreamTaskNetworkInput.java:174) ~[flink-dist_2.11-1.12.2-h0.cbu.dli.233.r28.jar:1.12.2-h0.cbu.dli.233.r28]
	at org.apache.flink.streaming.runtime.io.StreamOneInputProcessor.processInput(StreamOneInputProcessor.java:65) ~[flink-dist_2.11-1.12.2-h0.cbu.dli.233.r28.jar:1.12.2-h0.cbu.dli.233.r28]
	at org.apache.flink.streaming.runtime.tasks.StreamTask.processInput(StreamTask.java:412) ~[flink-dist_2.11-1.12.2-h0.cbu.dli.233.r28.jar:1.12.2-h0.cbu.dli.233.r28]
	at org.apache.flink.streaming.runtime.tasks.mailbox.MailboxProcessor.runMailboxLoop(MailboxProcessor.java:199) ~[flink-dist_2.11-1.12.2-h0.cbu.dli.233.r28.jar:1.12.2-h0.cbu.dli.233.r28]
	at org.apache.flink.streaming.runtime.tasks.StreamTask.runMailboxLoop(StreamTask.java:634) ~[flink-dist_2.11-1.12.2-h0.cbu.dli.233.r28.jar:1.12.2-h0.cbu.dli.233.r28]
	at org.apache.flink.streaming.runtime.tasks.StreamTask.invoke(StreamTask.java:598) ~[flink-dist_2.11-1.12.2-h0.cbu.dli.233.r28.jar:1.12.2-h0.cbu.dli.233.r28]
	at org.apache.flink.runtime.taskmanager.Task.doRun(Task.java:765) [flink-dist_2.11-1.12.2-h0.cbu.dli.233.r28.jar:1.12.2-h0.cbu.dli.233.r28]
	at org.apache.flink.runtime.taskmanager.Task.run(Task.java:580) [flink-dist_2.11-1.12.2-h0.cbu.dli.233.r28.jar:1.12.2-h0.cbu.dli.233.r28]
	at java.lang.Thread.run(Thread.java:750) [?:1.8.0_372]
	Suppressed: org.elasticsearch.client.ResponseException: method [POST], host [http://10.37.19.116:9200], URI [/device_status_snapshot_history_202408/_doc?timeout=1m], status line [HTTP/1.1 400 Bad Request]
{"error":{"root_cause":[{"type":"illegal_argument_exception","reason":"Limit of total fields [1000] in index [device_status_snapshot_history_202408] has been exceeded"}],"type":"illegal_argument_exception","reason":"Limit of total fields [1000] in index [device_status_snapshot_history_202408] has been exceeded"},"status":400}
		at org.elasticsearch.client.RestClient.convertResponse(RestClient.java:318) ~[blob_p-ba3774ed58ac44003b7e38716877c41d2a9a31f5-2f4e02b4169d9934c84e916e8cbb11b7:?]
		at org.elasticsearch.client.RestClient.performRequest(RestClient.java:288) ~[blob_p-ba3774ed58ac44003b7e38716877c41d2a9a31f5-2f4e02b4169d9934c84e916e8cbb11b7:?]
		at org.elasticsearch.client.RestClient.performRequest(RestClient.java:262) ~[blob_p-ba3774ed58ac44003b7e38716877c41d2a9a31f5-2f4e02b4169d9934c84e916e8cbb11b7:?]
		at org.elasticsearch.client.RestHighLevelClient.internalPerformRequest(RestHighLevelClient.java:1628) ~[blob_p-ba3774ed58ac44003b7e38716877c41d2a9a31f5-2f4e02b4169d9934c84e916e8cbb11b7:?]
		at org.elasticsearch.client.RestHighLevelClient.performRequest(RestHighLevelClient.java:1598) ~[blob_p-ba3774ed58ac44003b7e38716877c41d2a9a31f5-2f4e02b4169d9934c84e916e8cbb11b7:?]
		at org.elasticsearch.client.RestHighLevelClient.performRequestAndParseEntity(RestHighLevelClient.java:1568) ~[blob_p-ba3774ed58ac44003b7e38716877c41d2a9a31f5-2f4e02b4169d9934c84e916e8cbb11b7:?]
		at org.elasticsearch.client.RestHighLevelClient.index(RestHighLevelClient.java:985) ~[blob_p-ba3774ed58ac44003b7e38716877c41d2a9a31f5-2f4e02b4169d9934c84e916e8cbb11b7:?]
		at com.xxx.sink.DeviceStatusEsHistorySinkFunction.invoke(DeviceStatusEsHistorySinkFunction.java:87) ~[blob_p-ba3774ed58ac44003b7e38716877c41d2a9a31f5-2f4e02b4169d9934c84e916e8cbb11b7:?]
		at com.xxx.sink.DeviceStatusEsHistorySinkFunction.invoke(DeviceStatusEsHistorySinkFunction.java:25) ~[blob_p-ba3774ed58ac44003b7e38716877c41d2a9a31f5-2f4e02b4169d9934c84e916e8cbb11b7:?]
		at org.apache.flink.streaming.api.operators.StreamSink.processElement(StreamSink.java:57) ~[flink-dist_2.11-1.12.2-h0.cbu.dli.233.r28.jar:1.12.2-h0.cbu.dli.233.r28]
		at org.apache.flink.streaming.runtime.tasks.OneInputStreamTask$StreamTaskNetworkOutput.emitRecord(OneInputStreamTask.java:191) ~[flink-dist_2.11-1.12.2-h0.cbu.dli.233.r28.jar:1.12.2-h0.cbu.dli.233.r28]
		at org.apache.flink.streaming.runtime.io.StreamTaskNetworkInput.processElement(StreamTaskNetworkInput.java:204) ~[flink-dist_2.11-1.12.2-h0.cbu.dli.233.r28.jar:1.12.2-h0.cbu.dli.233.r28]
		at org.apache.flink.streaming.runtime.io.StreamTaskNetworkInput.emitNext(StreamTaskNetworkInput.java:174) ~[flink-dist_2.11-1.12.2-h0.cbu.dli.233.r28.jar:1.12.2-h0.cbu.dli.233.r28]
		at org.apache.flink.streaming.runtime.io.StreamOneInputProcessor.processInput(StreamOneInputProcessor.java:65) ~[flink-dist_2.11-1.12.2-h0.cbu.dli.233.r28.jar:1.12.2-h0.cbu.dli.233.r28]
		at org.apache.flink.streaming.runtime.tasks.StreamTask.processInput(StreamTask.java:412) ~[flink-dist_2.11-1.12.2-h0.cbu.dli.233.r28.jar:1.12.2-h0.cbu.dli.233.r28]
		at org.apache.flink.streaming.runtime.tasks.mailbox.MailboxProcessor.runMailboxLoop(MailboxProcessor.java:199) ~[flink-dist_2.11-1.12.2-h0.cbu.dli.233.r28.jar:1.12.2-h0.cbu.dli.233.r28]
		at org.apache.flink.streaming.runtime.tasks.StreamTask.runMailboxLoop(StreamTask.java:634) ~[flink-dist_2.11-1.12.2-h0.cbu.dli.233.r28.jar:1.12.2-h0.cbu.dli.233.r28]
		at org.apache.flink.streaming.runtime.tasks.StreamTask.invoke(StreamTask.java:598) ~[flink-dist_2.11-1.12.2-h0.cbu.dli.233.r28.jar:1.12.2-h0.cbu.dli.233.r28]
		at org.apache.flink.runtime.taskmanager.Task.doRun(Task.java:765) [flink-dist_2.11-1.12.2-h0.cbu.dli.233.r28.jar:1.12.2-h0.cbu.dli.233.r28]
		at org.apache.flink.runtime.taskmanager.Task.run(Task.java:580) [flink-dist_2.11-1.12.2-h0.cbu.dli.233.r28.jar:1.12.2-h0.cbu.dli.233.r28]
		at java.lang.Thread.run(Thread.java:750) [?:1.8.0_372]
2024-08-26 10:37:06,776 WARN  org.apache.flink.runtime.taskmanager.Task                    [] - Call stack:
    at java.lang.Thread.getStackTrace(Thread.java:1564)
    at org.apache.flink.runtime.taskmanager.Task.transitionState(Task.java:1065)
    at org.apache.flink.runtime.taskmanager.Task.doRun(Task.java:837)
    at org.apache.flink.runtime.taskmanager.Task.run(Task.java:580)
    at java.lang.Thread.run(Thread.java:750)
...

问题分析&解决方法

方法1：放大字段数的限制

• 设置索引的最大字段数为 2000

PUT {myIndexName}/_settings
{
	"index.mapping.total_fields.limit": 2000
} 

设置成功的响应:

{
  "acknowledged" : true
}

查看是否设置成功

GET {myIndexName}/_settings

X 参考文献

• 添加es索引出现异常:Limit of total fields [1000] in index [issue] has been exceeded - 博客园

Q: 基于curl命令的shell，实现对ES的网络连通性、可用性、用户认证、数据查询的检查

apk add openssh

HOST=192.167.12.34
PORT=9200

# USER=admin
# PASSWORD="xxxxx"
# 不推荐此方式↓（linux的base64，部分字符的编码不可靠，尚未找到原因）
# AUTH_BASE64=$(echo "$USER:$PASSWORD" | base64)

AUTH_BASE64="YWRtaW46aGVsbG8="
echo $AUTH_BASE64

echo '[1] 探测网络连通性:'
ssh $HOST -p $PORT -v

echo '[2] 测验：可用性健康检查、用户认证、数据查询(基于 opendistro 插件):'
curl --location "http://$HOST:$PORT/_opendistro/_sql?format=json" \
--header 'Content-Type: application/json' \
--header "Authorization: Basic $AUTH_BASE64" \
--data '{
    "query": "SELECT * FROM dwd_device_status where deviceUuid = '\''xxxxxx'\''"
}'

out

{"took":2,"timed_out":false,"_shards":{"total":3,"successful":3,"skipped":0,"failed":0},"hits":{"total":{"value":0,"relation":"eq"},"max_score":null,"hits":[]}}

X 参考文献