k8s离线部署Elasticsearch集群
1.Elasticsearch集群介绍
Elasticsearch集群通过分布式架构实现高可用性、水平扩展和自动容错,支持海量数据的实时高效处理与搜索。
2.部署环境
| IP | 节点 | 操作系统 | k8s版本 |
elasticsearch版本 |
docker版本 |
| 172.16.4.85 | master1 | centos7.8 | 1.23.17 | 20.10.9 | |
| 172.16.4.86 | node1 | centos7.8 | 1.23.17 | 7.1.1 | 20.10.9 |
| 172.16.4.87 | node2 | centos7.8 | 1.23.17 | 7.1.1 | 20.10.9 |
| 172.16.4.89 | node3 | centos7.8 | 1.23.17 | 20.10.9 | |
| 172.16.4.90 | node4 | centos7.8 | 1.23.17 | 7.1.1 | 20.10.9 |
3.Elasticsearch集群部署
3.1 nfs部署
- centos7安装nfs
yum install -y nfs-utils
- 创建nfs共享目录
mkdir -p /nfs_share/k8s/es/pv{1..3}
chmod 777 /nfs_share/k8s/es/pv{1..3} -R
- nfs配置文件编辑
[root@localhost minio]# cat /etc/exports
/nfs_share/k8s/es/pv1 *(rw,sync,no_subtree_check,no_root_squash)
/nfs_share/k8s/es/pv2 *(rw,sync,no_subtree_check,no_root_squash)
/nfs_share/k8s/es/pv3 *(rw,sync,no_subtree_check,no_root_squash)
- 启动nfs服务
# 启动 NFS 服务
systemctl start nfs-server
# 设置 NFS 服务在系统启动时自动启动
systemctl enable nfs-server
- 加载配置文件,并输出
[root@localhost es]# exportfs -r
[root@localhost es]# exportfs -v
/nfs_share/k8s/es/pv1
<world>(sync,wdelay,hide,no_subtree_check,sec=sys,rw,secure,no_root_squash,no_all_squash)
/nfs_share/k8s/es/pv2
<world>(sync,wdelay,hide,no_subtree_check,sec=sys,rw,secure,no_root_squash,no_all_squash)
/nfs_share/k8s/es/pv3
<world>(sync,wdelay,hide,no_subtree_check,sec=sys,rw,secure,no_root_squash,no_all_squash)
3.2 宿主机配置
- 一定要在所有的宿主机添加下边配置,否则在启动es集群的时候会报错:ERROR: [1] bootstrap checks failed [1]: max virtual memory areas vm.max_map_count [65530] is too low, increase to at least [262144]
cat /etc/sysctl.conf
# 添加ES配置
vm.max_map_count=262144
执行生效
sysctl -p
3.3 创建namespace
apiVersion: v1
kind: Namespace
metadata:
name: es
kubectl apply -f es-ns.yaml
3.4 elasticsearch部署pv
apiVersion: v1
kind: PersistentVolume
metadata:
name: es-pv1
labels:
type: nfs-es-pv # 用于PVC匹配
spec:
capacity:
storage: 10Gi
accessModes:
- ReadWriteOnce
persistentVolumeReclaimPolicy: Retain
storageClassName: nfs-es-storage
nfs:
server: 172.16.4.60
path: /nfs_share/k8s/es/pv1 # 确保路径已存在且有读写权限
---
apiVersion: v1
kind: PersistentVolume
metadata:
name: es-pv2
labels:
type: nfs-es-pv
spec:
capacity:
storage: 10Gi
accessModes:
- ReadWriteOnce
persistentVolumeReclaimPolicy: Retain
storageClassName: nfs-es-storage
nfs:
server: 172.16.4.60
path: /nfs_share/k8s/es/pv2
---
apiVersion: v1
kind: PersistentVolume
metadata:
name: es-pv3
labels:
type: nfs-es-pv
spec:
capacity:
storage: 10Gi
accessModes:
- ReadWriteOnce
persistentVolumeReclaimPolicy: Retain
storageClassName: nfs-es-storage
nfs:
server: 172.16.4.60
path: /nfs_share/k8s/es/pv3
kubectl apply -f es-pv.yaml
3.5 elasticsearch部署证书
- 生成证书:使用Elasticsearch 证书工具elasticsearch-certutil生成 TLS 证书
https://www.cnblogs.com/Jeffrey1172417122/p/16718354.html
- 创建 Secret:将证书上传到 Kubernetes,注意引用证书的路径
# 假设证书文件存放在当前目录的 certs/ 文件夹中
kubectl create secret generic elasticsearch-certs -n es \
--from-file=certs/ks-es.key \
--from-file=certs/ks-es.crt \
--from-file=certs/ca.crt
3.6 elasticsearch部署configmap
- k8s的configmap可以参考如下配置文件,这是我之前在宿主机上部署的有证书的集群,也是3台es组成集群,使用nginx做代理。
集群elasticsearch.yml配置
################################ es1 ################################
[root@localhost config]# cat elasticsearch.yml
# 集群名称
cluster.name: escluster
# 集群初始化 选择该节点为主节点 master
cluster.initial_master_nodes : "es1"
# 节点名称
node.name: es1
# 是否允许该节点参加master 选举
node.master: true
# 允许该节点存储数据(默认开启)
node.data: true
# 默认情况下,ElasticSearch使用0.0.0.0地址,并为http传输开启9200-9300端口,为节点到节点的通信开启9300-9400端口,也可以自行设置IP地址
network.host: 0.0.0.0
# publish_host设置其他节点连接此节点的地址,如果不设置的话,则自动获取,publish_host的地址必须为真实地址
network.publish_host: 172.17.22.65
# 该节点与其他节点交互的端口
transport.tcp.port: 29300
# Http传输监听端口
http.port: 29200
# 是否支持跨域,默认为false
http.cors.enabled: true
# 当设置允许跨域,默认为*,表示支持所有域名
http.cors.allow-origin: "*"
# 跨域允许设置的头信息
http.cors.allow-headers: Authorization
# 节点内部通信地址 9200 是默认端口 ,如有修改则补全端口号
discovery.zen.ping.unicast.hosts: ["172.17.22.65:29300","172.17.22.66:29300","172.17.22.67:29300"]
discovery.zen.minimum_master_nodes: 2
discovery.zen.ping_timeout: 5s
#cluster.initial_master_nodes: ["192.168.1.240:29300","192.168.1.125:29300","192.168.1.141:29300"]
# X-Pack是Elastic Stack扩展功能,提供安全性,警报,监视,报告,机器学习和许多其他功能 】
# 默认为true,启用节点上ES的XPACK安全功能,相当于总开关
#xpack.security.enabled: true
# 传输层的认证设置
#xpack.security.transport.ssl.enabled: true
#xpack.security.transport.ssl.verification_mode: certificate
#xpack.security.transport.ssl.keystore.path: certs/
#xpack.security.transport.ssl.truststore.path: certs/elastic-certificates.p12
xpack.security.transport.ssl.enabled: true
xpack.security.enabled: true
xpack.security.transport.ssl.verification_mode: certificate
xpack.security.transport.ssl.key: certs/ks-es.key
xpack.security.transport.ssl.certificate: certs/ks-es.crt
xpack.security.transport.ssl.certificate_authorities: certs/ca.crt
# 最大子查询个数 ,默认1024 目前系统是以相机为主要检索 如果超过 1024个相机 将会检索不到结果
indices.query.bool.max_clause_count: 10240
################################ es2 ################################
[root@localhost config]# cat elasticsearch.yml
# 集群名称
cluster.name: escluster
# 集群初始化 选择该节点为主节点 master
#cluster.initial_master_nodes : "es1"
# 节点名称
node.name: es2
# 是否允许该节点参加master 选举
node.master: true
# 允许该节点存储数据(默认开启)
node.data: true
# 默认情况下,ElasticSearch使用0.0.0.0地址,并为http传输开启9200-9300端口,为节点到节点的通信开启9300-9400端口,也可以自行设置IP地址
network.host: 0.0.0.0
# publish_host设置其他节点连接此节点的地址,如果不设置的话,则自动获取,publish_host的地址必须为真实地址
network.publish_host: 172.17.22.66
# 该节点与其他节点交互的端口
transport.tcp.port: 29300
# Http传输监听端口
http.port: 29200
# 是否支持跨域,默认为false
http.cors.enabled: true
# 当设置允许跨域,默认为*,表示支持所有域名
http.cors.allow-origin: "*"
# 跨域允许设置的头信息
http.cors.allow-headers: Authorization
# 节点内部通信地址 9200 是默认端口 ,如有修改则补全端口号
discovery.zen.ping.unicast.hosts: ["172.17.22.65:29300","172.17.22.66:29300","172.17.22.67:29300"]
discovery.zen.minimum_master_nodes: 2
discovery.zen.ping_timeout: 5s
#cluster.initial_master_nodes: ["192.168.1.240:29300","192.168.1.125:29300","192.168.1.141:29300"]
# X-Pack是Elastic Stack扩展功能,提供安全性,警报,监视,报告,机器学习和许多其他功能 】
# 默认为true,启用节点上ES的XPACK安全功能,相当于总开关
#xpack.security.enabled: true
# 传输层的认证设置
#xpack.security.transport.ssl.enabled: true
#xpack.security.transport.ssl.verification_mode: certificate
#xpack.security.transport.ssl.keystore.path: certs/
#xpack.security.transport.ssl.truststore.path: certs/elastic-certificates.p12
xpack.security.transport.ssl.enabled: true
xpack.security.enabled: true
xpack.security.transport.ssl.verification_mode: certificate
xpack.security.transport.ssl.key: certs/ks-es.key
xpack.security.transport.ssl.certificate: certs/ks-es.crt
xpack.security.transport.ssl.certificate_authorities: certs/ca.crt
# 最大子查询个数 ,默认1024 目前系统是以相机为主要检索 如果超过 1024个相机 将会检索不到结果
indices.query.bool.max_clause_count: 10240
################################ es3 ################################
[root@localhost config]# cat elasticsearch.yml
# 集群名称
cluster.name: escluster
# 集群初始化 选择该节点为主节点 master
#cluster.initial_master_nodes : "es1"
# 节点名称
node.name: es3
# 是否允许该节点参加master 选举
node.master: true
# 允许该节点存储数据(默认开启)
node.data: true
# 默认情况下,ElasticSearch使用0.0.0.0地址,并为http传输开启9200-9300端口,为节点到节点的通信开启9300-9400端口,也可以自行设置IP地址
network.host: 0.0.0.0
# publish_host设置其他节点连接此节点的地址,如果不设置的话,则自动获取,publish_host的地址必须为真实地址
network.publish_host: 172.17.22.67
# 该节点与其他节点交互的端口
transport.tcp.port: 29300
# Http传输监听端口
http.port: 29200
# 是否支持跨域,默认为false
http.cors.enabled: true
# 当设置允许跨域,默认为*,表示支持所有域名
http.cors.allow-origin: "*"
# 跨域允许设置的头信息
http.cors.allow-headers: Authorization
# 节点内部通信地址 9200 是默认端口 ,如有修改则补全端口号
discovery.zen.ping.unicast.hosts: ["172.17.22.65:29300","172.17.22.66:29300","172.17.22.67:29300"]
discovery.zen.minimum_master_nodes: 2
discovery.zen.ping_timeout: 5s
#cluster.initial_master_nodes: ["192.168.1.240:29300","192.168.1.125:29300","192.168.1.141:29300"]
# X-Pack是Elastic Stack扩展功能,提供安全性,警报,监视,报告,机器学习和许多其他功能 】
# 默认为true,启用节点上ES的XPACK安全功能,相当于总开关
#xpack.security.enabled: true
# 传输层的认证设置
#xpack.security.transport.ssl.enabled: true
#xpack.security.transport.ssl.verification_mode: certificate
#xpack.security.transport.ssl.keystore.path: certs/
#xpack.security.transport.ssl.truststore.path: certs/elastic-certificates.p12
xpack.security.transport.ssl.enabled: true
xpack.security.enabled: true
xpack.security.transport.ssl.verification_mode: certificate
xpack.security.transport.ssl.key: certs/ks-es.key
xpack.security.transport.ssl.certificate: certs/ks-es.crt
xpack.security.transport.ssl.certificate_authorities: certs/ca.crt
# 最大子查询个数 ,默认1024 目前系统是以相机为主要检索 如果超过 1024个相机 将会检索不到结果
indices.query.bool.max_clause_count: 10240
- nginx.conf配置代理
nginx代理es集群配置
#cat nginx.conf
http {
upstream es {
server 172.17.22.65:29200; # es1
server 172.17.22.66:29200; # es2
server 172.17.22.67:29200; # es3
}
server {
listen 29201;
server_name localhost;
client_body_timeout 5s;
client_header_timeout 5s;
send_timeout 30s;
location / {
proxy_pass http://es;
proxy_buffers 16 1024k;
proxy_busy_buffers_size 2048k;
proxy_temp_file_write_size 2048k;
client_max_body_size 4000m;
proxy_set_header Host $host:80;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
error_page 500 502 503 504 /50x.html;
location = /50x.html {
root html;
}
}
}
- elasticsearch configmap,需要注意cm中的证书路径和sts挂载保持一致
apiVersion: v1
kind: ConfigMap
metadata:
name: elasticsearch-config
namespace: es
labels:
app: elasticsearch
data:
elasticsearch.yml: |-
cluster.name: es-production
node.name: ${HOSTNAME}
network.host: 0.0.0.0
http.port: 29200
transport.port: 29300
http.cors.enabled: true
http.cors.allow-origin: "*"
discovery.seed_hosts:
- "es-cluster-0.es-cluster.es.svc.cluster.local:29300"
- "es-cluster-1.es-cluster.es.svc.cluster.local:29300"
- "es-cluster-2.es-cluster.es.svc.cluster.local:29300"
cluster.initial_master_nodes:
- "es-cluster-0"
- "es-cluster-1"
- "es-cluster-2"
xpack.security.enabled: true
xpack.security.transport.ssl.enabled: true
xpack.security.transport.ssl.verification_mode: certificate
xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/certs/ks-es.key
xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/certs/ks-es.crt
xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/config/certs/ca.crt
indices.query.bool.max_clause_count: 10240
kubectl apply -f es-cm.yaml
3.7 elasticsearch部署headless+statefulset
- 注意:自定义端口、自定义密码、配置MEM/CPU等
- 172.16.4.17:8090/public/elastic:7.1.1镜像是hub.docker.com中找的,然后私有化
- 探针 readinessProbe: 部分的 - name: Authorization 下的 value 说明
#探针内容
readinessProbe:
httpGet:
path: /_cluster/health?local
port: http
scheme: HTTP
httpHeaders:
- name: Authorization
value: "Basic ZWxhc3RpYzpMdHp4RWxhc3RpY0AxNjM4NA=="
# 确定用户名和密码
用户名: elastic(Elasticsearch 默认超级用户)
密码: 123456(由环境变量 ELASTIC_PASSWORD 定义)
组合为字符串:elastic:123456
# 生成 Base64 编码
在 Linux 终端中执行以下命令:
echo -n "elastic:123456" | base64
输出:
ZWxhc3RpYzpMdHp4RWxhc3RpY0AxNjM4NA==
# 更新 Readiness Probe 配置
将生成的 Base64 字符串替换到 readinessProbe 的 value 字段:
- 部署headless和statefulset
apiVersion: v1
kind: Service
metadata:
name: es-cluster
namespace: es
labels:
app: elasticsearch
spec:
clusterIP: None
ports:
- name: http
port: 29200
targetPort: 29200
- name: transport
port: 29300
targetPort: 29300
selector:
app: elasticsearch
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: es-cluster
namespace: es
spec:
serviceName: es-cluster
replicas: 3
updateStrategy:
type: RollingUpdate
podManagementPolicy: Parallel
selector:
matchLabels:
app: elasticsearch
template:
metadata:
labels:
app: elasticsearch
spec:
terminationGracePeriodSeconds: 120
securityContext:
runAsUser: 1000
fsGroup: 1000
fsGroupChangePolicy: "OnRootMismatch"
affinity:
podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchExpressions:
- key: app
operator: In
values: ["elasticsearch"]
topologyKey: "kubernetes.io/hostname"
containers:
- name: elasticsearch
image: 172.16.4.17:8090/public/elastic:7.1.1
securityContext:
capabilities:
add: ["SYS_RESOURCE"]
volumeMounts:
- name: elasticsearch-config
mountPath: /usr/share/elasticsearch/config/elasticsearch.yml
subPath: elasticsearch.yml
- name: data
mountPath: /usr/share/elasticsearch/data
- name: certs
mountPath: /usr/share/elasticsearch/config/certs
readOnly: true
env:
- name: ES_JAVA_OPTS
value: "-Xms4g -Xmx4g -Dlog4j2.formatMsgNoLookups=true -Des.bootstrap.memory_lock=true"
- name: ELASTIC_PASSWORD
value: "123456"
ports:
- containerPort: 29200
name: http
- containerPort: 29300
name: transport
resources:
requests:
memory: "5Gi"
cpu: "2"
limits:
memory: "5Gi"
cpu: "4"
livenessProbe:
exec:
command:
- /bin/bash
- -c
- |
curl -sS --max-time 5 -u elastic:123456 http://localhost:29200/_cluster/health || exit 1
initialDelaySeconds: 300
periodSeconds: 20
readinessProbe:
httpGet:
path: /_cluster/health?local
port: http
scheme: HTTP
httpHeaders:
- name: Authorization
value: "Basic ZWxhc3RpYzp5dHhAMTIzNA=="
initialDelaySeconds: 120
timeoutSeconds: 10
volumes:
- name: elasticsearch-config
configMap:
name: elasticsearch-config
items:
- key: elasticsearch.yml
path: elasticsearch.yml
- name: certs
secret:
secretName: elasticsearch-certs
volumeClaimTemplates:
- metadata:
name: data
spec:
storageClassName: nfs-es-storage
accessModes: [ "ReadWriteOnce" ]
resources:
requests:
storage: 10Gi
kubectl apply -f es-ss.yaml
3.8 elasticsearch部署service
- 有负载均衡能力,替代nginx
apiVersion: v1
kind: Service
metadata:
name: es-nodeport # NodePort 服务名称
namespace: es
spec:
type: NodePort
ports:
- name: http
port: 29200 # Service 端口(集群内访问端口)
targetPort: 29200 # 容器暴露的 HTTP 端口
nodePort: 30920 # 节点外部端口(范围 30000-32767)
selector:
app: elasticsearch # 与 StatefulSet 的 Pod 标签一致
kubectl apply -f es-svc.yaml
3.9 elasticsearch状态查看
[root@master1 es]# kubectl get pv | grep es-cluster
es-pv1 10Gi RWO Retain Bound es/data-es-cluster-1 nfs-es-storage 3h2m
es-pv2 10Gi RWO Retain Bound es/data-es-cluster-2 nfs-es-storage 3h2m
es-pv3 10Gi RWO Retain Bound es/data-es-cluster-0 nfs-es-storage 3h2m
[root@master1 es]# kubectl get pvc -n es
NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE
data-es-cluster-0 Bound es-pv3 10Gi RWO nfs-es-storage 3h1m
data-es-cluster-1 Bound es-pv1 10Gi RWO nfs-es-storage 3h1m
data-es-cluster-2 Bound es-pv2 10Gi RWO nfs-es-storage 3h1m
[root@master1 es]# kubectl get secret -n es
NAME TYPE DATA AGE
default-token-6f7pg kubernetes.io/service-account-token 3 23h
elasticsearch-certs Opaque 3 159m
[root@master1 es]# kubectl get cm -n es
NAME DATA AGE
elasticsearch-config 1 111m
kube-root-ca.crt 1 23h
[root@master1 es]# kubectl get sts -n es
NAME READY AGE
es-cluster 3/3 110m
[root@master1 es]# kubectl get pods -n es -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
es-cluster-0 1/1 Running 0 110m 10.244.3.84 node4 <none> <none>
es-cluster-1 1/1 Running 0 110m 10.244.166.190 node1 <none> <none>
es-cluster-2 1/1 Running 0 110m 10.244.104.22 node2 <none> <none>
[root@master1 es]# kubectl get svc -n es
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
es-cluster ClusterIP None <none> 29200/TCP,29300/TCP 110m
es-nodeport NodePort 10.98.45.191 <none> 29200:30920/TCP 100m
3.10 elasticsearch集群验证
一、查看 Elasticsearch 集群节点状态
#容器内
[root@master1 es]# kubectl exec -n es -it es-cluster-0 -- curl -u elastic:123456 http://es-cluster.es:29200/_cat/nodes?v
ip heap.percent ram.percent cpu load_1m load_5m load_15m node.role master name
10.244.3.84 7 99 6 0.28 0.65 0.77 mdi * es-cluster-0
10.244.104.22 5 99 4 0.08 0.39 0.57 mdi - es-cluster-2
10.244.166.190 6 99 4 0.10 0.40 0.51 mdi - es-cluster-1
#宿主机
[root@master1 es]# curl -u elastic:123456 http://localhost:30920/_cat/nodes?v
ip heap.percent ram.percent cpu load_1m load_5m load_15m node.role master name
10.244.3.84 7 99 6 0.28 0.65 0.77 mdi * es-cluster-0
10.244.104.22 5 99 4 0.08 0.39 0.57 mdi - es-cluster-2
10.244.166.190 6 99 4 0.10 0.40 0.51 mdi - es-cluster-1
二、查看分片分配状态
#查看,新环境没有分片,创建分片
[root@master1 es]# kubectl exec -n es -it es-cluster-0 -- curl -u elastic:123456 http://es-cluster.es:29200/_cat/shards?v
index shard prirep state docs store ip node
#创建分片
[root@master1 es]# kubectl exec -n es -it es-cluster-0 -- curl -XPUT -u elastic:123456 http://es-cluster.es:29200/test_index
{"acknowledged":true,"shards_acknowledged":true,"index":"test_index"}
#再次查看分片状态
[root@master1 es]# kubectl exec -n es -it es-cluster-0 -- curl -u elastic:123456 http://es-cluster.es:29200/_cat/shards?v
index shard prirep state docs store ip node
test_index 0 p STARTED 0 230b 10.244.104.22 es-cluster-2
test_index 0 r STARTED 0 230b 10.244.3.84 es-cluster-0
三、集群外验证集群状态
[root@master1 es]# curl --user elastic:ytx@1234 -XGET http://localhost:30920/_cluster/health?pretty=true
{
"cluster_name" : "es-production",
"status" : "green",
"timed_out" : false,
"number_of_nodes" : 3,
"number_of_data_nodes" : 3,
"active_primary_shards" : 1,
"active_shards" : 2,
"relocating_shards" : 0,
"initializing_shards" : 0,
"unassigned_shards" : 0,
"delayed_unassigned_shards" : 0,
"number_of_pending_tasks" : 0,
"number_of_in_flight_fetch" : 0,
"task_max_waiting_in_queue_millis" : 0,
"active_shards_percent_as_number" : 100.0
}
四、查询索引信息
[root@master1 es]# curl --user elastic:ytx@1234 -XGET http://localhost:30920/_cat/indices?v
health status index uuid pri rep docs.count docs.deleted store.size pri.store.size
green open test_index l8RTxJz2Te-evDhxhHIHNA 1 1 0 0 566b 283b
3.11 elasticsearch集群地址
- 内部集群域名地址
es-nodeport.es.svc.cluster.local 29200
- 外部nodeport地址(svc-nodeport)
10.98.45.191 30920
至此elasticsearch集群就部署完了!!!

浙公网安备 33010602011771号