kubernetes-1.35.3的基于Operator实现Elasticsearch高可用集群
案例：基于 Operator 实现 ECK（ Elastic Cloud on Kubernetes )
注意：Worker 节点内存需要4G以上
官方安装说明
https://www.elastic.co/guide/en/cloud-on-k8s/current/k8s-quickstart.html
https://www.elastic.co/guide/en/cloud-on-k8s/current/k8s-deploy-eck.html
https://www.elastic.co/guide/en/cloud-on-k8s/current/k8s-deployelasticsearch.html
安装 Operator
#安装Operator
kubectl create -f https://download.elastic.co/downloads/eck/3.3.2/crds.yaml
#自动创建相关CRD
#安装operator相关RBAC 规则
kubectl apply -f https://download.elastic.co/downloads/eck/3.3.2/operator.yaml
kubectl apply -f operator.yaml
#在elastic-system 名称空间查看相关资源
kubectl get all -n elastic-system
[root@master1 stateful]# kubectl get pod -n elastic-system NAME READY STATUS RESTARTS AGE elastic-operator-0 1/1 Running 5 (93s ago) 9m36s [root@master1 stateful]# [root@master1 stateful]# kubectl get all -n elastic-system NAME READY STATUS RESTARTS AGE pod/elastic-operator-0 1/1 Running 5 (96s ago) 9m39s NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE service/elastic-webhook-server ClusterIP 10.101.108.45 <none> 443/TCP 9m39s NAME READY AGE statefulset.apps/elastic-operator 1/1 9m39s
部署 Elasticsearch
#准备业务的名称空间
kubectl create ns demo
#准备elasticsearch-cluster清单文件
[root@master1 stateful]# cat operator-elasticsearch-cluster.yaml apiVersion: elasticsearch.k8s.elastic.co/v1 kind: Elasticsearch metadata: name: my-es-cluster #namespace: elastic-system namespace: demo spec: #version: 8.13.2 #version: 8.14.0 version: 9.2.1 nodeSets: - name: default count: 3 #3个节点的集群 config: node.store.allow_mmap: false volumeClaimTemplates: - metadata: name: elasticsearch-data spec: accessModes: ["ReadWriteOnce"] resources: requests: storage: 2Gi storageClassName: sc-nfs #需要提前准备sc-nfs的storageClass
#注意：节点内存需要4G以上
kubectl apply -f operator-elasticsearch-cluster.yaml
#查看结果
下载镜像慢每个节点 docker pull docker.elastic.co/elasticsearch/elasticsearch:9.2.1
pvc pv自动创建，绑定了
root@master1 stateful]# kubectl get pvc -n demo NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS VOLUMEATTRIBUTESCLASS AGE elasticsearch-data-my-es-cluster-es-default-0 Bound pvc-dd064d85-89ee-442c-9d59-b7969384ad66 2Gi RWO sc-nfs <unset> 5m43s elasticsearch-data-my-es-cluster-es-default-1 Bound pvc-f1807028-bf85-4ffc-9fbf-8e952dcd929e 2Gi RWO sc-nfs <unset> 5m43s elasticsearch-data-my-es-cluster-es-default-2 Bound pvc-4dcf51ea-a7d3-41df-8cf9-0f64196b7d72 2Gi RWO sc-nfs <unset> 5m43s
[root@master1 stateful]# [root@master1 stateful]# kubectl get pv -n demo NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS VOLUMEATTRIBUTESCLASS REASON AGE pvc-4dcf51ea-a7d3-41df-8cf9-0f64196b7d72 2Gi RWO Delete Bound demo/elasticsearch-data-my-es-cluster-es-default-2 sc-nfs <unset> 5m58s pvc-dd064d85-89ee-442c-9d59-b7969384ad66 2Gi RWO Delete Bound demo/elasticsearch-data-my-es-cluster-es-default-0 sc-nfs <unset> 5m59s pvc-f1807028-bf85-4ffc-9fbf-8e952dcd929e 2Gi RWO Delete Bound demo/elasticsearch-data-my-es-cluster-es-default-1 sc-nfs <unset> 5m59s [root@master1 stateful]#
[root@master1 stateful]# ls /data/sc-nfs/ demo-elasticsearch-data-my-es-cluster-es-default-0-pvc-dd064d85-89ee-442c-9d59-b7969384ad66 demo-elasticsearch-data-my-es-cluster-es-default-1-pvc-f1807028-bf85-4ffc-9fbf-8e952dcd929e demo-elasticsearch-data-my-es-cluster-es-default-2-pvc-4dcf51ea-a7d3-41df-8cf9-0f64196b7d72
[root@master1 stateful]# kubectl get svc -n demo NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE my-es-cluster-es-default ClusterIP None <none> 9200/TCP 9m21s my-es-cluster-es-http ClusterIP 10.105.204.18 <none> 9200/TCP 9m22s my-es-cluster-es-internal-http ClusterIP 10.97.59.239 <none> 9200/TCP 9m22s my-es-cluster-es-transport ClusterIP None <none> 9300/TCP 9m22s
[root@master1 stateful]# kubectl get pod -n demo -o wide NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES my-es-cluster-es-default-0 1/1 Running 0 33m 10.244.3.72 node3.org <none> <none> my-es-cluster-es-default-1 1/1 Running 0 33m 10.244.1.64 node1.org <none> <none> my-es-cluster-es-default-2 1/1 Running 0 33m 10.244.2.51 node2.org <none> <none>
[root@master1 stateful]# kubectl get elasticsearch -n demo NAME HEALTH NODES VERSION PHASE AGE my-es-cluster green 3 9.2.1 Ready 35m
[root@master1 stateful]# kubectl get secret -n demo NAME TYPE DATA AGE my-es-cluster-es-default-es-config Opaque 1 13m my-es-cluster-es-default-es-transport-certs Opaque 1 13m my-es-cluster-es-elastic-user Opaque 1 13m my-es-cluster-es-file-settings Opaque 1 13m my-es-cluster-es-http-ca-internal Opaque 2 13m my-es-cluster-es-http-certs-internal Opaque 3 13m my-es-cluster-es-http-certs-public Opaque 2 13m my-es-cluster-es-internal-users Opaque 5 13m my-es-cluster-es-remote-ca Opaque 1 13m my-es-cluster-es-transport-ca-internal Opaque 2 13m my-es-cluster-es-transport-certs-public Opaque 1 13m my-es-cluster-es-xpack-file-realm Opaque 4 13m
#取出ES的访问密码
PASSWORD=$(kubectl get secret my-es-cluster-es-elastic-user -n demo -o go-template='{{.data.elastic | base64decode}}')
#或者
PASSWORD=$(kubectl get secret my-es-cluster-es-elastic-user -n demo -o jsonpath={.data.elastic} -n demo|base64 -d)
root@master1 stateful]# PASSWORD=$(kubectl get secret my-es-cluster-es-elastic-user -n demo -o go-template='{{.data.elastic | base64decode}}') [root@master1 stateful]# echo $PASSWORD MUlbz7QQbT3tmEirYT6PVpNc
#开启一个新的Pod测试访问ES Cluster
kubectl run --env="PASSWORD=$PASSWORD" client-$RANDOM --image registry.cn-beijing.aliyuncs.com/wangxiaochun/admin-box:v0.1 -it --rm --restart=Never --command -- /bin/bash
[root@master1 ~]# PASSWORD=$(kubectl get secret my-es-cluster-es-elastic-user -n > demo -o jsonpath={.data.elastic} -n demo|base64 -d) error: flag needs an argument: 'n' in -n See 'kubectl get --help' for usage. -bash: demo: command not found [root@master1 ~]# PASSWORD=$(kubectl get secret my-es-cluster-es-elastic-user -n demo -o go-template='{{.data.elastic | base64decode}}') [root@master1 ~]# [root@master1 ~]# kubectl run --env="PASSWORD=$PASSWORD" client-$RANDOM --image registry.cn-beijing.aliyuncs.com/wangxiaochun/admin-box:v0.1 -it --rm --restart=Never --command -- /bin/bash All commands and output from this session will be recorded in container logs, including credentials and sensitive information passed through the command prompt. If you don't see a command prompt, try pressing enter. root@client-7340 /# root@client-7340 /#
curl -u "elastic:$PASSWORD" -k https://my-es-cluster-es-http.demo:9200
访问成功
root@client-7340 /# curl -u "elastic:$PASSWORD" -k https://my-es-cluster-es-http.demo:9200 { "name" : "my-es-cluster-es-default-0", "cluster_name" : "my-es-cluster", "cluster_uuid" : "McSD8kHlQmaBS5IhAXboGQ", "version" : { "number" : "9.2.1", "build_flavor" : "default", "build_type" : "docker", "build_hash" : "4ad0ef0e98a2e72fafbd79a19fa5cae2f026117d", "build_date" : "2025-11-06T22:07:39.673130621Z", "build_snapshot" : false, "lucene_version" : "10.3.1", "minimum_wire_compatibility_version" : "8.19.0", "minimum_index_compatibility_version" : "8.0.0" }, "tagline" : "You Know, for Search" } root@client-7340 /#
#当前没有索引
curl -u "elastic:$PASSWORD" -k https://my-es-cluster-es-http.demo:9200/_cat/indices
部署 Kibana
#部署Kibana
[root@master1 stateful]# cat operator-elasticsearch-cluster-kibana.yaml apiVersion: kibana.k8s.elastic.co/v1 kind: Kibana metadata: name: kibana namespace: demo spec: version: 9.2.1 # ← 改为和 ES 一致 count: 1 elasticsearchRef: name: "my-es-cluster" http: tls: selfSignedCertificate: disabled: true service: spec: type: LoadBalancer
kubectl apply -f operator-elasticsearch-cluster-kibana.yaml
docker pull docker.elastic.co/kibana/kibana:8.14.0
#查看状态
[root@master1 stateful]# kubectl get kibana -n demo NAME HEALTH NODES VERSION AGE kibana green 1 9.2.1 10m
[root@master1 stateful]# kubectl get pod -n demo NAME READY STATUS RESTARTS AGE kibana-kb-5fc95fc47-zpmlt 1/1 Running 0 5m55s my-es-cluster-es-default-0 1/1 Running 0 59m my-es-cluster-es-default-1 1/1 Running 0 59m my-es-cluster-es-default-2 1/1 Running 0 59m
#查看SVC
[root@master1 stateful]# kubectl get svc -n demo NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE kibana-kb-http LoadBalancer 10.105.110.152 192.168.3.11 5601:32304/TCP 2m39s my-es-cluster-es-default ClusterIP None <none> 9200/TCP 55m my-es-cluster-es-http ClusterIP 10.105.204.18 <none> 9200/TCP 55m my-es-cluster-es-internal-http ClusterIP 10.97.59.239 <none> 9200/TCP 55m my-es-cluster-es-transport ClusterIP None <none> 9300/TCP 55m
如何创建 TLS Secret
用自签名证书（测试环境）
mkdir ssl
cd ssl
# 生成私钥
openssl genrsa -out tls.key 2048
# 生成证书签名请求（CSR）
openssl req -new -key tls.key -out tls.csr -subj "/CN=kibana.ming.org"
# 生成自签名证书
openssl x509 -req -days 365 -in tls.csr -signkey tls.key -out tls.crt
# 创建 Secret
kubectl create secret tls tls-secret-name --cert=tls.crt --key=tls.key -n demo
[root@master1 stateful]# cd ssl/ [root@master1 ssl]# ll total 0 [root@master1 ssl]# [root@master1 ssl]# openssl genrsa -out tls.key 2048 [root@master1 ssl]# [root@master1 ssl]# openssl req -new -key tls.key -out tls.csr -subj "/CN=kibana.ming.org" [root@master1 ssl]# ll total 8 -rw-r--r-- 1 root root 899 May 29 02:01 tls.csr -rw------- 1 root root 1704 May 29 02:01 tls.key [root@master1 ssl]# [root@master1 ssl]# openssl x509 -req -days 365 -in tls.csr -signkey tls.key -out tls.crt Certificate request self-signature ok subject=CN=kibana.ming.org
kubectl create secret tls tls-secret-name --cert=tls.crt --key=tls.key -n demo
[root@master1 stateful]# cat kibana-ingress.yaml --- # Ingress 启用 TLS apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: kibana namespace: demo spec: ingressClassName: nginx tls: - hosts: - kibana.ming.org secretName: tls-secret-name # ← Ingress 用的证书 rules: - host: kibana.ming.org http: paths: - backend: service: name: kibana-kb-http port: number: 5601 path: / pathType: Prefix
root@master1 stateful]# kubectl get ingress -n demo NAME CLASS HOSTS ADDRESS PORTS AGE kibana nginx kibana.ming.org 192.168.3.10 80, 443 52s
部署 Filebeat
#部署Filebeat
[root@master1 stateful]# cat operator-elasticsearch-cluster-filebeat.yaml apiVersion: beat.k8s.elastic.co/v1beta1 kind: Beat metadata: name: filebeat namespace: demo spec: type: filebeat version: 9.2.1 elasticsearchRef: name: "my-es-cluster" kibanaRef: name: "kibana" config: filebeat: autodiscover: providers: - type: kubernetes node: ${NODE_NAME} hints: enabled: true default_config: type: container paths: - /var/log/containers/*${data.kubernetes.container.id}.log processors: - add_kubernetes_metadata: host: ${NODE_NAME} matchers: - logs_path: logs_path: "/var/log/containers/" - drop_event.when: or: - equals: kubernetes.namespace: "kube-system" - equals: kubernetes.namespace: "logging" - equals: kubernetes.namespace: "ingress-nginx" - equals: kubernetes.namespace: "kube-node-lease" - equals: kubernetes.namespace: "elastic-system" output: elasticsearch: hosts: ['https://my-es-cluster-es-http.demo.svc.cluster.local:9200'] ssl: verification_mode: none daemonSet: podTemplate: spec: serviceAccountName: filebeat automountServiceAccountToken: true terminationGracePeriodSeconds: 30 dnsPolicy: ClusterFirstWithHostNet hostNetwork: true containers: - name: filebeat securityContext: runAsUser: 0 volumeMounts: - name: varlogcontainers mountPath: /var/log/containers - name: varlogpods mountPath: /var/log/pods - name: varlibdockercontainers mountPath: /var/lib/docker/containers env: - name: NODE_NAME valueFrom: fieldRef: fieldPath: spec.nodeName volumes: - name: varlogcontainers hostPath: path: /var/log/containers - name: varlogpods hostPath: path: /var/log/pods - name: varlibdockercontainers hostPath: path: /var/lib/docker/containers --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: filebeat rules: - apiGroups: [""] resources: - namespaces - pods - nodes verbs: - get - watch - list --- apiVersion: v1 kind: ServiceAccount metadata: name: filebeat namespace: demo --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: filebeat subjects: - kind: ServiceAccount name: filebeat namespace: demo roleRef: kind: ClusterRole name: filebeat apiGroup: rbac.authorization.k8s.io
kubectl apply -f operator-elasticsearch-cluster-filebeat.yaml
[root@master1 stateful]# kubectl get pod -n demo -o wide NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES filebeat-beat-filebeat-8l5j2 1/1 Running 0 6m40s 192.168.3.64 node2.org <none> <none> filebeat-beat-filebeat-f7fnw 1/1 Running 0 6m41s 192.168.3.63 node1.org <none> <none> filebeat-beat-filebeat-mnkc8 1/1 Running 0 6m40s 192.168.3.65 node3.org <none> <none> kibana-kb-679c8b8d4c-tqsf9 1/1 Running 0 12m 10.244.3.77 node3.org <none> <none> my-es-cluster-es-default-0 1/1 Running 0 101m 10.244.3.72 node3.org <none> <none> my-es-cluster-es-default-1 1/1 Running 0 101m 10.244.1.64 node1.org <none> <none> my-es-cluster-es-default-2 1/1 Running 0 101m 10.244.2.51 node2.org <none> <none>
[root@master1 stateful]# kubectl get beats -n demo NAME HEALTH AVAILABLE EXPECTED TYPE VERSION AGE filebeat green 3 3 filebeat 9.2.1 7m17s
测试访问 Kibana
#使用ElasticSearch部署时生成的用户elastic及其密码即可登录service的LoadBalancer
#用前面的命令取出密码
[root@master1 ~]# PASSWORD=$(kubectl get secret my-es-cluster-es-elastic-user -n demo -o jsonpath={.data.elastic} -n demo|base64 -d)
[root@master1 ~]# echo $PASSWORD
MUlbz7QQbT3tmEirYT6PVpNc
通过Kibana的LoadBalancer SVC的IP直接访问
[root@master1 ~]# kubectl get svc -n demo NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE kibana-kb-http LoadBalancer 10.111.212.32 192.168.3.11 5601:32685/TCP 15m my-es-cluster-es-default ClusterIP None <none> 9200/TCP 105m my-es-cluster-es-http ClusterIP 10.105.204.18 <none> 9200/TCP 105m my-es-cluster-es-internal-http ClusterIP 10.97.59.239 <none> 9200/TCP 105m my-es-cluster-es-transport ClusterIP None <none> 9300/TCP 105m
http://192.168.3.11:5601/
用ingress nginx LoadBalancer 80 443访问
192.168.3.10 kibana.ming.org
https://kibana.ming.org/
Kibana会自动加载由filebeat生成的index pattern
因此直接进行Discover页面即可查看到数据
#查看到索引
PASSWORD=$(kubectl get secret my-es-cluster-es-elastic-user -n demo -o jsonpath={.data.elastic} -n demo|base64 -d)
kubectl run --env="PASSWORD=$PASSWORD" client-$RANDOM --image wangxiaochun/admin-box:v0.1 -it --rm --restart=Never --command -- /bin/bash
curl -u "elastic:$PASSWORD" -k https://my-es-cluster-es-http.demo:9200/_cat/indices
root@client-15351 /# curl -u "elastic:$PASSWORD" -k https://my-es-cluster-es-http.demo:9200/_cat/indices green open .internal.alerts-transform.health.alerts-default-000001 LisipK3qQRWorAIDPutpxQ 1 1 0 0 498b 249b 249b green open .internal.alerts-observability.logs.alerts-default-000001 rC2l1mx2QfqNh9eEePogvQ 1 1 0 0 498b 249b 249b green open .internal.alerts-observability.uptime.alerts-default-000001 lFX1l9dWQxuuiXw46YB_Iw 1 1 0 0 498b 249b 249b green open .internal.alerts-ml.anomaly-detection.alerts-default-000001 1C6hhwhlQS--bKez_D6CPA 1 1 0 0 498b 249b 249b green open .internal.alerts-observability.slo.alerts-default-000001 LhEBqyYQT260H6NicnACTw 1 1 0 0 498b 249b 249b green open .internal.alerts-default.alerts-default-000001 uxW_Kzu7R2G5utoGzDBb5w 1 1 0 0 498b 249b 249b green open .internal.alerts-streams.alerts-default-000001 wqhTlFNDRaagMvaxrx8qgg 1 1 0 0 498b 249b 249b green open .internal.alerts-observability.apm.alerts-default-000001 dDkomNgfQlyg6NZTUlxejw 1 1 0 0 498b 249b 249b green open .internal.alerts-security.attack.discovery.alerts-default-000001 bRNBXUlCTnianu4rUMmdYA 1 1 0 0 498b 249b 249b green open .internal.alerts-observability.metrics.alerts-default-000001 LDOoAnzIRFe-JAsizHysTw 1 1 0 0 498b 249b 249b green open .internal.alerts-ml.anomaly-detection-health.alerts-default-000001 jVTf5Pp_Tz2LMisUWlYEGA 1 1 0 0 498b 249b 249b green open .internal.alerts-observability.threshold.alerts-default-000001 UPpq9fQGRB2o0wM5QDeHtQ 1 1 0 0 498b 249b 249b green open .internal.alerts-security.alerts-default-000001 z3SoAFPLSGyUTBQaA5ZCww 1 1 0 0 498b 249b 249b green open .internal.alerts-dataset.quality.alerts-default-000001 XitRxpKNRYioNBYd95hbNA 1 1 0 0 498b 249b 249b green open .internal.alerts-stack.alerts-default-000001 Zm-YRokWRB-XIcO3kT4EQw 1 1 0 0 498b 249b 249b
浙公网安备 33010602011771号