Kubernetes 之 Cilium + Hubble + Prometheus + Grafana【2024-02-29 测试成功】
配置Cilium & Hubble
root@ubuntu-k8s-master01:~# cilium uninstall
🔥 Deleting pods in cilium-test namespace...
🔥 Deleting cilium-test namespace...
root@ubuntu-k8s-master01:~# kubectl get pods -n kube-system
NAME READY STATUS RESTARTS AGE
coredns-774bbd8588-5qx92 1/1 Running 0 160m
coredns-774bbd8588-ggfjp 1/1 Running 0 160m
etcd-ubuntu-k8s-master01 1/1 Running 0 160m
etcd-ubuntu-k8s-node01 1/1 Running 0 128m
etcd-ubuntu-k8s-node02 1/1 Running 0 124m
kube-apiserver-ubuntu-k8s-master01 1/1 Running 0 160m
kube-apiserver-ubuntu-k8s-node01 1/1 Running 0 128m
kube-apiserver-ubuntu-k8s-node02 1/1 Running 0 124m
kube-controller-manager-ubuntu-k8s-master01 1/1 Running 1 (128m ago) 160m
kube-controller-manager-ubuntu-k8s-node01 1/1 Running 0 128m
kube-controller-manager-ubuntu-k8s-node02 1/1 Running 0 124m
kube-scheduler-ubuntu-k8s-master01 1/1 Running 1 (128m ago) 160m
kube-scheduler-ubuntu-k8s-node01 1/1 Running 0 128m
kube-scheduler-ubuntu-k8s-node02 1/1 Running 0 124m
#直接部署时 安装hubble和让Prometheus采集
vxlan隧道模式
#这个配置端有点问题 用下面的
cilium install \
--set kubeProxyReplacement=strict \
--set ipam.mode=kubernetes \
--set routingMode=tunnel \
--set tunnelProtocol=vxlan \
--set ipam.operator.clusterPoolIPv4PodCIDRList=10.244.0.0/16 \
--set ipam.Operator.ClusterPoolIPv4MaskSize=24 \
--set hubble.enabled="true" \
--set hubble.listenAddress=":4244" \
--set hubble.relay.enabled="true" \
--set hubble.ui.enabled="true" \
--set prometheus.enabled=true \
--set operator.prometheus.enabled=true \
--set hubble.metrics.port=9665 \
--set hubble.metrics.enableOpenMetrics=true \
--set hubble.metrics.enabled="{dns,drop,tcp,flow,port-distribution,icmp,httpV2:exemplars=true;labelsContext=source_ip\,source_namespace\,source_workload\,destination_ip\,destination_namespace\,destination_workload\,traffic_direction}"
root@ubuntu-k8s-master01:~# cilium install --set kubeProxyReplacement=strict --set ipam.mode=kubernetes --set routingMode=tunnel --set tunnelProtocol=vxlan --set ipam.operator.clusterPoolIPv4PodCIDRList=10.244.0.0/16 --set ipam.Operator.ClusterPoolIPv4MaskSize=24 --set hubble.enabled="true" --set hubble.listenAddress=":4244" --set hubble.relay.enabled="true"--set hubble.ui.enabled="true" --set prometheus.enabled=true --set operator.prometheus.enabled=true --set hubble.metrics.port=9665 --set hubble.metrics.enableOpenMetrics=true --set hubble.metrics.enabled="{dns,drop,tcp,flow,port-distribution,icmp,httpV2:exemplars=true;labelsContext=source_ip\,source_namespace\,source_workload\,destination_ip\,destination_namespace\,destination_workload\,traffic_direction}"
ℹ️ Using Cilium version 1.15.0
🔮 Auto-detected cluster name: kubernetes
🔮 Auto-detected kube-proxy has not been installed
ℹ️ Cilium will fully replace all functionalities of kube-proxy
root@ubuntu-k8s-master01:~# cilium status
/¯¯\
/¯¯\__/¯¯\ Cilium: OK
\__/¯¯\__/ Operator: OK
/¯¯\__/¯¯\ Envoy DaemonSet: disabled (using embedded mode)
\__/¯¯\__/ Hubble Relay: OK
\__/ ClusterMesh: disabled
DaemonSet cilium Desired: 3, Ready: 3/3, Available: 3/3
Deployment hubble-relay Desired: 1, Ready: 1/1, Available: 1/1
Deployment cilium-operator Desired: 1, Ready: 1/1, Available: 1/1
Containers: cilium Running: 3
hubble-relay Running: 1
cilium-operator Running: 1
Cluster Pods: 6/6 managed by Cilium
Helm chart version: 1.15.0
Image versions cilium quay.io/cilium/cilium:v1.15.0@sha256:9cfd6a0a3a964780e73a11159f93cc363e616f7d9783608f62af6cfdf3759619: 3
hubble-relay quay.io/cilium/hubble-relay:v1.15.0@sha256:45b3ea70b73aee01644f800b8f6138c36446bfb130d2b88b0f75775ebe6a9ab6: 1
cilium-operator quay.io/cilium/operator-generic:v1.15.0@sha256:e26ecd316e742e4c8aa1e302ba8b577c2d37d114583d6c4cdd2b638493546a79: 1
#打开Hubble UI
root@ubuntu-k8s-master01:~# cilium hubble enable --ui
root@ubuntu-k8s-master01:~# kubectl get pods -n kube-system | grep ui
hubble-ui-6b4d867c59-ltncs 2/2 Running 0 12s
root@ubuntu-k8s-master01:~# cilium status
/¯¯\
/¯¯\__/¯¯\ Cilium: OK
\__/¯¯\__/ Operator: OK
/¯¯\__/¯¯\ Envoy DaemonSet: disabled (using embedded mode)
\__/¯¯\__/ Hubble Relay: OK
\__/ ClusterMesh: disabled
Deployment hubble-ui Desired: 1, Ready: 1/1, Available: 1/1
DaemonSet cilium Desired: 3, Ready: 3/3, Available: 3/3
Deployment hubble-relay Desired: 1, Ready: 1/1, Available: 1/1
Deployment cilium-operator Desired: 1, Ready: 1/1, Available: 1/1
Containers: hubble-ui Running: 1
hubble-relay Running: 1
cilium-operator Running: 1
cilium Running: 3
Cluster Pods: 7/7 managed by Cilium
Helm chart version: 1.15.0
Image versions cilium quay.io/cilium/cilium:v1.15.0@sha256:9cfd6a0a3a964780e73a11159f93cc363e616f7d9783608f62af6cfdf3759619: 3
hubble-ui quay.io/cilium/hubble-ui:v0.12.3@sha256:e6b825302fc1e406b1305363fe0bcd1fdf95730b32c2b99a2b36dfa37bdaeec2: 1
hubble-ui quay.io/cilium/hubble-ui-backend:v0.12.3@sha256:1cd84251cec46e20f9e839ee0afba9b51c8de59d35681234f701d7f42062f138: 1
hubble-relay quay.io/cilium/hubble-relay:v1.15.0@sha256:45b3ea70b73aee01644f800b8f6138c36446bfb130d2b88b0f75775ebe6a9ab6: 1
cilium-operator quay.io/cilium/operator-generic:v1.15.0@sha256:e26ecd316e742e4c8aa1e302ba8b577c2d37d114583d6c4cdd2b638493546a79: 1
配置Prometheus&Grafana
#访问暴露的metrics
http://192.168.40.132:9665/metrics
#配置Prometheus
#参考URL:https://www.cnblogs.com/birkhoffxia/articles/17275357.html 二进制部署
#
root@ubuntu-k8s-master01:~/software# vim /apps/prometheus/prometheus.yml
- job_name: "prometheus-cilium"
static_configs:
- targets: ["localhost:9665"]
#配置Grafana
#Ubuntu 版本 安装Grafana
sudo apt-get install -y adduser libfontconfig1 musl
wget https://dl.grafana.com/enterprise/release/grafana-enterprise_10.3.3_amd64.deb
sudo dpkg -i grafana-enterprise_10.3.3_amd64.deb
root@ubuntu-k8s-master01:~/software# systemctl start grafana-server
root@ubuntu-k8s-master01:~/software# systemctl restart grafana-server
#添加 Dashboard ID
15515