Kubernetes 之 Cilium + Hubble + Prometheus + Grafana【2024-02-29 测试成功】

配置Cilium & Hubble

root@ubuntu-k8s-master01:~# cilium uninstall
🔥 Deleting pods in cilium-test namespace...
🔥 Deleting cilium-test namespace...

root@ubuntu-k8s-master01:~# kubectl get pods -n kube-system
NAME                                          READY   STATUS    RESTARTS       AGE
coredns-774bbd8588-5qx92                      1/1     Running   0              160m
coredns-774bbd8588-ggfjp                      1/1     Running   0              160m
etcd-ubuntu-k8s-master01                      1/1     Running   0              160m
etcd-ubuntu-k8s-node01                        1/1     Running   0              128m
etcd-ubuntu-k8s-node02                        1/1     Running   0              124m
kube-apiserver-ubuntu-k8s-master01            1/1     Running   0              160m
kube-apiserver-ubuntu-k8s-node01              1/1     Running   0              128m
kube-apiserver-ubuntu-k8s-node02              1/1     Running   0              124m
kube-controller-manager-ubuntu-k8s-master01   1/1     Running   1 (128m ago)   160m
kube-controller-manager-ubuntu-k8s-node01     1/1     Running   0              128m
kube-controller-manager-ubuntu-k8s-node02     1/1     Running   0              124m
kube-scheduler-ubuntu-k8s-master01            1/1     Running   1 (128m ago)   160m
kube-scheduler-ubuntu-k8s-node01              1/1     Running   0              128m
kube-scheduler-ubuntu-k8s-node02              1/1     Running   0              124m

#直接部署时 安装hubble和让Prometheus采集
vxlan隧道模式
#这个配置端有点问题 用下面的
cilium install \
        --set kubeProxyReplacement=strict \
        --set ipam.mode=kubernetes \
        --set routingMode=tunnel \
        --set tunnelProtocol=vxlan \
        --set ipam.operator.clusterPoolIPv4PodCIDRList=10.244.0.0/16 \
        --set ipam.Operator.ClusterPoolIPv4MaskSize=24 \     
        --set hubble.enabled="true" \
        --set hubble.listenAddress=":4244" \
        --set hubble.relay.enabled="true" \
        --set hubble.ui.enabled="true" \
        --set prometheus.enabled=true \
        --set operator.prometheus.enabled=true \
        --set hubble.metrics.port=9665 \
        --set hubble.metrics.enableOpenMetrics=true \
        --set hubble.metrics.enabled="{dns,drop,tcp,flow,port-distribution,icmp,httpV2:exemplars=true;labelsContext=source_ip\,source_namespace\,source_workload\,destination_ip\,destination_namespace\,destination_workload\,traffic_direction}"

root@ubuntu-k8s-master01:~#     cilium install --set kubeProxyReplacement=strict --set ipam.mode=kubernetes  --set routingMode=tunnel --set tunnelProtocol=vxlan  --set ipam.operator.clusterPoolIPv4PodCIDRList=10.244.0.0/16  --set ipam.Operator.ClusterPoolIPv4MaskSize=24  --set hubble.enabled="true" --set hubble.listenAddress=":4244" --set hubble.relay.enabled="true"--set hubble.ui.enabled="true" --set prometheus.enabled=true --set operator.prometheus.enabled=true --set hubble.metrics.port=9665 --set hubble.metrics.enableOpenMetrics=true  --set hubble.metrics.enabled="{dns,drop,tcp,flow,port-distribution,icmp,httpV2:exemplars=true;labelsContext=source_ip\,source_namespace\,source_workload\,destination_ip\,destination_namespace\,destination_workload\,traffic_direction}"
ℹ️  Using Cilium version 1.15.0
🔮 Auto-detected cluster name: kubernetes
🔮 Auto-detected kube-proxy has not been installed
ℹ️  Cilium will fully replace all functionalities of kube-proxy

 root@ubuntu-k8s-master01:~# cilium status
    /¯¯\
 /¯¯\__/¯¯\    Cilium:             OK
 \__/¯¯\__/    Operator:           OK
 /¯¯\__/¯¯\    Envoy DaemonSet:    disabled (using embedded mode)
 \__/¯¯\__/    Hubble Relay:       OK
    \__/       ClusterMesh:        disabled

DaemonSet              cilium             Desired: 3, Ready: 3/3, Available: 3/3
Deployment             hubble-relay       Desired: 1, Ready: 1/1, Available: 1/1
Deployment             cilium-operator    Desired: 1, Ready: 1/1, Available: 1/1
Containers:            cilium             Running: 3
                       hubble-relay       Running: 1
                       cilium-operator    Running: 1
Cluster Pods:          6/6 managed by Cilium
Helm chart version:    1.15.0
Image versions         cilium             quay.io/cilium/cilium:v1.15.0@sha256:9cfd6a0a3a964780e73a11159f93cc363e616f7d9783608f62af6cfdf3759619: 3
                       hubble-relay       quay.io/cilium/hubble-relay:v1.15.0@sha256:45b3ea70b73aee01644f800b8f6138c36446bfb130d2b88b0f75775ebe6a9ab6: 1
                       cilium-operator    quay.io/cilium/operator-generic:v1.15.0@sha256:e26ecd316e742e4c8aa1e302ba8b577c2d37d114583d6c4cdd2b638493546a79: 1
                      
#打开Hubble UI
root@ubuntu-k8s-master01:~# cilium hubble enable --ui
root@ubuntu-k8s-master01:~# kubectl get pods -n kube-system | grep ui
hubble-ui-6b4d867c59-ltncs                    2/2     Running   0              12s

root@ubuntu-k8s-master01:~# cilium status
    /¯¯\
 /¯¯\__/¯¯\    Cilium:             OK
 \__/¯¯\__/    Operator:           OK
 /¯¯\__/¯¯\    Envoy DaemonSet:    disabled (using embedded mode)
 \__/¯¯\__/    Hubble Relay:       OK
    \__/       ClusterMesh:        disabled

Deployment             hubble-ui          Desired: 1, Ready: 1/1, Available: 1/1
DaemonSet              cilium             Desired: 3, Ready: 3/3, Available: 3/3
Deployment             hubble-relay       Desired: 1, Ready: 1/1, Available: 1/1
Deployment             cilium-operator    Desired: 1, Ready: 1/1, Available: 1/1
Containers:            hubble-ui          Running: 1
                       hubble-relay       Running: 1
                       cilium-operator    Running: 1
                       cilium             Running: 3
Cluster Pods:          7/7 managed by Cilium
Helm chart version:    1.15.0
Image versions         cilium             quay.io/cilium/cilium:v1.15.0@sha256:9cfd6a0a3a964780e73a11159f93cc363e616f7d9783608f62af6cfdf3759619: 3
                       hubble-ui          quay.io/cilium/hubble-ui:v0.12.3@sha256:e6b825302fc1e406b1305363fe0bcd1fdf95730b32c2b99a2b36dfa37bdaeec2: 1
                       hubble-ui          quay.io/cilium/hubble-ui-backend:v0.12.3@sha256:1cd84251cec46e20f9e839ee0afba9b51c8de59d35681234f701d7f42062f138: 1
                       hubble-relay       quay.io/cilium/hubble-relay:v1.15.0@sha256:45b3ea70b73aee01644f800b8f6138c36446bfb130d2b88b0f75775ebe6a9ab6: 1
                       cilium-operator    quay.io/cilium/operator-generic:v1.15.0@sha256:e26ecd316e742e4c8aa1e302ba8b577c2d37d114583d6c4cdd2b638493546a79: 1

配置Prometheus&Grafana

#访问暴露的metrics
http://192.168.40.132:9665/metrics

#配置Prometheus
#参考URL：https://www.cnblogs.com/birkhoffxia/articles/17275357.html 二进制部署

#
root@ubuntu-k8s-master01:~/software# vim /apps/prometheus/prometheus.yml
  - job_name: "prometheus-cilium"
    static_configs:
      - targets: ["localhost:9665"]
      
#配置Grafana
#Ubuntu 版本 安装Grafana
sudo apt-get install -y adduser libfontconfig1 musl
wget https://dl.grafana.com/enterprise/release/grafana-enterprise_10.3.3_amd64.deb
sudo dpkg -i grafana-enterprise_10.3.3_amd64.deb
root@ubuntu-k8s-master01:~/software# systemctl start grafana-server
root@ubuntu-k8s-master01:~/software# systemctl restart grafana-server

#添加 Dashboard ID
15515