35-Prometheus监控k8s集群(Operator)
一、基于Operator部署Prometheus实现K8S监控
1.下载源代码
wget https://github.com/prometheus-operator/kube-prometheus/archive/refs/tags/v0.11.0.tar.gz
2.解压目录
[root@master231 02-prometheus]# tar xf kube-prometheus-0.11.0.tar.gz
[root@master231 02-prometheus]#
[root@master231 02-prometheus]# cd kube-prometheus-0.11.0/
[root@master231 kube-prometheus-0.11.0]#
3.安装Prometheus-Operator
kubectl apply --server-side -f manifests/setup
kubectl wait \
--for condition=Established \
--all CustomResourceDefinition \
--namespace=monitoring
kubectl apply -f manifests/
4.检查Prometheus是否部署成功
[root@master231 kube-prometheus-0.11.0]# kubectl get pods -n monitoring -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
alertmanager-main-0 2/2 Running 0 3m34s 10.100.2.93 worker233 <none> <none>
alertmanager-main-1 2/2 Running 0 3m34s 10.100.1.11 worker232 <none> <none>
alertmanager-main-2 2/2 Running 0 3m34s 10.100.2.94 worker233 <none> <none>
blackbox-exporter-746c64fd88-g7w5k 3/3 Running 0 4m40s 10.100.2.89 worker233 <none> <none>
grafana-5fc7f9f55d-zk5tk 1/1 Running 0 4m39s 10.100.2.91 worker233 <none> <none>
kube-state-metrics-6c8846558c-wbs66 3/3 Running 0 4m39s 10.100.2.90 worker233 <none> <none>
node-exporter-mttjn 2/2 Running 0 4m39s 10.0.0.232 worker232 <none> <none>
node-exporter-szr25 2/2 Running 0 4m39s 10.0.0.231 master231 <none> <none>
node-exporter-wdkjk 2/2 Running 0 4m39s 10.0.0.233 worker233 <none> <none>
prometheus-adapter-6455646bdc-m9qwv 1/1 Running 0 4m38s 10.100.2.92 worker233 <none> <none>
prometheus-adapter-6455646bdc-wbjqd 1/1 Running 0 4m38s 10.100.1.9 worker232 <none> <none>
prometheus-k8s-0 2/2 Running 0 3m33s 10.100.1.12 worker232 <none> <none>
prometheus-k8s-1 1/2 Running 0 3m33s 10.100.2.95 worker233 <none> <none>
prometheus-operator-f59c8b954-ttjth 2/2 Running 0 4m38s 10.100.1.10 worker232 <none> <none>
[root@master231 kube-prometheus-0.11.0]#
5.修改Grafana的svc
[root@master231 kube-prometheus-0.11.0]# cat manifests/grafana-service.yaml
apiVersion: v1
kind: Service
metadata:
...
name: grafana
namespace: monitoring
spec:
type: LoadBalancer
...
[root@master231 kube-prometheus-0.11.0]#
[root@master231 kube-prometheus-0.11.0]# kubectl apply -f manifests/grafana-service.yaml
service/grafana configured
[root@master231 kube-prometheus-0.11.0]#
[root@master231 kube-prometheus-0.11.0]# kubectl get -f manifests/grafana-service.yaml
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
grafana LoadBalancer 10.200.231.129 10.0.0.154 3000:37754/TCP 7m23s
[root@master231 kube-prometheus-0.11.0]#
6.访问Grafana的WebUI
http://10.0.0.154:3000/
默认的用户名和密码： admin
二、暴露Prometheus的服务WebUI
1.基于NodePort方式暴露
[root@master231 kube-prometheus-0.11.0]# cat manifests/prometheus-service.yaml
apiVersion: v1
kind: Service
metadata:
...
name: prometheus-k8s
namespace: monitoring
spec:
type: NodePort
ports:
- name: web
port: 9090
nodePort: 9090
targetPort: web
...
[root@master231 kube-prometheus-0.11.0]#
[root@master231 kube-prometheus-0.11.0]# kubectl apply -f manifests/prometheus-service.yaml
service/prometheus-k8s configured
[root@master231 kube-prometheus-0.11.0]#
[root@master231 kube-prometheus-0.11.0]# kubectl get -f manifests/prometheus-service.yaml
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
prometheus-k8s NodePort 10.200.246.30 <none> 9090:9090/TCP,8080:39234/TCP 75m
[root@master231 kube-prometheus-0.11.0]#
2.基于LoadBalancer
[root@master231 Prometheus]# cat kube-prometheus-0.11.0/manifests/prometheus-service.yaml | grep LoadBalancer -A 3 -B 1
spec:
#添加如下字段
type: LoadBalancer
ports:
- name: web
port: 9090
3.基于端口转发
[root@master231 ~]# kubectl port-forward sts/prometheus-k8s 19090:9090 -n monitoring --address=0.0.0.0
Forwarding from 0.0.0.0:19090 -> 9090
Handling connection for 19090
Handling connection for 19090
4.Ingress实现
三、Prometheus监控云原生应用etcd案例
1.测试ectd metrics接口
1.1 查看etcd证书存储路径
[root@master231 yinzhengjie]# egrep "\--key-file|--cert-file" /etc/kubernetes/manifests/etcd.yaml
- --cert-file=/etc/kubernetes/pki/etcd/server.crt
- --key-file=/etc/kubernetes/pki/etcd/server.key
[root@master231 yinzhengjie]#
1.2 测试etcd证书访问的metrics接口
[root@master231 yinzhengjie]# curl -s --cert /etc/kubernetes/pki/etcd/server.crt --key /etc/kubernetes/pki/etcd/server.key https://10.0.0.231:2379/metrics -k | tail
# TYPE process_virtual_memory_max_bytes gauge
process_virtual_memory_max_bytes 1.8446744073709552e+19
# HELP promhttp_metric_handler_requests_in_flight Current number of scrapes being served.
# TYPE promhttp_metric_handler_requests_in_flight gauge
promhttp_metric_handler_requests_in_flight 1
# HELP promhttp_metric_handler_requests_total Total number of scrapes by HTTP status code.
# TYPE promhttp_metric_handler_requests_total counter
promhttp_metric_handler_requests_total{code="200"} 4
promhttp_metric_handler_requests_total{code="500"} 0
promhttp_metric_handler_requests_total{code="503"} 0
[root@master231 yinzhengjie]#
2.创建etcd证书的secrets并挂载到Prometheus server
2.1 查找需要挂载etcd的证书文件路径
[root@master231 yinzhengjie]# egrep "\--key-file|--cert-file|--trusted-ca-file" /etc/kubernetes/manifests/etcd.yaml
- --cert-file=/etc/kubernetes/pki/etcd/server.crt
- --key-file=/etc/kubernetes/pki/etcd/server.key
- --trusted-ca-file=/etc/kubernetes/pki/etcd/ca.crt
[root@master231 yinzhengjie]#
2.2 根据etcd的实际存储路径创建secrets
[root@master231 yinzhengjie]# kubectl create secret generic etcd-tls --from-file=/etc/kubernetes/pki/etcd/server.crt --from-file=/etc/kubernetes/pki/etcd/server.key --from-file=/etc/kubernetes/pki/etcd/ca.crt -n monitoring
secret/etcd-tls created
[root@master231 yinzhengjie]#
[root@master231 yinzhengjie]# kubectl -n monitoring get secrets etcd-tls
NAME TYPE DATA AGE
etcd-tls Opaque 3 12s
[root@master231 yinzhengjie]#
2.3 修改Prometheus的资源，修改后会自动重启
[root@master231 yinzhengjie]# kubectl -n monitoring edit prometheus k8s
...
spec:
secrets:
- etcd-tls
...
[root@master231 yinzhengjie]# kubectl -n monitoring get pods -l app.kubernetes.io/component=prometheus -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
prometheus-k8s-0 2/2 Running 0 74s 10.100.1.57 worker232 <none> <none>
prometheus-k8s-1 2/2 Running 0 92s 10.100.2.28 worker233 <none> <none>
[root@master231 yinzhengjie]#
2.4.查看证书是否挂载成功
[root@master231 yinzhengjie]# kubectl -n monitoring exec prometheus-k8s-0 -c prometheus -- ls -l /etc/prometheus/secrets/etcd-tls
total 0
lrwxrwxrwx 1 root 2000 13 Jan 24 14:07 ca.crt -> ..data/ca.crt
lrwxrwxrwx 1 root 2000 17 Jan 24 14:07 server.crt -> ..data/server.crt
lrwxrwxrwx 1 root 2000 17 Jan 24 14:07 server.key -> ..data/server.key
[root@master231 yinzhengjie]#
[root@master231 yinzhengjie]# kubectl -n monitoring exec prometheus-k8s-1 -c prometheus -- ls -l /etc/prometheus/secrets/etcd-tls
total 0
lrwxrwxrwx 1 root 2000 13 Jan 24 14:07 ca.crt -> ..data/ca.crt
lrwxrwxrwx 1 root 2000 17 Jan 24 14:07 server.crt -> ..data/server.crt
lrwxrwxrwx 1 root 2000 17 Jan 24 14:07 server.key -> ..data/server.key
[root@master231 yinzhengjie]#
3.编写资源清单
[root@master231 servicemonitors]# cat 01-smon-etcd.yaml
apiVersion: v1
kind: Endpoints
metadata:
name: etcd-k8s
namespace: kube-system
subsets:
- addresses:
- ip: 10.0.0.231
ports:
- name: https-metrics
port: 2379
protocol: TCP
---
apiVersion: v1
kind: Service
metadata:
name: etcd-k8s
namespace: kube-system
labels:
apps: etcd
spec:
ports:
- name: https-metrics
port: 2379
targetPort: 2379
type: ClusterIP
---
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
name: dingzhiyan-etcd-smon
namespace: monitoring
spec:
# 指定job的标签，可以不设置。
jobLabel: kubeadm-etcd-k8s-yinzhengjie
# 指定监控后端目标的策略
endpoints:
# 监控数据抓取的时间间隔
- interval: 30s
# 指定metrics端口，这个port对应Services.spec.ports.name
port: https-metrics
# Metrics接口路径
path: /metrics
# Metrics接口的协议
scheme: https
# 指定用于连接etcd的证书文件
tlsConfig:
# 指定etcd的CA的证书文件
caFile: /etc/prometheus/secrets/etcd-tls/ca.crt
# 指定etcd的证书文件
certFile: /etc/prometheus/secrets/etcd-tls/server.crt
# 指定etcd的私钥文件
keyFile: /etc/prometheus/secrets/etcd-tls/server.key
# 关闭证书校验，毕竟咱们是自建的证书，而非官方授权的证书文件。
insecureSkipVerify: true
# 监控目标Service所在的命名空间
namespaceSelector:
matchNames:
- kube-system
# 监控目标Service目标的标签。
selector:
# 注意，这个标签要和etcd的service的标签保持一致哟
matchLabels:
apps: etcd
[root@master231 servicemonitors]#
4.Prometheus查看数据
etcd_cluster_version
5.Grafana导入模板
3070
四、Prometheus监控非云原生应用MySQL案例
1.编写资源清单
[root@master231 servicemonitors]# cat 02-smon-mysqld.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: mysql80-deployment
spec:
replicas: 1
selector:
matchLabels:
apps: mysql80
template:
metadata:
labels:
apps: mysql80
spec:
containers:
- name: mysql
image: harbor250.dingzhiyan.com/dingzhiyan-wp/mysql:8.0.36-oracle
ports:
- containerPort: 3306
env:
- name: MYSQL_ROOT_PASSWORD
value: yinzhengjie
- name: MYSQL_USER
value: linux96
- name: MYSQL_PASSWORD
value: "dingzhiyan"
---
apiVersion: v1
kind: Service
metadata:
name: mysql80-service
spec:
selector:
apps: mysql80
ports:
- protocol: TCP
port: 3306
targetPort: 3306
---
apiVersion: v1
kind: ConfigMap
metadata:
name: my.cnf
data:
.my.cnf: |-
[client]
user = linux96
password = dingzhiyan
[client.servers]
user = linux96
password = dingzhiyan
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: mysql-exporter-deployment
spec:
replicas: 1
selector:
matchLabels:
apps: mysql-exporter
template:
metadata:
labels:
apps: mysql-exporter
spec:
volumes:
- name: data
configMap:
name: my.cnf
items:
- key: .my.cnf
path: .my.cnf
containers:
- name: mysql-exporter
image: registry.cn-hangzhou.aliyuncs.com/yinzhengjie-k8s/mysqld-exporter:v0.15.1
command:
- mysqld_exporter
- --config.my-cnf=/root/my.cnf
- --mysqld.address=mysql80-service.default.svc.dingzhiyan.com:3306
securityContext:
runAsUser: 0
ports:
- containerPort: 9104
#env:
#- name: DATA_SOURCE_NAME
# value: mysql_exporter:yinzhengjie@(mysql80-service.default.svc.yinzhengjie.com:3306)
volumeMounts:
- name: data
mountPath: /root/my.cnf
subPath: .my.cnf
---
apiVersion: v1
kind: Service
metadata:
name: mysql-exporter-service
labels:
apps: mysqld
spec:
selector:
apps: mysql-exporter
ports:
- protocol: TCP
port: 9104
targetPort: 9104
name: mysql80
---
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
name: dingzhiyan-mysql-smon
spec:
jobLabel: kubeadm-mysql-k8s-yinzhengjie
endpoints:
- interval: 3s
# 这里的端口可以写svc的端口号，也可以写svc的名称。
# 但我推荐写svc端口名称，这样svc就算修改了端口号，只要不修改svc端口的名称，那么我们此处就不用再次修改哟。
# port: 9104
port: mysql80
path: /metrics
scheme: http
namespaceSelector:
matchNames:
- default
selector:
matchLabels:
apps: mysqld
[root@master231 servicemonitors]#
2.Prometheus访问测试
mysql_up
3.Grafana导入模板
7362
本文来自博客园，作者：丁志岩，转载请注明原文链接：https://www.cnblogs.com/dezyan/p/18888864
浙公网安备 33010602011771号