Promethus-day3-监控携带metriecs接口的服务

监控携带metriecs接口的服务

普罗米修斯的两种监控服务：

1.携带metrics接口的服务

可以通过metrics接口来获取服务的监控项、监控信息

2.不携带metrics接口的服务

通过EndPoint造一个metrict接口

PS：metrics（接口）

1.监控流程

通过EndPoint获取需要监控ETCD服务的IP地址
创建Service，给集群内部的ServiceMonintor用
创建ServiceMonitor，部署需要的证书进行服务注入
重启普罗米修斯以监控Pod，载入监控项
测试服务
加入Grafana，做大屏展示

2.监控ETCD服务

监控携带metriecs接口的服务ETCD

ADM安装的K8s集群：

ETCD在集群内部；

只能通过EndPoint指定IP的方法获取；

二进制安装的K8s集群：

ETCD在集群外部；

既能通过IP、也能通过Service别名（ExternalName）获取；

1）测试ETCD服务的metrics接口

1> 查看etcd的端口

[root@k8s-master1 ~]# netstat -lntp |grep etcd
tcp        0      0 192.168.12.11:2379      0.0.0.0:*               LISTEN      85425/etcd          
tcp        0      0 127.0.0.1:2379          0.0.0.0:*               LISTEN      85425/etcd          
tcp        0      0 192.168.12.11:2380      0.0.0.0:*               LISTEN      85425/etcd          
tcp        0      0 127.0.0.1:2381          0.0.0.0:*               LISTEN      85425/etcd   

#2379是用于集群外部连接etcd集群的端口
#2380是集群内部端口，集群是高可用的，即分布式的，相当于多台主机组合成一台，用于内部沟通的端口

2> 查看可用的监控项

可用的监控项，即etcd服务的metrics接口

[root@k8s-master1 ~]# curl -k --cert /etc/kubernetes/pki/apiserver-etcd-client.crt --key /etc/kubernetes/pki/apiserver-etcd-client.key https://127.0.0.1:2379/metrics
# TYPE promhttp_metric_handler_requests_total counter
promhttp_metric_handler_requests_total{code="200"} 0
promhttp_metric_handler_requests_total{code="500"} 0
promhttp_metric_handler_requests_total{code="503"} 0

-k	# 忽略证书
https://127.0.0.1:2379		# url+端口
/etc/kubernetes/pki/apiserver-etcd-client.crt		# 连接etcd证书
/etc/kubernetes/pki/apiserver-etcd-client.key		# etcd证书私钥

2）通过endpoint获取需要监控的etcd的地址

1> 编辑配置清单

cat > etcd-endpoints.yaml <<EOF
kind: Endpoints
apiVersion: v1
metadata:
  namespace: kube-system  #注意命名空间
  name: etcd-monitor
  labels:
    k8s: etcd
subsets:
  - addresses:
    - ip: "192.168.12.11"  #etcd节点对应的主机ip，有几台就写几台
    ports:
      - port: 2379  #etcd集群的端口号
        protocol: TCP
        name: etcd
EOF

2> 部署查看

# 部署
[root@k8s-master1 ~]# kubectl apply -f etcd-endpoints.yaml 
endpoints/etcd-monitor created

# 查看结果
[root@k8s-master1 ~]# kubectl get endpoints -n kube-system 
NAME           ENDPOINTS                                 AGE
etcd-monitor   192.168.12.11:2379                        33s

3）创建service，给集群内部的servicemonitor使用

1> 编写Service配置清单

cat > etcd-service.yaml <<EOF
kind: Service
apiVersion: v1
metadata:
  namespace: kube-system
  name: etcd-monitor
  labels:
    k8s: etcd
spec:
  ports:
    - port: 2379
      targetPort: 2379
      name: etcd
      protocol: TCP
EOF

2> 部署查看

# 部署
[root@k8s-master1 ~]# kubectl apply  -f etcd-service.yaml 
service/etcd-monitor created

# 查看分配的IP地址
[root@k8s-master1 ~]# kubectl get svc -n kube-system 
NAME           TYPE        CLUSTER-IP    EXTERNAL-IP   PORT(S)                        AGE
etcd-monitor   ClusterIP   10.110.7.57   <none>        2379/TCP                       16s
kube-dns       ClusterIP   10.96.0.10    <none>        53/UDP,53/TCP,9153/TCP         6d22h
kubelet        ClusterIP   None          <none>        10250/TCP,10255/TCP,4194/TCP   27h

3> 测试连接结果

通过分配的集群内部的IP地址l来测试连接结果

[root@k8s-master1 ~]# curl -k --cert /etc/kubernetes/pki/apiserver-etcd-client.crt --key /etc/kubernetes/pki/apiserver-etcd-client.key https://10.111.200.179:2379/metrics
# TYPE promhttp_metric_handler_requests_total counter
promhttp_metric_handler_requests_total{code="200"} 45
promhttp_metric_handler_requests_total{code="500"} 0
promhttp_metric_handler_requests_total{code="503"} 0


PS：获取到可用的监控项就说明上述操作没有问题

4）创建ServiceMonitor

创建servicemonitor，以部署需要的访问证书

1> 编辑配置清单

cat > etcd-servicemontior.yaml <<EOF
kind: ServiceMonitor
apiVersion: monitoring.coreos.com/v1
metadata:
  namespace: monitoring
  name: etcd-monitor
  labels:
    k8s: etcd
spec:
  endpoints:
  - interval: 3s
    port: etcd
    scheme: https
    tlsConfig:
      caFile: /etc/prometheus/secrets/etcd-certs/ca.crt
      certFile: /etc/prometheus/secrets/etcd-certs/peer.crt
      keyFile: /etc/prometheus/secrets/etcd-certs/peer.key
      insecureSkipVerify: true
  selector:
    matchLabels:
      k8s: etcd
  namespaceSelector:
    matchNames:
      - "kube-system"
EOF

2> 部署查看

# 部署
[root@k8s-master1 ~]# kubectl apply -f etcd-servicemontior.yaml 
servicemonitor.monitoring.coreos.com/etcd-monitor created

# 查看部署结果
[root@k8s-master1 ~]# kubectl get servicemonitors -n monitoring 
NAME                      AGE
etcd-monitor              110s

5）创建一个secrets，用来保存prometheus监控的etcd的证书

[root@k8s-master1 ~]# kubectl create secret generic etcd-certs -n monitoring --from-file=/etc/kubernetes/pki/etcd/ca.crt --from-file=/etc/kubernetes/pki/etcd/peer.crt --from-file=/etc/kubernetes/pki/etcd/peer.key
secret/etcd-certs created

6）重启普罗米修斯监控pod(prometheus-k8s-0),载入监控项

1> 编辑配置清单

cat > prometheus-k8s.yaml <<EOF
kind: Prometheus
apiVersion: monitoring.coreos.com/v1
metadata:
  labels:
    prometheus: k8s
  name: k8s
  namespace: monitoring
spec:
  alerting:
    alertmanagers:
      - name: alertmanager-main
        namespace: monitoring
        port: web
      - name: alertmanager-main-etcd
        namespace: kube-system
        port: etcd
  image: quay.io/prometheus/prometheus:v2.15.2
  nodeSelector:
    kubernetes.io/os: linux
  podMonitorNamespaceSelector: {}
  podMonitorSelector: {}
  replicas: 2
  resources:
    requests:
      memory: 400Mi
  ruleSelector:
    matchLabels:
      prometheus: k8s
      role: alert-rules
  securityContext:
    fsGroup: 2000
    runAsNonRoot: true
    runAsUser: 1000
  serviceAccountName: prometheus-k8s
  serviceMonitorNamespaceSelector: {}
  serviceMonitorSelector: {}
  version: v2.15.2
  secrets:
    - etcd-certs
EOF

2> 部署查看

# 部署
[root@k8s-master1 ~]# kubectl apply -f prometheus-k8s.yaml 
prometheus.monitoring.coreos.com/k8s configured

# 2.查看
[root@k8s-master1 ~]# kubectl get pods -n monitoring 
NAME                                   READY   STATUS    RESTARTS   AGE
prometheus-k8s-0                       3/3     Running   1          16s
prometheus-k8s-1                       3/3     Running   1          16s