Kubernetes 之 Prometheus 监控 Tomcat、Redis、Mysql、Haproxy、Nginx、Ingress Nginx （五）
一、Tomcat-Java exporter
[root@xksmaster1 tomcat-image]# ll
total 473756
-rw-r--r-- 1 root root 185 Apr 28 09:50 build-command.sh
-rw-r--r-- 1 root root 692 Apr 26 15:50 Dockerfile
-rw-r--r-- 1 root root 3405 Dec 17 2021 metrics.war
drwxr-xr-x 2 root root 52 Sep 11 2022 myapp
-rw-r--r-- 1 root root 175 Jul 16 2021 myapp.tar.gz
-rwxr-xr-x 1 root root 125 Jul 16 2021 run_tomcat.sh
-rw-r--r-- 1 root root 7590 Apr 26 15:50 server.xml
-rw-r--r-- 1 root root 59477 Dec 17 2021 simpleclient-0.8.0.jar
-rw-r--r-- 1 root root 5840 Dec 17 2021 simpleclient_common-0.8.0.jar
-rw-r--r-- 1 root root 21767 Dec 17 2021 simpleclient_hotspot-0.8.0.jar
-rw-r--r-- 1 root root 7104 Dec 17 2021 simpleclient_servlet-0.8.0.jar
-rw-r--r-- 1 root root 484967424 Dec 17 2021 tomcat-8.5.73-jdk11-corretto.tar.gz
-rw-r--r-- 1 root root 19582 Dec 17 2021 tomcat_exporter_client-0.0.12.jar
-rw-r--r-- 1 root root 3405 Dec 17 2021 tomcat_exporter_servlet-0.0.12.war
#build-command.sh
[root@xksmaster1 tomcat-image]# vim build-command.sh
#!/bin/bash
docker build -t harbor.magedu.net/magedu/tomcat-app1:v1 .
#Docker File
[root@xksmaster1 tomcat-image]# cat Dockerfile
FROM tomcat:8.5.73
LABEL maintainer="jack 2973707860@qq.com"
ADD server.xml /usr/local/tomcat/conf/server.xml
RUN mkdir /data/tomcat/webapps -p
ADD myapp /data/tomcat/webapps/myapp
ADD metrics.war /data/tomcat/webapps
ADD simpleclient-0.8.0.jar /usr/local/tomcat/lib/
ADD simpleclient_common-0.8.0.jar /usr/local/tomcat/lib/
ADD simpleclient_hotspot-0.8.0.jar /usr/local/tomcat/lib/
ADD simpleclient_servlet-0.8.0.jar /usr/local/tomcat/lib/
ADD tomcat_exporter_client-0.0.12.jar /usr/local/tomcat/lib/
#ADD run_tomcat.sh /apps/tomcat/bin/
EXPOSE 8080 8443 8009
#CMD ["/apps/tomcat/bin/catalina.sh","run"]
#CMD ["/apps/tomcat/bin/run_tomcat.sh"]
bash build-command.sh
#测试是否正常可以访问 192.168.40.180:8082/myapp 正常回显 ： tomcat app1
#测试是否正常可以访问 192.168.40.180:8082/metrics 正常回显 ： 数据
docker run -it --rm -p 8082:8080 harbor.magedu.net/magedu/tomcat-app1:v1
#配置Prometheus 文件
- job_name: "prometheus-tomcat-metrics"
static_configs:
- targets: ["192.168.19.180:8082"]
#导入模板
https://github.com/nlighten/tomcat_exporter/tree/master/dashboard
=========================================================================================
以下是部署到k8s 配置yaml文件
[root@xksmaster1 tomcat-image]# docker images
REPOSITORY TAG IMAGE ID CREATED SIZE
harbor.magedu.net/magedu/tomcat-app1 v1 c77e9fe51c18 41 hours ago 1.16GB
kubernetesui/dashboard v2.7.0 07655ddf2eeb 7 months ago 246MB
kubernetesui/metrics-scraper v1.0.8 115053965e86 11 months ago 43.8MB
tomcat 8.5.73 2d2bccf89f53 16 months ago 678MB
oliver006/redis_exporter latest eeb98a1e7665 16 months ago 8.83MB
kubernetesui/dashboard v2.0.0-beta8 eb51a3597525 3 years ago 90.8MB
kubernetesui/metrics-scraper v1.0.1 709901356c11 3 years ago 40.1MB
#生成的容器 保存传入到其他两个节点进行部署
[root@xksmaster1 tomcat-image]# docker save -o tomcat-app1.tar.gz harbor.magedu.net/magedu/tomcat-app1:v1
[root@xksmaster1 tomcat-image]# scp tomcat-app1.tar.gz xksnode1:/root
tomcat-app1.tar.gz 100% 1122MB 66.0MB/s 00:17
[root@xksmaster1 tomcat-image]# scp tomcat-app1.tar.gz xksnode2:/root
tomcat-app1.tar.gz
#部署Yaml文件 metrics Nodeport开放 31080端口继续访问
[root@xksmaster1 yaml]# pwd
/root/prometheus/20220911/prometheus-case-files/prometheus-files/case/app-monitor-case/tomcat/yaml
[root@xksmaster1 yaml]# ll
total 8
-rw-r--r-- 1 root root 626 Apr 28 10:09 tomcat-deploy.yaml
-rw-r--r-- 1 root root 259 Sep 8 2022 tomcat-svc.yaml
[root@xksmaster1 yaml]# cat tomcat-deploy.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: tomcat-deployment
namespace: default
spec:
selector:
matchLabels:
app: tomcat
replicas: 1 # tells deployment to run 2 pods matching the template
template: # create pods using pod definition in this template
metadata:
labels:
app: tomcat
annotations:
prometheus.io/scrape: 'true'
spec:
containers:
- name: tomcat
image: harbor.magedu.net/magedu/tomcat-app1:v1
imagePullPolicy: IfNotPresent
ports:
- containerPort: 8080
securityContext:
privileged: true
[root@xksmaster1 yaml]# cat tomcat-svc.yaml
kind: Service #service 类型
apiVersion: v1
metadata:
annotations:
prometheus.io/scrape: 'true'
name: tomcat-service
spec:
selector:
app: tomcat
ports:
- nodePort: 31080
port: 80
protocol: TCP
targetPort: 8080
type: NodePort
[root@xksmaster1 yaml]# kubectl apply -f .
deployment.apps/tomcat-deployment created
service/tomcat-service created
然后进行访问
二、mysqld_exporter
#安装mariadb数据库
yum install -y mariadb-server
systemctl start mariadb
systemctl status mariadb
mysql
#授权监控个账户权限
CREATE USER 'mysql_exporter'@'localhost' IDENTIFIED BY '123456' WITH MAX_USER_CONNECTIONS 3;
GRANT PROCESS, REPLICATION CLIENT, SELECT ON *.* TO 'mysql_exporter'@'localhost';
#验证权限
mysql -u mysql_exporter -p123456 -hlocalhost
#下载准备mysqld_exporter环境
cd /usr/local/src
https://github.com/prometheus/mysqld_exporter/releases
https://github.com/prometheus/mysqld_exporter/releases/download/v0.14.0/mysqld_exporter-0.14.0.linux-amd64.tar.gz
tar zxvf mysqld_exporter-0.14.0.linux-amd64.tar.gz
cd mysqld_exporter-0.14.0.linux-amd64/
cp mysqld_exporter /usr/local/bin/
#免密码登录配置:
# vim /root/.my.cnf
[client]
user=mysql_exporter
password=123456
#验证权限
mysql
#启动 mysql_exporter:
# vim/etc/systemd/system/mysqld_exporter.service
[Unit]
Description=Prometheus Node Exporter
After=network.target
[Service]
ExecStart=/usr/local/bin/mysqld_exporter --config.my-cnf=/root/.my.cnf
[Install]
WantedBy=multi-user.target
# systemctl daemon-reload
# systemctl restart mysqld_exporter
# systemctl enable mysqld_exporter
[root@xksmaster1 ~]# netstat -nltp
tcp6 0 0 :::9104 :::* LISTEN 25513/mysqld_export
#http://192.168.19.180:9104/
配置Prometheus文件
[root@xksmaster1 prometheus]# vim prometheus.yml
- job_name: "prometheus-mysqld_exporter"
static_configs:
- targets: ["192.168.19.180:9104"]
systemctl restart prometheus.service
添加导入Grafana 模板-11323-13106
三、Haproxy_exporter
#1、安装配置haproxy
yum install -y haproxy
systemctl start haproxy
systemctl status haproxy
systemctl enable haproxy
#2、配置haproxy.cfg配置文件
[root@xksnode1 prometheus]# vim /etc/haproxy/haproxy.cfg
#添加socket文件配置
#turn on stats unix socket
#stats socket /var/lib/haproxy/stats
stats socket /run/haproxy/admin.sock mode 660 level admin
#stats socket /run/haproxy/admin.sock mode 660 level admin expose-fd listeners#添加测试页
listen stats
bind :9999
stats enable
stats uri /haproxy-status
stats realm HAPorxy Stats\ Page
stats auth haadmin:123456
stats auth admin:123456
#添加代理配置
listen prometheu-server-9092
bind :9092
mode http
server 192.168.19.180 192.168.19.180:9090 check inter 3s fall 3 rise 5
#手动创建一下这个文件 否则启动会报错
[root@xksnode1 prometheus]# mkdir /run/haproxy
[root@xksnode1 prometheus]# touch /run/haproxy/admin.sock
#3、重启haproxy服务
[root@xksnode1 prometheus]# systemctl restart haproxy
[root@xksnode1 prometheus]# systemctl status haproxy
#4、下载haproxy_exporter
https://github.com/prometheus/haproxy_exporter/releases/download/v0.13.0/haproxy_exporter-0.13.0.linux-amd64.tar.gz
tar zxvf haproxy_exporter-0.13.0.linux-amd64.tar.gz
cp haproxy_exporter-0.13.0.linux-amd64/haproxy_exporter /usr/local/bin
#5、启动haproxy_exporter
方法一：
haproxy_exporter --haproxy.scrape-uri=unix:/run/haproxy/admin.sock
方法二：
[root@xksnode1 prometheus]# haproxy_exporter --haproxy.scrape-uri="http://haadmin:123456@192.168.19.181:9999/haproxy-status?stats;csv" &
ts=2023-04-27T04:00:22.932Z caller=haproxy_exporter.go:584 level=info msg="Starting haproxy_exporter" version="(version=0.13.0, branch=HEAD, revision=c5c72aa059b69c18ab38fd63777653c13eddaa7f)"
ts=2023-04-27T04:00:22.932Z caller=haproxy_exporter.go:585 level=info msg="Build context" context="(go=go1.17.3, user=root@77b4a325967c, date=20211126-09:54:08)"
ts=2023-04-27T04:00:22.932Z caller=haproxy_exporter.go:603 level=info msg="Listening on address" address=:9101
ts=2023-04-27T04:00:22.932Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false
#6、检测 haproxy_exporter是否打开端口 9101并且访问
[root@xksnode1 ~]# netstat -ntlp | grep 9101
tcp6 0 0 :::9101 :::* LISTEN 12628/haproxy_expor
浏览器访问：
http://192.168.19.181:9101/
#7、配置Prometheus文件
[root@xksmaster1 exporter]# vim /apps/prometheus/prometheus.yml
- job_name: "prometheus-haproxy_exporter"
static_configs:
- targets: ["192.168.19.181:9101"]
#8、Grafana导入模板367 2428
四、Redis监控
https://github.com/oliver006/redis_exporter
# pwd
/root/prometheus-files/case/app-monitor-case/redis/yaml
# １、kubectl create ns studylinux-net
# ２、redis-deployment.yaml 两个容器 一个ｒｅｄｉｓ服务端口为6379、另一个ｒｅｄｉｓ－ｅｘｐｏｒｔｅｒ服务端口：9121
[root@xianchaomaster1 yaml]# cat redis-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: redis
namespace: studylinux-net
spec:
replicas: 1
selector:
matchLabels:
app: redis
template:
metadata:
labels:
app: redis
spec:
containers:
- name: redis
image: redis:4.0.14
resources:
requests:
cpu: 100m
memory: 100Mi
ports:
- containerPort: 6379
- name: redis-exporter
image: oliver006/redis_exporter:latest
resources:
requests:
cpu: 100m
memory: 100Mi
ports:
- containerPort: 9121
# ３、redis-redis-svc.yaml 暴露外部访问端口 redis服务端口：31081
[root@xianchaomaster1 yaml]# cat redis-redis-svc.yaml
kind: Service #service 类型
apiVersion: v1
metadata:
# annotations:
# prometheus.io/scrape: 'false'
name: redis-redis-service
namespace: studylinux-net
spec:
selector:
app: redis
ports:
- nodePort: 31081
name: redis
port: 6379
protocol: TCP
targetPort: 6379
type: NodePort
# ４、redis-exporter-svc.yaml 暴露外部redis-exporter端口：31082
[root@xianchaomaster1 yaml]# cat redis-exporter-svc.yaml
kind: Service #service 类型
apiVersion: v1
metadata:
annotations:
prometheus.io/scrape: 'true'
prometheus.io/port: "9121"
name: redis-exporter-service
namespace: studylinux-net
spec:
selector:
app: redis
ports:
- nodePort: 31082
name: prom
port: 9121
protocol: TCP
targetPort: 9121
type: NodePort
# ５、kubectl apply -f
deployment.apps/redis created
service/redis-exporter-service created
service/redis-redis-service created
# ６、验证ｒｅｄｉｓ－ｅｘｐｏｒｔｅｒ ｍｅｔｒｉｃｓ
http://192.168.40.180:31082/metrics
# ７、Prometheus配置文件
[root@xianchaomaster1 yaml]# cat /apps/prometheus/prometheus.yml
- job_name: "prometheus-redis"
static_configs:
- targets: ["192.168.40.180:31082"]
# ８、grafana导入模板并验证－１４６１５模板
http://192.168.40.182:3000/dashboard/import
# ９、验证ｒｅｄｉｓ服务是否正常
[root@xianchaomaster1 yaml]# cat redis-redis-svc.yaml ^C
[root@xianchaomaster1 yaml]# telnet 192.168.40.180 31081
Trying 192.168.40.180...
Connected to 192.168.40.180.
Escape character is '^]'.
set name xiaenting
+OK
set age 31
+OK
set nums 310
+OK
key *
-ERR unknown command `key`, with args beginning with: `*`,
keys *
*3
$4
name
$4
nums
$3
age
五、Nginx
# Nginx virtual host traffic status module(nginx-module-vts)
#1、克隆nginx-module-vts 模块
[root@xksnode2 prometheus]# git clone https://github.com/vozlt/nginx-module-vts.git
Cloning into 'nginx-module-vts'...
remote: Enumerating objects: 1042, done.
remote: Counting objects: 100% (241/241), done.
remote: Compressing objects: 100% (79/79), done.
remote: Total 1042 (delta 172), reused 188 (delta 156), pack-reused 801
Receiving objects: 100% (1042/1042), 1.14 MiB | 373.00 KiB/s, done.
Resolving deltas: 100% (695/695), done.
#2、下载nginx 源码包
[root@xksnode2 prometheus]# wget http://nginx.org/download/nginx-1.20.2.tar.gz
--2023-04-28 11:46:25-- http://nginx.org/download/nginx-1.20.2.tar.gz
Resolving nginx.org (nginx.org)... 52.58.199.22, 3.125.197.172, 2a05:d014:edb:5704::6, ...
Connecting to nginx.org (nginx.org)|52.58.199.22|:80... connected.
HTTP request sent, awaiting response... 200 OK
Length: 1062124 (1.0M) [application/octet-stream]
Saving to: ‘nginx-1.20.2.tar.gz’
100%[================================================================================================================================================================================================================================================================================>] 1,062,124 390KB/s in 2.7s
2023-04-28 11:46:33 (390 KB/s) - ‘nginx-1.20.2.tar.gz’ saved [1062124/1062124]
[root@xksnode2 prometheus]# ll
total 100512
-rw-r--r-- 1 root root 1062124 Nov 16 2021 nginx-1.20.2.tar.gz
drwxr-xr-x 9 root root 177 Apr 28 11:45 nginx-module-vts
#3、进行编译nginx模板
[root@xksnode2 nginx-1.20.2]#tar zxvf nginx-1.20.2.tar.gz
[root@xksnode2 nginx-1.20.2]#cd nginx-1.20.2
[root@xksnode2 nginx-1.20.2]#yum -y install make zlib zlib-devel gcc-c++ libtool openssl openssl-devel pcre-devel
[root@xksnode2 nginx-1.20.2]#
./configure --prefix=/apps/nginx \
--with-http_ssl_module \
--with-http_v2_module \
--with-http_realip_module \
--with-http_stub_status_module \
--with-http_gzip_static_module \
--with-pcre \
--with-file-aio \
--with-stream \
--with-stream_ssl_module \
--with-stream_realip_module \
--add-module=/root/prometheus/nginx-module-vts/
[root@xksnode2 nginx-1.20.2]# make & make install
#4、配置状态页
[root@xksnode2 ~]# vim /apps/nginx/conf/nginx.conf
#gzip on;
vhost_traffic_status_zone; #启动状态页
location / {
root html;
index index.html index.htm;
proxy_pass http://192.168.19.180:9090;#反向代理到 Prometheus
}
location /status{
vhost_traffic_status_display;
vhost_traffic_status_display_format html;
}
[root@xksnode2 ~]# /apps/nginx/sbin/nginx -t
nginx: the configuration file /apps/nginx/conf/nginx.conf syntax is ok
nginx: configuration file /apps/nginx/conf/nginx.conf test is successful
[root@xksnode2 ~]# /apps/nginx/sbin/nginx
#5、验证访问状态页
#访问 status状态页 、 反向代理80端口到prometheus
http://192.168.40.182:80
http://192.168.40.182/status
#6、安装nginx exporter
GitHub：https://github.com/hnlq715/nginx-vts-exporter/releases
包下载地址：https://github.com/hnlq715/nginx-vts-exporter/releases/download/v0.10.3/nginx-vts-exporter-0.10.3.linux-amd64.tar.gz
[root@xksnode2 prometheus]# wget https://github.com/hnlq715/nginx-vts-exporter/releases/download/v0.10.3/nginx-vts-exporter-0.10.3.linux-amd64.tar.gz
[root@xksnode2 prometheus]# tar zxvf nginx-vts-exporter-0.10.3.linux-amd64.tar.gz
nginx-vts-exporter-0.10.3.linux-amd64/
nginx-vts-exporter-0.10.3.linux-amd64/LICENSE
nginx-vts-exporter-0.10.3.linux-amd64/nginx-vts-exporter
[root@xksnode2 prometheus]# cp nginx-vts-exporter-0.10.3.linux-amd64/nginx-vts-exporter /usr/local/bin/
[root@xksnode2 prometheus]# nginx-vts-exporter -nginx.scrape_uri http://192.168.40.182/status/format/json
2023/04/28 12:17:44 Starting nginx_vts_exporter (version=0.10.3, branch=HEAD, revision=8aa2881c7050d9b28f2312d7ce99d93458611d04)
2023/04/28 12:17:44 Build context (go=go1.10, user=root@56ca8763ee48, date=20180328-05:47:47)
2023/04/28 12:17:44 Starting Server at : :9913
2023/04/28 12:17:44 Metrics endpoint: /metrics
2023/04/28 12:17:44 Metrics namespace: nginx
2023/04/28 12:17:44 Scraping information from : http://192.168.19.182/status/format/json
#7、打开url 访问是否正常metics 端口：9913
[root@xksnode2 ~]# netstat -nltp
Proto Recv-Q Send-Q Local Address Foreign Address State PID/Program name
tcp6 0 0 :::9913 :::* LISTEN 113780/nginx-vts-ex
http://192.168.19.182:9913/metrics
#8、配置prometheus
[root@xksmaster1 yaml]# vim /apps/prometheus/prometheus.yml
- job_name: "prometheus-nginx"
static_configs:
- targets: ["192.168.19.182:9913"]
[root@xksmaster1 yaml]# systemctl restart prometheus
#9、配置Grafana
ID：https://grafana.com/grafana/dashboards/2949
六、Ingress Nginx
#部署 Ingress Nginx
#https://www.cnblogs.com/birkhoffxia/articles/17998192
#修改yaml文件 ports一段添加 10254端口
ports:
- containerPort: 10254
name: prometrics
protocol: TCP
#修改svc文件 添加下面配置 暴露10254到宿主机31254端口
[root@ca-k8s-master01 Ingress]# kubectl edit -n ingress-nginx svc ingress-nginx-controller^C
- appProtocol: http
name: prometheus-metrics-port
nodePort: 31254
port: 10254
protocol: TCP
targetPort: 10254
#192.168.40.51 -
[root@ca-k8s-master01 Ingress]# kubectl get svc -n ingress-nginx
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
ingress-nginx-controller LoadBalancer 10.105.225.205 192.168.40.51 80:30227/TCP,443:30795/TCP,10254:31254/TCP 8m40s
ingress-nginx-controller-admission ClusterIP 10.103.133.140 <none> 443/TCP 8m40s
#访问地址
http://192.168.40.51:10254/metrics
#配置Prometheus Ccofig文件
[root@ca-k8s-master01 case]# vim case3-1-prometheus-cfg.yaml
- job_name: "prometheus-Ingress-nginx"
static_configs:
- targets: ["192.168.40.51:10254"]
[root@ca-k8s-master01 case]# kubectl apply -f case3-1-prometheus-cfg.yaml
configmap/prometheus-config configured
[root@ca-k8s-master01 case]# kubectl delete -f case3-2-prometheus-deployment.yaml
deployment.apps "prometheus-server" deleted
[root@ca-k8s-master01 case]# kubectl apply -f case3-2-prometheus-deployment.yaml
deployment.apps/prometheus-server created
#添加Grafana 模板 9614 没有添加过Ingress 规则 无数据 可以自己手动添加 Ingress规则
浙公网安备 33010602011771号