Prometheus安装
cd /usr/src/
wget https://github.com/prometheus/prometheus/releases/download/v2.12.0/prometheus-2.12.0.linux-amd64.tar.gz
tar xf prometheus-2.12.0.linux-amd64.tar.gz -C /usr/local/
cd /usr/local/
ln -s prometheus-2.12.0.linux-amd64 prometheus
设置systemctl prometheus 启动
cat > /usr/lib/systemd/system/prometheus.service << EOF
[Unit]
Description=Prometheus: the monitoring system
Documentation=http://prometheus.io/docs/
[Service]
ExecStart=/usr/local/prometheus/prometheus --config.file=/usr/local/prometheus/prometheus.yml
Restart=always
StartLimitInterval=0
RestartSec=10
[Install]
WantedBy=multi-user.target
EOF
systemctl enable prometheus ##制作开机启动prometheus
systemctl start prometheus ##启动prometheus
systemctl status prometheus ##查看promethus状态
node_exporter安装部署 -> promethus依赖node_exporter来采集信息
cd /usr/src/
wget https://github.com/prometheus/node_exporter/releases/download/v0.18.1/node_exporter-0.18.1.linux-amd64.tar.gz
tar xf node_exporter-0.18.1.linux-amd64.tar.gz -C /usr/local/
cd /usr/local/
ln -s node_exporter-0.18.1.linux-amd64 node_exporter
制作systemctl方式启动node_exporter
cat > /usr/lib/systemd/system/node_exporter.service << EOF
[Unit]
Description=Prometheus node exporter
After=local-fs.target network-online.target network.target
Wants=local-fs.target network-online.target network.target
[Service]
Restart=on-failure
ExecStart="/usr/local/node_exporter/node_exporter"
[Install]
WantedBy=multi-user.target
EOF
制作node_exporter服务启动
systemctl enable node_exporter.service ##制作开机启动node_exporter
systemctl start node_exporter.service ##启动node_exporter
systemctl status node_exporter.service ##查看node_exporter状态
访问方式
http://localhost:9090
获取主机信息
curl http://localhost:9090/metrics
如需要设置报警面板显示和监控多台机器可参考142机器的promethus.yml
# my global config
global:
scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
# scrape_timeout is set to the global default (10s).
# Alertmanager configuration
alerting:
alertmanagers:
- static_configs:
- targets:
- 127.0.0.1:9093
# - alertmanager:9093
# Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
rule_files:
- "rules/*.yml"
# - "first_rules.yml"
# - "second_rules.yml"
# A scrape configuration containing exactly one endpoint to scrape:
# Here it's Prometheus itself.
scrape_configs:
# The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.
- job_name: 'prometheus'
# metrics_path defaults to '/metrics'
# scheme defaults to 'http'.
static_configs:
- targets: ['192.168.1.140:9090']
- job_name: 'promethues-node'
static_configs:
- targets:
- 192.168.1.140:9100
- 192.168.1.137:9100
- 192.168.1.57:9100
- 192.168.1.141:9100
- 192.168.1.60:9100
- 192.168.1.201:9100
- targets: ['192.168.1.59:9100']
labels:
instance: dataexa-insight-59
- job_name: 'jmx'
static_configs:
- targets:
- 192.168.1.59:3010
- job_name: pushgateway
static_configs:
- targets: ['192.168.191.159:9091']
labels:
instance: pushgateway
jvm 监控
# 资料来源 https://www.jianshu.com/p/adada9c1f7dd
wget https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.3.1/jmx_prometheus_javaagent-0.3.1.jar
# java -javaagent:/usr/local/prometheus/jmx_exporter/jmx_prometheus_javaagent-0.3.1.jar=3010:/usr/local/prometheus/jmx_exporter/jmx_exporter.yml -jar yourJar.jar
报警规则编写
需要在promethus.yml的同级目录下创建rules --> mkdir rules
cat warining.yml
groups:
- name: 主机状态-监控告警
rules:
- alert: 主机状态
expr: up == 0
for: 1m
labels:
status: 非常严重
annotations:
summary: "{{$labels.instance}}:服务器宕机"
description: "{{$labels.instance}}:服务器延时超过5分钟"
- alert: CPU使用情况
expr: 100-(avg(irate(node_cpu_seconds_total{mode="idle"}[5m])) by(instance)* 100) > 80
for: 1m
labels:
status: 一般告警
annotations:
summary: "{{$labels.mountpoint}} CPU使用率过高！"
description: "{{$labels.mountpoint }} CPU使用大于80%(目前使用:{{$value}}%)"
- alert: 内存使用
expr: 100 -(node_memory_MemTotal_bytes -node_memory_MemFree_bytes+node_memory_Buffers_bytes+node_memory_Cached_bytes ) / node_memory_MemTotal_bytes * 100> 80
for: 1m
labels:
status: 严重告警
annotations:
summary: "{{$labels.mountpoint}} 内存使用率过高！"
description: "{{$labels.mountpoint }} 内存使用大于80%(目前使用:{{$value}}%)"
- alert: IO性能
expr: 100-(avg(irate(node_disk_io_time_seconds_total[1m])) by(instance)* 100) < 60
for: 1m
labels:
status: 严重告警
annotations:
summary: "{{$labels.mountpoint}} 流入磁盘IO使用率过高！"
description: "{{$labels.mountpoint }} 流入磁盘IO大于60%(目前使用:{{$value}})"
- alert: 网络
expr: ((sum(rate (node_network_receive_bytes_total{device!~'tap.*|veth.*|br.*|docker.*|virbr*|lo*'}[5m])) by (instance)) / 100) > 102400
for: 1m
labels:
status: 严重告警
annotations:
summary: "{{$labels.mountpoint}} 流入网络带宽过高！"
description: "{{$labels.mountpoint }}流入网络带宽持续2分钟高于100M. RX带宽使用率{{$value}}"
- alert: 网络
expr: ((sum(rate (node_network_transmit_bytes_total{device!~'tap.*|veth.*|br.*|docker.*|virbr*|lo*'}[5m])) by (instance)) / 100) > 102400
for: 1m
labels:
status: 严重告警
annotations:
summary: "{{$labels.mountpoint}} 流出网络带宽过高！"
description: "{{$labels.mountpoint }}流出网络带宽持续2分钟高于100M. RX带宽使用率{{$value}}"
- alert: TCP会话
expr: node_netstat_Tcp_CurrEstab > 1000
for: 1m
labels:
status: 严重告警
annotations:
summary: "{{$labels.mountpoint}} TCP_ESTABLISHED过高！"
description: "{{$labels.mountpoint }} TCP_ESTABLISHED大于1000%(目前使用:{{$value}}%)"
- alert: 磁盘容量
expr: 100-(node_filesystem_free_bytes{fstype=~"ext4|xfs"}/node_filesystem_size_bytes {fstype=~"ext4|xfs"}*100) > 80
for: 1m
labels:
status: 严重告警
annotations:
summary: "{{$labels.mountpoint}} 磁盘分区使用率过高！"
description: "{{$labels.mountpoint }} 磁盘分区使用大于80%(目前使用:{{$value}}%)"
安装grafana
cd /usr/local/src/
wget https://dl.grafana.com/oss/release/grafana-5.4.3-1.x86_64.rpm
yum localinstall grafana-5.4.3-1.x86_64.rpm
#启动服务
systemctl start grafana-server
#查看服务是否正常启动
systemctl status grafana-server
#自启动
systemctl enable grafana-server
访问
浏览器访问http://localhost:3000
grafana网页操作
https://www.cnblogs.com/zhaojiedi1992/p/zhaojiedi_liunx_64_prometheus_granafa.html
监控gpu
url：https://github.com/NVIDIA/gpu-monitoring-tools/tree/master/exporters/prometheus-dcgm
实际操作：
docker run -d --runtime=nvidia --name=nvidia-dcgm-exporter -v /run/prometheus:/run/prometheus nvidia/dcgm-exporter
docker run -d --net="host" --pid="host" --volumes-from nvidia-dcgm-exporter:ro quay.io/prometheus/node-exporter --collector.textfile.directory="/run/prometheus"
启动的三个服务
systemctl start prometheus
systemctl start node_exporter
systemctl start grafana-server
alertmanager报警插件安装
wget https://github.com/prometheus/alertmanager/releases/download/v0.19.0/alertmanager-0.19.0.linux-amd64.tar.gz
tar xf alertmanager-0.19.0.linux-amd64.tar.gz -C /usr/local
mv alertmanager-0.19.0.linux-amd64 alertmanager
/usr/local/alertmanager/bin/ alertmanager #启动
# 配置报警文件
cat alertmanager.yml
global:
resolve_timeout: 5m
route:
group_by: ['alertname']
group_wait: 10s
group_interval: 10s
repeat_interval: 1h
receiver: 'web.hook'
receivers:
- name: 'web.hook'
webhook_configs:
- url: 'http://localhost:8060/dingtalk/webhook/send'
钉钉报警
下载dingtalk进行报警
资料来源 https://www.codetd.com/article/6798984
下载好之后 选择使用markdown格式的报警格式
cat > /usr/local/gocode/src/github.com/timonwong/prometheus-webhook-dingtalk/template << EOF
{{ define "__subject" }}[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ .GroupLabels.SortedPairs.Values | join " " }} {{ if gt (len .CommonLabels) (len .GroupLabels) }}({{ with .CommonLabels.Remove .GroupLabels.Names }}{{ .Values | join " " }}{{ end }}){{ end }}{{ end }}
{{ define "__alertmanagerURL" }}{{ .ExternalURL }}/#/alerts?receiver={{ .Receiver }}{{ end }}
{{ define "__text_alert_list" }}{{ range . }}
**Labels**
{{ range .Labels.SortedPairs }}> - {{ .Name }}: {{ .Value | markdown | html }}
{{ end }}
**Annotations**
{{ range .Annotations.SortedPairs }}> - {{ .Name }}: {{ .Value | markdown | html }}
{{ end }}
**Source:** [{{ .Annotations.summary }}]({{ .GeneratorURL }})
{{ end }}{{ end }}
{{ define "ding.link.title" }}{{ template "__subject" . }}{{ end }}
{{ define "ding.link.content" }}#### \[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}\] **[{{ index .GroupLabels "alertname" }}]({{ template "__alertmanagerURL" . }})**
{{ template "__text_alert_list" .Alerts.Firing }}
{{ end }}
#dingtalk插件 指定格式模板和钉钉接口来启动
nohup prometheus-webhook-dingtalk --template.file="/usr/local/gocode/src/github.com/timonwong/prometheus-webhook-dingtalk/template/default.tmpl" --ding.profile="webhook=https://oapi.dingtalk.com/robot/send?access_token=64a517b7a1d0ad2dc23exxxx00fe18b0b4e15491f179456f94b6ff5" 2>&1 1>dingding.log &
钉钉报警群设置
只需要设置好公网ip即可
自定义监控项pushgateway
wget https://github.com/prometheus/pushgateway/releases/download/v0.10.0/pushgateway-0.10.0.linux-amd64.tar.gz
tar xf pushgateway-0.10.0.linux-amd64.tar.gz -C /usr/local
mv /usr/local/pushgateway-0.10.0.linux-amd64 /usr/local/pushgateway
/usr/local/pushgateway/bin/pushgateway #启动
使用脚本来获取机器值
cat count_netstat_wait_connections.sh
#!/bin/bash
instance_name=`hostname -f | cut -d'.' -f1` #获取本机名，用于后面的的标签
label="count_netstat_wait_connections" #定义key名
count_netstat_wait_connections=`netstat -an | grep -i wait | wc -l` #获取数据的命令
echo "$label: $count_netstat_wait_connections"
echo "$label $count_netstat_wait_connections" | curl --data-binary @- http://localhost:9091/metrics/job/pushgateway_test/instance/$instance_name
#这里pushgateway_test就是prometheus主配置文件里job的名字，需要保持一致，这样数据就会推送给这个job。后面的instance则是指定机器名，使用的就是脚本里获取的那个
promethus页面查看值
在promethus页面查询这个变量 count_netstat_wait_connections 即可获取到值
浙公网安备 33010602011771号