Prometheus docker-compose安装部署文档

备注：以下操作均在root权限下进行的操作

安装前置环境：

已安装docker

已安装docker-compose

在/opt目录新建目录prom

在对应的prom目录下分别创建prometheus、alertmanager，grafana相关目录

  1. mkdir -p /opt/prom/{prometheus,prometheus/data,alertmanager,grafana}
  2. #prom目录结构
  3. prom/
  4. ├── alertmanager
  5. ├── grafana
  6. └── prometheus
  7.     └── data

赋予相关目录对应的权限

chmod 777 /opt/prom/{prometheus/data,grafana}

编写docker-compose.yml文件，放在目录/opt/prom下

  1. touch /opt/prom/docker-compose.yml
  2. vim /opt/prom/docker-compose.yml

docker-compose.yml文件内容

  1. version: '3.3'
  2.  
  3. services:
  4.   node-exporter:
  5.     image: prom/node-exporter:latest
  6.     ports:
  7.       - "9101:9100"         #node-exporter默认端口9100，不过由于metersphere的node-exporter已经占用了9100了，这里调整为9101
  8.     networks:
  9.       - prom
  10.  
  11.   dingtalk:
  12.     image: timonwong/prometheus-webhook-dingtalk:latest
  13.     volumes:
  14.       - type: bind
  15.         source: ./alertmanager/config.yml
  16.         target: /etc/prometheus-webhook-dingtalk/config.yml
  17.         read_only: true
  18.     ports:
  19.       - "8060:8060"
  20.     networks:
  21.       - prom
  22.  
  23.   alertmanager:
  24.     depends_on:
  25.       - dingtalk
  26.     image: prom/alertmanager:latest
  27.     volumes:
  28.       - type: bind
  29.         source: ./alertmanager/alertmanager.yml
  30.         target: /etc/alertmanager/alertmanager.yml
  31.         read_only: true
  32.     ports:
  33.       - "9093:9093"
  34.       - "9094:9094"
  35.     networks:
  36.       - prom
  37.  
  38.   prometheus:
  39.     depends_on:
  40.       - alertmanager
  41.     image: prom/prometheus:latest
  42.     volumes:
  43.       - type: bind
  44.         source: ./prometheus/prometheus.yml
  45.         target: /etc/prometheus/prometheus.yml
  46.         read_only: true
  47.       - type: bind
  48.         source: ./prometheus/alert-rules.yml
  49.         target: /etc/prometheus/alert-rules.yml
  50.         read_only: true
  51.       - type: volume
  52.         source: prometheus
  53.         target: /prometheus
  54.     ports:
  55.       - "9200:9090"         #prometheus默认端口9090，不过ms-prometheus占用该端口，这里换成9200
  56.     networks:
  57.       - prom
  58.  
  59.   grafana:
  60.     depends_on:
  61.       - prometheus
  62.     image: grafana/grafana:latest
  63.     volumes:
  64.       - type: volume
  65.         source: grafana
  66.         target: /var/lib/grafana
  67.     ports:
  68.       - "3000:3000"
  69.     networks:
  70.       - prom
  71.  
  72. volumes:
  73.   prometheus:
  74.     driver: local
  75.     driver_opts:
  76.       type: none
  77.       o: bind
  78.       device: /opt/prom/prometheus/data
  79.   grafana:
  80.     driver: local
  81.     driver_opts:
  82.       type: none
  83.       o: bind
  84.       device: /opt/prom/grafana
  85.  
  86. networks:
  87.   prom:
  88.     driver: bridge

编写prometheus的配置文件prometheus.yml，放在/opt/prom/prometheus目录下

  1. touch /opt/prom/prometheus/prometheus.yml
  2. vim /opt/prom/prometheus/prometheus.yml
  1. global:
  2.   scrape_interval:     15s
  3.   evaluation_interval: 15s
  4.  
  5. alerting:
  6.   alertmanagers:
  7.   - static_configs:
  8.     - targets:
  9.       - alertmanager:9093
  10.  
  11. rule_files:
  12.   - "*rules.yml"
  13.  
  14. scrape_configs:
  15.   - job_name: 'prometheus'
  16.     static_configs:
  17.     - targets: ['prometheus:9091']
  18.  
  19.   - job_name: 'node_prom_localhost'
  20.     static_configs:
  21.     - targets: ['node-exporter:9101']
  22.  
  23.   - job_name: 'alertmanager'
  24.     static_configs:
  25.     - targets: ['alertmanager:9093']

编写告警规则alert-rules.yml文件，放在目录/opt/prom/prometheus下

  1. touch /opt/prom/prometheus/alert-rules.yml
  2. vim /opt/prom/prometheus/alert-rules.yml
  1. groups:
  2.   - name: node-alert
  3.     rules:
  4.     - alert: NodeDown
  5.       expr: up{job="node"} == 0
  6.       for: 5m
  7.       labels:
  8.         severity: critical
  9.         instance: "{{ $labels.instance }}"
  10.       annotations:
  11.         summary: "instance: {{ $labels.instance }} down"
  12.         description: "Instance: {{ $labels.instance }} 已经宕机 5分钟"
  13.         value: "{{ $value }}"
  14.  
  15.     - alert: NodeCpuHigh
  16.       expr: (1 - avg by (instance) (irate(node_cpu_seconds_total{job="node",mode="idle"}[5m]))) * 100 > 80
  17.       for: 5m
  18.       labels:
  19.         severity: warning
  20.         instance: "{{ $labels.instance }}"
  21.       annotations:
  22.         summary: "instance: {{ $labels.instance }} cpu使用率过高"
  23.         description: "CPU 使用率超过 80%"
  24.         value: "{{ $value }}"
  25.  
  26.     - alert: NodeCpuIowaitHigh
  27.       expr: avg by (instance) (irate(node_cpu_seconds_total{job="node",mode="iowait"}[5m])) * 100 > 50
  28.       for: 5m
  29.       labels:
  30.         severity: warning
  31.         instance: "{{ $labels.instance }}"
  32.       annotations:
  33.         summary: "instance: {{ $labels.instance }} cpu iowait 使用率过高"
  34.         description: "CPU iowait 使用率超过 50%"
  35.         value: "{{ $value }}"
  36.  
  37.     - alert: NodeLoad5High
  38.       expr: node_load5 > (count by (instance) (node_cpu_seconds_total{job="node",mode='system'})) * 1.2
  39.       for: 5m
  40.       labels:
  41.         severity: warning
  42.         instance: "{{ $labels.instance }}"
  43.       annotations:
  44.         summary: "instance: {{ $labels.instance }} load(5m) 过高"
  45.         description: "Load(5m) 过高，超出cpu核数 1.2倍"
  46.         value: "{{ $value }}"
  47.  
  48.     - alert: NodeMemoryHigh
  49.       expr: (1 - node_memory_MemAvailable_bytes{job="node"} / node_memory_MemTotal_bytes{job="node"}) * 100 > 90
  50.       for: 5m
  51.       labels:
  52.         severity: warning
  53.         instance: "{{ $labels.instance }}"
  54.       annotations:
  55.         summary: "instance: {{ $labels.instance }} memory 使用率过高"
  56.         description: "Memory 使用率超过 90%"
  57.         value: "{{ $value }}"
  58.  
  59.     - alert: NodeDiskRootHigh
  60.       expr: (1 - node_filesystem_avail_bytes{job="node",fstype=~"ext.*|xfs",mountpoint ="/"} / node_filesystem_size_bytes{job="node",fstype=~"ext.*|xfs",mountpoint ="/"}) * 100 > 90
  61.       for: 10m
  62.       labels:
  63.         severity: warning
  64.         instance: "{{ $labels.instance }}"
  65.       annotations:
  66.         summary: "instance: {{ $labels.instance }} disk(/ 分区) 使用率过高"
  67.         description: "Disk(/ 分区) 使用率超过 90%"
  68.         value: "{{ $value }}"
  69.  
  70.     - alert: NodeDiskBootHigh
  71.       expr: (1 - node_filesystem_avail_bytes{job="node",fstype=~"ext.*|xfs",mountpoint ="/boot"} / node_filesystem_size_bytes{job="node",fstype=~"ext.*|xfs",mountpoint ="/boot"}) * 100 > 80
  72.       for: 10m
  73.       labels:
  74.         severity: warning
  75.         instance: "{{ $labels.instance }}"
  76.       annotations:
  77.         summary: "instance: {{ $labels.instance }} disk(/boot 分区) 使用率过高"
  78.         description: "Disk(/boot 分区) 使用率超过 80%"
  79.         value: "{{ $value }}"
  80.  
  81.     - alert: NodeDiskReadHigh
  82.       expr: irate(node_disk_read_bytes_total{job="node"}[5m]) > 20 * (1024 ^ 2)
  83.       for: 5m
  84.       labels:
  85.         severity: warning
  86.         instance: "{{ $labels.instance }}"
  87.       annotations:
  88.         summary: "instance: {{ $labels.instance }} disk 读取字节数 速率过高"
  89.         description: "Disk 读取字节数 速率超过 20 MB/s"
  90.         value: "{{ $value }}"
  91.  
  92.     - alert: NodeDiskWriteHigh
  93.       expr: irate(node_disk_written_bytes_total{job="node"}[5m]) > 20 * (1024 ^ 2)
  94.       for: 5m
  95.       labels:
  96.         severity: warning
  97.         instance: "{{ $labels.instance }}"
  98.       annotations:
  99.         summary: "instance: {{ $labels.instance }} disk 写入字节数 速率过高"
  100.         description: "Disk 写入字节数 速率超过 20 MB/s"
  101.         value: "{{ $value }}"
  102.  
  103.     - alert: NodeDiskReadRateCountHigh
  104.       expr: irate(node_disk_reads_completed_total{job="node"}[5m]) > 3000
  105.       for: 5m
  106.       labels:
  107.         severity: warning
  108.         instance: "{{ $labels.instance }}"
  109.       annotations:
  110.         summary: "instance: {{ $labels.instance }} disk iops 每秒读取速率过高"
  111.         description: "Disk iops 每秒读取速率超过 3000 iops"
  112.         value: "{{ $value }}"
  113.  
  114.     - alert: NodeDiskWriteRateCountHigh
  115.       expr: irate(node_disk_writes_completed_total{job="node"}[5m]) > 3000
  116.       for: 5m
  117.       labels:
  118.         severity: warning
  119.         instance: "{{ $labels.instance }}"
  120.       annotations:
  121.         summary: "instance: {{ $labels.instance }} disk iops 每秒写入速率过高"
  122.         description: "Disk iops 每秒写入速率超过 3000 iops"
  123.         value: "{{ $value }}"
  124.  
  125.     - alert: NodeInodeRootUsedPercentHigh
  126.       expr: (1 - node_filesystem_files_free{job="node",fstype=~"ext4|xfs",mountpoint="/"} / node_filesystem_files{job="node",fstype=~"ext4|xfs",mountpoint="/"}) * 100 > 80
  127.       for: 10m
  128.       labels:
  129.         severity: warning
  130.         instance: "{{ $labels.instance }}"
  131.       annotations:
  132.         summary: "instance: {{ $labels.instance }} disk(/ 分区) inode 使用率过高"
  133.         description: "Disk (/ 分区) inode 使用率超过 80%"
  134.         value: "{{ $value }}"
  135.  
  136.     - alert: NodeInodeBootUsedPercentHigh
  137.       expr: (1 - node_filesystem_files_free{job="node",fstype=~"ext4|xfs",mountpoint="/boot"} / node_filesystem_files{job="node",fstype=~"ext4|xfs",mountpoint="/boot"}) * 100 > 80
  138.       for: 10m
  139.       labels:
  140.         severity: warning
  141.         instance: "{{ $labels.instance }}"
  142.       annotations:
  143.         summary: "instance: {{ $labels.instance }} disk(/boot 分区) inode 使用率过高"
  144.         description: "Disk (/boot 分区) inode 使用率超过 80%"
  145.         value: "{{ $value }}"
  146.  
  147.     - alert: NodeFilefdAllocatedPercentHigh
  148.       expr: node_filefd_allocated{job="node"} / node_filefd_maximum{job="node"} * 100 > 80
  149.       for: 10m
  150.       labels:
  151.         severity: warning
  152.         instance: "{{ $labels.instance }}"
  153.       annotations:
  154.         summary: "instance: {{ $labels.instance }} filefd 打开百分比过高"
  155.         description: "Filefd 打开百分比 超过 80%"
  156.         value: "{{ $value }}"
  157.  
  158.     - alert: NodeNetworkNetinBitRateHigh
  159.       expr: avg by (instance) (irate(node_network_receive_bytes_total{device=~"eth0|eth1|ens33|ens37"}[1m]) * 8) > 20 * (1024 ^ 2) * 8
  160.       for: 3m
  161.       labels:
  162.         severity: warning
  163.         instance: "{{ $labels.instance }}"
  164.       annotations:
  165.         summary: "instance: {{ $labels.instance }} network 接收比特数 速率过高"
  166.         description: "Network 接收比特数 速率超过 20MB/s"
  167.         value: "{{ $value }}"
  168.  
  169.     - alert: NodeNetworkNetoutBitRateHigh
  170.       expr: avg by (instance) (irate(node_network_transmit_bytes_total{device=~"eth0|eth1|ens33|ens37"}[1m]) * 8) > 20 * (1024 ^ 2) * 8
  171.       for: 3m
  172.       labels:
  173.         severity: warning
  174.         instance: "{{ $labels.instance }}"
  175.       annotations:
  176.         summary: "instance: {{ $labels.instance }} network 发送比特数 速率过高"
  177.         description: "Network 发送比特数 速率超过 20MB/s"
  178.         value: "{{ $value }}"
  179.  
  180.     - alert: NodeNetworkNetinPacketErrorRateHigh
  181.       expr: avg by (instance) (irate(node_network_receive_errs_total{device=~"eth0|eth1|ens33|ens37"}[1m])) > 15
  182.       for: 3m
  183.       labels:
  184.         severity: warning
  185.         instance: "{{ $labels.instance }}"
  186.       annotations:
  187.         summary: "instance: {{ $labels.instance }} 接收错误包 速率过高"
  188.         description: "Network 接收错误包 速率超过 15个/秒"
  189.         value: "{{ $value }}"
  190.  
  191.     - alert: NodeNetworkNetoutPacketErrorRateHigh
  192.       expr: avg by (instance) (irate(node_network_transmit_packets_total{device=~"eth0|eth1|ens33|ens37"}[1m])) > 15
  193.       for: 3m
  194.       labels:
  195.         severity: warning
  196.         instance: "{{ $labels.instance }}"
  197.       annotations:
  198.         summary: "instance: {{ $labels.instance }} 发送错误包 速率过高"
  199.         description: "Network 发送错误包 速率超过 15个/秒"
  200.         value: "{{ $value }}"
  201.  
  202.     - alert: NodeProcessBlockedHigh
  203.       expr: node_procs_blocked{job="node"} > 10
  204.       for: 10m
  205.       labels:
  206.         severity: warning
  207.         instance: "{{ $labels.instance }}"
  208.       annotations:
  209.         summary: "instance: {{ $labels.instance }} 当前被阻塞的任务的数量过多"
  210.         description: "Process 当前被阻塞的任务的数量超过 10个"
  211.         value: "{{ $value }}"
  212.  
  213.     - alert: NodeTimeOffsetHigh
  214.       expr: abs(node_timex_offset_seconds{job="node"}) > 3 * 60
  215.       for: 2m
  216.       labels:
  217.         severity: info
  218.         instance: "{{ $labels.instance }}"
  219.       annotations:
  220.         summary: "instance: {{ $labels.instance }} 时间偏差过大"
  221.         description: "Time 节点的时间偏差超过 3m"
  222.         value: "{{ $value }}"

编写alertmanager配置文件alertmanager.yml

  1. touch /opt/prom/alertmanager/alertmanager.yml
  2. vim /opt/prom/alertmanager/alertmanager.yml
  1. global:
  2.   resolve_timeout: 5m
  3.   smtp_smarthost: 'smtp.163.com:465'                #邮箱smtp服务器代理，启用SSL发信, 端口一般是465
  4.   smtp_from: 'test@163.com'                         #发送邮箱名称
  5.   smtp_auth_username: 'test@163.com'                #邮箱名称
  6.   smtp_auth_password: 'test@123'                    #邮箱密码或授权码
  7.   smtp_require_tls: false
  8.  
  9. route:
  10.   receiver: 'default'
  11.   group_wait: 10s
  12.   group_interval: 1m
  13.   repeat_interval: 1h
  14.   group_by: ['alertname']
  15.  
  16. inhibit_rules:
  17. - source_match:
  18.     severity: 'critical'
  19.   target_match:
  20.     severity: 'warning'
  21.   equal: ['alertname', 'instance']
  22.  
  23. receivers:
  24. - name: 'default'
  25.   email_configs:
  26.   - to: 'receiver@163.com'
  27.     send_resolved: true
  28.   webhook_configs:
  29.   - url: 'http://dingtalk:8060/dingtalk/webhook/send'       #这里是钉钉的接口
  30.     send_resolved: true

编写钉钉告警相关配置文件config.yml放在/opt/prom/alertmanager/目录下

  1. touch /opt/prom/alertmanager/config.yml
  2. vim /opt/prom/alertmanager/config.yml
  1. targets:
  2.   webhook:
  3.     url: https://oapi.dingtalk.com/robot/send?access_token=xxxxxxxxxxxx             #修改为钉钉机器人的webhook
  4.     mention:
  5.       all: true

切到/opt/prom目录下，使用docker-compose up -d去起相关服务

docker-compose up -d

posted on 2022-09-23 18:14  砖头哥-  阅读(502)  评论(0)    收藏  举报
博客园  ©  2004-2025
浙公网安备 33010602011771号 浙ICP备2021040463号-3