prometheus服务的客户端

环境

服务端 192.168.164.110
客户端 192.168.164.111

1 添加主机到prometheus服务

# 客户端运行node-exporter 
docker run -itd \
-p 9100:9100 \
-v /:/rootfs:ro \
-v /sys:/sys:ro \
--name mon_node-exporter \
--hostname mon_node-exporter \
--restart always \
prom/node-exporter:latest
# 客户端检查是否收集了数据
http://192.168.164.111:9100/metrics
# 服务器端添加配置链接客户端配置
vim prometheus.yml
cat >prometheus.yml<<EOF
# 全局配置
global:
  scrape_interval: 15s # 采集数据间隔时间
  evaluation_interval: 15s # 评估规则间隔时间
# 报警配置
alerting:
  alertmanagers:
    - static_configs:
      - targets: ['192.168.164.110:9093']
# 规则文件
rule_files:
  - "node_down.yml"
# 定义监控的目标 每个 'job' 代表一组具有共同目的的目标。
scrape_configs:
  # 1 监控 Prometheus 服务自身
  - job_name: 'prometheus'
    static_configs:
      - targets: ['localhost:9090']
  # 2 监控 nodes node_exporter,node_exporter 提供操作系统和硬件相关的指标
  - job_name: 'mon_node-exporter'
    static_configs:
      - targets: ['192.168.164.110:9100']
      - targets: ['192.168.164.110:9100']
  # 3 监控 nodes docker节点容器
  - job_name: 'mon_docker-exporter'
    static_configs:
      - targets: ['192.168.164.110:8080']
      - targets: ['192.168.164.110:8080']
  # 4 监控 GPU 资源
  - job_name: 'mon_gpu-exporter'
    static_configs:
      - targets: ['192.168.164.110:9835']
   # 5 监控 mysql服务 
  - job_name: 'mon_mysql-exporter'
    static_configs:
      - targets: ['192.168.164.110:9104']
EOF
# 重启服务端prometheus容器
docker restart mon_prometheus
# 服务器端检查链接信息状态
http://192.168.164.110:9090/targets

# 下载 主机模板 
  https://grafana.com/grafana/dashboards
  搜索 Node Exporter 并下载
# prometheus服务端导入模板
  分类 --Manage-- import --Upload json file

2 添加docker主机到prometheus

# 服务器端检查链接信息状态 是否有docker主机
  http://192.168.164.110:9090/targets
# 下载docker模板 
  https://grafana.com/grafana/dashboards
  搜索 Docker monitoring with node selection 并下载json文件
# prometheus服务端导入模板
  分类 --Manage-- import --Upload json file

3 添加GPU主机到prometheus服务

# 前提安装gpu驱动
# 在有gpu主机安装docker-nvidia2 < docker-version 19.3
# 安装 docker-nvidia2 docker-version < 19.3
  curl -s -L https://nvidia.github.io/nvidia-docker/gpgkey | apt-key add -
distribution=$(. /etc/os-release;echo $ID$VERSION_ID)
  curl -s -L https://nvidia.github.io/nvidia-docker/$distribution/nvidia-docker.list |\
tee /etc/apt/sources.list.d/nvidia-docker.list
  apt-get update
  apt-get install -y nvidia-docker2
  pkill -SIGHUP dockerd
# 安装 nvidia-container-toolkit docker-version >= 19.3
  distribution=$(. /etc/os-release;echo $ID$VERSION_ID)
  curl -s -L https://nvidia.github.io/nvidia-docker/gpgkey | sudo apt-key add -
  curl -s -L https://nvidia.github.io/nvidia-docker/$distribution/nvidia-docker.list |\
tee /etc/apt/sources.list.d/nvidia-docker.list
 apt update
 apt install -y nvidia-container-toolkit
# 在GPU客户端 安装 nvidia_gpu_exporter
# 下载对应系统发行版的包
wget https://github.com/utkuozdemir/nvidia_gpu_exporter/releases/download/v1.2.0/nvidia_gpu_exporter_1.2.0_linux_x86_64.tar.gz
# 解压并运行
tar xf nvidia_gpu_exporter_1.2.0_linux_x86_64.tar.gz
mv nvidia_gpu_exporter /usr/local/gpu-exporter/
/usr/local/gpu-exporter/nvidia_gpu_exporter &
# 验证是否运行成功
curl localhost:9835/metrics
# 配置 Prometheus 抓取 GPU 数据
vim prometheus.yml
# 下载展示GPU模版 json file
  https://grafana.com/grafana/dashboards
  搜索 Nvidia GPU Metrics 并下载json文件
# prometheus服务端导入模板
  分类 --Manage-- import --Upload json file

4 添加mysql服务监控

# mysql主机运行监控容器
docker run -itd \
  -p 9104:9104 \
  --name mon_mysqld-exporter \
  --hostname mon_mysqld-exporter \
  --restart="always" \
  -e DATA_SOURCE_NAME="root:Kc@123456@(192.168.164.111:3306)/" \
  prom/mysqld-exporter
# 客户端检查
http://192.168.164.111:9104/metrics
# 服务端检查
# 下载模板 导入
搜索 MySQL Exporter Quickstart and Dashboard