loki 日志监控
1、创建目录
#loki数据目录
sudo mkdir -p /data/promthues/loki/loki-data
#采集agent 目录
sudo mkdir -p /data/promthues/loki/promtail/data
#存放告警规则目录
sudo mkdir -p /data/promthues/loki/loki-data/rules/fake
#告警规则临时目录
sudo mkdir -p /data/promthues/loki/loki-data/rules-temp2、创建compose.yml
cat docker-compose.yml
services:
loki:
image: grafana/loki:latest
container_name: loki
ports:
- "3100:3100" # Loki API 端口
volumes:
- ./loki-config.yaml:/etc/loki/loki-config.yaml
- ./loki-data:/loki
command: -config.file=/etc/loki/loki-config.yaml -target=all
restart: unless-stopped
promtail:
image: grafana/promtail:latest
container_name: promtail
volumes:
- ./promtail-config.yaml:/etc/promtail/promtail-config.yaml
- /var/log:/var/log:ro # 采集系统日志
- ./promtail/data:/tmp/promtail
- /app/docker/containers:/var/lib/docker/containers:ro # 如需采集容器日志
- /var/run/docker.sock:/var/run/docker.sock
command: -config.file=/etc/promtail/promtail-config.yaml
restart: unless-stopped
depends_on:
- loki3、编辑 loki-config.yaml
cat loki-config.yaml
auth_enabled: false
target: all
server:
http_listen_port: 3100
common:
ring:
instance_addr: 0.0.0.0
kvstore:
store: inmemory
replication_factor: 1
schema_config:
configs:
- from: 2026-07-01
store: tsdb
object_store: filesystem
schema: v13
index:
prefix: index_
period: 24h
storage_config:
tsdb_shipper:
active_index_directory: ./loki/tsdb-index
cache_location: ./loki/tsdb-cache
filesystem:
directory: ./loki/chunks
ingester:
wal:
enabled: true
dir: /loki/wal
lifecycler:
ring:
kvstore:
store: inmemory
replication_factor: 1
final_sleep: 0s
compactor:
working_directory: ./loki/compactor
limits_config:
reject_old_samples: true
reject_old_samples_max_age: 168h
table_manager:
retention_deletes_enabled: true
retention_period: 168h # 日志保留7天
# 如果你要用 Loki 的告警功能，取消下面注释
ruler:
storage:
type: local
local:
directory: /loki/rules
rule_path: /loki/rules-temp
alertmanager_url: http://10.10.0.39:9093
enable_api: true4、创建采集promtail-config.yaml
server:
http_listen_port: 9080
grpc_listen_port: 0
positions:
filename: /tmp/promtail/positions.yaml
clients:
- url: http://loki:3100/loki/api/v1/push
scrape_configs:
#################################
# Ubuntu日志
#################################
- job_name: ubuntu_system_logs
static_configs:
- targets:
- localhost
labels:
job: auth_logs
host: "10.10.0.39"
department: "总部"
log_type: auth
__path__: /var/log/auth.log
- targets:
- localhost
labels:
job: kernel_logs
host: "10.10.0.39"
department: "总部"
log_type: kernel
__path__: /var/log/kern.log
pipeline_stages:
# 提取syslog时间、主机、程序
- regex:
expression: '^(?P<timestamp>\w{3}\s+\d{1,2}\s+\d{2}:\d{2}:\d{2})\s+(?P<hostname>\S+)\s+(?P<program>[\w\-\/]+)(?:\[(?P<pid>\d+)\])?:\s*(?P<message>.*)$'
# 根据关键词判断日志级别
- match:
selector: '{log_type="auth"}'
stages:
- regex:
expression: '(?i)(?P<level>failed|failure|invalid|accepted|error|warning)'
- labels:
level:
- match:
selector: '{log_type="kernel"}'
stages:
- regex:
expression: '(?i)(?P<level>error|failed|failure|panic|critical)'
- labels:
level:
###################################################
# Docker容器日志
###################################################
- job_name: docker_container_logs
docker_sd_configs:
- host: unix:///var/run/docker.sock
relabel_configs:
#################################
# 容器名称
#################################
- source_labels:
- __meta_docker_container_name
regex: '/(.*)'
target_label: container
#################################
# 镜像名称
#################################
- source_labels:
- __meta_docker_container_image
target_label: image
#################################
# 容器ID
#################################
- source_labels:
- __meta_docker_container_id
target_label: container_id
#################################
# stdout/stderr
#################################
- source_labels:
- __meta_docker_container_log_stream
target_label: stream
#################################
# 固定标签
#################################
- target_label: host
replacement: "192.168.7.7"
- target_label: department
replacement: "事业7"
- target_label: log_type
replacement: docker
pipeline_stages:
#################################
# Docker JSON解析
#################################
- json:
expressions:
output: log
stream: stream
timestamp: time
#################################
# 提取日志级别
#################################
- json:
expressions:
level: level
- labels:
level:5、编写告警规则
cat system.yaml
groups:
- name: ubuntu_system_logs_alerts
rules:
# 1. SSH登录失败告警（来自auth.log）
- alert: SSHFailedLogin
expr: |
count_over_time(
{job="auth_logs"}
|~ "(?i)Failed password|authentication failure"
[5m]
) > 5
for: 1m
labels:
severity: 警告
category: security
annotations:
summary: "SSH登录失败次数过多"
description: |
服务器: {{ $labels.host }}
部门: {{ $labels.department }}
日志类型: {{ $labels.log_type }}
告警内容: 过去5分钟内SSH登录失败次数超过5次（当前: {{ $value }}次），可能存在暴力破解攻击，请及时检查。
# 2. sudo命令执行告警（来自auth.log）
- alert: SudoCommandExecuted
expr: |
count_over_time(
{job="auth_logs"}
|~ "(?i)sudo.*COMMAND"
[1m]
) > 0
for: 0m
labels:
severity: info
category: audit
annotations:
summary: "sudo命令被执行"
description: |
服务器: {{ $labels.host }}
部门: {{ $labels.department }}
日志类型: {{ $labels.log_type }}
告警内容: 检测到sudo命令执行，请确认是否为授权操作。
# 3. 系统内核错误告警（来自kern.log）
- alert: KernelPanic
expr: |
count_over_time(
{job="kernel_logs"}
|~ "(?i)panic|Oops|BUG:"
[1m]
) > 0
for: 1m
labels:
severity: 严重告警
category: system
annotations:
summary: "内核发生严重错误"
description: |
服务器: {{ $labels.host }}
部门: {{ $labels.department }}
日志类型: {{ $labels.log_type }}
告警内容: 检测到内核panic/Oops/BUG错误，系统可能面临崩溃风险，请立即介入处理。
# 4. 硬件错误告警（来自kern.log）
- alert: HardwareError
expr: |
count_over_time(
{job="kernel_logs"}
|~ "(?i)hardware error|PCIe.*error|EDAC|MCE"
[5m]
) > 0
for: 2m
labels:
severity: 严重告警
category: hardware
annotations:
summary: "检测到硬件错误"
description: |
服务器: {{ $labels.host }}
部门: {{ $labels.department }}
日志类型: {{ $labels.log_type }}
告警内容: 检测到硬件相关错误（PCIe/EDAC/MCE等），请检查服务器硬件状态。
# 5. 系统关机/重启告警（来自kern.log）
- alert: SystemReboot
expr: |
count_over_time(
{job="kernel_logs"}
|~ "(?i)system reboot|Restarting system|shutdown"
[1m]
) > 0
for: 0m
labels:
severity: 警告
category: system
annotations:
summary: "系统发生重启或关机"
description: |
服务器: {{ $labels.host }}
部门: {{ $labels.department }}
日志类型: {{ $labels.log_type }}
告警内容: 检测到系统重启/关机事件，请确认是否为计划内操作。
# 6. 内核OOM告警（来自kern.log）
- alert: OutOfMemory
expr: |
count_over_time(
{job="kernel_logs"}
|~ "(?i)Out of memory|Kill process|oom-killer"
[2m]
) > 0
for: 1m
labels:
severity: 严重告警
category: performance
annotations:
summary: "系统发生OOM（内存溢出）"
description: |
服务器: {{ $labels.host }}
部门: {{ $labels.department }}
日志类型: {{ $labels.log_type }}
告警内容: 检测到OOM事件，系统内存不足导致进程被终止，请检查内存使用情况。
# 7. 综合ERROR级别日志告警（涵盖所有日志类型）
- alert: SystemErrorLog
expr: |
count_over_time(
{job=~".+"}
|~ "(?i)error|fail|fatal"
[5m]
) > 10
for: 3m
labels:
severity: 警告
category: general
annotations:
summary: "系统ERROR日志数量过多"
description: |
服务器: {{ $labels.host }}
部门: {{ $labels.department }}
日志类型: {{ $labels.log_type }}
告警内容: 过去5分钟内ERROR/FATAL/FAIL日志超过10条（当前: {{ $value }}条），请检查系统运行状态。6、容器日志告警规则
groups:
- name: docker_app_logs_sy7
rules:
- alert: AppErrorRate
expr: |
count_over_time(
{log_type="docker", container!="", container!="promtail", container!="loki", container!="kibana0", host!="10.10.0.153", host!="10.10.0.68", host!="10.10.0.69", host!="10.10.0.70"}
!= "查询老师失败"
!= "插入或更新课程章节信息异常"
!= "插入或更新课程信息异常"
!= "msgprocess no rs"
!= "'failed': 0"
!= "hits"
!= "WARN"
!= "Warn"
!= "warn"
!= "communication"
!= "data/media/"
!= "CTRL-CHAR"
|~ "(?i)(error|exception|fatal|failure|panic)"
!~ `(?i)"level": *"(info|warn|warning|debug|trace)"`
[5m]
) > 35
for: 2m
labels:
severity: 警告
category: application
annotations:
summary: "应用错误日志过多"
description: |
容器: {{ $labels.container }}
服务器: {{ $labels.host }}
部门: {{ $labels.department }}
告警内容: 过去5分钟内错误日志超过35条（当前: {{ $value }}条）
建议: 请检查应用运行状态和业务日志
# 2. Docker容器频繁重启告警（新增）
- alert: ContainerRestart
expr: |
count_over_time(
{log_type="docker", container!="loki", host!="10.10.0.68", host!="10.10.0.69", host!="10.10.0.70"}
|~ "(?i)restart|reload|stop|start"
|~ "(?i)container"
[10m]
) > 2
for: 0m
labels:
severity: 警告
category: container
annotations:
summary: "容器频繁重启"
description: |
容器: {{ $labels.container }}
服务器: {{ $labels.host }}
部门: {{ $labels.department }}
镜像: {{ $labels.image }}
告警内容: 过去10分钟内检测到容器重启/启停操作超过2次 restart|reload|stop|start
建议: 请检查容器健康检查和OOM情况
# 3. 数据库连接失败告警（应用级）
- alert: DBConnectionFailed
expr: |
count_over_time(
{log_type="docker", container!="loki", container!="promtail", host!="10.10.0.68", host!="10.10.0.69", host!="10.10.0.70"}
|~ "(?i)connection refused|connection timeout|SQLSTATE|database down|DB error"
!~ `(?i)"level": *"(info|warn|warning|debug|trace)"`
[3m]
) > 3
for: 1m
labels:
severity: 严重告警
category: dependency
annotations:
summary: "数据库连接异常"
description: |
容器: {{ $labels.container }}
服务器: {{ $labels.host }}
部门: {{ $labels.department }}
镜像: {{ $labels.image }}
告警内容: 检测到数据库连接失败或超时错误 出现 'connection refused|connection timeout|SQLSTATE|database down|DB error'
建议: 请检查数据库服务是否正常运行
# 4. API响应错误告警（HTTP 5xx）
- alert: Http5xxErrors
expr: |
count_over_time(
{log_type="docker", container!="loki", host!="10.10.0.68", host!="10.10.0.69", host!="10.10.0.70"}
|~ "(?i)Internal Server Error|Bad Gateway|Gateway Timeout"
!~ `(?i)"level": *"(info|warn|warning|debug|trace)"`
[5m]
) > 5
for: 2m
labels:
severity: 严重告警
category: application
annotations:
summary: "API服务返回5xx错误"
description: |
容器: {{ $labels.container }}
服务器: {{ $labels.host }}
部门: {{ $labels.department }}
镜像: {{ $labels.image }}
告警内容: 过去5分钟内HTTP 出现Internal Server Error、Bad Gateway、Gateway Timeout
建议: 请检查应用逻辑和上游服务状态
# 6. 特定业务错误码告警（示例）
- alert: BusinessErrorCode
expr: |
count_over_time(
{log_type="docker", container!="loki", host!="10.10.0.69", host!="10.10.0.68", host!="10.10.0.70"}
|~ "(?i)错误码|errorcode|业务异常|系统异常|BadSqlGrammarException|SQLSyntaxErrorException|NullPointerException|RuntimeException"
!~ `(?i)"level": *"(info|warn|warning|debug|trace)"`
[5m]
) > 5
for: 1m
labels:
severity: 警告
category: business
annotations:
summary: "业务错误码频繁出现"
description: |
容器: {{ $labels.container }}
服务器: {{ $labels.host }}
部门: {{ $labels.department }}
镜像: {{ $labels.image }}
告警内容: 过去5分钟内出现业务错误码 |errorcode|业务异常|系统异常|BadSqlGrammarException|SQLSyntaxErrorException|NullPointerException|RuntimeException
建议: 请结合业务日志分析根本原因
浙公网安备 33010602011771号