Loki部署
一、安装grafana
[root@loki ~]# yum install -y https://dl.grafana.com/enterprise/release/grafana-enterprise-10.0.0-1.x86_64.rpm
二、安装loki
[root@loki ~]# wget https://github.com/grafana/loki/releases/download/v2.8.2/loki-linux-amd64.zip
[root@loki ~]# unzip loki-linux-amd64.zip
[root@loki ~]# mkdir -p /etc/loki
[root@loki ~]# mv loki-linux-amd64 /etc/loki
[root@loki ~]# vim /etc/loki/loki.yaml
auth_enabled: false server: http_listen_port: 3100 grpc_listen_port: 9096 common: path_prefix: /data/loki storage: filesystem: chunks_directory: /data/loki/chunks rules_directory: /data/loki/rules replication_factor: 1 ring: instance_addr: 127.0.0.1 kvstore: store: inmemory schema_config: configs: - from: 2023-07-18 store: boltdb-shipper object_store: filesystem schema: v11 index: prefix: index_ period: 24h ruler: alertmanager_url: http://localhost:9093
[root@loki ~]# /etc/loki/loki-linux-amd64 -config.file loki.yaml
[root@loki ~]# vim /lib/systemd/system/loki.service
[Unit] Description=Loki service After=network.target [Service] Type=simple User=root ExecStart=/etc/loki/loki-linux-amd64 -config.file /etc/loki/loki.yaml [Install] WantedBy=multi-user.target
三、安装promtail
[root@loki ~]# wget https://github.com/grafana/loki/releases/download/v2.8.2/promtail-linux-amd64.zip
[root@loki ~]# unzip promtail-linux-amd64.zip
[root@loki ~]# mkdir -p /etc/promtail
[root@loki ~]# mv promtail-linux-amd64 /etc/promtail/
[root@loki ~]# vim /etc/promtail/promtail.yaml
server: http_listen_port: 9080 grpc_listen_port: 0 positions: filename: /tmp/positions.yaml clients: - url: http://127.0.0.1:3100/loki/api/v1/push scrape_configs: - job_name: r1 static_configs: - targets: - 127.0.0.1 labels: job: 网络设备日志 location: 办公室机房 vendor: 华为 hostname: R1 __path__: /var/log/network/r1/*.log - job_name: r2 static_configs: - targets: - 127.0.0.1 labels: job: 网络设备日志 location: 数据中心 vendor: 思科 hostname: R2 __path__: /var/log/network/r2/*.log
[root@loki ~]# /etc/promtail/promtail-linux-amd64 -config.file promtail.yaml
[root@loki ~]# vim /lib/systemd/system/promtail.service
[Unit] Description=Promtail service After=network.target [Service] Type=simple User=root ExecStart=/etc/promtail/promtail-linux-amd64 -config.file /etc/promtail/promtail.yaml [Install] WantedBy=multi-user.target
四、启用服务
[root@loki ~]# systemctl start promtail.service
[root@loki ~]# systemctl start loki.service
[root@loki ~]# systemctl start grafana-server.service
五、加入开机启动
[root@loki ~]# ystemctl enable promtail.service
[root@loki ~]# systemctl enable loki.service
[root@loki ~]# systemctl enable grafana-server.service
六、配置rsyslog
1、添加hosts解析
[root@loki ~]# cat /etc/hosts
127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 ::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 192.168.140.130 R1 192.168.140.131 R2
2、修改/etc/rsyslog.conf文件
[root@loki ~]# vim /etc/rsyslog.conf
$ModLoad imudp $UDPServerRun 514 $WorkDirectory /var/lib/rsyslog $ActionFileDefaultTemplate RSYSLOG_TraditionalFileFormat $IncludeConfig /etc/rsyslog.d/*.conf $OmitLocalLogging on $IMJournalStateFile imjournal.state *.info;mail.none;authpriv.none;cron.none;local5.none;local6.none /var/log/messages authpriv.* /var/log/secure mail.* -/var/log/maillog cron.* /var/log/cron *.emerg :omusrmsg:* uucp,news.crit /var/log/spooler local7.* /var/log/boot.log
3、添加/etc/rsyslog.d/network.conf文件
[root@loki ~]# vim /etc/rsyslog.d/network.conf # 以下为示例配置，按需使用
# 设置日志文件和路径所属用户和组，以及文件和路径权限 $FileOwner user1 $FileGroup user1 $DirOwner user1 $DirGroup user1 $FileCreateMode 0600 $DirCreateMode 0600 # $Umask 0022 $template NETWORK,"接收日志时间:%timegenerated:1:10:date-rfc3339% %timereported:12:19:date-rfc3339% %fromhost%:%fromhost-ip% msg:%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\n" # %$now%等于%$YEAR%-%$MONTH%-%$DAY%，%fromhost%获取设备名称(hosts文件中需要有相关的主机名和IP记录) $template NETWORKDIR,"/var/log/network/%fromhost%/%$now%.log" # local5.* /var/log/network/network.log;NETWORK # local5.* ?NETWORKDIR;NETWORK # local5.* action(type="omfile" FileGroup="promtail" FileOwner="promtail" FileCreateMode="0755" dirCreateMode="0755" dynaFile="NETWORKDIR" template="NETWORK") # local5.* if $fromhost-ip != '127.0.0.1' and $fromhost-ip != '10.122.8.10' then ?NETWORKDIR;NETWORK # local5.* if not ($msg contains("up")) then ?NETWORKDIR;NETWORK local5.* if $fromhost-ip == '192.168.140.140' then action(type="omfile" FileGroup="user1" FileOwner="user1" FileCreateMode="0755" dirCreateMode="0755" dynaFile="NETWORKDIR" template="NETWORK") # IP为127.0.0.1和10.122.8.10不使用NETWORKDIR模板记录日志 if $fromhost-ip != '127.0.0.1' and $fromhost-ip != '10.122.8.10' then ?NETWORKDIR;NETWORK $template VPNRemoteLogsSystemformat,"%timegenerated:1:10:date-rfc3339% %timereported:12:19:date-rfc3339% User:%msg:F,32:4% Src:%msg:F,32:13% dst:%msg:F,32:17%\n" $template VPNRemoteLogsSessionformat,"%timegenerated:1:10:date-rfc3339% %timereported:12:19:date-rfc3339% %msg:F,59:4% %msg:F,59:5% %msg:F,59:6% %msg:F,59:7%\n" $template VPNRemoteLogsSystem,"/var/log/network/%fromhost%/System-%$YEAR%-%$MONTH%-%$DAY%.log" $template VPNRemoteLogsSession,"/var/log/network/%fromhost%/Session-%$YEAR%-%$MONTH%-%$DAY%.log" # 匹配日志调用相关的日志格式、存放路径以及设置文件和文件夹的权限 # contains：包含；startswith：开始；contains_i和startswith_i：不区分大小写 # :msg,contains,"VsysId:1" ?VPNRemoteLogsSession # :msg,contains_i,"vsysid:1" ?VPNRemoteLogsSession # :msg,contains,"10SSLVPN/6/SSLVPN_IP_RESOURCE_PERMIT" ?VPNRemoteLogsSystem :msg,contains,"VsysId:1" action(type="omfile" FileGroup="promtail" FileOwner="promtail" FileCreateMode="0600" dirCreateMode="0600" dynaFile="VPNRemoteLogsSession" template="VPNRemoteLogsSessionformat") :msg,contains,"SSLVPN/6/SSLVPN_IP" action(type="omfile" FileGroup="promtail" FileOwner="promtail" FileCreateMode="0600" dirCreateMode="0600" dynaFile="VPNRemoteLogsSystem" template="VPNRemoteLogsSystemformat") # 忽略之前所有的日志，远程主机日志记录完之后不再继续往下记录 & ~
4、检查rsyslog.conf文件是否有语法错误
[root@loki ~]# rsyslogd -f /etc/rsyslog.conf -N1
5、重启rsyslog服务
[root@loki ~]# systemctl restart rsyslog.service
6、格式化日志
"％msg:1:2％" # 提取消息文本的前两个字符 "％msg:80:$％" # 从第80个字符开始一直截取到末尾的剩下的文本 "％msg:::drop-last-lf％" #获取日志消息的整个消息文本，并删除其最后一个换行符 "%msg:R:Inter.*--end%\n" # 正则匹配从字符Inter开始到该行结尾，--end为固定搭配 "%msg:F,32:5%\n" # 以空格为分隔符取第五列（32在ASCII码中表示空格） "%msg:F,59,1:5,12%\n" # 以;为分隔符，取第五列中的第一个字符到第十二个字符（59在ASCII码中表示分号，5表示第五列）
七、思科路由器配置
R1(config)#clock timezone BJ +8 R1#clock set 18:48:00 18 Aug 2026 R1(config)#service timestamps log datetime localtime show-timezone year R1(config)#logging host x.x.x.x R1(config)#logging facility local5 R1(config)#hostname cs-cmcc-gly-r1 cs-cmcc-gly-r1(config)#logging origin-id hostname # 使发出的Syslog报文携带主机名信息(默认可能不显示或仅显示IP,修改名称需把该命令重新配置一下才生效) cs-cmcc-gly-r1(config)#logging on
八、vector日志解析示例配置
# ------------------------------------------------------------------------------ # Website: https://vector.dev # Docs: https://vector.dev/docs # Chat: https://chat.vector.dev # Install: curl --proto '=https' --tlsv1.2 -sSfL https://sh.vector.dev | bash # Run: /root/.vector/bin/vector --config /root/.vector/config/vector.yaml # Vector Remap Language: https://playground.vrl.dev/ # ------------------------------------------------------------------------------ # Change this to use a non-default directory for Vector data storage: # data_dir: "/var/lib/vector" sources: syslog_input: type: socket address: 0.0.0.0:514 mode: udp decoding: codec: bytes # 改为 bytes，查看原始数据 #codec: syslog file_input: type: file data_dir: /tmp/vector_data include: - /var/log/messages transforms: parse_syslog: type: "remap" inputs: ["syslog_input"] source: | .host_ip = .host # source_msg解析前的完整日志 .source_msg = .message # 使用正则解析 Cisco 非标准 Syslog 格式 # 匹配: 序列号，主机名，时间戳，消息内容 parsed, err = parse_regex(.message, r'<(?P<priority>\d+)>(?P<seq>\d+): (?P<hostname>.*): (?P<timestamp>.*): %(?P<message>.*)') # 只有解析成功时，才进行字段提取和后续匹配 if err == null { .priority = to_int!(parsed.priority) .sequence = parsed.seq .message = parsed.message # 处理非标准时间戳 (将 "BJ" 替换为 "+0800" 并格式化) cleaned_ts, ts_err = replace(parsed.timestamp, "BJ", "+0800") if ts_err == null { parsed_time, parse_err = parse_timestamp(cleaned_ts, "%b %d %Y %H:%M:%S %z") if parse_err == null { #.timestamp = format_timestamp!(parsed_time, "%Y-%m-%d %H:%M:%S+8") .timestamp = format_timestamp!(parsed_time, "%Y-%m-%d %H:%M:%S", "Asia/Shanghai") } else { # 时间解析失败时，保留原始时间戳 .timestamp = parsed.timestamp } } else { .timestamp = parsed.timestamp } # 提取主机名用于后续匹配 hostname = string!(parsed.hostname) # case_sensitive:false,不区分大小写 .room = if starts_with(hostname, "CS", case_sensitive:false) { "长沙" } else if contains(hostname, "ZZ", case_sensitive: false) { "株洲" } else if starts_with(hostname, "XT", case_sensitive:false) { "湘潭" } else { "未知" } # (?!)，不区分大小写 .network = if match(hostname, to_regex!("(?i)CMCC")) { "中国移动机房" } else if match(hostname, to_regex!("(?i)CUCC")) { "中国联通机房" } else if match(hostname, to_regex!("(?i)CTCC")) { "中国电信机房" } else { "未知" } .domain = if ends_with(hostname, "GLY-R1", case_sensitive:false) { "管理域" } else if ends_with(hostname, "YWY-R1", case_sensitive:false) { "业务域" } else { "未知" } # 解析成功后，删除默认的host字段 del(.host) } else { # 如果正则匹配失败，保留原始消息并标记为解析失败 .parse_error = string!(err) } filter_syslog: type: "filter" inputs: ["parse_syslog"] condition: type: "vrl" source: | # to_string(.message) ?? ""：安全转换；null/其它类型转为字符串，失败给默认空字符串，不会抛异常适合filter条件 msg = to_string(.message) ?? "" .message != null && !contains(msg, "up") && !contains(msg, "administratively") filter_file: type: "filter" inputs: ["file_input"] condition: type: "vrl" source: | contains(string!(.message), "kernel") sinks: # 在终端打印输出 print: type: "console" inputs: ["filter_file", "filter_syslog"] encoding: codec: "json" json: pretty: true # 攒够10条或者等待2秒，打包成一条 `[obj1,obj2,...]` POST发送出去 http_sink: type: "http" inputs: ["filter_syslog"] uri: "http://127.0.0.1:8080/receive" encoding: codec: "json" batch: max_events: 10 timeout_secs: 2 # Vector's API (disabled by default) # Uncomment to try it out with the `vector top` command api: enabled: true address: "127.0.0.1:8686"
参考链接：
https://www.cnblogs.com/eeexu123/p/13441149.html # 轻量日志系统Loki
https://zhuanlan.zhihu.com/p/601612530 # 使用loki收集网络设备日志
https://cloud.tencent.com/developer/article/1889851 # 使用loki收集网络设备日志
https://cloud.tencent.com/developer/article/2115403 # rsyslogd_Syslog
https://zhuanlan.zhihu.com/p/602336166 # Rsyslog-日志收集快速上手
https://www.rsyslog.com/doc/v8-stable/ # rsyslog服务官方文档
https://grafana.com/docs/loki/latest/configuration/examples/ # loki.yaml配置示例
https://grafana.com/docs/loki/latest/operations/storage/table-manager/ # Loki日志保留时间设置
https://grafana.com/docs/loki/latest/clients/promtail/ # promtail.yaml
https://www.cnblogs.com/guoew/p/16417060.html # logcli命令工具查询loki日志
https://grafana.com/docs/loki/v2.9.x/query/logcli/ # logcli命令文档
https://github.com/grafana/loki/releases # logcli包下载
https://www.cnblogs.com/gered/p/17312748.html#autoid-1-5-0 # prometheus blackbox_exporter 监控组件
https://cloud.tencent.com/developer/article/2115410 # prometheus blackbox_exporter 监控组件
https://github.com/prometheus/blackbox_exporter/blob/master/example.yml
https://www.cnblogs.com/v-fan/category/1881118.html # prometheus
https://blog.csdn.net/qq_42249813/article/details/130740462 # loki 配置文件详解
https://blog.csdn.net/weixin_43381723/article/details/133971173 # 解决当limit过大时，rpc报错的问题
https://www.cpolar.com/blog/detailed-explanation-of-plg-log-system-integrated-deployment-practice-of-promtail-loki-grafana
浙公网安备 33010602011771号