通过rsyslog收集haproxy日志

一、通过rsyslog收集haproxy日志

在centos6及之前的版本叫做syslog，centos7开始叫做rsyslog，根据官方的介绍，rsyslog（2013年版本）可以带到每秒转发百万条日志的级别，官方网址：http://www.rsyslog.com/,确认系统安装版本命令如下：

[root@study62 ~]# yum list syslog
Installed Packages
rsyslog.x86_64                    8.24.0-34.el7                         @anaconda

1.1.1 编译安装haproxy(study63)

1.1.1.1 编译安装

[root@study62 ~]# cd /opt/src/
[root@study62 src]# ls | grep haproxy
haproxy-2.0.14.tar.gz
# 安装依赖包
[root@study62 src]# tar xf haproxy-2.0.14.tar.gz 
[root@study62 src]# cd haproxy-2.0.14/
[root@study62 haproxy-2.0.14]# yum install gcc pcre pcre-devel openssl openssl-devel -y
# make安装
[root@study62 haproxy-2.0.14]# make TARGET=linux-glibc USE_PCRE=1 USE_OPENSSL=1 USE_ZLIB=1 ARCH=x86_64 PREFIX=/usr/local/haproxy 
[root@study62 haproxy-2.0.14]# make install PREFIX=/usr/local/haproxy
# 确认版本
[root@study62 haproxy-2.0.14]# /usr/local/haproxy/sbin/haproxy -v
HA-Proxy version 2.0.14 2020/04/02 - https://haproxy.org/

1.1.1.2 准备启动脚本

[root@study62 haproxy-2.0.14]# cp examples/haproxy.init /etc/init.d/haproxy
[root@study62 haproxy-2.0.14]# chmod 0755 /etc/init.d/haproxy

1.1.1.3 准备配置文件

[root@study62 haproxy-2.0.14]# mkdir /etc/haproxy
[root@study62 haproxy-2.0.14]# vim /etc/haproxy/haproxy.cfg
global
maxconn 100000
chroot /usr/local/haproxy
uid 99
gid 99
daemon
nbproc 1
pidfile /usr/local/haproxy/run/haproxy.pid
log 127.0.0.1 local6 info

defaults
option http-keep-alive
option  forwardfor
maxconn 100000
mode http
timeout connect 300000ms
timeout client  300000ms
timeout server  300000ms

listen stats
 mode http
 bind 0.0.0.0:9999
 stats enable
 log global
 stats uri     /haproxy-status
 stats auth    haadmin:123456

#frontend web_port
frontend web_port
        bind 0.0.0.0:80
        mode http
        option httplog
        log global
        option  forwardfor
###################ACL Setting##########################
        acl pc          hdr_dom(host) -i www.elk.com
        acl mobile      hdr_dom(host) -i m.elk.com
###################USE ACL##############################
        use_backend     pc_host        if  pc
        use_backend     mobile_host    if  mobile
########################################################

backend pc_host
        mode    http
        option  httplog
        balance source
        server web1  10.0.0.62:80 check inter 2000 rise 3 fall 2 weight 1

backend mobile_host
        mode    http
        option  httplog
        balance source
        server web1  10.0.0.62:80 check inter 2000 rise 3 fall 2 weight 1

1.1.1.4 启动服务

[root@study63 ~]# systemctl daemon-reload
[root@study63 ~]# systemctl start haproxy
[root@study62 haproxy-2.0.14]# lsof -i:80
浏览器访问http://10.0.0.63:9999/haproxy-status查看haproxy监控页面

1.1.2 编辑rsyslog服务配置文件

14 # Provides UDP syslog reception
15 $ModLoad imudp
16 $UDPServerRun 514
17 
18 # Provides TCP syslog reception
19 $ModLoad imtcp
20 $InputTCPServerRun 514
# 在配置文件最下边添加配置
#local6.*       /var/log/haproxy.log # 添加到本地
# 添加到logstash
local6.*     @@10.0.0.62:5160 # 指定给Logstash
[root@study63 ~]# systemctl restart haproxy.service 
[root@study63 ~]# systemctl restart rsyslog.service

1.1.3 配置rsyslog记录haproxy日志

1.1.3.1 测试配置

[root@study62 ~]# vim /etc/logstash/conf.d/rsyslog.conf 

input {
  syslog {
    type => "haproxy063"
    port => "5160"
  }
}

output {
  stdout { codec => "rubydebug"
  }
}
[root@study62 ~]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/rsyslog.conf -t

1.1.3.2 通过logstash将haproxy日志打入es中

[root@study62 ~]# vim /etc/logstash/conf.d/rsyslog.conf
input {
  syslog {
    type => "rsyslog-haproxy063"
    port => "5160"
  }
}

output {
  if [type] == "rsyslog-haproxy063" {
    elasticsearch {
      hosts => ["10.0.0.62:9200"]
      index => "logstash-rsyslog-haproxy063-%{+YYYY.MM.dd}"
  }}
}

[root@study62 ~]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/rsyslog.conf -t
[root@study62 ~]# systemctl restart logstash.service