1 filebeat收集tomcat,nginx日志发送给redis-->logstash-->elasticsearch实现日志分类缓存及写入到Redis不同的index
1.1 环境
es1 #10.0.0.7 es2 #10.0.0.27 es3 #10.0.0.37 es-vip #10.0.0.248 nginx #10.0.0.47 tomcat #10.0.0.57 redis #10.0.0.67 logstash预处理主机 #10.0.0.77 logstash读取redis主机 #10.0.0.17
1.2 logstash
#安装redis
#安装tomcat
#rpm安装logstash-7.6.1 # rpm -ivh /usr/local/src/logstash-7.6.1.rpm #修改配置文件 # vi /etc/logstash/conf.d/tomcat_message-to-redis.conf input { file { path => "/var/log/messages" #注意文件及上级目录权限 start_position => "beginning" stat_interval => "3" type => "message" } file { path => "/usr/local/tomcat/logs/tomcat_access_log.*.log" #注意文件及上级目录权限 start_position => "beginning" stat_interval => "3" type => "tomcat_access" codec => "json" } } output { if [type] == "message" { redis { host => ["10.0.0.67:6379"] #redis-server地址 password => "root" key => "message" data_type => "list" db => "1" #10.0.0.57的日志写入redis的db1 timeout => "10" } } if [type] == "tomcat_access" { redis { host => ["10.0.0.67:6379"] #redis-server地址 password => "root" key => "tomcat-access" data_type => "list" db => "1" timeout => "10" } } } #授权logstash用户权限 # setfacl -m u:logstash:r /var/log/messages # setfacl -m u:logstash:rx /usr/local/tomcat/logs #配置文件语法检测 # /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/tomcat_message-to-redis.conf -t #以服务方式启动logstash # systemctl enable --now logstash #测试写入日志，测试刷新nginx-web页面 # logger "test-message_v5" #查看logstash日志输出 # tail -f /var/log/logstash/logstash-plain.log #redis-server验证 # redis-cli -a root -n 1 2> /dev/null keys \* # redis-cli -a root -n 1 2> /dev/null type message # redis-cli -a root -n 1 2> /dev/null llen message # redis-cli -a root -n 1 2> /dev/null llen tomcat-access # redis-cli -a root -n 1 2> /dev/null lrange tomcat-access 0 0
1.3
#yum安装filebeat # yum -y install filebeat-7.6.1-x86_64.rpm #修改配置文件 # vi /etc/filebeat/filebeat.yml filebeat.inputs: - type: log enabled: true #默认false paths: - /var/log/messages #exclude_lines: ['^DBG'] #inlude_lines: ['^ERR', '^WARN'] #exclude_files: ['.gz$'] #一般日志路径全模糊时使用 fields: type: message host: 10.0.0.47 #level: debug #review: 1 #Multiline options #也支持多行合并 - type: log enabled: true #默认false paths: - /data/www/www.testou.access.log fields: type: nginx-access host: 10.0.0.47 - type: log enabled: true #默认false paths: - /data/www/www.testou.error.log fields: type: nginx-error host: 10.0.0.47 output.logstash: hosts: ["10.0.0.77:5044", "10.0.0.77:5045"] #模拟filebeat轮询写入两个不同logstash loadbalance: true worker: 5 #默认每个主机开3个进程 #启动服务并设置开机自启 # systemctl enable --now filebeat
1.4 logstash 收集远程主机 nginx 日志及系统日志到 redis
#安装jdk #rpm安装logstash-7.6.1 # rpm -ivh /usr/local/src/logstash-7.6.1.rpm #修改配置文件 # vi /etc/logstash/conf.d/logstash-nginx_message-to-redis.conf input { beats { port => "5044" #模拟filebeat轮询写入两个不同logstash codec => "json" } beats { port => "5045" codec => "json" } } output { if [fields][type] == "message" { redis { host => ["10.0.0.67:6379"] #redis-server地址 password => "root" key => "message" data_type => "list" db => "0" #10.0.0.47的日志写入redis的db0 timeout => "10" } } if [fields][type] == "nginx-access" { redis { host => ["10.0.0.67:6379"] #redis-server地址 password => "root" key => "nginx-access" data_type => "list" db => "0" timeout => "10" } } if [fields][type] == "nginx-error" { redis { host => ["10.0.0.67:6379"] #redis-server地址 password => "root" key => "nginx-error" data_type => "list" db => "0" timeout => "10" } } } #配置文件语法检测 # /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/logstash-nginx_message-to-redis.conf -t #以服务方式启动logstash并设置开机自启 # systemctl enable --now logstash #查看logstash日志输出 # tail -f /var/log/logstash/logstash-plain.log #redis-server验证 # redis-cli -a root 2> /dev/null keys \* # redis-cli -a root 2> /dev/null type message # redis-cli -a root 2> /dev/null llen message # redis-cli -a root 2> /dev/null llen nginx-access # redis-cli -a root 2> /dev/null llen nginx-error # redis-cli -a root 2> /dev/null lrange nginx-access 0 0
1.5 logstash 读取 redis 数据写入 es
#安装jdk #rpm安装logstash-7.6.1 # rpm -ivh /usr/local/src/logstash-7.6.1.rpm #修改配置文件 # vi /etc/logstash/conf.d/redis-to-es.conf input { redis { host => "10.0.0.67" #redis-server地址 port => "6379" password => "root" key => "message" data_type => "list" db => "0" timeout => "10" } redis { host => "10.0.0.67" #redis-server地址 port => "6379" password => "root" key => "nginx-access" data_type => "list" db => "0" timeout => "10" codec => "json" #nginx访问日志json格式 } redis { host => "10.0.0.67" #redis-server地址 port => "6379" password => "root" key => "nginx-error" data_type => "list" db => "0" timeout => "10" } redis { host => "10.0.0.67" #redis-server地址 port => "6379" password => "root" key => "message" data_type => "list" db => "1" timeout => "10" } redis { host => "10.0.0.67" #redis-server地址 port => "6379" password => "root" key => "tomcat-access" data_type => "list" db => "1" timeout => "10" codec => "json" #tomcat访问日志json格式 } } output { if [fields][type] == "message" { elasticsearch { hosts => ["10.0.0.248:9200"] #es集群vip index => "rs-message-0.47-%{+YYYY.MM.dd}" } } if [fields][type] == "nginx-access" { elasticsearch { hosts => ["10.0.0.248:9200"] #es集群vip index => "rs-nginx_access-0.47-%{+YYYY.MM.dd}" } } if [fields][type] == "nginx-error" { elasticsearch { hosts => ["10.0.0.248:9200"] #es集群vip index => "rs-nginx_error-0.47-%{+YYYY.ww}" } } if [type] == "message" { elasticsearch { hosts => ["10.0.0.248:9200"] #es集群vip index => "rs-message-0.57-%{+YYYY.MM.dd}" } } if [type] == "tomcat_access" { elasticsearch { hosts => ["10.0.0.248:9200"] #es集群vip index => "rs-tomcat_access-0.57-%{+YYYY.MM.dd}" } } } #配置文件语法检测 # /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/redis-to-es.conf -t #以服务方式启动logstash并设置开机自启 # systemctl enable --now logstash #查看logstash日志输出 # tail -f /var/log/logstash/logstash-plain.log #kibana创建相关索引模式，查看日志输出
