1.简介
Filebeat附带预构建的模块，这些模块包含收集、解析、充实和可视化各种日志文件格式数据所需的配置，每个Filebeat模块由一个或多个文件集组成，这些文件集包含摄取节点管道、Elasticsearch模板、Filebeat勘探者配置和Kibana仪表盘。
Filebeat模块很好的入门，它是轻量级单用途的日志收集工具，用于在没有安装java的服务器上专门收集日志，可以将日志转发到logstash、elasticsearch或redis等场景中进行下一步处理。
Filebeat与logstash作用是一样
Logstash是java写的，需要java环境
filebeat是go写的，不需要安装java环境，并且非常的轻量
2.安装filebeat
#上传包
[root@web01 ~]# rz filebeat-6.6.0-x86_64.rpm
#安装
[root@web01 ~]# rpm -ivh filebeat-6.6.0-x86_64.rpm
3.配置文件
[root@web01 ~]# rpm -qc filebeat
/etc/filebeat/filebeat.yml
filebeat收集本地日志到文件
1.配置
[root@web01 ~]# vim /etc/filebeat/filebeat.yml
filebeat.inputs:
- type: log
enable: true
paths:
- /var/log/messages
output.file:
path: "/tmp"
filename: "filebeat_messages.log"
2.启动
[root@web01 ~]# systemctl start filebeat.service
#验证
[root@web01 ~]# ps -ef | grep filebeat
3.测试
[root@web01 ~]# tail -f /tmp/filebeat_messages.log
#输入内容
[root@web01 ~]# echo "123" >> /var/log/messages
filebeat收集本地日志到ES
1.配置
filebeat.inputs:
- type: log
enable: true
paths:
- /var/log/nginx/access.log
output.elasticsearch:
hosts: ["10.0.0.51:9200"]
2.启动
[root@web01 ~]# systemctl restart filebeat.service
#验证
[root@web01 ~]# ps -ef | grep filebeat
3.测试
#访问Nginx
http://10.0.0.7:8081/
#查看ES页面
一、Filebeat收集单个日志
1.配置收集日志到文件
[root@web01 ~]# vim /etc/filebeat/filebeat.yml
filebeat.inputs:
- type: log
enable: true
paths:
- /var/log/nginx/access.log
output.file:
path: "/tmp"
filename: "filebeat.log"
2.配置收集日志到ES
[root@web01 ~]# vim /etc/filebeat/filebeat.yml
filebeat.inputs:
- type: log
enable: true
paths:
- /var/log/nginx/access.log
output.elasticsearch:
hosts: ["10.0.0.51:9200"]
3.配置收集日志为json格式
1）配置
#由于收集日志内容还是写到了message，没有办法作图
[root@web01 ~]# vim /etc/filebeat/filebeat.yml
filebeat.inputs:
- type: log
enable: true
paths:
- /var/log/nginx/access.log
json.keys_under_root: true
json.overwrite_keys: true
output.elasticsearch:
hosts: ["10.0.0.51:9200"]
2）修改Nginx日志格式
#filebeat只支持某种json格式写法
[root@web01 ~]# vim /etc/nginx/nginx.conf
... ...
log_format log_json '{ "time_local": "$time_local", '
'"remote_addr": "$remote_addr", '
'"referer": "$http_referer", '
'"request": "$request", '
'"status": $status, '
'"bytes": $body_bytes_sent, '
'"agent": "$http_user_agent", '
'"x_forwarded": "$http_x_forwarded_for", '
'"up_addr": "$upstream_addr",'
'"up_host": "$upstream_http_host",'
'"upstream_time": "$upstream_response_time",'
'"request_time": "$request_time" }';
... ...
3）重启
1.重启Nginx
2.重启Filebeat
3.删除原来的索引
4.清空Nginx日志
4.收集日志配置指定索引名称
1）配置
[root@web01 ~]# vim /etc/filebeat/filebeat.yml
filebeat.inputs:
- type: log
enable: true
paths:
- /var/log/nginx/access.log
json.keys_under_root: true
json.overwrite_keys: true
output.elasticsearch:
hosts: ["10.0.0.51:9200"]
index: "nginx_log_%{+YYYY-MM-dd}"
setup.template.enabled: false
setup.template.name: "nginx"
setup.template.pattern: "nginx-*"
2）指定分片数
[root@web01 ~]# vim /etc/filebeat/filebeat.yml.bak
setup.template.settings:
index.number_of_shards: 3
5.收集日志到redis
1）配置
[root@web01 ~]# vim /etc/filebeat/filebeat.yml
filebeat.inputs:
- type: log
enable: true
paths:
- /var/log/nginx/access.log
json.keys_under_root: true
json.overwrite_keys: true
output.redis:
hosts: ["172.16.1.51"]
port: "6379"
key: "nginx_access"
db: 0
2）查看redis
#访问Nginx页面后，查看redis是否有数据
127.0.0.1:6379> keys *
1) "nginx_access"
127.0.0.1:6379> TYPE nginx_access
list
127.0.0.1:6379> LLEN nginx_access
(integer) 8
127.0.0.1:6379> LRANGE nginx_access 0 -1
6.使用logstash将redis数据取出到ES
[root@web01 conf.d]# vim redis_to_es.conf
input {
redis {
host => "172.16.1.51"
port => "6379"
db => "0"
data_type => "list"
key => "nginx_access"
}
}
output {
elasticsearch {
hosts => ["10.0.0.51:9200"]
index => "nginx_access_%{+YYYY-MM-dd}"
}
}
7.filebeat收集日志到logstash
1）配置收集日志到logstash
[root@web01 ~]# vim /etc/filebeat/filebeat.yml
filebeat.inputs:
- type: log
enable: true
paths:
- /var/log/nginx/access.log
json.keys_under_root: true
json.overwrite_keys: true
output.logstash:
hosts: ["172.16.1.52:3456"]
#如果启动失败，查看日志，应该是172.16.1.52服务器的3456端口没有启动，需要先启动52的logstash
2）配置logstash收集日志到ES
[root@db02 ~]# vim /etc/logstash/conf.d/filebeat_logstash_es.conf
input {
beats {
port => 3456
codec => "json"
}
}
output {
elasticsearch {
hosts => ["10.0.0.51:9200"]
index => "nginx_filebeat_logstash_es"
}
}
3）查看es数据
二、filebeat收集多日志
1.收集多日志到ES
1）方式一：
[root@web01 ~]# vim /etc/filebeat/filebeat.yml
filebeat.inputs:
- type: log
enable: true
paths:
- /var/log/nginx/access.log
json.keys_under_root: true
json.overwrite_keys: true
- type: log
enable: true
paths:
- /var/log/messages
output.elasticsearch:
hosts: ["10.0.0.51:9200"]
indices:
- index: "nginx_%{+YYYY-MM-dd}"
when.contains:
source: "/var/log/nginx/access.log"
- index: "message_%{+YYYY-MM-dd}"
when.contains:
source: "/var/log/messages"
setup.template.enabled: false
setup.template.name: "nginx"
setup.template.pattern: "nginx-*"
2）方式二：
[root@web01 ~]# vim /etc/filebeat/filebeat.yml
filebeat.inputs:
- type: log
enable: true
paths:
- /var/log/nginx/access.log
json.keys_under_root: true
json.overwrite_keys: true
tags: ["nginx"]
- type: log
enable: true
paths:
- /var/log/messages
tags: ["messages"]
output.elasticsearch:
hosts: ["10.0.0.51:9200"]
indices:
- index: "nginx_%{+YYYY-MM-dd}"
when.contains:
tags: "nginx"
- index: "message_%{+YYYY-MM-dd}"
when.contains:
tags: "messages"
setup.template.enabled: false
setup.template.name: "nginx"
setup.template.pattern: "nginx-*"
三、filebeat收集java报错
1）配置
[root@web01 ~]# vim /etc/filebeat/filebeat.yml
filebeat.inputs:
- type: log
enable: true
paths:
- /var/log/nginx/access.log
multiline.pattern: '^\['
multiline.negate: true
multiline.match: after
output.elasticsearch:
hosts: ["10.0.0.51:9200"]
index: "tomca_error_%{+YYYY-MM-dd}"
setup.template.enabled: false
setup.template.name: "nginx"
setup.template.pattern: "nginx-*"
2）导入错误日志查看
浙公网安备 33010602011771号