一.Logstash收集日志
1.Logstash的配置文件
[root@web01 ~]# vim /etc/logstash/logstash.yml
path.config: /etc/logstash/conf.d
2.logstash收集日志文件到文件
[root@web01 ~]# vim /etc/logstash/conf.d/file_file.conf
input {
file {
path => "/var/log/messages"
start_position => "beginning"
}
}
output {
file {
path => "/tmp/messages_%{+YYYY-MM-dd}.log"
}
}
3.logstash收集日志文件到ES
[root@web01 ~]# vim /etc/logstash/conf.d/file_es.conf
input {
file {
path => "/var/log/messages"
start_position => "beginning"
}
}
output {
elasticsearch {
hosts => ["172.16.1.51:9200"]
index => "messages_%{+YYYY-MM-dd}.log"
}
}
4.Logstash收集多日志到文件
[root@web01 ~]# vim /etc/logstash/conf.d/file_file.conf
input {
file {
type => "messages_log"
path => "/var/log/messages"
start_position => "beginning"
}
file {
type => "secure_log"
path => "/var/log/secure"
start_position => "beginning"
}
}
output {
if [type] == "messages_log" {
file {
path => "/tmp/messages_%{+YYYY-MM-dd}"
}
}
if [type] == "secure_log" {
file {
path => "/tmp/secure_%{+YYYY-MM-dd}"
}
}
}
5.Logstash收集多日志到ES
1）方法一：
[root@web01 ~]# vim /etc/logstash/conf.d/more_es.conf
input {
file {
type => "messages_log"
path => "/var/log/messages"
start_position => "beginning"
}
file {
type => "secure_log"
path => "/var/log/secure"
start_position => "beginning"
}
}
output {
if [type] == "messages_log" {
elasticsearch {
hosts => ["10.0.0.51:9200"]
index => "messages_%{+YYYY-MM-dd}"
}
}
if [type] == "secure_log" {
elasticsearch {
hosts => ["10.0.0.51:9200"]
index => "secure_%{+YYYY-MM-dd}"
}
}
}
[root@web01 ~]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/more_es.conf &
#启动后查看页面
2）方法二：
[root@web01 ~]# vim /etc/logstash/conf.d/more_es_2.conf
input {
file {
type => "messages_log"
path => "/var/log/messages"
start_position => "beginning"
}
file {
type => "secure_log"
path => "/var/log/secure"
start_position => "beginning"
}
}
output {
elasticsearch {
hosts => ["10.0.0.51:9200"]
index => "%{type}_%{+YYYY-MM-dd}"
}
}
[root@web01 ~]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/more_es_2.conf --path.data=/data/logstash/more_es_2 &
3）启动多实例
#创建不同的数据目录
[root@web01 ~]# mkdir /data/logstash/more_es_2
[root@web01 ~]# mkdir /data/logstash/more_es
#启动时使用--path.data指定数据目录
[root@web01 ~]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/more_es.conf --path.data=/data/logstash/more_es &
[root@web01 ~]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/more_es_2.conf --path.data=/data/logstash/more_es_2 &
#如果资源充足，可以使用多实例收集多日志，如果服务器资源不足，启动不了多实例，配置一个文件收集多日志启动
二、Logstash收集Tomcat日志
1.安装Tomcat
1.安装java环境
[root@web01 ~]# rpm -ivh jdk-8u181-linux-x64.rpm
2.上传包
[root@web01 ~]# rz apache-tomcat-10.0.0-M7.tar.gz
3.解压
[root@web01 ~]# tar xf apache-tomcat-10.0.0-M7.tar.gz -C /usr/local/
4.做软连接
[root@web01 ~]# ln -s /usr/local/apache-tomcat-10.0.0-M7 /usr/local/tomcat
5.启动Tomcat
[root@web01 ~]# /usr/local/tomcat/bin/startup.sh
6.访问页面 10.0.0.7:8080
2.配置Logstash收集Tomcat日志到文件
[root@web01 ~]# vim /etc/logstash/conf.d/tomcat_file.conf
input {
file {
path => "/usr/local/tomcat/logs/localhost_access_log.*.txt"
start_position => "beginning"
}
}
output {
file {
path => "/tmp/tomcat_%{+YYYY-MM-dd}.log"
}
}
3.配置Logstash收集Tomcat日志到ES
[root@web01 ~]# vim /etc/logstash/conf.d/tomcat_es.conf
input {
file {
path => "/usr/local/tomcat/logs/localhost_access_log.*.txt"
start_position => "beginning"
}
}
output {
elasticsearch {
hosts => ["10.0.0.51:9200"]
index => "tomcat_%{+YYYY-MM-dd}.log"
}
}
三、收集Tomcat日志修改格式
#收集tomcat日志，当遇到报错时，一条报错会被分割成很多条数据，不方便查看
解决方法：
1.修改tomcat日志格式为json
1）开发修改输出日志为json
2）修改tomcat配置，日志格式为json
2.使用logstash的input插件下的mutiline模块
1.方法一：修改tomcat日志格式
1）配置tomcat日志为json格式
[root@web01 ~]# vim /usr/local/tomcat/conf/server.xml
#把原来的日志格式注释，添加我们的格式
<Valve className="org.apache.catalina.valves.AccessLogValve" directory="logs"
prefix="tomcat_access_json" suffix=".log"
pattern="{"clientip":"%h","ClientUser":"%l","authenticated":"%u","AccessTime":"%t","method":"%r","status":"%s","SendBytes":"%b","Query?string":"%q","partner":"%{Referer}i","AgentVersion":"%{User-Agent}i"}"/>
2）重启tomcat
[root@web01 ~]# /usr/local/tomcat/bin/shutdown.sh
[root@web01 ~]# /usr/local/tomcat/bin/startup.sh
3）配置收集新的tomcat日志
[root@web01 ~]# vim /etc/logstash/conf.d/tomcat_json_es.conf
input {
file {
path => "/usr/local/tomcat/logs/tomcat_access_json.*.log"
start_position => "beginning"
}
}
output {
elasticsearch {
hosts => ["10.0.0.51:9200"]
index => "tomcat_json_%{+YYYY-MM-dd}.log"
}
}
2.方法二：使用mutiline模块收集日志
1）配置收集日志测试
[root@web01 ~]# vim /etc/logstash/conf.d/test_mutiline.conf
input {
stdin {
codec => multiline {
#以[开头
pattern => "^\["
#匹配到
negate => true
#向上合并，向下合并是next
what => "previous"
}
}
}
output {
stdout {
codec => json
}
}
#测试，输入内容不会直接输出，当遇到以 [ 开头才会收集以上的日志
2）配置收集tomcat错误日志
[root@web01 ~]# vim /etc/logstash/conf.d/tomcat_mutiline.conf
input {
file {
path => "/usr/local/tomcat/logs/tomcat_access_json.*.log"
start_position => "beginning"
codec => multiline {
pattern => "^\["
negate => true
what => "previous"
}
}
}
output {
elasticsearch {
hosts => ["10.0.0.51:9200"]
index => "tomcat_json_%{+YYYY-MM-dd}"
codec => "json"
}
}
3）将错误日志写入
[root@web01 ~]# cat 1.txt >> /usr/local/tomcat/logs/tomcat_access_json.2020-08-14.log
4）页面查看数据
四、收集Nginx日志
1.安装Nginx
[root@web01 ~]# yum install -y nginx
2.配置Nginx日志格式
[root@web01 ~]# vim /etc/nginx/nginx.conf
... ...
http {
log_format json '{"@timestamp":"$time_iso8601",'
'"host":"$server_addr",'
'"clientip":"$remote_addr",'
'"size":$body_bytes_sent,'
'"responsetime":$request_time,'
'"upstreamtime":"$upstream_response_time",'
'"upstreamhost":"$upstream_addr",'
'"http_host":"$host",'
'"url":"$uri",'
'"referer":"$http_referer",'
'"agent":"$http_user_agent",'
'"status":"$status"}';
access_log /var/log/nginx/access.log json;
... ...
3.配置收集Nginx日志
[root@web01 ~]# vim /etc/logstash/conf.d/nginx_json.conf
input {
file {
path => "/var/log/nginx/access.log"
start_position => "beginning"
}
}
output {
elasticsearch {
hosts => ["10.0.0.51:9200"]
index => "nginx_json_%{+YYYY-MM-dd}.log"
}
}
五、获取的日志参数分离
1.方法一：
1）修改tomcat日志收集配置
[root@web01 ~]# vim /etc/logstash/conf.d/tomcat_json_es.conf
input {
file {
path => "/usr/local/tomcat/logs/tomcat_access_json.*.log"
start_position => "beginning"
}
}
#把收集到的数据进行处理
filter {
json {
source => "message"
}
}
output {
elasticsearch {
hosts => ["10.0.0.51:9200"]
index => "tomcat_json_%{+YYYY-MM-dd}.log"
}
}
2）去掉多余数据
#message数据已经拆分，数据还在，去掉message数据
filter {
json {
source => "message"
remove_field => ["message"]
}
}
2.方法二：
1）修改收集Nginx日志的配置
#nginx不需要配置修改获取日志，只需要收集同时修改格式即可
[root@web01 ~]# vim /etc/logstash/conf.d/nginx_json.conf
input {
file {
path => "/var/log/nginx/access.log"
start_position => "beginning"
codec => "json"
}
}
output {
elasticsearch {
hosts => ["10.0.0.51:9200"]
index => "nginx_json_%{+YYYY-MM-dd}.log"
}
}
六、Logstash收集日志写入redis
1.安装redis
2.配置将数据写入redis
[root@web01 ~]# vim /etc/logstash/conf.d/nginx_to_redis.conf
input {
file {
path => "/var/log/nginx/access.log"
start_position => "beginning"
codec => "json"
}
}
output {
redis {
host => "172.16.1.51"
port => "6379"
data_type => "list"
db => "0"
key => "nginx_log"
}
}
4.收集Nginx和tomcat日志到redis
[root@web01 ~]# vim /etc/logstash/conf.d/more_to_redis.conf
input {
file {
type => "nginx_log"
path => "/var/log/nginx/access.log"
start_position => "beginning"
codec => "json"
}
file {
type => "tomcat_log"
path => "/usr/local/tomcat/logs/tomcat_access_json.*.log"
start_position => "beginning"
codec => "json"
}
}
output {
if [type] == "nginx_log" {
redis {
host => "172.16.1.51"
port => "6379"
data_type => "list"
db => "0"
key => "nginx_log"
}
}
if [type] == "tomcat_log" {
redis {
host => "172.16.1.51"
port => "6379"
data_type => "list"
db => "1"
key => "tomcat_log"
}
}
}
#验证：访问Nginx和tomcat页面，查看redis里面有没有key
127.0.0.1:6379> LLEN nginx_log
(integer) 1
127.0.0.1:6379> LLEN nginx_log
(integer) 888
127.0.0.1:6379> LRANGE nginx_log 0 -1
5.配置将redis取出，写入ES
[root@db02 ~]# yum localinstall -y logstash-6.6.0.rpm
[root@db02 ~]# vim /etc/logstash/conf.d/redis_to_es.conf
input {
redis {
host => "172.16.1.51"
port => "6379"
db => "0"
data_type => "list"
key => "nginx_log"
}
redis {
host => "172.16.1.51"
port => "6379"
db => "1"
data_type => "list"
key => "tomcat_log"
}
}
output {
if [type] == "nginx_log" {
elasticsearch {
hosts => ["10.0.0.51:9200"]
index => "nginx_log_%{+YYYY-MM-dd}"
}
}
if [type] == "tomcat_log" {
elasticsearch {
hosts => ["10.0.0.51:9200"]
index => "tomcat_log_%{+YYYY-MM-dd}"
}
}
}
七、通过TCP/UDP收集日志
1.配置收集日志
[root@web01 ~]# vim /etc/logstash/conf.d/tcp.conf
input {
tcp {
port => "1234"
mode => "server"
}
}
output {
stdout {}
}
2.使用telnet测试
[root@db02 ~]# telnet 172.16.1.7 1234
Trying 172.16.1.7...
Connected to 172.16.1.7.
Escape character is '^]'.
123
345
#输出内容
{
"@timestamp" => 2020-08-17T02:23:05.833Z,
"host" => "172.16.1.52",
"port" => 33002,
"message" => "\r",
"@version" => "1"
}
{
"@timestamp" => 2020-08-17T02:23:32.562Z,
"host" => "172.16.1.52",
"port" => 33002,
"message" => "123\r",
"@version" => "1"
}
{
"@timestamp" => 2020-08-17T02:23:38.300Z,
"host" => "172.16.1.52",
"port" => 33002,
"message" => "345\r",
"@version" => "1"
}
3.使用nc工具测试
#安装
[root@db02 ~]# yum install -y nc
#使用nc工具
[root@db02 ~]# nc 172.16.1.7 1234
123
456
#使用nc工具收集日志到logstash的服务器
[root@web01 ~]# tail -f /var/log/nginx/access.log | nc 10.0.0.7 1234 &
[1] 29595
#发送伪设备数据
[root@web01 ~]# echo "伪设备测试" > /dev/tcp/10.0.0.7/1234
4.收集日志到ES
[root@web01 ~]# vim /etc/logstash/conf.d/tcp.conf
input {
tcp {
port => "1234"
mode => "server"
}
}
output {
elasticsearch {
hosts => ["10.0.0.51:9200"]
index => "tcp_log_%{+YYYY-MM-dd}"
}
}
八、Logstash配合rsyslog收集haproxy日志
1.rsyslog介绍
在centos 6及之前的版本叫做syslog，centos 7开始叫做rsyslog，根据官方的介绍，rsyslog(2013年版本)可以达到每秒转发百万条日志的级别，官方网址：http://www.rsyslog.com/
2.安装
[root@web01 ~]# yum isntall -y rsyslog
3.配置rsyslog
[root@web01 ~]# vim /etc/rsyslog.conf
#打开注释
$ModLoad imudp
$UDPServerRun 514
$ModLoad imtcp
$InputTCPServerRun 514
#添加日志收集级别
local6.* @@172.16.1.52:2222
4.安装haproxy
[root@web01 ~]# yum install -y haproxy
5.配置haproxy
[root@web01 ~]# vim /etc/haproxy/haproxy.cfg
global
maxconn 100000
chroot /var/lib/haproxy
uid 99
gid 99
daemon
nbproc 1
pidfile /var/run/haproxy.pid
log 127.0.0.1 local6 info
defaults
option http-keep-alive
option forwardfor
maxconn 100000
mode http
timeout connect 300000ms
timeout client 300000ms
timeout server 300000ms
listen stats
mode http
bind 0.0.0.0:9999
stats enable
log global
stats uri /haproxy-status
stats auth haadmin:123456
#frontend web_port
frontend web_port
bind 0.0.0.0:80
mode http
option httplog
log global
option forwardfor
###################ACL Setting##########################
acl pc hdr_dom(host) -i www.elk.com
acl mobile hdr_dom(host) -i m.elk.com
###################USE ACL##############################
use_backend pc_host if pc
use_backend mobile_host if mobile
########################################################
backend pc_host
mode http
option httplog
balance source
server web1 10.0.0.53:8081 check inter 2000 rise 3 fall 2 weight 1
backend mobile_host
mode http
option httplog
balance source
server web1 10.0.0.53:8080 check inter 2000 rise 3 fall 2 weight 1
[root@web01 ~]# vim /etc/haproxy/haproxy.cfg
#全局配置
global
#最大并发
maxconn 100000
#安全机制
chroot /var/lib/haproxy
#指定启动的用户和组
uid 99
gid 99
#守护进程
daemon
#haproxy的进程数
nbproc 1
#指定pid文件
pidfile /var/run/haproxy.pid
#指定日志级别
log 127.0.0.1 local6 info
#默认配置
defaults
#开启长连接
option http-keep-alive
#获取用户真实IP
option forwardfor
#最大连接数
maxconn 100000
#支持http协议
mode http
#设置连接超时时间
timeout connect 300000ms
timeout client 300000ms
timeout server 300000ms
#监控状态
listen status
#支持http
mode http
#监听端口
bind 0.0.0.0:9999
#启动
stats enable
#日志级别
log global
#访问uri地址
stats uri /haproxy-status
#状态页用户名和密码
stats auth haadmin:123456
#frontend web_port
frontend web_port
bind 0.0.0.0:80
mode http
option httplog
log global
option forwardfor
###################ACL Setting##########################
acl nginx hdr_dom(host) -i www.nginx.com
acl tomcat hdr_dom(host) -i www.tomcat.com
###################USE ACL##############################
use_backend nginx_host if nginx
use_backend tomcat_host if tomcat
########################################################
backend nginx_host
mode http
option httplog
balance source
server web01 10.0.0.7:8081 check inter 2000 rise 3 fall 2 weight 1
backend tomcat_host
mode http
option httplog
balance source
server web01 10.0.0.7:8080 check inter 2000 rise 3 fall 2 weight 1
6.修改Nginx启动端口
[root@web01 ~]# vim /etc/nginx/nginx.conf
server {
listen 8081 default_server;
...
7.启动服务
#启动haproxy
[root@web01 ~]# systemctl start haproxy.service
#启动rsyslog
[root@web01 ~]# systemctl start rsyslog
#验证
[root@web01 ~]# netstat -lntp
8.访问状态页面
http://10.0.0.7:9999/haproxy-status
haadmin
123456
9.测试访问Nginx和tomcat
#配置本地hosts
10.0.0.7 www.nginx.com
10.0.0.7 www.tomcat.com
#访问页面
http://www.nginx.com/
http://www.tomcat.com/
10.测试配置收集proxy日志
[root@db02 ~]# vim /etc/logstash/conf.d/haproxy.conf
input {
syslog {
port => "2222"
}
}
output {
stdout {}
}
#访问haproxy的页面，查看有无输出
11.配置收集proxy日志到ES
[root@db02 ~]# vim /etc/logstash/conf.d/haproxy_es.conf
input {
syslog {
port => "2222"
}
}
output {
elasticsearch {
hosts => ["10.0.0.51:9200"]
index => "haproxy_log_%{+YYYY-MM-dd}"
}
}
浙公网安备 33010602011771号