ELK收集日志2

一、使用logstash收集nginx日志

1.安装nginx

2.配置nginx日志为json格式

[root@logstash ~]# vim /etc/nginx/nginx.conf 
http {
    ... ...
    log_format json   '{"@timestamp":"$time_iso8601",'
                      '"host":"$server_addr",'
                      '"clientip":"$remote_addr",'
                      '"size":$body_bytes_sent,'
                      '"responsetime":$request_time,'
                      '"upstreamtime":"$upstream_response_time",'
                      '"upstreamhost":"$upstream_addr",'
                      '"http_host":"$host",'
                      '"url":"$uri",'
                      '"domain":"$host",'
                      '"xff":"$http_x_forwarded_for",'
                      '"referer":"$http_referer",'
                      '"status":"$status"}';

    access_log  /var/log/nginx/nginx_json.log  json;
    ... ...
}

3.启动nginx

[root@logstash ~]# nginx -t
nginx: the configuration file /etc/nginx/nginx.conf syntax is ok
nginx: configuration file /etc/nginx/nginx.conf test is successful
[root@logstash ~]# nginx

4.配置logstash收集nginx日志

[root@logstash ~]# vim /etc/logstash/conf.d/nginx_json_es.conf 
input {
  file {
    path => "/var/log/nginx/nginx_json.log"
    start_position => "beginning"
  }
}

output {
  elasticsearch {
    hosts => ["10.0.0.51:9200"]
    index => "nginx_%{+YYYY-MM-dd}.log"
  }
}

5.启动logstash

[root@logstash ~]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/nginx_json_es.conf

6.ES页面查看数据

7.kibana查看数据

二、修改nginx日志为json格式展示

1.配置nginx日志格式为json

1）方法一：解决日志message无法调用问题

[root@logstash ~]# vim /etc/logstash/conf.d/nginx_json_es.conf 
input {
  file {
    path => "/var/log/nginx/nginx_json.log"
    start_position => "beginning"
  }
}

filter {                             #将inout的内容进行处理
  json {
    source => "message"				#将指定内容跟转化成json格式
    remove_field => ["message"]		 #移除message部分
  }
}

output {
  elasticsearch {
    hosts => ["10.0.0.51:9200"]
    index => "nginx_json_%{+YYYY-MM-dd}.log"
  } 
}

[root@logstash ~]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/nginx_json_es.conf

2）方式二：

[root@logstash ~]# vim /etc/logstash/conf.d/nginx_jsonlog_es.conf 
input {
  file {
    path => "/var/log/nginx/nginx_json.log"
    start_position => "beginning"
    codec => "json"					#收集日志为json格式
  }
}

output {
  elasticsearch {
    hosts => ["10.0.0.51:9200"]
    index => "nginx_json_%{+YYYY-MM-dd}.log"
  } 
}

[root@logstash ~]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/nginx_jsonlog_es.conf

2.画饼图

三、使用logstash将日志写入redis

1.准备机器

主机	IP	服务
logstash	10.0.0.54	logstash、nginx
db01	10.0.0.51	ES、kibana
db02	10.0.0.52	ES
db03	10.0.0.53	ES、redis(6381)

2.安装redis

#等等
[root@db03 ~]# redis-server /server/redis/6381/redis.conf

3.配置收集nginx日志到redis

[root@logstash ~]# vim /etc/logstash/conf.d/nginx_redis.conf 
input {
  file {
    path => "/var/log/nginx/nginx_json.log"
    start_position => "beginning"
    codec => "json"
  }
}

output {
  redis {
    host => "172.16.1.53"
    port => "6381"
    data_type => "list"
    key => "nginx_json_log"
    db => "0"
    #password => "123"    如果有密码加上password
  }
}

[root@logstash ~]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/nginx_redis.conf

#后台启动
[root@logstash ~]# mkdir /data/logstash/nginx_redis
[root@logstash ~]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/nginx_redis.conf --path.data=/data/logstash/nginx_redis &

4.访问页面，查看redis中数据

#刷新页面后到redis中查看
[root@db03 ~]# redis-cli -p 6381
127.0.0.1:6381> keys *
1) "nginx_json_log"

#查看数据长度（有多少条数据）
127.0.0.1:6381> LLEN nginx_json_log
(integer) 19

#查看所有数据
127.0.0.1:6381> LRANGE nginx_json_log 0 -1

5.配置将redis中的数据放到ES

[root@logstash ~]# vim /etc/logstash/conf.d/redis_es.conf
input {
  redis {
    host => "172.16.1.53"
    port => "6381"
    db => "0"
    data_type => "list"
    key => "nginx_json_log"
  }
}

output {
  elasticsearch {
    hosts => ["10.0.0.51:9200"]
    index => "nginx_redis_es_%{+YYYY-MM-dd}"
  }
}

[root@logstash ~]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/nginx_redis.conf --path.data=/data/logstash/redis_es &

四、通过TCP和UDP收集数据

1.配置收集远端通过tcp发来的消息

[root@logstash ~]# vim /etc/logstash/conf.d/tcp.conf
input {
  tcp {
    port => 1234
    type => "tcplog"
    mode => "server"
  }
}

output {
  stdout {
    codec => rubydebug
  }
}

[root@logstash ~]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/tcp.conf

2.测试收集数据

1）使用telnet测试

[root@db02 ~]# telnet 10.0.0.54 1234    #telnet 退出 连接使用 ctrl+ ] 然后输入quit
Trying 10.0.0.54...
Connected to 10.0.0.54.
Escape character is '^]'.
123
456

#查看收集日志的机器
{
      "@version" => "1",
    "@timestamp" => 2020-07-20T09:40:49.974Z,
          "host" => "10.0.0.52",
          "port" => 50492,
          "type" => "tcplog",
       "message" => "123\r"
}


{
      "@version" => "1",
    "@timestamp" => 2020-07-20T09:41:36.764Z,
          "host" => "10.0.0.52",
          "port" => 50492,
          "type" => "tcplog",
       "message" => "456\r"
}

2）使用nc工具测试

#使用yum安装nc
[root@db04 ~]# yum install -y nc

#使用nc传输数据
[root@db02 ~]# echo "zls test nc" | nc 10.0.0.54 1234

#查看输出的数据
{
      "@version" => "1",
    "@timestamp" => 2020-07-20T09:46:57.706Z,
          "host" => "10.0.0.52",
          "port" => 50520,
          "type" => "tcplog",
       "message" => "zls test nc"
}


#可以用来实时监控日志
[root@db02 ~]# tail -f /var/log/nginx/access.log | nc 10.0.0.54 1234 &

[root@db02 ~]# echo "123" >> /var/log/nginx/access.log
[root@db02 ~]# echo "123" >> /var/log/nginx/access.log

#查看输出的日志
{
      "@version" => "1",
    "@timestamp" => 2020-07-20T09:48:28.181Z,
          "host" => "10.0.0.52",
          "port" => 50524,
          "type" => "tcplog",
       "message" => "123"
}
{
      "@version" => "1",
    "@timestamp" => 2020-07-20T09:48:29.503Z,
          "host" => "10.0.0.52",
          "port" => 50524,
          "type" => "tcplog",
       "message" => "123"
}

3.通过伪设备的方式发送日志

#发送伪设备数据
[root@db02 ~]# echo "伪设备 测试"  > /dev/tcp/10.0.0.54/1234

#查看伪设备
{
      "@version" => "1",
    "@timestamp" => 2020-07-20T09:57:25.807Z,
          "host" => "10.0.0.52",
          "port" => 50526,
          "type" => "tcplog",
       "message" => "伪设备 测试"
}