linux系统ElK基础filebeat收集日志(4)

一、Filebeat收集单个日志

1.配置收集日志到文件

[root@web01 ~]# vim /etc/filebeat/filebeat.yml
filebeat.inputs:
- type: log
  enable: true
  paths:
    - /var/log/nginx/access.log
output.file:
  path: "/tmp"
  filename: "filebeat.log"

2.配置收集日志到ES

[root@web01 ~]# vim /etc/filebeat/filebeat.yml
filebeat.inputs:
- type: log
  enable: true
  paths:
    - /var/log/nginx/access.log
output.elasticsearch:
  hosts: ["10.0.0.51:9200"]

3.配置收集日志为json格式

1）配置

#由于收集日志内容还是写到了message，没有办法作图
[root@web01 ~]# vim /etc/filebeat/filebeat.yml
filebeat.inputs:
- type: log
  enable: true
  paths:
    - /var/log/nginx/access.log
  json.keys_under_root: true
  json.overwrite_keys: true
output.elasticsearch:
  hosts: ["10.0.0.51:9200"]

2）修改Nginx日志格式

#filebeat只支持某种json格式写法
[root@web01 ~]# vim /etc/nginx/nginx.conf
... ...
    log_format log_json '{ "time_local": "$time_local", '
                        '"remote_addr": "$remote_addr", '
                        '"referer": "$http_referer", '
                        '"request": "$request", '
                        '"status": $status, '
                        '"bytes": $body_bytes_sent, '
                        '"agent": "$http_user_agent", '
                        '"x_forwarded": "$http_x_forwarded_for", '
                        '"up_addr": "$upstream_addr",'
                        '"up_host": "$upstream_http_host",'
                        '"upstream_time": "$upstream_response_time",'
                        '"request_time": "$request_time" }';
 ... ...

3）重启

1.重启Nginx
2.重启Filebeat
3.删除原来的索引
4.清空Nginx日志

4.收集日志配置指定索引名称

1）配置

[root@web01 ~]# vim /etc/filebeat/filebeat.yml
filebeat.inputs:
- type: log
  enable: true
  paths:
    - /var/log/nginx/access.log
  json.keys_under_root: true
  json.overwrite_keys: true
output.elasticsearch:
  hosts: ["10.0.0.51:9200"]
  index: "nginx_log_%{+YYYY-MM-dd}"
setup.template.enabled: false
setup.template.name: "nginx"
setup.template.pattern: "nginx-*"

2）指定分片数

[root@web01 ~]# vim /etc/filebeat/filebeat.yml.bak 
setup.template.settings:
  index.number_of_shards: 3

5.收集日志到redis

1）配置

[root@web01 ~]# vim /etc/filebeat/filebeat.yml
filebeat.inputs:
- type: log
  enable: true
  paths:
    - /var/log/nginx/access.log
  json.keys_under_root: true
  json.overwrite_keys: true
output.redis:
  hosts: ["172.16.1.51"]
  port: "6379"
  key: "nginx_access"
  db: 0

2）查看redis

#访问Nginx页面后，查看redis是否有数据
127.0.0.1:6379> keys *
1) "nginx_access"
127.0.0.1:6379> TYPE nginx_access
list
127.0.0.1:6379> LLEN nginx_access
(integer) 8
127.0.0.1:6379> LRANGE nginx_access 0 -1

6.使用logstash将redis数据取出到ES

[root@web01 conf.d]# vim redis_to_es.conf 
input {
  redis {
    host => "172.16.1.51"
    port => "6379"
    db => "0"
    data_type => "list"
    key => "nginx_access"
  }
}
output {
  elasticsearch {
    hosts => ["10.0.0.51:9200"]
    index => "nginx_access_%{+YYYY-MM-dd}"
  }
}

7.filebeat收集日志到logstash

1）配置收集日志到logstash

[root@web01 ~]# vim /etc/filebeat/filebeat.yml
filebeat.inputs:
- type: log
  enable: true
  paths:
    - /var/log/nginx/access.log
  json.keys_under_root: true
  json.overwrite_keys: true
output.logstash:
  hosts: ["172.16.1.52:3456"]
  
#如果启动失败，查看日志，应该是172.16.1.52服务器的3456端口没有启动，需要先启动52的logstash

2）配置logstash收集日志到ES

[root@db02 ~]# vim /etc/logstash/conf.d/filebeat_logstash_es.conf
input {
  beats {
    port => 3456
    codec => "json"
  }
}
output {
  elasticsearch {
    hosts => ["10.0.0.51:9200"]
    index => "nginx_filebeat_logstash_es"
  }
}

3）查看es数据

二、filebeat收集多日志

1.收集多日志到ES

1）方式一：

[root@web01 ~]# vim /etc/filebeat/filebeat.yml
filebeat.inputs:
- type: log
  enable: true
  paths:
    - /var/log/nginx/access.log
  json.keys_under_root: true
  json.overwrite_keys: true
- type: log
  enable: true
  paths:
    - /var/log/messages

output.elasticsearch:
  hosts: ["10.0.0.51:9200"]
  indices:
    - index: "nginx_%{+YYYY-MM-dd}"
      when.contains:
        source: "/var/log/nginx/access.log"
    - index: "message_%{+YYYY-MM-dd}"
      when.contains:
        source: "/var/log/messages"
setup.template.enabled: false
setup.template.name: "nginx"
setup.template.pattern: "nginx-*"

2）方式二：

[root@web01 ~]# vim /etc/filebeat/filebeat.yml

filebeat.inputs:
- type: log
  enable: true
  paths:
    - /var/log/nginx/access.log
  json.keys_under_root: true
  json.overwrite_keys: true
  tags: ["nginx"]

- type: log
  enable: true
  paths:
    - /var/log/messages
  tags: ["messages"]

output.elasticsearch:
  hosts: ["10.0.0.51:9200"]
  indices:
    - index: "nginx_%{+YYYY-MM-dd}"
      when.contains:
        tags: "nginx"
    - index: "message_%{+YYYY-MM-dd}"
      when.contains:
        tags: "messages"
setup.template.enabled: false
setup.template.name: "nginx"
setup.template.pattern: "nginx-*"

三、filebeat收集java报错

1）配置

[root@web01 ~]# vim /etc/filebeat/filebeat.yml
filebeat.inputs:
- type: log
  enable: true
  paths:
    - /var/log/nginx/access.log
  multiline.pattern: '^\['
  multiline.negate: true
  multiline.match: after

output.elasticsearch:
  hosts: ["10.0.0.51:9200"]
  index: "tomca_error_%{+YYYY-MM-dd}"
setup.template.enabled: false
setup.template.name: "nginx"
setup.template.pattern: "nginx-*"

2）导入错误日志查看

四、kibana画图

1.先收集一个json格式的日志

2.写入数据

3.画图

五、使用地图统计地区IP

1.安装geoip

[root@web01 logstash]# rz ingest-geoip-6.6.0.zip
[root@web01 logstash]# unzip ingest-geoip-6.6.0.zip

#下载地址
Logstash2版本下载地址：http://geolite.maxmind.com/download/geoip/database/GeoLiteCity.dat.gz
logstash5版本下载地址：http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz

2.配置

[root@web01 logstash]# cat conf.d/geoip.conf 
input {
  file {
    path => "/var/log/nginx/access.log"
    type => "nginx_access_log"
    start_position => "end"
    codec => "json"
  }
}

filter {
  json {
    source => "message"
    remove_field => ["message"]
  }
  geoip {
    source => "clientip"
    target => "geoip"
    database => "/etc/logstash/config/GeoLite2-City.mmdb"
    add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]
    add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}"  ]
  }
  mutate {
    convert => [ "[geoip][coordinates]", "float"]
  }
}

output {
  elasticsearch {
    hosts => ["10.0.0.51:9200"]
    index => "logstash-%{type}-%{+YYYY.MM.dd}"
  }
}

3.启动

[root@web01 logstash]# logstash -f conf.d/geoip.conf

4.插入数据

#北京公网IP
[root@elkstack03 conf.d]# echo '{"@timestamp":"2019-04-11T20:27:25+08:00","host":"222.28.0.112","clientip":"222.28.0.112","size":0,"responsetime":0.000,"upstreamtime":"-","upstreamhost":"-","http_host":"www.elk.com","url":"/index.html","domain":"www.elk.com","xff":"10.0.0.1","referer":"-","status":"304"}' >> /usr/local/nginx/logs/access_json.log

#海南公网IP
[root@elkstack03 conf.d]# echo '{"@timestamp":"2019-04-11T20:40:24+08:00","host":" 124.225.0.13","clientip":"124.225.0.13","size":0,"responsetime":0.000,"upstreamtime":"-","upstreamhost":"-","http_host":"www.elk.com","url":"/index.html","domain":"www.elk.com","xff":"10.0.0.1","referer":"-","status":"304"}' >> /usr/local/nginx/logs/access_json.log

#吉林公网IP
[root@elkstack03 conf.d]# echo '{"@timestamp":"2019-04-11T20:45:24+08:00","host":" 124.234.0.12","clientip":"124.234.0.12","size":0,"responsetime":0.000,"upstreamtime":"-","upstreamhost":"-","http_host":"www.elk.com","url":"/index.html","domain":"www.elk.com","xff":"10.0.0.1","referer":"-","status":"304"}' >> /usr/local/nginx/logs/access_json.log

#黑龙江公网IP
[root@elkstack03 conf.d]# echo '{"@timestamp":"2019-04-11T20:46:24+08:00","host":" 123.164.0.18","clientip":"123.164.0.18","size":0,"responsetime":0.000,"upstreamtime":"-","upstreamhost":"-","http_host":"www.elk.com","url":"/index.html","domain":"www.elk.com","xff":"10.0.0.1","referer":"-","status":"304"}' >> /usr/local/nginx/logs/access_json.log