filebeat配置

yum 源配置

1
2
3
4
5
6
7
8
9
[root@kafka01 bin]# cat /etc/yum.repos.d/filebeat.repo
[filebeat-6.x]
name=Elasticsearch repository for 6.x packages
baseurl=https://artifacts.elastic.co/packages/6.x/yum
gpgcheck=1
gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch
enabled=1
autorefresh=1
type=rpm-md

 

其他没有用到的参数我都删掉了！

1
2
3
4
5
6
7
8
9
filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /var/log/messages
output.kafka:
    enabled: true
    hosts: ["192.168.1.7:9092","192.168.1.8:9092","192.168.1.9:9092"]
    topic: messages

 收集多个日志路径和kafka的topic配置

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
filebeat.inputs:
#messages
- type: log
  enabled: true
  paths:
    - /var/log/messages
  fields:
    log_topics: messages
 
#secure
- type: log
  enabled: true
  paths:
    - /var/log/secure
  fields:
    log_topics: secure
 
output.kafka:
    enabled: true
    hosts: ["192.168.1.7:9092","192.168.1.8:9092","192.168.1.9:9092"]
    topic: '%{[fields][log_topics]}'

 logstash配置

"/etc/logstash/conf.d/messages.conf"

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
input {
        kafka {
                bootstrap_servers => ["192.168.1.7:9092,192.168.1.8:9092,192.168.1.9:9092"]
                group_id => "logstash"
                topics => "messages"
                consumer_threads => 5
        }
}
 
output {
        elasticsearch {
                hosts => "192.168.1.7:9200"
                index => "messages-%{+YYYY.MM.dd}"
        }
 
}