ELK汇集zeek数据

1.配置filebeat用户读取zeek告警

1.新增filebeat用户
getent passwd filebeat || echo "filebeat user missing"
sudo groupadd -r filebeat 2>/dev/null || true
sudo useradd  -r -g filebeat -s /sbin/nologin -M filebeat 2>/dev/null || true
getent passwd filebeat

2.设置filebeat用户读取zeek告警目录
sudo setfacl -R -m u:filebeat:rx /usr/local/zeek/logs
sudo setfacl -R -m d:u:filebeat:rx /usr/local/zeek/logs

3.filebeat查看zeek告警
sudo -u filebeat head -n 2 /usr/local/zeek/logs/current/notice.log

2.配置filebeat

1.编辑filebeat.yml
filebeat.config.modules:
  path: ${path.config}/modules.d/*.yml
  reload.enabled: false

filebeat.inputs:
  - type: filestream
    id: slips-alerts
    enabled: true
    paths:
      - /opt/slips/output/**/alerts.json
      - /opt/slips/output/alerts.json
    fields:
      event.module: slips
      slips.format: idea0
      observer.ip: 10.10.10.2
    fields_under_root: true

  - type: filestream
    id: suricata-eve
    enabled: true
    paths:
      - /usr/local/soft/suricata/var/log/suricata/eve.json
    parsers:
      - ndjson:
          target: ""
          add_error_key: true
    fields:
      event.module: suricata
      observer.name: suricata-10.10.10.2
      observer.ip: 10.10.10.2
    fields_under_root: true

output.logstash:
  hosts: ["10.10.10.1:5044"]

2.开启filebeat zeek module ingest pipeline
filebeat modules enable zeek
filebeat modules list | grep zeek 

3.配置zeek.yml
- module: zeek
  capture_loss:
    enabled: true
    var.paths: 
      - /usr/local/zeek/logs/current/capture_loss.log
  connection:
    enabled: true
    var.paths: 
      - /usr/local/zeek/logs/current/conn.log
  dns:
    enabled: true
    var.paths: 
      - /usr/local/zeek/logs/current/dns.log
  files:
    enabled: true
    var.paths: 
      - /usr/local/zeek/logs/current/files.log
  http:
    enabled: true
    var.paths: 
      - /usr/local/zeek/logs/current/http.log
  notice:
    enabled: true
    var.paths: 
      - /usr/local/zeek/logs/current/notice.log
  stats:
    enabled: true
    var.paths: 
      - /usr/local/zeek/logs/current/stats.log
  weird:
    enabled: true
    var.paths: 
      - /usr/local/zeek/logs/current/weird.log

4.重启filebeat
systemctl restart filebeat
filebeat test config
filebeat test output
journalctl -u filebeat -n 200 --no-pager

3.配置logstash

1.配置logstash的ingest-beats.conf
input {
  beats { port => 5044 }
}

filter {
  if [event][module] == "suricata" {
    if [timestamp] {
      date { match => ["timestamp","ISO8601"] target => "@timestamp" }
    }
    mutate { add_field => { "pipeline" => "suricata" } }
  }
  else if [event][module] == "slips" {
    mutate { add_field => { "pipeline" => "slips" } }
  }
  else if [event][module] == "zeek" {
    mutate { add_field => { "pipeline" => "zeek" } }
  }
}

output {
  if [event][module] == "suricata" {
    elasticsearch {
      hosts => ["https://10.10.10.1:9200"]
      data_stream => true
      data_stream_type => "logs"
      data_stream_dataset => "suricata.eve"
      data_stream_namespace => "default"
      ecs_compatibility => "v8"
      ssl_enabled => true
      ssl_certificate_authorities => ["/etc/logstash/certs/http_ca.crt"]
      user => "elastic"
      password => "jq5SSXIrIUXW=xDUWQRP"
    }
  }
  else if [event][module] == "slips" {
    elasticsearch {
      hosts => ["https://10.10.10.1:9200"]
      data_stream => false
      index => "slips-alerts-%{+YYYY.MM.dd}"
      ssl_enabled => true
      ssl_certificate_authorities => ["/etc/logstash/certs/http_ca.crt"]
      user => "elastic"
      password => "jq5SSXIrIUXW=xDUWQRP"
    }
  }
  else if [event][module] == "zeek" {
    if [@metadata][pipeline] {
      elasticsearch {
        hosts => ["https://10.10.10.1:9200"]
        ssl_enabled => true
        ssl_certificate_authorities => ["/etc/logstash/certs/http_ca.crt"]
        user => "elastic"
        password => "jq5SSXIrIUXW=xDUWQRP"
        ecs_compatibility => "v8"
        pipeline => "%{[@metadata][pipeline]}"
        data_stream => false
        index => "zeek-alerts-%{+YYYY.MM.dd}"
      }
    } else {
      elasticsearch {
        hosts => ["https://10.10.10.1:9200"]
        ssl_enabled => true
        ssl_certificate_authorities => ["/etc/logstash/certs/http_ca.crt"]
        user => "elastic"
        password => "jq5SSXIrIUXW=xDUWQRP"
        ecs_compatibility => "v8"
        data_stream => false
        index => "zeek-alerts-%{+YYYY.MM.dd}"
      }
    }
  }
}

2.检查logstash配置是否生效
sudo -u logstash /usr/share/logstash/bin/logstash --path.settings /etc/logstash -t

3.重启logstash
systemctl restart logstash
ss -lntp | egrep '5044'
journalctl -u logstash -n 200 --no-pager

4.创建ES索引模板

1.创建zeek ingest pipeline的索引模板
scp root@10.10.10.1:/etc/logstash/certs/http_ca.crt /etc/filebeat/certs/http_ca.crt
filebeat setup --pipelines --modules zeek \
  -c /etc/filebeat/filebeat.yml \
  -M "zeek.connection.enabled=true" \
  -M "zeek.weird.enabled=true" \
  -M "zeek.notice.enabled=true" \
  -E output.logstash.enabled=false \
  -E output.elasticsearch.hosts='["https://10.10.10.1:9200"]' \
  -E output.elasticsearch.username='elastic' \
  -E output.elasticsearch.password='jq5SSXIrIUXW=xDUWQRP' \
  -E output.elasticsearch.ssl.certificate_authorities='["/etc/filebeat/certs/http_ca.crt"]'

2.查看创建的索引
curl -k -u elastic:'jq5SSXIrIUXW=xDUWQRP' "https://10.10.10.1:9200/_cat/indices/zeek-alerts-*?v"