EFLK架构实例01-模拟生成电商指标日志并采集

ES集群地址

10.0.0.91:9200
10.0.0.92:9200
10.0.0.93:9200

编写Python脚本生成测试日志

[root@elk91 ~]# cat gen-log.py
#!/usr/bin/env python
# -*- coding: UTF-8 -*-

import datetime
import random
import logging
import time
import sys

LOG_FORMAT = "%(levelname)s %(asctime)s [com.dezyan.%(module)s] - %(message)s "
DATE_FORMAT = "%Y-%m-%d %H:%M:%S"

# 配置root的logging.Logger实例的基本配置
logging.basicConfig(level=logging.INFO, format=LOG_FORMAT, datefmt=DATE_FORMAT, filename=sys.argv[1]
, filemode='a',)
actions = ["浏览页面", "评论商品", "加入收藏", "加入购物车", "提交订单", "使用优惠券", "领取优惠券",
 "搜索", "查看订单", "付款", "清空购物车"]

while True:
    time.sleep(random.randint(1, 5))
    user_id = random.randint(1, 10000)
    # 对生成的浮点数保留2位有效数字.
    price = round(random.uniform(15000, 30000),2)
    action = random.choice(actions)
    svip = random.choice([0,1,2])
    logging.info("DAU|{0}|{1}|{2}|{3}".format(user_id, action,svip,price))
[root@elk91 ~]# 
[root@elk91 ~]# 
[root@elk91 ~]# python3 gen-log.py /tmp/apps.log

编写Logstash实例

root@elk91:~# vim /etc/logstash/conf.d/02-beats_apps-to-es.conf
input { 
  beats {
    port => 9999
  }
} 


filter {

  mutate {
    # 将message字段按照"|"进行切分
    split => { "message" => "|" }

    # 添加字段
    add_field => { 
      "other" => "%{[message][0]}" 
      "userId" => "%{[message][1]}" 
      "action" => "%{[message][2]}" 
      "svip" => "%{[message][3]}" 
      "price" => "%{[message][4]}" 
    }

  }

  mutate {

    split => { "other" => " " }

    add_field => {
       datetime => "%{[other][1]} %{[other][2]}"
    }
    
    convert => {
       "price" => "float"
     }

    remove_field => [ "@version","host","agent","ecs","tags","input","log","message","other"]
  }


  date {
    # "2025-03-14 11:32:58"
    match => [ "datetime", "yyyy-MM-dd HH:mm:ss" ]
  }
}

output { 
 # stdout { 
 #   codec => rubydebug 
 # }

  elasticsearch {
     index => "dezyan-logstash-elfk-apps"
     hosts => ["http://10.0.0.91:9200","http://10.0.0.92:9200","http://10.0.0.93:9200"]
  }
}

root@elk91:~# rm -f /usr/share/logstash/data/plugins/inputs/file/.sincedb*
root@elk91:~# logstash -rf 08-nginx-to-es.conf

编写Filebeat实例

[root@elk91 ~]# cat /etc/filebeat/config/9-filestream-to-logstash.yml
filebeat.inputs:
- type: filestream
  paths:
    - /tmp/apps.log

output.logstash:
  hosts: ["10.0.0.93:9999"]
  
[root@elk91 ~]# rm -rf /var/lib/filebeat/
[root@elk91 ~]# 
[root@elk91 ~]# filebeat -e -c /etc/filebeat/config/9-filestream-to-logstash.yml