logstash 收集nginx 日志 windows

nginx 配置：

 

    log_format  main  'remote_user=$remote_user&ip=$remote_addr&real_ip=$http_x_forwarded_for&log_time=$time_local&request_time=$request_time&host=$http_host&$args&status=$status&body_bytes_sent=$body_bytes_sent&referer=$http_referer&user_agent=$http_user_agent&forwarded_for=$http_x_forwarded_for';
                        

    access_log  logs/access.log  main;

　　

 location = /s.gif {    
            empty_gif;
        }

logstash 配置：

 

input {

    file{
            path => "E:/test/nginx-1.18.0/logs/access.log"
            start_position => "beginning"
            stat_interval => "2"

        }


}

filter{
   urldecode{
      field => message
   } 
   kv{
      field_split => "&"
   }

　　   ruby{
　　　　code => "

　　　　　　event.set('tts',(Time.now.to_f.round(3)*1000).to_i)  #毫秒时间戳

　　　　　　value = event.get('oaid')

　　　　　　require 'digest/md5'

　　　　　　md5 = Digest::MD5::hexdigest(value)
　　　　　　event.set('oaid', md5)   # oaid MD5 加密

　　　　　　"
　　　}

  }

output {
    jdbc {
      # 配置数据库信息
      connection_string => "jdbc:mysql://localhost:3306/logstash_test?useUnicode=true&characterEncoding=UTF-8&serverTimezone=Asia/Shanghai"
      driver_class => "com.mysql.cj.jdbc.Driver"
      username => "root"
      password => "123456"
   
      driver_jar_path => "E:/test/logstash-7.14.2/mysql/mysql-connector-java-8.0.25.jar"

      statement => ["insert into log_coll(timestamp,args,remote_addr,name,age) VALUES(?,?,?,?,?)", "timestamp","user_agent","remote_addr","name","age"]

    }
    stdout {
     codec => rubydebug
   }
}

　　启动 bin 目录    .\logstash -f .\logstash.conf

 

      输出output 添加http 

     nginx请求 ：http://127.0.0.1/s.gif?name=123%E5%93%88%E5%93%88&ket=sd&age=9989&et=10

input {

    file{
            path => "E:/test/nginx-1.18.0/logs/access.log"
            start_position => "beginning"
            stat_interval => "2"

        }


}

filter{
   urldecode{
      field => message
   } 
   kv{
      field_split => "&"
   }
  }

output {
    jdbc {
      # 配置数据库信息
      connection_string => "jdbc:mysql://localhost:3306/logstash_test?useUnicode=true&characterEncoding=UTF-8&serverTimezone=Asia/Shanghai"
      driver_class => "com.mysql.cj.jdbc.Driver"
      username => "root"
      password => "123456"
   
      driver_jar_path => "E:/test/logstash-7.14.2/mysql/mysql-connector-java-8.0.25.jar"

      statement => ["insert into log_coll(timestamp,args,remote_addr,name,age) VALUES(?,?,?,?,?)", "timestamp","user_agent","remote_addr","name","age"]

    }

    if[et] == "10" {

        http {

            http_method => "post"                                   
            url => "http://localhost:8779/device/profit"     
            format => "json"
            mapping => { 
                    "name" => "%{name}"
                    "age" => "%{age}"

            }
        }

    }    
    

    stdout {
     codec => rubydebug
   }
}