logstash 日志收集

# Sample Logstash configuration for creating a simple
# Beats -> Logstash -> Elasticsearch pipeline.

#input{
#   kafka{
#    bootstrap_servers => ["localhost:9092"]
#        client_id => "mysqlslowlogs"
#        group_id => "mysqlslowlogs"
#        auto_offset_reset => "latest"
#        consumer_threads => 5
#        decorate_events => true
#        topics => ["mysqlslowlogs"]
#        type => "mysqlslowlogs"
#  }
#}

input{
   kafka{
    bootstrap_servers => ["10.168.6.89:9092"]
        client_id => "accesslog"
        group_id => "accesslog"
        auto_offset_reset => "latest"
        consumer_threads => 5
        decorate_events => true
        topics => ["accesslog"]
        type => "accesslog"
  }
}


filter {
        if ([message]== "")
        {
            drop {}
        }
    
    if ([type] == "mysqlslowlogs") {
      json {
            source => "message"
      }
      grok {
               match => [ "message", "^#\s+User@Host:\s+%{USER:user}\[[^\]]+\]\s+@\s+(?:(?<clienthost>\S*) )?\[(?:%{IP:clientip})?\]\s+Id:\s+%{NUMBER:id}\n# Query_time: %{NUMBER:query_time}\s+Lock_time: %{NUMBER:lock_time}\s+Rows_sent: %{NUMBER:rows_sent}\s+Rows_examined: %{NUMBER:rows_examined}\nuse\s(?<dbname>\w+);\nSET\s+timestamp=%{NUMBER:timestamp_mysql};\n(?<query>[\s\S]*)" ]
               match => [ "message", "^#\s+User@Host:\s+%{USER:user}\[[^\]]+\]\s+@\s+(?:(?<clienthost>\S*) )?\[(?:%{IP:clientip})?\]\s+Id:\s+%{NUMBER:id}\n# Query_time: %{NUMBER:query_time}\s+Lock_time: %{NUMBER:lock_time}\s+Rows_sent: %{NUMBER:rows_sent}\s+Rows_examined: %{NUMBER:rows_examined}\nSET\s+timestamp=%{NUMBER:timestamp_mysql};\n(?<query>[\s\S]*)" ]
            match => [ "message", "^#\s+User@Host:\s+%{USER:user}\[[^\]]+\]\s+@\s+(?:(?<clienthost>\S*) )?\[(?:%{IP:clientip})?\]\n# Query_time: %{NUMBER:query_time}\s+Lock_time: %{NUMBER:lock_time}\s+Rows_sent: %{NUMBER:rows_sent}\s+Rows_examined: %{NUMBER:rows_examined}\nuse\s(?<dbname>\w+);\nSET\s+timestamp=%{NUMBER:timestamp_mysql};\n(?<query>[\s\S]*)" ]
               match => [ "message", "^#\s+User@Host:\s+%{USER:user}\[[^\]]+\]\s+@\s+(?:(?<clienthost>\S*) )?\[(?:%{IP:clientip})?\]\n# Query_time: %{NUMBER:query_time}\s+Lock_time: %{NUMBER:lock_time}\s+Rows_sent: %{NUMBER:rows_sent}\s+Rows_examined: %{NUMBER:rows_examined}\nSET\s+timestamp=%{NUMBER:timestamp_mysql};\n(?<query>[\s\S]*)" ]
          }
          date {
              match => ["timestamp_mysql","UNIX"]
              target => "@timestamp"
          }
          mutate {
                add_tag => ["mysqlslowlogs"]
                remove_field => [ "agent","log","@version","ecs","host" ]
                remove_field => "@version"
                remove_field => "message"
          }
    }

      if ([type] == "accesslog") {
          json {
            source => "message"
          }
          grok {
     match => [ "message", '%{WORD:xforwardedproto} (?<site>([\S+]*)) (?:(%{IPORHOST:http_baidu_cdn_real_ip}|%{DATA:http_baidu_cdn_real_ip})) (?:(%{IPORHOST:http_ali_cdn_real_ip}|%{DATA:http_ali_cdn_real_ip})) \[%{HTTPDATE:timestamp}\] %{WORD:method} (?<urlpath>([\S+]*)) HTTP/%{NUMBER:httpversion} %{NUMBER:status} %{NUMBER:bytes} (?:(%{DATA:referer}|-)) "%{DATA:useragent}" %{IPV4:clientip} (?:(%{NUMBER:duration}|%{DATA:duration}))' ]
     match => [ "message", '%{WORD:xforwardedproto} (?<site>([\S+]*)) (?:(%{IPORHOST:http_baidu_cdn_real_ip}|%{DATA:http_baidu_cdn_real_ip})) (?:(%{IPORHOST:http_ali_cdn_real_ip}|%{DATA:http_ali_cdn_real_ip})) \[%{HTTPDATE:timestamp}\] (?<urlpath>([\S+]*)) %{NUMBER:status} %{NUMBER:bytes} (?:(%{DATA:referer}|-)) "%{DATA:useragent}" %{IPV4:clientip} (?:(%{NUMBER:duration}|-))' ]
     match => [ "message", '%{WORD:xforwardedproto} (?<site>([\S+]*)) (?:(%{IPORHOST:http_baidu_cdn_real_ip}|%{DATA:http_baidu_cdn_real_ip})) (?:(%{IPORHOST:http_ali_cdn_real_ip}|%{DATA:http_ali_cdn_real_ip})) (?:(%{DATA:clientip}|%{IPORHOST:clientip})) \[%{HTTPDATE:timestamp}\] %{WORD:method} (?<urlpath>([\S+]*)) HTTP/%{NUMBER:httpversion} %{NUMBER:status} %{NUMBER:bytes} (?:(%{DATA:referer}|-)) "%{DATA:useragent}" (?<http_x_forwarded_for>([\d.,]*)) (?:(%{NUMBER:duration}|%{DATA:duration}))' ]
              }
          date {
               match => [ "timestamp" , "dd/MMM/YYYY:HH:mm:ss Z" ]
               target => "@timestamp"
          }
          mutate {
                add_tag => ["accesslog"]
        remove_field => [ "agent","log","@version","ecs","host","timestamp" ]
                remove_field => "message"
          }
      }


}
output {
  if "_grokparsefailure" in [tags] {
    file { "path" => "E:/lyf/tmp/grok_failures.txt" }
  } else {
    if "mysqlslowlogs" in [tags] {
          elasticsearch {
          hosts => ["10.168.6.89:9800"]                  # ElasticSearch的地址加端口
          index => "mysqlslowlogs"            # ElasticSearch的保存文档的index名称，
      }
    }
 
   if "accesslog" in [tags] {
          elasticsearch {
          hosts => ["10.168.6.89:9800"]                  # ElasticSearch的地址加端口
          index => "accesslog"            # ElasticSearch的保存文档的index名称，
      }
    }
}


}

 

nginx日志格式：

log_format bdformat '$scheme $host $http_x_real_ip $http_ali_cdn_real_ip $remote_addr [$time_local] $request $status $body_bytes_sent $http_referer "$http_user_agent" $http_x_forwarded_for $upstream_response_time';

 

log_format bdformat '$scheme $host $http_x_real_ip $http_ali_cdn_real_ip $remote_addr [$time_local] $request $status $body_bytes_sent $http_referer "$http_user_agent" $http_x_forwarded_for $upstream_response_time';