Logstash部署_Nginx

Logstash部署_Nginx

 

部署JDK环境

$ tar xvf jdk-8u191-linux-x64.tar.gz -C /usr/local/
$ mv /usr/local/jdk1.8.0_191/ /usr/local/java
$ vim /etc/profile.d/elk.sh
export JAVA_HOME=/usr/local/java
export PATH=$JAVA_HOME:$PATH:/usr/local/filebeat/
$ source /etc/profile

 

下载Logstash源码包　　

 wget https://artifacts.elastic.co/downloads/logstash/logstash-6.6.0.tar.gz

　　

解压源码包

tar xvf logstash-6.6.0.tar.gz -C /usr/local/
mv /usr/local/logstash-6.6.0/ /usr/local/logstash

安装插件
/usr/local/logstash/bin/logstash-plugin install  logstash-input-jdbc

　　

logstash-sample.conf  默认配置

[root@210 ]# cat logstash-sample.conf   默认配置
# Sample Logstash configuration for creating a simple
# Beats -> Logstash -> Elasticsearch pipeline.

input {
  beats {
    port => 5044
  }
}

output {
  elasticsearch {
    hosts => ["http://localhost:9200"]
    index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}"
    #user => "elastic"
    #password => "changeme"
  }
}

　

测试logstash 是否可以正常使用

测试logstash 是否可以正常使用
logstash -e 'input { stdin {} } output { stdout { codec => rubydebug} }'         
/usr/share/logstash/bin/logstash -e 'input { stdin {} } output { stdout { codec => rubydebug} }'    屏幕输出
/usr/share/logstash/bin/logstash -e 'input { stdin {} } output { file { path => "/tmp/test-%{+YYYY.MM.dd}.log"} }'       保存在/tmp 目录里面
/usr/share/logstash/bin/logstash -e 'input { stdin {} } output { elasticsearch { hosts => ["192.168.192.100:9200"] index => "logstash-test-%{+YYYY.MM.dd}" } }'   测试数据库发往 es 集群

　　

建议使用yum 安装

yum install -y https://artifacts.elastic.co/downloads/logstash/logstash-6.6.0.rpm

　

Logstash_Nginx.conf 配置文件

修改配置文件

input {
  beats {
    port => 5044
  }
}
output {
  stdout {
    codec => rubydebug
  }
  elasticsearch {
    hosts => ["http://192.168.1.126:9200"]
    index => "test"
  }
}

　　

logstash nginx规则匹配

nginx.conf 参数配置
input { 
	beats { port => 5044	 
		 codec => "json" 	
         		} 
} 

filter{ 
if "nginx_log" in [tags] { 
if "hc_access_log" in [tags] or "xy_access_log" in [tags] or "sj_access_log" in [tags]{ 
   geoip { 
     source => "clientip" 
  target => "geoip" 
                  database => "/etc/logstash/GeoLite2-City.mmdb" 
                  add_field => ["[geoip][coordinates]" , "%{[geoip][longitude]}"] 
                  add_field => ["[geoip][coordinates]" , "%{[geoip][latitude]}"]
                } 
                mutate { 
                        convert => [ "[geoip][coordinates]", "float"] 
                } 
   } 
else if "hc_error_log"  in [tags] or "xy_error_log" in [tags] or "sj_error_log" in [tags]{ 

grok { 
 match => { 
        "message" => [ 

            "(?<timestamp>\d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2}) \[%{DATA:err_severity}\] (%{NUMBER:pid:int}#%{NUMBER}: \*%{NUMBER}|\*%{NUMBER}) %{DATA:err_message}(?:, client: (?<client_ip>%{IP}|%{HOSTNAME}))(?:, server: %{IPORHOST:server})(?:, request: %{QS:request})?(?:, referrer: \"%{URI:referrer})?","%{DATESTAMP:timestamp} \[%{DATA:err_severity}\] %{GREEDYDATA:err_message}" 

       ] 
    } 
       } 
        date{ 
            match=>["timestamp","yyyy/MM/dd HH:mm:ss"] 
            target=>"logdate" 
        } 
        ruby{ 
            code => "event.set('logdateunix',event.get('logdate').to_i)" 
        }	 
} 
}  
} 
output { 
if "sys_log" in [tags] { 
elasticsearch { 
hosts => ["192.168.192.100:9200"] 
index => "system-%{+YYYY.MM.dd}" 
} 
} 

if "nginx_log" in [tags] { 
if "hc_access_log" in [tags] { 
stdout { codec => rubydebug } 
elasticsearch { 
                        hosts => ["192.168.192.100"] 
manage_template => true 
                        index => "logstash-nginx-hc_access-%{+YYYY.MM.dd}" 
                } 
} 

else if "xy_access_log" in [tags]{ 
stdout { codec => rubydebug } 

elasticsearch { 
                                hosts => ["192.168.192.100"] 
                                manage_template => true 
                                index => "logstash-nginx-xy_access-%{+YYYY.MM.dd}" 
                        }	 
} 

else if "sj_access_log" in [tags]{ 
                        stdout { codec => rubydebug } 
                        elasticsearch { 
                                hosts => ["192.168.192.100"] 
                                manage_template => true 
                                index => "logstash-nginx-sj_access-%{+YYYY.MM.dd}" 
                        } 
                } 

else if "hc_error_log" in [tags] { 

elasticsearch { 
hosts => ["192.168.192.100"] 
                                manage_template => true 
                                index => "logstash-nginx-hc_error-%{+YYYY.MM.dd}" 
                        } 
} 

else if "xy_error_log" in [tags] { 
elasticsearch { 
                                hosts => ["192.168.192.100"] 
                                manage_template => true 
                                index => "logstash-nginx-xy_error-%{+YYYY.MM.dd}" 
                        } 
} 

else if "sj_error_log" in [tags] { 
                        elasticsearch { 
                                hosts => ["192.168.192.100"] 
                                manage_template => true 
                                index => "logstash-nginx-sj_error-%{+YYYY.MM.dd}" 
                        } 
                } 
} 
}