elk之logstash,nginx日志

1.建立logstash配置文件，/etc/logstash/conf.d/log.conf

# ==============================INPUT==============================================
input {

	# nginx admin log
	file {
		path => "/usr/local/nginx/logs/admin251_json.log"
		type => "nginx_admin_access"
		start_position => "beginning"
		sincedb_path => "/etc/logstash/conf.d/sincedbadmin"
		codec => "json"
	}

	# nginx wap log
	file {
		path => "/usr/local/nginx/logs/wap251_json.log"
		type => "nginx_wap_access"
		start_position => "beginning"
		sincedb_path => "/etc/logstash/conf.d/sincedbwap"
		codec => "json"
	}

	# nginx web
	file {
                path => "/usr/local/nginx/logs/www251_json.log"
                type => "nginx_web_access"
                start_position => "beginning"
		sincedb_path => "/etc/logstash/conf.d/sincedbweb"
                codec => "json"
        }

	# log
	#file {
        #        path => "/etc/logstash/conf.d/test.log"
        #        type => "test"
        #        start_position => "beginning"
	#	sincedb_path => "/etc/logstash/conf.d/sincedbtest"
        #        codec => "json"
        #}
}

# ==================================redis============================================
#output {
#	redis {
#		host => "10.10.10.10"
#		port => "6379"
#		db => "6"
#		data_type => "list"
#		key => "nginx_access"
#	}
#	stdout { codec => "rubydebug" }
#}

# ==================================filter===========================================

filter {
	# IP地址
	if [type] =~ "(nginx_admin_access|nginx_wap_access|nginx_web_access)" {
	#if [type] == "nginx_web_access" {
    		if [client_ip] !~ "^127\.|^192\.168\.|^172\.1[6-9]\.|^172\.2[0-9]\.|^172\.3[01]\.|^10\." {
		geoip {
			source => "client_ip"
			target => "geoip"
			#database => "/etc/logstash/GeoLiteCity.dat"
			add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]
			add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}" ]
		}

		# 字符类型转换，转成浮点
		mutate {
	  		convert => [ "[geoip][coordinates]", "float" ]
			#convert => [ "response","integer" ]
			#convert => [ "bytes","integer" ]
			#replace => { "type" => "nginx_access" }
			#remove_field => "message"
			}
	  
		# 日期转换
	  	#date {
	  		#match => [ "timestamp","dd/MMM/yyyy:HH:mm:ss Z"]
	  	#}
	 
		# 移除timestamp字段
	  	#mutate {
	  		#remove_field => "timestamp"
	  	#}
		}
	}

	# 用户UA信息转换
	if [user_agent] != "-" {
		useragent {
		source => "user_agent"
		target => "ua"	
		}
	}
}

# ==================================OUTPUT===========================================

output {

	# admin
	if [type] == "nginx_admin_access" {
		elasticsearch {
	  	hosts => ["112.124.37.252:9200"]
		#user => "elastic"
		#password => "changeme"
	  	index => "nginx-admin-%{+YYYY.MM.dd}"
		}
	        #stdout { codec => "rubydebug" }
	}

	# wap
        if [type] == "nginx_wap_access" {
                elasticsearch {
                hosts => ["112.124.37.252:9200"]
		#user => "elastic"
		#password => "changeme"
                index => "nginx-wap-%{+YYYY.MM.dd}"
                }
		#stdout { codec => "rubydebug" }
        }

	# web
        if [type] == "nginx_web_access" {
                elasticsearch {
                hosts => ["112.124.37.252:9200"]
		#user => "elastic"
		#password => "changeme"
                index => "nginx-web-%{+YYYY.MM.dd}"
                }
		#stdout { codec => "rubydebug" }
        }
	# test
        #if [type] == "test" {
        #        elasticsearch {
        #        hosts => ["112.124.37.252:9200"]
	#	#user => "elastic"
	#	#password => "changeme"
        #        index => "test-%{+YYYY.MM.dd}"
        #        }
	#	#stdout { codec => "rubydebug" }
        #}
}

# =====================================end===============================================