logstash的grok配置范例
日志范例：
172.59.225.30 - - [06/Sep/2017:10:42:24 +0800] "GET /fed/user/spsloosso?doneURL=http%3A%2F%2Fportal.croo.com.cn%2Foam%2Fserver%2Flogout HTTP/1.1" 302 1945 25575
/fmsauth/ssologin
172.132.6.46 - - [06/Sep/2017:10:18:20 +0800] "GET /fmsauth/ssologin HTTP/1.1" 302 321 155963
192.168.105.22 - - [06/Sep/2017:11:21:33 +0800] "GET /fmsauth/ssologin?requestUrl=http%3A%2F%2Ffmsjtap.croo.com.cn%3A8008%2FOA_HTML%2Fjsp%2Ffnd%2Fclose.jsp&cancelUrl=http%3A%2F%2Ffmsjtap.croo.com.cn%3A8008%2FOA_HTML%2FAppsLogin HTTP/1.1" 302 783 117057
172.62.64.137 - - [06/Sep/2017:11:29:47 +0800] "GET /fed/user/authnoam?refid=id-ZWgb5IXmL5U1DD3sCVN0RocuwGE- HTTP/1.1" 302 458 1392
1、配置input
######################################################
###################### input ########################
######################################################
input {
file {
path => "/var/log/messages"
sincedb_path => "/usr/share/logstash/data/logstash.sincedb"
sincedb_write_interval => 15
type => "linuxmessages"
start_position => "end"
max_open_files => 4095
stat_interval => 1
add_field => { "hostip" => "172.0.129.52" }
add_field => { "hostname" => "test12317" }
codec => multiline {
pattern => "\[<"
negate => true
what => "previous"
}
#codec => json { charset => "UTF-8" }
}
file {
path => "/var/log/secure"
sincedb_path => "/usr/share/logstash/data/logstash.sincedb"
sincedb_write_interval => 15
type => "linuxsecure"
start_position => "end"
max_open_files => 4095
stat_interval => 1
add_field => { "hostip" => "172.0.129.52" }
add_field => { "hostname" => "test12317" }
#codec => json { charset => "UTF-8" }
}
file {
path => "/var/log/dmesg"
sincedb_path => "/usr/share/logstash/data/logstash.sincedb"
sincedb_write_interval => 15
type => "linuxdmesg"
start_position => "end"
max_open_files => 4095
stat_interval => 1
add_field => { "hostip" => "172.0.129.52" }
add_field => { "hostname" => "test12317" }
#codec => json { charset => "UTF-8" }
}
file {
path => "/var/log/history_audit.log"
sincedb_path => "/usr/share/logstash/data/logstash.sincedb"
sincedb_write_interval => 15
type => "linuxhistory"
start_position => "end"
max_open_files => 4095
stat_interval => 1
add_field => { "hostip" => "172.0.129.52" }
add_field => { "hostname" => "test12317" }
#codec => json { charset => "UTF-8" }
}
file {
path => "/usr/share/logstash/bin/chk.out"
sincedb_path => "/usr/share/logstash/data/logstash.sincedb"
sincedb_write_interval => 15
type => "logstashcheck"
start_position => "end"
max_open_files => 4095
stat_interval => 1
add_field => { "hostip" => "172.0.129.52" }
add_field => { "hostname" => "test12317" }
#codec => json { charset => "UTF-8" }
}
file {
path => "/oracle/web/apache2/logs/*access_log"
sincedb_path => "/usr/share/logstash/data/logstash.sincedb"
sincedb_write_interval => 15
type => "ldapapacheaccess"
start_position => "end"
max_open_files => 4095
stat_interval => 1
add_field => { "hostip" => "172.0.129.52" }
}
sincedb_path => "/usr/share/logstash/data/logstash.sincedb"
sincedb_write_interval => 15
type => "ldapoamaccess"
start_position => "end"
max_open_files => 4095
stat_interval => 1
add_field => { "hostip" => "172.0.129.52" }
add_field => { "hostname" => "test12317" }
#codec => json { charset => "UTF-8" }
}
}
2、配置filter
###################### linuxmessages ########################
## log- model: Jul 27 22:37:02 testhvpra01 snmpd[1184]: Connection from UDP: [172.0.89.110]:60150->[172.0.89.116]:161 ##
if [type] == "linuxmessages" {
grok {
patterns_dir => "/usr/share/logstash/patterns/"
match => { "message" => "%{LINUXMESSSLOG}" }
}
drop{}
}
if [lmesg_message] =~ ":" {
grok {
match => { "lmesg_message" => "%{DATA:lmesg_program}(?:\[%{POSINT:lmesg_pid}\])?: %{GREEDYDATA:lmesg_mesg}" }
}
}
timezone => "Asia/Shanghai"
match => [ "lmesg_timestamp", "MMM d HH:mm:ss", "MMM dd HH:mm:ss", "ISO8601" ]
target => "@timestamp"
}
# json { source => "message" }
}
###################### linuxsecure ########################
## log- model: Jul 27 23:00:34 testhvpra01 su: pam_unix(su-l:session): session opened for user appuser by root(uid=0) ##
if [type] == "linuxsecure" {
grok {
break_on_match => true
match => [
]
}
if [message] =~ /(?i)session opened/{
# #Mar 6 09:44:52 zdhdpppcd01 su: pam_unix(su-l:session): session opened for user gpadmin by (uid=0)
mutate {
add_field => { "lsecure_pam_session" => "opened" }
match => [
"message", "session opened for user %{WORD:lsecure_pam_user}"
]
}
}
else if [message] =~ /(?i)session closed/{
#Mar 6 09:42:06 zdhdpppcd07 su: pam_unix(su-l:session): session closed for user root
mutate {
add_field => { "lsecure_pam_session" => "closed" }
}
grok {
match => [
"message", "session closed for user %{WORD:lsecure_pam_user}"
]
}
}
else if [message] =~ /(?i)expire/{
#Mar 6 10:10:01 kyisvprd01 crond[29756]: pam_unix(crond:account): password for user EPM will expire in 6 days
mutate {
add_field => { "lsecure_pam_session" => "expire" }
}
grok {
match => [
"message", "password for user %{WORD:lsecure_pam_user} will expire in %{NUMBER:lsecure_expire_days:int}"
]
}
}
else if [message] =~ /(?i)authentication failure/{
mutate {
add_field => { "lsecure_pam_session" => "authfailure" }
}
grok {
match => [
"message", ".*%{IPV4:lsecure_pam_rhost}\D+%{WORD:lsecure_pam_user}"
]
}
}
else if [message] =~ /(?i)Accepted password/{
# #Mar 6 01:09:03 testhvpra04 sshd[30112]: Accepted password for root from 172.0.89.166 port 37452 ssh2
mutate {
add_field => { "lsecure_pam_session" => "accepted" }
}
grok {
match => [
"message", "Accepted password for %{WORD:lsecure_pam_user}\D+%{IPV4:lsecure_pam_rhost}"
]
}
}
add_field => { "lsecure_pam_session" => "changed" }
}
grok {
match => ["message", "password changed for %{WORD:lsecure_pam_user}"]
}
}
else if [message] =~ /(?i)Failed password/{
#Mar 7 13:10:12 xyizvpra01 sshd[15227]: Failed password for appuser from 172.53.96.35 port 64910 ssh2
mutate {
add_field => { "lsecure_pam_session" => "failed" }
}
grok {
match => ["message", "password for %{WORD:lsecure_pam_user}\D+%{IPV4:lsecure_pam_rhost}"]
}
}
date {
locale => "en_US"
timezone => "Asia/Shanghai"
match => [ "lsecure_timestamp", "MMM d HH:mm:ss", "MMM dd HH:mm:ss", "ISO8601" ]
target => "@timestamp"
}
}
###################### linuxhistory ########################
if [type] == "linuxhistory" {
grok {
patterns_dir => "/usr/share/logstash/patterns/"
match => { "message" => "%{LINUXHISTORYLOG}" }
add_field => ["exectime", "%{lhist_timestamp}"]
}
if [lhist_command] =~ /(?i)rm|poweroff|shutdown|reboot|passwd|halt/{
mutate {
add_tag => "lhistalert"
add_field => { "lhist_alert" => "alert" }
}
}
date {
locale => "en_US"
timezone => "Asia/Shanghai"
match => [ "lhist_timestamp", "yyyy/MM/dd HH:mm:ss", "ISO8601" ]
target => "@timestamp"
}
# json { source => "message" }
}
###################### linuxdmesg ########################
### log- model: 2017/07/27 19:29:27 root(root)@[IP:(172.52.192.26)][PWD:/app/filebeat][LOGIN:2017-07-27 19:05 .] --- 2017/07/27 19:29:07 vim config/filebeat.yml ##
if [type] == "linuxdmesg" {
grok {
patterns_dir => "/usr/share/logstash/patterns/"
match => { "message" => "%{LINUXDMESG}" }
}
# json { source => "message" }
}
###################### logstashcheck ########################
if [type] == "logstashcheck" {
json { source => "message" }
mutate {
convert => { "state" => "integer" }
convert => { "checktimes" => "integer" }
}
}
###################### ldapapacheaccess ##########################
if [type] == "ldapapacheaccess" {
translate {
override => true
exact => false
field => "message"
destination => "message"
dictionary => [
"%253D", "=",
"%253F", "?",
"%253B", ";",
"%252F", "/",
"%2520", " ",
"%2521", "!",
"%2523", "#",
"%257E", "~",
"%252E", ".",
"%252B", "+",
"%2524", "$",
"%252A", "*",
"%252D", "-",
"%2526", "&",
"%253A", ":",
"%2F", "/",
"%23", "#",
"%25", "%",
"%7E", "~",
"%2E", ".",
"%2B", "+",
"%24", "$",
"%28", "(",
"%29", ")",
"%3C", "<",
"%3E", ">",
"%7B", "{",
"%7D", "}",
"%2A", "*",
"%2D", "-",
"%40", "@",
# "%5C%5C", "\\",
# "%22", ""',
"%26", "&"
]
#remove_field => [ "" ]
}
grok {
break_on_match => true
match => [
]
}
mutate {
convert => { "ldpapa_response_time" => "integer" }
convert => { "ldpapa_bytes" => "integer" }
convert => { "ldpapa_status" => "integer" }
convert => { "ldpapa_httpversion" => "float" }
split => { "ldpapa_request_content" => " " }
}
if [ ldpapa_username ] =~ /(?i)-/ {
mutate {
add_field => { "ldpapa_username2" => "nobody" }
add_tag => "nobody"
}
}
date {
locale => "en_US"
timezone => "Asia/Shanghai"
match => [ "ldpapa_timestamp", "dd/MMM/yyyy:HH:mm:ss Z", "ISO8601" ]
target => "@timestamp"
}
grok {
match => [ "ldpapa_request_content", "%{WORD:ldapa_request_app1}/%{WORD:ldpada_request_app2}" ]
}
kv {
source => "ldpapa_request_content"
target => "ldpapa_kv_trans"
# field_split => "&?"
}
# json { source => "message" }
}
############# refuse before 5 days events ##########################
ruby {
code =>"event.cancel if 2 * 24 * 3600 < (event.get('@timestamp')-::Time.now).abs"
}
######################################################
###################### output ########################
######################################################
output {
kafka {
topic_id => "inputData"
codec => json
bootstrap_servers => "172.0.89.144:9092"
id => "logstashinput"
}
#output {
# file {
# path => "/tmp/test.logstash"
# }
#
# stdout { codec => rubydebug }
#
}
浙公网安备 33010602011771号