ELK线上系统配置整理-logstash（一）

配置文件：

/usr/local/logstash-6.2.3/config/redis-input.conf

#redis-input
input {
  beats {
    port => 5044
  }
}

filter {
  if [fields][logtype] == "Artery" {
      grok {
        patterns_dir => ["/usr/local/logstash-6.2.3/patterns.d"]
        match => {
          "message" => "%{ARTERY}"
        }
      }
#Drop message
    if ([Level] =~ "DEBUG") and ("BeanParametersMapping" not in [LogMessage]) {
      if ([Level] =~ "DEBUG") and ("SQL" not in [LogMessage]) {
        if ([Level] =~ "DEBUG") and ("MapParametersMapping" not in [LogMessage]) {
    drop {}
    }
      }
    }


  } else if [fields][logtype] == "Tomcat-Fore" {
    grok {
      patterns_dir => ["/usr/local/logstash-6.2.3/patterns.d"]
      match => {
    "message" => "%{TOMCAT}"
      }
    }
#Drop message

    if ([Level] =~ "INFO") and ("com.izejin.fore.artery.ArteryService" not in [LogMessage]) {
    drop {}
    }


    if ([message] =~ "SetRequestTypeInterceptor.java") or ([message] =~ "DiskWriteTask.call") {
    drop {}
    }

    if  ([message] =~ "ThreadPoolAsynchronousRunner$DeadlockDetector@32eedbae") {
    drop {}
    }


  } else if [fields][logtype] == "Tomcat-Back" {
    grok {
      patterns_dir => ["/usr/local/logstash-6.2.3/patterns.d"]
      match => {
        "message" => "%{TOMCAT}"
      }
    }
#Drop Message
  if ([message] =~ "DiskWriteTask.call") or ([message] =~ "ThreadPoolAsynchronousRunner") {
    drop {}
    }

    if ([Level] == "INFO") {
    drop {}
    }
  } else if [fields][logtype] == "Nginx-access"{
        date {
          match => [ "timestamp","dd/MMM/yyyy:HH:mm:ss Z"]
        }
}
#LogTime Replace TimeStamp
  date {
    match => ["DateTime", "yyyy-MM-dd HH:mm:ss,SSS"]
    target => "@timestamp"
  }
}
output {
  if ("ERROR" in [Level]) and ("今日错误次数达5次账户将被锁定无法登陆" not in [LogMessage]) {
    if ("ERROR" in [Level]) and ("操作频繁" not in [LogMessage]) {
      if ("ERROR" in [Level]) and ("请选择应付账款批量导入模板" not in [LogMessage]) {
        if ("ERROR" in [Level]) and ("java.net.SocketException: Broken pipe" not in [LogMessage]) {
      if ("ERROR" in [Level]) and ("get-notice-list.json" not in [LogMessage]) {
        if ("ERROR" in [Level]) and ("com.izejin.base.utils.Utils$Files.doSftp" not in [LogMessage]) {
          if ("ERROR" in [Level]) and ("login/ukey-check.json" not in [LogMessage]) {
            if ("ERROR" in [Level]) and ("get-payables-list.json" not in [LogMessage]) {
          if ("ERROR" in [Level]) and ("支付日期不能为空" not in [LogMessage]) {
            if ("ERROR" in [Level]) and ("收单企业不能为空" not in [LogMessage]) {

    email {
    port => "587"
    address => "smtp.live.com"
#    address => "smtp.office365.com"
    username => "zejin_elk@hotmail.com"
#    username => "zejin-elk@hotmail.com"
    password => "Zejin2018"
    authentication => "login"
    use_tls => true
    from => "zejin_elk@hotmail.com"
#    from => "zejin-elk@hotmail.com"
    subject => "%{fields} Warning: You have a Error"
    to => "huangzhenqiang@izejin.com, xuyanhui@izejin.com, wuxueping@izejin.com, chenting@izejin.com"
    via => "smtp"
    body => "%{message}"
    }
            }
          }
        }
          }
        }
      }
        }
      }
    }
  }

#  stdout {
#    codec => rubydebug
#  }
  if [fields][logtype] == "Tomcat-Fore" {
    redis {
      host => '172.31.5.5'
      port => 6379
      data_type => 'list'
      key => "tomcat-fore"
    }
  }

  if [fields][logtype] == "Tomcat-Back" {
    redis {
      host => '172.31.5.5'
      port => 6379
      data_type => 'list'
      key => "tomcat-back"
    }
  }

  if [fields][logtype] == "Artery" {
    redis {
      host => '172.31.5.5'
      port => 6379
      data_type => 'list'
      key => "artery"
    }
  }

  if [fields][logtype] == "Nginx-access" {
    redis {
      host => '172.31.5.5'
      port => 6379
      data_type => 'list'
      key => "artery"
    }
  }
}

 

/data/ela/logstash-6.2.3/config/redis-output.conf

#redis-input
input {
  redis {
    data_type => "list"
    key => "tomcat-fore"
    host => "172.31.5.5"
    batch_count => 1
    port => 6379
    threads => 5
  }

  redis {
    data_type => "list"
    key => "tomcat-back"
    host => "172.31.5.5"
    batch_count => 1
    port => 6379
    threads => 5
  }

  redis {
    data_type => "list"
    key => "artery"
    host => "172.31.5.5"
    batch_count => 1
    port => 6379
    threads => 5
  }

  redis {
    data_type => "list"
    key => "Nginx-access"
    host => "172.31.5.5"
    batch_count => 1
    port => 6379
    threads => 5
  }
}

output {
#  stdout { codec => rubydebug }
  if [fields][logtype] == "Tomcat" {
    elasticsearch {
      hosts => ["127.0.0.1:9200"]
      index => "tomcat-%{+YYYY.MM.dd}"
    }
  }
  if [fields][logtype] == "Artery" {
    elasticsearch {
      hosts => ["127.0.0.1:9200"]
      index => "artery-%{+YYYY.MM.dd}"
    }
  }
  if [fields][logtype] == "Tomcat-Fore" {
    elasticsearch {
      hosts => ["127.0.0.1:9200"]
      index => "tomcat-fore-%{+YYYY.MM.dd}"
    }
  }
  if [fields][logtype] == "Tomcat-Back" {
    elasticsearch {
      hosts => ["127.0.0.1:9200"]
      index => "tomcat-back-%{+YYYY.MM.dd}"
    }
  }
  if [fields][logtype] == "Nginx-access" {
    elasticsearch {
      hosts => ["127.0.0.1:9200"]
      index => "nginx-access-%{+YYYY.MM.dd}"
    }
  }
}