如何通过logstash 把json log 同步给elasticsearch

如何通过logstash 把json log 同步给elasticsearch

# Sample Logstash configuration for creating a simple
# Beats -> Logstash -> Elasticsearch pipeline.

input { 
      file {
          path => "Z:/logs/rade_2022_05_24.log"
          start_position => "beginning"
          sincedb_path => "Z:/logs/sincedb"
          codec => json {
            charset => "GB2312"
        }      
      }
}
output {
  elasticsearch {
    action => "index"
    hosts => ["https://17.31.26.66:9200/"]
    index => "orderdb"
    workers => 1
    user => "elastic"
    password => "ZDO8ohu43oUdu"
    cacert => "D:\soft\elasticsearch-8.2.0\aa.cer"
    ssl => true
    ssl_certificate_verification => false
  }
  stdout {
         codec => rubydebug
         #codec => json_lines
     }
}

Logstash如何读取logs 到mysql

1. 先安装logstash-output-jdbc
 .\logstash-plugin.bat install logstash-output-jdbc
2. 下载mysql-connector-java-6.0.6-bin.jar 复制到/vendor/jar/jdbc下面
https://downloads.mysql.com/archives/c-j/

config如下

# Sample Logstash configuration for creating a simple
# Beats -> Logstash -> Elasticsearch pipeline.

input { 
      file {
          path => "Z:/erplogs/2022_05_24.log"
          start_position => "beginning"
          sincedb_path => "Z:/erplogs_sincedb/mysql_sincedb"
          codec => json {
            charset => "GB2312"
        }      
      }
}
filter {
#删除空值
  if(![id]){
    drop{}
  }
  grok {
    remove_field => ["message"]
  }
}
output {
    jdbc {
        driver_jar_path => "D:/soft/logstash-8.2.0/vendor/jar/mysql-connector-java-5.1.49.jar"
        driver_class => "com.mysql.jdbc.Driver"
        connection_string => "jdbc:mysql://5.22.145.87:3306/test_db"
        username => "root"
        password =>"3234"
        statement => [ "INSERT INTO test(id,name) values(?,?)", "[id]", "[name]"  ]
    stdout {
        #codec => rubydebug
        codec => json_lines
    }
}