java批量数据写入到es

 

1.pom.xml配置

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>org.example</groupId>
    <artifactId>myjava_proj</artifactId>
    <version>1.0-SNAPSHOT</version>

    <properties>
        <maven.compiler.source>21</maven.compiler.source>
        <maven.compiler.target>21</maven.compiler.target>
        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
    </properties>

    <dependencies>
        <!--mysql驱动 -->
        <dependency>
            <groupId>mysql</groupId>
            <artifactId>mysql-connector-java</artifactId>
            <version>5.1.49</version>
        </dependency>

        <!--es驱动 -->
        <dependency>
            <groupId>org.elasticsearch.client</groupId>
            <artifactId>elasticsearch-rest-high-level-client</artifactId>
            <version>6.8.5</version>
        </dependency>

    </dependencies>

</project>

 

2.创建索引

curl -u elastic:elastic -X PUT "192.168.1.134:19200/metric_pl?pretty" -H 'Content-Type: application/json' -d'
{}
'

curl -u elastic:elastic -H 'Content-Type: application/json' -XPOST "http://192.168.1.134:19200/metric_pl/_doc/_mapping?pretty" -d '
{
        "properties" : {
          "app" : {
            "type" : "text",
            "fields" : {
              "keyword" : {
                "type" : "keyword",
                "ignore_above" : 256
              }
            }
          },
          "blocked_qps" : {
            "type" : "long"
          },
          "count" : {
            "type" : "long"
          },
          "exception_qps" : {
            "type" : "long"
          },
          "gmt_create" : {
            "type" : "text",
            "fields" : {
              "keyword" : {
                "type" : "keyword",
                "ignore_above" : 256
              }
            }
          },
          "gmt_modified" : {
            "type" : "text",
            "fields" : {
              "keyword" : {
                "type" : "keyword",
                "ignore_above" : 256
              }
            }
          },
          "id" : {
            "type" : "long"
          },
          "machine_ip" : {
            "type" : "text",
            "fields" : {
              "keyword" : {
                "type" : "keyword",
                "ignore_above" : 256
              }
            }
          },
          "month" : {
            "type" : "date"
          },
          "pass_qps" : {
            "type" : "long"
          },
          "resource" : {
            "type" : "text",
            "fields" : {
              "keyword" : {
                "type" : "keyword",
                "ignore_above" : 256
              }
            }
          },
          "resource_code" : {
            "type" : "long"
          },
          "rt" : {
            "type" : "long"
          },
          "success_qps" : {
            "type" : "long"
          },
          "timestamp" : {
            "type" : "date",
            "store" : true,
            "format" : "yyyy-MM-dd HH:mm:ss"
          }
          }
}'

 

3.java 代码

 

package org.hxl.es;

import org.apache.http.HttpHost;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.common.xcontent.XContentType;

import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.Random;

public class InsertDataEsBulk {

    public static RestHighLevelClient getConnect(){
        BasicCredentialsProvider credentialsProvider = new BasicCredentialsProvider();
        credentialsProvider.setCredentials(
                AuthScope.ANY,
                new UsernamePasswordCredentials("elastic", "elastic"));
        //创建ES客户端
        RestHighLevelClient esClient = new RestHighLevelClient(
                RestClient.builder(new HttpHost("192.168.1.134",19200,"http"))
                        .setHttpClientConfigCallback(
                        httpClientBuilder -> httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider)
                                            .setMaxConnTotal(100)
                                            .setMaxConnPerRoute(20)
                                            .setKeepAliveStrategy((response, context) -> 60000)) //保持连接时间
                        .setRequestConfigCallback(requestConfigBuilder -> requestConfigBuilder
                                            .setConnectTimeout(5000) // 连接超时时间
                                            .setSocketTimeout(60000)) // 套接字超时时间 // 保持连接时间
        );
        return esClient;
    }


    public static void InsertData() throws IOException {
        RestHighLevelClient esClient = getConnect();

        SimpleDateFormat sdfdt = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
        SimpleDateFormat sdfdt_mon = new SimpleDateFormat("yyyy-MM-dd");

        BulkRequest request = new BulkRequest();

        Date now = new Date();

        for (int i = 1; i <= 998; i++) {
            //Date now = new Date();
            //IndexRequest ir = new IndexRequest("user").type("_doc").source(XContentType.JSON, "name", "张起灵", "sex", "boy", "age", i);
            String now_day = sdfdt_mon.format(now);

            Random rand = new Random();
            int rnd = rand.nextInt();

            String app = "digitalclinic-hospital-web";
            int blocked_qps = 0;
            int count = 1;
            int exception_qps = 0;
            String gmt_create = sdfdt.format(now);
            String gmt_modified = sdfdt.format(now);
            int id=i;
            String machine_ip="127.0.0.1";
            int pass_qps = i * rnd;
            String resource = "com.digitalclinic.preparation.service.InformTagService";
            String resource_code="1443828042";
            int rt = 100;
            int success_qps = 200;
            String timestamp=sdfdt.format(now);
            String month=now_day;

            IndexRequest ir = new IndexRequest("metric_pl").type("_doc").
                    source(XContentType.JSON,
                            "app", app,
                            "blocked_qps",blocked_qps,
                            "count", count,
                            "exception_qps", exception_qps,
                            "gmt_create", gmt_create,
                            "gmt_modified", gmt_modified,
                            "id",id,
                            "machine_ip", machine_ip,
                            "pass_qps", pass_qps,
                            "resource", resource,
                            "resource_code", resource_code,
                            "rt", rt,
                            "success_qps", success_qps,
                            "timestamp", timestamp,
                            "month", month
                    );

            request.add(ir);

            if (i%100 == 0){
                System.out.println("ok");
                System.out.println(request.numberOfActions());
                System.out.println(request.estimatedSizeInBytes());
                try {
                    BulkResponse response = esClient.bulk(request, RequestOptions.DEFAULT);
                    request = new BulkRequest();
                }catch (IOException e){
                    e.printStackTrace();
                }
            }
        }

        System.out.println(request.numberOfActions());
        System.out.println(request.estimatedSizeInBytes());

        if (request.numberOfActions() != 0) {
            try {
                BulkResponse response = esClient.bulk(request, RequestOptions.DEFAULT);
            } catch (IOException e) {
                e.printStackTrace();
            } finally {
                esClient.close();
            }
        }

        esClient.close();
    }

    public static void main(String[] args) throws IOException {
        SimpleDateFormat sdfdt = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
        Date now = new Date();
        String now_time = sdfdt.format(now);
        System.out.println("开始时间 :" + now_time);

        InsertData();

        Date endtime = new Date();
        String end_time = sdfdt.format(endtime);
        System.out.println("结束时间:" + end_time);
    }
}