springboot 实现数据库到ES数据的迁移

1.配置数据库和ES连接，以及相应的数据库信息

2.增加如下代码

package com.thunisoft.dzsjfcg.db2es;

import com.alibaba.fastjson.JSONObject;
import com.google.common.collect.ImmutableMap;
import org.apache.commons.lang3.time.FastDateFormat;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentType;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;

import javax.annotation.PostConstruct;
import java.util.List;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicLong;

/**
 * @Author ZhPJ
 * @Date 2019/10/11 001119:58
 * @Version 1.0
 * @Description:
 *
 *
 * ------------------------------------------
 * 谨慎运行：如果进行迁移，那么放开@Component注解即可
 * ----------------------------------------
 *
 */
@Component
public class DB2ES {

    @Autowired
    private DB2ESMapper db2ESMapper;
    // 导入以前数据的线程池大小
    private final Integer FIX_POOL_SIZE = 10;
//    @Value("${elasticSearch.esIndexKey}")
    private String index = "test";
    // 以前的数据多少页
    private AtomicLong page = new AtomicLong(-1);
    // 以前的数据总数
    private AtomicLong countTotal = new AtomicLong(0);
    // 线程池Count
    private final CountDownLatch countDownLatch = new CountDownLatch(FIX_POOL_SIZE);
    // 查询的长度，要和limit的一致，现在是5000
    private final int size = 5000;

    @Autowired
    private RestHighLevelClient restHighLevelClient;

    @PostConstruct
    public void db2es() throws Exception {
        runOldDatas2DB()；
    }

    private synchronized void setPage() {
        page.addAndGet(1);
    }

    /**
     * @param flag
     * @throws Exception
     */
    private void runOldDatas2DB( flag) throws Exception{
        final long count = db2ESMapper.countDbToday();
        final long startTime = System.currentTimeMillis();
        long tempTotalPage = count/size;
        if (count % size > 0) {
            tempTotalPage ++;
        }
        final long totalPage = tempTotalPage;
        final ExecutorService pool = Executors.newFixedThreadPool(FIX_POOL_SIZE);
        for (int i = 0; i < FIX_POOL_SIZE; i ++) {
            Thread.sleep(100);
            pool.execute(() -> {
                while (page.longValue() <= totalPage) {
                    setPage();
                    System.out.println("++++++++++++++++++++++++++++++++++++++" + page + "++++++++++++++++++++++++++++++++++++++");
                    final List<Db2EsBean> offsetListDB = db2ESMapper.queryDbCzxxToday(ImmutableMap.of("offset", page.longValue() * size));

                    if (!CollectionUtils.isEmpty(offsetListDB)) {
                        BulkRequest bulkRequest = new BulkRequest();
                        for (Db2EsBean db2EsBean : offsetListDB) {
　　　　　　　　　　　　　　     // db2EsBean.getcBh() 是数据库中的主键，更改成ES的_id
                            bulkRequest.add(new IndexRequest(index).id(db2EsBean.getcBh()).source(JSONObject.toJSONString(db2EsBean), XContentType.JSON).timeout(TimeValue.timeValueSeconds(500)));
                        }
                        try {
                            final BulkResponse bulk = restHighLevelClient.bulk(bulkRequest, RequestOptions.DEFAULT);
                            System.out.println(countTotal.addAndGet(bulk.getItems().length));
                        } catch (Exception e) {
                            e.printStackTrace();
                        }
                    }
                }
                countDownLatch.countDown();
            });
        }
        countDownLatch.await();
        System.out.println("-------共："+countTotal.get()+"条数据----------消耗时间：" + (System.currentTimeMillis() - startTime) + "----------");
        pool.shutdown();
    }

}

3. 连接数据库的mapper

package com.thunisoft.dzsjfcg.db2es;

import org.apache.ibatis.annotations.Mapper;
import org.apache.ibatis.annotations.Select;

import java.util.List;
import java.util.Map;

/**
 * @Author ZhPJ
 * @Date 2019/10/11 001119:59
 * @Version 1.0
 * @Description:
 */
@Mapper
public interface DB2ESMapper {

    String QUERY_FILED = "c_bh AS cBh";

    @Select("select " + QUERY_FILED + " from t order by c_bh desc limit 10000 offset #{offset}")
    List<Db2EsBean> queryDbCzxxToday(Map<String, Long> params);

    @Select("select count(*) from t")
    int countDbToday();
}