elasticSearch用xxl-job方式从mysql导入数据

结合公司实际业务记录这次的导数流程

  1 //注册到执行器工厂：添加“@JobHandler(value="自定义jobhandler名称")”注解，注解value值对应的是调度中心新建任务的JobHandler属性的值。
  2 @JobHandler(value = "BiddingHandler")
  3 @Component
  4 public class BiddingHandler extends IJobHandler {
  5    @Autowired
  6     SaleManageDao saleManageDao;
  7     @Autowired(required = false)
  8     RestHighLevelClient restHighLevelClient;
  9 
 10     @Override
 11     public ReturnT<String> execute(String param) throws Exception {
 12         try {
 13             XxlJobLogger.log("##任务开始##");
 14             //程序计数器,初始值为2,代表有两个子线程结束后才执行主线程
 15             final CountDownLatch latch = new CountDownLatch(2);
 16             //单线程池
 17             ExecutorService es2 = Executors.newSingleThreadExecutor();
 18             es2.execute(new Runnable() {
 19                 @Override
 20                 public void run() {
 21                     //主要业务代码,从sql库查出数据,经过业务逻辑处理数据,再插入到es
 22                     XxlJobLogger.log("供应商："+"执行");
 23                     BulkRequest bulkRequest = new BulkRequest();
 24                     boolean flag = true;
 25                     int startNum = 0; // 开始下标数量
 26                     int requestNum = 2000; // 请求条数
 27                     while(flag){
 28                         //从mysql里查询数据
 29                         List<JSONObject> supplier = saleManageDao.supplier(startNum,requestNum);
 30                         //对数据进行业务处理,码代码省略
 31                         if(ValidateObject.hasValueInCollection(supplier)){
 32                             XxlJobLogger.log("供应商结束获取信息..."+"【"+startNum+","+requestNum+"】");
 33                             for (JSONObject jsonObject : supplier) {
 34                                 //设置这条数据的索引,类型,主键
 35                                 IndexRequest indexRequest = new IndexRequest("bidding","bidder_search", MD5Util.MD5(jsonObject.get("companyName").toString()));
 36                                 //添加本条数据
 37                                 indexRequest.source(jsonObject);
 38                                 //将本条数据添加到批处理中
 39                                 bulkRequest.add(indexRequest);
 40                             }
 41                             BulkResponse bulk = null;
 42                             try {
 43                                 //执行批处理
 44                                 bulk = restHighLevelClient.bulk(bulkRequest, RequestOptions.DEFAULT);
 45                             } catch (IOException e) {
 46                                 e.printStackTrace();
 47                             }
 48                             XxlJobLogger.log("供应商hasFailures:---->  "+bulk.hasFailures());
 49                             System.out.println(startNum+","+requestNum+"供应商数据录入状态:---->  "+bulk.status().getStatus());
 50                             XxlJobLogger.log(startNum+","+requestNum+"供应商数据录入状态:---->  "+bulk.status().getStatus());
 51                         } else {
 52                             flag = false;
 53                         }
 54 
 55                         if(supplier.size() == requestNum){
 56                             startNum += requestNum;
 57                             flag = true;
 58                         } else {
 59                             flag = false;
 60                         }
 61                     }
 62                     //子线程结束计数器减1
 63                     latch.countDown();
 64                 }
 65             });
 66             //关闭线程池
 67             es2.shutdown();
 68 
 69             ExecutorService es1 = Executors.newSingleThreadExecutor();
 70             es1.execute(new Runnable() {
 71                 @Override
 72                 public void run() {
 73                     XxlJobLogger.log("采购方："+"执行");
 74                     BulkRequest bulkRequestP = new BulkRequest();
 75                     boolean flagP = true;
 76                     int startNumP = 0; // 开始下标数量
 77                     int requestNumP = 1000; // 请求条数
 78                     while(flagP){
 79                         List<JSONObject> purchaser = saleManageDao.purchaser(startNumP,requestNumP);
 80                         if(ValidateObject.hasValueInCollection(purchaser)){
 81                             XxlJobLogger.log("采购方结束获取信息..."+"【"+startNumP+","+requestNumP+"】");
 82                             for (JSONObject jsonObject : purchaser) {
 83                                 IndexRequest indexRequest = new IndexRequest("bidding","bidder_search",MD5Util.MD5(jsonObject.get("companyName").toString()));
 84                                 indexRequest.source(jsonObject);
 85                                 bulkRequestP.add(indexRequest);
 86                             }
 87                             BulkResponse bulkP = null;
 88                             try {
 89                                 bulkP = restHighLevelClient.bulk(bulkRequestP, RequestOptions.DEFAULT);
 90                             } catch (IOException e) {
 91                                 e.printStackTrace();
 92                             }
 93                             XxlJobLogger.log("采购方hasFailures:---->  "+bulkP.hasFailures());
 94                             XxlJobLogger.log(startNumP+","+requestNumP+"采购方数据录入状态:---->  "+bulkP.status().getStatus());
 95                         }else {
 96                             flagP = false;
 97                         }
 98 
 99                         if(purchaser.size() == requestNumP){
100                             startNumP += requestNumP;
101                             flagP = true;
102                         } else {
103                             flagP = false;
104                         }
105                     }
106                     latch.countDown();
107                 }
108             });
109             es1.shutdown();
110             try {
111                 latch.await();
112             } catch (InterruptedException e) {
113                 e.printStackTrace();
114             }
115 
116         } catch (Exception e) {
117             e.printStackTrace();
118             XxlJobLogger.log("##异常##"+"-->"+e.getMessage());
119             return ReturnT.FAIL;
120         }
121         XxlJobLogger.log("##任务结束##");
122         return ReturnT.SUCCESS;
123 
124     }
125 
126 }

 

RestHighLevelClient配置类代码:
@Configuration
public class ElasticsearchConfig {
 @Bean
    public static RestHighLevelClient restHighLevelClient() {
        final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
        credentialsProvider.setCredentials(AuthScope.ANY,
                new UsernamePasswordCredentials("用户名", "密码"));
        RestHighLevelClient client = new RestHighLevelClient(
                RestClient.builder(new HttpHost("地址", 端口))
                        .setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
                            public HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder) {
                                return httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
                            }
                        }));
        return client;
    }
}

 

posted @ 2022-04-08 11:00  杨吃羊  阅读(289)  评论(0)    收藏  举报