es 实战

ApplicationContext介绍

 

 

 

 

ApplicationContextAware使用理解

 

 

 

 

 

 

{

   private static ElaeticSearchConfig esConfig;
    // Elasticsearch查询服务器使用9200端口，我们可以通过RESTful API直接查询数据库。
    private static final int PORT_ONE = 9200;
    // REST服务器使用9201端口，外部客户端可以使用它来连接和执行操作。
//    private static final int PORT_TWO = 9201;
    // 通信方式
    private static final String SCHEME = "http";
    // 高级客户端实例
    private static RestHighLevelClient restHighLevelClient;

    private static Logger logger=LoggerFactory.getLogger(EsHighLevelClientLatest.class);
    
    /** 超时时间设为1分钟 */
//    private static final int TIME_OUT = 60 * 1000;
   
   private static final String ID = "_id";

   private static final int STEP = 1000; // 不可超过10000
    
    private static List<String> indexNames = new ArrayList<String>();
    
    private static Map<String,Object> DEFAULT_SETTING ;
    
    private static RestHighLevelClient differentClient;

  
    public static boolean addIndexName(String indexName) {
      if (StringUtils.isEmpty(indexName) || indexNames.contains(indexName)) {
            return false;
      }
      indexNames.add(indexName);
      return true;
   }
  
   public static boolean delIndexName(String indexName) {
      if (StringUtils.isEmpty(indexName) || !indexNames.contains(indexName)) {
            return false;
      }
      indexNames.remove(indexName);
      return true;
   }
  
   /**
    * @return the indexNames
    */
   public static List<String> getIndexNames() {
      return indexNames;
   }
    
    /**
     * 实现了单例模式
     * 不会为ES创建多个连接，从而节省大量内存，并保证线程安全
     * @return RestHighLevelClient
     */
   private static RestHighLevelClient getClient() {
       if(restHighLevelClient == null) {
          if (null==ComponentContext.getApplicationContext()) {
            return null;
         }
          esConfig = ComponentContext.getEsConfig();
         String[] hosts = esConfig.getEsHost().split(",");
         HttpHost[] hs = new HttpHost[hosts.length];
         for (int i = 0; i < hosts.length; i++) {
            String[] port = hosts[i].split(":");
            if (port.length==2) {
               hs[i] = new HttpHost(port[0], Integer.valueOf(port[1]), SCHEME);
            }else {
               hs[i] = new HttpHost(port[0], PORT_ONE, SCHEME);
            }
         }
         final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
         if (StringUtils.isNotEmpty(esConfig.getUserName())) {
              credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(esConfig.getUserName(), esConfig.getPassWord()));
         }
         // 主要关于异步httpclient的连接延时配置
         restHighLevelClient = new RestHighLevelClient(RestClient.builder(hs).setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() {
               @Override
               public RequestConfig.Builder customizeRequestConfig(RequestConfig.Builder requestConfigBuilder) {
                   requestConfigBuilder.setConnectTimeout(5000);
                   requestConfigBuilder.setSocketTimeout(40000);
                   requestConfigBuilder.setConnectionRequestTimeout(1000);
                   return requestConfigBuilder;
               }
                   // 主要关于异步httpclient的连接数配置
           }).setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
               @Override
               public HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder) {
                   httpClientBuilder.disableAuthCaching();
                   return httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
               }
           }));
           differentClient = restHighLevelClient;
//       HttpHost[] test = new HttpHost[2];
//       test[0] = new HttpHost("10.1.5.202", 9200, SCHEME);
//       test[1] = new HttpHost("10.6.1.87", 9200, SCHEME);
//         differentClient = new RestHighLevelClient(RestClient.builder(test).setMaxRetryTimeoutMillis(TIME_OUT));
           logger.info("初始化ES-client成功：ip={}",esConfig.getEsHost());
       }
       return restHighLevelClient;
   }
   
   /**
    * <p>Description: 查询的公共语句</p>
    */
   public static SearchResponse searchData(SearchRequest request) {
      if (request == null) {
         return null;
      }
      STimer createTimer = TimeStatUtil.createTimer("查询ES");
      //new SearchSourceBuilder();
      SearchSourceBuilder sourceBuilder = request.source();
      sourceBuilder.trackTotalHits(true);
      String indexName=request.indices()[0];
      //第几页
      int from = sourceBuilder.from();
      int size = sourceBuilder.size();// > 5000 ? 5000 : sourceBuilder.size();
      int index = from + size;
      if (index >= 10000) { // from + size > 10000时需要使用Search After方法
//       if (index > 20000) {//ES扛不住，限定只能2w内
//          return null;
//       }
         sourceBuilder.collapse(null);//cannot use `collapse` in conjunction with `search_after`
         SearchResponse result = searchAfter(request, from, size);
         createTimer.stop("Search After查询,原from="+from+",size="+size + request.toString());
         return result;
      } else {
         try{
            RestHighLevelClient client = getClient();
            if (indexNames.contains(indexName)) {
               client=differentClient;
            }
            SearchResponse result = client.search(request, RequestOptions.DEFAULT);
            return result;
         } catch (Exception e) {
            LogFormatter.buildError(logger,"查询ES报错,q:{}"+request.toString(), e);
         } finally {
            createTimer.stopLogTimeOut("indexName="+indexName+",source="+sourceBuilder.toString());
         }
      }
      return null;
   }
   
   /**
    * <p>Description: 快速对数据进行反序列化</p>
    * 
    * @param <T>
    * @param response
    * @param clazz
    * @return the List<T>
    */
   public static <T> List<T> getDataListByResponse(SearchResponse response,Class<T> clazz){
      List<T> list= new ArrayList<T>();
      try {
         SearchHit[] hits = response.getHits().getHits();
         for (SearchHit searchHit : hits) {
            T data = JacksonUtil.getWriteMapper().readValue(searchHit.getSourceAsString(), clazz);
            list.add(data);
         }
      } catch (Exception e) {
         LogFormatter.buildError(logger,"解析ES报错", e);
      }
      return list;
   }
   
   /**
    * <p>Description: 获取数据去重后的放inner_hits里的数据</p>
    * 
    * @param <T>
    * @param response
    * @param clazz
    * @param collapseNames
    * @return the List<T>
    */
   public static <T> List<T> getCollapseInnerDataOfResponse(SearchResponse response,Class<T> clazz,String... collapseNames){
      List<T> list = new ArrayList<T>();
      try {
         SearchHit[] hits = response.getHits().getHits();
         for (SearchHit searchHit : hits) {
            Map<String, SearchHits> innerHits = searchHit.getInnerHits();
            if (null==innerHits || null==collapseNames || collapseNames.length<1) {
               T data= JacksonUtil.getWriteMapper().readValue(searchHit.getSourceAsString(), clazz);
               list.add(data);
               continue;
            }
            for (int i = 0; i < collapseNames.length; i++) {
               SearchHits searchHits = innerHits.get(collapseNames[i]);
               SearchHit hits2 = searchHits.getHits()[0];
               T data = JacksonUtil.getWriteMapper().readValue(hits2.getSourceAsString(), clazz);
               list.add(data);
            }
         }
      } catch (Exception e) {
         LogFormatter.buildError(logger, "获取去重后的数据失败",e);
      }
      return list;
   }
   

   /**
    * <p>Description: 判断索引是否已构建</p>
    * 
    * @param indexName
    * @return
    * @throws IOException the boolean
    */
   public static boolean indexExists(String indexName) throws IOException {
      RestHighLevelClient client = getClient();
      GetIndexRequest request=new GetIndexRequest(indexName);
        return client.indices().exists(request, RequestOptions.DEFAULT);
   }
   
   /**
    * <p>Description: 获取索引的Mapping</p>
    * 
    * @param indexName
    * @return
    * @throws IOException the boolean
    */
   public static GetMappingsResponse getIndexMapping(String indexName) throws IOException {
      RestHighLevelClient client = getClient();
      GetMappingsRequest request=new GetMappingsRequest().indices(indexName);
        return client.indices().getMapping(request, RequestOptions.DEFAULT);
   }
   
   /**
    * <p>Description: Mapping增加字段</p>
    * 
    * @param indexName
    * @return
    * @throws IOException the GetMappingsResponse
    */
   @SuppressWarnings({ "unchecked", "rawtypes" })
   public static boolean addMapping(String indexName,Object mapping) throws IOException {
      RestHighLevelClient client = getClient();
      PutMappingRequest request = new PutMappingRequest(indexName);
      if (mapping instanceof String) {
         request.source(String.valueOf(mapping), XContentType.JSON);
      } else if (mapping instanceof Map) {
         request.source((Map) mapping);
      } else if (mapping instanceof XContentBuilder) {
         request.source((XContentBuilder) mapping);
      }
        AcknowledgedResponse response = client.indices().putMapping(request, RequestOptions.DEFAULT);
        return response.isAcknowledged();
   }
   
   /**
    * <p>Description: 创建索引</p>
    * 
    * @param indexName
    * @param mapping
    * @return
    * @throws IOException the CreateIndexResponse
    */
   public static CreateIndexResponse createIndex(String indexName, Object mapping) throws IOException {
      if (null==DEFAULT_SETTING) {
         DEFAULT_SETTING = new HashMap<String, Object>();
         DEFAULT_SETTING.put("number_of_shards", 5);
      }
      return createIndex(indexName, mapping, DEFAULT_SETTING);
   }
   
   /**
    * <p>Description: 创建索引</p>
    * 
    * @param indexName
    * @param mapping
    * @param setting
    * @return
    * @throws IOException the CreateIndexResponse
    */
   @SuppressWarnings({ "unchecked", "rawtypes" })
   public static CreateIndexResponse createIndex(String indexName, Object mapping,Object setting) throws IOException {
      RestHighLevelClient client = getClient();
      CreateIndexRequest request = new CreateIndexRequest(indexName);
      if (null!=setting) {
         if (setting instanceof String) {
            request.settings(String.valueOf(setting), XContentType.JSON);
         } else if (setting instanceof Map) {
            request.settings((Map) setting);
         } else if (setting instanceof XContentBuilder) {
            request.settings((XContentBuilder) setting);
         } else if (setting instanceof Settings) {
            request.settings((Settings) setting);
         }
      }
      if (mapping instanceof String) {
         request.mapping(String.valueOf(mapping), XContentType.JSON);
      } else if (mapping instanceof Map) {
         request.mapping((Map) mapping);
      } else if (mapping instanceof XContentBuilder) {
         request.mapping((XContentBuilder) mapping);
      }
      return client.indices().create(request, RequestOptions.DEFAULT);
   }
   
   /**
    * @param index
    * @param obj
    * @return
    * @throws IOException
    */
   public static IndexResponse indexObj(String index,String id,Object obj) throws IOException {
      if (StringUtils.isEmpty(index) || null==obj) {
         return null;
      }
      IndexRequest request = new IndexRequest(index);
      request.id(id);
      RestHighLevelClient client = getClient();
      request.source(JacksonUtil.getJson(obj),XContentType.JSON);
      return client.index(request,RequestOptions.DEFAULT);
   }

   public static BulkResponse bulkRequest(String index, BulkRequest request) throws IOException {
      if (StringUtils.isEmpty(index) || null==request) {
         return null;
      }
      RestHighLevelClient client = getClient();
      return client.bulk(request,RequestOptions.DEFAULT);
   }
   
   /**
    * @param index
    * @param request
    * @return
    * @throws IOException
    */
   public static BulkResponse indexBulkRequest(String index,BulkRequest request) throws IOException {
      if (StringUtils.isEmpty(index) || null==request) {
         return null;
      }
      RestHighLevelClient client = getClient();
      return client.bulk(request,RequestOptions.DEFAULT);
   }

   /**
    * @param index
    * @param obj
    * @return
    * @throws IOException
    */
   public static BulkResponse delByIds(String index,List<? extends Object> obj) throws IOException {
      if (StringUtils.isEmpty(index) || null==obj) {
         return null;
      }
      BulkRequest bulkRequest = new BulkRequest();
      for(Object ob:obj) {
         DeleteRequest request = new DeleteRequest(index);
         request.id(String.valueOf(ob));
         bulkRequest.add(request);
      }
      RestHighLevelClient client = getClient();
      if(client != null){
         return client.bulk(bulkRequest,RequestOptions.DEFAULT);
      }
      return null;
   }
   
   /**
    * <p>Description: 根据条件进行删除</p>
    * 
    * @param request
    * @return
    * @throws IOException the BulkByScrollResponse
    */
   public static BulkByScrollResponse delByQuery(DeleteByQueryRequest request) throws IOException {
      RestHighLevelClient client = getClient();
      if(client != null){
         return client.deleteByQuery(request, RequestOptions.DEFAULT);
      }
      return null;
   }
   
   /**
    * <p>Description: 更新一条数据的具体某些值</p>
    * 
    * @param indexName
    * @param id
    * @param param
    * @return
    * @throws IOException the UpdateResponse
    */
   public static UpdateResponse updateData(String indexName,String id,Map<String,Object> param) throws IOException {
      UpdateRequest request = new UpdateRequest(indexName,id);
      request.doc(param);
//    request.docAsUpsert(false);
      RestHighLevelClient client = getClient();
      UpdateResponse response = client.update(request, RequestOptions.DEFAULT);
      return response;
   }
    
    /**
     * <p>Description: 关闭连接</p>
     * 
     * @throws IOException the void
     */
    public static synchronized void closeConnection() throws IOException {
        restHighLevelClient.close();
        restHighLevelClient = null;
    }
    
    /**
    * @Description ES查询公共语句，当from + size > 10000时使用该方法
    */
   public static SearchResponse searchAfter(SearchRequest searchRequest, int from, int size) {
      if (searchRequest == null) {
         return null;
      }
      RestHighLevelClient client = getClient();
      if (indexNames.contains(searchRequest.indices()[0])) {
         client = differentClient;
      }
      try {
         searchRequest = searchToFrom(searchRequest, from, size);
         return client.search(searchRequest, RequestOptions.DEFAULT);
      } catch (Exception e) {
         logger.error("ES Search After查询失败, q={}, from={}, size={}", searchRequest.toString(), from, size, e);
      }
      return null;
   }
   
    /**
     * @Description 当size过小时，依据Search After查询，先按照size放大倍数查询至from起点
     */
    private static SearchRequest searchToFrom(SearchRequest searchRequest, int from, int size) {
       if (searchRequest == null) {
         return null;
      }
      RestHighLevelClient client = getClient();
       SearchSourceBuilder sourceBuilder = searchRequest.source();
      try {
         sourceBuilder.sort(ID, SortOrder.ASC);//299 Elasticsearch-6.2.4-ccec39f "Fielddata access on the _uid field is deprecated, use _id instead"
         /*
          * 依据Search After的请求方式，请求第{page}页的数据，需要请求page次，当page过大时，会导致耗时较长，资源占据较多的情况
          */
         int step = size(size);
          sourceBuilder.from(0); // 使用SearchAfter必须将from设置为0或者-1
         sourceBuilder.size(step);
         searchRequest.source(sourceBuilder);
         int i = 0;
         while (((i + 1) * step) <= from) {
            SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
            Object[] sortValue = sortValue(searchResponse);
            if (sortValue != null) {
               sourceBuilder.searchAfter(sortValue);
            }
            searchRequest.source(sourceBuilder);
            i++;
         }
         sourceBuilder.size(size); // 将size设置为用户所需要的
         int surplus = from - (i * step); // 距离from的剩余长度
         if (surplus > 0) {
            sourceBuilder.size(surplus);
            searchRequest.source(sourceBuilder);
            SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
            Object[] sortValue = sortValue(searchResponse);
            if (sortValue != null) {
               sourceBuilder.searchAfter(sortValue);
            }
            sourceBuilder.size(size); // 将size设置为用户所需要的
            searchRequest.source(sourceBuilder);
         }
           return searchRequest;
      } catch (Exception e) {
         logger.error("ES Search After查询失败, q={}", searchRequest.toString(), e);
      }
      return searchRequest;
    }
    
    /**
    * @Description 取出查询结果中的Sort值供Search After使用
    */
   private static Object[] sortValue(SearchResponse searchResponse) {
      if (searchResponse != null) {
         SearchHits hits = searchResponse.getHits();
         if (hits == null || hits.getHits() == null) {
            return null;
         }
         int length = hits.getHits().length;
         if (length <= 0) {
            return null;
         }
         SearchHit last = hits.getAt(length - 1);
         return last.getSortValues();
      }
      return null;
   }
   
   /**
    * @Description 当size过小时，按倍数放大size
    * 
    * @param size 用户要求的每页数量
    */
   private static int size(int size) {
      if (size > 0) {
         int proportion = STEP / size;
         if (proportion > 1) {
            return size * proportion;
         } else {
            return size;
         }
      }
      return 0;
   }
    
}