ElasticSearch分组统计数据入库

作者应用场景：

根据公司的业务需求，需要对用户调用本司的API做次数统计和计费统计，需要对网关日志进行解析，并将统计到的数据入库保存。公司的网关日志采用ELK架构，正好适用于ES来做解析。日志索引当天日期命名的，所以每天按日期调用一次解析服务，当天统计前一天的调用量，实现按日统计存储入库。后续将根据源数据做相应的业务快发，实现API调用的统计数据展示，便于客服查看用户的调用量和计费数据，也便于客户查看自己的API调用情况。

package com.maijia.cms.business.user.job;

import com.maijia.cms.business.user.domain.ApiRequestRecord;
import com.maijia.ucenter.utils.CalendarUtils;
import org.apache.commons.lang3.StringUtils;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.aggregations.Aggregation;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.bucket.terms.ParsedStringTerms;
import org.elasticsearch.search.aggregations.bucket.terms.Terms;
import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.springframework.stereotype.Component;

import javax.annotation.Resource;
import java.text.SimpleDateFormat;
import java.util.*;

/**
 * @Date 2022/3/04
 * @Author Dylan
 */
@Component
public class SearchHelp {

    @Resource
    private ElasticQueryDSLTemplates elasticQueryDslTemplates;

    //索引前缀，索引格式为gw-invoking-log-yyyy-MM-dd
    private static final String ES_INDEX_NAME_PREFIX = "gw-invoking-log-";

    private static final SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");

    /**
     * ES检索日志数据
     * @param appKey   appKey
     * @param dateStr  日期 yyyy-MM-dd
     * @param failureFlag  不为空时查询调用失败的记录
     */
    public List<ApiRequestRecord> logSearch(String appKey, String dateStr, String failureFlag) {
        //获取日期
        dateStr = getRequestDate(dateStr);
        SearchRequest searchRequest = new SearchRequest(ES_INDEX_NAME_PREFIX + dateStr);
        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
        BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
        //构造ES查询条件
        buildItemQuery(searchSourceBuilder, boolQueryBuilder, appKey, failureFlag);
        QueryResult<?> qr = new QueryResult<>();
        elasticQueryDslTemplates.excuteQuery(searchRequest, searchSourceBuilder, qr);
        //解析分组统计数据
        List<ApiRequestRecord> resultList = new ArrayList<>();
        Map<String, Aggregation> aggMap = qr.getAggMap();
        if(null != aggMap && !aggMap.isEmpty()){
            ParsedStringTerms serviceTerms = (ParsedStringTerms) aggMap.get("group_by_service");
            Iterator<Terms.Bucket> serviceBucketIt = (Iterator<Terms.Bucket>)serviceTerms.getBuckets().iterator();
            //返回结果
            Date createDate = CalendarUtils.getDate(dateStr,"yyyy-MM-dd");
            while (serviceBucketIt.hasNext()){
                Terms.Bucket bucket = serviceBucketIt.next();
                ApiRequestRecord record = new ApiRequestRecord();
                record.setAppKey(appKey);
                record.setCreateDate(createDate);
                record.setApiMethod(bucket.getKeyAsString());
                //请求失败次数
                if(StringUtils.isNotBlank(failureFlag)){
                    record.setFailureTotal(bucket.getDocCount());
                }else{
                    //请求总次数
                    record.setRequestTotal(bucket.getDocCount());
                }
                resultList.add(record);
            }
        }

        return resultList;
    }

    private void buildItemQuery(SearchSourceBuilder searchSourceBuilder, BoolQueryBuilder boolQueryBuilder, String appKey, String failureFlag) {
        //匹配某个appKey
        if (StringUtils.isNotBlank(appKey)) {
            boolQueryBuilder.must(QueryBuilders.termQuery("appKey", appKey));
        }
        //匹配调用失败的
        if(StringUtils.isNotBlank(failureFlag)){
            boolQueryBuilder.must(QueryBuilders.termQuery("rpcStatus", "error"));
        }
        //根据method分组统计
        TermsAggregationBuilder termsAggregationBuilder = AggregationBuilders.terms("group_by_service").field("service");
        searchSourceBuilder.aggregation(termsAggregationBuilder);
        searchSourceBuilder.query(boolQueryBuilder);
    }

    /**
     * dateStr为空时，默认获取前一天日期
     * @param dateStr
     * @return
     */
    private String getRequestDate(String dateStr){
        if(StringUtils.isNotBlank(dateStr)){
            return dateStr;
        }
        Calendar calendar = Calendar.getInstance();
        calendar.setTime(new Date());
        //当前时间往前推一天
        calendar.add(Calendar.DAY_OF_MONTH, -1);
        return sdf.format(calendar.getTime());
    }

}

GET gw-invoking-log-2022-03-02/_search
{
  "query": {
    "bool": {
      "must": [
        {"match": {
          "appKey": "5711393"
        }}
      ] 
        
    }
    },
  "aggs":{
    "group_by_service":{
      "terms": {
        "field": "service"
      }
    }
  }
}

//返回结果
"aggregations" : {
    "group_by_service" : {
      "doc_count_error_upper_bound" : 0,
      "sum_other_doc_count" : 0,
      "buckets" : [
        {
          "key" : "tsj.shopee.item.skuChart",
          "doc_count" : 446
        },
        {
          "key" : "tsj.shopee.shop.detail",
          "doc_count" : 296
        },
        {
          "key" : "tsj.shopee.item.detail",
          "doc_count" : 65
        },
        {
          "key" : "tsj.shopee.item.milestoneList",
          "doc_count" : 6
        }
      ]
    }
  }

package com.maijia.cms.business.user.job;

import com.maijia.cms.business.user.common.util.ExecuteTimeUtils;
import org.apache.log4j.Logger;
import org.elasticsearch.action.DocWriteRequest;
import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequest;
import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse;
import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.delete.DeleteRequest;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.reindex.DeleteByQueryRequest;
import org.elasticsearch.index.reindex.UpdateByQueryRequest;
import org.elasticsearch.script.Script;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.aggregations.Aggregations;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.springframework.stereotype.Component;

import javax.annotation.Resource;
import java.io.IOException;
import java.util.*;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;

@Component
public class ElasticQueryDSLTemplates {

    private static Logger logger = Logger.getLogger(ElasticQueryDSLTemplates.class);

    @Resource
    private RestHighLevelClient restHighLevelClient;

    public Set<String> esAnalyze(String content, String indexName) {
        AnalyzeRequest analyzeRequest = new AnalyzeRequest(indexName).text(content);

        AnalyzeResponse analyze = null;
        try {
            analyze = restHighLevelClient.indices().analyze(analyzeRequest, RequestOptions.DEFAULT);
        } catch (IOException e) {
            logger.error("es分词出错，content="+content);
        }

        if (analyze != null) {
            return analyze.getTokens().stream().map(AnalyzeResponse.AnalyzeToken::getTerm).collect(Collectors.toSet());
        }

        return new HashSet<>();
    }

    public boolean esForceMerge(String... indexNames) {
        ForceMergeRequest forceMergeRequest = new ForceMergeRequest(indexNames);
        boolean flag = Boolean.TRUE;
        try {
            restHighLevelClient.indices().forcemerge(forceMergeRequest, RequestOptions.DEFAULT);
        } catch (IOException e) {
            logger.error("es ForceMerge出错,indexNames="+indexNames+","+ e.getStackTrace());
            flag = Boolean.FALSE;
        }

        return flag;
    }

    public boolean bulkAddDocument(String indexName, List<Map<String, ?>> docList) {
        long l = System.currentTimeMillis();
        boolean flag = Boolean.TRUE;
        if (docList.size() < 1) {
            return true;
        }
        try {
            BulkRequest bulkRequest = new BulkRequest();
            docList.forEach(map -> bulkRequest.add(new IndexRequest(indexName).id(map.get("id") + "")
                    .source(map).opType(DocWriteRequest.OpType.INDEX)));
            restHighLevelClient.bulk(bulkRequest, RequestOptions.DEFAULT);
        } catch (Exception e) {
            logger.error("创建索引=【"+indexName+"】异常，索引内容="+e);
            flag = Boolean.FALSE;
        }
        //logger.info("批量创建{}{}条索引时:{}", indexName, docList.size(), (System.currentTimeMillis() - l));
        return flag;
    }

    public boolean bulkDeleteDocument(String indexName, QueryBuilder queryBuilder) {
        long l = System.currentTimeMillis();
        boolean flag = Boolean.TRUE;
        try {
            DeleteByQueryRequest deleteByQueryRequest = new DeleteByQueryRequest(indexName);
            deleteByQueryRequest.setQuery(queryBuilder);
            deleteByQueryRequest.setBatchSize(10000);
            deleteByQueryRequest.setConflicts("proceed");
            restHighLevelClient.deleteByQuery(deleteByQueryRequest, RequestOptions.DEFAULT);
        } catch (Exception e) {
            //logger.error("批量删除索=【{}】异常，索引内容={}", indexName, JsonNodeUtils.object2String(queryBuilder), e);
            flag = Boolean.FALSE;
        }
        //logger.info("批量删除{}索耗时:{}", indexName, (System.currentTimeMillis() - l));
        return flag;
    }

    public boolean bulkDeleteDocument(String indexName, List<Map<String, ?>> docList) {
        long l = System.currentTimeMillis();
        boolean flag = Boolean.TRUE;
        if (docList.size() < 1) {
            return true;
        }
        try {
            BulkRequest bulkRequest = new BulkRequest();
            docList.forEach(map -> bulkRequest.add(new DeleteRequest(indexName, map.get("id") + "")));
            restHighLevelClient.bulk(bulkRequest, RequestOptions.DEFAULT);
        } catch (Exception e) {
            //logger.error("批量删除索=【{}】异常，索引内容={}", indexName, JsonNodeUtils.object2String(docList), e);
            flag = Boolean.FALSE;
        }
        //logger.biz("批量删除{}索耗时:{}", indexName, (System.currentTimeMillis() - l));
        return flag;
    }

    public boolean bulkUpdateByQuery(String indexName, Map<String, Object> docDataMap, QueryBuilder queryBuilder) {
        boolean flag = Boolean.TRUE;
        if (docDataMap.size() < 1) {
            return Boolean.TRUE;
        }
        long l = System.currentTimeMillis();
        try {
            UpdateByQueryRequest updateByQueryRequest = new UpdateByQueryRequest(indexName);
            updateByQueryRequest.setConflicts("proceed");
            updateByQueryRequest.setQuery(queryBuilder);
            StringBuilder stringBuilder = new StringBuilder();
            docDataMap.forEach((key, value) -> stringBuilder.append("ctx._source.").append(key).append("=").append("params.").append(key).append(";"));

            Script script = new Script(Script.DEFAULT_SCRIPT_TYPE, Script.DEFAULT_SCRIPT_LANG, stringBuilder.toString(), docDataMap);

            updateByQueryRequest.setScript(script);
            restHighLevelClient.updateByQuery(updateByQueryRequest, RequestOptions.DEFAULT);
        } catch (Exception e) {
            //logger.error("批量修改索引【{}】异常，索引内容={}", indexName, "", e);
            flag = Boolean.FALSE;
        }

        return flag;
    }

    public boolean bulkUpdateListByQuery(String indexName, Map<String, Object> docDataMap, QueryBuilder queryBuilder) {
        boolean flag = Boolean.TRUE;
        if (docDataMap.size() < 1) {
            return Boolean.TRUE;
        }
        long l = System.currentTimeMillis();
        try {
            UpdateByQueryRequest updateByQueryRequest = new UpdateByQueryRequest(indexName);
            updateByQueryRequest.setConflicts("proceed");
            updateByQueryRequest.setQuery(queryBuilder);

            Script script = new Script(Script.DEFAULT_SCRIPT_TYPE,
                    Script.DEFAULT_SCRIPT_LANG, "for(int i=0; i<params.lives.size(); i++){int y=1; for(int x=0;x<ctx._source.lives.size();x++){if (params.lives[i].liveId==ctx._source.lives[x].liveId){y=0; break;}}if (y==1){ctx._source.lives.add(params.lives[i])    }}", docDataMap);

            updateByQueryRequest.setScript(script);
            restHighLevelClient.updateByQuery(updateByQueryRequest, RequestOptions.DEFAULT);
        } catch (Exception e) {
            //logger.error("批量修改索引【{}】异常，索引内容={}", indexName, JsonNodeUtils.object2String(docDataMap), e);
            flag = Boolean.FALSE;
        }
        return flag;
    }

    private void append(StringBuilder stringBuilder, Object value) {
        if (value instanceof String) {
            stringBuilder.append("'").append(value).append("'");
            return;
        }
        if (value instanceof List) {
            stringBuilder.append("[");
            List value1 = (List) value;
            for (int i = 0; i < value1.size(); i++) {
                if (i == value1.size() - 1) {
                    append(stringBuilder, value1.get(i));
                } else {
                    append(stringBuilder, value1.get(i));
                    stringBuilder.append(",");
                }
            }

            stringBuilder.append("]");
            return;
        }
        stringBuilder.append(value);
    }

    /**
     * 根据id获取文档
     *
     * @param indexName 索引名称
     * @param id        文档id
     * @param includes  需要返回的字段 空数组为所有字段返回
     * @param excludes  需要排除的字段
     * @return
     */
    public QueryResult<?> getDocumentById(String indexName, String id, String[] includes, String[] excludes) {
        SearchRequest searchRequest = new SearchRequest(indexName);
        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();

        searchSourceBuilder.query(QueryBuilders.termQuery("id", id));
        searchSourceBuilder.fetchSource(includes, excludes);
        QueryResult<?> qr = new QueryResult<>();
        excuteQuery(searchRequest, searchSourceBuilder, qr);
        return qr;
    }

    public SearchResponse excuteQuery(SearchRequest searchRequest, SearchSourceBuilder sourceBuilder, QueryResult<?> qr) {
        String searchId = UUID.randomUUID().toString().replaceAll("-", "");
        sourceBuilder.trackTotalHits(true);
        sourceBuilder.timeout(new TimeValue(60, TimeUnit.SECONDS));
        searchRequest.source(sourceBuilder);

        //logger.biz("searchId={},提交es搜索参数,={}", searchId, JsonNodeUtils.object2String(sourceBuilder));
        SearchResponse searchResponse = new SearchResponse();
        ExecuteTimeUtils instance = ExecuteTimeUtils.getInstance();
        instance.init();
        try {
            searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
            logger.info("searchId="+searchId+",es搜索的耗时 "+instance.spentLastExecuteTime());
        } catch (IOException e) {
            logger.error("searchId="+searchId+",es搜索出错"+e);
            qr.setSuccessful(false);
            return searchResponse;
        }
        doQueryResult(searchResponse, qr);
        return searchResponse;
    }


    private void doQueryResult(SearchResponse searchResponse, QueryResult<?> qr) {
        // 查询耗时
        TimeValue took = searchResponse.getTook();
        qr.setTook(took.getMillis());
        // 是否超时
        qr.setTimeout(searchResponse.isTimedOut());
        // 查询总数
        qr.setHitsTotal(searchResponse.getHits().getTotalHits().value);
        // 最高评分
        qr.setMaxScore(searchResponse.getHits().getMaxScore());
        List<Map<String, Object>> hitsBody = new ArrayList<>();
        SearchHits searchHits = searchResponse.getHits();
        searchHits.forEach(searchHit -> {
            Map<String, Object> hitMap = searchHit.getSourceAsMap();
            if (hitMap != null) {
                hitsBody.add(hitMap);
            }
        });
        Aggregations aggregations = searchResponse.getAggregations();
        if (aggregations != null) {
            aggregations.forEach(aggregation -> qr.getAggregation().add(aggregation));
            qr.setAggMap(aggregations.asMap());
        }

        qr.setHitsBody(hitsBody);
    }

    public QueryResult<?> getDocumentByIds(String indexName, Collection<String> list, String[] includes, String[] excludes) {

        SearchRequest searchRequest = new SearchRequest(indexName);
        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();

        searchSourceBuilder.query(QueryBuilders.termsQuery("id", list));
        searchSourceBuilder.fetchSource(includes, excludes);
        QueryResult<?> qr = new QueryResult<>();
        excuteQuery(searchRequest, searchSourceBuilder, qr);
        return qr;
    }
}

package com.maijia.cms.business.user.job;

import org.elasticsearch.search.aggregations.Aggregation;

import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;

/**
 * @author qiguoqiang
 * create 2019/7/3
 */
public class QueryResult<T> implements Serializable {

    /**
     * 序列化id
     */
    private static final long serialVersionUID = 7246688743186445516L;

    /**
     * 查询是否成功true=成功，false=失败
     */
    private boolean successful = true;

    /**
     * 查询耗时
     */
    private long took;

    /**
     * 是否超时
     */
    private boolean timeout;

    /**
     * 查询总数
     */
    private long hitsTotal;

    /**
     * 最高评分
     */
    private float maxScore;

    /**
     * 查询结果
     */
    private List<Map<String, Object>> hitsBody = new ArrayList<>();

    /**
     * 结果
     */
    private List<?> list = new ArrayList<>();

    /**
     * 聚合结果
     */
    private List<Aggregation> aggregation = new ArrayList<>();

    /**
     * 聚合结果Map
     */
    Map<String, Aggregation> aggMap;

    public List<?> getList() {
        return list;
    }

    public void setList(List<?> list) {
        this.list = list;
    }

    public List<Aggregation> getAggregation() {
        return aggregation;
    }

    public void setAggregation(List<Aggregation> aggregation) {
        this.aggregation = aggregation;
    }

    public QueryResult() {

    }

    public boolean isSuccessful() {
        return successful;
    }

    public void setSuccessful(boolean successful) {
        this.successful = successful;
    }

    public long getTook() {
        return took;
    }

    public void setTook(long took) {
        this.took = took;
    }

    public boolean isTimeout() {
        return timeout;
    }

    public void setTimeout(boolean timeout) {
        this.timeout = timeout;
    }

    public long getHitsTotal() {
        return hitsTotal;
    }

    public void setHitsTotal(long hitsTotal) {
        this.hitsTotal = hitsTotal;
    }

    public float getMaxScore() {
        return maxScore;
    }

    public void setMaxScore(float maxScore) {
        this.maxScore = maxScore;
    }

    public List<Map<String, Object>> getHitsBody() {
        return hitsBody;
    }

    public void setHitsBody(List<Map<String, Object>> hitsBody) {
        this.hitsBody = hitsBody;
    }

    public Map<String, Aggregation> getAggMap() {
        return aggMap;
    }

    public void setAggMap(Map<String, Aggregation> aggMap) {
        this.aggMap = aggMap;
    }
}

package com.maijia.cms.business.user.domain;

import lombok.Data;

import java.util.Date;

@Data
public class ApiRequestRecord {
    private Long id;
    /**
     * 开放平台appKey
     */
    private String appKey;
    /**
     * API服务名
     */
    private String apiMethod;
    /**
     * API服务名描述
     */
    private String apiDesc;
    /**
     * 调用成功次数
     */
    private Long successTotal;
    /**
     * 调用失败次数
     */
    private Long failureTotal;
    /**
     * 总调用次数
     */
    private Long requestTotal;
    /**
     * 调用成功计费(分)
     */
    private Long amount;
    /**
     * api类型：A、B、C、D
     */
    private String apiType;
    /**
     * 创建时间
     */
    private Date createDate;
}