Es search流程
Es 的search流程
1、协调节点接收到search请求后封装查询request
2、从clusterMata中获取索引有多少分片 设置相关参数，循环发送到分片所在的节点
3、分片接收到meaasge后判断是否要走cache
4、不需要走cache的话交交给lucene去查，查询完滞后判断是否要rescore aggregation suggese
5、封装查询结果返回给协调节点，如果协调节点发现失败了找该分片的下一个分片
6、协调节点就收到response后开始执行fetch阶段
7、发送get到索引所在的node
8、node接收到消息后执行executeFetchPhase从lucene中获取数据返回给协调节点
9、协调节点判断是否已经全部返回了（fetch时 协调节点创建countdown ，发请求时不管成功与否都会-1）当为0时执行finishPhase
10、协调节点返回结构给客户端
search流程很复杂，至今没找到协调节点接收到所有hits之后在哪里做全局排序排序
协调节点入口 RestSearchAction @Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { SearchRequest searchRequest = new SearchRequest(); /* * We have to pull out the call to `source().size(size)` because * _update_by_query and _delete_by_query uses this same parsing * path but sets a different variable when it sees the `size` * url parameter. * * Note that we can't use `searchRequest.source()::size` because * `searchRequest.source()` is null right now. We don't have to * guard against it being null in the IntConsumer because it can't * be null later. If that is confusing to you then you are in good * company. */ IntConsumer setSize = size -> searchRequest.source().size(size); request.withContentOrSourceParamParserOrNull(parser -> parseSearchRequest(searchRequest, request, parser, setSize)); return channel -> client.search(searchRequest, new RestStatusToXContentListener<>(channel)); } 协调节点发送请求 @Override protected void doExecute(Task task, SearchRequest searchRequest, ActionListener<SearchResponse> listener) { final long absoluteStartMillis = System.currentTimeMillis(); final long relativeStartNanos = System.nanoTime(); final SearchTimeProvider timeProvider = new SearchTimeProvider(absoluteStartMillis, relativeStartNanos, System::nanoTime); ActionListener<SearchSourceBuilder> rewriteListener = ActionListener.wrap(source -> { if (source != searchRequest.source()) { // only set it if it changed - we don't allow null values to be set but it might be already null be we want to catch // situations when it possible due to a bug changes to null searchRequest.source(source); } final ClusterState clusterState = clusterService.state();//获取集群状态 //远程集群 跨集群搜索 https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-cross-cluster-search.html final Map<String, OriginalIndices> remoteClusterIndices = remoteClusterService.groupIndices(searchRequest.indicesOptions(), searchRequest.indices(), idx -> indexNameExpressionResolver.hasIndexOrAlias(idx, clusterState)); OriginalIndices localIndices = remoteClusterIndices.remove(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY); if (remoteClusterIndices.isEmpty()) {//为空的话 就是本机群搜索 executeSearch((SearchTask)task, timeProvider, searchRequest, localIndices, remoteClusterIndices, Collections.emptyList(), (clusterName, nodeId) -> null, clusterState, Collections.emptyMap(), listener, clusterState.getNodes() .getDataNodes().size()); } else { remoteClusterService.collectSearchShards(searchRequest.indicesOptions(), searchRequest.preference(), searchRequest.routing(), remoteClusterIndices, ActionListener.wrap((searchShardsResponses) -> { List<SearchShardIterator> remoteShardIterators = new ArrayList<>(); Map<String, AliasFilter> remoteAliasFilters = new HashMap<>(); BiFunction<String, String, DiscoveryNode> clusterNodeLookup = processRemoteShards(searchShardsResponses, remoteClusterIndices, remoteShardIterators, remoteAliasFilters); int numNodesInvovled = searchShardsResponses.values().stream().mapToInt(r -> r.getNodes().length).sum() + clusterState.getNodes().getDataNodes().size(); executeSearch((SearchTask) task, timeProvider, searchRequest, localIndices, remoteClusterIndices, remoteShardIterators, clusterNodeLookup, clusterState, remoteAliasFilters, listener, numNodesInvovled); }, listener::onFailure)); } }, listener::onFailure); if (searchRequest.source() == null) { rewriteListener.onResponse(searchRequest.source()); } else { Rewriteable.rewriteAndFetch(searchRequest.source(), searchService.getRewriteContext(timeProvider::getAbsoluteStartMillis), rewriteListener); } } 封装查询的request private void executeSearch(SearchTask task, SearchTimeProvider timeProvider, SearchRequest searchRequest, OriginalIndices localIndices, Map<String, OriginalIndices> remoteClusterIndices, List<SearchShardIterator> remoteShardIterators, BiFunction<String, String, DiscoveryNode> remoteConnections, ClusterState clusterState, Map<String, AliasFilter> remoteAliasMap, ActionListener<SearchResponse> listener, int nodeCount) { clusterState.blocks().globalBlockedRaiseException(ClusterBlockLevel.READ); // TODO: I think startTime() should become part of ActionRequest and that should be used both for index name // date math expressions and $now in scripts. This way all apis will deal with now in the same way instead // of just for the _search api final Index[] indices; if (localIndices.indices().length == 0 && remoteClusterIndices.isEmpty() == false) { indices = Index.EMPTY_ARRAY; // don't search on _all if only remote indices were specified } else { indices = indexNameExpressionResolver.concreteIndices(clusterState, searchRequest.indicesOptions(), timeProvider.getAbsoluteStartMillis(), localIndices.indices());//找到可使用的别名和index } Map<String, AliasFilter> aliasFilter = buildPerIndexAliasFilter(searchRequest, clusterState, indices, remoteAliasMap); Map<String, Set<String>> routingMap = indexNameExpressionResolver.resolveSearchRouting(clusterState, searchRequest.routing(), searchRequest.indices());//如果没有设置routing信息 routingmap为空 String[] concreteIndices = new String[indices.length]; for (int i = 0; i < indices.length; i++) { concreteIndices[i] = indices[i].getName(); } GroupShardsIterator<ShardIterator> localShardsIterator = clusterService.operationRouting().searchShards(clusterState, concreteIndices, routingMap, searchRequest.preference()); GroupShardsIterator<SearchShardIterator> shardIterators = mergeShardsIterators(localShardsIterator, localIndices, remoteShardIterators);//索引所在的三个分片 failIfOverShardCountLimit(clusterService, shardIterators.size());//如果超出规定的分片数 默认是long.maxvalue Map<String, Float> concreteIndexBoosts = resolveIndexBoosts(searchRequest, clusterState);//查询权重 // optimize search type for cases where there is only one shard group to search on if (shardIterators.size() == 1) {//如果只有一个分片的话 也就不存在需要到各个分片查完然后打分汇总了 直接用这个查出来的就是排序之后的结果了 // if we only have one group, then we always want Q_A_F, no need for DFS, and no need to do THEN since we hit one shard searchRequest.searchType(QUERY_THEN_FETCH); } if (searchRequest.isSuggestOnly()) {//如果只用做suggest 不需要全局排序 // disable request cache if we have only suggest searchRequest.requestCache(false); switch (searchRequest.searchType()) { case DFS_QUERY_THEN_FETCH: // convert to Q_T_F if we have only suggest searchRequest.searchType(QUERY_THEN_FETCH); break; } } final DiscoveryNodes nodes = clusterState.nodes(); BiFunction<String, String, Transport.Connection> connectionLookup = (clusterName, nodeId) -> {//构建了connection的集合 final DiscoveryNode discoveryNode = clusterName == null ? nodes.get(nodeId) : remoteConnections.apply(clusterName, nodeId); if (discoveryNode == null) { throw new IllegalStateException("no node found for id: " + nodeId); } return searchTransportService.getConnection(clusterName, discoveryNode); }; if (searchRequest.isMaxConcurrentShardRequestsSet() == false) { // we try to set a default of max concurrent shard requests based on // the node count but upper-bound it by 256 by default to keep it sane. A single // search request that fans out lots of shards should hit a cluster too hard while 256 is already a lot // we multiply is by the default number of shards such that a single request in a cluster of 1 would hit all shards of a // default index. searchRequest.setMaxConcurrentShardRequests(Math.min(256, nodeCount * IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.getDefault(Settings.EMPTY)));//设置最多请求多少次 默认是5个分片 n台机器的话 最多请求就是 5n因为失败的时候会往其他分片的发查询 } boolean preFilterSearchShards = shouldPreFilterSearchShards(searchRequest, shardIterators); searchAsyncAction(task, searchRequest, shardIterators, timeProvider, connectionLookup, clusterState.version(), Collections.unmodifiableMap(aliasFilter), concreteIndexBoosts, listener, preFilterSearchShards).start(); } private AbstractSearchAsyncAction searchAsyncAction(SearchTask task, SearchRequest searchRequest, GroupShardsIterator<SearchShardIterator> shardIterators, SearchTimeProvider timeProvider, BiFunction<String, String, Transport.Connection> connectionLookup, long clusterStateVersion, Map<String, AliasFilter> aliasFilter, Map<String, Float> concreteIndexBoosts, ActionListener<SearchResponse> listener, boolean preFilter) { Executor executor = threadPool.executor(ThreadPool.Names.SEARCH); if (preFilter) { return new CanMatchPreFilterSearchPhase(logger, searchTransportService, connectionLookup, aliasFilter, concreteIndexBoosts, executor, searchRequest, listener, shardIterators, timeProvider, clusterStateVersion, task, (iter) -> { AbstractSearchAsyncAction action = searchAsyncAction(task, searchRequest, iter, timeProvider, connectionLookup, clusterStateVersion, aliasFilter, concreteIndexBoosts, listener, false); return new SearchPhase(action.getName()) { @Override public void run() throws IOException { action.start(); } }; }); } else { AbstractSearchAsyncAction searchAsyncAction;//根据类型构建action 然后执行 switch (searchRequest.searchType()) { case DFS_QUERY_THEN_FETCH: searchAsyncAction = new SearchDfsQueryThenFetchAsyncAction(logger, searchTransportService, connectionLookup, aliasFilter, concreteIndexBoosts, searchPhaseController, executor, searchRequest, listener, shardIterators, timeProvider, clusterStateVersion, task); break; case QUERY_AND_FETCH: case QUERY_THEN_FETCH: searchAsyncAction = new SearchQueryThenFetchAsyncAction(logger, searchTransportService, connectionLookup, aliasFilter, concreteIndexBoosts, searchPhaseController, executor, searchRequest, listener, shardIterators, timeProvider, clusterStateVersion, task); break; default: throw new IllegalStateException("Unknown search type: [" + searchRequest.searchType() + "]"); } return searchAsyncAction; } } @Override public final void run() throws IOException { for (final SearchShardIterator iterator : toSkipShardsIts) { assert iterator.skip(); skipShard(iterator); } if (shardsIts.size() > 0) { int maxConcurrentShardRequests = Math.min(this.maxConcurrentShardRequests, shardsIts.size()); final boolean success = shardExecutionIndex.compareAndSet(0, maxConcurrentShardRequests); assert success; for (int index = 0; index < maxConcurrentShardRequests; index++) {//从索引所在的分片中开始query final SearchShardIterator shardRoutings = shardsIts.get(index); assert shardRoutings.skip() == false; performPhaseOnShard(index, shardRoutings, shardRoutings.nextOrNull());//向分片所在的节点发送request } } } private void performPhaseOnShard(final int shardIndex, final SearchShardIterator shardIt, final ShardRouting shard) { /* * We capture the thread that this phase is starting on. When we are called back after executing the phase, we are either on the * same thread (because we never went async, or the same thread was selected from the thread pool) or a different thread. If we * continue on the same thread in the case that we never went async and this happens repeatedly we will end up recursing deeply and * could stack overflow. To prevent this, we fork if we are called back on the same thread that execution started on and otherwise * we can continue (cf. InitialSearchPhase#maybeFork). */ final Thread thread = Thread.currentThread(); if (shard == null) { fork(() -> onShardFailure(shardIndex, null, null, shardIt, new NoShardAvailableActionException(shardIt.shardId()))); } else { try { executePhaseOnShard(shardIt, shard, new SearchActionListener<FirstResult>(new SearchShardTarget(shard.currentNodeId(), shardIt.shardId(), shardIt.getClusterAlias(), shardIt.getOriginalIndices()), shardIndex) { @Override public void innerOnResponse(FirstResult result) { maybeFork(thread, () -> onShardResult(result, shardIt)); } @Override public void onFailure(Exception t) {//失败处理 从下一个分片里去找找 maybeFork(thread, () -> onShardFailure(shardIndex, shard, shard.currentNodeId(), shardIt, t)); } }); } catch (final Exception e) { /* * It is possible to run into connection exceptions here because we are getting the connection early and might run in to * nodes that are not connected. In this case, on shard failure will move us to the next shard copy. */ fork(() -> onShardFailure(shardIndex, shard, shard.currentNodeId(), shardIt, e)); } } } private void onShardFailure(final int shardIndex, @Nullable ShardRouting shard, @Nullable String nodeId, final SearchShardIterator shardIt, Exception e) { // we always add the shard failure for a specific shard instance // we do make sure to clean it on a successful response from a shard SearchShardTarget shardTarget = new SearchShardTarget(nodeId, shardIt.shardId(), shardIt.getClusterAlias(), shardIt.getOriginalIndices()); onShardFailure(shardIndex, shardTarget, e); if (totalOps.incrementAndGet() == expectedTotalOps) {//如果执行已经完成了 开始取回 每成功一次totalOps会+1 和预期的数相等时 说明已经执行完成 expectedTotalOps 应该是该索引的主副分片数的和 if (logger.isDebugEnabled()) { if (e != null && !TransportActions.isShardNotAvailableException(e)) { logger.debug( (Supplier<?>) () -> new ParameterizedMessage( "{}: Failed to execute [{}]", shard != null ? shard.shortSummary() : shardIt.shardId(), request), e); } else if (logger.isTraceEnabled()) { logger.trace((Supplier<?>) () -> new ParameterizedMessage("{}: Failed to execute [{}]", shard, request), e); } } onPhaseDone();//执行取回阶段 } else { final ShardRouting nextShard = shardIt.nextOrNull();//获取该分片的下一个分片 final boolean lastShard = nextShard == null; // trace log this exception logger.trace( (Supplier<?>) () -> new ParameterizedMessage( "{}: Failed to execute [{}] lastShard [{}]", shard != null ? shard.shortSummary() : shardIt.shardId(), request, lastShard), e); if (!lastShard) {//如果有下一个分片 在下一个分片上重新执行搜索 performPhaseOnShard(shardIndex, shardIt, nextShard); } else { maybeExecuteNext(); // move to the next execution if needed // no more shards active, add a failure if (logger.isDebugEnabled() && !logger.isTraceEnabled()) { // do not double log this exception if (e != null && !TransportActions.isShardNotAvailableException(e)) { logger.debug( (Supplier<?>) () -> new ParameterizedMessage( "{}: Failed to execute [{}] lastShard [{}]", shard != null ? shard.shortSummary() : shardIt.shardId(), request, lastShard), e); } } } } } 以上为协调节点query阶段 查询节点接收到message后
主要阶段在executeQueryPhase（） 查询阶段 public void executeQueryPhase(ShardSearchRequest request, SearchTask task, ActionListener<SearchPhaseResult> listener) { rewriteShardRequest(request, new ActionListener<ShardSearchRequest>() { @Override public void onResponse(ShardSearchRequest request) { try { listener.onResponse(executeQueryPhase(request, task)); } catch (Exception e) { onFailure(e); } } @Override public void onFailure(Exception e) { listener.onFailure(e); } }); } SearchPhaseResult executeQueryPhase(ShardSearchRequest request, SearchTask task) throws IOException { final SearchContext context = createAndPutContext(request); final SearchOperationListener operationListener = context.indexShard().getSearchOperationListener(); context.incRef(); boolean queryPhaseSuccess = false; try { context.setTask(task); operationListener.onPreQueryPhase(context); long time = System.nanoTime(); contextProcessing(context); loadOrExecuteQueryPhase(request, context);//执行查询 if (context.queryResult().hasSearchContext() == false && context.scrollContext() == null) { freeContext(context.id()); } else { contextProcessedSuccessfully(context); } final long afterQueryTime = System.nanoTime(); queryPhaseSuccess = true; operationListener.onQueryPhase(context, afterQueryTime - time); if (request.numberOfShards() == 1) { return executeFetchPhase(context, operationListener, afterQueryTime); } return context.queryResult(); //返回查询后的结果 } catch (Exception e) { // execution exception can happen while loading the cache, strip it if (e instanceof ExecutionException) { e = (e.getCause() == null || e.getCause() instanceof Exception) ? (Exception) e.getCause() : new ElasticsearchException(e.getCause()); } if (!queryPhaseSuccess) { operationListener.onFailedQueryPhase(context); } logger.trace("Query phase failed", e); processFailure(context, e); throw ExceptionsHelper.convertToRuntime(e); } finally { cleanContext(context); } } final SearchContext createAndPutContext(ShardSearchRequest request) throws IOException { SearchContext context = createContext(request, null);//将es 查询语句转换成lucene的 boolean success = false; try { putContext(context); if (request.scroll() != null) { context.indexShard().getSearchOperationListener().onNewScrollContext(context); } context.indexShard().getSearchOperationListener().onNewContext(context); success = true; return context; } finally { if (!success) { freeContext(context.id()); } } } /** * Try to load the query results from the cache or execute the query phase directly if the cache cannot be used. */ private void loadOrExecuteQueryPhase(final ShardSearchRequest request, final SearchContext context) throws Exception { final boolean canCache = indicesService.canCache(request, context); context.getQueryShardContext().freezeContext(); if (canCache) {//如果可以缓存 从缓存中着 没有的话 先查后写 indicesService.loadIntoContext(request, context, queryPhase); } else { queryPhase.execute(context); } } @Override public void execute(SearchContext searchContext) throws QueryPhaseExecutionException { if (searchContext.hasOnlySuggest()) { suggestPhase.execute(searchContext); // TODO: fix this once we can fetch docs for suggestions searchContext.queryResult().topDocs( new TopDocs(0, Lucene.EMPTY_SCORE_DOCS, 0), new DocValueFormat[0]); return; } // Pre-process aggregations as late as possible. In the case of a DFS_Q_T_F // request, preProcess is called on the DFS phase phase, this is why we pre-process them // here to make sure it happens during the QUERY phase aggregationPhase.preProcess(searchContext);//聚合 一个aggregation就是一个collector Sort indexSort = searchContext.mapperService().getIndexSettings().getIndexSortConfig() .buildIndexSort(searchContext.mapperService()::fullName, searchContext::getForField); //排序方式 final ContextIndexSearcher searcher = searchContext.searcher(); boolean rescore = execute(searchContext, searchContext.searcher(), searcher::setCheckCancelled, indexSort);//这里去查询了 结果在searchContext里面 if (rescore) { // only if we do a regular search 如果需要对结果重新打分 rescorePhase.execute(searchContext); } suggestPhase.execute(searchContext); aggregationPhase.execute(searchContext); //如果是聚合 会把在search 一下 把结果放到collector里面 lucene的search接口中 可以传 （q,from,size）也可以传（q,collector） , 前面一种进去后也会转换成一个默认的collector 叫 TopDocCollector if (searchContext.getProfilers() != null) { ProfileShardResult shardResults = SearchProfileShardResults .buildShardResults(searchContext.getProfilers()); searchContext.queryResult().profileResults(shardResults); } } 协调节点执行下一阶段 @Override public final void executeNextPhase(SearchPhase currentPhase, SearchPhase nextPhase) { /* This is the main search phase transition where we move to the next phase. At this point we check if there is * at least one successful operation left and if so we move to the next phase. If not we immediately fail the * search phase as "all shards failed"*/ if (successfulOps.get() == 0) { // 如果都没有执行成功 we have 0 successful results that means we shortcut stuff and return a failure if (logger.isDebugEnabled()) { final ShardOperationFailedException[] shardSearchFailures = ExceptionsHelper.groupBy(buildShardFailures()); Throwable cause = shardSearchFailures.length == 0 ? null : ElasticsearchException.guessRootCauses(shardSearchFailures[0].getCause())[0]; logger.debug((Supplier<?>) () -> new ParameterizedMessage("All shards failed for phase: [{}]", getName()), cause); } onPhaseFailure(currentPhase, "all shards failed", null); } else { if (logger.isTraceEnabled()) { final String resultsFrom = results.getSuccessfulResults() .map(r -> r.getSearchShardTarget().toString()).collect(Collectors.joining(",")); logger.trace("[{}] Moving to next phase: [{}], based on results from: {} (cluster state version: {})", currentPhase.getName(), nextPhase.getName(), resultsFrom, clusterStateVersion); } executePhase(nextPhase); } } private void innerRun() throws IOException {//结果收集 每收到一次 就count-1 final int numShards = context.getNumShards(); final boolean isScrollSearch = context.getRequest().scroll() != null; List<SearchPhaseResult> phaseResults = queryResults.asList(); String scrollId = isScrollSearch ? TransportSearchHelper.buildScrollId(queryResults) : null; final SearchPhaseController.ReducedQueryPhase reducedQueryPhase = resultConsumer.reduce(); final boolean queryAndFetchOptimization = queryResults.length() == 1; final Runnable finishPhase = () -> moveToNextPhase(searchPhaseController, scrollId, reducedQueryPhase, queryAndFetchOptimization ? queryResults : fetchResults); if (queryAndFetchOptimization) { assert phaseResults.isEmpty() || phaseResults.get(0).fetchResult() != null : "phaseResults empty [" + phaseResults.isEmpty() + "], single result: " + phaseResults.get(0).fetchResult(); // query AND fetch optimization finishPhase.run(); } else { final IntArrayList[] docIdsToLoad = searchPhaseController.fillDocIdsToLoad(numShards, reducedQueryPhase.scoreDocs); if (reducedQueryPhase.scoreDocs.length == 0) { // no docs to fetch -- sidestep everything and return phaseResults.stream() .map(SearchPhaseResult::queryResult) .forEach(this::releaseIrrelevantSearchContext); // we have to release contexts here to free up resources finishPhase.run(); } else { final ScoreDoc[] lastEmittedDocPerShard = isScrollSearch ? searchPhaseController.getLastEmittedDocPerShard(reducedQueryPhase, numShards) : null; final CountedCollector<FetchSearchResult> counter = new CountedCollector<>(r -> fetchResults.set(r.getShardIndex(), r), docIdsToLoad.length, // we count down every shard in the result no matter if we got any results or not finishPhase, context); for (int i = 0; i < docIdsToLoad.length; i++) { IntArrayList entry = docIdsToLoad[i]; SearchPhaseResult queryResult = queryResults.get(i); if (entry == null) { // docIdsToLoad 是查询到的分片的结果集合 如果某个分片上没结果 也就不需要进行fetch了 if (queryResult != null) { // if we got some hits from this shard we have to release the context there // we do this as we go since it will free up resources and passing on the request on the // transport layer is cheap. releaseIrrelevantSearchContext(queryResult.queryResult()); } // in any case we count down this result since we don't talk to this shard anymore counter.countDown();//-1 } else { SearchShardTarget searchShardTarget = queryResult.getSearchShardTarget(); Transport.Connection connection = context.getConnection(searchShardTarget.getClusterAlias(), searchShardTarget.getNodeId());//使用的是query then fatch query完之后开始fetch阶段 ShardFetchSearchRequest fetchSearchRequest = createFetchRequest(queryResult.queryResult().getRequestId(), i, entry, lastEmittedDocPerShard, searchShardTarget.getOriginalIndices());//执行召回 executeFetch(i, searchShardTarget, counter, fetchSearchRequest, queryResult.queryResult(), connection); //发送请求 fetch } } } } } indices:data/read/search[phase/fetch/id] 接收到消息后 开始对消息处理 transportService.registerRequestHandler(FETCH_ID_ACTION_NAME, ShardFetchSearchRequest::new, ThreadPool.Names.SEARCH, new TaskAwareTransportRequestHandler<ShardFetchSearchRequest>() { @Override public void messageReceived(ShardFetchSearchRequest request, TransportChannel channel, Task task) throws Exception { FetchSearchResult result = searchService.executeFetchPhase(request, (SearchTask)task); channel.sendResponse(result);//返回 } }); private void executeFetch(final int shardIndex, final SearchShardTarget shardTarget, final CountedCollector<FetchSearchResult> counter, final ShardFetchSearchRequest fetchSearchRequest, final QuerySearchResult querySearchResult, final Transport.Connection connection) { context.getSearchTransport().sendExecuteFetch(connection, fetchSearchRequest, context.getTask(), new SearchActionListener<FetchSearchResult>(shardTarget, shardIndex) { @Override public void innerOnResponse(FetchSearchResult result) { counter.onResult(result);//成功的话count -1 失败也会-1 } @Override public void onFailure(Exception e) { try { if (logger.isDebugEnabled()) { logger.debug((Supplier<?>) () -> new ParameterizedMessage("[{}] Failed to execute fetch phase", fetchSearchRequest.id()), e); } counter.onFailure(shardIndex, shardTarget, e); } finally { // the search context might not be cleared on the node where the fetch was executed for example // because the action was rejected by the thread pool. in this case we need to send a dedicated // request to clear the search context. releaseIrrelevantSearchContext(querySearchResult); } } }); } public FetchSearchResult executeFetchPhase(ShardFetchRequest request, SearchTask task) { final SearchContext context = findContext(request.id(), request); final SearchOperationListener operationListener = context.indexShard().getSearchOperationListener(); context.incRef(); try { context.setTask(task); contextProcessing(context); if (request.lastEmittedDoc() != null) { context.scrollContext().lastEmittedDoc = request.lastEmittedDoc(); } context.docIdsToLoad(request.docIds(), 0, request.docIdsSize()); operationListener.onPreFetchPhase(context); long time = System.nanoTime(); fetchPhase.execute(context);//获取数据 if (fetchPhaseShouldFreeContext(context)) { freeContext(request.id()); } else { contextProcessedSuccessfully(context); } operationListener.onFetchPhase(context, System.nanoTime() - time); return context.fetchResult(); } catch (Exception e) { operationListener.onFailedFetchPhase(context); logger.trace("Fetch phase failed", e); processFailure(context, e); throw ExceptionsHelper.convertToRuntime(e); } finally { cleanContext(context); } } 通过使用docId从lucene中取数据 @Override public void execute(SearchContext context) { final FieldsVisitor fieldsVisitor; Set<String> fieldNames = null; List<String> fieldNamePatterns = null; StoredFieldsContext storedFieldsContext = context.storedFieldsContext(); if (storedFieldsContext == null) { // no fields specified, default to return source if no explicit indication if (!context.hasScriptFields() && !context.hasFetchSourceContext()) { context.fetchSourceContext(new FetchSourceContext(true)); } fieldsVisitor = new FieldsVisitor(context.sourceRequested()); } else if (storedFieldsContext.fetchFields() == false) { // disable stored fields entirely fieldsVisitor = null; } else { for (String fieldName : context.storedFieldsContext().fieldNames()) { if (fieldName.equals(SourceFieldMapper.NAME)) { FetchSourceContext fetchSourceContext = context.hasFetchSourceContext() ? context.fetchSourceContext() : FetchSourceContext.FETCH_SOURCE; context.fetchSourceContext(new FetchSourceContext(true, fetchSourceContext.includes(), fetchSourceContext.excludes())); continue; } if (Regex.isSimpleMatchPattern(fieldName)) { if (fieldNamePatterns == null) { fieldNamePatterns = new ArrayList<>(); } fieldNamePatterns.add(fieldName); } else { MappedFieldType fieldType = context.smartNameFieldType(fieldName); if (fieldType == null) { // Only fail if we know it is a object field, missing paths / fields shouldn't fail. if (context.getObjectMapper(fieldName) != null) { throw new IllegalArgumentException("field [" + fieldName + "] isn't a leaf field"); } } if (fieldNames == null) { fieldNames = new HashSet<>(); } fieldNames.add(fieldName); } } boolean loadSource = context.sourceRequested(); if (fieldNames == null && fieldNamePatterns == null) { // empty list specified, default to disable _source if no explicit indication fieldsVisitor = new FieldsVisitor(loadSource); } else { fieldsVisitor = new CustomFieldsVisitor(fieldNames == null ? Collections.emptySet() : fieldNames, fieldNamePatterns == null ? Collections.emptyList() : fieldNamePatterns, loadSource); } } try { SearchHit[] hits = new SearchHit[context.docIdsToLoadSize()]; FetchSubPhase.HitContext hitContext = new FetchSubPhase.HitContext(); for (int index = 0; index < context.docIdsToLoadSize(); index++) { if (context.isCancelled()) { throw new TaskCancelledException("cancelled"); } int docId = context.docIdsToLoad()[context.docIdsToLoadFrom() + index]; int readerIndex = ReaderUtil.subIndex(docId, context.searcher().getIndexReader().leaves()); LeafReaderContext subReaderContext = context.searcher().getIndexReader().leaves().get(readerIndex); int subDocId = docId - subReaderContext.docBase; final SearchHit searchHit; int rootDocId = findRootDocumentIfNested(context, subReaderContext, subDocId); if (rootDocId != -1) { searchHit = createNestedSearchHit(context, docId, subDocId, rootDocId, fieldNames, fieldNamePatterns, subReaderContext); } else { searchHit = createSearchHit(context, fieldsVisitor, docId, subDocId, subReaderContext); } hits[index] = searchHit; hitContext.reset(searchHit, subReaderContext, subDocId, context.searcher()); for (FetchSubPhase fetchSubPhase : fetchSubPhases) { fetchSubPhase.hitExecute(context, hitContext); } } for (FetchSubPhase fetchSubPhase : fetchSubPhases) { fetchSubPhase.hitsExecute(context, hits); } context.fetchResult().hits(new SearchHits(hits, context.queryResult().getTotalHits(), context.queryResult().getMaxScore())); } catch (IOException e) { throw ExceptionsHelper.convertToElastic(e); } }
浙公网安备 33010602011771号