Skip to content

Apache Lucene open-source search software

Performance improvements to MatchHighlighter and MatchRegionRetriever (#12881)

Dawid Weiss755de5aae331
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 0ebe24ac8ea..4ebcc35d418 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -73,6 +73,12 @@ API Changes
* GITHUB#12243: Remove TermInSetQuery ctors taking varargs param. SortedSetDocValuesField#newSlowSetQuery,
SortedDocValuesField#newSlowSetQuery, KeywordField#newSetQuery, KeywordField#newSetQuery now take a collection. (Jakub Slowinski)
+* GITHUB#12881: Performance improvements to MatchHighlighter and MatchRegionRetriever. MatchRegionRetriever can be
+ configured to not load matches (or content) of certain fields and to force-load other fields so that stored fields
+ of a document are accessed once. A configurable limit of field matches placed in the priority queue was added
+ (allows handling long fields with lots of hits more gracefully). MatchRegionRetriever utilizes IndexSearcher's
+ executor to extract hit offsets concurrently. (Dawid Weiss)
+
New Features
---------------------
diff --git a/lucene/MIGRATE.md b/lucene/MIGRATE.md
index ff80dd5d442..348a9d2e84e 100644
--- a/lucene/MIGRATE.md
+++ b/lucene/MIGRATE.md
@@ -19,6 +19,11 @@
## Migration from Lucene 9.x to Lucene 10.0
+### Minor API changes in MatchHighlighter and MatchRegionRetriever. (GITHUB#12881)
+
+The API of interfaces for accepting highlights has changed to allow performance improvements. Look at the issue and the PR diff to get
+a sense of what's changed (changes are minor).
+
### Removed deprecated IndexSearcher.doc, IndexReader.document, IndexReader.getTermVectors (GITHUB#11998)
The deprecated Stored Fields and Term Vectors apis relied upon threadlocal storage and have been removed.
diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/FieldValueHighlighters.java b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/FieldValueHighlighters.java
index d04318a3175..36ab79291be 100644
--- a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/FieldValueHighlighters.java
+++ b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/FieldValueHighlighters.java
@@ -61,7 +61,7 @@ public final class FieldValueHighlighters {
@Override
public List<String> format(
String field,
- String[] values,
+ List<String> values,
String contiguousValue,
List<OffsetRange> valueRanges,
List<MatchHighlighter.QueryOffsetRange> matchOffsets) {
@@ -99,7 +99,7 @@ public final class FieldValueHighlighters {
@Override
public List<String> format(
String field,
- String[] values,
+ List<String> values,
String contiguousValue,
List<OffsetRange> valueRanges,
List<MatchHighlighter.QueryOffsetRange> matchOffsets) {
@@ -128,11 +128,11 @@ public final class FieldValueHighlighters {
@Override
public List<String> format(
String field,
- String[] values,
+ List<String> values,
String contiguousValue,
List<OffsetRange> valueRanges,
List<MatchHighlighter.QueryOffsetRange> matchOffsets) {
- return Arrays.asList(values);
+ return values;
}
};
}
@@ -146,7 +146,7 @@ public final class FieldValueHighlighters {
@Override
public List<String> format(
String field,
- String[] values,
+ List<String> values,
String contiguousValue,
List<OffsetRange> valueRanges,
List<MatchHighlighter.QueryOffsetRange> matchOffsets) {
diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/MatchHighlighter.java b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/MatchHighlighter.java
index 156d0e30a11..2c87fbdd62f 100644
--- a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/MatchHighlighter.java
+++ b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/MatchHighlighter.java
@@ -17,7 +17,6 @@
package org.apache.lucene.search.matchhighlight;
import java.io.IOException;
-import java.io.UncheckedIOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -29,10 +28,6 @@ import java.util.Objects;
import java.util.function.Predicate;
import java.util.stream.Stream;
import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.DocumentStoredFieldVisitor;
-import org.apache.lucene.index.FieldInfo;
-import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
@@ -40,12 +35,12 @@ import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TopDocs;
/**
- * An example highlighter that combines several lower-level highlighting utilities in this package
- * into a fully featured, ready-to-use component.
+ * An example highlighter that combines several lower-level utility classes in this package into a
+ * fully featured, ready-to-use component.
*
- * <p>Note that if you need to customize or tweak the details of highlighting, it is better to
- * assemble your own highlighter using those low-level building blocks, rather than extend or modify
- * this one.
+ * <p>Note: if you need to customize or tweak the details of highlighting, it is better to assemble
+ * your own highlighter using those low-level building blocks, rather than extend or modify this
+ * one.
*/
public class MatchHighlighter {
private final IndexSearcher searcher;
@@ -71,16 +66,16 @@ public class MatchHighlighter {
*/
boolean isApplicable(String field, boolean hasMatches);
- /** Do format field values appropriately. */
+ /** Format field values into a list of final "highlights". */
List<String> format(
String field,
- String[] values,
+ List<String> values,
String contiguousValue,
List<OffsetRange> valueRanges,
List<QueryOffsetRange> matchOffsets);
/**
- * @return Returns a set of fields that must be fetched for each document, regardless of whether
+ * @return Returns a set of fields that must be loaded from each document, regardless of whether
* they had matches or not. This is useful to load and return certain fields that should
* always be included (identifiers, document titles, etc.).
*/
@@ -106,7 +101,7 @@ public class MatchHighlighter {
@Override
public List<String> format(
String field,
- String[] values,
+ List<String> values,
String contiguousValue,
List<OffsetRange> valueRanges,
List<QueryOffsetRange> matchOffsets) {
@@ -169,14 +164,14 @@ public class MatchHighlighter {
private static class DocHit {
final int docId;
- private final LeafReader leafReader;
- private final int leafDocId;
private final LinkedHashMap<String, List<QueryOffsetRange>> matchRanges = new LinkedHashMap<>();
+ private final LinkedHashMap<String, List<String>> fieldValues = new LinkedHashMap<>();
- DocHit(int docId, LeafReader leafReader, int leafDocId) {
+ DocHit(int docId, MatchRegionRetriever.FieldValueProvider fieldValueProvider) {
this.docId = docId;
- this.leafReader = leafReader;
- this.leafDocId = leafDocId;
+ for (var fieldName : fieldValueProvider) {
+ fieldValues.put(fieldName, fieldValueProvider.getValues(fieldName));
+ }
}
void addMatches(Query query, Map<String, List<OffsetRange>> hits) {
@@ -187,22 +182,6 @@ public class MatchHighlighter {
offsets.forEach(o -> target.add(new QueryOffsetRange(query, o.from, o.to)));
});
}
-
- Document document(Predicate<String> needsField) throws IOException {
- // Only load the fields that have a chance to be highlighted.
- DocumentStoredFieldVisitor visitor =
- new DocumentStoredFieldVisitor() {
- @Override
- public Status needsField(FieldInfo fieldInfo) {
- return (matchRanges.containsKey(fieldInfo.name) || needsField.test(fieldInfo.name))
- ? Status.YES
- : Status.NO;
- }
- };
-
- leafReader.storedFields().document(leafDocId, visitor);
- return visitor.getDocument();
- }
}
public MatchHighlighter(IndexSearcher searcher, Analyzer analyzer) {
@@ -223,25 +202,44 @@ public class MatchHighlighter {
public Stream<DocHighlights> highlight(TopDocs topDocs, Query... queries) throws IOException {
// We want to preserve topDocs document ordering and MatchRegionRetriever is optimized
- // for streaming, so we'll just prepopulate the map in proper order.
+ // for streaming, so we'll just populate the map in proper order.
LinkedHashMap<Integer, DocHit> docHits = new LinkedHashMap<>();
for (ScoreDoc scoreDoc : topDocs.scoreDocs) {
docHits.put(scoreDoc.doc, null);
}
+ Predicate<String> fieldsToLoadUnconditionally = fieldsAlwaysReturned::contains;
+ Predicate<String> fieldsToLoadIfWithHits =
+ fieldName -> {
+ // We're interested in any fields for which existing highlighters are applicable (with or
+ // without hits).
+ return fieldHighlighters.stream()
+ .anyMatch(
+ highlighter ->
+ highlighter.isApplicable(fieldName, true)
+ || highlighter.isApplicable(fieldName, false));
+ };
+
// Collect match ranges for each query and associate each range to the origin query.
for (Query q : queries) {
MatchRegionRetriever highlighter =
- new MatchRegionRetriever(searcher, searcher.rewrite(q), offsetsRetrievalStrategies);
+ new MatchRegionRetriever(
+ searcher,
+ searcher.rewrite(q),
+ offsetsRetrievalStrategies,
+ fieldsToLoadUnconditionally,
+ fieldsToLoadIfWithHits);
+
highlighter.highlightDocuments(
topDocs,
(int docId,
LeafReader leafReader,
int leafDocId,
+ MatchRegionRetriever.FieldValueProvider fieldValueProvider,
Map<String, List<OffsetRange>> hits) -> {
DocHit docHit = docHits.get(docId);
if (docHit == null) {
- docHit = new DocHit(docId, leafReader, leafDocId);
+ docHit = new DocHit(docId, fieldValueProvider);
docHits.put(docId, docHit);
}
docHit.addMatches(q, hits);
@@ -254,23 +252,11 @@ public class MatchHighlighter {
}
private DocHighlights computeDocFieldValues(DocHit docHit) {
- Document doc;
- try {
- doc = docHit.document(fieldsAlwaysReturned::contains);
- } catch (IOException e) {
- throw new UncheckedIOException(e);
- }
-
DocHighlights docHighlights = new DocHighlights(docHit.docId);
- HashSet<String> unique = new HashSet<>();
- for (IndexableField indexableField : doc) {
- String field = indexableField.name();
- if (!unique.add(field)) {
- continue;
- }
-
- String[] values = doc.getValues(field);
+ for (var e : docHit.fieldValues.entrySet()) {
+ String field = e.getKey();
+ List<String> values = e.getValue();
String contiguousValue = contiguousFieldValue(field, values);
List<OffsetRange> valueRanges = computeValueRanges(field, values);
List<QueryOffsetRange> offsets = docHit.matchRanges.get(field);
@@ -287,7 +273,7 @@ public class MatchHighlighter {
return docHighlights;
}
- private List<OffsetRange> computeValueRanges(String field, String[] values) {
+ private List<OffsetRange> computeValueRanges(String field, List<String> values) {
ArrayList<OffsetRange> valueRanges = new ArrayList<>();
int offset = 0;
for (CharSequence v : values) {
@@ -298,10 +284,10 @@ public class MatchHighlighter {
return valueRanges;
}
- private String contiguousFieldValue(String field, String[] values) {
+ private String contiguousFieldValue(String field, List<String> values) {
String value;
- if (values.length == 1) {
- value = values[0];
+ if (values.size() == 1) {
+ value = values.get(0);
} else {
// TODO: This can be inefficient if offset gap is large but the logic
// of applying offsets would get much more complicated so leaving for now
diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/MatchRegionRetriever.java b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/MatchRegionRetriever.java
index 8b228eebee7..ce49c4d9c95 100644
--- a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/MatchRegionRetriever.java
+++ b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/MatchRegionRetriever.java
@@ -17,34 +17,40 @@
package org.apache.lucene.search.matchhighlight;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
-import java.util.HashSet;
import java.util.Iterator;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
-import java.util.Objects;
import java.util.PrimitiveIterator;
-import java.util.Set;
import java.util.TreeMap;
import java.util.TreeSet;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ForkJoinPool;
import java.util.function.Predicate;
+import java.util.function.ToIntFunction;
import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.document.Document;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.StoredFieldVisitor;
+import org.apache.lucene.index.StoredFields;
+import org.apache.lucene.search.FilterMatchesIterator;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Matches;
import org.apache.lucene.search.MatchesIterator;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.QueryVisitor;
import org.apache.lucene.search.ScoreMode;
+import org.apache.lucene.search.TaskExecutor;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.search.Weight;
-import org.apache.lucene.util.IOSupplier;
+import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.IOConsumer;
/**
* Utility class to compute a list of "match regions" for a given query, searcher and document(s)
@@ -53,57 +59,116 @@ import org.apache.lucene.util.IOSupplier;
public class MatchRegionRetriever {
private final List<LeafReaderContext> leaves;
private final Weight weight;
- private final TreeSet<String> affectedFields;
private final Map<String, OffsetsRetrievalStrategy> offsetStrategies;
- private final Set<String> preloadFields;
+ private final TreeSet<String> queryAffectedHighlightedFields;
+ private final Predicate<String> shouldLoadStoredField;
+ private final IndexSearcher searcher;
/**
- * A callback for accepting a single document (and its associated leaf reader, leaf document ID)
- * and its match offset ranges, as indicated by the {@link Matches} interface retrieved for the
- * query.
+ * A callback invoked for each document selected by the query. The callback receives a list of hit
+ * ranges, document field value accessor, the leaf reader and document ID of the document.
*/
@FunctionalInterface
public interface MatchOffsetsConsumer {
+ /**
+ * @param docId Document id (global).
+ * @param leafReader Document's {@link LeafReader}.
+ * @param leafDocId Document id (within the {@link LeafReader}).
+ * @param fieldValueProvider Access to preloaded document fields. See the {@link
+ * MatchRegionRetriever#MatchRegionRetriever(IndexSearcher, Query,
+ * OffsetsRetrievalStrategySupplier, Predicate, Predicate)} constructor's documentation for
+ * guidelines on which fields are available through this interface.
+ * @param hits A map of field names and offset ranges with query hits.
+ */
void accept(
- int docId, LeafReader leafReader, int leafDocId, Map<String, List<OffsetRange>> hits)
+ int docId,
+ LeafReader leafReader,
+ int leafDocId,
+ FieldValueProvider fieldValueProvider,
+ Map<String, List<OffsetRange>> hits)
throws IOException;
}
/**
- * An abstraction that provides document values for a given field. The default implementation just
- * reaches to a preloaded {@link Document}. It is possible to write a more efficient
- * implementation on top of a reusable character buffer (that reuses the buffer while retrieving
- * hit regions for documents).
+ * Access to field values of the highlighted document. See the {@link
+ * MatchRegionRetriever#MatchRegionRetriever(IndexSearcher, Query,
+ * OffsetsRetrievalStrategySupplier, Predicate, Predicate)} constructor's documentation for
+ * guidelines on which fields are available through this interface.
*/
- @FunctionalInterface
- public interface FieldValueProvider {
- List<CharSequence> getValues(String field) throws IOException;
+ public interface FieldValueProvider extends Iterable<String> {
+ /**
+ * @return Return a list of values for the provided field name or {@code null} if the field is
+ * not loaded or does not exist for the field.
+ */
+ List<String> getValues(String field);
}
/**
- * A constructor with the default offset strategy supplier.
+ * This constructor uses the default offset strategy supplier from {@link
+ * #computeOffsetRetrievalStrategies(IndexReader, Analyzer)}.
*
+ * @param searcher The {@link IndexSearcher} used to execute the query. The index searcher's
+ * {@linkplain IndexSearcher#getTaskExecutor() task executor} is also used for computing
+ * highlights concurrently.
+ * @param query The query for which highlights should be returned.
* @param analyzer An analyzer that may be used to reprocess (retokenize) document fields in the
* absence of position offsets in the index. Note that the analyzer must return tokens
* (positions and offsets) identical to the ones stored in the index.
+ * @param fieldsToLoadUnconditionally A custom predicate that should return {@code true} for any
+ * field that should be preloaded and made available through {@link FieldValueProvider},
+ * regardless of whether the query affected the field or not. This predicate can be used to
+ * load additional fields during field highlighting, making them available to {@link
+ * MatchOffsetsConsumer}s.
+ * @param fieldsToLoadIfWithHits A custom predicate that should return {@code true} for fields
+ * that should be highlighted. Typically, this would always return {@code true} indicating any
+ * field affected by the query should be highlighted. However, sometimes highlights may not be
+ * needed: for example, if they affect fields that are only used for filtering purposes.
+ * Returning {@code false} for such fields saves the costs of loading those fields into memory
+ * and scanning through field matches.
*/
- public MatchRegionRetriever(IndexSearcher searcher, Query query, Analyzer analyzer)
+ public MatchRegionRetriever(
+ IndexSearcher searcher,
+ Query query,
+ Analyzer analyzer,
+ Predicate<String> fieldsToLoadUnconditionally,
+ Predicate<String> fieldsToLoadIfWithHits)
throws IOException {
- this(searcher, query, computeOffsetRetrievalStrategies(searcher.getIndexReader(), analyzer));
+ this(
+ searcher,
+ query,
+ computeOffsetRetrievalStrategies(searcher.getIndexReader(), analyzer),
+ fieldsToLoadUnconditionally,
+ fieldsToLoadIfWithHits);
}
/**
- * @param searcher Index searcher to be used for retrieving matches.
+ * @param searcher The {@link IndexSearcher} used to execute the query. The index searcher's
+ * {@linkplain IndexSearcher#getTaskExecutor() task executor} is also used for computing
+ * highlights concurrently.
* @param query The query for which matches should be retrieved. The query should be rewritten
* against the provided searcher.
* @param fieldOffsetStrategySupplier A custom supplier of per-field {@link
* OffsetsRetrievalStrategy} instances.
+ * @param fieldsToLoadUnconditionally A custom predicate that should return {@code true} for any
+ * field that should be preloaded and made available through {@link FieldValueProvider},
+ * regardless of whether the query affected the field or not. This predicate can be used to
+ * load additional fields during field highlighting, making them available to {@link
+ * MatchOffsetsConsumer}s.
+ * @param fieldsToLoadIfWithHits A custom predicate that should return {@code true} for fields
+ * that should be highlighted. Typically, this would always return {@code true} indicating any
+ * field affected by the query should be highlighted. However, sometimes highlights may not be
+ * needed: for example, if they affect fields that are only used for filtering purposes.
+ * Returning {@code false} for such fields saves the costs of loading those fields into memory
+ * and scanning through field matches.
*/
public MatchRegionRetriever(
IndexSearcher searcher,
Query query,
- OffsetsRetrievalStrategySupplier fieldOffsetStrategySupplier)
+ OffsetsRetrievalStrategySupplier fieldOffsetStrategySupplier,
+ Predicate<String> fieldsToLoadUnconditionally,
+ Predicate<String> fieldsToLoadIfWithHits)
throws IOException {
+ this.searcher = searcher;
leaves = searcher.getIndexReader().leaves();
assert checkOrderConsistency(leaves);
@@ -111,83 +176,213 @@ public class MatchRegionRetriever {
// (no optimizations in Boolean queries take place).
weight = searcher.createWeight(query, ScoreMode.COMPLETE, 0);
- // Compute the subset of fields affected by this query so that we don't load or scan
- // fields that are irrelevant.
- affectedFields = new TreeSet<>();
+ // Compute a subset of fields affected by this query and for which highlights should be
+ // returned, so that we don't load or scan fields that are irrelevant.
+ queryAffectedHighlightedFields = new TreeSet<>();
query.visit(
new QueryVisitor() {
@Override
public boolean acceptField(String field) {
- affectedFields.add(field);
+ if (fieldsToLoadIfWithHits.test(field)) {
+ queryAffectedHighlightedFields.add(field);
+ }
return false;
}
});
// Compute value offset retrieval strategy for all affected fields.
offsetStrategies = new HashMap<>();
- for (String field : affectedFields) {
+ for (String field : queryAffectedHighlightedFields) {
offsetStrategies.put(field, fieldOffsetStrategySupplier.apply(field));
}
- // Ask offset strategies if they'll need field values.
- preloadFields = new HashSet<>();
- offsetStrategies.forEach(
- (field, strategy) -> {
- preloadFields.add(field);
- });
-
- // Only preload those field values that can be affected by the query and are required
- // by strategies.
- preloadFields.retainAll(affectedFields);
+ shouldLoadStoredField =
+ (field) -> {
+ return fieldsToLoadUnconditionally.test(field)
+ || queryAffectedHighlightedFields.contains(field);
+ };
}
+ /**
+ * Processes {@link TopDocs} with reasonable defaults. See variants of this method for low-level
+ * tuning parameters.
+ *
+ * @see #highlightDocuments(PrimitiveIterator.OfInt, MatchOffsetsConsumer, ToIntFunction, int,
+ * int)
+ * @param topDocs Search results.
+ * @param consumer A streaming consumer for document-hits pairs.
+ */
public void highlightDocuments(TopDocs topDocs, MatchOffsetsConsumer consumer)
throws IOException {
highlightDocuments(
Arrays.stream(topDocs.scoreDocs).mapToInt(scoreDoc -> scoreDoc.doc).sorted().iterator(),
- consumer);
+ consumer,
+ field -> Integer.MAX_VALUE);
}
/**
- * Low-level, high-efficiency method for highlighting large numbers of documents at once in a
- * streaming fashion.
+ * Low-level, high-efficiency method for highlighting large numbers of documents at once.
*
* @param docIds A stream of <em>sorted</em> document identifiers for which hit ranges should be
* returned.
* @param consumer A streaming consumer for document-hits pairs.
+ * @param maxHitsPerField A predicate that should, for the provided field, return the maximum
+ * number of hit regions to consider when scoring passages. The predicate should return {@link
+ * Integer#MAX_VALUE} for all hits to be considered, although typically 3-10 hits are
+ * sufficient and lead to performance savings in long fields with large numbers of hit ranges.
+ * @see #highlightDocuments(PrimitiveIterator.OfInt, MatchOffsetsConsumer, ToIntFunction, int,
+ * int)
*/
- public void highlightDocuments(PrimitiveIterator.OfInt docIds, MatchOffsetsConsumer consumer)
+ public void highlightDocuments(
+ PrimitiveIterator.OfInt docIds,
+ MatchOffsetsConsumer consumer,
+ ToIntFunction<String> maxHitsPerField)
+ throws IOException {
+ // Typically enough to saturate a single processing thread and be large enough to
+ // compensate for overhead of concurrency.
+ final int DEFAULT_MAX_BLOCK_SIZE = 50;
+ highlightDocuments(
+ docIds,
+ consumer,
+ maxHitsPerField,
+ DEFAULT_MAX_BLOCK_SIZE,
+ ForkJoinPool.getCommonPoolParallelism());
+ }
+
+ /**
+ * Low-level, high-efficiency method for highlighting large numbers of documents at once.
+ *
+ * <p>Document IDs are grouped into sequential "blocks". For each block, highlights are computed
+ * (this can use parallel threads, if {@link IndexSearcher#getTaskExecutor()}) can execute tasks
+ * in parallel. Finally, processed highlights are passed to the {@code consumer}.
+ *
+ * @param docIds A stream of <em>sorted</em> document identifiers for which hit ranges should be
+ * returned.
+ * @param consumer A streaming consumer for document-query hits pairs. This consumer will be
+ * called sequentially, with document ordering corresponding to that of the query results.
+ * @param maxHitsPerField A predicate that should, for the provided field, return the maximum
+ * number of hit regions to consider when scoring passages. The predicate should return {@link
+ * Integer#MAX_VALUE} for all hits to be considered, although typically 3-10 hits are
+ * sufficient and lead to performance savings in long fields with large numbers of hit ranges.
+ * @param maxBlockSize The maximum size of a single contiguous "block" of documents. Each block
+ * can be processed in parallel, using the index searcher's task executor.
+ * @param maxBlocksProcessedInParallel Maximum number of queued document "blocks"; when reached,
+ * the queue is processed (possibly concurrently) and then passed to the {@code consumer}. Set
+ * this value to {@code 1} to process blocks sequentially.
+ */
+ public void highlightDocuments(
+ PrimitiveIterator.OfInt docIds,
+ MatchOffsetsConsumer consumer,
+ ToIntFunction<String> maxHitsPerField,
+ int maxBlockSize,
+ int maxBlocksProcessedInParallel)
throws IOException {
if (leaves.isEmpty()) {
return;
}
- Iterator<LeafReaderContext> ctx = leaves.iterator();
- LeafReaderContext currentContext = ctx.next();
+ ArrayList<Callable<DocHighlightData[]>> blockQueue = new ArrayList<>();
+
+ TaskExecutor taskExecutor = searcher.getTaskExecutor();
+ IOConsumer<ArrayList<Callable<DocHighlightData[]>>> drainQueue;
+ if (maxBlocksProcessedInParallel == 1) {
+ // Sequential, own-thread processing.
+ drainQueue =
+ (queue) -> {
+ for (var callable : queue) {
+ try {
+ processBlock(callable.call(), consumer);
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+ }
+ queue.clear();
+ };
+ } else {
+ // Potentially concurrent processing via IndexSearcher's TaskExecutor.
+ drainQueue =
+ (queue) -> {
+ for (var highlightData : taskExecutor.invokeAll(queue)) {
+ processBlock(highlightData, consumer);
+ }
+ queue.clear();
+ };
+ }
+
+ // Collect blocks.
int previousDocId = -1;
- Map<String, List<OffsetRange>> highlights = new TreeMap<>();
+ int[] block = new int[maxBlockSize];
+ int blockPos = 0;
while (docIds.hasNext()) {
int docId = docIds.nextInt();
-
if (docId < previousDocId) {
throw new RuntimeException("Input document IDs must be sorted (increasing).");
}
previousDocId = docId;
- while (docId >= currentContext.docBase + currentContext.reader().maxDoc()) {
+ block[blockPos++] = docId;
+ if (blockPos >= maxBlockSize || !docIds.hasNext()) {
+ final int[] idBlock = ArrayUtil.copyOfSubArray(block, 0, blockPos);
+ blockQueue.add(() -> prepareBlock(idBlock, maxHitsPerField));
+ blockPos = 0;
+
+ if (blockQueue.size() >= maxBlocksProcessedInParallel) {
+ drainQueue.accept(blockQueue);
+ }
+ }
+ }
+
+ // Finalize any remaining blocks.
+ if (!blockQueue.isEmpty()) {
+ drainQueue.accept(blockQueue);
+ }
+ }
+
+ private record DocHighlightData(
+ int docId,
+ LeafReader leafReader,
+ int leafDocId,
+ FieldValueProvider fieldValueProvider,
+ Map<String, List<OffsetRange>> hits) {}
+
+ private DocHighlightData[] prepareBlock(int[] idBlock, ToIntFunction<String> maxHitsPerField)
+ throws IOException {
+ DocHighlightData[] docData = new DocHighlightData[idBlock.length];
+
+ Iterator<LeafReaderContext> ctx = leaves.iterator();
+ LeafReaderContext currentContext = ctx.next();
+ LeafReader reader = currentContext.reader();
+
+ for (int i = 0; i < idBlock.length; i++) {
+ final int docId = idBlock[i];
+
+ while (docId >= currentContext.docBase + reader.maxDoc()) {
currentContext = ctx.next();
+ reader = currentContext.reader();
}
int contextRelativeDocId = docId - currentContext.docBase;
- // Only preload fields we may potentially need.
- FieldValueProvider docFieldsSupplier =
- new DocumentFieldValueProvider(currentContext, contextRelativeDocId, preloadFields);
+ var fieldVisitor = new StoredFieldsVisitor(shouldLoadStoredField);
+ StoredFields storedFields = reader.storedFields();
+ storedFields.document(contextRelativeDocId, fieldVisitor);
- highlights.clear();
+ Map<String, List<OffsetRange>> highlights = new TreeMap<>();
highlightDocument(
- currentContext, contextRelativeDocId, docFieldsSupplier, (field) -> true, highlights);
- consumer.accept(docId, currentContext.reader(), contextRelativeDocId, highlights);
+ currentContext, contextRelativeDocId, fieldVisitor, maxHitsPerField, highlights);
+
+ docData[i] =
+ new DocHighlightData(docId, reader, contextRelativeDocId, fieldVisitor, highlights);
+ }
+
+ return docData;
+ }
+
+ private void processBlock(DocHighlightData[] docHighlightData, MatchOffsetsConsumer consumer)
+ throws IOException {
+ for (var data : docHighlightData) {
+ consumer.accept(
+ data.docId, data.leafReader, data.leafDocId, data.fieldValueProvider, data.hits);
}
}
@@ -199,7 +394,7 @@ public class MatchRegionRetriever {
LeafReaderContext leafReaderContext,
int contextDocId,
FieldValueProvider doc,
- Predicate<String> acceptField,
+ ToIntFunction<String> maxHitsPerField,
Map<String, List<OffsetRange>> outputHighlights)
throws IOException {
Matches matches = weight.matches(leafReaderContext, contextDocId);
@@ -207,29 +402,59 @@ public class MatchRegionRetriever {
return;
}
- for (String field : affectedFields) {
- if (acceptField.test(field)) {
- MatchesIterator matchesIterator = matches.getMatches(field);
- if (matchesIterator == null) {
- // No matches on this field, even though the field was part of the query. This may be
- // possible
- // with complex queries that source non-text fields (have no "hit regions" in any textual
- // representation). Skip.
- } else {
- OffsetsRetrievalStrategy offsetStrategy = offsetStrategies.get(field);
- if (offsetStrategy == null) {
- throw new IOException(
- "Non-empty matches but no offset retrieval strategy for field: " + field);
- }
- List<OffsetRange> ranges = offsetStrategy.get(matchesIterator, doc);
- if (!ranges.isEmpty()) {
- outputHighlights.put(field, ranges);
- }
+ for (String field : queryAffectedHighlightedFields) {
+ MatchesIterator matchesIterator = matches.getMatches(field);
+ if (matchesIterator == null) {
+ // No matches on this field, even though the field was part of the query. This may be
+ // possible
+ // with complex queries that source non-text fields (have no "hit regions" in any textual
+ // representation). Skip.
+ } else {
+ OffsetsRetrievalStrategy offsetStrategy = offsetStrategies.get(field);
+ if (offsetStrategy == null) {
+ throw new IOException(
+ "Non-empty matches but no offset retrieval strategy for field: " + field);
+ }
+ var delegate = offsetStrategy;
+
+ // Limit the number of hits so that we're not extracting dozens just to trim them to a few
+ // in the end.
+ final int maxHits = maxHitsPerField.applyAsInt(field);
+ if (maxHits != Integer.MAX_VALUE) {
+ offsetStrategy =
+ (matchesIterator1, doc1) ->
+ delegate.get(new MatchesIteratorWithLimit(matchesIterator1, maxHits), doc1);
+ }
+
+ List<OffsetRange> ranges = offsetStrategy.get(matchesIterator, doc);
+ if (!ranges.isEmpty()) {
+ outputHighlights.put(field, ranges);
}
}
}
}
+ private static class MatchesIteratorWithLimit extends FilterMatchesIterator {
+ private int limit;
+
+ public MatchesIteratorWithLimit(MatchesIterator matchesIterator, int limit) {
+ super(matchesIterator);
+ if (limit < 0) {
+ throw new IllegalArgumentException();
+ }
+ this.limit = limit;
+ }
+
+ @Override
+ public boolean next() throws IOException {
+ if (limit == 0) {
+ return false;
+ }
+ limit--;
+ return super.next();
+ }
+ }
+
private boolean checkOrderConsistency(List<LeafReaderContext> leaves) {
for (int i = 1; i < leaves.size(); i++) {
LeafReaderContext prev = leaves.get(i - 1);
@@ -286,23 +511,58 @@ public class MatchRegionRetriever {
};
}
- /** Implements {@link FieldValueProvider} wrapping a preloaded {@link Document}. */
- private static final class DocumentFieldValueProvider implements FieldValueProvider {
- private final IOSupplier<Document> docSupplier;
- private Document doc;
+ private static class StoredFieldsVisitor extends StoredFieldVisitor
+ implements FieldValueProvider {
+ private final Predicate<String> needsField;
+ private final LinkedHashMap<String, List<String>> fieldValues = new LinkedHashMap<>();
- public DocumentFieldValueProvider(
- LeafReaderContext currentContext, int docId, Set<String> preloadFields) {
- docSupplier = () -> currentContext.reader().storedFields().document(docId, preloadFields);
+ public StoredFieldsVisitor(Predicate<String> shouldLoadStoredField) {
+ this.needsField = shouldLoadStoredField;
}
@Override
- public List<CharSequence> getValues(String field) throws IOException {
- if (doc == null) {
- doc = Objects.requireNonNull(docSupplier.get());
- }
+ public Status needsField(FieldInfo fieldInfo) throws IOException {
+ return needsField.test(fieldInfo.name) ? Status.YES : Status.NO;
+ }
+
+ @Override
+ public List<String> getValues(String field) {
+ List<String> values = fieldValues.get(field);
+ return values == null ? List.of() : values;
+ }
+
+ @Override
+ public void stringField(FieldInfo fieldInfo, String value) throws IOException {
+ addField(fieldInfo, value);
+ }
+
+ @Override
+ public void intField(FieldInfo fieldInfo, int value) throws IOException {
+ addField(fieldInfo, Integer.toString(value));
+ }
+
+ @Override
+ public void longField(FieldInfo fieldInfo, long value) throws IOException {
+ addField(fieldInfo, Long.toString(value));
+ }
+
+ @Override
+ public void floatField(FieldInfo fieldInfo, float value) throws IOException {
+ addField(fieldInfo, Float.toString(value));
+ }
- return Arrays.asList(doc.getValues(field));
+ @Override
+ public void doubleField(FieldInfo fieldInfo, double value) throws IOException {
+ addField(fieldInfo, Double.toString(value));
+ }
+
+ private void addField(FieldInfo field, String value) {
+ fieldValues.computeIfAbsent(field.name, v -> new ArrayList<>()).add(value);
+ }
+
+ @Override
+ public Iterator<String> iterator() {
+ return fieldValues.keySet().iterator();
}
}
}
diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/OffsetsFromPositions.java b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/OffsetsFromPositions.java
index 20a67194299..a8d33a52a25 100644
--- a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/OffsetsFromPositions.java
+++ b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/OffsetsFromPositions.java
@@ -60,7 +60,7 @@ public final class OffsetsFromPositions implements OffsetsRetrievalStrategy {
}
List<OffsetRange> convertPositionsToOffsets(
- ArrayList<OffsetRange> positionRanges, List<CharSequence> values) throws IOException {
+ ArrayList<OffsetRange> positionRanges, List<String> values) throws IOException {
if (positionRanges.isEmpty()) {
return positionRanges;
@@ -94,7 +94,7 @@ public final class OffsetsFromPositions implements OffsetsRetrievalStrategy {
int position = -1;
int valueOffset = 0;
for (int valueIndex = 0, max = values.size(); valueIndex < max; valueIndex++) {
- final String value = values.get(valueIndex).toString();
+ final String value = values.get(valueIndex);
final boolean lastValue = valueIndex + 1 == max;
TokenStream ts = analyzer.tokenStream(field, value);
diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/OffsetsFromTokens.java b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/OffsetsFromTokens.java
index 3861c4e8cc0..307e9d34576 100644
--- a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/OffsetsFromTokens.java
+++ b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/OffsetsFromTokens.java
@@ -51,7 +51,7 @@ public final class OffsetsFromTokens implements OffsetsRetrievalStrategy {
public List<OffsetRange> get(
MatchesIterator matchesIterator, MatchRegionRetriever.FieldValueProvider doc)
throws IOException {
- List<CharSequence> values = doc.getValues(field);
+ List<String> values = doc.getValues(field);
Set<BytesRef> matchTerms = new HashSet<>();
while (matchesIterator.next()) {
@@ -72,7 +72,7 @@ public final class OffsetsFromTokens implements OffsetsRetrievalStrategy {
ArrayList<OffsetRange> ranges = new ArrayList<>();
int valueOffset = 0;
for (int valueIndex = 0, max = values.size(); valueIndex < max; valueIndex++) {
- final String value = values.get(valueIndex).toString();
+ final String value = values.get(valueIndex);
TokenStream ts = analyzer.tokenStream(field, value);
OffsetAttribute offsetAttr = ts.getAttribute(OffsetAttribute.class);
diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/OffsetsFromValues.java b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/OffsetsFromValues.java
index edb68936260..72bacb711a4 100644
--- a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/OffsetsFromValues.java
+++ b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/OffsetsFromValues.java
@@ -44,7 +44,7 @@ public final class OffsetsFromValues implements OffsetsRetrievalStrategy {
public List<OffsetRange> get(
MatchesIterator matchesIterator, MatchRegionRetriever.FieldValueProvider doc)
throws IOException {
- List<CharSequence> values = doc.getValues(field);
+ List<String> values = doc.getValues(field);
ArrayList<OffsetRange> ranges = new ArrayList<>();
int valueOffset = 0;
diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/matchhighlight/TestMatchHighlighter.java b/lucene/highlighter/src/test/org/apache/lucene/search/matchhighlight/TestMatchHighlighter.java
index 00ca8d7b1c1..ef1f038248e 100644
--- a/lucene/highlighter/src/test/org/apache/lucene/search/matchhighlight/TestMatchHighlighter.java
+++ b/lucene/highlighter/src/test/org/apache/lucene/search/matchhighlight/TestMatchHighlighter.java
@@ -613,16 +613,15 @@ public class TestMatchHighlighter extends LuceneTestCase {
@Override
public List<String> format(
String field,
- String[] values,
+ List<String> values,
String contiguousValue,
List<OffsetRange> valueRanges,
List<MatchHighlighter.QueryOffsetRange> matchOffsets) {
- if (values.length <= limit) {
- return Arrays.asList(values);
+ if (values.size() <= limit) {
+ return values;
} else {
- List<String> collected =
- Stream.of(values).limit(limit).collect(Collectors.toList());
- int remaining = values.length - collected.size();
+ List<String> collected = values.subList(0, limit);
+ int remaining = values.size() - collected.size();
collected.add(String.format(Locale.ROOT, "[%d omitted]", remaining));
return collected;
}
@@ -729,7 +728,7 @@ public class TestMatchHighlighter extends LuceneTestCase {
@Override
public List<String> format(
String field,
- String[] values,
+ List<String> values,
String contiguousValue,
List<OffsetRange> valueRanges,
List<MatchHighlighter.QueryOffsetRange> matchOffsets) {
diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/matchhighlight/TestMatchRegionRetriever.java b/lucene/highlighter/src/test/org/apache/lucene/search/matchhighlight/TestMatchRegionRetriever.java
index 8db2641f089..cd46ea15b14 100644
--- a/lucene/highlighter/src/test/org/apache/lucene/search/matchhighlight/TestMatchRegionRetriever.java
+++ b/lucene/highlighter/src/test/org/apache/lucene/search/matchhighlight/TestMatchRegionRetriever.java
@@ -21,6 +21,7 @@ import static org.hamcrest.Matchers.containsInAnyOrder;
import com.carrotsearch.randomizedtesting.RandomizedTest;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
@@ -28,6 +29,7 @@ import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import java.util.function.BiFunction;
+import java.util.function.Predicate;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.lucene.analysis.Analyzer;
@@ -319,7 +321,7 @@ public class TestMatchRegionRetriever extends LuceneTestCase {
.build(
analyzer,
reader -> {
- assertThat(
+ MatcherAssert.assertThat(
highlights(
reader,
new IntervalQuery(
@@ -746,7 +748,7 @@ public class TestMatchRegionRetriever extends LuceneTestCase {
AsciiMatchRangeHighlighter formatter = new AsciiMatchRangeHighlighter(analyzer);
MatchRegionRetriever.MatchOffsetsConsumer highlightCollector =
- (docId, leafReader, leafDocId, fieldHighlights) -> {
+ (docId, leafReader, leafDocId, fieldValueProvider, fieldHighlights) -> {
StringBuilder sb = new StringBuilder();
Document document = leafReader.storedFields().document(leafDocId);
@@ -765,9 +767,23 @@ public class TestMatchRegionRetriever extends LuceneTestCase {
}
};
+ Predicate<String> fieldsToLoadUnconditionally = fieldName -> false;
+ Predicate<String> fieldsToLoadIfWithHits = fieldName -> true;
MatchRegionRetriever highlighter =
- new MatchRegionRetriever(searcher, rewrittenQuery, offsetsStrategySupplier);
- highlighter.highlightDocuments(topDocs, highlightCollector);
+ new MatchRegionRetriever(
+ searcher,
+ rewrittenQuery,
+ offsetsStrategySupplier,
+ fieldsToLoadUnconditionally,
+ fieldsToLoadIfWithHits);
+ int maxBlocksProcessedInParallel = RandomizedTest.randomIntBetween(1, 5);
+ int maxBlockSize = RandomizedTest.randomFrom(new int[] {2, 10, 100});
+ highlighter.highlightDocuments(
+ Arrays.stream(topDocs.scoreDocs).mapToInt(scoreDoc -> scoreDoc.doc).sorted().iterator(),
+ highlightCollector,
+ field -> Integer.MAX_VALUE,
+ maxBlockSize,
+ maxBlocksProcessedInParallel);
return highlights;
}