Apache Lucene open-source search software
Don't try to highlight very long terms (#11808)
Alan Woodward188a78d76975
diff --git a/lucene/core/src/java/org/apache/lucene/util/automaton/DaciukMihovAutomatonBuilder.java b/lucene/core/src/java/org/apache/lucene/util/automaton/DaciukMihovAutomatonBuilder.javaindex 26952b678e3..d96a88904d0 100644--- a/lucene/core/src/java/org/apache/lucene/util/automaton/DaciukMihovAutomatonBuilder.java+++ b/lucene/core/src/java/org/apache/lucene/util/automaton/DaciukMihovAutomatonBuilder.java@@ -35,7 +35,7 @@ public final class DaciukMihovAutomatonBuilder {* This builder rejects terms that are more than 1k chars long since it then uses recursion based* on the length of the string, which might cause stack overflows.*/- static final int MAX_TERM_LENGTH = 1_000;+ public static final int MAX_TERM_LENGTH = 1_000;/** The default constructor is private. Use static methods directly. */private DaciukMihovAutomatonBuilder() {diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/MemoryIndexOffsetStrategy.java b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/MemoryIndexOffsetStrategy.javaindex 79dbcbea44b..7237be57807 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/MemoryIndexOffsetStrategy.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/MemoryIndexOffsetStrategy.java@@ -28,6 +28,8 @@ import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;import org.apache.lucene.index.LeafReader;import org.apache.lucene.index.memory.MemoryIndex;import org.apache.lucene.queries.spans.SpanQuery;+import org.apache.lucene.util.BytesRef;+import org.apache.lucene.util.automaton.DaciukMihovAutomatonBuilder;/*** Uses an {@link Analyzer} on content to get offsets and then populates a {@link MemoryIndex}.@@ -60,7 +62,13 @@ public class MemoryIndexOffsetStrategy extends AnalysisOffsetStrategy {List<CharArrayMatcher> allAutomata = new ArrayList<>();if (components.getTerms().length > 0) {- allAutomata.add(CharArrayMatcher.fromTerms(Arrays.asList(components.getTerms())));+ // Filter out any long terms that would otherwise cause exceptions if we tried+ // to build an automaton on them+ List<BytesRef> filteredTerms =+ Arrays.stream(components.getTerms())+ .filter(b -> b.length < DaciukMihovAutomatonBuilder.MAX_TERM_LENGTH)+ .toList();+ allAutomata.add(CharArrayMatcher.fromTerms(filteredTerms));}Collections.addAll(allAutomata, components.getAutomata());for (SpanQuery spanQuery : components.getPhraseHelper().getSpanQueries()) {diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestUnifiedHighlighter.java b/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestUnifiedHighlighter.javaindex 482644d4243..6bad8ddbe5e 100644--- a/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestUnifiedHighlighter.java+++ b/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestUnifiedHighlighter.java@@ -58,6 +58,7 @@ import org.apache.lucene.tests.analysis.MockAnalyzer;import org.apache.lucene.tests.analysis.MockTokenizer;import org.apache.lucene.tests.index.RandomIndexWriter;import org.apache.lucene.tests.util.LuceneTestCase;+import org.apache.lucene.util.automaton.DaciukMihovAutomatonBuilder;import org.junit.After;import org.junit.Before;@@ -1659,4 +1660,32 @@ public class TestUnifiedHighlighter extends LuceneTestCase {ir.close();}++ public void testQueryWithLongTerm() throws IOException {+ IndexReader ir = indexSomeFields();+ IndexSearcher searcher = newSearcher(ir);+ UnifiedHighlighter highlighter =+ randomUnifiedHighlighter(+ searcher, indexAnalyzer, EnumSet.of(HighlightFlag.WEIGHT_MATCHES), true);++ Query query =+ new BooleanQuery.Builder()+ .add(+ new TermQuery(+ new Term("title", "a".repeat(DaciukMihovAutomatonBuilder.MAX_TERM_LENGTH))),+ BooleanClause.Occur.SHOULD)+ .add(+ new TermQuery(+ new Term("title", "a".repeat(DaciukMihovAutomatonBuilder.MAX_TERM_LENGTH + 1))),+ BooleanClause.Occur.SHOULD)+ .add(new TermQuery(new Term("title", "title")), BooleanClause.Occur.SHOULD)+ .build();++ TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);++ String[] snippets = highlighter.highlight("title", query, topDocs);+ assertArrayEquals(new String[] {"This is the <b>title</b> field."}, snippets);++ ir.close();+ }}