Skip to content

Apache Lucene open-source search software

UnifiedHighlighter highlight on multiple fields (#13268)

Mayya Sharipova0345fcabb3ce
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 1b68f73b8f6..8f6b1aadfa1 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -249,6 +249,10 @@ New Features
* GITHUB#13197: Expand support for new scalar bit levels for HNSW vectors. This includes 4-bit vectors and an option
to compress them to gain a 50% reduction in memory usage. (Ben Trent)
+* GITHUB#13268: Add ability for UnifiedHighlighter to highlight a field based on combined matches from multiple fields.
+ (Mayya Sharipova, Jim Ferenczi)
+
+
Improvements
---------------------
diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/MultiFieldsOffsetStrategy.java b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/MultiFieldsOffsetStrategy.java
new file mode 100644
index 00000000000..94331f7770e
--- /dev/null
+++ b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/MultiFieldsOffsetStrategy.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.search.uhighlight;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.lucene.index.LeafReader;
+
+/**
+ * FieldOffsetStrategy that combines offsets from multiple fields. Used to highlight a single field
+ * based on matches from multiple fields.
+ *
+ * @lucene.internal
+ */
+public class MultiFieldsOffsetStrategy extends FieldOffsetStrategy {
+ private final List<FieldOffsetStrategy> fieldsOffsetStrategies;
+
+ public MultiFieldsOffsetStrategy(List<FieldOffsetStrategy> fieldsOffsetStrategies) {
+ super(null);
+ this.fieldsOffsetStrategies = fieldsOffsetStrategies;
+ }
+
+ @Override
+ public String getField() {
+ throw new IllegalStateException("MultiFieldsOffsetStrategy does not have a single field.");
+ }
+
+ @Override
+ public UnifiedHighlighter.OffsetSource getOffsetSource() {
+ // TODO: what should be returned here as offset source?
+ return fieldsOffsetStrategies.getFirst().getOffsetSource();
+ }
+
+ @Override
+ public OffsetsEnum getOffsetsEnum(LeafReader reader, int docId, String content)
+ throws IOException {
+ List<OffsetsEnum> fieldsOffsetsEnums = new ArrayList<>(fieldsOffsetStrategies.size());
+ for (FieldOffsetStrategy fieldOffsetStrategy : fieldsOffsetStrategies) {
+ OffsetsEnum offsetsEnum = fieldOffsetStrategy.getOffsetsEnum(reader, docId, content);
+ if (offsetsEnum != OffsetsEnum.EMPTY) {
+ fieldsOffsetsEnums.add(offsetsEnum);
+ }
+ }
+ return new OffsetsEnum.MultiOffsetsEnum(fieldsOffsetsEnums);
+ }
+}
diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/UnifiedHighlighter.java b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/UnifiedHighlighter.java
index dfedf7974fd..69065933d7c 100644
--- a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/UnifiedHighlighter.java
+++ b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/UnifiedHighlighter.java
@@ -31,6 +31,7 @@ import java.util.Objects;
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;
+import java.util.function.Function;
import java.util.function.Predicate;
import java.util.function.Supplier;
import org.apache.lucene.analysis.Analyzer;
@@ -122,6 +123,8 @@ public class UnifiedHighlighter {
private Predicate<String> fieldMatcher;
+ private final Function<String, Set<String>> maskedFieldsFunc;
+
private Set<HighlightFlag> flags;
// e.g. wildcards
@@ -162,6 +165,7 @@ public class UnifiedHighlighter {
Objects.requireNonNull(
indexAnalyzer,
"indexAnalyzer is required" + " (even if in some circumstances it isn't used)");
+ this.maskedFieldsFunc = null;
}
@Deprecated
@@ -256,6 +260,8 @@ public class UnifiedHighlighter {
private final Analyzer indexAnalyzer;
private Predicate<String> fieldMatcher;
+
+ private Function<String, Set<String>> maskedFieldsFunc;
private Set<HighlightFlag> flags;
private boolean handleMultiTermQuery = DEFAULT_ENABLE_MULTI_TERM_QUERY;
private boolean highlightPhrasesStrictly = DEFAULT_ENABLE_HIGHLIGHT_PHRASES_STRICTLY;
@@ -360,6 +366,22 @@ public class UnifiedHighlighter {
return this;
}
+ /**
+ * Set up a function that given a field retuns a set of masked fields whose matches are combined
+ * to highlight the given field. Masked fields should not include the original field. This is
+ * useful when you want to highlight a field based on matches from several fields.
+ *
+ * <p>Note: All masked fields must share the same source as the field being highlighted,
+ * otherwise their offsets will not correspond to the highlighted field.
+ *
+ * <p>Note: Only the field being highlighted must provide an original source value (e.g. through
+ * stored field), other masked fields don't need it.
+ */
+ public Builder withMaskedFieldsFunc(Function<String, Set<String>> maskedFieldsFunc) {
+ this.maskedFieldsFunc = maskedFieldsFunc;
+ return this;
+ }
+
public Builder withScorer(PassageScorer value) {
this.scorer = value;
return this;
@@ -436,6 +458,7 @@ public class UnifiedHighlighter {
this.maxLength = builder.maxLength;
this.breakIterator = builder.breakIterator;
this.fieldMatcher = builder.fieldMatcher;
+ this.maskedFieldsFunc = builder.maskedFieldsFunc;
this.scorer = builder.scorer;
this.formatter = builder.formatter;
this.maxNoHighlightPassages = builder.maxNoHighlightPassages;
@@ -543,6 +566,10 @@ public class UnifiedHighlighter {
}
}
+ protected Set<String> getMaskedFields(String field) {
+ return maskedFieldsFunc == null ? null : maskedFieldsFunc.apply(field);
+ }
+
/** Returns the {@link HighlightFlag}s applicable for the current UH instance. */
protected Set<HighlightFlag> getFlags(String field) {
// If a builder is used for initializing a UH object, then flags will never be null.
@@ -1065,11 +1092,29 @@ public class UnifiedHighlighter {
protected FieldHighlighter getFieldHighlighter(
String field, Query query, Set<Term> allTerms, int maxPassages) {
- UHComponents components = getHighlightComponents(field, query, allTerms);
- OffsetSource offsetSource = getOptimizedOffsetSource(components);
+ Set<String> maskedFields = getMaskedFields(field);
+ FieldOffsetStrategy fieldOffsetStrategy;
+ if (maskedFields == null || maskedFields.isEmpty()) {
+ UHComponents components = getHighlightComponents(field, query, allTerms);
+ OffsetSource offsetSource = getOptimizedOffsetSource(components);
+ fieldOffsetStrategy = getOffsetStrategy(offsetSource, components);
+ } else {
+ List<FieldOffsetStrategy> fieldsOffsetStrategies = new ArrayList<>(maskedFields.size() + 1);
+ for (String maskedField : maskedFields) {
+ UHComponents components = getHighlightComponents(maskedField, query, allTerms);
+ OffsetSource offsetSource = getOptimizedOffsetSource(components);
+ fieldsOffsetStrategies.add(getOffsetStrategy(offsetSource, components));
+ }
+ // adding original field as well
+ UHComponents components = getHighlightComponents(field, query, allTerms);
+ OffsetSource offsetSource = getOptimizedOffsetSource(components);
+ fieldsOffsetStrategies.add(getOffsetStrategy(offsetSource, components));
+
+ fieldOffsetStrategy = new MultiFieldsOffsetStrategy(fieldsOffsetStrategies);
+ }
return newFieldHighlighter(
field,
- getOffsetStrategy(offsetSource, components),
+ fieldOffsetStrategy,
new SplittingBreakIterator(getBreakIterator(field), UnifiedHighlighter.MULTIVAL_SEP_CHAR),
getScorer(field),
maxPassages,
diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestUnifiedHighlighter.java b/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestUnifiedHighlighter.java
index 091af3595f1..01d5420522f 100644
--- a/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestUnifiedHighlighter.java
+++ b/lucene/highlighter/src/test/org/apache/lucene/search/uhighlight/TestUnifiedHighlighter.java
@@ -30,13 +30,20 @@ import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
+import java.util.TreeMap;
+import java.util.function.Function;
import java.util.function.Predicate;
import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.DelegatingAnalyzerWrapper;
+import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
+import org.apache.lucene.analysis.en.EnglishAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
+import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
@@ -54,8 +61,13 @@ import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.search.Weight;
import org.apache.lucene.search.uhighlight.UnifiedHighlighter.HighlightFlag;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.tests.analysis.MockAnalyzer;
import org.apache.lucene.tests.index.RandomIndexWriter;
+import org.apache.lucene.util.QueryBuilder;
import org.apache.lucene.util.automaton.Automata;
+import org.apache.lucene.util.automaton.CharacterRunAutomaton;
+import org.apache.lucene.util.automaton.RegExp;
public class TestUnifiedHighlighter extends UnifiedHighlighterTestBase {
@ParametersFactory
@@ -1337,6 +1349,121 @@ public class TestUnifiedHighlighter extends UnifiedHighlighterTestBase {
ir.close();
}
+ public void testMaskedFields() throws IOException {
+ final Map<String, Analyzer> fieldAnalyzers = new TreeMap<>();
+ fieldAnalyzers.put("field", new WhitespaceAnalyzer());
+ fieldAnalyzers.put("field_english", new EnglishAnalyzer()); // English stemming and stopwords
+ fieldAnalyzers.put( // Each letter is a token
+ "field_characters",
+ new MockAnalyzer(random(), new CharacterRunAutomaton(new RegExp(".").toAutomaton()), true));
+ fieldAnalyzers.put( // Every three letters is a token
+ "field_tripples",
+ new MockAnalyzer(
+ random(), new CharacterRunAutomaton(new RegExp("...").toAutomaton()), true));
+ Analyzer analyzer =
+ new DelegatingAnalyzerWrapper(Analyzer.PER_FIELD_REUSE_STRATEGY) {
+ @Override
+ public Analyzer getWrappedAnalyzer(String fieldName) {
+ return fieldAnalyzers.get(fieldName);
+ }
+ };
+ FieldType fieldTypeMatched = new FieldType(fieldType);
+ fieldTypeMatched.setStored(false); // matched fields don't need to be stored
+ fieldTypeMatched.freeze();
+
+ try (Directory dir = newDirectory()) {
+ try (IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(analyzer))) {
+ Document doc = new Document();
+ doc.add(new Field("field", "dance with star", fieldType));
+ doc.add(new Field("field_english", "dance with star", fieldTypeMatched));
+ doc.add(new Field("field_characters", "dance with star", fieldTypeMatched));
+ doc.add(new Field("field_tripples", "dance with star", fieldTypeMatched));
+ writer.addDocument(doc);
+ }
+
+ try (IndexReader reader = DirectoryReader.open(dir)) {
+ IndexSearcher searcher = newSearcher(reader);
+ // field is highlighted based on the matches from the "field_english"
+ maskedFieldsTestCase(
+ analyzer,
+ searcher,
+ "field",
+ Set.of("field_english"),
+ "dancing with the stars",
+ "<b>dance with star</b>",
+ "<b>dance</b> with <b>star</b>");
+
+ // field is highlighted based on the matches from the "field_characters"
+ maskedFieldsTestCase(
+ analyzer,
+ searcher,
+ "field",
+ Set.of("field_characters"),
+ "danc",
+ "<b>danc</b>e with star",
+ "<b>d</b><b>a</b><b>n</b><b>c</b>e with star");
+
+ // field is highlighted based on the matches from the "field_tripples"
+ maskedFieldsTestCase(
+ analyzer,
+ searcher,
+ "field",
+ Set.of("field_tripples"),
+ "danc",
+ "<b>dan</b>ce with star",
+ "<b>dan</b>ce with star");
+
+ // field is highlighted based on the matches from the "field_characters" and
+ // "field_tripples"
+ maskedFieldsTestCase(
+ analyzer,
+ searcher,
+ "field",
+ Set.of("field_tripples", "field_characters"),
+ "danc",
+ "<b>danc</b>e with star",
+ "<b>da</b><b>n</b><b>c</b>e with star");
+ }
+ }
+ }
+
+ private static void maskedFieldsTestCase(
+ Analyzer analyzer,
+ IndexSearcher searcher,
+ String field,
+ Set<String> maskedFields,
+ String queryText,
+ String expectedSnippetWithWeightMatches,
+ String expectedSnippetWithoutWeightMatches)
+ throws IOException {
+ QueryBuilder queryBuilder = new QueryBuilder(analyzer);
+ BooleanQuery.Builder boolQueryBuilder = new BooleanQuery.Builder();
+ Query fieldPhraseQuery = queryBuilder.createPhraseQuery(field, queryText, 2);
+ boolQueryBuilder.add(fieldPhraseQuery, BooleanClause.Occur.SHOULD);
+ for (String maskedField : maskedFields) {
+ fieldPhraseQuery = queryBuilder.createPhraseQuery(maskedField, queryText, 2);
+ boolQueryBuilder.add(fieldPhraseQuery, BooleanClause.Occur.SHOULD);
+ }
+ Query query = boolQueryBuilder.build();
+ TopDocs topDocs = searcher.search(query, 10);
+ assertEquals(1, topDocs.totalHits.value);
+
+ Function<String, Set<String>> maskedFieldsFunc =
+ fieldName -> fieldName.equals(field) ? maskedFields : Collections.emptySet();
+ UnifiedHighlighter.Builder uhBuilder =
+ new UnifiedHighlighter.Builder(searcher, analyzer).withMaskedFieldsFunc(maskedFieldsFunc);
+ UnifiedHighlighter highlighter =
+ randomUnifiedHighlighter(
+ uhBuilder, EnumSet.of(HighlightFlag.PHRASES), random().nextBoolean());
+ String[] snippets = highlighter.highlight(field, query, topDocs, 10);
+ String expectedSnippet =
+ highlighter.getFlags(field).contains(HighlightFlag.WEIGHT_MATCHES)
+ ? expectedSnippetWithWeightMatches
+ : expectedSnippetWithoutWeightMatches;
+ assertEquals(1, snippets.length);
+ assertEquals(expectedSnippet, snippets[0]);
+ }
+
public void testMatchesSlopBug() throws IOException {
IndexReader ir = indexSomeFields();
IndexSearcher searcher = newSearcher(ir);