Skip to content

Apache Lucene open-source search software

Don't let merged passages push out lower-scoring ones (#11990)

Alan Woodward72ff140f5a65
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 529b07de390..0c3acec854c 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -166,6 +166,10 @@ Bug Fixes
* GITHUB#11950: Fix NPE in BinaryRangeFieldRangeQuery variants when the queried field doesn't exist
in a segment or is of the wrong type. (Greg Miller)
+* GITHUB#11990: PassageSelector now has a larger minimum size for its priority queue,
+ so that subsequent passage merges don't mean that we return too few passages in
+ total. (Alan Woodward, Dawid Weiss)
+
Optimizations
---------------------
* GITHUB#11738: Optimize MultiTermQueryConstantScoreWrapper when a term is present that matches all
diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/PassageSelector.java b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/PassageSelector.java
index cdb07d9e611..05cba2f1b1e 100644
--- a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/PassageSelector.java
+++ b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/PassageSelector.java
@@ -88,9 +88,13 @@ public class PassageSelector {
return Collections.emptyList();
}
+ // minimum priority queue size of 16 so that small maxPassages values don't
+ // return too few passages due to subsequent passage merges
+ int pqSize = Math.max(16, maxPassages);
+
// Best passages so far.
PriorityQueue<Passage> pq =
- new PriorityQueue<>(maxPassages) {
+ new PriorityQueue<>(pqSize) {
@Override
protected boolean lessThan(Passage a, Passage b) {
return passageScorer.compare(a, b) < 0;
@@ -215,6 +219,7 @@ public class PassageSelector {
}
// Remove nullified slots.
+ last = Math.min(last, maxPassages);
if (passages.length != last) {
passages = ArrayUtil.copyOfSubArray(passages, 0, last);
}
diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/matchhighlight/TestPassageSelector.java b/lucene/highlighter/src/test/org/apache/lucene/search/matchhighlight/TestPassageSelector.java
index 95fa8730991..55e58a86f94 100644
--- a/lucene/highlighter/src/test/org/apache/lucene/search/matchhighlight/TestPassageSelector.java
+++ b/lucene/highlighter/src/test/org/apache/lucene/search/matchhighlight/TestPassageSelector.java
@@ -103,6 +103,18 @@ public class TestPassageSelector extends LuceneTestCase {
checkPassages("...>123456<...", value, 6, 1, new OffsetRange(1, value.length()));
}
+ @Test
+ public void overlappingResultsDontPushOutLowerScoringResults() {
+ String value =
+ "a fair amount of space and then two matches here and then two big long spaces for matches";
+ checkPassages(
+ "...and then >two< >matches<...|...and then >two< big lon...",
+ value,
+ 20,
+ 2,
+ ranges(new OffsetRange(32, 35), new OffsetRange(36, 43), new OffsetRange(58, 61)));
+ }
+
@Test
public void highlightLargerThanWindowWithSubranges() {
String value = "0123456789a";