Apache Lucene open-source search software
Respect minCompetitiveScore in BlockMaxConjunctionBulkScorer (#14751)
Guo Fengd242a403f9d1
diff --git a/lucene/core/src/java/org/apache/lucene/search/BlockMaxConjunctionBulkScorer.java b/lucene/core/src/java/org/apache/lucene/search/BlockMaxConjunctionBulkScorer.javaindex 31904eeff7e..0ea42029787 100644--- a/lucene/core/src/java/org/apache/lucene/search/BlockMaxConjunctionBulkScorer.java+++ b/lucene/core/src/java/org/apache/lucene/search/BlockMaxConjunctionBulkScorer.java@@ -22,6 +22,7 @@ import java.util.Comparator;import java.util.List;import org.apache.lucene.search.Weight.DefaultBulkScorer;import org.apache.lucene.util.Bits;+import org.apache.lucene.util.MathUtil;/*** BulkScorer implementation of {@link BlockMaxConjunctionScorer} that focuses on top-level@@ -34,6 +35,8 @@ import org.apache.lucene.util.Bits;*/final class BlockMaxConjunctionBulkScorer extends BulkScorer {+ private static final int MAX_WINDOW_SIZE = 65536;+private final Scorer[] scorers;private final Scorable[] scorables;private final DocIdSetIterator[] iterators;@@ -81,18 +84,18 @@ final class BlockMaxConjunctionBulkScorer extends BulkScorer {public int score(LeafCollector collector, Bits acceptDocs, int min, int max) throws IOException {collector.setScorer(scorable);- int windowMin = Math.max(lead.docID(), min);+ int windowMin = scoreDocFirstUntilDynamicPruning(collector, acceptDocs, min, max);+while (windowMin < max) {// Use impacts of the least costly scorer to compute windows// NOTE: windowMax is inclusiveint windowMax = Math.min(scorers[0].advanceShallow(windowMin), max - 1);+ // Ensure the scoring window not too big, this especially works for the default implementation+ // of `Scorer#advanceShallow` which may return `DocIdSetIterator#NO_MORE_DOCS`.+ windowMax = MathUtil.unsignedMin(windowMax, windowMin + MAX_WINDOW_SIZE);- if (0 < scorable.minCompetitiveScore) {- float maxWindowScore = computeMaxScore(windowMin, windowMax);- scoreWindowScoreFirst(collector, acceptDocs, windowMin, windowMax + 1, maxWindowScore);- } else {- scoreWindowDocFirst(collector, acceptDocs, windowMin, windowMax + 1);- }+ float maxWindowScore = computeMaxScore(windowMin, windowMax);+ scoreWindowScoreFirst(collector, acceptDocs, windowMin, windowMax + 1, maxWindowScore);windowMin = Math.max(lead.docID(), windowMax + 1);}@@ -100,11 +103,11 @@ final class BlockMaxConjunctionBulkScorer extends BulkScorer {}/**- * Score a window of doc IDs by first finding agreement between all iterators, and only then- * compute scores and call the collector.+ * Score a window of doc IDs by first finding agreement between all iterators and only then+ * compute scores and call the collector until dynamic pruning kicks in.*/- private void scoreWindowDocFirst(LeafCollector collector, Bits acceptDocs, int min, int max)- throws IOException {+ private int scoreDocFirstUntilDynamicPruning(+ LeafCollector collector, Bits acceptDocs, int min, int max) throws IOException {int doc = lead.docID();if (doc < min) {doc = lead.advance(min);@@ -131,9 +134,13 @@ final class BlockMaxConjunctionBulkScorer extends BulkScorer {}scorable.score = (float) score;collector.collect(doc);+ if (scorable.minCompetitiveScore > 0) {+ return lead.nextDoc();+ }}doc = lead.nextDoc();}+ return doc;}/**@@ -166,11 +173,13 @@ final class BlockMaxConjunctionBulkScorer extends BulkScorer {docAndScoreAccBuffer.copyFrom(docAndScoreBuffer);for (int i = 1; i < scorers.length; ++i) {- ScorerUtil.filterCompetitiveHits(- docAndScoreAccBuffer,- sumOfOtherClauses[i],- scorable.minCompetitiveScore,- scorers.length);+ double sumOfOtherClause = sumOfOtherClauses[i];+ if (sumOfOtherClause != sumOfOtherClauses[i - 1]) {+ // two equal consecutive values mean that the first clause always returns a score of zero,+ // so we don't need to filter hits by score again.+ ScorerUtil.filterCompetitiveHits(+ docAndScoreAccBuffer, sumOfOtherClause, scorable.minCompetitiveScore, scorers.length);+ }ScorerUtil.applyRequiredClause(docAndScoreAccBuffer, iterators[i], scorables[i]);}diff --git a/lucene/core/src/java/org/apache/lucene/search/Scorer.java b/lucene/core/src/java/org/apache/lucene/search/Scorer.javaindex 86207f5c18c..fc540c30cc4 100644--- a/lucene/core/src/java/org/apache/lucene/search/Scorer.java+++ b/lucene/core/src/java/org/apache/lucene/search/Scorer.java@@ -93,7 +93,7 @@ public abstract class Scorer extends Scorable {* <p>The default implementation is provided below:** <pre class="prettyprint">- * int batchSize = 16; // arbitrary+ * int batchSize = 64; // arbitrary* buffer.growNoCopy(batchSize);* int size = 0;* DocIdSetIterator iterator = iterator();@@ -117,7 +117,7 @@ public abstract class Scorer extends Scorable {*/public void nextDocsAndScores(int upTo, Bits liveDocs, DocAndFloatFeatureBuffer buffer)throws IOException {- int batchSize = 16; // arbitrary+ int batchSize = 64; // arbitrarybuffer.growNoCopy(batchSize);int size = 0;DocIdSetIterator iterator = iterator();diff --git a/lucene/core/src/java/org/apache/lucene/search/ScorerUtil.java b/lucene/core/src/java/org/apache/lucene/search/ScorerUtil.javaindex 19a42b5b29d..5c677d7a464 100644--- a/lucene/core/src/java/org/apache/lucene/search/ScorerUtil.java+++ b/lucene/core/src/java/org/apache/lucene/search/ScorerUtil.java@@ -116,11 +116,22 @@ class ScorerUtil {}}+ /**+ * Filters competitive hits from the provided {@link DocAndScoreAccBuffer}.+ *+ * <p>This method removes documents from the buffer that cannot possibly have a score competitive+ * enough to exceed the minimum competitive score, given the maximum remaining score and the+ * number of scorers.+ */static void filterCompetitiveHits(DocAndScoreAccBuffer buffer,double maxRemainingScore,float minCompetitiveScore,int numScorers) {+ if ((float) MathUtil.sumUpperBound(maxRemainingScore, numScorers) >= minCompetitiveScore) {+ return;+ }+int newSize = 0;for (int i = 0; i < buffer.size; ++i) {float maxPossibleScore =