Apache Lucene open-source search software
Make MaxScoreBulkScorer repartition scorers when the min competitive increases. (#13800)
Adrien Grandcf29597fecb8
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex 48a7e784481..5d9343bca2a 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -47,8 +47,12 @@ ImprovementsOptimizations---------------------+* GITHUB#13828: Reduce long[] array allocation for bitset in readBitSetIterator. (Zhang Chao)+* GITHUB#13800: MaxScoreBulkScorer now recomputes scorer partitions when the+ minimum competitive allows for a more favorable partitioning. (Adrien Grand)+Bug Fixes---------------------* GITHUB#13832: Fixed an issue where the DefaultPassageFormatter.format method did not format passages as intendeddiff --git a/lucene/core/src/java/org/apache/lucene/search/MaxScoreBulkScorer.java b/lucene/core/src/java/org/apache/lucene/search/MaxScoreBulkScorer.javaindex 8786343ccec..18f5b83e93a 100644--- a/lucene/core/src/java/org/apache/lucene/search/MaxScoreBulkScorer.java+++ b/lucene/core/src/java/org/apache/lucene/search/MaxScoreBulkScorer.java@@ -40,6 +40,8 @@ final class MaxScoreBulkScorer extends BulkScorer {// Index of the first scorer that is required, this scorer and all following scorers are required// for a document to match.int firstRequiredScorer;+ // The minimum value of minCompetitiveScore that would produce a more favorable partitioning.+ float nextMinCompetitiveScore;private final long cost;float minCompetitiveScore;private final Score scorable = new Score();@@ -114,9 +116,14 @@ final class MaxScoreBulkScorer extends BulkScorer {while (top.doc < outerWindowMax) {scoreInnerWindow(collector, acceptDocs, outerWindowMax);top = essentialQueue.top();+ if (minCompetitiveScore >= nextMinCompetitiveScore) {+ // The minimum competitive score increased substantially, so we can now partition scorers+ // in a more favorable way.+ break;+ }}- outerWindowMin = outerWindowMax;+ outerWindowMin = Math.min(top.doc, outerWindowMax);}return nextCandidate(max);@@ -337,6 +344,7 @@ final class MaxScoreBulkScorer extends BulkScorer {});double maxScoreSum = 0;firstEssentialScorer = 0;+ nextMinCompetitiveScore = Float.POSITIVE_INFINITY;for (int i = 0; i < allScorers.length; ++i) {final DisiWrapper w = scratch[i];double newMaxScoreSum = maxScoreSum + w.maxWindowScore;@@ -349,6 +357,7 @@ final class MaxScoreBulkScorer extends BulkScorer {firstEssentialScorer++;} else {allScorers[allScorers.length - 1 - (i - firstEssentialScorer)] = w;+ nextMinCompetitiveScore = Math.min(maxScoreSumFloat, nextMinCompetitiveScore);}}diff --git a/lucene/core/src/test/org/apache/lucene/search/TestMaxScoreBulkScorer.java b/lucene/core/src/test/org/apache/lucene/search/TestMaxScoreBulkScorer.javaindex c6920403c91..6973cc0025a 100644--- a/lucene/core/src/test/org/apache/lucene/search/TestMaxScoreBulkScorer.java+++ b/lucene/core/src/test/org/apache/lucene/search/TestMaxScoreBulkScorer.java@@ -38,23 +38,6 @@ import org.apache.lucene.util.Bits;// These basic tests are similar to some of the tests in TestWANDScorer, and may not need to be keptpublic class TestMaxScoreBulkScorer extends LuceneTestCase {- private static class CapMaxScoreWindowAt2048Scorer extends FilterScorer {-- public CapMaxScoreWindowAt2048Scorer(Scorer in) {- super(in);- }-- @Override- public int advanceShallow(int target) throws IOException {- return Math.min(target | 0x7FF, in.advanceShallow(target));- }-- @Override- public float getMaxScore(int upTo) throws IOException {- return in.getMaxScore(upTo);- }- }-private void writeDocuments(Directory dir) throws IOException {try (IndexWriter w =new IndexWriter(dir, newIndexWriterConfig().setMergePolicy(newLogMergePolicy()))) {@@ -96,12 +79,10 @@ public class TestMaxScoreBulkScorer extends LuceneTestCase {searcher.createWeight(searcher.rewrite(clause1), ScoreMode.TOP_SCORES, 1f).scorer(context);- scorer1 = new CapMaxScoreWindowAt2048Scorer(scorer1);Scorer scorer2 =searcher.createWeight(searcher.rewrite(clause2), ScoreMode.TOP_SCORES, 1f).scorer(context);- scorer2 = new CapMaxScoreWindowAt2048Scorer(scorer2);BulkScorer scorer =new MaxScoreBulkScorer(context.reader().maxDoc(), Arrays.asList(scorer1, scorer2));@@ -168,12 +149,10 @@ public class TestMaxScoreBulkScorer extends LuceneTestCase {searcher.createWeight(searcher.rewrite(clause1), ScoreMode.TOP_SCORES, 1f).scorer(context);- scorer1 = new CapMaxScoreWindowAt2048Scorer(scorer1);Scorer scorer2 =searcher.createWeight(searcher.rewrite(clause2), ScoreMode.TOP_SCORES, 1f).scorer(context);- scorer2 = new CapMaxScoreWindowAt2048Scorer(scorer2);BulkScorer scorer =new MaxScoreBulkScorer(context.reader().maxDoc(), Arrays.asList(scorer1, scorer2));@@ -237,17 +216,14 @@ public class TestMaxScoreBulkScorer extends LuceneTestCase {searcher.createWeight(searcher.rewrite(clause1), ScoreMode.TOP_SCORES, 1f).scorer(context);- scorer1 = new CapMaxScoreWindowAt2048Scorer(scorer1);Scorer scorer2 =searcher.createWeight(searcher.rewrite(clause2), ScoreMode.TOP_SCORES, 1f).scorer(context);- scorer2 = new CapMaxScoreWindowAt2048Scorer(scorer2);Scorer scorer3 =searcher.createWeight(searcher.rewrite(clause3), ScoreMode.TOP_SCORES, 1f).scorer(context);- scorer3 = new CapMaxScoreWindowAt2048Scorer(scorer3);BulkScorer scorer =new MaxScoreBulkScorer(@@ -317,17 +293,14 @@ public class TestMaxScoreBulkScorer extends LuceneTestCase {searcher.createWeight(searcher.rewrite(clause1), ScoreMode.TOP_SCORES, 1f).scorer(context);- scorer1 = new CapMaxScoreWindowAt2048Scorer(scorer1);Scorer scorer2 =searcher.createWeight(searcher.rewrite(clause2), ScoreMode.TOP_SCORES, 1f).scorer(context);- scorer2 = new CapMaxScoreWindowAt2048Scorer(scorer2);Scorer scorer3 =searcher.createWeight(searcher.rewrite(clause3), ScoreMode.TOP_SCORES, 1f).scorer(context);- scorer3 = new CapMaxScoreWindowAt2048Scorer(scorer3);BulkScorer scorer =new MaxScoreBulkScorer(