Apache Lucene open-source search software
Sometimes intersect the essential clause and the best non-essential clause. (#12589)
Adrien Grand611bbbd9512d
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex b21c6d807da..eea75bc7104 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -101,6 +101,9 @@ Optimizations* GITHUB#12552: Make FSTPostingsFormat load FSTs off-heap. (Tony X)+* GITHUB#12589: Disjunctions now sometimes run as conjunctions when the minimum+ competitive score requires multiple clauses to match. (Adrien Grand)+Bug Fixes---------------------diff --git a/lucene/core/src/java/org/apache/lucene/search/MaxScoreBulkScorer.java b/lucene/core/src/java/org/apache/lucene/search/MaxScoreBulkScorer.javaindex 3c3f3db7491..026bf1f7d53 100644--- a/lucene/core/src/java/org/apache/lucene/search/MaxScoreBulkScorer.java+++ b/lucene/core/src/java/org/apache/lucene/search/MaxScoreBulkScorer.java@@ -30,19 +30,21 @@ final class MaxScoreBulkScorer extends BulkScorer {private final int maxDoc;// All scorers, sorted by increasing max score.- private final DisiWrapper[] allScorers;+ final DisiWrapper[] allScorers;private final DisiWrapper[] scratch;// These are the last scorers from `allScorers` that are "essential", ie. required for a match to// have a competitive score.private final DisiPriorityQueue essentialQueue;// Index of the first essential scorer, ie. essentialQueue contains all scorers from// allScorers[firstEssentialScorer:]. All scorers below this index are non-essential.- private int firstEssentialScorer;+ int firstEssentialScorer;+ // Index of the first scorer that is required, this scorer and all following scorers are required+ // for a document to match.+ int firstRequiredScorer;private final long cost;- private float minCompetitiveScore;- private boolean minCompetitiveScoreUpdated;+ float minCompetitiveScore;private Score scorable = new Score();- private final double[] maxScoreSums;+ final double[] maxScoreSums;private final long[] windowMatches = new long[FixedBitSet.bits2words(INNER_WINDOW_SIZE)];private final double[] windowScores = new double[INNER_WINDOW_SIZE];@@ -113,25 +115,8 @@ final class MaxScoreBulkScorer extends BulkScorer {while (top.doc < outerWindowMax) {scoreInnerWindow(collector, acceptDocs, outerWindowMax);top = essentialQueue.top();-- if (minCompetitiveScoreUpdated) {- minCompetitiveScoreUpdated = false;- if (partitionScorers() == false) {- outerWindowMin = outerWindowMax;- continue outer;- } else {- // Partitioning may have swapped essential and non-essential scorers, and some of the- // non-essential scorers may be behind the last scored doc. So let's advance to the next- // candidate match.- final int nextCandidateMatch = top.doc;- top = essentialQueue.top();- while (top.doc < nextCandidateMatch) {- top.doc = top.iterator.advance(nextCandidateMatch);- top = essentialQueue.updateTop();- }- }- }}+outerWindowMin = outerWindowMax;}@@ -140,17 +125,20 @@ final class MaxScoreBulkScorer extends BulkScorer {private void scoreInnerWindow(LeafCollector collector, Bits acceptDocs, int max)throws IOException {- DisiWrapper top = essentialQueue.top();-- DisiWrapper top2 = essentialQueue.top2();- if (top2 == null) {- scoreInnerWindowSingleEssentialClause(collector, acceptDocs, max);- } else if (top2.doc - INNER_WINDOW_SIZE / 2 >= top.doc) {- // The first half of the window would match a single clause. Let's collect this single clause- // until the next doc ID of the next clause.- scoreInnerWindowSingleEssentialClause(collector, acceptDocs, Math.min(max, top2.doc));+ if (allScorers.length - firstRequiredScorer >= 2) {+ scoreInnerWindowAsConjunction(collector, acceptDocs, max);} else {- scoreInnerWindowMultipleEssentialClauses(collector, acceptDocs, max);+ DisiWrapper top = essentialQueue.top();+ DisiWrapper top2 = essentialQueue.top2();+ if (top2 == null) {+ scoreInnerWindowSingleEssentialClause(collector, acceptDocs, max);+ } else if (top2.doc - INNER_WINDOW_SIZE / 2 >= top.doc) {+ // The first half of the window would match a single clause. Let's collect this single+ // clause until the next doc ID of the next clause.+ scoreInnerWindowSingleEssentialClause(collector, acceptDocs, Math.min(max, top2.doc));+ } else {+ scoreInnerWindowMultipleEssentialClauses(collector, acceptDocs, max);+ }}}@@ -164,17 +152,79 @@ final class MaxScoreBulkScorer extends BulkScorer {if (acceptDocs != null && acceptDocs.get(doc) == false) {continue;}- scoreNonEssentialClauses(collector, doc, top.scorer.score());- if (minCompetitiveScoreUpdated) {- // force scorers to be partitioned again before collecting more hits- top.iterator.nextDoc();- break;- }+ scoreNonEssentialClauses(collector, doc, top.scorer.score(), firstEssentialScorer);}top.doc = top.iterator.docID();essentialQueue.updateTop();}+ private void scoreInnerWindowAsConjunction(LeafCollector collector, Bits acceptDocs, int max)+ throws IOException {+ assert firstEssentialScorer == allScorers.length - 1;+ assert firstRequiredScorer <= allScorers.length - 2;+ DisiWrapper lead1 = allScorers[allScorers.length - 1];+ assert essentialQueue.size() == 1;+ assert lead1 == essentialQueue.top();+ DisiWrapper lead2 = allScorers[allScorers.length - 2];+ if (lead1.doc < lead2.doc) {+ lead1.doc = lead1.iterator.advance(Math.min(lead2.doc, max));+ }+ // maximum score contribution of all scorers but the lead+ double maxScoreSumAtLead2 = maxScoreSums[allScorers.length - 2];++ outer:+ while (lead1.doc < max) {++ if (acceptDocs != null && acceptDocs.get(lead1.doc) == false) {+ lead1.doc = lead1.iterator.nextDoc();+ continue;+ }++ double score = lead1.scorer.score();++ // We specialize handling the second best scorer, which seems to help a bit with performance.+ // But this is the exact same logic as in the below for loop.+ if ((float) MathUtil.sumUpperBound(score + maxScoreSumAtLead2, allScorers.length)+ < minCompetitiveScore) {+ // a competitive match is not possible according to max scores, skip to the next candidate+ lead1.doc = lead1.iterator.nextDoc();+ continue;+ }++ if (lead2.doc < lead1.doc) {+ lead2.doc = lead2.iterator.advance(lead1.doc);+ }+ if (lead2.doc != lead1.doc) {+ lead1.doc = lead1.iterator.advance(Math.min(lead2.doc, max));+ continue;+ }++ score += lead2.scorer.score();++ for (int i = allScorers.length - 3; i >= firstRequiredScorer; --i) {+ if ((float) MathUtil.sumUpperBound(score + maxScoreSums[i], allScorers.length)+ < minCompetitiveScore) {+ // a competitive match is not possible according to max scores, skip to the next candidate+ lead1.doc = lead1.iterator.nextDoc();+ continue outer;+ }++ DisiWrapper w = allScorers[i];+ if (w.doc < lead1.doc) {+ w.doc = w.iterator.advance(lead1.doc);+ }+ if (w.doc != lead1.doc) {+ lead1.doc = lead1.iterator.advance(Math.min(w.doc, max));+ continue outer;+ }+ score += w.scorer.score();+ }++ scoreNonEssentialClauses(collector, lead1.doc, score, firstRequiredScorer);+ lead1.doc = lead1.iterator.nextDoc();+ }+ }+private void scoreInnerWindowMultipleEssentialClauses(LeafCollector collector, Bits acceptDocs, int max) throws IOException {DisiWrapper top = essentialQueue.top();@@ -206,7 +256,7 @@ final class MaxScoreBulkScorer extends BulkScorer {double score = windowScores[index];windowScores[index] = 0d;- scoreNonEssentialClauses(collector, doc, score);+ scoreNonEssentialClauses(collector, doc, score, firstEssentialScorer);}}}@@ -230,7 +280,7 @@ final class MaxScoreBulkScorer extends BulkScorer {return windowMax;}- private void updateMaxWindowScores(int windowMin, int windowMax) throws IOException {+ void updateMaxWindowScores(int windowMin, int windowMax) throws IOException {for (DisiWrapper scorer : allScorers) {if (scorer.doc < windowMax) {if (scorer.doc < windowMin) {@@ -246,10 +296,11 @@ final class MaxScoreBulkScorer extends BulkScorer {}}- private void scoreNonEssentialClauses(LeafCollector collector, int doc, double essentialScore)+ private void scoreNonEssentialClauses(+ LeafCollector collector, int doc, double essentialScore, int numNonEssentialClauses)throws IOException {double score = essentialScore;- for (int i = firstEssentialScorer - 1; i >= 0; --i) {+ for (int i = numNonEssentialClauses - 1; i >= 0; --i) {float maxPossibleScore =(float) MathUtil.sumUpperBound(score + maxScoreSums[i], allScorers.length);if (maxPossibleScore < minCompetitiveScore) {@@ -270,7 +321,7 @@ final class MaxScoreBulkScorer extends BulkScorer {collector.collect(doc);}- private boolean partitionScorers() {+ boolean partitionScorers() {// Partitioning scorers is an optimization problem: the optimal set of non-essential scorers is// the subset of scorers whose sum of max window scores is less than the minimum competitive// score that maximizes the sum of costs.@@ -303,6 +354,8 @@ final class MaxScoreBulkScorer extends BulkScorer {}}+ firstRequiredScorer = allScorers.length;+if (firstEssentialScorer == allScorers.length) {return false;}@@ -311,6 +364,33 @@ final class MaxScoreBulkScorer extends BulkScorer {for (int i = firstEssentialScorer; i < allScorers.length; ++i) {essentialQueue.add(allScorers[i]);}++ if (firstEssentialScorer == allScorers.length - 1) { // single essential clause+ // If there is a single essential clause and matching it plus all non-essential clauses but+ // the best one is not enough to yield a competitive match, the we know that hits must match+ // both the essential clause and the best non-essential clause. Here are some examples when+ // this optimization would kick in:+ // `quick fox` when maxscore(quick) = 1, maxscore(fox) = 1, minCompetitiveScore = 1.5+ // `the quick fox` when maxscore (the) = 0.1, maxscore(quick) = 1, maxscore(fox) = 1,+ // minCompetitiveScore = 1.5+ firstRequiredScorer = allScorers.length - 1;+ double maxRequiredScore = allScorers[firstEssentialScorer].maxWindowScore;++ while (firstRequiredScorer > 0) {+ double maxPossibleScoreWithoutPreviousClause = maxRequiredScore;+ if (firstRequiredScorer > 1) {+ maxPossibleScoreWithoutPreviousClause += maxScoreSums[firstRequiredScorer - 2];+ }+ if ((float) maxPossibleScoreWithoutPreviousClause >= minCompetitiveScore) {+ break;+ }+ // The sum of maximum scores ignoring the previous clause is less than the minimum+ // competitive+ --firstRequiredScorer;+ maxRequiredScore += allScorers[firstRequiredScorer].maxWindowScore;+ }+ }+return true;}@@ -348,7 +428,6 @@ final class MaxScoreBulkScorer extends BulkScorer {@Overridepublic void setMinCompetitiveScore(float minScore) throws IOException {MaxScoreBulkScorer.this.minCompetitiveScore = minScore;- minCompetitiveScoreUpdated = true;}}}diff --git a/lucene/core/src/test/org/apache/lucene/search/TestMaxScoreBulkScorer.java b/lucene/core/src/test/org/apache/lucene/search/TestMaxScoreBulkScorer.javaindex 3ec87d14000..c000c82eb13 100644--- a/lucene/core/src/test/org/apache/lucene/search/TestMaxScoreBulkScorer.java+++ b/lucene/core/src/test/org/apache/lucene/search/TestMaxScoreBulkScorer.java@@ -18,19 +18,43 @@ package org.apache.lucene.search;import java.io.IOException;import java.util.Arrays;+import java.util.Collections;import org.apache.lucene.document.Document;import org.apache.lucene.document.Field;+import org.apache.lucene.document.Field.Store;import org.apache.lucene.document.StringField;import org.apache.lucene.index.DirectoryReader;import org.apache.lucene.index.IndexReader;import org.apache.lucene.index.IndexWriter;+import org.apache.lucene.index.IndexWriterConfig;+import org.apache.lucene.index.IndexableField;+import org.apache.lucene.index.LeafReaderContext;import org.apache.lucene.index.Term;+import org.apache.lucene.search.BooleanClause.Occur;import org.apache.lucene.store.Directory;import org.apache.lucene.tests.util.LuceneTestCase;+import org.apache.lucene.util.Bits;// These basic tests are similar to some of the tests in TestWANDScorer, and may not need to be keptpublic class TestMaxScoreBulkScorer extends LuceneTestCase {+ private static class CapMaxScoreWindowAt2048Scorer extends FilterScorer {++ public CapMaxScoreWindowAt2048Scorer(Scorer in) {+ super(in);+ }++ @Override+ public int advanceShallow(int target) throws IOException {+ return Math.min(target | 0x7FF, in.advanceShallow(target));+ }++ @Override+ public float getMaxScore(int upTo) throws IOException {+ return in.getMaxScore(upTo);+ }+ }+private void writeDocuments(Directory dir) throws IOException {try (IndexWriter w =new IndexWriter(dir, newIndexWriterConfig().setMergePolicy(newLogMergePolicy()))) {@@ -64,20 +88,23 @@ public class TestMaxScoreBulkScorer extends LuceneTestCase {try (IndexReader reader = DirectoryReader.open(dir)) {IndexSearcher searcher = newSearcher(reader);- Query query =- new BooleanQuery.Builder()- .add(- new BoostQuery(new ConstantScoreQuery(new TermQuery(new Term("foo", "A"))), 2),- BooleanClause.Occur.SHOULD)- .add(- new ConstantScoreQuery(new TermQuery(new Term("foo", "B"))),- BooleanClause.Occur.SHOULD)- .build();+ Query clause1 =+ new BoostQuery(new ConstantScoreQuery(new TermQuery(new Term("foo", "A"))), 2);+ Query clause2 = new ConstantScoreQuery(new TermQuery(new Term("foo", "B")));+ LeafReaderContext context = searcher.getIndexReader().leaves().get(0);+ Scorer scorer1 =+ searcher+ .createWeight(searcher.rewrite(clause1), ScoreMode.TOP_SCORES, 1f)+ .scorer(context);+ scorer1 = new CapMaxScoreWindowAt2048Scorer(scorer1);+ Scorer scorer2 =+ searcher+ .createWeight(searcher.rewrite(clause2), ScoreMode.TOP_SCORES, 1f)+ .scorer(context);+ scorer2 = new CapMaxScoreWindowAt2048Scorer(scorer2);BulkScorer scorer =- searcher- .createWeight(searcher.rewrite(query), ScoreMode.TOP_SCORES, 1)- .bulkScorer(searcher.getIndexReader().leaves().get(0));+ new MaxScoreBulkScorer(context.reader().maxDoc(), Arrays.asList(scorer1, scorer2));scorer.score(new LeafCollector() {@@ -131,20 +158,23 @@ public class TestMaxScoreBulkScorer extends LuceneTestCase {try (IndexReader reader = DirectoryReader.open(dir)) {IndexSearcher searcher = newSearcher(reader);- Query query =- new BooleanQuery.Builder()- .add(- new BoostQuery(new ConstantScoreQuery(new TermQuery(new Term("foo", "A"))), 2),- BooleanClause.Occur.SHOULD)- .add(- new ConstantScoreQuery(new TermQuery(new Term("foo", "B"))),- BooleanClause.Occur.SHOULD)- .build();+ Query clause1 =+ new BoostQuery(new ConstantScoreQuery(new TermQuery(new Term("foo", "A"))), 2);+ Query clause2 = new ConstantScoreQuery(new TermQuery(new Term("foo", "B")));+ LeafReaderContext context = searcher.getIndexReader().leaves().get(0);+ Scorer scorer1 =+ searcher+ .createWeight(searcher.rewrite(clause1), ScoreMode.TOP_SCORES, 1f)+ .scorer(context);+ scorer1 = new CapMaxScoreWindowAt2048Scorer(scorer1);+ Scorer scorer2 =+ searcher+ .createWeight(searcher.rewrite(clause2), ScoreMode.TOP_SCORES, 1f)+ .scorer(context);+ scorer2 = new CapMaxScoreWindowAt2048Scorer(scorer2);BulkScorer scorer =- searcher- .createWeight(searcher.rewrite(query), ScoreMode.TOP_SCORES, 1)- .bulkScorer(searcher.getIndexReader().leaves().get(0));+ new MaxScoreBulkScorer(context.reader().maxDoc(), Arrays.asList(scorer1, scorer2));scorer.score(new LeafCollector() {@@ -193,23 +223,31 @@ public class TestMaxScoreBulkScorer extends LuceneTestCase {try (IndexReader reader = DirectoryReader.open(dir)) {IndexSearcher searcher = newSearcher(reader);- Query query =- new BooleanQuery.Builder()- .add(- new BoostQuery(new ConstantScoreQuery(new TermQuery(new Term("foo", "A"))), 2),- BooleanClause.Occur.SHOULD)- .add(- new ConstantScoreQuery(new TermQuery(new Term("foo", "B"))),- BooleanClause.Occur.SHOULD)- .add(- new BoostQuery(new ConstantScoreQuery(new TermQuery(new Term("foo", "C"))), 3),- BooleanClause.Occur.SHOULD)- .build();+ Query clause1 =+ new BoostQuery(new ConstantScoreQuery(new TermQuery(new Term("foo", "A"))), 2);+ Query clause2 = new ConstantScoreQuery(new TermQuery(new Term("foo", "B")));+ Query clause3 =+ new BoostQuery(new ConstantScoreQuery(new TermQuery(new Term("foo", "C"))), 3);+ LeafReaderContext context = searcher.getIndexReader().leaves().get(0);+ Scorer scorer1 =+ searcher+ .createWeight(searcher.rewrite(clause1), ScoreMode.TOP_SCORES, 1f)+ .scorer(context);+ scorer1 = new CapMaxScoreWindowAt2048Scorer(scorer1);+ Scorer scorer2 =+ searcher+ .createWeight(searcher.rewrite(clause2), ScoreMode.TOP_SCORES, 1f)+ .scorer(context);+ scorer2 = new CapMaxScoreWindowAt2048Scorer(scorer2);+ Scorer scorer3 =+ searcher+ .createWeight(searcher.rewrite(clause3), ScoreMode.TOP_SCORES, 1f)+ .scorer(context);+ scorer3 = new CapMaxScoreWindowAt2048Scorer(scorer3);BulkScorer scorer =- searcher- .createWeight(searcher.rewrite(query), ScoreMode.TOP_SCORES, 1)- .bulkScorer(searcher.getIndexReader().leaves().get(0));+ new MaxScoreBulkScorer(+ context.reader().maxDoc(), Arrays.asList(scorer1, scorer2, scorer3));scorer.score(new LeafCollector() {@@ -263,23 +301,31 @@ public class TestMaxScoreBulkScorer extends LuceneTestCase {try (IndexReader reader = DirectoryReader.open(dir)) {IndexSearcher searcher = newSearcher(reader);- Query query =- new BooleanQuery.Builder()- .add(- new BoostQuery(new ConstantScoreQuery(new TermQuery(new Term("foo", "A"))), 2),- BooleanClause.Occur.SHOULD)- .add(- new ConstantScoreQuery(new TermQuery(new Term("foo", "B"))),- BooleanClause.Occur.SHOULD)- .add(- new BoostQuery(new ConstantScoreQuery(new TermQuery(new Term("foo", "C"))), 3),- BooleanClause.Occur.SHOULD)- .build();+ Query clause1 =+ new BoostQuery(new ConstantScoreQuery(new TermQuery(new Term("foo", "A"))), 2);+ Query clause2 = new ConstantScoreQuery(new TermQuery(new Term("foo", "B")));+ Query clause3 =+ new BoostQuery(new ConstantScoreQuery(new TermQuery(new Term("foo", "C"))), 3);+ LeafReaderContext context = searcher.getIndexReader().leaves().get(0);+ Scorer scorer1 =+ searcher+ .createWeight(searcher.rewrite(clause1), ScoreMode.TOP_SCORES, 1f)+ .scorer(context);+ scorer1 = new CapMaxScoreWindowAt2048Scorer(scorer1);+ Scorer scorer2 =+ searcher+ .createWeight(searcher.rewrite(clause2), ScoreMode.TOP_SCORES, 1f)+ .scorer(context);+ scorer2 = new CapMaxScoreWindowAt2048Scorer(scorer2);+ Scorer scorer3 =+ searcher+ .createWeight(searcher.rewrite(clause3), ScoreMode.TOP_SCORES, 1f)+ .scorer(context);+ scorer3 = new CapMaxScoreWindowAt2048Scorer(scorer3);BulkScorer scorer =- searcher- .createWeight(searcher.rewrite(query), ScoreMode.TOP_SCORES, 1)- .bulkScorer(searcher.getIndexReader().leaves().get(0));+ new MaxScoreBulkScorer(+ context.reader().maxDoc(), Arrays.asList(scorer1, scorer2, scorer3));scorer.score(new LeafCollector() {@@ -325,4 +371,311 @@ public class TestMaxScoreBulkScorer extends LuceneTestCase {}}}++ private static class FakeWeight extends Weight {++ protected FakeWeight() {+ super(null);+ }++ @Override+ public boolean isCacheable(LeafReaderContext ctx) {+ return false;+ }++ @Override+ public Explanation explain(LeafReaderContext context, int doc) throws IOException {+ throw new UnsupportedOperationException();+ }++ @Override+ public Scorer scorer(LeafReaderContext context) throws IOException {+ throw new UnsupportedOperationException();+ }+ }++ private static class FakeScorer extends Scorer {++ final String toString;+ int docID = -1;+ int maxScoreUpTo = DocIdSetIterator.NO_MORE_DOCS;+ float maxScore = 1f;+ int cost = 10;++ protected FakeScorer(String toString) {+ super(new FakeWeight());+ this.toString = toString;+ }++ @Override+ public int docID() {+ return docID;+ }++ @Override+ public DocIdSetIterator iterator() {+ return DocIdSetIterator.all(cost); // just so that it exposes the right cost+ }++ @Override+ public int advanceShallow(int target) throws IOException {+ return maxScoreUpTo;+ }++ @Override+ public float getMaxScore(int upTo) throws IOException {+ return maxScore;+ }++ @Override+ public float score() throws IOException {+ throw new UnsupportedOperationException();+ }++ @Override+ public String toString() {+ return toString;+ }+ }++ public void testDeletes() throws IOException {++ Directory dir = newDirectory();+ IndexWriterConfig iwc = newIndexWriterConfig().setMergePolicy(newLogMergePolicy());+ IndexWriter w = new IndexWriter(dir, iwc);+ Document doc1 = new Document();+ doc1.add(new StringField("field", "foo", Store.NO));+ doc1.add(new StringField("field", "bar", Store.NO));+ doc1.add(new StringField("field", "quux", Store.NO));+ Document doc2 = new Document();+ Document doc3 = new Document();+ for (IndexableField field : doc1) {+ doc2.add(field);+ doc3.add(field);+ }+ doc1.add(new StringField("id", "1", Store.NO));+ doc2.add(new StringField("id", "2", Store.NO));+ doc3.add(new StringField("id", "3", Store.NO));+ w.addDocument(doc1);+ w.addDocument(doc2);+ w.addDocument(doc3);++ w.forceMerge(1);++ IndexReader reader = DirectoryReader.open(w);+ w.close();++ Query query =+ new BooleanQuery.Builder()+ .add(+ new BoostQuery(new ConstantScoreQuery(new TermQuery(new Term("field", "foo"))), 1f),+ Occur.SHOULD)+ .add(+ new BoostQuery(+ new ConstantScoreQuery(new TermQuery(new Term("field", "bar"))), 1.5f),+ Occur.SHOULD)+ .add(+ new BoostQuery(+ new ConstantScoreQuery(new TermQuery(new Term("field", "quux"))), 0.1f),+ Occur.SHOULD)+ .build();++ IndexSearcher searcher = newSearcher(reader);+ Weight weight = searcher.createWeight(searcher.rewrite(query), ScoreMode.TOP_SCORES, 1f);++ Bits liveDocs =+ new Bits() {+ @Override+ public boolean get(int index) {+ return index == 1;+ }++ @Override+ public int length() {+ return 3;+ }+ };++ // Test min competitive scores that exercise different execution modes+ for (float minCompetitiveScore :+ new float[] {+ 0f, // 3 essential clauses+ 1f, // 2 essential clauses+ 1.2f, // 1 essential clause+ 2f // two required clauses+ }) {+ BulkScorer scorer = weight.bulkScorer(searcher.getIndexReader().leaves().get(0));+ LeafCollector collector =+ new LeafCollector() {++ int i = 0;++ @Override+ public void setScorer(Scorable scorer) throws IOException {+ scorer.setMinCompetitiveScore(minCompetitiveScore);+ }++ @Override+ public void collect(int doc) throws IOException {+ assertEquals(1, doc);+ assertEquals(0, i++);+ }++ @Override+ public void finish() throws IOException {+ assertEquals(1, i);+ }+ };+ scorer.score(collector, liveDocs);+ collector.finish();+ }++ reader.close();+ dir.close();+ }++ // This test simulates what happens over time for the query `the quick fox` as collection+ // progresses and the minimum competitive score increases.+ public void testPartition() throws IOException {+ FakeScorer the = new FakeScorer("the");+ the.cost = 9_000;+ the.maxScore = 0.1f;+ FakeScorer quick = new FakeScorer("quick");+ quick.cost = 1_000;+ quick.maxScore = 1f;+ FakeScorer fox = new FakeScorer("fox");+ fox.cost = 900;+ fox.maxScore = 1.1f;++ MaxScoreBulkScorer scorer = new MaxScoreBulkScorer(10_000, Arrays.asList(the, quick, fox));+ the.docID = 4;+ the.maxScoreUpTo = 130;+ quick.docID = 4;+ quick.maxScoreUpTo = 999;+ fox.docID = 10;+ fox.maxScoreUpTo = 1_200;++ Collections.shuffle(Arrays.asList(scorer.allScorers), random());+ scorer.updateMaxWindowScores(4, 100);+ assertTrue(scorer.partitionScorers());+ assertEquals(0, scorer.firstEssentialScorer); // all clauses are essential+ assertEquals(3, scorer.firstRequiredScorer); // no required clauses++ // less than the minimum score of every clause+ scorer.minCompetitiveScore = 0.09f;+ Collections.shuffle(Arrays.asList(scorer.allScorers), random());+ scorer.updateMaxWindowScores(4, 100);+ assertTrue(scorer.partitionScorers());+ assertEquals(0, scorer.firstEssentialScorer); // all clauses are still essential+ assertEquals(3, scorer.firstRequiredScorer); // no required clauses++ // equal to the maximum score of `the`+ scorer.minCompetitiveScore = 0.1f;+ Collections.shuffle(Arrays.asList(scorer.allScorers), random());+ scorer.updateMaxWindowScores(4, 100);+ assertTrue(scorer.partitionScorers());+ assertEquals(0, scorer.firstEssentialScorer); // all clauses are still essential+ assertEquals(3, scorer.firstRequiredScorer); // no required clauses++ // gt than the minimum score of `the`+ scorer.minCompetitiveScore = 0.11f;+ Collections.shuffle(Arrays.asList(scorer.allScorers), random());+ scorer.updateMaxWindowScores(4, 100);+ assertTrue(scorer.partitionScorers());+ assertEquals(1, scorer.firstEssentialScorer); // the is non essential+ assertEquals(3, scorer.firstRequiredScorer); // no required clauses+ assertSame(the, scorer.allScorers[0].scorer);++ // equal to the sum of the max scores of the and quick+ scorer.minCompetitiveScore = 1.1f;+ Collections.shuffle(Arrays.asList(scorer.allScorers), random());+ scorer.updateMaxWindowScores(4, 100);+ assertTrue(scorer.partitionScorers());+ assertEquals(1, scorer.firstEssentialScorer); // the is non essential+ assertEquals(3, scorer.firstRequiredScorer); // no required clauses+ assertSame(the, scorer.allScorers[0].scorer);++ // greater than the sum of the max scores of the and quick+ scorer.minCompetitiveScore = 1.11f;+ Collections.shuffle(Arrays.asList(scorer.allScorers), random());+ scorer.updateMaxWindowScores(4, 100);+ assertTrue(scorer.partitionScorers());+ assertEquals(2, scorer.firstEssentialScorer); // the and quick are non essential+ assertEquals(2, scorer.firstRequiredScorer); // fox is required+ assertSame(the, scorer.allScorers[0].scorer);+ assertSame(quick, scorer.allScorers[1].scorer);+ assertSame(fox, scorer.allScorers[2].scorer);++ // equal to the sum of the max scores of the and fox+ scorer.minCompetitiveScore = 1.2f;+ Collections.shuffle(Arrays.asList(scorer.allScorers), random());+ scorer.updateMaxWindowScores(4, 100);+ assertTrue(scorer.partitionScorers());+ assertEquals(2, scorer.firstEssentialScorer); // the and quick are non essential+ assertEquals(2, scorer.firstRequiredScorer); // fox is required+ assertSame(the, scorer.allScorers[0].scorer);+ assertSame(quick, scorer.allScorers[1].scorer);+ assertSame(fox, scorer.allScorers[2].scorer);++ // greater than the sum of the max scores of the and fox+ scorer.minCompetitiveScore = 1.21f;+ Collections.shuffle(Arrays.asList(scorer.allScorers), random());+ scorer.updateMaxWindowScores(4, 100);+ assertTrue(scorer.partitionScorers());+ assertEquals(2, scorer.firstEssentialScorer); // the and quick are non essential+ assertEquals(1, scorer.firstRequiredScorer); // quick and fox are required+ assertSame(the, scorer.allScorers[0].scorer);+ assertSame(quick, scorer.allScorers[1].scorer);+ assertSame(fox, scorer.allScorers[2].scorer);++ // equal to the sum of the max scores of quick and fox+ scorer.minCompetitiveScore = 2.1f;+ Collections.shuffle(Arrays.asList(scorer.allScorers), random());+ scorer.updateMaxWindowScores(4, 100);+ assertTrue(scorer.partitionScorers());+ assertEquals(2, scorer.firstEssentialScorer); // the and quick are non essential+ assertEquals(1, scorer.firstRequiredScorer); // quick and fox are required+ assertSame(the, scorer.allScorers[0].scorer);+ assertSame(quick, scorer.allScorers[1].scorer);+ assertSame(fox, scorer.allScorers[2].scorer);++ // greater than the sum of the max scores of quick and fox+ scorer.minCompetitiveScore = 2.11f;+ Collections.shuffle(Arrays.asList(scorer.allScorers), random());+ scorer.updateMaxWindowScores(4, 100);+ assertTrue(scorer.partitionScorers());+ assertEquals(2, scorer.firstEssentialScorer); // the and quick are non essential+ assertEquals(0, scorer.firstRequiredScorer); // all terms are required+ assertSame(the, scorer.allScorers[0].scorer);+ assertSame(quick, scorer.allScorers[1].scorer);+ assertSame(fox, scorer.allScorers[2].scorer);++ // greater than the sum of the max scores of quick and fox+ scorer.minCompetitiveScore = 2.11f;+ Collections.shuffle(Arrays.asList(scorer.allScorers), random());+ scorer.updateMaxWindowScores(4, 100);+ assertTrue(scorer.partitionScorers());+ assertEquals(2, scorer.firstEssentialScorer); // the and quick are non essential+ assertEquals(0, scorer.firstRequiredScorer); // all terms are required+ assertSame(the, scorer.allScorers[0].scorer);+ assertSame(quick, scorer.allScorers[1].scorer);+ assertSame(fox, scorer.allScorers[2].scorer);++ // equal to the sum of the max scores of all terms+ scorer.minCompetitiveScore = 2.2f;+ Collections.shuffle(Arrays.asList(scorer.allScorers), random());+ scorer.updateMaxWindowScores(4, 100);+ assertTrue(scorer.partitionScorers());+ assertEquals(2, scorer.firstEssentialScorer); // the and quick are non essential+ assertEquals(0, scorer.firstRequiredScorer); // all terms are required+ assertSame(the, scorer.allScorers[0].scorer);+ assertSame(quick, scorer.allScorers[1].scorer);+ assertSame(fox, scorer.allScorers[2].scorer);++ // greater than the sum of the max scores of all terms+ scorer.minCompetitiveScore = 2.21f;+ Collections.shuffle(Arrays.asList(scorer.allScorers), random());+ scorer.updateMaxWindowScores(4, 100);+ assertFalse(scorer.partitionScorers()); // no possible match in this window+ }}