Apache Lucene open-source search software
Apply filter before scoring in MaxScoreBulkScorer (#16515)
kkewweie2e9c7b05b7e
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex 00580cb83a1..c702f80ea5d 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -505,6 +505,8 @@ Optimizations* GITHUB#16588: AcceptDocs builds its bit set from the raw filter iterator so thatDocIdSetIterator#intoBitSet is used on segments with deletions. (John Maurice)+* GITHUB#16515: Apply filter before scoring in MaxScoreBulkScorer. (Ke Wei)+* GITHUB#16582: Rewrite sorted-numeric doc-values range queries that cover every value of a field to aFieldExistsQuery, and collapse an IndexOrDocValuesQuery whose two sides rewrite equally. (Houston Putman)diff --git a/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/MaxScoreBulkScorerFilterBenchmark.java b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/MaxScoreBulkScorerFilterBenchmark.javaindex 4a7e37b10da..34b8565c7d9 100644--- a/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/MaxScoreBulkScorerFilterBenchmark.java+++ b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/MaxScoreBulkScorerFilterBenchmark.java@@ -77,7 +77,7 @@ public class MaxScoreBulkScorerFilterBenchmark {public int numClauses;/** Filter density: 1.0 = match-all, 0.8 = 80% dense */- @Param({"1.0", "0.8"})+ @Param({"0.01", "0.05", "0.1", "0.2", "0.3", "0.8", "1.0"})public double filterDensity;private Directory dir;@@ -96,7 +96,7 @@ public class MaxScoreBulkScorerFilterBenchmark {// so multiple clauses often match the same doc, forcing PQ activity.for (int t = 0; t < numClauses; t++) {int period = 7 + t * 11; // 7, 18, 29, 40, ... ~14% avg density- int offset = (t * 7919) % period;+ int offset = (int) ((t * 7919L) % period);if ((i + offset) % period == 0) {doc.add(new TextField("text", "word" + t, Field.Store.NO));}@@ -106,7 +106,7 @@ public class MaxScoreBulkScorerFilterBenchmark {doc.add(new StringField("filter", "yes", Field.Store.NO));} else {int filterThreshold = (int) (filterDensity * 1000);- if ((i * 7919) % 1000 < filterThreshold) {+ if ((i * 7919L) % 1000 < filterThreshold) {doc.add(new StringField("filter", "yes", Field.Store.NO));} else {doc.add(new StringField("filter", "no", Field.Store.NO));diff --git a/lucene/core/src/java/org/apache/lucene/search/MaxScoreBulkScorer.java b/lucene/core/src/java/org/apache/lucene/search/MaxScoreBulkScorer.javaindex 8d517092bc8..bb39bca70ea 100644--- a/lucene/core/src/java/org/apache/lucene/search/MaxScoreBulkScorer.java+++ b/lucene/core/src/java/org/apache/lucene/search/MaxScoreBulkScorer.java@@ -49,7 +49,7 @@ final class MaxScoreBulkScorer extends BulkScorer {private final FixedBitSet windowMatches = new FixedBitSet(INNER_WINDOW_SIZE);private final double[] windowScores = new double[INNER_WINDOW_SIZE];- private FixedBitSet filterMatches = null;+ private OffsetBits filterMatchesBits = null;private final DocAndFloatFeatureBuffer docAndScoreBuffer = new DocAndFloatFeatureBuffer();private final DocAndScoreAccBuffer docAndScoreAccBuffer = new DocAndScoreAccBuffer();@@ -82,7 +82,7 @@ final class MaxScoreBulkScorer extends BulkScorer {// filter advance()if (minScorerCost >= this.filter.cost|| (allScorers.length > 4 && this.cost >= this.filter.cost)) {- this.filterMatches = new FixedBitSet(INNER_WINDOW_SIZE);+ this.filterMatchesBits = new OffsetBits(new FixedBitSet(INNER_WINDOW_SIZE), maxDoc);}}}@@ -216,7 +216,7 @@ final class MaxScoreBulkScorer extends BulkScorer {int innerWindowMax = MathUtil.unsignedMin(max, innerWindowMin + INNER_WINDOW_SIZE);docAndScoreAccBuffer.size = 0;- if (filterMatches == null) {+ if (filterMatchesBits == null) {fillScoreBufferViaLeapFrog(top, acceptDocs, innerWindowMax);} else {fillScoreBufferViaBitSet(top, acceptDocs, innerWindowMax);@@ -227,24 +227,25 @@ final class MaxScoreBulkScorer extends BulkScorer {private void fillScoreBufferViaBitSet(DisiWrapper top, Bits acceptDocs, int innerWindowMax)throws IOException {- filterMatches.clear();+ filterMatchesBits.bits.clear();int innerWindowMin = top.doc;if (filter.doc < innerWindowMax) {if (filter.doc < innerWindowMin) {filter.doc = filter.approximation.advance(innerWindowMin);}if (filter.doc < innerWindowMax) {- filter.approximation.intoBitSet(innerWindowMax, filterMatches, innerWindowMin);+ filter.approximation.intoBitSet(innerWindowMax, filterMatchesBits.bits, innerWindowMin);filter.doc = filter.approximation.docID();}}if (acceptDocs != null) {- acceptDocs.applyMask(filterMatches, innerWindowMin);+ acceptDocs.applyMask(filterMatchesBits.bits, innerWindowMin);}+ filterMatchesBits.setOffset(innerWindowMin);int innerWindowSize = innerWindowMax - innerWindowMin;// Collect matches of essential clauses into a bitset, checking filter via bitset lookup- collectEssentialScoresIntoWindow(top, innerWindowMax, innerWindowMin, null, filterMatches);+ collectEssentialScoresIntoWindow(top, innerWindowMax, innerWindowMin, filterMatchesBits);flushWindowToDocAndScoreAccBuffer(innerWindowMin, innerWindowSize);}@@ -290,25 +291,15 @@ final class MaxScoreBulkScorer extends BulkScorer {* caller is responsible for populating {@link #docAndScoreAccBuffer} from the window afterwards.** @param acceptDocs docs to accept, passed to {@link Scorer#nextDocsAndScores}- * @param filterMatches if non-null, only docs whose corresponding bit is set in this bitset will- * be collected; if null, all docs are collected*/private void collectEssentialScoresIntoWindow(- DisiWrapper top,- int innerWindowMax,- int innerWindowMin,- Bits acceptDocs,- FixedBitSet filterMatches)- throws IOException {+ DisiWrapper top, int innerWindowMax, int innerWindowMin, Bits acceptDocs) throws IOException {do {for (top.scorer.nextDocsAndScores(innerWindowMax, acceptDocs, docAndScoreBuffer);docAndScoreBuffer.size > 0;top.scorer.nextDocsAndScores(innerWindowMax, acceptDocs, docAndScoreBuffer)) {for (int index = 0; index < docAndScoreBuffer.size; ++index) {final int doc = docAndScoreBuffer.docs[index];- if (filterMatches != null && filterMatches.get(doc - innerWindowMin) == false) {- continue;- }final float score = docAndScoreBuffer.features[index];final int i = doc - innerWindowMin;windowMatches.set(i);@@ -366,12 +357,40 @@ final class MaxScoreBulkScorer extends BulkScorer {int innerWindowSize = innerWindowMax - innerWindowMin;// Collect matches of essential clauses into a bitset- collectEssentialScoresIntoWindow(top, innerWindowMax, innerWindowMin, acceptDocs, null);+ collectEssentialScoresIntoWindow(top, innerWindowMax, innerWindowMin, acceptDocs);flushWindowToDocAndScoreAccBuffer(innerWindowMin, innerWindowSize);scoreNonEssentialClauses(collector, docAndScoreAccBuffer, firstEssentialScorer);}+ /** A wrapper around {@link FixedBitSet} that supports setting an offset. */+ private static final class OffsetBits implements Bits {+ private final FixedBitSet bits;+ private final int maxDoc;+ private int offset;++ OffsetBits(FixedBitSet bits, int maxDoc) {+ this.bits = bits;+ this.maxDoc = maxDoc;+ }++ void setOffset(int offset) {+ assert offset < maxDoc;+ this.offset = offset;+ }++ @Override+ public boolean get(int index) {+ assert index >= 0 && offset <= index && index - offset < bits.length() && index < maxDoc;+ return bits.get(index - offset);+ }++ @Override+ public int length() {+ return maxDoc;+ }+ }+private int computeOuterWindowMax(int windowMin) throws IOException {// Only use essential scorers to compute the window's max doc ID, in order to avoid constantly// recomputing max scores over small windowsdiff --git a/lucene/core/src/test/org/apache/lucene/search/TestMaxScoreBulkScorer.java b/lucene/core/src/test/org/apache/lucene/search/TestMaxScoreBulkScorer.javaindex 834da692c97..e484f9ffbc0 100644--- a/lucene/core/src/test/org/apache/lucene/search/TestMaxScoreBulkScorer.java+++ b/lucene/core/src/test/org/apache/lucene/search/TestMaxScoreBulkScorer.java@@ -1167,6 +1167,7 @@ public class TestMaxScoreBulkScorer extends LuceneTestCase {IndexWriter w = new IndexWriter(dir, new IndexWriterConfig());for (int i = 0; i < 10000; i++) {Document doc = new Document();+ doc.add(new StringField("id", Integer.toString(i), Field.Store.NO));if (i % 2 == 0) {doc.add(new TextField("body", "dense1", Field.Store.NO));}@@ -1178,15 +1179,38 @@ public class TestMaxScoreBulkScorer extends LuceneTestCase {}w.addDocument(doc);}- w.close();+ w.commit();+ // Test without-delete pathsDirectoryReader reader = DirectoryReader.open(dir);+ assertEquals(10000, reader.numDocs());+ assertDenseScorersUseBitSet(reader, 500);++ // Test with-delete paths+ w.deleteDocuments(+ // first window+ new Term("id", "20"), new Term("id", "21"),+ // another window+ new Term("id", "4101"), new Term("id", "4102"));+ w.commit();+ DirectoryReader newReader = DirectoryReader.openIfChanged(reader);+ reader.close();+ assertEquals(9996, newReader.numDocs());+ assertDenseScorersUseBitSet(newReader, 498);+ newReader.close();+ w.close();+ dir.close();+ }++ private void assertDenseScorersUseBitSet(DirectoryReader reader, int expectedCollectedDocs)+ throws IOException {IndexSearcher searcher = new IndexSearcher(reader);searcher.setQueryCache(null);// minScorerCost = min(5000, 5000) = 5000 >= filter.cost(500) → bitset pathint[] intoBitSetCalls = {0};int[] advanceCalls = {0};+ int[] scoredDocs = {0};Query filterQuery =new CountingFilterQuery(@@ -1194,8 +1218,20 @@ public class TestMaxScoreBulkScorer extends LuceneTestCase {BooleanQuery innerOr =new BooleanQuery.Builder()- .add(new TermQuery(new Term("body", "dense1")), Occur.SHOULD)- .add(new TermQuery(new Term("body", "dense2")), Occur.SHOULD)+ .add(+ new CountingScorerQuery(+ new TermQuery(new Term("body", "dense1")),+ new int[] {Integer.MAX_VALUE},+ new int[] {Integer.MAX_VALUE},+ scoredDocs),+ Occur.SHOULD)+ .add(+ new CountingScorerQuery(+ new TermQuery(new Term("body", "dense2")),+ new int[] {Integer.MAX_VALUE},+ new int[] {Integer.MAX_VALUE},+ scoredDocs),+ Occur.SHOULD).build();BooleanQuery outerQuery =@@ -1203,6 +1239,7 @@ public class TestMaxScoreBulkScorer extends LuceneTestCase {Query rewritten = searcher.rewrite(outerQuery);Weight weight = searcher.createWeight(rewritten, ScoreMode.TOP_SCORES, 1f);+ int[] collectedDocs = {0};for (LeafReaderContext ctx : reader.leaves()) {ScorerSupplier ss = weight.scorerSupplier(ctx);if (ss != null) {@@ -1216,9 +1253,12 @@ public class TestMaxScoreBulkScorer extends LuceneTestCase {public void setScorer(Scorable scorer) {}@Override- public void collect(int doc) {}+ public void collect(int doc) {+ assertEquals(1, doc % 20);+ collectedDocs[0]++;+ }},- null,+ ctx.reader().getLiveDocs(),0,DocIdSetIterator.NO_MORE_DOCS);}@@ -1230,9 +1270,8 @@ public class TestMaxScoreBulkScorer extends LuceneTestCase {+ " advance="+ advanceCalls[0],intoBitSetCalls[0] > 0);-- reader.close();- dir.close();+ assertEquals(expectedCollectedDocs, collectedDocs[0]);+ assertEquals(collectedDocs[0], scoredDocs[0]);}/**@@ -1358,7 +1397,10 @@ public class TestMaxScoreBulkScorer extends LuceneTestCase {Query dense1 =new CountingScorerQuery(- new TermQuery(new Term("body", "dense1")), minAdvanceShallowTarget, minGetMaxScoreUpTo);+ new TermQuery(new Term("body", "dense1")),+ minAdvanceShallowTarget,+ minGetMaxScoreUpTo,+ new int[1]);Query dense2 = new TermQuery(new Term("body", "dense2"));BooleanQuery innerOr =@@ -1414,11 +1456,14 @@ public class TestMaxScoreBulkScorer extends LuceneTestCase {private final Query delegate;private final int[] minAdvanceShallowTarget;private final int[] minGetMaxScoreUpTo;+ private final int[] scoredDocs;- CountingScorerQuery(Query delegate, int[] minAdvanceShallowTarget, int[] minGetMaxScoreUpTo) {+ private CountingScorerQuery(+ Query delegate, int[] minAdvanceShallowTarget, int[] minGetMaxScoreUpTo, int[] scoredDocs) {this.delegate = delegate;this.minAdvanceShallowTarget = minAdvanceShallowTarget;this.minGetMaxScoreUpTo = minGetMaxScoreUpTo;+ this.scoredDocs = scoredDocs;}@Override@@ -1446,6 +1491,13 @@ public class TestMaxScoreBulkScorer extends LuceneTestCase {minGetMaxScoreUpTo[0] = Math.min(minGetMaxScoreUpTo[0], upTo);return in.getMaxScore(upTo);}++ @Override+ public void nextDocsAndScores(+ int upTo, Bits liveDocs, DocAndFloatFeatureBuffer buffer) throws IOException {+ in.nextDocsAndScores(upTo, liveDocs, buffer);+ scoredDocs[0] += buffer.size;+ }};}