Skip to content

Apache Lucene open-source search software

Apply filter before scoring in MaxScoreBulkScorer (#16515)

kkewweie2e9c7b05b7e
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 00580cb83a1..c702f80ea5d 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -505,6 +505,8 @@ Optimizations
* GITHUB#16588: AcceptDocs builds its bit set from the raw filter iterator so that
DocIdSetIterator#intoBitSet is used on segments with deletions. (John Maurice)
+* GITHUB#16515: Apply filter before scoring in MaxScoreBulkScorer. (Ke Wei)
+
* GITHUB#16582: Rewrite sorted-numeric doc-values range queries that cover every value of a field to a
FieldExistsQuery, and collapse an IndexOrDocValuesQuery whose two sides rewrite equally. (Houston Putman)
diff --git a/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/MaxScoreBulkScorerFilterBenchmark.java b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/MaxScoreBulkScorerFilterBenchmark.java
index 4a7e37b10da..34b8565c7d9 100644
--- a/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/MaxScoreBulkScorerFilterBenchmark.java
+++ b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/MaxScoreBulkScorerFilterBenchmark.java
@@ -77,7 +77,7 @@ public class MaxScoreBulkScorerFilterBenchmark {
public int numClauses;
/** Filter density: 1.0 = match-all, 0.8 = 80% dense */
- @Param({"1.0", "0.8"})
+ @Param({"0.01", "0.05", "0.1", "0.2", "0.3", "0.8", "1.0"})
public double filterDensity;
private Directory dir;
@@ -96,7 +96,7 @@ public class MaxScoreBulkScorerFilterBenchmark {
// so multiple clauses often match the same doc, forcing PQ activity.
for (int t = 0; t < numClauses; t++) {
int period = 7 + t * 11; // 7, 18, 29, 40, ... ~14% avg density
- int offset = (t * 7919) % period;
+ int offset = (int) ((t * 7919L) % period);
if ((i + offset) % period == 0) {
doc.add(new TextField("text", "word" + t, Field.Store.NO));
}
@@ -106,7 +106,7 @@ public class MaxScoreBulkScorerFilterBenchmark {
doc.add(new StringField("filter", "yes", Field.Store.NO));
} else {
int filterThreshold = (int) (filterDensity * 1000);
- if ((i * 7919) % 1000 < filterThreshold) {
+ if ((i * 7919L) % 1000 < filterThreshold) {
doc.add(new StringField("filter", "yes", Field.Store.NO));
} else {
doc.add(new StringField("filter", "no", Field.Store.NO));
diff --git a/lucene/core/src/java/org/apache/lucene/search/MaxScoreBulkScorer.java b/lucene/core/src/java/org/apache/lucene/search/MaxScoreBulkScorer.java
index 8d517092bc8..bb39bca70ea 100644
--- a/lucene/core/src/java/org/apache/lucene/search/MaxScoreBulkScorer.java
+++ b/lucene/core/src/java/org/apache/lucene/search/MaxScoreBulkScorer.java
@@ -49,7 +49,7 @@ final class MaxScoreBulkScorer extends BulkScorer {
private final FixedBitSet windowMatches = new FixedBitSet(INNER_WINDOW_SIZE);
private final double[] windowScores = new double[INNER_WINDOW_SIZE];
- private FixedBitSet filterMatches = null;
+ private OffsetBits filterMatchesBits = null;
private final DocAndFloatFeatureBuffer docAndScoreBuffer = new DocAndFloatFeatureBuffer();
private final DocAndScoreAccBuffer docAndScoreAccBuffer = new DocAndScoreAccBuffer();
@@ -82,7 +82,7 @@ final class MaxScoreBulkScorer extends BulkScorer {
// filter advance()
if (minScorerCost >= this.filter.cost
|| (allScorers.length > 4 && this.cost >= this.filter.cost)) {
- this.filterMatches = new FixedBitSet(INNER_WINDOW_SIZE);
+ this.filterMatchesBits = new OffsetBits(new FixedBitSet(INNER_WINDOW_SIZE), maxDoc);
}
}
}
@@ -216,7 +216,7 @@ final class MaxScoreBulkScorer extends BulkScorer {
int innerWindowMax = MathUtil.unsignedMin(max, innerWindowMin + INNER_WINDOW_SIZE);
docAndScoreAccBuffer.size = 0;
- if (filterMatches == null) {
+ if (filterMatchesBits == null) {
fillScoreBufferViaLeapFrog(top, acceptDocs, innerWindowMax);
} else {
fillScoreBufferViaBitSet(top, acceptDocs, innerWindowMax);
@@ -227,24 +227,25 @@ final class MaxScoreBulkScorer extends BulkScorer {
private void fillScoreBufferViaBitSet(DisiWrapper top, Bits acceptDocs, int innerWindowMax)
throws IOException {
- filterMatches.clear();
+ filterMatchesBits.bits.clear();
int innerWindowMin = top.doc;
if (filter.doc < innerWindowMax) {
if (filter.doc < innerWindowMin) {
filter.doc = filter.approximation.advance(innerWindowMin);
}
if (filter.doc < innerWindowMax) {
- filter.approximation.intoBitSet(innerWindowMax, filterMatches, innerWindowMin);
+ filter.approximation.intoBitSet(innerWindowMax, filterMatchesBits.bits, innerWindowMin);
filter.doc = filter.approximation.docID();
}
}
if (acceptDocs != null) {
- acceptDocs.applyMask(filterMatches, innerWindowMin);
+ acceptDocs.applyMask(filterMatchesBits.bits, innerWindowMin);
}
+ filterMatchesBits.setOffset(innerWindowMin);
int innerWindowSize = innerWindowMax - innerWindowMin;
// Collect matches of essential clauses into a bitset, checking filter via bitset lookup
- collectEssentialScoresIntoWindow(top, innerWindowMax, innerWindowMin, null, filterMatches);
+ collectEssentialScoresIntoWindow(top, innerWindowMax, innerWindowMin, filterMatchesBits);
flushWindowToDocAndScoreAccBuffer(innerWindowMin, innerWindowSize);
}
@@ -290,25 +291,15 @@ final class MaxScoreBulkScorer extends BulkScorer {
* caller is responsible for populating {@link #docAndScoreAccBuffer} from the window afterwards.
*
* @param acceptDocs docs to accept, passed to {@link Scorer#nextDocsAndScores}
- * @param filterMatches if non-null, only docs whose corresponding bit is set in this bitset will
- * be collected; if null, all docs are collected
*/
private void collectEssentialScoresIntoWindow(
- DisiWrapper top,
- int innerWindowMax,
- int innerWindowMin,
- Bits acceptDocs,
- FixedBitSet filterMatches)
- throws IOException {
+ DisiWrapper top, int innerWindowMax, int innerWindowMin, Bits acceptDocs) throws IOException {
do {
for (top.scorer.nextDocsAndScores(innerWindowMax, acceptDocs, docAndScoreBuffer);
docAndScoreBuffer.size > 0;
top.scorer.nextDocsAndScores(innerWindowMax, acceptDocs, docAndScoreBuffer)) {
for (int index = 0; index < docAndScoreBuffer.size; ++index) {
final int doc = docAndScoreBuffer.docs[index];
- if (filterMatches != null && filterMatches.get(doc - innerWindowMin) == false) {
- continue;
- }
final float score = docAndScoreBuffer.features[index];
final int i = doc - innerWindowMin;
windowMatches.set(i);
@@ -366,12 +357,40 @@ final class MaxScoreBulkScorer extends BulkScorer {
int innerWindowSize = innerWindowMax - innerWindowMin;
// Collect matches of essential clauses into a bitset
- collectEssentialScoresIntoWindow(top, innerWindowMax, innerWindowMin, acceptDocs, null);
+ collectEssentialScoresIntoWindow(top, innerWindowMax, innerWindowMin, acceptDocs);
flushWindowToDocAndScoreAccBuffer(innerWindowMin, innerWindowSize);
scoreNonEssentialClauses(collector, docAndScoreAccBuffer, firstEssentialScorer);
}
+ /** A wrapper around {@link FixedBitSet} that supports setting an offset. */
+ private static final class OffsetBits implements Bits {
+ private final FixedBitSet bits;
+ private final int maxDoc;
+ private int offset;
+
+ OffsetBits(FixedBitSet bits, int maxDoc) {
+ this.bits = bits;
+ this.maxDoc = maxDoc;
+ }
+
+ void setOffset(int offset) {
+ assert offset < maxDoc;
+ this.offset = offset;
+ }
+
+ @Override
+ public boolean get(int index) {
+ assert index >= 0 && offset <= index && index - offset < bits.length() && index < maxDoc;
+ return bits.get(index - offset);
+ }
+
+ @Override
+ public int length() {
+ return maxDoc;
+ }
+ }
+
private int computeOuterWindowMax(int windowMin) throws IOException {
// Only use essential scorers to compute the window's max doc ID, in order to avoid constantly
// recomputing max scores over small windows
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestMaxScoreBulkScorer.java b/lucene/core/src/test/org/apache/lucene/search/TestMaxScoreBulkScorer.java
index 834da692c97..e484f9ffbc0 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestMaxScoreBulkScorer.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestMaxScoreBulkScorer.java
@@ -1167,6 +1167,7 @@ public class TestMaxScoreBulkScorer extends LuceneTestCase {
IndexWriter w = new IndexWriter(dir, new IndexWriterConfig());
for (int i = 0; i < 10000; i++) {
Document doc = new Document();
+ doc.add(new StringField("id", Integer.toString(i), Field.Store.NO));
if (i % 2 == 0) {
doc.add(new TextField("body", "dense1", Field.Store.NO));
}
@@ -1178,15 +1179,38 @@ public class TestMaxScoreBulkScorer extends LuceneTestCase {
}
w.addDocument(doc);
}
- w.close();
+ w.commit();
+ // Test without-delete paths
DirectoryReader reader = DirectoryReader.open(dir);
+ assertEquals(10000, reader.numDocs());
+ assertDenseScorersUseBitSet(reader, 500);
+
+ // Test with-delete paths
+ w.deleteDocuments(
+ // first window
+ new Term("id", "20"), new Term("id", "21"),
+ // another window
+ new Term("id", "4101"), new Term("id", "4102"));
+ w.commit();
+ DirectoryReader newReader = DirectoryReader.openIfChanged(reader);
+ reader.close();
+ assertEquals(9996, newReader.numDocs());
+ assertDenseScorersUseBitSet(newReader, 498);
+ newReader.close();
+ w.close();
+ dir.close();
+ }
+
+ private void assertDenseScorersUseBitSet(DirectoryReader reader, int expectedCollectedDocs)
+ throws IOException {
IndexSearcher searcher = new IndexSearcher(reader);
searcher.setQueryCache(null);
// minScorerCost = min(5000, 5000) = 5000 >= filter.cost(500) → bitset path
int[] intoBitSetCalls = {0};
int[] advanceCalls = {0};
+ int[] scoredDocs = {0};
Query filterQuery =
new CountingFilterQuery(
@@ -1194,8 +1218,20 @@ public class TestMaxScoreBulkScorer extends LuceneTestCase {
BooleanQuery innerOr =
new BooleanQuery.Builder()
- .add(new TermQuery(new Term("body", "dense1")), Occur.SHOULD)
- .add(new TermQuery(new Term("body", "dense2")), Occur.SHOULD)
+ .add(
+ new CountingScorerQuery(
+ new TermQuery(new Term("body", "dense1")),
+ new int[] {Integer.MAX_VALUE},
+ new int[] {Integer.MAX_VALUE},
+ scoredDocs),
+ Occur.SHOULD)
+ .add(
+ new CountingScorerQuery(
+ new TermQuery(new Term("body", "dense2")),
+ new int[] {Integer.MAX_VALUE},
+ new int[] {Integer.MAX_VALUE},
+ scoredDocs),
+ Occur.SHOULD)
.build();
BooleanQuery outerQuery =
@@ -1203,6 +1239,7 @@ public class TestMaxScoreBulkScorer extends LuceneTestCase {
Query rewritten = searcher.rewrite(outerQuery);
Weight weight = searcher.createWeight(rewritten, ScoreMode.TOP_SCORES, 1f);
+ int[] collectedDocs = {0};
for (LeafReaderContext ctx : reader.leaves()) {
ScorerSupplier ss = weight.scorerSupplier(ctx);
if (ss != null) {
@@ -1216,9 +1253,12 @@ public class TestMaxScoreBulkScorer extends LuceneTestCase {
public void setScorer(Scorable scorer) {}
@Override
- public void collect(int doc) {}
+ public void collect(int doc) {
+ assertEquals(1, doc % 20);
+ collectedDocs[0]++;
+ }
},
- null,
+ ctx.reader().getLiveDocs(),
0,
DocIdSetIterator.NO_MORE_DOCS);
}
@@ -1230,9 +1270,8 @@ public class TestMaxScoreBulkScorer extends LuceneTestCase {
+ " advance="
+ advanceCalls[0],
intoBitSetCalls[0] > 0);
-
- reader.close();
- dir.close();
+ assertEquals(expectedCollectedDocs, collectedDocs[0]);
+ assertEquals(collectedDocs[0], scoredDocs[0]);
}
/**
@@ -1358,7 +1397,10 @@ public class TestMaxScoreBulkScorer extends LuceneTestCase {
Query dense1 =
new CountingScorerQuery(
- new TermQuery(new Term("body", "dense1")), minAdvanceShallowTarget, minGetMaxScoreUpTo);
+ new TermQuery(new Term("body", "dense1")),
+ minAdvanceShallowTarget,
+ minGetMaxScoreUpTo,
+ new int[1]);
Query dense2 = new TermQuery(new Term("body", "dense2"));
BooleanQuery innerOr =
@@ -1414,11 +1456,14 @@ public class TestMaxScoreBulkScorer extends LuceneTestCase {
private final Query delegate;
private final int[] minAdvanceShallowTarget;
private final int[] minGetMaxScoreUpTo;
+ private final int[] scoredDocs;
- CountingScorerQuery(Query delegate, int[] minAdvanceShallowTarget, int[] minGetMaxScoreUpTo) {
+ private CountingScorerQuery(
+ Query delegate, int[] minAdvanceShallowTarget, int[] minGetMaxScoreUpTo, int[] scoredDocs) {
this.delegate = delegate;
this.minAdvanceShallowTarget = minAdvanceShallowTarget;
this.minGetMaxScoreUpTo = minGetMaxScoreUpTo;
+ this.scoredDocs = scoredDocs;
}
@Override
@@ -1446,6 +1491,13 @@ public class TestMaxScoreBulkScorer extends LuceneTestCase {
minGetMaxScoreUpTo[0] = Math.min(minGetMaxScoreUpTo[0], upTo);
return in.getMaxScore(upTo);
}
+
+ @Override
+ public void nextDocsAndScores(
+ int upTo, Bits liveDocs, DocAndFloatFeatureBuffer buffer) throws IOException {
+ in.nextDocsAndScores(upTo, liveDocs, buffer);
+ scoredDocs[0] += buffer.size;
+ }
};
}