Apache Lucene open-source search software
Further improve filtering by score. (#14970)
Adrien Grand7fc9fd36095b
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex 56974c03cd5..a53e5892cb9 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -218,6 +218,8 @@ Optimizations* GITHUB#14976: Utilize docIdRunEnd on ReqExclBulkScorer. (Ge Song)+* GITHUB#14970: Further speed up filtering hits by score. (Adrien Grand)+Changes in Runtime Behavior---------------------* GITHUB#14823: Decrease TieredMergePolicy's default number of segments perdiff --git a/lucene/core/src/java/org/apache/lucene/internal/vectorization/DefaultVectorUtilSupport.java b/lucene/core/src/java/org/apache/lucene/internal/vectorization/DefaultVectorUtilSupport.javaindex 7a4e4efb277..afece999f2e 100644--- a/lucene/core/src/java/org/apache/lucene/internal/vectorization/DefaultVectorUtilSupport.java+++ b/lucene/core/src/java/org/apache/lucene/internal/vectorization/DefaultVectorUtilSupport.java@@ -310,6 +310,22 @@ final class DefaultVectorUtilSupport implements VectorUtilSupport {}}+ @Override+ public int filterByScore(+ int[] docBuffer, float[] scoreBuffer, float minScoreInclusive, int upTo) {+ int newSize = 0;+ for (int i = 0; i < upTo; ++i) {+ int doc = docBuffer[i];+ float score = scoreBuffer[i];+ docBuffer[newSize] = doc;+ scoreBuffer[newSize] = score;+ if (score >= minScoreInclusive) {+ newSize++;+ }+ }+ return newSize;+ }+@Overridepublic int filterByScore(int[] docBuffer, double[] scoreBuffer, double minScoreInclusive, int upTo) {diff --git a/lucene/core/src/java/org/apache/lucene/internal/vectorization/VectorUtilSupport.java b/lucene/core/src/java/org/apache/lucene/internal/vectorization/VectorUtilSupport.javaindex 1aef96a9d1c..7eec50a3b32 100644--- a/lucene/core/src/java/org/apache/lucene/internal/vectorization/VectorUtilSupport.java+++ b/lucene/core/src/java/org/apache/lucene/internal/vectorization/VectorUtilSupport.java@@ -101,6 +101,20 @@ public interface VectorUtilSupport {float minQuantile,float maxQuantile);+ /**+ * filter both {@code docBuffer} and {@code scoreBuffer} with {@code minScoreInclusive}, each+ * {@code docBuffer} and {@code scoreBuffer} of the same index forms a pair, pairs with score not+ * greater than or equal to {@code minScoreInclusive} will be filtered out from the array.+ *+ * @param docBuffer doc buffer contains docs (or some other value forms a pair with {@code+ * scoreBuffer})+ * @param scoreBuffer score buffer contains scores to be compared with {@code minScoreInclusive}+ * @param minScoreInclusive minimal required score to not be filtered out+ * @param upTo where the filter should end+ * @return how many pairs left after filter+ */+ int filterByScore(int[] docBuffer, float[] scoreBuffer, float minScoreInclusive, int upTo);+/*** filter both {@code docBuffer} and {@code scoreBuffer} with {@code minScoreInclusive}, each* {@code docBuffer} and {@code scoreBuffer} of the same index forms a pair, pairs with score notdiff --git a/lucene/core/src/java/org/apache/lucene/search/BatchScoreBulkScorer.java b/lucene/core/src/java/org/apache/lucene/search/BatchScoreBulkScorer.javaindex d1412fb5e45..34f377f3987 100644--- a/lucene/core/src/java/org/apache/lucene/search/BatchScoreBulkScorer.java+++ b/lucene/core/src/java/org/apache/lucene/search/BatchScoreBulkScorer.java@@ -18,6 +18,7 @@ package org.apache.lucene.search;import java.io.IOException;import org.apache.lucene.util.Bits;+import org.apache.lucene.util.VectorUtil;/*** A bulk scorer used when {@link ScoreMode#needsScores()} is true and {@link@@ -49,11 +50,16 @@ class BatchScoreBulkScorer extends BulkScorer {for (scorer.nextDocsAndScores(max, acceptDocs, buffer);buffer.size > 0;scorer.nextDocsAndScores(max, acceptDocs, buffer)) {++ // The collector already filters hits whose scores is less than the minimum competitive score,+ // but doing it here is a bit more efficient.+ buffer.size =+ VectorUtil.filterByScore(+ buffer.docs, buffer.features, scorable.minCompetitiveScore, buffer.size);+for (int i = 0, size = buffer.size; i < size; i++) {- float score = scorable.score = buffer.features[i];- if (score >= scorable.minCompetitiveScore) {- collector.collect(buffer.docs[i]);- }+ scorable.score = buffer.features[i];+ collector.collect(buffer.docs[i]);}scorer.setMinCompetitiveScore(scorable.minCompetitiveScore);}diff --git a/lucene/core/src/java/org/apache/lucene/search/BlockMaxConjunctionBulkScorer.java b/lucene/core/src/java/org/apache/lucene/search/BlockMaxConjunctionBulkScorer.javaindex d10c0bb41e7..c5d567a4373 100644--- a/lucene/core/src/java/org/apache/lucene/search/BlockMaxConjunctionBulkScorer.java+++ b/lucene/core/src/java/org/apache/lucene/search/BlockMaxConjunctionBulkScorer.java@@ -166,15 +166,24 @@ final class BlockMaxConjunctionBulkScorer extends BulkScorer {return;}+ // two equal consecutive values mean that the first clause always returns a score of zero, so we+ // don't need to filter hits by score again.+ boolean leadingClauseHasZeroScores = sumOfOtherClauses[1] == sumOfOtherClauses[0];+for (scorers[0].nextDocsAndScores(max, acceptDocs, docAndScoreBuffer);docAndScoreBuffer.size > 0;scorers[0].nextDocsAndScores(max, acceptDocs, docAndScoreBuffer)) {+ if (leadingClauseHasZeroScores == false) {+ ScorerUtil.filterCompetitiveHits(+ docAndScoreBuffer, sumOfOtherClauses[1], scorable.minCompetitiveScore, scorers.length);+ }+docAndScoreAccBuffer.copyFrom(docAndScoreBuffer);for (int i = 1; i < scorers.length; ++i) {double sumOfOtherClause = sumOfOtherClauses[i];- if (sumOfOtherClause != sumOfOtherClauses[i - 1]) {+ if (i > 1 && sumOfOtherClause != sumOfOtherClauses[i - 1]) {// two equal consecutive values mean that the first clause always returns a score of zero,// so we don't need to filter hits by score again.ScorerUtil.filterCompetitiveHits(diff --git a/lucene/core/src/java/org/apache/lucene/search/MaxScoreBulkScorer.java b/lucene/core/src/java/org/apache/lucene/search/MaxScoreBulkScorer.javaindex c723cbe00f1..b9c6c530fe7 100644--- a/lucene/core/src/java/org/apache/lucene/search/MaxScoreBulkScorer.java+++ b/lucene/core/src/java/org/apache/lucene/search/MaxScoreBulkScorer.java@@ -22,6 +22,7 @@ import java.util.List;import org.apache.lucene.util.Bits;import org.apache.lucene.util.FixedBitSet;import org.apache.lucene.util.MathUtil;+import org.apache.lucene.util.VectorUtil;final class MaxScoreBulkScorer extends BulkScorer {@@ -40,6 +41,8 @@ final class MaxScoreBulkScorer extends BulkScorer {// Index of the first scorer that is required, this scorer and all following scorers are required// for a document to match.int firstRequiredScorer;+ // Index of the first scorer that may produce positive scores on this window.+ int firstNonNullScorer;// The minimum value of minCompetitiveScore that would produce a more favorable partitioning.float nextMinCompetitiveScore;private final long cost;@@ -230,8 +233,39 @@ final class MaxScoreBulkScorer extends BulkScorer {docAndScoreBuffer.size > 0;top.scorer.nextDocsAndScores(upTo, acceptDocs, docAndScoreBuffer)) {- docAndScoreAccBuffer.copyFrom(docAndScoreBuffer);- scoreNonEssentialClauses(collector, docAndScoreAccBuffer, firstEssentialScorer);+ if (firstNonNullScorer >= firstEssentialScorer) {+ // Note: firstNonNullScorer may be > firstEssentialScorer if minCompetitiveScore=0 since+ // hits with a score of 0 are still competitive+ // There are no non-essential clauses, filter non-competitive hits and collect directly++ int[] docs = docAndScoreBuffer.docs;+ float[] scores = docAndScoreBuffer.features;+ int size = docAndScoreBuffer.size;+ size = VectorUtil.filterByScore(docs, scores, scorable.minCompetitiveScore, size);++ for (int i = 0; i < size; ++i) {+ scorable.score = scores[i];+ collector.collect(docs[i]);+ }+ } else {+ // Filter based on float scores before promoting them to doubles so that vectorization can+ // work on 2x more values at once.+ ScorerUtil.filterCompetitiveHits(+ docAndScoreBuffer,+ maxScoreSums[firstEssentialScorer - 1],+ scorable.minCompetitiveScore,+ allScorers.length);++ docAndScoreAccBuffer.copyFrom(docAndScoreBuffer);++ // Apply the last non-essential clause here instead of delegating it to+ // `scoreNonEssentialClauses` so that it doesn't re-do filtering by score.+ DisiWrapper scorer = allScorers[firstEssentialScorer - 1];+ ScorerUtil.applyOptionalClause(docAndScoreAccBuffer, scorer.iterator, scorer.scorable);+ scorer.doc = scorer.iterator.docID();++ scoreNonEssentialClauses(collector, docAndScoreAccBuffer, firstEssentialScorer - 1);+ }}top.doc = top.iterator.docID();@@ -250,11 +284,19 @@ final class MaxScoreBulkScorer extends BulkScorer {docAndScoreBuffer.size > 0;lead1.scorer.nextDocsAndScores(max, acceptDocs, docAndScoreBuffer)) {+ // Filter based on float scores before promoting them to doubles so that vectorization can+ // work on 2x more values at once.+ ScorerUtil.filterCompetitiveHits(+ docAndScoreBuffer,+ maxScoreSums[allScorers.length - 2],+ scorable.minCompetitiveScore,+ allScorers.length);+docAndScoreAccBuffer.copyFrom(docAndScoreBuffer);for (int i = allScorers.length - 2; i >= firstRequiredScorer; --i) {- if (scorable.minCompetitiveScore > 0) {+ if (i < allScorers.length - 2 && scorable.minCompetitiveScore > 0) {ScorerUtil.filterCompetitiveHits(docAndScoreAccBuffer,maxScoreSums[i],@@ -371,7 +413,7 @@ final class MaxScoreBulkScorer extends BulkScorer {throws IOException {numCandidates += buffer.size;- for (int i = numNonEssentialClauses - 1; i >= 0; --i) {+ for (int i = numNonEssentialClauses - 1; i >= firstNonNullScorer; --i) {DisiWrapper scorer = allScorers[i];assert scorable.minCompetitiveScore > 0: "All clauses are essential if minCompetitiveScore is equal to zero";@@ -381,9 +423,20 @@ final class MaxScoreBulkScorer extends BulkScorer {scorer.doc = scorer.iterator.docID();}- for (int i = 0; i < buffer.size; ++i) {- scorable.score = (float) buffer.scores[i];- collector.collect(buffer.docs[i]);+ // The collector already filters hits whose score is less than the min competitive score, but+ // doing it here is a bit more efficient.+ int size = buffer.size;+ int[] docs = buffer.docs;+ docAndScoreBuffer.growNoCopy(size);+ float[] scores = docAndScoreBuffer.features;+ for (int i = 0; i < size; ++i) {+ scores[i] = (float) buffer.scores[i];+ }+ size = VectorUtil.filterByScore(docs, scores, scorable.minCompetitiveScore, size);++ for (int i = 0; i < size; ++i) {+ scorable.score = scores[i];+ collector.collect(docs[i]);}}@@ -408,10 +461,14 @@ final class MaxScoreBulkScorer extends BulkScorer {(double) scorer2.maxWindowScore / Math.max(1L, scorer2.cost));});double maxScoreSum = 0;+ firstNonNullScorer = 0;firstEssentialScorer = 0;nextMinCompetitiveScore = Float.POSITIVE_INFINITY;for (int i = 0; i < allScorers.length; ++i) {final DisiWrapper w = scratch[i];+ if (w.maxWindowScore == 0f) {+ firstNonNullScorer = i + 1;+ }double newMaxScoreSum = maxScoreSum + w.maxWindowScore;float maxScoreSumFloat =(float) MathUtil.sumUpperBound(newMaxScoreSum, firstEssentialScorer + 1);diff --git a/lucene/core/src/java/org/apache/lucene/search/ScorerUtil.java b/lucene/core/src/java/org/apache/lucene/search/ScorerUtil.javaindex 8562ee84c01..ebd6d98dfc5 100644--- a/lucene/core/src/java/org/apache/lucene/search/ScorerUtil.java+++ b/lucene/core/src/java/org/apache/lucene/search/ScorerUtil.java@@ -138,6 +138,34 @@ class ScorerUtil {return minRequiredScore;}+ /**+ * Filters competitive hits from the provided {@link DocAndFloatFeatureBuffer}.+ *+ * <p>This method removes documents from the buffer that cannot possibly have a score competitive+ * enough to exceed the minimum competitive score, given the maximum remaining score and the+ * number of scorers.+ */+ static void filterCompetitiveHits(+ DocAndFloatFeatureBuffer buffer,+ double maxRemainingScore,+ float minCompetitiveScore,+ int numScorers) {+ double minRequiredScoreDouble =+ minRequiredScore(maxRemainingScore, minCompetitiveScore, numScorers);+ float minRequiredScoreFloat = (float) minRequiredScoreDouble;+ if ((double) minRequiredScoreFloat > minRequiredScoreDouble) { // the cast rounded up+ minRequiredScoreFloat = Math.nextDown(minRequiredScoreFloat);+ }+ assert (double) minRequiredScoreFloat <= minRequiredScoreDouble;++ if (minRequiredScoreFloat <= 0) {+ return;+ }++ buffer.size =+ VectorUtil.filterByScore(buffer.docs, buffer.features, minRequiredScoreFloat, buffer.size);+ }+/*** Filters competitive hits from the provided {@link DocAndScoreAccBuffer}.*diff --git a/lucene/core/src/java/org/apache/lucene/util/VectorUtil.java b/lucene/core/src/java/org/apache/lucene/util/VectorUtil.javaindex c03b71eb9ec..27572b49321 100644--- a/lucene/core/src/java/org/apache/lucene/util/VectorUtil.java+++ b/lucene/core/src/java/org/apache/lucene/util/VectorUtil.java@@ -377,6 +377,27 @@ public final class VectorUtil {vector, oldAlpha, oldMinQuantile, scale, alpha, minQuantile, maxQuantile);}+ /**+ * filter both {@code docBuffer} and {@code scoreBuffer} with {@code minScoreInclusive}, each+ * {@code docBuffer} and {@code scoreBuffer} of the same index forms a pair, pairs with score not+ * greater than or equal to {@code minScoreInclusive} will be filtered out from the array.+ *+ * @param docBuffer doc buffer contains docs (or some other value forms a pair with {@code+ * scoreBuffer})+ * @param scoreBuffer score buffer contains scores to be compared with {@code minScoreInclusive}+ * @param minScoreInclusive minimal required score to not be filtered out+ * @param upTo where the filter should end+ * @return how many pairs left after filter+ */+ public static int filterByScore(+ int[] docBuffer, float[] scoreBuffer, float minScoreInclusive, int upTo) {+ if (docBuffer.length < upTo || scoreBuffer.length < upTo) {+ throw new IllegalArgumentException(+ "docBuffer and scoreBuffer should be at least as long as upTo");+ }+ return IMPL.filterByScore(docBuffer, scoreBuffer, minScoreInclusive, upTo);+ }+/*** filter both {@code docBuffer} and {@code scoreBuffer} with {@code minScoreInclusive}, each* {@code docBuffer} and {@code scoreBuffer} of the same index forms a pair, pairs with score not@@ -391,9 +412,9 @@ public final class VectorUtil {*/public static int filterByScore(int[] docBuffer, double[] scoreBuffer, double minScoreInclusive, int upTo) {- if (docBuffer.length != scoreBuffer.length || docBuffer.length < upTo) {+ if (docBuffer.length < upTo || scoreBuffer.length < upTo) {throw new IllegalArgumentException(- "docBuffer and scoreBuffer should keep same length and at least as long as upTo");+ "docBuffer and scoreBuffer should be at least as long as upTo");}return IMPL.filterByScore(docBuffer, scoreBuffer, minScoreInclusive, upTo);}diff --git a/lucene/core/src/java24/org/apache/lucene/internal/vectorization/PanamaVectorUtilSupport.java b/lucene/core/src/java24/org/apache/lucene/internal/vectorization/PanamaVectorUtilSupport.javaindex efd91aad448..f9223ab64db 100644--- a/lucene/core/src/java24/org/apache/lucene/internal/vectorization/PanamaVectorUtilSupport.java+++ b/lucene/core/src/java24/org/apache/lucene/internal/vectorization/PanamaVectorUtilSupport.java@@ -1005,6 +1005,35 @@ final class PanamaVectorUtilSupport implements VectorUtilSupport {return correction;}+ @SuppressForbidden(reason = "Uses compress and cast only where fast and carefully contained")+ @Override+ public int filterByScore(+ int[] docBuffer, float[] scoreBuffer, float minScoreInclusive, int upTo) {+ int newUpto = 0;+ int i = 0;+ if (Constants.HAS_FAST_COMPRESS_MASK_CAST) {+ for (int bound = FLOAT_SPECIES.loopBound(upTo); i < bound; i += FLOAT_SPECIES.length()) {+ FloatVector scoreVector = FloatVector.fromArray(FLOAT_SPECIES, scoreBuffer, i);+ IntVector docVector = IntVector.fromArray(INT_SPECIES, docBuffer, i);+ VectorMask<Float> mask = scoreVector.compare(VectorOperators.GE, minScoreInclusive);+ scoreVector.compress(mask).intoArray(scoreBuffer, newUpto);+ docVector.compress(mask.cast(INT_SPECIES)).intoArray(docBuffer, newUpto);+ newUpto += mask.trueCount();+ }+ }++ for (; i < upTo; ++i) {+ int doc = docBuffer[i];+ float score = scoreBuffer[i];+ docBuffer[newUpto] = doc;+ scoreBuffer[newUpto] = score;+ if (score >= minScoreInclusive) {+ newUpto++;+ }+ }+ return newUpto;+ }+@SuppressForbidden(reason = "Uses compress and cast only where fast and carefully contained")@Overridepublic int filterByScore(diff --git a/lucene/core/src/test/org/apache/lucene/util/TestVectorUtil.java b/lucene/core/src/test/org/apache/lucene/util/TestVectorUtil.javaindex 8b1ad330b91..1edbfb50dc0 100644--- a/lucene/core/src/test/org/apache/lucene/util/TestVectorUtil.java+++ b/lucene/core/src/test/org/apache/lucene/util/TestVectorUtil.java@@ -23,6 +23,7 @@ import java.util.Random;import org.apache.lucene.index.VectorSimilarityFunction;import org.apache.lucene.internal.vectorization.BaseVectorizationTestCase;import org.apache.lucene.internal.vectorization.VectorizationProvider;+import org.apache.lucene.search.DocAndFloatFeatureBuffer;import org.apache.lucene.search.DocAndScoreAccBuffer;import org.apache.lucene.tests.util.LuceneTestCase;import org.apache.lucene.tests.util.TestUtil;@@ -391,7 +392,46 @@ public class TestVectorUtil extends LuceneTestCase {return length;}- public void testFilterByScore() {+ public void testFilterByFloatScore() {+ for (int iter = 0; iter < 1_000; ++iter) {+ int padding = TestUtil.nextInt(random(), 0, 5);+ DocAndFloatFeatureBuffer b1 = new DocAndFloatFeatureBuffer();+ DocAndFloatFeatureBuffer b2 = new DocAndFloatFeatureBuffer();+ b1.growNoCopy(128 + padding);+ b2.growNoCopy(128 + padding);++ int doc = 0;+ for (int i = 0; i < 128 + padding; ++i) {+ doc += TestUtil.nextInt(random(), 1, 1000);+ b1.docs[i] = b2.docs[i] = doc;+ b1.features[i] = b2.features[i] = random().nextFloat();+ }++ float minScoreInclusive = random().nextFloat();+ int upTo = TestUtil.nextInt(random(), 0, 127);+ b1.size = slowFilterByScore(b1.docs, b1.features, minScoreInclusive, upTo);+ b2.size = VectorUtil.filterByScore(b2.docs, b2.features, minScoreInclusive, upTo);+ assertEquals(b1.size, b2.size);+ assertTrue(Arrays.equals(b1.docs, 0, b1.size, b2.docs, 0, b2.size));+ // two float array should be exactly the same, so just use simple Arrays.equals+ assertTrue(Arrays.equals(b1.features, 0, b1.size, b2.features, 0, b2.size));+ }+ }++ private static int slowFilterByScore(+ int[] docBuffer, float[] scoreBuffer, float minScoreInclusive, int upTo) {+ int newSize = 0;+ for (int i = 0; i < upTo; i++) {+ if (scoreBuffer[i] >= minScoreInclusive) {+ docBuffer[newSize] = docBuffer[i];+ scoreBuffer[newSize] = scoreBuffer[i];+ newSize++;+ }+ }+ return newSize;+ }++ public void testFilterByDoubleScore() {for (int iter = 0; iter < 1_000; ++iter) {int padding = TestUtil.nextInt(random(), 0, 5);DocAndScoreAccBuffer b1 = new DocAndScoreAccBuffer();