Skip to content

Apache Lucene open-source search software

Fix block-max support of DoubleValuesSource (#16697)

Zhang Chaod150ba27d119
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index fd562625fe5..448bf39d782 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -201,9 +201,8 @@ Improvements
Optimizations
---------------------
-
-* GITHUB#16431: Add support for Block-Max WAND dynamic score skipping in FunctionScoreQuery
- and DoubleValuesSource using DocValuesSkipper. (Rajat Jain)
+* GITHUB#16431, GITHUB#16697: Add support for Block-Max WAND dynamic score skipping in FunctionScoreQuery
+ and DoubleValuesSource using DocValuesSkipper. (Rajat Jain, Zhang Chao)
* GITHUB#16382: WildcardQuery.toAutomaton now collapses consecutive unescaped '*'
wildcards into a single any-string automaton, avoiding a quadratic blow-up in
diff --git a/lucene/core/src/java/org/apache/lucene/search/DoubleValuesSource.java b/lucene/core/src/java/org/apache/lucene/search/DoubleValuesSource.java
index f7099484106..94e2bb6fc82 100644
--- a/lucene/core/src/java/org/apache/lucene/search/DoubleValuesSource.java
+++ b/lucene/core/src/java/org/apache/lucene/search/DoubleValuesSource.java
@@ -541,20 +541,42 @@ public abstract class DoubleValuesSource implements SegmentCacheable {
@Override
public int advanceShallow(int target) throws IOException {
if (skipper != null) {
- skipper.advance(target);
+ if (target > skipper.maxDocID(0)) {
+ skipper.advance(target);
+ }
return skipper.maxDocID(0);
}
return DocIdSetIterator.NO_MORE_DOCS;
}
+ /**
+ * Return the first level that includes all doc IDs up to {@code upTo}, or -1 if there is no
+ * such level.
+ */
+ private int getLevel(int upTo) {
+ final int maxDocID = skipper.maxDocID(0);
+ if (maxDocID == -1 || maxDocID == DocIdSetIterator.NO_MORE_DOCS) {
+ // the skipper's values may not be read before it is advanced or after it is exhausted
+ return -1;
+ }
+ for (int level = 0, numLevels = skipper.numLevels(); level < numLevels; ++level) {
+ final int skipperUpTo = skipper.maxDocID(level);
+ if (upTo <= skipperUpTo) {
+ return level;
+ }
+ }
+ return -1;
+ }
+
@Override
public float getMaxScore(int upTo) throws IOException {
if (skipper != null && monotonicity != Monotonicity.NONE) {
- if (skipper.minDocID(0) <= upTo) {
+ int level = getLevel(upTo);
+ if (level >= 0) {
long rawBound =
(monotonicity == Monotonicity.INCREASING)
- ? skipper.maxValue(0)
- : skipper.minValue(0);
+ ? skipper.maxValue(level)
+ : skipper.minValue(level);
return (float) decoder.applyAsDouble(rawBound);
}
}
@@ -564,11 +586,12 @@ public abstract class DoubleValuesSource implements SegmentCacheable {
@Override
public float getMinScore(int upTo) throws IOException {
if (skipper != null && monotonicity != Monotonicity.NONE) {
- if (skipper.minDocID(0) <= upTo) {
+ int level = getLevel(upTo);
+ if (level >= 0) {
long rawBound =
(monotonicity == Monotonicity.INCREASING)
- ? skipper.minValue(0)
- : skipper.maxValue(0);
+ ? skipper.minValue(level)
+ : skipper.maxValue(level);
return (float) decoder.applyAsDouble(rawBound);
}
}
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestDoubleValuesSource.java b/lucene/core/src/test/org/apache/lucene/search/TestDoubleValuesSource.java
index b95162a2727..b041426549b 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestDoubleValuesSource.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestDoubleValuesSource.java
@@ -21,15 +21,20 @@ import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
+import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.DoubleDocValuesField;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FloatDocValuesField;
import org.apache.lucene.document.NumericDocValuesField;
+import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.Term;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.tests.index.AssertingDirectoryReader;
import org.apache.lucene.tests.index.RandomIndexWriter;
import org.apache.lucene.tests.search.CheckHits;
import org.apache.lucene.tests.util.English;
@@ -48,6 +53,9 @@ public class TestDoubleValuesSource extends LuceneTestCase {
private static IndexReader reader;
private static IndexSearcher searcher;
+ private static Directory skipIndexDir;
+ private static DirectoryReader skipIndexReader;
+
@BeforeClass
public static void beforeClass() throws Exception {
dir = newDirectory();
@@ -72,15 +80,22 @@ public class TestDoubleValuesSource extends LuceneTestCase {
reader = iw.getReader();
iw.close();
searcher = newSearcher(reader);
+
+ skipIndexDir = indexWithSkipIndex();
+ skipIndexReader = new AssertingDirectoryReader(DirectoryReader.open(skipIndexDir));
}
@AfterClass
public static void afterClass() throws Exception {
reader.close();
dir.close();
+ skipIndexReader.close();
+ skipIndexDir.close();
searcher = null;
reader = null;
dir = null;
+ skipIndexReader = null;
+ skipIndexDir = null;
}
public void testSortMissingZeroDefault() throws Exception {
@@ -365,4 +380,54 @@ public class TestDoubleValuesSource extends LuceneTestCase {
}
});
}
+
+ private static Directory indexWithSkipIndex() throws IOException {
+ Directory directory = newDirectory();
+ IndexWriterConfig conf = newIndexWriterConfig();
+ conf.setCodec(TestUtil.alwaysDocValuesFormat(new Lucene90DocValuesFormat(16)));
+ conf.setMergePolicy(newLogMergePolicy()); // we rely on docids following indexing order
+ try (IndexWriter writer = new IndexWriter(directory, conf)) {
+ for (int i = 0; i < 200; i++) {
+ Document doc = new Document();
+ doc.add(NumericDocValuesField.indexedField("field", i));
+ writer.addDocument(doc);
+ }
+ writer.forceMerge(1);
+ }
+ return directory;
+ }
+
+ private static DoubleValues skipIndexValues() throws IOException {
+ LeafReaderContext context = skipIndexReader.leaves().get(0);
+ return DoubleValuesSource.fromLongField("field").getValues(context, null);
+ }
+
+ public void testAdvanceShallow() throws Exception {
+ DoubleValues values = skipIndexValues();
+ int blockEnd = values.advanceShallow(0);
+ assertEquals(blockEnd, values.advanceShallow(blockEnd));
+ assertTrue(values.advanceShallow(blockEnd + 1) > blockEnd);
+ }
+
+ public void testMaxScoreBeyondTheCurrentBlock() throws Exception {
+ DoubleValues values = skipIndexValues();
+ int upTo = values.advanceShallow(0) + 1;
+ assertTrue(values.advanceExact(upTo));
+ double score = values.doubleValue();
+ assertTrue(values.getMinScore(upTo) <= score);
+ assertTrue(score <= values.getMaxScore(upTo));
+
+ assertEquals(Float.POSITIVE_INFINITY, values.getMaxScore(DocIdSetIterator.NO_MORE_DOCS), 0f);
+ assertEquals(Float.NEGATIVE_INFINITY, values.getMinScore(DocIdSetIterator.NO_MORE_DOCS), 0f);
+ }
+
+ public void testMaxScoreAfterTheSkipperIsExhausted() throws Exception {
+ DoubleValues values = skipIndexValues();
+ values.advanceShallow(0);
+
+ assertEquals(
+ DocIdSetIterator.NO_MORE_DOCS, values.advanceShallow(DocIdSetIterator.NO_MORE_DOCS));
+ assertEquals(Float.POSITIVE_INFINITY, values.getMaxScore(DocIdSetIterator.NO_MORE_DOCS), 0f);
+ assertEquals(Float.NEGATIVE_INFINITY, values.getMinScore(DocIdSetIterator.NO_MORE_DOCS), 0f);
+ }
}