Apache Lucene open-source search software
Fix block-max support of DoubleValuesSource (#16697)
Zhang Chaod150ba27d119
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex fd562625fe5..448bf39d782 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -201,9 +201,8 @@ ImprovementsOptimizations-----------------------* GITHUB#16431: Add support for Block-Max WAND dynamic score skipping in FunctionScoreQuery- and DoubleValuesSource using DocValuesSkipper. (Rajat Jain)+* GITHUB#16431, GITHUB#16697: Add support for Block-Max WAND dynamic score skipping in FunctionScoreQuery+ and DoubleValuesSource using DocValuesSkipper. (Rajat Jain, Zhang Chao)* GITHUB#16382: WildcardQuery.toAutomaton now collapses consecutive unescaped '*'wildcards into a single any-string automaton, avoiding a quadratic blow-up indiff --git a/lucene/core/src/java/org/apache/lucene/search/DoubleValuesSource.java b/lucene/core/src/java/org/apache/lucene/search/DoubleValuesSource.javaindex f7099484106..94e2bb6fc82 100644--- a/lucene/core/src/java/org/apache/lucene/search/DoubleValuesSource.java+++ b/lucene/core/src/java/org/apache/lucene/search/DoubleValuesSource.java@@ -541,20 +541,42 @@ public abstract class DoubleValuesSource implements SegmentCacheable {@Overridepublic int advanceShallow(int target) throws IOException {if (skipper != null) {- skipper.advance(target);+ if (target > skipper.maxDocID(0)) {+ skipper.advance(target);+ }return skipper.maxDocID(0);}return DocIdSetIterator.NO_MORE_DOCS;}+ /**+ * Return the first level that includes all doc IDs up to {@code upTo}, or -1 if there is no+ * such level.+ */+ private int getLevel(int upTo) {+ final int maxDocID = skipper.maxDocID(0);+ if (maxDocID == -1 || maxDocID == DocIdSetIterator.NO_MORE_DOCS) {+ // the skipper's values may not be read before it is advanced or after it is exhausted+ return -1;+ }+ for (int level = 0, numLevels = skipper.numLevels(); level < numLevels; ++level) {+ final int skipperUpTo = skipper.maxDocID(level);+ if (upTo <= skipperUpTo) {+ return level;+ }+ }+ return -1;+ }+@Overridepublic float getMaxScore(int upTo) throws IOException {if (skipper != null && monotonicity != Monotonicity.NONE) {- if (skipper.minDocID(0) <= upTo) {+ int level = getLevel(upTo);+ if (level >= 0) {long rawBound =(monotonicity == Monotonicity.INCREASING)- ? skipper.maxValue(0)- : skipper.minValue(0);+ ? skipper.maxValue(level)+ : skipper.minValue(level);return (float) decoder.applyAsDouble(rawBound);}}@@ -564,11 +586,12 @@ public abstract class DoubleValuesSource implements SegmentCacheable {@Overridepublic float getMinScore(int upTo) throws IOException {if (skipper != null && monotonicity != Monotonicity.NONE) {- if (skipper.minDocID(0) <= upTo) {+ int level = getLevel(upTo);+ if (level >= 0) {long rawBound =(monotonicity == Monotonicity.INCREASING)- ? skipper.minValue(0)- : skipper.maxValue(0);+ ? skipper.minValue(level)+ : skipper.maxValue(level);return (float) decoder.applyAsDouble(rawBound);}}diff --git a/lucene/core/src/test/org/apache/lucene/search/TestDoubleValuesSource.java b/lucene/core/src/test/org/apache/lucene/search/TestDoubleValuesSource.javaindex b95162a2727..b041426549b 100644--- a/lucene/core/src/test/org/apache/lucene/search/TestDoubleValuesSource.java+++ b/lucene/core/src/test/org/apache/lucene/search/TestDoubleValuesSource.java@@ -21,15 +21,20 @@ import java.io.IOException;import java.util.Arrays;import java.util.Collection;import java.util.Collections;+import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat;import org.apache.lucene.document.Document;import org.apache.lucene.document.DoubleDocValuesField;import org.apache.lucene.document.Field;import org.apache.lucene.document.FloatDocValuesField;import org.apache.lucene.document.NumericDocValuesField;+import org.apache.lucene.index.DirectoryReader;import org.apache.lucene.index.IndexReader;+import org.apache.lucene.index.IndexWriter;+import org.apache.lucene.index.IndexWriterConfig;import org.apache.lucene.index.LeafReaderContext;import org.apache.lucene.index.Term;import org.apache.lucene.store.Directory;+import org.apache.lucene.tests.index.AssertingDirectoryReader;import org.apache.lucene.tests.index.RandomIndexWriter;import org.apache.lucene.tests.search.CheckHits;import org.apache.lucene.tests.util.English;@@ -48,6 +53,9 @@ public class TestDoubleValuesSource extends LuceneTestCase {private static IndexReader reader;private static IndexSearcher searcher;+ private static Directory skipIndexDir;+ private static DirectoryReader skipIndexReader;+@BeforeClasspublic static void beforeClass() throws Exception {dir = newDirectory();@@ -72,15 +80,22 @@ public class TestDoubleValuesSource extends LuceneTestCase {reader = iw.getReader();iw.close();searcher = newSearcher(reader);++ skipIndexDir = indexWithSkipIndex();+ skipIndexReader = new AssertingDirectoryReader(DirectoryReader.open(skipIndexDir));}@AfterClasspublic static void afterClass() throws Exception {reader.close();dir.close();+ skipIndexReader.close();+ skipIndexDir.close();searcher = null;reader = null;dir = null;+ skipIndexReader = null;+ skipIndexDir = null;}public void testSortMissingZeroDefault() throws Exception {@@ -365,4 +380,54 @@ public class TestDoubleValuesSource extends LuceneTestCase {}});}++ private static Directory indexWithSkipIndex() throws IOException {+ Directory directory = newDirectory();+ IndexWriterConfig conf = newIndexWriterConfig();+ conf.setCodec(TestUtil.alwaysDocValuesFormat(new Lucene90DocValuesFormat(16)));+ conf.setMergePolicy(newLogMergePolicy()); // we rely on docids following indexing order+ try (IndexWriter writer = new IndexWriter(directory, conf)) {+ for (int i = 0; i < 200; i++) {+ Document doc = new Document();+ doc.add(NumericDocValuesField.indexedField("field", i));+ writer.addDocument(doc);+ }+ writer.forceMerge(1);+ }+ return directory;+ }++ private static DoubleValues skipIndexValues() throws IOException {+ LeafReaderContext context = skipIndexReader.leaves().get(0);+ return DoubleValuesSource.fromLongField("field").getValues(context, null);+ }++ public void testAdvanceShallow() throws Exception {+ DoubleValues values = skipIndexValues();+ int blockEnd = values.advanceShallow(0);+ assertEquals(blockEnd, values.advanceShallow(blockEnd));+ assertTrue(values.advanceShallow(blockEnd + 1) > blockEnd);+ }++ public void testMaxScoreBeyondTheCurrentBlock() throws Exception {+ DoubleValues values = skipIndexValues();+ int upTo = values.advanceShallow(0) + 1;+ assertTrue(values.advanceExact(upTo));+ double score = values.doubleValue();+ assertTrue(values.getMinScore(upTo) <= score);+ assertTrue(score <= values.getMaxScore(upTo));++ assertEquals(Float.POSITIVE_INFINITY, values.getMaxScore(DocIdSetIterator.NO_MORE_DOCS), 0f);+ assertEquals(Float.NEGATIVE_INFINITY, values.getMinScore(DocIdSetIterator.NO_MORE_DOCS), 0f);+ }++ public void testMaxScoreAfterTheSkipperIsExhausted() throws Exception {+ DoubleValues values = skipIndexValues();+ values.advanceShallow(0);++ assertEquals(+ DocIdSetIterator.NO_MORE_DOCS, values.advanceShallow(DocIdSetIterator.NO_MORE_DOCS));+ assertEquals(Float.POSITIVE_INFINITY, values.getMaxScore(DocIdSetIterator.NO_MORE_DOCS), 0f);+ assertEquals(Float.NEGATIVE_INFINITY, values.getMinScore(DocIdSetIterator.NO_MORE_DOCS), 0f);+ }}