Apache Lucene open-source search software
Use a single two-phase strategy for doc-values range queries (#16177)
Jim Ferenczi8fe014e8e82f
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex c2dd58432c8..4e1db85be82 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -432,8 +432,10 @@ Optimizations* GITHUB#16001: IndexSearcher.count() was calling query.rewrite twice, a regression since v9.10 (David Smiley)-* GITHUB#16050: Add SIMD-accelerated bulk range evaluation for dense numeric doc values via- BatchDocValuesRangeIterator and DocValuesRangeSupport. (Sagar Upadhyaya)+* GITHUB#16050, GITHUB#16177: Doc-values range queries are now always two-phase iterators, with+ SIMD-accelerated bulk range evaluation for dense numeric doc values exposed through+ TwoPhaseIterator#intoBitSet (overridable for SIMD block evaluation) instead of a separate plain+ DocIdSetIterator, speeding up conjunctions of range filters. (Sagar Upadhyaya, Jim Ferenczi)* GITHUB#16061, GITHUB#16070, GITHUB#16085, GITHUB#16120: Improve cost estimation in SortedSetDocValuesRangeQueryand SortedNumericDocValuesRangeQuery when using DocValuesSkipper, the field is dense and is the primary sort ofdiff --git a/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/MultiFieldDocValuesRangeBenchmark.java b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/MultiFieldDocValuesRangeBenchmark.javaindex 35c44e91d50..e617fb16929 100644--- a/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/MultiFieldDocValuesRangeBenchmark.java+++ b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/MultiFieldDocValuesRangeBenchmark.java@@ -33,6 +33,10 @@ import org.apache.lucene.index.IndexWriterConfig;import org.apache.lucene.search.BooleanClause.Occur;import org.apache.lucene.search.BooleanQuery;import org.apache.lucene.search.IndexSearcher;+import org.apache.lucene.search.Query;+import org.apache.lucene.search.Sort;+import org.apache.lucene.search.TopDocs;+import org.apache.lucene.search.TopFieldCollectorManager;import org.apache.lucene.store.Directory;import org.apache.lucene.store.MMapDirectory;import org.openjdk.jmh.annotations.Benchmark;@@ -76,11 +80,19 @@ public class MultiFieldDocValuesRangeBenchmark {private static final String MIXED = "mixed";private static final String SORTED = "sorted";private static final String RANDOM = "random";+ // All fields random; every clause matches ~70% so the intersection stays dense (stresses the+ // bit-set intersection path).+ private static final String DENSE = "dense";+ // field0 matches ~all docs (a dense clause), every other field matches ~0.1% (very sparse), so a+ // tiny surviving set must be confirmed against a dense clause (stresses survivor pruning).+ private static final String DENSE_SPARSE = "dense_sparse";private Directory dir;private IndexReader reader;private Path path;private BooleanQuery query;+ private Query singleRange;+ private Query sparseNoSkipRange;@State(Scope.Benchmark)public static class Params {@@ -90,7 +102,7 @@ public class MultiFieldDocValuesRangeBenchmark {@Param({"1", "3", "5"})public int fieldCount;- @Param({CLUSTERED, MIXED, RANDOM, SORTED})+ @Param({CLUSTERED, MIXED, RANDOM, SORTED, DENSE, DENSE_SPARSE})public String dataPattern;}@@ -116,6 +128,13 @@ public class MultiFieldDocValuesRangeBenchmark {long value = generateValue(params.dataPattern, f, i, params.docCount, r);doc.add(NumericDocValuesField.indexedField("field" + f, value));}+ // A sparsely populated field WITHOUT a skip index (plain NumericDocValuesField). Its range+ // iterator is a DocValuesValueRangeIterator whose approximation reports a real (sparse) cost,+ // so a range on it routes to ConstantScoreBulkScorer's two-phase path rather than+ // DenseConjunctionBulkScorer (which is what the skip-indexed fields above always hit).+ if (i % 100 == 0) {+ doc.add(new NumericDocValuesField("sparse", i));+ }w.addDocument(doc);}w.forceMerge(1);@@ -141,6 +160,12 @@ public class MultiFieldDocValuesRangeBenchmark {bqBuilder.add(new org.apache.lucene.search.MatchAllDocsQuery(), Occur.FILTER);}query = bqBuilder.build();++ long[] range0 = getQueryRange(params.dataPattern, 0, params.docCount);+ singleRange = SortedNumericDocValuesField.newSlowRangeQuery("field0", range0[0], range0[1]);++ sparseNoSkipRange =+ SortedNumericDocValuesField.newSlowRangeQuery("sparse", 0, params.docCount / 2);}private static long generateValue(@@ -167,6 +192,8 @@ public class MultiFieldDocValuesRangeBenchmark {return r.nextLong(0, docCount);}case RANDOM:+ case DENSE:+ case DENSE_SPARSE:return r.nextLong(0, docCount);default:throw new IllegalArgumentException("Unknown pattern: " + pattern);@@ -205,6 +232,18 @@ public class MultiFieldDocValuesRangeBenchmark {long rangeSize4 = docCount / 5;long lower4 = (docCount - rangeSize4) / 2;return new long[] {lower4, lower4 + rangeSize4};+ case DENSE:+ // Each clause matches ~70% of docs, so the intersection stays dense.+ return new long[] {0, (long) (docCount * 0.7)};+ case DENSE_SPARSE:+ if (fieldIdx == 0) {+ // Dense clause: matches all docs.+ return new long[] {0, docCount};+ }+ // Very sparse clause: matches ~0.1% of docs.+ long rangeSize5 = Math.max(1, docCount / 1000);+ long lower5 = (docCount - rangeSize5) / 2;+ return new long[] {lower5, lower5 + rangeSize5};default:throw new IllegalArgumentException("Unknown pattern: " + pattern);}@@ -236,4 +275,28 @@ public class MultiFieldDocValuesRangeBenchmark {IndexSearcher searcher = new IndexSearcher(reader);return searcher.count(query);}++ // A bare single range, driven through the bulk scorer. We collect (index-order, no early+ // termination) rather than count() because SortedNumericDocValuesRangeQuery#count()+ // short-circuits+ // via the skip index and never builds a scorer. Selectivity follows the data pattern:+ // clustered ~ very sparse, random ~ 20%, dense ~ 70%.+ @Benchmark+ public TopDocs searchSingleRange() throws IOException {+ IndexSearcher searcher = new IndexSearcher(reader);+ return searcher.search(+ singleRange, new TopFieldCollectorManager(Sort.INDEXORDER, 10, Integer.MAX_VALUE));+ }++ // A range on the sparse, non-skip-indexed field. Its DocValuesValueRangeIterator reports a real+ // (sparse) cost, so this is the one shape that routes to ConstantScoreBulkScorer's two-phase path+ // (ours) / DefaultBulkScorer (#16143's fallback) instead of DenseConjunctionBulkScorer. Forced+ // through the bulk scorer via a collector (count() would fall back to scoring anyway with no skip+ // index, but the collector makes the path unambiguous).+ @Benchmark+ public TopDocs searchSparseNoSkipRange() throws IOException {+ IndexSearcher searcher = new IndexSearcher(reader);+ return searcher.search(+ sparseNoSkipRange, new TopFieldCollectorManager(Sort.INDEXORDER, 10, Integer.MAX_VALUE));+ }}diff --git a/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/PhraseScorerBenchmark.java b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/PhraseScorerBenchmark.javaindex 2b0ba46b948..c073fee3166 100644--- a/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/PhraseScorerBenchmark.java+++ b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/PhraseScorerBenchmark.java@@ -21,14 +21,22 @@ import java.io.IOException;import java.util.concurrent.TimeUnit;import org.apache.lucene.document.Document;import org.apache.lucene.document.Field;+import org.apache.lucene.document.NumericDocValuesField;+import org.apache.lucene.document.SortedNumericDocValuesField;+import org.apache.lucene.document.StringField;import org.apache.lucene.document.TextField;import org.apache.lucene.index.DirectoryReader;import org.apache.lucene.index.IndexReader;import org.apache.lucene.index.IndexWriter;import org.apache.lucene.index.IndexWriterConfig;+import org.apache.lucene.index.Term;+import org.apache.lucene.search.BooleanClause.Occur;+import org.apache.lucene.search.BooleanQuery;import org.apache.lucene.search.IndexSearcher;import org.apache.lucene.search.PhraseQuery;+import org.apache.lucene.search.Query;import org.apache.lucene.search.Sort;+import org.apache.lucene.search.TermQuery;import org.apache.lucene.search.TopDocs;import org.apache.lucene.search.TopFieldCollectorManager;import org.apache.lucene.search.TopScoreDocCollectorManager;@@ -90,6 +98,11 @@ public class PhraseScorerBenchmark {// 50% of docs: no query terms at alldoc.add(new TextField("text", "unrelated words", Field.Store.NO));}+ doc.add(NumericDocValuesField.indexedField("num", i));+ // A term present in every doc, used as a fully-dense lead clause in conjunctions. Unlike+ // MatchAllDocsQuery (which BooleanQuery#rewrite strips from multi-clause FILTERs), a real+ // TermQuery survives rewrite, so the conjunction path is actually exercised.+ doc.add(new StringField("all", "1", Field.Store.NO));writer.addDocument(doc);}}@@ -105,6 +118,56 @@ public class PhraseScorerBenchmark {dir.close();}+ // A constant-score conjunction of a phrase FILTER clause with a fully-dense term clause routes+ // through DenseConjunctionBulkScorer (the phrase is a two-phase clause whose approximation+ // matches ~50% but whose phrase matches ~0.1%), so this exercises the unified two-phase+ // bit-set/survivor path. The "all" term is present in every doc and forms the dense lead+ // iterator; it is a real TermQuery rather than MatchAllDocsQuery, which BooleanQuery#rewrite+ // would strip from a multi-clause FILTER, collapsing this back to the bare phrase.+ @Benchmark+ public int benchmarkPhraseFilterConjunction() throws IOException {+ Query q =+ new BooleanQuery.Builder()+ .add(exactQuery, Occur.FILTER)+ .add(new TermQuery(new Term("all", "1")), Occur.FILTER)+ .build();+ return searcher.count(q);+ }++ // Two phrase clauses (both two-phase) confirming each other.+ @Benchmark+ public int benchmarkPhraseConjunction() throws IOException {+ Query q =+ new BooleanQuery.Builder()+ .add(exactQuery, Occur.FILTER)+ .add(new PhraseQuery("text", "lazy", "dog"), Occur.FILTER)+ .build();+ return searcher.count(q);+ }++ // Phrase (two-phase) ∩ term (plain, dense): the term is the dense clause, the phrase confirms.+ @Benchmark+ public int benchmarkPhraseTermConjunction() throws IOException {+ Query q =+ new BooleanQuery.Builder()+ .add(exactQuery, Occur.FILTER)+ .add(new TermQuery(new Term("text", "quick")), Occur.FILTER)+ .build();+ return searcher.count(q);+ }++ // Cross-type: phrase (two-phase) ∩ doc-values range (two-phase) — both two-phase clauses of+ // different kinds pruning each other through the unified path.+ @Benchmark+ public int benchmarkPhraseRangeConjunction() throws IOException {+ Query q =+ new BooleanQuery.Builder()+ .add(exactQuery, Occur.FILTER)+ .add(SortedNumericDocValuesField.newSlowRangeQuery("num", 0, 500_000), Occur.FILTER)+ .build();+ return searcher.count(q);+ }+@Benchmarkpublic TopDocs benchmarkExactTopScores() throws IOException {return searcher.search(exactQuery, 10);diff --git a/lucene/core/src/java/org/apache/lucene/document/SortedNumericDocValuesRangeQuery.java b/lucene/core/src/java/org/apache/lucene/document/SortedNumericDocValuesRangeQuery.javaindex 173673c6514..f00a4e72c35 100644--- a/lucene/core/src/java/org/apache/lucene/document/SortedNumericDocValuesRangeQuery.java+++ b/lucene/core/src/java/org/apache/lucene/document/SortedNumericDocValuesRangeQuery.java@@ -25,7 +25,6 @@ import org.apache.lucene.index.LeafReader;import org.apache.lucene.index.LeafReaderContext;import org.apache.lucene.index.NumericDocValues;import org.apache.lucene.index.SortedNumericDocValues;-import org.apache.lucene.search.BatchDocValuesRangeIterator;import org.apache.lucene.search.ConstantScoreScorerSupplier;import org.apache.lucene.search.ConstantScoreWeight;import org.apache.lucene.search.DocIdSetIterator;@@ -141,17 +140,13 @@ final class SortedNumericDocValuesRangeQuery extends NumericDocValuesRangeQueryfinal SortField primarySortField;if (singleton != null) {- if (skipper != null) {- if ((primarySortField = densePrimarySort(context.reader(), skipper)) != null) {- return getScorerSupplierFromDensePrimarySort(singleton, skipper, primarySortField);- }- // Use batch iterator for bulk block evaluation via intoBitSet()- return ConstantScoreScorerSupplier.fromIterator(- new BatchDocValuesRangeIterator(singleton, skipper, lowerValue, upperValue),- score(),- scoreMode,- maxDoc);+ if (skipper != null+ && (primarySortField = densePrimarySort(context.reader(), skipper)) != null) {+ return getScorerSupplierFromDensePrimarySort(singleton, skipper, primarySortField);}+ // A single two-phase iterator covers every density: its approximation rides the skipper+ // (no over-scan), its intoBitSet bulk-evaluates dense blocks, and YES runs collect as+ // ranges. The bulk scorer unwraps it and picks the right strategy from there.return ConstantScoreScorerSupplier.fromIterator(TwoPhaseIterator.asDocIdSetIterator(DocValuesRangeIterator.forRange(singleton, skipper, lowerValue, upperValue)),diff --git a/lucene/core/src/java/org/apache/lucene/search/BatchDocValuesRangeIterator.java b/lucene/core/src/java/org/apache/lucene/search/BatchDocValuesRangeIterator.javadeleted file mode 100644index 9ad12fc2015..00000000000--- a/lucene/core/src/java/org/apache/lucene/search/BatchDocValuesRangeIterator.java+++ /dev/null@@ -1,177 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.search;--import java.io.IOException;-import org.apache.lucene.index.DocValuesSkipper;-import org.apache.lucene.index.NumericDocValues;-import org.apache.lucene.util.FixedBitSet;--/**- * A {@link DocIdSetIterator} for numeric doc values range queries that batch-evaluates values for- * MAYBE blocks. Instead of checking one doc at a time through a {@link- * org.apache.lucene.search.TwoPhaseIterator}, this iterator reads values in a tight loop and sets- * bits directly in a {@link FixedBitSet}, enabling the {@link DenseConjunctionBulkScorer} to use- * the faster bitset intersection path.- *- * <p>This is used for single-valued numeric fields with a skip index.- */-public final class BatchDocValuesRangeIterator extends DocIdSetIterator {-- private final SkipBlockRangeIterator blockIterator;- private final NumericDocValues values;- private final long minValue;- private final long maxValue;- private int doc = -1;-- public BatchDocValuesRangeIterator(- NumericDocValues values, DocValuesSkipper skipper, long minValue, long maxValue) {- this.blockIterator = new SkipBlockRangeIterator(skipper, minValue, maxValue);- this.values = values;- this.minValue = minValue;- this.maxValue = maxValue;- }-- @Override- public int docID() {- return doc;- }-- @Override- public int nextDoc() throws IOException {- return advance(doc + 1);- }-- @Override- public int advance(int target) throws IOException {- int blockDoc = blockIterator.docID();- if (blockDoc < target) {- blockDoc = blockIterator.advance(target);- }- if (blockDoc == NO_MORE_DOCS) {- return doc = NO_MORE_DOCS;- }-- // For YES blocks, all docs have values in range, so return the first doc- if (blockIterator.getMatch() == SkipBlockRangeIterator.Match.YES) {- return doc = blockDoc;- }-- // Scan forward through YES_IF_PRESENT and MAYBE blocks to find a matching doc.- // - YES_IF_PRESENT: all values are in range, but some docs may not have a value.- // We only need to check presence via advanceExact().- // - MAYBE: docs may or may not have a value, and values may or may not be in range.- // We need both a presence check (advanceExact) and a range check (longValue).- int docToCheck = Math.max(target, blockDoc);- int currentBlockEnd = blockIterator.blockEnd();- while (docToCheck != NO_MORE_DOCS) {- if (values.advanceExact(docToCheck)) {- // If we landed in a YES_IF_PRESENT block, skip the range check- if (blockIterator.getMatch() == SkipBlockRangeIterator.Match.YES_IF_PRESENT) {- return doc = docToCheck;- }- // This is a MAYBE block. We need to verify if the value is in range- long v = values.longValue();- if (v >= minValue && v <= maxValue) {- return doc = docToCheck;- }- }- docToCheck++;- // Check if we've left the current block- if (docToCheck >= currentBlockEnd) {- // Move to next matching block- blockDoc = blockIterator.advance(docToCheck);- if (blockDoc == NO_MORE_DOCS) {- return doc = NO_MORE_DOCS;- }- docToCheck = blockDoc;- if (blockIterator.getMatch() == SkipBlockRangeIterator.Match.YES) {- return doc = docToCheck;- }- currentBlockEnd = blockIterator.blockEnd();- }- }- return doc = NO_MORE_DOCS;- }-- @Override- public long cost() {- return values.cost();- }-- @Override- public int docIDRunEnd() throws IOException {- return blockIterator.docIDRunEnd();- }-- @Override- public void intoBitSet(int upTo, FixedBitSet bitSet, int offset) throws IOException {- while (doc < upTo) {- // Advance block iterator if needed- if (blockIterator.docID() < doc) {- blockIterator.advance(doc);- }- if (blockIterator.docID() >= upTo || blockIterator.docID() == NO_MORE_DOCS) {- doc = blockIterator.docID() == NO_MORE_DOCS ? NO_MORE_DOCS : blockIterator.docID();- return;- }-- int blockStart = Math.max(doc, blockIterator.docID());- SkipBlockRangeIterator.Match match = blockIterator.getMatch();-- // Use blockEnd() for MAYBE blocks since docIDRunEnd() is conservative (returns doc+1)- int blockEnd =- match == SkipBlockRangeIterator.Match.MAYBE- ? Math.min(upTo, blockIterator.blockEnd())- : Math.min(upTo, blockIterator.docIDRunEnd());-- switch (match) {- case YES:- // All docs in this range match — set all bits- bitSet.set(blockStart - offset, blockEnd - offset);- break;-- case YES_IF_PRESENT:- // All values in this block are in range, but the field is sparse so some docs- // may not have a value. No range check needed here.- for (int d = blockStart; d < blockEnd; d++) {- if (values.advanceExact(d)) {- bitSet.set(d - offset);- }- }- break;-- case MAYBE:- // Use rangeIntoBitSet — SIMD bulk evaluation for the full block.- // For dense fields, this bypasses advanceExact() overhead entirely.- values.rangeIntoBitSet(blockStart, blockEnd, minValue, maxValue, bitSet, offset);- break;- }-- // Move past this block- doc = blockEnd;- if (doc <= upTo) {- blockIterator.advance(doc);- if (blockIterator.docID() == NO_MORE_DOCS) {- doc = NO_MORE_DOCS;- return;- }- doc = blockIterator.docID();- }- }- }-}diff --git a/lucene/core/src/java/org/apache/lucene/search/ConstantScoreBulkScorer.java b/lucene/core/src/java/org/apache/lucene/search/ConstantScoreBulkScorer.javaindex 024e2f911a4..79c04200997 100644--- a/lucene/core/src/java/org/apache/lucene/search/ConstantScoreBulkScorer.java+++ b/lucene/core/src/java/org/apache/lucene/search/ConstantScoreBulkScorer.java@@ -28,17 +28,38 @@ import org.apache.lucene.util.MathUtil;final class ConstantScoreBulkScorer extends BulkScorer {private final Scorer scorer;private final DocIdSetIterator iterator;+ // When non-null, matches of the approximation must be confirmed via twoPhase#matches() /+ // twoPhase#intoBitSet rather than taken as-is.+ private final TwoPhaseIterator twoPhase;private final FixedBitSet windowMatches = new FixedBitSet(DenseConjunctionBulkScorer.WINDOW_SIZE);ConstantScoreBulkScorer(float score, ScoreMode scoreMode, DocIdSetIterator iterator) {+ this(score, scoreMode, iterator, null);+ }++ /**+ * Variant that confirms the matches of a {@link TwoPhaseIterator}. It is worthwhile when the+ * two-phase iterator overrides {@link TwoPhaseIterator#intoBitSet} with a bulk implementation, so+ * a single clause can be collected window-by-window instead of confirmed one doc at a time.+ */+ ConstantScoreBulkScorer(float score, ScoreMode scoreMode, TwoPhaseIterator twoPhase) {+ this(score, scoreMode, twoPhase.approximation(), twoPhase);+ }++ private ConstantScoreBulkScorer(+ float score, ScoreMode scoreMode, DocIdSetIterator iterator, TwoPhaseIterator twoPhase) {if (scoreMode.needsScores()) {throw new IllegalArgumentException("ScoreMode must not need scores: " + scoreMode);}- if (TwoPhaseIterator.unwrap(iterator) != null) {+ if (twoPhase == null && TwoPhaseIterator.unwrap(iterator) != null) {throw new IllegalArgumentException("Iterator must not wrap a TwoPhaseIterator");}- this.scorer = new ConstantScoreScorer(score, scoreMode, iterator);+ this.scorer =+ twoPhase == null+ ? new ConstantScoreScorer(score, scoreMode, iterator)+ : new ConstantScoreScorer(score, scoreMode, twoPhase);this.iterator = iterator;+ this.twoPhase = twoPhase;}@Override@@ -46,7 +67,7 @@ final class ConstantScoreBulkScorer extends BulkScorer {collector.setScorer(scorer);DocIdSetIterator competitiveIterator = collector.competitiveIterator();if (competitiveIterator != null) {- scoreIterator(collector, acceptDocs, iterator, competitiveIterator, min, max);+ scoreIterator(collector, acceptDocs, iterator, twoPhase, competitiveIterator, min, max);} else {scoreIteratorIntoBitSet(collector, acceptDocs, iterator, min, max);}@@ -62,6 +83,7 @@ final class ConstantScoreBulkScorer extends BulkScorer {LeafCollector collector,Bits acceptDocs,DocIdSetIterator iterator,+ TwoPhaseIterator twoPhase,DocIdSetIterator competitiveIterator,int min,int max)@@ -86,7 +108,7 @@ final class ConstantScoreBulkScorer extends BulkScorer {}}- if (acceptDocs == null || acceptDocs.get(doc)) {+ if ((acceptDocs == null || acceptDocs.get(doc)) && (twoPhase == null || twoPhase.matches())) {collector.collect(doc);}@@ -110,7 +132,11 @@ final class ConstantScoreBulkScorer extends BulkScorer {MathUtil.unsignedMin(max, windowBase + DenseConjunctionBulkScorer.WINDOW_SIZE);assert windowMatches.scanIsEmpty();- iterator.intoBitSet(windowMax, windowMatches, windowBase);+ if (twoPhase == null) {+ iterator.intoBitSet(windowMax, windowMatches, windowBase);+ } else {+ twoPhase.intoBitSet(windowMax, windowMatches, windowBase);+ }if (acceptDocs != null) {acceptDocs.applyMask(windowMatches, windowBase);diff --git a/lucene/core/src/java/org/apache/lucene/search/ConstantScoreScorerSupplier.java b/lucene/core/src/java/org/apache/lucene/search/ConstantScoreScorerSupplier.javaindex 48239201443..a8ff43ef9f2 100644--- a/lucene/core/src/java/org/apache/lucene/search/ConstantScoreScorerSupplier.java+++ b/lucene/core/src/java/org/apache/lucene/search/ConstantScoreScorerSupplier.java@@ -90,8 +90,13 @@ public abstract class ConstantScoreScorerSupplier extends ScorerSupplier {twoPhases = Collections.singletonList(twoPhase);}return new DenseConjunctionBulkScorer(iterators, twoPhases, maxDoc, score);- } else if (scoreMode.needsScores() == false && twoPhase == null) {- return new ConstantScoreBulkScorer(score, scoreMode, iterator);+ } else if (scoreMode.needsScores() == false) {+ // Collect window-by-window via intoBitSet. For a two-phase iterator this confirms matches in+ // its (possibly bulk) intoBitSet; the only overhead over a plain leap-frog is the reusable+ // window bit set, which buys batched live-docs masking and bulk collection in return.+ return twoPhase == null+ ? new ConstantScoreBulkScorer(score, scoreMode, iterator)+ : new ConstantScoreBulkScorer(score, scoreMode, twoPhase);} else {Scorer scorer =twoPhase == nulldiff --git a/lucene/core/src/java/org/apache/lucene/search/DenseConjunctionBulkScorer.java b/lucene/core/src/java/org/apache/lucene/search/DenseConjunctionBulkScorer.javaindex 08c71a81d7d..46d2c36394c 100644--- a/lucene/core/src/java/org/apache/lucene/search/DenseConjunctionBulkScorer.java+++ b/lucene/core/src/java/org/apache/lucene/search/DenseConjunctionBulkScorer.java@@ -51,6 +51,14 @@ final class DenseConjunctionBulkScorer extends BulkScorer {return twoPhase().docIDRunEnd();}}++ void intoBitSet(int upTo, FixedBitSet bitSet, int offset) throws IOException {+ if (twoPhase() == null) {+ approximation().intoBitSet(upTo, bitSet, offset);+ } else {+ twoPhase().intoBitSet(upTo, bitSet, offset);+ }+ }}// Use a small-ish window size to make sure that we can take advantage of gaps in the postings of@@ -67,8 +75,7 @@ final class DenseConjunctionBulkScorer extends BulkScorer {private final FixedBitSet windowMatches = new FixedBitSet(WINDOW_SIZE);private final FixedBitSet clauseWindowMatches = new FixedBitSet(WINDOW_SIZE);- private final List<DocIdSetIterator> windowApproximations = new ArrayList<>();- private final List<TwoPhaseIterator> windowTwoPhases = new ArrayList<>();+ private final List<DisiWrapper> windowClauses = new ArrayList<>();static DenseConjunctionBulkScorer of(List<Scorer> filters, int maxDoc, float constantScore) {List<DocIdSetIterator> iterators = new ArrayList<>();@@ -181,16 +188,13 @@ final class DenseConjunctionBulkScorer extends BulkScorer {for (DisiWrapper w : iterators) {int docIdRunEnd = w.docIDRunEnd();if (w.docID() > min || docIdRunEnd < minRunEndThreshold) {- windowApproximations.add(w.approximation());- if (w.twoPhase() != null) {- windowTwoPhases.add(w.twoPhase());- }+ windowClauses.add(w);} else {minDocIDRunEnd = Math.min(minDocIDRunEnd, docIdRunEnd);}}- if (acceptDocs == null && windowApproximations.isEmpty()) {+ if (acceptDocs == null && windowClauses.isEmpty()) {// We have a large range of doc IDs that all match.collector.collectRange(min, minDocIDRunEnd);return minDocIDRunEnd;@@ -198,15 +202,8 @@ final class DenseConjunctionBulkScorer extends BulkScorer {int bitsetWindowMax = MathUtil.unsignedMin(minDocIDRunEnd, WINDOW_SIZE + min);- if (windowTwoPhases.isEmpty()) {- scoreWindowUsingBitSet(collector, acceptDocs, windowApproximations, min, bitsetWindowMax);- } else {- windowTwoPhases.sort(Comparator.comparingDouble(TwoPhaseIterator::matchCost));- scoreWindowUsingLeapFrog(- collector, acceptDocs, windowApproximations, windowTwoPhases, min, bitsetWindowMax);- windowTwoPhases.clear();- }- windowApproximations.clear();+ scoreWindowUsingBitSet(collector, acceptDocs, windowClauses, min, bitsetWindowMax);+ windowClauses.clear();return bitsetWindowMax;}@@ -214,7 +211,7 @@ final class DenseConjunctionBulkScorer extends BulkScorer {private void scoreWindowUsingBitSet(LeafCollector collector,Bits acceptDocs,- List<DocIdSetIterator> iterators,+ List<DisiWrapper> iterators,int windowBase,int windowMax)throws IOException {@@ -226,9 +223,9 @@ final class DenseConjunctionBulkScorer extends BulkScorer {// This happens if all clauses fully matched the window and there are deleted docs.windowMatches.set(0, windowMax - windowBase);} else {- DocIdSetIterator lead = iterators.get(0);+ DisiWrapper lead = iterators.get(0);if (lead.docID() < windowBase) {- lead.advance(windowBase);+ lead.approximation().advance(windowBase);}lead.intoBitSet(windowMax, windowMatches, windowBase);}@@ -240,15 +237,42 @@ final class DenseConjunctionBulkScorer extends BulkScorer {int windowSize = windowMax - windowBase;int threshold = windowSize / DENSITY_THRESHOLD_INVERSE;+ // Above this many surviving docs, decoding a two-phase clause's whole window in one shot beats+ // confirming each survivor one at a time; below it we confirm survivors only.+ int bulkConfirmThreshold = windowSize / 4;int upTo = 1; // the leading clause at index 0 is already applied- for (; upTo < iterators.size() && windowMatches.cardinality() >= threshold; upTo++) {- DocIdSetIterator other = iterators.get(upTo);+ for (int cardinality = windowMatches.cardinality();+ upTo < iterators.size() && cardinality >= threshold;+ upTo++, cardinality = windowMatches.cardinality()) {+ DisiWrapper other = iterators.get(upTo);if (other.docID() < windowBase) {- other.advance(windowBase);+ other.approximation().advance(windowBase);+ }+ TwoPhaseIterator twoPhase = other.twoPhase();+ if (twoPhase != null && cardinality < bulkConfirmThreshold) {+ // Sparse survivors + per-doc confirmation: confirm only the docs that survived the cheaper+ // clauses (the bit set gates matches()), never decoding a doc another clause excluded.+ DocIdSetIterator approximation = other.approximation();+ for (int windowMatch = windowMatches.nextSetBit(0);+ windowMatch != DocIdSetIterator.NO_MORE_DOCS; ) {+ int doc = windowBase + windowMatch;+ int otherDoc = approximation.docID();+ if (otherDoc < doc) {+ otherDoc = approximation.advance(doc);+ }+ if (otherDoc != doc || twoPhase.matches() == false) {+ windowMatches.clear(windowMatch);+ }+ windowMatch = advance(windowMatches, windowMatch + 1);+ }+ } else {+ // Dense survivors, or a plain iterator: load this clause's matches in bulk and intersect.+ // For a two-phase clause this still confirms matches() via its (possibly vectorized)+ // intoBitSet.+ other.intoBitSet(windowMax, clauseWindowMatches, windowBase);+ windowMatches.and(clauseWindowMatches);+ clauseWindowMatches.clear();}- other.intoBitSet(windowMax, clauseWindowMatches, windowBase);- windowMatches.and(clauseWindowMatches);- clauseWindowMatches.clear();}if (upTo < iterators.size()) {@@ -256,13 +280,15 @@ final class DenseConjunctionBulkScorer extends BulkScorer {// after applying a few clauses, we finish evaluating the intersection using the traditional// leap-frog approach. This proved important with a query such as "+secretary +of +state" on// wikibigall, where the intersection becomes sparse after intersecting "secretary" and- // "state".+ // "state". As the leap-frog only visits surviving docs, two-phase clauses confirm matches()+ // here only on docs that no cheaper clause already excluded.advanceHead:for (int windowMatch = windowMatches.nextSetBit(0);windowMatch != DocIdSetIterator.NO_MORE_DOCS; ) {int doc = windowBase + windowMatch;+ // First confirm every remaining approximation is on doc...for (int i = upTo; i < iterators.size(); ++i) {- DocIdSetIterator other = iterators.get(i);+ DocIdSetIterator other = iterators.get(i).approximation();int otherDoc = other.docID();if (otherDoc < doc) {otherDoc = other.advance(doc);@@ -272,6 +298,14 @@ final class DenseConjunctionBulkScorer extends BulkScorer {continue advanceHead;}}+ // ...then run the (more expensive) two-phase confirmations, only on surviving docs.+ for (int i = upTo; i < iterators.size(); ++i) {+ TwoPhaseIterator twoPhase = iterators.get(i).twoPhase();+ if (twoPhase != null && twoPhase.matches() == false) {+ windowMatch = advance(windowMatches, windowMatch + 1);+ continue advanceHead;+ }+ }collector.collect(doc);windowMatch = advance(windowMatches, windowMatch + 1);}@@ -282,76 +316,6 @@ final class DenseConjunctionBulkScorer extends BulkScorer {windowMatches.clear();}- private static void scoreWindowUsingLeapFrog(- LeafCollector collector,- Bits acceptDocs,- List<DocIdSetIterator> approximations,- List<TwoPhaseIterator> twoPhases,- int min,- int max)- throws IOException {- assert twoPhases.size() > 0;- assert approximations.size() >= twoPhases.size();-- if (approximations.size() == 1) {- // scoreWindowUsingLeapFrog is only used if there is at least one two-phase iterator, so our- // single clause is a two-phase iterator- assert twoPhases.size() == 1;- DocIdSetIterator approximation = approximations.get(0);- TwoPhaseIterator twoPhase = twoPhases.get(0);- if (approximation.docID() < min) {- approximation.advance(min);- }- for (int doc = approximation.docID(); doc < max; doc = approximation.nextDoc()) {- if ((acceptDocs == null || acceptDocs.get(doc)) && twoPhase.matches()) {- collector.collect(doc);- }- }- } else {- DocIdSetIterator lead1 = approximations.get(0);- DocIdSetIterator lead2 = approximations.get(1);-- if (lead1.docID() < min) {- lead1.advance(min);- }-- advanceHead:- for (int doc = lead1.docID(); doc < max; ) {- if (acceptDocs != null && acceptDocs.get(doc) == false) {- doc = lead1.nextDoc();- continue;- }- int doc2 = lead2.docID();- if (doc2 < doc) {- doc2 = lead2.advance(doc);- }- if (doc != doc2) {- doc = lead1.advance(Math.min(doc2, max));- continue;- }- for (int i = 2; i < approximations.size(); ++i) {- DocIdSetIterator other = approximations.get(i);- int docN = other.docID();- if (docN < doc) {- docN = other.advance(doc);- }- if (doc != docN) {- doc = lead1.advance(Math.min(docN, max));- continue advanceHead;- }- }- for (TwoPhaseIterator twoPhase : twoPhases) {- if (twoPhase.matches() == false) {- doc = lead1.nextDoc();- continue advanceHead;- }- }- collector.collect(doc);- doc = lead1.nextDoc();- }- }- }-@Overridepublic long cost() {return iterators.get(0).approximation().cost();diff --git a/lucene/core/src/java/org/apache/lucene/search/DocValuesRangeIterator.java b/lucene/core/src/java/org/apache/lucene/search/DocValuesRangeIterator.javaindex 918e7ffa580..c20c7ed7cd7 100644--- a/lucene/core/src/java/org/apache/lucene/search/DocValuesRangeIterator.java+++ b/lucene/core/src/java/org/apache/lucene/search/DocValuesRangeIterator.java@@ -23,6 +23,7 @@ import org.apache.lucene.index.SortedDocValues;import org.apache.lucene.index.SortedNumericDocValues;import org.apache.lucene.index.SortedSetDocValues;import org.apache.lucene.index.TermsEnum;+import org.apache.lucene.util.FixedBitSet;import org.apache.lucene.util.IOBooleanSupplier;import org.apache.lucene.util.LongBitSet;@@ -52,7 +53,14 @@ public abstract sealed class DocValuesRangeIterator extends TwoPhaseIterator {return skipper == null? new DocValuesValueRangeIterator(values, check, 2): new DocValuesBlockRangeIterator(- values, new SkipBlockRangeIterator(skipper, min, max), check, 2, false);+ values,+ new SkipBlockRangeIterator(skipper, min, max),+ check,+ 2,+ false,+ values,+ min,+ max);}/**@@ -245,6 +253,11 @@ public abstract sealed class DocValuesRangeIterator extends TwoPhaseIterator {private final IOBooleanSupplier predicate;private final float matchCost;private final boolean alwaysCheckPredicate;+ // Non-null only for single-valued numeric doc values, in which case a whole block can be+ // range-evaluated in one shot in intoBitSet rather than confirming matches() one doc at a time.+ private final NumericDocValues numericValues;+ private final long minValue;+ private final long maxValue;private DocValuesBlockRangeIterator(DocIdSetIterator disi,@@ -252,12 +265,27 @@ public abstract sealed class DocValuesRangeIterator extends TwoPhaseIterator {IOBooleanSupplier predicate,float matchCost,boolean alwaysCheckPredicate) {+ this(disi, blockIterator, predicate, matchCost, alwaysCheckPredicate, null, 0, 0);+ }++ private DocValuesBlockRangeIterator(+ DocIdSetIterator disi,+ SkipBlockRangeIterator blockIterator,+ IOBooleanSupplier predicate,+ float matchCost,+ boolean alwaysCheckPredicate,+ NumericDocValues numericValues,+ long minValue,+ long maxValue) {super(blockIterator);this.disi = disi;this.blockIterator = blockIterator;this.predicate = predicate;this.matchCost = matchCost;this.alwaysCheckPredicate = alwaysCheckPredicate;+ this.numericValues = numericValues;+ this.minValue = minValue;+ this.maxValue = maxValue;}private boolean advanceDisi(int target) throws IOException {@@ -287,6 +315,44 @@ public abstract sealed class DocValuesRangeIterator extends TwoPhaseIterator {return blockIterator.docIDRunEnd();}+ @Override+ public void intoBitSet(int upTo, FixedBitSet bitSet, int offset) throws IOException {+ if (numericValues == null) {+ // Generic doc values: confirm each candidate one doc at a time (the TwoPhaseIterator+ // default). Only single-valued numeric fields can be range-evaluated block at a time.+ super.intoBitSet(upTo, bitSet, offset);+ return;+ }+ while (blockIterator.docID() < upTo) {+ int blockStart = blockIterator.docID();+ SkipBlockRangeIterator.Match match = blockIterator.getMatch();+ // For MAYBE blocks docIDRunEnd() is conservative (doc+1), so use the full block boundary to+ // evaluate the whole block at once.+ int blockEnd =+ match == SkipBlockRangeIterator.Match.MAYBE+ ? Math.min(upTo, blockIterator.blockEnd())+ : Math.min(upTo, blockIterator.docIDRunEnd());+ switch (match) {+ case YES -> bitSet.set(blockStart - offset, blockEnd - offset);+ case YES_IF_PRESENT -> {+ // All present values are in range, but the field is sparse: set a bit for every+ // doc that has a value. Delegate to intoBitSet so dense codecs can bulk-set the run+ // rather than probing one doc at a time. Only advance forward; a preceding YES block+ // leaves the iterator behind blockStart, while MAYBE/YES_IF_PRESENT blocks leave it+ // at or past it.+ if (numericValues.docID() < blockStart) {+ numericValues.advance(blockStart);+ }+ numericValues.intoBitSet(blockEnd, bitSet, offset);+ }+ case MAYBE ->+ numericValues.rangeIntoBitSet(+ blockStart, blockEnd, minValue, maxValue, bitSet, offset);+ }+ blockIterator.advance(blockEnd);+ }+ }+@Overridepublic float matchCost() {return matchCost;diff --git a/lucene/core/src/java/org/apache/lucene/search/TwoPhaseIterator.java b/lucene/core/src/java/org/apache/lucene/search/TwoPhaseIterator.javaindex b36c0d6f5c1..f4b7410ec9f 100644--- a/lucene/core/src/java/org/apache/lucene/search/TwoPhaseIterator.java+++ b/lucene/core/src/java/org/apache/lucene/search/TwoPhaseIterator.java@@ -18,6 +18,7 @@ package org.apache.lucene.search;import java.io.IOException;import java.util.Objects;+import org.apache.lucene.util.FixedBitSet;/*** Returned by {@link Scorer#twoPhaseIterator()} to expose an approximation of a {@link@@ -121,4 +122,26 @@ public abstract class TwoPhaseIterator {public int docIDRunEnd() throws IOException {return approximation().docID();}++ /**+ * Load the doc IDs that both belong to the {@link #approximation()} and {@link #matches() match},+ * and are in {@code [approximation().docID(), upTo)}, into {@code bitSet}, the document whose ID+ * is {@code i} being stored at bit {@code i - offset}. Upon return the {@link #approximation()}+ * is positioned on the first doc that is {@code >= upTo}, mirroring {@link+ * DocIdSetIterator#intoBitSet}.+ *+ * <p>The default implementation walks the {@link #approximation()} and confirms each doc with+ * {@link #matches()} — i.e. it is functionally identical to the leap-frog evaluation, just+ * writing into a bit set. Implementations that can confirm matches in bulk — for instance a+ * vectorized scan over a columnar block — should override it; the bit set window is then+ * collected in one shot rather than one doc at a time.+ */+ public void intoBitSet(int upTo, FixedBitSet bitSet, int offset) throws IOException {+ DocIdSetIterator approximation = approximation();+ for (int doc = approximation.docID(); doc < upTo; doc = approximation.nextDoc()) {+ if (matches()) {+ bitSet.set(doc - offset);+ }+ }+ }}diff --git a/lucene/core/src/test/org/apache/lucene/search/TestBooleanScorer.java b/lucene/core/src/test/org/apache/lucene/search/TestBooleanScorer.javaindex 53d0e7961de..c3f084bc5f0 100644--- a/lucene/core/src/test/org/apache/lucene/search/TestBooleanScorer.java+++ b/lucene/core/src/test/org/apache/lucene/search/TestBooleanScorer.java@@ -260,7 +260,7 @@ public class TestBooleanScorer extends LuceneTestCase {1f, ScoreMode.COMPLETE, new CountingDocIdSetIterator(new int[] {1})));}- public void testConstantScoreScorerSupplierPreservesTwoPhaseIterator() throws IOException {+ public void testConstantScoreScorerSupplierBulkCollectsTwoPhaseWithMatcher() throws IOException {int[] docs = {1, 2, 3};CountingDocIdSetIterator approximation = new CountingDocIdSetIterator(docs);DocIdSetIterator iterator =@@ -290,17 +290,21 @@ public class TestBooleanScorer extends LuceneTestCase {}@Override- public void collect(DocIdStream stream) {- fail("two-phase iterators must preserve per-doc confirmation");+ public void collect(DocIdStream stream) throws IOException {+ stream.forEach(doc -> collected[count[0]++] = doc);}};+ // A no-score two-phase iterator is bulk-collected through ConstantScoreBulkScorer; its matcher+ // is preserved because TwoPhaseIterator#intoBitSet confirms matches() while loading the window.BulkScorer bulkScorer =ConstantScoreScorerSupplier.fromIterator(iterator, 1f, ScoreMode.COMPLETE_NO_SCORES, 9000).bulkScorer();- assertThat(bulkScorer, instanceOf(DefaultBulkScorer.class));+ assertThat(bulkScorer, instanceOf(ConstantScoreBulkScorer.class));assertEquals(DocIdSetIterator.NO_MORE_DOCS, bulkScorer.score(collector, null, 0, 9000));+ // doc 2 is excluded by matches(), so the matcher is honored even via the bulk path.assertArrayEquals(new int[] {1, 3}, Arrays.copyOf(collected, count[0]));+ // Confirmation runs through matches(), not the approximation's own bulk intoBitSet.assertEquals(0, approximation.intoBitSetCalls);}diff --git a/lucene/core/src/test/org/apache/lucene/search/TestConstantScoreBulkScorer.java b/lucene/core/src/test/org/apache/lucene/search/TestConstantScoreBulkScorer.javanew file mode 100644index 00000000000..290503742da--- /dev/null+++ b/lucene/core/src/test/org/apache/lucene/search/TestConstantScoreBulkScorer.java@@ -0,0 +1,217 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.search;++import java.io.IOException;+import org.apache.lucene.tests.search.AssertingBulkScorer;+import org.apache.lucene.tests.search.RandomApproximationQuery.RandomTwoPhaseView;+import org.apache.lucene.tests.util.LuceneTestCase;+import org.apache.lucene.tests.util.TestUtil;+import org.apache.lucene.util.BitSetIterator;+import org.apache.lucene.util.Bits;+import org.apache.lucene.util.FixedBitSet;++public class TestConstantScoreBulkScorer extends LuceneTestCase {++ /**+ * A two-phase iterator that advertises a bulk {@link TwoPhaseIterator#intoBitSet}. Its+ * approximation is a superset of the actual matches, so {@link #matches()} genuinely filters.+ */+ private static class BulkTwoPhase extends TwoPhaseIterator {+ private final FixedBitSet matchSet;++ BulkTwoPhase(FixedBitSet approximation, FixedBitSet matchSet) {+ super(new BitSetIterator(approximation, approximation.approximateCardinality()));+ this.matchSet = matchSet;+ }++ @Override+ public boolean matches() {+ return matchSet.get(approximation.docID());+ }++ @Override+ public float matchCost() {+ return 1f;+ }++ @Override+ public void intoBitSet(int upTo, FixedBitSet bitSet, int offset) throws IOException {+ for (int doc = approximation.docID(); doc < upTo; doc = approximation.nextDoc()) {+ if (matchSet.get(doc)) {+ bitSet.set(doc - offset);+ }+ }+ }+ }++ private FixedBitSet modBitSet(int maxDoc, int step) {+ FixedBitSet bits = new FixedBitSet(maxDoc);+ for (int i = random().nextInt(step); i < maxDoc; i += step) {+ bits.set(i);+ }+ return bits;+ }++ private FixedBitSet supersetOf(FixedBitSet matchSet, int maxDoc) {+ FixedBitSet approx = matchSet.clone();+ approx.or(modBitSet(maxDoc, TestUtil.nextInt(random(), 2, 8)));+ return approx;+ }++ private static FixedBitSet collect(BulkScorer scorer, int maxDoc, Bits acceptDocs)+ throws IOException {+ FixedBitSet result = new FixedBitSet(maxDoc);+ scorer.score(+ new LeafCollector() {+ @Override+ public void setScorer(Scorable scorer) throws IOException {}++ @Override+ public void collect(int doc) throws IOException {+ result.set(doc);+ }+ },+ acceptDocs,+ 0,+ DocIdSetIterator.NO_MORE_DOCS);+ return result;+ }++ public void testBulkTwoPhaseCollectsExactMatches() throws IOException {+ int maxDoc = 100_000;+ FixedBitSet match = modBitSet(maxDoc, TestUtil.nextInt(random(), 2, 50));+ BulkScorer scorer =+ new ConstantScoreBulkScorer(+ 1f, ScoreMode.COMPLETE_NO_SCORES, new BulkTwoPhase(supersetOf(match, maxDoc), match));+ // AssertingBulkScorer randomly splits the scored range into smaller ranges.+ scorer = AssertingBulkScorer.wrap(random(), scorer, maxDoc);+ assertEquals(match, collect(scorer, maxDoc, null));+ }++ public void testBulkTwoPhaseWithAcceptDocs() throws IOException {+ int maxDoc = 100_000;+ FixedBitSet match = modBitSet(maxDoc, 5);+ FixedBitSet acceptDocs = modBitSet(maxDoc, 2);+ BulkScorer scorer =+ new ConstantScoreBulkScorer(+ 1f, ScoreMode.COMPLETE_NO_SCORES, new BulkTwoPhase(supersetOf(match, maxDoc), match));+ scorer = AssertingBulkScorer.wrap(random(), scorer, maxDoc);+ FixedBitSet expected = match.clone();+ expected.and(acceptDocs);+ assertEquals(expected, collect(scorer, maxDoc, acceptDocs));+ }++ /** The competitive-iterator path must confirm matches() before collecting. */+ public void testBulkTwoPhaseWithCompetitiveIterator() throws IOException {+ int maxDoc = 100_000;+ FixedBitSet match = modBitSet(maxDoc, 5);+ FixedBitSet competitive = modBitSet(maxDoc, 7);+ BulkScorer scorer =+ new ConstantScoreBulkScorer(+ 1f, ScoreMode.COMPLETE_NO_SCORES, new BulkTwoPhase(supersetOf(match, maxDoc), match));++ FixedBitSet result = new FixedBitSet(maxDoc);+ scorer.score(+ new LeafCollector() {+ private final DocIdSetIterator competitiveIt =+ new BitSetIterator(competitive, competitive.approximateCardinality());++ @Override+ public void setScorer(Scorable scorer) throws IOException {}++ @Override+ public void collect(int doc) throws IOException {+ result.set(doc);+ }++ @Override+ public DocIdSetIterator competitiveIterator() {+ return competitiveIt;+ }+ },+ null,+ 0,+ DocIdSetIterator.NO_MORE_DOCS);++ FixedBitSet expected = match.clone();+ expected.and(competitive);+ assertEquals(expected, result);+ }++ public void testSupplierRoutesSparseBulkTwoPhase() throws IOException {+ int maxDoc = 100_000;+ // Sparse approximation (cardinality well below maxDoc/32) so it is not routed to+ // DenseConjunctionBulkScorer.+ FixedBitSet match = modBitSet(maxDoc, 1000);++ ConstantScoreScorerSupplier supplier =+ ConstantScoreScorerSupplier.fromIterator(+ TwoPhaseIterator.asDocIdSetIterator(new BulkTwoPhase(match.clone(), match)),+ 1f,+ ScoreMode.COMPLETE_NO_SCORES,+ maxDoc);+ BulkScorer bulkScorer = supplier.bulkScorer();+ assertTrue(+ "expected ConstantScoreBulkScorer but got " + bulkScorer.getClass().getName(),+ bulkScorer instanceof ConstantScoreBulkScorer);+ assertEquals(match, collect(bulkScorer, maxDoc, null));+ }++ public void testSupplierRoutesSparseNonBulkTwoPhase() throws IOException {+ int maxDoc = 100_000;+ FixedBitSet match = modBitSet(maxDoc, 1000);++ // Even a two-phase iterator that doesn't override intoBitSet is routed to+ // ConstantScoreBulkScorer:+ // its default intoBitSet confirms matches one doc at a time, so this is no worse than leap-frog+ // while still batching live-docs masking and collection.+ ConstantScoreScorerSupplier supplier =+ ConstantScoreScorerSupplier.fromIterator(+ TwoPhaseIterator.asDocIdSetIterator(+ new RandomTwoPhaseView(+ random(), new BitSetIterator(match, match.approximateCardinality()))),+ 1f,+ ScoreMode.COMPLETE_NO_SCORES,+ maxDoc);+ BulkScorer bulkScorer = supplier.bulkScorer();+ assertTrue(+ "expected ConstantScoreBulkScorer but got " + bulkScorer.getClass().getName(),+ bulkScorer instanceof ConstantScoreBulkScorer);+ assertEquals(match, collect(bulkScorer, maxDoc, null));+ }++ /**+ * The plain-iterator constructor must reject a {@link DocIdSetIterator} that wraps a {@link+ * TwoPhaseIterator}: such an iterator has to go through the two-phase constructor so its+ * (upTo-bounded) {@link TwoPhaseIterator#intoBitSet} confirms matches. If it slipped through as a+ * plain iterator we would scan the bare approximation and lose the over-scan guarantee. {@link+ * ConstantScoreScorerSupplier} already unwraps unconditionally, but this guard makes the+ * invariant explicit and fail-fast for any future caller.+ */+ public void testRejectsWrappedTwoPhaseOnPlainConstructor() {+ int maxDoc = 100_000;+ FixedBitSet match = modBitSet(maxDoc, 5);+ DocIdSetIterator wrapped =+ TwoPhaseIterator.asDocIdSetIterator(new BulkTwoPhase(supersetOf(match, maxDoc), match));+ IllegalArgumentException e =+ expectThrows(+ IllegalArgumentException.class,+ () -> new ConstantScoreBulkScorer(1f, ScoreMode.COMPLETE_NO_SCORES, wrapped));+ assertEquals("Iterator must not wrap a TwoPhaseIterator", e.getMessage());+ }+}diff --git a/lucene/core/src/test/org/apache/lucene/search/TestDenseConjunctionBulkScorer.java b/lucene/core/src/test/org/apache/lucene/search/TestDenseConjunctionBulkScorer.javaindex ff220e2d0e2..c05cb11a543 100644--- a/lucene/core/src/test/org/apache/lucene/search/TestDenseConjunctionBulkScorer.java+++ b/lucene/core/src/test/org/apache/lucene/search/TestDenseConjunctionBulkScorer.java@@ -528,6 +528,68 @@ public class TestDenseConjunctionBulkScorer extends LuceneTestCase {DocIdSetIterator.NO_MORE_DOCS);}+ /**+ * A two-phase clause that reports a real {@link TwoPhaseIterator#docIDRunEnd() run} — as {@link+ * DocValuesRangeIterator}'s block iterator does for fully-matching YES blocks — must be collected+ * via {@link LeafCollector#collectRange}, never materialized into a bit set or visited per-doc.+ * This guards the collectRange fast path for doc-values ranges: the two-phase unification routes+ * ranges through this scorer as two-phase clauses, so the run has to survive {@code+ * DisiWrapper#docIDRunEnd()} delegating to the two-phase. The existing two-phase tests use {@code+ * RandomTwoPhaseView}, whose approximation reports no run, so they exercise only the bit-set+ * path.+ */+ public void testTwoPhaseRunCollectedAsRange() throws IOException {+ int maxDoc = 100_000;+ TwoPhaseIterator twoPhase =+ new TwoPhaseIterator(DocIdSetIterator.range(10_000, 60_000)) {+ @Override+ public boolean matches() {+ return true; // every doc in the run matches, like a YES block+ }++ @Override+ public float matchCost() {+ return 1f;+ }++ @Override+ public int docIDRunEnd() throws IOException {+ return approximation().docIDRunEnd();+ }+ };+ BulkScorer scorer =+ new DenseConjunctionBulkScorer(+ Collections.emptyList(), Collections.singletonList(twoPhase), maxDoc, 0f);+ int[] collectedViaRange = {0};+ scorer.score(+ new LeafCollector() {+ @Override+ public void setScorer(Scorable scorer) throws IOException {}++ @Override+ public void collect(int doc) throws IOException {+ fail("a fully-matching run must be collected via collectRange, not per-doc");+ }++ @Override+ public void collect(DocIdStream stream) throws IOException {+ fail(+ "a fully-matching run must be collected via collectRange, not as a bit-set stream");+ }++ @Override+ public void collectRange(int min, int max) throws IOException {+ assertTrue(min >= 10_000);+ assertTrue(max <= 60_000);+ collectedViaRange[0] += max - min;+ }+ },+ null,+ 0,+ DocIdSetIterator.NO_MORE_DOCS);+ assertEquals(50_000, collectedViaRange[0]);+ }+private static class CountingLeafCollector implements LeafCollector {int count;@@ -958,11 +1020,10 @@ public class TestDenseConjunctionBulkScorer extends LuceneTestCase {BulkScorer scorer =new DenseConjunctionBulkScorer(Collections.emptyList(), clauses, maxDoc, 0f);- // Matches are collected as a single DocIdStream+ // Matches arrive in order, as ranges and/or bit-set windows.scorer.score(new LeafCollector() {- private boolean called = false;private int expected = 30_000;@Override@@ -975,8 +1036,6 @@ public class TestDenseConjunctionBulkScorer extends LuceneTestCase {@Overridepublic void collect(DocIdStream stream) throws IOException {- assertFalse(called);- called = true;stream.forEach(doc -> {assertEquals(expected++, doc);@@ -985,7 +1044,6 @@ public class TestDenseConjunctionBulkScorer extends LuceneTestCase {@Overridepublic void finish() throws IOException {- assertTrue(called);assertEquals(60_001, expected);}},@@ -1023,11 +1081,10 @@ public class TestDenseConjunctionBulkScorer extends LuceneTestCase {new RandomTwoPhaseView(random(), clause3)),maxDoc,0f);- // Matches are collected as a single DocIdStream+ // Matches arrive in order, as ranges and/or bit-set windows.scorer.score(new LeafCollector() {- private boolean called = false;private int expected = 30_000;@Override@@ -1040,8 +1097,6 @@ public class TestDenseConjunctionBulkScorer extends LuceneTestCase {@Overridepublic void collect(DocIdStream stream) throws IOException {- assertFalse(called);- called = true;stream.forEach(doc -> {assertEquals(expected++, doc);@@ -1050,7 +1105,6 @@ public class TestDenseConjunctionBulkScorer extends LuceneTestCase {@Overridepublic void finish() throws IOException {- assertTrue(called);assertEquals(60_001, expected);}},diff --git a/lucene/core/src/test/org/apache/lucene/search/TestSkipBlockRangeIteratorIntoBitSet.java b/lucene/core/src/test/org/apache/lucene/search/TestSkipBlockRangeIteratorIntoBitSet.javaindex 6bc9d2e23db..858b185e29b 100644--- a/lucene/core/src/test/org/apache/lucene/search/TestSkipBlockRangeIteratorIntoBitSet.java+++ b/lucene/core/src/test/org/apache/lucene/search/TestSkipBlockRangeIteratorIntoBitSet.java@@ -34,8 +34,8 @@ import org.apache.lucene.store.Directory;import org.apache.lucene.util.FixedBitSet;/**- * Tests correctness of {@link BatchDocValuesRangeIterator} and its {@code intoBitSet()} path,- * including YES, YES_IF_PRESENT, and MAYBE block states.+ * Tests the {@code intoBitSet()} bulk path of {@link DocValuesRangeIterator} over single-valued+ * numeric doc values, including YES, YES_IF_PRESENT, and MAYBE block states.*/public class TestSkipBlockRangeIteratorIntoBitSet extends BaseDocValuesSkipperTests {@@ -77,10 +77,10 @@ public class TestSkipBlockRangeIteratorIntoBitSet extends BaseDocValuesSkipperTe}/**- * Verifies that SortedNumericDocValuesRangeQuery wires BatchDocValuesRangeIterator when the field- * has a skip index and is single-valued.+ * Verifies that SortedNumericDocValuesRangeQuery wires a DocValuesRangeIterator two-phase when+ * the field has a skip index and is single-valued.*/- public void testBatchIteratorIsWired() throws Exception {+ public void testTwoPhaseIteratorIsWired() throws Exception {LeafReaderContext ctx = reader.leaves().get(0);// Use a range that won't be rewritten to MatchAll or MatchNoneQuery q = SortedNumericDocValuesField.newSlowRangeQuery("age", 20, 40);@@ -92,11 +92,12 @@ public class TestSkipBlockRangeIteratorIntoBitSet extends BaseDocValuesSkipperTe"ScorerSupplier must be a ConstantScoreScorerSupplier",ss instanceof ConstantScoreScorerSupplier);DocIdSetIterator iter = ((ConstantScoreScorerSupplier) ss).iterator(Long.MAX_VALUE);+ TwoPhaseIterator twoPhase = TwoPhaseIterator.unwrap(iter);assertTrue(- "Range query on single-valued field with skip index must use BatchDocValuesRangeIterator"+ "Range query on single-valued field with skip index must use a DocValuesRangeIterator,"+ " but got: "- + iter.getClass().getSimpleName(),- iter instanceof BatchDocValuesRangeIterator);+ + (twoPhase == null ? iter.getClass() : twoPhase.getClass()).getSimpleName(),+ twoPhase instanceof DocValuesRangeIterator);}public void testSingleFieldRangeCorrectness() throws Exception {@@ -171,17 +172,19 @@ public class TestSkipBlockRangeIteratorIntoBitSet extends BaseDocValuesSkipperTeNumericDocValues values = docValues(queryMin, queryMax);DocValuesSkipper skipper = docValuesSkipper(queryMin, queryMax, true);- BatchDocValuesRangeIterator iter =- new BatchDocValuesRangeIterator(values, skipper, queryMin, queryMax);- assertEquals(0, iter.nextDoc());+ DocValuesRangeIterator iter =+ DocValuesRangeIterator.forRange(values, skipper, queryMin, queryMax);+ DocIdSetIterator approximation = iter.approximation();+ assertEquals(0, approximation.nextDoc());FixedBitSet bitSet = new FixedBitSet(2048);iter.intoBitSet(128, bitSet, 0);assertEquals("All docs in the YES block must match", 128, bitSet.cardinality());assertTrue(- "docID must skip the NO block that starts at the block boundary, but was " + iter.docID(),- iter.docID() >= 512);+ "approximation must skip the NO block that starts at the block boundary, but was "+ + approximation.docID(),+ approximation.docID() >= 512);// Continue filling the bitset in a second window, as DenseConjunctionBulkScorer does.iter.intoBitSet(2048, bitSet, 0);@@ -232,8 +235,8 @@ public class TestSkipBlockRangeIteratorIntoBitSet extends BaseDocValuesSkipperTeNumericDocValues dv = ctx.reader().getNumericDocValues("age");assertNotNull("Field must have a skip index", skipper);- BatchDocValuesRangeIterator iter = new BatchDocValuesRangeIterator(dv, skipper, 20, 40);- iter.nextDoc();+ DocValuesRangeIterator iter = DocValuesRangeIterator.forRange(dv, skipper, 20, 40);+ iter.approximation().nextDoc();FixedBitSet actual = new FixedBitSet(windowSize);iter.intoBitSet(Math.min(maxDoc, windowSize), actual, 0);@@ -270,8 +273,8 @@ public class TestSkipBlockRangeIteratorIntoBitSet extends BaseDocValuesSkipperTeNumericDocValues dv = ctx.reader().getNumericDocValues("age");assertNotNull(skipper);- BatchDocValuesRangeIterator iter = new BatchDocValuesRangeIterator(dv, skipper, 0, 99);- iter.nextDoc();+ DocValuesRangeIterator iter = DocValuesRangeIterator.forRange(dv, skipper, 0, 99);+ iter.approximation().nextDoc();FixedBitSet bitSet = new FixedBitSet(maxDoc);iter.intoBitSet(maxDoc, bitSet, 0);@@ -302,8 +305,8 @@ public class TestSkipBlockRangeIteratorIntoBitSet extends BaseDocValuesSkipperTeNumericDocValues dv = ctx.reader().getNumericDocValues("age");assertNotNull(skipper);- BatchDocValuesRangeIterator iter = new BatchDocValuesRangeIterator(dv, skipper, 200, 300);- iter.nextDoc();+ DocValuesRangeIterator iter = DocValuesRangeIterator.forRange(dv, skipper, 200, 300);+ iter.approximation().nextDoc();FixedBitSet bitSet = new FixedBitSet(maxDoc);iter.intoBitSet(maxDoc, bitSet, 0);@@ -345,8 +348,8 @@ public class TestSkipBlockRangeIteratorIntoBitSet extends BaseDocValuesSkipperTeNumericDocValues dv = ctx.reader().getNumericDocValues("sparse");if (skipper == null) return;- BatchDocValuesRangeIterator iter = new BatchDocValuesRangeIterator(dv, skipper, 20, 40);- iter.nextDoc();+ DocValuesRangeIterator iter = DocValuesRangeIterator.forRange(dv, skipper, 20, 40);+ iter.approximation().nextDoc();FixedBitSet bitSet = new FixedBitSet(numDocs);iter.intoBitSet(numDocs, bitSet, 0);@@ -394,7 +397,8 @@ public class TestSkipBlockRangeIteratorIntoBitSet extends BaseDocValuesSkipperTeNumericDocValues dv = ctx.reader().getNumericDocValues("age");assertNotNull(skipper);- BatchDocValuesRangeIterator iter = new BatchDocValuesRangeIterator(dv, skipper, 20, 40);+ DocIdSetIterator iter =+ TwoPhaseIterator.asDocIdSetIterator(DocValuesRangeIterator.forRange(dv, skipper, 20, 40));List<Integer> actual = new ArrayList<>();for (int d = iter.nextDoc(); d != DocIdSetIterator.NO_MORE_DOCS; d = iter.nextDoc()) {actual.add(d);@@ -406,7 +410,8 @@ public class TestSkipBlockRangeIteratorIntoBitSet extends BaseDocValuesSkipperTe// verifying each advance lands on the correct next matching doc.skipper = ctx.reader().getDocValuesSkipper("age");dv = ctx.reader().getNumericDocValues("age");- iter = new BatchDocValuesRangeIterator(dv, skipper, 20, 40);+ iter =+ TwoPhaseIterator.asDocIdSetIterator(DocValuesRangeIterator.forRange(dv, skipper, 20, 40));Random rng = random();int idx = 0;while (idx < expected.size()) {@@ -422,7 +427,8 @@ public class TestSkipBlockRangeIteratorIntoBitSet extends BaseDocValuesSkipperTe// advance returns the next matching doc after the target.skipper = ctx.reader().getDocValuesSkipper("age");dv = ctx.reader().getNumericDocValues("age");- iter = new BatchDocValuesRangeIterator(dv, skipper, 20, 40);+ iter =+ TwoPhaseIterator.asDocIdSetIterator(DocValuesRangeIterator.forRange(dv, skipper, 20, 40));int prevDoc = -1;for (int i = 0; i < expected.size() && i < 20; i++) {int matchDoc = expected.get(i);@@ -441,7 +447,8 @@ public class TestSkipBlockRangeIteratorIntoBitSet extends BaseDocValuesSkipperTe// Test 4: advance() past all docs returns NO_MORE_DOCSskipper = ctx.reader().getDocValuesSkipper("age");dv = ctx.reader().getNumericDocValues("age");- iter = new BatchDocValuesRangeIterator(dv, skipper, 20, 40);+ iter =+ TwoPhaseIterator.asDocIdSetIterator(DocValuesRangeIterator.forRange(dv, skipper, 20, 40));int doc = iter.advance(ctx.reader().maxDoc());assertEquals(DocIdSetIterator.NO_MORE_DOCS, doc);}@@ -460,9 +467,9 @@ public class TestSkipBlockRangeIteratorIntoBitSet extends BaseDocValuesSkipperTe}/**- * Tests BatchDocValuesRangeIterator using the fake NumericDocValues and DocValuesSkipper from- * BaseDocValuesSkipperTests, which exercises all block types (YES, NO, MAYBE, YES_IF_PRESENT)- * across both dense and sparse regions.+ * Tests the DocValuesRangeIterator iteration and intoBitSet paths using the fake NumericDocValues+ * and DocValuesSkipper from BaseDocValuesSkipperTests, which exercises all block types (YES, NO,+ * MAYBE, YES_IF_PRESENT) across both dense and sparse regions.*/public void testAllBlockTypesWithFakeSkipper() throws Exception {long queryMin = 10;@@ -484,8 +491,9 @@ public class TestSkipBlockRangeIteratorIntoBitSet extends BaseDocValuesSkipperTe// Test nextDoc() through all block typesNumericDocValues values = docValues(queryMin, queryMax);DocValuesSkipper skipper = docValuesSkipper(queryMin, queryMax, true);- BatchDocValuesRangeIterator iter =- new BatchDocValuesRangeIterator(values, skipper, queryMin, queryMax);+ DocIdSetIterator iter =+ TwoPhaseIterator.asDocIdSetIterator(+ DocValuesRangeIterator.forRange(values, skipper, queryMin, queryMax));List<Integer> actual = new ArrayList<>();for (int d = iter.nextDoc(); d != DocIdSetIterator.NO_MORE_DOCS; d = iter.nextDoc()) {actual.add(d);@@ -500,7 +508,9 @@ public class TestSkipBlockRangeIteratorIntoBitSet extends BaseDocValuesSkipperTe// - advance past endvalues = docValues(queryMin, queryMax);skipper = docValuesSkipper(queryMin, queryMax, true);- iter = new BatchDocValuesRangeIterator(values, skipper, queryMin, queryMax);+ iter =+ TwoPhaseIterator.asDocIdSetIterator(+ DocValuesRangeIterator.forRange(values, skipper, queryMin, queryMax));// advance into YES block — should land exactly on targetassertEquals(50, iter.advance(50));@@ -520,11 +530,12 @@ public class TestSkipBlockRangeIteratorIntoBitSet extends BaseDocValuesSkipperTe// Test intoBitSet() across all block typesvalues = docValues(queryMin, queryMax);skipper = docValuesSkipper(queryMin, queryMax, true);- iter = new BatchDocValuesRangeIterator(values, skipper, queryMin, queryMax);- int firstDoc = iter.nextDoc();+ DocValuesRangeIterator rangeIter =+ DocValuesRangeIterator.forRange(values, skipper, queryMin, queryMax);+ int firstDoc = rangeIter.approximation().nextDoc();FixedBitSet bitSet = new FixedBitSet(2048);- iter.intoBitSet(2048, bitSet, 0);+ rangeIter.intoBitSet(2048, bitSet, 0);// All expected docs after firstDoc should be setfor (int expectedDoc : expected) {