Apache Lucene open-source search software
Revert TwoPhaseIterator#docIDRunEnd default to approximation().docID()
Jim Ferenczi6f3c1d7ad69a
diff --git a/lucene/core/src/java/org/apache/lucene/search/ReqExclBulkScorer.java b/lucene/core/src/java/org/apache/lucene/search/ReqExclBulkScorer.javaindex 42f7110cd8e..2fbe3ea5558 100644--- a/lucene/core/src/java/org/apache/lucene/search/ReqExclBulkScorer.java+++ b/lucene/core/src/java/org/apache/lucene/search/ReqExclBulkScorer.java@@ -61,9 +61,10 @@ final class ReqExclBulkScorer extends BulkScorer {// from upTo to docIdRunEnd() are excluded, so we scored up to docIdRunEnd()upTo = Math.min(exclApproximation.docIDRunEnd(), max);} else if (exclTwoPhase.matches()) {- // upTo is excluded; skip the whole run of consecutive excluded docs at once, like the- // non-two-phase branch above. Clamp to at least upTo+1 to guarantee progress in case a- // TwoPhaseIterator returns a run end that does not move past the current doc.+ // upTo is excluded (matches() just confirmed it), so skip the whole run of consecutive+ // excluded docs at once, like the non-two-phase branch above. The default+ // TwoPhaseIterator#docIDRunEnd() conservatively returns the current doc, so clamp to at+ // least upTo+1 to guarantee progress when the run end is not overridden.upTo = Math.max(upTo + 1, Math.min(exclTwoPhase.docIDRunEnd(), max));}exclDoc = exclApproximation.nextDoc();diff --git a/lucene/core/src/java/org/apache/lucene/search/TwoPhaseIterator.java b/lucene/core/src/java/org/apache/lucene/search/TwoPhaseIterator.javaindex 91988acd37e..0f495ec2f0c 100644--- a/lucene/core/src/java/org/apache/lucene/search/TwoPhaseIterator.java+++ b/lucene/core/src/java/org/apache/lucene/search/TwoPhaseIterator.java@@ -117,11 +117,18 @@ public abstract class TwoPhaseIterator {* <p><b>Note</b>: It is illegal to call this method when the approximation is exhausted or not* positioned.*- * <p>The default implementation assumes runs of a single doc ID and returns the current doc ID of- * the approximation plus one, mirroring {@link DocIdSetIterator#docIDRunEnd()}.+ * <p>The default implementation is conservative and returns the current doc ID of the+ * approximation, i.e. it reports no matching run. Unlike a positioned {@link DocIdSetIterator} --+ * whose current doc is by definition a match -- a positioned {@link TwoPhaseIterator} has not yet+ * confirmed that the current doc matches, so the default cannot claim it as part of a matching+ * run without a (potentially side-effecting) {@link #matches()} call. Callers must therefore+ * treat a return value equal to {@code approximation().docID()} as "no known matching run" and+ * fall back to per-doc {@link #matches()} confirmation. Implementations that can cheaply prove+ * that a run of docs matches -- e.g. a doc-values skipper reporting a fully-matching block --+ * should override this to return the real run end.*/public int docIDRunEnd() throws IOException {- return approximation().docID() + 1;+ return approximation().docID();}/**diff --git a/lucene/core/src/test/org/apache/lucene/search/TestDenseConjunctionBulkScorer.java b/lucene/core/src/test/org/apache/lucene/search/TestDenseConjunctionBulkScorer.javaindex c23e94d7ab0..e482b8e129c 100644--- a/lucene/core/src/test/org/apache/lucene/search/TestDenseConjunctionBulkScorer.java+++ b/lucene/core/src/test/org/apache/lucene/search/TestDenseConjunctionBulkScorer.java@@ -20,6 +20,7 @@ import java.io.IOException;import java.util.Arrays;import java.util.Collections;import java.util.List;+import java.util.function.IntPredicate;import org.apache.lucene.tests.search.AssertingBulkScorer;import org.apache.lucene.tests.search.RandomApproximationQuery.RandomTwoPhaseView;import org.apache.lucene.tests.util.LuceneTestCase;@@ -590,6 +591,80 @@ public class TestDenseConjunctionBulkScorer extends LuceneTestCase {assertEquals(50_000, collectedViaRange[0]);}+ /**+ * A two-phase clause that relies on the default {@link TwoPhaseIterator#docIDRunEnd()} — which+ * reports no matching run, because the current approximation doc is only a candidate until {@link+ * TwoPhaseIterator#matches()} confirms it — must produce the same matches however the doc space+ * is split into scoring windows. That partition invariance is what intra-segment search+ * concurrency relies on: a segment scored in one pass and the same segment scored as adjacent+ * sub-ranges must agree. A per-doc (width-1) partition is the strongest such split and forces+ * every window to go through {@link TwoPhaseIterator#matches()} confirmation rather than the+ * {@link LeafCollector#collectRange} fast path, which must never collect an unconfirmed doc.+ */+ public void testTwoPhaseDefaultRunEndIsPartitionInvariant() throws IOException {+ int maxDoc = 5_000;+ // The approximation is positioned on every doc; matches() confirms a non-trivial subset.+ IntPredicate matching = doc -> doc % 3 != 0;++ FixedBitSet expected = new FixedBitSet(maxDoc);+ for (int doc = 0; doc < maxDoc; doc++) {+ if (matching.test(doc)) {+ expected.set(doc);+ }+ }++ // Whole range in one pass.+ FixedBitSet singlePass = new FixedBitSet(maxDoc);+ newTwoPhaseScorer(maxDoc, matching)+ .score(collectInto(singlePass), null, 0, DocIdSetIterator.NO_MORE_DOCS);+ assertEquals(expected, singlePass);++ // Same range as adjacent width-1 windows must collect exactly the same docs.+ FixedBitSet perDoc = new FixedBitSet(maxDoc);+ BulkScorer scorer = newTwoPhaseScorer(maxDoc, matching);+ LeafCollector collector = collectInto(perDoc);+ for (int doc = 0; doc < maxDoc; doc++) {+ scorer.score(collector, null, doc, doc + 1);+ }+ assertEquals(expected, perDoc);+ }++ /** A single dense two-phase clause using the default (unoverridden) {@code docIDRunEnd()}. */+ private static BulkScorer newTwoPhaseScorer(int maxDoc, IntPredicate matching) {+ TwoPhaseIterator twoPhase =+ new TwoPhaseIterator(DocIdSetIterator.all(maxDoc)) {+ @Override+ public boolean matches() {+ return matching.test(approximation().docID());+ }++ @Override+ public float matchCost() {+ return 1f;+ }+ };+ return new DenseConjunctionBulkScorer(+ Collections.emptyList(), Collections.singletonList(twoPhase), maxDoc, 0f);+ }++ /** A collector that records every collected doc, whichever collection path delivers it. */+ private static LeafCollector collectInto(FixedBitSet bitSet) {+ return new LeafCollector() {+ @Override+ public void setScorer(Scorable scorer) throws IOException {}++ @Override+ public void collect(int doc) throws IOException {+ bitSet.set(doc);+ }++ @Override+ public void collect(DocIdStream stream) throws IOException {+ stream.forEach(bitSet::set);+ }+ };+ }+private static class CountingLeafCollector implements LeafCollector {int count;