Skip to content

Apache Lucene open-source search software

Revert TwoPhaseIterator#docIDRunEnd default to approximation().docID()

Jim Ferenczi6f3c1d7ad69a
diff --git a/lucene/core/src/java/org/apache/lucene/search/ReqExclBulkScorer.java b/lucene/core/src/java/org/apache/lucene/search/ReqExclBulkScorer.java
index 42f7110cd8e..2fbe3ea5558 100644
--- a/lucene/core/src/java/org/apache/lucene/search/ReqExclBulkScorer.java
+++ b/lucene/core/src/java/org/apache/lucene/search/ReqExclBulkScorer.java
@@ -61,9 +61,10 @@ final class ReqExclBulkScorer extends BulkScorer {
// from upTo to docIdRunEnd() are excluded, so we scored up to docIdRunEnd()
upTo = Math.min(exclApproximation.docIDRunEnd(), max);
} else if (exclTwoPhase.matches()) {
- // upTo is excluded; skip the whole run of consecutive excluded docs at once, like the
- // non-two-phase branch above. Clamp to at least upTo+1 to guarantee progress in case a
- // TwoPhaseIterator returns a run end that does not move past the current doc.
+ // upTo is excluded (matches() just confirmed it), so skip the whole run of consecutive
+ // excluded docs at once, like the non-two-phase branch above. The default
+ // TwoPhaseIterator#docIDRunEnd() conservatively returns the current doc, so clamp to at
+ // least upTo+1 to guarantee progress when the run end is not overridden.
upTo = Math.max(upTo + 1, Math.min(exclTwoPhase.docIDRunEnd(), max));
}
exclDoc = exclApproximation.nextDoc();
diff --git a/lucene/core/src/java/org/apache/lucene/search/TwoPhaseIterator.java b/lucene/core/src/java/org/apache/lucene/search/TwoPhaseIterator.java
index 91988acd37e..0f495ec2f0c 100644
--- a/lucene/core/src/java/org/apache/lucene/search/TwoPhaseIterator.java
+++ b/lucene/core/src/java/org/apache/lucene/search/TwoPhaseIterator.java
@@ -117,11 +117,18 @@ public abstract class TwoPhaseIterator {
* <p><b>Note</b>: It is illegal to call this method when the approximation is exhausted or not
* positioned.
*
- * <p>The default implementation assumes runs of a single doc ID and returns the current doc ID of
- * the approximation plus one, mirroring {@link DocIdSetIterator#docIDRunEnd()}.
+ * <p>The default implementation is conservative and returns the current doc ID of the
+ * approximation, i.e. it reports no matching run. Unlike a positioned {@link DocIdSetIterator} --
+ * whose current doc is by definition a match -- a positioned {@link TwoPhaseIterator} has not yet
+ * confirmed that the current doc matches, so the default cannot claim it as part of a matching
+ * run without a (potentially side-effecting) {@link #matches()} call. Callers must therefore
+ * treat a return value equal to {@code approximation().docID()} as "no known matching run" and
+ * fall back to per-doc {@link #matches()} confirmation. Implementations that can cheaply prove
+ * that a run of docs matches -- e.g. a doc-values skipper reporting a fully-matching block --
+ * should override this to return the real run end.
*/
public int docIDRunEnd() throws IOException {
- return approximation().docID() + 1;
+ return approximation().docID();
}
/**
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestDenseConjunctionBulkScorer.java b/lucene/core/src/test/org/apache/lucene/search/TestDenseConjunctionBulkScorer.java
index c23e94d7ab0..e482b8e129c 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestDenseConjunctionBulkScorer.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestDenseConjunctionBulkScorer.java
@@ -20,6 +20,7 @@ import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
+import java.util.function.IntPredicate;
import org.apache.lucene.tests.search.AssertingBulkScorer;
import org.apache.lucene.tests.search.RandomApproximationQuery.RandomTwoPhaseView;
import org.apache.lucene.tests.util.LuceneTestCase;
@@ -590,6 +591,80 @@ public class TestDenseConjunctionBulkScorer extends LuceneTestCase {
assertEquals(50_000, collectedViaRange[0]);
}
+ /**
+ * A two-phase clause that relies on the default {@link TwoPhaseIterator#docIDRunEnd()} — which
+ * reports no matching run, because the current approximation doc is only a candidate until {@link
+ * TwoPhaseIterator#matches()} confirms it — must produce the same matches however the doc space
+ * is split into scoring windows. That partition invariance is what intra-segment search
+ * concurrency relies on: a segment scored in one pass and the same segment scored as adjacent
+ * sub-ranges must agree. A per-doc (width-1) partition is the strongest such split and forces
+ * every window to go through {@link TwoPhaseIterator#matches()} confirmation rather than the
+ * {@link LeafCollector#collectRange} fast path, which must never collect an unconfirmed doc.
+ */
+ public void testTwoPhaseDefaultRunEndIsPartitionInvariant() throws IOException {
+ int maxDoc = 5_000;
+ // The approximation is positioned on every doc; matches() confirms a non-trivial subset.
+ IntPredicate matching = doc -> doc % 3 != 0;
+
+ FixedBitSet expected = new FixedBitSet(maxDoc);
+ for (int doc = 0; doc < maxDoc; doc++) {
+ if (matching.test(doc)) {
+ expected.set(doc);
+ }
+ }
+
+ // Whole range in one pass.
+ FixedBitSet singlePass = new FixedBitSet(maxDoc);
+ newTwoPhaseScorer(maxDoc, matching)
+ .score(collectInto(singlePass), null, 0, DocIdSetIterator.NO_MORE_DOCS);
+ assertEquals(expected, singlePass);
+
+ // Same range as adjacent width-1 windows must collect exactly the same docs.
+ FixedBitSet perDoc = new FixedBitSet(maxDoc);
+ BulkScorer scorer = newTwoPhaseScorer(maxDoc, matching);
+ LeafCollector collector = collectInto(perDoc);
+ for (int doc = 0; doc < maxDoc; doc++) {
+ scorer.score(collector, null, doc, doc + 1);
+ }
+ assertEquals(expected, perDoc);
+ }
+
+ /** A single dense two-phase clause using the default (unoverridden) {@code docIDRunEnd()}. */
+ private static BulkScorer newTwoPhaseScorer(int maxDoc, IntPredicate matching) {
+ TwoPhaseIterator twoPhase =
+ new TwoPhaseIterator(DocIdSetIterator.all(maxDoc)) {
+ @Override
+ public boolean matches() {
+ return matching.test(approximation().docID());
+ }
+
+ @Override
+ public float matchCost() {
+ return 1f;
+ }
+ };
+ return new DenseConjunctionBulkScorer(
+ Collections.emptyList(), Collections.singletonList(twoPhase), maxDoc, 0f);
+ }
+
+ /** A collector that records every collected doc, whichever collection path delivers it. */
+ private static LeafCollector collectInto(FixedBitSet bitSet) {
+ return new LeafCollector() {
+ @Override
+ public void setScorer(Scorable scorer) throws IOException {}
+
+ @Override
+ public void collect(int doc) throws IOException {
+ bitSet.set(doc);
+ }
+
+ @Override
+ public void collect(DocIdStream stream) throws IOException {
+ stream.forEach(bitSet::set);
+ }
+ };
+ }
+
private static class CountingLeafCollector implements LeafCollector {
int count;