Skip to content

Apache Lucene open-source search software

Add docIDRunEnd contract tests for DV formats (#16513)

Tim Allison511cc8bf7231
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index df93df3dc8a..ee6598db13a 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -281,6 +281,8 @@ Build
Other
---------------------
+* GITHUB#16513: Add docIDRunEnd contract tests for DV formats to BaseDocValuesFormatTestCase. (Tim Allison)
+
* GITHUB#16512: AssertingScorer's two-phase iterator now delegates docIDRunEnd(),
so it exercises that contract in randomized tests. (Tim Allison)
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseDocValuesFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseDocValuesFormatTestCase.java
index 70bff46d76c..83f4e2ffe81 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseDocValuesFormatTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseDocValuesFormatTestCase.java
@@ -53,13 +53,17 @@ import org.apache.lucene.index.Term;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.index.TermsEnum.SeekStatus;
import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.DocValuesRangeIterator;
+import org.apache.lucene.search.TwoPhaseIterator;
import org.apache.lucene.store.Directory;
import org.apache.lucene.tests.analysis.MockAnalyzer;
import org.apache.lucene.tests.util.TestUtil;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.FixedBitSet;
+import org.apache.lucene.util.IOBooleanSupplier;
import org.apache.lucene.util.IOFunction;
import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.LongBitSet;
/**
* Extends {@link LegacyBaseDocValuesFormatTestCase} and adds checks for {@link DocValuesSkipper}.
@@ -1089,4 +1093,242 @@ public abstract class BaseDocValuesFormatTestCase extends LegacyBaseDocValuesFor
}
}
}
+
+ public void testDocIDRunEndNumericRange() throws Exception {
+ doTestDocIDRunEnd(
+ false,
+ (doc, v) -> doc.add(NumericDocValuesField.indexedField("f", v)),
+ (leaf, domain) -> {
+ NumericDocValues values = leaf.getNumericDocValues("f");
+ if (values == null) {
+ return;
+ }
+ long a = TestUtil.nextLong(random(), 0, domain);
+ long b = TestUtil.nextLong(random(), 0, domain);
+ long min = Math.min(a, b);
+ long max = Math.max(a, b);
+ TwoPhaseIterator tpi =
+ DocValuesRangeIterator.forRange(values, leaf.getDocValuesSkipper("f"), min, max);
+ NumericDocValues truth = leaf.getNumericDocValues("f");
+ FixedBitSet expected =
+ expectedMatches(
+ truth, () -> truth.longValue() >= min && truth.longValue() <= max, leaf.maxDoc());
+ assertDocIDRunEnds(tpi, expected, leaf.maxDoc());
+ });
+ }
+
+ public void testDocIDRunEndSortedNumericRange() throws Exception {
+ doTestDocIDRunEnd(
+ true,
+ (doc, v) -> doc.add(SortedNumericDocValuesField.indexedField("f", v)),
+ (leaf, domain) -> {
+ SortedNumericDocValues values = leaf.getSortedNumericDocValues("f");
+ if (values == null) {
+ return;
+ }
+ long a = TestUtil.nextLong(random(), 0, domain);
+ long b = TestUtil.nextLong(random(), 0, domain);
+ long min = Math.min(a, b);
+ long max = Math.max(a, b);
+ TwoPhaseIterator tpi =
+ DocValuesRangeIterator.forRange(values, leaf.getDocValuesSkipper("f"), min, max);
+ SortedNumericDocValues truth = leaf.getSortedNumericDocValues("f");
+ FixedBitSet expected =
+ expectedMatches(
+ truth,
+ () -> {
+ for (int j = 0; j < truth.docValueCount(); j++) {
+ long v = truth.nextValue();
+ if (v >= min && v <= max) {
+ return true;
+ }
+ }
+ return false;
+ },
+ leaf.maxDoc());
+ assertDocIDRunEnds(tpi, expected, leaf.maxDoc());
+ });
+ }
+
+ public void testDocIDRunEndSortedOrdinalRange() throws Exception {
+ doTestDocIDRunEnd(
+ false,
+ (doc, v) -> doc.add(SortedDocValuesField.indexedField("f", fixedWidthTerm(v))),
+ (leaf, _) -> {
+ SortedDocValues values = leaf.getSortedDocValues("f");
+ if (values == null || values.getValueCount() == 0) {
+ return;
+ }
+ int a = random().nextInt(values.getValueCount());
+ int b = random().nextInt(values.getValueCount());
+ int minOrd = Math.min(a, b);
+ int maxOrd = Math.max(a, b);
+ TwoPhaseIterator tpi =
+ DocValuesRangeIterator.forOrdinalRange(
+ values, leaf.getDocValuesSkipper("f"), minOrd, maxOrd);
+ SortedDocValues truth = leaf.getSortedDocValues("f");
+ FixedBitSet expected =
+ expectedMatches(
+ truth,
+ () -> truth.ordValue() >= minOrd && truth.ordValue() <= maxOrd,
+ leaf.maxDoc());
+ assertDocIDRunEnds(tpi, expected, leaf.maxDoc());
+ });
+ }
+
+ public void testDocIDRunEndSortedSetOrdinalRange() throws Exception {
+ doTestDocIDRunEnd(
+ true,
+ (doc, v) -> doc.add(SortedSetDocValuesField.indexedField("f", fixedWidthTerm(v))),
+ (leaf, _) -> checkSortedSetRunEnds(leaf, false));
+ }
+
+ public void testDocIDRunEndSortedSetOrdinalSet() throws Exception {
+ doTestDocIDRunEnd(
+ true,
+ (doc, v) -> doc.add(SortedSetDocValuesField.indexedField("f", fixedWidthTerm(v))),
+ (leaf, _) -> checkSortedSetRunEnds(leaf, true));
+ }
+
+ private void checkSortedSetRunEnds(LeafReader leaf, boolean ordinalSet) throws IOException {
+ SortedSetDocValues values = leaf.getSortedSetDocValues("f");
+ if (values == null || values.getValueCount() == 0) {
+ return;
+ }
+ long valueCount = values.getValueCount();
+ LongBitSet ords = new LongBitSet(valueCount);
+ long minOrd = -1;
+ long maxOrd = -1;
+ if (ordinalSet) {
+ // every other ordinal: non-contiguous, the GH#16450 shape
+ for (long o = random().nextInt(2); o < valueCount; o += 2) {
+ ords.set(o);
+ if (minOrd == -1) {
+ minOrd = o;
+ }
+ maxOrd = o;
+ }
+ if (minOrd == -1) {
+ return;
+ }
+ } else {
+ long a = TestUtil.nextLong(random(), 0, valueCount - 1);
+ long b = TestUtil.nextLong(random(), 0, valueCount - 1);
+ minOrd = Math.min(a, b);
+ maxOrd = Math.max(a, b);
+ ords.set(minOrd, maxOrd + 1);
+ }
+ TwoPhaseIterator tpi =
+ ordinalSet
+ ? DocValuesRangeIterator.forOrdinalSet(
+ values, leaf.getDocValuesSkipper("f"), minOrd, maxOrd, ords)
+ : DocValuesRangeIterator.forOrdinalRange(
+ values, leaf.getDocValuesSkipper("f"), minOrd, maxOrd);
+ SortedSetDocValues truth = leaf.getSortedSetDocValues("f");
+ FixedBitSet expected =
+ expectedMatches(
+ truth,
+ () -> {
+ for (int j = 0; j < truth.docValueCount(); j++) {
+ if (ords.get(truth.nextOrd())) {
+ return true;
+ }
+ }
+ return false;
+ },
+ leaf.maxDoc());
+ assertDocIDRunEnds(tpi, expected, leaf.maxDoc());
+ }
+
+ /** Brute-force oracle: walks {@code truth} and records every doc where {@code matches} holds. */
+ private static FixedBitSet expectedMatches(
+ DocIdSetIterator truth, IOBooleanSupplier matches, int maxDoc) throws IOException {
+ FixedBitSet expected = new FixedBitSet(maxDoc);
+ for (int d = truth.nextDoc(); d != NO_MORE_DOCS; d = truth.nextDoc()) {
+ if (matches.get()) {
+ expected.set(d);
+ }
+ }
+ return expected;
+ }
+
+ private interface RunEndDocPopulator {
+ void addValue(Document doc, long value);
+ }
+
+ private interface RunEndChecker {
+ void check(LeafReader leaf, long domain) throws IOException;
+ }
+
+ /**
+ * Indexes a random skip-indexed field whose values are drawn from a small domain (random or
+ * clustered in doc-ID order, with random gaps) and runs {@code checker} against every leaf.
+ */
+ private void doTestDocIDRunEnd(
+ boolean multiValued, RunEndDocPopulator populate, RunEndChecker checker) throws Exception {
+ int totalDocs = TestUtil.nextInt(random(), 1, 1500);
+ long domain = TestUtil.nextInt(random(), 5, 100);
+ boolean clustered = random().nextBoolean();
+ int missingOneIn = TestUtil.nextInt(random(), 0, 4);
+ try (Directory dir = newDirectory();
+ RandomIndexWriter w = new RandomIndexWriter(random(), dir)) {
+ for (int i = 0; i < totalDocs; i++) {
+ Document doc = new Document();
+ if (missingOneIn == 0 || random().nextInt(missingOneIn + 1) != 0) {
+ int count = multiValued ? TestUtil.nextInt(random(), 1, 2) : 1;
+ for (int j = 0; j < count; j++) {
+ long v =
+ clustered && j == 0
+ ? i * domain / totalDocs
+ : TestUtil.nextLong(random(), 0, domain);
+ populate.addValue(doc, v);
+ }
+ }
+ w.addDocument(doc);
+ }
+ if (random().nextBoolean()) {
+ w.forceMerge(1);
+ }
+ try (IndexReader r = w.getReader()) {
+ for (LeafReaderContext ctx : r.leaves()) {
+ for (int iter = 0; iter < 10; iter++) {
+ checker.check(ctx.reader(), domain);
+ }
+ }
+ }
+ }
+ }
+
+ private static BytesRef fixedWidthTerm(long value) {
+ // offset keeps the string width fixed so ordinal order matches value order
+ return new BytesRef(Long.toString(1000000L + value));
+ }
+
+ /**
+ * Verifies the {@link TwoPhaseIterator#docIDRunEnd()} contract: every doc in a reported run
+ * {@code [docID, runEnd)} truly matches, the call does not move the approximation, and outside
+ * runs {@link TwoPhaseIterator#matches()} agrees with brute force.
+ */
+ private static void assertDocIDRunEnds(TwoPhaseIterator tpi, FixedBitSet expected, int maxDoc)
+ throws IOException {
+ DocIdSetIterator approx = tpi.approximation();
+ for (int doc = approx.nextDoc(); doc != NO_MORE_DOCS; ) {
+ int runEnd = tpi.docIDRunEnd();
+ assertEquals("docIDRunEnd() must not move the approximation", doc, approx.docID());
+ assertTrue("docIDRunEnd() " + runEnd + " is below the current doc " + doc, runEnd >= doc);
+ assertTrue("docIDRunEnd() " + runEnd + " is beyond maxDoc " + maxDoc, runEnd <= maxDoc);
+ for (int d = doc; d < runEnd; d++) {
+ assertTrue(
+ "doc " + d + " in reported run [" + doc + ", " + runEnd + ") is not a true match",
+ expected.get(d));
+ }
+ if (runEnd > doc) {
+ doc = approx.advance(runEnd);
+ } else {
+ assertEquals(
+ "matches() disagrees with brute force on doc " + doc, expected.get(doc), tpi.matches());
+ doc = approx.nextDoc();
+ }
+ }
+ }
}