Apache Lucene open-source search software
Add docIDRunEnd contract tests for DV formats (#16513)
Tim Allison511cc8bf7231
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex df93df3dc8a..ee6598db13a 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -281,6 +281,8 @@ BuildOther---------------------+* GITHUB#16513: Add docIDRunEnd contract tests for DV formats to BaseDocValuesFormatTestCase. (Tim Allison)+* GITHUB#16512: AssertingScorer's two-phase iterator now delegates docIDRunEnd(),so it exercises that contract in randomized tests. (Tim Allison)diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseDocValuesFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseDocValuesFormatTestCase.javaindex 70bff46d76c..83f4e2ffe81 100644--- a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseDocValuesFormatTestCase.java+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseDocValuesFormatTestCase.java@@ -53,13 +53,17 @@ import org.apache.lucene.index.Term;import org.apache.lucene.index.TermsEnum;import org.apache.lucene.index.TermsEnum.SeekStatus;import org.apache.lucene.search.DocIdSetIterator;+import org.apache.lucene.search.DocValuesRangeIterator;+import org.apache.lucene.search.TwoPhaseIterator;import org.apache.lucene.store.Directory;import org.apache.lucene.tests.analysis.MockAnalyzer;import org.apache.lucene.tests.util.TestUtil;import org.apache.lucene.util.BytesRef;import org.apache.lucene.util.FixedBitSet;+import org.apache.lucene.util.IOBooleanSupplier;import org.apache.lucene.util.IOFunction;import org.apache.lucene.util.IOUtils;+import org.apache.lucene.util.LongBitSet;/*** Extends {@link LegacyBaseDocValuesFormatTestCase} and adds checks for {@link DocValuesSkipper}.@@ -1089,4 +1093,242 @@ public abstract class BaseDocValuesFormatTestCase extends LegacyBaseDocValuesFor}}}++ public void testDocIDRunEndNumericRange() throws Exception {+ doTestDocIDRunEnd(+ false,+ (doc, v) -> doc.add(NumericDocValuesField.indexedField("f", v)),+ (leaf, domain) -> {+ NumericDocValues values = leaf.getNumericDocValues("f");+ if (values == null) {+ return;+ }+ long a = TestUtil.nextLong(random(), 0, domain);+ long b = TestUtil.nextLong(random(), 0, domain);+ long min = Math.min(a, b);+ long max = Math.max(a, b);+ TwoPhaseIterator tpi =+ DocValuesRangeIterator.forRange(values, leaf.getDocValuesSkipper("f"), min, max);+ NumericDocValues truth = leaf.getNumericDocValues("f");+ FixedBitSet expected =+ expectedMatches(+ truth, () -> truth.longValue() >= min && truth.longValue() <= max, leaf.maxDoc());+ assertDocIDRunEnds(tpi, expected, leaf.maxDoc());+ });+ }++ public void testDocIDRunEndSortedNumericRange() throws Exception {+ doTestDocIDRunEnd(+ true,+ (doc, v) -> doc.add(SortedNumericDocValuesField.indexedField("f", v)),+ (leaf, domain) -> {+ SortedNumericDocValues values = leaf.getSortedNumericDocValues("f");+ if (values == null) {+ return;+ }+ long a = TestUtil.nextLong(random(), 0, domain);+ long b = TestUtil.nextLong(random(), 0, domain);+ long min = Math.min(a, b);+ long max = Math.max(a, b);+ TwoPhaseIterator tpi =+ DocValuesRangeIterator.forRange(values, leaf.getDocValuesSkipper("f"), min, max);+ SortedNumericDocValues truth = leaf.getSortedNumericDocValues("f");+ FixedBitSet expected =+ expectedMatches(+ truth,+ () -> {+ for (int j = 0; j < truth.docValueCount(); j++) {+ long v = truth.nextValue();+ if (v >= min && v <= max) {+ return true;+ }+ }+ return false;+ },+ leaf.maxDoc());+ assertDocIDRunEnds(tpi, expected, leaf.maxDoc());+ });+ }++ public void testDocIDRunEndSortedOrdinalRange() throws Exception {+ doTestDocIDRunEnd(+ false,+ (doc, v) -> doc.add(SortedDocValuesField.indexedField("f", fixedWidthTerm(v))),+ (leaf, _) -> {+ SortedDocValues values = leaf.getSortedDocValues("f");+ if (values == null || values.getValueCount() == 0) {+ return;+ }+ int a = random().nextInt(values.getValueCount());+ int b = random().nextInt(values.getValueCount());+ int minOrd = Math.min(a, b);+ int maxOrd = Math.max(a, b);+ TwoPhaseIterator tpi =+ DocValuesRangeIterator.forOrdinalRange(+ values, leaf.getDocValuesSkipper("f"), minOrd, maxOrd);+ SortedDocValues truth = leaf.getSortedDocValues("f");+ FixedBitSet expected =+ expectedMatches(+ truth,+ () -> truth.ordValue() >= minOrd && truth.ordValue() <= maxOrd,+ leaf.maxDoc());+ assertDocIDRunEnds(tpi, expected, leaf.maxDoc());+ });+ }++ public void testDocIDRunEndSortedSetOrdinalRange() throws Exception {+ doTestDocIDRunEnd(+ true,+ (doc, v) -> doc.add(SortedSetDocValuesField.indexedField("f", fixedWidthTerm(v))),+ (leaf, _) -> checkSortedSetRunEnds(leaf, false));+ }++ public void testDocIDRunEndSortedSetOrdinalSet() throws Exception {+ doTestDocIDRunEnd(+ true,+ (doc, v) -> doc.add(SortedSetDocValuesField.indexedField("f", fixedWidthTerm(v))),+ (leaf, _) -> checkSortedSetRunEnds(leaf, true));+ }++ private void checkSortedSetRunEnds(LeafReader leaf, boolean ordinalSet) throws IOException {+ SortedSetDocValues values = leaf.getSortedSetDocValues("f");+ if (values == null || values.getValueCount() == 0) {+ return;+ }+ long valueCount = values.getValueCount();+ LongBitSet ords = new LongBitSet(valueCount);+ long minOrd = -1;+ long maxOrd = -1;+ if (ordinalSet) {+ // every other ordinal: non-contiguous, the GH#16450 shape+ for (long o = random().nextInt(2); o < valueCount; o += 2) {+ ords.set(o);+ if (minOrd == -1) {+ minOrd = o;+ }+ maxOrd = o;+ }+ if (minOrd == -1) {+ return;+ }+ } else {+ long a = TestUtil.nextLong(random(), 0, valueCount - 1);+ long b = TestUtil.nextLong(random(), 0, valueCount - 1);+ minOrd = Math.min(a, b);+ maxOrd = Math.max(a, b);+ ords.set(minOrd, maxOrd + 1);+ }+ TwoPhaseIterator tpi =+ ordinalSet+ ? DocValuesRangeIterator.forOrdinalSet(+ values, leaf.getDocValuesSkipper("f"), minOrd, maxOrd, ords)+ : DocValuesRangeIterator.forOrdinalRange(+ values, leaf.getDocValuesSkipper("f"), minOrd, maxOrd);+ SortedSetDocValues truth = leaf.getSortedSetDocValues("f");+ FixedBitSet expected =+ expectedMatches(+ truth,+ () -> {+ for (int j = 0; j < truth.docValueCount(); j++) {+ if (ords.get(truth.nextOrd())) {+ return true;+ }+ }+ return false;+ },+ leaf.maxDoc());+ assertDocIDRunEnds(tpi, expected, leaf.maxDoc());+ }++ /** Brute-force oracle: walks {@code truth} and records every doc where {@code matches} holds. */+ private static FixedBitSet expectedMatches(+ DocIdSetIterator truth, IOBooleanSupplier matches, int maxDoc) throws IOException {+ FixedBitSet expected = new FixedBitSet(maxDoc);+ for (int d = truth.nextDoc(); d != NO_MORE_DOCS; d = truth.nextDoc()) {+ if (matches.get()) {+ expected.set(d);+ }+ }+ return expected;+ }++ private interface RunEndDocPopulator {+ void addValue(Document doc, long value);+ }++ private interface RunEndChecker {+ void check(LeafReader leaf, long domain) throws IOException;+ }++ /**+ * Indexes a random skip-indexed field whose values are drawn from a small domain (random or+ * clustered in doc-ID order, with random gaps) and runs {@code checker} against every leaf.+ */+ private void doTestDocIDRunEnd(+ boolean multiValued, RunEndDocPopulator populate, RunEndChecker checker) throws Exception {+ int totalDocs = TestUtil.nextInt(random(), 1, 1500);+ long domain = TestUtil.nextInt(random(), 5, 100);+ boolean clustered = random().nextBoolean();+ int missingOneIn = TestUtil.nextInt(random(), 0, 4);+ try (Directory dir = newDirectory();+ RandomIndexWriter w = new RandomIndexWriter(random(), dir)) {+ for (int i = 0; i < totalDocs; i++) {+ Document doc = new Document();+ if (missingOneIn == 0 || random().nextInt(missingOneIn + 1) != 0) {+ int count = multiValued ? TestUtil.nextInt(random(), 1, 2) : 1;+ for (int j = 0; j < count; j++) {+ long v =+ clustered && j == 0+ ? i * domain / totalDocs+ : TestUtil.nextLong(random(), 0, domain);+ populate.addValue(doc, v);+ }+ }+ w.addDocument(doc);+ }+ if (random().nextBoolean()) {+ w.forceMerge(1);+ }+ try (IndexReader r = w.getReader()) {+ for (LeafReaderContext ctx : r.leaves()) {+ for (int iter = 0; iter < 10; iter++) {+ checker.check(ctx.reader(), domain);+ }+ }+ }+ }+ }++ private static BytesRef fixedWidthTerm(long value) {+ // offset keeps the string width fixed so ordinal order matches value order+ return new BytesRef(Long.toString(1000000L + value));+ }++ /**+ * Verifies the {@link TwoPhaseIterator#docIDRunEnd()} contract: every doc in a reported run+ * {@code [docID, runEnd)} truly matches, the call does not move the approximation, and outside+ * runs {@link TwoPhaseIterator#matches()} agrees with brute force.+ */+ private static void assertDocIDRunEnds(TwoPhaseIterator tpi, FixedBitSet expected, int maxDoc)+ throws IOException {+ DocIdSetIterator approx = tpi.approximation();+ for (int doc = approx.nextDoc(); doc != NO_MORE_DOCS; ) {+ int runEnd = tpi.docIDRunEnd();+ assertEquals("docIDRunEnd() must not move the approximation", doc, approx.docID());+ assertTrue("docIDRunEnd() " + runEnd + " is below the current doc " + doc, runEnd >= doc);+ assertTrue("docIDRunEnd() " + runEnd + " is beyond maxDoc " + maxDoc, runEnd <= maxDoc);+ for (int d = doc; d < runEnd; d++) {+ assertTrue(+ "doc " + d + " in reported run [" + doc + ", " + runEnd + ") is not a true match",+ expected.get(d));+ }+ if (runEnd > doc) {+ doc = approx.advance(runEnd);+ } else {+ assertEquals(+ "matches() disagrees with brute force on doc " + doc, expected.get(doc), tpi.matches());+ doc = approx.nextDoc();+ }+ }+ }}