Apache Lucene open-source search software
Harden BaseDocValuesFormatTestCase (#13346)
Nhat Nguyen06973f5723e9
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseDocValuesFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseDocValuesFormatTestCase.javaindex 8c412946b87..07c4507ae92 100644--- a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseDocValuesFormatTestCase.java+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseDocValuesFormatTestCase.java@@ -1414,6 +1414,98 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTesir.close();}+ protected void compareStoredFieldWithSortedNumericsDV(+ DirectoryReader directoryReader, String storedField, String dvField) throws IOException {+ for (LeafReaderContext leaf : directoryReader.leaves()) {+ LeafReader reader = leaf.reader();+ StoredFields storedFields = reader.storedFields();+ SortedNumericDocValues docValues = reader.getSortedNumericDocValues(dvField);+ if (docValues == null) {+ // no stored values at all+ for (int doc = 0; doc < reader.maxDoc(); doc++) {+ assertArrayEquals(new String[0], storedFields.document(doc).getValues(storedField));+ }+ continue;+ }+ for (int doc = 0; doc < reader.maxDoc(); doc++) {+ String[] storedValues = storedFields.document(doc).getValues(storedField);+ if (storedValues.length == 0) {+ assertFalse(docValues.advanceExact(doc));+ continue;+ }+ switch (random().nextInt(3)) {+ case 0 -> assertEquals(doc, docValues.nextDoc());+ case 1 -> assertEquals(doc, docValues.advance(doc));+ default -> assertTrue(docValues.advanceExact(doc));+ }+ assertEquals(doc, docValues.docID());+ int repeats = 1 + random().nextInt(3);+ for (int r = 0; r < repeats; r++) {+ if (r > 0 || random().nextBoolean()) {+ assertTrue(docValues.advanceExact(doc));+ }+ assertEquals(storedValues.length, docValues.docValueCount());+ for (int v = 0; v < docValues.docValueCount(); v++) {+ assertEquals(storedValues[v], Long.toString(docValues.nextValue()));+ }+ }+ }+ // jump with advanceExact+ int iters = 1 + random().nextInt(3);+ for (int i = 0; i < iters; i++) {+ docValues = reader.getSortedNumericDocValues(dvField);+ for (int doc = random().nextInt(leaf.reader().maxDoc()); doc < reader.maxDoc(); doc++) {+ String[] storedValues = storedFields.document(doc).getValues(storedField);+ if (docValues.advanceExact(doc)) {+ assertEquals(doc, docValues.docID());+ int repeats = 1 + random().nextInt(3);+ for (int r = 0; r < repeats; r++) {+ if (r > 0 || random().nextBoolean()) {+ assertTrue(docValues.advanceExact(doc));+ }+ assertEquals(storedValues.length, docValues.docValueCount());+ for (int v = 0; v < docValues.docValueCount(); v++) {+ assertEquals(storedValues[v], Long.toString(docValues.nextValue()));+ }+ }+ } else {+ assertArrayEquals(new String[0], storedValues);+ }+ doc += random().nextInt(5); // skip some docs+ }+ }+ // jump with advance+ for (int i = 0; i < iters; i++) {+ docValues = reader.getSortedNumericDocValues(dvField);+ int doc = random().nextInt(leaf.reader().maxDoc());+ while (doc != NO_MORE_DOCS) {+ int nextDoc = docValues.advance(doc);+ // no stored fields in between+ for (int d = doc; d < (nextDoc == NO_MORE_DOCS ? reader.maxDoc() : nextDoc); d++) {+ String[] storedValues = storedFields.document(d).getValues(storedField);+ assertArrayEquals(new String[0], storedValues);+ }+ doc = nextDoc;+ if (doc != NO_MORE_DOCS) {+ String[] storedValues = storedFields.document(doc).getValues(storedField);+ int repeats = 1 + random().nextInt(3);+ for (int r = 0; r < repeats; r++) {+ if (r > 0 || random().nextBoolean()) {+ assertTrue(docValues.advanceExact(doc));+ }+ assertEquals(storedValues.length, docValues.docValueCount());+ for (int v = 0; v < docValues.docValueCount(); v++) {+ assertEquals(storedValues[v], Long.toString(docValues.nextValue()));+ }+ }+ doc = nextDoc + 1;+ doc += random().nextInt(5); // skip some docs+ }+ }+ }+ }+ }+private void doTestSortedNumericsVsStoredFields(LongSupplier counts, LongSupplier values)throws Exception {Directory dir = newDirectory();@@ -1452,38 +1544,18 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTesint id = random().nextInt(numDocs);writer.deleteDocuments(new Term("id", Integer.toString(id)));}-+ try (DirectoryReader reader = maybeWrapWithMergingReader(DirectoryReader.open(dir))) {+ TestUtil.checkReader(reader);+ compareStoredFieldWithSortedNumericsDV(reader, "stored", "dv");+ }// merge some segments and ensure that at least one of them has more than// 256 valueswriter.forceMerge(numDocs / 256);-- writer.close();-- // compare- DirectoryReader ir = maybeWrapWithMergingReader(DirectoryReader.open(dir));- TestUtil.checkReader(ir);- for (LeafReaderContext context : ir.leaves()) {- LeafReader r = context.reader();- StoredFields storedFields = r.storedFields();- SortedNumericDocValues docValues = DocValues.getSortedNumeric(r, "dv");- for (int i = 0; i < r.maxDoc(); i++) {- if (i > docValues.docID()) {- docValues.nextDoc();- }- String[] expected = storedFields.document(i).getValues("stored");- if (i < docValues.docID()) {- assertEquals(0, expected.length);- } else {- String[] actual = new String[docValues.docValueCount()];- for (int j = 0; j < actual.length; j++) {- actual[j] = Long.toString(docValues.nextValue());- }- assertArrayEquals(expected, actual);- }- }+ try (DirectoryReader reader = maybeWrapWithMergingReader(DirectoryReader.open(dir))) {+ TestUtil.checkReader(reader);+ compareStoredFieldWithSortedNumericsDV(reader, "stored", "dv");}- ir.close();- dir.close();+ IOUtils.close(writer, dir);}public void testBooleanNumericsVsStoredFields() throws Exception {@@ -2266,6 +2338,101 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTesdirectory.close();}+ protected void compareStoredFieldWithSortedSetDV(+ DirectoryReader directoryReader, String storedField, String dvField) throws IOException {+ for (LeafReaderContext leaf : directoryReader.leaves()) {+ LeafReader reader = leaf.reader();+ StoredFields storedFields = reader.storedFields();+ SortedSetDocValues docValues = reader.getSortedSetDocValues(dvField);+ if (docValues == null) {+ // no stored values at all+ for (int doc = 0; doc < reader.maxDoc(); doc++) {+ assertArrayEquals(new String[0], storedFields.document(doc).getValues(storedField));+ }+ continue;+ }+ // sequentially+ for (int doc = 0; doc < reader.maxDoc(); doc++) {+ String[] storedValues = storedFields.document(doc).getValues(storedField);+ if (storedValues.length == 0) {+ assertFalse(docValues.advanceExact(doc));+ continue;+ }+ switch (random().nextInt(3)) {+ case 0 -> assertEquals(doc, docValues.nextDoc());+ case 1 -> assertEquals(doc, docValues.advance(doc));+ default -> assertTrue(docValues.advanceExact(doc));+ }+ assertEquals(doc, docValues.docID());+ assertEquals(storedValues.length, docValues.docValueCount());+ int repeats = 1 + random().nextInt(3);+ for (int r = 0; r < repeats; r++) {+ if (r > 0 || random().nextBoolean()) {+ assertTrue(docValues.advanceExact(doc));+ }+ for (int v = 0; v < docValues.docValueCount(); v++) {+ long ord = docValues.nextOrd();+ assertEquals(storedValues[v], docValues.lookupOrd(ord).utf8ToString());+ }+ }+ }+ // jump with advanceExact+ int iters = 1 + random().nextInt(3);+ for (int i = 0; i < iters; i++) {+ docValues = reader.getSortedSetDocValues(dvField);+ for (int doc = random().nextInt(leaf.reader().maxDoc()); doc < reader.maxDoc(); doc++) {+ String[] storedValues = storedFields.document(doc).getValues(storedField);+ if (docValues.advanceExact(doc)) {+ assertEquals(doc, docValues.docID());+ assertEquals(storedValues.length, docValues.docValueCount());+ int repeats = 1 + random().nextInt(3);+ for (int r = 0; r < repeats; r++) {+ if (r > 0 || random().nextBoolean()) {+ assertTrue(docValues.advanceExact(doc));+ }+ for (int v = 0; v < docValues.docValueCount(); v++) {+ long ord = docValues.nextOrd();+ assertEquals(storedValues[v], docValues.lookupOrd(ord).utf8ToString());+ }+ }+ } else {+ assertArrayEquals(new String[0], storedValues);+ }+ doc += random().nextInt(5); // skip some docs+ }+ }+ // jump with advance+ for (int i = 0; i < iters; i++) {+ docValues = reader.getSortedSetDocValues(dvField);+ int doc = random().nextInt(leaf.reader().maxDoc());+ while (doc != NO_MORE_DOCS) {+ int nextDoc = docValues.advance(doc);+ // no stored fields in between+ for (int d = doc; d < (nextDoc == NO_MORE_DOCS ? reader.maxDoc() : nextDoc); d++) {+ String[] storedValues = storedFields.document(d).getValues(storedField);+ assertArrayEquals(new String[0], storedValues);+ }+ doc = nextDoc;+ if (doc != NO_MORE_DOCS) {+ int repeats = 1 + random().nextInt(3);+ String[] storedValues = storedFields.document(doc).getValues(storedField);+ for (int r = 0; r < repeats; r++) {+ if (r > 0 || random().nextBoolean()) {+ assertTrue(docValues.advanceExact(doc));+ }+ for (int v = 0; v < docValues.docValueCount(); v++) {+ long ord = docValues.nextOrd();+ assertEquals(storedValues[v], docValues.lookupOrd(ord).utf8ToString());+ }+ }+ doc = nextDoc + 1;+ doc += random().nextInt(5); // skip some docs+ }+ }+ }+ }+ }+protected void doTestSortedSetVsStoredFields(int numDocs, int minLength, int maxLength, int maxValuesPerDoc, int maxUniqueValues)throws Exception {@@ -2312,91 +2479,23 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTeswriter.commit();}}-// delete some docsint numDeletions = random().nextInt(numDocs / 10);- if (VERBOSE) {- System.out.println("\nTEST: now delete " + numDeletions + " docs");- }for (int i = 0; i < numDeletions; i++) {int id = random().nextInt(numDocs);writer.deleteDocuments(new Term("id", Integer.toString(id)));}- // compare- if (VERBOSE) {- System.out.println("\nTEST: now get reader");- }- DirectoryReader ir = writer.getReader();- TestUtil.checkReader(ir);- for (LeafReaderContext context : ir.leaves()) {- LeafReader r = context.reader();- StoredFields storedFields = r.storedFields();- SortedSetDocValues docValues = r.getSortedSetDocValues("dv");- for (int i = 0; i < r.maxDoc(); i++) {- String[] stringValues = storedFields.document(i).getValues("stored");- if (docValues != null) {- if (docValues.docID() < i) {- docValues.nextDoc();- }- }- if (docValues != null && stringValues.length > 0) {- assertEquals(i, docValues.docID());- assertEquals(stringValues.length, docValues.docValueCount());- for (String stringValue : stringValues) {- assert docValues != null;- long ord = docValues.nextOrd();- BytesRef scratch = docValues.lookupOrd(ord);- assertEquals(stringValue, scratch.utf8ToString());- }- }- }- }- if (VERBOSE) {- System.out.println("\nTEST: now close reader");- }- ir.close();- if (VERBOSE) {- System.out.println("TEST: force merge");+ try (DirectoryReader reader = writer.getReader()) {+ TestUtil.checkReader(reader);+ compareStoredFieldWithSortedSetDV(reader, "stored", "dv");}writer.forceMerge(1);-- // compare again- ir = writer.getReader();- TestUtil.checkReader(ir);- for (LeafReaderContext context : ir.leaves()) {- LeafReader r = context.reader();- StoredFields storedFields = r.storedFields();- SortedSetDocValues docValues = r.getSortedSetDocValues("dv");- for (int i = 0; i < r.maxDoc(); i++) {- String[] stringValues = storedFields.document(i).getValues("stored");- if (docValues.docID() < i) {- docValues.nextDoc();- }- if (stringValues.length > 0) {- assertEquals(i, docValues.docID());- assertEquals(stringValues.length, docValues.docValueCount());- for (String stringValue : stringValues) {- assert docValues != null;- long ord = docValues.nextOrd();- BytesRef scratch = docValues.lookupOrd(ord);- assertEquals(stringValue, scratch.utf8ToString());- }- }- }- }- if (VERBOSE) {- System.out.println("TEST: close reader");- }- ir.close();- if (VERBOSE) {- System.out.println("TEST: close writer");- }- writer.close();- if (VERBOSE) {- System.out.println("TEST: close dir");+ try (DirectoryReader reader = writer.getReader()) {+ TestUtil.checkReader(reader);+ compareStoredFieldWithSortedSetDV(reader, "stored", "dv");}- dir.close();+ IOUtils.close(writer, dir);}public void testSortedSetFixedLengthVsStoredFields() throws Exception {