Apache Lucene open-source search software
Improve testing of mismatched field numbers. (#13812)
Adrien Grandda1f954601c2
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene90/TestLucene90HnswVectorsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene90/TestLucene90HnswVectorsFormat.javaindex 2c689d5c0e5..37e2745c247 100644--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene90/TestLucene90HnswVectorsFormat.java+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene90/TestLucene90HnswVectorsFormat.java@@ -83,4 +83,9 @@ public class TestLucene90HnswVectorsFormat extends BaseKnnVectorsFormatTestCasepublic void testMergingWithDifferentByteKnnFields() {// unimplemented}++ @Override+ public void testMismatchedFields() throws Exception {+ // requires byte support+ }}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene91/TestLucene91HnswVectorsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene91/TestLucene91HnswVectorsFormat.javaindex df79316db0a..7bf2d426eac 100644--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene91/TestLucene91HnswVectorsFormat.java+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene91/TestLucene91HnswVectorsFormat.java@@ -82,4 +82,9 @@ public class TestLucene91HnswVectorsFormat extends BaseKnnVectorsFormatTestCasepublic void testMergingWithDifferentByteKnnFields() {// unimplemented}++ @Override+ public void testMismatchedFields() throws Exception {+ // requires byte support+ }}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene92/TestLucene92HnswVectorsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene92/TestLucene92HnswVectorsFormat.javaindex 0e003dafc3b..192f70a6397 100644--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene92/TestLucene92HnswVectorsFormat.java+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene92/TestLucene92HnswVectorsFormat.java@@ -72,4 +72,9 @@ public class TestLucene92HnswVectorsFormat extends BaseKnnVectorsFormatTestCasepublic void testMergingWithDifferentByteKnnFields() {// unimplemented}++ @Override+ public void testMismatchedFields() throws Exception {+ // requires byte support+ }}diff --git a/lucene/core/src/java/org/apache/lucene/codecs/DocValuesConsumer.java b/lucene/core/src/java/org/apache/lucene/codecs/DocValuesConsumer.javaindex cbb906788e5..08c08ec5075 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/DocValuesConsumer.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/DocValuesConsumer.java@@ -613,7 +613,7 @@ public abstract class DocValuesConsumer implements Closeable {if (docValuesProducer != null) {FieldInfo readerFieldInfo = mergeState.fieldInfos[i].fieldInfo(fieldInfo.name);if (readerFieldInfo != null && readerFieldInfo.getDocValuesType() == DocValuesType.SORTED) {- values = docValuesProducer.getSorted(fieldInfo);+ values = docValuesProducer.getSorted(readerFieldInfo);}}if (values == null) {diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90PointsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90PointsReader.javaindex d3f256cbf00..82910e23ab9 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90PointsReader.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90PointsReader.java@@ -33,9 +33,9 @@ import org.apache.lucene.util.bkd.BKDReader;/** Reads point values previously written with {@link Lucene90PointsWriter} */public class Lucene90PointsReader extends PointsReader {- final IndexInput indexIn, dataIn;- final SegmentReadState readState;- final IntObjectHashMap<PointValues> readers = new IntObjectHashMap<>();+ private final IndexInput indexIn, dataIn;+ private final SegmentReadState readState;+ private final IntObjectHashMap<PointValues> readers = new IntObjectHashMap<>();/** Sole constructor */public Lucene90PointsReader(SegmentReadState readState) throws IOException {diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90PointsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90PointsWriter.javaindex e50d6a0fdb5..45a946e8ac4 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90PointsWriter.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90PointsWriter.java@@ -253,7 +253,7 @@ public class Lucene90PointsWriter extends PointsWriter {FieldInfos readerFieldInfos = mergeState.fieldInfos[i];FieldInfo readerFieldInfo = readerFieldInfos.fieldInfo(fieldInfo.name);if (readerFieldInfo != null && readerFieldInfo.getPointDimensionCount() > 0) {- PointValues aPointValues = reader90.readers.get(readerFieldInfo.number);+ PointValues aPointValues = reader90.getValues(readerFieldInfo.name);if (aPointValues != null) {pointValues.add(aPointValues);docMaps.add(mergeState.docMaps[i]);diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/asserting/AssertingDocValuesFormat.java b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/asserting/AssertingDocValuesFormat.javaindex 619b5b02b80..046fd850304 100644--- a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/asserting/AssertingDocValuesFormat.java+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/asserting/AssertingDocValuesFormat.java@@ -27,6 +27,7 @@ import org.apache.lucene.index.DocValuesSkipIndexType;import org.apache.lucene.index.DocValuesSkipper;import org.apache.lucene.index.DocValuesType;import org.apache.lucene.index.FieldInfo;+import org.apache.lucene.index.FieldInfos;import org.apache.lucene.index.NumericDocValues;import org.apache.lucene.index.SegmentReadState;import org.apache.lucene.index.SegmentWriteState;@@ -59,7 +60,8 @@ public class AssertingDocValuesFormat extends DocValuesFormat {assert state.fieldInfos.hasDocValues();DocValuesProducer producer = in.fieldsProducer(state);assert producer != null;- return new AssertingDocValuesProducer(producer, state.segmentInfo.maxDoc(), false);+ return new AssertingDocValuesProducer(+ producer, state.fieldInfos, state.segmentInfo.maxDoc(), false);}static class AssertingDocValuesConsumer extends DocValuesConsumer {@@ -214,12 +216,15 @@ public class AssertingDocValuesFormat extends DocValuesFormat {static class AssertingDocValuesProducer extends DocValuesProducer {private final DocValuesProducer in;+ private final FieldInfos fieldInfos;private final int maxDoc;private final boolean merging;private final Thread creationThread;- AssertingDocValuesProducer(DocValuesProducer in, int maxDoc, boolean merging) {+ AssertingDocValuesProducer(+ DocValuesProducer in, FieldInfos fieldInfos, int maxDoc, boolean merging) {this.in = in;+ this.fieldInfos = fieldInfos;this.maxDoc = maxDoc;this.merging = merging;this.creationThread = Thread.currentThread();@@ -229,6 +234,7 @@ public class AssertingDocValuesFormat extends DocValuesFormat {@Overridepublic NumericDocValues getNumeric(FieldInfo field) throws IOException {+ assert fieldInfos.fieldInfo(field.name).number == field.number;if (merging) {AssertingCodec.assertThread("DocValuesProducer", creationThread);}@@ -240,6 +246,7 @@ public class AssertingDocValuesFormat extends DocValuesFormat {@Overridepublic BinaryDocValues getBinary(FieldInfo field) throws IOException {+ assert fieldInfos.fieldInfo(field.name).number == field.number;if (merging) {AssertingCodec.assertThread("DocValuesProducer", creationThread);}@@ -251,6 +258,7 @@ public class AssertingDocValuesFormat extends DocValuesFormat {@Overridepublic SortedDocValues getSorted(FieldInfo field) throws IOException {+ assert fieldInfos.fieldInfo(field.name).number == field.number;if (merging) {AssertingCodec.assertThread("DocValuesProducer", creationThread);}@@ -262,6 +270,7 @@ public class AssertingDocValuesFormat extends DocValuesFormat {@Overridepublic SortedNumericDocValues getSortedNumeric(FieldInfo field) throws IOException {+ assert fieldInfos.fieldInfo(field.name).number == field.number;if (merging) {AssertingCodec.assertThread("DocValuesProducer", creationThread);}@@ -273,6 +282,7 @@ public class AssertingDocValuesFormat extends DocValuesFormat {@Overridepublic SortedSetDocValues getSortedSet(FieldInfo field) throws IOException {+ assert fieldInfos.fieldInfo(field.name).number == field.number;if (merging) {AssertingCodec.assertThread("DocValuesProducer", creationThread);}@@ -284,6 +294,7 @@ public class AssertingDocValuesFormat extends DocValuesFormat {@Overridepublic DocValuesSkipper getSkipper(FieldInfo field) throws IOException {+ assert fieldInfos.fieldInfo(field.name).number == field.number;assert field.docValuesSkipIndexType() != DocValuesSkipIndexType.NONE;DocValuesSkipper skipper = in.getSkipper(field);assert skipper != null;@@ -303,7 +314,7 @@ public class AssertingDocValuesFormat extends DocValuesFormat {@Overridepublic DocValuesProducer getMergeInstance() {- return new AssertingDocValuesProducer(in.getMergeInstance(), maxDoc, true);+ return new AssertingDocValuesProducer(in.getMergeInstance(), fieldInfos, maxDoc, true);}@Overridediff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseDocValuesFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseDocValuesFormatTestCase.javaindex a312b42a910..9b99aeecba6 100644--- a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseDocValuesFormatTestCase.java+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseDocValuesFormatTestCase.java@@ -24,6 +24,7 @@ import java.io.IOException;import java.io.PrintStream;import java.util.function.Supplier;import org.apache.lucene.analysis.Analyzer;+import org.apache.lucene.document.BinaryDocValuesField;import org.apache.lucene.document.Document;import org.apache.lucene.document.Field;import org.apache.lucene.document.NumericDocValuesField;@@ -31,22 +32,26 @@ import org.apache.lucene.document.SortedDocValuesField;import org.apache.lucene.document.SortedNumericDocValuesField;import org.apache.lucene.document.SortedSetDocValuesField;import org.apache.lucene.document.StringField;+import org.apache.lucene.index.BinaryDocValues;import org.apache.lucene.index.CheckIndex;import org.apache.lucene.index.CheckIndex.Status.DocValuesStatus;import org.apache.lucene.index.CodecReader;import org.apache.lucene.index.DirectoryReader;import org.apache.lucene.index.DocValuesSkipper;import org.apache.lucene.index.IndexReader;+import org.apache.lucene.index.IndexWriter;import org.apache.lucene.index.IndexWriterConfig;import org.apache.lucene.index.LeafReader;import org.apache.lucene.index.LeafReaderContext;import org.apache.lucene.index.NumericDocValues;+import org.apache.lucene.index.SerialMergeScheduler;import org.apache.lucene.index.SortedDocValues;import org.apache.lucene.index.SortedNumericDocValues;import org.apache.lucene.index.SortedSetDocValues;import org.apache.lucene.index.Term;import org.apache.lucene.index.TermsEnum;import org.apache.lucene.index.TermsEnum.SeekStatus;+import org.apache.lucene.search.DocIdSetIterator;import org.apache.lucene.store.Directory;import org.apache.lucene.tests.analysis.MockAnalyzer;import org.apache.lucene.tests.util.TestUtil;@@ -832,4 +837,74 @@ public abstract class BaseDocValuesFormatTestCase extends LegacyBaseDocValuesForint docID();}++ public void testMismatchedFields() throws Exception {+ Directory dir1 = newDirectory();+ IndexWriter w1 = new IndexWriter(dir1, newIndexWriterConfig());+ Document doc = new Document();+ doc.add(new BinaryDocValuesField("binary", new BytesRef("lucene")));+ doc.add(new NumericDocValuesField("numeric", 0L));+ doc.add(new SortedDocValuesField("sorted", new BytesRef("search")));+ doc.add(new SortedNumericDocValuesField("sorted_numeric", 1L));+ doc.add(new SortedSetDocValuesField("sorted_set", new BytesRef("engine")));+ w1.addDocument(doc);++ Directory dir2 = newDirectory();+ IndexWriter w2 =+ new IndexWriter(dir2, newIndexWriterConfig().setMergeScheduler(new SerialMergeScheduler()));+ w2.addDocument(doc);+ w2.commit();++ DirectoryReader reader = DirectoryReader.open(w1);+ w1.close();+ w2.addIndexes(new MismatchedCodecReader((CodecReader) getOnlyLeafReader(reader), random()));+ reader.close();+ w2.forceMerge(1);+ reader = DirectoryReader.open(w2);+ w2.close();++ LeafReader leafReader = getOnlyLeafReader(reader);++ BinaryDocValues bdv = leafReader.getBinaryDocValues("binary");+ assertNotNull(bdv);+ assertEquals(0, bdv.nextDoc());+ assertEquals(new BytesRef("lucene"), bdv.binaryValue());+ assertEquals(1, bdv.nextDoc());+ assertEquals(new BytesRef("lucene"), bdv.binaryValue());+ assertEquals(DocIdSetIterator.NO_MORE_DOCS, bdv.nextDoc());++ NumericDocValues ndv = leafReader.getNumericDocValues("numeric");+ assertNotNull(ndv);+ assertEquals(0, ndv.nextDoc());+ assertEquals(0, ndv.longValue());+ assertEquals(1, ndv.nextDoc());+ assertEquals(0, ndv.longValue());+ assertEquals(DocIdSetIterator.NO_MORE_DOCS, ndv.nextDoc());++ SortedDocValues sdv = leafReader.getSortedDocValues("sorted");+ assertNotNull(sdv);+ assertEquals(0, sdv.nextDoc());+ assertEquals(new BytesRef("search"), sdv.lookupOrd(sdv.ordValue()));+ assertEquals(1, sdv.nextDoc());+ assertEquals(new BytesRef("search"), sdv.lookupOrd(sdv.ordValue()));+ assertEquals(DocIdSetIterator.NO_MORE_DOCS, sdv.nextDoc());++ SortedNumericDocValues sndv = leafReader.getSortedNumericDocValues("sorted_numeric");+ assertNotNull(sndv);+ assertEquals(0, sndv.nextDoc());+ assertEquals(1, sndv.nextValue());+ assertEquals(1, sndv.nextDoc());+ assertEquals(1, sndv.nextValue());+ assertEquals(DocIdSetIterator.NO_MORE_DOCS, sndv.nextDoc());++ SortedSetDocValues ssdv = leafReader.getSortedSetDocValues("sorted_set");+ assertNotNull(ssdv);+ assertEquals(0, ssdv.nextDoc());+ assertEquals(new BytesRef("engine"), ssdv.lookupOrd(ssdv.nextOrd()));+ assertEquals(1, ssdv.nextDoc());+ assertEquals(new BytesRef("engine"), ssdv.lookupOrd(ssdv.nextOrd()));+ assertEquals(DocIdSetIterator.NO_MORE_DOCS, ssdv.nextDoc());++ IOUtils.close(reader, w2, dir1, dir2);+ }}diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseKnnVectorsFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseKnnVectorsFormatTestCase.javaindex 4c9165e1a10..fd8259e2636 100644--- a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseKnnVectorsFormatTestCase.java+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseKnnVectorsFormatTestCase.java@@ -63,6 +63,7 @@ import org.apache.lucene.index.MergeTrigger;import org.apache.lucene.index.NoMergePolicy;import org.apache.lucene.index.SegmentInfo;import org.apache.lucene.index.SegmentWriteState;+import org.apache.lucene.index.SerialMergeScheduler;import org.apache.lucene.index.StoredFields;import org.apache.lucene.index.Term;import org.apache.lucene.index.VectorEncoding;@@ -78,6 +79,7 @@ import org.apache.lucene.store.FSDirectory;import org.apache.lucene.tests.util.TestUtil;import org.apache.lucene.util.Bits;import org.apache.lucene.util.BytesRef;+import org.apache.lucene.util.IOUtils;import org.apache.lucene.util.InfoStream;import org.apache.lucene.util.StringHelper;import org.apache.lucene.util.VectorUtil;@@ -1810,4 +1812,51 @@ public abstract class BaseKnnVectorsFormatTestCase extends BaseIndexFileFormatTe}}}++ public void testMismatchedFields() throws Exception {+ Directory dir1 = newDirectory();+ IndexWriter w1 = new IndexWriter(dir1, newIndexWriterConfig());+ Document doc = new Document();+ doc.add(new KnnFloatVectorField("float", new float[] {1f}));+ doc.add(new KnnByteVectorField("byte", new byte[] {42}));+ w1.addDocument(doc);++ Directory dir2 = newDirectory();+ IndexWriter w2 =+ new IndexWriter(dir2, newIndexWriterConfig().setMergeScheduler(new SerialMergeScheduler()));+ w2.addDocument(doc);+ w2.commit();++ DirectoryReader reader = DirectoryReader.open(w1);+ w1.close();+ w2.addIndexes(new MismatchedCodecReader((CodecReader) getOnlyLeafReader(reader), random()));+ reader.close();+ w2.forceMerge(1);+ reader = DirectoryReader.open(w2);+ w2.close();++ LeafReader leafReader = getOnlyLeafReader(reader);++ ByteVectorValues byteVectors = leafReader.getByteVectorValues("byte");+ assertNotNull(byteVectors);+ assertEquals(0, byteVectors.nextDoc());+ assertArrayEquals(new byte[] {42}, byteVectors.vectorValue());+ assertEquals(1, byteVectors.nextDoc());+ assertArrayEquals(new byte[] {42}, byteVectors.vectorValue());+ assertEquals(DocIdSetIterator.NO_MORE_DOCS, byteVectors.nextDoc());++ FloatVectorValues floatVectors = leafReader.getFloatVectorValues("float");+ assertNotNull(floatVectors);+ assertEquals(0, floatVectors.nextDoc());+ float[] vector = floatVectors.vectorValue();+ assertEquals(1, vector.length);+ assertEquals(1f, vector[0], 0f);+ assertEquals(1, floatVectors.nextDoc());+ vector = floatVectors.vectorValue();+ assertEquals(1, vector.length);+ assertEquals(1f, vector[0], 0f);+ assertEquals(DocIdSetIterator.NO_MORE_DOCS, floatVectors.nextDoc());++ IOUtils.close(reader, w2, dir1, dir2);+ }}diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BasePointsFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BasePointsFormatTestCase.javaindex 265e3f073be..a15dd07a79e 100644--- a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BasePointsFormatTestCase.java+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BasePointsFormatTestCase.java@@ -30,6 +30,7 @@ import org.apache.lucene.document.Document;import org.apache.lucene.document.Field;import org.apache.lucene.document.FieldType;import org.apache.lucene.document.IntPoint;+import org.apache.lucene.document.LongPoint;import org.apache.lucene.document.NumericDocValuesField;import org.apache.lucene.document.StringField;import org.apache.lucene.index.CodecReader;@@ -46,6 +47,7 @@ import org.apache.lucene.index.NumericDocValues;import org.apache.lucene.index.PointValues;import org.apache.lucene.index.PointValues.IntersectVisitor;import org.apache.lucene.index.PointValues.Relation;+import org.apache.lucene.index.SerialMergeScheduler;import org.apache.lucene.index.Term;import org.apache.lucene.internal.tests.ConcurrentMergeSchedulerAccess;import org.apache.lucene.internal.tests.TestSecrets;@@ -1408,4 +1410,80 @@ public abstract class BasePointsFormatTestCase extends BaseIndexFileFormatTestCa}};}++ public void testMismatchedFields() throws Exception {+ Directory dir1 = newDirectory();+ IndexWriter w1 = new IndexWriter(dir1, newIndexWriterConfig());+ Document doc = new Document();+ doc.add(new LongPoint("f", 1L));+ doc.add(new LongPoint("g", 42L, 43L));+ w1.addDocument(doc);++ Directory dir2 = newDirectory();+ IndexWriter w2 =+ new IndexWriter(dir2, newIndexWriterConfig().setMergeScheduler(new SerialMergeScheduler()));+ w2.addDocument(doc);+ w2.commit();++ DirectoryReader reader = DirectoryReader.open(w1);+ w1.close();+ w2.addIndexes(new MismatchedCodecReader((CodecReader) getOnlyLeafReader(reader), random()));+ reader.close();+ w2.forceMerge(1);+ reader = DirectoryReader.open(w2);+ w2.close();++ LeafReader leafReader = getOnlyLeafReader(reader);+ assertEquals(2, leafReader.maxDoc());++ PointValues fPoints = leafReader.getPointValues("f");+ assertEquals(2, fPoints.size());+ fPoints.intersect(+ new IntersectVisitor() {++ int expectedDoc = 0;++ @Override+ public void visit(int docID, byte[] packedValue) throws IOException {+ assertEquals(LongPoint.pack(1L), new BytesRef(packedValue));+ assertEquals(expectedDoc++, docID);+ }++ @Override+ public void visit(int docID) throws IOException {+ throw new UnsupportedOperationException();+ }++ @Override+ public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) {+ return Relation.CELL_CROSSES_QUERY;+ }+ });++ PointValues gPoints = leafReader.getPointValues("g");+ assertEquals(2, fPoints.size());+ gPoints.intersect(+ new IntersectVisitor() {++ int expectedDoc = 0;++ @Override+ public void visit(int docID, byte[] packedValue) throws IOException {+ assertEquals(LongPoint.pack(42L, 43L), new BytesRef(packedValue));+ assertEquals(expectedDoc++, docID);+ }++ @Override+ public void visit(int docID) throws IOException {+ throw new UnsupportedOperationException();+ }++ @Override+ public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) {+ return Relation.CELL_CROSSES_QUERY;+ }+ });++ IOUtils.close(reader, w2, dir1, dir2);+ }}diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BasePostingsFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BasePostingsFormatTestCase.javaindex 8f8233ee680..8e0292b3f8d 100644--- a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BasePostingsFormatTestCase.java+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BasePostingsFormatTestCase.java@@ -42,6 +42,7 @@ import org.apache.lucene.document.Field.Store;import org.apache.lucene.document.FieldType;import org.apache.lucene.document.StringField;import org.apache.lucene.document.TextField;+import org.apache.lucene.index.CodecReader;import org.apache.lucene.index.DirectoryReader;import org.apache.lucene.index.Fields;import org.apache.lucene.index.IndexOptions;@@ -54,6 +55,7 @@ import org.apache.lucene.index.MultiTerms;import org.apache.lucene.index.PostingsEnum;import org.apache.lucene.index.SegmentReadState;import org.apache.lucene.index.SegmentWriteState;+import org.apache.lucene.index.SerialMergeScheduler;import org.apache.lucene.index.Term;import org.apache.lucene.index.Terms;import org.apache.lucene.index.TermsEnum;@@ -71,6 +73,7 @@ import org.apache.lucene.tests.util.LineFileDocs;import org.apache.lucene.tests.util.RamUsageTester;import org.apache.lucene.tests.util.TestUtil;import org.apache.lucene.util.BytesRef;+import org.apache.lucene.util.IOUtils;import org.junit.AfterClass;import org.junit.BeforeClass;@@ -1728,4 +1731,41 @@ public abstract class BasePostingsFormatTestCase extends BaseIndexFileFormatTestTestUtil.checkIndex(dir);}}++ public void testMismatchedFields() throws Exception {+ Directory dir1 = newDirectory();+ IndexWriter w1 = new IndexWriter(dir1, newIndexWriterConfig());+ Document doc = new Document();+ doc.add(new StringField("f", "a", Store.NO));+ doc.add(new StringField("g", "b", Store.NO));+ w1.addDocument(doc);++ Directory dir2 = newDirectory();+ IndexWriter w2 =+ new IndexWriter(dir2, newIndexWriterConfig().setMergeScheduler(new SerialMergeScheduler()));+ w2.addDocument(doc);+ w2.commit();++ DirectoryReader reader = DirectoryReader.open(w1);+ w1.close();+ w2.addIndexes(new MismatchedCodecReader((CodecReader) getOnlyLeafReader(reader), random()));+ reader.close();+ w2.forceMerge(1);+ reader = DirectoryReader.open(w2);+ w2.close();++ LeafReader leafReader = getOnlyLeafReader(reader);++ TermsEnum te = leafReader.terms("f").iterator();+ assertEquals("a", te.next().utf8ToString());+ assertEquals(2, te.docFreq());+ assertNull(te.next());++ te = leafReader.terms("g").iterator();+ assertEquals("b", te.next().utf8ToString());+ assertEquals(2, te.docFreq());+ assertNull(te.next());++ IOUtils.close(reader, w2, dir1, dir2);+ }}diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/index/MismatchedCodecReader.java b/lucene/test-framework/src/java/org/apache/lucene/tests/index/MismatchedCodecReader.javanew file mode 100644index 00000000000..8c856aafcba--- /dev/null+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/index/MismatchedCodecReader.java@@ -0,0 +1,216 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.tests.index;++import java.io.IOException;+import java.util.Objects;+import java.util.Random;+import org.apache.lucene.codecs.DocValuesProducer;+import org.apache.lucene.codecs.NormsProducer;+import org.apache.lucene.codecs.StoredFieldsReader;+import org.apache.lucene.index.BinaryDocValues;+import org.apache.lucene.index.CodecReader;+import org.apache.lucene.index.DocValuesSkipper;+import org.apache.lucene.index.FieldInfo;+import org.apache.lucene.index.FieldInfos;+import org.apache.lucene.index.FilterCodecReader;+import org.apache.lucene.index.NumericDocValues;+import org.apache.lucene.index.SortedDocValues;+import org.apache.lucene.index.SortedNumericDocValues;+import org.apache.lucene.index.SortedSetDocValues;+import org.apache.lucene.index.StoredFieldVisitor;++/**+ * Shuffles field numbers around to try to trip bugs where field numbers are assumed to always be+ * consistent across segments.+ */+public class MismatchedCodecReader extends FilterCodecReader {++ private final FieldInfos shuffled;++ /** Sole constructor. */+ public MismatchedCodecReader(CodecReader in, Random random) {+ super(in);+ shuffled = MismatchedLeafReader.shuffleInfos(in.getFieldInfos(), random);+ }++ @Override+ public FieldInfos getFieldInfos() {+ return shuffled;+ }++ @Override+ public CacheHelper getCoreCacheHelper() {+ return in.getCoreCacheHelper();+ }++ @Override+ public CacheHelper getReaderCacheHelper() {+ return in.getReaderCacheHelper();+ }++ @Override+ public StoredFieldsReader getFieldsReader() {+ StoredFieldsReader in = super.getFieldsReader();+ if (in == null) {+ return null;+ }+ return new MismatchedStoredFieldsReader(in, shuffled);+ }++ private static class MismatchedStoredFieldsReader extends StoredFieldsReader {++ private final StoredFieldsReader in;+ private final FieldInfos shuffled;++ MismatchedStoredFieldsReader(StoredFieldsReader in, FieldInfos shuffled) {+ this.in = Objects.requireNonNull(in);+ this.shuffled = shuffled;+ }++ @Override+ public void close() throws IOException {+ in.close();+ }++ @Override+ public StoredFieldsReader clone() {+ return new MismatchedStoredFieldsReader(in.clone(), shuffled);+ }++ @Override+ public void checkIntegrity() throws IOException {+ in.checkIntegrity();+ }++ @Override+ public void document(int docID, StoredFieldVisitor visitor) throws IOException {+ in.document(docID, new MismatchedLeafReader.MismatchedVisitor(visitor, shuffled));+ }+ }++ @Override+ public DocValuesProducer getDocValuesReader() {+ DocValuesProducer in = super.getDocValuesReader();+ if (in == null) {+ return null;+ }+ return new MismatchedDocValuesProducer(in, shuffled, super.getFieldInfos());+ }++ private static class MismatchedDocValuesProducer extends DocValuesProducer {++ private final DocValuesProducer in;+ private final FieldInfos shuffled;+ private final FieldInfos orig;++ MismatchedDocValuesProducer(DocValuesProducer in, FieldInfos shuffled, FieldInfos orig) {+ this.in = Objects.requireNonNull(in);+ this.shuffled = shuffled;+ this.orig = orig;+ }++ @Override+ public void close() throws IOException {+ in.close();+ }++ private FieldInfo remapFieldInfo(FieldInfo field) {+ FieldInfo fi = shuffled.fieldInfo(field.name);+ assert fi != null && fi.number == field.number;+ return orig.fieldInfo(field.name);+ }++ @Override+ public NumericDocValues getNumeric(FieldInfo field) throws IOException {+ return in.getNumeric(remapFieldInfo(field));+ }++ @Override+ public BinaryDocValues getBinary(FieldInfo field) throws IOException {+ return in.getBinary(remapFieldInfo(field));+ }++ @Override+ public SortedDocValues getSorted(FieldInfo field) throws IOException {+ return in.getSorted(remapFieldInfo(field));+ }++ @Override+ public SortedNumericDocValues getSortedNumeric(FieldInfo field) throws IOException {+ return in.getSortedNumeric(remapFieldInfo(field));+ }++ @Override+ public SortedSetDocValues getSortedSet(FieldInfo field) throws IOException {+ return in.getSortedSet(remapFieldInfo(field));+ }++ @Override+ public DocValuesSkipper getSkipper(FieldInfo field) throws IOException {+ return in.getSkipper(remapFieldInfo(field));+ }++ @Override+ public void checkIntegrity() throws IOException {+ in.checkIntegrity();+ }+ }++ @Override+ public NormsProducer getNormsReader() {+ NormsProducer in = super.getNormsReader();+ if (in == null) {+ return null;+ }+ return new MismatchedNormsProducer(in, shuffled, super.getFieldInfos());+ }++ private static class MismatchedNormsProducer extends NormsProducer {++ private final NormsProducer in;+ private final FieldInfos shuffled;+ private final FieldInfos orig;++ MismatchedNormsProducer(NormsProducer in, FieldInfos shuffled, FieldInfos orig) {+ this.in = Objects.requireNonNull(in);+ this.shuffled = shuffled;+ this.orig = orig;+ }++ @Override+ public void close() throws IOException {+ in.close();+ }++ private FieldInfo remapFieldInfo(FieldInfo field) {+ FieldInfo fi = shuffled.fieldInfo(field.name);+ assert fi != null && fi.number == field.number;+ return orig.fieldInfo(field.name);+ }++ @Override+ public NumericDocValues getNorms(FieldInfo field) throws IOException {+ return in.getNorms(remapFieldInfo(field));+ }++ @Override+ public void checkIntegrity() throws IOException {+ in.checkIntegrity();+ }+ }+}diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/index/MismatchedLeafReader.java b/lucene/test-framework/src/java/org/apache/lucene/tests/index/MismatchedLeafReader.javaindex eddee35240f..46404f514c6 100644--- a/lucene/test-framework/src/java/org/apache/lucene/tests/index/MismatchedLeafReader.java+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/index/MismatchedLeafReader.java@@ -28,8 +28,6 @@ import org.apache.lucene.index.FilterLeafReader;import org.apache.lucene.index.LeafReader;import org.apache.lucene.index.StoredFieldVisitor;import org.apache.lucene.index.StoredFields;-import org.apache.lucene.search.KnnCollector;-import org.apache.lucene.util.Bits;/*** Shuffles field numbers around to try to trip bugs where field numbers are assumed to always be@@ -55,7 +53,7 @@ public class MismatchedLeafReader extends FilterLeafReader {return new StoredFields() {@Overridepublic void document(int docID, StoredFieldVisitor visitor) throws IOException {- inStoredFields.document(docID, new MismatchedVisitor(visitor));+ inStoredFields.document(docID, new MismatchedVisitor(visitor, shuffled));}};}@@ -70,18 +68,6 @@ public class MismatchedLeafReader extends FilterLeafReader {return in.getReaderCacheHelper();}- @Override- public void searchNearestVectors(- String field, float[] target, KnnCollector knnCollector, Bits acceptDocs) throws IOException {- in.searchNearestVectors(field, target, knnCollector, acceptDocs);- }-- @Override- public void searchNearestVectors(- String field, byte[] target, KnnCollector knnCollector, Bits acceptDocs) throws IOException {- in.searchNearestVectors(field, target, knnCollector, acceptDocs);- }-static FieldInfos shuffleInfos(FieldInfos infos, Random random) {// first, shuffle the orderList<FieldInfo> shuffled = new ArrayList<>();@@ -124,11 +110,13 @@ public class MismatchedLeafReader extends FilterLeafReader {/** StoredFieldsVisitor that remaps actual field numbers to our new shuffled ones. */// TODO: its strange this part of our IR api exposes FieldInfo,// no other "user-accessible" codec apis do this?- class MismatchedVisitor extends StoredFieldVisitor {+ static class MismatchedVisitor extends StoredFieldVisitor {final StoredFieldVisitor in;+ final FieldInfos shuffled;- MismatchedVisitor(StoredFieldVisitor in) {+ MismatchedVisitor(StoredFieldVisitor in, FieldInfos shuffled) {this.in = in;+ this.shuffled = shuffled;}@Overridediff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/index/MockRandomMergePolicy.java b/lucene/test-framework/src/java/org/apache/lucene/tests/index/MockRandomMergePolicy.javaindex 74f3b87ed5d..d3f202ad9dc 100644--- a/lucene/test-framework/src/java/org/apache/lucene/tests/index/MockRandomMergePolicy.java+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/index/MockRandomMergePolicy.java@@ -237,8 +237,7 @@ public class MockRandomMergePolicy extends MergePolicy {"NOTE: MockRandomMergePolicy now swaps in a MismatchedLeafReader for merging reader="+ reader);}- return SlowCodecReaderWrapper.wrap(- new MismatchedLeafReader(new MergeReaderWrapper(reader), r));+ return new MismatchedCodecReader(reader, r);} else {// otherwise, reader is unchangedreturn reader;diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/util/LuceneTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/tests/util/LuceneTestCase.javaindex 26dd29e27b9..84fa120b88b 100644--- a/lucene/test-framework/src/java/org/apache/lucene/tests/util/LuceneTestCase.java+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/util/LuceneTestCase.java@@ -182,6 +182,7 @@ import org.apache.lucene.tests.index.AssertingLeafReader;import org.apache.lucene.tests.index.FieldFilterLeafReader;import org.apache.lucene.tests.index.MergingCodecReader;import org.apache.lucene.tests.index.MergingDirectoryReaderWrapper;+import org.apache.lucene.tests.index.MismatchedCodecReader;import org.apache.lucene.tests.index.MismatchedDirectoryReader;import org.apache.lucene.tests.index.MismatchedLeafReader;import org.apache.lucene.tests.index.MockIndexWriterEventListener;@@ -1746,12 +1747,14 @@ public abstract class LuceneTestCase extends Assert {System.out.println("NOTE: LuceneTestCase.wrapReader: wrapping previous reader="+ r- + " with MismatchedLeaf/DirectoryReader");+ + " with MismatchedLeaf/Directory/CodecReader");}if (r instanceof LeafReader) {r = new MismatchedLeafReader((LeafReader) r, random);} else if (r instanceof DirectoryReader) {r = new MismatchedDirectoryReader((DirectoryReader) r, random);+ } else if (r instanceof CodecReader) {+ r = new MismatchedCodecReader((CodecReader) r, random);}break;case 4: