Apache Solr open-source search software
Improve backwards compatibility tests for sorted indexes. (#2276)
Julie Tibshirani902ce0809dd5
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestBackwardsCompatibility.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestBackwardsCompatibility.javaindex 9a3fb7496d9..d89b6481471 100644--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestBackwardsCompatibility.java+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestBackwardsCompatibility.java@@ -60,7 +60,6 @@ import org.apache.lucene.document.TextField;import org.apache.lucene.index.BinaryDocValues;import org.apache.lucene.index.CheckIndex;import org.apache.lucene.index.DirectoryReader;-import org.apache.lucene.index.FieldInfos;import org.apache.lucene.index.Fields;import org.apache.lucene.index.IndexCommit;import org.apache.lucene.index.IndexFormatTooOldException;@@ -93,11 +92,16 @@ import org.apache.lucene.index.Term;import org.apache.lucene.index.Terms;import org.apache.lucene.index.TermsEnum;import org.apache.lucene.search.DocIdSetIterator;+import org.apache.lucene.search.DocValuesFieldExistsQuery;+import org.apache.lucene.search.FieldDoc;import org.apache.lucene.search.IndexSearcher;+import org.apache.lucene.search.MatchAllDocsQuery;+import org.apache.lucene.search.NormsFieldExistsQuery;import org.apache.lucene.search.ScoreDoc;import org.apache.lucene.search.Sort;import org.apache.lucene.search.SortField;import org.apache.lucene.search.TermQuery;+import org.apache.lucene.search.TopDocs;import org.apache.lucene.store.BaseDirectoryWrapper;import org.apache.lucene.store.ByteBuffersDirectory;import org.apache.lucene.store.Directory;@@ -1057,21 +1061,8 @@ public class TestBackwardsCompatibility extends LuceneTestCase {TestUtil.checkIndex(dir);- // true if this is a 4.0+ index- final boolean is40Index = FieldInfos.getMergedFieldInfos(reader).fieldInfo("content5") != null;- // true if this is a 4.2+ index- final boolean is42Index =- FieldInfos.getMergedFieldInfos(reader).fieldInfo("dvSortedSet") != null;- // true if this is a 4.9+ index- final boolean is49Index =- FieldInfos.getMergedFieldInfos(reader).fieldInfo("dvSortedNumeric") != null;- // true if this index has points (>= 6.0)- final boolean hasPoints =- FieldInfos.getMergedFieldInfos(reader).fieldInfo("intPoint1d") != null;-- assert is40Index;-final Bits liveDocs = MultiBits.getLiveDocs(reader);+ assertNotNull(liveDocs);for (int i = 0; i < 35; i++) {if (liveDocs.get(i)) {@@ -1079,8 +1070,7 @@ public class TestBackwardsCompatibility extends LuceneTestCase {List<IndexableField> fields = d.getFields();boolean isProxDoc = d.getField("content3") == null;if (isProxDoc) {- final int numFields = is40Index ? 7 : 5;- assertEquals(numFields, fields.size());+ assertEquals(7, fields.size());IndexableField f = d.getField("id");assertEquals("" + i, f.stringValue());@@ -1109,87 +1099,78 @@ public class TestBackwardsCompatibility extends LuceneTestCase {}}- if (is40Index) {- // check docvalues fields- NumericDocValues dvByte = MultiDocValues.getNumericValues(reader, "dvByte");- BinaryDocValues dvBytesDerefFixed =- MultiDocValues.getBinaryValues(reader, "dvBytesDerefFixed");- BinaryDocValues dvBytesDerefVar = MultiDocValues.getBinaryValues(reader, "dvBytesDerefVar");- SortedDocValues dvBytesSortedFixed =- MultiDocValues.getSortedValues(reader, "dvBytesSortedFixed");- SortedDocValues dvBytesSortedVar = MultiDocValues.getSortedValues(reader, "dvBytesSortedVar");- BinaryDocValues dvBytesStraightFixed =- MultiDocValues.getBinaryValues(reader, "dvBytesStraightFixed");- BinaryDocValues dvBytesStraightVar =- MultiDocValues.getBinaryValues(reader, "dvBytesStraightVar");- NumericDocValues dvDouble = MultiDocValues.getNumericValues(reader, "dvDouble");- NumericDocValues dvFloat = MultiDocValues.getNumericValues(reader, "dvFloat");- NumericDocValues dvInt = MultiDocValues.getNumericValues(reader, "dvInt");- NumericDocValues dvLong = MultiDocValues.getNumericValues(reader, "dvLong");- NumericDocValues dvPacked = MultiDocValues.getNumericValues(reader, "dvPacked");- NumericDocValues dvShort = MultiDocValues.getNumericValues(reader, "dvShort");- SortedSetDocValues dvSortedSet = null;- if (is42Index) {- dvSortedSet = MultiDocValues.getSortedSetValues(reader, "dvSortedSet");- }- SortedNumericDocValues dvSortedNumeric = null;- if (is49Index) {- dvSortedNumeric = MultiDocValues.getSortedNumericValues(reader, "dvSortedNumeric");- }+ // check docvalues fields+ NumericDocValues dvByte = MultiDocValues.getNumericValues(reader, "dvByte");+ BinaryDocValues dvBytesDerefFixed = MultiDocValues.getBinaryValues(reader, "dvBytesDerefFixed");+ BinaryDocValues dvBytesDerefVar = MultiDocValues.getBinaryValues(reader, "dvBytesDerefVar");+ SortedDocValues dvBytesSortedFixed =+ MultiDocValues.getSortedValues(reader, "dvBytesSortedFixed");+ SortedDocValues dvBytesSortedVar = MultiDocValues.getSortedValues(reader, "dvBytesSortedVar");+ BinaryDocValues dvBytesStraightFixed =+ MultiDocValues.getBinaryValues(reader, "dvBytesStraightFixed");+ BinaryDocValues dvBytesStraightVar =+ MultiDocValues.getBinaryValues(reader, "dvBytesStraightVar");+ NumericDocValues dvDouble = MultiDocValues.getNumericValues(reader, "dvDouble");+ NumericDocValues dvFloat = MultiDocValues.getNumericValues(reader, "dvFloat");+ NumericDocValues dvInt = MultiDocValues.getNumericValues(reader, "dvInt");+ NumericDocValues dvLong = MultiDocValues.getNumericValues(reader, "dvLong");+ NumericDocValues dvPacked = MultiDocValues.getNumericValues(reader, "dvPacked");+ NumericDocValues dvShort = MultiDocValues.getNumericValues(reader, "dvShort");++ SortedSetDocValues dvSortedSet = MultiDocValues.getSortedSetValues(reader, "dvSortedSet");+ SortedNumericDocValues dvSortedNumeric =+ MultiDocValues.getSortedNumericValues(reader, "dvSortedNumeric");- for (int i = 0; i < 35; i++) {- int id = Integer.parseInt(reader.document(i).get("id"));- assertEquals(i, dvByte.nextDoc());- assertEquals(id, dvByte.longValue());-- byte bytes[] =- new byte[] {(byte) (id >>> 24), (byte) (id >>> 16), (byte) (id >>> 8), (byte) id};- BytesRef expectedRef = new BytesRef(bytes);-- assertEquals(i, dvBytesDerefFixed.nextDoc());- BytesRef term = dvBytesDerefFixed.binaryValue();- assertEquals(expectedRef, term);- assertEquals(i, dvBytesDerefVar.nextDoc());- term = dvBytesDerefVar.binaryValue();- assertEquals(expectedRef, term);- assertEquals(i, dvBytesSortedFixed.nextDoc());- term = dvBytesSortedFixed.binaryValue();- assertEquals(expectedRef, term);- assertEquals(i, dvBytesSortedVar.nextDoc());- term = dvBytesSortedVar.binaryValue();- assertEquals(expectedRef, term);- assertEquals(i, dvBytesStraightFixed.nextDoc());- term = dvBytesStraightFixed.binaryValue();- assertEquals(expectedRef, term);- assertEquals(i, dvBytesStraightVar.nextDoc());- term = dvBytesStraightVar.binaryValue();- assertEquals(expectedRef, term);-- assertEquals(i, dvDouble.nextDoc());- assertEquals((double) id, Double.longBitsToDouble(dvDouble.longValue()), 0D);- assertEquals(i, dvFloat.nextDoc());- assertEquals((float) id, Float.intBitsToFloat((int) dvFloat.longValue()), 0F);- assertEquals(i, dvInt.nextDoc());- assertEquals(id, dvInt.longValue());- assertEquals(i, dvLong.nextDoc());- assertEquals(id, dvLong.longValue());- assertEquals(i, dvPacked.nextDoc());- assertEquals(id, dvPacked.longValue());- assertEquals(i, dvShort.nextDoc());- assertEquals(id, dvShort.longValue());- if (is42Index) {- assertEquals(i, dvSortedSet.nextDoc());- long ord = dvSortedSet.nextOrd();- assertEquals(SortedSetDocValues.NO_MORE_ORDS, dvSortedSet.nextOrd());- term = dvSortedSet.lookupOrd(ord);- assertEquals(expectedRef, term);- }- if (is49Index) {- assertEquals(i, dvSortedNumeric.nextDoc());- assertEquals(1, dvSortedNumeric.docValueCount());- assertEquals(id, dvSortedNumeric.nextValue());- }- }+ for (int i = 0; i < 35; i++) {+ int id = Integer.parseInt(reader.document(i).get("id"));+ assertEquals(i, dvByte.nextDoc());+ assertEquals(id, dvByte.longValue());++ byte bytes[] =+ new byte[] {(byte) (id >>> 24), (byte) (id >>> 16), (byte) (id >>> 8), (byte) id};+ BytesRef expectedRef = new BytesRef(bytes);++ assertEquals(i, dvBytesDerefFixed.nextDoc());+ BytesRef term = dvBytesDerefFixed.binaryValue();+ assertEquals(expectedRef, term);+ assertEquals(i, dvBytesDerefVar.nextDoc());+ term = dvBytesDerefVar.binaryValue();+ assertEquals(expectedRef, term);+ assertEquals(i, dvBytesSortedFixed.nextDoc());+ term = dvBytesSortedFixed.binaryValue();+ assertEquals(expectedRef, term);+ assertEquals(i, dvBytesSortedVar.nextDoc());+ term = dvBytesSortedVar.binaryValue();+ assertEquals(expectedRef, term);+ assertEquals(i, dvBytesStraightFixed.nextDoc());+ term = dvBytesStraightFixed.binaryValue();+ assertEquals(expectedRef, term);+ assertEquals(i, dvBytesStraightVar.nextDoc());+ term = dvBytesStraightVar.binaryValue();+ assertEquals(expectedRef, term);++ assertEquals(i, dvDouble.nextDoc());+ assertEquals((double) id, Double.longBitsToDouble(dvDouble.longValue()), 0D);+ assertEquals(i, dvFloat.nextDoc());+ assertEquals((float) id, Float.intBitsToFloat((int) dvFloat.longValue()), 0F);+ assertEquals(i, dvInt.nextDoc());+ assertEquals(id, dvInt.longValue());+ assertEquals(i, dvLong.nextDoc());+ assertEquals(id, dvLong.longValue());+ assertEquals(i, dvPacked.nextDoc());+ assertEquals(id, dvPacked.longValue());+ assertEquals(i, dvShort.nextDoc());+ assertEquals(id, dvShort.longValue());++ assertEquals(i, dvSortedSet.nextDoc());+ long ord = dvSortedSet.nextOrd();+ assertEquals(SortedSetDocValues.NO_MORE_ORDS, dvSortedSet.nextOrd());+ term = dvSortedSet.lookupOrd(ord);+ assertEquals(expectedRef, term);++ assertEquals(i, dvSortedNumeric.nextDoc());+ assertEquals(1, dvSortedNumeric.docValueCount());+ assertEquals(id, dvSortedNumeric.nextValue());}ScoreDoc[] hits =@@ -1201,17 +1182,13 @@ public class TestBackwardsCompatibility extends LuceneTestCase {doTestHits(hits, 34, searcher.getIndexReader());- if (is40Index) {- hits =- searcher.search(new TermQuery(new Term(new String("content5"), "aaa")), 1000).scoreDocs;+ hits = searcher.search(new TermQuery(new Term(new String("content5"), "aaa")), 1000).scoreDocs;- doTestHits(hits, 34, searcher.getIndexReader());+ doTestHits(hits, 34, searcher.getIndexReader());- hits =- searcher.search(new TermQuery(new Term(new String("content6"), "aaa")), 1000).scoreDocs;+ hits = searcher.search(new TermQuery(new Term(new String("content6"), "aaa")), 1000).scoreDocs;- doTestHits(hits, 34, searcher.getIndexReader());- }+ doTestHits(hits, 34, searcher.getIndexReader());hits = searcher.search(new TermQuery(new Term("utf8", "\u0000")), 1000).scoreDocs;assertEquals(34, hits.length);@@ -1223,72 +1200,67 @@ public class TestBackwardsCompatibility extends LuceneTestCase {hits = searcher.search(new TermQuery(new Term("utf8", "ab\ud917\udc17cd")), 1000).scoreDocs;assertEquals(34, hits.length);- if (hasPoints) {- doTestHits(- searcher.search(IntPoint.newRangeQuery("intPoint1d", 0, 34), 1000).scoreDocs,- 34,- searcher.getIndexReader());- doTestHits(- searcher.search(- IntPoint.newRangeQuery("intPoint2d", new int[] {0, 0}, new int[] {34, 68}), 1000)- .scoreDocs,- 34,- searcher.getIndexReader());- doTestHits(- searcher.search(FloatPoint.newRangeQuery("floatPoint1d", 0f, 34f), 1000).scoreDocs,- 34,- searcher.getIndexReader());- doTestHits(- searcher.search(- FloatPoint.newRangeQuery(- "floatPoint2d", new float[] {0f, 0f}, new float[] {34f, 68f}),- 1000)- .scoreDocs,- 34,- searcher.getIndexReader());- doTestHits(- searcher.search(LongPoint.newRangeQuery("longPoint1d", 0, 34), 1000).scoreDocs,- 34,- searcher.getIndexReader());- doTestHits(- searcher.search(- LongPoint.newRangeQuery("longPoint2d", new long[] {0, 0}, new long[] {34, 68}),- 1000)- .scoreDocs,- 34,- searcher.getIndexReader());- doTestHits(- searcher.search(DoublePoint.newRangeQuery("doublePoint1d", 0.0, 34.0), 1000).scoreDocs,- 34,- searcher.getIndexReader());- doTestHits(- searcher.search(- DoublePoint.newRangeQuery(- "doublePoint2d", new double[] {0.0, 0.0}, new double[] {34.0, 68.0}),- 1000)- .scoreDocs,- 34,- searcher.getIndexReader());-- byte[] bytes1 = new byte[4];- byte[] bytes2 = new byte[] {0, 0, 0, (byte) 34};- doTestHits(- searcher.search(BinaryPoint.newRangeQuery("binaryPoint1d", bytes1, bytes2), 1000)- .scoreDocs,- 34,- searcher.getIndexReader());- byte[] bytes3 = new byte[] {0, 0, 0, (byte) 68};- doTestHits(- searcher.search(- BinaryPoint.newRangeQuery(- "binaryPoint2d",- new byte[][] {bytes1, bytes1},- new byte[][] {bytes2, bytes3}),- 1000)- .scoreDocs,- 34,- searcher.getIndexReader());- }+ doTestHits(+ searcher.search(IntPoint.newRangeQuery("intPoint1d", 0, 34), 1000).scoreDocs,+ 34,+ searcher.getIndexReader());+ doTestHits(+ searcher.search(+ IntPoint.newRangeQuery("intPoint2d", new int[] {0, 0}, new int[] {34, 68}), 1000)+ .scoreDocs,+ 34,+ searcher.getIndexReader());+ doTestHits(+ searcher.search(FloatPoint.newRangeQuery("floatPoint1d", 0f, 34f), 1000).scoreDocs,+ 34,+ searcher.getIndexReader());+ doTestHits(+ searcher.search(+ FloatPoint.newRangeQuery(+ "floatPoint2d", new float[] {0f, 0f}, new float[] {34f, 68f}),+ 1000)+ .scoreDocs,+ 34,+ searcher.getIndexReader());+ doTestHits(+ searcher.search(LongPoint.newRangeQuery("longPoint1d", 0, 34), 1000).scoreDocs,+ 34,+ searcher.getIndexReader());+ doTestHits(+ searcher.search(+ LongPoint.newRangeQuery("longPoint2d", new long[] {0, 0}, new long[] {34, 68}),+ 1000)+ .scoreDocs,+ 34,+ searcher.getIndexReader());+ doTestHits(+ searcher.search(DoublePoint.newRangeQuery("doublePoint1d", 0.0, 34.0), 1000).scoreDocs,+ 34,+ searcher.getIndexReader());+ doTestHits(+ searcher.search(+ DoublePoint.newRangeQuery(+ "doublePoint2d", new double[] {0.0, 0.0}, new double[] {34.0, 68.0}),+ 1000)+ .scoreDocs,+ 34,+ searcher.getIndexReader());++ byte[] bytes1 = new byte[4];+ byte[] bytes2 = new byte[] {0, 0, 0, (byte) 34};+ doTestHits(+ searcher.search(BinaryPoint.newRangeQuery("binaryPoint1d", bytes1, bytes2), 1000).scoreDocs,+ 34,+ searcher.getIndexReader());+ byte[] bytes3 = new byte[] {0, 0, 0, (byte) 68};+ doTestHits(+ searcher.search(+ BinaryPoint.newRangeQuery(+ "binaryPoint2d", new byte[][] {bytes1, bytes1}, new byte[][] {bytes2, bytes3}),+ 1000)+ .scoreDocs,+ 34,+ searcher.getIndexReader());reader.close();}@@ -1791,9 +1763,13 @@ public class TestBackwardsCompatibility extends LuceneTestCase {Path oldIndexDir = createTempDir("moreterms");TestUtil.unzip(getDataInputStream(moreTermsIndex), oldIndexDir);Directory dir = newFSDirectory(oldIndexDir);+ DirectoryReader reader = DirectoryReader.open(dir);+verifyUsesDefaultCodec(dir, moreTermsIndex);- // TODO: more testsTestUtil.checkIndex(dir);+ searchExampleIndex(reader);++ reader.close();dir.close();}@@ -1978,22 +1954,60 @@ public class TestBackwardsCompatibility extends LuceneTestCase {assertNotNull("Sorted index index " + name + " not found", resource);TestUtil.unzip(resource, path);- // TODO: more testsDirectory dir = newFSDirectory(path);-DirectoryReader reader = DirectoryReader.open(dir);+assertEquals(1, reader.leaves().size());Sort sort = reader.leaves().get(0).reader().getMetaData().getSort();assertNotNull(sort);assertEquals("<long: \"dateDV\">!", sort.toString());- reader.close();- // this will confirm the docs really are sorted:+ // This will confirm the docs are really sortedTestUtil.checkIndex(dir);++ searchExampleIndex(reader);++ reader.close();dir.close();}}+ private void searchExampleIndex(DirectoryReader reader) throws IOException {+ IndexSearcher searcher = newSearcher(reader);++ TopDocs topDocs = searcher.search(new NormsFieldExistsQuery("titleTokenized"), 10);+ assertEquals(50, topDocs.totalHits.value);++ topDocs = searcher.search(new DocValuesFieldExistsQuery("titleDV"), 10);+ assertEquals(50, topDocs.totalHits.value);++ topDocs = searcher.search(new TermQuery(new Term("body", "ja")), 10);+ assertTrue(topDocs.totalHits.value > 0);++ topDocs =+ searcher.search(+ IntPoint.newRangeQuery("docid_int", 42, 44),+ 10,+ new Sort(new SortField("docid_intDV", SortField.Type.INT)));+ assertEquals(3, topDocs.totalHits.value);+ assertEquals(3, topDocs.scoreDocs.length);+ assertEquals(42, ((FieldDoc) topDocs.scoreDocs[0]).fields[0]);+ assertEquals(43, ((FieldDoc) topDocs.scoreDocs[1]).fields[0]);+ assertEquals(44, ((FieldDoc) topDocs.scoreDocs[2]).fields[0]);++ topDocs = searcher.search(new TermQuery(new Term("body", "the")), 5);+ assertTrue(topDocs.totalHits.value > 0);++ topDocs =+ searcher.search(+ new MatchAllDocsQuery(), 5, new Sort(new SortField("dateDV", SortField.Type.LONG)));+ assertEquals(50, topDocs.totalHits.value);+ assertEquals(5, topDocs.scoreDocs.length);+ long firstDate = (Long) ((FieldDoc) topDocs.scoreDocs[0]).fields[0];+ long lastDate = (Long) ((FieldDoc) topDocs.scoreDocs[4]).fields[0];+ assertTrue(firstDate <= lastDate);+ }+static long getValue(BinaryDocValues bdv) throws IOException {BytesRef term = bdv.binaryValue();int idx = term.offset;