Skip to content

armgit / solr

Public Imported from GitHub
0

Apache Solr open-source search software

Improve backwards compatibility tests for sorted indexes. (#2276)

Julie Tibshirani902ce0809dd5
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestBackwardsCompatibility.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestBackwardsCompatibility.java
index 9a3fb7496d9..d89b6481471 100644
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestBackwardsCompatibility.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestBackwardsCompatibility.java
@@ -60,7 +60,6 @@ import org.apache.lucene.document.TextField;
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.CheckIndex;
import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.Fields;
import org.apache.lucene.index.IndexCommit;
import org.apache.lucene.index.IndexFormatTooOldException;
@@ -93,11 +92,16 @@ import org.apache.lucene.index.Term;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.DocValuesFieldExistsQuery;
+import org.apache.lucene.search.FieldDoc;
import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.NormsFieldExistsQuery;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.BaseDirectoryWrapper;
import org.apache.lucene.store.ByteBuffersDirectory;
import org.apache.lucene.store.Directory;
@@ -1057,21 +1061,8 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
TestUtil.checkIndex(dir);
- // true if this is a 4.0+ index
- final boolean is40Index = FieldInfos.getMergedFieldInfos(reader).fieldInfo("content5") != null;
- // true if this is a 4.2+ index
- final boolean is42Index =
- FieldInfos.getMergedFieldInfos(reader).fieldInfo("dvSortedSet") != null;
- // true if this is a 4.9+ index
- final boolean is49Index =
- FieldInfos.getMergedFieldInfos(reader).fieldInfo("dvSortedNumeric") != null;
- // true if this index has points (>= 6.0)
- final boolean hasPoints =
- FieldInfos.getMergedFieldInfos(reader).fieldInfo("intPoint1d") != null;
-
- assert is40Index;
-
final Bits liveDocs = MultiBits.getLiveDocs(reader);
+ assertNotNull(liveDocs);
for (int i = 0; i < 35; i++) {
if (liveDocs.get(i)) {
@@ -1079,8 +1070,7 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
List<IndexableField> fields = d.getFields();
boolean isProxDoc = d.getField("content3") == null;
if (isProxDoc) {
- final int numFields = is40Index ? 7 : 5;
- assertEquals(numFields, fields.size());
+ assertEquals(7, fields.size());
IndexableField f = d.getField("id");
assertEquals("" + i, f.stringValue());
@@ -1109,87 +1099,78 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
}
}
- if (is40Index) {
- // check docvalues fields
- NumericDocValues dvByte = MultiDocValues.getNumericValues(reader, "dvByte");
- BinaryDocValues dvBytesDerefFixed =
- MultiDocValues.getBinaryValues(reader, "dvBytesDerefFixed");
- BinaryDocValues dvBytesDerefVar = MultiDocValues.getBinaryValues(reader, "dvBytesDerefVar");
- SortedDocValues dvBytesSortedFixed =
- MultiDocValues.getSortedValues(reader, "dvBytesSortedFixed");
- SortedDocValues dvBytesSortedVar = MultiDocValues.getSortedValues(reader, "dvBytesSortedVar");
- BinaryDocValues dvBytesStraightFixed =
- MultiDocValues.getBinaryValues(reader, "dvBytesStraightFixed");
- BinaryDocValues dvBytesStraightVar =
- MultiDocValues.getBinaryValues(reader, "dvBytesStraightVar");
- NumericDocValues dvDouble = MultiDocValues.getNumericValues(reader, "dvDouble");
- NumericDocValues dvFloat = MultiDocValues.getNumericValues(reader, "dvFloat");
- NumericDocValues dvInt = MultiDocValues.getNumericValues(reader, "dvInt");
- NumericDocValues dvLong = MultiDocValues.getNumericValues(reader, "dvLong");
- NumericDocValues dvPacked = MultiDocValues.getNumericValues(reader, "dvPacked");
- NumericDocValues dvShort = MultiDocValues.getNumericValues(reader, "dvShort");
- SortedSetDocValues dvSortedSet = null;
- if (is42Index) {
- dvSortedSet = MultiDocValues.getSortedSetValues(reader, "dvSortedSet");
- }
- SortedNumericDocValues dvSortedNumeric = null;
- if (is49Index) {
- dvSortedNumeric = MultiDocValues.getSortedNumericValues(reader, "dvSortedNumeric");
- }
+ // check docvalues fields
+ NumericDocValues dvByte = MultiDocValues.getNumericValues(reader, "dvByte");
+ BinaryDocValues dvBytesDerefFixed = MultiDocValues.getBinaryValues(reader, "dvBytesDerefFixed");
+ BinaryDocValues dvBytesDerefVar = MultiDocValues.getBinaryValues(reader, "dvBytesDerefVar");
+ SortedDocValues dvBytesSortedFixed =
+ MultiDocValues.getSortedValues(reader, "dvBytesSortedFixed");
+ SortedDocValues dvBytesSortedVar = MultiDocValues.getSortedValues(reader, "dvBytesSortedVar");
+ BinaryDocValues dvBytesStraightFixed =
+ MultiDocValues.getBinaryValues(reader, "dvBytesStraightFixed");
+ BinaryDocValues dvBytesStraightVar =
+ MultiDocValues.getBinaryValues(reader, "dvBytesStraightVar");
+ NumericDocValues dvDouble = MultiDocValues.getNumericValues(reader, "dvDouble");
+ NumericDocValues dvFloat = MultiDocValues.getNumericValues(reader, "dvFloat");
+ NumericDocValues dvInt = MultiDocValues.getNumericValues(reader, "dvInt");
+ NumericDocValues dvLong = MultiDocValues.getNumericValues(reader, "dvLong");
+ NumericDocValues dvPacked = MultiDocValues.getNumericValues(reader, "dvPacked");
+ NumericDocValues dvShort = MultiDocValues.getNumericValues(reader, "dvShort");
+
+ SortedSetDocValues dvSortedSet = MultiDocValues.getSortedSetValues(reader, "dvSortedSet");
+ SortedNumericDocValues dvSortedNumeric =
+ MultiDocValues.getSortedNumericValues(reader, "dvSortedNumeric");
- for (int i = 0; i < 35; i++) {
- int id = Integer.parseInt(reader.document(i).get("id"));
- assertEquals(i, dvByte.nextDoc());
- assertEquals(id, dvByte.longValue());
-
- byte bytes[] =
- new byte[] {(byte) (id >>> 24), (byte) (id >>> 16), (byte) (id >>> 8), (byte) id};
- BytesRef expectedRef = new BytesRef(bytes);
-
- assertEquals(i, dvBytesDerefFixed.nextDoc());
- BytesRef term = dvBytesDerefFixed.binaryValue();
- assertEquals(expectedRef, term);
- assertEquals(i, dvBytesDerefVar.nextDoc());
- term = dvBytesDerefVar.binaryValue();
- assertEquals(expectedRef, term);
- assertEquals(i, dvBytesSortedFixed.nextDoc());
- term = dvBytesSortedFixed.binaryValue();
- assertEquals(expectedRef, term);
- assertEquals(i, dvBytesSortedVar.nextDoc());
- term = dvBytesSortedVar.binaryValue();
- assertEquals(expectedRef, term);
- assertEquals(i, dvBytesStraightFixed.nextDoc());
- term = dvBytesStraightFixed.binaryValue();
- assertEquals(expectedRef, term);
- assertEquals(i, dvBytesStraightVar.nextDoc());
- term = dvBytesStraightVar.binaryValue();
- assertEquals(expectedRef, term);
-
- assertEquals(i, dvDouble.nextDoc());
- assertEquals((double) id, Double.longBitsToDouble(dvDouble.longValue()), 0D);
- assertEquals(i, dvFloat.nextDoc());
- assertEquals((float) id, Float.intBitsToFloat((int) dvFloat.longValue()), 0F);
- assertEquals(i, dvInt.nextDoc());
- assertEquals(id, dvInt.longValue());
- assertEquals(i, dvLong.nextDoc());
- assertEquals(id, dvLong.longValue());
- assertEquals(i, dvPacked.nextDoc());
- assertEquals(id, dvPacked.longValue());
- assertEquals(i, dvShort.nextDoc());
- assertEquals(id, dvShort.longValue());
- if (is42Index) {
- assertEquals(i, dvSortedSet.nextDoc());
- long ord = dvSortedSet.nextOrd();
- assertEquals(SortedSetDocValues.NO_MORE_ORDS, dvSortedSet.nextOrd());
- term = dvSortedSet.lookupOrd(ord);
- assertEquals(expectedRef, term);
- }
- if (is49Index) {
- assertEquals(i, dvSortedNumeric.nextDoc());
- assertEquals(1, dvSortedNumeric.docValueCount());
- assertEquals(id, dvSortedNumeric.nextValue());
- }
- }
+ for (int i = 0; i < 35; i++) {
+ int id = Integer.parseInt(reader.document(i).get("id"));
+ assertEquals(i, dvByte.nextDoc());
+ assertEquals(id, dvByte.longValue());
+
+ byte bytes[] =
+ new byte[] {(byte) (id >>> 24), (byte) (id >>> 16), (byte) (id >>> 8), (byte) id};
+ BytesRef expectedRef = new BytesRef(bytes);
+
+ assertEquals(i, dvBytesDerefFixed.nextDoc());
+ BytesRef term = dvBytesDerefFixed.binaryValue();
+ assertEquals(expectedRef, term);
+ assertEquals(i, dvBytesDerefVar.nextDoc());
+ term = dvBytesDerefVar.binaryValue();
+ assertEquals(expectedRef, term);
+ assertEquals(i, dvBytesSortedFixed.nextDoc());
+ term = dvBytesSortedFixed.binaryValue();
+ assertEquals(expectedRef, term);
+ assertEquals(i, dvBytesSortedVar.nextDoc());
+ term = dvBytesSortedVar.binaryValue();
+ assertEquals(expectedRef, term);
+ assertEquals(i, dvBytesStraightFixed.nextDoc());
+ term = dvBytesStraightFixed.binaryValue();
+ assertEquals(expectedRef, term);
+ assertEquals(i, dvBytesStraightVar.nextDoc());
+ term = dvBytesStraightVar.binaryValue();
+ assertEquals(expectedRef, term);
+
+ assertEquals(i, dvDouble.nextDoc());
+ assertEquals((double) id, Double.longBitsToDouble(dvDouble.longValue()), 0D);
+ assertEquals(i, dvFloat.nextDoc());
+ assertEquals((float) id, Float.intBitsToFloat((int) dvFloat.longValue()), 0F);
+ assertEquals(i, dvInt.nextDoc());
+ assertEquals(id, dvInt.longValue());
+ assertEquals(i, dvLong.nextDoc());
+ assertEquals(id, dvLong.longValue());
+ assertEquals(i, dvPacked.nextDoc());
+ assertEquals(id, dvPacked.longValue());
+ assertEquals(i, dvShort.nextDoc());
+ assertEquals(id, dvShort.longValue());
+
+ assertEquals(i, dvSortedSet.nextDoc());
+ long ord = dvSortedSet.nextOrd();
+ assertEquals(SortedSetDocValues.NO_MORE_ORDS, dvSortedSet.nextOrd());
+ term = dvSortedSet.lookupOrd(ord);
+ assertEquals(expectedRef, term);
+
+ assertEquals(i, dvSortedNumeric.nextDoc());
+ assertEquals(1, dvSortedNumeric.docValueCount());
+ assertEquals(id, dvSortedNumeric.nextValue());
}
ScoreDoc[] hits =
@@ -1201,17 +1182,13 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
doTestHits(hits, 34, searcher.getIndexReader());
- if (is40Index) {
- hits =
- searcher.search(new TermQuery(new Term(new String("content5"), "aaa")), 1000).scoreDocs;
+ hits = searcher.search(new TermQuery(new Term(new String("content5"), "aaa")), 1000).scoreDocs;
- doTestHits(hits, 34, searcher.getIndexReader());
+ doTestHits(hits, 34, searcher.getIndexReader());
- hits =
- searcher.search(new TermQuery(new Term(new String("content6"), "aaa")), 1000).scoreDocs;
+ hits = searcher.search(new TermQuery(new Term(new String("content6"), "aaa")), 1000).scoreDocs;
- doTestHits(hits, 34, searcher.getIndexReader());
- }
+ doTestHits(hits, 34, searcher.getIndexReader());
hits = searcher.search(new TermQuery(new Term("utf8", "\u0000")), 1000).scoreDocs;
assertEquals(34, hits.length);
@@ -1223,72 +1200,67 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
hits = searcher.search(new TermQuery(new Term("utf8", "ab\ud917\udc17cd")), 1000).scoreDocs;
assertEquals(34, hits.length);
- if (hasPoints) {
- doTestHits(
- searcher.search(IntPoint.newRangeQuery("intPoint1d", 0, 34), 1000).scoreDocs,
- 34,
- searcher.getIndexReader());
- doTestHits(
- searcher.search(
- IntPoint.newRangeQuery("intPoint2d", new int[] {0, 0}, new int[] {34, 68}), 1000)
- .scoreDocs,
- 34,
- searcher.getIndexReader());
- doTestHits(
- searcher.search(FloatPoint.newRangeQuery("floatPoint1d", 0f, 34f), 1000).scoreDocs,
- 34,
- searcher.getIndexReader());
- doTestHits(
- searcher.search(
- FloatPoint.newRangeQuery(
- "floatPoint2d", new float[] {0f, 0f}, new float[] {34f, 68f}),
- 1000)
- .scoreDocs,
- 34,
- searcher.getIndexReader());
- doTestHits(
- searcher.search(LongPoint.newRangeQuery("longPoint1d", 0, 34), 1000).scoreDocs,
- 34,
- searcher.getIndexReader());
- doTestHits(
- searcher.search(
- LongPoint.newRangeQuery("longPoint2d", new long[] {0, 0}, new long[] {34, 68}),
- 1000)
- .scoreDocs,
- 34,
- searcher.getIndexReader());
- doTestHits(
- searcher.search(DoublePoint.newRangeQuery("doublePoint1d", 0.0, 34.0), 1000).scoreDocs,
- 34,
- searcher.getIndexReader());
- doTestHits(
- searcher.search(
- DoublePoint.newRangeQuery(
- "doublePoint2d", new double[] {0.0, 0.0}, new double[] {34.0, 68.0}),
- 1000)
- .scoreDocs,
- 34,
- searcher.getIndexReader());
-
- byte[] bytes1 = new byte[4];
- byte[] bytes2 = new byte[] {0, 0, 0, (byte) 34};
- doTestHits(
- searcher.search(BinaryPoint.newRangeQuery("binaryPoint1d", bytes1, bytes2), 1000)
- .scoreDocs,
- 34,
- searcher.getIndexReader());
- byte[] bytes3 = new byte[] {0, 0, 0, (byte) 68};
- doTestHits(
- searcher.search(
- BinaryPoint.newRangeQuery(
- "binaryPoint2d",
- new byte[][] {bytes1, bytes1},
- new byte[][] {bytes2, bytes3}),
- 1000)
- .scoreDocs,
- 34,
- searcher.getIndexReader());
- }
+ doTestHits(
+ searcher.search(IntPoint.newRangeQuery("intPoint1d", 0, 34), 1000).scoreDocs,
+ 34,
+ searcher.getIndexReader());
+ doTestHits(
+ searcher.search(
+ IntPoint.newRangeQuery("intPoint2d", new int[] {0, 0}, new int[] {34, 68}), 1000)
+ .scoreDocs,
+ 34,
+ searcher.getIndexReader());
+ doTestHits(
+ searcher.search(FloatPoint.newRangeQuery("floatPoint1d", 0f, 34f), 1000).scoreDocs,
+ 34,
+ searcher.getIndexReader());
+ doTestHits(
+ searcher.search(
+ FloatPoint.newRangeQuery(
+ "floatPoint2d", new float[] {0f, 0f}, new float[] {34f, 68f}),
+ 1000)
+ .scoreDocs,
+ 34,
+ searcher.getIndexReader());
+ doTestHits(
+ searcher.search(LongPoint.newRangeQuery("longPoint1d", 0, 34), 1000).scoreDocs,
+ 34,
+ searcher.getIndexReader());
+ doTestHits(
+ searcher.search(
+ LongPoint.newRangeQuery("longPoint2d", new long[] {0, 0}, new long[] {34, 68}),
+ 1000)
+ .scoreDocs,
+ 34,
+ searcher.getIndexReader());
+ doTestHits(
+ searcher.search(DoublePoint.newRangeQuery("doublePoint1d", 0.0, 34.0), 1000).scoreDocs,
+ 34,
+ searcher.getIndexReader());
+ doTestHits(
+ searcher.search(
+ DoublePoint.newRangeQuery(
+ "doublePoint2d", new double[] {0.0, 0.0}, new double[] {34.0, 68.0}),
+ 1000)
+ .scoreDocs,
+ 34,
+ searcher.getIndexReader());
+
+ byte[] bytes1 = new byte[4];
+ byte[] bytes2 = new byte[] {0, 0, 0, (byte) 34};
+ doTestHits(
+ searcher.search(BinaryPoint.newRangeQuery("binaryPoint1d", bytes1, bytes2), 1000).scoreDocs,
+ 34,
+ searcher.getIndexReader());
+ byte[] bytes3 = new byte[] {0, 0, 0, (byte) 68};
+ doTestHits(
+ searcher.search(
+ BinaryPoint.newRangeQuery(
+ "binaryPoint2d", new byte[][] {bytes1, bytes1}, new byte[][] {bytes2, bytes3}),
+ 1000)
+ .scoreDocs,
+ 34,
+ searcher.getIndexReader());
reader.close();
}
@@ -1791,9 +1763,13 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
Path oldIndexDir = createTempDir("moreterms");
TestUtil.unzip(getDataInputStream(moreTermsIndex), oldIndexDir);
Directory dir = newFSDirectory(oldIndexDir);
+ DirectoryReader reader = DirectoryReader.open(dir);
+
verifyUsesDefaultCodec(dir, moreTermsIndex);
- // TODO: more tests
TestUtil.checkIndex(dir);
+ searchExampleIndex(reader);
+
+ reader.close();
dir.close();
}
@@ -1978,22 +1954,60 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
assertNotNull("Sorted index index " + name + " not found", resource);
TestUtil.unzip(resource, path);
- // TODO: more tests
Directory dir = newFSDirectory(path);
-
DirectoryReader reader = DirectoryReader.open(dir);
+
assertEquals(1, reader.leaves().size());
Sort sort = reader.leaves().get(0).reader().getMetaData().getSort();
assertNotNull(sort);
assertEquals("<long: \"dateDV\">!", sort.toString());
- reader.close();
- // this will confirm the docs really are sorted:
+ // This will confirm the docs are really sorted
TestUtil.checkIndex(dir);
+
+ searchExampleIndex(reader);
+
+ reader.close();
dir.close();
}
}
+ private void searchExampleIndex(DirectoryReader reader) throws IOException {
+ IndexSearcher searcher = newSearcher(reader);
+
+ TopDocs topDocs = searcher.search(new NormsFieldExistsQuery("titleTokenized"), 10);
+ assertEquals(50, topDocs.totalHits.value);
+
+ topDocs = searcher.search(new DocValuesFieldExistsQuery("titleDV"), 10);
+ assertEquals(50, topDocs.totalHits.value);
+
+ topDocs = searcher.search(new TermQuery(new Term("body", "ja")), 10);
+ assertTrue(topDocs.totalHits.value > 0);
+
+ topDocs =
+ searcher.search(
+ IntPoint.newRangeQuery("docid_int", 42, 44),
+ 10,
+ new Sort(new SortField("docid_intDV", SortField.Type.INT)));
+ assertEquals(3, topDocs.totalHits.value);
+ assertEquals(3, topDocs.scoreDocs.length);
+ assertEquals(42, ((FieldDoc) topDocs.scoreDocs[0]).fields[0]);
+ assertEquals(43, ((FieldDoc) topDocs.scoreDocs[1]).fields[0]);
+ assertEquals(44, ((FieldDoc) topDocs.scoreDocs[2]).fields[0]);
+
+ topDocs = searcher.search(new TermQuery(new Term("body", "the")), 5);
+ assertTrue(topDocs.totalHits.value > 0);
+
+ topDocs =
+ searcher.search(
+ new MatchAllDocsQuery(), 5, new Sort(new SortField("dateDV", SortField.Type.LONG)));
+ assertEquals(50, topDocs.totalHits.value);
+ assertEquals(5, topDocs.scoreDocs.length);
+ long firstDate = (Long) ((FieldDoc) topDocs.scoreDocs[0]).fields[0];
+ long lastDate = (Long) ((FieldDoc) topDocs.scoreDocs[4]).fields[0];
+ assertTrue(firstDate <= lastDate);
+ }
+
static long getValue(BinaryDocValues bdv) throws IOException {
BytesRef term = bdv.binaryValue();
int idx = term.offset;