Skip to content

armgit / solr

Public Imported from GitHub
0

Apache Solr open-source search software

Further tune Lucene87StoredFieldsFormat for small documents. (#1888)

Adrien Grande0a64908d837
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsReader.java
index cd9197f1a34..555fa04ca22 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsReader.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsReader.java
@@ -69,7 +69,7 @@ import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BitUtil;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
-import org.apache.lucene.util.IntsRef;
+import org.apache.lucene.util.LongsRef;
import org.apache.lucene.util.packed.PackedInts;
/**
@@ -399,8 +399,8 @@ public final class CompressingStoredFieldsReader extends StoredFieldsReader {
// whether the block has been sliced, this happens for large documents
private boolean sliced;
- private int[] offsets = IntsRef.EMPTY_INTS;
- private int[] numStoredFields = IntsRef.EMPTY_INTS;
+ private long[] offsets = LongsRef.EMPTY_LONGS;
+ private long[] numStoredFields = LongsRef.EMPTY_LONGS;
// the start pointer at which you can read the compressed documents
private long startPointer;
@@ -469,9 +469,11 @@ public final class CompressingStoredFieldsReader extends StoredFieldsReader {
} else if (bitsPerStoredFields > 31) {
throw new CorruptIndexException("bitsPerStoredFields=" + bitsPerStoredFields, fieldsStream);
} else {
- final PackedInts.ReaderIterator it = PackedInts.getReaderIteratorNoHeader(fieldsStream, PackedInts.Format.PACKED, packedIntsVersion, chunkDocs, bitsPerStoredFields, 1);
- for (int i = 0; i < chunkDocs; ++i) {
- numStoredFields[i] = (int) it.next();
+ final PackedInts.ReaderIterator it = PackedInts.getReaderIteratorNoHeader(fieldsStream, PackedInts.Format.PACKED, packedIntsVersion, chunkDocs, bitsPerStoredFields, 1024);
+ for (int i = 0; i < chunkDocs; ) {
+ final LongsRef next = it.next(Integer.MAX_VALUE);
+ System.arraycopy(next.longs, next.offset, numStoredFields, i, next.length);
+ i += next.length;
}
}
@@ -486,9 +488,11 @@ public final class CompressingStoredFieldsReader extends StoredFieldsReader {
} else if (bitsPerStoredFields > 31) {
throw new CorruptIndexException("bitsPerLength=" + bitsPerLength, fieldsStream);
} else {
- final PackedInts.ReaderIterator it = PackedInts.getReaderIteratorNoHeader(fieldsStream, PackedInts.Format.PACKED, packedIntsVersion, chunkDocs, bitsPerLength, 1);
- for (int i = 0; i < chunkDocs; ++i) {
- offsets[i + 1] = (int) it.next();
+ final PackedInts.ReaderIterator it = PackedInts.getReaderIteratorNoHeader(fieldsStream, PackedInts.Format.PACKED, packedIntsVersion, chunkDocs, bitsPerLength, 1024);
+ for (int i = 0; i < chunkDocs; ) {
+ final LongsRef next = it.next(Integer.MAX_VALUE);
+ System.arraycopy(next.longs, next.offset, offsets, i + 1, next.length);
+ i += next.length;
}
for (int i = 0; i < chunkDocs; ++i) {
offsets[i + 1] += offsets[i];
@@ -497,8 +501,8 @@ public final class CompressingStoredFieldsReader extends StoredFieldsReader {
// Additional validation: only the empty document has a serialized length of 0
for (int i = 0; i < chunkDocs; ++i) {
- final int len = offsets[i + 1] - offsets[i];
- final int storedFields = numStoredFields[i];
+ final long len = offsets[i + 1] - offsets[i];
+ final long storedFields = numStoredFields[i];
if ((len == 0) != (storedFields == 0)) {
throw new CorruptIndexException("length=" + len + ", numStoredFields=" + storedFields, fieldsStream);
}
@@ -509,7 +513,7 @@ public final class CompressingStoredFieldsReader extends StoredFieldsReader {
startPointer = fieldsStream.getFilePointer();
if (merging) {
- final int totalLength = offsets[chunkDocs];
+ final int totalLength = Math.toIntExact(offsets[chunkDocs]);
// decompress eagerly
if (sliced) {
bytes.offset = bytes.length = 0;
@@ -540,10 +544,10 @@ public final class CompressingStoredFieldsReader extends StoredFieldsReader {
}
final int index = docID - docBase;
- final int offset = offsets[index];
- final int length = offsets[index+1] - offset;
- final int totalLength = offsets[chunkDocs];
- final int numStoredFields = this.numStoredFields[index];
+ final int offset = Math.toIntExact(offsets[index]);
+ final int length = Math.toIntExact(offsets[index+1]) - offset;
+ final int totalLength = Math.toIntExact(offsets[chunkDocs]);
+ final int numStoredFields = Math.toIntExact(this.numStoredFields[index]);
final BytesRef bytes;
if (merging) {
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene87/Lucene87StoredFieldsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene87/Lucene87StoredFieldsFormat.java
index fc5b6193f03..56648c7be04 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene87/Lucene87StoredFieldsFormat.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene87/Lucene87StoredFieldsFormat.java
@@ -144,9 +144,9 @@ public class Lucene87StoredFieldsFormat extends StoredFieldsFormat {
StoredFieldsFormat impl(Mode mode) {
switch (mode) {
case BEST_SPEED:
- return new CompressingStoredFieldsFormat("Lucene87StoredFieldsFastData", BEST_SPEED_MODE, BEST_SPEED_BLOCK_LENGTH, 512, 10);
+ return new CompressingStoredFieldsFormat("Lucene87StoredFieldsFastData", BEST_SPEED_MODE, BEST_SPEED_BLOCK_LENGTH, 1024, 10);
case BEST_COMPRESSION:
- return new CompressingStoredFieldsFormat("Lucene87StoredFieldsHighData", BEST_COMPRESSION_MODE, BEST_COMPRESSION_BLOCK_LENGTH, 512, 10);
+ return new CompressingStoredFieldsFormat("Lucene87StoredFieldsHighData", BEST_COMPRESSION_MODE, BEST_COMPRESSION_BLOCK_LENGTH, 4096, 10);
default: throw new AssertionError();
}
}