Apache Solr open-source search software
Further tune Lucene87StoredFieldsFormat for small documents. (#1888)
Adrien Grande0a64908d837
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsReader.javaindex cd9197f1a34..555fa04ca22 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsReader.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsReader.java@@ -69,7 +69,7 @@ import org.apache.lucene.util.ArrayUtil;import org.apache.lucene.util.BitUtil;import org.apache.lucene.util.BytesRef;import org.apache.lucene.util.IOUtils;-import org.apache.lucene.util.IntsRef;+import org.apache.lucene.util.LongsRef;import org.apache.lucene.util.packed.PackedInts;/**@@ -399,8 +399,8 @@ public final class CompressingStoredFieldsReader extends StoredFieldsReader {// whether the block has been sliced, this happens for large documentsprivate boolean sliced;- private int[] offsets = IntsRef.EMPTY_INTS;- private int[] numStoredFields = IntsRef.EMPTY_INTS;+ private long[] offsets = LongsRef.EMPTY_LONGS;+ private long[] numStoredFields = LongsRef.EMPTY_LONGS;// the start pointer at which you can read the compressed documentsprivate long startPointer;@@ -469,9 +469,11 @@ public final class CompressingStoredFieldsReader extends StoredFieldsReader {} else if (bitsPerStoredFields > 31) {throw new CorruptIndexException("bitsPerStoredFields=" + bitsPerStoredFields, fieldsStream);} else {- final PackedInts.ReaderIterator it = PackedInts.getReaderIteratorNoHeader(fieldsStream, PackedInts.Format.PACKED, packedIntsVersion, chunkDocs, bitsPerStoredFields, 1);- for (int i = 0; i < chunkDocs; ++i) {- numStoredFields[i] = (int) it.next();+ final PackedInts.ReaderIterator it = PackedInts.getReaderIteratorNoHeader(fieldsStream, PackedInts.Format.PACKED, packedIntsVersion, chunkDocs, bitsPerStoredFields, 1024);+ for (int i = 0; i < chunkDocs; ) {+ final LongsRef next = it.next(Integer.MAX_VALUE);+ System.arraycopy(next.longs, next.offset, numStoredFields, i, next.length);+ i += next.length;}}@@ -486,9 +488,11 @@ public final class CompressingStoredFieldsReader extends StoredFieldsReader {} else if (bitsPerStoredFields > 31) {throw new CorruptIndexException("bitsPerLength=" + bitsPerLength, fieldsStream);} else {- final PackedInts.ReaderIterator it = PackedInts.getReaderIteratorNoHeader(fieldsStream, PackedInts.Format.PACKED, packedIntsVersion, chunkDocs, bitsPerLength, 1);- for (int i = 0; i < chunkDocs; ++i) {- offsets[i + 1] = (int) it.next();+ final PackedInts.ReaderIterator it = PackedInts.getReaderIteratorNoHeader(fieldsStream, PackedInts.Format.PACKED, packedIntsVersion, chunkDocs, bitsPerLength, 1024);+ for (int i = 0; i < chunkDocs; ) {+ final LongsRef next = it.next(Integer.MAX_VALUE);+ System.arraycopy(next.longs, next.offset, offsets, i + 1, next.length);+ i += next.length;}for (int i = 0; i < chunkDocs; ++i) {offsets[i + 1] += offsets[i];@@ -497,8 +501,8 @@ public final class CompressingStoredFieldsReader extends StoredFieldsReader {// Additional validation: only the empty document has a serialized length of 0for (int i = 0; i < chunkDocs; ++i) {- final int len = offsets[i + 1] - offsets[i];- final int storedFields = numStoredFields[i];+ final long len = offsets[i + 1] - offsets[i];+ final long storedFields = numStoredFields[i];if ((len == 0) != (storedFields == 0)) {throw new CorruptIndexException("length=" + len + ", numStoredFields=" + storedFields, fieldsStream);}@@ -509,7 +513,7 @@ public final class CompressingStoredFieldsReader extends StoredFieldsReader {startPointer = fieldsStream.getFilePointer();if (merging) {- final int totalLength = offsets[chunkDocs];+ final int totalLength = Math.toIntExact(offsets[chunkDocs]);// decompress eagerlyif (sliced) {bytes.offset = bytes.length = 0;@@ -540,10 +544,10 @@ public final class CompressingStoredFieldsReader extends StoredFieldsReader {}final int index = docID - docBase;- final int offset = offsets[index];- final int length = offsets[index+1] - offset;- final int totalLength = offsets[chunkDocs];- final int numStoredFields = this.numStoredFields[index];+ final int offset = Math.toIntExact(offsets[index]);+ final int length = Math.toIntExact(offsets[index+1]) - offset;+ final int totalLength = Math.toIntExact(offsets[chunkDocs]);+ final int numStoredFields = Math.toIntExact(this.numStoredFields[index]);final BytesRef bytes;if (merging) {diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene87/Lucene87StoredFieldsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene87/Lucene87StoredFieldsFormat.javaindex fc5b6193f03..56648c7be04 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene87/Lucene87StoredFieldsFormat.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene87/Lucene87StoredFieldsFormat.java@@ -144,9 +144,9 @@ public class Lucene87StoredFieldsFormat extends StoredFieldsFormat {StoredFieldsFormat impl(Mode mode) {switch (mode) {case BEST_SPEED:- return new CompressingStoredFieldsFormat("Lucene87StoredFieldsFastData", BEST_SPEED_MODE, BEST_SPEED_BLOCK_LENGTH, 512, 10);+ return new CompressingStoredFieldsFormat("Lucene87StoredFieldsFastData", BEST_SPEED_MODE, BEST_SPEED_BLOCK_LENGTH, 1024, 10);case BEST_COMPRESSION:- return new CompressingStoredFieldsFormat("Lucene87StoredFieldsHighData", BEST_COMPRESSION_MODE, BEST_COMPRESSION_BLOCK_LENGTH, 512, 10);+ return new CompressingStoredFieldsFormat("Lucene87StoredFieldsHighData", BEST_COMPRESSION_MODE, BEST_COMPRESSION_BLOCK_LENGTH, 4096, 10);default: throw new AssertionError();}}