Skip to content

Apache Lucene open-source search software

Faster bulk numeric reads from BufferedIndexInput (#12453)

Armin Braun20e97fbd0060
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 3d3172c6d1a..21825fb7b5c 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -159,6 +159,8 @@ Optimizations
* GITHUB#12381: Speed up NumericDocValuesWriter with index sorting. (Chao Zhang)
+* GITHUB#12453: Faster bulk numeric reads from BufferedIndexInput (Armin Braun)
+
Bug Fixes
---------------------
diff --git a/lucene/core/src/java/org/apache/lucene/store/BufferedIndexInput.java b/lucene/core/src/java/org/apache/lucene/store/BufferedIndexInput.java
index 50dde6aa4c9..290c829a4b9 100644
--- a/lucene/core/src/java/org/apache/lucene/store/BufferedIndexInput.java
+++ b/lucene/core/src/java/org/apache/lucene/store/BufferedIndexInput.java
@@ -159,6 +159,63 @@ public abstract class BufferedIndexInput extends IndexInput implements RandomAcc
}
}
+ @Override
+ public void readFloats(float[] dst, int offset, int len) throws IOException {
+ int remainingDst = len;
+ while (remainingDst > 0) {
+ int cnt = Math.min(buffer.remaining() / Float.BYTES, remainingDst);
+ buffer.asFloatBuffer().get(dst, offset + len - remainingDst, cnt);
+ buffer.position(buffer.position() + Float.BYTES * cnt);
+ remainingDst -= cnt;
+ if (remainingDst > 0) {
+ if (buffer.hasRemaining()) {
+ dst[offset + len - remainingDst] = Float.intBitsToFloat(readInt());
+ --remainingDst;
+ } else {
+ refill();
+ }
+ }
+ }
+ }
+
+ @Override
+ public void readLongs(long[] dst, int offset, int len) throws IOException {
+ int remainingDst = len;
+ while (remainingDst > 0) {
+ int cnt = Math.min(buffer.remaining() / Long.BYTES, remainingDst);
+ buffer.asLongBuffer().get(dst, offset + len - remainingDst, cnt);
+ buffer.position(buffer.position() + Long.BYTES * cnt);
+ remainingDst -= cnt;
+ if (remainingDst > 0) {
+ if (buffer.hasRemaining()) {
+ dst[offset + len - remainingDst] = readLong();
+ --remainingDst;
+ } else {
+ refill();
+ }
+ }
+ }
+ }
+
+ @Override
+ public void readInts(int[] dst, int offset, int len) throws IOException {
+ int remainingDst = len;
+ while (remainingDst > 0) {
+ int cnt = Math.min(buffer.remaining() / Integer.BYTES, remainingDst);
+ buffer.asIntBuffer().get(dst, offset + len - remainingDst, cnt);
+ buffer.position(buffer.position() + Integer.BYTES * cnt);
+ remainingDst -= cnt;
+ if (remainingDst > 0) {
+ if (buffer.hasRemaining()) {
+ dst[offset + len - remainingDst] = readInt();
+ --remainingDst;
+ } else {
+ refill();
+ }
+ }
+ }
+ }
+
// Computes an offset into the current buffer from an absolute position to read
// `width` bytes from. If the buffer does not contain the position, then we
// readjust the bufferStart and refill.
diff --git a/lucene/core/src/test/org/apache/lucene/store/TestBufferedIndexInput.java b/lucene/core/src/test/org/apache/lucene/store/TestBufferedIndexInput.java
index d19b2058e07..8f02bd9cd1a 100644
--- a/lucene/core/src/test/org/apache/lucene/store/TestBufferedIndexInput.java
+++ b/lucene/core/src/test/org/apache/lucene/store/TestBufferedIndexInput.java
@@ -209,6 +209,104 @@ public class TestBufferedIndexInput extends LuceneTestCase {
"Expected 4 or 3, got " + input.readCount, input.readCount == 4 || input.readCount == 3);
}
+ public void testReadFloats() throws IOException {
+ final int length = 1024 * 8;
+ MyBufferedIndexInput input = new MyBufferedIndexInput(length);
+ ByteBuffer bb = ByteBuffer.allocate(Float.BYTES).order(ByteOrder.LITTLE_ENDIAN);
+ final int bufferLength = 128;
+ float[] floatBuffer = new float[bufferLength];
+
+ for (int alignment = 0; alignment < Float.BYTES; alignment++) {
+ input.seek(0);
+ for (int i = 0; i < alignment; i++) {
+ input.readByte();
+ }
+ final int bulkReads = length / (bufferLength * Float.BYTES) - 1;
+ for (int i = 0; i < bulkReads; i++) {
+ int pos = alignment + i * bufferLength * Float.BYTES;
+ final int floatOffset = random().nextInt(3);
+ input.skipBytes(floatOffset * Float.BYTES);
+ input.readFloats(floatBuffer, floatOffset, bufferLength - floatOffset);
+ for (int idx = floatOffset; idx < bufferLength; idx++) {
+ final int offset = pos + idx * Float.BYTES;
+ bb.position(0)
+ .put(byten(offset))
+ .put(byten(offset + 1))
+ .put(byten(offset + 2))
+ .put(byten(offset + 3));
+ assertEquals(
+ Float.floatToRawIntBits(bb.getFloat(0)), Float.floatToRawIntBits(floatBuffer[idx]));
+ }
+ }
+ }
+ }
+
+ public void testReadInts() throws IOException {
+ final int length = 1024 * 8;
+ MyBufferedIndexInput input = new MyBufferedIndexInput(length);
+ ByteBuffer bb = ByteBuffer.allocate(Integer.BYTES).order(ByteOrder.LITTLE_ENDIAN);
+ final int bufferLength = 128;
+ int[] intBuffer = new int[bufferLength];
+
+ for (int alignment = 0; alignment < Integer.BYTES; alignment++) {
+ input.seek(0);
+ for (int i = 0; i < alignment; i++) {
+ input.readByte();
+ }
+ final int bulkReads = length / (bufferLength * Integer.BYTES) - 1;
+ for (int i = 0; i < bulkReads; i++) {
+ int pos = alignment + i * bufferLength * Integer.BYTES;
+ final int intOffset = random().nextInt(3);
+ input.skipBytes(intOffset * Integer.BYTES);
+ input.readInts(intBuffer, intOffset, bufferLength - intOffset);
+ for (int idx = intOffset; idx < bufferLength; idx++) {
+ final int offset = pos + idx * Integer.BYTES;
+ bb.position(0)
+ .put(byten(offset))
+ .put(byten(offset + 1))
+ .put(byten(offset + 2))
+ .put(byten(offset + 3));
+ assertEquals(bb.getInt(0), intBuffer[idx]);
+ }
+ }
+ }
+ }
+
+ public void testReadLongs() throws IOException {
+ final int length = 1024 * 8;
+ MyBufferedIndexInput input = new MyBufferedIndexInput(length);
+ ByteBuffer bb = ByteBuffer.allocate(Long.BYTES).order(ByteOrder.LITTLE_ENDIAN);
+ final int bufferLength = 128;
+ long[] longBuffer = new long[bufferLength];
+
+ for (int alignment = 0; alignment < Long.BYTES; alignment++) {
+ input.seek(0);
+ for (int i = 0; i < alignment; i++) {
+ input.readByte();
+ }
+ final int bulkReads = length / (bufferLength * Long.BYTES) - 1;
+ for (int i = 0; i < bulkReads; i++) {
+ int pos = alignment + i * bufferLength * Long.BYTES;
+ final int longOffset = random().nextInt(3);
+ input.skipBytes(longOffset * Long.BYTES);
+ input.readLongs(longBuffer, longOffset, bufferLength - longOffset);
+ for (int idx = longOffset; idx < bufferLength; idx++) {
+ final int offset = pos + idx * Long.BYTES;
+ bb.position(0)
+ .put(byten(offset))
+ .put(byten(offset + 1))
+ .put(byten(offset + 2))
+ .put(byten(offset + 3))
+ .put(byten(offset + 4))
+ .put(byten(offset + 5))
+ .put(byten(offset + 6))
+ .put(byten(offset + 7));
+ assertEquals(bb.getLong(0), longBuffer[idx]);
+ }
+ }
+ }
+ }
+
// byten emulates a file - byten(n) returns the n'th byte in that file.
// MyBufferedIndexInput reads this "file".
private static byte byten(long n) {