Apache Lucene open-source search software
Refactor GroupVIntUtil functional interface lambda which does not inline correctly in MemorySegmentIndexInput (#15089)
Uwe Schindlerf245bed35d47
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex e843976c7f6..5c08661643f 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -166,6 +166,9 @@ Improvements* GITHUB#14816: Expose search strategy in KNN query+* GITHUB#15089, GITHUB#15079: Refactor GroupVIntUtil functional interface lambda+ which does not inline correctly in MemorySegmentIndexInput (Uwe Schindler, Robert Muir)+Optimizations---------------------* GITHUB#14932: Switched to GroupVarInt Encoding for HNSW Graph edges, added backwards compatibility (Akira Lonske)diff --git a/lucene/core/src/java/org/apache/lucene/store/BufferedIndexInput.java b/lucene/core/src/java/org/apache/lucene/store/BufferedIndexInput.javaindex cd47500a2df..e45ffee5946 100644--- a/lucene/core/src/java/org/apache/lucene/store/BufferedIndexInput.java+++ b/lucene/core/src/java/org/apache/lucene/store/BufferedIndexInput.java@@ -154,7 +154,13 @@ public abstract class BufferedIndexInput extends IndexInput implements RandomAccpublic void readGroupVInt(int[] dst, int offset) throws IOException {final int len =GroupVIntUtil.readGroupVInt(- this, buffer.remaining(), p -> buffer.getInt((int) p), buffer.position(), dst, offset);+ this,+ buffer.remaining(),+ GroupVIntUtil.VH_BUFFER_GET_INT,+ buffer,+ buffer.position(),+ dst,+ offset);if (len > 0) {buffer.position(buffer.position() + len);}diff --git a/lucene/core/src/java/org/apache/lucene/store/ByteBuffersDataInput.java b/lucene/core/src/java/org/apache/lucene/store/ByteBuffersDataInput.javaindex dee5c8e3a73..00c9d51eaed 100644--- a/lucene/core/src/java/org/apache/lucene/store/ByteBuffersDataInput.java+++ b/lucene/core/src/java/org/apache/lucene/store/ByteBuffersDataInput.java@@ -38,6 +38,7 @@ import org.apache.lucene.util.RamUsageEstimator;*/public final class ByteBuffersDataInput extends DataInputimplements Accountable, RandomAccessInput {+private final ByteBuffer[] blocks;private final FloatBuffer[] floatBuffers;private final LongBuffer[] longBuffers;@@ -215,7 +216,8 @@ public final class ByteBuffersDataInput extends DataInputGroupVIntUtil.readGroupVInt(this,block.limit() - blockOffset,- p -> block.getInt((int) p),+ GroupVIntUtil.VH_BUFFER_GET_INT,+ block,blockOffset,dst,offset);diff --git a/lucene/core/src/java/org/apache/lucene/store/MemorySegmentIndexInput.java b/lucene/core/src/java/org/apache/lucene/store/MemorySegmentIndexInput.javaindex a44232b3674..28f2be0b216 100644--- a/lucene/core/src/java/org/apache/lucene/store/MemorySegmentIndexInput.java+++ b/lucene/core/src/java/org/apache/lucene/store/MemorySegmentIndexInput.java@@ -23,6 +23,9 @@ import java.io.IOException;import java.lang.foreign.Arena;import java.lang.foreign.MemorySegment;import java.lang.foreign.ValueLayout;+import java.lang.invoke.MethodHandles;+import java.lang.invoke.MethodType;+import java.lang.invoke.VarHandle;import java.nio.ByteOrder;import java.util.Arrays;import java.util.Objects;@@ -52,6 +55,13 @@ abstract class MemorySegmentIndexInput extends IndexInput implements MemorySegmeValueLayout.JAVA_FLOAT_UNALIGNED.withOrder(ByteOrder.LITTLE_ENDIAN);private static final Optional<NativeAccess> NATIVE_ACCESS = NativeAccess.getImplementation();+ private static final VarHandle VH_MEMSEG_GET_INT =+ MethodHandles.filterCoordinates(+ LAYOUT_LE_INT.varHandle(),+ 0,+ MethodHandles.identity(Object.class)+ .asType(MethodType.methodType(MemorySegment.class, Object.class)));+final long length;final long chunkSizeMask;final int chunkSizePower;@@ -454,7 +464,8 @@ abstract class MemorySegmentIndexInput extends IndexInput implements MemorySegmeGroupVIntUtil.readGroupVInt(this,curSegment.byteSize() - curPosition,- p -> curSegment.get(LAYOUT_LE_INT, p),+ VH_MEMSEG_GET_INT,+ curSegment,curPosition,dst,offset);diff --git a/lucene/core/src/java/org/apache/lucene/util/GroupVIntUtil.java b/lucene/core/src/java/org/apache/lucene/util/GroupVIntUtil.javaindex e95e2eee4db..6967c12df86 100644--- a/lucene/core/src/java/org/apache/lucene/util/GroupVIntUtil.java+++ b/lucene/core/src/java/org/apache/lucene/util/GroupVIntUtil.java@@ -17,6 +17,11 @@package org.apache.lucene.util;import java.io.IOException;+import java.lang.invoke.MethodHandles;+import java.lang.invoke.MethodType;+import java.lang.invoke.VarHandle;+import java.nio.ByteBuffer;+import java.nio.ByteOrder;import org.apache.lucene.store.DataInput;import org.apache.lucene.store.DataOutput;@@ -33,6 +38,23 @@ public final class GroupVIntUtil {private static final long[] LONG_MASKS = new long[] {0xFFL, 0xFFFFL, 0xFFFFFFL, 0xFFFFFFFFL};private static final int[] INT_MASKS = new int[] {0xFF, 0xFFFF, 0xFFFFFF, ~0};+ /**+ * A {@link VarHandle} which allows to read ints from a {@link ByteBuffer} using {@code long}+ * offsets. The handle can be used with the {@code readGroupVInt()} methods taking a {@code+ * VarHandle} and {@code ByteBuffer} storage parameter.+ *+ * @see #readGroupVInt(DataInput, long, VarHandle, Object, long, int[], int)+ * @see #readGroupVInt(DataInput, long, VarHandle, Object, long, long[], int)+ */+ public static final VarHandle VH_BUFFER_GET_INT =+ MethodHandles.filterCoordinates(+ MethodHandles.byteBufferViewVarHandle(int[].class, ByteOrder.LITTLE_ENDIAN),+ 0,+ MethodHandles.identity(Object.class)+ .asType(MethodType.methodType(ByteBuffer.class, Object.class)),+ MethodHandles.explicitCastArguments(+ MethodHandles.identity(long.class), MethodType.methodType(int.class, long.class)));+/*** Read all the group varints, including the tail vints. we need a long[] because this is what* postings are using, all longs are actually required to be integers.@@ -126,21 +148,15 @@ public final class GroupVIntUtil {}/**- * Provides an abstraction for read int values, so that decoding logic can be reused in different- * DataInput.- */- @FunctionalInterface- public static interface IntReader {- int read(long v);- }-- /**- * Faster implementation of read single group, It read values from the buffer that would not cross- * boundaries.+ * Faster implementation of read single group, It read values from {@link VarHandle} that would+ * not cross boundaries.** @param in the input to use to read data.* @param remaining the number of remaining bytes allowed to read for current block/segment.- * @param reader the supplier of read int.+ * @param vh the varhandle which has the coordinates {@code (Object, long)}. The first coordinate+ * must accept the {@code storage} parameter, the second coordinate must be the long offset.+ * @param storage the reference to the backing storage (e.g., one of {@code byte[], ByteBuffer,+ * MemorySegment})* @param pos the start pos to read from the reader.* @param dst the array to read ints into.* @param offset the offset in the array to start storing ints.@@ -149,7 +165,7 @@ public final class GroupVIntUtil {* #MAX_LENGTH_PER_GROUP}*/public static int readGroupVInt(- DataInput in, long remaining, IntReader reader, long pos, long[] dst, int offset)+ DataInput in, long remaining, VarHandle vh, Object storage, long pos, long[] dst, int offset)throws IOException {if (remaining < MAX_LENGTH_PER_GROUP) {readGroupVInt(in, dst, offset);@@ -163,24 +179,27 @@ public final class GroupVIntUtil {final int n4Minus1 = flag & 0x03;// This code path has fewer conditionals and tends to be significantly faster in benchmarks- dst[offset] = reader.read(pos) & LONG_MASKS[n1Minus1];+ dst[offset] = (int) vh.get(storage, pos) & LONG_MASKS[n1Minus1];pos += 1 + n1Minus1;- dst[offset + 1] = reader.read(pos) & LONG_MASKS[n2Minus1];+ dst[offset + 1] = (int) vh.get(storage, pos) & LONG_MASKS[n2Minus1];pos += 1 + n2Minus1;- dst[offset + 2] = reader.read(pos) & LONG_MASKS[n3Minus1];+ dst[offset + 2] = (int) vh.get(storage, pos) & LONG_MASKS[n3Minus1];pos += 1 + n3Minus1;- dst[offset + 3] = reader.read(pos) & LONG_MASKS[n4Minus1];+ dst[offset + 3] = (int) vh.get(storage, pos) & LONG_MASKS[n4Minus1];pos += 1 + n4Minus1;return (int) (pos - posStart);}/**- * Faster implementation of read single group, It read values from the buffer that would not cross- * boundaries.+ * Faster implementation of read single group, It read values from a {@link VarHandle} that would+ * not cross boundaries.** @param in the input to use to read data.* @param remaining the number of remaining bytes allowed to read for current block/segment.- * @param reader the supplier of read int.+ * @param vh the varhandle which has the coordinates {@code (Object, long)}. The first coordinate+ * must accept the {@code storage} parameter, the second coordinate must be the long offset.+ * @param storage the reference to the backing storage (e.g., one of {@code byte[], ByteBuffer,+ * MemorySegment})* @param pos the start pos to read from the reader.* @param dst the array to read ints into.* @param offset the offset in the array to start storing ints.@@ -189,7 +208,7 @@ public final class GroupVIntUtil {* #MAX_LENGTH_PER_GROUP}*/public static int readGroupVInt(- DataInput in, long remaining, IntReader reader, long pos, int[] dst, int offset)+ DataInput in, long remaining, VarHandle vh, Object storage, long pos, int[] dst, int offset)throws IOException {if (remaining < MAX_LENGTH_PER_GROUP) {readGroupVInt(in, dst, offset);@@ -203,13 +222,13 @@ public final class GroupVIntUtil {final int n4Minus1 = flag & 0x03;// This code path has fewer conditionals and tends to be significantly faster in benchmarks- dst[offset] = reader.read(pos) & INT_MASKS[n1Minus1];+ dst[offset] = (int) vh.get(storage, pos) & INT_MASKS[n1Minus1];pos += 1 + n1Minus1;- dst[offset + 1] = reader.read(pos) & INT_MASKS[n2Minus1];+ dst[offset + 1] = (int) vh.get(storage, pos) & INT_MASKS[n2Minus1];pos += 1 + n2Minus1;- dst[offset + 2] = reader.read(pos) & INT_MASKS[n3Minus1];+ dst[offset + 2] = (int) vh.get(storage, pos) & INT_MASKS[n3Minus1];pos += 1 + n3Minus1;- dst[offset + 3] = reader.read(pos) & INT_MASKS[n4Minus1];+ dst[offset + 3] = (int) vh.get(storage, pos) & INT_MASKS[n4Minus1];pos += 1 + n4Minus1;return (int) (pos - posStart);}