Skip to content

Apache Lucene open-source search software

Fix GroupVIntBenchmark and some tests to actually use DataInput#readGroupVInt; deprecate/remove obsolete long[] code (#15104)

Adrien Grand2c5650592742
diff --git a/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/GroupVIntBenchmark.java b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/GroupVIntBenchmark.java
index 48b95570694..6b40870c375 100644
--- a/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/GroupVIntBenchmark.java
+++ b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/GroupVIntBenchmark.java
@@ -86,12 +86,12 @@ public class GroupVIntBenchmark {
};
final int maxSize = 256;
- final long[] docs = new long[maxSize];
- final long[] values = new long[maxSize];
+ final int[] docs = new int[maxSize];
+ final int[] values = new int[maxSize];
- IndexInput byteBufferGVIntIn;
+ IndexInput mmapGVIntIn;
IndexInput nioGVIntIn;
- IndexInput byteBufferVIntIn;
+ IndexInput mmapVIntIn;
ByteBuffersDataInput byteBuffersGVIntIn;
ByteArrayDataInput byteArrayVIntIn;
@@ -103,7 +103,7 @@ public class GroupVIntBenchmark {
@Param({"64"})
public int size;
- void initArrayInput(long[] docs) throws Exception {
+ void initArrayInput(int[] docs) throws Exception {
byte[] gVIntBytes = new byte[Integer.BYTES * maxSize * 2];
byte[] vIntBytes = new byte[Integer.BYTES * maxSize * 2];
ByteArrayDataOutput vIntOut = new ByteArrayDataOutput(vIntBytes);
@@ -116,7 +116,7 @@ public class GroupVIntBenchmark {
byteArrayGVIntIn = new ByteArrayDataInput(gVIntBytes);
}
- void initNioInput(long[] docs) throws Exception {
+ void initNioInput(int[] docs) throws Exception {
Directory dir = new NIOFSDirectory(Files.createTempDirectory("groupvintdata"));
IndexOutput out = dir.createOutput("gvint", IOContext.DEFAULT);
out.writeGroupVInts(docs, docs.length);
@@ -124,13 +124,13 @@ public class GroupVIntBenchmark {
nioGVIntIn = dir.openInput("gvint", IOContext.DEFAULT);
}
- void initByteBuffersInput(long[] docs) throws Exception {
+ void initByteBuffersInput(int[] docs) throws Exception {
ByteBuffersDataOutput buffer = new ByteBuffersDataOutput();
buffer.writeGroupVInts(docs, docs.length);
byteBuffersGVIntIn = buffer.toDataInput();
}
- void initByteBufferInput(long[] docs) throws Exception {
+ void initMMapInput(int[] docs) throws Exception {
Directory dir = new MMapDirectory(Files.createTempDirectory("groupvintdata"));
IndexOutput vintOut = dir.createOutput("vint", IOContext.DEFAULT);
IndexOutput gvintOut = dir.createOutput("gvint", IOContext.DEFAULT);
@@ -141,11 +141,11 @@ public class GroupVIntBenchmark {
}
vintOut.close();
gvintOut.close();
- byteBufferGVIntIn = dir.openInput("gvint", IOContext.DEFAULT);
- byteBufferVIntIn = dir.openInput("vint", IOContext.DEFAULT);
+ mmapGVIntIn = dir.openInput("gvint", IOContext.DEFAULT);
+ mmapVIntIn = dir.openInput("vint", IOContext.DEFAULT);
}
- private void readGroupVIntsBaseline(DataInput in, long[] dst, int limit) throws IOException {
+ private void readGroupVIntsBaseline(DataInput in, int[] dst, int limit) throws IOException {
int i;
for (i = 0; i <= limit - 4; i += 4) {
GroupVIntUtil.readGroupVInt(in, dst, i);
@@ -168,7 +168,7 @@ public class GroupVIntBenchmark {
}
docs[i] = r.nextInt(1 << (numBits - 1), 1 << numBits);
}
- initByteBufferInput(docs);
+ initMMapInput(docs);
initArrayInput(docs);
initNioInput(docs);
initByteBuffersInput(docs);
@@ -176,24 +176,24 @@ public class GroupVIntBenchmark {
@Benchmark
public void benchMMapDirectoryInputs_readVInt(Blackhole bh) throws IOException {
- byteBufferVIntIn.seek(0);
+ mmapVIntIn.seek(0);
for (int i = 0; i < size; i++) {
- values[i] = byteBufferVIntIn.readVInt();
+ values[i] = mmapVIntIn.readVInt();
}
bh.consume(values);
}
@Benchmark
public void benchMMapDirectoryInputs_readGroupVInt(Blackhole bh) throws IOException {
- byteBufferGVIntIn.seek(0);
- GroupVIntUtil.readGroupVInts(byteBufferGVIntIn, values, size);
+ mmapGVIntIn.seek(0);
+ GroupVIntUtil.readGroupVInts(mmapGVIntIn, values, size);
bh.consume(values);
}
@Benchmark
public void benchMMapDirectoryInputs_readGroupVIntBaseline(Blackhole bh) throws IOException {
- byteBufferGVIntIn.seek(0);
- this.readGroupVIntsBaseline(byteBufferGVIntIn, values, size);
+ mmapGVIntIn.seek(0);
+ this.readGroupVIntsBaseline(mmapGVIntIn, values, size);
bh.consume(values);
}
diff --git a/lucene/core/src/java/org/apache/lucene/store/DataOutput.java b/lucene/core/src/java/org/apache/lucene/store/DataOutput.java
index 543ff175867..3d856cbc5f6 100644
--- a/lucene/core/src/java/org/apache/lucene/store/DataOutput.java
+++ b/lucene/core/src/java/org/apache/lucene/store/DataOutput.java
@@ -333,7 +333,9 @@ public abstract class DataOutput {
* @param values the values to write
* @param limit the number of values to write.
* @lucene.experimental
+ * @deprecated This method is preserved only for backwards codecs
*/
+ @Deprecated
public void writeGroupVInts(long[] values, int limit) throws IOException {
if (groupVIntBytes == null) {
groupVIntBytes = new byte[GroupVIntUtil.MAX_LENGTH_PER_GROUP];
diff --git a/lucene/core/src/java/org/apache/lucene/util/GroupVIntUtil.java b/lucene/core/src/java/org/apache/lucene/util/GroupVIntUtil.java
index e95e2eee4db..a1917d87a17 100644
--- a/lucene/core/src/java/org/apache/lucene/util/GroupVIntUtil.java
+++ b/lucene/core/src/java/org/apache/lucene/util/GroupVIntUtil.java
@@ -29,28 +29,8 @@ public final class GroupVIntUtil {
// the maximum length of a single group-varint is 4 integers + 1 byte flag.
public static final int MAX_LENGTH_PER_GROUP = 17;
- // we use long array instead of int array to make negative integer to be read as positive long.
- private static final long[] LONG_MASKS = new long[] {0xFFL, 0xFFFFL, 0xFFFFFFL, 0xFFFFFFFFL};
private static final int[] INT_MASKS = new int[] {0xFF, 0xFFFF, 0xFFFFFF, ~0};
- /**
- * Read all the group varints, including the tail vints. we need a long[] because this is what
- * postings are using, all longs are actually required to be integers.
- *
- * @param dst the array to read ints into.
- * @param limit the number of int values to read.
- * @lucene.experimental
- */
- public static void readGroupVInts(DataInput in, long[] dst, int limit) throws IOException {
- int i;
- for (i = 0; i <= limit - 4; i += 4) {
- readGroupVInt(in, dst, i);
- }
- for (; i < limit; ++i) {
- dst[i] = in.readVInt() & 0xFFFFFFFFL;
- }
- }
-
/**
* Read all the group varints, including the tail vints.
*
@@ -68,28 +48,6 @@ public final class GroupVIntUtil {
}
}
- /**
- * Default implementation of read single group, for optimal performance, you should use {@link
- * GroupVIntUtil#readGroupVInts(DataInput, long[], int)} instead.
- *
- * @param in the input to use to read data.
- * @param dst the array to read ints into.
- * @param offset the offset in the array to start storing ints.
- */
- public static void readGroupVInt(DataInput in, long[] dst, int offset) throws IOException {
- final int flag = in.readByte() & 0xFF;
-
- final int n1Minus1 = flag >> 6;
- final int n2Minus1 = (flag >> 4) & 0x03;
- final int n3Minus1 = (flag >> 2) & 0x03;
- final int n4Minus1 = flag & 0x03;
-
- dst[offset] = readIntInGroup(in, n1Minus1) & 0xFFFFFFFFL;
- dst[offset + 1] = readIntInGroup(in, n2Minus1) & 0xFFFFFFFFL;
- dst[offset + 2] = readIntInGroup(in, n3Minus1) & 0xFFFFFFFFL;
- dst[offset + 3] = readIntInGroup(in, n4Minus1) & 0xFFFFFFFFL;
- }
-
/**
* Default implementation of read single group, for optimal performance, you should use {@link
* GroupVIntUtil#readGroupVInts(DataInput, int[], int)} instead.
@@ -134,46 +92,6 @@ public final class GroupVIntUtil {
int read(long v);
}
- /**
- * Faster implementation of read single group, It read values from the buffer that would not cross
- * boundaries.
- *
- * @param in the input to use to read data.
- * @param remaining the number of remaining bytes allowed to read for current block/segment.
- * @param reader the supplier of read int.
- * @param pos the start pos to read from the reader.
- * @param dst the array to read ints into.
- * @param offset the offset in the array to start storing ints.
- * @return the number of bytes read excluding the flag. this indicates the number of positions
- * should to be increased for caller, it is 0 or positive number and less than {@link
- * #MAX_LENGTH_PER_GROUP}
- */
- public static int readGroupVInt(
- DataInput in, long remaining, IntReader reader, long pos, long[] dst, int offset)
- throws IOException {
- if (remaining < MAX_LENGTH_PER_GROUP) {
- readGroupVInt(in, dst, offset);
- return 0;
- }
- final int flag = in.readByte() & 0xFF;
- final long posStart = ++pos; // exclude the flag bytes, the position has updated via readByte().
- final int n1Minus1 = flag >> 6;
- final int n2Minus1 = (flag >> 4) & 0x03;
- final int n3Minus1 = (flag >> 2) & 0x03;
- final int n4Minus1 = flag & 0x03;
-
- // This code path has fewer conditionals and tends to be significantly faster in benchmarks
- dst[offset] = reader.read(pos) & LONG_MASKS[n1Minus1];
- pos += 1 + n1Minus1;
- dst[offset + 1] = reader.read(pos) & LONG_MASKS[n2Minus1];
- pos += 1 + n2Minus1;
- dst[offset + 2] = reader.read(pos) & LONG_MASKS[n3Minus1];
- pos += 1 + n3Minus1;
- dst[offset + 3] = reader.read(pos) & LONG_MASKS[n4Minus1];
- pos += 1 + n4Minus1;
- return (int) (pos - posStart);
- }
-
/**
* Faster implementation of read single group, It read values from the buffer that would not cross
* boundaries.
@@ -230,26 +148,26 @@ public final class GroupVIntUtil {
* The implementation for group-varint encoding, It uses a maximum of {@link
* #MAX_LENGTH_PER_GROUP} bytes scratch buffer.
*/
- public static void writeGroupVInts(DataOutput out, byte[] scratch, long[] values, int limit)
+ public static void writeGroupVInts(DataOutput out, byte[] scratch, int[] values, int limit)
throws IOException {
int readPos = 0;
// encode each group
while ((limit - readPos) >= 4) {
int writePos = 0;
- final int n1Minus1 = numBytes(toInt(values[readPos])) - 1;
- final int n2Minus1 = numBytes(toInt(values[readPos + 1])) - 1;
- final int n3Minus1 = numBytes(toInt(values[readPos + 2])) - 1;
- final int n4Minus1 = numBytes(toInt(values[readPos + 3])) - 1;
+ final int n1Minus1 = numBytes(values[readPos]) - 1;
+ final int n2Minus1 = numBytes(values[readPos + 1]) - 1;
+ final int n3Minus1 = numBytes(values[readPos + 2]) - 1;
+ final int n4Minus1 = numBytes(values[readPos + 3]) - 1;
int flag = (n1Minus1 << 6) | (n2Minus1 << 4) | (n3Minus1 << 2) | (n4Minus1);
scratch[writePos++] = (byte) flag;
- BitUtil.VH_LE_INT.set(scratch, writePos, (int) (values[readPos++]));
+ BitUtil.VH_LE_INT.set(scratch, writePos, values[readPos++]);
writePos += n1Minus1 + 1;
- BitUtil.VH_LE_INT.set(scratch, writePos, (int) (values[readPos++]));
+ BitUtil.VH_LE_INT.set(scratch, writePos, values[readPos++]);
writePos += n2Minus1 + 1;
- BitUtil.VH_LE_INT.set(scratch, writePos, (int) (values[readPos++]));
+ BitUtil.VH_LE_INT.set(scratch, writePos, values[readPos++]);
writePos += n3Minus1 + 1;
- BitUtil.VH_LE_INT.set(scratch, writePos, (int) (values[readPos++]));
+ BitUtil.VH_LE_INT.set(scratch, writePos, values[readPos++]);
writePos += n4Minus1 + 1;
out.writeBytes(scratch, writePos);
@@ -257,34 +175,78 @@ public final class GroupVIntUtil {
// tail vints
for (; readPos < limit; readPos++) {
- out.writeVInt(toInt(values[readPos]));
+ out.writeVInt(values[readPos]);
+ }
+ }
+
+ /**
+ * Read all the group varints, including the tail vints to a long[].
+ *
+ * @param dst the array to read ints into.
+ * @param limit the number of int values to read.
+ * @lucene.experimental
+ * @deprecated Only for backwards codecs
+ */
+ @Deprecated
+ public static void readGroupVInts(DataInput in, long[] dst, int limit) throws IOException {
+ int i;
+ for (i = 0; i <= limit - 4; i += 4) {
+ readGroupVInt(in, dst, i);
+ }
+ for (; i < limit; ++i) {
+ dst[i] = in.readVInt() & 0xFFFFFFFFL;
}
}
+ /**
+ * Default implementation of read single group, for optimal performance, you should use {@link
+ * GroupVIntUtil#readGroupVInts(DataInput, long[], int)} instead.
+ *
+ * @param in the input to use to read data.
+ * @param dst the array to read ints into.
+ * @param offset the offset in the array to start storing ints.
+ * @deprecated Only for backwards codecs
+ */
+ @Deprecated
+ public static void readGroupVInt(DataInput in, long[] dst, int offset) throws IOException {
+ final int flag = in.readByte() & 0xFF;
+
+ final int n1Minus1 = flag >> 6;
+ final int n2Minus1 = (flag >> 4) & 0x03;
+ final int n3Minus1 = (flag >> 2) & 0x03;
+ final int n4Minus1 = flag & 0x03;
+
+ dst[offset] = readIntInGroup(in, n1Minus1) & 0xFFFFFFFFL;
+ dst[offset + 1] = readIntInGroup(in, n2Minus1) & 0xFFFFFFFFL;
+ dst[offset + 2] = readIntInGroup(in, n3Minus1) & 0xFFFFFFFFL;
+ dst[offset + 3] = readIntInGroup(in, n4Minus1) & 0xFFFFFFFFL;
+ }
+
/**
* The implementation for group-varint encoding, It uses a maximum of {@link
* #MAX_LENGTH_PER_GROUP} bytes scratch buffer.
*/
- public static void writeGroupVInts(DataOutput out, byte[] scratch, int[] values, int limit)
+ @Deprecated
+ public static void writeGroupVInts(DataOutput out, byte[] scratch, long[] values, int limit)
throws IOException {
int readPos = 0;
// encode each group
while ((limit - readPos) >= 4) {
int writePos = 0;
- final int n1Minus1 = numBytes(values[readPos]) - 1;
- final int n2Minus1 = numBytes(values[readPos + 1]) - 1;
- final int n3Minus1 = numBytes(values[readPos + 2]) - 1;
- final int n4Minus1 = numBytes(values[readPos + 3]) - 1;
+ final int n1Minus1 = numBytes(toInt(values[readPos])) - 1;
+ final int n2Minus1 = numBytes(toInt(values[readPos + 1])) - 1;
+ final int n3Minus1 = numBytes(toInt(values[readPos + 2])) - 1;
+ final int n4Minus1 = numBytes(toInt(values[readPos + 3])) - 1;
int flag = (n1Minus1 << 6) | (n2Minus1 << 4) | (n3Minus1 << 2) | (n4Minus1);
scratch[writePos++] = (byte) flag;
- BitUtil.VH_LE_INT.set(scratch, writePos, values[readPos++]);
+ BitUtil.VH_LE_INT.set(scratch, writePos, (int) (values[readPos++]));
writePos += n1Minus1 + 1;
- BitUtil.VH_LE_INT.set(scratch, writePos, values[readPos++]);
+ BitUtil.VH_LE_INT.set(scratch, writePos, (int) (values[readPos++]));
writePos += n2Minus1 + 1;
- BitUtil.VH_LE_INT.set(scratch, writePos, values[readPos++]);
+ BitUtil.VH_LE_INT.set(scratch, writePos, (int) (values[readPos++]));
writePos += n3Minus1 + 1;
- BitUtil.VH_LE_INT.set(scratch, writePos, values[readPos++]);
+ BitUtil.VH_LE_INT.set(scratch, writePos, (int) (values[readPos++]));
writePos += n4Minus1 + 1;
out.writeBytes(scratch, writePos);
@@ -292,7 +254,7 @@ public final class GroupVIntUtil {
// tail vints
for (; readPos < limit; readPos++) {
- out.writeVInt(values[readPos]);
+ out.writeVInt(toInt(values[readPos]));
}
}
}
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/store/BaseChunkedDirectoryTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/tests/store/BaseChunkedDirectoryTestCase.java
index 8de332eeec9..7d259d88f3b 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/store/BaseChunkedDirectoryTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/store/BaseChunkedDirectoryTestCase.java
@@ -61,7 +61,7 @@ public abstract class BaseChunkedDirectoryTestCase extends BaseDirectoryTestCase
public void testCloneClose() throws Exception {
Directory dir = getDirectory(createTempDir("testCloneClose"));
IndexOutput io = dir.createOutput("bytes", newIOContext(random()));
- final long[] values = new long[] {0, 7, 11, 9};
+ final int[] values = new int[] {0, 7, 11, 9};
io.writeVInt(5);
io.writeGroupVInts(values, values.length);
io.close();
@@ -89,7 +89,7 @@ public abstract class BaseChunkedDirectoryTestCase extends BaseDirectoryTestCase
public void testCloneSliceClose() throws Exception {
Directory dir = getDirectory(createTempDir("testCloneSliceClose"));
IndexOutput io = dir.createOutput("bytes", newIOContext(random()));
- final long[] values = new long[] {0, 7, 11, 9};
+ final int[] values = new int[] {0, 7, 11, 9};
io.writeInt(1);
io.writeInt(2);
io.writeGroupVInts(values, values.length); // will write 5 bytes
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/store/BaseDirectoryTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/tests/store/BaseDirectoryTestCase.java
index 7b74a6b0dc8..27bfa3c72f1 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/store/BaseDirectoryTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/store/BaseDirectoryTestCase.java
@@ -1458,7 +1458,7 @@ public abstract class BaseDirectoryTestCase extends LuceneTestCase {
}
public void testDataTypes() throws IOException {
- final long[] values = new long[] {43, 12345, 123456, 1234567890};
+ final int[] values = new int[] {43, 12345, 123456, 1234567890};
try (Directory dir = getDirectory(createTempDir("testDataTypes"))) {
IndexOutput out = dir.createOutput("test", IOContext.DEFAULT);
out.writeByte((byte) 43);
@@ -1468,7 +1468,7 @@ public abstract class BaseDirectoryTestCase extends LuceneTestCase {
out.writeLong(1234567890123456789L);
out.close();
- long[] restored = new long[4];
+ int[] restored = new int[4];
IndexInput in = dir.openInput("test", IOContext.DEFAULT);
assertEquals(43, in.readByte());
assertEquals(12345, in.readShort());
@@ -1480,6 +1480,7 @@ public abstract class BaseDirectoryTestCase extends LuceneTestCase {
}
}
+ @Deprecated
public void testGroupVIntOverflow() throws IOException {
try (Directory dir = getDirectory(createTempDir("testGroupVIntOverflow"))) {
final int size = 32;
@@ -1526,7 +1527,7 @@ public abstract class BaseDirectoryTestCase extends LuceneTestCase {
protected void doTestGroupVInt(
Directory dir, int iterations, int minBpv, int maxBpv, int maxNumValues) throws IOException {
- long[] values = new long[maxNumValues];
+ int[] values = new int[maxNumValues];
int[] numValuesArray = new int[iterations];
IndexOutput groupVIntOut = dir.createOutput("group-varint", IOContext.DEFAULT);
IndexOutput vIntOut = dir.createOutput("vint", IOContext.DEFAULT);
@@ -1537,7 +1538,7 @@ public abstract class BaseDirectoryTestCase extends LuceneTestCase {
numValuesArray[iter] = TestUtil.nextInt(random(), 1, maxNumValues);
for (int j = 0; j < numValuesArray[iter]; j++) {
values[j] = RandomNumbers.randomIntBetween(random(), 0, (int) PackedInts.maxValue(bpv));
- vIntOut.writeVInt((int) values[j]);
+ vIntOut.writeVInt(values[j]);
}
groupVIntOut.writeGroupVInts(values, numValuesArray[iter]);
}