Apache Lucene open-source search software
Fix GroupVIntBenchmark and some tests to actually use DataInput#readGroupVInt; deprecate/remove obsolete long[] code (#15104)
Adrien Grand2c5650592742
diff --git a/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/GroupVIntBenchmark.java b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/GroupVIntBenchmark.javaindex 48b95570694..6b40870c375 100644--- a/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/GroupVIntBenchmark.java+++ b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/GroupVIntBenchmark.java@@ -86,12 +86,12 @@ public class GroupVIntBenchmark {};final int maxSize = 256;- final long[] docs = new long[maxSize];- final long[] values = new long[maxSize];+ final int[] docs = new int[maxSize];+ final int[] values = new int[maxSize];- IndexInput byteBufferGVIntIn;+ IndexInput mmapGVIntIn;IndexInput nioGVIntIn;- IndexInput byteBufferVIntIn;+ IndexInput mmapVIntIn;ByteBuffersDataInput byteBuffersGVIntIn;ByteArrayDataInput byteArrayVIntIn;@@ -103,7 +103,7 @@ public class GroupVIntBenchmark {@Param({"64"})public int size;- void initArrayInput(long[] docs) throws Exception {+ void initArrayInput(int[] docs) throws Exception {byte[] gVIntBytes = new byte[Integer.BYTES * maxSize * 2];byte[] vIntBytes = new byte[Integer.BYTES * maxSize * 2];ByteArrayDataOutput vIntOut = new ByteArrayDataOutput(vIntBytes);@@ -116,7 +116,7 @@ public class GroupVIntBenchmark {byteArrayGVIntIn = new ByteArrayDataInput(gVIntBytes);}- void initNioInput(long[] docs) throws Exception {+ void initNioInput(int[] docs) throws Exception {Directory dir = new NIOFSDirectory(Files.createTempDirectory("groupvintdata"));IndexOutput out = dir.createOutput("gvint", IOContext.DEFAULT);out.writeGroupVInts(docs, docs.length);@@ -124,13 +124,13 @@ public class GroupVIntBenchmark {nioGVIntIn = dir.openInput("gvint", IOContext.DEFAULT);}- void initByteBuffersInput(long[] docs) throws Exception {+ void initByteBuffersInput(int[] docs) throws Exception {ByteBuffersDataOutput buffer = new ByteBuffersDataOutput();buffer.writeGroupVInts(docs, docs.length);byteBuffersGVIntIn = buffer.toDataInput();}- void initByteBufferInput(long[] docs) throws Exception {+ void initMMapInput(int[] docs) throws Exception {Directory dir = new MMapDirectory(Files.createTempDirectory("groupvintdata"));IndexOutput vintOut = dir.createOutput("vint", IOContext.DEFAULT);IndexOutput gvintOut = dir.createOutput("gvint", IOContext.DEFAULT);@@ -141,11 +141,11 @@ public class GroupVIntBenchmark {}vintOut.close();gvintOut.close();- byteBufferGVIntIn = dir.openInput("gvint", IOContext.DEFAULT);- byteBufferVIntIn = dir.openInput("vint", IOContext.DEFAULT);+ mmapGVIntIn = dir.openInput("gvint", IOContext.DEFAULT);+ mmapVIntIn = dir.openInput("vint", IOContext.DEFAULT);}- private void readGroupVIntsBaseline(DataInput in, long[] dst, int limit) throws IOException {+ private void readGroupVIntsBaseline(DataInput in, int[] dst, int limit) throws IOException {int i;for (i = 0; i <= limit - 4; i += 4) {GroupVIntUtil.readGroupVInt(in, dst, i);@@ -168,7 +168,7 @@ public class GroupVIntBenchmark {}docs[i] = r.nextInt(1 << (numBits - 1), 1 << numBits);}- initByteBufferInput(docs);+ initMMapInput(docs);initArrayInput(docs);initNioInput(docs);initByteBuffersInput(docs);@@ -176,24 +176,24 @@ public class GroupVIntBenchmark {@Benchmarkpublic void benchMMapDirectoryInputs_readVInt(Blackhole bh) throws IOException {- byteBufferVIntIn.seek(0);+ mmapVIntIn.seek(0);for (int i = 0; i < size; i++) {- values[i] = byteBufferVIntIn.readVInt();+ values[i] = mmapVIntIn.readVInt();}bh.consume(values);}@Benchmarkpublic void benchMMapDirectoryInputs_readGroupVInt(Blackhole bh) throws IOException {- byteBufferGVIntIn.seek(0);- GroupVIntUtil.readGroupVInts(byteBufferGVIntIn, values, size);+ mmapGVIntIn.seek(0);+ GroupVIntUtil.readGroupVInts(mmapGVIntIn, values, size);bh.consume(values);}@Benchmarkpublic void benchMMapDirectoryInputs_readGroupVIntBaseline(Blackhole bh) throws IOException {- byteBufferGVIntIn.seek(0);- this.readGroupVIntsBaseline(byteBufferGVIntIn, values, size);+ mmapGVIntIn.seek(0);+ this.readGroupVIntsBaseline(mmapGVIntIn, values, size);bh.consume(values);}diff --git a/lucene/core/src/java/org/apache/lucene/store/DataOutput.java b/lucene/core/src/java/org/apache/lucene/store/DataOutput.javaindex 543ff175867..3d856cbc5f6 100644--- a/lucene/core/src/java/org/apache/lucene/store/DataOutput.java+++ b/lucene/core/src/java/org/apache/lucene/store/DataOutput.java@@ -333,7 +333,9 @@ public abstract class DataOutput {* @param values the values to write* @param limit the number of values to write.* @lucene.experimental+ * @deprecated This method is preserved only for backwards codecs*/+ @Deprecatedpublic void writeGroupVInts(long[] values, int limit) throws IOException {if (groupVIntBytes == null) {groupVIntBytes = new byte[GroupVIntUtil.MAX_LENGTH_PER_GROUP];diff --git a/lucene/core/src/java/org/apache/lucene/util/GroupVIntUtil.java b/lucene/core/src/java/org/apache/lucene/util/GroupVIntUtil.javaindex e95e2eee4db..a1917d87a17 100644--- a/lucene/core/src/java/org/apache/lucene/util/GroupVIntUtil.java+++ b/lucene/core/src/java/org/apache/lucene/util/GroupVIntUtil.java@@ -29,28 +29,8 @@ public final class GroupVIntUtil {// the maximum length of a single group-varint is 4 integers + 1 byte flag.public static final int MAX_LENGTH_PER_GROUP = 17;- // we use long array instead of int array to make negative integer to be read as positive long.- private static final long[] LONG_MASKS = new long[] {0xFFL, 0xFFFFL, 0xFFFFFFL, 0xFFFFFFFFL};private static final int[] INT_MASKS = new int[] {0xFF, 0xFFFF, 0xFFFFFF, ~0};- /**- * Read all the group varints, including the tail vints. we need a long[] because this is what- * postings are using, all longs are actually required to be integers.- *- * @param dst the array to read ints into.- * @param limit the number of int values to read.- * @lucene.experimental- */- public static void readGroupVInts(DataInput in, long[] dst, int limit) throws IOException {- int i;- for (i = 0; i <= limit - 4; i += 4) {- readGroupVInt(in, dst, i);- }- for (; i < limit; ++i) {- dst[i] = in.readVInt() & 0xFFFFFFFFL;- }- }-/*** Read all the group varints, including the tail vints.*@@ -68,28 +48,6 @@ public final class GroupVIntUtil {}}- /**- * Default implementation of read single group, for optimal performance, you should use {@link- * GroupVIntUtil#readGroupVInts(DataInput, long[], int)} instead.- *- * @param in the input to use to read data.- * @param dst the array to read ints into.- * @param offset the offset in the array to start storing ints.- */- public static void readGroupVInt(DataInput in, long[] dst, int offset) throws IOException {- final int flag = in.readByte() & 0xFF;-- final int n1Minus1 = flag >> 6;- final int n2Minus1 = (flag >> 4) & 0x03;- final int n3Minus1 = (flag >> 2) & 0x03;- final int n4Minus1 = flag & 0x03;-- dst[offset] = readIntInGroup(in, n1Minus1) & 0xFFFFFFFFL;- dst[offset + 1] = readIntInGroup(in, n2Minus1) & 0xFFFFFFFFL;- dst[offset + 2] = readIntInGroup(in, n3Minus1) & 0xFFFFFFFFL;- dst[offset + 3] = readIntInGroup(in, n4Minus1) & 0xFFFFFFFFL;- }-/*** Default implementation of read single group, for optimal performance, you should use {@link* GroupVIntUtil#readGroupVInts(DataInput, int[], int)} instead.@@ -134,46 +92,6 @@ public final class GroupVIntUtil {int read(long v);}- /**- * Faster implementation of read single group, It read values from the buffer that would not cross- * boundaries.- *- * @param in the input to use to read data.- * @param remaining the number of remaining bytes allowed to read for current block/segment.- * @param reader the supplier of read int.- * @param pos the start pos to read from the reader.- * @param dst the array to read ints into.- * @param offset the offset in the array to start storing ints.- * @return the number of bytes read excluding the flag. this indicates the number of positions- * should to be increased for caller, it is 0 or positive number and less than {@link- * #MAX_LENGTH_PER_GROUP}- */- public static int readGroupVInt(- DataInput in, long remaining, IntReader reader, long pos, long[] dst, int offset)- throws IOException {- if (remaining < MAX_LENGTH_PER_GROUP) {- readGroupVInt(in, dst, offset);- return 0;- }- final int flag = in.readByte() & 0xFF;- final long posStart = ++pos; // exclude the flag bytes, the position has updated via readByte().- final int n1Minus1 = flag >> 6;- final int n2Minus1 = (flag >> 4) & 0x03;- final int n3Minus1 = (flag >> 2) & 0x03;- final int n4Minus1 = flag & 0x03;-- // This code path has fewer conditionals and tends to be significantly faster in benchmarks- dst[offset] = reader.read(pos) & LONG_MASKS[n1Minus1];- pos += 1 + n1Minus1;- dst[offset + 1] = reader.read(pos) & LONG_MASKS[n2Minus1];- pos += 1 + n2Minus1;- dst[offset + 2] = reader.read(pos) & LONG_MASKS[n3Minus1];- pos += 1 + n3Minus1;- dst[offset + 3] = reader.read(pos) & LONG_MASKS[n4Minus1];- pos += 1 + n4Minus1;- return (int) (pos - posStart);- }-/*** Faster implementation of read single group, It read values from the buffer that would not cross* boundaries.@@ -230,26 +148,26 @@ public final class GroupVIntUtil {* The implementation for group-varint encoding, It uses a maximum of {@link* #MAX_LENGTH_PER_GROUP} bytes scratch buffer.*/- public static void writeGroupVInts(DataOutput out, byte[] scratch, long[] values, int limit)+ public static void writeGroupVInts(DataOutput out, byte[] scratch, int[] values, int limit)throws IOException {int readPos = 0;// encode each groupwhile ((limit - readPos) >= 4) {int writePos = 0;- final int n1Minus1 = numBytes(toInt(values[readPos])) - 1;- final int n2Minus1 = numBytes(toInt(values[readPos + 1])) - 1;- final int n3Minus1 = numBytes(toInt(values[readPos + 2])) - 1;- final int n4Minus1 = numBytes(toInt(values[readPos + 3])) - 1;+ final int n1Minus1 = numBytes(values[readPos]) - 1;+ final int n2Minus1 = numBytes(values[readPos + 1]) - 1;+ final int n3Minus1 = numBytes(values[readPos + 2]) - 1;+ final int n4Minus1 = numBytes(values[readPos + 3]) - 1;int flag = (n1Minus1 << 6) | (n2Minus1 << 4) | (n3Minus1 << 2) | (n4Minus1);scratch[writePos++] = (byte) flag;- BitUtil.VH_LE_INT.set(scratch, writePos, (int) (values[readPos++]));+ BitUtil.VH_LE_INT.set(scratch, writePos, values[readPos++]);writePos += n1Minus1 + 1;- BitUtil.VH_LE_INT.set(scratch, writePos, (int) (values[readPos++]));+ BitUtil.VH_LE_INT.set(scratch, writePos, values[readPos++]);writePos += n2Minus1 + 1;- BitUtil.VH_LE_INT.set(scratch, writePos, (int) (values[readPos++]));+ BitUtil.VH_LE_INT.set(scratch, writePos, values[readPos++]);writePos += n3Minus1 + 1;- BitUtil.VH_LE_INT.set(scratch, writePos, (int) (values[readPos++]));+ BitUtil.VH_LE_INT.set(scratch, writePos, values[readPos++]);writePos += n4Minus1 + 1;out.writeBytes(scratch, writePos);@@ -257,34 +175,78 @@ public final class GroupVIntUtil {// tail vintsfor (; readPos < limit; readPos++) {- out.writeVInt(toInt(values[readPos]));+ out.writeVInt(values[readPos]);+ }+ }++ /**+ * Read all the group varints, including the tail vints to a long[].+ *+ * @param dst the array to read ints into.+ * @param limit the number of int values to read.+ * @lucene.experimental+ * @deprecated Only for backwards codecs+ */+ @Deprecated+ public static void readGroupVInts(DataInput in, long[] dst, int limit) throws IOException {+ int i;+ for (i = 0; i <= limit - 4; i += 4) {+ readGroupVInt(in, dst, i);+ }+ for (; i < limit; ++i) {+ dst[i] = in.readVInt() & 0xFFFFFFFFL;}}+ /**+ * Default implementation of read single group, for optimal performance, you should use {@link+ * GroupVIntUtil#readGroupVInts(DataInput, long[], int)} instead.+ *+ * @param in the input to use to read data.+ * @param dst the array to read ints into.+ * @param offset the offset in the array to start storing ints.+ * @deprecated Only for backwards codecs+ */+ @Deprecated+ public static void readGroupVInt(DataInput in, long[] dst, int offset) throws IOException {+ final int flag = in.readByte() & 0xFF;++ final int n1Minus1 = flag >> 6;+ final int n2Minus1 = (flag >> 4) & 0x03;+ final int n3Minus1 = (flag >> 2) & 0x03;+ final int n4Minus1 = flag & 0x03;++ dst[offset] = readIntInGroup(in, n1Minus1) & 0xFFFFFFFFL;+ dst[offset + 1] = readIntInGroup(in, n2Minus1) & 0xFFFFFFFFL;+ dst[offset + 2] = readIntInGroup(in, n3Minus1) & 0xFFFFFFFFL;+ dst[offset + 3] = readIntInGroup(in, n4Minus1) & 0xFFFFFFFFL;+ }+/*** The implementation for group-varint encoding, It uses a maximum of {@link* #MAX_LENGTH_PER_GROUP} bytes scratch buffer.*/- public static void writeGroupVInts(DataOutput out, byte[] scratch, int[] values, int limit)+ @Deprecated+ public static void writeGroupVInts(DataOutput out, byte[] scratch, long[] values, int limit)throws IOException {int readPos = 0;// encode each groupwhile ((limit - readPos) >= 4) {int writePos = 0;- final int n1Minus1 = numBytes(values[readPos]) - 1;- final int n2Minus1 = numBytes(values[readPos + 1]) - 1;- final int n3Minus1 = numBytes(values[readPos + 2]) - 1;- final int n4Minus1 = numBytes(values[readPos + 3]) - 1;+ final int n1Minus1 = numBytes(toInt(values[readPos])) - 1;+ final int n2Minus1 = numBytes(toInt(values[readPos + 1])) - 1;+ final int n3Minus1 = numBytes(toInt(values[readPos + 2])) - 1;+ final int n4Minus1 = numBytes(toInt(values[readPos + 3])) - 1;int flag = (n1Minus1 << 6) | (n2Minus1 << 4) | (n3Minus1 << 2) | (n4Minus1);scratch[writePos++] = (byte) flag;- BitUtil.VH_LE_INT.set(scratch, writePos, values[readPos++]);+ BitUtil.VH_LE_INT.set(scratch, writePos, (int) (values[readPos++]));writePos += n1Minus1 + 1;- BitUtil.VH_LE_INT.set(scratch, writePos, values[readPos++]);+ BitUtil.VH_LE_INT.set(scratch, writePos, (int) (values[readPos++]));writePos += n2Minus1 + 1;- BitUtil.VH_LE_INT.set(scratch, writePos, values[readPos++]);+ BitUtil.VH_LE_INT.set(scratch, writePos, (int) (values[readPos++]));writePos += n3Minus1 + 1;- BitUtil.VH_LE_INT.set(scratch, writePos, values[readPos++]);+ BitUtil.VH_LE_INT.set(scratch, writePos, (int) (values[readPos++]));writePos += n4Minus1 + 1;out.writeBytes(scratch, writePos);@@ -292,7 +254,7 @@ public final class GroupVIntUtil {// tail vintsfor (; readPos < limit; readPos++) {- out.writeVInt(values[readPos]);+ out.writeVInt(toInt(values[readPos]));}}}diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/store/BaseChunkedDirectoryTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/tests/store/BaseChunkedDirectoryTestCase.javaindex 8de332eeec9..7d259d88f3b 100644--- a/lucene/test-framework/src/java/org/apache/lucene/tests/store/BaseChunkedDirectoryTestCase.java+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/store/BaseChunkedDirectoryTestCase.java@@ -61,7 +61,7 @@ public abstract class BaseChunkedDirectoryTestCase extends BaseDirectoryTestCasepublic void testCloneClose() throws Exception {Directory dir = getDirectory(createTempDir("testCloneClose"));IndexOutput io = dir.createOutput("bytes", newIOContext(random()));- final long[] values = new long[] {0, 7, 11, 9};+ final int[] values = new int[] {0, 7, 11, 9};io.writeVInt(5);io.writeGroupVInts(values, values.length);io.close();@@ -89,7 +89,7 @@ public abstract class BaseChunkedDirectoryTestCase extends BaseDirectoryTestCasepublic void testCloneSliceClose() throws Exception {Directory dir = getDirectory(createTempDir("testCloneSliceClose"));IndexOutput io = dir.createOutput("bytes", newIOContext(random()));- final long[] values = new long[] {0, 7, 11, 9};+ final int[] values = new int[] {0, 7, 11, 9};io.writeInt(1);io.writeInt(2);io.writeGroupVInts(values, values.length); // will write 5 bytesdiff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/store/BaseDirectoryTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/tests/store/BaseDirectoryTestCase.javaindex 7b74a6b0dc8..27bfa3c72f1 100644--- a/lucene/test-framework/src/java/org/apache/lucene/tests/store/BaseDirectoryTestCase.java+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/store/BaseDirectoryTestCase.java@@ -1458,7 +1458,7 @@ public abstract class BaseDirectoryTestCase extends LuceneTestCase {}public void testDataTypes() throws IOException {- final long[] values = new long[] {43, 12345, 123456, 1234567890};+ final int[] values = new int[] {43, 12345, 123456, 1234567890};try (Directory dir = getDirectory(createTempDir("testDataTypes"))) {IndexOutput out = dir.createOutput("test", IOContext.DEFAULT);out.writeByte((byte) 43);@@ -1468,7 +1468,7 @@ public abstract class BaseDirectoryTestCase extends LuceneTestCase {out.writeLong(1234567890123456789L);out.close();- long[] restored = new long[4];+ int[] restored = new int[4];IndexInput in = dir.openInput("test", IOContext.DEFAULT);assertEquals(43, in.readByte());assertEquals(12345, in.readShort());@@ -1480,6 +1480,7 @@ public abstract class BaseDirectoryTestCase extends LuceneTestCase {}}+ @Deprecatedpublic void testGroupVIntOverflow() throws IOException {try (Directory dir = getDirectory(createTempDir("testGroupVIntOverflow"))) {final int size = 32;@@ -1526,7 +1527,7 @@ public abstract class BaseDirectoryTestCase extends LuceneTestCase {protected void doTestGroupVInt(Directory dir, int iterations, int minBpv, int maxBpv, int maxNumValues) throws IOException {- long[] values = new long[maxNumValues];+ int[] values = new int[maxNumValues];int[] numValuesArray = new int[iterations];IndexOutput groupVIntOut = dir.createOutput("group-varint", IOContext.DEFAULT);IndexOutput vIntOut = dir.createOutput("vint", IOContext.DEFAULT);@@ -1537,7 +1538,7 @@ public abstract class BaseDirectoryTestCase extends LuceneTestCase {numValuesArray[iter] = TestUtil.nextInt(random(), 1, maxNumValues);for (int j = 0; j < numValuesArray[iter]; j++) {values[j] = RandomNumbers.randomIntBetween(random(), 0, (int) PackedInts.maxValue(bpv));- vIntOut.writeVInt((int) values[j]);+ vIntOut.writeVInt(values[j]);}groupVIntOut.writeGroupVInts(values, numValuesArray[iter]);}