Skip to content

Apache Lucene open-source search software

Add missing test for Lucene912PostingsFormat.

Adrien Grand82210189ee84
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene912/TestForDeltaUtil.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene912/TestForDeltaUtil.java
new file mode 100644
index 00000000000..3c201ce6835
--- /dev/null
+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene912/TestForDeltaUtil.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.codecs.lucene912;
+
+import com.carrotsearch.randomizedtesting.generators.RandomNumbers;
+import java.io.IOException;
+import java.util.Arrays;
+import org.apache.lucene.store.ByteBuffersDirectory;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.tests.util.LuceneTestCase;
+import org.apache.lucene.tests.util.TestUtil;
+import org.apache.lucene.util.packed.PackedInts;
+
+public class TestForDeltaUtil extends LuceneTestCase {
+
+ public void testEncodeDecode() throws IOException {
+ final int iterations = RandomNumbers.randomIntBetween(random(), 50, 1000);
+ final int[] values = new int[iterations * ForUtil.BLOCK_SIZE];
+
+ for (int i = 0; i < iterations; ++i) {
+ final int bpv = TestUtil.nextInt(random(), 1, 31 - 7);
+ for (int j = 0; j < ForUtil.BLOCK_SIZE; ++j) {
+ values[i * ForUtil.BLOCK_SIZE + j] =
+ RandomNumbers.randomIntBetween(random(), 1, (int) PackedInts.maxValue(bpv));
+ }
+ }
+
+ final Directory d = new ByteBuffersDirectory();
+ final long endPointer;
+
+ {
+ // encode
+ IndexOutput out = d.createOutput("test.bin", IOContext.DEFAULT);
+ final ForDeltaUtil forDeltaUtil = new ForDeltaUtil(new ForUtil());
+
+ for (int i = 0; i < iterations; ++i) {
+ long[] source = new long[ForUtil.BLOCK_SIZE];
+ for (int j = 0; j < ForUtil.BLOCK_SIZE; ++j) {
+ source[j] = values[i * ForUtil.BLOCK_SIZE + j];
+ }
+ forDeltaUtil.encodeDeltas(source, out);
+ }
+ endPointer = out.getFilePointer();
+ out.close();
+ }
+
+ {
+ // decode
+ IndexInput in = d.openInput("test.bin", IOContext.READONCE);
+ final ForDeltaUtil forDeltaUtil = new ForDeltaUtil(new ForUtil());
+ for (int i = 0; i < iterations; ++i) {
+ long base = 0;
+ final long[] restored = new long[ForUtil.BLOCK_SIZE];
+ forDeltaUtil.decodeAndPrefixSum(in, base, restored);
+ final long[] expected = new long[ForUtil.BLOCK_SIZE];
+ for (int j = 0; j < ForUtil.BLOCK_SIZE; ++j) {
+ expected[j] = values[i * ForUtil.BLOCK_SIZE + j];
+ if (j > 0) {
+ expected[j] += expected[j - 1];
+ } else {
+ expected[j] += base;
+ }
+ }
+ assertArrayEquals(Arrays.toString(restored), expected, restored);
+ }
+ assertEquals(endPointer, in.getFilePointer());
+ in.close();
+ }
+
+ d.close();
+ }
+}
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene912/TestForUtil.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene912/TestForUtil.java
new file mode 100644
index 00000000000..114a9d0415c
--- /dev/null
+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene912/TestForUtil.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.codecs.lucene912;
+
+import com.carrotsearch.randomizedtesting.generators.RandomNumbers;
+import java.io.IOException;
+import java.util.Arrays;
+import org.apache.lucene.store.ByteBuffersDirectory;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.tests.util.LuceneTestCase;
+import org.apache.lucene.tests.util.TestUtil;
+import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.packed.PackedInts;
+
+public class TestForUtil extends LuceneTestCase {
+
+ public void testEncodeDecode() throws IOException {
+ final int iterations = RandomNumbers.randomIntBetween(random(), 50, 1000);
+ final int[] values = new int[iterations * ForUtil.BLOCK_SIZE];
+
+ for (int i = 0; i < iterations; ++i) {
+ final int bpv = TestUtil.nextInt(random(), 1, 31);
+ for (int j = 0; j < ForUtil.BLOCK_SIZE; ++j) {
+ values[i * ForUtil.BLOCK_SIZE + j] =
+ RandomNumbers.randomIntBetween(random(), 0, (int) PackedInts.maxValue(bpv));
+ }
+ }
+
+ final Directory d = new ByteBuffersDirectory();
+ final long endPointer;
+
+ {
+ // encode
+ IndexOutput out = d.createOutput("test.bin", IOContext.DEFAULT);
+ final ForUtil forUtil = new ForUtil();
+
+ for (int i = 0; i < iterations; ++i) {
+ long[] source = new long[ForUtil.BLOCK_SIZE];
+ long or = 0;
+ for (int j = 0; j < ForUtil.BLOCK_SIZE; ++j) {
+ source[j] = values[i * ForUtil.BLOCK_SIZE + j];
+ or |= source[j];
+ }
+ final int bpv = PackedInts.bitsRequired(or);
+ out.writeByte((byte) bpv);
+ forUtil.encode(source, bpv, out);
+ }
+ endPointer = out.getFilePointer();
+ out.close();
+ }
+
+ {
+ // decode
+ IndexInput in = d.openInput("test.bin", IOContext.READONCE);
+ final ForUtil forUtil = new ForUtil();
+ for (int i = 0; i < iterations; ++i) {
+ final int bitsPerValue = in.readByte();
+ final long currentFilePointer = in.getFilePointer();
+ final long[] restored = new long[ForUtil.BLOCK_SIZE];
+ forUtil.decode(bitsPerValue, in, restored);
+ int[] ints = new int[ForUtil.BLOCK_SIZE];
+ for (int j = 0; j < ForUtil.BLOCK_SIZE; ++j) {
+ ints[j] = Math.toIntExact(restored[j]);
+ }
+ assertArrayEquals(
+ Arrays.toString(ints),
+ ArrayUtil.copyOfSubArray(values, i * ForUtil.BLOCK_SIZE, (i + 1) * ForUtil.BLOCK_SIZE),
+ ints);
+ assertEquals(forUtil.numBytes(bitsPerValue), in.getFilePointer() - currentFilePointer);
+ }
+ assertEquals(endPointer, in.getFilePointer());
+ in.close();
+ }
+
+ d.close();
+ }
+}
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene912/TestPForUtil.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene912/TestPForUtil.java
new file mode 100644
index 00000000000..08fec7a3a33
--- /dev/null
+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene912/TestPForUtil.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.codecs.lucene912;
+
+import com.carrotsearch.randomizedtesting.generators.RandomNumbers;
+import java.io.IOException;
+import java.util.Arrays;
+import org.apache.lucene.store.ByteBuffersDirectory;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.tests.util.LuceneTestCase;
+import org.apache.lucene.tests.util.TestUtil;
+import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.packed.PackedInts;
+
+public class TestPForUtil extends LuceneTestCase {
+
+ public void testEncodeDecode() throws IOException {
+ final int iterations = RandomNumbers.randomIntBetween(random(), 50, 1000);
+ final int[] values = createTestData(iterations, 31);
+
+ final Directory d = new ByteBuffersDirectory();
+ final long endPointer = encodeTestData(iterations, values, d);
+
+ IndexInput in = d.openInput("test.bin", IOContext.READONCE);
+ final PForUtil pforUtil = new PForUtil(new ForUtil());
+ for (int i = 0; i < iterations; ++i) {
+ if (random().nextInt(5) == 0) {
+ pforUtil.skip(in);
+ continue;
+ }
+ final long[] restored = new long[ForUtil.BLOCK_SIZE];
+ pforUtil.decode(in, restored);
+ int[] ints = new int[ForUtil.BLOCK_SIZE];
+ for (int j = 0; j < ForUtil.BLOCK_SIZE; ++j) {
+ ints[j] = Math.toIntExact(restored[j]);
+ }
+ assertArrayEquals(
+ Arrays.toString(ints),
+ ArrayUtil.copyOfSubArray(values, i * ForUtil.BLOCK_SIZE, (i + 1) * ForUtil.BLOCK_SIZE),
+ ints);
+ }
+ assertEquals(endPointer, in.getFilePointer());
+ in.close();
+
+ d.close();
+ }
+
+ private int[] createTestData(int iterations, int maxBpv) {
+ final int[] values = new int[iterations * ForUtil.BLOCK_SIZE];
+
+ for (int i = 0; i < iterations; ++i) {
+ final int bpv = TestUtil.nextInt(random(), 0, maxBpv);
+ for (int j = 0; j < ForUtil.BLOCK_SIZE; ++j) {
+ values[i * ForUtil.BLOCK_SIZE + j] =
+ RandomNumbers.randomIntBetween(random(), 0, (int) PackedInts.maxValue(bpv));
+ if (random().nextInt(100) == 0) {
+ final int exceptionBpv;
+ if (random().nextInt(10) == 0) {
+ exceptionBpv = Math.min(bpv + TestUtil.nextInt(random(), 9, 16), maxBpv);
+ } else {
+ exceptionBpv = Math.min(bpv + TestUtil.nextInt(random(), 1, 8), maxBpv);
+ }
+ values[i * ForUtil.BLOCK_SIZE + j] |= random().nextInt(1 << (exceptionBpv - bpv)) << bpv;
+ }
+ }
+ }
+
+ return values;
+ }
+
+ private long encodeTestData(int iterations, int[] values, Directory d) throws IOException {
+ IndexOutput out = d.createOutput("test.bin", IOContext.DEFAULT);
+ final PForUtil pforUtil = new PForUtil(new ForUtil());
+
+ for (int i = 0; i < iterations; ++i) {
+ long[] source = new long[ForUtil.BLOCK_SIZE];
+ for (int j = 0; j < ForUtil.BLOCK_SIZE; ++j) {
+ source[j] = values[i * ForUtil.BLOCK_SIZE + j];
+ }
+ pforUtil.encode(source, out);
+ }
+ final long endPointer = out.getFilePointer();
+ out.close();
+
+ return endPointer;
+ }
+}
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene912/TestPostingsUtil.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene912/TestPostingsUtil.java
new file mode 100644
index 00000000000..870f6e94e98
--- /dev/null
+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene912/TestPostingsUtil.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.codecs.lucene912;
+
+import java.io.IOException;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.tests.util.LuceneTestCase;
+
+public class TestPostingsUtil extends LuceneTestCase {
+
+ // checks for bug described in https://github.com/apache/lucene/issues/13373
+ public void testIntegerOverflow() throws IOException {
+ final int size = random().nextInt(1, ForUtil.BLOCK_SIZE);
+ final long[] docDeltaBuffer = new long[size];
+ final long[] freqBuffer = new long[size];
+
+ final int delta = 1 << 30;
+ docDeltaBuffer[0] = delta;
+ try (Directory dir = newDirectory()) {
+ try (IndexOutput out = dir.createOutput("test", IOContext.DEFAULT)) {
+ // In old implementation, this would cause integer overflow exception.
+ PostingsUtil.writeVIntBlock(out, docDeltaBuffer, freqBuffer, size, true);
+ }
+ long[] restoredDocs = new long[size];
+ long[] restoredFreqs = new long[size];
+ try (IndexInput in = dir.openInput("test", IOContext.DEFAULT)) {
+ PostingsUtil.readVIntBlock(in, restoredDocs, restoredFreqs, size, true, true);
+ }
+ assertEquals(delta, restoredDocs[0]);
+ }
+ }
+}