Apache Lucene open-source search software
Add missing test for Lucene912PostingsFormat.
Adrien Grand82210189ee84
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene912/TestForDeltaUtil.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene912/TestForDeltaUtil.javanew file mode 100644index 00000000000..3c201ce6835--- /dev/null+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene912/TestForDeltaUtil.java@@ -0,0 +1,89 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.codecs.lucene912;++import com.carrotsearch.randomizedtesting.generators.RandomNumbers;+import java.io.IOException;+import java.util.Arrays;+import org.apache.lucene.store.ByteBuffersDirectory;+import org.apache.lucene.store.Directory;+import org.apache.lucene.store.IOContext;+import org.apache.lucene.store.IndexInput;+import org.apache.lucene.store.IndexOutput;+import org.apache.lucene.tests.util.LuceneTestCase;+import org.apache.lucene.tests.util.TestUtil;+import org.apache.lucene.util.packed.PackedInts;++public class TestForDeltaUtil extends LuceneTestCase {++ public void testEncodeDecode() throws IOException {+ final int iterations = RandomNumbers.randomIntBetween(random(), 50, 1000);+ final int[] values = new int[iterations * ForUtil.BLOCK_SIZE];++ for (int i = 0; i < iterations; ++i) {+ final int bpv = TestUtil.nextInt(random(), 1, 31 - 7);+ for (int j = 0; j < ForUtil.BLOCK_SIZE; ++j) {+ values[i * ForUtil.BLOCK_SIZE + j] =+ RandomNumbers.randomIntBetween(random(), 1, (int) PackedInts.maxValue(bpv));+ }+ }++ final Directory d = new ByteBuffersDirectory();+ final long endPointer;++ {+ // encode+ IndexOutput out = d.createOutput("test.bin", IOContext.DEFAULT);+ final ForDeltaUtil forDeltaUtil = new ForDeltaUtil(new ForUtil());++ for (int i = 0; i < iterations; ++i) {+ long[] source = new long[ForUtil.BLOCK_SIZE];+ for (int j = 0; j < ForUtil.BLOCK_SIZE; ++j) {+ source[j] = values[i * ForUtil.BLOCK_SIZE + j];+ }+ forDeltaUtil.encodeDeltas(source, out);+ }+ endPointer = out.getFilePointer();+ out.close();+ }++ {+ // decode+ IndexInput in = d.openInput("test.bin", IOContext.READONCE);+ final ForDeltaUtil forDeltaUtil = new ForDeltaUtil(new ForUtil());+ for (int i = 0; i < iterations; ++i) {+ long base = 0;+ final long[] restored = new long[ForUtil.BLOCK_SIZE];+ forDeltaUtil.decodeAndPrefixSum(in, base, restored);+ final long[] expected = new long[ForUtil.BLOCK_SIZE];+ for (int j = 0; j < ForUtil.BLOCK_SIZE; ++j) {+ expected[j] = values[i * ForUtil.BLOCK_SIZE + j];+ if (j > 0) {+ expected[j] += expected[j - 1];+ } else {+ expected[j] += base;+ }+ }+ assertArrayEquals(Arrays.toString(restored), expected, restored);+ }+ assertEquals(endPointer, in.getFilePointer());+ in.close();+ }++ d.close();+ }+}diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene912/TestForUtil.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene912/TestForUtil.javanew file mode 100644index 00000000000..114a9d0415c--- /dev/null+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene912/TestForUtil.java@@ -0,0 +1,94 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.codecs.lucene912;++import com.carrotsearch.randomizedtesting.generators.RandomNumbers;+import java.io.IOException;+import java.util.Arrays;+import org.apache.lucene.store.ByteBuffersDirectory;+import org.apache.lucene.store.Directory;+import org.apache.lucene.store.IOContext;+import org.apache.lucene.store.IndexInput;+import org.apache.lucene.store.IndexOutput;+import org.apache.lucene.tests.util.LuceneTestCase;+import org.apache.lucene.tests.util.TestUtil;+import org.apache.lucene.util.ArrayUtil;+import org.apache.lucene.util.packed.PackedInts;++public class TestForUtil extends LuceneTestCase {++ public void testEncodeDecode() throws IOException {+ final int iterations = RandomNumbers.randomIntBetween(random(), 50, 1000);+ final int[] values = new int[iterations * ForUtil.BLOCK_SIZE];++ for (int i = 0; i < iterations; ++i) {+ final int bpv = TestUtil.nextInt(random(), 1, 31);+ for (int j = 0; j < ForUtil.BLOCK_SIZE; ++j) {+ values[i * ForUtil.BLOCK_SIZE + j] =+ RandomNumbers.randomIntBetween(random(), 0, (int) PackedInts.maxValue(bpv));+ }+ }++ final Directory d = new ByteBuffersDirectory();+ final long endPointer;++ {+ // encode+ IndexOutput out = d.createOutput("test.bin", IOContext.DEFAULT);+ final ForUtil forUtil = new ForUtil();++ for (int i = 0; i < iterations; ++i) {+ long[] source = new long[ForUtil.BLOCK_SIZE];+ long or = 0;+ for (int j = 0; j < ForUtil.BLOCK_SIZE; ++j) {+ source[j] = values[i * ForUtil.BLOCK_SIZE + j];+ or |= source[j];+ }+ final int bpv = PackedInts.bitsRequired(or);+ out.writeByte((byte) bpv);+ forUtil.encode(source, bpv, out);+ }+ endPointer = out.getFilePointer();+ out.close();+ }++ {+ // decode+ IndexInput in = d.openInput("test.bin", IOContext.READONCE);+ final ForUtil forUtil = new ForUtil();+ for (int i = 0; i < iterations; ++i) {+ final int bitsPerValue = in.readByte();+ final long currentFilePointer = in.getFilePointer();+ final long[] restored = new long[ForUtil.BLOCK_SIZE];+ forUtil.decode(bitsPerValue, in, restored);+ int[] ints = new int[ForUtil.BLOCK_SIZE];+ for (int j = 0; j < ForUtil.BLOCK_SIZE; ++j) {+ ints[j] = Math.toIntExact(restored[j]);+ }+ assertArrayEquals(+ Arrays.toString(ints),+ ArrayUtil.copyOfSubArray(values, i * ForUtil.BLOCK_SIZE, (i + 1) * ForUtil.BLOCK_SIZE),+ ints);+ assertEquals(forUtil.numBytes(bitsPerValue), in.getFilePointer() - currentFilePointer);+ }+ assertEquals(endPointer, in.getFilePointer());+ in.close();+ }++ d.close();+ }+}diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene912/TestPForUtil.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene912/TestPForUtil.javanew file mode 100644index 00000000000..08fec7a3a33--- /dev/null+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene912/TestPForUtil.java@@ -0,0 +1,104 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.codecs.lucene912;++import com.carrotsearch.randomizedtesting.generators.RandomNumbers;+import java.io.IOException;+import java.util.Arrays;+import org.apache.lucene.store.ByteBuffersDirectory;+import org.apache.lucene.store.Directory;+import org.apache.lucene.store.IOContext;+import org.apache.lucene.store.IndexInput;+import org.apache.lucene.store.IndexOutput;+import org.apache.lucene.tests.util.LuceneTestCase;+import org.apache.lucene.tests.util.TestUtil;+import org.apache.lucene.util.ArrayUtil;+import org.apache.lucene.util.packed.PackedInts;++public class TestPForUtil extends LuceneTestCase {++ public void testEncodeDecode() throws IOException {+ final int iterations = RandomNumbers.randomIntBetween(random(), 50, 1000);+ final int[] values = createTestData(iterations, 31);++ final Directory d = new ByteBuffersDirectory();+ final long endPointer = encodeTestData(iterations, values, d);++ IndexInput in = d.openInput("test.bin", IOContext.READONCE);+ final PForUtil pforUtil = new PForUtil(new ForUtil());+ for (int i = 0; i < iterations; ++i) {+ if (random().nextInt(5) == 0) {+ pforUtil.skip(in);+ continue;+ }+ final long[] restored = new long[ForUtil.BLOCK_SIZE];+ pforUtil.decode(in, restored);+ int[] ints = new int[ForUtil.BLOCK_SIZE];+ for (int j = 0; j < ForUtil.BLOCK_SIZE; ++j) {+ ints[j] = Math.toIntExact(restored[j]);+ }+ assertArrayEquals(+ Arrays.toString(ints),+ ArrayUtil.copyOfSubArray(values, i * ForUtil.BLOCK_SIZE, (i + 1) * ForUtil.BLOCK_SIZE),+ ints);+ }+ assertEquals(endPointer, in.getFilePointer());+ in.close();++ d.close();+ }++ private int[] createTestData(int iterations, int maxBpv) {+ final int[] values = new int[iterations * ForUtil.BLOCK_SIZE];++ for (int i = 0; i < iterations; ++i) {+ final int bpv = TestUtil.nextInt(random(), 0, maxBpv);+ for (int j = 0; j < ForUtil.BLOCK_SIZE; ++j) {+ values[i * ForUtil.BLOCK_SIZE + j] =+ RandomNumbers.randomIntBetween(random(), 0, (int) PackedInts.maxValue(bpv));+ if (random().nextInt(100) == 0) {+ final int exceptionBpv;+ if (random().nextInt(10) == 0) {+ exceptionBpv = Math.min(bpv + TestUtil.nextInt(random(), 9, 16), maxBpv);+ } else {+ exceptionBpv = Math.min(bpv + TestUtil.nextInt(random(), 1, 8), maxBpv);+ }+ values[i * ForUtil.BLOCK_SIZE + j] |= random().nextInt(1 << (exceptionBpv - bpv)) << bpv;+ }+ }+ }++ return values;+ }++ private long encodeTestData(int iterations, int[] values, Directory d) throws IOException {+ IndexOutput out = d.createOutput("test.bin", IOContext.DEFAULT);+ final PForUtil pforUtil = new PForUtil(new ForUtil());++ for (int i = 0; i < iterations; ++i) {+ long[] source = new long[ForUtil.BLOCK_SIZE];+ for (int j = 0; j < ForUtil.BLOCK_SIZE; ++j) {+ source[j] = values[i * ForUtil.BLOCK_SIZE + j];+ }+ pforUtil.encode(source, out);+ }+ final long endPointer = out.getFilePointer();+ out.close();++ return endPointer;+ }+}diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene912/TestPostingsUtil.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene912/TestPostingsUtil.javanew file mode 100644index 00000000000..870f6e94e98--- /dev/null+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene912/TestPostingsUtil.java@@ -0,0 +1,49 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.codecs.lucene912;++import java.io.IOException;+import org.apache.lucene.store.Directory;+import org.apache.lucene.store.IOContext;+import org.apache.lucene.store.IndexInput;+import org.apache.lucene.store.IndexOutput;+import org.apache.lucene.tests.util.LuceneTestCase;++public class TestPostingsUtil extends LuceneTestCase {++ // checks for bug described in https://github.com/apache/lucene/issues/13373+ public void testIntegerOverflow() throws IOException {+ final int size = random().nextInt(1, ForUtil.BLOCK_SIZE);+ final long[] docDeltaBuffer = new long[size];+ final long[] freqBuffer = new long[size];++ final int delta = 1 << 30;+ docDeltaBuffer[0] = delta;+ try (Directory dir = newDirectory()) {+ try (IndexOutput out = dir.createOutput("test", IOContext.DEFAULT)) {+ // In old implementation, this would cause integer overflow exception.+ PostingsUtil.writeVIntBlock(out, docDeltaBuffer, freqBuffer, size, true);+ }+ long[] restoredDocs = new long[size];+ long[] restoredFreqs = new long[size];+ try (IndexInput in = dir.openInput("test", IOContext.DEFAULT)) {+ PostingsUtil.readVIntBlock(in, restoredDocs, restoredFreqs, size, true, true);+ }+ assertEquals(delta, restoredDocs[0]);+ }+ }+}