Apache Lucene open-source search software
Introduce TestLucene90DocValuesFormatVariableSkipInterval for testing docvalues skipper index (#13550)
Ignacio Vera392ddc154f22
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesConsumer.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesConsumer.javaindex 63e4891960c..021eacacd3b 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesConsumer.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesConsumer.java@@ -19,7 +19,6 @@ package org.apache.lucene.codecs.lucene90;import static org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat.DIRECT_MONOTONIC_BLOCK_SHIFT;import static org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat.NUMERIC_BLOCK_SHIFT;import static org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat.NUMERIC_BLOCK_SIZE;-import static org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat.SKIP_INDEX_INTERVAL_SIZE;import java.io.IOException;import java.util.Arrays;@@ -63,10 +62,12 @@ final class Lucene90DocValuesConsumer extends DocValuesConsumer {IndexOutput data, meta;final int maxDoc;private byte[] termsDictBuffer;+ private final int skipIndexIntervalSize;/** expert: Creates a new writer */public Lucene90DocValuesConsumer(SegmentWriteState state,+ int skipIndexIntervalSize,String dataCodec,String dataExtension,String metaCodec,@@ -96,6 +97,7 @@ final class Lucene90DocValuesConsumer extends DocValuesConsumer {state.segmentInfo.getId(),state.segmentSuffix);maxDoc = state.segmentInfo.maxDoc();+ this.skipIndexIntervalSize = skipIndexIntervalSize;success = true;} finally {if (!success) {@@ -239,7 +241,7 @@ final class Lucene90DocValuesConsumer extends DocValuesConsumer {for (int i = 0, end = values.docValueCount(); i < end; ++i) {accumulator.accumulate(values.nextValue());}- if (++counter == SKIP_INDEX_INTERVAL_SIZE) {+ if (++counter == skipIndexIntervalSize) {globalMaxValue = Math.max(globalMaxValue, accumulator.maxValue);globalMinValue = Math.min(globalMinValue, accumulator.minValue);globalDocCount += accumulator.docCount;diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesFormat.javaindex 847a5341584..0ae0a7ac2aa 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesFormat.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesFormat.java@@ -138,15 +138,27 @@ import org.apache.lucene.util.packed.DirectWriter;*/public final class Lucene90DocValuesFormat extends DocValuesFormat {+ private final int skipIndexIntervalSize;+/** Default constructor. */public Lucene90DocValuesFormat() {+ this(DEFAULT_SKIP_INDEX_INTERVAL_SIZE);+ }++ /** Doc values fields format with specified skipIndexIntervalSize. */+ public Lucene90DocValuesFormat(int skipIndexIntervalSize) {super("Lucene90");+ if (skipIndexIntervalSize < 2) {+ throw new IllegalArgumentException(+ "skipIndexIntervalSize must be > 1, got [" + skipIndexIntervalSize + "]");+ }+ this.skipIndexIntervalSize = skipIndexIntervalSize;}@Overridepublic DocValuesConsumer fieldsConsumer(SegmentWriteState state) throws IOException {return new Lucene90DocValuesConsumer(- state, DATA_CODEC, DATA_EXTENSION, META_CODEC, META_EXTENSION);+ state, skipIndexIntervalSize, DATA_CODEC, DATA_EXTENSION, META_CODEC, META_EXTENSION);}@Override@@ -182,6 +194,5 @@ public final class Lucene90DocValuesFormat extends DocValuesFormat {static final int TERMS_DICT_REVERSE_INDEX_SIZE = 1 << TERMS_DICT_REVERSE_INDEX_SHIFT;static final int TERMS_DICT_REVERSE_INDEX_MASK = TERMS_DICT_REVERSE_INDEX_SIZE - 1;- static final int SKIP_INDEX_INTERVAL_SHIFT = 12;- static final int SKIP_INDEX_INTERVAL_SIZE = 1 << SKIP_INDEX_INTERVAL_SHIFT;+ private static final int DEFAULT_SKIP_INDEX_INTERVAL_SIZE = 4096;}diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90DocValuesFormatVariableSkipInterval.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90DocValuesFormatVariableSkipInterval.javanew file mode 100644index 00000000000..94204fb2a8d--- /dev/null+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90DocValuesFormatVariableSkipInterval.java@@ -0,0 +1,38 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.codecs.lucene90;++import org.apache.lucene.codecs.Codec;+import org.apache.lucene.tests.index.BaseDocValuesFormatTestCase;+import org.apache.lucene.tests.util.TestUtil;++/** Tests Lucene90DocValuesFormat */+public class TestLucene90DocValuesFormatVariableSkipInterval extends BaseDocValuesFormatTestCase {++ @Override+ protected Codec getCodec() {+ return TestUtil.alwaysDocValuesFormat(new Lucene90DocValuesFormat(random().nextInt(2, 1024)));+ }++ public void testSkipIndexIntervalSize() {+ IllegalArgumentException ex =+ expectThrows(+ IllegalArgumentException.class,+ () -> new Lucene90DocValuesFormat(random().nextInt(Integer.MIN_VALUE, 2)));+ assertTrue(ex.getMessage().contains("skipIndexIntervalSize must be > 1"));+ }+}