Skip to content

Apache Lucene open-source search software

Introduce TestLucene90DocValuesFormatVariableSkipInterval for testing docvalues skipper index (#13550)

Ignacio Vera392ddc154f22
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesConsumer.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesConsumer.java
index 63e4891960c..021eacacd3b 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesConsumer.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesConsumer.java
@@ -19,7 +19,6 @@ package org.apache.lucene.codecs.lucene90;
import static org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat.DIRECT_MONOTONIC_BLOCK_SHIFT;
import static org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat.NUMERIC_BLOCK_SHIFT;
import static org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat.NUMERIC_BLOCK_SIZE;
-import static org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat.SKIP_INDEX_INTERVAL_SIZE;
import java.io.IOException;
import java.util.Arrays;
@@ -63,10 +62,12 @@ final class Lucene90DocValuesConsumer extends DocValuesConsumer {
IndexOutput data, meta;
final int maxDoc;
private byte[] termsDictBuffer;
+ private final int skipIndexIntervalSize;
/** expert: Creates a new writer */
public Lucene90DocValuesConsumer(
SegmentWriteState state,
+ int skipIndexIntervalSize,
String dataCodec,
String dataExtension,
String metaCodec,
@@ -96,6 +97,7 @@ final class Lucene90DocValuesConsumer extends DocValuesConsumer {
state.segmentInfo.getId(),
state.segmentSuffix);
maxDoc = state.segmentInfo.maxDoc();
+ this.skipIndexIntervalSize = skipIndexIntervalSize;
success = true;
} finally {
if (!success) {
@@ -239,7 +241,7 @@ final class Lucene90DocValuesConsumer extends DocValuesConsumer {
for (int i = 0, end = values.docValueCount(); i < end; ++i) {
accumulator.accumulate(values.nextValue());
}
- if (++counter == SKIP_INDEX_INTERVAL_SIZE) {
+ if (++counter == skipIndexIntervalSize) {
globalMaxValue = Math.max(globalMaxValue, accumulator.maxValue);
globalMinValue = Math.min(globalMinValue, accumulator.minValue);
globalDocCount += accumulator.docCount;
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesFormat.java
index 847a5341584..0ae0a7ac2aa 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesFormat.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesFormat.java
@@ -138,15 +138,27 @@ import org.apache.lucene.util.packed.DirectWriter;
*/
public final class Lucene90DocValuesFormat extends DocValuesFormat {
+ private final int skipIndexIntervalSize;
+
/** Default constructor. */
public Lucene90DocValuesFormat() {
+ this(DEFAULT_SKIP_INDEX_INTERVAL_SIZE);
+ }
+
+ /** Doc values fields format with specified skipIndexIntervalSize. */
+ public Lucene90DocValuesFormat(int skipIndexIntervalSize) {
super("Lucene90");
+ if (skipIndexIntervalSize < 2) {
+ throw new IllegalArgumentException(
+ "skipIndexIntervalSize must be > 1, got [" + skipIndexIntervalSize + "]");
+ }
+ this.skipIndexIntervalSize = skipIndexIntervalSize;
}
@Override
public DocValuesConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
return new Lucene90DocValuesConsumer(
- state, DATA_CODEC, DATA_EXTENSION, META_CODEC, META_EXTENSION);
+ state, skipIndexIntervalSize, DATA_CODEC, DATA_EXTENSION, META_CODEC, META_EXTENSION);
}
@Override
@@ -182,6 +194,5 @@ public final class Lucene90DocValuesFormat extends DocValuesFormat {
static final int TERMS_DICT_REVERSE_INDEX_SIZE = 1 << TERMS_DICT_REVERSE_INDEX_SHIFT;
static final int TERMS_DICT_REVERSE_INDEX_MASK = TERMS_DICT_REVERSE_INDEX_SIZE - 1;
- static final int SKIP_INDEX_INTERVAL_SHIFT = 12;
- static final int SKIP_INDEX_INTERVAL_SIZE = 1 << SKIP_INDEX_INTERVAL_SHIFT;
+ private static final int DEFAULT_SKIP_INDEX_INTERVAL_SIZE = 4096;
}
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90DocValuesFormatVariableSkipInterval.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90DocValuesFormatVariableSkipInterval.java
new file mode 100644
index 00000000000..94204fb2a8d
--- /dev/null
+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90DocValuesFormatVariableSkipInterval.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.codecs.lucene90;
+
+import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.tests.index.BaseDocValuesFormatTestCase;
+import org.apache.lucene.tests.util.TestUtil;
+
+/** Tests Lucene90DocValuesFormat */
+public class TestLucene90DocValuesFormatVariableSkipInterval extends BaseDocValuesFormatTestCase {
+
+ @Override
+ protected Codec getCodec() {
+ return TestUtil.alwaysDocValuesFormat(new Lucene90DocValuesFormat(random().nextInt(2, 1024)));
+ }
+
+ public void testSkipIndexIntervalSize() {
+ IllegalArgumentException ex =
+ expectThrows(
+ IllegalArgumentException.class,
+ () -> new Lucene90DocValuesFormat(random().nextInt(Integer.MIN_VALUE, 2)));
+ assertTrue(ex.getMessage().contains("skipIndexIntervalSize must be > 1"));
+ }
+}