Skip to content

Apache Lucene open-source search software

Align doc value skipper interval boundaries when an interval contains a constant value (#13597)

Ignacio Vera7709f575efec
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index f1c687c8969..6de4246129f 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -135,6 +135,9 @@ New Features
* GITHUB#13563: Add levels to doc values skip index. (Ignacio Vera)
+* GITHUB#13597: Align doc value skipper interval boundaries when an interval contains a constant
+ value. (Ignacio Vera)
+
Improvements
---------------------
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesConsumer.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesConsumer.java
index 3a8f0a2c8a3..c9f2e774233 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesConsumer.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesConsumer.java
@@ -205,6 +205,20 @@ final class Lucene90DocValuesConsumer extends DocValuesConsumer {
docCount = 0;
}
+ boolean isDone(int skipIndexIntervalSize, int valueCount, long nextValue, int nextDoc) {
+ if (docCount < skipIndexIntervalSize) {
+ return false;
+ }
+ // Once we reach the interval size, we will keep accepting documents if
+ // - next doc value is not a multi-value
+ // - current accumulator only contains a single value and next value is the same value
+ // - the accumulator is dense and the next doc keeps the density (no gaps)
+ return valueCount > 1
+ || minValue != maxValue
+ || minValue != nextValue
+ || docCount != nextDoc - minDocID;
+ }
+
void accumulate(long value) {
minValue = Math.min(minValue, value);
maxValue = Math.max(maxValue, value);
@@ -245,15 +259,9 @@ final class Lucene90DocValuesConsumer extends DocValuesConsumer {
SkipAccumulator accumulator = null;
final int maxAccumulators = 1 << (SKIP_INDEX_LEVEL_SHIFT * (SKIP_INDEX_MAX_LEVEL - 1));
for (int doc = values.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = values.nextDoc()) {
- if (accumulator == null) {
- accumulator = new SkipAccumulator(doc);
- accumulators.add(accumulator);
- }
- accumulator.nextDoc(doc);
- for (int i = 0, end = values.docValueCount(); i < end; ++i) {
- accumulator.accumulate(values.nextValue());
- }
- if (accumulator.docCount == skipIndexIntervalSize) {
+ final long firstValue = values.nextValue();
+ if (accumulator != null
+ && accumulator.isDone(skipIndexIntervalSize, values.docValueCount(), firstValue, doc)) {
globalMaxValue = Math.max(globalMaxValue, accumulator.maxValue);
globalMinValue = Math.min(globalMinValue, accumulator.minValue);
globalDocCount += accumulator.docCount;
@@ -264,15 +272,22 @@ final class Lucene90DocValuesConsumer extends DocValuesConsumer {
accumulators.clear();
}
}
+ if (accumulator == null) {
+ accumulator = new SkipAccumulator(doc);
+ accumulators.add(accumulator);
+ }
+ accumulator.nextDoc(doc);
+ accumulator.accumulate(firstValue);
+ for (int i = 1, end = values.docValueCount(); i < end; ++i) {
+ accumulator.accumulate(values.nextValue());
+ }
}
if (accumulators.isEmpty() == false) {
- if (accumulator != null) {
- globalMaxValue = Math.max(globalMaxValue, accumulator.maxValue);
- globalMinValue = Math.min(globalMinValue, accumulator.minValue);
- globalDocCount += accumulator.docCount;
- maxDocId = accumulator.maxDocID;
- }
+ globalMaxValue = Math.max(globalMaxValue, accumulator.maxValue);
+ globalMinValue = Math.min(globalMinValue, accumulator.minValue);
+ globalDocCount += accumulator.docCount;
+ maxDocId = accumulator.maxDocID;
writeLevels(accumulators);
}
meta.writeLong(start); // record the start in meta
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90DocValuesFormatVariableSkipInterval.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90DocValuesFormatVariableSkipInterval.java
index e452313fbd7..b5fc29440b7 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90DocValuesFormatVariableSkipInterval.java
+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90DocValuesFormatVariableSkipInterval.java
@@ -16,11 +16,24 @@
*/
package org.apache.lucene.codecs.lucene90;
+import java.io.IOException;
+import java.util.Arrays;
import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.NumericDocValuesField;
+import org.apache.lucene.document.SortedNumericDocValuesField;
+import org.apache.lucene.index.DocValuesSkipper;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.Sort;
+import org.apache.lucene.search.SortField;
+import org.apache.lucene.store.Directory;
import org.apache.lucene.tests.index.BaseDocValuesFormatTestCase;
+import org.apache.lucene.tests.index.RandomIndexWriter;
import org.apache.lucene.tests.util.TestUtil;
-/** Tests Lucene90DocValuesFormat */
+/** Tests Lucene90DocValuesFormat with custom skipper interval size */
public class TestLucene90DocValuesFormatVariableSkipInterval extends BaseDocValuesFormatTestCase {
@Override
@@ -36,4 +49,159 @@ public class TestLucene90DocValuesFormatVariableSkipInterval extends BaseDocValu
() -> new Lucene90DocValuesFormat(random().nextInt(Integer.MIN_VALUE, 2)));
assertTrue(ex.getMessage().contains("skipIndexIntervalSize must be > 1"));
}
+
+ public void testSkipperAllEqualValue() throws IOException {
+ final IndexWriterConfig config = new IndexWriterConfig().setCodec(getCodec());
+ try (Directory directory = newDirectory();
+ RandomIndexWriter writer = new RandomIndexWriter(random(), directory, config)) {
+ final int numDocs = atLeast(100);
+ for (int i = 0; i < numDocs; i++) {
+ final Document doc = new Document();
+ doc.add(NumericDocValuesField.indexedField("dv", 0L));
+ writer.addDocument(doc);
+ }
+ writer.forceMerge(1);
+ try (IndexReader reader = writer.getReader()) {
+ assertEquals(1, reader.leaves().size());
+ final DocValuesSkipper skipper = reader.leaves().get(0).reader().getDocValuesSkipper("dv");
+ assertNotNull(skipper);
+ skipper.advance(0);
+ assertEquals(0L, skipper.minValue(0));
+ assertEquals(0L, skipper.maxValue(0));
+ assertEquals(numDocs, skipper.docCount(0));
+ skipper.advance(skipper.maxDocID(0) + 1);
+ assertEquals(DocIdSetIterator.NO_MORE_DOCS, skipper.minDocID(0));
+ }
+ }
+ }
+
+ // break on different value
+ public void testSkipperFewValuesSorted() throws IOException {
+ final IndexWriterConfig config = new IndexWriterConfig().setCodec(getCodec());
+ boolean reverse = random().nextBoolean();
+ config.setIndexSort(new Sort(new SortField("dv", SortField.Type.LONG, reverse)));
+ try (Directory directory = newDirectory();
+ RandomIndexWriter writer = new RandomIndexWriter(random(), directory, config)) {
+ final int intervals = random().nextInt(2, 10);
+ final int[] numDocs = new int[intervals];
+ for (int i = 0; i < intervals; i++) {
+ numDocs[i] = random().nextInt(10) + 16;
+ for (int j = 0; j < numDocs[i]; j++) {
+ final Document doc = new Document();
+ doc.add(NumericDocValuesField.indexedField("dv", i));
+ writer.addDocument(doc);
+ }
+ }
+ writer.forceMerge(1);
+ try (IndexReader reader = writer.getReader()) {
+ assertEquals(1, reader.leaves().size());
+ final DocValuesSkipper skipper = reader.leaves().get(0).reader().getDocValuesSkipper("dv");
+ assertNotNull(skipper);
+ assertEquals(Arrays.stream(numDocs).sum(), skipper.docCount());
+ skipper.advance(0);
+ if (reverse) {
+ for (int i = intervals - 1; i >= 0; i--) {
+ assertEquals(i, skipper.minValue(0));
+ assertEquals(i, skipper.maxValue(0));
+ assertEquals(numDocs[i], skipper.docCount(0));
+ skipper.advance(skipper.maxDocID(0) + 1);
+ }
+ } else {
+ for (int i = 0; i < intervals; i++) {
+ assertEquals(i, skipper.minValue(0));
+ assertEquals(i, skipper.maxValue(0));
+ assertEquals(numDocs[i], skipper.docCount(0));
+ skipper.advance(skipper.maxDocID(0) + 1);
+ }
+ }
+ assertEquals(DocIdSetIterator.NO_MORE_DOCS, skipper.minDocID(0));
+ }
+ }
+ }
+
+ // break on empty doc values
+ public void testSkipperAllEqualValueWithGaps() throws IOException {
+ final IndexWriterConfig config = new IndexWriterConfig().setCodec(getCodec());
+ config.setIndexSort(new Sort(new SortField("sort", SortField.Type.LONG, false)));
+ try (Directory directory = newDirectory();
+ RandomIndexWriter writer = new RandomIndexWriter(random(), directory, config)) {
+ final int gaps = random().nextInt(2, 10);
+ final int[] numDocs = new int[gaps];
+ long totaldocs = 0;
+ for (int i = 0; i < gaps; i++) {
+ numDocs[i] = random().nextInt(10) + 16;
+ for (int j = 0; j < numDocs[i]; j++) {
+ final Document doc = new Document();
+ doc.add(new NumericDocValuesField("sort", totaldocs++));
+ doc.add(SortedNumericDocValuesField.indexedField("dv", 0L));
+ writer.addDocument(doc);
+ }
+ // add doc with empty "dv"
+ final Document doc = new Document();
+ doc.add(new NumericDocValuesField("sort", totaldocs++));
+ writer.addDocument(doc);
+ }
+ writer.forceMerge(1);
+ try (IndexReader reader = writer.getReader()) {
+ assertEquals(1, reader.leaves().size());
+ final DocValuesSkipper skipper = reader.leaves().get(0).reader().getDocValuesSkipper("dv");
+ assertNotNull(skipper);
+ assertEquals(Arrays.stream(numDocs).sum(), skipper.docCount());
+ skipper.advance(0);
+ for (int i = 0; i < gaps; i++) {
+ assertEquals(0L, skipper.minValue(0));
+ assertEquals(0L, skipper.maxValue(0));
+ assertEquals(numDocs[i], skipper.docCount(0));
+ skipper.advance(skipper.maxDocID(0) + 1);
+ }
+ assertEquals(DocIdSetIterator.NO_MORE_DOCS, skipper.minDocID(0));
+ }
+ }
+ }
+
+ // break on multi-values
+ public void testSkipperAllEqualValueWithMultiValues() throws IOException {
+ final IndexWriterConfig config = new IndexWriterConfig().setCodec(getCodec());
+ config.setIndexSort(new Sort(new SortField("sort", SortField.Type.LONG, false)));
+ try (Directory directory = newDirectory();
+ RandomIndexWriter writer = new RandomIndexWriter(random(), directory, config)) {
+ final int gaps = random().nextInt(2, 10);
+ final int[] numDocs = new int[gaps];
+ long totaldocs = 0;
+ for (int i = 0; i < gaps; i++) {
+ int docs = random().nextInt(10) + 16;
+ numDocs[i] += docs;
+ for (int j = 0; j < docs; j++) {
+ final Document doc = new Document();
+ doc.add(new NumericDocValuesField("sort", totaldocs++));
+ doc.add(SortedNumericDocValuesField.indexedField("dv", 0L));
+ writer.addDocument(doc);
+ }
+ if (i != gaps - 1) {
+ // add doc with mutivalues
+ final Document doc = new Document();
+ doc.add(new NumericDocValuesField("sort", totaldocs++));
+ doc.add(SortedNumericDocValuesField.indexedField("dv", 0L));
+ doc.add(SortedNumericDocValuesField.indexedField("dv", 0L));
+ writer.addDocument(doc);
+ numDocs[i + 1] = 1;
+ }
+ }
+ writer.forceMerge(1);
+ try (IndexReader reader = writer.getReader()) {
+ assertEquals(1, reader.leaves().size());
+ final DocValuesSkipper skipper = reader.leaves().get(0).reader().getDocValuesSkipper("dv");
+ assertNotNull(skipper);
+ assertEquals(Arrays.stream(numDocs).sum(), skipper.docCount());
+ skipper.advance(0);
+ for (int i = 0; i < gaps; i++) {
+ assertEquals(0L, skipper.minValue(0));
+ assertEquals(0L, skipper.maxValue(0));
+ assertEquals(numDocs[i], skipper.docCount(0));
+ skipper.advance(skipper.maxDocID(0) + 1);
+ }
+ assertEquals(DocIdSetIterator.NO_MORE_DOCS, skipper.minDocID(0));
+ }
+ }
+ }
}