Apache Lucene open-source search software
GITHUB#13449: Sparse index, optional skip list on top of doc values (#13449)
Ignacio Vera048770205cb0
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex 8dd40060450..2cc172bd316 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -129,6 +129,10 @@ New Features* GITHUB#13233: Add RomanianNormalizationFilter (Trey Jones, Robert Muir)+* GITHUB#13449: Sparse index: optional skip list on top of doc values which is exposed via the+ DocValuesSkipper abstraction. A new flag is added to FieldType.java that configures whether+ to create a "skip index" for doc values. (Ignacio Vera)+Improvements---------------------diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene60/Lucene60FieldInfosFormat.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene60/Lucene60FieldInfosFormat.javaindex a3e09db8ae9..9f7f0b83e41 100644--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene60/Lucene60FieldInfosFormat.java+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene60/Lucene60FieldInfosFormat.java@@ -209,6 +209,7 @@ public final class Lucene60FieldInfosFormat extends FieldInfosFormat {storePayloads,indexOptions,docValuesType,+ false,dvGen,attributes,pointDataDimensionCount,diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene80/Lucene80DocValuesProducer.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene80/Lucene80DocValuesProducer.javaindex 40e955dfeb4..c5754e5d1e5 100644--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene80/Lucene80DocValuesProducer.java+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene80/Lucene80DocValuesProducer.java@@ -28,6 +28,7 @@ import org.apache.lucene.index.BaseTermsEnum;import org.apache.lucene.index.BinaryDocValues;import org.apache.lucene.index.CorruptIndexException;import org.apache.lucene.index.DocValues;+import org.apache.lucene.index.DocValuesSkipper;import org.apache.lucene.index.FieldInfo;import org.apache.lucene.index.FieldInfos;import org.apache.lucene.index.ImpactsEnum;@@ -1677,6 +1678,11 @@ final class Lucene80DocValuesProducer extends DocValuesProducer {}}+ @Override+ public DocValuesSkipper getSkipper(FieldInfo field) {+ return null;+ }+@Overridepublic void checkIntegrity() throws IOException {CodecUtil.checksumEntireFile(data);diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90FieldInfosFormat.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90FieldInfosFormat.javaindex 22eb4558e3b..c4e3dca2873 100644--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90FieldInfosFormat.java+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90FieldInfosFormat.java@@ -186,6 +186,7 @@ public final class Lucene90FieldInfosFormat extends FieldInfosFormat {storePayloads,indexOptions,docValuesType,+ false,dvGen,attributes,pointDataDimensionCount,diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesReader.java b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesReader.javaindex 916b696020d..f58ff0873ca 100644--- a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesReader.java+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesReader.java@@ -16,13 +16,16 @@*/package org.apache.lucene.codecs.simpletext;+import static org.apache.lucene.codecs.simpletext.SimpleTextDocValuesWriter.DOCCOUNT;import static org.apache.lucene.codecs.simpletext.SimpleTextDocValuesWriter.END;import static org.apache.lucene.codecs.simpletext.SimpleTextDocValuesWriter.FIELD;import static org.apache.lucene.codecs.simpletext.SimpleTextDocValuesWriter.LENGTH;import static org.apache.lucene.codecs.simpletext.SimpleTextDocValuesWriter.MAXLENGTH;+import static org.apache.lucene.codecs.simpletext.SimpleTextDocValuesWriter.MAXVALUE;import static org.apache.lucene.codecs.simpletext.SimpleTextDocValuesWriter.MINVALUE;import static org.apache.lucene.codecs.simpletext.SimpleTextDocValuesWriter.NUMVALUES;import static org.apache.lucene.codecs.simpletext.SimpleTextDocValuesWriter.ORDPATTERN;+import static org.apache.lucene.codecs.simpletext.SimpleTextDocValuesWriter.ORIGIN;import static org.apache.lucene.codecs.simpletext.SimpleTextDocValuesWriter.PATTERN;import static org.apache.lucene.codecs.simpletext.SimpleTextDocValuesWriter.TYPE;@@ -40,6 +43,7 @@ import java.util.function.IntFunction;import org.apache.lucene.codecs.DocValuesProducer;import org.apache.lucene.index.BinaryDocValues;import org.apache.lucene.index.CorruptIndexException;+import org.apache.lucene.index.DocValuesSkipper;import org.apache.lucene.index.DocValuesType;import org.apache.lucene.index.FieldInfo;import org.apache.lucene.index.IndexFileNames;@@ -59,12 +63,15 @@ import org.apache.lucene.util.StringHelper;class SimpleTextDocValuesReader extends DocValuesProducer {static class OneField {+ int docCount;long dataStartFilePointer;String pattern;String ordPattern;int maxLength;boolean fixedLength;+ long origin;long minValue;+ long maxValue;long numValues;}@@ -99,17 +106,34 @@ class SimpleTextDocValuesReader extends DocValuesProducer {DocValuesType dvType = DocValuesType.valueOf(stripPrefix(TYPE));assert dvType != DocValuesType.NONE;- if (dvType == DocValuesType.NUMERIC) {++ if (dvType == DocValuesType.NUMERIC || dvType == DocValuesType.SORTED_NUMERIC) {readLine();assert startsWith(MINVALUE): "got " + scratch.get().utf8ToString() + " field=" + fieldName + " ext=" + ext;field.minValue = Long.parseLong(stripPrefix(MINVALUE));readLine();+ assert startsWith(MAXVALUE)+ : "got " + scratch.get().utf8ToString() + " field=" + fieldName + " ext=" + ext;+ field.maxValue = Long.parseLong(stripPrefix(MAXVALUE));+ }++ readLine();+ assert startsWith(DOCCOUNT)+ : "got " + scratch.get().utf8ToString() + " field=" + fieldName + " ext=" + ext;+ field.docCount = Integer.parseInt(stripPrefix(DOCCOUNT));++ if (dvType == DocValuesType.NUMERIC) {+ readLine();+ assert startsWith(ORIGIN)+ : "got " + scratch.get().utf8ToString() + " field=" + fieldName + " ext=" + ext;+ field.origin = Long.parseLong(stripPrefix(ORIGIN));+ readLine();assert startsWith(PATTERN);field.pattern = stripPrefix(PATTERN);field.dataStartFilePointer = data.getFilePointer();data.seek(data.getFilePointer() + (1 + field.pattern.length() + 2) * (long) maxDoc);- } else if (dvType == DocValuesType.BINARY) {+ } else if (dvType == DocValuesType.BINARY || dvType == DocValuesType.SORTED_NUMERIC) {readLine();assert startsWith(MAXLENGTH);field.maxLength = Integer.parseInt(stripPrefix(MAXLENGTH));@@ -225,7 +249,7 @@ class SimpleTextDocValuesReader extends DocValuesProducer {throw new CorruptIndexException("failed to parse BigDecimal value", in, pe);}SimpleTextUtil.readLine(in, scratch); // read the line telling us if it's real or not- return BigInteger.valueOf(field.minValue).add(bd.toBigIntegerExact()).longValue();+ return BigInteger.valueOf(field.origin).add(bd.toBigIntegerExact()).longValue();} catch (IOException ioe) {throw new RuntimeException(ioe);}@@ -824,4 +848,82 @@ class SimpleTextDocValuesReader extends DocValuesProducer {}}}++ @Override+ public DocValuesSkipper getSkipper(FieldInfo fieldInfo) {+ final boolean numeric =+ fieldInfo.getDocValuesType() == DocValuesType.NUMERIC+ || fieldInfo.getDocValuesType() == DocValuesType.SORTED_NUMERIC;+ final OneField field = fields.get(fieldInfo.name);++ // SegmentCoreReaders already verifies this field is+ // valid:+ assert field != null;++ return new DocValuesSkipper() {+ int doc = -1;++ @Override+ public int numLevels() {+ return 1;+ }++ @Override+ public long minValue(int level) {+ return minValue();+ }++ @Override+ public long maxValue(int level) {+ return maxValue();+ }++ @Override+ public int docCount(int level) {+ return docCount();+ }++ @Override+ public long minValue() {+ return numeric ? field.minValue : 0;+ }++ @Override+ public long maxValue() {+ return numeric ? field.maxValue : field.numValues - 1;+ }++ @Override+ public int docCount() {+ return field.docCount;+ }++ @Override+ public int minDocID(int level) {+ if (doc == -1) {+ return -1;+ } else if (doc >= maxDoc || field.docCount == 0) {+ return DocIdSetIterator.NO_MORE_DOCS;+ } else {+ return 0;+ }+ }++ @Override+ public int maxDocID(int level) {+ if (doc == -1) {+ return -1;+ } else if (doc >= maxDoc || field.docCount == 0) {+ return DocIdSetIterator.NO_MORE_DOCS;+ } else {+ return maxDoc;+ }+ }++ @Override+ public void advance(int target) {+ doc = target;+ }+ };+ }}diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesWriter.java b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesWriter.javaindex a49041d2730..760895624da 100644--- a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesWriter.java+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesWriter.java@@ -46,8 +46,13 @@ class SimpleTextDocValuesWriter extends DocValuesConsumer {static final BytesRef END = new BytesRef("END");static final BytesRef FIELD = new BytesRef("field ");static final BytesRef TYPE = new BytesRef(" type ");+ static final BytesRef DOCCOUNT = new BytesRef(" doccount ");// used for numerics- static final BytesRef MINVALUE = new BytesRef(" minvalue ");+ static final BytesRef ORIGIN = new BytesRef(" origin "); // for deltas++ static final BytesRef MINVALUE = new BytesRef(" minalue ");+ static final BytesRef MAXVALUE = new BytesRef(" maxvalue ");+static final BytesRef PATTERN = new BytesRef(" pattern ");// used for bytesstatic final BytesRef LENGTH = new BytesRef("length ");@@ -97,13 +102,27 @@ class SimpleTextDocValuesWriter extends DocValuesConsumer {maxValue = Math.max(maxValue, v);numValues++;}++ // write absolute min and max for skipper+ SimpleTextUtil.write(data, MINVALUE);+ SimpleTextUtil.write(data, Long.toString(minValue), scratch);+ SimpleTextUtil.writeNewline(data);++ SimpleTextUtil.write(data, MAXVALUE);+ SimpleTextUtil.write(data, Long.toString(maxValue), scratch);+ SimpleTextUtil.writeNewline(data);++ SimpleTextUtil.write(data, DOCCOUNT);+ SimpleTextUtil.write(data, Integer.toString(numValues), scratch);+ SimpleTextUtil.writeNewline(data);+if (numValues != numDocs) {minValue = Math.min(minValue, 0);maxValue = Math.max(maxValue, 0);}// write our minimum value to the .dat, all entries are deltas from that- SimpleTextUtil.write(data, MINVALUE);+ SimpleTextUtil.write(data, ORIGIN);SimpleTextUtil.write(data, Long.toString(minValue), scratch);SimpleTextUtil.writeNewline(data);@@ -161,6 +180,7 @@ class SimpleTextDocValuesWriter extends DocValuesConsumer {public void addBinaryField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException {assert fieldSeen(field.name);assert field.getDocValuesType() == DocValuesType.BINARY;+ writeFieldEntry(field, DocValuesType.BINARY);doAddBinaryField(field, valuesProducer);}@@ -168,10 +188,15 @@ class SimpleTextDocValuesWriter extends DocValuesConsumer {throws IOException {int maxLength = 0;BinaryDocValues values = valuesProducer.getBinary(field);+ int docCount = 0;for (int doc = values.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = values.nextDoc()) {+ ++docCount;maxLength = Math.max(maxLength, values.binaryValue().toString().length());}- writeFieldEntry(field, DocValuesType.BINARY);++ SimpleTextUtil.write(data, DOCCOUNT);+ SimpleTextUtil.write(data, Integer.toString(docCount), scratch);+ SimpleTextUtil.writeNewline(data);// write maxLengthSimpleTextUtil.write(data, MAXLENGTH);@@ -232,6 +257,15 @@ class SimpleTextDocValuesWriter extends DocValuesConsumer {assert field.getDocValuesType() == DocValuesType.SORTED;writeFieldEntry(field, DocValuesType.SORTED);+ int docCount = 0;+ SortedDocValues values = valuesProducer.getSorted(field);+ for (int doc = values.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = values.nextDoc()) {+ ++docCount;+ }+ SimpleTextUtil.write(data, DOCCOUNT);+ SimpleTextUtil.write(data, Integer.toString(docCount), scratch);+ SimpleTextUtil.writeNewline(data);+int valueCount = 0;int maxLength = -1;TermsEnum terms = valuesProducer.getSorted(field).termsEnum();@@ -301,7 +335,7 @@ class SimpleTextDocValuesWriter extends DocValuesConsumer {assert valuesSeen == valueCount;- SortedDocValues values = valuesProducer.getSorted(field);+ values = valuesProducer.getSorted(field);for (int i = 0; i < numDocs; ++i) {if (values.docID() < i) {values.nextDoc();@@ -321,6 +355,28 @@ class SimpleTextDocValuesWriter extends DocValuesConsumer {throws IOException {assert fieldSeen(field.name);assert field.getDocValuesType() == DocValuesType.SORTED_NUMERIC;+ writeFieldEntry(field, DocValuesType.SORTED_NUMERIC);++ long minValue = Long.MAX_VALUE;+ long maxValue = Long.MIN_VALUE;+ SortedNumericDocValues values = valuesProducer.getSortedNumeric(field);+ for (int doc = values.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = values.nextDoc()) {+ for (int i = 0; i < values.docValueCount(); ++i) {+ long v = values.nextValue();+ minValue = Math.min(minValue, v);+ maxValue = Math.max(maxValue, v);+ }+ }++ // write absolute min and max for skipper+ SimpleTextUtil.write(data, MINVALUE);+ SimpleTextUtil.write(data, Long.toString(minValue), scratch);+ SimpleTextUtil.writeNewline(data);++ SimpleTextUtil.write(data, MAXVALUE);+ SimpleTextUtil.write(data, Long.toString(maxValue), scratch);+ SimpleTextUtil.writeNewline(data);+doAddBinaryField(field,new EmptyDocValuesProducer() {@@ -395,6 +451,15 @@ class SimpleTextDocValuesWriter extends DocValuesConsumer {assert field.getDocValuesType() == DocValuesType.SORTED_SET;writeFieldEntry(field, DocValuesType.SORTED_SET);+ int docCount = 0;+ SortedSetDocValues values = valuesProducer.getSortedSet(field);+ for (int doc = values.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = values.nextDoc()) {+ ++docCount;+ }+ SimpleTextUtil.write(data, DOCCOUNT);+ SimpleTextUtil.write(data, Integer.toString(docCount), scratch);+ SimpleTextUtil.writeNewline(data);+long valueCount = 0;int maxLength = 0;TermsEnum terms = valuesProducer.getSortedSet(field).termsEnum();@@ -430,7 +495,7 @@ class SimpleTextDocValuesWriter extends DocValuesConsumer {// lengthint maxOrdListLength = 0;StringBuilder sb2 = new StringBuilder();- SortedSetDocValues values = valuesProducer.getSortedSet(field);+ values = valuesProducer.getSortedSet(field);for (int doc = values.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = values.nextDoc()) {sb2.setLength(0);for (int i = 0; i < values.docValueCount(); i++) {diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosFormat.java b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosFormat.javaindex 21cfe9b613f..655938ac67a 100644--- a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosFormat.java+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosFormat.java@@ -60,6 +60,7 @@ public class SimpleTextFieldInfosFormat extends FieldInfosFormat {static final BytesRef PAYLOADS = new BytesRef(" payloads ");static final BytesRef NORMS = new BytesRef(" norms ");static final BytesRef DOCVALUES = new BytesRef(" doc values ");+ static final BytesRef DOCVALUES_SKIP_INDEX = new BytesRef(" doc values skip index");static final BytesRef DOCVALUES_GEN = new BytesRef(" doc values gen ");static final BytesRef INDEXOPTIONS = new BytesRef(" index options ");static final BytesRef NUM_ATTS = new BytesRef(" attributes ");@@ -122,6 +123,11 @@ public class SimpleTextFieldInfosFormat extends FieldInfosFormat {String dvType = readString(DOCVALUES.length, scratch);final DocValuesType docValuesType = docValuesType(dvType);+ SimpleTextUtil.readLine(input, scratch);+ assert StringHelper.startsWith(scratch.get(), DOCVALUES_SKIP_INDEX);+ boolean docValueSkipper =+ Boolean.parseBoolean(readString(DOCVALUES_SKIP_INDEX.length, scratch));+SimpleTextUtil.readLine(input, scratch);assert StringHelper.startsWith(scratch.get(), DOCVALUES_GEN);final long dvGen = Long.parseLong(readString(DOCVALUES_GEN.length, scratch));@@ -184,6 +190,7 @@ public class SimpleTextFieldInfosFormat extends FieldInfosFormat {storePayloads,indexOptions,docValuesType,+ docValueSkipper,dvGen,Collections.unmodifiableMap(atts),dimensionalCount,@@ -276,6 +283,10 @@ public class SimpleTextFieldInfosFormat extends FieldInfosFormat {SimpleTextUtil.write(out, getDocValuesType(fi.getDocValuesType()), scratch);SimpleTextUtil.writeNewline(out);+ SimpleTextUtil.write(out, DOCVALUES_SKIP_INDEX);+ SimpleTextUtil.write(out, Boolean.toString(fi.hasDocValuesSkipIndex()), scratch);+ SimpleTextUtil.writeNewline(out);+SimpleTextUtil.write(out, DOCVALUES_GEN);SimpleTextUtil.write(out, Long.toString(fi.getDocValuesGen()), scratch);SimpleTextUtil.writeNewline(out);diff --git a/lucene/codecs/src/test/org/apache/lucene/codecs/simpletext/TestSimpleTextDocValuesFormat.java b/lucene/codecs/src/test/org/apache/lucene/codecs/simpletext/TestSimpleTextDocValuesFormat.javaindex 5c1504679ec..dd3cdf7d768 100644--- a/lucene/codecs/src/test/org/apache/lucene/codecs/simpletext/TestSimpleTextDocValuesFormat.java+++ b/lucene/codecs/src/test/org/apache/lucene/codecs/simpletext/TestSimpleTextDocValuesFormat.java@@ -37,6 +37,12 @@ import org.apache.lucene.util.BytesRef;public class TestSimpleTextDocValuesFormat extends BaseDocValuesFormatTestCase {private final Codec codec = new SimpleTextCodec();+ @Override+ protected boolean skipperHasAccurateDocBounds() {+ // This format always returns minDocID = 0 and maxDocID = maxDoc - 1+ return false;+ }+@Overrideprotected Codec getCodec() {return codec;diff --git a/lucene/codecs/src/test/org/apache/lucene/codecs/uniformsplit/TestBlockWriter.java b/lucene/codecs/src/test/org/apache/lucene/codecs/uniformsplit/TestBlockWriter.javaindex 2ee6b8fd2d2..24f14448178 100644--- a/lucene/codecs/src/test/org/apache/lucene/codecs/uniformsplit/TestBlockWriter.java+++ b/lucene/codecs/src/test/org/apache/lucene/codecs/uniformsplit/TestBlockWriter.java@@ -111,6 +111,7 @@ public class TestBlockWriter extends LuceneTestCase {true,IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS,DocValuesType.NONE,+ false,-1,Collections.emptyMap(),0,diff --git a/lucene/codecs/src/test/org/apache/lucene/codecs/uniformsplit/sharedterms/TestSTBlockReader.java b/lucene/codecs/src/test/org/apache/lucene/codecs/uniformsplit/sharedterms/TestSTBlockReader.javaindex bf2b0133240..7a6524d77cd 100644--- a/lucene/codecs/src/test/org/apache/lucene/codecs/uniformsplit/sharedterms/TestSTBlockReader.java+++ b/lucene/codecs/src/test/org/apache/lucene/codecs/uniformsplit/sharedterms/TestSTBlockReader.java@@ -198,6 +198,7 @@ public class TestSTBlockReader extends LuceneTestCase {true,IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS,DocValuesType.NONE,+ false,-1,Collections.emptyMap(),0,diff --git a/lucene/core/src/java/org/apache/lucene/codecs/DocValuesProducer.java b/lucene/core/src/java/org/apache/lucene/codecs/DocValuesProducer.javaindex 2731bccd067..2c90448a39c 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/DocValuesProducer.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/DocValuesProducer.java@@ -19,6 +19,7 @@ package org.apache.lucene.codecs;import java.io.Closeable;import java.io.IOException;import org.apache.lucene.index.BinaryDocValues;+import org.apache.lucene.index.DocValuesSkipper;import org.apache.lucene.index.DocValuesType;import org.apache.lucene.index.FieldInfo;import org.apache.lucene.index.NumericDocValues;@@ -73,6 +74,13 @@ public abstract class DocValuesProducer implements Closeable {*/public abstract SortedSetDocValues getSortedSet(FieldInfo field) throws IOException;+ /**+ * Returns a {@link DocValuesSkipper} for this field. The returned instance need not be+ * thread-safe: it will only be used by a single thread. The return value is undefined if {@link+ * FieldInfo#hasDocValuesSkipIndex()} doesn't return {@code true}.+ */+ public abstract DocValuesSkipper getSkipper(FieldInfo field) throws IOException;+/*** Checks consistency of this producer*diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesConsumer.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesConsumer.javaindex d64dce985c0..63e4891960c 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesConsumer.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesConsumer.java@@ -19,6 +19,7 @@ package org.apache.lucene.codecs.lucene90;import static org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat.DIRECT_MONOTONIC_BLOCK_SHIFT;import static org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat.NUMERIC_BLOCK_SHIFT;import static org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat.NUMERIC_BLOCK_SIZE;+import static org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat.SKIP_INDEX_INTERVAL_SIZE;import java.io.IOException;import java.util.Arrays;@@ -43,6 +44,7 @@ import org.apache.lucene.search.SortedSetSelector;import org.apache.lucene.store.ByteArrayDataOutput;import org.apache.lucene.store.ByteBuffersDataOutput;import org.apache.lucene.store.ByteBuffersIndexOutput;+import org.apache.lucene.store.DataOutput;import org.apache.lucene.store.IndexOutput;import org.apache.lucene.util.ArrayUtil;import org.apache.lucene.util.BytesRef;@@ -129,16 +131,17 @@ final class Lucene90DocValuesConsumer extends DocValuesConsumer {throws IOException {meta.writeInt(field.number);meta.writeByte(Lucene90DocValuesFormat.NUMERIC);-- writeValues(- field,+ DocValuesProducer producer =new EmptyDocValuesProducer() {@Overridepublic SortedNumericDocValues getSortedNumeric(FieldInfo field) throws IOException {return DocValues.singleton(valuesProducer.getNumeric(field));}- },- false);+ };+ if (field.hasDocValuesSkipIndex()) {+ writeSkipIndex(field, producer);+ }+ writeValues(field, producer, false);}private static class MinMaxTracker {@@ -183,6 +186,84 @@ final class Lucene90DocValuesConsumer extends DocValuesConsumer {}}+ private static class SkipAccumulator {+ int minDocID;+ int maxDocID;+ int docCount;+ long minValue;+ long maxValue;++ SkipAccumulator(int docID) {+ minDocID = docID;+ minValue = Long.MAX_VALUE;+ maxValue = Long.MIN_VALUE;+ docCount = 0;+ }++ void accumulate(long value) {+ minValue = Math.min(minValue, value);+ maxValue = Math.max(maxValue, value);+ }++ void nextDoc(int docID) {+ maxDocID = docID;+ ++docCount;+ }++ void writeTo(DataOutput output) throws IOException {+ output.writeInt(maxDocID);+ output.writeInt(minDocID);+ output.writeLong(maxValue);+ output.writeLong(minValue);+ output.writeInt(docCount);+ }+ }++ private void writeSkipIndex(FieldInfo field, DocValuesProducer valuesProducer)+ throws IOException {+ assert field.hasDocValuesSkipIndex();+ // TODO: This disk compression once we introduce levels+ long start = data.getFilePointer();+ SortedNumericDocValues values = valuesProducer.getSortedNumeric(field);+ long globalMaxValue = Long.MIN_VALUE;+ long globalMinValue = Long.MAX_VALUE;+ int globalDocCount = 0;+ int maxDocId = -1;+ SkipAccumulator accumulator = null;+ int counter = 0;+ for (int doc = values.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = values.nextDoc()) {+ if (counter == 0) {+ accumulator = new SkipAccumulator(doc);+ }+ accumulator.nextDoc(doc);+ for (int i = 0, end = values.docValueCount(); i < end; ++i) {+ accumulator.accumulate(values.nextValue());+ }+ if (++counter == SKIP_INDEX_INTERVAL_SIZE) {+ globalMaxValue = Math.max(globalMaxValue, accumulator.maxValue);+ globalMinValue = Math.min(globalMinValue, accumulator.minValue);+ globalDocCount += accumulator.docCount;+ maxDocId = accumulator.maxDocID;+ accumulator.writeTo(data);+ counter = 0;+ }+ }++ if (counter > 0) {+ globalMaxValue = Math.max(globalMaxValue, accumulator.maxValue);+ globalMinValue = Math.min(globalMinValue, accumulator.minValue);+ globalDocCount += accumulator.docCount;+ maxDocId = accumulator.maxDocID;+ accumulator.writeTo(data);+ }+ meta.writeLong(start); // record the start in meta+ meta.writeLong(data.getFilePointer() - start); // record the length+ meta.writeLong(globalMaxValue);+ meta.writeLong(globalMinValue);+ meta.writeInt(globalDocCount);+ meta.writeInt(maxDocId);+ }+private long[] writeValues(FieldInfo field, DocValuesProducer valuesProducer, boolean ords)throws IOException {SortedNumericDocValues values = valuesProducer.getSortedNumeric(field);@@ -489,13 +570,12 @@ final class Lucene90DocValuesConsumer extends DocValuesConsumer {public void addSortedField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException {meta.writeInt(field.number);meta.writeByte(Lucene90DocValuesFormat.SORTED);- doAddSortedField(field, valuesProducer);+ doAddSortedField(field, valuesProducer, false);}- private void doAddSortedField(FieldInfo field, DocValuesProducer valuesProducer)- throws IOException {- writeValues(- field,+ private void doAddSortedField(+ FieldInfo field, DocValuesProducer valuesProducer, boolean addTypeByte) throws IOException {+ DocValuesProducer producer =new EmptyDocValuesProducer() {@Overridepublic SortedNumericDocValues getSortedNumeric(FieldInfo field) throws IOException {@@ -534,8 +614,14 @@ final class Lucene90DocValuesConsumer extends DocValuesConsumer {};return DocValues.singleton(sortedOrds);}- },- true);+ };+ if (field.hasDocValuesSkipIndex()) {+ writeSkipIndex(field, producer);+ }+ if (addTypeByte) {+ meta.writeByte((byte) 0); // multiValued (0 = singleValued)+ }+ writeValues(field, producer, true);addTermsDict(DocValues.singleton(valuesProducer.getSorted(field)));}@@ -702,6 +788,12 @@ final class Lucene90DocValuesConsumer extends DocValuesConsumer {private void doAddSortedNumericField(FieldInfo field, DocValuesProducer valuesProducer, boolean ords) throws IOException {+ if (field.hasDocValuesSkipIndex()) {+ writeSkipIndex(field, valuesProducer);+ }+ if (ords) {+ meta.writeByte((byte) 1); // multiValued (1 = multiValued)+ }long[] stats = writeValues(field, valuesProducer, ords);int numDocsWithField = Math.toIntExact(stats[0]);long numValues = stats[1];@@ -753,7 +845,7 @@ final class Lucene90DocValuesConsumer extends DocValuesConsumer {meta.writeByte(Lucene90DocValuesFormat.SORTED_SET);if (isSingleValued(valuesProducer.getSortedSet(field))) {- meta.writeByte((byte) 0); // multiValued (0 = singleValued)+doAddSortedField(field,new EmptyDocValuesProducer() {@@ -762,10 +854,10 @@ final class Lucene90DocValuesConsumer extends DocValuesConsumer {return SortedSetSelector.wrap(valuesProducer.getSortedSet(field), SortedSetSelector.Type.MIN);}- });+ },+ true);return;}- meta.writeByte((byte) 1); // multiValued (1 = multiValued)doAddSortedNumericField(field,diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesFormat.javaindex 7a59baf90f3..847a5341584 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesFormat.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesFormat.java@@ -181,4 +181,7 @@ public final class Lucene90DocValuesFormat extends DocValuesFormat {static final int TERMS_DICT_REVERSE_INDEX_SHIFT = 10;static final int TERMS_DICT_REVERSE_INDEX_SIZE = 1 << TERMS_DICT_REVERSE_INDEX_SHIFT;static final int TERMS_DICT_REVERSE_INDEX_MASK = TERMS_DICT_REVERSE_INDEX_SIZE - 1;++ static final int SKIP_INDEX_INTERVAL_SHIFT = 12;+ static final int SKIP_INDEX_INTERVAL_SIZE = 1 << SKIP_INDEX_INTERVAL_SHIFT;}diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesProducer.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesProducer.javaindex 4078a2dc977..ad467fbc87f 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesProducer.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesProducer.java@@ -27,6 +27,7 @@ import org.apache.lucene.index.BaseTermsEnum;import org.apache.lucene.index.BinaryDocValues;import org.apache.lucene.index.CorruptIndexException;import org.apache.lucene.index.DocValues;+import org.apache.lucene.index.DocValuesSkipper;import org.apache.lucene.index.FieldInfo;import org.apache.lucene.index.FieldInfos;import org.apache.lucene.index.ImpactsEnum;@@ -39,6 +40,7 @@ import org.apache.lucene.index.SortedNumericDocValues;import org.apache.lucene.index.SortedSetDocValues;import org.apache.lucene.index.TermsEnum;import org.apache.lucene.index.TermsEnum.SeekStatus;+import org.apache.lucene.search.DocIdSetIterator;import org.apache.lucene.store.ByteArrayDataInput;import org.apache.lucene.store.ChecksumIndexInput;import org.apache.lucene.store.DataInput;@@ -59,6 +61,7 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {private final Map<String, SortedEntry> sorted;private final Map<String, SortedSetEntry> sortedSets;private final Map<String, SortedNumericEntry> sortedNumerics;+ private final Map<String, DocValuesSkipperEntry> skippers;private final IndexInput data;private final int maxDoc;private int version = -1;@@ -80,6 +83,7 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {sorted = new HashMap<>();sortedSets = new HashMap<>();sortedNumerics = new HashMap<>();+ skippers = new HashMap<>();merging = false;// read in the entries from the metadata file.@@ -147,6 +151,7 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {Map<String, SortedEntry> sorted,Map<String, SortedSetEntry> sortedSets,Map<String, SortedNumericEntry> sortedNumerics,+ Map<String, DocValuesSkipperEntry> skippers,IndexInput data,int maxDoc,int version,@@ -156,6 +161,7 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {this.sorted = sorted;this.sortedSets = sortedSets;this.sortedNumerics = sortedNumerics;+ this.skippers = skippers;this.data = data.clone();this.maxDoc = maxDoc;this.version = version;@@ -165,7 +171,16 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {@Overridepublic DocValuesProducer getMergeInstance() {return new Lucene90DocValuesProducer(- numerics, binaries, sorted, sortedSets, sortedNumerics, data, maxDoc, version, true);+ numerics,+ binaries,+ sorted,+ sortedSets,+ sortedNumerics,+ skippers,+ data,+ maxDoc,+ version,+ true);}private void readFields(IndexInput meta, FieldInfos infos) throws IOException {@@ -175,6 +190,9 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {throw new CorruptIndexException("Invalid field number: " + fieldNumber, meta);}byte type = meta.readByte();+ if (info.hasDocValuesSkipIndex()) {+ skippers.put(info.name, readDocValueSkipperMeta(meta));+ }if (type == Lucene90DocValuesFormat.NUMERIC) {numerics.put(info.name, readNumeric(meta));} else if (type == Lucene90DocValuesFormat.BINARY) {@@ -197,6 +215,17 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {return entry;}+ private DocValuesSkipperEntry readDocValueSkipperMeta(IndexInput meta) throws IOException {+ long offset = meta.readLong();+ long length = meta.readLong();+ long maxValue = meta.readLong();+ long minValue = meta.readLong();+ int docCount = meta.readInt();+ int maxDocID = meta.readInt();++ return new DocValuesSkipperEntry(offset, length, minValue, maxValue, docCount, maxDocID);+ }+private void readNumeric(IndexInput meta, NumericEntry entry) throws IOException {entry.docsWithFieldOffset = meta.readLong();entry.docsWithFieldLength = meta.readLong();@@ -326,6 +355,9 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {data.close();}+ private record DocValuesSkipperEntry(+ long offset, long length, long minValue, long maxValue, int docCount, int maxDocId) {}+private static class NumericEntry {long[] table;int blockShift;@@ -1749,4 +1781,88 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {return mul * values.get(index & mask) + delta;}}++ @Override+ public DocValuesSkipper getSkipper(FieldInfo field) throws IOException {+ final DocValuesSkipperEntry entry = skippers.get(field.name);++ final IndexInput input = data.slice("doc value skipper", entry.offset, entry.length);+ // Prefetch the first page of data. Following pages are expected to get prefetched through+ // read-ahead.+ if (input.length() > 0) {+ input.prefetch(0, 1);+ }+ return new DocValuesSkipper() {+ int minDocID = -1;+ int maxDocID = -1;+ long minValue, maxValue;+ int docCount;++ @Override+ public void advance(int target) throws IOException {+ if (target > entry.maxDocId) {+ minDocID = DocIdSetIterator.NO_MORE_DOCS;+ maxDocID = DocIdSetIterator.NO_MORE_DOCS;+ } else {+ while (true) {+ maxDocID = input.readInt();+ if (maxDocID >= target) {+ minDocID = input.readInt();+ maxValue = input.readLong();+ minValue = input.readLong();+ docCount = input.readInt();+ break;+ } else {+ input.skipBytes(24);+ }+ }+ }+ }++ @Override+ public int numLevels() {+ return 1;+ }++ @Override+ public int minDocID(int level) {+ return minDocID;+ }++ @Override+ public int maxDocID(int level) {+ return maxDocID;+ }++ @Override+ public long minValue(int level) {+ return minValue;+ }++ @Override+ public long maxValue(int level) {+ return maxValue;+ }++ @Override+ public int docCount(int level) {+ return docCount;+ }++ @Override+ public long minValue() {+ return entry.minValue;+ }++ @Override+ public long maxValue() {+ return entry.maxValue;+ }++ @Override+ public int docCount() {+ return entry.docCount;+ }+ };+ }}diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene94/Lucene94FieldInfosFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene94/Lucene94FieldInfosFormat.javaindex 341e28c36f5..481148cbd0d 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene94/Lucene94FieldInfosFormat.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene94/Lucene94FieldInfosFormat.java@@ -163,8 +163,10 @@ public final class Lucene94FieldInfosFormat extends FieldInfosFormat {boolean isSoftDeletesField = (bits & SOFT_DELETES_FIELD) != 0;boolean isParentField =format >= FORMAT_PARENT_FIELD ? (bits & PARENT_FIELD_FIELD) != 0 : false;+ boolean hasDocValuesSkipIndex =+ format >= FORMAT_DOCVALUE_SKIPPER ? (bits & DOCVALUES_SKIPPER) != 0 : false;- if ((bits & 0xE0) != 0) {+ if ((bits & 0xC0) != 0) {throw new CorruptIndexException("unused bits are set \"" + Integer.toBinaryString(bits) + "\"", input);}@@ -173,6 +175,13 @@ public final class Lucene94FieldInfosFormat extends FieldInfosFormat {"parent field bit is set but shouldn't \"" + Integer.toBinaryString(bits) + "\"",input);}+ if (format < FORMAT_DOCVALUE_SKIPPER && (bits & DOCVALUES_SKIPPER) != 0) {+ throw new CorruptIndexException(+ "doc values skipper bit is set but shouldn't \""+ + Integer.toBinaryString(bits)+ + "\"",+ input);+ }final IndexOptions indexOptions = getIndexOptions(input, input.readByte());@@ -208,6 +217,7 @@ public final class Lucene94FieldInfosFormat extends FieldInfosFormat {storePayloads,indexOptions,docValuesType,+ hasDocValuesSkipIndex,dvGen,attributes,pointDataDimensionCount,@@ -394,6 +404,7 @@ public final class Lucene94FieldInfosFormat extends FieldInfosFormat {if (fi.hasPayloads()) bits |= STORE_PAYLOADS;if (fi.isSoftDeletesField()) bits |= SOFT_DELETES_FIELD;if (fi.isParentField()) bits |= PARENT_FIELD_FIELD;+ if (fi.hasDocValuesSkipIndex()) bits |= DOCVALUES_SKIPPER;output.writeByte(bits);output.writeByte(indexOptionsByte(fi.getIndexOptions()));@@ -423,7 +434,8 @@ public final class Lucene94FieldInfosFormat extends FieldInfosFormat {static final int FORMAT_START = 0;// this doesn't actually change the file format but uses up one more bit an existing bit patternstatic final int FORMAT_PARENT_FIELD = 1;- static final int FORMAT_CURRENT = FORMAT_PARENT_FIELD;+ static final int FORMAT_DOCVALUE_SKIPPER = 2;+ static final int FORMAT_CURRENT = FORMAT_DOCVALUE_SKIPPER;// Field flagsstatic final byte STORE_TERMVECTOR = 0x1;@@ -431,4 +443,5 @@ public final class Lucene94FieldInfosFormat extends FieldInfosFormat {static final byte STORE_PAYLOADS = 0x4;static final byte SOFT_DELETES_FIELD = 0x8;static final byte PARENT_FIELD_FIELD = 0x10;+ static final byte DOCVALUES_SKIPPER = 0x20;}diff --git a/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldDocValuesFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldDocValuesFormat.javaindex 0e8352da445..2496278fe7a 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldDocValuesFormat.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldDocValuesFormat.java@@ -28,6 +28,7 @@ import org.apache.lucene.codecs.DocValuesConsumer;import org.apache.lucene.codecs.DocValuesFormat;import org.apache.lucene.codecs.DocValuesProducer;import org.apache.lucene.index.BinaryDocValues;+import org.apache.lucene.index.DocValuesSkipper;import org.apache.lucene.index.DocValuesType;import org.apache.lucene.index.FieldInfo;import org.apache.lucene.index.MergeState;@@ -346,6 +347,12 @@ public abstract class PerFieldDocValuesFormat extends DocValuesFormat {return producer == null ? null : producer.getSortedSet(field);}+ @Override+ public DocValuesSkipper getSkipper(FieldInfo field) throws IOException {+ DocValuesProducer producer = fields.get(field.name);+ return producer == null ? null : producer.getSkipper(field);+ }+@Overridepublic void close() throws IOException {IOUtils.close(formats.values());diff --git a/lucene/core/src/java/org/apache/lucene/document/DocValuesRangeIterator.java b/lucene/core/src/java/org/apache/lucene/document/DocValuesRangeIterator.javanew file mode 100644index 00000000000..fbefe128cca--- /dev/null+++ b/lucene/core/src/java/org/apache/lucene/document/DocValuesRangeIterator.java@@ -0,0 +1,171 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.document;++import java.io.IOException;+import org.apache.lucene.index.DocValuesSkipper;+import org.apache.lucene.search.DocIdSetIterator;+import org.apache.lucene.search.TwoPhaseIterator;++/**+ * Wrapper around a {@link TwoPhaseIterator} for a doc-values range query that speeds things up by+ * taking advantage of a {@link DocValuesSkipper}.+ */+final class DocValuesRangeIterator extends TwoPhaseIterator {++ enum Match {+ /** None of the documents in the range match */+ NO,+ /** Document values need to be checked to verify matches */+ MAYBE,+ /** All documents in the range that have a value match */+ IF_DOC_HAS_VALUE,+ /** All docs in the range match */+ YES;+ }++ private final Approximation approximation;+ private final TwoPhaseIterator innerTwoPhase;++ DocValuesRangeIterator(+ TwoPhaseIterator twoPhase, DocValuesSkipper skipper, long lowerValue, long upperValue) {+ super(new Approximation(twoPhase.approximation(), skipper, lowerValue, upperValue));+ this.approximation = (Approximation) approximation();+ this.innerTwoPhase = twoPhase;+ }++ static class Approximation extends DocIdSetIterator {++ private final DocIdSetIterator innerApproximation;+ private final DocValuesSkipper skipper;+ private final long lowerValue;+ private final long upperValue;++ private int doc = -1;++ // Track a decision for all doc IDs between the current doc ID and upTo inclusive.+ Match match = Match.MAYBE;+ int upTo = -1;++ Approximation(+ DocIdSetIterator innerApproximation,+ DocValuesSkipper skipper,+ long lowerValue,+ long upperValue) {+ this.innerApproximation = innerApproximation;+ this.skipper = skipper;+ this.lowerValue = lowerValue;+ this.upperValue = upperValue;+ }++ @Override+ public int docID() {+ return doc;+ }++ @Override+ public int nextDoc() throws IOException {+ return advance(docID() + 1);+ }++ @Override+ public int advance(int target) throws IOException {+ while (true) {+ if (target > upTo) {+ skipper.advance(target);+ // If target doesn't have a value and is between two blocks, it is possible that advance()+ // moved to a block that doesn't contain `target`.+ target = Math.max(target, skipper.minDocID(0));+ if (target == NO_MORE_DOCS) {+ return doc = NO_MORE_DOCS;+ }+ upTo = skipper.maxDocID(0);+ match = match(0);++ // If we have a YES or NO decision, see if we still have the same decision on a higher+ // level (= on a wider range of doc IDs)+ int nextLevel = 1;+ while (match != Match.MAYBE+ && nextLevel < skipper.numLevels()+ && match == match(nextLevel)) {+ upTo = skipper.maxDocID(nextLevel);+ nextLevel++;+ }+ }+ switch (match) {+ case YES:+ return doc = target;+ case MAYBE:+ case IF_DOC_HAS_VALUE:+ if (target > innerApproximation.docID()) {+ target = innerApproximation.advance(target);+ }+ if (target <= upTo) {+ return doc = target;+ }+ // Otherwise we are breaking the invariant that `doc` must always be <= upTo, so let+ // the loop run one more iteration to advance the skipper.+ break;+ case NO:+ if (upTo == DocIdSetIterator.NO_MORE_DOCS) {+ return doc = NO_MORE_DOCS;+ }+ target = upTo + 1;+ break;+ default:+ throw new AssertionError("Unknown enum constant: " + match);+ }+ }+ }++ @Override+ public long cost() {+ return innerApproximation.cost();+ }++ private Match match(int level) {+ long minValue = skipper.minValue(level);+ long maxValue = skipper.maxValue(level);+ if (minValue > upperValue || maxValue < lowerValue) {+ return Match.NO;+ } else if (minValue >= lowerValue && maxValue <= upperValue) {+ if (skipper.docCount(level) == skipper.maxDocID(level) - skipper.minDocID(level) + 1) {+ return Match.YES;+ } else {+ return Match.IF_DOC_HAS_VALUE;+ }+ } else {+ return Match.MAYBE;+ }+ }+ }++ @Override+ public final boolean matches() throws IOException {+ return switch (approximation.match) {+ case YES -> true;+ case IF_DOC_HAS_VALUE -> true;+ case MAYBE -> innerTwoPhase.matches();+ case NO -> throw new IllegalStateException("Unpositioned approximation");+ };+ }++ @Override+ public float matchCost() {+ return innerTwoPhase.matchCost();+ }+}diff --git a/lucene/core/src/java/org/apache/lucene/document/FieldType.java b/lucene/core/src/java/org/apache/lucene/document/FieldType.javaindex 5b37955cc14..db4b37f6711 100644--- a/lucene/core/src/java/org/apache/lucene/document/FieldType.java+++ b/lucene/core/src/java/org/apache/lucene/document/FieldType.java@@ -22,6 +22,7 @@ import java.util.Objects;import org.apache.lucene.analysis.Analyzer; // javadocsimport org.apache.lucene.index.DocValuesType;import org.apache.lucene.index.IndexOptions;+import org.apache.lucene.index.IndexWriterConfig;import org.apache.lucene.index.IndexableFieldType;import org.apache.lucene.index.PointValues;import org.apache.lucene.index.VectorEncoding;@@ -40,6 +41,7 @@ public class FieldType implements IndexableFieldType {private IndexOptions indexOptions = IndexOptions.NONE;private boolean frozen;private DocValuesType docValuesType = DocValuesType.NONE;+ private boolean docValuesSkipIndex;private int dimensionCount;private int indexDimensionCount;private int dimensionNumBytes;@@ -59,6 +61,7 @@ public class FieldType implements IndexableFieldType {this.omitNorms = ref.omitNorms();this.indexOptions = ref.indexOptions();this.docValuesType = ref.docValuesType();+ this.docValuesSkipIndex = ref.hasDocValuesSkipIndex();this.dimensionCount = ref.pointDimensionCount();this.indexDimensionCount = ref.pointIndexDimensionCount();this.dimensionNumBytes = ref.pointNumBytes();@@ -504,6 +507,22 @@ public class FieldType implements IndexableFieldType {docValuesType = type;}+ @Override+ public boolean hasDocValuesSkipIndex() {+ return docValuesSkipIndex;+ }++ /**+ * Set whether to enable a skip index for doc values on this field. This is typically useful on+ * fields that are part of the {@link IndexWriterConfig#setIndexSort index sort}, or that+ * correlate with fields that are part of the index sort, so that values can be expected to be+ * clustered in the doc ID space.+ */+ public void setDocValuesSkipIndex(boolean docValuesSkipIndex) {+ checkIfFrozen();+ this.docValuesSkipIndex = docValuesSkipIndex;+ }+@Overridepublic int hashCode() {final int prime = 31;@@ -512,6 +531,7 @@ public class FieldType implements IndexableFieldType {result = prime * result + indexDimensionCount;result = prime * result + dimensionNumBytes;result = prime * result + ((docValuesType == null) ? 0 : docValuesType.hashCode());+ result = prime * result + Boolean.hashCode(docValuesSkipIndex);result = prime * result + indexOptions.hashCode();result = prime * result + (omitNorms ? 1231 : 1237);result = prime * result + (storeTermVectorOffsets ? 1231 : 1237);@@ -533,6 +553,7 @@ public class FieldType implements IndexableFieldType {if (indexDimensionCount != other.indexDimensionCount) return false;if (dimensionNumBytes != other.dimensionNumBytes) return false;if (docValuesType != other.docValuesType) return false;+ if (docValuesSkipIndex != other.docValuesSkipIndex) return false;if (indexOptions != other.indexOptions) return false;if (omitNorms != other.omitNorms) return false;if (storeTermVectorOffsets != other.storeTermVectorOffsets) return false;diff --git a/lucene/core/src/java/org/apache/lucene/document/NumericDocValuesField.java b/lucene/core/src/java/org/apache/lucene/document/NumericDocValuesField.javaindex 6c3c3825e33..95ed6eb0711 100644--- a/lucene/core/src/java/org/apache/lucene/document/NumericDocValuesField.java+++ b/lucene/core/src/java/org/apache/lucene/document/NumericDocValuesField.java@@ -35,9 +35,27 @@ public class NumericDocValuesField extends Field {/** Type for numeric DocValues. */public static final FieldType TYPE = new FieldType();+ private static final FieldType INDEXED_TYPE;+static {TYPE.setDocValuesType(DocValuesType.NUMERIC);TYPE.freeze();++ INDEXED_TYPE = new FieldType(TYPE);+ INDEXED_TYPE.setDocValuesSkipIndex(true);+ INDEXED_TYPE.freeze();+ }++ /**+ * Creates a new {@link NumericDocValuesField} with the specified 64-bit long value that also+ * creates a {@link FieldType#hasDocValuesSkipIndex() skip index}.+ *+ * @param name field name+ * @param value 64-bit long value+ * @throws IllegalArgumentException if the field name is null+ */+ public static NumericDocValuesField indexedField(String name, long value) {+ return new NumericDocValuesField(name, value, INDEXED_TYPE);}/**@@ -60,7 +78,11 @@ public class NumericDocValuesField extends Field {* @throws IllegalArgumentException if the field name is null*/public NumericDocValuesField(String name, Long value) {- super(name, TYPE);+ this(name, value, TYPE);+ }++ private NumericDocValuesField(String name, Long value, FieldType fieldType) {+ super(name, fieldType);fieldsData = value;}diff --git a/lucene/core/src/java/org/apache/lucene/document/SortedDocValuesField.java b/lucene/core/src/java/org/apache/lucene/document/SortedDocValuesField.javaindex 61476e87a2b..2ed6956b717 100644--- a/lucene/core/src/java/org/apache/lucene/document/SortedDocValuesField.java+++ b/lucene/core/src/java/org/apache/lucene/document/SortedDocValuesField.java@@ -41,9 +41,27 @@ public class SortedDocValuesField extends Field {/** Type for sorted bytes DocValues */public static final FieldType TYPE = new FieldType();+ private static final FieldType INDEXED_TYPE;+static {TYPE.setDocValuesType(DocValuesType.SORTED);TYPE.freeze();++ INDEXED_TYPE = new FieldType(TYPE);+ INDEXED_TYPE.setDocValuesSkipIndex(true);+ INDEXED_TYPE.freeze();+ }++ /**+ * Creates a new {@link SortedDocValuesField} with the specified 64-bit long value that also+ * creates a {@link FieldType#hasDocValuesSkipIndex() skip index}.+ *+ * @param name field name+ * @param bytes binary content+ * @throws IllegalArgumentException if the field name is null+ */+ public static SortedDocValuesField indexedField(String name, BytesRef bytes) {+ return new SortedDocValuesField(name, bytes, INDEXED_TYPE);}/**@@ -54,7 +72,11 @@ public class SortedDocValuesField extends Field {* @throws IllegalArgumentException if the field name is null*/public SortedDocValuesField(String name, BytesRef bytes) {- super(name, TYPE);+ this(name, bytes, TYPE);+ }++ private SortedDocValuesField(String name, BytesRef bytes, FieldType fieldType) {+ super(name, fieldType);fieldsData = bytes;}diff --git a/lucene/core/src/java/org/apache/lucene/document/SortedNumericDocValuesField.java b/lucene/core/src/java/org/apache/lucene/document/SortedNumericDocValuesField.javaindex 091634cfb40..2d635462a22 100644--- a/lucene/core/src/java/org/apache/lucene/document/SortedNumericDocValuesField.java+++ b/lucene/core/src/java/org/apache/lucene/document/SortedNumericDocValuesField.java@@ -43,9 +43,27 @@ public class SortedNumericDocValuesField extends Field {/** Type for sorted numeric DocValues. */public static final FieldType TYPE = new FieldType();+ private static final FieldType INDEXED_TYPE;+static {TYPE.setDocValuesType(DocValuesType.SORTED_NUMERIC);TYPE.freeze();++ INDEXED_TYPE = new FieldType(TYPE);+ INDEXED_TYPE.setDocValuesSkipIndex(true);+ INDEXED_TYPE.freeze();+ }++ /**+ * Creates a new {@link SortedNumericDocValuesField} with the specified 64-bit long value that+ * also creates a {@link FieldType#hasDocValuesSkipIndex() skip index}.+ *+ * @param name field name+ * @param value 64-bit long value+ * @throws IllegalArgumentException if the field name is null+ */+ public static SortedNumericDocValuesField indexedField(String name, long value) {+ return new SortedNumericDocValuesField(name, value, INDEXED_TYPE);}/**@@ -56,8 +74,12 @@ public class SortedNumericDocValuesField extends Field {* @throws IllegalArgumentException if the field name is null*/public SortedNumericDocValuesField(String name, long value) {- super(name, TYPE);- fieldsData = Long.valueOf(value);+ this(name, Long.valueOf(value), TYPE);+ }++ private SortedNumericDocValuesField(String name, Long value, FieldType fieldType) {+ super(name, fieldType);+ fieldsData = value;}/**diff --git a/lucene/core/src/java/org/apache/lucene/document/SortedNumericDocValuesRangeQuery.java b/lucene/core/src/java/org/apache/lucene/document/SortedNumericDocValuesRangeQuery.javaindex abbbeae571c..31ec0319db8 100644--- a/lucene/core/src/java/org/apache/lucene/document/SortedNumericDocValuesRangeQuery.java+++ b/lucene/core/src/java/org/apache/lucene/document/SortedNumericDocValuesRangeQuery.java@@ -19,6 +19,7 @@ package org.apache.lucene.document;import java.io.IOException;import java.util.Objects;import org.apache.lucene.index.DocValues;+import org.apache.lucene.index.DocValuesSkipper;import org.apache.lucene.index.LeafReaderContext;import org.apache.lucene.index.NumericDocValues;import org.apache.lucene.index.SortedNumericDocValues;@@ -109,9 +110,17 @@ final class SortedNumericDocValuesRangeQuery extends Query {if (context.reader().getFieldInfos().fieldInfo(field) == null) {return null;}++ DocValuesSkipper skipper = context.reader().getDocValuesSkipper(field);+ if (skipper != null) {+ if (skipper.minValue() > upperValue || skipper.maxValue() < lowerValue) {+ return null;+ }+ }+SortedNumericDocValues values = DocValues.getSortedNumeric(context.reader(), field);final NumericDocValues singleton = DocValues.unwrapSingleton(values);- final TwoPhaseIterator iterator;+ TwoPhaseIterator iterator;if (singleton != null) {iterator =new TwoPhaseIterator(singleton) {@@ -149,6 +158,9 @@ final class SortedNumericDocValuesRangeQuery extends Query {}};}+ if (skipper != null) {+ iterator = new DocValuesRangeIterator(iterator, skipper, lowerValue, upperValue);+ }final var scorer = new ConstantScoreScorer(score(), scoreMode, iterator);return new DefaultScorerSupplier(scorer);}diff --git a/lucene/core/src/java/org/apache/lucene/document/SortedSetDocValuesField.java b/lucene/core/src/java/org/apache/lucene/document/SortedSetDocValuesField.javaindex 9bec5459664..74ae5dc8043 100644--- a/lucene/core/src/java/org/apache/lucene/document/SortedSetDocValuesField.java+++ b/lucene/core/src/java/org/apache/lucene/document/SortedSetDocValuesField.java@@ -42,9 +42,27 @@ public class SortedSetDocValuesField extends Field {/** Type for sorted bytes DocValues */public static final FieldType TYPE = new FieldType();+ private static final FieldType INDEXED_TYPE;+static {TYPE.setDocValuesType(DocValuesType.SORTED_SET);TYPE.freeze();++ INDEXED_TYPE = new FieldType(TYPE);+ INDEXED_TYPE.setDocValuesSkipIndex(true);+ INDEXED_TYPE.freeze();+ }++ /**+ * Creates a new {@link SortedSetDocValuesField} with the specified 64-bit long value that also+ * creates a {@link FieldType#hasDocValuesSkipIndex() skip index}.+ *+ * @param name field name+ * @param bytes binary content+ * @throws IllegalArgumentException if the field name is null+ */+ public static SortedSetDocValuesField indexedField(String name, BytesRef bytes) {+ return new SortedSetDocValuesField(name, bytes, INDEXED_TYPE);}/**@@ -55,7 +73,11 @@ public class SortedSetDocValuesField extends Field {* @throws IllegalArgumentException if the field name is null*/public SortedSetDocValuesField(String name, BytesRef bytes) {- super(name, TYPE);+ this(name, bytes, TYPE);+ }++ private SortedSetDocValuesField(String name, BytesRef bytes, FieldType fieldType) {+ super(name, fieldType);fieldsData = bytes;}diff --git a/lucene/core/src/java/org/apache/lucene/document/SortedSetDocValuesRangeQuery.java b/lucene/core/src/java/org/apache/lucene/document/SortedSetDocValuesRangeQuery.javaindex 0c9959b8756..d03daac0689 100644--- a/lucene/core/src/java/org/apache/lucene/document/SortedSetDocValuesRangeQuery.java+++ b/lucene/core/src/java/org/apache/lucene/document/SortedSetDocValuesRangeQuery.java@@ -19,6 +19,7 @@ package org.apache.lucene.document;import java.io.IOException;import java.util.Objects;import org.apache.lucene.index.DocValues;+import org.apache.lucene.index.DocValuesSkipper;import org.apache.lucene.index.LeafReaderContext;import org.apache.lucene.index.SortedDocValues;import org.apache.lucene.index.SortedSetDocValues;@@ -113,6 +114,7 @@ final class SortedSetDocValuesRangeQuery extends Query {if (context.reader().getFieldInfos().fieldInfo(field) == null) {return null;}+ DocValuesSkipper skipper = context.reader().getDocValuesSkipper(field);SortedSetDocValues values = DocValues.getSortedSet(context.reader(), field);// implement ScorerSupplier, since we do some expensive stuff to make a scorer@@ -149,12 +151,15 @@ final class SortedSetDocValuesRangeQuery extends Query {}// no terms matched in this segment- if (minOrd > maxOrd) {+ // no terms matched in this segment+ if (minOrd > maxOrd+ || (skipper != null+ && (minOrd > skipper.maxValue() || maxOrd < skipper.minValue()))) {return new ConstantScoreScorer(score(), scoreMode, DocIdSetIterator.empty());}final SortedDocValues singleton = DocValues.unwrapSingleton(values);- final TwoPhaseIterator iterator;+ TwoPhaseIterator iterator;if (singleton != null) {iterator =new TwoPhaseIterator(singleton) {@@ -192,6 +197,9 @@ final class SortedSetDocValuesRangeQuery extends Query {}};}+ if (skipper != null) {+ iterator = new DocValuesRangeIterator(iterator, skipper, minOrd, maxOrd);+ }return new ConstantScoreScorer(score(), scoreMode, iterator);}diff --git a/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java b/lucene/core/src/java/org/apache/lucene/index/CheckIndex.javaindex f4e3d1248d8..db813d231e6 100644--- a/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java+++ b/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java@@ -365,6 +365,9 @@ public final class CheckIndex implements Closeable {/** Total number of sortedset fields */public long totalSortedSetFields;+ /** Total number of skipping index tested. */+ public long totalSkippingIndex;+/** Exception thrown during doc values test (null on success) */public Throwable error;}@@ -3228,13 +3231,14 @@ public final class CheckIndex implements Closeable {infoStream,String.format(Locale.ROOT,- "OK [%d docvalues fields; %d BINARY; %d NUMERIC; %d SORTED; %d SORTED_NUMERIC; %d SORTED_SET] [took %.3f sec]",+ "OK [%d docvalues fields; %d BINARY; %d NUMERIC; %d SORTED; %d SORTED_NUMERIC; %d SORTED_SET; %d SKIPPING INDEX] [took %.3f sec]",status.totalValueFields,status.totalBinaryFields,status.totalNumericFields,status.totalSortedFields,status.totalSortedNumericFields,status.totalSortedSetFields,+ status.totalSkippingIndex,nsToSec(System.nanoTime() - startNS)));} catch (Throwable e) {if (failFast) {@@ -3254,6 +3258,94 @@ public final class CheckIndex implements Closeable {DocValuesIterator get(FieldInfo fi) throws IOException;}+ private static void checkDocValueSkipper(FieldInfo fi, DocValuesSkipper skipper)+ throws IOException {+ String fieldName = fi.name;+ if (skipper.maxDocID(0) != -1) {+ throw new CheckIndexException(+ "binary dv iterator for field: "+ + fieldName+ + " should start at docID=-1, but got "+ + skipper.maxDocID(0));+ }+ if (skipper.docCount() > 0 && skipper.minValue() > skipper.maxValue()) {+ throw new CheckIndexException(+ "skipper dv iterator for field: "+ + fieldName+ + " reports wrong global value range, got "+ + skipper.minValue()+ + " > "+ + skipper.maxValue());+ }+ int docCount = 0;+ int doc;+ while (true) {+ doc = skipper.maxDocID(0) + 1;+ skipper.advance(doc);+ if (skipper.maxDocID(0) == NO_MORE_DOCS) {+ break;+ }+ int levels = skipper.numLevels();+ for (int level = 0; level < levels; level++) {+ if (skipper.minDocID(level) < doc) {+ throw new CheckIndexException(+ "skipper dv iterator for field: "+ + fieldName+ + " reports wrong minDocID, got "+ + skipper.minDocID(level)+ + " < "+ + doc);+ }+ if (skipper.minDocID(level) > skipper.maxDocID(level)) {+ throw new CheckIndexException(+ "skipper dv iterator for field: "+ + fieldName+ + " reports wrong doc range, got "+ + skipper.minDocID(level)+ + " > "+ + skipper.maxDocID(level));+ }+ if (skipper.minValue() > skipper.minValue(level)) {+ throw new CheckIndexException(+ "skipper dv iterator for field: "+ + fieldName+ + " : global minValue "+ + skipper.minValue()+ + " , got "+ + skipper.minValue(level));+ }+ if (skipper.maxValue() < skipper.maxValue(level)) {+ throw new CheckIndexException(+ "skipper dv iterator for field: "+ + fieldName+ + " : global maxValue "+ + skipper.maxValue()+ + " , got "+ + skipper.maxValue(level));+ }+ if (skipper.minValue(level) > skipper.maxValue(level)) {+ throw new CheckIndexException(+ "skipper dv iterator for field: "+ + fieldName+ + " reports wrong value range, got "+ + skipper.minValue(level)+ + " > "+ + skipper.maxValue(level));+ }+ }+ docCount += skipper.docCount(0);+ }+ if (skipper.docCount() != docCount) {+ throw new CheckIndexException(+ "skipper dv iterator for field: "+ + fieldName+ + " inconsistent docCount, got "+ + skipper.docCount()+ + " != "+ + docCount);+ }+ }+private static void checkDVIterator(FieldInfo fi, DocValuesIteratorSupplier producer)throws IOException {String field = fi.name;@@ -3627,6 +3719,10 @@ public final class CheckIndex implements Closeable {private static void checkDocValues(FieldInfo fi, DocValuesProducer dvReader, DocValuesStatus status) throws Exception {+ if (fi.hasDocValuesSkipIndex()) {+ status.totalSkippingIndex++;+ checkDocValueSkipper(fi, dvReader.getSkipper(fi));+ }switch (fi.getDocValuesType()) {case SORTED:status.totalSortedFields++;diff --git a/lucene/core/src/java/org/apache/lucene/index/CodecReader.java b/lucene/core/src/java/org/apache/lucene/index/CodecReader.javaindex 980f2dd9582..bec27c5176e 100644--- a/lucene/core/src/java/org/apache/lucene/index/CodecReader.java+++ b/lucene/core/src/java/org/apache/lucene/index/CodecReader.java@@ -196,6 +196,16 @@ public abstract class CodecReader extends LeafReader {return getDocValuesReader().getSortedSet(fi);}+ @Override+ public final DocValuesSkipper getDocValuesSkipper(String field) throws IOException {+ ensureOpen();+ FieldInfo fi = getFieldInfos().fieldInfo(field);+ if (fi == null || fi.hasDocValuesSkipIndex() == false) {+ return null;+ }+ return getDocValuesReader().getSkipper(fi);+ }+@Overridepublic final NumericDocValues getNormValues(String field) throws IOException {ensureOpen();diff --git a/lucene/core/src/java/org/apache/lucene/index/DocValuesLeafReader.java b/lucene/core/src/java/org/apache/lucene/index/DocValuesLeafReader.javaindex c238cea9d60..3504c7429a5 100644--- a/lucene/core/src/java/org/apache/lucene/index/DocValuesLeafReader.java+++ b/lucene/core/src/java/org/apache/lucene/index/DocValuesLeafReader.java@@ -108,4 +108,9 @@ abstract class DocValuesLeafReader extends LeafReader {public final CacheHelper getReaderCacheHelper() {throw new UnsupportedOperationException();}++ @Override+ public DocValuesSkipper getDocValuesSkipper(String field) throws IOException {+ throw new UnsupportedOperationException();+ }}diff --git a/lucene/core/src/java/org/apache/lucene/index/DocValuesSkipper.java b/lucene/core/src/java/org/apache/lucene/index/DocValuesSkipper.javanew file mode 100644index 00000000000..15d3c67c34d--- /dev/null+++ b/lucene/core/src/java/org/apache/lucene/index/DocValuesSkipper.java@@ -0,0 +1,101 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.index;++import java.io.IOException;+import org.apache.lucene.search.DocIdSetIterator;++/**+ * Skipper for {@link DocValues}.+ *+ * <p>A skipper has a position that can only be advanced via {@link #advance(int)}. The next advance+ * position must be greater than {@link #maxDocID(int)} at level 0. A skipper's position, along with+ * a {@code level}, determines the interval at which the skipper is currently situated.+ */+public abstract class DocValuesSkipper {++ /**+ * Advance this skipper so that all levels contain the next document on or after {@code target}.+ *+ * <p><b>NOTE</b>: The behavior is undefined if {@code target} is less than or equal to {@code+ * maxDocID(0)}.+ *+ * <p><b>NOTE</b>: {@code minDocID(0)} may return a doc ID that is greater than {@code target} if+ * the target document doesn't have a value.+ */+ public abstract void advance(int target) throws IOException;++ /** Return the number of levels. This number may change when moving to a different interval. */+ public abstract int numLevels();++ /**+ * Return the minimum doc ID of the interval on the given level, inclusive. This returns {@code+ * -1} if {@link #advance(int)} has not been called yet and {@link DocIdSetIterator#NO_MORE_DOCS}+ * if the iterator is exhausted. This method is non-increasing when {@code level} increases. Said+ * otherwise {@code minDocID(level+1) <= minDocId(level)}.+ */+ public abstract int minDocID(int level);++ /**+ * Return the maximum doc ID of the interval on the given level, inclusive. This returns {@code+ * -1} if {@link #advance(int)} has not been called yet and {@link DocIdSetIterator#NO_MORE_DOCS}+ * if the iterator is exhausted. This method is non-decreasing when {@code level} decreases. Said+ * otherwise {@code maxDocID(level+1) >= maxDocId(level)}.+ */+ public abstract int maxDocID(int level);++ /**+ * Return the minimum value of the interval at the given level, inclusive.+ *+ * <p><b>NOTE</b>: It is only guaranteed that values in this interval are greater than or equal+ * the returned value. There is no guarantee that one document actually has this value.+ */+ public abstract long minValue(int level);++ /**+ * Return the maximum value of the interval at the given level, inclusive.+ *+ * <p><b>NOTE</b>: It is only guaranteed that values in this interval are less than or equal the+ * returned value. There is no guarantee that one document actually has this value.+ */+ public abstract long maxValue(int level);++ /**+ * Return the number of documents that have a value in the interval associated with the given+ * level.+ */+ public abstract int docCount(int level);++ /**+ * Return the global minimum value.+ *+ * <p><b>NOTE</b>: It is only guaranteed that values are greater than or equal the returned value.+ * There is no guarantee that one document actually has this value.+ */+ public abstract long minValue();++ /**+ * Return the global maximum value.+ *+ * <p><b>NOTE</b>: It is only guaranteed that values are less than or equal the returned value.+ * There is no guarantee that one document actually has this value.+ */+ public abstract long maxValue();++ /** Return the global number of documents with a value for the field. */+ public abstract int docCount();+}diff --git a/lucene/core/src/java/org/apache/lucene/index/DocValuesType.java b/lucene/core/src/java/org/apache/lucene/index/DocValuesType.javaindex 6a14772481b..f40132907a6 100644--- a/lucene/core/src/java/org/apache/lucene/index/DocValuesType.java+++ b/lucene/core/src/java/org/apache/lucene/index/DocValuesType.java@@ -22,31 +22,37 @@ package org.apache.lucene.index;*/public enum DocValuesType {/** No doc values for this field. */- NONE,+ NONE(false),/** A per-document Number */- NUMERIC,+ NUMERIC(true),/*** A per-document byte[]. Values may be larger than 32766 bytes, but different codecs may enforce* their own limits.*/- BINARY,+ BINARY(false),/*** A pre-sorted byte[]. Fields with this type only store distinct byte values and store an* additional offset pointer per document to dereference the shared byte[]. The stored byte[] is* presorted and allows access via document id, ordinal and by-value. Values must be {@code <=* 32766} bytes.*/- SORTED,+ SORTED(true),/*** A pre-sorted Number[]. Fields with this type store numeric values in sorted order according to* {@link Long#compare(long, long)}.*/- SORTED_NUMERIC,+ SORTED_NUMERIC(true),/*** A pre-sorted Set<byte[]>. Fields with this type only store distinct byte values and store* additional offset pointers per document to dereference the shared byte[]s. The stored byte[] is* presorted and allows access via document id, ordinal and by-value. Values must be {@code <=* 32766} bytes.*/- SORTED_SET,+ SORTED_SET(true);++ final boolean supportsSkipIndex; // pkg-private for use in FieldInfo++ DocValuesType(boolean supportsSkipIndex) {+ this.supportsSkipIndex = supportsSkipIndex;+ }}diff --git a/lucene/core/src/java/org/apache/lucene/index/EmptyDocValuesProducer.java b/lucene/core/src/java/org/apache/lucene/index/EmptyDocValuesProducer.javaindex 89a034e1c41..8e6df43c43a 100644--- a/lucene/core/src/java/org/apache/lucene/index/EmptyDocValuesProducer.java+++ b/lucene/core/src/java/org/apache/lucene/index/EmptyDocValuesProducer.java@@ -51,6 +51,11 @@ public abstract class EmptyDocValuesProducer extends DocValuesProducer {throw new UnsupportedOperationException();}+ @Override+ public DocValuesSkipper getSkipper(FieldInfo field) {+ throw new UnsupportedOperationException();+ }+@Overridepublic void checkIntegrity() {throw new UnsupportedOperationException();diff --git a/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java b/lucene/core/src/java/org/apache/lucene/index/FieldInfo.javaindex 610ea689e3c..862c6b37993 100644--- a/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java+++ b/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java@@ -28,13 +28,16 @@ import java.util.Objects;* threads accessing this object.*/public final class FieldInfo {+/** Field's name */public final String name;/** Internal field number */public final int number;- private DocValuesType docValuesType;+ private DocValuesType docValuesType = DocValuesType.NONE;++ private final boolean docValuesSkipIndex;// True if any document indexed term vectorsprivate boolean storeTermVector;@@ -80,6 +83,7 @@ public final class FieldInfo {boolean storePayloads,IndexOptions indexOptions,DocValuesType docValues,+ boolean hasDocValuesSkipIndex,long dvGen,Map<String, String> attributes,int pointDimensionCount,@@ -95,6 +99,7 @@ public final class FieldInfo {this.docValuesType =Objects.requireNonNull(docValues, "DocValuesType must not be null (field: \"" + name + "\")");+ this.docValuesSkipIndex = hasDocValuesSkipIndex;this.indexOptions =Objects.requireNonNull(indexOptions, "IndexOptions must not be null (field: \"" + name + "\")");@@ -152,6 +157,13 @@ public final class FieldInfo {if (docValuesType == null) {throw new IllegalArgumentException("DocValuesType must not be null (field: '" + name + "')");}+ if (docValuesType.supportsSkipIndex == false && docValuesSkipIndex) {+ throw new IllegalArgumentException(+ "field '"+ + name+ + "' cannot have docValuesSkipIndex set to true with doc values type "+ + docValuesType);+ }if (dvGen != -1 && docValuesType == DocValuesType.NONE) {throw new IllegalArgumentException("field '"@@ -235,6 +247,7 @@ public final class FieldInfo {verifySameStoreTermVectors(fieldName, this.storeTermVector, o.storeTermVector);}verifySameDocValuesType(fieldName, this.docValuesType, o.docValuesType);+ verifySameDocValuesSkipIndex(fieldName, this.docValuesSkipIndex, o.docValuesSkipIndex);verifySamePointsOptions(fieldName,this.pointDimensionCount,@@ -289,6 +302,24 @@ public final class FieldInfo {}}+ /**+ * Verify that the provided docValues type are the same+ *+ * @throws IllegalArgumentException if they are not the same+ */+ static void verifySameDocValuesSkipIndex(+ String fieldName, boolean hasDocValuesSkipIndex1, boolean hasDocValuesSkipIndex2) {+ if (hasDocValuesSkipIndex1 != hasDocValuesSkipIndex2) {+ throw new IllegalArgumentException(+ "cannot change field \""+ + fieldName+ + "\" from docValuesSkipIndex="+ + hasDocValuesSkipIndex1+ + " to inconsistent docValuesSkipIndex="+ + hasDocValuesSkipIndex2);+ }+ }+/*** Verify that the provided store term vectors options are the same*@@ -557,6 +588,11 @@ public final class FieldInfo {return docValuesType;}+ /** Returns true if, and only if, this field has a skip index. */+ public boolean hasDocValuesSkipIndex() {+ return docValuesSkipIndex;+ }+/** Sets the docValues generation of this field. */void setDocValuesGen(long dvGen) {this.dvGen = dvGen;diff --git a/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java b/lucene/core/src/java/org/apache/lucene/index/FieldInfos.javaindex 628cdd12844..fd0338bfe68 100644--- a/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java+++ b/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java@@ -16,6 +16,7 @@*/package org.apache.lucene.index;+import static org.apache.lucene.index.FieldInfo.verifySameDocValuesSkipIndex;import static org.apache.lucene.index.FieldInfo.verifySameDocValuesType;import static org.apache.lucene.index.FieldInfo.verifySameIndexOptions;import static org.apache.lucene.index.FieldInfo.verifySameOmitNorms;@@ -364,6 +365,7 @@ public class FieldInfos implements Iterable<FieldInfo> {IndexOptions indexOptions,IndexOptionsProperties indexOptionsProperties,DocValuesType docValuesType,+ boolean docValuesSkipIndex,FieldDimensions fieldDimensions,FieldVectorProperties fieldVectorProperties) {}@@ -442,6 +444,7 @@ public class FieldInfos implements Iterable<FieldInfo> {? new IndexOptionsProperties(fi.hasVectors(), fi.omitsNorms()): null,fi.getDocValuesType(),+ fi.hasDocValuesSkipIndex(),new FieldDimensions(fi.getPointDimensionCount(),fi.getPointIndexDimensionCount(),@@ -521,6 +524,9 @@ public class FieldInfos implements Iterable<FieldInfo> {DocValuesType currentDVType = fieldProperties.docValuesType;verifySameDocValuesType(fieldName, currentDVType, fi.getDocValuesType());+ boolean currentDocValuesSkipIndex = fieldProperties.docValuesSkipIndex;+ verifySameDocValuesSkipIndex(+ fieldName, currentDocValuesSkipIndex, fi.hasDocValuesSkipIndex());FieldDimensions dims = fieldProperties.fieldDimensions;verifySamePointsOptions(@@ -576,6 +582,7 @@ public class FieldInfos implements Iterable<FieldInfo> {false,IndexOptions.NONE,dvType,+ false,-1,new HashMap<>(),0,@@ -602,6 +609,15 @@ public class FieldInfos implements Iterable<FieldInfo> {+ fieldDvType+ "].");}+ boolean hasDocValuesSkipIndex = fieldProperties.docValuesSkipIndex;+ if (hasDocValuesSkipIndex) {+ throw new IllegalArgumentException(+ "Can't update ["+ + dvType+ + "] doc values; the field ["+ + fieldName+ + "] must be doc values only field, bit it has doc values skip index");+ }FieldDimensions fdimensions = fieldProperties.fieldDimensions;if (fdimensions != null && fdimensions.dimensionCount != 0) {throw new IllegalArgumentException(@@ -660,6 +676,7 @@ public class FieldInfos implements Iterable<FieldInfo> {false,IndexOptions.NONE,dvType,+ false,-1,new HashMap<>(),0,@@ -780,6 +797,7 @@ public class FieldInfos implements Iterable<FieldInfo> {fi.hasPayloads(),fi.getIndexOptions(),fi.getDocValuesType(),+ fi.hasDocValuesSkipIndex(),dvGen,// original attributes is UnmodifiableMapnew HashMap<>(fi.attributes()),diff --git a/lucene/core/src/java/org/apache/lucene/index/FilterLeafReader.java b/lucene/core/src/java/org/apache/lucene/index/FilterLeafReader.javaindex 38d0ac48d46..4935237178a 100644--- a/lucene/core/src/java/org/apache/lucene/index/FilterLeafReader.java+++ b/lucene/core/src/java/org/apache/lucene/index/FilterLeafReader.java@@ -441,6 +441,12 @@ public abstract class FilterLeafReader extends LeafReader {return in.getSortedSetDocValues(field);}+ @Override+ public DocValuesSkipper getDocValuesSkipper(String field) throws IOException {+ ensureOpen();+ return in.getDocValuesSkipper(field);+ }+@Overridepublic NumericDocValues getNormValues(String field) throws IOException {ensureOpen();diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexableFieldType.java b/lucene/core/src/java/org/apache/lucene/index/IndexableFieldType.javaindex 5d32e5dbcf9..006828e98a2 100644--- a/lucene/core/src/java/org/apache/lucene/index/IndexableFieldType.java+++ b/lucene/core/src/java/org/apache/lucene/index/IndexableFieldType.java@@ -86,6 +86,9 @@ public interface IndexableFieldType {/** DocValues {@link DocValuesType}: how the field's value will be indexed into docValues. */DocValuesType docValuesType();+ /** Whether a skip index for doc values should be created on this field. */+ boolean hasDocValuesSkipIndex();+/*** If this is positive (representing the number of point dimensions), the field is indexed as a* point.diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexingChain.java b/lucene/core/src/java/org/apache/lucene/index/IndexingChain.javaindex 701fa16f6b3..4e61afbc2b4 100644--- a/lucene/core/src/java/org/apache/lucene/index/IndexingChain.java+++ b/lucene/core/src/java/org/apache/lucene/index/IndexingChain.java@@ -680,6 +680,7 @@ final class IndexingChain implements Accountable {false,s.indexOptions,s.docValuesType,+ s.hasDocValuesSkipIndex,-1,s.attributes,s.pointDimensionCount,@@ -831,7 +832,12 @@ final class IndexingChain implements Accountable {verifyUnIndexedFieldType(fieldName, fieldType);}if (fieldType.docValuesType() != DocValuesType.NONE) {- schema.setDocValues(fieldType.docValuesType());+ schema.setDocValues(fieldType.docValuesType(), fieldType.hasDocValuesSkipIndex());+ } else if (fieldType.hasDocValuesSkipIndex()) {+ throw new IllegalArgumentException(+ "field '"+ + schema.name+ + "' cannot have docValuesSkipIndex set to true without doc values");}if (fieldType.pointDimensionCount() != 0) {schema.setPoints(@@ -1432,6 +1438,7 @@ final class IndexingChain implements Accountable {private boolean storeTermVector = false;private IndexOptions indexOptions = IndexOptions.NONE;private DocValuesType docValuesType = DocValuesType.NONE;+ private boolean hasDocValuesSkipIndex = false;private int pointDimensionCount = 0;private int pointIndexDimensionCount = 0;private int pointNumBytes = 0;@@ -1497,11 +1504,13 @@ final class IndexingChain implements Accountable {}}- void setDocValues(DocValuesType newDocValuesType) {+ void setDocValues(DocValuesType newDocValuesType, boolean newHasDocValuesSkipIndex) {if (docValuesType == DocValuesType.NONE) {this.docValuesType = newDocValuesType;+ this.hasDocValuesSkipIndex = newHasDocValuesSkipIndex;} else {assertSame("doc values type", docValuesType, newDocValuesType);+ assertSame("doc values skip index", hasDocValuesSkipIndex, newHasDocValuesSkipIndex);}}@@ -1549,6 +1558,7 @@ final class IndexingChain implements Accountable {assertSame("omit norms", fi.omitsNorms(), omitNorms);assertSame("store term vector", fi.hasVectors(), storeTermVector);assertSame("doc values type", fi.getDocValuesType(), docValuesType);+ assertSame("doc values skip index", fi.hasDocValuesSkipIndex(), hasDocValuesSkipIndex);assertSame("vector similarity function", fi.getVectorSimilarityFunction(), vectorSimilarityFunction);assertSame("vector encoding", fi.getVectorEncoding(), vectorEncoding);diff --git a/lucene/core/src/java/org/apache/lucene/index/LeafReader.java b/lucene/core/src/java/org/apache/lucene/index/LeafReader.javaindex 5c3c9f8da5e..0f39d1ae1e8 100644--- a/lucene/core/src/java/org/apache/lucene/index/LeafReader.java+++ b/lucene/core/src/java/org/apache/lucene/index/LeafReader.java@@ -202,6 +202,13 @@ public abstract non-sealed class LeafReader extends IndexReader {*/public abstract NumericDocValues getNormValues(String field) throws IOException;+ /**+ * Returns a {@link DocValuesSkipper} allowing skipping ranges of doc IDs that are not of+ * interest, or {@code null} if a skip index was not indexed. The returned instance should be+ * confined to the thread that created it.+ */+ public abstract DocValuesSkipper getDocValuesSkipper(String field) throws IOException;+/*** Returns {@link FloatVectorValues} for this field, or null if no {@link FloatVectorValues} were* indexed. The returned instance should only be used by a single thread.diff --git a/lucene/core/src/java/org/apache/lucene/index/ParallelLeafReader.java b/lucene/core/src/java/org/apache/lucene/index/ParallelLeafReader.javaindex 19c2fd6bd4e..1f1e2dba9c1 100644--- a/lucene/core/src/java/org/apache/lucene/index/ParallelLeafReader.java+++ b/lucene/core/src/java/org/apache/lucene/index/ParallelLeafReader.java@@ -399,6 +399,13 @@ public class ParallelLeafReader extends LeafReader {return reader == null ? null : reader.getSortedSetDocValues(field);}+ @Override+ public DocValuesSkipper getDocValuesSkipper(String field) throws IOException {+ ensureOpen();+ LeafReader reader = fieldToReader.get(field);+ return reader == null ? null : reader.getDocValuesSkipper(field);+ }+@Overridepublic NumericDocValues getNormValues(String field) throws IOException {ensureOpen();diff --git a/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java b/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.javaindex ed635c68038..9d4f79a5586 100644--- a/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java+++ b/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java@@ -713,6 +713,7 @@ final class ReadersAndUpdates {fi.hasPayloads(),fi.getIndexOptions(),fi.getDocValuesType(),+ fi.hasDocValuesSkipIndex(),fi.getDocValuesGen(),new HashMap<>(fi.attributes()),fi.getPointDimensionCount(),diff --git a/lucene/core/src/java/org/apache/lucene/index/SegmentDocValuesProducer.java b/lucene/core/src/java/org/apache/lucene/index/SegmentDocValuesProducer.javaindex 63baddb4174..1d9878fe0db 100644--- a/lucene/core/src/java/org/apache/lucene/index/SegmentDocValuesProducer.java+++ b/lucene/core/src/java/org/apache/lucene/index/SegmentDocValuesProducer.java@@ -124,6 +124,13 @@ class SegmentDocValuesProducer extends DocValuesProducer {return dvProducer.getSortedSet(field);}+ @Override+ public DocValuesSkipper getSkipper(FieldInfo field) throws IOException {+ DocValuesProducer dvProducer = dvProducersByField.get(field.name);+ assert dvProducer != null;+ return dvProducer.getSkipper(field);+ }+@Overridepublic void checkIntegrity() throws IOException {for (DocValuesProducer producer : dvProducers) {diff --git a/lucene/core/src/java/org/apache/lucene/index/SlowCodecReaderWrapper.java b/lucene/core/src/java/org/apache/lucene/index/SlowCodecReaderWrapper.javaindex 497fa1162bd..4d05d241e69 100644--- a/lucene/core/src/java/org/apache/lucene/index/SlowCodecReaderWrapper.java+++ b/lucene/core/src/java/org/apache/lucene/index/SlowCodecReaderWrapper.java@@ -240,6 +240,11 @@ public final class SlowCodecReaderWrapper {return reader.getSortedSetDocValues(field.name);}+ @Override+ public DocValuesSkipper getSkipper(FieldInfo field) throws IOException {+ return reader.getDocValuesSkipper(field.name);+ }+@Overridepublic void checkIntegrity() throws IOException {// We already checkIntegrity the entire reader up frontdiff --git a/lucene/core/src/java/org/apache/lucene/index/SlowCompositeCodecReaderWrapper.java b/lucene/core/src/java/org/apache/lucene/index/SlowCompositeCodecReaderWrapper.javaindex 5809a9aa4f4..fc6c1d9b294 100644--- a/lucene/core/src/java/org/apache/lucene/index/SlowCompositeCodecReaderWrapper.java+++ b/lucene/core/src/java/org/apache/lucene/index/SlowCompositeCodecReaderWrapper.java@@ -494,6 +494,11 @@ final class SlowCompositeCodecReaderWrapper extends CodecReader {}return new MultiSortedSetDocValues(values, docStarts, map, totalCost);}++ @Override+ public DocValuesSkipper getSkipper(FieldInfo field) throws IOException {+ throw new UnsupportedOperationException("This method is for searching not for merging");+ }}@Overridediff --git a/lucene/core/src/java/org/apache/lucene/index/SortingCodecReader.java b/lucene/core/src/java/org/apache/lucene/index/SortingCodecReader.javaindex 67c76227385..ff88e30de4a 100644--- a/lucene/core/src/java/org/apache/lucene/index/SortingCodecReader.java+++ b/lucene/core/src/java/org/apache/lucene/index/SortingCodecReader.java@@ -637,6 +637,12 @@ public final class SortingCodecReader extends FilterCodecReader {public void close() throws IOException {delegate.close();}++ @Override+ public DocValuesSkipper getSkipper(FieldInfo field) throws IOException {+ // We can hardly return information about min/max values if doc IDs have been reordered.+ return null;+ }};}diff --git a/lucene/core/src/test/org/apache/lucene/document/TestDocValuesRangeIterator.java b/lucene/core/src/test/org/apache/lucene/document/TestDocValuesRangeIterator.javanew file mode 100644index 00000000000..5b24608320a--- /dev/null+++ b/lucene/core/src/test/org/apache/lucene/document/TestDocValuesRangeIterator.java@@ -0,0 +1,273 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.document;++import java.io.IOException;+import java.util.concurrent.atomic.AtomicBoolean;+import org.apache.lucene.index.DocValuesSkipper;+import org.apache.lucene.index.NumericDocValues;+import org.apache.lucene.search.DocIdSetIterator;+import org.apache.lucene.search.TwoPhaseIterator;+import org.apache.lucene.tests.util.LuceneTestCase;++public class TestDocValuesRangeIterator extends LuceneTestCase {++ public void testSingleLevel() throws IOException {+ doTestBasics(false);+ }++ public void testMultipleLevels() throws IOException {+ doTestBasics(true);+ }++ private void doTestBasics(boolean doLevels) throws IOException {+ long queryMin = 10;+ long queryMax = 20;++ // Fake numeric doc values so that:+ // docs 0-256 all match+ // docs in 256-512 are all greater than queryMax+ // docs in 512-768 are all less than queryMin+ // docs in 768-1024 have some docs that match the range, others not+ // docs in 1024-2048 follow a similar pattern as docs in 0-1024 except that not all docs have a+ // value+ NumericDocValues values =+ new NumericDocValues() {++ int doc = -1;++ @Override+ public boolean advanceExact(int target) throws IOException {+ throw new UnsupportedOperationException();+ }++ @Override+ public int docID() {+ return doc;+ }++ @Override+ public int nextDoc() throws IOException {+ return advance(doc + 1);+ }++ @Override+ public int advance(int target) throws IOException {+ if (target < 1024) {+ // dense up to 1024+ return doc = target;+ } else if (doc < 2047) {+ // 50% docs have a value up to 2048+ return doc = target + (target & 1);+ } else {+ return doc = DocIdSetIterator.NO_MORE_DOCS;+ }+ }++ @Override+ public long longValue() throws IOException {+ int d = doc % 1024;+ if (d < 128) {+ return (queryMin + queryMax) >> 1;+ } else if (d < 256) {+ return queryMax + 1;+ } else if (d < 512) {+ return queryMin - 1;+ } else {+ return switch ((d / 2) % 3) {+ case 0 -> queryMin - 1;+ case 1 -> queryMax + 1;+ case 2 -> (queryMin + queryMax) >> 1;+ default -> throw new AssertionError();+ };+ }+ }++ @Override+ public long cost() {+ return 42;+ }+ };++ AtomicBoolean twoPhaseCalled = new AtomicBoolean();+ TwoPhaseIterator twoPhase =+ new TwoPhaseIterator(values) {++ @Override+ public boolean matches() throws IOException {+ twoPhaseCalled.set(true);+ long v = values.longValue();+ return v >= queryMin && v <= queryMax;+ }++ @Override+ public float matchCost() {+ return 2f; // 2 comparisons+ }+ };++ DocValuesSkipper skipper =+ new DocValuesSkipper() {++ int doc = -1;++ @Override+ public void advance(int target) throws IOException {+ doc = target;+ }++ @Override+ public int numLevels() {+ return doLevels ? 3 : 1;+ }++ @Override+ public int minDocID(int level) {+ int rangeLog = 9 - numLevels() + level;++ // the level is the log2 of the interval+ if (doc < 0) {+ return -1;+ } else if (doc >= 2048) {+ return DocIdSetIterator.NO_MORE_DOCS;+ } else {+ int mask = (1 << rangeLog) - 1;+ // prior multiple of 2^level+ return doc & ~mask;+ }+ }++ @Override+ public int maxDocID(int level) {+ int rangeLog = 9 - numLevels() + level;++ int minDocID = minDocID(level);+ return switch (minDocID) {+ case -1 -> -1;+ case DocIdSetIterator.NO_MORE_DOCS -> DocIdSetIterator.NO_MORE_DOCS;+ default -> minDocID + (1 << rangeLog) - 1;+ };+ }++ @Override+ public long minValue(int level) {+ int d = doc % 1024;+ if (d < 128) {+ return queryMin;+ } else if (d < 256) {+ return queryMax + 1;+ } else if (d < 768) {+ return queryMin - 1;+ } else {+ return queryMin - 1;+ }+ }++ @Override+ public long maxValue(int level) {+ int d = doc % 1024;+ if (d < 128) {+ return queryMax;+ } else if (d < 256) {+ return queryMax + 1;+ } else if (d < 768) {+ return queryMin - 1;+ } else {+ return queryMax + 1;+ }+ }++ @Override+ public int docCount(int level) {+ int rangeLog = 9 - numLevels() + level;++ if (doc < 1024) {+ return 1 << rangeLog;+ } else {+ // half docs have a value+ return 1 << rangeLog >> 1;+ }+ }++ @Override+ public long minValue() {+ return Long.MIN_VALUE;+ }++ @Override+ public long maxValue() {+ return Long.MAX_VALUE;+ }++ @Override+ public int docCount() {+ return 1024 + 1024 / 2;+ }+ };++ DocValuesRangeIterator rangeIterator =+ new DocValuesRangeIterator(twoPhase, skipper, queryMin, queryMax);+ DocValuesRangeIterator.Approximation rangeApproximation =+ (DocValuesRangeIterator.Approximation) rangeIterator.approximation();++ assertEquals(100, rangeApproximation.advance(100));+ assertEquals(DocValuesRangeIterator.Match.YES, rangeApproximation.match);+ assertEquals(255, rangeApproximation.upTo);+ assertTrue(rangeIterator.matches());+ assertTrue(values.docID() < rangeApproximation.docID()); // we did not advance doc values+ assertFalse(twoPhaseCalled.get());++ assertEquals(768, rangeApproximation.advance(300));+ assertEquals(DocValuesRangeIterator.Match.MAYBE, rangeApproximation.match);+ if (doLevels) {+ assertEquals(831, rangeApproximation.upTo);+ } else {+ assertEquals(1023, rangeApproximation.upTo);+ }+ for (int i = 0; i < 10; ++i) {+ assertEquals(values.docID(), rangeApproximation.docID());+ assertEquals(twoPhase.matches(), rangeIterator.matches());+ assertTrue(twoPhaseCalled.get());+ twoPhaseCalled.set(false);+ rangeApproximation.nextDoc();+ }++ assertEquals(1100, rangeApproximation.advance(1099));+ assertEquals(DocValuesRangeIterator.Match.IF_DOC_HAS_VALUE, rangeApproximation.match);+ assertEquals(1024 + 256 - 1, rangeApproximation.upTo);+ assertEquals(values.docID(), rangeApproximation.docID());+ assertTrue(rangeIterator.matches());+ assertFalse(twoPhaseCalled.get());++ assertEquals(1024 + 768, rangeApproximation.advance(1024 + 300));+ assertEquals(DocValuesRangeIterator.Match.MAYBE, rangeApproximation.match);+ if (doLevels) {+ assertEquals(1024 + 831, rangeApproximation.upTo);+ } else {+ assertEquals(2047, rangeApproximation.upTo);+ }+ for (int i = 0; i < 10; ++i) {+ assertEquals(values.docID(), rangeApproximation.docID());+ assertEquals(twoPhase.matches(), rangeIterator.matches());+ assertTrue(twoPhaseCalled.get());+ twoPhaseCalled.set(false);+ rangeApproximation.nextDoc();+ }++ assertEquals(DocIdSetIterator.NO_MORE_DOCS, rangeApproximation.advance(2048));+ }+}diff --git a/lucene/core/src/test/org/apache/lucene/index/TestCheckIndex.java b/lucene/core/src/test/org/apache/lucene/index/TestCheckIndex.javaindex 96e545e6011..496c1b78ff2 100644--- a/lucene/core/src/test/org/apache/lucene/index/TestCheckIndex.java+++ b/lucene/core/src/test/org/apache/lucene/index/TestCheckIndex.java@@ -105,6 +105,9 @@ public class TestCheckIndex extends BaseTestCheckIndex {// doc valuedoc.add(new NumericDocValuesField("dv", random().nextLong()));+ // doc value with skip index+ doc.add(NumericDocValuesField.indexedField("dv_skip", random().nextLong()));+// point valuebyte[] point = new byte[4];NumericUtils.intToSortableBytes(random().nextInt(), point, 0);@@ -154,7 +157,7 @@ public class TestCheckIndex extends BaseTestCheckIndex {assertNull(segStatus.liveDocStatus.error);// confirm field infos testing status- assertEquals(8, segStatus.fieldInfoStatus.totFields);+ assertEquals(9, segStatus.fieldInfoStatus.totFields);assertTrue(output.toString(UTF_8).contains("test: field infos"));assertNull(segStatus.fieldInfoStatus.error);@@ -184,7 +187,8 @@ public class TestCheckIndex extends BaseTestCheckIndex {assertNull(segStatus.termVectorStatus.error);// confirm doc values testing status- assertEquals(2, segStatus.docValuesStatus.totalNumericFields);+ assertEquals(3, segStatus.docValuesStatus.totalNumericFields);+ assertEquals(1, segStatus.docValuesStatus.totalSkippingIndex);assertTrue(output.toString(UTF_8).contains("test: docvalues"));assertNull(segStatus.docValuesStatus.error);diff --git a/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java b/lucene/core/src/test/org/apache/lucene/index/TestCodecs.javaindex eff9cbe763f..1759271012d 100644--- a/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java+++ b/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java@@ -106,6 +106,7 @@ public class TestCodecs extends LuceneTestCase {storePayloads,indexOptions,DocValuesType.NONE,+ false,-1,new HashMap<>(),0,diff --git a/lucene/core/src/test/org/apache/lucene/index/TestFieldInfos.java b/lucene/core/src/test/org/apache/lucene/index/TestFieldInfos.javaindex e647b6d2b33..e19855bbdda 100644--- a/lucene/core/src/test/org/apache/lucene/index/TestFieldInfos.java+++ b/lucene/core/src/test/org/apache/lucene/index/TestFieldInfos.java@@ -250,6 +250,7 @@ public class TestFieldInfos extends LuceneTestCase {false,IndexOptions.NONE,DocValuesType.NONE,+ false,-1,new HashMap<>(),0,@@ -271,6 +272,7 @@ public class TestFieldInfos extends LuceneTestCase {false,IndexOptions.NONE,DocValuesType.NONE,+ false,-1,new HashMap<>(),0,@@ -294,6 +296,7 @@ public class TestFieldInfos extends LuceneTestCase {false,IndexOptions.NONE,DocValuesType.NONE,+ false,-1,new HashMap<>(),0,diff --git a/lucene/core/src/test/org/apache/lucene/index/TestFieldsReader.java b/lucene/core/src/test/org/apache/lucene/index/TestFieldsReader.javaindex b526f04f39d..15d6dddcb58 100644--- a/lucene/core/src/test/org/apache/lucene/index/TestFieldsReader.java+++ b/lucene/core/src/test/org/apache/lucene/index/TestFieldsReader.java@@ -58,6 +58,7 @@ public class TestFieldsReader extends LuceneTestCase {false,ift.indexOptions(),ift.docValuesType(),+ ift.hasDocValuesSkipIndex(),-1,new HashMap<>(),0,diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.javaindex 163bb21d67d..8ef60beefd4 100644--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java@@ -4976,4 +4976,62 @@ public class TestIndexWriter extends LuceneTestCase {}}}++ public void testDocValuesMixedSkippingIndex() throws Exception {+ try (Directory dir = newDirectory()) {+ try (IndexWriter writer =+ new IndexWriter(dir, new IndexWriterConfig(new MockAnalyzer(random())))) {+ Document doc1 = new Document();+ doc1.add(SortedNumericDocValuesField.indexedField("test", random().nextLong()));+ writer.addDocument(doc1);++ Document doc2 = new Document();+ doc2.add(new SortedNumericDocValuesField("test", random().nextLong()));+ IllegalArgumentException ex =+ expectThrows(IllegalArgumentException.class, () -> writer.addDocument(doc2));+ assertEquals(+ "Inconsistency of field data structures across documents for field [test] of doc [1]. doc values skip index: expected 'true', but it has 'false'.",+ ex.getMessage());+ }+ }+ try (Directory dir = newDirectory()) {+ try (IndexWriter writer =+ new IndexWriter(dir, new IndexWriterConfig(new MockAnalyzer(random())))) {+ Document doc1 = new Document();+ doc1.add(new SortedSetDocValuesField("test", TestUtil.randomBinaryTerm(random())));+ writer.addDocument(doc1);++ Document doc2 = new Document();+ doc2.add(SortedSetDocValuesField.indexedField("test", TestUtil.randomBinaryTerm(random())));+ IllegalArgumentException ex =+ expectThrows(IllegalArgumentException.class, () -> writer.addDocument(doc2));+ assertEquals(+ "Inconsistency of field data structures across documents for field [test] of doc [1]. doc values skip index: expected 'false', but it has 'true'.",+ ex.getMessage());+ }+ }+ }++ public void testDocValuesSkippingIndexWithoutDocValues() throws Exception {+ for (DocValuesType docValuesType :+ new DocValuesType[] {DocValuesType.NONE, DocValuesType.BINARY}) {+ FieldType fieldType = new FieldType();+ fieldType.setStored(true);+ fieldType.setDocValuesType(docValuesType);+ fieldType.setDocValuesSkipIndex(true);+ fieldType.freeze();+ try (Directory dir = newMockDirectory()) {+ try (IndexWriter writer =+ new IndexWriter(dir, new IndexWriterConfig(new MockAnalyzer(random())))) {+ Document doc1 = new Document();+ doc1.add(new Field("test", new byte[10], fieldType));+ IllegalArgumentException ex =+ expectThrows(IllegalArgumentException.class, () -> writer.addDocument(doc1));+ assertTrue(+ ex.getMessage()+ .startsWith("field 'test' cannot have docValuesSkipIndex set to true"));+ }+ }+ }+ }}diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexableField.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexableField.javaindex 886417fabc8..339368da1f8 100644--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexableField.java+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexableField.java@@ -95,6 +95,11 @@ public class TestIndexableField extends LuceneTestCase {return DocValuesType.NONE;}+ @Override+ public boolean hasDocValuesSkipIndex() {+ return false;+ }+@Overridepublic int pointDimensionCount() {return 0;diff --git a/lucene/core/src/test/org/apache/lucene/index/TestPendingSoftDeletes.java b/lucene/core/src/test/org/apache/lucene/index/TestPendingSoftDeletes.javaindex 3c6d57c5d2f..cb57e983691 100644--- a/lucene/core/src/test/org/apache/lucene/index/TestPendingSoftDeletes.java+++ b/lucene/core/src/test/org/apache/lucene/index/TestPendingSoftDeletes.java@@ -191,6 +191,7 @@ public class TestPendingSoftDeletes extends TestPendingDeletes {false,IndexOptions.NONE,DocValuesType.NUMERIC,+ false,0,Collections.emptyMap(),0,@@ -230,6 +231,7 @@ public class TestPendingSoftDeletes extends TestPendingDeletes {false,IndexOptions.NONE,DocValuesType.NUMERIC,+ false,1,Collections.emptyMap(),0,@@ -295,6 +297,7 @@ public class TestPendingSoftDeletes extends TestPendingDeletes {false,IndexOptions.NONE,DocValuesType.NUMERIC,+ false,segmentInfo.getNextDocValuesGen(),Collections.emptyMap(),0,@@ -365,6 +368,7 @@ public class TestPendingSoftDeletes extends TestPendingDeletes {false,IndexOptions.NONE,DocValuesType.NUMERIC,+ false,segmentInfo.getNextDocValuesGen(),Collections.emptyMap(),0,@@ -403,6 +407,7 @@ public class TestPendingSoftDeletes extends TestPendingDeletes {false,IndexOptions.NONE,DocValuesType.NUMERIC,+ false,segmentInfo.getNextDocValuesGen(),Collections.emptyMap(),0,diff --git a/lucene/core/src/test/org/apache/lucene/index/TestSegmentToThreadMapping.java b/lucene/core/src/test/org/apache/lucene/index/TestSegmentToThreadMapping.javaindex f6aee2f3f27..609dd0359ab 100644--- a/lucene/core/src/test/org/apache/lucene/index/TestSegmentToThreadMapping.java+++ b/lucene/core/src/test/org/apache/lucene/index/TestSegmentToThreadMapping.java@@ -101,6 +101,11 @@ public class TestSegmentToThreadMapping extends LuceneTestCase {return null;}+ @Override+ public DocValuesSkipper getDocValuesSkipper(String field) {+ return null;+ }+@Overridepublic PointValues getPointValues(String field) {return null;diff --git a/lucene/core/src/test/org/apache/lucene/search/TestSortOptimization.java b/lucene/core/src/test/org/apache/lucene/search/TestSortOptimization.javaindex 401bcb3e5c6..ad955e668ed 100644--- a/lucene/core/src/test/org/apache/lucene/search/TestSortOptimization.java+++ b/lucene/core/src/test/org/apache/lucene/search/TestSortOptimization.java@@ -1289,6 +1289,7 @@ public class TestSortOptimization extends LuceneTestCase {false,IndexOptions.NONE,fi.getDocValuesType(),+ fi.hasDocValuesSkipIndex(),fi.getDocValuesGen(),fi.attributes(),0,diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TermVectorLeafReader.java b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TermVectorLeafReader.javaindex 17ba68777fb..f60c7966f98 100644--- a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TermVectorLeafReader.java+++ b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TermVectorLeafReader.java@@ -21,6 +21,7 @@ import java.util.Collections;import java.util.Iterator;import org.apache.lucene.index.BinaryDocValues;import org.apache.lucene.index.ByteVectorValues;+import org.apache.lucene.index.DocValuesSkipper;import org.apache.lucene.index.DocValuesType;import org.apache.lucene.index.FieldInfo;import org.apache.lucene.index.FieldInfos;@@ -95,6 +96,7 @@ public class TermVectorLeafReader extends LeafReader {terms.hasPayloads(),indexOptions,DocValuesType.NONE,+ false,-1,Collections.emptyMap(),0,@@ -141,6 +143,11 @@ public class TermVectorLeafReader extends LeafReader {return null;}+ @Override+ public DocValuesSkipper getDocValuesSkipper(String field) throws IOException {+ return null;+ }+@Overridepublic NumericDocValues getNormValues(String field) throws IOException {return null; // Is this needed? See MemoryIndex for a way to do it.diff --git a/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java b/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.javaindex 2bd50c5355c..ff65bca6a16 100644--- a/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java+++ b/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java@@ -728,6 +728,7 @@ public class MemoryIndex {storePayloads,indexOptions,fieldType.docValuesType(),+ false,-1,Collections.emptyMap(),fieldType.pointDimensionCount(),@@ -782,6 +783,7 @@ public class MemoryIndex {info.fieldInfo.hasPayloads(),info.fieldInfo.getIndexOptions(),docValuesType,+ false,-1,info.fieldInfo.attributes(),info.fieldInfo.getPointDimensionCount(),@@ -1622,6 +1624,12 @@ public class MemoryIndex {}}+ @Override+ public DocValuesSkipper getDocValuesSkipper(String field) throws IOException {+ // Skipping isn't needed on a 1-doc index.+ return null;+ }+@Overridepublic PointValues getPointValues(String fieldName) {Info info = fields.get(fieldName);diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/asserting/AssertingDocValuesFormat.java b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/asserting/AssertingDocValuesFormat.javaindex f33521fc10a..3355d925eca 100644--- a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/asserting/AssertingDocValuesFormat.java+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/asserting/AssertingDocValuesFormat.java@@ -23,6 +23,7 @@ import org.apache.lucene.codecs.DocValuesConsumer;import org.apache.lucene.codecs.DocValuesFormat;import org.apache.lucene.codecs.DocValuesProducer;import org.apache.lucene.index.BinaryDocValues;+import org.apache.lucene.index.DocValuesSkipper;import org.apache.lucene.index.DocValuesType;import org.apache.lucene.index.FieldInfo;import org.apache.lucene.index.NumericDocValues;@@ -280,6 +281,14 @@ public class AssertingDocValuesFormat extends DocValuesFormat {return AssertingLeafReader.AssertingSortedSetDocValues.create(values, maxDoc);}+ @Override+ public DocValuesSkipper getSkipper(FieldInfo field) throws IOException {+ assert field.hasDocValuesSkipIndex();+ DocValuesSkipper skipper = in.getSkipper(field);+ assert skipper != null;+ return new AssertingLeafReader.AssertingDocValuesSkipper(skipper);+ }+@Overridepublic void close() throws IOException {in.close();diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/index/AssertingLeafReader.java b/lucene/test-framework/src/java/org/apache/lucene/tests/index/AssertingLeafReader.javaindex 8f55dfb5322..fc02ae82de2 100644--- a/lucene/test-framework/src/java/org/apache/lucene/tests/index/AssertingLeafReader.java+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/index/AssertingLeafReader.java@@ -23,6 +23,7 @@ import java.util.List;import java.util.Objects;import org.apache.lucene.index.BinaryDocValues;import org.apache.lucene.index.DocValues;+import org.apache.lucene.index.DocValuesSkipper;import org.apache.lucene.index.DocValuesType;import org.apache.lucene.index.FieldInfo;import org.apache.lucene.index.Fields;@@ -1155,6 +1156,109 @@ public class AssertingLeafReader extends FilterLeafReader {}}+ /** Wraps a DocValuesSkipper but with additional asserts */+ public static class AssertingDocValuesSkipper extends DocValuesSkipper {++ private final Thread creationThread = Thread.currentThread();+ private final DocValuesSkipper in;++ /** Sole constructor */+ public AssertingDocValuesSkipper(DocValuesSkipper in) {+ this.in = in;+ assert minDocID(0) == -1;+ assert maxDocID(0) == -1;+ }++ @Override+ public void advance(int target) throws IOException {+ assertThread("Doc values skipper", creationThread);+ assert target > maxDocID(0)+ : "Illegal to call advance() on a target that is not beyond the current interval";+ in.advance(target);+ assert in.minDocID(0) <= in.maxDocID(0);+ }++ private boolean iterating() {+ return maxDocID(0) != -1+ && minDocID(0) != -1+ && maxDocID(0) != DocIdSetIterator.NO_MORE_DOCS+ && minDocID(0) != DocIdSetIterator.NO_MORE_DOCS;+ }++ @Override+ public int numLevels() {+ assertThread("Doc values skipper", creationThread);+ return in.numLevels();+ }++ @Override+ public int minDocID(int level) {+ assertThread("Doc values skipper", creationThread);+ Objects.checkIndex(level, numLevels());+ int minDocID = in.minDocID(level);+ assert minDocID <= in.maxDocID(level);+ if (level > 0) {+ assert minDocID <= in.minDocID(level - 1);+ }+ return minDocID;+ }++ @Override+ public int maxDocID(int level) {+ assertThread("Doc values skipper", creationThread);+ Objects.checkIndex(level, numLevels());+ int maxDocID = in.maxDocID(level);++ assert maxDocID >= in.minDocID(level);+ if (level > 0) {+ assert maxDocID >= in.maxDocID(level - 1);+ }+ return maxDocID;+ }++ @Override+ public long minValue(int level) {+ assertThread("Doc values skipper", creationThread);+ assert iterating() : "Unpositioned iterator";+ Objects.checkIndex(level, numLevels());+ return in.minValue(level);+ }++ @Override+ public long maxValue(int level) {+ assertThread("Doc values skipper", creationThread);+ assert iterating() : "Unpositioned iterator";+ Objects.checkIndex(level, numLevels());+ return in.maxValue(level);+ }++ @Override+ public int docCount(int level) {+ assertThread("Doc values skipper", creationThread);+ assert iterating() : "Unpositioned iterator";+ Objects.checkIndex(level, numLevels());+ return in.docCount(level);+ }++ @Override+ public long minValue() {+ assertThread("Doc values skipper", creationThread);+ return in.minValue();+ }++ @Override+ public long maxValue() {+ assertThread("Doc values skipper", creationThread);+ return in.maxValue();+ }++ @Override+ public int docCount() {+ assertThread("Doc values skipper", creationThread);+ return in.docCount();+ }+ }+/** Wraps a SortedSetDocValues but with additional asserts */public static class AssertingPointValues extends PointValues {private final Thread creationThread = Thread.currentThread();@@ -1483,6 +1587,19 @@ public class AssertingLeafReader extends FilterLeafReader {}}+ @Override+ public DocValuesSkipper getDocValuesSkipper(String field) throws IOException {+ DocValuesSkipper skipper = super.getDocValuesSkipper(field);+ FieldInfo fi = getFieldInfos().fieldInfo(field);+ if (skipper != null) {+ assert fi.hasDocValuesSkipIndex();+ return new AssertingDocValuesSkipper(skipper);+ } else {+ assert fi == null || fi.hasDocValuesSkipIndex() == false;+ return null;+ }+ }+@Overridepublic NumericDocValues getNormValues(String field) throws IOException {NumericDocValues dv = super.getNormValues(field);diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseDocValuesFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseDocValuesFormatTestCase.javaindex 07c4507ae92..15118f7f10b 100644--- a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseDocValuesFormatTestCase.java+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseDocValuesFormatTestCase.java@@ -56,6 +56,7 @@ import org.apache.lucene.index.CheckIndex.Status.DocValuesStatus;import org.apache.lucene.index.CodecReader;import org.apache.lucene.index.DirectoryReader;import org.apache.lucene.index.DocValues;+import org.apache.lucene.index.DocValuesSkipper;import org.apache.lucene.index.IndexReader;import org.apache.lucene.index.IndexWriter;import org.apache.lucene.index.IndexWriterConfig;@@ -101,6 +102,24 @@ import org.apache.lucene.util.automaton.RegExp;*/public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTestCase {+ /**+ * Override and return {@code false} if the {@link DocValuesSkipper} produced by this format+ * sometimes returns documents in {@link DocValuesSkipper#minDocID(int)} or {@link+ * DocValuesSkipper#maxDocID(int)} that may not have a value.+ */+ protected boolean skipperHasAccurateDocBounds() {+ return true;+ }++ /**+ * Override and return {@code false} if the {@link DocValuesSkipper} produced by this format+ * sometimes returns values in {@link DocValuesSkipper#minValue(int)} or {@link+ * DocValuesSkipper#maxValue(int)} that none of the documents in the range have.+ */+ protected boolean skipperHasAccurateValueBounds() {+ return true;+ }+@Overrideprotected void addRandomFields(Document doc) {if (usually()) {@@ -889,7 +908,7 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTesiwriter.addDocument(doc);doc = new Document();doc.add(new StringField("id", "1", Field.Store.NO));- doc.add(new SortedDocValuesField("field", newBytesRef("hello")));+ doc.add(SortedDocValuesField.indexedField("field", newBytesRef("hello")));iwriter.addDocument(doc);iwriter.commit();iwriter.deleteDocuments(new Term("id", "1"));@@ -901,6 +920,11 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTesSortedDocValues dv = getOnlyLeafReader(ireader).getSortedDocValues("field");assertEquals(NO_MORE_DOCS, dv.nextDoc());+ DocValuesSkipper skipper = getOnlyLeafReader(ireader).getDocValuesSkipper("field");+ assertEquals(0, skipper.docCount());+ skipper.advance(0);+ assertEquals(NO_MORE_DOCS, skipper.minDocID(0));+TermsEnum termsEnum = dv.termsEnum();assertFalse(termsEnum.seekExact(new BytesRef("lucene")));assertEquals(SeekStatus.END, termsEnum.seekCeil(new BytesRef("lucene")));@@ -2234,7 +2258,7 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTesiwriter.addDocument(doc);doc = new Document();doc.add(new StringField("id", "1", Field.Store.NO));- doc.add(new SortedSetDocValuesField("field", newBytesRef("hello")));+ doc.add(SortedSetDocValuesField.indexedField("field", newBytesRef("hello")));iwriter.addDocument(doc);iwriter.commit();iwriter.deleteDocuments(new Term("id", "1"));@@ -2246,6 +2270,11 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTesSortedSetDocValues dv = getOnlyLeafReader(ireader).getSortedSetDocValues("field");assertEquals(0, dv.getValueCount());+ DocValuesSkipper skipper = getOnlyLeafReader(ireader).getDocValuesSkipper("field");+ assertEquals(0, skipper.docCount());+ skipper.advance(0);+ assertEquals(NO_MORE_DOCS, skipper.minDocID(0));+TermsEnum termsEnum = dv.termsEnum();assertFalse(termsEnum.seekExact(new BytesRef("lucene")));assertEquals(SeekStatus.END, termsEnum.seekCeil(new BytesRef("lucene")));@@ -3211,7 +3240,7 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTesiwriter.addDocument(doc);doc = new Document();doc.add(new StringField("id", "1", Field.Store.NO));- doc.add(new NumericDocValuesField("field", 5));+ doc.add(NumericDocValuesField.indexedField("field", 5));iwriter.addDocument(doc);iwriter.commit();iwriter.deleteDocuments(new Term("id", "1"));@@ -3223,6 +3252,11 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTesNumericDocValues dv = getOnlyLeafReader(ireader).getNumericDocValues("field");assertEquals(NO_MORE_DOCS, dv.nextDoc());+ DocValuesSkipper skipper = getOnlyLeafReader(ireader).getDocValuesSkipper("field");+ assertEquals(0, skipper.docCount());+ skipper.advance(0);+ assertEquals(NO_MORE_DOCS, skipper.minDocID(0));+ireader.close();directory.close();}@@ -3337,7 +3371,7 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTesiwriter.addDocument(doc);doc = new Document();doc.add(new StringField("id", "1", Field.Store.NO));- doc.add(new SortedNumericDocValuesField("field", 5));+ doc.add(SortedNumericDocValuesField.indexedField("field", 5));iwriter.addDocument(doc);iwriter.commit();iwriter.deleteDocuments(new Term("id", "1"));@@ -3349,6 +3383,11 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTesSortedNumericDocValues dv = getOnlyLeafReader(ireader).getSortedNumericDocValues("field");assertEquals(NO_MORE_DOCS, dv.nextDoc());+ DocValuesSkipper skipper = getOnlyLeafReader(ireader).getDocValuesSkipper("field");+ assertEquals(0, skipper.docCount());+ skipper.advance(0);+ assertEquals(NO_MORE_DOCS, skipper.minDocID(0));+ireader.close();directory.close();}@@ -3499,7 +3538,7 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTesDocument doc = new Document();doc.add(new StringField("id", "1", Field.Store.NO));- doc.add(new SortedDocValuesField("field", newBytesRef("hello")));+ doc.add(SortedDocValuesField.indexedField("field", newBytesRef("hello")));iwriter.addDocument(doc);final int numEmptyDocs = atLeast(1024);for (int i = 0; i < numEmptyDocs; ++i) {@@ -3515,6 +3554,11 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTesSortedDocValues dv = getOnlyLeafReader(ireader).getSortedDocValues("field");assertEquals(NO_MORE_DOCS, dv.nextDoc());+ DocValuesSkipper skipper = getOnlyLeafReader(ireader).getDocValuesSkipper("field");+ assertEquals(0, skipper.docCount());+ skipper.advance(0);+ assertEquals(NO_MORE_DOCS, skipper.minDocID(0));+TermsEnum termsEnum = dv.termsEnum();assertFalse(termsEnum.seekExact(new BytesRef("lucene")));assertEquals(SeekStatus.END, termsEnum.seekCeil(new BytesRef("lucene")));@@ -3534,7 +3578,7 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTesDocument doc = new Document();doc.add(new StringField("id", "1", Field.Store.NO));- doc.add(new SortedSetDocValuesField("field", newBytesRef("hello")));+ doc.add(SortedSetDocValuesField.indexedField("field", newBytesRef("hello")));iwriter.addDocument(doc);final int numEmptyDocs = atLeast(1024);for (int i = 0; i < numEmptyDocs; ++i) {@@ -3550,6 +3594,11 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTesSortedSetDocValues dv = getOnlyLeafReader(ireader).getSortedSetDocValues("field");assertEquals(NO_MORE_DOCS, dv.nextDoc());+ DocValuesSkipper skipper = getOnlyLeafReader(ireader).getDocValuesSkipper("field");+ assertEquals(0, skipper.docCount());+ skipper.advance(0);+ assertEquals(NO_MORE_DOCS, skipper.minDocID(0));+TermsEnum termsEnum = dv.termsEnum();assertFalse(termsEnum.seekExact(new BytesRef("lucene")));assertEquals(SeekStatus.END, termsEnum.seekCeil(new BytesRef("lucene")));@@ -3569,7 +3618,7 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTesDocument doc = new Document();doc.add(new StringField("id", "1", Field.Store.NO));- doc.add(new NumericDocValuesField("field", 42L));+ doc.add(NumericDocValuesField.indexedField("field", 42L));iwriter.addDocument(doc);final int numEmptyDocs = atLeast(1024);for (int i = 0; i < numEmptyDocs; ++i) {@@ -3585,6 +3634,11 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTesNumericDocValues dv = getOnlyLeafReader(ireader).getNumericDocValues("field");assertEquals(NO_MORE_DOCS, dv.nextDoc());+ DocValuesSkipper skipper = getOnlyLeafReader(ireader).getDocValuesSkipper("field");+ assertEquals(0, skipper.docCount());+ skipper.advance(0);+ assertEquals(NO_MORE_DOCS, skipper.minDocID(0));+ireader.close();directory.close();}@@ -3600,7 +3654,7 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTesDocument doc = new Document();doc.add(new StringField("id", "1", Field.Store.NO));- doc.add(new SortedNumericDocValuesField("field", 42L));+ doc.add(SortedNumericDocValuesField.indexedField("field", 42L));iwriter.addDocument(doc);final int numEmptyDocs = atLeast(1024);for (int i = 0; i < numEmptyDocs; ++i) {@@ -3616,6 +3670,11 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTesSortedNumericDocValues dv = getOnlyLeafReader(ireader).getSortedNumericDocValues("field");assertEquals(NO_MORE_DOCS, dv.nextDoc());+ DocValuesSkipper skipper = getOnlyLeafReader(ireader).getDocValuesSkipper("field");+ assertEquals(0, skipper.docCount());+ skipper.advance(0);+ assertEquals(NO_MORE_DOCS, skipper.minDocID(0));+ireader.close();directory.close();}@@ -3807,4 +3866,457 @@ public abstract class BaseDocValuesFormatTestCase extends BaseIndexFileFormatTesprotected boolean codecAcceptsHugeBinaryValues(String field) {return true;}++ public void testNumericDocValuesWithSkipperSmall() throws Exception {+ doTestNumericDocValuesWithSkipper(random().nextInt(1, 1000));+ }++ public void testNumericDocValuesWithSkipperMedium() throws Exception {+ doTestNumericDocValuesWithSkipper(random().nextInt(1000, 20000));+ }++ @Nightly+ public void testNumericDocValuesWithSkipperBig() throws Exception {+ doTestNumericDocValuesWithSkipper(random().nextInt(50000, 100000));+ }++ private void doTestNumericDocValuesWithSkipper(int totalDocs) throws Exception {+ assertDocValuesWithSkipper(+ totalDocs,+ new TestDocValueSkipper() {+ @Override+ public void populateDoc(Document doc) {+ doc.add(NumericDocValuesField.indexedField("test", random().nextLong()));+ }++ @Override+ public DocValuesWrapper docValuesWrapper(LeafReader leafReader) throws IOException {+ NumericDocValues numericDocValues = leafReader.getNumericDocValues("test");+ return new DocValuesWrapper() {++ @Override+ public int advance(int target) throws IOException {+ return numericDocValues.advance(target);+ }++ @Override+ public boolean advanceExact(int target) throws IOException {+ return numericDocValues.advanceExact(target);+ }++ @Override+ public long maxValue() throws IOException {+ return numericDocValues.longValue();+ }++ @Override+ public long minValue() throws IOException {+ return numericDocValues.longValue();+ }++ @Override+ public int docID() {+ return numericDocValues.docID();+ }+ };+ }++ @Override+ public DocValuesSkipper docValuesSkipper(LeafReader leafReader) throws IOException {+ return leafReader.getDocValuesSkipper("test");+ }+ });+ }++ public void testSortedNumericDocValuesWithSkipperSmall() throws Exception {+ doTestSortedNumericDocValuesWithSkipper(random().nextInt(1, 1000));+ }++ public void testSortedNumericDocValuesWithSkipperMedium() throws Exception {+ doTestSortedNumericDocValuesWithSkipper(random().nextInt(1000, 20000));+ }++ @Nightly+ public void testSortedNumericDocValuesWithSkipperBig() throws Exception {+ doTestSortedNumericDocValuesWithSkipper(random().nextInt(50000, 100000));+ }++ private void doTestSortedNumericDocValuesWithSkipper(int totalDocs) throws Exception {+ assertDocValuesWithSkipper(+ totalDocs,+ new TestDocValueSkipper() {+ @Override+ public void populateDoc(Document doc) {+ for (int j = 0; j < random().nextInt(1, 5); j++) {+ doc.add(SortedNumericDocValuesField.indexedField("test", random().nextLong()));+ }+ }++ @Override+ public DocValuesWrapper docValuesWrapper(LeafReader leafReader) throws IOException {+ SortedNumericDocValues sortedNumericDocValues =+ leafReader.getSortedNumericDocValues("test");+ return new DocValuesWrapper() {+ long max;+ long min;++ @Override+ public int advance(int target) throws IOException {+ int doc = sortedNumericDocValues.advance(target);+ if (doc != NO_MORE_DOCS) {+ readValues();+ }+ return doc;+ }++ @Override+ public boolean advanceExact(int target) throws IOException {+ if (sortedNumericDocValues.advanceExact(target)) {+ readValues();+ return true;+ }+ return false;+ }++ private void readValues() throws IOException {+ max = Long.MIN_VALUE;+ min = Long.MAX_VALUE;+ for (int i = 0; i < sortedNumericDocValues.docValueCount(); i++) {+ long value = sortedNumericDocValues.nextValue();+ max = Math.max(max, value);+ min = Math.min(min, value);+ }+ }++ @Override+ public long maxValue() {+ return max;+ }++ @Override+ public long minValue() {+ return min;+ }++ @Override+ public int docID() {+ return sortedNumericDocValues.docID();+ }+ };+ }++ @Override+ public DocValuesSkipper docValuesSkipper(LeafReader leafReader) throws IOException {+ return leafReader.getDocValuesSkipper("test");+ }+ });+ }++ public void testSortedDocValuesWithSkipperSmall() throws Exception {+ doTestSortedDocValuesWithSkipper(random().nextInt(1, 1000));+ }++ public void testSortedDocValuesWithSkipperMedium() throws Exception {+ doTestSortedDocValuesWithSkipper(random().nextInt(1000, 20000));+ }++ @Nightly+ public void testSortedDocValuesWithSkipperBig() throws Exception {+ doTestSortedDocValuesWithSkipper(random().nextInt(50000, 100000));+ }++ private void doTestSortedDocValuesWithSkipper(int totalDocs) throws Exception {+ assertDocValuesWithSkipper(+ totalDocs,+ new TestDocValueSkipper() {+ @Override+ public void populateDoc(Document doc) {+ doc.add(SortedDocValuesField.indexedField("test", TestUtil.randomBinaryTerm(random())));+ }++ @Override+ public DocValuesWrapper docValuesWrapper(LeafReader leafReader) throws IOException {+ SortedDocValues sortedDocValues = leafReader.getSortedDocValues("test");+ return new DocValuesWrapper() {++ @Override+ public int advance(int target) throws IOException {+ return sortedDocValues.advance(target);+ }++ @Override+ public boolean advanceExact(int target) throws IOException {+ return sortedDocValues.advanceExact(target);+ }++ @Override+ public long maxValue() throws IOException {+ return sortedDocValues.ordValue();+ }++ @Override+ public long minValue() throws IOException {+ return sortedDocValues.ordValue();+ }++ @Override+ public int docID() {+ return sortedDocValues.docID();+ }+ };+ }++ @Override+ public DocValuesSkipper docValuesSkipper(LeafReader leafReader) throws IOException {+ return leafReader.getDocValuesSkipper("test");+ }+ });+ }++ public void testSortedSetDocValuesWithSkipperSmall() throws Exception {+ doTestSortedSetDocValuesWithSkipper(random().nextInt(1, 1000));+ }++ public void testSortedSetDocValuesWithSkipperMedium() throws Exception {+ doTestSortedSetDocValuesWithSkipper(random().nextInt(10000, 20000));+ }++ @Nightly+ public void testSortedSetDocValuesWithSkipperBig() throws Exception {+ doTestSortedSetDocValuesWithSkipper(random().nextInt(50000, 100000));+ }++ private void doTestSortedSetDocValuesWithSkipper(int totalDocs) throws Exception {+ assertDocValuesWithSkipper(+ totalDocs,+ new TestDocValueSkipper() {+ @Override+ public void populateDoc(Document doc) {+ for (int j = 0; j < random().nextInt(1, 5); j++) {+ doc.add(+ SortedSetDocValuesField.indexedField(+ "test", TestUtil.randomBinaryTerm(random())));+ }+ }++ @Override+ public DocValuesWrapper docValuesWrapper(LeafReader leafReader) throws IOException {+ SortedSetDocValues sortedSetDocValues = leafReader.getSortedSetDocValues("test");+ return new DocValuesWrapper() {+ long max;+ long min;++ @Override+ public int advance(int target) throws IOException {+ int doc = sortedSetDocValues.advance(target);+ if (doc != NO_MORE_DOCS) {+ readValues();+ }+ return doc;+ }++ @Override+ public boolean advanceExact(int target) throws IOException {+ if (sortedSetDocValues.advanceExact(target)) {+ readValues();+ return true;+ }+ return false;+ }++ private void readValues() throws IOException {+ max = Long.MIN_VALUE;+ min = Long.MAX_VALUE;+ for (int i = 0; i < sortedSetDocValues.docValueCount(); i++) {+ long value = sortedSetDocValues.nextOrd();+ max = Math.max(max, value);+ min = Math.min(min, value);+ }+ }++ @Override+ public long maxValue() {+ return max;+ }++ @Override+ public long minValue() {+ return min;+ }++ @Override+ public int docID() {+ return sortedSetDocValues.docID();+ }+ };+ }++ @Override+ public DocValuesSkipper docValuesSkipper(LeafReader leafReader) throws IOException {+ return leafReader.getDocValuesSkipper("test");+ }+ });+ }++ private void assertDocValuesWithSkipper(int totalDocs, TestDocValueSkipper testDocValueSkipper)+ throws Exception {+ Supplier<Boolean> booleanSupplier;+ switch (random().nextInt(3)) {+ case 0 -> booleanSupplier = () -> true;+ case 1 -> booleanSupplier = () -> random().nextBoolean();+ case 2 -> booleanSupplier = () -> random().nextBoolean() && random().nextBoolean();+ default -> throw new AssertionError();+ }+ Directory directory = newDirectory();+ RandomIndexWriter writer = new RandomIndexWriter(random(), directory);+ int numDocs = 0;+ for (int i = 0; i < totalDocs; i++) {+ Document doc = new Document();+ if (booleanSupplier.get()) {+ testDocValueSkipper.populateDoc(doc);+ numDocs++;+ }+ writer.addDocument(doc);+ if (rarely()) {+ writer.commit();+ }+ }+ writer.flush();++ if (random().nextBoolean()) {+ writer.forceMerge(1);+ }++ IndexReader r = writer.getReader();+ int readDocs = 0;+ for (LeafReaderContext readerContext : r.leaves()) {+ LeafReader reader = readerContext.reader();+ ByteArrayOutputStream bos = new ByteArrayOutputStream(1024);+ PrintStream infoStream = new PrintStream(bos, false, UTF_8);+ DocValuesStatus status = CheckIndex.testDocValues((CodecReader) reader, infoStream, true);+ if (status.error != null) {+ throw new Exception(status.error);+ }+ readDocs +=+ assertDocValuesSkipSequential(+ testDocValueSkipper.docValuesWrapper(reader),+ testDocValueSkipper.docValuesSkipper(reader));+ for (int i = 0; i < 10; i++) {+ assertDocValuesSkipRandom(+ testDocValueSkipper.docValuesWrapper(reader),+ testDocValueSkipper.docValuesSkipper(reader),+ reader.maxDoc());+ }+ }+ assertEquals(numDocs, readDocs);+ IOUtils.close(r, writer, directory);+ }++ private int assertDocValuesSkipSequential(DocValuesWrapper iterator, DocValuesSkipper skipper)+ throws IOException {+ if (skipper == null) {+ return 0;+ }++ assertEquals(-1, iterator.docID());+ assertEquals(-1, skipper.minDocID(0));+ assertEquals(-1, skipper.maxDocID(0));++ iterator.advance(0);+ int docCount = 0;+ while (true) {+ int previousMaxDoc = skipper.maxDocID(0);+ skipper.advance(previousMaxDoc + 1);+ assertTrue(skipper.minDocID(0) > previousMaxDoc);+ if (skipperHasAccurateDocBounds()) {+ assertEquals(iterator.docID(), skipper.minDocID(0));+ } else {+ assertTrue(+ "Expected: " + iterator.docID() + " but got " + skipper.minDocID(0),+ skipper.minDocID(0) <= iterator.docID());+ }++ if (skipper.minDocID(0) == NO_MORE_DOCS) {+ assertEquals(NO_MORE_DOCS, skipper.maxDocID(0));+ break;+ }+ assertTrue(skipper.docCount(0) > 0);++ int maxDoc = -1;+ long minVal = Long.MAX_VALUE;+ long maxVal = Long.MIN_VALUE;+ for (int i = 0; i < skipper.docCount(0); ++i) {+ assertNotEquals(NO_MORE_DOCS, iterator.docID());+ maxDoc = Math.max(maxDoc, iterator.docID());+ minVal = Math.min(minVal, iterator.minValue());+ maxVal = Math.max(maxVal, iterator.maxValue());+ iterator.advance(iterator.docID() + 1);+ }+ if (skipperHasAccurateDocBounds()) {+ assertEquals(maxDoc, skipper.maxDocID(0));+ } else {+ assertTrue(+ "Expected: " + maxDoc + " but got " + skipper.maxDocID(0),+ skipper.maxDocID(0) >= maxDoc);+ }+ if (skipperHasAccurateValueBounds()) {+ assertEquals(minVal, skipper.minValue(0));+ assertEquals(maxVal, skipper.maxValue(0));+ } else {+ assertTrue(+ "Expected: " + minVal + " but got " + skipper.minValue(0),+ minVal >= skipper.minValue(0));+ assertTrue(+ "Expected: " + maxVal + " but got " + skipper.maxValue(0),+ maxVal <= skipper.maxValue(0));+ }+ docCount += skipper.docCount(0);+ }++ assertEquals(docCount, skipper.docCount());+ return docCount;+ }++ private static void assertDocValuesSkipRandom(+ DocValuesWrapper iterator, DocValuesSkipper skipper, int maxDoc) throws IOException {+ if (skipper == null) {+ return;+ }+ while (true) {+ int doc = random().nextInt(skipper.maxDocID(0), maxDoc + 1) + 1;+ skipper.advance(doc);+ if (skipper.minDocID(0) == NO_MORE_DOCS) {+ assertEquals(NO_MORE_DOCS, skipper.maxDocID(0));+ return;+ }+ if (iterator.advanceExact(doc)) {+ assertTrue(iterator.docID() >= skipper.minDocID(0));+ assertTrue(iterator.docID() <= skipper.maxDocID(0));+ assertTrue(iterator.minValue() >= skipper.minValue(0));+ assertTrue(iterator.maxValue() <= skipper.maxValue(0));+ }+ }+ }++ private interface TestDocValueSkipper {++ void populateDoc(Document doc);++ DocValuesWrapper docValuesWrapper(LeafReader leafReader) throws IOException;++ DocValuesSkipper docValuesSkipper(LeafReader leafReader) throws IOException;+ }++ private interface DocValuesWrapper {++ int advance(int target) throws IOException;++ boolean advanceExact(int target) throws IOException;++ long maxValue() throws IOException;++ long minValue() throws IOException;++ int docID();+ }}diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseFieldInfoFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseFieldInfoFormatTestCase.javaindex a8af8b03a6b..11cb0acc733 100644--- a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseFieldInfoFormatTestCase.java+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseFieldInfoFormatTestCase.java@@ -19,6 +19,7 @@ package org.apache.lucene.tests.index;import com.carrotsearch.randomizedtesting.generators.RandomPicks;import java.io.IOException;import java.util.Collections;+import java.util.EnumSet;import java.util.HashMap;import java.util.HashSet;import java.util.Map;@@ -295,6 +296,15 @@ public abstract class BaseFieldInfoFormatTestCase extends BaseIndexFileFormatTesstorePayloads = random().nextBoolean();}}+ boolean hasDocValuesSkipIndex = false;+ if (EnumSet.of(+ DocValuesType.NUMERIC,+ DocValuesType.SORTED,+ DocValuesType.SORTED_NUMERIC,+ DocValuesType.SORTED_SET)+ .contains(fieldType.docValuesType())) {+ hasDocValuesSkipIndex = fieldType.hasDocValuesSkipIndex();+ }FieldInfo fi =new FieldInfo(field,@@ -304,6 +314,7 @@ public abstract class BaseFieldInfoFormatTestCase extends BaseIndexFileFormatTesstorePayloads,fieldType.indexOptions(),fieldType.docValuesType(),+ hasDocValuesSkipIndex,-1,new HashMap<>(),fieldType.pointDimensionCount(),@@ -349,8 +360,15 @@ public abstract class BaseFieldInfoFormatTestCase extends BaseIndexFileFormatTes}if (r.nextBoolean()) {- DocValuesType values[] = DocValuesType.values();+ DocValuesType[] values = DocValuesType.values();+ DocValuesType current = values[r.nextInt(values.length)];type.setDocValuesType(values[r.nextInt(values.length)]);+ if (current == DocValuesType.NUMERIC+ || current == DocValuesType.SORTED_NUMERIC+ || current == DocValuesType.SORTED+ || current == DocValuesType.SORTED_SET) {+ type.setDocValuesSkipIndex(random().nextBoolean());+ }}if (r.nextBoolean()) {@@ -389,6 +407,7 @@ public abstract class BaseFieldInfoFormatTestCase extends BaseIndexFileFormatTesassertEquals(expected.number, actual.number);assertEquals(expected.name, actual.name);assertEquals(expected.getDocValuesType(), actual.getDocValuesType());+ assertEquals(expected.hasDocValuesSkipIndex(), actual.hasDocValuesSkipIndex());assertEquals(expected.getIndexOptions(), actual.getIndexOptions());assertEquals(expected.hasNorms(), actual.hasNorms());assertEquals(expected.hasPayloads(), actual.hasPayloads());@@ -429,6 +448,7 @@ public abstract class BaseFieldInfoFormatTestCase extends BaseIndexFileFormatTesfalse,TextField.TYPE_STORED.indexOptions(),DocValuesType.NONE,+ false,-1,new HashMap<>(),0,diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseIndexFileFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseIndexFileFormatTestCase.javaindex 10e748363d7..6f2bc38ff64 100644--- a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseIndexFileFormatTestCase.java+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseIndexFileFormatTestCase.java@@ -357,6 +357,7 @@ public abstract class BaseIndexFileFormatTestCase extends LuceneTestCase {proto.hasPayloads(),proto.getIndexOptions(),proto.getDocValuesType(),+ proto.hasDocValuesSkipIndex(),proto.getDocValuesGen(),new HashMap<>(),proto.getPointDimensionCount(),diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/index/MergeReaderWrapper.java b/lucene/test-framework/src/java/org/apache/lucene/tests/index/MergeReaderWrapper.javaindex cc3084fec07..3fee110f783 100644--- a/lucene/test-framework/src/java/org/apache/lucene/tests/index/MergeReaderWrapper.java+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/index/MergeReaderWrapper.java@@ -26,6 +26,7 @@ import org.apache.lucene.codecs.TermVectorsReader;import org.apache.lucene.index.BinaryDocValues;import org.apache.lucene.index.ByteVectorValues;import org.apache.lucene.index.CodecReader;+import org.apache.lucene.index.DocValuesSkipper;import org.apache.lucene.index.DocValuesType;import org.apache.lucene.index.FieldInfo;import org.apache.lucene.index.FieldInfos;@@ -187,6 +188,17 @@ class MergeReaderWrapper extends LeafReader {return norms.getNorms(fi);}+ @Override+ public DocValuesSkipper getDocValuesSkipper(String field) throws IOException {+ ensureOpen();+ FieldInfo fi = getFieldInfos().fieldInfo(field);+ if (fi == null) {+ // Field does not exist+ return null;+ }+ return docValues.getSkipper(fi);+ }+@Overridepublic FieldInfos getFieldInfos() {return in.getFieldInfos();diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/index/MismatchedLeafReader.java b/lucene/test-framework/src/java/org/apache/lucene/tests/index/MismatchedLeafReader.javaindex e18c25ee0cf..ab907b76802 100644--- a/lucene/test-framework/src/java/org/apache/lucene/tests/index/MismatchedLeafReader.java+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/index/MismatchedLeafReader.java@@ -103,6 +103,7 @@ public class MismatchedLeafReader extends FilterLeafReader {oldInfo.hasPayloads(), // storePayloadsoldInfo.getIndexOptions(), // indexOptionsoldInfo.getDocValuesType(), // docValuesType+ oldInfo.hasDocValuesSkipIndex(), // hasDocValuesSkipIndexoldInfo.getDocValuesGen(), // dvGenoldInfo.attributes(), // attributesoldInfo.getPointDimensionCount(), // data dimension countdiff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/index/RandomPostingsTester.java b/lucene/test-framework/src/java/org/apache/lucene/tests/index/RandomPostingsTester.javaindex f8fe88b5cf2..5dea79f203a 100644--- a/lucene/test-framework/src/java/org/apache/lucene/tests/index/RandomPostingsTester.java+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/index/RandomPostingsTester.java@@ -157,6 +157,7 @@ public class RandomPostingsTester {true,IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS,DocValuesType.NONE,+ false,-1,new HashMap<>(),0,@@ -731,6 +732,7 @@ public class RandomPostingsTester {doPayloads,indexOptions,DocValuesType.NONE,+ false,-1,new HashMap<>(),0,diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/search/QueryUtils.java b/lucene/test-framework/src/java/org/apache/lucene/tests/search/QueryUtils.javaindex 0df805b01e4..efd13121d93 100644--- a/lucene/test-framework/src/java/org/apache/lucene/tests/search/QueryUtils.java+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/search/QueryUtils.java@@ -25,6 +25,7 @@ import java.util.List;import java.util.Random;import org.apache.lucene.index.BinaryDocValues;import org.apache.lucene.index.ByteVectorValues;+import org.apache.lucene.index.DocValuesSkipper;import org.apache.lucene.index.FieldInfos;import org.apache.lucene.index.FloatVectorValues;import org.apache.lucene.index.IndexReader;@@ -227,6 +228,11 @@ public class QueryUtils {return null;}+ @Override+ public DocValuesSkipper getDocValuesSkipper(String field) throws IOException {+ return null;+ }+@Overridepublic FloatVectorValues getFloatVectorValues(String field) throws IOException {return null;