Apache Lucene open-source search software
Dynamic pruning with DocValueSkipper (#14672)
Guo Feng9a7bcc9b5905
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex 8a6f1939079..b298eec2c62 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -100,6 +100,8 @@ Improvements* GITHUB#14602: Refactor the expressions compiler to use official ClassData BSM with indexed lookup(Uwe Schindler)+* GITHUB#14672: Numeric sort queries can take advantage of DocValuesSkipper to dynamic prune. (Guo Feng)+* GITHUB#14443: Support adaptive refresh in Searcher Managers (Vigya Sharma)* GITHUB#14737: Clean up redundant code and comments in query node classes. (Stefan Vodita)diff --git a/lucene/core/src/java/org/apache/lucene/search/DocValuesRangeIterator.java b/lucene/core/src/java/org/apache/lucene/search/DocValuesRangeIterator.javaindex 4b50e53e5c2..72066bc5f3e 100644--- a/lucene/core/src/java/org/apache/lucene/search/DocValuesRangeIterator.java+++ b/lucene/core/src/java/org/apache/lucene/search/DocValuesRangeIterator.java@@ -67,7 +67,7 @@ public final class DocValuesRangeIterator extends TwoPhaseIterator {// Track a decision for all doc IDs between the current doc ID and upTo inclusive.Match match = Match.MAYBE;- int upTo = -1;+ int upTo;Approximation(DocIdSetIterator innerApproximation,@@ -78,6 +78,7 @@ public final class DocValuesRangeIterator extends TwoPhaseIterator {this.skipper = skipper;this.lowerValue = lowerValue;this.upperValue = upperValue;+ this.upTo = skipper.maxDocID(0);}@Overridediff --git a/lucene/core/src/java/org/apache/lucene/search/comparators/NumericComparator.java b/lucene/core/src/java/org/apache/lucene/search/comparators/NumericComparator.javaindex 353d7ab39bf..37efe32ec7d 100644--- a/lucene/core/src/java/org/apache/lucene/search/comparators/NumericComparator.java+++ b/lucene/core/src/java/org/apache/lucene/search/comparators/NumericComparator.java@@ -19,16 +19,20 @@ package org.apache.lucene.search.comparators;import java.io.IOException;import org.apache.lucene.index.DocValues;+import org.apache.lucene.index.DocValuesSkipper;import org.apache.lucene.index.FieldInfo;+import org.apache.lucene.index.LeafReader;import org.apache.lucene.index.LeafReaderContext;import org.apache.lucene.index.NumericDocValues;import org.apache.lucene.index.PointValues;import org.apache.lucene.search.DocIdSetIterator;+import org.apache.lucene.search.DocValuesRangeIterator;import org.apache.lucene.search.FieldComparator;import org.apache.lucene.search.LeafFieldComparator;import org.apache.lucene.search.Pruning;import org.apache.lucene.search.Scorable;import org.apache.lucene.search.Scorer;+import org.apache.lucene.search.TwoPhaseIterator;import org.apache.lucene.util.DocIdSetBuilder;import org.apache.lucene.util.IntsRef;@@ -95,64 +99,25 @@ public abstract class NumericComparator<T extends Number> extends FieldComparatopublic abstract class NumericLeafComparator implements LeafFieldComparator {private final LeafReaderContext context;protected final NumericDocValues docValues;- private final PointValues pointValues;- // lazily constructed to avoid performance overhead when this is not used- private PointValues.PointTree pointTree;- // if skipping functionality should be enabled on this segment- private final boolean enableSkipping;- private final int maxDoc;-- /** According to {@link FieldComparator#setTopValue}, topValueSet is final in leafComparator */- private final boolean leafTopSet = topValueSet;-- private long minValueAsLong = Long.MIN_VALUE;- private long maxValueAsLong = Long.MAX_VALUE;-- private final UpdateableDocIdSetIterator competitiveIterator;- private long iteratorCost = -1;- private int maxDocVisited = -1;- private int updateCounter = 0;- private int currentSkipInterval = MIN_SKIP_INTERVAL;- // helps to be conservative about increasing the sampling interval- private int tryUpdateFailCount = 0;+ private final CompetitiveDISIBuilder competitiveDISIBuilder;public NumericLeafComparator(LeafReaderContext context) throws IOException {this.context = context;this.docValues = getNumericDocValues(context, field);- this.pointValues = pruning != Pruning.NONE ? context.reader().getPointValues(field) : null;- if (pointValues != null) {- FieldInfo info = context.reader().getFieldInfos().fieldInfo(field);- if (info == null || info.getPointDimensionCount() == 0) {- throw new IllegalStateException(- "Field "- + field- + " doesn't index points according to FieldInfos yet returns non-null PointValues");- } else if (info.getPointDimensionCount() > 1) {- throw new IllegalArgumentException(- "Field " + field + " is indexed with multiple dimensions, sorting is not supported");- } else if (info.getPointNumBytes() != bytesCount) {- throw new IllegalArgumentException(- "Field "- + field- + " is indexed with "- + info.getPointNumBytes()- + " bytes per dimension, but "- + NumericComparator.this- + " expected "- + bytesCount);- }- this.enableSkipping = true; // skipping is enabled when points are available- this.maxDoc = context.reader().maxDoc();- this.competitiveIterator = new UpdateableDocIdSetIterator();- this.competitiveIterator.update(DocIdSetIterator.all(maxDoc));- if (leafTopSet) {- encodeTop();+ CompetitiveDISIBuilder builder = null;+ if (pruning != Pruning.NONE) {+ LeafReader reader = context.reader();+ PointValues pointValues = reader.getPointValues(field);+ if (pointValues != null) {+ builder = new PointsCompetitiveDISIBuilder(pointValues, this);+ } else {+ DocValuesSkipper skipper = reader.getDocValuesSkipper(field);+ if (skipper != null) {+ builder = new DVSkipperCompetitiveDISIBuilder(skipper, this);+ }}- } else {- this.enableSkipping = false;- this.competitiveIterator = null;- this.maxDoc = 0;}+ competitiveDISIBuilder = builder;}/**@@ -175,42 +140,82 @@ public abstract class NumericComparator<T extends Number> extends FieldComparato@Overridepublic void setBottom(int slot) throws IOException {queueFull = true; // if we are setting bottom, it means that we have collected enough hits- updateCompetitiveIterator(); // update an iterator if we set a new bottom+ if (competitiveDISIBuilder != null) {+ competitiveDISIBuilder.updateCompetitiveIterator();+ }}@Overridepublic void copy(int slot, int doc) throws IOException {- maxDocVisited = doc;+ if (competitiveDISIBuilder != null) {+ competitiveDISIBuilder.setMaxDocVisited(doc);+ }}@Overridepublic void setScorer(Scorable scorer) throws IOException {- if (iteratorCost == -1) {- if (scorer instanceof Scorer) {- iteratorCost =- ((Scorer) scorer).iterator().cost(); // starting iterator cost is the scorer's cost- } else {- iteratorCost = maxDoc;- }- updateCompetitiveIterator(); // update an iterator when we have a new segment+ if (competitiveDISIBuilder != null) {+ competitiveDISIBuilder.setScorer(scorer);}}@Overridepublic void setHitsThresholdReached() throws IOException {hitsThresholdReached = true;- updateCompetitiveIterator();+ if (competitiveDISIBuilder != null) {+ competitiveDISIBuilder.updateCompetitiveIterator();+ }}- // update its iterator to include possibly only docs that are "stronger" than the current bottom- // entry- private void updateCompetitiveIterator() throws IOException {- if (enableSkipping == false- || hitsThresholdReached == false- || (leafTopSet == false && queueFull == false)) return;+ @Override+ public DocIdSetIterator competitiveIterator() {+ return competitiveDISIBuilder == null ? null : competitiveDISIBuilder.competitiveIterator;+ }++ protected abstract long bottomAsComparableLong();++ protected abstract long topAsComparableLong();+ }++ private abstract class CompetitiveDISIBuilder {++ final int maxDoc;+ final NumericLeafComparator leafComparator;++ /** According to {@link FieldComparator#setTopValue}, topValueSet is final in leafComparator */+ final boolean leafTopSet = topValueSet;++ final UpdateableDocIdSetIterator competitiveIterator = new UpdateableDocIdSetIterator();+ long minValueAsLong = Long.MIN_VALUE;+ long maxValueAsLong = Long.MAX_VALUE;+ int maxDocVisited = -1;+ int updateCounter = 0;+ int currentSkipInterval = MIN_SKIP_INTERVAL;++ CompetitiveDISIBuilder(NumericLeafComparator leafComparator) {+ this.leafComparator = leafComparator;+ this.maxDoc = leafComparator.context.reader().maxDoc();+ this.competitiveIterator.update(DocIdSetIterator.all(maxDoc));+ if (leafTopSet) {+ encodeTop();+ }+ }++ void setScorer(Scorable scorer) throws IOException {}++ abstract int docCount();++ final void updateCompetitiveIterator() throws IOException {+ if (hitsThresholdReached == false) {+ return;+ }+ if (leafTopSet == false && queueFull == false) {+ return;+ }+// if some documents have missing points, check that missing values prohibits optimization- if ((pointValues.getDocCount() < maxDoc) && isMissingValueCompetitive()) {- return; // we can't filter out documents, as documents with missing values are competitive+ if (docCount() < maxDoc && isMissingValueCompetitive()) {+ return;}updateCounter++;@@ -224,6 +229,143 @@ public abstract class NumericComparator<T extends Number> extends FieldComparatoencodeBottom();}+ doUpdateCompetitiveIterator();+ }++ abstract void doUpdateCompetitiveIterator() throws IOException;++ private void setMaxDocVisited(int maxDocVisited) {+ this.maxDocVisited = maxDocVisited;+ }++ /**+ * If {@link NumericComparator#pruning} equals {@link Pruning#GREATER_THAN_OR_EQUAL_TO}, we+ * could better tune the {@link #maxValueAsLong}/{@link #minValueAsLong}. For instance, if the+ * sort is ascending and bottom value is 5, we will use a range on [MIN_VALUE, 4].+ */+ private void encodeBottom() {+ if (reverse == false) {+ maxValueAsLong = leafComparator.bottomAsComparableLong();+ if (pruning == Pruning.GREATER_THAN_OR_EQUAL_TO && maxValueAsLong != Long.MIN_VALUE) {+ maxValueAsLong--;+ }+ } else {+ minValueAsLong = leafComparator.bottomAsComparableLong();+ if (pruning == Pruning.GREATER_THAN_OR_EQUAL_TO && minValueAsLong != Long.MAX_VALUE) {+ minValueAsLong++;+ }+ }+ }++ /**+ * If {@link NumericComparator#pruning} equals {@link Pruning#GREATER_THAN_OR_EQUAL_TO}, we+ * could better tune the {@link #minValueAsLong}/{@link #minValueAsLong}. For instance, if the+ * sort is ascending and top value is 3, we will use a range on [4, MAX_VALUE].+ */+ private void encodeTop() {+ if (reverse == false) {+ minValueAsLong = leafComparator.topAsComparableLong();+ if (singleSort+ && pruning == Pruning.GREATER_THAN_OR_EQUAL_TO+ && queueFull+ && minValueAsLong != Long.MAX_VALUE) {+ minValueAsLong++;+ }+ } else {+ maxValueAsLong = leafComparator.topAsComparableLong();+ if (singleSort+ && pruning == Pruning.GREATER_THAN_OR_EQUAL_TO+ && queueFull+ && maxValueAsLong != Long.MIN_VALUE) {+ maxValueAsLong--;+ }+ }+ }++ boolean isMissingValueCompetitive() {+ // if queue is full, compare with bottom first,+ // if competitive, then check if we can compare with topValue+ if (queueFull) {+ int result = Long.compare(missingValueAsLong, leafComparator.bottomAsComparableLong());+ // in reverse (desc) sort missingValue is competitive when it's greater or equal to bottom,+ // in asc sort missingValue is competitive when it's smaller or equal to bottom+ final boolean competitive =+ reverse+ ? (pruning == Pruning.GREATER_THAN_OR_EQUAL_TO ? result > 0 : result >= 0)+ : (pruning == Pruning.GREATER_THAN_OR_EQUAL_TO ? result < 0 : result <= 0);+ if (competitive == false) {+ return false;+ }+ }++ if (leafTopSet) {+ int result = Long.compare(missingValueAsLong, leafComparator.topAsComparableLong());+ // in reverse (desc) sort missingValue is competitive when it's smaller or equal to+ // topValue,+ // in asc sort missingValue is competitive when it's greater or equal to topValue+ return reverse ? (result <= 0) : (result >= 0);+ }++ // by default competitive+ return true;+ }+ }++ private class PointsCompetitiveDISIBuilder extends CompetitiveDISIBuilder {++ private final PointValues pointValues;+ // lazily constructed to avoid performance overhead when this is not used+ private PointValues.PointTree pointTree;+ private long iteratorCost = -1;+ // helps to be conservative about increasing the sampling interval+ private int tryUpdateFailCount = 0;++ public PointsCompetitiveDISIBuilder(PointValues pointValues, NumericLeafComparator comparator) {+ super(comparator);+ LeafReaderContext context = comparator.context;+ FieldInfo info = context.reader().getFieldInfos().fieldInfo(field);+ if (info == null || info.getPointDimensionCount() == 0) {+ throw new IllegalStateException(+ "Field "+ + field+ + " doesn't index points according to FieldInfos yet returns non-null PointValues");+ } else if (info.getPointDimensionCount() > 1) {+ throw new IllegalArgumentException(+ "Field " + field + " is indexed with multiple dimensions, sorting is not supported");+ } else if (info.getPointNumBytes() != bytesCount) {+ throw new IllegalArgumentException(+ "Field "+ + field+ + " is indexed with "+ + info.getPointNumBytes()+ + " bytes per dimension, but "+ + NumericComparator.this+ + " expected "+ + bytesCount);+ }+ this.pointValues = pointValues;+ }++ @Override+ void setScorer(Scorable scorer) throws IOException {+ if (iteratorCost == -1) {+ if (scorer instanceof Scorer) {+ iteratorCost =+ ((Scorer) scorer).iterator().cost(); // starting iterator cost is the scorer's cost+ } else {+ iteratorCost = maxDoc;+ }+ updateCompetitiveIterator(); // update an iterator when we have a new segment+ }+ }++ @Override+ int docCount() {+ return pointValues.getDocCount();+ }++ @Override+ void doUpdateCompetitiveIterator() throws IOException {DocIdSetBuilder result = new DocIdSetBuilder(maxDoc);PointValues.IntersectVisitor visitor =new PointValues.IntersectVisitor() {@@ -295,7 +437,8 @@ public abstract class NumericComparator<T extends Number> extends FieldComparatoupdateSkipInterval(false);if (pointValues.getDocCount() < iteratorCost) {// Use the set of doc with values to help drive iteration- competitiveIterator.update(getNumericDocValues(context, field));+ competitiveIterator.update(+ leafComparator.getNumericDocValues(leafComparator.context, field));iteratorCost = pointValues.getDocCount();}return;@@ -328,88 +471,44 @@ public abstract class NumericComparator<T extends Number> extends FieldComparato}}}+ }- /**- * If {@link NumericComparator#pruning} equals {@link Pruning#GREATER_THAN_OR_EQUAL_TO}, we- * could better tune the {@link NumericLeafComparator#maxValueAsLong}/{@link- * NumericLeafComparator#minValueAsLong}. For instance, if the sort is ascending and bottom- * value is 5, we will use a range on [MIN_VALUE, 4].- */- private void encodeBottom() {- if (reverse == false) {- maxValueAsLong = bottomAsComparableLong();- if (pruning == Pruning.GREATER_THAN_OR_EQUAL_TO && maxValueAsLong != Long.MIN_VALUE) {- maxValueAsLong--;- }- } else {- minValueAsLong = bottomAsComparableLong();- if (pruning == Pruning.GREATER_THAN_OR_EQUAL_TO && minValueAsLong != Long.MAX_VALUE) {- minValueAsLong++;- }- }- }+ private class DVSkipperCompetitiveDISIBuilder extends CompetitiveDISIBuilder {- /**- * If {@link NumericComparator#pruning} equals {@link Pruning#GREATER_THAN_OR_EQUAL_TO}, we- * could better tune the {@link NumericLeafComparator#minValueAsLong}/{@link- * NumericLeafComparator#minValueAsLong}. For instance, if the sort is ascending and top value- * is 3, we will use a range on [4, MAX_VALUE].- */- private void encodeTop() {- if (reverse == false) {- minValueAsLong = topAsComparableLong();- if (singleSort- && pruning == Pruning.GREATER_THAN_OR_EQUAL_TO- && queueFull- && minValueAsLong != Long.MAX_VALUE) {- minValueAsLong++;- }- } else {- maxValueAsLong = topAsComparableLong();- if (singleSort- && pruning == Pruning.GREATER_THAN_OR_EQUAL_TO- && queueFull- && maxValueAsLong != Long.MIN_VALUE) {- maxValueAsLong--;- }- }- }-- private boolean isMissingValueCompetitive() {- // if queue is full, compare with bottom first,- // if competitive, then check if we can compare with topValue- if (queueFull) {- int result = Long.compare(missingValueAsLong, bottomAsComparableLong());- // in reverse (desc) sort missingValue is competitive when it's greater or equal to bottom,- // in asc sort missingValue is competitive when it's smaller or equal to bottom- final boolean competitive =- reverse- ? (pruning == Pruning.GREATER_THAN_OR_EQUAL_TO ? result > 0 : result >= 0)- : (pruning == Pruning.GREATER_THAN_OR_EQUAL_TO ? result < 0 : result <= 0);- if (competitive == false) {- return false;- }- }+ private final DocValuesSkipper skipper;+ private final TwoPhaseIterator innerTwoPhase;- if (leafTopSet) {- int result = Long.compare(missingValueAsLong, topAsComparableLong());- // in reverse (desc) sort missingValue is competitive when it's smaller or equal to- // topValue,- // in asc sort missingValue is competitive when it's greater or equal to topValue- return reverse ? (result <= 0) : (result >= 0);- }+ public DVSkipperCompetitiveDISIBuilder(+ DocValuesSkipper skipper, NumericLeafComparator leafComparator) throws IOException {+ super(leafComparator);+ this.skipper = skipper;+ NumericDocValues docValues =+ leafComparator.getNumericDocValues(leafComparator.context, field);+ innerTwoPhase =+ new TwoPhaseIterator(docValues) {+ @Override+ public boolean matches() throws IOException {+ final long value = docValues.longValue();+ return value >= minValueAsLong && value <= maxValueAsLong;+ }- // by default competitive- return true;+ @Override+ public float matchCost() {+ return 2; // 2 comparisons+ }+ };}@Override- public DocIdSetIterator competitiveIterator() {- return competitiveIterator;+ int docCount() {+ return skipper.docCount();}- protected abstract long bottomAsComparableLong();-- protected abstract long topAsComparableLong();+ @Override+ void doUpdateCompetitiveIterator() {+ TwoPhaseIterator twoPhaseIterator =+ new DocValuesRangeIterator(innerTwoPhase, skipper, minValueAsLong, maxValueAsLong, false);+ competitiveIterator.update(TwoPhaseIterator.asDocIdSetIterator(twoPhaseIterator));+ }}}diff --git a/lucene/core/src/test/org/apache/lucene/search/TestSortOptimization.java b/lucene/core/src/test/org/apache/lucene/search/TestSortOptimization.javaindex bc8a6e0125a..3bc0cc1d024 100644--- a/lucene/core/src/test/org/apache/lucene/search/TestSortOptimization.java+++ b/lucene/core/src/test/org/apache/lucene/search/TestSortOptimization.java@@ -24,6 +24,7 @@ import java.io.IOException;import java.util.ArrayList;import java.util.Collections;import java.util.List;+import java.util.function.BiFunction;import org.apache.lucene.document.Document;import org.apache.lucene.document.Field;import org.apache.lucene.document.FloatDocValuesField;@@ -45,6 +46,7 @@ import org.apache.lucene.index.IndexOptions;import org.apache.lucene.index.IndexReader;import org.apache.lucene.index.IndexWriter;import org.apache.lucene.index.IndexWriterConfig;+import org.apache.lucene.index.IndexableField;import org.apache.lucene.index.LeafReader;import org.apache.lucene.index.PointValues;import org.apache.lucene.index.StoredFields;@@ -65,8 +67,19 @@ import org.apache.lucene.util.IOUtils;public class TestSortOptimization extends LuceneTestCase {- public void testLongSortOptimization() throws IOException {+ public void testLongSortOptimizationPointIndex() throws Exception {+ testLongSortOptimization(+ (field, value) ->+ List.of(new LongPoint(field, value), new NumericDocValuesField(field, value)));+ }++ public void testLongSortOptimizationSkipperIndex() throws Exception {+ testLongSortOptimization(+ (field, value) -> List.of(NumericDocValuesField.indexedField(field, value)));+ }+ private void testLongSortOptimization(+ BiFunction<String, Integer, List<IndexableField>> fieldsBuilder) throws IOException {final Directory dir = newDirectory();IndexWriterConfig config =new IndexWriterConfig()@@ -77,8 +90,7 @@ public class TestSortOptimization extends LuceneTestCase {final int numDocs = atLeast(10000);for (int i = 0; i < numDocs; ++i) {final Document doc = new Document();- doc.add(new NumericDocValuesField("my_field", i));- doc.add(new LongPoint("my_field", i));+ fieldsBuilder.apply("my_field", i).forEach(doc::add);writer.addDocument(doc);if (i == 7000) writer.flush(); // two segments}@@ -175,7 +187,19 @@ public class TestSortOptimization extends LuceneTestCase {dir.close();}- public void testSortOptimizationWithMissingValues() throws IOException {+ public void testSortOptimizationWithMissingValuesPointIndex() throws Exception {+ testSortOptimizationWithMissingValues(+ (field, value) ->+ List.of(new LongPoint(field, value), new NumericDocValuesField(field, value)));+ }++ public void testSortOptimizationWithMissingValuesSkipperIndex() throws Exception {+ testSortOptimizationWithMissingValues(+ (field, value) -> List.of(NumericDocValuesField.indexedField(field, value)));+ }++ private void testSortOptimizationWithMissingValues(+ BiFunction<String, Integer, List<IndexableField>> fieldsBuilder) throws IOException {final Directory dir = newDirectory();IndexWriterConfig config =new IndexWriterConfig()@@ -187,8 +211,7 @@ public class TestSortOptimization extends LuceneTestCase {for (int i = 0; i < numDocs; ++i) {final Document doc = new Document();if ((i % 500) != 0) { // miss values on every 500th document- doc.add(new NumericDocValuesField("my_field", i));- doc.add(new LongPoint("my_field", i));+ fieldsBuilder.apply("my_field", i).forEach(doc::add);}writer.addDocument(doc);if (i == 7000) writer.flush(); // two segments@@ -269,18 +292,26 @@ public class TestSortOptimization extends LuceneTestCase {assertEquals(afterValue + 1 + i, fieldDoc.fields[0]);}assertEquals(TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO, topDocs.totalHits.relation());- // expect to skip all but the first leaf in the BKD tree in the first segment as well as the- // second segment- // doc-0 has no target field, so we need to minus 1- final int expectedSkipped = (7001 - 512 - 1) + (numDocs - 7001);- assertNonCompetitiveHitsAreSkipped(topDocs.totalHits.value(), numDocs - expectedSkipped + 1);+ assertNonCompetitiveHitsAreSkipped(topDocs.totalHits.value(), numDocs);}reader.close();dir.close();}- public void testNumericDocValuesOptimizationWithMissingValues() throws IOException {+ public void testNumericDocValuesOptimizationWithMissingValuesPointIndex() throws Exception {+ testNumericDocValuesOptimizationWithMissingValues(+ (field, value) ->+ List.of(new LongPoint(field, value), new NumericDocValuesField(field, value)));+ }++ public void testNumericDocValuesOptimizationWithMissingValuesSkipperIndex() throws Exception {+ testNumericDocValuesOptimizationWithMissingValues(+ (field, value) -> List.of(NumericDocValuesField.indexedField(field, value)));+ }++ private void testNumericDocValuesOptimizationWithMissingValues(+ BiFunction<String, Integer, List<IndexableField>> fieldsBuilder) throws IOException {final Directory dir = newDirectory();IndexWriterConfig config =new IndexWriterConfig()@@ -294,8 +325,7 @@ public class TestSortOptimization extends LuceneTestCase {final Document doc = new Document();if (i <= missValuesNumDocs) { // missing value document} else {- doc.add(new NumericDocValuesField("my_field", i));- doc.add(new LongPoint("my_field", i));+ fieldsBuilder.apply("my_field", i).forEach(doc::add);}writer.addDocument(doc);}@@ -348,7 +378,19 @@ public class TestSortOptimization extends LuceneTestCase {dir.close();}- public void testSortOptimizationEqualValues() throws IOException {+ public void testSortOptimizationEqualValuesPointIndex() throws Exception {+ testSortOptimizationEqualValues(+ (field, value) ->+ List.of(new IntPoint(field, value), new NumericDocValuesField(field, value)));+ }++ public void testSortOptimizationEqualValuesSkipperIndex() throws Exception {+ testSortOptimizationEqualValues(+ (field, value) -> List.of(NumericDocValuesField.indexedField(field, value)));+ }++ private void testSortOptimizationEqualValues(+ BiFunction<String, Integer, List<IndexableField>> fieldsBuilder) throws IOException {final Directory dir = newDirectory();IndexWriterConfig config =new IndexWriterConfig()@@ -359,9 +401,7 @@ public class TestSortOptimization extends LuceneTestCase {final int numDocs = atLeast(TEST_NIGHTLY ? 50_000 : 10_000);for (int i = 1; i <= numDocs; ++i) {final Document doc = new Document();- doc.add(- new NumericDocValuesField("my_field1", 100)); // all docs have the same value of my_field1- doc.add(new IntPoint("my_field1", 100));+ fieldsBuilder.apply("my_field1", 100).forEach(doc::add);doc.add(new NumericDocValuesField("my_field2", numDocs - i)); // diff values for the field my_field2@@ -429,15 +469,26 @@ public class TestSortOptimization extends LuceneTestCase {dir.close();}- public void testFloatSortOptimization() throws IOException {+ public void testFloatSortOptimizationPointIndex() throws Exception {+ testFloatSortOptimization(+ (field, value) ->+ List.of(new FloatPoint(field, value), new FloatDocValuesField(field, value)));+ }++ public void testFloatSortOptimizationSkpperIndex() throws Exception {+ testFloatSortOptimization(+ (field, value) ->+ List.of(FloatDocValuesField.indexedField(field, Float.floatToRawIntBits(value))));+ }++ private void testFloatSortOptimization(+ BiFunction<String, Float, List<IndexableField>> fieldsBuilder) throws IOException {final Directory dir = newDirectory();final IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig());final int numDocs = atLeast(10000);for (int i = 0; i < numDocs; ++i) {final Document doc = new Document();- float f = 1f * i;- doc.add(new FloatDocValuesField("my_field", f));- doc.add(new FloatPoint("my_field", i));+ fieldsBuilder.apply("my_field", 1f * i).forEach(doc::add);writer.addDocument(doc);}final DirectoryReader reader = DirectoryReader.open(writer);