Skip to content

Apache Lucene open-source search software

Smarter use of sort-field optimizations in DV range queries (#16424)

Alan Woodward693a7234bad2
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index c54fc9a987c..44924919b4f 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -391,6 +391,9 @@ Optimizations
* GITHUB#16380: Speed up filtered disjunction queries when the filter is part of the
index sort by advancing the filter before calculating impacts. (Alan Woodward)
+* GITHUB#16424: Use secondary or lower sorts to accelerate XXDocValuesRangeQueries if the primary
+ sort field is empty or only contains a single value. (Alan Woodward)
+
* GITHUB#16180: Add bulk intoBitSet for SortedDocValues ordinal ranges. (Prithvi S)
Bug Fixes
diff --git a/lucene/core/src/java/org/apache/lucene/document/SortedNumericDocValuesRangeQuery.java b/lucene/core/src/java/org/apache/lucene/document/SortedNumericDocValuesRangeQuery.java
index f00a4e72c35..16ccd65c679 100644
--- a/lucene/core/src/java/org/apache/lucene/document/SortedNumericDocValuesRangeQuery.java
+++ b/lucene/core/src/java/org/apache/lucene/document/SortedNumericDocValuesRangeQuery.java
@@ -138,11 +138,13 @@ final class SortedNumericDocValuesRangeQuery extends NumericDocValuesRangeQuery
final NumericDocValues singleton = DocValues.unwrapSingleton(values);
final DocValuesSkipper skipper = context.reader().getDocValuesSkipper(field);
- final SortField primarySortField;
if (singleton != null) {
- if (skipper != null
- && (primarySortField = densePrimarySort(context.reader(), skipper)) != null) {
- return getScorerSupplierFromDensePrimarySort(singleton, skipper, primarySortField);
+ if (skipper != null) {
+ SortField primarySortField =
+ canUsePrimarySortShortcut(context.reader(), field, skipper);
+ if (primarySortField != null) {
+ return getScorerSupplierFromDensePrimarySort(singleton, skipper, primarySortField);
+ }
}
// A single two-phase iterator covers every density: its approximation rides the skipper
// (no over-scan), its intoBitSet bulk-evaluates dense blocks, and YES runs collect as
@@ -163,7 +165,7 @@ final class SortedNumericDocValuesRangeQuery extends NumericDocValuesRangeQuery
}
@Override
- public int count(LeafReaderContext context) throws IOException {
+ public int count(LeafReaderContext context) {
int maxDoc = context.reader().maxDoc();
int cnt = docCountIgnoringDeletes(context);
if (cnt == maxDoc) {
@@ -177,7 +179,7 @@ final class SortedNumericDocValuesRangeQuery extends NumericDocValuesRangeQuery
* # docs within the query range ignoring any deleted documents
* -1 if # docs cannot be determined efficiently
*/
- private int docCountIgnoringDeletes(LeafReaderContext context) throws IOException {
+ private int docCountIgnoringDeletes(LeafReaderContext context) {
final DocValuesSkipper skipper = context.reader().getDocValuesSkipper(field);
if (skipper != null) {
if (skipper.minValue() > upperValue || skipper.maxValue() < lowerValue) {
@@ -224,16 +226,23 @@ final class SortedNumericDocValuesRangeQuery extends NumericDocValuesRangeQuery
};
}
- private SortField densePrimarySort(LeafReader reader, DocValuesSkipper skipper) {
- if (skipper.docCount() != reader.maxDoc()) {
+ /**
+ * Returns the primary sort field if it is safe to use the {@link SortedSkipperScorerSupplier}
+ * range shortcuts for this query, or {@code null} if the shortcut cannot be applied.
+ *
+ * <p>The shortcuts ({@code skipperMinDocId=0} and {@code skipperMaxDocId=docCount}) assume that
+ * docs without a value do not appear within or before the matching doc-ID range. This requires
+ * that the field is dense (every doc has a value, {@code docCount == reader.maxDoc()}).
+ */
+ private static SortField canUsePrimarySortShortcut(
+ LeafReader reader, String field, DocValuesSkipper skipper) {
+ SortField sf = Sort.getPrimarySortField(reader);
+ if (sf == null || sf.getField().equals(field) == false) {
return null;
}
- final Sort indexSort = reader.getMetaData().sort();
- if (indexSort == null
- || indexSort.getSort().length == 0
- || indexSort.getSort()[0].getField().equals(field) == false) {
+ if (skipper.docCount() != reader.maxDoc()) {
return null;
}
- return indexSort.getSort()[0];
+ return sf;
}
}
diff --git a/lucene/core/src/java/org/apache/lucene/document/SortedSetDocValuesRangeQuery.java b/lucene/core/src/java/org/apache/lucene/document/SortedSetDocValuesRangeQuery.java
index 3022ade0f85..a4228bd3453 100644
--- a/lucene/core/src/java/org/apache/lucene/document/SortedSetDocValuesRangeQuery.java
+++ b/lucene/core/src/java/org/apache/lucene/document/SortedSetDocValuesRangeQuery.java
@@ -121,12 +121,13 @@ final class SortedSetDocValuesRangeQuery extends Query {
DocValuesSkipper skipper = context.reader().getDocValuesSkipper(field);
SortedSetDocValues values = DocValues.getSortedSet(context.reader(), field);
final SortedDocValues singleton = DocValues.unwrapSingleton(values);
- final SortField primarySortField;
- if (singleton != null
- && skipper != null
- && (primarySortField = densePrimarySort(context.reader(), skipper)) != null) {
- return getScorerSupplierFromDensePrimarySort(
- singleton, values, skipper, primarySortField);
+ if (singleton != null && skipper != null) {
+ SortField primarySortField =
+ canUsePrimarySortShortcut(context.reader(), field, skipper.docCount());
+ if (primarySortField != null) {
+ return getScorerSupplierFromDensePrimarySort(
+ singleton, values, skipper, primarySortField);
+ }
}
// implement ScorerSupplier, since we do some expensive stuff to make a scorer
return new ConstantScoreScorerSupplier(score(), scoreMode, context.reader().maxDoc()) {
@@ -276,16 +277,23 @@ final class SortedSetDocValuesRangeQuery extends Query {
return maxOrd;
}
- private SortField densePrimarySort(LeafReader reader, DocValuesSkipper skipper) {
- if (skipper.docCount() != reader.maxDoc()) {
+ /**
+ * Returns the primary sort field if it is safe to use the {@link SortedSkipperScorerSupplier}
+ * range shortcuts for this query, or {@code null} if the shortcut cannot be applied.
+ *
+ * <p>The shortcuts assume that docs without a value do not appear before or within the matching
+ * doc-ID range. This requires that the field is dense (every doc has a value, {@code docCount ==
+ * reader.maxDoc()}).
+ */
+ private static SortField canUsePrimarySortShortcut(LeafReader reader, String field, int docCount)
+ throws IOException {
+ SortField sf = Sort.getPrimarySortField(reader);
+ if (sf == null || sf.getField().equals(field) == false) {
return null;
}
- final Sort indexSort = reader.getMetaData().sort();
- if (indexSort == null
- || indexSort.getSort().length == 0
- || indexSort.getSort()[0].getField().equals(field) == false) {
+ if (docCount != reader.maxDoc()) {
return null;
}
- return indexSort.getSort()[0];
+ return sf;
}
}
diff --git a/lucene/core/src/java/org/apache/lucene/search/IndexSortSortedNumericDocValuesRangeQuery.java b/lucene/core/src/java/org/apache/lucene/search/IndexSortSortedNumericDocValuesRangeQuery.java
index fb4afd4c795..83e907dcd50 100644
--- a/lucene/core/src/java/org/apache/lucene/search/IndexSortSortedNumericDocValuesRangeQuery.java
+++ b/lucene/core/src/java/org/apache/lucene/search/IndexSortSortedNumericDocValuesRangeQuery.java
@@ -190,21 +190,19 @@ public class IndexSortSortedNumericDocValuesRangeQuery extends NumericDocValuesR
}
// use index sort optimization if possible
- Sort indexSort = reader.getMetaData().sort();
- if (indexSort != null
- && indexSort.getSort().length > 0
- && indexSort.getSort()[0].getField().equals(field)) {
- final SortField sortField = indexSort.getSort()[0];
- final SortField.Type sortFieldType = getSortFieldType(sortField);
+ SortField primarySortField = Sort.getPrimarySortField(reader);
+ if (primarySortField != null && primarySortField.getField().equals(field)) {
+ final SortField.Type sortFieldType = getSortFieldType(primarySortField);
// The index sort optimization is only supported for Type.INT and Type.LONG
if (sortFieldType == Type.INT || sortFieldType == Type.LONG) {
- Object missingValue = sortField.getMissingValue();
+ Object missingValue = primarySortField.getMissingValue();
final long missingLongValue =
missingValue == null ? 0L : ((Number) missingValue).longValue();
// all documents have docValues or missing value falls outside the range
if ((pointValues != null && pointValues.getDocCount() == reader.maxDoc())
|| (missingLongValue < lowerValue || missingLongValue > upperValue)) {
- itAndCount = getDocIdSetIterator(sortField, sortFieldType, context, numericValues);
+ itAndCount =
+ getDocIdSetIterator(primarySortField, sortFieldType, context, numericValues);
}
if (itAndCount != null && itAndCount.count != -1) {
return itAndCount.count;
@@ -419,14 +417,12 @@ public class IndexSortSortedNumericDocValuesRangeQuery extends NumericDocValuesR
private IteratorAndCount getDocIdSetIteratorOrNullFromBkd(
LeafReaderContext context, DocIdSetIterator delegate) throws IOException {
- Sort indexSort = context.reader().getMetaData().sort();
- if (indexSort == null
- || indexSort.getSort().length == 0
- || indexSort.getSort()[0].getField().equals(field) == false) {
+ SortField primarySortField = Sort.getPrimarySortField(context.reader());
+ if (primarySortField == null || primarySortField.getField().equals(field) == false) {
return null;
}
- final boolean reverse = indexSort.getSort()[0].getReverse();
+ final boolean reverse = primarySortField.getReverse();
PointValues points = context.reader().getPointValues(field);
if (points == null) {
@@ -519,16 +515,12 @@ public class IndexSortSortedNumericDocValuesRangeQuery extends NumericDocValuesR
if (itAndCount != null) {
return itAndCount;
}
- Sort indexSort = context.reader().getMetaData().sort();
- if (indexSort != null
- && indexSort.getSort().length > 0
- && indexSort.getSort()[0].getField().equals(field)) {
-
- final SortField sortField = indexSort.getSort()[0];
- final SortField.Type sortFieldType = getSortFieldType(sortField);
+ SortField primarySortField = Sort.getPrimarySortField(context.reader());
+ if (primarySortField != null && primarySortField.getField().equals(field)) {
+ final SortField.Type sortFieldType = getSortFieldType(primarySortField);
// The index sort optimization is only supported for Type.INT and Type.LONG
if (sortFieldType == Type.INT || sortFieldType == Type.LONG) {
- return getDocIdSetIterator(sortField, sortFieldType, context, numericValues);
+ return getDocIdSetIterator(primarySortField, sortFieldType, context, numericValues);
}
}
}
diff --git a/lucene/core/src/java/org/apache/lucene/search/Sort.java b/lucene/core/src/java/org/apache/lucene/search/Sort.java
index f46515e8ea3..a037e0751ed 100644
--- a/lucene/core/src/java/org/apache/lucene/search/Sort.java
+++ b/lucene/core/src/java/org/apache/lucene/search/Sort.java
@@ -18,6 +18,8 @@ package org.apache.lucene.search;
import java.io.IOException;
import java.util.Arrays;
+import org.apache.lucene.index.DocValuesSkipper;
+import org.apache.lucene.index.LeafReader;
/**
* Encapsulates sort criteria for returned hits.
@@ -132,4 +134,41 @@ public final class Sort {
}
return false;
}
+
+ /**
+ * Returns the primary index sort field for the given LeafReader, or null if none is configured.
+ *
+ * <p>If sorting against a field would have no effect then that SortField is skipped over and the
+ * next SortField in the list is returned. This can happen for fields with no values in the
+ * segment, or fields that have only a single distinct value.
+ */
+ public static SortField getPrimarySortField(LeafReader reader) {
+ Sort sort = reader.getMetaData().sort();
+ if (sort == null) {
+ return null;
+ }
+ for (SortField sf : sort.fields) {
+ String field = sf.getField();
+ if (field == null) {
+ // Custom field that we don't know anything about, so return it as primary
+ return sf;
+ }
+ if (reader.getFieldInfos().fieldInfo(field) == null) {
+ // Field has no values in this segment, so sorting by it has no effect.
+ continue;
+ }
+ // If the field has a skip index, check whether all values are identical,
+ // in which case sorting by this field is a no-op for this segment. Note
+ // that we also need to check for denseness, as missing values add an
+ // implicit second sort group.
+ DocValuesSkipper skipper = reader.getDocValuesSkipper(field);
+ if (skipper != null
+ && skipper.docCount() == reader.maxDoc()
+ && skipper.minValue() == skipper.maxValue()) {
+ continue;
+ }
+ return sf;
+ }
+ return null;
+ }
}
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestDocValuesQueries.java b/lucene/core/src/test/org/apache/lucene/search/TestDocValuesQueries.java
index a932dc0061c..cb70682fece 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestDocValuesQueries.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestDocValuesQueries.java
@@ -1122,4 +1122,252 @@ public class TestDocValuesQueries extends LuceneTestCaseJupiter {
reader.close();
dir.close();
}
+
+ /**
+ * Verifies that SortedNumericDocValuesRangeQuery can use a secondary sort field to accelerate
+ * querying if the primary sort field is single-valued and therefore a no-op
+ */
+ @Test
+ public void testSortedNumericRangeQueryOptimizesWithConstantPrimary(Random random)
+ throws IOException {
+ Directory dir = newDirectory();
+ IndexWriterConfig config = new IndexWriterConfig().setCodec(getCodec(random));
+ config.setIndexSort(
+ new Sort(
+ new SortField("primary", SortField.Type.LONG), // constant → no-op
+ new SortField("secondary", SortField.Type.LONG))); // varying → effective primary
+ IndexWriter iw = new IndexWriter(dir, config);
+ for (int i = 0; i < 10; i++) {
+ Document doc = new Document();
+ doc.add(NumericDocValuesField.indexedField("primary", 42)); // constant, skip index present
+ doc.add(NumericDocValuesField.indexedField("secondary", i)); // varying
+ iw.addDocument(doc);
+ }
+ iw.forceMerge(1);
+ iw.close();
+ DirectoryReader reader = DirectoryReader.open(dir);
+ IndexSearcher searcher = new IndexSearcher(reader);
+
+ // primary has a constant value, so getPrimarySortField skips it and returns "secondary".
+ // The query on "secondary" should therefore use SortedSkipperScorerSupplier,
+ // which yields a plain range iterator with no two-phase iterator.
+ Query query = SortedNumericDocValuesField.newSlowRangeQuery("secondary", 3, 7);
+ Weight weight = query.createWeight(searcher, ScoreMode.COMPLETE, 1.0f);
+ Scorer scorer = weight.scorer(reader.leaves().get(0));
+ assertNotNull(scorer);
+ assertNull(
+ "SortedSkipperScorerSupplier should be used when primary sort is a no-op",
+ scorer.twoPhaseIterator());
+
+ reader.close();
+ dir.close();
+ }
+
+ /**
+ * Verifies that SortedNumericDocValuesRangeQuery can use a secondary sort field to accelerate
+ * querying if the primary sort field is empty and therefore a no-op
+ */
+ @Test
+ public void testSortedNumericRangeQueryOptimizesWithEmptyPrimary(Random random)
+ throws IOException {
+ Directory dir = newDirectory();
+ IndexWriterConfig config = new IndexWriterConfig().setCodec(getCodec(random));
+ config.setIndexSort(
+ new Sort(
+ new SortField("primary", SortField.Type.LONG), // empty → no-op
+ new SortField("secondary", SortField.Type.LONG))); // varying → effective primary
+ IndexWriter iw = new IndexWriter(dir, config);
+ for (int i = 0; i < 10; i++) {
+ Document doc = new Document();
+ // nothing added for the primary sort field
+ doc.add(NumericDocValuesField.indexedField("secondary", i)); // varying
+ iw.addDocument(doc);
+ }
+ iw.forceMerge(1);
+ iw.close();
+ DirectoryReader reader = DirectoryReader.open(dir);
+ IndexSearcher searcher = new IndexSearcher(reader);
+
+ // primary has a constant value, so getPrimarySortField skips it and returns "secondary".
+ // The query on "secondary" should therefore use SortedSkipperScorerSupplier,
+ // which yields a plain range iterator with no two-phase iterator.
+ Query query = SortedNumericDocValuesField.newSlowRangeQuery("secondary", 3, 7);
+ Weight weight = query.createWeight(searcher, ScoreMode.COMPLETE, 1.0f);
+ Scorer scorer = weight.scorer(reader.leaves().get(0));
+ assertNotNull(scorer);
+ assertNull(
+ "SortedSkipperScorerSupplier should be used when primary sort is a no-op",
+ scorer.twoPhaseIterator());
+
+ reader.close();
+ dir.close();
+ }
+
+ /**
+ * Verifies that SortedNumericDocValuesRangeQuery does not use a secondary sort field to
+ * accelerate querying if the primary sort field is a genuine sort field.
+ */
+ @Test
+ public void testSortedNumericRangeQueryDoesNotOptimizeWithActivePrimary(Random random)
+ throws IOException {
+ Directory dir = newDirectory();
+ IndexWriterConfig config = new IndexWriterConfig().setCodec(getCodec(random));
+ config.setIndexSort(
+ new Sort(
+ new SortField("primary", SortField.Type.LONG), // multiple values → effective primary
+ new SortField("secondary", SortField.Type.LONG)));
+ IndexWriter iw = new IndexWriter(dir, config);
+ for (int i = 0; i < 10; i++) {
+ Document doc = new Document();
+ doc.add(NumericDocValuesField.indexedField("primary", i)); // varying
+ doc.add(NumericDocValuesField.indexedField("secondary", i)); // varying
+ iw.addDocument(doc);
+ }
+ iw.forceMerge(1);
+ iw.close();
+ DirectoryReader reader = DirectoryReader.open(dir);
+ IndexSearcher searcher = new IndexSearcher(reader);
+
+ // primary is active, so getPrimarySortField returns "primary", not "secondary".
+ // The query on "secondary" must fall back to the TwoPhaseIterator path.
+ Query query = SortedNumericDocValuesField.newSlowRangeQuery("secondary", 3, 7);
+ Weight weight = query.createWeight(searcher, ScoreMode.COMPLETE, 1.0f);
+ Scorer scorer = weight.scorer(reader.leaves().get(0));
+ assertNotNull(scorer);
+ assertNotNull(
+ "TwoPhaseIterator fallback should be used when secondary is not the effective primary",
+ scorer.twoPhaseIterator());
+
+ reader.close();
+ dir.close();
+ }
+
+ /**
+ * Verifies that SortedSetDocValuesRangeQuery can use a secondary sort field to accelerate
+ * querying if the primary sort field is single-valued and therefore a no-op
+ */
+ @Test
+ public void testSortedSetRangeQueryOptimizesWithConstantPrimary(Random random)
+ throws IOException {
+ Directory dir = newDirectory();
+ IndexWriterConfig config = new IndexWriterConfig().setCodec(getCodec(random));
+ config.setIndexSort(
+ new Sort(
+ new SortField("primary", SortField.Type.STRING), // constant → no-op
+ new SortField("secondary", SortField.Type.STRING))); // varying → effective primary
+ IndexWriter iw = new IndexWriter(dir, config);
+ for (int i = 0; i < 10; i++) {
+ Document doc = new Document();
+ doc.add(SortedDocValuesField.indexedField("primary", newBytesRef("constant")));
+ doc.add(
+ SortedDocValuesField.indexedField(
+ "secondary", newBytesRef(String.format(Locale.ROOT, "%03d", i))));
+ iw.addDocument(doc);
+ }
+ iw.forceMerge(1);
+ iw.close();
+ DirectoryReader reader = DirectoryReader.open(dir);
+ IndexSearcher searcher = new IndexSearcher(reader);
+
+ // primary has a constant ordinal, so getPrimarySortField skips it and returns "secondary".
+ // The query on "secondary" should therefore use SortedSkipperScorerSupplier,
+ // which yields a plain range iterator with no two-phase iterator.
+ Query query =
+ SortedSetDocValuesField.newSlowRangeQuery(
+ "secondary", newBytesRef("003"), newBytesRef("007"), true, true);
+ Weight weight = query.createWeight(searcher, ScoreMode.COMPLETE, 1.0f);
+ Scorer scorer = weight.scorer(reader.leaves().get(0));
+ assertNotNull(scorer);
+ assertNull(
+ "SortedSkipperScorerSupplier should be used when primary sort is a no-op",
+ scorer.twoPhaseIterator());
+
+ reader.close();
+ dir.close();
+ }
+
+ /**
+ * Verifies that SortedSetDocValuesRangeQuery can use a secondary sort field to accelerate
+ * querying if the primary sort field is empty and therefore a no-op
+ */
+ @Test
+ public void testSortedSetRangeQueryOptimizesWithEmptyPrimary(Random random) throws IOException {
+ Directory dir = newDirectory();
+ IndexWriterConfig config = new IndexWriterConfig().setCodec(getCodec(random));
+ config.setIndexSort(
+ new Sort(
+ new SortField("primary", SortField.Type.STRING), // constant → no-op
+ new SortField("secondary", SortField.Type.STRING))); // varying → effective primary
+ IndexWriter iw = new IndexWriter(dir, config);
+ for (int i = 0; i < 10; i++) {
+ Document doc = new Document();
+ // no values added for primary field
+ doc.add(
+ SortedDocValuesField.indexedField(
+ "secondary", newBytesRef(String.format(Locale.ROOT, "%03d", i))));
+ iw.addDocument(doc);
+ }
+ iw.forceMerge(1);
+ iw.close();
+ DirectoryReader reader = DirectoryReader.open(dir);
+ IndexSearcher searcher = new IndexSearcher(reader);
+
+ // primary has a constant ordinal, so getPrimarySortField skips it and returns "secondary".
+ // The query on "secondary" should therefore use SortedSkipperScorerSupplier,
+ // which yields a plain range iterator with no two-phase iterator.
+ Query query =
+ SortedSetDocValuesField.newSlowRangeQuery(
+ "secondary", newBytesRef("003"), newBytesRef("007"), true, true);
+ Weight weight = query.createWeight(searcher, ScoreMode.COMPLETE, 1.0f);
+ Scorer scorer = weight.scorer(reader.leaves().get(0));
+ assertNotNull(scorer);
+ assertNull(
+ "SortedSkipperScorerSupplier should be used when primary sort is a no-op",
+ scorer.twoPhaseIterator());
+
+ reader.close();
+ dir.close();
+ }
+
+ /**
+ * Verifies that SortedSetDocValuesRangeQuery does not use a secondary sort field to accelerate
+ * querying if the primary sort field is a genuine sort field.
+ */
+ @Test
+ public void testSortedSetRangeQueryDoesNotOptimizeWithActivePrimary(Random random)
+ throws IOException {
+ Directory dir = newDirectory();
+ IndexWriterConfig config = new IndexWriterConfig().setCodec(getCodec(random));
+ config.setIndexSort(
+ new Sort(
+ new SortField("primary", SortField.Type.STRING), // multiple values → effective primary
+ new SortField("secondary", SortField.Type.STRING)));
+ IndexWriter iw = new IndexWriter(dir, config);
+ for (int i = 0; i < 10; i++) {
+ String val = String.format(Locale.ROOT, "%03d", i);
+ Document doc = new Document();
+ doc.add(SortedDocValuesField.indexedField("primary", newBytesRef(val))); // varying
+ doc.add(SortedDocValuesField.indexedField("secondary", newBytesRef(val))); // varying
+ iw.addDocument(doc);
+ }
+ iw.forceMerge(1);
+ iw.close();
+ DirectoryReader reader = DirectoryReader.open(dir);
+ IndexSearcher searcher = new IndexSearcher(reader);
+
+ // primary is active, so getPrimarySortField returns "primary", not "secondary".
+ // The query on "secondary" must fall back to the TwoPhaseIterator path.
+ Query query =
+ SortedSetDocValuesField.newSlowRangeQuery(
+ "secondary", newBytesRef("003"), newBytesRef("007"), true, true);
+ Weight weight = query.createWeight(searcher, ScoreMode.COMPLETE, 1.0f);
+ Scorer scorer = weight.scorer(reader.leaves().get(0));
+ assertNotNull(scorer);
+ assertNotNull(
+ "TwoPhaseIterator fallback should be used when secondary is not the effective primary",
+ scorer.twoPhaseIterator());
+
+ reader.close();
+ dir.close();
+ }
}
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestGetPrimarySortField.java b/lucene/core/src/test/org/apache/lucene/search/TestGetPrimarySortField.java
new file mode 100644
index 00000000000..e860aaeef43
--- /dev/null
+++ b/lucene/core/src/test/org/apache/lucene/search/TestGetPrimarySortField.java
@@ -0,0 +1,312 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.search;
+
+import java.io.IOException;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.NumericDocValuesField;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.NoMergePolicy;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.tests.util.LuceneTestCase;
+
+/**
+ * Tests for {@link Sort#getPrimarySortField(LeafReader)}, which returns the effective primary sort
+ * field for a segment, skipping fields that are no-ops because they have no values in the segment
+ * or have only a single distinct value according to their skip index.
+ */
+public class TestGetPrimarySortField extends LuceneTestCase {
+
+ /** A reader with no index sort configured always returns null. */
+ public void testNoIndexSort() throws IOException {
+ Directory dir = newDirectory();
+ IndexWriter iw = new IndexWriter(dir, new IndexWriterConfig());
+ Document doc = new Document();
+ doc.add(NumericDocValuesField.indexedField("field", 42));
+ iw.addDocument(doc);
+ iw.forceMerge(1);
+ iw.close();
+ DirectoryReader reader = DirectoryReader.open(dir);
+ LeafReader leafReader = reader.leaves().get(0).reader();
+ assertNull(Sort.getPrimarySortField(leafReader));
+ reader.close();
+ dir.close();
+ }
+
+ /**
+ * When the sort field has no DocValues skip index (plain DV field), there is no skipper to detect
+ * a single-value no-op, but the field exists in FieldInfos so it is returned as-is.
+ */
+ public void testSortFieldWithoutSkipperIsReturned() throws IOException {
+ Directory dir = newDirectory();
+ SortField sortField = new SortField("field", SortField.Type.LONG);
+ IndexWriter iw =
+ new IndexWriter(dir, new IndexWriterConfig().setIndexSort(new Sort(sortField)));
+ for (int i = 0; i < 10; i++) {
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("field", i)); // plain DV, no skip index
+ iw.addDocument(doc);
+ }
+ iw.forceMerge(1);
+ iw.close();
+ DirectoryReader reader = DirectoryReader.open(dir);
+ LeafReader leafReader = reader.leaves().get(0).reader();
+ SortField result = Sort.getPrimarySortField(leafReader);
+ assertNotNull(result);
+ assertEquals("field", result.getField());
+ reader.close();
+ dir.close();
+ }
+
+ /**
+ * A sort field with a skip index and multiple distinct values is returned as-is: it is not a
+ * no-op because the skipper's min and max values differ.
+ */
+ public void testSortFieldWithMultipleDistinctValuesIsReturned() throws IOException {
+ Directory dir = newDirectory();
+ SortField sortField = new SortField("field", SortField.Type.LONG);
+ IndexWriter iw =
+ new IndexWriter(dir, new IndexWriterConfig().setIndexSort(new Sort(sortField)));
+ for (int i = 0; i < 10; i++) {
+ Document doc = new Document();
+ doc.add(NumericDocValuesField.indexedField("field", i)); // indexed DV, skipper present
+ iw.addDocument(doc);
+ }
+ iw.forceMerge(1);
+ iw.close();
+ DirectoryReader reader = DirectoryReader.open(dir);
+ LeafReader leafReader = reader.leaves().get(0).reader();
+ SortField result = Sort.getPrimarySortField(leafReader);
+ assertNotNull(result);
+ assertEquals("field", result.getField());
+ reader.close();
+ dir.close();
+ }
+
+ /**
+ * A sort field with a skip index and one distinct value but with documents with no value is
+ * returned as-is: it is not a no-op because documents with no value will group together and form
+ * an implicitly-valued second group.
+ */
+ public void testSparseSortFieldValuesIsReturned() throws IOException {
+ Directory dir = newDirectory();
+ SortField primary = new SortField("field1", SortField.Type.LONG);
+ SortField secondary = new SortField("field2", SortField.Type.LONG);
+ IndexWriter iw =
+ new IndexWriter(dir, new IndexWriterConfig().setIndexSort(new Sort(primary, secondary)));
+ for (int i = 0; i < 10; i++) {
+ Document doc = new Document();
+ if (i != 7) {
+ doc.add(NumericDocValuesField.indexedField("field1", 42)); // constant → no-op
+ }
+ doc.add(NumericDocValuesField.indexedField("field2", i));
+ iw.addDocument(doc);
+ }
+ iw.forceMerge(1);
+ iw.close();
+ DirectoryReader reader = DirectoryReader.open(dir);
+ LeafReader leafReader = reader.leaves().get(0).reader();
+ SortField result = Sort.getPrimarySortField(leafReader);
+ assertNotNull(result);
+ assertEquals("field1", result.getField());
+ reader.close();
+ dir.close();
+ }
+
+ /**
+ * When the primary sort field has no values at all in a segment, its FieldInfo is absent and it
+ * is treated as a no-op. The next sort field becomes the effective primary.
+ */
+ public void testPrimaryWithNoValuesInSegmentIsSkipped() throws IOException {
+ Directory dir = newDirectory();
+ SortField primary = new SortField("field1", SortField.Type.LONG);
+ SortField secondary = new SortField("field2", SortField.Type.LONG);
+ IndexWriter iw =
+ new IndexWriter(dir, new IndexWriterConfig().setIndexSort(new Sort(primary, secondary)));
+ for (int i = 0; i < 10; i++) {
+ Document doc = new Document();
+ // field1: no doc values → FieldInfos has no entry for "field1" → skipped
+ doc.add(NumericDocValuesField.indexedField("field2", i));
+ iw.addDocument(doc);
+ }
+ iw.forceMerge(1);
+ iw.close();
+ DirectoryReader reader = DirectoryReader.open(dir);
+ LeafReader leafReader = reader.leaves().get(0).reader();
+ SortField result = Sort.getPrimarySortField(leafReader);
+ assertNotNull(result);
+ assertEquals("field2", result.getField());
+ reader.close();
+ dir.close();
+ }
+
+ /**
+ * When the primary sort field has a skip index with a single distinct value (skipper min == max),
+ * it is a no-op and the next sort field becomes the effective primary.
+ */
+ public void testSingleValuePrimaryIsSkippedToSecondary() throws IOException {
+ Directory dir = newDirectory();
+ SortField primary = new SortField("field1", SortField.Type.LONG);
+ SortField secondary = new SortField("field2", SortField.Type.LONG);
+ IndexWriter iw =
+ new IndexWriter(dir, new IndexWriterConfig().setIndexSort(new Sort(primary, secondary)));
+ for (int i = 0; i < 10; i++) {
+ Document doc = new Document();
+ doc.add(NumericDocValuesField.indexedField("field1", 42)); // constant → no-op
+ doc.add(NumericDocValuesField.indexedField("field2", i)); // multiple distinct values
+ iw.addDocument(doc);
+ }
+ iw.forceMerge(1);
+ iw.close();
+ DirectoryReader reader = DirectoryReader.open(dir);
+ LeafReader leafReader = reader.leaves().get(0).reader();
+ SortField result = Sort.getPrimarySortField(leafReader);
+ assertNotNull(result);
+ assertEquals("field2", result.getField());
+ reader.close();
+ dir.close();
+ }
+
+ /**
+ * When all sort fields have only a single distinct value in the segment, every field is a no-op
+ * and null is returned.
+ */
+ public void testAllSingleValueFieldsReturnNull() throws IOException {
+ Directory dir = newDirectory();
+ SortField primary = new SortField("field1", SortField.Type.LONG);
+ SortField secondary = new SortField("field2", SortField.Type.LONG);
+ IndexWriter iw =
+ new IndexWriter(dir, new IndexWriterConfig().setIndexSort(new Sort(primary, secondary)));
+ for (int i = 0; i < 10; i++) {
+ Document doc = new Document();
+ doc.add(NumericDocValuesField.indexedField("field1", 42)); // constant
+ doc.add(NumericDocValuesField.indexedField("field2", 7)); // constant
+ iw.addDocument(doc);
+ }
+ iw.forceMerge(1);
+ iw.close();
+ DirectoryReader reader = DirectoryReader.open(dir);
+ LeafReader leafReader = reader.leaves().get(0).reader();
+ assertNull(Sort.getPrimarySortField(leafReader));
+ reader.close();
+ dir.close();
+ }
+
+ /**
+ * With NoMergePolicy ensuring a fixed three-segment layout, each leaf independently returns its
+ * own effective primary sort field: segment 0 has a real primary, segment 1 has an absent primary
+ * (no FieldInfo), and segment 2 has a constant primary (single distinct value).
+ */
+ public void testPerSegmentEffectivePrimaryWithNoMergePolicy() throws IOException {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig();
+ iwc.setMergePolicy(NoMergePolicy.INSTANCE);
+ iwc.setIndexSort(
+ new Sort(
+ new SortField("field1", SortField.Type.LONG), // primary
+ new SortField("field2", SortField.Type.LONG))); // secondary
+ IndexWriter iw = new IndexWriter(dir, iwc);
+
+ // Segment 0: field1 has two distinct values → effective primary is field1
+ Document doc = new Document();
+ doc.add(NumericDocValuesField.indexedField("field1", 1));
+ doc.add(NumericDocValuesField.indexedField("field2", 10));
+ iw.addDocument(doc);
+ doc = new Document();
+ doc.add(NumericDocValuesField.indexedField("field1", 2));
+ doc.add(NumericDocValuesField.indexedField("field2", 20));
+ iw.addDocument(doc);
+ iw.commit();
+
+ // Segment 1: field1 absent (no FieldInfo) → effective primary is field2
+ doc = new Document();
+ doc.add(NumericDocValuesField.indexedField("field2", 30));
+ iw.addDocument(doc);
+ doc = new Document();
+ doc.add(NumericDocValuesField.indexedField("field2", 31));
+ iw.addDocument(doc);
+ iw.commit();
+
+ // Segment 2: field1 is constant (skipper min == max) → no-op, effective primary is field2
+ doc = new Document();
+ doc.add(NumericDocValuesField.indexedField("field1", 42));
+ doc.add(NumericDocValuesField.indexedField("field2", 40));
+ iw.addDocument(doc);
+ doc = new Document();
+ doc.add(NumericDocValuesField.indexedField("field1", 42));
+ doc.add(NumericDocValuesField.indexedField("field2", 50));
+ iw.addDocument(doc);
+ iw.commit();
+
+ iw.close();
+
+ DirectoryReader reader = DirectoryReader.open(dir);
+ assertEquals(3, reader.leaves().size());
+
+ // Segment 0: field1 has multiple distinct values → effective primary is field1
+ LeafReader leaf0 = reader.leaves().get(0).reader();
+ SortField sf0 = Sort.getPrimarySortField(leaf0);
+ assertNotNull(sf0);
+ assertEquals("field1", sf0.getField());
+
+ // Segment 1: field1 absent → effective primary is field2
+ LeafReader leaf1 = reader.leaves().get(1).reader();
+ SortField sf1 = Sort.getPrimarySortField(leaf1);
+ assertNotNull(sf1);
+ assertEquals("field2", sf1.getField());
+
+ // Segment 2: field1 constant → no-op, effective primary is field2
+ LeafReader leaf2 = reader.leaves().get(2).reader();
+ SortField sf2 = Sort.getPrimarySortField(leaf2);
+ assertNotNull(sf2);
+ assertEquals("field2", sf2.getField());
+
+ reader.close();
+ dir.close();
+ }
+
+ /**
+ * The reverse attribute of the effective primary sort field is correctly preserved when the
+ * initial primary sort field is skipped as a no-op.
+ */
+ public void testSecondaryReverseAttributeIsPreserved() throws IOException {
+ Directory dir = newDirectory();
+ SortField primary = new SortField("field1", SortField.Type.LONG, false); // ascending
+ SortField secondary = new SortField("field2", SortField.Type.LONG, true); // descending
+ IndexWriter iw =
+ new IndexWriter(dir, new IndexWriterConfig().setIndexSort(new Sort(primary, secondary)));
+ for (int i = 0; i < 10; i++) {
+ Document doc = new Document();
+ doc.add(NumericDocValuesField.indexedField("field1", 42)); // constant → no-op
+ doc.add(NumericDocValuesField.indexedField("field2", i));
+ iw.addDocument(doc);
+ }
+ iw.forceMerge(1);
+ iw.close();
+ DirectoryReader reader = DirectoryReader.open(dir);
+ LeafReader leafReader = reader.leaves().get(0).reader();
+ SortField result = Sort.getPrimarySortField(leafReader);
+ assertNotNull(result);
+ assertEquals("field2", result.getField());
+ assertTrue("secondary sort field should be reversed", result.getReverse());
+ reader.close();
+ dir.close();
+ }
+}
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestIndexSortSortedNumericDocValuesRangeQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestIndexSortSortedNumericDocValuesRangeQuery.java
index 4adb4a3cb3c..69b667241c0 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestIndexSortSortedNumericDocValuesRangeQuery.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestIndexSortSortedNumericDocValuesRangeQuery.java
@@ -24,6 +24,7 @@ import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.IntPoint;
import org.apache.lucene.document.LongPoint;
+import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.SortedNumericDocValuesField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.index.DirectoryReader;
@@ -855,6 +856,91 @@ public class TestIndexSortSortedNumericDocValuesRangeQuery extends LuceneTestCas
dir.close();
}
+ /**
+ * Verifies that the binary-search optimization activates when the primary index sort field has a
+ * single constant value (detected as a no-op via its skip index), promoting the secondary field
+ * to effective primary and enabling direct doc-ID range iteration without a two-phase iterator.
+ */
+ public void testNoOpPrimarySort() throws IOException {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ Sort indexSort =
+ new Sort(
+ new SortField("field1", SortField.Type.LONG), // constant → no-op via skipper
+ new SortField("field2", SortField.Type.LONG)); // varying → effective primary
+ iwc.setIndexSort(indexSort);
+ RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
+ for (int i = 0; i < 10; i++) {
+ Document doc = new Document();
+ doc.add(NumericDocValuesField.indexedField("field1", 42)); // constant; skip index present
+ doc.add(NumericDocValuesField.indexedField("field2", i)); // varying
+ iw.addDocument(doc);
+ }
+ iw.forceMerge(1);
+ DirectoryReader reader = iw.getReader();
+ IndexSearcher searcher = newSearcher(reader);
+ iw.close();
+
+ // field1 has a single distinct value, so getPrimarySortField skips it and returns field2.
+ // The binary-search optimization should activate for the query on field2, producing a plain
+ // range iterator with no two-phase iterator.
+ Query fallback = SortedNumericDocValuesField.newSlowRangeQuery("field2", 3, 7);
+ Query query = new IndexSortSortedNumericDocValuesRangeQuery("field2", 3, 7, fallback);
+ Weight weight = query.createWeight(searcher, ScoreMode.TOP_SCORES, 1.0F);
+ for (LeafReaderContext context : searcher.getIndexReader().leaves()) {
+ Scorer scorer = weight.scorer(context);
+ assertNotNull(scorer);
+ assertNull(
+ "binary-search optimization should be active when primary sort is a no-op",
+ scorer.twoPhaseIterator());
+ }
+
+ reader.close();
+ dir.close();
+ }
+
+ /**
+ * Verifies that the binary-search optimization activates when the primary index sort field has no
+ * values in the segment at all (detected as a no-op via FieldInfos), promoting the secondary
+ * field to effective primary.
+ */
+ public void testMissingPrimarySort() throws IOException {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ Sort indexSort =
+ new Sort(
+ new SortField("field1", SortField.Type.LONG), // absent → no-op via FieldInfos check
+ new SortField("field2", SortField.Type.LONG)); // varying → effective primary
+ iwc.setIndexSort(indexSort);
+ RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
+ for (int i = 0; i < 10; i++) {
+ Document doc = new Document();
+ // field1 intentionally absent: no FieldInfo in segment → detected as no-op
+ doc.add(NumericDocValuesField.indexedField("field2", i));
+ iw.addDocument(doc);
+ }
+ iw.forceMerge(1);
+ DirectoryReader reader = iw.getReader();
+ IndexSearcher searcher = newSearcher(reader);
+ iw.close();
+
+ // field1 has no FieldInfo, so getPrimarySortField skips it and returns field2.
+ // The binary-search optimization should activate for the query on field2.
+ Query fallback = SortedNumericDocValuesField.newSlowRangeQuery("field2", 3, 7);
+ Query query = new IndexSortSortedNumericDocValuesRangeQuery("field2", 3, 7, fallback);
+ Weight weight = query.createWeight(searcher, ScoreMode.TOP_SCORES, 1.0F);
+ for (LeafReaderContext context : searcher.getIndexReader().leaves()) {
+ Scorer scorer = weight.scorer(context);
+ assertNotNull(scorer);
+ assertNull(
+ "binary-search optimization should be active when primary sort field has no values",
+ scorer.twoPhaseIterator());
+ }
+
+ reader.close();
+ dir.close();
+ }
+
private void addDocWithBkd(RandomIndexWriter indexWriter, String field, long value, int repeat)
throws IOException {
for (int i = 0; i < repeat; i++) {