Apache Lucene open-source search software
Smarter use of sort-field optimizations in DV range queries (#16424)
Alan Woodward693a7234bad2
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex c54fc9a987c..44924919b4f 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -391,6 +391,9 @@ Optimizations* GITHUB#16380: Speed up filtered disjunction queries when the filter is part of theindex sort by advancing the filter before calculating impacts. (Alan Woodward)+* GITHUB#16424: Use secondary or lower sorts to accelerate XXDocValuesRangeQueries if the primary+ sort field is empty or only contains a single value. (Alan Woodward)+* GITHUB#16180: Add bulk intoBitSet for SortedDocValues ordinal ranges. (Prithvi S)Bug Fixesdiff --git a/lucene/core/src/java/org/apache/lucene/document/SortedNumericDocValuesRangeQuery.java b/lucene/core/src/java/org/apache/lucene/document/SortedNumericDocValuesRangeQuery.javaindex f00a4e72c35..16ccd65c679 100644--- a/lucene/core/src/java/org/apache/lucene/document/SortedNumericDocValuesRangeQuery.java+++ b/lucene/core/src/java/org/apache/lucene/document/SortedNumericDocValuesRangeQuery.java@@ -138,11 +138,13 @@ final class SortedNumericDocValuesRangeQuery extends NumericDocValuesRangeQueryfinal NumericDocValues singleton = DocValues.unwrapSingleton(values);final DocValuesSkipper skipper = context.reader().getDocValuesSkipper(field);- final SortField primarySortField;if (singleton != null) {- if (skipper != null- && (primarySortField = densePrimarySort(context.reader(), skipper)) != null) {- return getScorerSupplierFromDensePrimarySort(singleton, skipper, primarySortField);+ if (skipper != null) {+ SortField primarySortField =+ canUsePrimarySortShortcut(context.reader(), field, skipper);+ if (primarySortField != null) {+ return getScorerSupplierFromDensePrimarySort(singleton, skipper, primarySortField);+ }}// A single two-phase iterator covers every density: its approximation rides the skipper// (no over-scan), its intoBitSet bulk-evaluates dense blocks, and YES runs collect as@@ -163,7 +165,7 @@ final class SortedNumericDocValuesRangeQuery extends NumericDocValuesRangeQuery}@Override- public int count(LeafReaderContext context) throws IOException {+ public int count(LeafReaderContext context) {int maxDoc = context.reader().maxDoc();int cnt = docCountIgnoringDeletes(context);if (cnt == maxDoc) {@@ -177,7 +179,7 @@ final class SortedNumericDocValuesRangeQuery extends NumericDocValuesRangeQuery* # docs within the query range ignoring any deleted documents* -1 if # docs cannot be determined efficiently*/- private int docCountIgnoringDeletes(LeafReaderContext context) throws IOException {+ private int docCountIgnoringDeletes(LeafReaderContext context) {final DocValuesSkipper skipper = context.reader().getDocValuesSkipper(field);if (skipper != null) {if (skipper.minValue() > upperValue || skipper.maxValue() < lowerValue) {@@ -224,16 +226,23 @@ final class SortedNumericDocValuesRangeQuery extends NumericDocValuesRangeQuery};}- private SortField densePrimarySort(LeafReader reader, DocValuesSkipper skipper) {- if (skipper.docCount() != reader.maxDoc()) {+ /**+ * Returns the primary sort field if it is safe to use the {@link SortedSkipperScorerSupplier}+ * range shortcuts for this query, or {@code null} if the shortcut cannot be applied.+ *+ * <p>The shortcuts ({@code skipperMinDocId=0} and {@code skipperMaxDocId=docCount}) assume that+ * docs without a value do not appear within or before the matching doc-ID range. This requires+ * that the field is dense (every doc has a value, {@code docCount == reader.maxDoc()}).+ */+ private static SortField canUsePrimarySortShortcut(+ LeafReader reader, String field, DocValuesSkipper skipper) {+ SortField sf = Sort.getPrimarySortField(reader);+ if (sf == null || sf.getField().equals(field) == false) {return null;}- final Sort indexSort = reader.getMetaData().sort();- if (indexSort == null- || indexSort.getSort().length == 0- || indexSort.getSort()[0].getField().equals(field) == false) {+ if (skipper.docCount() != reader.maxDoc()) {return null;}- return indexSort.getSort()[0];+ return sf;}}diff --git a/lucene/core/src/java/org/apache/lucene/document/SortedSetDocValuesRangeQuery.java b/lucene/core/src/java/org/apache/lucene/document/SortedSetDocValuesRangeQuery.javaindex 3022ade0f85..a4228bd3453 100644--- a/lucene/core/src/java/org/apache/lucene/document/SortedSetDocValuesRangeQuery.java+++ b/lucene/core/src/java/org/apache/lucene/document/SortedSetDocValuesRangeQuery.java@@ -121,12 +121,13 @@ final class SortedSetDocValuesRangeQuery extends Query {DocValuesSkipper skipper = context.reader().getDocValuesSkipper(field);SortedSetDocValues values = DocValues.getSortedSet(context.reader(), field);final SortedDocValues singleton = DocValues.unwrapSingleton(values);- final SortField primarySortField;- if (singleton != null- && skipper != null- && (primarySortField = densePrimarySort(context.reader(), skipper)) != null) {- return getScorerSupplierFromDensePrimarySort(- singleton, values, skipper, primarySortField);+ if (singleton != null && skipper != null) {+ SortField primarySortField =+ canUsePrimarySortShortcut(context.reader(), field, skipper.docCount());+ if (primarySortField != null) {+ return getScorerSupplierFromDensePrimarySort(+ singleton, values, skipper, primarySortField);+ }}// implement ScorerSupplier, since we do some expensive stuff to make a scorerreturn new ConstantScoreScorerSupplier(score(), scoreMode, context.reader().maxDoc()) {@@ -276,16 +277,23 @@ final class SortedSetDocValuesRangeQuery extends Query {return maxOrd;}- private SortField densePrimarySort(LeafReader reader, DocValuesSkipper skipper) {- if (skipper.docCount() != reader.maxDoc()) {+ /**+ * Returns the primary sort field if it is safe to use the {@link SortedSkipperScorerSupplier}+ * range shortcuts for this query, or {@code null} if the shortcut cannot be applied.+ *+ * <p>The shortcuts assume that docs without a value do not appear before or within the matching+ * doc-ID range. This requires that the field is dense (every doc has a value, {@code docCount ==+ * reader.maxDoc()}).+ */+ private static SortField canUsePrimarySortShortcut(LeafReader reader, String field, int docCount)+ throws IOException {+ SortField sf = Sort.getPrimarySortField(reader);+ if (sf == null || sf.getField().equals(field) == false) {return null;}- final Sort indexSort = reader.getMetaData().sort();- if (indexSort == null- || indexSort.getSort().length == 0- || indexSort.getSort()[0].getField().equals(field) == false) {+ if (docCount != reader.maxDoc()) {return null;}- return indexSort.getSort()[0];+ return sf;}}diff --git a/lucene/core/src/java/org/apache/lucene/search/IndexSortSortedNumericDocValuesRangeQuery.java b/lucene/core/src/java/org/apache/lucene/search/IndexSortSortedNumericDocValuesRangeQuery.javaindex fb4afd4c795..83e907dcd50 100644--- a/lucene/core/src/java/org/apache/lucene/search/IndexSortSortedNumericDocValuesRangeQuery.java+++ b/lucene/core/src/java/org/apache/lucene/search/IndexSortSortedNumericDocValuesRangeQuery.java@@ -190,21 +190,19 @@ public class IndexSortSortedNumericDocValuesRangeQuery extends NumericDocValuesR}// use index sort optimization if possible- Sort indexSort = reader.getMetaData().sort();- if (indexSort != null- && indexSort.getSort().length > 0- && indexSort.getSort()[0].getField().equals(field)) {- final SortField sortField = indexSort.getSort()[0];- final SortField.Type sortFieldType = getSortFieldType(sortField);+ SortField primarySortField = Sort.getPrimarySortField(reader);+ if (primarySortField != null && primarySortField.getField().equals(field)) {+ final SortField.Type sortFieldType = getSortFieldType(primarySortField);// The index sort optimization is only supported for Type.INT and Type.LONGif (sortFieldType == Type.INT || sortFieldType == Type.LONG) {- Object missingValue = sortField.getMissingValue();+ Object missingValue = primarySortField.getMissingValue();final long missingLongValue =missingValue == null ? 0L : ((Number) missingValue).longValue();// all documents have docValues or missing value falls outside the rangeif ((pointValues != null && pointValues.getDocCount() == reader.maxDoc())|| (missingLongValue < lowerValue || missingLongValue > upperValue)) {- itAndCount = getDocIdSetIterator(sortField, sortFieldType, context, numericValues);+ itAndCount =+ getDocIdSetIterator(primarySortField, sortFieldType, context, numericValues);}if (itAndCount != null && itAndCount.count != -1) {return itAndCount.count;@@ -419,14 +417,12 @@ public class IndexSortSortedNumericDocValuesRangeQuery extends NumericDocValuesRprivate IteratorAndCount getDocIdSetIteratorOrNullFromBkd(LeafReaderContext context, DocIdSetIterator delegate) throws IOException {- Sort indexSort = context.reader().getMetaData().sort();- if (indexSort == null- || indexSort.getSort().length == 0- || indexSort.getSort()[0].getField().equals(field) == false) {+ SortField primarySortField = Sort.getPrimarySortField(context.reader());+ if (primarySortField == null || primarySortField.getField().equals(field) == false) {return null;}- final boolean reverse = indexSort.getSort()[0].getReverse();+ final boolean reverse = primarySortField.getReverse();PointValues points = context.reader().getPointValues(field);if (points == null) {@@ -519,16 +515,12 @@ public class IndexSortSortedNumericDocValuesRangeQuery extends NumericDocValuesRif (itAndCount != null) {return itAndCount;}- Sort indexSort = context.reader().getMetaData().sort();- if (indexSort != null- && indexSort.getSort().length > 0- && indexSort.getSort()[0].getField().equals(field)) {-- final SortField sortField = indexSort.getSort()[0];- final SortField.Type sortFieldType = getSortFieldType(sortField);+ SortField primarySortField = Sort.getPrimarySortField(context.reader());+ if (primarySortField != null && primarySortField.getField().equals(field)) {+ final SortField.Type sortFieldType = getSortFieldType(primarySortField);// The index sort optimization is only supported for Type.INT and Type.LONGif (sortFieldType == Type.INT || sortFieldType == Type.LONG) {- return getDocIdSetIterator(sortField, sortFieldType, context, numericValues);+ return getDocIdSetIterator(primarySortField, sortFieldType, context, numericValues);}}}diff --git a/lucene/core/src/java/org/apache/lucene/search/Sort.java b/lucene/core/src/java/org/apache/lucene/search/Sort.javaindex f46515e8ea3..a037e0751ed 100644--- a/lucene/core/src/java/org/apache/lucene/search/Sort.java+++ b/lucene/core/src/java/org/apache/lucene/search/Sort.java@@ -18,6 +18,8 @@ package org.apache.lucene.search;import java.io.IOException;import java.util.Arrays;+import org.apache.lucene.index.DocValuesSkipper;+import org.apache.lucene.index.LeafReader;/*** Encapsulates sort criteria for returned hits.@@ -132,4 +134,41 @@ public final class Sort {}return false;}++ /**+ * Returns the primary index sort field for the given LeafReader, or null if none is configured.+ *+ * <p>If sorting against a field would have no effect then that SortField is skipped over and the+ * next SortField in the list is returned. This can happen for fields with no values in the+ * segment, or fields that have only a single distinct value.+ */+ public static SortField getPrimarySortField(LeafReader reader) {+ Sort sort = reader.getMetaData().sort();+ if (sort == null) {+ return null;+ }+ for (SortField sf : sort.fields) {+ String field = sf.getField();+ if (field == null) {+ // Custom field that we don't know anything about, so return it as primary+ return sf;+ }+ if (reader.getFieldInfos().fieldInfo(field) == null) {+ // Field has no values in this segment, so sorting by it has no effect.+ continue;+ }+ // If the field has a skip index, check whether all values are identical,+ // in which case sorting by this field is a no-op for this segment. Note+ // that we also need to check for denseness, as missing values add an+ // implicit second sort group.+ DocValuesSkipper skipper = reader.getDocValuesSkipper(field);+ if (skipper != null+ && skipper.docCount() == reader.maxDoc()+ && skipper.minValue() == skipper.maxValue()) {+ continue;+ }+ return sf;+ }+ return null;+ }}diff --git a/lucene/core/src/test/org/apache/lucene/search/TestDocValuesQueries.java b/lucene/core/src/test/org/apache/lucene/search/TestDocValuesQueries.javaindex a932dc0061c..cb70682fece 100644--- a/lucene/core/src/test/org/apache/lucene/search/TestDocValuesQueries.java+++ b/lucene/core/src/test/org/apache/lucene/search/TestDocValuesQueries.java@@ -1122,4 +1122,252 @@ public class TestDocValuesQueries extends LuceneTestCaseJupiter {reader.close();dir.close();}++ /**+ * Verifies that SortedNumericDocValuesRangeQuery can use a secondary sort field to accelerate+ * querying if the primary sort field is single-valued and therefore a no-op+ */+ @Test+ public void testSortedNumericRangeQueryOptimizesWithConstantPrimary(Random random)+ throws IOException {+ Directory dir = newDirectory();+ IndexWriterConfig config = new IndexWriterConfig().setCodec(getCodec(random));+ config.setIndexSort(+ new Sort(+ new SortField("primary", SortField.Type.LONG), // constant → no-op+ new SortField("secondary", SortField.Type.LONG))); // varying → effective primary+ IndexWriter iw = new IndexWriter(dir, config);+ for (int i = 0; i < 10; i++) {+ Document doc = new Document();+ doc.add(NumericDocValuesField.indexedField("primary", 42)); // constant, skip index present+ doc.add(NumericDocValuesField.indexedField("secondary", i)); // varying+ iw.addDocument(doc);+ }+ iw.forceMerge(1);+ iw.close();+ DirectoryReader reader = DirectoryReader.open(dir);+ IndexSearcher searcher = new IndexSearcher(reader);++ // primary has a constant value, so getPrimarySortField skips it and returns "secondary".+ // The query on "secondary" should therefore use SortedSkipperScorerSupplier,+ // which yields a plain range iterator with no two-phase iterator.+ Query query = SortedNumericDocValuesField.newSlowRangeQuery("secondary", 3, 7);+ Weight weight = query.createWeight(searcher, ScoreMode.COMPLETE, 1.0f);+ Scorer scorer = weight.scorer(reader.leaves().get(0));+ assertNotNull(scorer);+ assertNull(+ "SortedSkipperScorerSupplier should be used when primary sort is a no-op",+ scorer.twoPhaseIterator());++ reader.close();+ dir.close();+ }++ /**+ * Verifies that SortedNumericDocValuesRangeQuery can use a secondary sort field to accelerate+ * querying if the primary sort field is empty and therefore a no-op+ */+ @Test+ public void testSortedNumericRangeQueryOptimizesWithEmptyPrimary(Random random)+ throws IOException {+ Directory dir = newDirectory();+ IndexWriterConfig config = new IndexWriterConfig().setCodec(getCodec(random));+ config.setIndexSort(+ new Sort(+ new SortField("primary", SortField.Type.LONG), // empty → no-op+ new SortField("secondary", SortField.Type.LONG))); // varying → effective primary+ IndexWriter iw = new IndexWriter(dir, config);+ for (int i = 0; i < 10; i++) {+ Document doc = new Document();+ // nothing added for the primary sort field+ doc.add(NumericDocValuesField.indexedField("secondary", i)); // varying+ iw.addDocument(doc);+ }+ iw.forceMerge(1);+ iw.close();+ DirectoryReader reader = DirectoryReader.open(dir);+ IndexSearcher searcher = new IndexSearcher(reader);++ // primary has a constant value, so getPrimarySortField skips it and returns "secondary".+ // The query on "secondary" should therefore use SortedSkipperScorerSupplier,+ // which yields a plain range iterator with no two-phase iterator.+ Query query = SortedNumericDocValuesField.newSlowRangeQuery("secondary", 3, 7);+ Weight weight = query.createWeight(searcher, ScoreMode.COMPLETE, 1.0f);+ Scorer scorer = weight.scorer(reader.leaves().get(0));+ assertNotNull(scorer);+ assertNull(+ "SortedSkipperScorerSupplier should be used when primary sort is a no-op",+ scorer.twoPhaseIterator());++ reader.close();+ dir.close();+ }++ /**+ * Verifies that SortedNumericDocValuesRangeQuery does not use a secondary sort field to+ * accelerate querying if the primary sort field is a genuine sort field.+ */+ @Test+ public void testSortedNumericRangeQueryDoesNotOptimizeWithActivePrimary(Random random)+ throws IOException {+ Directory dir = newDirectory();+ IndexWriterConfig config = new IndexWriterConfig().setCodec(getCodec(random));+ config.setIndexSort(+ new Sort(+ new SortField("primary", SortField.Type.LONG), // multiple values → effective primary+ new SortField("secondary", SortField.Type.LONG)));+ IndexWriter iw = new IndexWriter(dir, config);+ for (int i = 0; i < 10; i++) {+ Document doc = new Document();+ doc.add(NumericDocValuesField.indexedField("primary", i)); // varying+ doc.add(NumericDocValuesField.indexedField("secondary", i)); // varying+ iw.addDocument(doc);+ }+ iw.forceMerge(1);+ iw.close();+ DirectoryReader reader = DirectoryReader.open(dir);+ IndexSearcher searcher = new IndexSearcher(reader);++ // primary is active, so getPrimarySortField returns "primary", not "secondary".+ // The query on "secondary" must fall back to the TwoPhaseIterator path.+ Query query = SortedNumericDocValuesField.newSlowRangeQuery("secondary", 3, 7);+ Weight weight = query.createWeight(searcher, ScoreMode.COMPLETE, 1.0f);+ Scorer scorer = weight.scorer(reader.leaves().get(0));+ assertNotNull(scorer);+ assertNotNull(+ "TwoPhaseIterator fallback should be used when secondary is not the effective primary",+ scorer.twoPhaseIterator());++ reader.close();+ dir.close();+ }++ /**+ * Verifies that SortedSetDocValuesRangeQuery can use a secondary sort field to accelerate+ * querying if the primary sort field is single-valued and therefore a no-op+ */+ @Test+ public void testSortedSetRangeQueryOptimizesWithConstantPrimary(Random random)+ throws IOException {+ Directory dir = newDirectory();+ IndexWriterConfig config = new IndexWriterConfig().setCodec(getCodec(random));+ config.setIndexSort(+ new Sort(+ new SortField("primary", SortField.Type.STRING), // constant → no-op+ new SortField("secondary", SortField.Type.STRING))); // varying → effective primary+ IndexWriter iw = new IndexWriter(dir, config);+ for (int i = 0; i < 10; i++) {+ Document doc = new Document();+ doc.add(SortedDocValuesField.indexedField("primary", newBytesRef("constant")));+ doc.add(+ SortedDocValuesField.indexedField(+ "secondary", newBytesRef(String.format(Locale.ROOT, "%03d", i))));+ iw.addDocument(doc);+ }+ iw.forceMerge(1);+ iw.close();+ DirectoryReader reader = DirectoryReader.open(dir);+ IndexSearcher searcher = new IndexSearcher(reader);++ // primary has a constant ordinal, so getPrimarySortField skips it and returns "secondary".+ // The query on "secondary" should therefore use SortedSkipperScorerSupplier,+ // which yields a plain range iterator with no two-phase iterator.+ Query query =+ SortedSetDocValuesField.newSlowRangeQuery(+ "secondary", newBytesRef("003"), newBytesRef("007"), true, true);+ Weight weight = query.createWeight(searcher, ScoreMode.COMPLETE, 1.0f);+ Scorer scorer = weight.scorer(reader.leaves().get(0));+ assertNotNull(scorer);+ assertNull(+ "SortedSkipperScorerSupplier should be used when primary sort is a no-op",+ scorer.twoPhaseIterator());++ reader.close();+ dir.close();+ }++ /**+ * Verifies that SortedSetDocValuesRangeQuery can use a secondary sort field to accelerate+ * querying if the primary sort field is empty and therefore a no-op+ */+ @Test+ public void testSortedSetRangeQueryOptimizesWithEmptyPrimary(Random random) throws IOException {+ Directory dir = newDirectory();+ IndexWriterConfig config = new IndexWriterConfig().setCodec(getCodec(random));+ config.setIndexSort(+ new Sort(+ new SortField("primary", SortField.Type.STRING), // constant → no-op+ new SortField("secondary", SortField.Type.STRING))); // varying → effective primary+ IndexWriter iw = new IndexWriter(dir, config);+ for (int i = 0; i < 10; i++) {+ Document doc = new Document();+ // no values added for primary field+ doc.add(+ SortedDocValuesField.indexedField(+ "secondary", newBytesRef(String.format(Locale.ROOT, "%03d", i))));+ iw.addDocument(doc);+ }+ iw.forceMerge(1);+ iw.close();+ DirectoryReader reader = DirectoryReader.open(dir);+ IndexSearcher searcher = new IndexSearcher(reader);++ // primary has a constant ordinal, so getPrimarySortField skips it and returns "secondary".+ // The query on "secondary" should therefore use SortedSkipperScorerSupplier,+ // which yields a plain range iterator with no two-phase iterator.+ Query query =+ SortedSetDocValuesField.newSlowRangeQuery(+ "secondary", newBytesRef("003"), newBytesRef("007"), true, true);+ Weight weight = query.createWeight(searcher, ScoreMode.COMPLETE, 1.0f);+ Scorer scorer = weight.scorer(reader.leaves().get(0));+ assertNotNull(scorer);+ assertNull(+ "SortedSkipperScorerSupplier should be used when primary sort is a no-op",+ scorer.twoPhaseIterator());++ reader.close();+ dir.close();+ }++ /**+ * Verifies that SortedSetDocValuesRangeQuery does not use a secondary sort field to accelerate+ * querying if the primary sort field is a genuine sort field.+ */+ @Test+ public void testSortedSetRangeQueryDoesNotOptimizeWithActivePrimary(Random random)+ throws IOException {+ Directory dir = newDirectory();+ IndexWriterConfig config = new IndexWriterConfig().setCodec(getCodec(random));+ config.setIndexSort(+ new Sort(+ new SortField("primary", SortField.Type.STRING), // multiple values → effective primary+ new SortField("secondary", SortField.Type.STRING)));+ IndexWriter iw = new IndexWriter(dir, config);+ for (int i = 0; i < 10; i++) {+ String val = String.format(Locale.ROOT, "%03d", i);+ Document doc = new Document();+ doc.add(SortedDocValuesField.indexedField("primary", newBytesRef(val))); // varying+ doc.add(SortedDocValuesField.indexedField("secondary", newBytesRef(val))); // varying+ iw.addDocument(doc);+ }+ iw.forceMerge(1);+ iw.close();+ DirectoryReader reader = DirectoryReader.open(dir);+ IndexSearcher searcher = new IndexSearcher(reader);++ // primary is active, so getPrimarySortField returns "primary", not "secondary".+ // The query on "secondary" must fall back to the TwoPhaseIterator path.+ Query query =+ SortedSetDocValuesField.newSlowRangeQuery(+ "secondary", newBytesRef("003"), newBytesRef("007"), true, true);+ Weight weight = query.createWeight(searcher, ScoreMode.COMPLETE, 1.0f);+ Scorer scorer = weight.scorer(reader.leaves().get(0));+ assertNotNull(scorer);+ assertNotNull(+ "TwoPhaseIterator fallback should be used when secondary is not the effective primary",+ scorer.twoPhaseIterator());++ reader.close();+ dir.close();+ }}diff --git a/lucene/core/src/test/org/apache/lucene/search/TestGetPrimarySortField.java b/lucene/core/src/test/org/apache/lucene/search/TestGetPrimarySortField.javanew file mode 100644index 00000000000..e860aaeef43--- /dev/null+++ b/lucene/core/src/test/org/apache/lucene/search/TestGetPrimarySortField.java@@ -0,0 +1,312 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.search;++import java.io.IOException;+import org.apache.lucene.document.Document;+import org.apache.lucene.document.NumericDocValuesField;+import org.apache.lucene.index.DirectoryReader;+import org.apache.lucene.index.IndexWriter;+import org.apache.lucene.index.IndexWriterConfig;+import org.apache.lucene.index.LeafReader;+import org.apache.lucene.index.NoMergePolicy;+import org.apache.lucene.store.Directory;+import org.apache.lucene.tests.util.LuceneTestCase;++/**+ * Tests for {@link Sort#getPrimarySortField(LeafReader)}, which returns the effective primary sort+ * field for a segment, skipping fields that are no-ops because they have no values in the segment+ * or have only a single distinct value according to their skip index.+ */+public class TestGetPrimarySortField extends LuceneTestCase {++ /** A reader with no index sort configured always returns null. */+ public void testNoIndexSort() throws IOException {+ Directory dir = newDirectory();+ IndexWriter iw = new IndexWriter(dir, new IndexWriterConfig());+ Document doc = new Document();+ doc.add(NumericDocValuesField.indexedField("field", 42));+ iw.addDocument(doc);+ iw.forceMerge(1);+ iw.close();+ DirectoryReader reader = DirectoryReader.open(dir);+ LeafReader leafReader = reader.leaves().get(0).reader();+ assertNull(Sort.getPrimarySortField(leafReader));+ reader.close();+ dir.close();+ }++ /**+ * When the sort field has no DocValues skip index (plain DV field), there is no skipper to detect+ * a single-value no-op, but the field exists in FieldInfos so it is returned as-is.+ */+ public void testSortFieldWithoutSkipperIsReturned() throws IOException {+ Directory dir = newDirectory();+ SortField sortField = new SortField("field", SortField.Type.LONG);+ IndexWriter iw =+ new IndexWriter(dir, new IndexWriterConfig().setIndexSort(new Sort(sortField)));+ for (int i = 0; i < 10; i++) {+ Document doc = new Document();+ doc.add(new NumericDocValuesField("field", i)); // plain DV, no skip index+ iw.addDocument(doc);+ }+ iw.forceMerge(1);+ iw.close();+ DirectoryReader reader = DirectoryReader.open(dir);+ LeafReader leafReader = reader.leaves().get(0).reader();+ SortField result = Sort.getPrimarySortField(leafReader);+ assertNotNull(result);+ assertEquals("field", result.getField());+ reader.close();+ dir.close();+ }++ /**+ * A sort field with a skip index and multiple distinct values is returned as-is: it is not a+ * no-op because the skipper's min and max values differ.+ */+ public void testSortFieldWithMultipleDistinctValuesIsReturned() throws IOException {+ Directory dir = newDirectory();+ SortField sortField = new SortField("field", SortField.Type.LONG);+ IndexWriter iw =+ new IndexWriter(dir, new IndexWriterConfig().setIndexSort(new Sort(sortField)));+ for (int i = 0; i < 10; i++) {+ Document doc = new Document();+ doc.add(NumericDocValuesField.indexedField("field", i)); // indexed DV, skipper present+ iw.addDocument(doc);+ }+ iw.forceMerge(1);+ iw.close();+ DirectoryReader reader = DirectoryReader.open(dir);+ LeafReader leafReader = reader.leaves().get(0).reader();+ SortField result = Sort.getPrimarySortField(leafReader);+ assertNotNull(result);+ assertEquals("field", result.getField());+ reader.close();+ dir.close();+ }++ /**+ * A sort field with a skip index and one distinct value but with documents with no value is+ * returned as-is: it is not a no-op because documents with no value will group together and form+ * an implicitly-valued second group.+ */+ public void testSparseSortFieldValuesIsReturned() throws IOException {+ Directory dir = newDirectory();+ SortField primary = new SortField("field1", SortField.Type.LONG);+ SortField secondary = new SortField("field2", SortField.Type.LONG);+ IndexWriter iw =+ new IndexWriter(dir, new IndexWriterConfig().setIndexSort(new Sort(primary, secondary)));+ for (int i = 0; i < 10; i++) {+ Document doc = new Document();+ if (i != 7) {+ doc.add(NumericDocValuesField.indexedField("field1", 42)); // constant → no-op+ }+ doc.add(NumericDocValuesField.indexedField("field2", i));+ iw.addDocument(doc);+ }+ iw.forceMerge(1);+ iw.close();+ DirectoryReader reader = DirectoryReader.open(dir);+ LeafReader leafReader = reader.leaves().get(0).reader();+ SortField result = Sort.getPrimarySortField(leafReader);+ assertNotNull(result);+ assertEquals("field1", result.getField());+ reader.close();+ dir.close();+ }++ /**+ * When the primary sort field has no values at all in a segment, its FieldInfo is absent and it+ * is treated as a no-op. The next sort field becomes the effective primary.+ */+ public void testPrimaryWithNoValuesInSegmentIsSkipped() throws IOException {+ Directory dir = newDirectory();+ SortField primary = new SortField("field1", SortField.Type.LONG);+ SortField secondary = new SortField("field2", SortField.Type.LONG);+ IndexWriter iw =+ new IndexWriter(dir, new IndexWriterConfig().setIndexSort(new Sort(primary, secondary)));+ for (int i = 0; i < 10; i++) {+ Document doc = new Document();+ // field1: no doc values → FieldInfos has no entry for "field1" → skipped+ doc.add(NumericDocValuesField.indexedField("field2", i));+ iw.addDocument(doc);+ }+ iw.forceMerge(1);+ iw.close();+ DirectoryReader reader = DirectoryReader.open(dir);+ LeafReader leafReader = reader.leaves().get(0).reader();+ SortField result = Sort.getPrimarySortField(leafReader);+ assertNotNull(result);+ assertEquals("field2", result.getField());+ reader.close();+ dir.close();+ }++ /**+ * When the primary sort field has a skip index with a single distinct value (skipper min == max),+ * it is a no-op and the next sort field becomes the effective primary.+ */+ public void testSingleValuePrimaryIsSkippedToSecondary() throws IOException {+ Directory dir = newDirectory();+ SortField primary = new SortField("field1", SortField.Type.LONG);+ SortField secondary = new SortField("field2", SortField.Type.LONG);+ IndexWriter iw =+ new IndexWriter(dir, new IndexWriterConfig().setIndexSort(new Sort(primary, secondary)));+ for (int i = 0; i < 10; i++) {+ Document doc = new Document();+ doc.add(NumericDocValuesField.indexedField("field1", 42)); // constant → no-op+ doc.add(NumericDocValuesField.indexedField("field2", i)); // multiple distinct values+ iw.addDocument(doc);+ }+ iw.forceMerge(1);+ iw.close();+ DirectoryReader reader = DirectoryReader.open(dir);+ LeafReader leafReader = reader.leaves().get(0).reader();+ SortField result = Sort.getPrimarySortField(leafReader);+ assertNotNull(result);+ assertEquals("field2", result.getField());+ reader.close();+ dir.close();+ }++ /**+ * When all sort fields have only a single distinct value in the segment, every field is a no-op+ * and null is returned.+ */+ public void testAllSingleValueFieldsReturnNull() throws IOException {+ Directory dir = newDirectory();+ SortField primary = new SortField("field1", SortField.Type.LONG);+ SortField secondary = new SortField("field2", SortField.Type.LONG);+ IndexWriter iw =+ new IndexWriter(dir, new IndexWriterConfig().setIndexSort(new Sort(primary, secondary)));+ for (int i = 0; i < 10; i++) {+ Document doc = new Document();+ doc.add(NumericDocValuesField.indexedField("field1", 42)); // constant+ doc.add(NumericDocValuesField.indexedField("field2", 7)); // constant+ iw.addDocument(doc);+ }+ iw.forceMerge(1);+ iw.close();+ DirectoryReader reader = DirectoryReader.open(dir);+ LeafReader leafReader = reader.leaves().get(0).reader();+ assertNull(Sort.getPrimarySortField(leafReader));+ reader.close();+ dir.close();+ }++ /**+ * With NoMergePolicy ensuring a fixed three-segment layout, each leaf independently returns its+ * own effective primary sort field: segment 0 has a real primary, segment 1 has an absent primary+ * (no FieldInfo), and segment 2 has a constant primary (single distinct value).+ */+ public void testPerSegmentEffectivePrimaryWithNoMergePolicy() throws IOException {+ Directory dir = newDirectory();+ IndexWriterConfig iwc = new IndexWriterConfig();+ iwc.setMergePolicy(NoMergePolicy.INSTANCE);+ iwc.setIndexSort(+ new Sort(+ new SortField("field1", SortField.Type.LONG), // primary+ new SortField("field2", SortField.Type.LONG))); // secondary+ IndexWriter iw = new IndexWriter(dir, iwc);++ // Segment 0: field1 has two distinct values → effective primary is field1+ Document doc = new Document();+ doc.add(NumericDocValuesField.indexedField("field1", 1));+ doc.add(NumericDocValuesField.indexedField("field2", 10));+ iw.addDocument(doc);+ doc = new Document();+ doc.add(NumericDocValuesField.indexedField("field1", 2));+ doc.add(NumericDocValuesField.indexedField("field2", 20));+ iw.addDocument(doc);+ iw.commit();++ // Segment 1: field1 absent (no FieldInfo) → effective primary is field2+ doc = new Document();+ doc.add(NumericDocValuesField.indexedField("field2", 30));+ iw.addDocument(doc);+ doc = new Document();+ doc.add(NumericDocValuesField.indexedField("field2", 31));+ iw.addDocument(doc);+ iw.commit();++ // Segment 2: field1 is constant (skipper min == max) → no-op, effective primary is field2+ doc = new Document();+ doc.add(NumericDocValuesField.indexedField("field1", 42));+ doc.add(NumericDocValuesField.indexedField("field2", 40));+ iw.addDocument(doc);+ doc = new Document();+ doc.add(NumericDocValuesField.indexedField("field1", 42));+ doc.add(NumericDocValuesField.indexedField("field2", 50));+ iw.addDocument(doc);+ iw.commit();++ iw.close();++ DirectoryReader reader = DirectoryReader.open(dir);+ assertEquals(3, reader.leaves().size());++ // Segment 0: field1 has multiple distinct values → effective primary is field1+ LeafReader leaf0 = reader.leaves().get(0).reader();+ SortField sf0 = Sort.getPrimarySortField(leaf0);+ assertNotNull(sf0);+ assertEquals("field1", sf0.getField());++ // Segment 1: field1 absent → effective primary is field2+ LeafReader leaf1 = reader.leaves().get(1).reader();+ SortField sf1 = Sort.getPrimarySortField(leaf1);+ assertNotNull(sf1);+ assertEquals("field2", sf1.getField());++ // Segment 2: field1 constant → no-op, effective primary is field2+ LeafReader leaf2 = reader.leaves().get(2).reader();+ SortField sf2 = Sort.getPrimarySortField(leaf2);+ assertNotNull(sf2);+ assertEquals("field2", sf2.getField());++ reader.close();+ dir.close();+ }++ /**+ * The reverse attribute of the effective primary sort field is correctly preserved when the+ * initial primary sort field is skipped as a no-op.+ */+ public void testSecondaryReverseAttributeIsPreserved() throws IOException {+ Directory dir = newDirectory();+ SortField primary = new SortField("field1", SortField.Type.LONG, false); // ascending+ SortField secondary = new SortField("field2", SortField.Type.LONG, true); // descending+ IndexWriter iw =+ new IndexWriter(dir, new IndexWriterConfig().setIndexSort(new Sort(primary, secondary)));+ for (int i = 0; i < 10; i++) {+ Document doc = new Document();+ doc.add(NumericDocValuesField.indexedField("field1", 42)); // constant → no-op+ doc.add(NumericDocValuesField.indexedField("field2", i));+ iw.addDocument(doc);+ }+ iw.forceMerge(1);+ iw.close();+ DirectoryReader reader = DirectoryReader.open(dir);+ LeafReader leafReader = reader.leaves().get(0).reader();+ SortField result = Sort.getPrimarySortField(leafReader);+ assertNotNull(result);+ assertEquals("field2", result.getField());+ assertTrue("secondary sort field should be reversed", result.getReverse());+ reader.close();+ dir.close();+ }+}diff --git a/lucene/core/src/test/org/apache/lucene/search/TestIndexSortSortedNumericDocValuesRangeQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestIndexSortSortedNumericDocValuesRangeQuery.javaindex 4adb4a3cb3c..69b667241c0 100644--- a/lucene/core/src/test/org/apache/lucene/search/TestIndexSortSortedNumericDocValuesRangeQuery.java+++ b/lucene/core/src/test/org/apache/lucene/search/TestIndexSortSortedNumericDocValuesRangeQuery.java@@ -24,6 +24,7 @@ import org.apache.lucene.document.Document;import org.apache.lucene.document.Field;import org.apache.lucene.document.IntPoint;import org.apache.lucene.document.LongPoint;+import org.apache.lucene.document.NumericDocValuesField;import org.apache.lucene.document.SortedNumericDocValuesField;import org.apache.lucene.document.StringField;import org.apache.lucene.index.DirectoryReader;@@ -855,6 +856,91 @@ public class TestIndexSortSortedNumericDocValuesRangeQuery extends LuceneTestCasdir.close();}+ /**+ * Verifies that the binary-search optimization activates when the primary index sort field has a+ * single constant value (detected as a no-op via its skip index), promoting the secondary field+ * to effective primary and enabling direct doc-ID range iteration without a two-phase iterator.+ */+ public void testNoOpPrimarySort() throws IOException {+ Directory dir = newDirectory();+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));+ Sort indexSort =+ new Sort(+ new SortField("field1", SortField.Type.LONG), // constant → no-op via skipper+ new SortField("field2", SortField.Type.LONG)); // varying → effective primary+ iwc.setIndexSort(indexSort);+ RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);+ for (int i = 0; i < 10; i++) {+ Document doc = new Document();+ doc.add(NumericDocValuesField.indexedField("field1", 42)); // constant; skip index present+ doc.add(NumericDocValuesField.indexedField("field2", i)); // varying+ iw.addDocument(doc);+ }+ iw.forceMerge(1);+ DirectoryReader reader = iw.getReader();+ IndexSearcher searcher = newSearcher(reader);+ iw.close();++ // field1 has a single distinct value, so getPrimarySortField skips it and returns field2.+ // The binary-search optimization should activate for the query on field2, producing a plain+ // range iterator with no two-phase iterator.+ Query fallback = SortedNumericDocValuesField.newSlowRangeQuery("field2", 3, 7);+ Query query = new IndexSortSortedNumericDocValuesRangeQuery("field2", 3, 7, fallback);+ Weight weight = query.createWeight(searcher, ScoreMode.TOP_SCORES, 1.0F);+ for (LeafReaderContext context : searcher.getIndexReader().leaves()) {+ Scorer scorer = weight.scorer(context);+ assertNotNull(scorer);+ assertNull(+ "binary-search optimization should be active when primary sort is a no-op",+ scorer.twoPhaseIterator());+ }++ reader.close();+ dir.close();+ }++ /**+ * Verifies that the binary-search optimization activates when the primary index sort field has no+ * values in the segment at all (detected as a no-op via FieldInfos), promoting the secondary+ * field to effective primary.+ */+ public void testMissingPrimarySort() throws IOException {+ Directory dir = newDirectory();+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));+ Sort indexSort =+ new Sort(+ new SortField("field1", SortField.Type.LONG), // absent → no-op via FieldInfos check+ new SortField("field2", SortField.Type.LONG)); // varying → effective primary+ iwc.setIndexSort(indexSort);+ RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);+ for (int i = 0; i < 10; i++) {+ Document doc = new Document();+ // field1 intentionally absent: no FieldInfo in segment → detected as no-op+ doc.add(NumericDocValuesField.indexedField("field2", i));+ iw.addDocument(doc);+ }+ iw.forceMerge(1);+ DirectoryReader reader = iw.getReader();+ IndexSearcher searcher = newSearcher(reader);+ iw.close();++ // field1 has no FieldInfo, so getPrimarySortField skips it and returns field2.+ // The binary-search optimization should activate for the query on field2.+ Query fallback = SortedNumericDocValuesField.newSlowRangeQuery("field2", 3, 7);+ Query query = new IndexSortSortedNumericDocValuesRangeQuery("field2", 3, 7, fallback);+ Weight weight = query.createWeight(searcher, ScoreMode.TOP_SCORES, 1.0F);+ for (LeafReaderContext context : searcher.getIndexReader().leaves()) {+ Scorer scorer = weight.scorer(context);+ assertNotNull(scorer);+ assertNull(+ "binary-search optimization should be active when primary sort field has no values",+ scorer.twoPhaseIterator());+ }++ reader.close();+ dir.close();+ }+private void addDocWithBkd(RandomIndexWriter indexWriter, String field, long value, int repeat)throws IOException {for (int i = 0; i < repeat; i++) {