Skip to content

Apache Lucene open-source search software

Follow up #16061. Tighten score supplier cost estimation and introduce SortedSkipperScorerSupplier (#16070)

Ignacio Veraa16bca0fbdaf
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 23246c59772..3f65f1956cb 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -376,8 +376,8 @@ Optimizations
* GITHUB#16001: IndexSearcher.count() was calling query.rewrite twice, a regression since v9.10 (David Smiley)
-* GITHUB16061#: Improve cost estimation in SortedSetDocValuesRangeQuery when using DocValuesSkipper and the field
- is dense and is the primary sort of the index and reduce the number of doc values visited. (Ignacio Vera)
+* GITHUB#16061, GITHUB#16070: Improve cost estimation in SortedSetDocValuesRangeQuery when using DocValuesSkipper
+ and the field is dense and is the primary sort of the index to reduce the number of doc values visited. (Ignacio Vera)
Bug Fixes
diff --git a/lucene/core/src/java/org/apache/lucene/document/SortedSetDocValuesRangeQuery.java b/lucene/core/src/java/org/apache/lucene/document/SortedSetDocValuesRangeQuery.java
index 63d200e265e..66acf63277a 100644
--- a/lucene/core/src/java/org/apache/lucene/document/SortedSetDocValuesRangeQuery.java
+++ b/lucene/core/src/java/org/apache/lucene/document/SortedSetDocValuesRangeQuery.java
@@ -17,7 +17,6 @@
package org.apache.lucene.document;
import java.io.IOException;
-import java.io.UncheckedIOException;
import java.util.Objects;
import java.util.function.LongPredicate;
import org.apache.lucene.index.DocValues;
@@ -37,6 +36,7 @@ import org.apache.lucene.search.QueryVisitor;
import org.apache.lucene.search.ScoreMode;
import org.apache.lucene.search.ScorerSupplier;
import org.apache.lucene.search.Sort;
+import org.apache.lucene.search.SortField;
import org.apache.lucene.search.TwoPhaseIterator;
import org.apache.lucene.search.Weight;
import org.apache.lucene.util.BytesRef;
@@ -121,8 +121,12 @@ final class SortedSetDocValuesRangeQuery extends Query {
DocValuesSkipper skipper = context.reader().getDocValuesSkipper(field);
SortedSetDocValues values = DocValues.getSortedSet(context.reader(), field);
final SortedDocValues singleton = DocValues.unwrapSingleton(values);
- if (singleton != null && skipper != null && isDensePrimarySort(context.reader(), skipper)) {
- return getScorerSupplierFromDensePrimarySort(context, singleton, values, skipper);
+ final SortField primarySortField;
+ if (singleton != null
+ && skipper != null
+ && (primarySortField = densePrimarySort(context.reader(), skipper)) != null) {
+ return getScorerSupplierFromDensePrimarySort(
+ context, singleton, values, skipper, primarySortField);
}
// implement ScorerSupplier, since we do some expensive stuff to make a scorer
return new ConstantScoreScorerSupplier(score(), scoreMode, context.reader().maxDoc()) {
@@ -166,121 +170,40 @@ final class SortedSetDocValuesRangeQuery extends Query {
LeafReaderContext context,
SortedDocValues singleton,
SortedSetDocValues values,
- DocValuesSkipper skipper) {
- final Sort indexSort = context.reader().getMetaData().sort();
- return new ConstantScoreScorerSupplier(score(), scoreMode, context.reader().maxDoc()) {
- int skipperMinDocId = -1, skipperMaxDocId = -1;
- long minOrd, maxOrd;
- boolean skipperMinDocIdExact = false, skipperMaxDocIdExact = false;
+ DocValuesSkipper skipper,
+ SortField sortField) {
+ return new SortedSkipperScorerSupplier(
+ skipper, sortField, score(), scoreMode, context.reader().maxDoc()) {
+ long minOrd = -1, maxOrd = -1;
@Override
- public DocIdSetIterator iterator(long leadCost) throws IOException {
- if (skipperMinDocId == -1) {
- computeSkipperDocIds();
- }
- final int minDocID;
- final int maxDocID;
- if (indexSort.getSort()[0].getReverse()) {
- minDocID =
- skipperMinDocIdExact
- ? skipperMinDocId
- : nextDoc(skipperMinDocId, singleton, l -> l <= maxOrd);
- maxDocID =
- skipperMaxDocIdExact
- ? skipperMaxDocId
- : nextDoc(skipperMaxDocId, singleton, l -> l < minOrd);
- } else {
- minDocID =
- skipperMinDocIdExact
- ? skipperMinDocId
- : nextDoc(skipperMinDocId, singleton, l -> l >= minOrd);
- maxDocID =
- skipperMaxDocIdExact
- ? skipperMaxDocId
- : nextDoc(skipperMaxDocId, singleton, l -> l > maxOrd);
+ protected long getLowerValue() throws IOException {
+ if (minOrd == -1) {
+ minOrd = minOrd(values);
}
- return minDocID == maxDocID
- ? DocIdSetIterator.empty()
- : DocIdSetIterator.range(minDocID, maxDocID);
+ return minOrd;
}
@Override
- public long cost() {
- if (skipperMinDocId == -1) {
- try {
- // Similar to PointValues, IOExceptions needs to be caught and rethrown as
- // UncheckedIOException
- computeSkipperDocIds();
- } catch (IOException e) {
- throw new UncheckedIOException(e);
- }
+ protected long getUpperValue() throws IOException {
+ if (maxOrd == -1) {
+ maxOrd = maxOrd(values);
}
- if (skipperMinDocIdExact && skipperMaxDocIdExact) {
- return skipperMaxDocId - skipperMinDocId;
- }
- // TODO: expose skipper block size here?
- return Math.min(context.reader().maxDoc(), 4096 + skipperMaxDocId - skipperMinDocId);
+ return maxOrd;
}
- private void computeSkipperDocIds() throws IOException {
- minOrd = minOrd(values);
- maxOrd = upperValue != null && upperValue.equals(lowerValue) ? minOrd : maxOrd(values);
- if (minOrd > maxOrd || minOrd > skipper.maxValue() || maxOrd < skipper.minValue()) {
- skipperMinDocId = skipperMaxDocId = DocIdSetIterator.NO_MORE_DOCS;
- skipperMinDocIdExact = skipperMaxDocIdExact = true;
- return;
- }
- if (skipper.minValue() >= minOrd && skipper.maxValue() <= maxOrd) {
- skipperMinDocId = 0;
- skipperMaxDocId = skipper.docCount();
- skipperMinDocIdExact = skipperMaxDocIdExact = true;
- return;
+ @Override
+ protected int nextDoc(int startDocId, LongPredicate predicate) throws IOException {
+ int doc = singleton.docID();
+ if (startDocId > doc) {
+ doc = singleton.advance(startDocId);
}
- if (indexSort.getSort()[0].getReverse()) {
- if (skipper.maxValue() <= maxOrd) {
- skipperMinDocId = 0;
- skipperMinDocIdExact = true;
- } else {
- skipper.advance(Long.MIN_VALUE, maxOrd);
- skipperMinDocId = skipper.minDocID(0);
- skipperMinDocIdExact = skipper.maxValue(0) == maxOrd;
- }
- if (skipper.minValue() >= minOrd) {
- skipperMaxDocId = skipper.docCount();
- skipperMaxDocIdExact = true;
- } else {
- skipper.advance(Long.MIN_VALUE, minOrd);
- skipperMaxDocId =
- skipper.minValue(0) == minOrd ? skipper.maxDocID(0) + 1 : skipper.minDocID(0);
- // we can read the next block, if the maxValue is different to minOrd, then we
- // should be done, we
- // don't need to visit the doc values. But what is more expensive, visit one doc
- // value or one skipper block?
- skipperMaxDocIdExact = false;
- }
- } else {
- if (skipper.minValue() >= minOrd) {
- skipperMinDocId = 0;
- skipperMinDocIdExact = true;
- } else {
- skipper.advance(minOrd, Long.MAX_VALUE);
- skipperMinDocId = skipper.minDocID(0);
- skipperMinDocIdExact = skipper.minValue(0) == minOrd;
- }
- if (skipper.maxValue() <= maxOrd) {
- skipperMaxDocId = skipper.docCount();
- skipperMaxDocIdExact = true;
- } else {
- skipper.advance(maxOrd, Long.MAX_VALUE);
- skipperMaxDocId =
- skipper.maxValue(0) == maxOrd ? skipper.maxDocID(0) + 1 : skipper.minDocID(0);
- // we can read the next block, if the minValue is different to maxOrd, then we
- // should be done, we
- // don't need to visit the doc values. But what is more expensive, visit one doc
- // value or one skipper block?
- skipperMaxDocIdExact = false;
+ for (; doc < DocIdSetIterator.NO_MORE_DOCS; doc = singleton.nextDoc()) {
+ if (predicate.test(singleton.ordValue())) {
+ break;
}
}
+ return doc;
}
};
}
@@ -326,30 +249,16 @@ final class SortedSetDocValuesRangeQuery extends Query {
return maxOrd;
}
- private boolean isDensePrimarySort(LeafReader reader, DocValuesSkipper skipper) {
+ private SortField densePrimarySort(LeafReader reader, DocValuesSkipper skipper) {
if (skipper.docCount() != reader.maxDoc()) {
- return false;
+ return null;
}
final Sort indexSort = reader.getMetaData().sort();
if (indexSort == null
|| indexSort.getSort().length == 0
|| indexSort.getSort()[0].getField().equals(field) == false) {
- return false;
- }
- return true;
- }
-
- private static int nextDoc(int startDoc, SortedDocValues docValues, LongPredicate predicate)
- throws IOException {
- int doc = docValues.docID();
- if (startDoc > doc) {
- doc = docValues.advance(startDoc);
- }
- for (; doc < DocIdSetIterator.NO_MORE_DOCS; doc = docValues.nextDoc()) {
- if (predicate.test(docValues.ordValue())) {
- break;
- }
+ return null;
}
- return doc;
+ return indexSort.getSort()[0];
}
}
diff --git a/lucene/core/src/java/org/apache/lucene/document/SortedSkipperScorerSupplier.java b/lucene/core/src/java/org/apache/lucene/document/SortedSkipperScorerSupplier.java
new file mode 100644
index 00000000000..d840a75c8fe
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/document/SortedSkipperScorerSupplier.java
@@ -0,0 +1,179 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.document;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.function.LongPredicate;
+import org.apache.lucene.index.DocValuesSkipper;
+import org.apache.lucene.search.ConstantScoreScorerSupplier;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.ScoreMode;
+import org.apache.lucene.search.SortField;
+
+/**
+ * Constant-score supplier that uses a {@link DocValuesSkipper} to approximate the document interval
+ * matching a numeric or ordinal range when the leaf's primary {@link SortField} aligns with that
+ * field.
+ *
+ * <p>The skipper yields coarse min/max doc IDs that contain all matching documents. When the
+ * skipper interval at a boundary is not exact for the query range, {@link #nextDoc} scans forward
+ * from that doc to the first document whose value satisfies the appropriate comparison, producing a
+ * tight {@link DocIdSetIterator#range} iterator.
+ */
+abstract class SortedSkipperScorerSupplier extends ConstantScoreScorerSupplier {
+
+ private final DocValuesSkipper skipper;
+ private final SortField sortField;
+ private int skipperMinDocId = -1, skipperMaxDocId = -1;
+ private boolean skipperMinDocIdExact = false, skipperMaxDocIdExact = false;
+
+ /**
+ * @param skipper skipper for the doc values range field (same field as {@code sortField})
+ * @param sortField leaf primary sort; {@link SortField#getReverse()} controls how range endpoints
+ * map to ordinal or value predicates
+ * @param score constant score assigned to hits
+ * @param scoreMode scoring mode passed to {@link ConstantScoreScorerSupplier}
+ * @param maxDoc exclusive upper bound of doc IDs on this leaf (typically {@link
+ * org.apache.lucene.index.LeafReader#maxDoc()})
+ */
+ SortedSkipperScorerSupplier(
+ DocValuesSkipper skipper, SortField sortField, float score, ScoreMode scoreMode, int maxDoc) {
+ super(score, scoreMode, maxDoc);
+ this.skipper = skipper;
+ this.sortField = sortField;
+ }
+
+ /** Lower bound of the query range in the same coordinate space as the skipper values. */
+ protected abstract long getLowerValue() throws IOException;
+
+ /** Upper bound of the query range in the same coordinate space as the skipper values. */
+ protected abstract long getUpperValue() throws IOException;
+
+ /**
+ * Advance doc values from {@code startDocID} and return the first document whose value makes
+ * {@code predicate} true, or {@link DocIdSetIterator#NO_MORE_DOCS} if there is none. Used to
+ * refine skipper-derived boundaries when the interval min/max is not exact for the range.
+ *
+ * @param startDocID first doc ID to examine (inclusive)
+ * @param predicate tests the per-document value (ordinal or numeric) at the current doc
+ */
+ protected abstract int nextDoc(int startDocID, LongPredicate predicate) throws IOException;
+
+ @Override
+ public DocIdSetIterator iterator(long leadCost) throws IOException {
+ if (skipperMinDocId == -1) {
+ computeSkipperDocIds();
+ }
+ long minOrd = getLowerValue();
+ long maxOrd = getUpperValue();
+ final int minDocID;
+ final int maxDocID;
+ if (sortField.getReverse()) {
+ minDocID =
+ skipperMinDocIdExact ? skipperMinDocId : nextDoc(skipperMinDocId, l -> l <= maxOrd);
+ maxDocID = skipperMaxDocIdExact ? skipperMaxDocId : nextDoc(skipperMaxDocId, l -> l < minOrd);
+ } else {
+ minDocID =
+ skipperMinDocIdExact ? skipperMinDocId : nextDoc(skipperMinDocId, l -> l >= minOrd);
+ maxDocID = skipperMaxDocIdExact ? skipperMaxDocId : nextDoc(skipperMaxDocId, l -> l > maxOrd);
+ }
+ return minDocID == maxDocID
+ ? DocIdSetIterator.empty()
+ : DocIdSetIterator.range(minDocID, maxDocID);
+ }
+
+ @Override
+ public long cost() {
+ if (skipperMinDocId == -1) {
+ try {
+ // Similar to PointValues, IOExceptions needs to be caught and rethrown as
+ // UncheckedIOException
+ computeSkipperDocIds();
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+ if (skipperMinDocIdExact && skipperMaxDocIdExact) {
+ return skipperMaxDocId - skipperMinDocId;
+ }
+ return skipperMaxDocId - skipperMinDocId + skipper.docCount(0);
+ }
+
+ private void computeSkipperDocIds() throws IOException {
+ long minOrd = getLowerValue();
+ long maxOrd = getUpperValue();
+ if (minOrd > maxOrd || minOrd > skipper.maxValue() || maxOrd < skipper.minValue()) {
+ skipperMinDocId = skipperMaxDocId = DocIdSetIterator.NO_MORE_DOCS;
+ skipperMinDocIdExact = skipperMaxDocIdExact = true;
+ return;
+ }
+ if (skipper.minValue() >= minOrd && skipper.maxValue() <= maxOrd) {
+ skipperMinDocId = 0;
+ skipperMaxDocId = skipper.docCount();
+ skipperMinDocIdExact = skipperMaxDocIdExact = true;
+ return;
+ }
+ if (sortField.getReverse()) {
+ if (skipper.maxValue() <= maxOrd) {
+ skipperMinDocId = 0;
+ skipperMinDocIdExact = true;
+ } else {
+ skipper.advance(Long.MIN_VALUE, maxOrd);
+ skipperMinDocId = skipper.minDocID(0);
+ skipperMinDocIdExact = skipper.maxValue(0) == maxOrd;
+ }
+ if (skipper.minValue() >= minOrd) {
+ skipperMaxDocId = skipper.docCount();
+ skipperMaxDocIdExact = true;
+ } else {
+ skipper.advance(Long.MIN_VALUE, minOrd);
+ if (skipper.minValue(0) == minOrd) {
+ skipperMaxDocId = skipper.maxDocID(0) + 1;
+ skipper.advance(skipperMaxDocId);
+ skipperMaxDocIdExact = skipper.maxValue(0) != minOrd;
+ } else {
+ skipperMaxDocId = skipper.minDocID(0);
+ skipperMaxDocIdExact = false;
+ }
+ }
+ } else {
+ if (skipper.minValue() >= minOrd) {
+ skipperMinDocId = 0;
+ skipperMinDocIdExact = true;
+ } else {
+ skipper.advance(minOrd, Long.MAX_VALUE);
+ skipperMinDocId = skipper.minDocID(0);
+ skipperMinDocIdExact = skipper.minValue(0) == minOrd;
+ }
+ if (skipper.maxValue() <= maxOrd) {
+ skipperMaxDocId = skipper.docCount();
+ skipperMaxDocIdExact = true;
+ } else {
+ skipper.advance(maxOrd, Long.MAX_VALUE);
+ if (skipper.maxValue(0) == maxOrd) {
+ skipperMaxDocId = skipper.maxDocID(0) + 1;
+ skipper.advance(skipperMaxDocId);
+ skipperMaxDocIdExact = skipper.minValue(0) != maxOrd;
+ } else {
+ skipperMaxDocId = skipper.minDocID(0);
+ skipperMaxDocIdExact = false;
+ }
+ }
+ }
+ }
+}
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestDocValuesQueries.java b/lucene/core/src/test/org/apache/lucene/search/TestDocValuesQueries.java
index 1bcc196cd97..cb3e28e0816 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestDocValuesQueries.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestDocValuesQueries.java
@@ -58,6 +58,11 @@ public class TestDocValuesQueries extends LuceneTestCase {
return TestUtil.alwaysDocValuesFormat(new Lucene90DocValuesFormat(random().nextInt(4, 16)));
}
+ private Codec getCodec(int skipIntervalSize) {
+ // small interval size to test with many intervals
+ return TestUtil.alwaysDocValuesFormat(new Lucene90DocValuesFormat(skipIntervalSize));
+ }
+
public void testDuelPointRangeSortedNumericRangeQuery() throws IOException {
doTestDuelPointRangeNumericRangeQuery(true, 1, false);
}
@@ -757,15 +762,14 @@ public class TestDocValuesQueries extends LuceneTestCase {
public void testPrimarySortDenseSortedDocValuesExactMatch() throws IOException {
Directory dir = newDirectory();
- IndexWriterConfig config = new IndexWriterConfig().setCodec(getCodec());
+ int skipIntervalSize = random().nextInt(4, 4096);
+ IndexWriterConfig config = new IndexWriterConfig().setCodec(getCodec(skipIntervalSize));
config.setIndexSort(
new Sort(new SortField("dv", SortField.Type.STRING, random().nextBoolean())));
int numBlocks = random().nextInt(4, 16);
int[] sizes = new int[numBlocks];
- int totalDocs = 0;
for (int i = 0; i < numBlocks; i++) {
sizes[i] = random().nextInt(1, 250);
- totalDocs += sizes[i];
}
RandomIndexWriter iw = new RandomIndexWriter(random(), dir, config);
for (int i = 0; i < numBlocks; i++) {
@@ -796,8 +800,7 @@ public class TestDocValuesQueries extends LuceneTestCase {
Weight weight = rewritten.createWeight(searcher, ScoreMode.COMPLETE_NO_SCORES, 1.0f);
ScorerSupplier supplier = weight.scorerSupplier(ctx);
assertThat(supplier.cost(), greaterThanOrEqualTo((long) sizes[i]));
- assertThat(
- supplier.cost(), lessThanOrEqualTo(Math.min((long) sizes[i] + 2 * 4096, totalDocs)));
+ assertThat(supplier.cost(), lessThanOrEqualTo(sizes[i] + 2L * skipIntervalSize));
}
reader.close();
dir.close();