Apache Lucene open-source search software
Follow up #16061. Tighten score supplier cost estimation and introduce SortedSkipperScorerSupplier (#16070)
Ignacio Veraa16bca0fbdaf
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex 23246c59772..3f65f1956cb 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -376,8 +376,8 @@ Optimizations* GITHUB#16001: IndexSearcher.count() was calling query.rewrite twice, a regression since v9.10 (David Smiley)-* GITHUB16061#: Improve cost estimation in SortedSetDocValuesRangeQuery when using DocValuesSkipper and the field- is dense and is the primary sort of the index and reduce the number of doc values visited. (Ignacio Vera)+* GITHUB#16061, GITHUB#16070: Improve cost estimation in SortedSetDocValuesRangeQuery when using DocValuesSkipper+ and the field is dense and is the primary sort of the index to reduce the number of doc values visited. (Ignacio Vera)Bug Fixesdiff --git a/lucene/core/src/java/org/apache/lucene/document/SortedSetDocValuesRangeQuery.java b/lucene/core/src/java/org/apache/lucene/document/SortedSetDocValuesRangeQuery.javaindex 63d200e265e..66acf63277a 100644--- a/lucene/core/src/java/org/apache/lucene/document/SortedSetDocValuesRangeQuery.java+++ b/lucene/core/src/java/org/apache/lucene/document/SortedSetDocValuesRangeQuery.java@@ -17,7 +17,6 @@package org.apache.lucene.document;import java.io.IOException;-import java.io.UncheckedIOException;import java.util.Objects;import java.util.function.LongPredicate;import org.apache.lucene.index.DocValues;@@ -37,6 +36,7 @@ import org.apache.lucene.search.QueryVisitor;import org.apache.lucene.search.ScoreMode;import org.apache.lucene.search.ScorerSupplier;import org.apache.lucene.search.Sort;+import org.apache.lucene.search.SortField;import org.apache.lucene.search.TwoPhaseIterator;import org.apache.lucene.search.Weight;import org.apache.lucene.util.BytesRef;@@ -121,8 +121,12 @@ final class SortedSetDocValuesRangeQuery extends Query {DocValuesSkipper skipper = context.reader().getDocValuesSkipper(field);SortedSetDocValues values = DocValues.getSortedSet(context.reader(), field);final SortedDocValues singleton = DocValues.unwrapSingleton(values);- if (singleton != null && skipper != null && isDensePrimarySort(context.reader(), skipper)) {- return getScorerSupplierFromDensePrimarySort(context, singleton, values, skipper);+ final SortField primarySortField;+ if (singleton != null+ && skipper != null+ && (primarySortField = densePrimarySort(context.reader(), skipper)) != null) {+ return getScorerSupplierFromDensePrimarySort(+ context, singleton, values, skipper, primarySortField);}// implement ScorerSupplier, since we do some expensive stuff to make a scorerreturn new ConstantScoreScorerSupplier(score(), scoreMode, context.reader().maxDoc()) {@@ -166,121 +170,40 @@ final class SortedSetDocValuesRangeQuery extends Query {LeafReaderContext context,SortedDocValues singleton,SortedSetDocValues values,- DocValuesSkipper skipper) {- final Sort indexSort = context.reader().getMetaData().sort();- return new ConstantScoreScorerSupplier(score(), scoreMode, context.reader().maxDoc()) {- int skipperMinDocId = -1, skipperMaxDocId = -1;- long minOrd, maxOrd;- boolean skipperMinDocIdExact = false, skipperMaxDocIdExact = false;+ DocValuesSkipper skipper,+ SortField sortField) {+ return new SortedSkipperScorerSupplier(+ skipper, sortField, score(), scoreMode, context.reader().maxDoc()) {+ long minOrd = -1, maxOrd = -1;@Override- public DocIdSetIterator iterator(long leadCost) throws IOException {- if (skipperMinDocId == -1) {- computeSkipperDocIds();- }- final int minDocID;- final int maxDocID;- if (indexSort.getSort()[0].getReverse()) {- minDocID =- skipperMinDocIdExact- ? skipperMinDocId- : nextDoc(skipperMinDocId, singleton, l -> l <= maxOrd);- maxDocID =- skipperMaxDocIdExact- ? skipperMaxDocId- : nextDoc(skipperMaxDocId, singleton, l -> l < minOrd);- } else {- minDocID =- skipperMinDocIdExact- ? skipperMinDocId- : nextDoc(skipperMinDocId, singleton, l -> l >= minOrd);- maxDocID =- skipperMaxDocIdExact- ? skipperMaxDocId- : nextDoc(skipperMaxDocId, singleton, l -> l > maxOrd);+ protected long getLowerValue() throws IOException {+ if (minOrd == -1) {+ minOrd = minOrd(values);}- return minDocID == maxDocID- ? DocIdSetIterator.empty()- : DocIdSetIterator.range(minDocID, maxDocID);+ return minOrd;}@Override- public long cost() {- if (skipperMinDocId == -1) {- try {- // Similar to PointValues, IOExceptions needs to be caught and rethrown as- // UncheckedIOException- computeSkipperDocIds();- } catch (IOException e) {- throw new UncheckedIOException(e);- }+ protected long getUpperValue() throws IOException {+ if (maxOrd == -1) {+ maxOrd = maxOrd(values);}- if (skipperMinDocIdExact && skipperMaxDocIdExact) {- return skipperMaxDocId - skipperMinDocId;- }- // TODO: expose skipper block size here?- return Math.min(context.reader().maxDoc(), 4096 + skipperMaxDocId - skipperMinDocId);+ return maxOrd;}- private void computeSkipperDocIds() throws IOException {- minOrd = minOrd(values);- maxOrd = upperValue != null && upperValue.equals(lowerValue) ? minOrd : maxOrd(values);- if (minOrd > maxOrd || minOrd > skipper.maxValue() || maxOrd < skipper.minValue()) {- skipperMinDocId = skipperMaxDocId = DocIdSetIterator.NO_MORE_DOCS;- skipperMinDocIdExact = skipperMaxDocIdExact = true;- return;- }- if (skipper.minValue() >= minOrd && skipper.maxValue() <= maxOrd) {- skipperMinDocId = 0;- skipperMaxDocId = skipper.docCount();- skipperMinDocIdExact = skipperMaxDocIdExact = true;- return;+ @Override+ protected int nextDoc(int startDocId, LongPredicate predicate) throws IOException {+ int doc = singleton.docID();+ if (startDocId > doc) {+ doc = singleton.advance(startDocId);}- if (indexSort.getSort()[0].getReverse()) {- if (skipper.maxValue() <= maxOrd) {- skipperMinDocId = 0;- skipperMinDocIdExact = true;- } else {- skipper.advance(Long.MIN_VALUE, maxOrd);- skipperMinDocId = skipper.minDocID(0);- skipperMinDocIdExact = skipper.maxValue(0) == maxOrd;- }- if (skipper.minValue() >= minOrd) {- skipperMaxDocId = skipper.docCount();- skipperMaxDocIdExact = true;- } else {- skipper.advance(Long.MIN_VALUE, minOrd);- skipperMaxDocId =- skipper.minValue(0) == minOrd ? skipper.maxDocID(0) + 1 : skipper.minDocID(0);- // we can read the next block, if the maxValue is different to minOrd, then we- // should be done, we- // don't need to visit the doc values. But what is more expensive, visit one doc- // value or one skipper block?- skipperMaxDocIdExact = false;- }- } else {- if (skipper.minValue() >= minOrd) {- skipperMinDocId = 0;- skipperMinDocIdExact = true;- } else {- skipper.advance(minOrd, Long.MAX_VALUE);- skipperMinDocId = skipper.minDocID(0);- skipperMinDocIdExact = skipper.minValue(0) == minOrd;- }- if (skipper.maxValue() <= maxOrd) {- skipperMaxDocId = skipper.docCount();- skipperMaxDocIdExact = true;- } else {- skipper.advance(maxOrd, Long.MAX_VALUE);- skipperMaxDocId =- skipper.maxValue(0) == maxOrd ? skipper.maxDocID(0) + 1 : skipper.minDocID(0);- // we can read the next block, if the minValue is different to maxOrd, then we- // should be done, we- // don't need to visit the doc values. But what is more expensive, visit one doc- // value or one skipper block?- skipperMaxDocIdExact = false;+ for (; doc < DocIdSetIterator.NO_MORE_DOCS; doc = singleton.nextDoc()) {+ if (predicate.test(singleton.ordValue())) {+ break;}}+ return doc;}};}@@ -326,30 +249,16 @@ final class SortedSetDocValuesRangeQuery extends Query {return maxOrd;}- private boolean isDensePrimarySort(LeafReader reader, DocValuesSkipper skipper) {+ private SortField densePrimarySort(LeafReader reader, DocValuesSkipper skipper) {if (skipper.docCount() != reader.maxDoc()) {- return false;+ return null;}final Sort indexSort = reader.getMetaData().sort();if (indexSort == null|| indexSort.getSort().length == 0|| indexSort.getSort()[0].getField().equals(field) == false) {- return false;- }- return true;- }-- private static int nextDoc(int startDoc, SortedDocValues docValues, LongPredicate predicate)- throws IOException {- int doc = docValues.docID();- if (startDoc > doc) {- doc = docValues.advance(startDoc);- }- for (; doc < DocIdSetIterator.NO_MORE_DOCS; doc = docValues.nextDoc()) {- if (predicate.test(docValues.ordValue())) {- break;- }+ return null;}- return doc;+ return indexSort.getSort()[0];}}diff --git a/lucene/core/src/java/org/apache/lucene/document/SortedSkipperScorerSupplier.java b/lucene/core/src/java/org/apache/lucene/document/SortedSkipperScorerSupplier.javanew file mode 100644index 00000000000..d840a75c8fe--- /dev/null+++ b/lucene/core/src/java/org/apache/lucene/document/SortedSkipperScorerSupplier.java@@ -0,0 +1,179 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.document;++import java.io.IOException;+import java.io.UncheckedIOException;+import java.util.function.LongPredicate;+import org.apache.lucene.index.DocValuesSkipper;+import org.apache.lucene.search.ConstantScoreScorerSupplier;+import org.apache.lucene.search.DocIdSetIterator;+import org.apache.lucene.search.ScoreMode;+import org.apache.lucene.search.SortField;++/**+ * Constant-score supplier that uses a {@link DocValuesSkipper} to approximate the document interval+ * matching a numeric or ordinal range when the leaf's primary {@link SortField} aligns with that+ * field.+ *+ * <p>The skipper yields coarse min/max doc IDs that contain all matching documents. When the+ * skipper interval at a boundary is not exact for the query range, {@link #nextDoc} scans forward+ * from that doc to the first document whose value satisfies the appropriate comparison, producing a+ * tight {@link DocIdSetIterator#range} iterator.+ */+abstract class SortedSkipperScorerSupplier extends ConstantScoreScorerSupplier {++ private final DocValuesSkipper skipper;+ private final SortField sortField;+ private int skipperMinDocId = -1, skipperMaxDocId = -1;+ private boolean skipperMinDocIdExact = false, skipperMaxDocIdExact = false;++ /**+ * @param skipper skipper for the doc values range field (same field as {@code sortField})+ * @param sortField leaf primary sort; {@link SortField#getReverse()} controls how range endpoints+ * map to ordinal or value predicates+ * @param score constant score assigned to hits+ * @param scoreMode scoring mode passed to {@link ConstantScoreScorerSupplier}+ * @param maxDoc exclusive upper bound of doc IDs on this leaf (typically {@link+ * org.apache.lucene.index.LeafReader#maxDoc()})+ */+ SortedSkipperScorerSupplier(+ DocValuesSkipper skipper, SortField sortField, float score, ScoreMode scoreMode, int maxDoc) {+ super(score, scoreMode, maxDoc);+ this.skipper = skipper;+ this.sortField = sortField;+ }++ /** Lower bound of the query range in the same coordinate space as the skipper values. */+ protected abstract long getLowerValue() throws IOException;++ /** Upper bound of the query range in the same coordinate space as the skipper values. */+ protected abstract long getUpperValue() throws IOException;++ /**+ * Advance doc values from {@code startDocID} and return the first document whose value makes+ * {@code predicate} true, or {@link DocIdSetIterator#NO_MORE_DOCS} if there is none. Used to+ * refine skipper-derived boundaries when the interval min/max is not exact for the range.+ *+ * @param startDocID first doc ID to examine (inclusive)+ * @param predicate tests the per-document value (ordinal or numeric) at the current doc+ */+ protected abstract int nextDoc(int startDocID, LongPredicate predicate) throws IOException;++ @Override+ public DocIdSetIterator iterator(long leadCost) throws IOException {+ if (skipperMinDocId == -1) {+ computeSkipperDocIds();+ }+ long minOrd = getLowerValue();+ long maxOrd = getUpperValue();+ final int minDocID;+ final int maxDocID;+ if (sortField.getReverse()) {+ minDocID =+ skipperMinDocIdExact ? skipperMinDocId : nextDoc(skipperMinDocId, l -> l <= maxOrd);+ maxDocID = skipperMaxDocIdExact ? skipperMaxDocId : nextDoc(skipperMaxDocId, l -> l < minOrd);+ } else {+ minDocID =+ skipperMinDocIdExact ? skipperMinDocId : nextDoc(skipperMinDocId, l -> l >= minOrd);+ maxDocID = skipperMaxDocIdExact ? skipperMaxDocId : nextDoc(skipperMaxDocId, l -> l > maxOrd);+ }+ return minDocID == maxDocID+ ? DocIdSetIterator.empty()+ : DocIdSetIterator.range(minDocID, maxDocID);+ }++ @Override+ public long cost() {+ if (skipperMinDocId == -1) {+ try {+ // Similar to PointValues, IOExceptions needs to be caught and rethrown as+ // UncheckedIOException+ computeSkipperDocIds();+ } catch (IOException e) {+ throw new UncheckedIOException(e);+ }+ }+ if (skipperMinDocIdExact && skipperMaxDocIdExact) {+ return skipperMaxDocId - skipperMinDocId;+ }+ return skipperMaxDocId - skipperMinDocId + skipper.docCount(0);+ }++ private void computeSkipperDocIds() throws IOException {+ long minOrd = getLowerValue();+ long maxOrd = getUpperValue();+ if (minOrd > maxOrd || minOrd > skipper.maxValue() || maxOrd < skipper.minValue()) {+ skipperMinDocId = skipperMaxDocId = DocIdSetIterator.NO_MORE_DOCS;+ skipperMinDocIdExact = skipperMaxDocIdExact = true;+ return;+ }+ if (skipper.minValue() >= minOrd && skipper.maxValue() <= maxOrd) {+ skipperMinDocId = 0;+ skipperMaxDocId = skipper.docCount();+ skipperMinDocIdExact = skipperMaxDocIdExact = true;+ return;+ }+ if (sortField.getReverse()) {+ if (skipper.maxValue() <= maxOrd) {+ skipperMinDocId = 0;+ skipperMinDocIdExact = true;+ } else {+ skipper.advance(Long.MIN_VALUE, maxOrd);+ skipperMinDocId = skipper.minDocID(0);+ skipperMinDocIdExact = skipper.maxValue(0) == maxOrd;+ }+ if (skipper.minValue() >= minOrd) {+ skipperMaxDocId = skipper.docCount();+ skipperMaxDocIdExact = true;+ } else {+ skipper.advance(Long.MIN_VALUE, minOrd);+ if (skipper.minValue(0) == minOrd) {+ skipperMaxDocId = skipper.maxDocID(0) + 1;+ skipper.advance(skipperMaxDocId);+ skipperMaxDocIdExact = skipper.maxValue(0) != minOrd;+ } else {+ skipperMaxDocId = skipper.minDocID(0);+ skipperMaxDocIdExact = false;+ }+ }+ } else {+ if (skipper.minValue() >= minOrd) {+ skipperMinDocId = 0;+ skipperMinDocIdExact = true;+ } else {+ skipper.advance(minOrd, Long.MAX_VALUE);+ skipperMinDocId = skipper.minDocID(0);+ skipperMinDocIdExact = skipper.minValue(0) == minOrd;+ }+ if (skipper.maxValue() <= maxOrd) {+ skipperMaxDocId = skipper.docCount();+ skipperMaxDocIdExact = true;+ } else {+ skipper.advance(maxOrd, Long.MAX_VALUE);+ if (skipper.maxValue(0) == maxOrd) {+ skipperMaxDocId = skipper.maxDocID(0) + 1;+ skipper.advance(skipperMaxDocId);+ skipperMaxDocIdExact = skipper.minValue(0) != maxOrd;+ } else {+ skipperMaxDocId = skipper.minDocID(0);+ skipperMaxDocIdExact = false;+ }+ }+ }+ }+}diff --git a/lucene/core/src/test/org/apache/lucene/search/TestDocValuesQueries.java b/lucene/core/src/test/org/apache/lucene/search/TestDocValuesQueries.javaindex 1bcc196cd97..cb3e28e0816 100644--- a/lucene/core/src/test/org/apache/lucene/search/TestDocValuesQueries.java+++ b/lucene/core/src/test/org/apache/lucene/search/TestDocValuesQueries.java@@ -58,6 +58,11 @@ public class TestDocValuesQueries extends LuceneTestCase {return TestUtil.alwaysDocValuesFormat(new Lucene90DocValuesFormat(random().nextInt(4, 16)));}+ private Codec getCodec(int skipIntervalSize) {+ // small interval size to test with many intervals+ return TestUtil.alwaysDocValuesFormat(new Lucene90DocValuesFormat(skipIntervalSize));+ }+public void testDuelPointRangeSortedNumericRangeQuery() throws IOException {doTestDuelPointRangeNumericRangeQuery(true, 1, false);}@@ -757,15 +762,14 @@ public class TestDocValuesQueries extends LuceneTestCase {public void testPrimarySortDenseSortedDocValuesExactMatch() throws IOException {Directory dir = newDirectory();- IndexWriterConfig config = new IndexWriterConfig().setCodec(getCodec());+ int skipIntervalSize = random().nextInt(4, 4096);+ IndexWriterConfig config = new IndexWriterConfig().setCodec(getCodec(skipIntervalSize));config.setIndexSort(new Sort(new SortField("dv", SortField.Type.STRING, random().nextBoolean())));int numBlocks = random().nextInt(4, 16);int[] sizes = new int[numBlocks];- int totalDocs = 0;for (int i = 0; i < numBlocks; i++) {sizes[i] = random().nextInt(1, 250);- totalDocs += sizes[i];}RandomIndexWriter iw = new RandomIndexWriter(random(), dir, config);for (int i = 0; i < numBlocks; i++) {@@ -796,8 +800,7 @@ public class TestDocValuesQueries extends LuceneTestCase {Weight weight = rewritten.createWeight(searcher, ScoreMode.COMPLETE_NO_SCORES, 1.0f);ScorerSupplier supplier = weight.scorerSupplier(ctx);assertThat(supplier.cost(), greaterThanOrEqualTo((long) sizes[i]));- assertThat(- supplier.cost(), lessThanOrEqualTo(Math.min((long) sizes[i] + 2 * 4096, totalDocs)));+ assertThat(supplier.cost(), lessThanOrEqualTo(sizes[i] + 2L * skipIntervalSize));}reader.close();dir.close();