Apache Lucene open-source search software
Use SkipBlockRangeIterator in TermOrdValComparator (#15696)
Alan Woodward3f8207849cd7
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex 6a70470c88d..7d9ec546401 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -359,7 +359,7 @@ Optimizations* GITHUB#15498: ExitableDirectoryReader keeps singleton SortedSetDocValues or SortedNumericDocValues as singletons. (Houston Putman)-* GITHUB#15511: Dynamic pruning for SORTED(_SET) fields with doc values skipper (Pan Guixin)+* GITHUB#15511, GITHUB#15696: Dynamic pruning for SORTED(_SET) fields with doc values skipper (Pan Guixin, Alan Woodward)* GITHUB#15560: Avoid unnecessary getGraph() call. (Shubham Chaudhary)diff --git a/lucene/core/src/java/org/apache/lucene/search/comparators/TermOrdValComparator.java b/lucene/core/src/java/org/apache/lucene/search/comparators/TermOrdValComparator.javaindex f02e9d4b3ae..7b6b45bfc49 100644--- a/lucene/core/src/java/org/apache/lucene/search/comparators/TermOrdValComparator.java+++ b/lucene/core/src/java/org/apache/lucene/search/comparators/TermOrdValComparator.java@@ -30,13 +30,12 @@ import org.apache.lucene.index.Terms;import org.apache.lucene.index.TermsEnum;import org.apache.lucene.search.AbstractDocIdSetIterator;import org.apache.lucene.search.DocIdSetIterator;-import org.apache.lucene.search.DocValuesRangeIterator;import org.apache.lucene.search.FieldComparator;import org.apache.lucene.search.IndexSearcher;import org.apache.lucene.search.LeafFieldComparator;import org.apache.lucene.search.Pruning;import org.apache.lucene.search.Scorable;-import org.apache.lucene.search.TwoPhaseIterator;+import org.apache.lucene.search.SkipBlockRangeIterator;import org.apache.lucene.util.BytesRef;import org.apache.lucene.util.BytesRefBuilder;import org.apache.lucene.util.PriorityQueue;@@ -493,7 +492,7 @@ public class TermOrdValComparator extends FieldComparator<BytesRef> {private record PostingsEnumAndOrd(PostingsEnum postings, int ord) {}- private abstract class CompetitiveState {+ private abstract static class CompetitiveState {final UpdateableDocIdSetIterator iterator;CompetitiveState(LeafReaderContext context) {@@ -625,41 +624,18 @@ public class TermOrdValComparator extends FieldComparator<BytesRef> {}}- private class SkipperBasedCompetitiveState extends CompetitiveState {+ private static class SkipperBasedCompetitiveState extends CompetitiveState {private final DocValuesSkipper skipper;- private final TwoPhaseIterator innerTwoPhase;- private int minOrd;- private int maxOrd;- SkipperBasedCompetitiveState(LeafReaderContext context, DocValuesSkipper skipper)- throws IOException {+ SkipperBasedCompetitiveState(LeafReaderContext context, DocValuesSkipper skipper) {super(context);this.skipper = skipper;this.iterator.update(DocIdSetIterator.all(context.reader().maxDoc()));- final SortedDocValues docValues = getSortedDocValues(context, field);- this.innerTwoPhase =- new TwoPhaseIterator(docValues) {- @Override- public boolean matches() throws IOException {- final int cur = docValues.ordValue();- return cur >= minOrd && cur <= maxOrd;- }-- @Override- public float matchCost() {- return 2;- }- };}@Override- public void update(int minOrd, int maxOrd) throws IOException {- this.minOrd = minOrd;- this.maxOrd = maxOrd;-- final TwoPhaseIterator twoPhaseIterator =- new DocValuesRangeIterator(innerTwoPhase, skipper, minOrd, maxOrd, false);- iterator.update(TwoPhaseIterator.asDocIdSetIterator(twoPhaseIterator));+ public void update(int minOrd, int maxOrd) {+ iterator.update(new SkipBlockRangeIterator(skipper, minOrd, maxOrd));}}}diff --git a/lucene/core/src/test/org/apache/lucene/search/TestSortOptimization.java b/lucene/core/src/test/org/apache/lucene/search/TestSortOptimization.javaindex bdbb1586772..d42d51936b8 100644--- a/lucene/core/src/test/org/apache/lucene/search/TestSortOptimization.java+++ b/lucene/core/src/test/org/apache/lucene/search/TestSortOptimization.java@@ -1016,7 +1016,9 @@ public class TestSortOptimization extends LuceneTestCase {final int numDocs = atLeast(10000);for (int i = 0; i < numDocs; ++i) {final Document doc = new Document();- final BytesRef value = new BytesRef(Integer.toString(random().nextInt(1000)));+ // Random values that roughly correlate with index order, to make skipper implementation+ // useful+ final BytesRef value = new BytesRef(Integer.toString(random().nextInt(i, i + 20)));doc.add(fieldsBuilder.apply("my_field", value));writer.addDocument(doc);if (i == 7000) writer.flush(); // multiple segments@@ -1043,14 +1045,18 @@ public class TestSortOptimization extends LuceneTestCase {final Directory dir = newDirectory();final IndexWriter writer =new IndexWriter(dir, new IndexWriterConfig().setMergePolicy(newLogMergePolicy()));- final int numDocs = atLeast(10000);+ // Larger number of documents as some of them are missing values and we still want+ // to have multiple skipper blocks of 4096 entries.+ final int numDocs = atLeast(30000);// one segment with all values missing to start withwriter.addDocument(new Document());for (int i = 0; i < numDocs - 2; ++i) {if (i == 7000) writer.flush(); // multiple segmentsfinal Document doc = new Document();if (random().nextInt(2) == 0) {- final BytesRef value = new BytesRef(Integer.toString(random().nextInt(1000)));+ // Random values that roughly correlate with index order, to make skipper implementation+ // useful+ final BytesRef value = new BytesRef(Integer.toString(random().nextInt(i, i + 20)));doc.add(fieldsBuilder.apply("my_field", value));}writer.addDocument(doc);