Apache Lucene open-source search software
Allow NumericComparator subclasses to implement their own competitive iterators (#15663)
Alan Woodwardc3bcae4c0261
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex d33b42ab774..3975f2b70b6 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -174,7 +174,8 @@ OtherAPI Changes----------------------(No changes)+* GITHUB#15663: Allow subclasses of NumericComparator to implement their own+ CompetitiveDISIBuilder subtypes. (Alan Woodward)New Features---------------------diff --git a/lucene/core.tests/src/test/org/apache/lucene/core/tests/TestComparatorSubclassing.java b/lucene/core.tests/src/test/org/apache/lucene/core/tests/TestComparatorSubclassing.javanew file mode 100644index 00000000000..63e1ad5c3e9--- /dev/null+++ b/lucene/core.tests/src/test/org/apache/lucene/core/tests/TestComparatorSubclassing.java@@ -0,0 +1,102 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.core.tests;++import java.io.IOException;+import org.apache.lucene.document.Document;+import org.apache.lucene.document.NumericDocValuesField;+import org.apache.lucene.index.LeafReaderContext;+import org.apache.lucene.search.DocIdSetIterator;+import org.apache.lucene.search.FieldComparator;+import org.apache.lucene.search.FieldComparatorSource;+import org.apache.lucene.search.IndexSearcher;+import org.apache.lucene.search.LeafFieldComparator;+import org.apache.lucene.search.MatchAllDocsQuery;+import org.apache.lucene.search.Pruning;+import org.apache.lucene.search.Sort;+import org.apache.lucene.search.SortField;+import org.apache.lucene.search.TopDocs;+import org.apache.lucene.search.TotalHits;+import org.apache.lucene.search.comparators.LongComparator;+import org.apache.lucene.search.comparators.NumericComparator;+import org.apache.lucene.tests.index.RandomIndexWriter;+import org.apache.lucene.tests.util.LuceneTestCase;++public class TestComparatorSubclassing extends LuceneTestCase {++ public void testPluggableCompetitiveIterators() throws IOException {++ boolean[] updated = new boolean[] {false};++ LongComparator extendedComparator =+ new LongComparator(10, "field", 0L, true, Pruning.NONE) {+ @Override+ public LeafFieldComparator getLeafComparator(LeafReaderContext context)+ throws IOException {+ return new LongLeafComparator(context) {+ @Override+ protected NumericComparator<Long>.CompetitiveDISIBuilder buildCompetitiveDISIBuilder()+ throws IOException {+ int maxDoc = context.reader().maxDoc();+ int jumpTo = Math.max(0, maxDoc - 10);+ return new CompetitiveDISIBuilder(this) {+ @Override+ protected boolean hasMissingDocs() {+ return false;+ }++ @Override+ protected void doUpdateCompetitiveIterator() {+ updateCompetitiveIterator(DocIdSetIterator.range(jumpTo, maxDoc));+ updated[0] = true;+ }+ };+ }+ };+ }+ };++ try (var dir = newDirectory();+ var writer = new RandomIndexWriter(random(), dir)) {+ for (int i = 0; i < 2000; i++) {+ Document doc = new Document();+ doc.add(NumericDocValuesField.indexedField("field", i));+ writer.addDocument(doc);+ }+ writer.commit();++ try (var reader = writer.getReader()) {+ IndexSearcher searcher = new IndexSearcher(reader);+ Sort sort =+ new Sort(+ new SortField(+ "field",+ new FieldComparatorSource() {+ @Override+ public FieldComparator<?> newComparator(+ String fieldname, int numHits, Pruning pruning, boolean reversed) {+ return extendedComparator;+ }+ }));++ TopDocs td = searcher.search(MatchAllDocsQuery.INSTANCE, 10, sort);+ assertEquals(TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO, td.totalHits.relation());+ assertTrue(updated[0]);+ }+ }+ }+}diff --git a/lucene/core/src/java/org/apache/lucene/search/comparators/NumericComparator.java b/lucene/core/src/java/org/apache/lucene/search/comparators/NumericComparator.javaindex 1aa704b36c5..5b62de7345b 100644--- a/lucene/core/src/java/org/apache/lucene/search/comparators/NumericComparator.java+++ b/lucene/core/src/java/org/apache/lucene/search/comparators/NumericComparator.java@@ -103,20 +103,23 @@ public abstract class NumericComparator<T extends Number> extends FieldComparatopublic NumericLeafComparator(LeafReaderContext context) throws IOException {this.context = context;this.docValues = getNumericDocValues(context, field);- CompetitiveDISIBuilder builder = null;- if (pruning != Pruning.NONE) {- LeafReader reader = context.reader();- PointValues pointValues = reader.getPointValues(field);- if (pointValues != null) {- builder = new PointsCompetitiveDISIBuilder(pointValues, this);- } else {- DocValuesSkipper skipper = reader.getDocValuesSkipper(field);- if (skipper != null) {- builder = new DVSkipperCompetitiveDISIBuilder(skipper, this);- }- }+ this.competitiveDISIBuilder = buildCompetitiveDISIBuilder();+ }++ protected CompetitiveDISIBuilder buildCompetitiveDISIBuilder() throws IOException {+ if (pruning == Pruning.NONE) {+ return null;}- competitiveDISIBuilder = builder;+ LeafReader reader = context.reader();+ PointValues pointValues = reader.getPointValues(field);+ if (pointValues != null) {+ return new PointsCompetitiveDISIBuilder(pointValues, this);+ }+ DocValuesSkipper skipper = reader.getDocValuesSkipper(field);+ if (skipper != null) {+ return new DVSkipperCompetitiveDISIBuilder(skipper, this);+ }+ return null;}/**@@ -176,22 +179,47 @@ public abstract class NumericComparator<T extends Number> extends FieldComparatoprotected abstract long topAsComparableLong();}- private abstract class CompetitiveDISIBuilder {+ /** Constructs a competitive iterator which can be updated during collection */+ protected abstract class CompetitiveDISIBuilder {++ /** maxDoc of the current segment */+ protected final int maxDoc;- final int maxDoc;- final NumericLeafComparator leafComparator;+ /** The LeafComparator for the current segment */+ protected final NumericLeafComparator leafComparator;/** According to {@link FieldComparator#setTopValue}, topValueSet is final in leafComparator */final boolean leafTopSet = topValueSet;- final UpdateableDocIdSetIterator competitiveIterator = new UpdateableDocIdSetIterator();- long minValueAsLong = Long.MIN_VALUE;- long maxValueAsLong = Long.MAX_VALUE;+ private final UpdateableDocIdSetIterator competitiveIterator = new UpdateableDocIdSetIterator();++ /** The current minimum value encoded as a long */+ protected long minValueAsLong = Long.MIN_VALUE;++ /** The current maximum value encoded as a long */+ protected long maxValueAsLong = Long.MAX_VALUE;+int maxDocVisited = -1;int updateCounter = 0;int currentSkipInterval = MIN_SKIP_INTERVAL;- CompetitiveDISIBuilder(NumericLeafComparator leafComparator) {+ /** Are there documents in this segment with no value for the comparator to use */+ protected abstract boolean hasMissingDocs();++ /**+ * Build a new DocIdSetIterator to use as a competitive iterator, if the new iterator will+ * filter out enough documents. Implementations should pass their newly built iterators to+ * {@link #updateCompetitiveIterator(DocIdSetIterator)}+ */+ protected abstract void doUpdateCompetitiveIterator() throws IOException;++ /** Use a new iterator as the competitive iterator for collection */+ protected final void updateCompetitiveIterator(DocIdSetIterator iterator) {+ this.competitiveIterator.update(iterator);+ }++ /** Create a new CompetitiveDISIBuilder */+ protected CompetitiveDISIBuilder(NumericLeafComparator leafComparator) {this.leafComparator = leafComparator;this.maxDoc = leafComparator.context.reader().maxDoc();this.competitiveIterator.update(DocIdSetIterator.all(maxDoc));@@ -202,8 +230,6 @@ public abstract class NumericComparator<T extends Number> extends FieldComparatovoid setScorer(Scorable scorer) throws IOException {}- abstract int docCount();-final void updateCompetitiveIterator() throws IOException {if (hitsThresholdReached == false) {return;@@ -213,7 +239,7 @@ public abstract class NumericComparator<T extends Number> extends FieldComparato}// if some documents have missing points, check that missing values prohibits optimization- if (docCount() < maxDoc && isMissingValueCompetitive()) {+ if (hasMissingDocs() && isMissingValueCompetitive()) {return;}@@ -231,8 +257,6 @@ public abstract class NumericComparator<T extends Number> extends FieldComparatodoUpdateCompetitiveIterator();}- abstract void doUpdateCompetitiveIterator() throws IOException;-private void setMaxDocVisited(int maxDocVisited) {this.maxDocVisited = maxDocVisited;}@@ -361,8 +385,8 @@ public abstract class NumericComparator<T extends Number> extends FieldComparato}@Override- int docCount() {- return pointValues.getDocCount();+ protected boolean hasMissingDocs() {+ return pointValues.getDocCount() != maxDoc;}/**@@ -375,22 +399,22 @@ public abstract class NumericComparator<T extends Number> extends FieldComparatoif (queueFull && hitsThresholdReached) {// if some documents have missing points, then check that missing values prohibits// optimization- if (docCount() < maxDoc && isMissingValueCompetitive()) {+ if (hasMissingDocs() && isMissingValueCompetitive()) {return;}long bottom = leafComparator.bottomAsComparableLong();long minValue = sortableBytesToLong(pointValues.getMinPackedValue());long maxValue = sortableBytesToLong(pointValues.getMaxPackedValue());if (reverse == false && bottom < minValue) {- competitiveIterator.update(DocIdSetIterator.empty());+ updateCompetitiveIterator(DocIdSetIterator.empty());} else if (reverse && bottom > maxValue) {- competitiveIterator.update(DocIdSetIterator.empty());+ updateCompetitiveIterator(DocIdSetIterator.empty());}}}@Override- void doUpdateCompetitiveIterator() throws IOException {+ protected void doUpdateCompetitiveIterator() throws IOException {DocIdSetBuilder result = new DocIdSetBuilder(maxDoc);PointValues.IntersectVisitor visitor =new PointValues.IntersectVisitor() {@@ -462,15 +486,16 @@ public abstract class NumericComparator<T extends Number> extends FieldComparatoupdateSkipInterval(false);if (pointValues.getDocCount() < iteratorCost) {// Use the set of doc with values to help drive iteration- competitiveIterator.update(+ updateCompetitiveIterator(leafComparator.getNumericDocValues(leafComparator.context, field));iteratorCost = pointValues.getDocCount();}return;}pointValues.intersect(visitor);- competitiveIterator.update(result.build().iterator());- iteratorCost = competitiveIterator.cost();+ DocIdSetIterator newIterator = result.build().iterator();+ updateCompetitiveIterator(newIterator);+ iteratorCost = newIterator.cost();updateSkipInterval(true);}@@ -510,8 +535,8 @@ public abstract class NumericComparator<T extends Number> extends FieldComparato}@Override- int docCount() {- return skipper.docCount();+ protected boolean hasMissingDocs() {+ return skipper.docCount() != maxDoc;}/**@@ -522,21 +547,21 @@ public abstract class NumericComparator<T extends Number> extends FieldComparatoif (queueFull && hitsThresholdReached) {// if some documents have missing doc values, check that missing values prohibits// optimization- if (docCount() < maxDoc && isMissingValueCompetitive()) {+ if (hasMissingDocs() && isMissingValueCompetitive()) {return;}long bottom = leafComparator.bottomAsComparableLong();if (reverse == false && bottom < skipper.minValue()) {- competitiveIterator.update(DocIdSetIterator.empty());+ updateCompetitiveIterator(DocIdSetIterator.empty());} else if (reverse && bottom > skipper.maxValue()) {- competitiveIterator.update(DocIdSetIterator.empty());+ updateCompetitiveIterator(DocIdSetIterator.empty());}}}@Override- void doUpdateCompetitiveIterator() {- competitiveIterator.update(+ protected void doUpdateCompetitiveIterator() {+ updateCompetitiveIterator(new SkipBlockRangeIterator(skipper, minValueAsLong, maxValueAsLong));}}