Apache Lucene open-source search software
LUCENE-10385: Implement Weight#count on IndexSortSortedNumericDocValuesRangeQuery (#635)
Luca Cavannabade484998e2
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex 727c0fa5e9d..d162714036f 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -128,6 +128,9 @@ New Featuresbased on TotalHitCountCollector that allows users to parallelize counting thenumber of hits. (Luca Cavanna, Adrien Grand)+* LUCENE-10385: Implement Weight#count on IndexSortSortedNumericDocValuesRangeQuery+ to speed up computing the number of hits when possible. (Luca Cavanna, Adrien Grand)+Improvements---------------------diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/search/IndexSortSortedNumericDocValuesRangeQuery.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/search/IndexSortSortedNumericDocValuesRangeQuery.javaindex 829ed712469..a35352564a9 100644--- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/search/IndexSortSortedNumericDocValuesRangeQuery.java+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/search/IndexSortSortedNumericDocValuesRangeQuery.java@@ -156,20 +156,9 @@ public class IndexSortSortedNumericDocValuesRangeQuery extends Query {return new ConstantScoreWeight(this, boost) {@Overridepublic Scorer scorer(LeafReaderContext context) throws IOException {- SortedNumericDocValues sortedNumericValues =- DocValues.getSortedNumeric(context.reader(), field);- NumericDocValues numericValues = DocValues.unwrapSingleton(sortedNumericValues);-- if (numericValues != null) {- Sort indexSort = context.reader().getMetaData().getSort();- if (indexSort != null- && indexSort.getSort().length > 0- && indexSort.getSort()[0].getField().equals(field)) {-- SortField sortField = indexSort.getSort()[0];- DocIdSetIterator disi = getDocIdSetIterator(sortField, context, numericValues);- return new ConstantScoreScorer(this, score(), scoreMode, disi);- }+ DocIdSetIterator disi = getDocIdSetIteratorOrNull(context);+ if (disi != null) {+ return new ConstantScoreScorer(this, score(), scoreMode, disi);}return fallbackWeight.scorer(context);}@@ -180,9 +169,36 @@ public class IndexSortSortedNumericDocValuesRangeQuery extends Query {// if the fallback query is cacheable.return fallbackWeight.isCacheable(ctx);}++ @Override+ public int count(LeafReaderContext context) throws IOException {+ BoundedDocSetIdIterator disi = getDocIdSetIteratorOrNull(context);+ if (disi != null) {+ return disi.lastDoc - disi.firstDoc;+ }+ return fallbackWeight.count(context);+ }};}+ private BoundedDocSetIdIterator getDocIdSetIteratorOrNull(LeafReaderContext context)+ throws IOException {+ SortedNumericDocValues sortedNumericValues =+ DocValues.getSortedNumeric(context.reader(), field);+ NumericDocValues numericValues = DocValues.unwrapSingleton(sortedNumericValues);+ if (numericValues != null) {+ Sort indexSort = context.reader().getMetaData().getSort();+ if (indexSort != null+ && indexSort.getSort().length > 0+ && indexSort.getSort()[0].getField().equals(field)) {++ SortField sortField = indexSort.getSort()[0];+ return getDocIdSetIterator(sortField, context, numericValues);+ }+ }+ return null;+ }+/*** Computes the document IDs that lie within the range [lowerValue, upperValue] by performing* binary search on the field's doc values.@@ -195,7 +211,7 @@ public class IndexSortSortedNumericDocValuesRangeQuery extends Query {* {@link DocIdSetIterator} makes sure to wrap the original docvalues to skip over documents with* no value.*/- private DocIdSetIterator getDocIdSetIterator(+ private BoundedDocSetIdIterator getDocIdSetIterator(SortField sortField, LeafReaderContext context, DocIdSetIterator delegate)throws IOException {long lower = sortField.getReverse() ? upperValue : lowerValue;diff --git a/lucene/sandbox/src/test/org/apache/lucene/sandbox/search/TestIndexSortSortedNumericDocValuesRangeQuery.java b/lucene/sandbox/src/test/org/apache/lucene/sandbox/search/TestIndexSortSortedNumericDocValuesRangeQuery.javaindex fb8ecf26007..f876dddd0bc 100644--- a/lucene/sandbox/src/test/org/apache/lucene/sandbox/search/TestIndexSortSortedNumericDocValuesRangeQuery.java+++ b/lucene/sandbox/src/test/org/apache/lucene/sandbox/search/TestIndexSortSortedNumericDocValuesRangeQuery.java@@ -38,6 +38,7 @@ import org.apache.lucene.search.Sort;import org.apache.lucene.search.SortField;import org.apache.lucene.search.SortedNumericSortField;import org.apache.lucene.search.TopDocs;+import org.apache.lucene.search.TotalHitCountCollectorManager;import org.apache.lucene.search.Weight;import org.apache.lucene.store.Directory;import org.apache.lucene.tests.analysis.MockAnalyzer;@@ -95,7 +96,7 @@ public class TestIndexSortSortedNumericDocValuesRangeQuery extends LuceneTestCas}}- private void assertSameHits(IndexSearcher searcher, Query q1, Query q2, boolean scores)+ private static void assertSameHits(IndexSearcher searcher, Query q1, Query q2, boolean scores)throws IOException {final int maxDoc = searcher.getIndexReader().maxDoc();final TopDocs td1 = searcher.search(q1, maxDoc, scores ? Sort.RELEVANCE : Sort.INDEXORDER);@@ -167,43 +168,50 @@ public class TestIndexSortSortedNumericDocValuesRangeQuery extends LuceneTestCasIndexSearcher searcher = newSearcher(reader);// Test ranges consisting of one value.- assertEquals(1, searcher.count(createQuery("field", -80, -80)));- assertEquals(1, searcher.count(createQuery("field", -5, -5)));- assertEquals(2, searcher.count(createQuery("field", 0, 0)));- assertEquals(1, searcher.count(createQuery("field", 30, 30)));- assertEquals(1, searcher.count(createQuery("field", 35, 35)));+ assertNumberOfHits(searcher, createQuery("field", -80, -80), 1);+ assertNumberOfHits(searcher, createQuery("field", -5, -5), 1);+ assertNumberOfHits(searcher, createQuery("field", 0, 0), 2);+ assertNumberOfHits(searcher, createQuery("field", 30, 30), 1);+ assertNumberOfHits(searcher, createQuery("field", 35, 35), 1);- assertEquals(0, searcher.count(createQuery("field", -90, -90)));- assertEquals(0, searcher.count(createQuery("field", 5, 5)));- assertEquals(0, searcher.count(createQuery("field", 40, 40)));+ assertNumberOfHits(searcher, createQuery("field", -90, -90), 0);+ assertNumberOfHits(searcher, createQuery("field", 5, 5), 0);+ assertNumberOfHits(searcher, createQuery("field", 40, 40), 0);// Test the lower end of the document value range.- assertEquals(2, searcher.count(createQuery("field", -90, -4)));- assertEquals(2, searcher.count(createQuery("field", -80, -4)));- assertEquals(1, searcher.count(createQuery("field", -70, -4)));- assertEquals(2, searcher.count(createQuery("field", -80, -5)));+ assertNumberOfHits(searcher, createQuery("field", -90, -4), 2);+ assertNumberOfHits(searcher, createQuery("field", -80, -4), 2);+ assertNumberOfHits(searcher, createQuery("field", -70, -4), 1);+ assertNumberOfHits(searcher, createQuery("field", -80, -5), 2);// Test the upper end of the document value range.- assertEquals(1, searcher.count(createQuery("field", 25, 34)));- assertEquals(2, searcher.count(createQuery("field", 25, 35)));- assertEquals(2, searcher.count(createQuery("field", 25, 36)));- assertEquals(2, searcher.count(createQuery("field", 30, 35)));+ assertNumberOfHits(searcher, createQuery("field", 25, 34), 1);+ assertNumberOfHits(searcher, createQuery("field", 25, 35), 2);+ assertNumberOfHits(searcher, createQuery("field", 25, 36), 2);+ assertNumberOfHits(searcher, createQuery("field", 30, 35), 2);// Test multiple occurrences of the same value.- assertEquals(2, searcher.count(createQuery("field", -4, 4)));- assertEquals(2, searcher.count(createQuery("field", -4, 0)));- assertEquals(2, searcher.count(createQuery("field", 0, 4)));- assertEquals(3, searcher.count(createQuery("field", 0, 30)));+ assertNumberOfHits(searcher, createQuery("field", -4, 4), 2);+ assertNumberOfHits(searcher, createQuery("field", -4, 0), 2);+ assertNumberOfHits(searcher, createQuery("field", 0, 4), 2);+ assertNumberOfHits(searcher, createQuery("field", 0, 30), 3);// Test ranges that span all documents.- assertEquals(6, searcher.count(createQuery("field", -80, 35)));- assertEquals(6, searcher.count(createQuery("field", -90, 40)));+ assertNumberOfHits(searcher, createQuery("field", -80, 35), 6);+ assertNumberOfHits(searcher, createQuery("field", -90, 40), 6);writer.close();reader.close();dir.close();}+ private static void assertNumberOfHits(IndexSearcher searcher, Query query, int numberOfHits)+ throws IOException {+ assertEquals(+ numberOfHits, searcher.search(query, new TotalHitCountCollectorManager()).intValue());+ assertEquals(numberOfHits, searcher.count(query));+ }+public void testIndexSortDocValuesWithOddLength() throws Exception {testIndexSortDocValuesWithOddLength(false);testIndexSortDocValuesWithOddLength(true);@@ -229,38 +237,38 @@ public class TestIndexSortSortedNumericDocValuesRangeQuery extends LuceneTestCasIndexSearcher searcher = newSearcher(reader);// Test ranges consisting of one value.- assertEquals(1, searcher.count(createQuery("field", -80, -80)));- assertEquals(1, searcher.count(createQuery("field", -5, -5)));- assertEquals(2, searcher.count(createQuery("field", 0, 0)));- assertEquals(1, searcher.count(createQuery("field", 5, 5)));- assertEquals(1, searcher.count(createQuery("field", 30, 30)));- assertEquals(1, searcher.count(createQuery("field", 35, 35)));+ assertNumberOfHits(searcher, createQuery("field", -80, -80), 1);+ assertNumberOfHits(searcher, createQuery("field", -5, -5), 1);+ assertNumberOfHits(searcher, createQuery("field", 0, 0), 2);+ assertNumberOfHits(searcher, createQuery("field", 5, 5), 1);+ assertNumberOfHits(searcher, createQuery("field", 30, 30), 1);+ assertNumberOfHits(searcher, createQuery("field", 35, 35), 1);- assertEquals(0, searcher.count(createQuery("field", -90, -90)));- assertEquals(0, searcher.count(createQuery("field", 6, 6)));- assertEquals(0, searcher.count(createQuery("field", 40, 40)));+ assertNumberOfHits(searcher, createQuery("field", -90, -90), 0);+ assertNumberOfHits(searcher, createQuery("field", 6, 6), 0);+ assertNumberOfHits(searcher, createQuery("field", 40, 40), 0);// Test the lower end of the document value range.- assertEquals(2, searcher.count(createQuery("field", -90, -4)));- assertEquals(2, searcher.count(createQuery("field", -80, -4)));- assertEquals(1, searcher.count(createQuery("field", -70, -4)));- assertEquals(2, searcher.count(createQuery("field", -80, -5)));+ assertNumberOfHits(searcher, createQuery("field", -90, -4), 2);+ assertNumberOfHits(searcher, createQuery("field", -80, -4), 2);+ assertNumberOfHits(searcher, createQuery("field", -70, -4), 1);+ assertNumberOfHits(searcher, createQuery("field", -80, -5), 2);// Test the upper end of the document value range.- assertEquals(1, searcher.count(createQuery("field", 25, 34)));- assertEquals(2, searcher.count(createQuery("field", 25, 35)));- assertEquals(2, searcher.count(createQuery("field", 25, 36)));- assertEquals(2, searcher.count(createQuery("field", 30, 35)));+ assertNumberOfHits(searcher, createQuery("field", 25, 34), 1);+ assertNumberOfHits(searcher, createQuery("field", 25, 35), 2);+ assertNumberOfHits(searcher, createQuery("field", 25, 36), 2);+ assertNumberOfHits(searcher, createQuery("field", 30, 35), 2);// Test multiple occurrences of the same value.- assertEquals(2, searcher.count(createQuery("field", -4, 4)));- assertEquals(2, searcher.count(createQuery("field", -4, 0)));- assertEquals(2, searcher.count(createQuery("field", 0, 4)));- assertEquals(4, searcher.count(createQuery("field", 0, 30)));+ assertNumberOfHits(searcher, createQuery("field", -4, 4), 2);+ assertNumberOfHits(searcher, createQuery("field", -4, 0), 2);+ assertNumberOfHits(searcher, createQuery("field", 0, 4), 2);+ assertNumberOfHits(searcher, createQuery("field", 0, 30), 4);// Test ranges that span all documents.- assertEquals(7, searcher.count(createQuery("field", -80, 35)));- assertEquals(7, searcher.count(createQuery("field", -90, 40)));+ assertNumberOfHits(searcher, createQuery("field", -80, 35), 7);+ assertNumberOfHits(searcher, createQuery("field", -90, 40), 7);writer.close();reader.close();@@ -285,10 +293,10 @@ public class TestIndexSortSortedNumericDocValuesRangeQuery extends LuceneTestCasDirectoryReader reader = writer.getReader();IndexSearcher searcher = newSearcher(reader);- assertEquals(1, searcher.count(createQuery("field", 42, 43)));- assertEquals(1, searcher.count(createQuery("field", 42, 42)));- assertEquals(0, searcher.count(createQuery("field", 41, 41)));- assertEquals(0, searcher.count(createQuery("field", 43, 43)));+ assertNumberOfHits(searcher, createQuery("field", 42, 43), 1);+ assertNumberOfHits(searcher, createQuery("field", 42, 42), 1);+ assertNumberOfHits(searcher, createQuery("field", 41, 41), 0);+ assertNumberOfHits(searcher, createQuery("field", 43, 43), 0);writer.close();reader.close();@@ -316,11 +324,11 @@ public class TestIndexSortSortedNumericDocValuesRangeQuery extends LuceneTestCasDirectoryReader reader = writer.getReader();IndexSearcher searcher = newSearcher(reader);- assertEquals(2, searcher.count(createQuery("field", -70, 0)));- assertEquals(2, searcher.count(createQuery("field", -2, 35)));+ assertNumberOfHits(searcher, createQuery("field", -70, 0), 2);+ assertNumberOfHits(searcher, createQuery("field", -2, 35), 2);- assertEquals(4, searcher.count(createQuery("field", -80, 35)));- assertEquals(4, searcher.count(createQuery("field", Long.MIN_VALUE, Long.MAX_VALUE)));+ assertNumberOfHits(searcher, createQuery("field", -80, 35), 4);+ assertNumberOfHits(searcher, createQuery("field", Long.MIN_VALUE, Long.MAX_VALUE), 4);writer.close();reader.close();@@ -450,6 +458,56 @@ public class TestIndexSortSortedNumericDocValuesRangeQuery extends LuceneTestCasreader.close();}+ public void testCount() throws IOException {+ Directory dir = newDirectory();+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));+ Sort indexSort = new Sort(new SortedNumericSortField("field", SortField.Type.LONG));+ iwc.setIndexSort(indexSort);+ RandomIndexWriter writer = new RandomIndexWriter(random(), dir, iwc);+ Document doc = new Document();+ doc.add(new SortedNumericDocValuesField("field", 10));+ writer.addDocument(doc);+ IndexReader reader = writer.getReader();+ IndexSearcher searcher = newSearcher(reader);++ Query fallbackQuery = LongPoint.newRangeQuery("field", 1, 42);+ Query query = new IndexSortSortedNumericDocValuesRangeQuery("field", 1, 42, fallbackQuery);+ Weight weight = query.createWeight(searcher, ScoreMode.COMPLETE, 1.0f);+ for (LeafReaderContext context : searcher.getLeafContexts()) {+ assertEquals(1, weight.count(context));+ }++ writer.close();+ reader.close();+ dir.close();+ }++ public void testFallbackCount() throws IOException {+ Directory dir = newDirectory();+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));+ Sort indexSort = new Sort(new SortedNumericSortField("field", SortField.Type.LONG));+ iwc.setIndexSort(indexSort);+ RandomIndexWriter writer = new RandomIndexWriter(random(), dir, iwc);+ Document doc = new Document();+ doc.add(new SortedNumericDocValuesField("field", 10));+ writer.addDocument(doc);+ IndexReader reader = writer.getReader();+ IndexSearcher searcher = newSearcher(reader);++ // we use an unrealistic query that exposes its own Weight#count+ Query fallbackQuery = new MatchNoDocsQuery();+ // the index is not sorted on this field, the fallback query is used+ Query query = new IndexSortSortedNumericDocValuesRangeQuery("another", 1, 42, fallbackQuery);+ Weight weight = query.createWeight(searcher, ScoreMode.COMPLETE, 1.0f);+ for (LeafReaderContext context : searcher.getLeafContexts()) {+ assertEquals(0, weight.count(context));+ }++ writer.close();+ reader.close();+ dir.close();+ }+private Document createDocument(String field, long value) {Document doc = new Document();doc.add(new SortedNumericDocValuesField(field, value));