Apache Lucene open-source search software
Rewrite full-coverage doc-values range queries to FieldExistsQuery (#16582)
Houston Putmana4d32c044e06
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex d0df2844676..00580cb83a1 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -427,6 +427,7 @@ ImprovementsOptimizations---------------------+* GITHUB#16593: DenseLiveDocs and SparseLiveDocs implement Bits#applyMask in bulk, restoring theword-at-a-time masking that live docs had before GITHUB#13084 exposed them as LiveDocs rather thanas a FixedBitSet, so AcceptDocs, the bulk scorers and CheckIndex no longer test one bit per@@ -504,6 +505,9 @@ Optimizations* GITHUB#16588: AcceptDocs builds its bit set from the raw filter iterator so thatDocIdSetIterator#intoBitSet is used on segments with deletions. (John Maurice)+* GITHUB#16582: Rewrite sorted-numeric doc-values range queries that cover every value of a field to a+ FieldExistsQuery, and collapse an IndexOrDocValuesQuery whose two sides rewrite equally. (Houston Putman)+Bug Fixes---------------------diff --git a/lucene/core/src/java/org/apache/lucene/document/SortedNumericDocValuesRangeQuery.java b/lucene/core/src/java/org/apache/lucene/document/SortedNumericDocValuesRangeQuery.javaindex 16ccd65c679..b4ac75288aa 100644--- a/lucene/core/src/java/org/apache/lucene/document/SortedNumericDocValuesRangeQuery.java+++ b/lucene/core/src/java/org/apache/lucene/document/SortedNumericDocValuesRangeQuery.java@@ -101,10 +101,13 @@ final class SortedNumericDocValuesRangeQuery extends NumericDocValuesRangeQueryif (lowerValue > stats.max() || upperValue < stats.min()) {return MatchNoDocsQuery.INSTANCE;}- if (lowerValue <= stats.min()- && upperValue >= stats.max()- && stats.docCount() == indexSearcher.getIndexReader().maxDoc()) {- return MatchAllDocsQuery.INSTANCE;+ if (lowerValue <= stats.min() && upperValue >= stats.max()) {+ // Every value in the field falls within [lowerValue, upperValue], so this matches exactly+ // the documents that have a value for the field.+ if (stats.docCount() == indexSearcher.getIndexReader().maxDoc()) {+ return MatchAllDocsQuery.INSTANCE;+ }+ return new FieldExistsQuery(field);}}return super.rewrite(indexSearcher);diff --git a/lucene/core/src/java/org/apache/lucene/search/IndexOrDocValuesQuery.java b/lucene/core/src/java/org/apache/lucene/search/IndexOrDocValuesQuery.javaindex e4e975bb567..3dc56376c1b 100644--- a/lucene/core/src/java/org/apache/lucene/search/IndexOrDocValuesQuery.java+++ b/lucene/core/src/java/org/apache/lucene/search/IndexOrDocValuesQuery.java@@ -121,6 +121,16 @@ public final class IndexOrDocValuesQuery extends Query {|| dvRewrite.getClass() == MatchNoDocsQuery.class) {return MatchNoDocsQuery.INSTANCE;}+ if (dvRewrite.getClass() == FieldExistsQuery.class) {+ return dvRewrite;+ } else if (indexRewrite.getClass() == FieldExistsQuery.class) {+ return indexRewrite;+ }+ // If both sides rewrite to the same query there is no longer a choice to make between an+ // index-based and a doc-values-based execution, so drop the wrapper.+ if (indexRewrite.equals(dvRewrite)) {+ return indexRewrite;+ }if (indexQuery != indexRewrite || dvQuery != dvRewrite) {return new IndexOrDocValuesQuery(indexRewrite, dvRewrite);}diff --git a/lucene/core/src/java/org/apache/lucene/search/IndexSortSortedNumericDocValuesRangeQuery.java b/lucene/core/src/java/org/apache/lucene/search/IndexSortSortedNumericDocValuesRangeQuery.javaindex 83e907dcd50..5de92f33c9f 100644--- a/lucene/core/src/java/org/apache/lucene/search/IndexSortSortedNumericDocValuesRangeQuery.java+++ b/lucene/core/src/java/org/apache/lucene/search/IndexSortSortedNumericDocValuesRangeQuery.java@@ -133,8 +133,13 @@ public class IndexSortSortedNumericDocValuesRangeQuery extends NumericDocValuesR}Query rewrittenFallback = fallbackQuery.rewrite(indexSearcher);- if (rewrittenFallback.getClass() == MatchAllDocsQuery.class) {- return MatchAllDocsQuery.INSTANCE;+ // This query matches the same documents as its fallback, so if the fallback simplifies to a+ // terminal query - match-all, match-none, or field-exists (e.g. a range covering every value of+ // a sparse field) - the index-sort optimization adds nothing and we return it directly.+ if (rewrittenFallback.getClass() == MatchAllDocsQuery.class+ || rewrittenFallback.getClass() == MatchNoDocsQuery.class+ || rewrittenFallback.getClass() == FieldExistsQuery.class) {+ return rewrittenFallback;}if (rewrittenFallback == fallbackQuery) {return this;diff --git a/lucene/core/src/java/org/apache/lucene/search/PointRangeQuery.java b/lucene/core/src/java/org/apache/lucene/search/PointRangeQuery.javaindex 84699ed81c8..4b349afe2b5 100644--- a/lucene/core/src/java/org/apache/lucene/search/PointRangeQuery.java+++ b/lucene/core/src/java/org/apache/lucene/search/PointRangeQuery.java@@ -566,17 +566,8 @@ public abstract class PointRangeQuery extends Query {};}- private boolean canRewriteToMatchAllQuery(IndexReader reader) throws IOException {- for (LeafReaderContext context : reader.leaves()) {- LeafReader leaf = context.reader();- PointValues values = leaf.getPointValues(field);-- if (values == null || values.getDocCount() != leaf.maxDoc()) {- return false;- }- }-- return true;+ private boolean canRewriteToMatchAllQuery(IndexReader reader) {+ return reader.maxDoc() == PointValues.getDocCount(reader, field);}private boolean canRewriteToFieldExistsQuery(IndexReader reader) {diff --git a/lucene/core/src/test/org/apache/lucene/search/TestDocValuesQueries.java b/lucene/core/src/test/org/apache/lucene/search/TestDocValuesQueries.javaindex cb70682fece..2a1e192198e 100644--- a/lucene/core/src/test/org/apache/lucene/search/TestDocValuesQueries.java+++ b/lucene/core/src/test/org/apache/lucene/search/TestDocValuesQueries.java@@ -980,12 +980,12 @@ public class TestDocValuesQueries extends LuceneTestCaseJupiter {searcher.rewrite(SortedNumericDocValuesField.newSlowRangeQuery("super_sparse", 250, 350)),instanceOf(MatchNoDocsQuery.class));- assertThat(- searcher- .rewrite(SortedNumericDocValuesField.newSlowRangeQuery("super_sparse", 174, 174))- .getClass()- .toString(),- containsString("SortedNumericDocValuesRangeQuery"));+ // A range that covers the field's only value but not every doc rewrites to a+ // FieldExistsQuery (super_sparse has a value on a single doc).+ assertEquals(+ new FieldExistsQuery("super_sparse"),+ searcher.rewrite(+ SortedNumericDocValuesField.newSlowRangeQuery("super_sparse", 174, 174)));assertThat(searcher.rewrite(SortedNumericDocValuesField.newSlowRangeQuery("with_index", 0, 150))@@ -1004,12 +1004,11 @@ public class TestDocValuesQueries extends LuceneTestCaseJupiter {.getClass().toString(),containsString("SortedNumericDocValuesRangeQuery"));- assertThat(- searcher- .rewrite(SortedNumericDocValuesField.newSlowRangeQuery("sparse", 0, 250))- .getClass()- .toString(),- containsString("SortedNumericDocValuesRangeQuery"));+ // A range that covers every value of a sparse field (a value on all but one doc) rewrites+ // to a FieldExistsQuery rather than staying a range query.+ assertEquals(+ new FieldExistsQuery("sparse"),+ searcher.rewrite(SortedNumericDocValuesField.newSlowRangeQuery("sparse", 0, 250)));}}}diff --git a/lucene/core/src/test/org/apache/lucene/search/TestIndexOrDocValuesQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestIndexOrDocValuesQuery.javaindex dc62b192ac3..176494251ba 100644--- a/lucene/core/src/test/org/apache/lucene/search/TestIndexOrDocValuesQuery.java+++ b/lucene/core/src/test/org/apache/lucene/search/TestIndexOrDocValuesQuery.java@@ -264,4 +264,34 @@ public class TestIndexOrDocValuesQuery extends LuceneTestCase {}}}++ public void testRewriteToFieldExistsWhenBothSidesCoverAllValues() throws Exception {+ var config = newIndexWriterConfig().setCodec(TestUtil.getDefaultCodec());+ try (Directory dir = newDirectory();+ IndexWriter w = new IndexWriter(dir, config)) {+ for (int i = 0; i < 10; ++i) {+ Document doc = new Document();+ // sparse field: a value on every doc but one, so it can't rewrite to MatchAllDocsQuery+ if (i != 5) {+ doc.add(new LongPoint("sparse", 100 + i));+ doc.add(SortedNumericDocValuesField.indexedField("sparse", 100 + i));+ }+ w.addDocument(doc);+ }+ w.forceMerge(1);++ try (IndexReader reader = DirectoryReader.open(w)) {+ IndexSearcher searcher = newSearcher(reader);+ // A range covering every value of the sparse field: both the point and the doc-values side+ // rewrite to a FieldExistsQuery, so the wrapper collapses to a single FieldExistsQuery+ // rather than re-wrapping two identical queries.+ IndexOrDocValuesQuery query =+ new IndexOrDocValuesQuery(+ LongPoint.newRangeQuery("sparse", 0, 250),+ SortedNumericDocValuesField.newSlowRangeQuery("sparse", 0, 250));+ QueryUtils.check(random(), query, searcher);+ assertEquals(new FieldExistsQuery("sparse"), query.rewrite(searcher));+ }+ }+ }}diff --git a/lucene/core/src/test/org/apache/lucene/search/TestIndexSortSortedNumericDocValuesRangeQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestIndexSortSortedNumericDocValuesRangeQuery.javaindex 69b667241c0..1885f7480a2 100644--- a/lucene/core/src/test/org/apache/lucene/search/TestIndexSortSortedNumericDocValuesRangeQuery.java+++ b/lucene/core/src/test/org/apache/lucene/search/TestIndexSortSortedNumericDocValuesRangeQuery.java@@ -31,6 +31,7 @@ import org.apache.lucene.index.DirectoryReader;import org.apache.lucene.index.IndexReader;import org.apache.lucene.index.IndexWriterConfig;import org.apache.lucene.index.LeafReaderContext;+import org.apache.lucene.index.Term;import org.apache.lucene.store.Directory;import org.apache.lucene.tests.analysis.MockAnalyzer;import org.apache.lucene.tests.index.RandomIndexWriter;@@ -412,24 +413,67 @@ public class TestIndexSortSortedNumericDocValuesRangeQuery extends LuceneTestCasdir.close();}- public void testRewriteFallbackQuery() throws IOException {+ public void testRewriteFallbackToMatchNone() throws IOException {Directory dir = newDirectory();RandomIndexWriter writer = new RandomIndexWriter(random(), dir);writer.addDocument(new Document());IndexReader reader = writer.getReader();- // Create an (unrealistic) fallback query that is sure to be rewritten.+ // An empty BooleanQuery rewrites to a MatchNoDocsQuery; the wrapper matches the same docs as+ // its+ // fallback, so it collapses to that MatchNoDocsQuery instead of re-wrapping it.Query fallbackQuery = new BooleanQuery.Builder().build();Query query = new IndexSortSortedNumericDocValuesRangeQuery("field", 1, 42, fallbackQuery);+ assertEquals(new MatchNoDocsQuery(), query.rewrite(newSearcher(reader)));++ writer.close();+ reader.close();+ dir.close();+ }++ public void testRewriteFallbackReWrapped() throws IOException {+ Directory dir = newDirectory();+ RandomIndexWriter writer = new RandomIndexWriter(random(), dir);+ writer.addDocument(new Document());+ IndexReader reader = writer.getReader();++ // A fallback that rewrites to a non-terminal query (BoostQuery with boost 1 unwraps to its+ // inner query) keeps the index-sort wrapper, now wrapping the rewritten fallback.+ TermQuery inner = new TermQuery(new Term("field", "x"));+ Query fallbackQuery = new BoostQuery(inner, 1.0f);+ Query query = new IndexSortSortedNumericDocValuesRangeQuery("field", 1, 42, fallbackQuery);Query rewrittenQuery = query.rewrite(newSearcher(reader));- assertNotEquals(query, rewrittenQuery);MatcherAssert.assertThat(rewrittenQuery, instanceOf(IndexSortSortedNumericDocValuesRangeQuery.class));+ assertEquals(+ inner, ((IndexSortSortedNumericDocValuesRangeQuery) rewrittenQuery).getFallbackQuery());++ writer.close();+ reader.close();+ dir.close();+ }++ public void testRewriteFallbackToFieldExists() throws IOException {+ Directory dir = newDirectory();+ RandomIndexWriter writer = new RandomIndexWriter(random(), dir);+ // Indexed (skip-indexed) sorted-numeric field on all docs but one, so the field is sparse and+ // its values span [100, 108].+ for (int i = 0; i < 10; i++) {+ Document doc = new Document();+ if (i != 5) {+ doc.add(SortedNumericDocValuesField.indexedField("field", 100 + i));+ }+ writer.addDocument(doc);+ }+ IndexReader reader = writer.getReader();- IndexSortSortedNumericDocValuesRangeQuery rangeQuery =- (IndexSortSortedNumericDocValuesRangeQuery) rewrittenQuery;- assertEquals(MatchNoDocsQuery.INSTANCE, rangeQuery.getFallbackQuery());+ // A range covering every value of the sparse field: the fallback rewrites to a+ // FieldExistsQuery,+ // which the index-sort query must return directly rather than re-wrapping.+ Query fallbackQuery = SortedNumericDocValuesField.newSlowRangeQuery("field", 0, 250);+ Query query = new IndexSortSortedNumericDocValuesRangeQuery("field", 0, 250, fallbackQuery);+ assertEquals(new FieldExistsQuery("field"), query.rewrite(newSearcher(reader)));writer.close();reader.close();