Skip to content

Apache Lucene open-source search software

Overrides rewrite in PointRangeQuery to optimize AllDocs/NoDocs cases (#14609)

Elliott Bradshaw416dd2fe7c37
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 843253bf167..e3ef26dd271 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -41,6 +41,8 @@ Optimizations
---------------------
* GITHUB#14011: Reduce allocation rate in HNSW concurrent merge. (Viliam Durina)
* GITHUB#14022: Optimize DFS marking of connected components in HNSW by reducing stack depth, improving performance and reducing allocations. (Viswanath Kuchibhotla)
+* GITHUB#14609: Optimizes PointRangeQuery by rewriting to MatchAllDocsQuery/FieldExistsQuery/MatchNoDocsQuery if all
+ docs in index are contained or excluded (Elliott Bradshaw)
Bug Fixes
---------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/PointRangeQuery.java b/lucene/core/src/java/org/apache/lucene/search/PointRangeQuery.java
index fddafdcaef7..fae9b363145 100644
--- a/lucene/core/src/java/org/apache/lucene/search/PointRangeQuery.java
+++ b/lucene/core/src/java/org/apache/lucene/search/PointRangeQuery.java
@@ -22,6 +22,9 @@ import java.util.Objects;
import java.util.function.BiFunction;
import java.util.function.Predicate;
import org.apache.lucene.document.IntPoint;
+import org.apache.lucene.index.DocValuesType;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.PointValues;
@@ -54,6 +57,7 @@ public abstract class PointRangeQuery extends Query {
final int bytesPerDim;
final byte[] lowerPoint;
final byte[] upperPoint;
+ final ByteArrayComparator comparator;
/**
* Expert: create a multidimensional range query for point values.
@@ -89,6 +93,8 @@ public abstract class PointRangeQuery extends Query {
this.lowerPoint = lowerPoint;
this.upperPoint = upperPoint;
+
+ this.comparator = ArrayUtil.getUnsignedComparator(bytesPerDim);
}
/**
@@ -125,8 +131,6 @@ public abstract class PointRangeQuery extends Query {
return new ConstantScoreWeight(this, boost) {
- private final ByteArrayComparator comparator = ArrayUtil.getUnsignedComparator(bytesPerDim);
-
private boolean matches(byte[] packedValue) {
int offset = 0;
for (int dim = 0; dim < numDims; dim++, offset += bytesPerDim) {
@@ -142,30 +146,6 @@ public abstract class PointRangeQuery extends Query {
return true;
}
- private Relation relate(byte[] minPackedValue, byte[] maxPackedValue) {
-
- boolean crosses = false;
- int offset = 0;
-
- for (int dim = 0; dim < numDims; dim++, offset += bytesPerDim) {
-
- if (comparator.compare(minPackedValue, offset, upperPoint, offset) > 0
- || comparator.compare(maxPackedValue, offset, lowerPoint, offset) < 0) {
- return Relation.CELL_OUTSIDE_QUERY;
- }
-
- crosses |=
- comparator.compare(minPackedValue, offset, lowerPoint, offset) < 0
- || comparator.compare(maxPackedValue, offset, upperPoint, offset) > 0;
- }
-
- if (crosses) {
- return Relation.CELL_CROSSES_QUERY;
- } else {
- return Relation.CELL_INSIDE_QUERY;
- }
- }
-
private IntersectVisitor getIntersectVisitor(DocIdSetBuilder result) {
return new IntersectVisitor() {
@@ -268,33 +248,6 @@ public abstract class PointRangeQuery extends Query {
};
}
- private boolean checkValidPointValues(PointValues values) throws IOException {
- if (values == null) {
- // No docs in this segment/field indexed any points
- return false;
- }
-
- if (values.getNumIndexDimensions() != numDims) {
- throw new IllegalArgumentException(
- "field=\""
- + field
- + "\" was indexed with numIndexDimensions="
- + values.getNumIndexDimensions()
- + " but this query has numDims="
- + numDims);
- }
- if (bytesPerDim != values.getBytesPerDimension()) {
- throw new IllegalArgumentException(
- "field=\""
- + field
- + "\" was indexed with bytesPerDim="
- + values.getBytesPerDimension()
- + " but this query has bytesPerDim="
- + bytesPerDim);
- }
- return true;
- }
-
@Override
public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOException {
LeafReader reader = context.reader();
@@ -402,7 +355,8 @@ public abstract class PointRangeQuery extends Query {
// docCount == size : counting according number of points in leaf node, so must be
// single-valued.
if (numDims == 1 && values.getDocCount() == values.size()) {
- return (int) pointCount(values.getPointTree(), this::relate, this::matches);
+ return (int)
+ pointCount(values.getPointTree(), PointRangeQuery.this::relate, this::matches);
}
}
return super.count(context);
@@ -580,4 +534,114 @@ public abstract class PointRangeQuery extends Query {
* @return human readable value for debugging
*/
protected abstract String toString(int dimension, byte[] value);
+
+ @Override
+ public Query rewrite(IndexSearcher searcher) throws IOException {
+ IndexReader reader = searcher.getIndexReader();
+
+ for (LeafReaderContext leaf : reader.leaves()) {
+ checkValidPointValues(leaf.reader().getPointValues(field));
+ }
+
+ // fetch the global min/max packed values across all segments
+ byte[] globalMinPacked = PointValues.getMinPackedValue(reader, getField());
+ byte[] globalMaxPacked = PointValues.getMaxPackedValue(reader, getField());
+
+ if (globalMinPacked == null || globalMaxPacked == null) {
+ return new MatchNoDocsQuery();
+ }
+
+ return switch (relate(globalMinPacked, globalMaxPacked)) {
+ case CELL_INSIDE_QUERY -> {
+ if (canRewriteToMatchAllQuery(reader)) {
+ yield new MatchAllDocsQuery();
+ } else if (canRewriteToFieldExistsQuery(reader)) {
+ yield new FieldExistsQuery(field);
+ } else {
+ yield super.rewrite(searcher);
+ }
+ }
+ case CELL_OUTSIDE_QUERY -> new MatchNoDocsQuery();
+ case CELL_CROSSES_QUERY -> super.rewrite(searcher);
+ };
+ }
+
+ private boolean canRewriteToMatchAllQuery(IndexReader reader) throws IOException {
+ for (LeafReaderContext context : reader.leaves()) {
+ LeafReader leaf = context.reader();
+ PointValues values = leaf.getPointValues(field);
+
+ if (values == null || values.getDocCount() != leaf.maxDoc()) {
+ return false;
+ }
+ }
+
+ return true;
+ }
+
+ private boolean canRewriteToFieldExistsQuery(IndexReader reader) {
+ for (LeafReaderContext leaf : reader.leaves()) {
+ FieldInfo info = leaf.reader().getFieldInfos().fieldInfo(field);
+
+ if (info != null
+ && info.getDocValuesType() == DocValuesType.NONE
+ && !info.hasNorms()
+ && info.getVectorDimension() == 0) {
+ // Can't use a FieldExistsQuery on this segment, so return false
+ return false;
+ }
+ }
+
+ return true;
+ }
+
+ private Relation relate(byte[] minPackedValue, byte[] maxPackedValue) {
+ boolean crosses = false;
+ int offset = 0;
+
+ for (int dim = 0; dim < numDims; dim++, offset += bytesPerDim) {
+
+ if (comparator.compare(minPackedValue, offset, upperPoint, offset) > 0
+ || comparator.compare(maxPackedValue, offset, lowerPoint, offset) < 0) {
+ return Relation.CELL_OUTSIDE_QUERY;
+ }
+
+ crosses |=
+ comparator.compare(minPackedValue, offset, lowerPoint, offset) < 0
+ || comparator.compare(maxPackedValue, offset, upperPoint, offset) > 0;
+ }
+
+ if (crosses) {
+ return Relation.CELL_CROSSES_QUERY;
+ } else {
+ return Relation.CELL_INSIDE_QUERY;
+ }
+ }
+
+ private boolean checkValidPointValues(PointValues values) throws IOException {
+ if (values == null) {
+ // No docs in this segment/field indexed any points
+ return false;
+ }
+
+ if (values.getNumIndexDimensions() != numDims) {
+ throw new IllegalArgumentException(
+ "field=\""
+ + field
+ + "\" was indexed with numIndexDimensions="
+ + values.getNumIndexDimensions()
+ + " but this query has numDims="
+ + numDims);
+ }
+ if (bytesPerDim != values.getBytesPerDimension()) {
+ throw new IllegalArgumentException(
+ "field=\""
+ + field
+ + "\" was indexed with bytesPerDim="
+ + values.getBytesPerDimension()
+ + " but this query has bytesPerDim="
+ + bytesPerDim);
+ }
+ return true;
+ }
}
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestLRUQueryCache.java b/lucene/core/src/test/org/apache/lucene/search/TestLRUQueryCache.java
index 8a7b604881f..8bb0762ebe9 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestLRUQueryCache.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestLRUQueryCache.java
@@ -1953,7 +1953,11 @@ public class TestLRUQueryCache extends LuceneTestCase {
doc2.add(new StringField("name", "alice", Store.YES));
doc2.add(new LongPoint("age", 20));
doc2.add(new SortedNumericDocValuesField("age", 20));
- w.addDocuments(Arrays.asList(doc1, doc2));
+ Document doc3 = new Document();
+ doc3.add(new StringField("name", "steve", Store.YES));
+ doc3.add(new LongPoint("age", 25));
+ doc3.add(new SortedNumericDocValuesField("age", 25));
+ w.addDocuments(Arrays.asList(doc1, doc2, doc3));
final IndexReader reader = w.getReader();
final IndexSearcher searcher = newSearcher(reader);
searcher.setQueryCachingPolicy(ALWAYS_CACHE);
@@ -1964,8 +1968,8 @@ public class TestLRUQueryCache extends LuceneTestCase {
TermQuery subQuery1 = new TermQuery(new Term("name", "tom"));
IndexOrDocValuesQuery subQuery2 =
new IndexOrDocValuesQuery(
- LongPoint.newRangeQuery("age", 10, 30),
- SortedNumericDocValuesField.newSlowRangeQuery("age", 10, 30));
+ LongPoint.newRangeQuery("age", 10, 20),
+ SortedNumericDocValuesField.newSlowRangeQuery("age", 10, 20));
BooleanQuery query = bq.add(subQuery1, Occur.FILTER).add(subQuery2, Occur.FILTER).build();
Set<Query> cacheSet = new HashSet<>();
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestPointQueries.java b/lucene/core/src/test/org/apache/lucene/search/TestPointQueries.java
index caedc1aa2dc..60906f919ee 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestPointQueries.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestPointQueries.java
@@ -772,6 +772,7 @@ public class TestPointQueries extends LuceneTestCase {
if (missing.get(id) == false) {
doc.add(new BinaryPoint("value", docValues[ord]));
+ doc.add(new SortedNumericDocValuesField("value", 1));
if (VERBOSE) {
System.out.println("id=" + id);
for (int dim = 0; dim < numDims; dim++) {
@@ -2465,6 +2466,66 @@ public class TestPointQueries extends LuceneTestCase {
dir.close();
}
+ public void testRangeQueryOptimizesRewrites() throws IOException {
+ Directory dir = newDirectory();
+ RandomIndexWriter w = new RandomIndexWriter(random(), dir, newIndexWriterConfig());
+
+ int numDims = randomIntValue(1, PointValues.MAX_INDEX_DIMENSIONS);
+ int[] point = new int[numDims];
+ int[] lower = new int[numDims];
+ int[] upper = new int[numDims];
+
+ for (int i = 0; i < numDims; i++) {
+ point[i] = randomIntValue(1, 10);
+ lower[i] = point[i] - randomIntValue(0, 5);
+ upper[i] = point[i] + randomIntValue(0, 5);
+ }
+
+ // Should rewrite to match all docs query if fully contained
+ w.addDocument(
+ List.of(new IntPoint("field", point), new SortedNumericDocValuesField("field", 1)));
+
+ IndexReader reader = w.getReader();
+ IndexSearcher searcher = new IndexSearcher(reader);
+
+ Query query = IntPoint.newRangeQuery("field", lower, upper);
+ Query rewritten = searcher.rewrite(query);
+ assertTrue(
+ "Expected MatchAllDocsQuery, but got [" + rewritten.getClass().getName() + "]",
+ rewritten instanceof MatchAllDocsQuery);
+ assertEquals(1, searcher.count(query));
+
+ // Should rewrite to FieldExistsQuery if fully contained but not all docs have values
+ w.addDocument(new Document());
+ w.forceMerge(1);
+
+ reader.close();
+ reader = w.getReader();
+ searcher = new IndexSearcher(reader);
+ rewritten = searcher.rewrite(query);
+ assertTrue(
+ "Expected FieldExistsQuery, but got [" + rewritten.getClass().getName() + "]",
+ rewritten instanceof FieldExistsQuery);
+ assertEquals(1, searcher.count(query));
+
+ // Should fallback to MatchNoDocsQuery if no docs have values
+ for (int i = 0; i < numDims; i++) {
+ lower[i] = point[i] - 3;
+ upper[i] = point[i] - 2;
+ }
+
+ query = IntPoint.newRangeQuery("field", lower, upper);
+ rewritten = searcher.rewrite(query);
+ assertTrue(
+ "Expected MatchNoDocsQuery, but got [" + rewritten.getClass().getName() + "]",
+ rewritten instanceof MatchNoDocsQuery);
+ assertEquals(0, searcher.count(query));
+
+ reader.close();
+ w.close();
+ dir.close();
+ }
+
public void testRangeQuerySkipsNonMatchingSegments() throws IOException {
Directory dir = newDirectory();
IndexWriter w = new IndexWriter(dir, newIndexWriterConfig());