Skip to content

Apache Lucene open-source search software

LUCENE-10084: Rewrite DocValuesFieldExistsQuery to MatchAllDocsQuery when all docs have the field (#677)

Vigya Sharmac132bbf677b5
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 903b9db4d8c..f92030bdb1b 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -226,6 +226,8 @@ Optimizations
* LUCENE-10408 Better encoding of doc Ids in vectors. (Mayya Sharipova, Julie Tibshirani, Adrien Grand)
+* LUCENE-10084: Rewrite DocValuesFieldExistsQuery to MatchAllDocsQuery whenever terms
+ or points have a docCount that is equal to maxDoc. (Vigya Sharma)
Changes in runtime behavior
---------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/DocValuesFieldExistsQuery.java b/lucene/core/src/java/org/apache/lucene/search/DocValuesFieldExistsQuery.java
index 303789d305f..c2939ea1708 100644
--- a/lucene/core/src/java/org/apache/lucene/search/DocValuesFieldExistsQuery.java
+++ b/lucene/core/src/java/org/apache/lucene/search/DocValuesFieldExistsQuery.java
@@ -22,8 +22,11 @@ import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.DocValuesType;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.PointValues;
+import org.apache.lucene.index.Terms;
/**
* A {@link Query} that matches documents that have a value for a given field as reported by doc
@@ -64,6 +67,24 @@ public final class DocValuesFieldExistsQuery extends Query {
}
}
+ @Override
+ public Query rewrite(IndexReader reader) throws IOException {
+ int rewritableReaders = 0;
+ for (LeafReaderContext context : reader.leaves()) {
+ LeafReader leaf = context.reader();
+ Terms terms = leaf.terms(field);
+ PointValues pointValues = leaf.getPointValues(field);
+ if ((terms != null && terms.getDocCount() == leaf.maxDoc())
+ || (pointValues != null && pointValues.getDocCount() == leaf.maxDoc())) {
+ rewritableReaders++;
+ }
+ }
+ if (rewritableReaders == reader.leaves().size()) {
+ return new MatchAllDocsQuery();
+ }
+ return super.rewrite(reader);
+ }
+
@Override
public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) {
return new ConstantScoreWeight(this, boost) {
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestDocValuesFieldExistsQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestDocValuesFieldExistsQuery.java
index c03be974684..32f163ba99e 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestDocValuesFieldExistsQuery.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestDocValuesFieldExistsQuery.java
@@ -17,6 +17,7 @@
package org.apache.lucene.search;
import java.io.IOException;
+import org.apache.lucene.document.BinaryPoint;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.document.LongPoint;
@@ -38,6 +39,92 @@ import org.apache.lucene.util.IOUtils;
public class TestDocValuesFieldExistsQuery extends LuceneTestCase {
+ public void testRewriteWithTermsPresent() throws IOException {
+ Directory dir = newDirectory();
+ RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
+ final int numDocs = atLeast(100);
+ for (int i = 0; i < numDocs; ++i) {
+ Document doc = new Document();
+ doc.add(new StringField("f", random().nextBoolean() ? "yes" : "no", Store.NO));
+ iw.addDocument(doc);
+ }
+ iw.commit();
+ final IndexReader reader = iw.getReader();
+ iw.close();
+
+ assertTrue((new DocValuesFieldExistsQuery("f")).rewrite(reader) instanceof MatchAllDocsQuery);
+ reader.close();
+ dir.close();
+ }
+
+ public void testRewriteWithPointValuesPresent() throws IOException {
+ Directory dir = newDirectory();
+ RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
+ final int numDocs = atLeast(100);
+ for (int i = 0; i < numDocs; ++i) {
+ Document doc = new Document();
+ doc.add(new BinaryPoint("dim", new byte[4], new byte[4]));
+ iw.addDocument(doc);
+ }
+ iw.commit();
+ final IndexReader reader = iw.getReader();
+ iw.close();
+
+ assertTrue((new DocValuesFieldExistsQuery("dim")).rewrite(reader) instanceof MatchAllDocsQuery);
+ reader.close();
+ dir.close();
+ }
+
+ public void testNoRewrite() throws IOException {
+ Directory dir = newDirectory();
+ RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
+ final int numDocs = atLeast(100);
+ for (int i = 0; i < numDocs; ++i) {
+ Document doc = new Document();
+ doc.add(new BinaryPoint("dim", new byte[4], new byte[4]));
+ iw.addDocument(doc);
+ }
+ for (int i = 0; i < numDocs; ++i) {
+ Document doc = new Document();
+ doc.add(new StringField("f", random().nextBoolean() ? "yes" : "no", Store.NO));
+ iw.addDocument(doc);
+ }
+ iw.commit();
+ final IndexReader reader = iw.getReader();
+ iw.close();
+
+ assertFalse(
+ (new DocValuesFieldExistsQuery("dim")).rewrite(reader) instanceof MatchAllDocsQuery);
+ assertFalse((new DocValuesFieldExistsQuery("f")).rewrite(reader) instanceof MatchAllDocsQuery);
+ reader.close();
+ dir.close();
+ }
+
+ public void testNoRewriteWithDocValues() throws IOException {
+ Directory dir = newDirectory();
+ RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
+ final int numDocs = atLeast(100);
+ for (int i = 0; i < numDocs; ++i) {
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("dv1", 1));
+ doc.add(new SortedNumericDocValuesField("dv2", 1));
+ doc.add(new SortedNumericDocValuesField("dv2", 2));
+ iw.addDocument(doc);
+ }
+ iw.commit();
+ final IndexReader reader = iw.getReader();
+ iw.close();
+
+ assertFalse(
+ (new DocValuesFieldExistsQuery("dv1")).rewrite(reader) instanceof MatchAllDocsQuery);
+ assertFalse(
+ (new DocValuesFieldExistsQuery("dv2")).rewrite(reader) instanceof MatchAllDocsQuery);
+ assertFalse(
+ (new DocValuesFieldExistsQuery("dv3")).rewrite(reader) instanceof MatchAllDocsQuery);
+ reader.close();
+ dir.close();
+ }
+
public void testRandom() throws IOException {
final int iters = atLeast(10);
for (int iter = 0; iter < iters; ++iter) {