Skip to content

Apache Lucene open-source search software

Optimize FieldExistsQuery to leverage index statistic in DocValuesSkipper (#14830)

panguixin51e6fbbdf952
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 95673d07d2a..01938ae6347 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -177,6 +177,8 @@ Optimizations
* GITHUB#14836: Sometimes use FixedBitSet when doing HNSW searches. This slightly improves HNSW
search performance on smaller graphs. (Ben Trent)
+* GITHUB#14830: Optimize FieldExistsQuery to leverage index statistic in DocValuesSkipper. (Pan Guixin)
+
Bug Fixes
---------------------
* GITHUB#14654: ValueSource.fromDoubleValuesSource(dvs).getSortField() would throw errors when
diff --git a/lucene/core/src/java/org/apache/lucene/search/FieldExistsQuery.java b/lucene/core/src/java/org/apache/lucene/search/FieldExistsQuery.java
index f695197a230..43f4924cda6 100644
--- a/lucene/core/src/java/org/apache/lucene/search/FieldExistsQuery.java
+++ b/lucene/core/src/java/org/apache/lucene/search/FieldExistsQuery.java
@@ -21,6 +21,8 @@ import java.util.Objects;
import org.apache.lucene.document.KnnFloatVectorField;
import org.apache.lucene.index.ByteVectorValues;
import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.DocValuesSkipIndexType;
+import org.apache.lucene.index.DocValuesSkipper;
import org.apache.lucene.index.DocValuesType;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
@@ -138,17 +140,14 @@ public class FieldExistsQuery extends Query {
!= DocValuesType.NONE) { // the field indexes doc values or points
// This optimization is possible due to LUCENE-9334 enforcing a field to always use the
- // same data structures (all or nothing). Since there's no index statistic to detect when
- // all documents have doc values for a specific field, FieldExistsQuery can only be
- // rewritten to MatchAllDocsQuery for doc values field, when that same field also indexes
- // terms or point values which do have index statistics, and those statistics confirm that
- // all documents in this segment have values terms or point values.
-
- Terms terms = leaf.terms(field);
- PointValues pointValues = leaf.getPointValues(field);
+ // same data structures (all or nothing).
+ final Terms terms = leaf.terms(field);
+ final PointValues pointValues = leaf.getPointValues(field);
+ final DocValuesSkipper docValuesSkipper = leaf.getDocValuesSkipper(field);
if ((terms == null || terms.getDocCount() != leaf.maxDoc())
- && (pointValues == null || pointValues.getDocCount() != leaf.maxDoc())) {
+ && (pointValues == null || pointValues.getDocCount() != leaf.maxDoc())
+ && (docValuesSkipper == null || docValuesSkipper.docCount() != leaf.maxDoc())) {
allReadersRewritable = false;
break;
}
@@ -248,6 +247,9 @@ public class FieldExistsQuery extends Query {
} else if (fieldInfo.getIndexOptions() != IndexOptions.NONE) {
Terms terms = reader.terms(field);
return terms == null ? 0 : terms.getDocCount();
+ } else if (fieldInfo.docValuesSkipIndexType() != DocValuesSkipIndexType.NONE) {
+ DocValuesSkipper docValuesSkipper = reader.getDocValuesSkipper(field);
+ return docValuesSkipper == null ? 0 : docValuesSkipper.docCount();
}
}
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestFieldExistsQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestFieldExistsQuery.java
index 75b438fac4b..e9839ed722b 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestFieldExistsQuery.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestFieldExistsQuery.java
@@ -88,6 +88,24 @@ public class TestFieldExistsQuery extends LuceneTestCase {
dir.close();
}
+ public void testDocValuesRewriteWithDocValuesSkipperPresent() throws IOException {
+ Directory dir = newDirectory();
+ RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
+ final int numDocs = atLeast(100);
+ for (int i = 0; i < numDocs; ++i) {
+ Document doc = new Document();
+ doc.add(NumericDocValuesField.indexedField("dim", 2));
+ iw.addDocument(doc);
+ }
+ iw.commit();
+ final IndexReader reader = iw.getReader();
+ iw.close();
+
+ assertEquals(new MatchAllDocsQuery(), new FieldExistsQuery("dim").rewrite(newSearcher(reader)));
+ reader.close();
+ dir.close();
+ }
+
public void testDocValuesNoRewrite() throws IOException {
Directory dir = newDirectory();
RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
@@ -741,6 +759,41 @@ public class TestFieldExistsQuery extends LuceneTestCase {
return v;
}
+ public void testDeleteDocValues() throws IOException {
+ try (Directory dir = newDirectory();
+ RandomIndexWriter iw = new RandomIndexWriter(random(), dir)) {
+ final int numDocs = atLeast(100);
+
+ boolean allDocsHaveValue = random().nextBoolean();
+ BitSet docWithValue = new FixedBitSet(numDocs);
+ for (int i = 0; i < numDocs; ++i) {
+ Document doc = new Document();
+ if (allDocsHaveValue || random().nextBoolean()) {
+ doc.add(NumericDocValuesField.indexedField("num", i));
+ docWithValue.set(i);
+ }
+ doc.add(new StringField("id", Integer.toString(i), Store.NO));
+ iw.addDocument(doc);
+ }
+
+ if (random().nextBoolean()) {
+ final int numDeleted = random().nextInt(numDocs) + 1;
+ for (int i = 0; i < numDeleted; ++i) {
+ int id = random().nextInt(numDocs);
+ iw.deleteDocuments(new Term("id", Integer.toString(id)));
+ docWithValue.clear(id);
+ }
+ }
+
+ try (IndexReader reader = iw.getReader()) {
+ final IndexSearcher searcher = newSearcher(reader);
+
+ final int count = searcher.count(new FieldExistsQuery("num"));
+ assertEquals(docWithValue.cardinality(), count);
+ }
+ }
+ }
+
public void testDeleteAllPointDocs() throws Exception {
try (Directory dir = newDirectory();
RandomIndexWriter iw = new RandomIndexWriter(random(), dir)) {