Apache Lucene open-source search software
Optimize FieldExistsQuery to leverage index statistic in DocValuesSkipper (#14830)
panguixin51e6fbbdf952
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex 95673d07d2a..01938ae6347 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -177,6 +177,8 @@ Optimizations* GITHUB#14836: Sometimes use FixedBitSet when doing HNSW searches. This slightly improves HNSWsearch performance on smaller graphs. (Ben Trent)+* GITHUB#14830: Optimize FieldExistsQuery to leverage index statistic in DocValuesSkipper. (Pan Guixin)+Bug Fixes---------------------* GITHUB#14654: ValueSource.fromDoubleValuesSource(dvs).getSortField() would throw errors whendiff --git a/lucene/core/src/java/org/apache/lucene/search/FieldExistsQuery.java b/lucene/core/src/java/org/apache/lucene/search/FieldExistsQuery.javaindex f695197a230..43f4924cda6 100644--- a/lucene/core/src/java/org/apache/lucene/search/FieldExistsQuery.java+++ b/lucene/core/src/java/org/apache/lucene/search/FieldExistsQuery.java@@ -21,6 +21,8 @@ import java.util.Objects;import org.apache.lucene.document.KnnFloatVectorField;import org.apache.lucene.index.ByteVectorValues;import org.apache.lucene.index.DocValues;+import org.apache.lucene.index.DocValuesSkipIndexType;+import org.apache.lucene.index.DocValuesSkipper;import org.apache.lucene.index.DocValuesType;import org.apache.lucene.index.FieldInfo;import org.apache.lucene.index.FieldInfos;@@ -138,17 +140,14 @@ public class FieldExistsQuery extends Query {!= DocValuesType.NONE) { // the field indexes doc values or points// This optimization is possible due to LUCENE-9334 enforcing a field to always use the- // same data structures (all or nothing). Since there's no index statistic to detect when- // all documents have doc values for a specific field, FieldExistsQuery can only be- // rewritten to MatchAllDocsQuery for doc values field, when that same field also indexes- // terms or point values which do have index statistics, and those statistics confirm that- // all documents in this segment have values terms or point values.-- Terms terms = leaf.terms(field);- PointValues pointValues = leaf.getPointValues(field);+ // same data structures (all or nothing).+ final Terms terms = leaf.terms(field);+ final PointValues pointValues = leaf.getPointValues(field);+ final DocValuesSkipper docValuesSkipper = leaf.getDocValuesSkipper(field);if ((terms == null || terms.getDocCount() != leaf.maxDoc())- && (pointValues == null || pointValues.getDocCount() != leaf.maxDoc())) {+ && (pointValues == null || pointValues.getDocCount() != leaf.maxDoc())+ && (docValuesSkipper == null || docValuesSkipper.docCount() != leaf.maxDoc())) {allReadersRewritable = false;break;}@@ -248,6 +247,9 @@ public class FieldExistsQuery extends Query {} else if (fieldInfo.getIndexOptions() != IndexOptions.NONE) {Terms terms = reader.terms(field);return terms == null ? 0 : terms.getDocCount();+ } else if (fieldInfo.docValuesSkipIndexType() != DocValuesSkipIndexType.NONE) {+ DocValuesSkipper docValuesSkipper = reader.getDocValuesSkipper(field);+ return docValuesSkipper == null ? 0 : docValuesSkipper.docCount();}}diff --git a/lucene/core/src/test/org/apache/lucene/search/TestFieldExistsQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestFieldExistsQuery.javaindex 75b438fac4b..e9839ed722b 100644--- a/lucene/core/src/test/org/apache/lucene/search/TestFieldExistsQuery.java+++ b/lucene/core/src/test/org/apache/lucene/search/TestFieldExistsQuery.java@@ -88,6 +88,24 @@ public class TestFieldExistsQuery extends LuceneTestCase {dir.close();}+ public void testDocValuesRewriteWithDocValuesSkipperPresent() throws IOException {+ Directory dir = newDirectory();+ RandomIndexWriter iw = new RandomIndexWriter(random(), dir);+ final int numDocs = atLeast(100);+ for (int i = 0; i < numDocs; ++i) {+ Document doc = new Document();+ doc.add(NumericDocValuesField.indexedField("dim", 2));+ iw.addDocument(doc);+ }+ iw.commit();+ final IndexReader reader = iw.getReader();+ iw.close();++ assertEquals(new MatchAllDocsQuery(), new FieldExistsQuery("dim").rewrite(newSearcher(reader)));+ reader.close();+ dir.close();+ }+public void testDocValuesNoRewrite() throws IOException {Directory dir = newDirectory();RandomIndexWriter iw = new RandomIndexWriter(random(), dir);@@ -741,6 +759,41 @@ public class TestFieldExistsQuery extends LuceneTestCase {return v;}+ public void testDeleteDocValues() throws IOException {+ try (Directory dir = newDirectory();+ RandomIndexWriter iw = new RandomIndexWriter(random(), dir)) {+ final int numDocs = atLeast(100);++ boolean allDocsHaveValue = random().nextBoolean();+ BitSet docWithValue = new FixedBitSet(numDocs);+ for (int i = 0; i < numDocs; ++i) {+ Document doc = new Document();+ if (allDocsHaveValue || random().nextBoolean()) {+ doc.add(NumericDocValuesField.indexedField("num", i));+ docWithValue.set(i);+ }+ doc.add(new StringField("id", Integer.toString(i), Store.NO));+ iw.addDocument(doc);+ }++ if (random().nextBoolean()) {+ final int numDeleted = random().nextInt(numDocs) + 1;+ for (int i = 0; i < numDeleted; ++i) {+ int id = random().nextInt(numDocs);+ iw.deleteDocuments(new Term("id", Integer.toString(id)));+ docWithValue.clear(id);+ }+ }++ try (IndexReader reader = iw.getReader()) {+ final IndexSearcher searcher = newSearcher(reader);++ final int count = searcher.count(new FieldExistsQuery("num"));+ assertEquals(docWithValue.cardinality(), count);+ }+ }+ }+public void testDeleteAllPointDocs() throws Exception {try (Directory dir = newDirectory();RandomIndexWriter iw = new RandomIndexWriter(random(), dir)) {