Skip to content

Apache Lucene open-source search software

Don't rely on .cost() to provide the number of docs with a value (SimpleText uses a heuristic). (#16566)

Dawid Weiss411013eebd53
diff --git a/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java b/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java
index 56ec93a3f40..4593447a4ff 100644
--- a/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java
+++ b/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java
@@ -35,6 +35,7 @@ import org.apache.lucene.codecs.DocValuesConsumer;
import org.apache.lucene.codecs.DocValuesFormat;
import org.apache.lucene.codecs.DocValuesProducer;
import org.apache.lucene.codecs.FieldInfosFormat;
+import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FlushInfo;
import org.apache.lucene.store.IOContext;
@@ -389,10 +390,13 @@ final class ReadersAndUpdates {
Long.toString(gen, Character.MAX_RADIX));
DocValuesProducer p = dvFormat.fieldsProducer(srs);
deltaProducers.add(p);
+ // Count docs-with-value by iteration: cost() is only an estimate (SimpleText reports
+ // maxDoc for sparse columns) and deltas are small by construction.
deltaCoverage +=
- type == DocValuesType.BINARY
- ? p.getBinary(fieldInfo).cost()
- : p.getNumeric(fieldInfo).cost();
+ docsWithValueCount(
+ type == DocValuesType.BINARY
+ ? p.getBinary(fieldInfo)
+ : p.getNumeric(fieldInfo));
}
} catch (Throwable t) {
// The try-with-resources below owns the normal close; close here if opening a delta fails
@@ -580,6 +584,15 @@ final class ReadersAndUpdates {
}
}
+ /** Exact number of docs with a value; the iterator is consumed. */
+ private static long docsWithValueCount(DocIdSetIterator it) throws IOException {
+ long count = 0;
+ while (it.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
+ count++;
+ }
+ return count;
+ }
+
/**
* Fresh overlay (newest generation first) over the given delta producers, used to fold deltas
* during compaction.