Skip to content

Apache Lucene open-source search software

Implement sparse LiveDocs to reduce memory by up to 8x (#15413)

Salvatore Campagna011703499168
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 28d338b5869..e34082d60ce 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -225,6 +225,8 @@ Optimizations
* GITHUB#14963: Bypass HNSW graph building for tiny segments. (Shubham Chaudhary, Ben Trent)
+* GITHUB#13084: Implement sparse LiveDocs format to reduce memory usage by up to 8x for segments with <=1% deletions. (Salvatore Campagna)
+
* GITHUB#15397: NumericComparator: immediately check whether a segment is competitive with the recorded bottom (Martijn van Groningen)
# GITHUB#15303: Speed up NumericUtils#{add,subtract} by operating on integers instead of bytes. (Kaival Parikh)
diff --git a/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/LiveDocsBenchmark.java b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/LiveDocsBenchmark.java
new file mode 100644
index 00000000000..89fa96d1857
--- /dev/null
+++ b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/LiveDocsBenchmark.java
@@ -0,0 +1,430 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.benchmark.jmh;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Random;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.util.DenseLiveDocs;
+import org.apache.lucene.util.FixedBitSet;
+import org.apache.lucene.util.SparseFixedBitSet;
+import org.apache.lucene.util.SparseLiveDocs;
+import org.openjdk.jmh.annotations.AuxCounters;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.infra.Blackhole;
+
+/**
+ * Benchmarks comparing {@link SparseLiveDocs} vs {@link DenseLiveDocs} performance across different
+ * deletion rates, patterns, and segment sizes.
+ *
+ * <p>This benchmark suite measures four key operations to evaluate the trade-offs between sparse
+ * and dense LiveDocs implementations:
+ *
+ * <ul>
+ * <li><b>Random access (get)</b> - O(1) for both, but sparse has additional indirection overhead
+ * <li><b>Deleted docs iteration</b> - O(deletedDocs) for sparse, O(maxDoc) for dense
+ * <li><b>Live docs iteration (full)</b> - O(maxDoc) for both, tests get() performance at scale
+ * <li><b>Live docs iteration (range)</b> - O(range) for both, tests advance() and get() on subset
+ * </ul>
+ *
+ * <h2>Benchmark Parameters</h2>
+ *
+ * <ul>
+ * <li><b>maxDoc</b> - Segment sizes: 100K, 1M, 10M documents
+ * <li><b>deletionRate</b> - Percentage of deleted documents: 0.1%, 1%, 5%, 10%, 20%, 30%
+ * <li><b>deletionPattern</b> - Distribution of deletions:
+ * <ul>
+ * <li>RANDOM: Deletions scattered uniformly across entire document space
+ * <li>CLUSTERED: Consecutive deletions at start of segment
+ * <li>UNIFORM: Deletions evenly spaced across segment
+ * </ul>
+ * </ul>
+ *
+ * <h2>Usage</h2>
+ *
+ * <p>Run all benchmarks:
+ *
+ * <pre>
+ * java -jar lucene-benchmark-jmh.jar "LiveDocsBenchmark"
+ * </pre>
+ *
+ * <p>Run specific operation for sparse only:
+ *
+ * <pre>
+ * java -jar lucene-benchmark-jmh.jar "LiveDocsBenchmark.sparseIterateDeleted"
+ * </pre>
+ *
+ * <p>Filter by specific parameters:
+ *
+ * <pre>
+ * java -jar lucene-benchmark-jmh.jar "LiveDocsBenchmark" -p deletionRate=0.01 -p deletionPattern=CLUSTERED
+ * </pre>
+ *
+ * @see SparseLiveDocs
+ * @see DenseLiveDocs
+ * @see LiveDocsPathologicalBenchmark
+ */
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.MICROSECONDS)
+@State(Scope.Benchmark)
+@Warmup(iterations = 3, time = 2)
+@Measurement(iterations = 5, time = 3)
+@Fork(
+ value = 1,
+ jvmArgsAppend = {"-Xmx2g", "-Xms2g"})
+public class LiveDocsBenchmark {
+
+ /** Number of documents in the segment (100K, 1M, or 10M). */
+ @Param({"100000", "1000000", "10000000"})
+ int maxDoc;
+
+ /** Percentage of documents to delete (0.1%, 1%, 5%, 10%, 20%, or 30%). */
+ @Param({"0.001", "0.01", "0.05", "0.10", "0.20", "0.30"})
+ double deletionRate;
+
+ /**
+ * Pattern for distributing deletions:
+ *
+ * <ul>
+ * <li>RANDOM - Deletions scattered uniformly across document space
+ * <li>CLUSTERED - Consecutive deletions at start of segment
+ * <li>UNIFORM - Deletions evenly spaced across segment
+ * </ul>
+ */
+ @Param({"RANDOM", "CLUSTERED", "UNIFORM"})
+ String deletionPattern;
+
+ /** Sparse LiveDocs implementation under test. */
+ private SparseLiveDocs sparseLiveDocs;
+
+ /** Dense LiveDocs implementation for comparison. */
+ private DenseLiveDocs denseLiveDocs;
+
+ /** Pre-generated random document IDs for random access benchmarks. */
+ private int[] randomDocIds;
+
+ /** Number of random accesses to perform in each benchmark iteration. */
+ private static final int RANDOM_ACCESS_SIZE = 10000;
+
+ /** Memory used by SparseLiveDocs in bytes. */
+ private long sparseBytes;
+
+ /** Memory used by DenseLiveDocs in bytes. */
+ private long denseBytes;
+
+ /** Memory overhead percentage (negative means sparse uses less memory). */
+ private double overheadPct;
+
+ /** Number of deleted documents. */
+ private int deleted;
+
+ /**
+ * JMH auxiliary counters for tracking memory metrics across benchmark runs.
+ *
+ * <p>These metrics are reported as secondary results in JMH output and include: deletion count,
+ * memory usage for both implementations, and overhead percentage.
+ */
+ @AuxCounters(AuxCounters.Type.EVENTS)
+ @State(Scope.Thread)
+ public static class LiveDocsMetrics {
+ /** Number of deleted documents. */
+ public int deleted;
+
+ /** Memory used by SparseLiveDocs in bytes. */
+ public long sparseBytes;
+
+ /** Memory used by DenseLiveDocs in bytes. */
+ public long denseBytes;
+
+ /** Memory overhead percentage (negative means sparse uses less memory). */
+ public double overheadPct;
+ }
+
+ /**
+ * Sets up the benchmark by creating both sparse and dense LiveDocs with identical deletion
+ * patterns.
+ *
+ * <p>This method is called once per trial (combination of parameters) before any benchmark
+ * iterations run.
+ */
+ @Setup(Level.Trial)
+ public void setup() {
+ Random random = new Random(42);
+ int numDeleted = (int) (maxDoc * deletionRate);
+
+ if (numDeleted == 0) {
+ throw new IllegalStateException(
+ "Benchmark requires at least one deletion. "
+ + "Current parameters: maxDoc="
+ + maxDoc
+ + ", deletionRate="
+ + deletionRate
+ + " result in zero deletions.");
+ }
+
+ SparseFixedBitSet sparseSet = new SparseFixedBitSet(maxDoc);
+ FixedBitSet fixedSet = new FixedBitSet(maxDoc);
+ fixedSet.set(0, maxDoc);
+
+ switch (deletionPattern) {
+ case "RANDOM":
+ Set<Integer> deletedSet = new HashSet<>();
+ while (deletedSet.size() < numDeleted) {
+ deletedSet.add(random.nextInt(maxDoc));
+ }
+ for (int docId : deletedSet) {
+ sparseSet.set(docId);
+ fixedSet.clear(docId);
+ }
+ break;
+
+ case "CLUSTERED":
+ for (int i = 0; i < numDeleted; i++) {
+ sparseSet.set(i);
+ fixedSet.clear(i);
+ }
+ break;
+
+ case "UNIFORM":
+ for (int i = 0; i < numDeleted; i++) {
+ int docId = (int) ((long) i * maxDoc / numDeleted);
+ sparseSet.set(docId);
+ fixedSet.clear(docId);
+ }
+ break;
+ }
+
+ sparseLiveDocs = SparseLiveDocs.builder(sparseSet, maxDoc).build();
+ denseLiveDocs = DenseLiveDocs.builder(fixedSet, maxDoc).build();
+
+ sparseBytes = sparseLiveDocs.ramBytesUsed();
+ denseBytes = denseLiveDocs.ramBytesUsed();
+ overheadPct = ((double) sparseBytes - denseBytes) / denseBytes * 100.0;
+ deleted = (int) (maxDoc * deletionRate);
+
+ randomDocIds = new int[RANDOM_ACCESS_SIZE];
+ for (int i = 0; i < RANDOM_ACCESS_SIZE; i++) {
+ randomDocIds[i] = random.nextInt(maxDoc);
+ }
+ }
+
+ /**
+ * Benchmarks random access (get) performance for {@link SparseLiveDocs}.
+ *
+ * <p>Tests 10,000 random get() operations on pre-generated random document IDs. Sparse
+ * implementation has additional indirection overhead (block lookup + word lookup) compared to
+ * dense.
+ *
+ * @param metrics JMH auxiliary counters for memory statistics
+ * @param blackhole JMH blackhole to prevent dead code elimination
+ */
+ @Benchmark
+ public void sparseRandomAccess(final LiveDocsMetrics metrics, final Blackhole blackhole) {
+ fillMetrics(metrics);
+
+ for (int docId : randomDocIds) {
+ blackhole.consume(sparseLiveDocs.get(docId));
+ }
+ }
+
+ /**
+ * Benchmarks random access (get) performance for {@link DenseLiveDocs}.
+ *
+ * <p>Tests 10,000 random get() operations on pre-generated random document IDs. Dense
+ * implementation uses simple array access with bit masking.
+ *
+ * @param metrics JMH auxiliary counters for memory statistics
+ * @param blackhole JMH blackhole to prevent dead code elimination
+ */
+ @Benchmark
+ public void denseRandomAccess(final LiveDocsMetrics metrics, final Blackhole blackhole) {
+ fillMetrics(metrics);
+
+ for (int docId : randomDocIds) {
+ blackhole.consume(denseLiveDocs.get(docId));
+ }
+ }
+
+ /**
+ * Benchmarks iteration over deleted documents for {@link SparseLiveDocs}.
+ *
+ * <p>This is the primary use case for sparse LiveDocs. Sparse implementation only iterates over
+ * actually deleted documents, making it much faster at low deletion rates.
+ *
+ * <p><b>Expected performance:</b>
+ *
+ * @param metrics JMH auxiliary counters for memory statistics
+ * @return number of deleted documents (for verification)
+ * @throws IOException if iteration fails
+ */
+ @Benchmark
+ public int sparseIterateDeleted(final LiveDocsMetrics metrics) throws IOException {
+ fillMetrics(metrics);
+
+ DocIdSetIterator it = sparseLiveDocs.deletedDocsIterator();
+ int count = 0;
+ for (int doc = it.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = it.nextDoc()) {
+ count++;
+ }
+ return count;
+ }
+
+ /**
+ * Benchmarks iteration over deleted documents for {@link DenseLiveDocs}.
+ *
+ * <p>Dense implementation must scan all maxDoc positions to find deleted documents, making it
+ * slower at low deletion rates but more predictable.
+ *
+ * @param metrics JMH auxiliary counters for memory statistics
+ * @return number of deleted documents (for verification)
+ * @throws IOException if iteration fails
+ */
+ @Benchmark
+ public int denseIterateDeleted(final LiveDocsMetrics metrics) throws IOException {
+ fillMetrics(metrics);
+
+ DocIdSetIterator it = denseLiveDocs.deletedDocsIterator();
+ int count = 0;
+ for (int doc = it.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = it.nextDoc()) {
+ count++;
+ }
+ return count;
+ }
+
+ /**
+ * Benchmarks full iteration over live (non-deleted) documents for {@link SparseLiveDocs}.
+ *
+ * <p>Tests iteration over all live documents in the segment. Performance depends on deletion
+ * pattern:
+ *
+ * <ul>
+ * <li>CLUSTERED: Excellent (3-5× faster than dense)
+ * <li>UNIFORM: Good (2-4× faster than dense)
+ * <li>RANDOM: Variable (4× faster at 0.1%, but can be 2.4× SLOWER at 30%)
+ * </ul>
+ *
+ * @param metrics JMH auxiliary counters for memory statistics
+ * @return number of live documents (for verification)
+ * @throws IOException if iteration fails
+ */
+ @Benchmark
+ public int sparseIterateLiveDocs(final LiveDocsMetrics metrics) throws IOException {
+ fillMetrics(metrics);
+
+ DocIdSetIterator it = sparseLiveDocs.liveDocsIterator();
+ int count = 0;
+ for (int doc = it.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = it.nextDoc()) {
+ count++;
+ }
+ return count;
+ }
+
+ /**
+ * Benchmarks full iteration over live (non-deleted) documents for {@link DenseLiveDocs}.
+ *
+ * <p>Dense implementation provides consistent, predictable performance regardless of deletion
+ * pattern or rate.
+ *
+ * @param metrics JMH auxiliary counters for memory statistics
+ * @return number of live documents (for verification)
+ * @throws IOException if iteration fails
+ */
+ @Benchmark
+ public int denseIterateLiveDocs(final LiveDocsMetrics metrics) throws IOException {
+ fillMetrics(metrics);
+
+ DocIdSetIterator it = denseLiveDocs.liveDocsIterator();
+ int count = 0;
+ for (int doc = it.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = it.nextDoc()) {
+ count++;
+ }
+ return count;
+ }
+
+ /**
+ * Benchmarks range iteration over live documents for {@link SparseLiveDocs}.
+ *
+ * <p>Tests iteration over live documents in a specific range (from maxDoc/4 to maxDoc/2),
+ * simulating range queries.
+ *
+ * @param metrics JMH auxiliary counters for memory statistics
+ * @return number of live documents in range (for verification)
+ * @throws IOException if iteration fails
+ */
+ @Benchmark
+ public int sparseIterateLiveDocsRange(final LiveDocsMetrics metrics) throws IOException {
+ fillMetrics(metrics);
+
+ int rangeStart = maxDoc / 4;
+ int rangeEnd = maxDoc / 2;
+ DocIdSetIterator it = sparseLiveDocs.liveDocsIterator();
+ int count = 0;
+ int doc = it.advance(rangeStart);
+ while (doc < rangeEnd) {
+ count++;
+ doc = it.nextDoc();
+ }
+ return count;
+ }
+
+ /**
+ * Benchmarks range iteration over live documents for {@link DenseLiveDocs}.
+ *
+ * <p>Tests iteration over live documents in a specific range (from maxDoc/4 to maxDoc/2),
+ * simulating range queries.
+ *
+ * @param metrics JMH auxiliary counters for memory statistics
+ * @return number of live documents in range (for verification)
+ * @throws IOException if iteration fails
+ */
+ @Benchmark
+ public int denseIterateLiveDocsRange(final LiveDocsMetrics metrics) throws IOException {
+ fillMetrics(metrics);
+
+ int rangeStart = maxDoc / 4;
+ int rangeEnd = maxDoc / 2;
+ DocIdSetIterator it = denseLiveDocs.liveDocsIterator();
+ int count = 0;
+ int doc = it.advance(rangeStart);
+ while (doc < rangeEnd) {
+ count++;
+ doc = it.nextDoc();
+ }
+ return count;
+ }
+
+ private void fillMetrics(final LiveDocsMetrics metrics) {
+ metrics.deleted = deleted;
+ metrics.sparseBytes = sparseBytes;
+ metrics.denseBytes = denseBytes;
+ metrics.overheadPct = overheadPct;
+ }
+}
diff --git a/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/LiveDocsPathologicalBenchmark.java b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/LiveDocsPathologicalBenchmark.java
new file mode 100644
index 00000000000..eb09f234f75
--- /dev/null
+++ b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/LiveDocsPathologicalBenchmark.java
@@ -0,0 +1,209 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.benchmark.jmh;
+
+import java.io.IOException;
+import java.util.concurrent.TimeUnit;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.util.DenseLiveDocs;
+import org.apache.lucene.util.FixedBitSet;
+import org.apache.lucene.util.SparseFixedBitSet;
+import org.apache.lucene.util.SparseLiveDocs;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+
+/**
+ * Benchmarks the worst-case (pathological) deletion pattern for {@link SparseLiveDocs}.
+ *
+ * <p>This benchmark tests the adversarial case that maximizes SparseFixedBitSet memory usage while
+ * maintaining a low deletion rate. The pattern is designed to force maximum memory allocation in
+ * the sparse structure.
+ *
+ * <h2>Pathological Deletion Pattern</h2>
+ *
+ * <p>The benchmark deletes exactly 64 documents per 4096-bit block, with one deletion per 64-bit
+ * long, spread across all blocks. This pattern:
+ *
+ * <ul>
+ * <li>Forces allocation of ALL blocks (because each block has at least one deletion)
+ * <li>Forces allocation of ALL 64 longs within each block (one deletion per long)
+ * <li>Results in ~1.56% deletion rate (64 deleted per 4096 docs per block)
+ * <li>Maximizes memory overhead while keeping deletion rate low
+ * </ul>
+ *
+ * <h2>Why This Pattern Is Pathological</h2>
+ *
+ * <p>SparseFixedBitSet uses a two-level sparse structure:
+ *
+ * <pre>
+ * Level 1: 4096-bit blocks (only allocated if block has deletions)
+ * Level 2: Within each block, 64 longs (only allocated if long has deletions)
+ * </pre>
+ *
+ * <p>This pattern defeats both levels of sparsity:
+ *
+ * <ul>
+ * <li>All blocks must be allocated (defeats level 1 sparsity)
+ * <li>All longs within blocks must be allocated (defeats level 2 sparsity)
+ * <li>Results in maximum possible memory overhead for ~1.56% deletion rate
+ * <li>May actually use MORE memory than dense representation
+ * </ul>
+ *
+ * <h2>Real-World Relevance</h2>
+ *
+ * <p>While highly unlikely in practice, this pattern could theoretically occur with:
+ *
+ * <ul>
+ * <li>Adversarial deletion patterns designed to defeat sparse structures
+ * <li>Certain systematic deletion patterns based on document ID modulo operations
+ * <li>Worst-case hash-based deletion patterns
+ * </ul>
+ *
+ * <h2>Usage</h2>
+ *
+ * <p>Run the pathological benchmark:
+ *
+ * <pre>
+ * java -jar lucene-benchmark-jmh.jar "LiveDocsPathologicalBenchmark"
+ * </pre>
+ *
+ * <p>Test specific segment size:
+ *
+ * <pre>
+ * java -jar lucene-benchmark-jmh.jar "LiveDocsPathologicalBenchmark" -p maxDoc=10000000
+ * </pre>
+ *
+ * @see SparseLiveDocs
+ * @see DenseLiveDocs
+ * @see LiveDocsBenchmark
+ */
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.MICROSECONDS)
+@State(Scope.Benchmark)
+@Warmup(iterations = 3, time = 2)
+@Measurement(iterations = 5, time = 3)
+@Fork(
+ value = 1,
+ jvmArgsAppend = {"-Xmx2g", "-Xms2g"})
+public class LiveDocsPathologicalBenchmark {
+
+ /**
+ * Number of documents in the segment. Tests across a wide range of segment sizes to evaluate how
+ * the pathological pattern scales from small (100K) to very large (100M) segments.
+ */
+ @Param({"100000", "500000", "1000000", "5000000", "10000000", "50000000", "100000000"})
+ int maxDoc;
+
+ /** Sparse LiveDocs implementation under test with pathological deletion pattern. */
+ private SparseLiveDocs sparseLiveDocs;
+
+ /** Dense LiveDocs implementation for comparison (unaffected by deletion patterns). */
+ private DenseLiveDocs denseLiveDocs;
+
+ /**
+ * Sets up the benchmark by creating both sparse and dense LiveDocs with the pathological deletion
+ * pattern.
+ *
+ * <p>Applies worst-case deletion pattern: deletes 64 documents per 4096-bit block (one deletion
+ * per 64-bit long), spread across all blocks. This maximizes memory allocation in the sparse
+ * structure while maintaining ~1.56% deletion rate.
+ */
+ @Setup(Level.Trial)
+ public void setup() {
+ SparseFixedBitSet sparseSet = new SparseFixedBitSet(maxDoc);
+ FixedBitSet fixedSet = new FixedBitSet(maxDoc);
+ fixedSet.set(0, maxDoc); // All docs initially live
+
+ // Apply worst-case deletion pattern:
+ // Delete 64 docs per block (one per long), spread across all blocks
+ // This maximizes both number of blocks AND number of longs per block
+ // SparseFixedBitSet has 4096-bit blocks with 64 longs per block
+ final int blockSize = 4096;
+ final int longsPerBlock = 64;
+ final int bitsPerLong = 64;
+
+ for (int blockIdx = 0; blockIdx * blockSize < maxDoc; blockIdx++) {
+ // Within each block, delete one document per long (spread across all 64 longs)
+ for (int longIdx = 0; longIdx < longsPerBlock; longIdx++) {
+ int docId = blockIdx * blockSize + longIdx * bitsPerLong;
+ if (docId < maxDoc) {
+ sparseSet.set(docId);
+ fixedSet.clear(docId);
+ }
+ }
+ }
+
+ sparseLiveDocs = SparseLiveDocs.builder(sparseSet, maxDoc).build();
+ denseLiveDocs = DenseLiveDocs.builder(fixedSet, maxDoc).build();
+ }
+
+ /**
+ * Benchmarks iteration over deleted documents for {@link SparseLiveDocs} with pathological
+ * pattern.
+ *
+ * <p>With the pathological deletion pattern (~1.56% deletion rate), sparse is forced to allocate
+ * all blocks and all longs within blocks, defeating both levels of sparsity. This results in:
+ *
+ * <ul>
+ * <li>Maximum memory overhead (may exceed dense representation)
+ * <li>Additional indirection overhead during iteration
+ * <li>No benefit from sparse structure
+ * </ul>
+ *
+ * @return number of deleted documents (for verification)
+ * @throws IOException if iteration fails
+ */
+ @Benchmark
+ public int sparseIterateDeleted() throws IOException {
+ DocIdSetIterator it = sparseLiveDocs.deletedDocsIterator();
+ int count = 0;
+ for (int doc = it.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = it.nextDoc()) {
+ count++;
+ }
+ return count;
+ }
+
+ /**
+ * Benchmarks iteration over deleted documents for {@link DenseLiveDocs} with pathological
+ * pattern.
+ *
+ * <p>Dense implementation is unaffected by deletion patterns. Provides consistent baseline
+ * performance for comparison.
+ *
+ * @return number of deleted documents (for verification)
+ * @throws IOException if iteration fails
+ */
+ @Benchmark
+ public int denseIterateDeleted() throws IOException {
+ DocIdSetIterator it = denseLiveDocs.deletedDocsIterator();
+ int count = 0;
+ for (int doc = it.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = it.nextDoc()) {
+ count++;
+ }
+ return count;
+ }
+}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90LiveDocsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90LiveDocsFormat.java
index 5f31a3ee6bd..c375739d822 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90LiveDocsFormat.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90LiveDocsFormat.java
@@ -30,7 +30,10 @@ import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.DenseLiveDocs;
import org.apache.lucene.util.FixedBitSet;
+import org.apache.lucene.util.SparseFixedBitSet;
+import org.apache.lucene.util.SparseLiveDocs;
/**
* Lucene 9.0 live docs format
@@ -59,6 +62,12 @@ public final class Lucene90LiveDocsFormat extends LiveDocsFormat {
private static final int VERSION_CURRENT = VERSION_START;
+ /**
+ * Deletion rate threshold for choosing sparse vs dense representation. If deletion rate is at or
+ * below this threshold, use SparseFixedBitSet; otherwise use dense FixedBitSet.
+ */
+ private static final double SPARSE_DENSE_THRESHOLD = 0.01; // 1%
+
/** Sole constructor. */
public Lucene90LiveDocsFormat() {}
@@ -67,7 +76,10 @@ public final class Lucene90LiveDocsFormat extends LiveDocsFormat {
throws IOException {
long gen = info.getDelGen();
String name = IndexFileNames.fileNameFromGeneration(info.info.name, EXTENSION, gen);
- final int length = info.info.maxDoc();
+ final int maxDoc = info.info.maxDoc();
+ final int delCount = info.getDelCount();
+ final double deletionRate = (double) delCount / maxDoc;
+
try (ChecksumIndexInput input = dir.openChecksumInput(name)) {
Throwable priorE = null;
try {
@@ -79,17 +91,7 @@ public final class Lucene90LiveDocsFormat extends LiveDocsFormat {
info.info.getId(),
Long.toString(gen, Character.MAX_RADIX));
- FixedBitSet fbs = readFixedBitSet(input, length);
-
- if (fbs.length() - fbs.cardinality() != info.getDelCount()) {
- throw new CorruptIndexException(
- "bits.deleted="
- + (fbs.length() - fbs.cardinality())
- + " info.delcount="
- + info.getDelCount(),
- input);
- }
- return fbs.asReadOnlyBits();
+ return readLiveDocs(input, maxDoc, deletionRate, delCount);
} catch (Throwable exception) {
priorE = exception;
} finally {
@@ -99,12 +101,66 @@ public final class Lucene90LiveDocsFormat extends LiveDocsFormat {
throw new AssertionError();
}
+ /**
+ * Reads live docs from input and chooses between sparse and dense representation based on
+ * deletion rate.
+ */
+ private Bits readLiveDocs(IndexInput input, int maxDoc, double deletionRate, int expectedDelCount)
+ throws IOException {
+ Bits liveDocs;
+ int actualDelCount;
+
+ if (deletionRate <= SPARSE_DENSE_THRESHOLD) {
+ SparseFixedBitSet sparse = readSparseFixedBitSet(input, maxDoc);
+ actualDelCount = sparse.cardinality();
+ liveDocs = SparseLiveDocs.builder(sparse, maxDoc).withDeletedCount(actualDelCount).build();
+ } else {
+ FixedBitSet dense = readFixedBitSet(input, maxDoc);
+ actualDelCount = maxDoc - dense.cardinality();
+ liveDocs = DenseLiveDocs.builder(dense, maxDoc).withDeletedCount(actualDelCount).build();
+ }
+
+ if (actualDelCount != expectedDelCount) {
+ throw new CorruptIndexException(
+ "bits.deleted=" + actualDelCount + " info.delcount=" + expectedDelCount, input);
+ }
+
+ return liveDocs;
+ }
+
private FixedBitSet readFixedBitSet(IndexInput input, int length) throws IOException {
long[] data = new long[FixedBitSet.bits2words(length)];
input.readLongs(data, 0, data.length);
return new FixedBitSet(data, length);
}
+ private SparseFixedBitSet readSparseFixedBitSet(IndexInput input, int length) throws IOException {
+ long[] data = new long[FixedBitSet.bits2words(length)];
+ input.readLongs(data, 0, data.length);
+
+ SparseFixedBitSet sparse = new SparseFixedBitSet(length);
+ for (int wordIndex = 0; wordIndex < data.length; wordIndex++) {
+ long word = data[wordIndex];
+ // Semantic inversion: disk format stores LIVE docs (bit=1 means live, bit=0 means deleted)
+ // but SparseLiveDocs stores DELETED docs (bit=1 means deleted).
+ // Skip words with all bits set (all docs live in disk format = no deletions to convert)
+ if (word == -1L) {
+ continue;
+ }
+ int baseDocId = wordIndex << 6;
+ int maxDocInWord = Math.min(baseDocId + 64, length);
+ for (int docId = baseDocId; docId < maxDocInWord; docId++) {
+ int bitIndex = docId & 63;
+ // If bit is 0 in disk format (deleted doc), set it in sparse representation (bit=1 means
+ // deleted)
+ if ((word & (1L << bitIndex)) == 0) {
+ sparse.set(docId);
+ }
+ }
+ }
+ return sparse;
+ }
+
@Override
public void writeLiveDocs(
Bits bits, Directory dir, SegmentCommitInfo info, int newDelCount, IOContext context)
diff --git a/lucene/core/src/java/org/apache/lucene/index/LeafReader.java b/lucene/core/src/java/org/apache/lucene/index/LeafReader.java
index 39ea90e182e..a608bdbe60d 100644
--- a/lucene/core/src/java/org/apache/lucene/index/LeafReader.java
+++ b/lucene/core/src/java/org/apache/lucene/index/LeafReader.java
@@ -25,6 +25,7 @@ import org.apache.lucene.search.TopDocsCollector;
import org.apache.lucene.search.TopKnnCollector;
import org.apache.lucene.search.TotalHits;
import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.LiveDocs;
/**
* {@code LeafReader} is an abstract class, providing an interface for accessing an index. Search of
@@ -392,6 +393,24 @@ public abstract non-sealed class LeafReader extends IndexReader {
*/
public abstract Bits getLiveDocs();
+ /**
+ * Returns a {@link LiveDocs} view with efficient iteration support, or null if not available.
+ * When non-null, provides O(deletedDocs) iteration via {@link LiveDocs#deletedDocsIterator()}
+ * instead of O(maxDoc) scanning. Callers should fall back to {@link #getLiveDocs()} when null.
+ *
+ * <p>This method enables consumers to opt into efficient deleted document iteration when the
+ * underlying implementation supports it (e.g., SparseLiveDocs or DenseLiveDocs).
+ *
+ * <p>Default implementation returns the {@link LiveDocs} instance if {@link #getLiveDocs()}
+ * implements that interface.
+ *
+ * @lucene.experimental
+ */
+ public LiveDocs getLiveDocsWithDeletedIterator() {
+ Bits bits = getLiveDocs();
+ return bits instanceof LiveDocs ? (LiveDocs) bits : null;
+ }
+
/**
* Returns the {@link PointValues} used for numeric or spatial searches for the given field, or
* null if there are no point fields.
diff --git a/lucene/core/src/java/org/apache/lucene/search/ScorerUtil.java b/lucene/core/src/java/org/apache/lucene/search/ScorerUtil.java
index 457a2cbeefe..40eec686834 100644
--- a/lucene/core/src/java/org/apache/lucene/search/ScorerUtil.java
+++ b/lucene/core/src/java/org/apache/lucene/search/ScorerUtil.java
@@ -93,6 +93,9 @@ class ScorerUtil {
return acceptDocs;
} else if (acceptDocs.getClass() == DEFAULT_ACCEPT_DOCS_CLASS) {
return acceptDocs;
+ } else if (acceptDocs instanceof org.apache.lucene.util.SparseLiveDocs
+ || acceptDocs instanceof org.apache.lucene.util.DenseLiveDocs) {
+ return acceptDocs;
} else {
return new FilterBits(acceptDocs);
}
diff --git a/lucene/core/src/java/org/apache/lucene/util/DenseLiveDocs.java b/lucene/core/src/java/org/apache/lucene/util/DenseLiveDocs.java
new file mode 100644
index 00000000000..133469fc8bb
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/util/DenseLiveDocs.java
@@ -0,0 +1,166 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.util;
+
+import java.util.Locale;
+import org.apache.lucene.search.DocIdSetIterator;
+
+/**
+ * {@link LiveDocs} implementation optimized for dense deletions.
+ *
+ * <p>This implementation stores LIVE documents using {@link FixedBitSet}, which is the traditional
+ * approach used by Lucene. This provides:
+ *
+ * <ul>
+ * <li>O(1) random access via {@link #get(int)}
+ * <li>Memory usage proportional to maxDoc
+ * <li>Efficient iteration over live documents
+ * </ul>
+ *
+ * <p>This is most efficient when deletions are dense. For sparser deletions, {@link SparseLiveDocs}
+ * should be used instead.
+ *
+ * <p><b>Standard semantics:</b> Set bits represent LIVE documents, which is the traditional Lucene
+ * approach:
+ *
+ * <ul>
+ * <li>{@link #get(int)} returns {@code true} if doc is LIVE (bit IS set in liveDocs)
+ * <li>{@link #deletedDocsIterator()} iterates documents where bit is NOT set in liveDocs
+ * </ul>
+ *
+ * <p><b>Immutability:</b> This class is immutable once constructed. Instances are typically created
+ * using the {@link Builder} via {@link #builder(FixedBitSet, int)}.
+ *
+ * @lucene.experimental
+ */
+public final class DenseLiveDocs implements LiveDocs {
+
+ private final FixedBitSet liveDocs;
+ private final int maxDoc;
+ // Cached at construction for performance. Safe because this class is immutable.
+ // Eliminates repeated cardinality() and subtraction operations.
+ private final int deletedCount;
+
+ /**
+ * Creates a builder for constructing DenseLiveDocs instances.
+ *
+ * @param liveDocs bit set where set bits represent LIVE documents
+ * @param maxDoc the maximum document ID (exclusive)
+ * @return a new builder instance
+ */
+ public static Builder builder(FixedBitSet liveDocs, int maxDoc) {
+ return new Builder(liveDocs, maxDoc);
+ }
+
+ /** Builder for creating DenseLiveDocs instances with optional pre-computed deleted count. */
+ public static final class Builder {
+ private final FixedBitSet liveDocs;
+ private final int maxDoc;
+ private Integer deletedCount;
+
+ private Builder(FixedBitSet liveDocs, int maxDoc) {
+ this.liveDocs = liveDocs;
+ this.maxDoc = maxDoc;
+ }
+
+ /**
+ * Sets the pre-computed deleted document count, avoiding cardinality computation.
+ *
+ * @param deletedCount the number of deleted documents
+ * @return this builder
+ */
+ public Builder withDeletedCount(int deletedCount) {
+ this.deletedCount = deletedCount;
+ return this;
+ }
+
+ /**
+ * Builds the DenseLiveDocs instance.
+ *
+ * @return a new DenseLiveDocs instance
+ * @throws IllegalArgumentException if deletedCount is outside valid range [0, maxDoc]
+ */
+ public DenseLiveDocs build() {
+ int count = deletedCount != null ? deletedCount : (maxDoc - liveDocs.cardinality());
+
+ if (count < 0 || count > maxDoc) {
+ throw new IllegalArgumentException(
+ "deletedCount=" + count + " is outside valid range [0, " + maxDoc + "]");
+ }
+
+ assert count == (maxDoc - liveDocs.cardinality())
+ : "deletedCount="
+ + count
+ + " does not match maxDoc - liveDocs.cardinality()="
+ + (maxDoc - liveDocs.cardinality());
+
+ return new DenseLiveDocs(liveDocs, maxDoc, count);
+ }
+ }
+
+ private DenseLiveDocs(final FixedBitSet liveDocs, int maxDoc, int deletedCount) {
+ assert liveDocs.length() >= maxDoc;
+ this.maxDoc = maxDoc;
+ this.liveDocs = liveDocs;
+ this.deletedCount = deletedCount;
+ }
+
+ @Override
+ public boolean get(int index) {
+ return liveDocs.get(index);
+ }
+
+ @Override
+ public int length() {
+ return maxDoc;
+ }
+
+ @Override
+ public DocIdSetIterator liveDocsIterator() {
+ return new BitSetIterator(liveDocs, maxDoc - deletedCount);
+ }
+
+ @Override
+ public DocIdSetIterator deletedDocsIterator() {
+ return new FilteredDocIdSetIterator(maxDoc, deletedCount, doc -> !liveDocs.get(doc));
+ }
+
+ @Override
+ public int deletedCount() {
+ return deletedCount;
+ }
+
+ /**
+ * Returns the memory usage in bytes.
+ *
+ * @return estimated memory usage in bytes
+ */
+ public long ramBytesUsed() {
+ return liveDocs.ramBytesUsed();
+ }
+
+ @Override
+ public String toString() {
+ return "DenseLiveDocs(maxDoc="
+ + maxDoc
+ + ", deleted="
+ + deletedCount()
+ + ", deletionRate="
+ + String.format(Locale.ROOT, "%.2f%%", 100.0 * deletedCount() / maxDoc)
+ + ")";
+ }
+}
diff --git a/lucene/core/src/java/org/apache/lucene/util/FilteredDocIdSetIterator.java b/lucene/core/src/java/org/apache/lucene/util/FilteredDocIdSetIterator.java
new file mode 100644
index 00000000000..bfac799ac26
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/util/FilteredDocIdSetIterator.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.util;
+
+import java.util.function.IntPredicate;
+import org.apache.lucene.search.DocIdSetIterator;
+
+/**
+ * {@link DocIdSetIterator} that returns documents matching a predicate by scanning all document
+ * positions.
+ *
+ * <p>This generic iterator can be used to iterate either live or deleted documents based on a
+ * predicate function. Iteration has O(maxDoc) complexity since it must scan all document positions.
+ *
+ * @lucene.internal
+ */
+final class FilteredDocIdSetIterator extends DocIdSetIterator {
+ private final int maxDoc;
+ private final int cost;
+ private final IntPredicate predicate;
+ private int doc = -1;
+
+ /**
+ * Creates a filtered iterator over documents.
+ *
+ * @param maxDoc the maximum document ID (exclusive)
+ * @param cost the expected number of documents to return
+ * @param predicate predicate to test each document ID; returns {@code true} if document should be
+ * included in iteration
+ */
+ FilteredDocIdSetIterator(int maxDoc, int cost, IntPredicate predicate) {
+ this.maxDoc = maxDoc;
+ this.cost = cost;
+ this.predicate = predicate;
+ }
+
+ @Override
+ public int docID() {
+ return doc;
+ }
+
+ @Override
+ public int nextDoc() {
+ return advance(doc + 1);
+ }
+
+ @Override
+ public int advance(int target) {
+ if (target >= maxDoc) {
+ doc = NO_MORE_DOCS;
+ return doc;
+ }
+
+ doc = target;
+ while (doc < maxDoc) {
+ if (predicate.test(doc)) {
+ return doc;
+ }
+ doc++;
+ }
+ doc = NO_MORE_DOCS;
+ return doc;
+ }
+
+ @Override
+ public long cost() {
+ return cost;
+ }
+}
diff --git a/lucene/core/src/java/org/apache/lucene/util/LiveDocs.java b/lucene/core/src/java/org/apache/lucene/util/LiveDocs.java
new file mode 100644
index 00000000000..29ebb91ee92
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/util/LiveDocs.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.util;
+
+import org.apache.lucene.search.DocIdSetIterator;
+
+/**
+ * Extension of {@link Bits} that provides efficient iteration over deleted documents.
+ *
+ * <p>This interface enables efficient sequential access to deleted documents, which is beneficial
+ * when deletions are sparse (typically for low deletion rates). For sparse deletions, iterating
+ * deleted documents directly in O(deletedDocs) time is much more efficient than scanning all
+ * documents in O(maxDoc) time.
+ *
+ * <p>Implementations should provide optimal iteration strategies based on deletion density:
+ *
+ * <ul>
+ * <li>Sparse deletions: Use {@link SparseFixedBitSet} to store deleted docs, enabling
+ * O(deletedDocs) iteration
+ * <li>Dense deletions: Use {@link FixedBitSet} to store live docs, maintaining current behavior
+ * </ul>
+ *
+ * @lucene.experimental
+ */
+public interface LiveDocs extends Bits {
+
+ /**
+ * Returns an iterator over live document IDs.
+ *
+ * <p>The returned iterator provides sequential access to all live documents in ascending order.
+ * The iteration complexity depends on the implementation:
+ *
+ * <ul>
+ * <li>For sparse deletions: O(maxDoc) - may need to scan all documents
+ * <li>For dense deletions: O(liveDocs) - only visits live documents
+ * </ul>
+ *
+ * <p>Callers can use {@link DocIdSetIterator#cost()} to determine the expected number of live
+ * documents.
+ *
+ * @return an iterator over live document IDs
+ */
+ DocIdSetIterator liveDocsIterator();
+
+ /**
+ * Returns an iterator over deleted document IDs.
+ *
+ * <p>The returned iterator provides sequential access to all deleted documents in ascending
+ * order. The iteration complexity depends on the implementation:
+ *
+ * <ul>
+ * <li>For sparse deletions: O(deletedDocs) - only visits deleted documents
+ * <li>For dense deletions: O(maxDoc) - may need to scan all documents
+ * </ul>
+ *
+ * <p>Callers can use {@link DocIdSetIterator#cost()} to determine if sparse iteration would be
+ * beneficial for their use case.
+ *
+ * @return an iterator over deleted document IDs, or an empty iterator if no documents are deleted
+ */
+ DocIdSetIterator deletedDocsIterator();
+
+ /**
+ * Returns the number of deleted documents.
+ *
+ * <p>This can be used to determine deletion density and choose appropriate algorithms.
+ *
+ * @return the number of deleted documents in this segment
+ */
+ int deletedCount();
+}
diff --git a/lucene/core/src/java/org/apache/lucene/util/SparseLiveDocs.java b/lucene/core/src/java/org/apache/lucene/util/SparseLiveDocs.java
new file mode 100644
index 00000000000..4220de9d51b
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/util/SparseLiveDocs.java
@@ -0,0 +1,166 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.util;
+
+import java.util.Locale;
+import org.apache.lucene.search.DocIdSetIterator;
+
+/**
+ * {@link LiveDocs} implementation optimized for sparse deletions.
+ *
+ * <p>This implementation stores DELETED documents using {@link SparseFixedBitSet}, which provides:
+ *
+ * <ul>
+ * <li>O(1) random access via {@link #get(int)}
+ * <li>O(deletedDocs) iteration via {@link #deletedDocsIterator()}
+ * <li>Memory usage proportional to number of deleted documents, not total documents
+ * </ul>
+ *
+ * <p>This is most efficient when deletions are sparse. For denser deletions, {@link DenseLiveDocs}
+ * should be used instead.
+ *
+ * <p><b>Inverted semantics:</b> Unlike typical live docs that store which documents are live, this
+ * stores which documents are DELETED. Therefore:
+ *
+ * <ul>
+ * <li>{@link #get(int)} returns {@code true} if doc is LIVE (bit is NOT set in deletedDocs)
+ * <li>{@link #deletedDocsIterator()} iterates documents where bit IS set in deletedDocs
+ * </ul>
+ *
+ * <p><b>Immutability:</b> This class is immutable once constructed. Instances are typically created
+ * using the {@link Builder} via {@link #builder(SparseFixedBitSet, int)}.
+ *
+ * @lucene.experimental
+ */
+public final class SparseLiveDocs implements LiveDocs {
+
+ private final SparseFixedBitSet deletedDocs;
+ private final int maxDoc;
+ // Cached at construction for performance. Safe because this class is immutable.
+ // Eliminates repeated O(n) cardinality() calls.
+ private final int deletedCount;
+
+ /**
+ * Creates a builder for constructing SparseLiveDocs instances.
+ *
+ * @param deletedDocs bit set where set bits represent DELETED documents
+ * @param maxDoc the maximum document ID (exclusive)
+ * @return a new builder instance
+ */
+ public static Builder builder(SparseFixedBitSet deletedDocs, int maxDoc) {
+ return new Builder(deletedDocs, maxDoc);
+ }
+
+ /** Builder for creating SparseLiveDocs instances with optional pre-computed deleted count. */
+ public static final class Builder {
+ private final SparseFixedBitSet deletedDocs;
+ private final int maxDoc;
+ private Integer deletedCount;
+
+ private Builder(SparseFixedBitSet deletedDocs, int maxDoc) {
+ this.deletedDocs = deletedDocs;
+ this.maxDoc = maxDoc;
+ }
+
+ /**
+ * Sets the pre-computed deleted document count, avoiding cardinality computation.
+ *
+ * @param deletedCount the number of deleted documents
+ * @return this builder
+ */
+ public Builder withDeletedCount(int deletedCount) {
+ this.deletedCount = deletedCount;
+ return this;
+ }
+
+ /**
+ * Builds the SparseLiveDocs instance.
+ *
+ * @return a new SparseLiveDocs instance
+ * @throws IllegalArgumentException if deletedCount is outside valid range [0, maxDoc]
+ */
+ public SparseLiveDocs build() {
+ int count = deletedCount != null ? deletedCount : deletedDocs.cardinality();
+
+ if (count < 0 || count > maxDoc) {
+ throw new IllegalArgumentException(
+ "deletedCount=" + count + " is outside valid range [0, " + maxDoc + "]");
+ }
+
+ assert count == deletedDocs.cardinality()
+ : "deletedCount="
+ + count
+ + " does not match deletedDocs.cardinality()="
+ + deletedDocs.cardinality();
+
+ return new SparseLiveDocs(deletedDocs, maxDoc, count);
+ }
+ }
+
+ private SparseLiveDocs(final SparseFixedBitSet deletedDocs, int maxDoc, int deletedCount) {
+ assert deletedDocs.length >= maxDoc;
+ this.maxDoc = maxDoc;
+ this.deletedDocs = deletedDocs;
+ this.deletedCount = deletedCount;
+ }
+
+ @Override
+ public boolean get(int index) {
+ return !deletedDocs.get(index);
+ }
+
+ @Override
+ public int length() {
+ return maxDoc;
+ }
+
+ @Override
+ public DocIdSetIterator liveDocsIterator() {
+ return new FilteredDocIdSetIterator(
+ maxDoc, maxDoc - deletedCount, doc -> !deletedDocs.get(doc));
+ }
+
+ @Override
+ public DocIdSetIterator deletedDocsIterator() {
+ return new BitSetIterator(deletedDocs, deletedCount);
+ }
+
+ @Override
+ public int deletedCount() {
+ return deletedCount;
+ }
+
+ /**
+ * Returns the memory usage in bytes.
+ *
+ * @return estimated memory usage in bytes
+ */
+ public long ramBytesUsed() {
+ return deletedDocs.ramBytesUsed();
+ }
+
+ @Override
+ public String toString() {
+ return "SparseLiveDocs(maxDoc="
+ + maxDoc
+ + ", deleted="
+ + deletedCount()
+ + ", deletionRate="
+ + String.format(Locale.ROOT, "%.2f%%", 100.0 * deletedCount() / maxDoc)
+ + ")";
+ }
+}
diff --git a/lucene/core/src/test/org/apache/lucene/util/TestLiveDocs.java b/lucene/core/src/test/org/apache/lucene/util/TestLiveDocs.java
new file mode 100644
index 00000000000..49767de9d76
--- /dev/null
+++ b/lucene/core/src/test/org/apache/lucene/util/TestLiveDocs.java
@@ -0,0 +1,599 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.util;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.tests.util.LuceneTestCase;
+import org.apache.lucene.tests.util.TestUtil;
+
+public class TestLiveDocs extends LuceneTestCase {
+
+ public void testSparseLiveDocsBasic() {
+ // GIVEN
+ final int maxDoc = 1000;
+ SparseFixedBitSet sparseSet = new SparseFixedBitSet(maxDoc);
+ sparseSet.set(10);
+ sparseSet.set(50);
+ sparseSet.set(100);
+
+ // WHEN
+ SparseLiveDocs liveDocs = SparseLiveDocs.builder(sparseSet, maxDoc).build();
+
+ // THEN
+ assertEquals(3, liveDocs.deletedCount());
+ assertFalse("Doc 10 should be deleted", liveDocs.get(10));
+ assertFalse("Doc 50 should be deleted", liveDocs.get(50));
+ assertFalse("Doc 100 should be deleted", liveDocs.get(100));
+ assertTrue("Doc 11 should be live", liveDocs.get(11));
+ assertTrue("Doc 51 should be live", liveDocs.get(51));
+ }
+
+ public void testDenseLiveDocsBasic() {
+ // GIVEN
+ final int maxDoc = 1000;
+ FixedBitSet fixedSet = new FixedBitSet(maxDoc);
+ fixedSet.set(0, maxDoc);
+ fixedSet.clear(10);
+ fixedSet.clear(50);
+ fixedSet.clear(100);
+
+ // WHEN
+ DenseLiveDocs liveDocs = DenseLiveDocs.builder(fixedSet, maxDoc).build();
+
+ // THEN
+ assertEquals(3, liveDocs.deletedCount());
+ assertFalse("Doc 10 should be deleted", liveDocs.get(10));
+ assertFalse("Doc 50 should be deleted", liveDocs.get(50));
+ assertFalse("Doc 100 should be deleted", liveDocs.get(100));
+ assertTrue("Doc 11 should be live", liveDocs.get(11));
+ assertTrue("Doc 51 should be live", liveDocs.get(51));
+ }
+
+ public void testSparseLiveDocsIterator() throws IOException {
+ // GIVEN
+ final int maxDoc = 1000;
+ SparseFixedBitSet sparseSet = new SparseFixedBitSet(maxDoc);
+ int[] deletedDocs = {5, 50, 100, 150, 500, 999};
+ for (int doc : deletedDocs) {
+ sparseSet.set(doc);
+ }
+ SparseLiveDocs liveDocs = SparseLiveDocs.builder(sparseSet, maxDoc).build();
+
+ // WHEN
+ DocIdSetIterator it = liveDocs.deletedDocsIterator();
+ List<Integer> iteratedDocs = new ArrayList<>();
+ for (int doc = it.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = it.nextDoc()) {
+ iteratedDocs.add(doc);
+ }
+
+ // THEN
+ assertEquals("Iterator cost should match deleted count", deletedDocs.length, it.cost());
+ assertEquals(
+ "Should iterate exact number of deleted docs", deletedDocs.length, iteratedDocs.size());
+ for (int i = 0; i < deletedDocs.length; i++) {
+ assertEquals(
+ "Deleted doc mismatch at position " + i, deletedDocs[i], iteratedDocs.get(i).intValue());
+ }
+ }
+
+ public void testDenseLiveDocsIterator() throws IOException {
+ // GIVEN
+ final int maxDoc = 1000;
+ FixedBitSet fixedSet = new FixedBitSet(maxDoc);
+ fixedSet.set(0, maxDoc);
+ int[] deletedDocs = {5, 50, 100, 150, 500, 999};
+ for (int doc : deletedDocs) {
+ fixedSet.clear(doc);
+ }
+ DenseLiveDocs liveDocs = DenseLiveDocs.builder(fixedSet, maxDoc).build();
+
+ // WHEN
+ DocIdSetIterator it = liveDocs.deletedDocsIterator();
+ List<Integer> iteratedDocs = new ArrayList<>();
+ for (int doc = it.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = it.nextDoc()) {
+ iteratedDocs.add(doc);
+ }
+
+ // THEN
+ assertEquals("Iterator cost should match deleted count", deletedDocs.length, it.cost());
+ assertEquals(
+ "Should iterate exact number of deleted docs", deletedDocs.length, iteratedDocs.size());
+ for (int i = 0; i < deletedDocs.length; i++) {
+ assertEquals(
+ "Deleted doc mismatch at position " + i, deletedDocs[i], iteratedDocs.get(i).intValue());
+ }
+ }
+
+ public void testSparseDenseEquivalence() throws IOException {
+ // GIVEN
+ final int maxDoc = 1000;
+ SparseFixedBitSet sparseSet = new SparseFixedBitSet(maxDoc);
+ FixedBitSet fixedSet = new FixedBitSet(maxDoc);
+ fixedSet.set(0, maxDoc);
+ int[] deletedDocs = {1, 10, 50, 100, 200, 500, 750, 999};
+ for (int doc : deletedDocs) {
+ sparseSet.set(doc);
+ fixedSet.clear(doc);
+ }
+
+ // WHEN
+ SparseLiveDocs sparse = SparseLiveDocs.builder(sparseSet, maxDoc).build();
+ DenseLiveDocs dense = DenseLiveDocs.builder(fixedSet, maxDoc).build();
+ DocIdSetIterator sparseIt = sparse.deletedDocsIterator();
+ DocIdSetIterator denseIt = dense.deletedDocsIterator();
+
+ // THEN
+ assertEquals("Deleted counts should match", sparse.deletedCount(), dense.deletedCount());
+ assertEquals("Lengths should match", sparse.length(), dense.length());
+ for (int i = 0; i < maxDoc; i++) {
+ assertEquals("get(" + i + ") should match", sparse.get(i), dense.get(i));
+ }
+ int sparseDoc, denseDoc;
+ do {
+ sparseDoc = sparseIt.nextDoc();
+ denseDoc = denseIt.nextDoc();
+ assertEquals("Iterators should return same documents", sparseDoc, denseDoc);
+ } while (sparseDoc != DocIdSetIterator.NO_MORE_DOCS);
+ }
+
+ public void testEmptyIterator() throws IOException {
+ // GIVEN
+ final int maxDoc = 1000;
+ SparseFixedBitSet sparseSet = new SparseFixedBitSet(maxDoc);
+ FixedBitSet fixedSet = new FixedBitSet(maxDoc);
+ fixedSet.set(0, maxDoc);
+ SparseLiveDocs sparse = SparseLiveDocs.builder(sparseSet, maxDoc).build();
+ DenseLiveDocs dense = DenseLiveDocs.builder(fixedSet, maxDoc).build();
+
+ // WHEN
+ DocIdSetIterator sparseIt = sparse.deletedDocsIterator();
+ DocIdSetIterator denseIt = dense.deletedDocsIterator();
+
+ // THEN
+ assertEquals(DocIdSetIterator.NO_MORE_DOCS, sparseIt.nextDoc());
+ assertEquals(DocIdSetIterator.NO_MORE_DOCS, denseIt.nextDoc());
+ }
+
+ public void testIteratorAdvance() throws IOException {
+ // GIVEN
+ final int maxDoc = 1000;
+ FixedBitSet fixedSet = new FixedBitSet(maxDoc);
+ fixedSet.set(0, maxDoc);
+ for (int i = 10; i <= 50; i += 10) {
+ fixedSet.clear(i);
+ }
+ DenseLiveDocs liveDocs = DenseLiveDocs.builder(fixedSet, maxDoc).build();
+
+ // WHEN
+ DocIdSetIterator it = liveDocs.deletedDocsIterator();
+
+ // THEN
+ assertEquals(20, it.advance(15));
+ assertEquals(30, it.nextDoc());
+ assertEquals(50, it.advance(45));
+ assertEquals(DocIdSetIterator.NO_MORE_DOCS, it.advance(1000));
+ }
+
+ public void testRandomDeletions() throws IOException {
+ // GIVEN
+ final int maxDoc = TestUtil.nextInt(random(), 1000, 10000);
+ final double deleteRatio = random().nextDouble() * 0.5;
+ SparseFixedBitSet sparseSet = new SparseFixedBitSet(maxDoc);
+ FixedBitSet fixedSet = new FixedBitSet(maxDoc);
+ fixedSet.set(0, maxDoc);
+ List<Integer> expectedDeleted = new ArrayList<>();
+ for (int i = 0; i < maxDoc; i++) {
+ if (random().nextDouble() < deleteRatio) {
+ sparseSet.set(i);
+ fixedSet.clear(i);
+ expectedDeleted.add(i);
+ }
+ }
+
+ // WHEN
+ SparseLiveDocs sparse = SparseLiveDocs.builder(sparseSet, maxDoc).build();
+ DenseLiveDocs dense = DenseLiveDocs.builder(fixedSet, maxDoc).build();
+ List<Integer> sparseDeleted = collectDocs(sparse.deletedDocsIterator());
+ List<Integer> denseDeleted = collectDocs(dense.deletedDocsIterator());
+
+ // THEN
+ assertEquals(
+ "Deleted counts should match expected", expectedDeleted.size(), sparse.deletedCount());
+ assertEquals(
+ "Sparse and dense counts should match", sparse.deletedCount(), dense.deletedCount());
+ for (int i = 0; i < maxDoc; i++) {
+ boolean expectedLive = !expectedDeleted.contains(i);
+ assertEquals("Sparse get(" + i + ") mismatch", expectedLive, sparse.get(i));
+ assertEquals("Dense get(" + i + ") mismatch", expectedLive, dense.get(i));
+ }
+ assertEquals("Sparse iterator should return all deleted docs", expectedDeleted, sparseDeleted);
+ assertEquals("Dense iterator should return all deleted docs", expectedDeleted, denseDeleted);
+ }
+
+ public void testMemoryUsage() {
+ // GIVEN
+ final int maxDoc = 1000000;
+ SparseFixedBitSet sparseSet = new SparseFixedBitSet(maxDoc);
+ FixedBitSet fixedSet = new FixedBitSet(maxDoc);
+ fixedSet.set(0, maxDoc);
+ for (int i = 0; i < maxDoc / 1000; i++) {
+ sparseSet.set(i * 1000);
+ fixedSet.clear(i * 1000);
+ }
+
+ // WHEN
+ SparseLiveDocs sparse = SparseLiveDocs.builder(sparseSet, maxDoc).build();
+ DenseLiveDocs dense = DenseLiveDocs.builder(fixedSet, maxDoc).build();
+ long sparseBytes = sparse.ramBytesUsed();
+ long denseBytes = dense.ramBytesUsed();
+
+ // THEN
+ assertTrue(
+ "Sparse should use less memory than dense for 0.1% deletions (sparse="
+ + sparseBytes
+ + ", dense="
+ + denseBytes
+ + ", ratio="
+ + String.format(Locale.ROOT, "%.2f%%", 100.0 * sparseBytes / denseBytes)
+ + ")",
+ sparseBytes < denseBytes);
+ assertTrue(
+ "Sparse should use significantly less memory (< 50%) for very sparse deletions (ratio="
+ + String.format(Locale.ROOT, "%.2f%%", 100.0 * sparseBytes / denseBytes)
+ + ")",
+ sparseBytes < denseBytes / 2);
+ }
+
+ public void testWrappingExistingBitSets() {
+ // GIVEN
+ final int maxDoc = 1000;
+ SparseFixedBitSet sparseDeleted = new SparseFixedBitSet(maxDoc);
+ sparseDeleted.set(10);
+ sparseDeleted.set(50);
+ sparseDeleted.set(100);
+ FixedBitSet liveSet = new FixedBitSet(maxDoc);
+ liveSet.set(0, maxDoc);
+ liveSet.clear(10);
+ liveSet.clear(50);
+ liveSet.clear(100);
+
+ // WHEN
+ SparseLiveDocs sparse = SparseLiveDocs.builder(sparseDeleted, maxDoc).build();
+ DenseLiveDocs dense = DenseLiveDocs.builder(liveSet, maxDoc).build();
+
+ // THEN
+ assertEquals(3, sparse.deletedCount());
+ assertFalse(sparse.get(10));
+ assertFalse(sparse.get(50));
+ assertTrue(sparse.get(11));
+ assertEquals(3, dense.deletedCount());
+ assertFalse(dense.get(10));
+ assertFalse(dense.get(50));
+ assertTrue(dense.get(11));
+ }
+
+ public void testSparseLiveDocsLiveIterator() throws IOException {
+ // GIVEN
+ final int maxDoc = 100;
+ SparseFixedBitSet sparseSet = new SparseFixedBitSet(maxDoc);
+ int[] deletedDocs = {5, 10, 50, 99};
+ for (int doc : deletedDocs) {
+ sparseSet.set(doc);
+ }
+ SparseLiveDocs liveDocs = SparseLiveDocs.builder(sparseSet, maxDoc).build();
+
+ // WHEN
+ DocIdSetIterator it = liveDocs.liveDocsIterator();
+ List<Integer> iteratedDocs = collectDocs(it);
+
+ // THEN
+ assertEquals("Iterator cost should match live count", maxDoc - deletedDocs.length, it.cost());
+ assertEquals(
+ "Should iterate exact number of live docs",
+ maxDoc - deletedDocs.length,
+ iteratedDocs.size());
+ for (int deletedDoc : deletedDocs) {
+ assertFalse(
+ "Deleted doc " + deletedDoc + " should not be in live iterator",
+ iteratedDocs.contains(deletedDoc));
+ }
+ for (int i = 1; i < iteratedDocs.size(); i++) {
+ assertTrue(
+ "Docs should be in ascending order", iteratedDocs.get(i) > iteratedDocs.get(i - 1));
+ }
+ }
+
+ public void testDenseLiveDocsLiveIterator() throws IOException {
+ // GIVEN
+ final int maxDoc = 100;
+ FixedBitSet fixedSet = new FixedBitSet(maxDoc);
+ fixedSet.set(0, maxDoc);
+ int[] deletedDocs = {5, 10, 50, 99};
+ for (int doc : deletedDocs) {
+ fixedSet.clear(doc);
+ }
+ DenseLiveDocs liveDocs = DenseLiveDocs.builder(fixedSet, maxDoc).build();
+
+ // WHEN
+ DocIdSetIterator it = liveDocs.liveDocsIterator();
+ List<Integer> iteratedDocs = collectDocs(it);
+
+ // THEN
+ assertEquals("Iterator cost should match live count", maxDoc - deletedDocs.length, it.cost());
+ assertEquals(
+ "Should iterate exact number of live docs",
+ maxDoc - deletedDocs.length,
+ iteratedDocs.size());
+ for (int deletedDoc : deletedDocs) {
+ assertFalse(
+ "Deleted doc " + deletedDoc + " should not be in live iterator",
+ iteratedDocs.contains(deletedDoc));
+ }
+ for (int i = 1; i < iteratedDocs.size(); i++) {
+ assertTrue(
+ "Docs should be in ascending order", iteratedDocs.get(i) > iteratedDocs.get(i - 1));
+ }
+ }
+
+ public void testLiveIteratorEquivalence() throws IOException {
+ // GIVEN
+ final int maxDoc = 1000;
+ SparseFixedBitSet sparseSet = new SparseFixedBitSet(maxDoc);
+ FixedBitSet fixedSet = new FixedBitSet(maxDoc);
+ fixedSet.set(0, maxDoc);
+ int[] deletedDocs = {1, 10, 50, 100, 200, 500, 750, 999};
+ for (int doc : deletedDocs) {
+ sparseSet.set(doc);
+ fixedSet.clear(doc);
+ }
+ SparseLiveDocs sparse = SparseLiveDocs.builder(sparseSet, maxDoc).build();
+ DenseLiveDocs dense = DenseLiveDocs.builder(fixedSet, maxDoc).build();
+
+ // WHEN
+ DocIdSetIterator sparseIt = sparse.liveDocsIterator();
+ DocIdSetIterator denseIt = dense.liveDocsIterator();
+
+ // THEN
+ int sparseDoc, denseDoc;
+ do {
+ sparseDoc = sparseIt.nextDoc();
+ denseDoc = denseIt.nextDoc();
+ assertEquals("Live iterators should return same documents", sparseDoc, denseDoc);
+ } while (sparseDoc != DocIdSetIterator.NO_MORE_DOCS);
+ }
+
+ public void testLiveIteratorFullIteration() throws IOException {
+ // GIVEN
+ final int maxDoc = 1000;
+ SparseFixedBitSet sparseSet = new SparseFixedBitSet(maxDoc);
+ FixedBitSet fixedSet = new FixedBitSet(maxDoc);
+ fixedSet.set(0, maxDoc);
+ SparseLiveDocs sparse = SparseLiveDocs.builder(sparseSet, maxDoc).build();
+ DenseLiveDocs dense = DenseLiveDocs.builder(fixedSet, maxDoc).build();
+
+ // WHEN
+ DocIdSetIterator sparseIt = sparse.liveDocsIterator();
+ DocIdSetIterator denseIt = dense.liveDocsIterator();
+ List<Integer> sparseDocs = collectDocs(sparseIt);
+ List<Integer> denseDocs = collectDocs(denseIt);
+
+ // THEN
+ assertEquals("Sparse iterator should return all docs", maxDoc, sparseDocs.size());
+ assertEquals("Dense iterator should return all docs", maxDoc, denseDocs.size());
+ for (int i = 0; i < maxDoc; i++) {
+ assertEquals("Sparse doc at position " + i, i, sparseDocs.get(i).intValue());
+ assertEquals("Dense doc at position " + i, i, denseDocs.get(i).intValue());
+ }
+ }
+
+ public void testLiveIteratorAdvance() throws IOException {
+ // GIVEN
+ final int maxDoc = 1000;
+ FixedBitSet fixedSet = new FixedBitSet(maxDoc);
+ fixedSet.set(0, maxDoc);
+ for (int i = 10; i <= 50; i += 10) {
+ fixedSet.clear(i);
+ }
+ DenseLiveDocs liveDocs = DenseLiveDocs.builder(fixedSet, maxDoc).build();
+
+ // WHEN
+ DocIdSetIterator it = liveDocs.liveDocsIterator();
+
+ // THEN
+ assertEquals(11, it.advance(10));
+ assertEquals(12, it.nextDoc());
+ assertEquals(21, it.advance(20));
+ assertEquals(500, it.advance(500));
+ assertEquals(DocIdSetIterator.NO_MORE_DOCS, it.advance(1000));
+ }
+
+ public void testRandomDeletionsLiveIterator() throws IOException {
+ // GIVEN
+ final int maxDoc = TestUtil.nextInt(random(), 1000, 10000);
+ final double deleteRatio = random().nextDouble() * 0.5;
+ SparseFixedBitSet sparseSet = new SparseFixedBitSet(maxDoc);
+ FixedBitSet fixedSet = new FixedBitSet(maxDoc);
+ fixedSet.set(0, maxDoc);
+ List<Integer> expectedLive = new ArrayList<>();
+ for (int i = 0; i < maxDoc; i++) {
+ if (random().nextDouble() < deleteRatio) {
+ sparseSet.set(i);
+ fixedSet.clear(i);
+ } else {
+ expectedLive.add(i);
+ }
+ }
+
+ // WHEN
+ SparseLiveDocs sparse = SparseLiveDocs.builder(sparseSet, maxDoc).build();
+ DenseLiveDocs dense = DenseLiveDocs.builder(fixedSet, maxDoc).build();
+ List<Integer> sparseLive = collectDocs(sparse.liveDocsIterator());
+ List<Integer> denseLive = collectDocs(dense.liveDocsIterator());
+
+ // THEN
+ assertEquals("Sparse live iterator should return all live docs", expectedLive, sparseLive);
+ assertEquals("Dense live iterator should return all live docs", expectedLive, denseLive);
+ }
+
+ private List<Integer> collectDocs(DocIdSetIterator it) throws IOException {
+ List<Integer> docs = new ArrayList<>();
+ for (int doc = it.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = it.nextDoc()) {
+ docs.add(doc);
+ }
+ return docs;
+ }
+
+ public void testSingleDocumentSegment() throws IOException {
+ // GIVEN
+ final int maxDoc = 1;
+ SparseFixedBitSet sparseSet = new SparseFixedBitSet(maxDoc);
+ FixedBitSet fixedSet = new FixedBitSet(maxDoc);
+ fixedSet.set(0, maxDoc);
+ sparseSet.set(0);
+ fixedSet.clear(0);
+
+ // WHEN
+ SparseLiveDocs sparse = SparseLiveDocs.builder(sparseSet, maxDoc).build();
+ DenseLiveDocs dense = DenseLiveDocs.builder(fixedSet, maxDoc).build();
+ List<Integer> sparseDeleted = collectDocs(sparse.deletedDocsIterator());
+ List<Integer> denseDeleted = collectDocs(dense.deletedDocsIterator());
+ List<Integer> sparseLive = collectDocs(sparse.liveDocsIterator());
+ List<Integer> denseLive = collectDocs(dense.liveDocsIterator());
+
+ // THEN
+ assertFalse("Doc 0 should be deleted", sparse.get(0));
+ assertFalse("Doc 0 should be deleted", dense.get(0));
+ assertEquals(1, sparse.deletedCount());
+ assertEquals(1, dense.deletedCount());
+ assertEquals(List.of(0), sparseDeleted);
+ assertEquals(List.of(0), denseDeleted);
+ assertEquals(0, sparseLive.size());
+ assertEquals(0, denseLive.size());
+ }
+
+ public void testAllDocumentsDeleted() throws IOException {
+ // GIVEN
+ final int maxDoc = 100;
+ SparseFixedBitSet sparseSet = new SparseFixedBitSet(maxDoc);
+ FixedBitSet fixedSet = new FixedBitSet(maxDoc);
+ fixedSet.set(0, maxDoc);
+ for (int i = 0; i < maxDoc; i++) {
+ sparseSet.set(i);
+ fixedSet.clear(i);
+ }
+
+ // WHEN
+ SparseLiveDocs sparse = SparseLiveDocs.builder(sparseSet, maxDoc).build();
+ DenseLiveDocs dense = DenseLiveDocs.builder(fixedSet, maxDoc).build();
+ List<Integer> sparseDeleted = collectDocs(sparse.deletedDocsIterator());
+ List<Integer> denseDeleted = collectDocs(dense.deletedDocsIterator());
+ List<Integer> sparseLive = collectDocs(sparse.liveDocsIterator());
+ List<Integer> denseLive = collectDocs(dense.liveDocsIterator());
+
+ // THEN
+ assertEquals(maxDoc, sparse.deletedCount());
+ assertEquals(maxDoc, dense.deletedCount());
+ for (int i = 0; i < maxDoc; i++) {
+ assertFalse("Doc " + i + " should be deleted", sparse.get(i));
+ assertFalse("Doc " + i + " should be deleted", dense.get(i));
+ }
+ assertEquals(maxDoc, sparseDeleted.size());
+ assertEquals(maxDoc, denseDeleted.size());
+ for (int i = 0; i < maxDoc; i++) {
+ assertEquals(i, sparseDeleted.get(i).intValue());
+ assertEquals(i, denseDeleted.get(i).intValue());
+ }
+ assertEquals(0, sparseLive.size());
+ assertEquals(0, denseLive.size());
+ }
+
+ public void testLargeSegment() throws IOException {
+ // GIVEN
+ final int maxDoc = 10_000_000;
+ SparseFixedBitSet sparseSet = new SparseFixedBitSet(maxDoc);
+ FixedBitSet fixedSet = new FixedBitSet(maxDoc);
+ fixedSet.set(0, maxDoc);
+ for (int i = 0; i < maxDoc; i += 1000) {
+ sparseSet.set(i);
+ fixedSet.clear(i);
+ }
+
+ // WHEN
+ SparseLiveDocs sparse = SparseLiveDocs.builder(sparseSet, maxDoc).build();
+ DenseLiveDocs dense = DenseLiveDocs.builder(fixedSet, maxDoc).build();
+ DocIdSetIterator sparseIt = sparse.deletedDocsIterator();
+ long sparseBytes = sparse.ramBytesUsed();
+ long denseBytes = dense.ramBytesUsed();
+
+ // THEN
+ assertEquals(10_000, sparse.deletedCount());
+ assertEquals(10_000, dense.deletedCount());
+ assertEquals(0, sparseIt.nextDoc());
+ sparseIt.advance(maxDoc - 1000);
+ assertEquals(maxDoc - 1000, sparseIt.docID());
+ assertTrue(
+ "Sparse should use less memory than dense for 0.1% deletions (sparse="
+ + sparseBytes
+ + ", dense="
+ + denseBytes
+ + ")",
+ sparseBytes < denseBytes);
+ assertTrue(
+ "Sparse should use significantly less memory (<50%) for very sparse deletions (ratio="
+ + String.format(Locale.ROOT, "%.2f%%", 100.0 * sparseBytes / denseBytes)
+ + ")",
+ sparseBytes < denseBytes / 2);
+ }
+
+ public void testFirstAndLastDocDeletion() throws IOException {
+ // GIVEN
+ final int maxDoc = 1000;
+ SparseFixedBitSet sparseSet = new SparseFixedBitSet(maxDoc);
+ FixedBitSet fixedSet = new FixedBitSet(maxDoc);
+ fixedSet.set(0, maxDoc);
+ sparseSet.set(0);
+ sparseSet.set(maxDoc - 1);
+ fixedSet.clear(0);
+ fixedSet.clear(maxDoc - 1);
+
+ // WHEN
+ SparseLiveDocs sparse = SparseLiveDocs.builder(sparseSet, maxDoc).build();
+ DenseLiveDocs dense = DenseLiveDocs.builder(fixedSet, maxDoc).build();
+ List<Integer> sparseDeleted = collectDocs(sparse.deletedDocsIterator());
+ List<Integer> denseDeleted = collectDocs(dense.deletedDocsIterator());
+ List<Integer> sparseLive = collectDocs(sparse.liveDocsIterator());
+ List<Integer> denseLive = collectDocs(dense.liveDocsIterator());
+
+ // THEN
+ assertFalse("First doc should be deleted", sparse.get(0));
+ assertFalse("Last doc should be deleted", sparse.get(maxDoc - 1));
+ assertFalse("First doc should be deleted", dense.get(0));
+ assertFalse("Last doc should be deleted", dense.get(maxDoc - 1));
+ assertEquals(2, sparseDeleted.size());
+ assertEquals(2, denseDeleted.size());
+ assertEquals(0, sparseDeleted.get(0).intValue());
+ assertEquals(maxDoc - 1, sparseDeleted.get(1).intValue());
+ assertEquals(maxDoc - 2, sparseLive.size());
+ assertEquals(maxDoc - 2, denseLive.size());
+ assertEquals(1, sparseLive.get(0).intValue());
+ assertEquals(maxDoc - 2, sparseLive.get(sparseLive.size() - 1).intValue());
+ }
+}
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/asserting/AssertingLiveDocsFormat.java b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/asserting/AssertingLiveDocsFormat.java
index e2152e45aa5..e7a05fd818a 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/asserting/AssertingLiveDocsFormat.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/asserting/AssertingLiveDocsFormat.java
@@ -20,11 +20,13 @@ import java.io.IOException;
import java.util.Collection;
import org.apache.lucene.codecs.LiveDocsFormat;
import org.apache.lucene.index.SegmentCommitInfo;
+import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.tests.util.TestUtil;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.FixedBitSet;
+import org.apache.lucene.util.LiveDocs;
/** Just like the default live docs format but with additional asserts. */
public class AssertingLiveDocsFormat extends LiveDocsFormat {
@@ -36,6 +38,9 @@ public class AssertingLiveDocsFormat extends LiveDocsFormat {
Bits raw = in.readLiveDocs(dir, info, context);
assert raw != null;
check(raw, info.info.maxDoc(), info.getDelCount());
+ if (raw instanceof LiveDocs) {
+ return new AssertingLiveDocs((LiveDocs) raw);
+ }
return new AssertingBits(raw);
}
@@ -100,4 +105,31 @@ public class AssertingLiveDocsFormat extends LiveDocsFormat {
return "Asserting(" + in + ")";
}
}
+
+ static class AssertingLiveDocs extends AssertingBits implements LiveDocs {
+ private final LiveDocs liveDocs;
+
+ AssertingLiveDocs(LiveDocs liveDocs) {
+ super(liveDocs);
+ this.liveDocs = liveDocs;
+ }
+
+ @Override
+ public int deletedCount() {
+ int count = liveDocs.deletedCount();
+ assert count >= 0;
+ assert count <= length();
+ return count;
+ }
+
+ @Override
+ public DocIdSetIterator liveDocsIterator() {
+ return liveDocs.liveDocsIterator();
+ }
+
+ @Override
+ public DocIdSetIterator deletedDocsIterator() {
+ return liveDocs.deletedDocsIterator();
+ }
+ }
}
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/index/AssertingLeafReader.java b/lucene/test-framework/src/java/org/apache/lucene/tests/index/AssertingLeafReader.java
index 1958cd4b958..ab4b3b143aa 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/index/AssertingLeafReader.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/index/AssertingLeafReader.java
@@ -57,6 +57,7 @@ import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.IOBooleanSupplier;
+import org.apache.lucene.util.LiveDocs;
import org.apache.lucene.util.VirtualMethod;
import org.apache.lucene.util.automaton.CompiledAutomaton;
@@ -1852,7 +1853,11 @@ public class AssertingLeafReader extends FilterLeafReader {
Bits liveDocs = super.getLiveDocs();
if (liveDocs != null) {
assert maxDoc() == liveDocs.length();
- liveDocs = new AssertingBits(liveDocs);
+ if (liveDocs instanceof LiveDocs) {
+ liveDocs = new AssertingLiveDocs((LiveDocs) liveDocs);
+ } else {
+ liveDocs = new AssertingBits(liveDocs);
+ }
} else {
assert maxDoc() == numDocs();
assert !hasDeletions();
@@ -1887,4 +1892,31 @@ public class AssertingLeafReader extends FilterLeafReader {
public CacheHelper getReaderCacheHelper() {
return in.getReaderCacheHelper();
}
+
+ static class AssertingLiveDocs extends AssertingBits implements LiveDocs {
+ private final LiveDocs liveDocs;
+
+ AssertingLiveDocs(LiveDocs liveDocs) {
+ super(liveDocs);
+ this.liveDocs = liveDocs;
+ }
+
+ @Override
+ public int deletedCount() {
+ int count = liveDocs.deletedCount();
+ assert count >= 0;
+ assert count <= length();
+ return count;
+ }
+
+ @Override
+ public DocIdSetIterator liveDocsIterator() {
+ return liveDocs.liveDocsIterator();
+ }
+
+ @Override
+ public DocIdSetIterator deletedDocsIterator() {
+ return liveDocs.deletedDocsIterator();
+ }
+ }
}