Skip to content

Apache Lucene open-source search software

Fix over-allocation of result bitset in MemoryAccountingBitsetCollectorManager (#16452)

sasilekhae13475a21f5f
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 48fceaff75a..4b2f2ebac95 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -493,6 +493,9 @@ Bug Fixes
* GITHUB#16173: FilterDirectory now delegates copyFrom() to the wrapped directory. (Prithvi S)
+* GITHUB#16452: Fix over-allocation in MemoryAccountingBitsetCollectorManager.Result#bitSet, which
+ is now sized to highestMatchedDoc + 1 instead of the full searched range. (Sasilekha R)
+
Other
---------------------
* GITHUB#16266: Remove deprecated search(Query, Collector) calls in QueryUtils by replacing
diff --git a/lucene/misc/src/java/org/apache/lucene/misc/search/MemoryAccountingBitsetCollector.java b/lucene/misc/src/java/org/apache/lucene/misc/search/MemoryAccountingBitsetCollector.java
index 386921a231d..f705f60bc24 100644
--- a/lucene/misc/src/java/org/apache/lucene/misc/search/MemoryAccountingBitsetCollector.java
+++ b/lucene/misc/src/java/org/apache/lucene/misc/search/MemoryAccountingBitsetCollector.java
@@ -34,6 +34,12 @@ public class MemoryAccountingBitsetCollector extends SimpleCollector {
int minDocBase = Integer.MAX_VALUE;
int maxDocEnd = 0;
+ // Highest bit index set in bitSet, or -1 if no doc has been collected. Docs are collected in
+ // strictly ascending order: within a leaf by the Collector contract, and across leaves for a
+ // given collector because IndexSearcher sorts partitions within a slice by docBase and rejects
+ // multiple partitions of the same leaf sharing a slice. So this is simply the position written
+ // by the most recent collect() call.
+ int highestSetBit = -1;
public MemoryAccountingBitsetCollector(CollectorMemoryTracker tracker) {
this.tracker = tracker;
@@ -57,7 +63,14 @@ public class MemoryAccountingBitsetCollector extends SimpleCollector {
@Override
public void collect(int doc) {
- bitSet.set(docBase - minDocBase + doc);
+ int local = docBase - minDocBase + doc;
+ assert local > highestSetBit
+ : "collect() must receive docs in strictly ascending order; got local="
+ + local
+ + " after highestSetBit="
+ + highestSetBit;
+ bitSet.set(local);
+ highestSetBit = local;
}
@Override
@@ -69,7 +82,7 @@ public class MemoryAccountingBitsetCollector extends SimpleCollector {
return minDocBase;
}
- int getMaxDocEnd() {
- return maxDocEnd;
+ int getHighestSetBit() {
+ return highestSetBit;
}
}
diff --git a/lucene/misc/src/java/org/apache/lucene/misc/search/MemoryAccountingBitsetCollectorManager.java b/lucene/misc/src/java/org/apache/lucene/misc/search/MemoryAccountingBitsetCollectorManager.java
index 85e8110843f..c9f58b4f5b8 100644
--- a/lucene/misc/src/java/org/apache/lucene/misc/search/MemoryAccountingBitsetCollectorManager.java
+++ b/lucene/misc/src/java/org/apache/lucene/misc/search/MemoryAccountingBitsetCollectorManager.java
@@ -24,14 +24,23 @@ import org.apache.lucene.util.FixedBitSet;
/**
* CollectorManager for MemoryAccountingBitsetCollector that supports concurrent search.
*
- * <p>Creates multiple collectors for concurrent execution, each collector only allocates bitset for
- * slices it processes, then merges with proper offset in reduce().
+ * <p>Creates multiple collectors for concurrent execution; each collector only allocates a bitset
+ * for the slices it processes, and {@link #reduce} merges them into a single {@link Result} sized
+ * to the highest matched document across all collectors.
*/
public class MemoryAccountingBitsetCollectorManager
implements CollectorManager<
MemoryAccountingBitsetCollector, MemoryAccountingBitsetCollectorManager.Result> {
- /** The result of a search, containing the matched document IDs and total memory used. */
+ /**
+ * The result of a search, containing the matched document IDs and total memory used.
+ *
+ * <p>The returned {@link FixedBitSet} has length {@code highestMatchedDoc + 1} when any document
+ * matched, or {@code 1} (an empty single-word bitset) otherwise, so that iteration via {@link
+ * FixedBitSet#nextSetBit(int)} from index {@code 0} is always safe. It is not padded to the
+ * searched index range; callers probing document ids beyond the last match should first check
+ * {@link FixedBitSet#length()} before calling {@link FixedBitSet#get(int)}.
+ */
public record Result(FixedBitSet bitSet, long totalBytesUsed) {}
private final CollectorMemoryTracker tracker;
@@ -47,21 +56,28 @@ public class MemoryAccountingBitsetCollectorManager
@Override
public Result reduce(Collection<MemoryAccountingBitsetCollector> collectors) {
- int globalMaxDocEnd = 0;
+ // Size the result to just cover the highest matched doc across all collectors. Each
+ // collector's maxDocEnd is inflated by doSetNextReader to the full leaf regardless of what
+ // actually matches, so keying off it can significantly over-allocate on selective queries or
+ // narrow intra-segment slices; use the actual high-water mark tracked at collect time.
+ int resultSize = 0;
for (MemoryAccountingBitsetCollector collector : collectors) {
- globalMaxDocEnd = Math.max(globalMaxDocEnd, collector.getMaxDocEnd());
+ int last = collector.getHighestSetBit();
+ if (last >= 0) {
+ resultSize = Math.max(resultSize, collector.getMinDocBase() + last + 1);
+ }
}
- // TODO: with intra-segment concurrency enabled, globalMaxDocEnd equals the full index maxDoc
- // even when only a portion of the index was searched, causing over-allocation of the result
- // bitset.
- FixedBitSet result = new FixedBitSet(globalMaxDocEnd);
+ // Allocate at least one bit even when no doc matched, so that the natural iteration idiom
+ // (FixedBitSet#nextSetBit(0)) on the result remains safe rather than reading past an empty
+ // long[] backing array.
+ FixedBitSet result = new FixedBitSet(Math.max(1, resultSize));
tracker.updateBytes(result.ramBytesUsed());
for (MemoryAccountingBitsetCollector collector : collectors) {
- if (collector.bitSet != null && collector.bitSet.length() > 0) {
- int length = collector.getMaxDocEnd() - collector.getMinDocBase();
- FixedBitSet.orRange(collector.bitSet, 0, result, collector.getMinDocBase(), length);
+ int last = collector.getHighestSetBit();
+ if (last >= 0) {
+ FixedBitSet.orRange(collector.bitSet, 0, result, collector.getMinDocBase(), last + 1);
}
}
diff --git a/lucene/misc/src/test/org/apache/lucene/misc/search/TestMemoryAccountingBitsetCollector.java b/lucene/misc/src/test/org/apache/lucene/misc/search/TestMemoryAccountingBitsetCollector.java
index 7bbbe52102a..4e6396b3cee 100644
--- a/lucene/misc/src/test/org/apache/lucene/misc/search/TestMemoryAccountingBitsetCollector.java
+++ b/lucene/misc/src/test/org/apache/lucene/misc/search/TestMemoryAccountingBitsetCollector.java
@@ -17,13 +17,20 @@
package org.apache.lucene.misc.search;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.SortedDocValuesField;
import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.Term;
import org.apache.lucene.misc.CollectorMemoryTracker;
+import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.TermQuery;
import org.apache.lucene.store.Directory;
import org.apache.lucene.tests.index.RandomIndexWriter;
import org.apache.lucene.tests.util.LuceneTestCase;
@@ -86,4 +93,91 @@ public class TestMemoryAccountingBitsetCollector extends LuceneTestCase {
// For collector with collecting only 1 doc, 80 bytes are required.
assertTrue(result.totalBytesUsed() >= 80);
}
+
+ public void testResultBitSetSizedToHighestMatchedDoc() throws Exception {
+ // Highly selective query: matches a single document early in the index. The result bitset
+ // must be sized tightly to (highestMatchedDoc + 1) rather than padded to the last visited
+ // leaf's docBase + maxDoc. Uses newSearcher() to get randomized executor/slicing coverage;
+ // the deterministic intra-segment case is exercised by the sister test below.
+ CollectorMemoryTracker tracker =
+ new CollectorMemoryTracker("testMemoryTracker", Long.MAX_VALUE);
+ MemoryAccountingBitsetCollectorManager bitsetCollectorManager =
+ new MemoryAccountingBitsetCollectorManager(tracker);
+
+ IndexSearcher searcher = newSearcher(reader);
+ MemoryAccountingBitsetCollectorManager.Result result =
+ searcher.search(new TermQuery(new Term("field", "5")), bitsetCollectorManager);
+
+ assertEquals(1, result.bitSet().cardinality());
+ int matchedDoc = result.bitSet().nextSetBit(0);
+ assertEquals(matchedDoc + 1, result.bitSet().length());
+ assertTrue(result.bitSet().length() < reader.maxDoc());
+ }
+
+ public void testResultBitSetSizedToHighestMatchedDocUnderIntraSegmentConcurrency()
+ throws Exception {
+ CollectorMemoryTracker tracker =
+ new CollectorMemoryTracker("testMemoryTracker", Long.MAX_VALUE);
+ MemoryAccountingBitsetCollectorManager bitsetCollectorManager =
+ new MemoryAccountingBitsetCollectorManager(tracker);
+
+ IndexSearcher searcher =
+ new IndexSearcher(reader, Runnable::run) {
+ @Override
+ protected LeafSlice[] slices(List<LeafReaderContext> leaves) {
+ // Split each leaf into two partitions, each in its own slice, to force
+ // intra-segment concurrency.
+ List<LeafSlice> slices = new ArrayList<>();
+ for (LeafReaderContext ctx : leaves) {
+ int maxDoc = ctx.reader().maxDoc();
+ if (maxDoc <= 1) {
+ slices.add(
+ new LeafSlice(
+ Collections.singletonList(
+ LeafReaderContextPartition.createForEntireSegment(ctx))));
+ } else {
+ int mid = maxDoc / 2;
+ slices.add(
+ new LeafSlice(
+ Collections.singletonList(
+ LeafReaderContextPartition.createFromAndTo(ctx, 0, mid))));
+ slices.add(
+ new LeafSlice(
+ Collections.singletonList(
+ LeafReaderContextPartition.createFromAndTo(ctx, mid, maxDoc))));
+ }
+ }
+ return slices.toArray(LeafSlice[]::new);
+ }
+ };
+
+ // Use a mid-index match so any given leaf's slice split lands the match in a non-trivial
+ // partition, and the highest-set bit differs from the other tests.
+ MemoryAccountingBitsetCollectorManager.Result result =
+ searcher.search(new TermQuery(new Term("field", "500")), bitsetCollectorManager);
+
+ assertEquals(1, result.bitSet().cardinality());
+ int matchedDoc = result.bitSet().nextSetBit(0);
+ assertEquals(matchedDoc + 1, result.bitSet().length());
+ assertTrue(result.bitSet().length() < reader.maxDoc());
+ }
+
+ public void testResultBitSetEmptyOnNoMatches() throws Exception {
+ CollectorMemoryTracker tracker =
+ new CollectorMemoryTracker("testMemoryTracker", Long.MAX_VALUE);
+ MemoryAccountingBitsetCollectorManager bitsetCollectorManager =
+ new MemoryAccountingBitsetCollectorManager(tracker);
+
+ IndexSearcher searcher = newSearcher(reader);
+ MemoryAccountingBitsetCollectorManager.Result result =
+ searcher.search(new TermQuery(new Term("field", "does-not-exist")), bitsetCollectorManager);
+
+ assertEquals(0, result.bitSet().cardinality());
+ // The bitset must have at least one bit so that the natural iteration idiom stays safe on
+ // empty results: without this guard, nextSetBit(0) would read past an empty long[] and throw
+ // ArrayIndexOutOfBoundsException (see #16452 review).
+ assertEquals(1, result.bitSet().length());
+ assertEquals(DocIdSetIterator.NO_MORE_DOCS, result.bitSet().nextSetBit(0));
+ assertFalse(result.bitSet().get(0));
+ }
}