Skip to content

Apache Lucene open-source search software

Introduce MemoryAccountingBitsetCollectorManager (#15795)

gaobinlongcd413bbfbb9f
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 1167dbf3b12..d58db69ce48 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -373,6 +373,8 @@ Improvements
* GITHUB#16000: Clarify that Accountable#ramBytesUsed() reports JVM heap memory only. (Luca Cavanna)
+* GITHUB#15795: Introduce MemoryAccountingBitsetCollectorManager to parallelize search when using MemoryAccountingBitsetCollector. (Binlong Gao)
+
* GITHUB#15574: Introduce FirstPassGroupingCollectorManager to parallelize search when using FirstPassGroupingCollector. (Binlong Gao)
* GITHUB#15660: Introduce LargeNumHitsTopDocsCollectorManager to parallelize search when using LargeNumHitsTopDocsCollector. (Binlong Gao)
diff --git a/lucene/misc/src/java/org/apache/lucene/misc/search/MemoryAccountingBitsetCollector.java b/lucene/misc/src/java/org/apache/lucene/misc/search/MemoryAccountingBitsetCollector.java
index a2337ab033e..386921a231d 100644
--- a/lucene/misc/src/java/org/apache/lucene/misc/search/MemoryAccountingBitsetCollector.java
+++ b/lucene/misc/src/java/org/apache/lucene/misc/search/MemoryAccountingBitsetCollector.java
@@ -24,7 +24,7 @@ import org.apache.lucene.search.ScoreMode;
import org.apache.lucene.search.SimpleCollector;
import org.apache.lucene.util.FixedBitSet;
-/** Bitset collector which supports memory tracking */
+/** Bitset collector which supports memory tracking. */
public class MemoryAccountingBitsetCollector extends SimpleCollector {
final CollectorMemoryTracker tracker;
@@ -32,6 +32,9 @@ public class MemoryAccountingBitsetCollector extends SimpleCollector {
int length = 0;
int docBase = 0;
+ int minDocBase = Integer.MAX_VALUE;
+ int maxDocEnd = 0;
+
public MemoryAccountingBitsetCollector(CollectorMemoryTracker tracker) {
this.tracker = tracker;
tracker.updateBytes(bitSet.ramBytesUsed());
@@ -40,7 +43,11 @@ public class MemoryAccountingBitsetCollector extends SimpleCollector {
@Override
protected void doSetNextReader(LeafReaderContext context) throws IOException {
docBase = context.docBase;
- length += context.reader().maxDoc();
+ int docEnd = docBase + context.reader().maxDoc();
+ minDocBase = Math.min(minDocBase, docBase);
+ maxDocEnd = Math.max(maxDocEnd, docEnd);
+ length = maxDocEnd - minDocBase;
+
FixedBitSet newBitSet = FixedBitSet.ensureCapacity(bitSet, length);
if (newBitSet != bitSet) {
tracker.updateBytes(newBitSet.ramBytesUsed() - bitSet.ramBytesUsed());
@@ -50,11 +57,19 @@ public class MemoryAccountingBitsetCollector extends SimpleCollector {
@Override
public void collect(int doc) {
- bitSet.set(docBase + doc);
+ bitSet.set(docBase - minDocBase + doc);
}
@Override
public ScoreMode scoreMode() {
return ScoreMode.COMPLETE_NO_SCORES;
}
+
+ int getMinDocBase() {
+ return minDocBase;
+ }
+
+ int getMaxDocEnd() {
+ return maxDocEnd;
+ }
}
diff --git a/lucene/misc/src/java/org/apache/lucene/misc/search/MemoryAccountingBitsetCollectorManager.java b/lucene/misc/src/java/org/apache/lucene/misc/search/MemoryAccountingBitsetCollectorManager.java
new file mode 100644
index 00000000000..85e8110843f
--- /dev/null
+++ b/lucene/misc/src/java/org/apache/lucene/misc/search/MemoryAccountingBitsetCollectorManager.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.misc.search;
+
+import java.util.Collection;
+import org.apache.lucene.misc.CollectorMemoryTracker;
+import org.apache.lucene.search.CollectorManager;
+import org.apache.lucene.util.FixedBitSet;
+
+/**
+ * CollectorManager for MemoryAccountingBitsetCollector that supports concurrent search.
+ *
+ * <p>Creates multiple collectors for concurrent execution, each collector only allocates bitset for
+ * slices it processes, then merges with proper offset in reduce().
+ */
+public class MemoryAccountingBitsetCollectorManager
+ implements CollectorManager<
+ MemoryAccountingBitsetCollector, MemoryAccountingBitsetCollectorManager.Result> {
+
+ /** The result of a search, containing the matched document IDs and total memory used. */
+ public record Result(FixedBitSet bitSet, long totalBytesUsed) {}
+
+ private final CollectorMemoryTracker tracker;
+
+ public MemoryAccountingBitsetCollectorManager(CollectorMemoryTracker tracker) {
+ this.tracker = tracker;
+ }
+
+ @Override
+ public MemoryAccountingBitsetCollector newCollector() {
+ return new MemoryAccountingBitsetCollector(tracker);
+ }
+
+ @Override
+ public Result reduce(Collection<MemoryAccountingBitsetCollector> collectors) {
+ int globalMaxDocEnd = 0;
+ for (MemoryAccountingBitsetCollector collector : collectors) {
+ globalMaxDocEnd = Math.max(globalMaxDocEnd, collector.getMaxDocEnd());
+ }
+
+ // TODO: with intra-segment concurrency enabled, globalMaxDocEnd equals the full index maxDoc
+ // even when only a portion of the index was searched, causing over-allocation of the result
+ // bitset.
+ FixedBitSet result = new FixedBitSet(globalMaxDocEnd);
+ tracker.updateBytes(result.ramBytesUsed());
+
+ for (MemoryAccountingBitsetCollector collector : collectors) {
+ if (collector.bitSet != null && collector.bitSet.length() > 0) {
+ int length = collector.getMaxDocEnd() - collector.getMinDocBase();
+ FixedBitSet.orRange(collector.bitSet, 0, result, collector.getMinDocBase(), length);
+ }
+ }
+
+ return new Result(result, tracker.getBytes());
+ }
+}
diff --git a/lucene/misc/src/test/org/apache/lucene/misc/search/TestMemoryAccountingBitsetCollector.java b/lucene/misc/src/test/org/apache/lucene/misc/search/TestMemoryAccountingBitsetCollector.java
index b642cf8f171..7bbbe52102a 100644
--- a/lucene/misc/src/test/org/apache/lucene/misc/search/TestMemoryAccountingBitsetCollector.java
+++ b/lucene/misc/src/test/org/apache/lucene/misc/search/TestMemoryAccountingBitsetCollector.java
@@ -57,31 +57,33 @@ public class TestMemoryAccountingBitsetCollector extends LuceneTestCase {
dir.close();
}
- public void testMemoryAccountingBitsetCollectorMemoryLimit() {
- long perCollectorMemoryLimit = 150;
+ public void testMemoryAccountingBitsetCollectorMemoryLimit() throws Exception {
+ long collectorMemoryLimit = 150;
CollectorMemoryTracker tracker =
- new CollectorMemoryTracker("testMemoryTracker", perCollectorMemoryLimit);
- MemoryAccountingBitsetCollector bitSetCollector = new MemoryAccountingBitsetCollector(tracker);
-
- IndexSearcher searcher = new IndexSearcher(reader);
+ new CollectorMemoryTracker("testMemoryTracker", collectorMemoryLimit);
+ MemoryAccountingBitsetCollectorManager bitsetCollectorManager =
+ new MemoryAccountingBitsetCollectorManager(tracker);
+ IndexSearcher searcher = newSearcher(reader);
expectThrows(
IllegalStateException.class,
- () -> {
- searcher.search(MatchAllDocsQuery.INSTANCE, bitSetCollector);
- });
+ () -> searcher.search(MatchAllDocsQuery.INSTANCE, bitsetCollectorManager));
}
public void testCollectedResult() throws Exception {
CollectorMemoryTracker tracker =
new CollectorMemoryTracker("testMemoryTracker", Long.MAX_VALUE);
- MemoryAccountingBitsetCollector collector = new MemoryAccountingBitsetCollector(tracker);
+ MemoryAccountingBitsetCollectorManager bitsetCollectorManager =
+ new MemoryAccountingBitsetCollectorManager(tracker);
- IndexSearcher searcher = new IndexSearcher(reader);
- searcher.search(MatchAllDocsQuery.INSTANCE, collector);
+ IndexSearcher searcher = newSearcher(reader);
+ MemoryAccountingBitsetCollectorManager.Result result =
+ searcher.search(MatchAllDocsQuery.INSTANCE, bitsetCollectorManager);
- assertEquals(1000, collector.bitSet.cardinality());
+ assertEquals(1000, result.bitSet().cardinality());
for (int i = 0; i < 1000; i++) {
- assertTrue(collector.bitSet.get(i));
+ assertTrue(result.bitSet().get(i));
}
+ // For collector with collecting only 1 doc, 80 bytes are required.
+ assertTrue(result.totalBytesUsed() >= 80);
}
}