Apache Lucene open-source search software
Introduce MemoryAccountingBitsetCollectorManager (#15795)
gaobinlongcd413bbfbb9f
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex 1167dbf3b12..d58db69ce48 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -373,6 +373,8 @@ Improvements* GITHUB#16000: Clarify that Accountable#ramBytesUsed() reports JVM heap memory only. (Luca Cavanna)+* GITHUB#15795: Introduce MemoryAccountingBitsetCollectorManager to parallelize search when using MemoryAccountingBitsetCollector. (Binlong Gao)+* GITHUB#15574: Introduce FirstPassGroupingCollectorManager to parallelize search when using FirstPassGroupingCollector. (Binlong Gao)* GITHUB#15660: Introduce LargeNumHitsTopDocsCollectorManager to parallelize search when using LargeNumHitsTopDocsCollector. (Binlong Gao)diff --git a/lucene/misc/src/java/org/apache/lucene/misc/search/MemoryAccountingBitsetCollector.java b/lucene/misc/src/java/org/apache/lucene/misc/search/MemoryAccountingBitsetCollector.javaindex a2337ab033e..386921a231d 100644--- a/lucene/misc/src/java/org/apache/lucene/misc/search/MemoryAccountingBitsetCollector.java+++ b/lucene/misc/src/java/org/apache/lucene/misc/search/MemoryAccountingBitsetCollector.java@@ -24,7 +24,7 @@ import org.apache.lucene.search.ScoreMode;import org.apache.lucene.search.SimpleCollector;import org.apache.lucene.util.FixedBitSet;-/** Bitset collector which supports memory tracking */+/** Bitset collector which supports memory tracking. */public class MemoryAccountingBitsetCollector extends SimpleCollector {final CollectorMemoryTracker tracker;@@ -32,6 +32,9 @@ public class MemoryAccountingBitsetCollector extends SimpleCollector {int length = 0;int docBase = 0;+ int minDocBase = Integer.MAX_VALUE;+ int maxDocEnd = 0;+public MemoryAccountingBitsetCollector(CollectorMemoryTracker tracker) {this.tracker = tracker;tracker.updateBytes(bitSet.ramBytesUsed());@@ -40,7 +43,11 @@ public class MemoryAccountingBitsetCollector extends SimpleCollector {@Overrideprotected void doSetNextReader(LeafReaderContext context) throws IOException {docBase = context.docBase;- length += context.reader().maxDoc();+ int docEnd = docBase + context.reader().maxDoc();+ minDocBase = Math.min(minDocBase, docBase);+ maxDocEnd = Math.max(maxDocEnd, docEnd);+ length = maxDocEnd - minDocBase;+FixedBitSet newBitSet = FixedBitSet.ensureCapacity(bitSet, length);if (newBitSet != bitSet) {tracker.updateBytes(newBitSet.ramBytesUsed() - bitSet.ramBytesUsed());@@ -50,11 +57,19 @@ public class MemoryAccountingBitsetCollector extends SimpleCollector {@Overridepublic void collect(int doc) {- bitSet.set(docBase + doc);+ bitSet.set(docBase - minDocBase + doc);}@Overridepublic ScoreMode scoreMode() {return ScoreMode.COMPLETE_NO_SCORES;}++ int getMinDocBase() {+ return minDocBase;+ }++ int getMaxDocEnd() {+ return maxDocEnd;+ }}diff --git a/lucene/misc/src/java/org/apache/lucene/misc/search/MemoryAccountingBitsetCollectorManager.java b/lucene/misc/src/java/org/apache/lucene/misc/search/MemoryAccountingBitsetCollectorManager.javanew file mode 100644index 00000000000..85e8110843f--- /dev/null+++ b/lucene/misc/src/java/org/apache/lucene/misc/search/MemoryAccountingBitsetCollectorManager.java@@ -0,0 +1,70 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.misc.search;++import java.util.Collection;+import org.apache.lucene.misc.CollectorMemoryTracker;+import org.apache.lucene.search.CollectorManager;+import org.apache.lucene.util.FixedBitSet;++/**+ * CollectorManager for MemoryAccountingBitsetCollector that supports concurrent search.+ *+ * <p>Creates multiple collectors for concurrent execution, each collector only allocates bitset for+ * slices it processes, then merges with proper offset in reduce().+ */+public class MemoryAccountingBitsetCollectorManager+ implements CollectorManager<+ MemoryAccountingBitsetCollector, MemoryAccountingBitsetCollectorManager.Result> {++ /** The result of a search, containing the matched document IDs and total memory used. */+ public record Result(FixedBitSet bitSet, long totalBytesUsed) {}++ private final CollectorMemoryTracker tracker;++ public MemoryAccountingBitsetCollectorManager(CollectorMemoryTracker tracker) {+ this.tracker = tracker;+ }++ @Override+ public MemoryAccountingBitsetCollector newCollector() {+ return new MemoryAccountingBitsetCollector(tracker);+ }++ @Override+ public Result reduce(Collection<MemoryAccountingBitsetCollector> collectors) {+ int globalMaxDocEnd = 0;+ for (MemoryAccountingBitsetCollector collector : collectors) {+ globalMaxDocEnd = Math.max(globalMaxDocEnd, collector.getMaxDocEnd());+ }++ // TODO: with intra-segment concurrency enabled, globalMaxDocEnd equals the full index maxDoc+ // even when only a portion of the index was searched, causing over-allocation of the result+ // bitset.+ FixedBitSet result = new FixedBitSet(globalMaxDocEnd);+ tracker.updateBytes(result.ramBytesUsed());++ for (MemoryAccountingBitsetCollector collector : collectors) {+ if (collector.bitSet != null && collector.bitSet.length() > 0) {+ int length = collector.getMaxDocEnd() - collector.getMinDocBase();+ FixedBitSet.orRange(collector.bitSet, 0, result, collector.getMinDocBase(), length);+ }+ }++ return new Result(result, tracker.getBytes());+ }+}diff --git a/lucene/misc/src/test/org/apache/lucene/misc/search/TestMemoryAccountingBitsetCollector.java b/lucene/misc/src/test/org/apache/lucene/misc/search/TestMemoryAccountingBitsetCollector.javaindex b642cf8f171..7bbbe52102a 100644--- a/lucene/misc/src/test/org/apache/lucene/misc/search/TestMemoryAccountingBitsetCollector.java+++ b/lucene/misc/src/test/org/apache/lucene/misc/search/TestMemoryAccountingBitsetCollector.java@@ -57,31 +57,33 @@ public class TestMemoryAccountingBitsetCollector extends LuceneTestCase {dir.close();}- public void testMemoryAccountingBitsetCollectorMemoryLimit() {- long perCollectorMemoryLimit = 150;+ public void testMemoryAccountingBitsetCollectorMemoryLimit() throws Exception {+ long collectorMemoryLimit = 150;CollectorMemoryTracker tracker =- new CollectorMemoryTracker("testMemoryTracker", perCollectorMemoryLimit);- MemoryAccountingBitsetCollector bitSetCollector = new MemoryAccountingBitsetCollector(tracker);-- IndexSearcher searcher = new IndexSearcher(reader);+ new CollectorMemoryTracker("testMemoryTracker", collectorMemoryLimit);+ MemoryAccountingBitsetCollectorManager bitsetCollectorManager =+ new MemoryAccountingBitsetCollectorManager(tracker);+ IndexSearcher searcher = newSearcher(reader);expectThrows(IllegalStateException.class,- () -> {- searcher.search(MatchAllDocsQuery.INSTANCE, bitSetCollector);- });+ () -> searcher.search(MatchAllDocsQuery.INSTANCE, bitsetCollectorManager));}public void testCollectedResult() throws Exception {CollectorMemoryTracker tracker =new CollectorMemoryTracker("testMemoryTracker", Long.MAX_VALUE);- MemoryAccountingBitsetCollector collector = new MemoryAccountingBitsetCollector(tracker);+ MemoryAccountingBitsetCollectorManager bitsetCollectorManager =+ new MemoryAccountingBitsetCollectorManager(tracker);- IndexSearcher searcher = new IndexSearcher(reader);- searcher.search(MatchAllDocsQuery.INSTANCE, collector);+ IndexSearcher searcher = newSearcher(reader);+ MemoryAccountingBitsetCollectorManager.Result result =+ searcher.search(MatchAllDocsQuery.INSTANCE, bitsetCollectorManager);- assertEquals(1000, collector.bitSet.cardinality());+ assertEquals(1000, result.bitSet().cardinality());for (int i = 0; i < 1000; i++) {- assertTrue(collector.bitSet.get(i));+ assertTrue(result.bitSet().get(i));}+ // For collector with collecting only 1 doc, 80 bytes are required.+ assertTrue(result.totalBytesUsed() >= 80);}}