Skip to content

Apache Lucene open-source search software

Reduce FST block size for BlockTreeTermsWriter (#12604)

gf212196052891e62b
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 93d2563b39d..e02da0a9f7d 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -167,6 +167,9 @@ Optimizations
* GITHUB#12591: Use stable radix sort to speed up the sorting of update terms. (Guo Feng)
+* GITHUB#12604: Estimate the block size of FST BytesStore in BlockTreeTermsWriter
+ to reduce GC load during indexing. (Guo Feng)
+
Changes in runtime behavior
---------------------
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/Lucene90BlockTreeTermsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/Lucene90BlockTreeTermsWriter.java
index a7eb438489f..1b0bd3568c9 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/Lucene90BlockTreeTermsWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/Lucene90BlockTreeTermsWriter.java
@@ -52,6 +52,7 @@ import org.apache.lucene.util.fst.BytesRefFSTEnum;
import org.apache.lucene.util.fst.FST;
import org.apache.lucene.util.fst.FSTCompiler;
import org.apache.lucene.util.fst.Util;
+import org.apache.lucene.util.packed.PackedInts;
/*
TODO:
@@ -490,10 +491,22 @@ public final class Lucene90BlockTreeTermsWriter extends FieldsConsumer {
}
}
+ long estimateSize = prefix.length;
+ for (PendingBlock block : blocks) {
+ if (block.subIndices != null) {
+ for (FST<BytesRef> subIndex : block.subIndices) {
+ estimateSize += subIndex.numBytes();
+ }
+ }
+ }
+ int estimateBitsRequired = PackedInts.bitsRequired(estimateSize);
+ int pageBits = Math.min(15, Math.max(6, estimateBitsRequired));
+
final ByteSequenceOutputs outputs = ByteSequenceOutputs.getSingleton();
final FSTCompiler<BytesRef> fstCompiler =
new FSTCompiler.Builder<>(FST.INPUT_TYPE.BYTE1, outputs)
.shouldShareNonSingletonNodes(false)
+ .bytesPageBits(pageBits)
.build();
// if (DEBUG) {
// System.out.println(" compile index for prefix=" + prefix);
diff --git a/lucene/core/src/java/org/apache/lucene/util/fst/FST.java b/lucene/core/src/java/org/apache/lucene/util/fst/FST.java
index 816a2857268..fb356c2c9c7 100644
--- a/lucene/core/src/java/org/apache/lucene/util/fst/FST.java
+++ b/lucene/core/src/java/org/apache/lucene/util/fst/FST.java
@@ -520,6 +520,10 @@ public final class FST<T> implements Accountable {
bytes.finish();
}
+ public long numBytes() {
+ return bytes.getPosition();
+ }
+
public T getEmptyOutput() {
return emptyOutput;
}