Skip to content

Apache Lucene open-source search software

Completion FSTs to be loaded off-heap at all times (#14364)

Luca Cavannafe86094f2f47
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 012f79c0d7c..c215696ad98 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -145,6 +145,7 @@ Improvements
* GITHUB#14213: Allowing indexing stored-only StoredField directly from DataInput. (Tim Brooks)
+* GITHUB#14364: Completion FSTs to be loaded off-heap at all times. (Luca Cavanna)
Optimizations
---------------------
diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion101PostingsFormat.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion101PostingsFormat.java
index 2a51f01fd7a..e9dde1705b9 100644
--- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion101PostingsFormat.java
+++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion101PostingsFormat.java
@@ -25,17 +25,9 @@ import org.apache.lucene.codecs.PostingsFormat;
* @lucene.experimental
*/
public class Completion101PostingsFormat extends CompletionPostingsFormat {
- /** Creates a {@link Completion101PostingsFormat} that will load the completion FST on-heap. */
+ /** Creates a {@link Completion101PostingsFormat}. */
public Completion101PostingsFormat() {
- this(FSTLoadMode.ON_HEAP);
- }
-
- /**
- * Creates a {@link Completion101PostingsFormat} that will use the provided <code>fstLoadMode
- * </code> to determine if the completion FST should be loaded on or off heap.
- */
- public Completion101PostingsFormat(FSTLoadMode fstLoadMode) {
- super("Completion101", fstLoadMode);
+ super("Completion101");
}
@Override
diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion50PostingsFormat.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion50PostingsFormat.java
index e53e991db22..08ed3802fa8 100644
--- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion50PostingsFormat.java
+++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion50PostingsFormat.java
@@ -27,17 +27,9 @@ import org.apache.lucene.codecs.PostingsFormat;
* @lucene.experimental
*/
public class Completion50PostingsFormat extends CompletionPostingsFormat {
- /** Creates a {@link Completion50PostingsFormat} that will load the completion FST on-heap. */
+ /** Creates a {@link Completion50PostingsFormat}. */
public Completion50PostingsFormat() {
- this(FSTLoadMode.ON_HEAP);
- }
-
- /**
- * Creates a {@link Completion50PostingsFormat} that will use the provided <code>fstLoadMode
- * </code> to determine if the completion FST should be loaded on or off heap.
- */
- public Completion50PostingsFormat(FSTLoadMode fstLoadMode) {
- super("completion", fstLoadMode);
+ super("completion");
}
@Override
diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion84PostingsFormat.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion84PostingsFormat.java
index 76e29811834..406f5d6affa 100644
--- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion84PostingsFormat.java
+++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion84PostingsFormat.java
@@ -27,17 +27,9 @@ import org.apache.lucene.codecs.PostingsFormat;
* @lucene.experimental
*/
public class Completion84PostingsFormat extends CompletionPostingsFormat {
- /** Creates a {@link Completion84PostingsFormat} that will load the completion FST on-heap. */
+ /** Creates a {@link Completion84PostingsFormat}. */
public Completion84PostingsFormat() {
- this(FSTLoadMode.ON_HEAP);
- }
-
- /**
- * Creates a {@link Completion84PostingsFormat} that will use the provided <code>fstLoadMode
- * </code> to determine if the completion FST should be loaded on or off heap.
- */
- public Completion84PostingsFormat(FSTLoadMode fstLoadMode) {
- super("Completion84", fstLoadMode);
+ super("Completion84");
}
@Override
diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion90PostingsFormat.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion90PostingsFormat.java
index db45303bcd7..58d6071a424 100644
--- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion90PostingsFormat.java
+++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion90PostingsFormat.java
@@ -27,17 +27,9 @@ import org.apache.lucene.codecs.PostingsFormat;
* @lucene.experimental
*/
public class Completion90PostingsFormat extends CompletionPostingsFormat {
- /** Creates a {@link Completion90PostingsFormat} that will load the completion FST on-heap. */
+ /** Creates a {@link Completion90PostingsFormat}. */
public Completion90PostingsFormat() {
- this(FSTLoadMode.ON_HEAP);
- }
-
- /**
- * Creates a {@link Completion90PostingsFormat} that will use the provided <code>fstLoadMode
- * </code> to determine if the completion FST should be loaded on or off heap.
- */
- public Completion90PostingsFormat(FSTLoadMode fstLoadMode) {
- super("Completion90", fstLoadMode);
+ super("Completion90");
}
@Override
diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion912PostingsFormat.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion912PostingsFormat.java
index 77a62443f31..3f170063149 100644
--- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion912PostingsFormat.java
+++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion912PostingsFormat.java
@@ -27,17 +27,9 @@ import org.apache.lucene.codecs.PostingsFormat;
* @lucene.experimental
*/
public class Completion912PostingsFormat extends CompletionPostingsFormat {
- /** Creates a {@link Completion912PostingsFormat} that will load the completion FST on-heap. */
+ /** Creates a {@link Completion912PostingsFormat}. */
public Completion912PostingsFormat() {
- this(FSTLoadMode.ON_HEAP);
- }
-
- /**
- * Creates a {@link Completion912PostingsFormat} that will use the provided <code>fstLoadMode
- * </code> to determine if the completion FST should be loaded on or off heap.
- */
- public Completion912PostingsFormat(FSTLoadMode fstLoadMode) {
- super("Completion912", fstLoadMode);
+ super("Completion912");
}
@Override
diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion99PostingsFormat.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion99PostingsFormat.java
index d3ea76d5425..2d3509ea4be 100644
--- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion99PostingsFormat.java
+++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion99PostingsFormat.java
@@ -27,17 +27,9 @@ import org.apache.lucene.codecs.PostingsFormat;
* @lucene.experimental
*/
public class Completion99PostingsFormat extends CompletionPostingsFormat {
- /** Creates a {@link Completion99PostingsFormat} that will load the completion FST on-heap. */
+ /** Creates a {@link Completion99PostingsFormat}. */
public Completion99PostingsFormat() {
- this(FSTLoadMode.ON_HEAP);
- }
-
- /**
- * Creates a {@link Completion99PostingsFormat} that will use the provided <code>fstLoadMode
- * </code> to determine if the completion FST should be loaded on or off heap.
- */
- public Completion99PostingsFormat(FSTLoadMode fstLoadMode) {
- super("Completion99", fstLoadMode);
+ super("Completion99");
}
@Override
diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/CompletionFieldsProducer.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/CompletionFieldsProducer.java
index 86a42af9fad..13f3df5c19e 100644
--- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/CompletionFieldsProducer.java
+++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/CompletionFieldsProducer.java
@@ -35,7 +35,6 @@ import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.Terms;
-import org.apache.lucene.search.suggest.document.CompletionPostingsFormat.FSTLoadMode;
import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.Accountable;
@@ -56,7 +55,7 @@ import org.apache.lucene.util.IOUtils;
final class CompletionFieldsProducer extends FieldsProducer implements Accountable {
private FieldsProducer delegateFieldsProducer;
- private Map<String, CompletionsTermsReader> readers;
+ private final Map<String, CompletionsTermsReader> readers;
private IndexInput dictIn;
// copy ctr for merge instance
@@ -66,8 +65,7 @@ final class CompletionFieldsProducer extends FieldsProducer implements Accountab
this.readers = readers;
}
- CompletionFieldsProducer(String codecName, SegmentReadState state, FSTLoadMode fstLoadMode)
- throws IOException {
+ CompletionFieldsProducer(String codecName, SegmentReadState state) throws IOException {
String indexFile =
IndexFileNames.segmentFileName(
state.segmentInfo.name, state.segmentSuffix, INDEX_EXTENSION);
@@ -114,8 +112,7 @@ final class CompletionFieldsProducer extends FieldsProducer implements Accountab
FieldInfo fieldInfo = state.fieldInfos.fieldInfo(fieldNumber);
// we don't load the FST yet
readers.put(
- fieldInfo.name,
- new CompletionsTermsReader(dictIn, offset, minWeight, maxWeight, type, fstLoadMode));
+ fieldInfo.name, new CompletionsTermsReader(dictIn, offset, minWeight, maxWeight, type));
}
CodecUtil.checkFooter(index);
success = true;
diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/CompletionPostingsFormat.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/CompletionPostingsFormat.java
index 146487d7774..278b2037822 100644
--- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/CompletionPostingsFormat.java
+++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/CompletionPostingsFormat.java
@@ -104,36 +104,9 @@ public abstract class CompletionPostingsFormat extends PostingsFormat {
static final String INDEX_EXTENSION = "cmp";
static final String DICT_EXTENSION = "lkp";
- /** An enum that allows to control if suggester FSTs are loaded into memory or read off-heap */
- public enum FSTLoadMode {
- /**
- * Always read FSTs from disk. NOTE: If this option is used the FST will be read off-heap even
- * if buffered directory implementations are used.
- */
- OFF_HEAP,
- /** Never read FSTs from disk ie. all suggest fields FSTs are loaded into memory */
- ON_HEAP,
- /**
- * Automatically make the decision if FSTs are read from disk depending if the segment read from
- * an MMAPDirectory
- */
- AUTO
- }
-
- private final FSTLoadMode fstLoadMode;
-
- /** Used only by core Lucene at read-time via Service Provider instantiation */
+ /** Creates a {@link CompletionPostingsFormat} with the given name. */
public CompletionPostingsFormat(String name) {
- this(name, FSTLoadMode.ON_HEAP);
- }
-
- /**
- * Creates a {@link CompletionPostingsFormat} that will use the provided <code>fstLoadMode</code>
- * to determine if the completion FST should be loaded on or off heap.
- */
- public CompletionPostingsFormat(String name, FSTLoadMode fstLoadMode) {
super(name);
- this.fstLoadMode = fstLoadMode;
}
/** Concrete implementation should specify the delegating postings format */
@@ -153,6 +126,6 @@ public abstract class CompletionPostingsFormat extends PostingsFormat {
@Override
public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {
- return new CompletionFieldsProducer(getName(), state, fstLoadMode);
+ return new CompletionFieldsProducer(getName(), state);
}
}
diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/CompletionsTermsReader.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/CompletionsTermsReader.java
index f8e22256f75..26e77b5f9c3 100644
--- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/CompletionsTermsReader.java
+++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/CompletionsTermsReader.java
@@ -19,7 +19,6 @@ package org.apache.lucene.search.suggest.document;
import java.io.IOException;
import java.util.Collection;
import java.util.Collections;
-import org.apache.lucene.search.suggest.document.CompletionPostingsFormat.FSTLoadMode;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.Accountable;
@@ -41,8 +40,6 @@ public final class CompletionsTermsReader implements Accountable {
private final IndexInput dictIn;
private final long offset;
- private final FSTLoadMode fstLoadMode;
-
private NRTSuggester suggester;
/**
@@ -50,12 +47,7 @@ public final class CompletionsTermsReader implements Accountable {
* </code> with <code>offset</code>
*/
CompletionsTermsReader(
- IndexInput dictIn,
- long offset,
- long minWeight,
- long maxWeight,
- byte type,
- FSTLoadMode fstLoadMode) {
+ IndexInput dictIn, long offset, long minWeight, long maxWeight, byte type) {
assert minWeight <= maxWeight;
assert offset >= 0l && offset < dictIn.length();
this.dictIn = dictIn;
@@ -63,7 +55,6 @@ public final class CompletionsTermsReader implements Accountable {
this.minWeight = minWeight;
this.maxWeight = maxWeight;
this.type = type;
- this.fstLoadMode = fstLoadMode;
}
/**
@@ -73,7 +64,7 @@ public final class CompletionsTermsReader implements Accountable {
public synchronized NRTSuggester suggester() throws IOException {
if (suggester == null) {
IndexInput indexInput = dictIn.slice("NRTSuggester", offset, dictIn.length() - offset);
- suggester = NRTSuggester.load(indexInput, fstLoadMode);
+ suggester = NRTSuggester.load(indexInput);
}
return suggester;
}
diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/NRTSuggester.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/NRTSuggester.java
index a923886d6ce..ef365682da8 100644
--- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/NRTSuggester.java
+++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/NRTSuggester.java
@@ -24,7 +24,6 @@ import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import org.apache.lucene.search.suggest.analyzing.FSTUtil;
-import org.apache.lucene.search.suggest.document.CompletionPostingsFormat.FSTLoadMode;
import org.apache.lucene.store.ByteArrayDataInput;
import org.apache.lucene.store.ByteArrayDataOutput;
import org.apache.lucene.store.IndexInput;
@@ -306,37 +305,15 @@ public final class NRTSuggester implements Accountable {
return (numDocs > 0) ? ((double) numDocs / maxDocs) : -1;
}
- private static boolean shouldLoadFSTOffHeap(IndexInput input, FSTLoadMode fstLoadMode) {
- switch (fstLoadMode) {
- case ON_HEAP:
- return false;
- case OFF_HEAP:
- return true;
- case AUTO:
- // TODO: Make this less hacky to maybe expose "off-heap" feature using a marker interface on
- // the IndexInput
- return input.getClass().getName().contains(".MemorySegmentIndexInput");
- default:
- throw new IllegalStateException("unknown enum constant: " + fstLoadMode);
- }
- }
-
- /**
- * Loads a {@link NRTSuggester} from {@link org.apache.lucene.store.IndexInput} on or off-heap
- * depending on the provided <code>fstLoadMode</code>
- */
- static NRTSuggester load(IndexInput input, FSTLoadMode fstLoadMode) throws IOException {
+ /** Loads a {@link NRTSuggester} from {@link org.apache.lucene.store.IndexInput} */
+ static NRTSuggester load(IndexInput input) throws IOException {
final FST<Pair<Long, BytesRef>> fst;
PairOutputs<Long, BytesRef> outputs =
new PairOutputs<>(PositiveIntOutputs.getSingleton(), ByteSequenceOutputs.getSingleton());
- if (shouldLoadFSTOffHeap(input, fstLoadMode)) {
- final FST.FSTMetadata<Pair<Long, BytesRef>> fstMetadata = FST.readMetadata(input, outputs);
- OffHeapFSTStore store = new OffHeapFSTStore(input, input.getFilePointer(), fstMetadata);
- fst = FST.fromFSTReader(fstMetadata, store);
- input.skipBytes(store.size());
- } else {
- fst = new FST<>(FST.readMetadata(input, outputs), input);
- }
+ final FST.FSTMetadata<Pair<Long, BytesRef>> fstMetadata = FST.readMetadata(input, outputs);
+ OffHeapFSTStore store = new OffHeapFSTStore(input, input.getFilePointer(), fstMetadata);
+ fst = FST.fromFSTReader(fstMetadata, store);
+ input.skipBytes(store.size());
/* read some meta info */
int maxAnalyzedPathsPerOutput = input.readVInt();
diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/NRTSuggesterBuilder.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/NRTSuggesterBuilder.java
index 7aec1568ad7..8d983a29a5e 100644
--- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/NRTSuggesterBuilder.java
+++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/NRTSuggesterBuilder.java
@@ -99,10 +99,7 @@ final class NRTSuggesterBuilder {
entries.clear();
}
- /**
- * Builds and stores a FST that can be loaded with {@link NRTSuggester#load(IndexInput,
- * CompletionPostingsFormat.FSTLoadMode)})}
- */
+ /** Builds and stores a FST that can be loaded with {@link NRTSuggester#load(IndexInput)})} */
public boolean store(DataOutput output) throws IOException {
final FST<PairOutputs.Pair<Long, BytesRef>> fst =
FST.fromFSTReader(fstCompiler.compile(), fstCompiler.getFSTReader());
diff --git a/lucene/suggest/src/test/org/apache/lucene/search/suggest/document/TestSuggestField.java b/lucene/suggest/src/test/org/apache/lucene/search/suggest/document/TestSuggestField.java
index 4a59d09d35c..1990baffe72 100644
--- a/lucene/suggest/src/test/org/apache/lucene/search/suggest/document/TestSuggestField.java
+++ b/lucene/suggest/src/test/org/apache/lucene/search/suggest/document/TestSuggestField.java
@@ -20,7 +20,6 @@ import static org.apache.lucene.tests.analysis.BaseTokenStreamTestCase.assertTok
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.startsWith;
-import com.carrotsearch.randomizedtesting.generators.RandomPicks;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.ArrayList;
@@ -949,9 +948,7 @@ public class TestSuggestField extends LuceneTestCase {
iwc.setMergePolicy(newLogMergePolicy());
Codec filterCodec =
new FilterCodec(TestUtil.getDefaultCodec().getName(), TestUtil.getDefaultCodec()) {
- final CompletionPostingsFormat.FSTLoadMode fstLoadMode =
- RandomPicks.randomFrom(random(), CompletionPostingsFormat.FSTLoadMode.values());
- final PostingsFormat postingsFormat = new Completion101PostingsFormat(fstLoadMode);
+ final PostingsFormat postingsFormat = new Completion101PostingsFormat();
@Override
public PostingsFormat postingsFormat() {