Apache Lucene open-source search software
Completion FSTs to be loaded off-heap at all times (#14364)
Luca Cavannafe86094f2f47
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex 012f79c0d7c..c215696ad98 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -145,6 +145,7 @@ Improvements* GITHUB#14213: Allowing indexing stored-only StoredField directly from DataInput. (Tim Brooks)+* GITHUB#14364: Completion FSTs to be loaded off-heap at all times. (Luca Cavanna)Optimizations---------------------diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion101PostingsFormat.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion101PostingsFormat.javaindex 2a51f01fd7a..e9dde1705b9 100644--- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion101PostingsFormat.java+++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion101PostingsFormat.java@@ -25,17 +25,9 @@ import org.apache.lucene.codecs.PostingsFormat;* @lucene.experimental*/public class Completion101PostingsFormat extends CompletionPostingsFormat {- /** Creates a {@link Completion101PostingsFormat} that will load the completion FST on-heap. */+ /** Creates a {@link Completion101PostingsFormat}. */public Completion101PostingsFormat() {- this(FSTLoadMode.ON_HEAP);- }-- /**- * Creates a {@link Completion101PostingsFormat} that will use the provided <code>fstLoadMode- * </code> to determine if the completion FST should be loaded on or off heap.- */- public Completion101PostingsFormat(FSTLoadMode fstLoadMode) {- super("Completion101", fstLoadMode);+ super("Completion101");}@Overridediff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion50PostingsFormat.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion50PostingsFormat.javaindex e53e991db22..08ed3802fa8 100644--- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion50PostingsFormat.java+++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion50PostingsFormat.java@@ -27,17 +27,9 @@ import org.apache.lucene.codecs.PostingsFormat;* @lucene.experimental*/public class Completion50PostingsFormat extends CompletionPostingsFormat {- /** Creates a {@link Completion50PostingsFormat} that will load the completion FST on-heap. */+ /** Creates a {@link Completion50PostingsFormat}. */public Completion50PostingsFormat() {- this(FSTLoadMode.ON_HEAP);- }-- /**- * Creates a {@link Completion50PostingsFormat} that will use the provided <code>fstLoadMode- * </code> to determine if the completion FST should be loaded on or off heap.- */- public Completion50PostingsFormat(FSTLoadMode fstLoadMode) {- super("completion", fstLoadMode);+ super("completion");}@Overridediff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion84PostingsFormat.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion84PostingsFormat.javaindex 76e29811834..406f5d6affa 100644--- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion84PostingsFormat.java+++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion84PostingsFormat.java@@ -27,17 +27,9 @@ import org.apache.lucene.codecs.PostingsFormat;* @lucene.experimental*/public class Completion84PostingsFormat extends CompletionPostingsFormat {- /** Creates a {@link Completion84PostingsFormat} that will load the completion FST on-heap. */+ /** Creates a {@link Completion84PostingsFormat}. */public Completion84PostingsFormat() {- this(FSTLoadMode.ON_HEAP);- }-- /**- * Creates a {@link Completion84PostingsFormat} that will use the provided <code>fstLoadMode- * </code> to determine if the completion FST should be loaded on or off heap.- */- public Completion84PostingsFormat(FSTLoadMode fstLoadMode) {- super("Completion84", fstLoadMode);+ super("Completion84");}@Overridediff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion90PostingsFormat.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion90PostingsFormat.javaindex db45303bcd7..58d6071a424 100644--- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion90PostingsFormat.java+++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion90PostingsFormat.java@@ -27,17 +27,9 @@ import org.apache.lucene.codecs.PostingsFormat;* @lucene.experimental*/public class Completion90PostingsFormat extends CompletionPostingsFormat {- /** Creates a {@link Completion90PostingsFormat} that will load the completion FST on-heap. */+ /** Creates a {@link Completion90PostingsFormat}. */public Completion90PostingsFormat() {- this(FSTLoadMode.ON_HEAP);- }-- /**- * Creates a {@link Completion90PostingsFormat} that will use the provided <code>fstLoadMode- * </code> to determine if the completion FST should be loaded on or off heap.- */- public Completion90PostingsFormat(FSTLoadMode fstLoadMode) {- super("Completion90", fstLoadMode);+ super("Completion90");}@Overridediff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion912PostingsFormat.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion912PostingsFormat.javaindex 77a62443f31..3f170063149 100644--- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion912PostingsFormat.java+++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion912PostingsFormat.java@@ -27,17 +27,9 @@ import org.apache.lucene.codecs.PostingsFormat;* @lucene.experimental*/public class Completion912PostingsFormat extends CompletionPostingsFormat {- /** Creates a {@link Completion912PostingsFormat} that will load the completion FST on-heap. */+ /** Creates a {@link Completion912PostingsFormat}. */public Completion912PostingsFormat() {- this(FSTLoadMode.ON_HEAP);- }-- /**- * Creates a {@link Completion912PostingsFormat} that will use the provided <code>fstLoadMode- * </code> to determine if the completion FST should be loaded on or off heap.- */- public Completion912PostingsFormat(FSTLoadMode fstLoadMode) {- super("Completion912", fstLoadMode);+ super("Completion912");}@Overridediff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion99PostingsFormat.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion99PostingsFormat.javaindex d3ea76d5425..2d3509ea4be 100644--- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion99PostingsFormat.java+++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion99PostingsFormat.java@@ -27,17 +27,9 @@ import org.apache.lucene.codecs.PostingsFormat;* @lucene.experimental*/public class Completion99PostingsFormat extends CompletionPostingsFormat {- /** Creates a {@link Completion99PostingsFormat} that will load the completion FST on-heap. */+ /** Creates a {@link Completion99PostingsFormat}. */public Completion99PostingsFormat() {- this(FSTLoadMode.ON_HEAP);- }-- /**- * Creates a {@link Completion99PostingsFormat} that will use the provided <code>fstLoadMode- * </code> to determine if the completion FST should be loaded on or off heap.- */- public Completion99PostingsFormat(FSTLoadMode fstLoadMode) {- super("Completion99", fstLoadMode);+ super("Completion99");}@Overridediff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/CompletionFieldsProducer.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/CompletionFieldsProducer.javaindex 86a42af9fad..13f3df5c19e 100644--- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/CompletionFieldsProducer.java+++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/CompletionFieldsProducer.java@@ -35,7 +35,6 @@ import org.apache.lucene.index.FieldInfo;import org.apache.lucene.index.IndexFileNames;import org.apache.lucene.index.SegmentReadState;import org.apache.lucene.index.Terms;-import org.apache.lucene.search.suggest.document.CompletionPostingsFormat.FSTLoadMode;import org.apache.lucene.store.ChecksumIndexInput;import org.apache.lucene.store.IndexInput;import org.apache.lucene.util.Accountable;@@ -56,7 +55,7 @@ import org.apache.lucene.util.IOUtils;final class CompletionFieldsProducer extends FieldsProducer implements Accountable {private FieldsProducer delegateFieldsProducer;- private Map<String, CompletionsTermsReader> readers;+ private final Map<String, CompletionsTermsReader> readers;private IndexInput dictIn;// copy ctr for merge instance@@ -66,8 +65,7 @@ final class CompletionFieldsProducer extends FieldsProducer implements Accountabthis.readers = readers;}- CompletionFieldsProducer(String codecName, SegmentReadState state, FSTLoadMode fstLoadMode)- throws IOException {+ CompletionFieldsProducer(String codecName, SegmentReadState state) throws IOException {String indexFile =IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, INDEX_EXTENSION);@@ -114,8 +112,7 @@ final class CompletionFieldsProducer extends FieldsProducer implements AccountabFieldInfo fieldInfo = state.fieldInfos.fieldInfo(fieldNumber);// we don't load the FST yetreaders.put(- fieldInfo.name,- new CompletionsTermsReader(dictIn, offset, minWeight, maxWeight, type, fstLoadMode));+ fieldInfo.name, new CompletionsTermsReader(dictIn, offset, minWeight, maxWeight, type));}CodecUtil.checkFooter(index);success = true;diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/CompletionPostingsFormat.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/CompletionPostingsFormat.javaindex 146487d7774..278b2037822 100644--- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/CompletionPostingsFormat.java+++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/CompletionPostingsFormat.java@@ -104,36 +104,9 @@ public abstract class CompletionPostingsFormat extends PostingsFormat {static final String INDEX_EXTENSION = "cmp";static final String DICT_EXTENSION = "lkp";- /** An enum that allows to control if suggester FSTs are loaded into memory or read off-heap */- public enum FSTLoadMode {- /**- * Always read FSTs from disk. NOTE: If this option is used the FST will be read off-heap even- * if buffered directory implementations are used.- */- OFF_HEAP,- /** Never read FSTs from disk ie. all suggest fields FSTs are loaded into memory */- ON_HEAP,- /**- * Automatically make the decision if FSTs are read from disk depending if the segment read from- * an MMAPDirectory- */- AUTO- }-- private final FSTLoadMode fstLoadMode;-- /** Used only by core Lucene at read-time via Service Provider instantiation */+ /** Creates a {@link CompletionPostingsFormat} with the given name. */public CompletionPostingsFormat(String name) {- this(name, FSTLoadMode.ON_HEAP);- }-- /**- * Creates a {@link CompletionPostingsFormat} that will use the provided <code>fstLoadMode</code>- * to determine if the completion FST should be loaded on or off heap.- */- public CompletionPostingsFormat(String name, FSTLoadMode fstLoadMode) {super(name);- this.fstLoadMode = fstLoadMode;}/** Concrete implementation should specify the delegating postings format */@@ -153,6 +126,6 @@ public abstract class CompletionPostingsFormat extends PostingsFormat {@Overridepublic FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {- return new CompletionFieldsProducer(getName(), state, fstLoadMode);+ return new CompletionFieldsProducer(getName(), state);}}diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/CompletionsTermsReader.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/CompletionsTermsReader.javaindex f8e22256f75..26e77b5f9c3 100644--- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/CompletionsTermsReader.java+++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/CompletionsTermsReader.java@@ -19,7 +19,6 @@ package org.apache.lucene.search.suggest.document;import java.io.IOException;import java.util.Collection;import java.util.Collections;-import org.apache.lucene.search.suggest.document.CompletionPostingsFormat.FSTLoadMode;import org.apache.lucene.store.IndexInput;import org.apache.lucene.util.Accountable;@@ -41,8 +40,6 @@ public final class CompletionsTermsReader implements Accountable {private final IndexInput dictIn;private final long offset;- private final FSTLoadMode fstLoadMode;-private NRTSuggester suggester;/**@@ -50,12 +47,7 @@ public final class CompletionsTermsReader implements Accountable {* </code> with <code>offset</code>*/CompletionsTermsReader(- IndexInput dictIn,- long offset,- long minWeight,- long maxWeight,- byte type,- FSTLoadMode fstLoadMode) {+ IndexInput dictIn, long offset, long minWeight, long maxWeight, byte type) {assert minWeight <= maxWeight;assert offset >= 0l && offset < dictIn.length();this.dictIn = dictIn;@@ -63,7 +55,6 @@ public final class CompletionsTermsReader implements Accountable {this.minWeight = minWeight;this.maxWeight = maxWeight;this.type = type;- this.fstLoadMode = fstLoadMode;}/**@@ -73,7 +64,7 @@ public final class CompletionsTermsReader implements Accountable {public synchronized NRTSuggester suggester() throws IOException {if (suggester == null) {IndexInput indexInput = dictIn.slice("NRTSuggester", offset, dictIn.length() - offset);- suggester = NRTSuggester.load(indexInput, fstLoadMode);+ suggester = NRTSuggester.load(indexInput);}return suggester;}diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/NRTSuggester.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/NRTSuggester.javaindex a923886d6ce..ef365682da8 100644--- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/NRTSuggester.java+++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/NRTSuggester.java@@ -24,7 +24,6 @@ import java.util.Collections;import java.util.Comparator;import java.util.List;import org.apache.lucene.search.suggest.analyzing.FSTUtil;-import org.apache.lucene.search.suggest.document.CompletionPostingsFormat.FSTLoadMode;import org.apache.lucene.store.ByteArrayDataInput;import org.apache.lucene.store.ByteArrayDataOutput;import org.apache.lucene.store.IndexInput;@@ -306,37 +305,15 @@ public final class NRTSuggester implements Accountable {return (numDocs > 0) ? ((double) numDocs / maxDocs) : -1;}- private static boolean shouldLoadFSTOffHeap(IndexInput input, FSTLoadMode fstLoadMode) {- switch (fstLoadMode) {- case ON_HEAP:- return false;- case OFF_HEAP:- return true;- case AUTO:- // TODO: Make this less hacky to maybe expose "off-heap" feature using a marker interface on- // the IndexInput- return input.getClass().getName().contains(".MemorySegmentIndexInput");- default:- throw new IllegalStateException("unknown enum constant: " + fstLoadMode);- }- }-- /**- * Loads a {@link NRTSuggester} from {@link org.apache.lucene.store.IndexInput} on or off-heap- * depending on the provided <code>fstLoadMode</code>- */- static NRTSuggester load(IndexInput input, FSTLoadMode fstLoadMode) throws IOException {+ /** Loads a {@link NRTSuggester} from {@link org.apache.lucene.store.IndexInput} */+ static NRTSuggester load(IndexInput input) throws IOException {final FST<Pair<Long, BytesRef>> fst;PairOutputs<Long, BytesRef> outputs =new PairOutputs<>(PositiveIntOutputs.getSingleton(), ByteSequenceOutputs.getSingleton());- if (shouldLoadFSTOffHeap(input, fstLoadMode)) {- final FST.FSTMetadata<Pair<Long, BytesRef>> fstMetadata = FST.readMetadata(input, outputs);- OffHeapFSTStore store = new OffHeapFSTStore(input, input.getFilePointer(), fstMetadata);- fst = FST.fromFSTReader(fstMetadata, store);- input.skipBytes(store.size());- } else {- fst = new FST<>(FST.readMetadata(input, outputs), input);- }+ final FST.FSTMetadata<Pair<Long, BytesRef>> fstMetadata = FST.readMetadata(input, outputs);+ OffHeapFSTStore store = new OffHeapFSTStore(input, input.getFilePointer(), fstMetadata);+ fst = FST.fromFSTReader(fstMetadata, store);+ input.skipBytes(store.size());/* read some meta info */int maxAnalyzedPathsPerOutput = input.readVInt();diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/NRTSuggesterBuilder.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/NRTSuggesterBuilder.javaindex 7aec1568ad7..8d983a29a5e 100644--- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/NRTSuggesterBuilder.java+++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/NRTSuggesterBuilder.java@@ -99,10 +99,7 @@ final class NRTSuggesterBuilder {entries.clear();}- /**- * Builds and stores a FST that can be loaded with {@link NRTSuggester#load(IndexInput,- * CompletionPostingsFormat.FSTLoadMode)})}- */+ /** Builds and stores a FST that can be loaded with {@link NRTSuggester#load(IndexInput)})} */public boolean store(DataOutput output) throws IOException {final FST<PairOutputs.Pair<Long, BytesRef>> fst =FST.fromFSTReader(fstCompiler.compile(), fstCompiler.getFSTReader());diff --git a/lucene/suggest/src/test/org/apache/lucene/search/suggest/document/TestSuggestField.java b/lucene/suggest/src/test/org/apache/lucene/search/suggest/document/TestSuggestField.javaindex 4a59d09d35c..1990baffe72 100644--- a/lucene/suggest/src/test/org/apache/lucene/search/suggest/document/TestSuggestField.java+++ b/lucene/suggest/src/test/org/apache/lucene/search/suggest/document/TestSuggestField.java@@ -20,7 +20,6 @@ import static org.apache.lucene.tests.analysis.BaseTokenStreamTestCase.assertTokimport static org.hamcrest.Matchers.containsString;import static org.hamcrest.Matchers.startsWith;-import com.carrotsearch.randomizedtesting.generators.RandomPicks;import java.io.ByteArrayOutputStream;import java.io.IOException;import java.util.ArrayList;@@ -949,9 +948,7 @@ public class TestSuggestField extends LuceneTestCase {iwc.setMergePolicy(newLogMergePolicy());Codec filterCodec =new FilterCodec(TestUtil.getDefaultCodec().getName(), TestUtil.getDefaultCodec()) {- final CompletionPostingsFormat.FSTLoadMode fstLoadMode =- RandomPicks.randomFrom(random(), CompletionPostingsFormat.FSTLoadMode.values());- final PostingsFormat postingsFormat = new Completion101PostingsFormat(fstLoadMode);+ final PostingsFormat postingsFormat = new Completion101PostingsFormat();@Overridepublic PostingsFormat postingsFormat() {