Apache Lucene open-source search software
hunspell: add cancellation checks to `WordStorage` (#16527)
Ilia Permiashkin173eef076172
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex 4b2f2ebac95..422332fcda5 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -196,6 +196,8 @@ Optimizations* GITHUB#15085, GITHUB#15092: Hunspell suggestions: Ensure candidate roots are not worse before updating. (Ilia Permiashkin)+* GITHUB#16527: Hunspell: check for cancellation while building the suggestible entry cache. (Ilia Permiashkin)+* GITHUB#15597, GITHUB#15777: Reduce memory usage of NeighborArray (Viliam Durina)* GITHUB#12324: ELiminate redundant work in AbstractKnnVectorQuery.rewrite (Mike Sokolov)diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Suggester.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Suggester.javaindex 4b361da8602..6bf134f9c1f 100644--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Suggester.java+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Suggester.java@@ -51,7 +51,8 @@ import org.apache.lucene.util.CharsRef;*/public class Suggester {private final Dictionary dictionary;- private final SuggestibleEntryCache suggestibleCache;+ private volatile SuggestibleEntryCache suggestibleCache;+ private final boolean useSuggestibleEntryCache;private final FragmentChecker fragmentChecker;private final boolean proceedPastRep;private final DictionarySuggester suggester;@@ -60,6 +61,7 @@ public class Suggester {this(dictionary,null,+ false,FragmentChecker.EVERYTHING_POSSIBLE,false,(speller, entryCache, word, originalCase, prevSuggestions) ->@@ -70,11 +72,13 @@ public class Suggester {private Suggester(Dictionary dictionary,SuggestibleEntryCache suggestibleCache,+ boolean useSuggestibleEntryCache,FragmentChecker checker,boolean proceedPastRep,DictionarySuggester suggester) {this.dictionary = dictionary;this.suggestibleCache = suggestibleCache;+ this.useSuggestibleEntryCache = useSuggestibleEntryCache;this.fragmentChecker = checker;this.proceedPastRep = proceedPastRep;this.suggester = suggester;@@ -86,8 +90,7 @@ public class Suggester {* entries are stored as fast-to-iterate plain words instead of highly compressed prefix trees.*/public Suggester withSuggestibleEntryCache() {- SuggestibleEntryCache cache = SuggestibleEntryCache.buildCache(dictionary.words);- return new Suggester(dictionary, cache, fragmentChecker, proceedPastRep, suggester);+ return new Suggester(dictionary, null, true, fragmentChecker, proceedPastRep, suggester);}/**@@ -95,7 +98,8 @@ public class Suggester {* the performance of the "Modification" phase performance.*/public Suggester withFragmentChecker(FragmentChecker checker) {- return new Suggester(dictionary, suggestibleCache, checker, proceedPastRep, suggester);+ return new Suggester(+ dictionary, suggestibleCache, useSuggestibleEntryCache, checker, proceedPastRep, suggester);}/**@@ -105,7 +109,8 @@ public class Suggester {* not "times", which could also be meant.*/public Suggester proceedPastRep() {- return new Suggester(dictionary, suggestibleCache, fragmentChecker, true, suggester);+ return new Suggester(+ dictionary, suggestibleCache, useSuggestibleEntryCache, fragmentChecker, true, suggester);}/**@@ -114,7 +119,12 @@ public class Suggester {*/public Suggester withDictionarySuggester(DictionarySuggester dictionarySuggester) {return new Suggester(- dictionary, suggestibleCache, fragmentChecker, proceedPastRep, dictionarySuggester);+ dictionary,+ suggestibleCache,+ useSuggestibleEntryCache,+ fragmentChecker,+ proceedPastRep,+ dictionarySuggester);}/**@@ -213,6 +223,7 @@ public class Suggester {.suggest();if (!hasGoodSuggestions && dictionary.maxNGramSuggestions > 0) {+ SuggestibleEntryCache suggestibleCache = getSuggestibleEntryCache(checkCanceled);List<String> generated =this.suggester.suggest(suggestionSpeller,@@ -233,6 +244,17 @@ public class Suggester {return postprocess(suggestions);}+ private SuggestibleEntryCache getSuggestibleEntryCache(Runnable checkCanceled) {+ if (useSuggestibleEntryCache && suggestibleCache == null) {+ synchronized (this) {+ if (suggestibleCache == null) {+ suggestibleCache = SuggestibleEntryCache.buildCache(dictionary.words, checkCanceled);+ }+ }+ }+ return suggestibleCache;+ }+private Runnable checkTimeLimit(String word, Set<Suggestion> suggestions, long timeLimitMs, Runnable checkCanceled) {return new Runnable() {diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/SuggestibleEntryCache.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/SuggestibleEntryCache.javaindex 01ecb3e5550..8d19ad9229a 100644--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/SuggestibleEntryCache.java+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/SuggestibleEntryCache.java@@ -42,7 +42,7 @@ public class SuggestibleEntryCache {}}- static SuggestibleEntryCache buildCache(WordStorage storage) {+ static SuggestibleEntryCache buildCache(WordStorage storage, Runnable checkCanceled) {var consumer =new Consumer<FlyweightEntry>() {final IntObjectHashMap<SectionBuilder> builders = new IntObjectHashMap<>();@@ -50,6 +50,7 @@ public class SuggestibleEntryCache {@Overridepublic void accept(FlyweightEntry entry) {+ checkCanceled.run();CharsRef root = entry.root();if (root.length > Short.MAX_VALUE) {throw new UnsupportedOperationException(diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestDictionary.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestDictionary.javaindex 3482a0215f3..05a81c3596f 100644--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestDictionary.java+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestDictionary.java@@ -111,7 +111,7 @@ public class TestDictionary extends LuceneTestCase {minLength, maxLength, e -> processed.add(e.root().toString()));Set<String> cached = new HashSet<>();- SuggestibleEntryCache.buildCache(dictionary.words)+ SuggestibleEntryCache.buildCache(dictionary.words, () -> {}).processSuggestibleWords(minLength, maxLength, e -> cached.add(e.root().toString()));assertEquals(processed, cached);