Skip to content

Apache Lucene open-source search software

hunspell: add cancellation checks to `WordStorage` (#16527)

Ilia Permiashkin173eef076172
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 4b2f2ebac95..422332fcda5 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -196,6 +196,8 @@ Optimizations
* GITHUB#15085, GITHUB#15092: Hunspell suggestions: Ensure candidate roots are not worse before updating. (Ilia Permiashkin)
+* GITHUB#16527: Hunspell: check for cancellation while building the suggestible entry cache. (Ilia Permiashkin)
+
* GITHUB#15597, GITHUB#15777: Reduce memory usage of NeighborArray (Viliam Durina)
* GITHUB#12324: ELiminate redundant work in AbstractKnnVectorQuery.rewrite (Mike Sokolov)
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Suggester.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Suggester.java
index 4b361da8602..6bf134f9c1f 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Suggester.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Suggester.java
@@ -51,7 +51,8 @@ import org.apache.lucene.util.CharsRef;
*/
public class Suggester {
private final Dictionary dictionary;
- private final SuggestibleEntryCache suggestibleCache;
+ private volatile SuggestibleEntryCache suggestibleCache;
+ private final boolean useSuggestibleEntryCache;
private final FragmentChecker fragmentChecker;
private final boolean proceedPastRep;
private final DictionarySuggester suggester;
@@ -60,6 +61,7 @@ public class Suggester {
this(
dictionary,
null,
+ false,
FragmentChecker.EVERYTHING_POSSIBLE,
false,
(speller, entryCache, word, originalCase, prevSuggestions) ->
@@ -70,11 +72,13 @@ public class Suggester {
private Suggester(
Dictionary dictionary,
SuggestibleEntryCache suggestibleCache,
+ boolean useSuggestibleEntryCache,
FragmentChecker checker,
boolean proceedPastRep,
DictionarySuggester suggester) {
this.dictionary = dictionary;
this.suggestibleCache = suggestibleCache;
+ this.useSuggestibleEntryCache = useSuggestibleEntryCache;
this.fragmentChecker = checker;
this.proceedPastRep = proceedPastRep;
this.suggester = suggester;
@@ -86,8 +90,7 @@ public class Suggester {
* entries are stored as fast-to-iterate plain words instead of highly compressed prefix trees.
*/
public Suggester withSuggestibleEntryCache() {
- SuggestibleEntryCache cache = SuggestibleEntryCache.buildCache(dictionary.words);
- return new Suggester(dictionary, cache, fragmentChecker, proceedPastRep, suggester);
+ return new Suggester(dictionary, null, true, fragmentChecker, proceedPastRep, suggester);
}
/**
@@ -95,7 +98,8 @@ public class Suggester {
* the performance of the "Modification" phase performance.
*/
public Suggester withFragmentChecker(FragmentChecker checker) {
- return new Suggester(dictionary, suggestibleCache, checker, proceedPastRep, suggester);
+ return new Suggester(
+ dictionary, suggestibleCache, useSuggestibleEntryCache, checker, proceedPastRep, suggester);
}
/**
@@ -105,7 +109,8 @@ public class Suggester {
* not "times", which could also be meant.
*/
public Suggester proceedPastRep() {
- return new Suggester(dictionary, suggestibleCache, fragmentChecker, true, suggester);
+ return new Suggester(
+ dictionary, suggestibleCache, useSuggestibleEntryCache, fragmentChecker, true, suggester);
}
/**
@@ -114,7 +119,12 @@ public class Suggester {
*/
public Suggester withDictionarySuggester(DictionarySuggester dictionarySuggester) {
return new Suggester(
- dictionary, suggestibleCache, fragmentChecker, proceedPastRep, dictionarySuggester);
+ dictionary,
+ suggestibleCache,
+ useSuggestibleEntryCache,
+ fragmentChecker,
+ proceedPastRep,
+ dictionarySuggester);
}
/**
@@ -213,6 +223,7 @@ public class Suggester {
.suggest();
if (!hasGoodSuggestions && dictionary.maxNGramSuggestions > 0) {
+ SuggestibleEntryCache suggestibleCache = getSuggestibleEntryCache(checkCanceled);
List<String> generated =
this.suggester.suggest(
suggestionSpeller,
@@ -233,6 +244,17 @@ public class Suggester {
return postprocess(suggestions);
}
+ private SuggestibleEntryCache getSuggestibleEntryCache(Runnable checkCanceled) {
+ if (useSuggestibleEntryCache && suggestibleCache == null) {
+ synchronized (this) {
+ if (suggestibleCache == null) {
+ suggestibleCache = SuggestibleEntryCache.buildCache(dictionary.words, checkCanceled);
+ }
+ }
+ }
+ return suggestibleCache;
+ }
+
private Runnable checkTimeLimit(
String word, Set<Suggestion> suggestions, long timeLimitMs, Runnable checkCanceled) {
return new Runnable() {
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/SuggestibleEntryCache.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/SuggestibleEntryCache.java
index 01ecb3e5550..8d19ad9229a 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/SuggestibleEntryCache.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/SuggestibleEntryCache.java
@@ -42,7 +42,7 @@ public class SuggestibleEntryCache {
}
}
- static SuggestibleEntryCache buildCache(WordStorage storage) {
+ static SuggestibleEntryCache buildCache(WordStorage storage, Runnable checkCanceled) {
var consumer =
new Consumer<FlyweightEntry>() {
final IntObjectHashMap<SectionBuilder> builders = new IntObjectHashMap<>();
@@ -50,6 +50,7 @@ public class SuggestibleEntryCache {
@Override
public void accept(FlyweightEntry entry) {
+ checkCanceled.run();
CharsRef root = entry.root();
if (root.length > Short.MAX_VALUE) {
throw new UnsupportedOperationException(
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestDictionary.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestDictionary.java
index 3482a0215f3..05a81c3596f 100644
--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestDictionary.java
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestDictionary.java
@@ -111,7 +111,7 @@ public class TestDictionary extends LuceneTestCase {
minLength, maxLength, e -> processed.add(e.root().toString()));
Set<String> cached = new HashSet<>();
- SuggestibleEntryCache.buildCache(dictionary.words)
+ SuggestibleEntryCache.buildCache(dictionary.words, () -> {})
.processSuggestibleWords(minLength, maxLength, e -> cached.add(e.root().toString()));
assertEquals(processed, cached);