Apache Lucene open-source search software
hunspell: add Suggester#proceedPastRep to avoid losing relevant suggestions (#13612)
Peter Gromov481ca2d30f53
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex b761a4fa058..703cef58580 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -253,6 +253,8 @@ API Changes* GITHUB#13469: Expose FlatVectorsFormat as a first-class format; can be configured using a custom Codec. (Michael Sokolov)+* GITHUB#13612: Hunspell: add Suggester#proceedPastRep to avoid losing relevant suggestions. (Peter Gromov)+New Features---------------------diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/ModifyingSuggester.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/ModifyingSuggester.javaindex 218150cb416..aa63a5ec80b 100644--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/ModifyingSuggester.java+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/ModifyingSuggester.java@@ -31,6 +31,7 @@ class ModifyingSuggester {private final String misspelled;private final WordCase wordCase;private final FragmentChecker fragmentChecker;+ private final boolean proceedPastRep;private final char[] tryChars;private final Hunspell speller;@@ -39,13 +40,15 @@ class ModifyingSuggester {LinkedHashSet<Suggestion> result,String misspelled,WordCase wordCase,- FragmentChecker checker) {+ FragmentChecker checker,+ boolean proceedPastRep) {this.speller = speller;tryChars = speller.dictionary.tryChars.toCharArray();this.result = result;this.misspelled = misspelled;this.wordCase = wordCase;fragmentChecker = checker;+ this.proceedPastRep = proceedPastRep;}/**@@ -125,9 +128,9 @@ class ModifyingSuggester {boolean hasGoodSuggestions = trySuggestion(word.toUpperCase(Locale.ROOT));GradedSuggestions repResult = tryRep(word);- if (repResult == GradedSuggestions.Best) return true;+ if (repResult == GradedSuggestions.Best && !proceedPastRep) return true;- hasGoodSuggestions |= repResult == GradedSuggestions.Normal;+ hasGoodSuggestions |= repResult != GradedSuggestions.None;if (!speller.dictionary.mapTable.isEmpty()) {enumerateMapReplacements(word, "", 0);diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Suggester.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Suggester.javaindex e6da3c90584..6d4c12d71b5 100644--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Suggester.java+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Suggester.java@@ -53,16 +53,21 @@ public class Suggester {private final Dictionary dictionary;private final SuggestibleEntryCache suggestibleCache;private final FragmentChecker fragmentChecker;+ private final boolean proceedPastRep;public Suggester(Dictionary dictionary) {- this(dictionary, null, FragmentChecker.EVERYTHING_POSSIBLE);+ this(dictionary, null, FragmentChecker.EVERYTHING_POSSIBLE, false);}private Suggester(- Dictionary dictionary, SuggestibleEntryCache suggestibleCache, FragmentChecker checker) {+ Dictionary dictionary,+ SuggestibleEntryCache suggestibleCache,+ FragmentChecker checker,+ boolean proceedPastRep) {this.dictionary = dictionary;this.suggestibleCache = suggestibleCache;this.fragmentChecker = checker;+ this.proceedPastRep = proceedPastRep;}/**@@ -71,8 +76,8 @@ public class Suggester {* entries are stored as fast-to-iterate plain words instead of highly compressed prefix trees.*/public Suggester withSuggestibleEntryCache() {- return new Suggester(- dictionary, SuggestibleEntryCache.buildCache(dictionary.words), fragmentChecker);+ SuggestibleEntryCache cache = SuggestibleEntryCache.buildCache(dictionary.words);+ return new Suggester(dictionary, cache, fragmentChecker, proceedPastRep);}/**@@ -80,7 +85,17 @@ public class Suggester {* the performance of the "Modification" phase performance.*/public Suggester withFragmentChecker(FragmentChecker checker) {- return new Suggester(dictionary, suggestibleCache, checker);+ return new Suggester(dictionary, suggestibleCache, checker, proceedPastRep);+ }++ /**+ * Returns a copy of this suggester instance that doesn't stop after encountering acceptable words+ * after applying REP rules. By default, Hunspell stops when it finds any, but this behavior may+ * not always be desirable, e.g., if we have "REP i ea", "tims" be replaced only by "teams" and+ * not "times", which could also be meant.+ */+ public Suggester proceedPastRep() {+ return new Suggester(dictionary, suggestibleCache, fragmentChecker, true);}/**@@ -174,7 +189,8 @@ public class Suggester {}boolean hasGoodSuggestions =- new ModifyingSuggester(suggestionSpeller, suggestions, word, wordCase, fragmentChecker)+ new ModifyingSuggester(+ suggestionSpeller, suggestions, word, wordCase, fragmentChecker, proceedPastRep).suggest();if (!hasGoodSuggestions && dictionary.maxNGramSuggestions > 0) {diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestSpellChecking.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestSpellChecking.javaindex 4ca262799c6..6426fcc7a26 100644--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestSpellChecking.java+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestSpellChecking.java@@ -59,6 +59,14 @@ public class TestSpellChecking extends LuceneTestCase {public void testRepSuggestions() throws Exception {doTest("rep");++ //noinspection DataFlowIssue+ Path aff = Path.of(getClass().getResource("rep.aff").toURI());+ Dictionary dictionary = TestAllDictionaries.loadDictionary(aff);+ Suggester suggester = new Suggester(dictionary);+ assertEquals(List.of("auto's"), suggester.suggestNoTimeout("autos", () -> {}));+ assertEquals(+ List.of("auto's", "auto"), suggester.proceedPastRep().suggestNoTimeout("autos", () -> {}));}public void testPhSuggestions() throws Exception {