Skip to content

Apache Lucene open-source search software

hunspell: add Suggester#proceedPastRep to avoid losing relevant suggestions (#13612)

Peter Gromov481ca2d30f53
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index b761a4fa058..703cef58580 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -253,6 +253,8 @@ API Changes
* GITHUB#13469: Expose FlatVectorsFormat as a first-class format; can be configured using a custom Codec. (Michael Sokolov)
+* GITHUB#13612: Hunspell: add Suggester#proceedPastRep to avoid losing relevant suggestions. (Peter Gromov)
+
New Features
---------------------
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/ModifyingSuggester.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/ModifyingSuggester.java
index 218150cb416..aa63a5ec80b 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/ModifyingSuggester.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/ModifyingSuggester.java
@@ -31,6 +31,7 @@ class ModifyingSuggester {
private final String misspelled;
private final WordCase wordCase;
private final FragmentChecker fragmentChecker;
+ private final boolean proceedPastRep;
private final char[] tryChars;
private final Hunspell speller;
@@ -39,13 +40,15 @@ class ModifyingSuggester {
LinkedHashSet<Suggestion> result,
String misspelled,
WordCase wordCase,
- FragmentChecker checker) {
+ FragmentChecker checker,
+ boolean proceedPastRep) {
this.speller = speller;
tryChars = speller.dictionary.tryChars.toCharArray();
this.result = result;
this.misspelled = misspelled;
this.wordCase = wordCase;
fragmentChecker = checker;
+ this.proceedPastRep = proceedPastRep;
}
/**
@@ -125,9 +128,9 @@ class ModifyingSuggester {
boolean hasGoodSuggestions = trySuggestion(word.toUpperCase(Locale.ROOT));
GradedSuggestions repResult = tryRep(word);
- if (repResult == GradedSuggestions.Best) return true;
+ if (repResult == GradedSuggestions.Best && !proceedPastRep) return true;
- hasGoodSuggestions |= repResult == GradedSuggestions.Normal;
+ hasGoodSuggestions |= repResult != GradedSuggestions.None;
if (!speller.dictionary.mapTable.isEmpty()) {
enumerateMapReplacements(word, "", 0);
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Suggester.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Suggester.java
index e6da3c90584..6d4c12d71b5 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Suggester.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Suggester.java
@@ -53,16 +53,21 @@ public class Suggester {
private final Dictionary dictionary;
private final SuggestibleEntryCache suggestibleCache;
private final FragmentChecker fragmentChecker;
+ private final boolean proceedPastRep;
public Suggester(Dictionary dictionary) {
- this(dictionary, null, FragmentChecker.EVERYTHING_POSSIBLE);
+ this(dictionary, null, FragmentChecker.EVERYTHING_POSSIBLE, false);
}
private Suggester(
- Dictionary dictionary, SuggestibleEntryCache suggestibleCache, FragmentChecker checker) {
+ Dictionary dictionary,
+ SuggestibleEntryCache suggestibleCache,
+ FragmentChecker checker,
+ boolean proceedPastRep) {
this.dictionary = dictionary;
this.suggestibleCache = suggestibleCache;
this.fragmentChecker = checker;
+ this.proceedPastRep = proceedPastRep;
}
/**
@@ -71,8 +76,8 @@ public class Suggester {
* entries are stored as fast-to-iterate plain words instead of highly compressed prefix trees.
*/
public Suggester withSuggestibleEntryCache() {
- return new Suggester(
- dictionary, SuggestibleEntryCache.buildCache(dictionary.words), fragmentChecker);
+ SuggestibleEntryCache cache = SuggestibleEntryCache.buildCache(dictionary.words);
+ return new Suggester(dictionary, cache, fragmentChecker, proceedPastRep);
}
/**
@@ -80,7 +85,17 @@ public class Suggester {
* the performance of the "Modification" phase performance.
*/
public Suggester withFragmentChecker(FragmentChecker checker) {
- return new Suggester(dictionary, suggestibleCache, checker);
+ return new Suggester(dictionary, suggestibleCache, checker, proceedPastRep);
+ }
+
+ /**
+ * Returns a copy of this suggester instance that doesn't stop after encountering acceptable words
+ * after applying REP rules. By default, Hunspell stops when it finds any, but this behavior may
+ * not always be desirable, e.g., if we have "REP i ea", "tims" be replaced only by "teams" and
+ * not "times", which could also be meant.
+ */
+ public Suggester proceedPastRep() {
+ return new Suggester(dictionary, suggestibleCache, fragmentChecker, true);
}
/**
@@ -174,7 +189,8 @@ public class Suggester {
}
boolean hasGoodSuggestions =
- new ModifyingSuggester(suggestionSpeller, suggestions, word, wordCase, fragmentChecker)
+ new ModifyingSuggester(
+ suggestionSpeller, suggestions, word, wordCase, fragmentChecker, proceedPastRep)
.suggest();
if (!hasGoodSuggestions && dictionary.maxNGramSuggestions > 0) {
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestSpellChecking.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestSpellChecking.java
index 4ca262799c6..6426fcc7a26 100644
--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestSpellChecking.java
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestSpellChecking.java
@@ -59,6 +59,14 @@ public class TestSpellChecking extends LuceneTestCase {
public void testRepSuggestions() throws Exception {
doTest("rep");
+
+ //noinspection DataFlowIssue
+ Path aff = Path.of(getClass().getResource("rep.aff").toURI());
+ Dictionary dictionary = TestAllDictionaries.loadDictionary(aff);
+ Suggester suggester = new Suggester(dictionary);
+ assertEquals(List.of("auto's"), suggester.suggestNoTimeout("autos", () -> {}));
+ assertEquals(
+ List.of("auto's", "auto"), suggester.proceedPastRep().suggestNoTimeout("autos", () -> {}));
}
public void testPhSuggestions() throws Exception {