Apache Solr open-source search software
LUCENE-9758: Hunspell: support NOSUGGEST option (#2340)
Peter Gromov7f9b1f991b86
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.javaindex e9a638441c1..3acf59ef951 100644--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java@@ -175,6 +175,7 @@ public class Dictionary {int maxDiff = 5;int maxNGramSuggestions = Integer.MAX_VALUE;boolean onlyMaxDiff;+ char noSuggest, subStandard;// FSTs used for ICONV/OCONV, output ord pointing to replacement textFST<CharsRef> iconv;@@ -431,6 +432,10 @@ public class Dictionary {onlyMaxDiff = true;} else if ("FORBIDDENWORD".equals(firstWord)) {forbiddenword = flagParsingStrategy.parseFlag(singleArgument(reader, line));+ } else if ("NOSUGGEST".equals(firstWord)) {+ noSuggest = flagParsingStrategy.parseFlag(singleArgument(reader, line));+ } else if ("SUBSTANDARD".equals(firstWord)) {+ subStandard = flagParsingStrategy.parseFlag(singleArgument(reader, line));} else if ("COMPOUNDMIN".equals(firstWord)) {compoundMin = Math.max(1, parseNum(reader, line));} else if ("COMPOUNDWORDMAX".equals(firstWord)) {diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/GeneratingSuggester.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/GeneratingSuggester.javaindex 9546aac2bb4..35eae7b9165 100644--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/GeneratingSuggester.java+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/GeneratingSuggester.java@@ -111,6 +111,7 @@ class GeneratingSuggester {for (int i = 0; i < forms.length; i += dictionary.formStep()) {int entryId = forms.ints[forms.offset + i];if (dictionary.hasFlag(entryId, dictionary.forbiddenword)+ || dictionary.hasFlag(entryId, dictionary.noSuggest)|| dictionary.hasFlag(entryId, Dictionary.HIDDEN_FLAG)|| dictionary.hasFlag(entryId, dictionary.onlyincompound)) {continue;diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/SpellChecker.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/SpellChecker.javaindex 790eca84a5c..97dea149a51 100644--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/SpellChecker.java+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/SpellChecker.java@@ -143,13 +143,19 @@ public class SpellChecker {length,originalCase,context,- (stem, forms, formID) -> {- result[0] = stem;+ (stem, formID, stemException) -> {+ if (acceptsStem(formID)) {+ result[0] = stem;+ }return false;});return result[0];}+ boolean acceptsStem(int formID) {+ return true;+ }+private boolean checkCompounds(CharsRef word, WordCase originalCase, CompoundPart prev) {if (prev != null && prev.index > dictionary.compoundMax - 2) return false;@@ -424,12 +430,20 @@ public class SpellChecker {}WordCase wordCase = WordCase.caseOf(word);- ModifyingSuggester modifier = new ModifyingSuggester(this);+ SpellChecker suggestionSpeller =+ new SpellChecker(dictionary) {+ @Override+ boolean acceptsStem(int formID) {+ return !dictionary.hasFlag(formID, dictionary.noSuggest)+ && !dictionary.hasFlag(formID, dictionary.subStandard);+ }+ };+ ModifyingSuggester modifier = new ModifyingSuggester(suggestionSpeller);Set<String> suggestions = modifier.suggest(word, wordCase);if (!modifier.hasGoodSuggestions && dictionary.maxNGramSuggestions > 0) {suggestions.addAll(- new GeneratingSuggester(this)+ new GeneratingSuggester(suggestionSpeller).suggest(dictionary.toLowerCase(word), wordCase, suggestions));}diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Stemmer.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Stemmer.javaindex 3c8e186761b..64b649d8cc3 100644--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Stemmer.java+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Stemmer.java@@ -100,8 +100,8 @@ final class Stemmer {List<CharsRef> list = new ArrayList<>();RootProcessor processor =- (stem, forms, formID) -> {- list.add(newStem(stem, forms, formID));+ (stem, formID, stemException) -> {+ list.add(newStem(stem, stemException));return true;};@@ -273,7 +273,7 @@ final class Stemmer {continue;}}- if (!processor.processRoot(new CharsRef(word, offset, length), forms, i)) {+ if (!callProcessor(word, offset, length, processor, forms, i)) {return false;}}@@ -344,23 +344,27 @@ final class Stemmer {}interface RootProcessor {- /** @return whether the processing should be continued */- boolean processRoot(CharsRef stem, IntsRef forms, int formID);+ /**+ * @param stem the text of the found dictionary entry+ * @param formID internal id of the dictionary entry, e.g. to be used in {@link+ * Dictionary#hasFlag(int, char)}+ * @param stemException "st:" morphological data if present, {@code null} otherwise+ * @return whether the processing should be continued+ */+ boolean processRoot(CharsRef stem, int formID, String stemException);}- private CharsRef newStem(CharsRef stem, IntsRef forms, int formID) {- final String exception;+ private String stemException(IntsRef forms, int formIndex) {if (dictionary.hasStemExceptions) {- int exceptionID = forms.ints[forms.offset + formID + 1];+ int exceptionID = forms.ints[forms.offset + formIndex + 1];if (exceptionID > 0) {- exception = dictionary.getStemException(exceptionID);- } else {- exception = null;+ return dictionary.getStemException(exceptionID);}- } else {- exception = null;}+ return null;+ }+ private CharsRef newStem(CharsRef stem, String exception) {if (dictionary.needsOutputCleaning) {scratchSegment.setLength(0);if (exception != null) {@@ -704,7 +708,7 @@ final class Stemmer {continue;}}- if (!processor.processRoot(new CharsRef(strippedWord, offset, length), forms, i)) {+ if (!callProcessor(strippedWord, offset, length, processor, forms, i)) {return false;}}@@ -757,6 +761,12 @@ final class Stemmer {return true;}+ private boolean callProcessor(+ char[] word, int offset, int length, RootProcessor processor, IntsRef forms, int i) {+ CharsRef stem = new CharsRef(word, offset, length);+ return processor.processRoot(stem, forms.ints[forms.offset + i], stemException(forms, i));+ }+private boolean needsAnotherAffix(int affix, int previousAffix, boolean isSuffix, int prefixId) {char circumfix = dictionary.circumfix;// if circumfix was previously set by a prefix, we must check this suffix,@@ -765,7 +775,6 @@ final class Stemmer {&& isFlagAppendedByAffix(prefixId, circumfix) != isFlagAppendedByAffix(affix, circumfix)) {return true;}-if (isFlagAppendedByAffix(affix, dictionary.needaffix)) {return !isSuffix|| previousAffix < 0diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/SpellCheckerTest.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/SpellCheckerTest.javaindex b71b6e30ef9..4dfcb61774f 100644--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/SpellCheckerTest.java+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/SpellCheckerTest.java@@ -196,6 +196,10 @@ public class SpellCheckerTest extends StemmerTestBase {doTest("map");}+ public void testNoSuggest() throws Exception {+ doTest("nosuggest");+ }+protected void doTest(String name) throws Exception {checkSpellCheckerExpectations(Path.of(getClass().getResource(name + ".aff").toURI()).getParent().resolve(name));diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/nosuggest.aff b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/nosuggest.affnew file mode 100644index 00000000000..c9361da4c66--- /dev/null+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/nosuggest.aff@@ -0,0 +1,5 @@+# don't suggest word with NOSUGGEST flag (for example vulgar or obscene words)+# See OpenOffice.org Issue #55498+# (nosuggest.sug is an empty file)+NOSUGGEST A+COMPOUNDFLAG Bdiff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/nosuggest.dic b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/nosuggest.dicnew file mode 100644index 00000000000..dc80c916d42--- /dev/null+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/nosuggest.dic@@ -0,0 +1,3 @@+1+foo/AB+bar/Bdiff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/nosuggest.good b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/nosuggest.goodnew file mode 100644index 00000000000..ad91a5e3138--- /dev/null+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/nosuggest.good@@ -0,0 +1,3 @@+foo+foobar+barfoodiff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/nosuggest.sug b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/nosuggest.sugnew file mode 100644index 00000000000..e69de29bb2ddiff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/nosuggest.wrong b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/nosuggest.wrongnew file mode 100644index 00000000000..89c7a1a9c0a--- /dev/null+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/nosuggest.wrong@@ -0,0 +1,3 @@+foox+foobarx+barfoox