Apache Solr open-source search software
LUCENE-9748: Hunspell: suggest inflected dictionary entries similar to the misspelled word (#2330)
Peter Gromovf2b7cdc49190
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.javaindex b175cdaef9b..f39575c704c 100644--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java@@ -122,7 +122,7 @@ public class Dictionary {// offsets in affixDatastatic final int AFFIX_FLAG = 0;static final int AFFIX_STRIP_ORD = 1;- static final int AFFIX_CONDITION = 2;+ private static final int AFFIX_CONDITION = 2;static final int AFFIX_APPEND = 3;// Default flag parsing strategy@@ -779,6 +779,14 @@ public class Dictionary {return affixData[affixIndex * 4 + offset];}+ boolean isCrossProduct(int affix) {+ return (affixData(affix, AFFIX_CONDITION) & 1) == 1;+ }++ int getAffixCondition(int affix) {+ return affixData(affix, AFFIX_CONDITION) >>> 1;+ }+private FST<CharsRef> parseConversions(LineNumberReader reader, int num)throws IOException, ParseException {Map<String, String> mappings = new TreeMap<>();diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/GeneratingSuggester.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/GeneratingSuggester.javaindex 6a423960b29..9546aac2bb4 100644--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/GeneratingSuggester.java+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/GeneratingSuggester.java@@ -16,15 +16,22 @@*/package org.apache.lucene.analysis.hunspell;+import static org.apache.lucene.analysis.hunspell.Dictionary.AFFIX_APPEND;+import static org.apache.lucene.analysis.hunspell.Dictionary.AFFIX_FLAG;+import static org.apache.lucene.analysis.hunspell.Dictionary.AFFIX_STRIP_ORD;+import java.io.IOException;import java.util.ArrayList;import java.util.EnumSet;+import java.util.LinkedHashSet;import java.util.List;import java.util.Objects;import java.util.Set;import java.util.TreeSet;+import java.util.function.BiConsumer;import java.util.stream.Collectors;import org.apache.lucene.util.IntsRef;+import org.apache.lucene.util.fst.FST;import org.apache.lucene.util.fst.IntsRefFSTEnum;/**@@ -33,44 +40,59 @@ import org.apache.lucene.util.fst.IntsRefFSTEnum;*/class GeneratingSuggester {private static final int MAX_ROOTS = 100;- private static final int MAX_GUESSES = 100;+ private static final int MAX_WORDS = 100;+ private static final int MAX_GUESSES = 200;private final Dictionary dictionary;+ private final SpellChecker speller;- GeneratingSuggester(Dictionary dictionary) {- this.dictionary = dictionary;+ GeneratingSuggester(SpellChecker speller) {+ this.dictionary = speller.dictionary;+ this.speller = speller;}List<String> suggest(String word, WordCase originalCase, Set<String> prevSuggestions) {- List<WeightedWord> roots = findSimilarDictionaryEntries(word, originalCase);- List<WeightedWord> expanded = expandRoots(word, roots);- TreeSet<WeightedWord> bySimilarity = rankBySimilarity(word, expanded);+ List<Weighted<DictEntry>> roots = findSimilarDictionaryEntries(word, originalCase);+ List<Weighted<String>> expanded = expandRoots(word, roots);+ TreeSet<Weighted<String>> bySimilarity = rankBySimilarity(word, expanded);return getMostRelevantSuggestions(bySimilarity, prevSuggestions);}- private List<WeightedWord> findSimilarDictionaryEntries(String word, WordCase originalCase) {- try {- IntsRefFSTEnum<IntsRef> fstEnum = new IntsRefFSTEnum<>(dictionary.words);- TreeSet<WeightedWord> roots = new TreeSet<>();+ private List<Weighted<DictEntry>> findSimilarDictionaryEntries(+ String word, WordCase originalCase) {+ TreeSet<Weighted<DictEntry>> roots = new TreeSet<>();+ processFST(+ dictionary.words,+ (key, forms) -> {+ if (Math.abs(key.length - word.length()) > 4) return;++ String root = toString(key);+ List<DictEntry> entries = filterSuitableEntries(root, forms);+ if (entries.isEmpty()) return;++ if (originalCase == WordCase.LOWER+ && WordCase.caseOf(root) == WordCase.TITLE+ && !dictionary.hasLanguage("de")) {+ return;+ }- IntsRefFSTEnum.InputOutput<IntsRef> mapping;- while ((mapping = fstEnum.next()) != null) {- IntsRef key = mapping.input;- if (Math.abs(key.length - word.length()) > 4 || !isSuitableRoot(mapping.output)) continue;-- String root = toString(key);- if (originalCase == WordCase.LOWER- && WordCase.caseOf(root) == WordCase.TITLE- && !dictionary.hasLanguage("de")) {- continue;- }+ String lower = dictionary.toLowerCase(root);+ int sc =+ ngram(3, word, lower, EnumSet.of(NGramOptions.LONGER_WORSE))+ + commonPrefix(word, root);- String lower = dictionary.toLowerCase(root);- int sc =- ngram(3, word, lower, EnumSet.of(NGramOptions.LONGER_WORSE)) + commonPrefix(word, root);+ entries.forEach(e -> roots.add(new Weighted<>(e, sc)));+ });+ return roots.stream().limit(MAX_ROOTS).collect(Collectors.toList());+ }- roots.add(new WeightedWord(root, sc));+ private void processFST(FST<IntsRef> fst, BiConsumer<IntsRef, IntsRef> keyValueConsumer) {+ if (fst == null) return;+ try {+ IntsRefFSTEnum<IntsRef> fstEnum = new IntsRefFSTEnum<>(fst);+ IntsRefFSTEnum.InputOutput<IntsRef> mapping;+ while ((mapping = fstEnum.next()) != null) {+ keyValueConsumer.accept(mapping.input, mapping.output);}- return roots.stream().limit(MAX_ROOTS).collect(Collectors.toList());} catch (IOException e) {throw new RuntimeException(e);}@@ -84,33 +106,34 @@ class GeneratingSuggester {return new String(chars);}- private boolean isSuitableRoot(IntsRef forms) {+ private List<DictEntry> filterSuitableEntries(String word, IntsRef forms) {+ List<DictEntry> result = new ArrayList<>();for (int i = 0; i < forms.length; i += dictionary.formStep()) {int entryId = forms.ints[forms.offset + i];- if (dictionary.hasFlag(entryId, dictionary.needaffix)- || dictionary.hasFlag(entryId, dictionary.forbiddenword)+ if (dictionary.hasFlag(entryId, dictionary.forbiddenword)|| dictionary.hasFlag(entryId, Dictionary.HIDDEN_FLAG)|| dictionary.hasFlag(entryId, dictionary.onlyincompound)) {continue;}- return true;+ result.add(new DictEntry(word, entryId));}- return false;+ return result;}- private List<WeightedWord> expandRoots(String word, List<WeightedWord> roots) {- int thresh = calcThreshold(word);+ private List<Weighted<String>> expandRoots(String misspelled, List<Weighted<DictEntry>> roots) {+ int thresh = calcThreshold(misspelled);- TreeSet<WeightedWord> expanded = new TreeSet<>();- for (WeightedWord weighted : roots) {- String guess = weighted.word;- String lower = dictionary.toLowerCase(guess);- int sc =- ngram(word.length(), word, lower, EnumSet.of(NGramOptions.ANY_MISMATCH))- + commonPrefix(word, guess);- if (sc > thresh) {- expanded.add(new WeightedWord(guess, sc));+ TreeSet<Weighted<String>> expanded = new TreeSet<>();+ for (Weighted<DictEntry> weighted : roots) {+ for (String guess : expandRoot(weighted.word, misspelled)) {+ String lower = dictionary.toLowerCase(guess);+ int sc =+ ngram(misspelled.length(), misspelled, lower, EnumSet.of(NGramOptions.ANY_MISMATCH))+ + commonPrefix(misspelled, guess);+ if (sc > thresh) {+ expanded.add(new Weighted<>(guess, sc));+ }}}return expanded.stream().limit(MAX_GUESSES).collect(Collectors.toList());@@ -132,14 +155,105 @@ class GeneratingSuggester {return thresh / 3 - 1;}- private TreeSet<WeightedWord> rankBySimilarity(String word, List<WeightedWord> expanded) {+ private List<String> expandRoot(DictEntry root, String misspelled) {+ List<String> crossProducts = new ArrayList<>();+ Set<String> result = new LinkedHashSet<>();++ if (!dictionary.hasFlag(root.entryId, dictionary.needaffix)) {+ result.add(root.word);+ }++ // suffixes+ processFST(+ dictionary.suffixes,+ (key, ids) -> {+ String suffix = new StringBuilder(toString(key)).reverse().toString();+ if (misspelled.length() <= suffix.length() || !misspelled.endsWith(suffix)) return;++ for (int i = 0; i < ids.length; i++) {+ int suffixId = ids.ints[ids.offset + i];+ if (!hasCompatibleFlags(root, suffixId) || !checkAffixCondition(suffixId, root.word)) {+ continue;+ }++ String withSuffix =+ root.word.substring(0, root.word.length() - affixStripLength(suffixId)) + suffix;+ result.add(withSuffix);+ if (dictionary.isCrossProduct(suffixId)) {+ crossProducts.add(withSuffix);+ }+ }+ });++ // cross-product prefixes+ processFST(+ dictionary.prefixes,+ (key, ids) -> {+ String prefix = toString(key);+ if (misspelled.length() <= prefix.length() || !misspelled.startsWith(prefix)) return;++ for (int i = 0; i < ids.length; i++) {+ int prefixId = ids.ints[ids.offset + i];+ if (!dictionary.hasFlag(root.entryId, dictionary.affixData(prefixId, AFFIX_FLAG))+ || !dictionary.isCrossProduct(prefixId)) {+ continue;+ }++ for (String suffixed : crossProducts) {+ if (checkAffixCondition(prefixId, suffixed)) {+ result.add(prefix + suffixed.substring(affixStripLength(prefixId)));+ }+ }+ }+ });++ // pure prefixes+ processFST(+ dictionary.prefixes,+ (key, ids) -> {+ String prefix = toString(key);+ if (misspelled.length() <= prefix.length() || !misspelled.startsWith(prefix)) return;++ for (int i = 0; i < ids.length; i++) {+ int prefixId = ids.ints[ids.offset + i];+ if (hasCompatibleFlags(root, prefixId) && checkAffixCondition(prefixId, root.word)) {+ result.add(prefix + root.word.substring(affixStripLength(prefixId)));+ }+ }+ });++ return result.stream().limit(MAX_WORDS).collect(Collectors.toList());+ }++ private boolean hasCompatibleFlags(DictEntry root, int affixId) {+ if (!dictionary.hasFlag(root.entryId, dictionary.affixData(affixId, AFFIX_FLAG))) {+ return false;+ }++ int append = dictionary.affixData(affixId, AFFIX_APPEND);+ return !dictionary.hasFlag(append, dictionary.needaffix)+ && !dictionary.hasFlag(append, dictionary.circumfix)+ && !dictionary.hasFlag(append, dictionary.onlyincompound);+ }++ private boolean checkAffixCondition(int suffixId, String stem) {+ int condition = dictionary.getAffixCondition(suffixId);+ return condition == 0 || dictionary.patterns.get(condition).run(stem);+ }++ private int affixStripLength(int affixId) {+ char stripOrd = dictionary.affixData(affixId, AFFIX_STRIP_ORD);+ return dictionary.stripOffsets[stripOrd + 1] - dictionary.stripOffsets[stripOrd];+ }++ private TreeSet<Weighted<String>> rankBySimilarity(String word, List<Weighted<String>> expanded) {double fact = (10.0 - dictionary.maxDiff) / 5.0;- TreeSet<WeightedWord> bySimilarity = new TreeSet<>();- for (WeightedWord weighted : expanded) {+ TreeSet<Weighted<String>> bySimilarity = new TreeSet<>();+ for (Weighted<String> weighted : expanded) {String guess = weighted.word;String lower = dictionary.toLowerCase(guess);if (lower.equals(word)) {- bySimilarity.add(new WeightedWord(guess, weighted.score + 2000));+ bySimilarity.add(new Weighted<>(guess, weighted.score + 2000));break;}@@ -155,16 +269,16 @@ class GeneratingSuggester {+ ngram(4, word, lower, EnumSet.of(NGramOptions.ANY_MISMATCH))+ re+ (re < (word.length() + lower.length()) * fact ? -1000 : 0);- bySimilarity.add(new WeightedWord(guess, score));+ bySimilarity.add(new Weighted<>(guess, score));}return bySimilarity;}private List<String> getMostRelevantSuggestions(- TreeSet<WeightedWord> bySimilarity, Set<String> prevSuggestions) {+ TreeSet<Weighted<String>> bySimilarity, Set<String> prevSuggestions) {List<String> result = new ArrayList<>();boolean hasExcellent = false;- for (WeightedWord weighted : bySimilarity) {+ for (Weighted<String> weighted : bySimilarity) {if (weighted.score > 1000) {hasExcellent = true;} else if (hasExcellent) {@@ -178,7 +292,8 @@ class GeneratingSuggester {}if (prevSuggestions.stream().noneMatch(weighted.word::contains)- && result.stream().noneMatch(weighted.word::contains)) {+ && result.stream().noneMatch(weighted.word::contains)+ && speller.checkWord(weighted.word)) {result.add(weighted.word);if (result.size() > dictionary.maxNGramSuggestions) {break;@@ -284,11 +399,11 @@ class GeneratingSuggester {ANY_MISMATCH}- private static class WeightedWord implements Comparable<WeightedWord> {- final String word;+ private static class Weighted<T extends Comparable<T>> implements Comparable<Weighted<T>> {+ final T word;final int score;- WeightedWord(String word, int score) {+ Weighted(T word, int score) {this.word = word;this.score = score;}@@ -296,8 +411,9 @@ class GeneratingSuggester {@Overridepublic boolean equals(Object o) {if (this == o) return true;- if (!(o instanceof WeightedWord)) return false;- WeightedWord that = (WeightedWord) o;+ if (!(o instanceof Weighted)) return false;+ @SuppressWarnings("unchecked")+ Weighted<T> that = (Weighted<T>) o;return score == that.score && word.equals(that.word);}@@ -312,9 +428,42 @@ class GeneratingSuggester {}@Override- public int compareTo(WeightedWord o) {+ public int compareTo(Weighted<T> o) {int cmp = Integer.compare(score, o.score);return cmp != 0 ? -cmp : word.compareTo(o.word);}}++ private static class DictEntry implements Comparable<DictEntry> {+ private final String word;+ private final int entryId;++ DictEntry(String word, int entryId) {+ this.word = word;+ this.entryId = entryId;+ }++ @Override+ public String toString() {+ return word;+ }++ @Override+ public boolean equals(Object o) {+ if (this == o) return true;+ if (!(o instanceof DictEntry)) return false;+ DictEntry dictEntry = (DictEntry) o;+ return entryId == dictEntry.entryId && word.equals(dictEntry.word);+ }++ @Override+ public int hashCode() {+ return Objects.hash(word, entryId);+ }++ @Override+ public int compareTo(DictEntry o) {+ return word.compareTo(o.word);+ }+ }}diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/SpellChecker.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/SpellChecker.javaindex 3fe849c4da2..7e383b3fe4e 100644--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/SpellChecker.java+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/SpellChecker.java@@ -423,7 +423,7 @@ public class SpellChecker {if (!modifier.hasGoodSuggestions && dictionary.maxNGramSuggestions > 0) {suggestions.addAll(- new GeneratingSuggester(dictionary)+ new GeneratingSuggester(this).suggest(dictionary.toLowerCase(word), wordCase, suggestions));}diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Stemmer.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Stemmer.javaindex 97c93012fb5..159642d4550 100644--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Stemmer.java+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Stemmer.java@@ -613,7 +613,7 @@ final class Stemmer {return context.isCompound() || !dictionary.hasFlag(append, dictionary.onlyincompound);}- if (isCrossProduct(affix)) {+ if (dictionary.isCrossProduct(affix)) {// cross check incoming continuation class (flag of previous affix) against list.if (context.isCompound() || !dictionary.hasFlag(append, dictionary.onlyincompound)) {return previousWasPrefix || dictionary.hasFlag(append, prevFlag);@@ -629,7 +629,7 @@ final class Stemmer {// but this is a little bit more complicated.private boolean checkCondition(int affix, char[] c1, int c1off, int c1len, char[] c2, int c2off, int c2len) {- int condition = dictionary.affixData(affix, Dictionary.AFFIX_CONDITION) >>> 1;+ int condition = dictionary.getAffixCondition(affix);if (condition != 0) {CharacterRunAutomaton pattern = dictionary.patterns.get(condition);int state = 0;@@ -736,7 +736,7 @@ final class Stemmer {circumfix = isFlagAppendedByAffix(affix, dictionary.circumfix);}- if (isCrossProduct(affix) && recursionDepth <= 1) {+ if (dictionary.isCrossProduct(affix) && recursionDepth <= 1) {boolean doPrefix;if (recursionDepth == 0) {if (prefix) {@@ -798,8 +798,4 @@ final class Stemmer {int appendId = dictionary.affixData(affixId, Dictionary.AFFIX_APPEND);return dictionary.hasFlag(appendId, flag);}-- private boolean isCrossProduct(int affix) {- return (dictionary.affixData(affix, Dictionary.AFFIX_CONDITION) & 1) == 1;- }}diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/allcaps.sug b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/allcaps.sugnew file mode 100644index 00000000000..d372ff23d63--- /dev/null+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/allcaps.sug@@ -0,0 +1,3 @@+OpenOffice.org+UNICEF+UNICEF's