Skip to content

armgit / solr

Public Imported from GitHub
0

Apache Solr open-source search software

LUCENE-9748: Hunspell: suggest inflected dictionary entries similar to the misspelled word (#2330)

Peter Gromovf2b7cdc49190
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java
index b175cdaef9b..f39575c704c 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java
@@ -122,7 +122,7 @@ public class Dictionary {
// offsets in affixData
static final int AFFIX_FLAG = 0;
static final int AFFIX_STRIP_ORD = 1;
- static final int AFFIX_CONDITION = 2;
+ private static final int AFFIX_CONDITION = 2;
static final int AFFIX_APPEND = 3;
// Default flag parsing strategy
@@ -779,6 +779,14 @@ public class Dictionary {
return affixData[affixIndex * 4 + offset];
}
+ boolean isCrossProduct(int affix) {
+ return (affixData(affix, AFFIX_CONDITION) & 1) == 1;
+ }
+
+ int getAffixCondition(int affix) {
+ return affixData(affix, AFFIX_CONDITION) >>> 1;
+ }
+
private FST<CharsRef> parseConversions(LineNumberReader reader, int num)
throws IOException, ParseException {
Map<String, String> mappings = new TreeMap<>();
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/GeneratingSuggester.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/GeneratingSuggester.java
index 6a423960b29..9546aac2bb4 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/GeneratingSuggester.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/GeneratingSuggester.java
@@ -16,15 +16,22 @@
*/
package org.apache.lucene.analysis.hunspell;
+import static org.apache.lucene.analysis.hunspell.Dictionary.AFFIX_APPEND;
+import static org.apache.lucene.analysis.hunspell.Dictionary.AFFIX_FLAG;
+import static org.apache.lucene.analysis.hunspell.Dictionary.AFFIX_STRIP_ORD;
+
import java.io.IOException;
import java.util.ArrayList;
import java.util.EnumSet;
+import java.util.LinkedHashSet;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.TreeSet;
+import java.util.function.BiConsumer;
import java.util.stream.Collectors;
import org.apache.lucene.util.IntsRef;
+import org.apache.lucene.util.fst.FST;
import org.apache.lucene.util.fst.IntsRefFSTEnum;
/**
@@ -33,44 +40,59 @@ import org.apache.lucene.util.fst.IntsRefFSTEnum;
*/
class GeneratingSuggester {
private static final int MAX_ROOTS = 100;
- private static final int MAX_GUESSES = 100;
+ private static final int MAX_WORDS = 100;
+ private static final int MAX_GUESSES = 200;
private final Dictionary dictionary;
+ private final SpellChecker speller;
- GeneratingSuggester(Dictionary dictionary) {
- this.dictionary = dictionary;
+ GeneratingSuggester(SpellChecker speller) {
+ this.dictionary = speller.dictionary;
+ this.speller = speller;
}
List<String> suggest(String word, WordCase originalCase, Set<String> prevSuggestions) {
- List<WeightedWord> roots = findSimilarDictionaryEntries(word, originalCase);
- List<WeightedWord> expanded = expandRoots(word, roots);
- TreeSet<WeightedWord> bySimilarity = rankBySimilarity(word, expanded);
+ List<Weighted<DictEntry>> roots = findSimilarDictionaryEntries(word, originalCase);
+ List<Weighted<String>> expanded = expandRoots(word, roots);
+ TreeSet<Weighted<String>> bySimilarity = rankBySimilarity(word, expanded);
return getMostRelevantSuggestions(bySimilarity, prevSuggestions);
}
- private List<WeightedWord> findSimilarDictionaryEntries(String word, WordCase originalCase) {
- try {
- IntsRefFSTEnum<IntsRef> fstEnum = new IntsRefFSTEnum<>(dictionary.words);
- TreeSet<WeightedWord> roots = new TreeSet<>();
+ private List<Weighted<DictEntry>> findSimilarDictionaryEntries(
+ String word, WordCase originalCase) {
+ TreeSet<Weighted<DictEntry>> roots = new TreeSet<>();
+ processFST(
+ dictionary.words,
+ (key, forms) -> {
+ if (Math.abs(key.length - word.length()) > 4) return;
+
+ String root = toString(key);
+ List<DictEntry> entries = filterSuitableEntries(root, forms);
+ if (entries.isEmpty()) return;
+
+ if (originalCase == WordCase.LOWER
+ && WordCase.caseOf(root) == WordCase.TITLE
+ && !dictionary.hasLanguage("de")) {
+ return;
+ }
- IntsRefFSTEnum.InputOutput<IntsRef> mapping;
- while ((mapping = fstEnum.next()) != null) {
- IntsRef key = mapping.input;
- if (Math.abs(key.length - word.length()) > 4 || !isSuitableRoot(mapping.output)) continue;
-
- String root = toString(key);
- if (originalCase == WordCase.LOWER
- && WordCase.caseOf(root) == WordCase.TITLE
- && !dictionary.hasLanguage("de")) {
- continue;
- }
+ String lower = dictionary.toLowerCase(root);
+ int sc =
+ ngram(3, word, lower, EnumSet.of(NGramOptions.LONGER_WORSE))
+ + commonPrefix(word, root);
- String lower = dictionary.toLowerCase(root);
- int sc =
- ngram(3, word, lower, EnumSet.of(NGramOptions.LONGER_WORSE)) + commonPrefix(word, root);
+ entries.forEach(e -> roots.add(new Weighted<>(e, sc)));
+ });
+ return roots.stream().limit(MAX_ROOTS).collect(Collectors.toList());
+ }
- roots.add(new WeightedWord(root, sc));
+ private void processFST(FST<IntsRef> fst, BiConsumer<IntsRef, IntsRef> keyValueConsumer) {
+ if (fst == null) return;
+ try {
+ IntsRefFSTEnum<IntsRef> fstEnum = new IntsRefFSTEnum<>(fst);
+ IntsRefFSTEnum.InputOutput<IntsRef> mapping;
+ while ((mapping = fstEnum.next()) != null) {
+ keyValueConsumer.accept(mapping.input, mapping.output);
}
- return roots.stream().limit(MAX_ROOTS).collect(Collectors.toList());
} catch (IOException e) {
throw new RuntimeException(e);
}
@@ -84,33 +106,34 @@ class GeneratingSuggester {
return new String(chars);
}
- private boolean isSuitableRoot(IntsRef forms) {
+ private List<DictEntry> filterSuitableEntries(String word, IntsRef forms) {
+ List<DictEntry> result = new ArrayList<>();
for (int i = 0; i < forms.length; i += dictionary.formStep()) {
int entryId = forms.ints[forms.offset + i];
- if (dictionary.hasFlag(entryId, dictionary.needaffix)
- || dictionary.hasFlag(entryId, dictionary.forbiddenword)
+ if (dictionary.hasFlag(entryId, dictionary.forbiddenword)
|| dictionary.hasFlag(entryId, Dictionary.HIDDEN_FLAG)
|| dictionary.hasFlag(entryId, dictionary.onlyincompound)) {
continue;
}
- return true;
+ result.add(new DictEntry(word, entryId));
}
- return false;
+ return result;
}
- private List<WeightedWord> expandRoots(String word, List<WeightedWord> roots) {
- int thresh = calcThreshold(word);
+ private List<Weighted<String>> expandRoots(String misspelled, List<Weighted<DictEntry>> roots) {
+ int thresh = calcThreshold(misspelled);
- TreeSet<WeightedWord> expanded = new TreeSet<>();
- for (WeightedWord weighted : roots) {
- String guess = weighted.word;
- String lower = dictionary.toLowerCase(guess);
- int sc =
- ngram(word.length(), word, lower, EnumSet.of(NGramOptions.ANY_MISMATCH))
- + commonPrefix(word, guess);
- if (sc > thresh) {
- expanded.add(new WeightedWord(guess, sc));
+ TreeSet<Weighted<String>> expanded = new TreeSet<>();
+ for (Weighted<DictEntry> weighted : roots) {
+ for (String guess : expandRoot(weighted.word, misspelled)) {
+ String lower = dictionary.toLowerCase(guess);
+ int sc =
+ ngram(misspelled.length(), misspelled, lower, EnumSet.of(NGramOptions.ANY_MISMATCH))
+ + commonPrefix(misspelled, guess);
+ if (sc > thresh) {
+ expanded.add(new Weighted<>(guess, sc));
+ }
}
}
return expanded.stream().limit(MAX_GUESSES).collect(Collectors.toList());
@@ -132,14 +155,105 @@ class GeneratingSuggester {
return thresh / 3 - 1;
}
- private TreeSet<WeightedWord> rankBySimilarity(String word, List<WeightedWord> expanded) {
+ private List<String> expandRoot(DictEntry root, String misspelled) {
+ List<String> crossProducts = new ArrayList<>();
+ Set<String> result = new LinkedHashSet<>();
+
+ if (!dictionary.hasFlag(root.entryId, dictionary.needaffix)) {
+ result.add(root.word);
+ }
+
+ // suffixes
+ processFST(
+ dictionary.suffixes,
+ (key, ids) -> {
+ String suffix = new StringBuilder(toString(key)).reverse().toString();
+ if (misspelled.length() <= suffix.length() || !misspelled.endsWith(suffix)) return;
+
+ for (int i = 0; i < ids.length; i++) {
+ int suffixId = ids.ints[ids.offset + i];
+ if (!hasCompatibleFlags(root, suffixId) || !checkAffixCondition(suffixId, root.word)) {
+ continue;
+ }
+
+ String withSuffix =
+ root.word.substring(0, root.word.length() - affixStripLength(suffixId)) + suffix;
+ result.add(withSuffix);
+ if (dictionary.isCrossProduct(suffixId)) {
+ crossProducts.add(withSuffix);
+ }
+ }
+ });
+
+ // cross-product prefixes
+ processFST(
+ dictionary.prefixes,
+ (key, ids) -> {
+ String prefix = toString(key);
+ if (misspelled.length() <= prefix.length() || !misspelled.startsWith(prefix)) return;
+
+ for (int i = 0; i < ids.length; i++) {
+ int prefixId = ids.ints[ids.offset + i];
+ if (!dictionary.hasFlag(root.entryId, dictionary.affixData(prefixId, AFFIX_FLAG))
+ || !dictionary.isCrossProduct(prefixId)) {
+ continue;
+ }
+
+ for (String suffixed : crossProducts) {
+ if (checkAffixCondition(prefixId, suffixed)) {
+ result.add(prefix + suffixed.substring(affixStripLength(prefixId)));
+ }
+ }
+ }
+ });
+
+ // pure prefixes
+ processFST(
+ dictionary.prefixes,
+ (key, ids) -> {
+ String prefix = toString(key);
+ if (misspelled.length() <= prefix.length() || !misspelled.startsWith(prefix)) return;
+
+ for (int i = 0; i < ids.length; i++) {
+ int prefixId = ids.ints[ids.offset + i];
+ if (hasCompatibleFlags(root, prefixId) && checkAffixCondition(prefixId, root.word)) {
+ result.add(prefix + root.word.substring(affixStripLength(prefixId)));
+ }
+ }
+ });
+
+ return result.stream().limit(MAX_WORDS).collect(Collectors.toList());
+ }
+
+ private boolean hasCompatibleFlags(DictEntry root, int affixId) {
+ if (!dictionary.hasFlag(root.entryId, dictionary.affixData(affixId, AFFIX_FLAG))) {
+ return false;
+ }
+
+ int append = dictionary.affixData(affixId, AFFIX_APPEND);
+ return !dictionary.hasFlag(append, dictionary.needaffix)
+ && !dictionary.hasFlag(append, dictionary.circumfix)
+ && !dictionary.hasFlag(append, dictionary.onlyincompound);
+ }
+
+ private boolean checkAffixCondition(int suffixId, String stem) {
+ int condition = dictionary.getAffixCondition(suffixId);
+ return condition == 0 || dictionary.patterns.get(condition).run(stem);
+ }
+
+ private int affixStripLength(int affixId) {
+ char stripOrd = dictionary.affixData(affixId, AFFIX_STRIP_ORD);
+ return dictionary.stripOffsets[stripOrd + 1] - dictionary.stripOffsets[stripOrd];
+ }
+
+ private TreeSet<Weighted<String>> rankBySimilarity(String word, List<Weighted<String>> expanded) {
double fact = (10.0 - dictionary.maxDiff) / 5.0;
- TreeSet<WeightedWord> bySimilarity = new TreeSet<>();
- for (WeightedWord weighted : expanded) {
+ TreeSet<Weighted<String>> bySimilarity = new TreeSet<>();
+ for (Weighted<String> weighted : expanded) {
String guess = weighted.word;
String lower = dictionary.toLowerCase(guess);
if (lower.equals(word)) {
- bySimilarity.add(new WeightedWord(guess, weighted.score + 2000));
+ bySimilarity.add(new Weighted<>(guess, weighted.score + 2000));
break;
}
@@ -155,16 +269,16 @@ class GeneratingSuggester {
+ ngram(4, word, lower, EnumSet.of(NGramOptions.ANY_MISMATCH))
+ re
+ (re < (word.length() + lower.length()) * fact ? -1000 : 0);
- bySimilarity.add(new WeightedWord(guess, score));
+ bySimilarity.add(new Weighted<>(guess, score));
}
return bySimilarity;
}
private List<String> getMostRelevantSuggestions(
- TreeSet<WeightedWord> bySimilarity, Set<String> prevSuggestions) {
+ TreeSet<Weighted<String>> bySimilarity, Set<String> prevSuggestions) {
List<String> result = new ArrayList<>();
boolean hasExcellent = false;
- for (WeightedWord weighted : bySimilarity) {
+ for (Weighted<String> weighted : bySimilarity) {
if (weighted.score > 1000) {
hasExcellent = true;
} else if (hasExcellent) {
@@ -178,7 +292,8 @@ class GeneratingSuggester {
}
if (prevSuggestions.stream().noneMatch(weighted.word::contains)
- && result.stream().noneMatch(weighted.word::contains)) {
+ && result.stream().noneMatch(weighted.word::contains)
+ && speller.checkWord(weighted.word)) {
result.add(weighted.word);
if (result.size() > dictionary.maxNGramSuggestions) {
break;
@@ -284,11 +399,11 @@ class GeneratingSuggester {
ANY_MISMATCH
}
- private static class WeightedWord implements Comparable<WeightedWord> {
- final String word;
+ private static class Weighted<T extends Comparable<T>> implements Comparable<Weighted<T>> {
+ final T word;
final int score;
- WeightedWord(String word, int score) {
+ Weighted(T word, int score) {
this.word = word;
this.score = score;
}
@@ -296,8 +411,9 @@ class GeneratingSuggester {
@Override
public boolean equals(Object o) {
if (this == o) return true;
- if (!(o instanceof WeightedWord)) return false;
- WeightedWord that = (WeightedWord) o;
+ if (!(o instanceof Weighted)) return false;
+ @SuppressWarnings("unchecked")
+ Weighted<T> that = (Weighted<T>) o;
return score == that.score && word.equals(that.word);
}
@@ -312,9 +428,42 @@ class GeneratingSuggester {
}
@Override
- public int compareTo(WeightedWord o) {
+ public int compareTo(Weighted<T> o) {
int cmp = Integer.compare(score, o.score);
return cmp != 0 ? -cmp : word.compareTo(o.word);
}
}
+
+ private static class DictEntry implements Comparable<DictEntry> {
+ private final String word;
+ private final int entryId;
+
+ DictEntry(String word, int entryId) {
+ this.word = word;
+ this.entryId = entryId;
+ }
+
+ @Override
+ public String toString() {
+ return word;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (!(o instanceof DictEntry)) return false;
+ DictEntry dictEntry = (DictEntry) o;
+ return entryId == dictEntry.entryId && word.equals(dictEntry.word);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(word, entryId);
+ }
+
+ @Override
+ public int compareTo(DictEntry o) {
+ return word.compareTo(o.word);
+ }
+ }
}
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/SpellChecker.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/SpellChecker.java
index 3fe849c4da2..7e383b3fe4e 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/SpellChecker.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/SpellChecker.java
@@ -423,7 +423,7 @@ public class SpellChecker {
if (!modifier.hasGoodSuggestions && dictionary.maxNGramSuggestions > 0) {
suggestions.addAll(
- new GeneratingSuggester(dictionary)
+ new GeneratingSuggester(this)
.suggest(dictionary.toLowerCase(word), wordCase, suggestions));
}
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Stemmer.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Stemmer.java
index 97c93012fb5..159642d4550 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Stemmer.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Stemmer.java
@@ -613,7 +613,7 @@ final class Stemmer {
return context.isCompound() || !dictionary.hasFlag(append, dictionary.onlyincompound);
}
- if (isCrossProduct(affix)) {
+ if (dictionary.isCrossProduct(affix)) {
// cross check incoming continuation class (flag of previous affix) against list.
if (context.isCompound() || !dictionary.hasFlag(append, dictionary.onlyincompound)) {
return previousWasPrefix || dictionary.hasFlag(append, prevFlag);
@@ -629,7 +629,7 @@ final class Stemmer {
// but this is a little bit more complicated.
private boolean checkCondition(
int affix, char[] c1, int c1off, int c1len, char[] c2, int c2off, int c2len) {
- int condition = dictionary.affixData(affix, Dictionary.AFFIX_CONDITION) >>> 1;
+ int condition = dictionary.getAffixCondition(affix);
if (condition != 0) {
CharacterRunAutomaton pattern = dictionary.patterns.get(condition);
int state = 0;
@@ -736,7 +736,7 @@ final class Stemmer {
circumfix = isFlagAppendedByAffix(affix, dictionary.circumfix);
}
- if (isCrossProduct(affix) && recursionDepth <= 1) {
+ if (dictionary.isCrossProduct(affix) && recursionDepth <= 1) {
boolean doPrefix;
if (recursionDepth == 0) {
if (prefix) {
@@ -798,8 +798,4 @@ final class Stemmer {
int appendId = dictionary.affixData(affixId, Dictionary.AFFIX_APPEND);
return dictionary.hasFlag(appendId, flag);
}
-
- private boolean isCrossProduct(int affix) {
- return (dictionary.affixData(affix, Dictionary.AFFIX_CONDITION) & 1) == 1;
- }
}
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/allcaps.sug b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/allcaps.sug
new file mode 100644
index 00000000000..d372ff23d63
--- /dev/null
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/allcaps.sug
@@ -0,0 +1,3 @@
+OpenOffice.org
+UNICEF
+UNICEF's