Skip to content

armgit / solr

Public Imported from GitHub
0

Apache Solr open-source search software

LUCENE-9701: Hunspell: implement simple REP-based suggestion algorithm (#2251)

Peter Gromov8a34cc7afd3f
diff --git a/gradle/validation/rat-sources.gradle b/gradle/validation/rat-sources.gradle
index 9454f0f6f75..5738d0142fd 100644
--- a/gradle/validation/rat-sources.gradle
+++ b/gradle/validation/rat-sources.gradle
@@ -56,6 +56,7 @@ configure(project(":lucene:analysis:common")) {
"**/*.dic",
"**/*.wrong",
"**/*.good",
+ "**/*.sug",
"**/charfilter/*.htm*",
"**/*LuceneResourcesWikiPage.html"
]
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java
index 94b3c323b84..9de67eb2f18 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java
@@ -152,6 +152,9 @@ public class Dictionary {
// ignored characters (dictionary, affix, inputs)
private char[] ignore;
+ String tryChars = "";
+ List<RepEntry> repTable = new ArrayList<>();
+
// FSTs used for ICONV/OCONV, output ord pointing to replacement text
FST<CharsRef> iconv;
FST<CharsRef> oconv;
@@ -383,6 +386,14 @@ public class Dictionary {
alternateCasing = langCode.equals("tr") || langCode.equals("az");
} else if ("BREAK".equals(firstWord)) {
breaks = parseBreaks(reader, line);
+ } else if ("TRY".equals(firstWord)) {
+ tryChars = singleArgument(reader, line);
+ } else if ("REP".equals(firstWord)) {
+ int count = Integer.parseInt(singleArgument(reader, line));
+ for (int i = 0; i < count; i++) {
+ String[] parts = splitBySpace(reader, reader.readLine(), 3);
+ repTable.add(new RepEntry(parts[1], parts[2]));
+ }
} else if ("FORBIDDENWORD".equals(firstWord)) {
forbiddenword = flagParsingStrategy.parseFlag(singleArgument(reader, line));
} else if ("COMPOUNDMIN".equals(firstWord)) {
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/ModifyingSuggester.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/ModifyingSuggester.java
new file mode 100644
index 00000000000..02fa0b47701
--- /dev/null
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/ModifyingSuggester.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.analysis.hunspell;
+
+import java.util.Arrays;
+import java.util.LinkedHashSet;
+
+class ModifyingSuggester {
+ private final LinkedHashSet<String> result = new LinkedHashSet<>();
+ private final char[] tryChars;
+ private final SpellChecker speller;
+
+ ModifyingSuggester(SpellChecker speller) {
+ this.speller = speller;
+ tryChars = speller.dictionary.tryChars.toCharArray();
+ }
+
+ LinkedHashSet<String> suggest(String word) {
+ tryRep(word);
+ tryAddingChar(word);
+ return result;
+ }
+
+ private void tryRep(String word) {
+ for (RepEntry entry : speller.dictionary.repTable) {
+ for (String candidate : entry.substitute(word)) {
+ if (trySuggestion(candidate)) {
+ continue;
+ }
+
+ if (candidate.contains(" ")
+ && Arrays.stream(candidate.split(" ")).allMatch(speller::checkWord)) {
+ result.add(candidate);
+ }
+ }
+ }
+ }
+
+ private void tryAddingChar(String word) {
+ for (int i = 0; i <= word.length(); i++) {
+ String prefix = word.substring(0, i);
+ String suffix = word.substring(i);
+ for (char toInsert : tryChars) {
+ trySuggestion(prefix + toInsert + suffix);
+ }
+ }
+ }
+
+ private boolean trySuggestion(String candidate) {
+ if (speller.checkWord(candidate)) {
+ result.add(candidate);
+ return true;
+ }
+ return false;
+ }
+}
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/RepEntry.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/RepEntry.java
new file mode 100644
index 00000000000..1cd5738b9ed
--- /dev/null
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/RepEntry.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.analysis.hunspell;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+class RepEntry {
+ private final String pattern;
+ private final String replacement;
+ private final boolean mustStart;
+ private final boolean mustEnd;
+ private final int patternLen;
+
+ RepEntry(String rawPattern, String rawReplacement) {
+ mustStart = rawPattern.startsWith("^");
+ mustEnd = rawPattern.endsWith("$");
+ pattern = rawPattern.substring(mustStart ? 1 : 0, rawPattern.length() - (mustEnd ? 1 : 0));
+ replacement = rawReplacement.replace('_', ' ');
+ patternLen = pattern.length();
+ }
+
+ List<String> substitute(String word) {
+ if (mustStart) {
+ boolean matches = mustEnd ? word.equals(pattern) : word.startsWith(pattern);
+ return matches
+ ? Collections.singletonList(replacement + word.substring(patternLen))
+ : Collections.emptyList();
+ }
+
+ if (mustEnd) {
+ return word.endsWith(pattern)
+ ? Collections.singletonList(word.substring(0, word.length() - patternLen) + replacement)
+ : Collections.emptyList();
+ }
+
+ int pos = word.indexOf(pattern);
+ if (pos < 0) return Collections.emptyList();
+
+ List<String> result = new ArrayList<>();
+ while (pos >= 0) {
+ result.add(word.substring(0, pos) + replacement + word.substring(pos + patternLen));
+ pos = word.indexOf(pattern, pos + 1);
+ }
+ return result;
+ }
+
+ @Override
+ public String toString() {
+ return (mustStart ? "^" : "") + pattern + (mustEnd ? "$" : "") + "->" + replacement;
+ }
+}
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/SpellChecker.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/SpellChecker.java
index 10cc206fe65..aa0aeac53ec 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/SpellChecker.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/SpellChecker.java
@@ -23,7 +23,9 @@ import static org.apache.lucene.analysis.hunspell.WordContext.COMPOUND_MIDDLE;
import static org.apache.lucene.analysis.hunspell.WordContext.SIMPLE_WORD;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
+import java.util.Set;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CharsRef;
import org.apache.lucene.util.IntsRef;
@@ -34,9 +36,9 @@ import org.apache.lucene.util.IntsRef;
* threads). Not all Hunspell features are supported yet.
*/
public class SpellChecker {
- private final Dictionary dictionary;
+ final Dictionary dictionary;
+ final Stemmer stemmer;
private final BytesRef scratch = new BytesRef();
- private final Stemmer stemmer;
public SpellChecker(Dictionary dictionary) {
this.dictionary = dictionary;
@@ -128,6 +130,10 @@ public class SpellChecker {
return false;
}
+ boolean checkWord(String word) {
+ return checkWord(word.toCharArray(), word.length(), null);
+ }
+
private boolean checkWord(char[] wordChars, int length, WordCase originalCase) {
if (dictionary.isForbiddenWord(wordChars, length, scratch)) {
return false;
@@ -329,4 +335,44 @@ public class SpellChecker {
&& spell(word.substring(0, breakPos))
&& spell(word.substring(breakPos + breakStr.length()));
}
+
+ public List<String> suggest(String word) {
+ if (word.length() >= 100) return Collections.emptyList();
+
+ if (dictionary.needsInputCleaning) {
+ word = dictionary.cleanInput(word, new StringBuilder()).toString();
+ }
+
+ ModifyingSuggester modifier = new ModifyingSuggester(this);
+ Set<String> result = modifier.suggest(word);
+
+ if (word.contains("-") && result.stream().noneMatch(s -> s.contains("-"))) {
+ result.addAll(modifyChunksBetweenDashes(word));
+ }
+
+ return new ArrayList<>(result);
+ }
+
+ private List<String> modifyChunksBetweenDashes(String word) {
+ List<String> result = new ArrayList<>();
+ int chunkStart = 0;
+ while (chunkStart < word.length()) {
+ int chunkEnd = word.indexOf('-', chunkStart);
+ if (chunkEnd < 0) {
+ chunkEnd = word.length();
+ }
+
+ if (chunkEnd > chunkStart) {
+ String chunk = word.substring(chunkStart, chunkEnd);
+ if (!spell(chunk)) {
+ for (String chunkSug : suggest(chunk)) {
+ result.add(word.substring(0, chunkStart) + chunkSug + word.substring(chunkEnd));
+ }
+ }
+ }
+
+ chunkStart = chunkEnd + 1;
+ }
+ return result;
+ }
}
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/SpellCheckerTest.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/SpellCheckerTest.java
index f870f380110..123ee181d82 100644
--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/SpellCheckerTest.java
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/SpellCheckerTest.java
@@ -20,7 +20,9 @@ import java.io.InputStream;
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Path;
+import java.util.List;
import java.util.Objects;
+import java.util.stream.Collectors;
import org.apache.lucene.store.ByteBuffersDirectory;
import org.apache.lucene.util.IOUtils;
import org.junit.Test;
@@ -46,6 +48,10 @@ public class SpellCheckerTest extends StemmerTestBase {
doTest("allcaps");
}
+ public void rep() throws Exception {
+ doTest("rep");
+ }
+
@Test
public void forceUCase() throws Exception {
doTest("forceucase");
@@ -178,10 +184,22 @@ public class SpellCheckerTest extends StemmerTestBase {
}
URL wrong = StemmerTestBase.class.getResource(name + ".wrong");
+ URL sug = StemmerTestBase.class.getResource(name + ".sug");
if (wrong != null) {
- for (String word : Files.readAllLines(Path.of(wrong.toURI()))) {
+ List<String> wrongWords = Files.readAllLines(Path.of(wrong.toURI()));
+ for (String word : wrongWords) {
assertFalse("Unexpectedly considered correct: " + word, speller.spell(word));
}
+ if (sug != null) {
+ String suggestions =
+ wrongWords.stream()
+ .map(s -> String.join(", ", speller.suggest(s)))
+ .filter(s -> !s.isEmpty())
+ .collect(Collectors.joining("\n"));
+ assertEquals(Files.readString(Path.of(sug.toURI())).trim(), suggestions);
+ }
+ } else {
+ assertNull(".sug file without .wrong file!", sug);
}
}
}
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/breakdefault.sug b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/breakdefault.sug
new file mode 100644
index 00000000000..d3b924cd89b
--- /dev/null
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/breakdefault.sug
@@ -0,0 +1,5 @@
+scott
+scot-free
+foo-bar
+foo-foo-bar
+foo-foo-foo
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/rep.aff b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/rep.aff
new file mode 100644
index 00000000000..4ccc6e20b65
--- /dev/null
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/rep.aff
@@ -0,0 +1,21 @@
+# With REP suggestions, we can fix typical language specific misspellings.
+
+# switch off ngram suggestion for testing
+MAXNGRAMSUGS 0
+
+REP 8
+REP f ph
+REP ph f
+REP shun$ tion
+REP ^alot$ a_lot # add the highest priority for "a lot" suggestion to "alot"
+REP ^foo$ bar
+REP ' _ # "un'alunno" -> "un alunno"
+REP ^vinte�n$ vinte_e_un
+REP s 's
+
+
+SFX A Y 1
+SFX A 0 's .
+
+
+WORDCHARS '
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/rep.dic b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/rep.dic
new file mode 100644
index 00000000000..f9a4c008b0a
--- /dev/null
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/rep.dic
@@ -0,0 +1,15 @@
+10
+form
+phantom
+vacation
+vacations
+a
+lot
+un
+alunno
+bar
+barbars
+vinte
+e
+un
+auto/A
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/rep.sug b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/rep.sug
new file mode 100644
index 00000000000..b48a5b80eb0
--- /dev/null
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/rep.sug
@@ -0,0 +1,8 @@
+form
+phantom
+vacation
+a lot, lot
+un alunno
+bar
+vinte e un
+auto's, auto
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/rep.wrong b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/rep.wrong
new file mode 100644
index 00000000000..b3254ac064e
--- /dev/null
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/rep.wrong
@@ -0,0 +1,11 @@
+phorm
+fantom
+vacashun
+vacashuns
+alot
+un'alunno
+foo
+foobars
+barfoos
+vinteún
+autos