Apache Solr open-source search software
LUCENE-9729: Hunspell: support CHECKCOMPOUNDREP flags (#2300)
Peter Gromov650f16ad5d8e
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/CheckCompoundPattern.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/CheckCompoundPattern.javaindex ac5208c1868..3d7059166d7 100644--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/CheckCompoundPattern.java+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/CheckCompoundPattern.java@@ -21,9 +21,9 @@ import org.apache.lucene.util.CharsRef;import org.apache.lucene.util.IntsRef;class CheckCompoundPattern {- private final char[] endChars;- private final char[] beginChars;- private final char[] replacement;+ private final String endChars;+ private final String beginChars;+ private final String replacement;private final char[] endFlags;private final char[] beginFlags;private final Dictionary dictionary;@@ -38,36 +38,33 @@ class CheckCompoundPattern {}int flagSep = parts[1].indexOf("/");- endChars = (flagSep < 0 ? parts[1] : parts[1].substring(0, flagSep)).toCharArray();+ endChars = flagSep < 0 ? parts[1] : parts[1].substring(0, flagSep);endFlags = flagSep < 0 ? new char[0] : strategy.parseFlags(parts[1].substring(flagSep + 1));flagSep = parts[2].indexOf("/");- beginChars = (flagSep < 0 ? parts[2] : parts[2].substring(0, flagSep)).toCharArray();+ beginChars = flagSep < 0 ? parts[2] : parts[2].substring(0, flagSep);beginFlags = flagSep < 0 ? new char[0] : strategy.parseFlags(parts[2].substring(flagSep + 1));- replacement = parts.length == 3 ? null : parts[3].toCharArray();+ replacement = parts.length == 3 ? null : parts[3];}@Overridepublic String toString() {- return new String(endChars)- + " "- + new String(beginChars)- + (replacement == null ? "" : " -> " + new String(replacement));+ return endChars + " " + beginChars + (replacement == null ? "" : " -> " + replacement);}boolean prohibitsCompounding(CharsRef word, int breakPos, CharsRef stemBefore, CharsRef stemAfter) {if (isNonAffixedPattern(endChars)) {- if (!charsMatch(word, breakPos - stemBefore.length, stemBefore.chars)) {+ if (!charsMatch(word, breakPos - stemBefore.length, stemBefore)) {return false;}- } else if (!charsMatch(word, breakPos - endChars.length, endChars)) {+ } else if (!charsMatch(word, breakPos - endChars.length(), endChars)) {return false;}if (isNonAffixedPattern(beginChars)) {- if (!charsMatch(word, breakPos, stemAfter.chars)) {+ if (!charsMatch(word, breakPos, stemAfter)) {return false;}} else if (!charsMatch(word, breakPos, beginChars)) {@@ -85,8 +82,8 @@ class CheckCompoundPattern {return true;}- private static boolean isNonAffixedPattern(char[] pattern) {- return pattern.length == 1 && pattern[0] == '0';+ private static boolean isNonAffixedPattern(String pattern) {+ return pattern.length() == 1 && pattern.charAt(0) == '0';}private boolean stemHasFlags(CharsRef stem, char[] flags) {@@ -107,25 +104,25 @@ class CheckCompoundPattern {if (replacement != null && charsMatch(word, breakPos, replacement)) {return new CharsRef(word.subSequence(0, breakPos)- + new String(endChars)- + new String(beginChars)- + word.subSequence(breakPos + replacement.length, word.length));+ + endChars+ + beginChars+ + word.subSequence(breakPos + replacement.length(), word.length));}return null;}int endLength() {- return endChars.length;+ return endChars.length();}- private static boolean charsMatch(CharsRef word, int offset, char[] pattern) {- int len = pattern.length;+ private static boolean charsMatch(CharsRef word, int offset, CharSequence pattern) {+ int len = pattern.length();if (word.length - offset < len || offset < 0 || offset > word.length) {return false;}for (int i = 0; i < len; i++) {- if (word.chars[word.offset + offset + i] != pattern[i]) {+ if (word.chars[word.offset + offset + i] != pattern.charAt(i)) {return false;}}diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.javaindex 567e26f519b..ebb76ee9eab 100644--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java@@ -152,7 +152,7 @@ public class Dictionary {char forbiddenword;char onlyincompound, compoundBegin, compoundMiddle, compoundEnd, compoundFlag;char compoundPermit, compoundForbid;- boolean checkCompoundCase, checkCompoundDup;+ boolean checkCompoundCase, checkCompoundDup, checkCompoundRep;boolean checkCompoundTriple, simplifiedTriple;int compoundMin = 3, compoundMax = Integer.MAX_VALUE;List<CompoundRule> compoundRules; // nullable@@ -425,6 +425,8 @@ public class Dictionary {checkCompoundCase = true;} else if ("CHECKCOMPOUNDDUP".equals(firstWord)) {checkCompoundDup = true;+ } else if ("CHECKCOMPOUNDREP".equals(firstWord)) {+ checkCompoundRep = true;} else if ("CHECKCOMPOUNDTRIPLE".equals(firstWord)) {checkCompoundTriple = true;} else if ("SIMPLIFIEDTRIPLE".equals(firstWord)) {diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/RepEntry.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/RepEntry.javaindex 1cd5738b9ed..52f8f5943a0 100644--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/RepEntry.java+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/RepEntry.java@@ -35,6 +35,10 @@ class RepEntry {patternLen = pattern.length();}+ boolean isMiddle() {+ return !mustStart && !mustEnd;+ }+List<String> substitute(String word) {if (mustStart) {boolean matches = mustEnd ? word.equals(pattern) : word.startsWith(pattern);diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/SpellChecker.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/SpellChecker.javaindex e59dee20419..1b0c2d3dc0c 100644--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/SpellChecker.java+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/SpellChecker.java@@ -26,7 +26,6 @@ import java.util.ArrayList;import java.util.Collections;import java.util.List;import java.util.Set;-import java.util.function.Predicate;import org.apache.lucene.util.BytesRef;import org.apache.lucene.util.CharsRef;import org.apache.lucene.util.IntsRef;@@ -150,7 +149,7 @@ public class SpellChecker {}if (dictionary.compoundBegin != FLAG_UNSET || dictionary.compoundFlag != FLAG_UNSET) {- return checkCompounds(new CharsRef(wordChars, 0, length), originalCase, 0, __ -> true);+ return checkCompounds(new CharsRef(wordChars, 0, length), originalCase, null);}return false;@@ -172,13 +171,12 @@ public class SpellChecker {return result[0];}- private boolean checkCompounds(- CharsRef word, WordCase originalCase, int depth, Predicate<CharsRef> checkPatterns) {- if (depth > dictionary.compoundMax - 2) return false;+ private boolean checkCompounds(CharsRef word, WordCase originalCase, CompoundPart prev) {+ if (prev != null && prev.index > dictionary.compoundMax - 2) return false;int limit = word.length - dictionary.compoundMin + 1;for (int breakPos = dictionary.compoundMin; breakPos < limit; breakPos++) {- WordContext context = depth == 0 ? COMPOUND_BEGIN : COMPOUND_MIDDLE;+ WordContext context = prev == null ? COMPOUND_BEGIN : COMPOUND_MIDDLE;int breakOffset = word.offset + breakPos;if (mayBreakIntoCompounds(word.chars, word.offset, word.length, breakOffset)) {CharsRef stem = findStem(word.chars, word.offset, breakPos, originalCase, context);@@ -187,15 +185,15 @@ public class SpellChecker {&& word.chars[breakOffset - 1] == word.chars[breakOffset]) {stem = findStem(word.chars, word.offset, breakPos + 1, originalCase, context);}- if (stem != null && checkPatterns.test(stem)) {- Predicate<CharsRef> nextCheck = checkNextPatterns(word, breakPos, stem);- if (checkCompoundsAfter(word, breakPos, originalCase, depth, stem, nextCheck)) {+ if (stem != null && (prev == null || prev.mayCompound(stem, breakPos, originalCase))) {+ CompoundPart part = new CompoundPart(prev, word, breakPos, stem, null);+ if (checkCompoundsAfter(originalCase, part)) {return true;}}}- if (checkCompoundPatternReplacements(word, breakPos, originalCase, depth)) {+ if (checkCompoundPatternReplacements(word, breakPos, originalCase, prev)) {return true;}}@@ -204,17 +202,16 @@ public class SpellChecker {}private boolean checkCompoundPatternReplacements(- CharsRef word, int pos, WordCase originalCase, int depth) {+ CharsRef word, int pos, WordCase originalCase, CompoundPart prev) {for (CheckCompoundPattern pattern : dictionary.checkCompoundPatterns) {CharsRef expanded = pattern.expandReplacement(word, pos);if (expanded != null) {- WordContext context = depth == 0 ? COMPOUND_BEGIN : COMPOUND_MIDDLE;+ WordContext context = prev == null ? COMPOUND_BEGIN : COMPOUND_MIDDLE;int breakPos = pos + pattern.endLength();CharsRef stem = findStem(expanded.chars, expanded.offset, breakPos, originalCase, context);if (stem != null) {- Predicate<CharsRef> nextCheck =- next -> pattern.prohibitsCompounding(expanded, breakPos, stem, next);- if (checkCompoundsAfter(expanded, breakPos, originalCase, depth, stem, nextCheck)) {+ CompoundPart part = new CompoundPart(prev, expanded, breakPos, stem, pattern);+ if (checkCompoundsAfter(originalCase, part)) {return true;}}@@ -223,32 +220,22 @@ public class SpellChecker {return false;}- private Predicate<CharsRef> checkNextPatterns(CharsRef word, int breakPos, CharsRef stems) {- return nextStem ->- dictionary.checkCompoundPatterns.stream()- .noneMatch(p -> p.prohibitsCompounding(word, breakPos, stems, nextStem));- }-- private boolean checkCompoundsAfter(- CharsRef word,- int breakPos,- WordCase originalCase,- int depth,- CharsRef prevStem,- Predicate<CharsRef> checkPatterns) {+ private boolean checkCompoundsAfter(WordCase originalCase, CompoundPart prev) {+ CharsRef word = prev.tail;+ int breakPos = prev.length;int remainingLength = word.length - breakPos;int breakOffset = word.offset + breakPos;CharsRef tailStem =findStem(word.chars, breakOffset, remainingLength, originalCase, COMPOUND_END);if (tailStem != null- && !(dictionary.checkCompoundDup && equalsIgnoreCase(prevStem, tailStem))+ && !(dictionary.checkCompoundDup && equalsIgnoreCase(prev.stem, tailStem))&& !hasForceUCaseProblem(word.chars, breakOffset, remainingLength, originalCase)- && checkPatterns.test(tailStem)) {+ && prev.mayCompound(tailStem, remainingLength, originalCase)) {return true;}CharsRef tail = new CharsRef(word.chars, breakOffset, remainingLength);- return checkCompounds(tail, originalCase, depth + 1, checkPatterns);+ return checkCompounds(tail, originalCase, prev);}private boolean hasForceUCaseProblem(@@ -260,14 +247,64 @@ public class SpellChecker {return forms != null && dictionary.hasFlag(forms, dictionary.forceUCase, scratch);}- private boolean intersectIgnoreCase(List<CharsRef> stems1, List<CharsRef> stems2) {- return stems1.stream().anyMatch(s1 -> stems2.stream().anyMatch(s2 -> equalsIgnoreCase(s1, s2)));- }-private boolean equalsIgnoreCase(CharsRef cr1, CharsRef cr2) {return cr1.toString().equalsIgnoreCase(cr2.toString());}+ private class CompoundPart {+ final CompoundPart prev;+ final int index, length;+ final CharsRef tail, stem;+ final CheckCompoundPattern enablingPattern;++ CompoundPart(+ CompoundPart prev, CharsRef tail, int length, CharsRef stem, CheckCompoundPattern enabler) {+ this.prev = prev;+ this.tail = tail;+ this.length = length;+ this.stem = stem;+ index = prev == null ? 1 : prev.index + 1;+ enablingPattern = enabler;+ }++ @Override+ public String toString() {+ return (prev == null ? "" : prev + "+") + tail.subSequence(0, length);+ }++ boolean mayCompound(CharsRef nextStem, int nextPartLength, WordCase originalCase) {+ boolean patternsOk =+ enablingPattern != null+ ? enablingPattern.prohibitsCompounding(tail, length, stem, nextStem)+ : dictionary.checkCompoundPatterns.stream()+ .noneMatch(p -> p.prohibitsCompounding(tail, length, stem, nextStem));+ if (!patternsOk) {+ return false;+ }++ //noinspection RedundantIfStatement+ if (dictionary.checkCompoundRep+ && isMisspelledSimpleWord(length + nextPartLength, originalCase)) {+ return false;+ }+ return true;+ }++ private boolean isMisspelledSimpleWord(int length, WordCase originalCase) {+ String word = new String(tail.chars, tail.offset, length);+ for (RepEntry entry : dictionary.repTable) {+ if (entry.isMiddle()) {+ for (String sug : entry.substitute(word)) {+ if (findStem(sug.toCharArray(), 0, sug.length(), originalCase, SIMPLE_WORD) != null) {+ return true;+ }+ }+ }+ }+ return false;+ }+ }+private boolean mayBreakIntoCompounds(char[] chars, int offset, int length, int breakPos) {if (dictionary.checkCompoundCase) {if (Character.isUpperCase(chars[breakPos - 1]) || Character.isUpperCase(chars[breakPos])) {diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/SpellCheckerTest.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/SpellCheckerTest.javaindex dd5ed0c5e74..a2399afcabe 100644--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/SpellCheckerTest.java+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/SpellCheckerTest.java@@ -120,6 +120,10 @@ public class SpellCheckerTest extends StemmerTestBase {doTest("breakoff");}+ public void testCheckCompoundRep() throws Exception {+ doTest("checkcompoundrep");+ }+public void testCompoundrule() throws Exception {doTest("compoundrule");}diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/checkcompoundrep.aff b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/checkcompoundrep.affnew file mode 100644index 00000000000..20545931c8a--- /dev/null+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/checkcompoundrep.aff@@ -0,0 +1,8 @@+// forbid compound word, if it is also a non-compound word with a REP fault+// In example: Hungarian `szerv�z' (szer+v�z) compound word is forbidden, because+// this word is also a dictionary word (szerviz) with typical fault (i->�)+CHECKCOMPOUNDREP+COMPOUNDFLAG A++REP 1+REP � idiff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/checkcompoundrep.dic b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/checkcompoundrep.dicnew file mode 100644index 00000000000..030bda916c3--- /dev/null+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/checkcompoundrep.dic@@ -0,0 +1,5 @@+3+szer/A+v�z/A+szerviz+kocsi/Adiff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/checkcompoundrep.good b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/checkcompoundrep.goodnew file mode 100644index 00000000000..80b19b27d3b--- /dev/null+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/checkcompoundrep.good@@ -0,0 +1,2 @@+vízszer+szerkocsidiff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/checkcompoundrep.wrong b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/checkcompoundrep.wrongnew file mode 100644index 00000000000..13ae6f60e00--- /dev/null+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/checkcompoundrep.wrong@@ -0,0 +1,3 @@+szervíz+szervízkocsi+kocsiszervíz