Skip to content

armgit / solr

Public Imported from GitHub
0

Apache Solr open-source search software

LUCENE-9729: Hunspell: support CHECKCOMPOUNDREP flags (#2300)

Peter Gromov650f16ad5d8e
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/CheckCompoundPattern.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/CheckCompoundPattern.java
index ac5208c1868..3d7059166d7 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/CheckCompoundPattern.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/CheckCompoundPattern.java
@@ -21,9 +21,9 @@ import org.apache.lucene.util.CharsRef;
import org.apache.lucene.util.IntsRef;
class CheckCompoundPattern {
- private final char[] endChars;
- private final char[] beginChars;
- private final char[] replacement;
+ private final String endChars;
+ private final String beginChars;
+ private final String replacement;
private final char[] endFlags;
private final char[] beginFlags;
private final Dictionary dictionary;
@@ -38,36 +38,33 @@ class CheckCompoundPattern {
}
int flagSep = parts[1].indexOf("/");
- endChars = (flagSep < 0 ? parts[1] : parts[1].substring(0, flagSep)).toCharArray();
+ endChars = flagSep < 0 ? parts[1] : parts[1].substring(0, flagSep);
endFlags = flagSep < 0 ? new char[0] : strategy.parseFlags(parts[1].substring(flagSep + 1));
flagSep = parts[2].indexOf("/");
- beginChars = (flagSep < 0 ? parts[2] : parts[2].substring(0, flagSep)).toCharArray();
+ beginChars = flagSep < 0 ? parts[2] : parts[2].substring(0, flagSep);
beginFlags = flagSep < 0 ? new char[0] : strategy.parseFlags(parts[2].substring(flagSep + 1));
- replacement = parts.length == 3 ? null : parts[3].toCharArray();
+ replacement = parts.length == 3 ? null : parts[3];
}
@Override
public String toString() {
- return new String(endChars)
- + " "
- + new String(beginChars)
- + (replacement == null ? "" : " -> " + new String(replacement));
+ return endChars + " " + beginChars + (replacement == null ? "" : " -> " + replacement);
}
boolean prohibitsCompounding(
CharsRef word, int breakPos, CharsRef stemBefore, CharsRef stemAfter) {
if (isNonAffixedPattern(endChars)) {
- if (!charsMatch(word, breakPos - stemBefore.length, stemBefore.chars)) {
+ if (!charsMatch(word, breakPos - stemBefore.length, stemBefore)) {
return false;
}
- } else if (!charsMatch(word, breakPos - endChars.length, endChars)) {
+ } else if (!charsMatch(word, breakPos - endChars.length(), endChars)) {
return false;
}
if (isNonAffixedPattern(beginChars)) {
- if (!charsMatch(word, breakPos, stemAfter.chars)) {
+ if (!charsMatch(word, breakPos, stemAfter)) {
return false;
}
} else if (!charsMatch(word, breakPos, beginChars)) {
@@ -85,8 +82,8 @@ class CheckCompoundPattern {
return true;
}
- private static boolean isNonAffixedPattern(char[] pattern) {
- return pattern.length == 1 && pattern[0] == '0';
+ private static boolean isNonAffixedPattern(String pattern) {
+ return pattern.length() == 1 && pattern.charAt(0) == '0';
}
private boolean stemHasFlags(CharsRef stem, char[] flags) {
@@ -107,25 +104,25 @@ class CheckCompoundPattern {
if (replacement != null && charsMatch(word, breakPos, replacement)) {
return new CharsRef(
word.subSequence(0, breakPos)
- + new String(endChars)
- + new String(beginChars)
- + word.subSequence(breakPos + replacement.length, word.length));
+ + endChars
+ + beginChars
+ + word.subSequence(breakPos + replacement.length(), word.length));
}
return null;
}
int endLength() {
- return endChars.length;
+ return endChars.length();
}
- private static boolean charsMatch(CharsRef word, int offset, char[] pattern) {
- int len = pattern.length;
+ private static boolean charsMatch(CharsRef word, int offset, CharSequence pattern) {
+ int len = pattern.length();
if (word.length - offset < len || offset < 0 || offset > word.length) {
return false;
}
for (int i = 0; i < len; i++) {
- if (word.chars[word.offset + offset + i] != pattern[i]) {
+ if (word.chars[word.offset + offset + i] != pattern.charAt(i)) {
return false;
}
}
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java
index 567e26f519b..ebb76ee9eab 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java
@@ -152,7 +152,7 @@ public class Dictionary {
char forbiddenword;
char onlyincompound, compoundBegin, compoundMiddle, compoundEnd, compoundFlag;
char compoundPermit, compoundForbid;
- boolean checkCompoundCase, checkCompoundDup;
+ boolean checkCompoundCase, checkCompoundDup, checkCompoundRep;
boolean checkCompoundTriple, simplifiedTriple;
int compoundMin = 3, compoundMax = Integer.MAX_VALUE;
List<CompoundRule> compoundRules; // nullable
@@ -425,6 +425,8 @@ public class Dictionary {
checkCompoundCase = true;
} else if ("CHECKCOMPOUNDDUP".equals(firstWord)) {
checkCompoundDup = true;
+ } else if ("CHECKCOMPOUNDREP".equals(firstWord)) {
+ checkCompoundRep = true;
} else if ("CHECKCOMPOUNDTRIPLE".equals(firstWord)) {
checkCompoundTriple = true;
} else if ("SIMPLIFIEDTRIPLE".equals(firstWord)) {
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/RepEntry.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/RepEntry.java
index 1cd5738b9ed..52f8f5943a0 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/RepEntry.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/RepEntry.java
@@ -35,6 +35,10 @@ class RepEntry {
patternLen = pattern.length();
}
+ boolean isMiddle() {
+ return !mustStart && !mustEnd;
+ }
+
List<String> substitute(String word) {
if (mustStart) {
boolean matches = mustEnd ? word.equals(pattern) : word.startsWith(pattern);
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/SpellChecker.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/SpellChecker.java
index e59dee20419..1b0c2d3dc0c 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/SpellChecker.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/SpellChecker.java
@@ -26,7 +26,6 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Set;
-import java.util.function.Predicate;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CharsRef;
import org.apache.lucene.util.IntsRef;
@@ -150,7 +149,7 @@ public class SpellChecker {
}
if (dictionary.compoundBegin != FLAG_UNSET || dictionary.compoundFlag != FLAG_UNSET) {
- return checkCompounds(new CharsRef(wordChars, 0, length), originalCase, 0, __ -> true);
+ return checkCompounds(new CharsRef(wordChars, 0, length), originalCase, null);
}
return false;
@@ -172,13 +171,12 @@ public class SpellChecker {
return result[0];
}
- private boolean checkCompounds(
- CharsRef word, WordCase originalCase, int depth, Predicate<CharsRef> checkPatterns) {
- if (depth > dictionary.compoundMax - 2) return false;
+ private boolean checkCompounds(CharsRef word, WordCase originalCase, CompoundPart prev) {
+ if (prev != null && prev.index > dictionary.compoundMax - 2) return false;
int limit = word.length - dictionary.compoundMin + 1;
for (int breakPos = dictionary.compoundMin; breakPos < limit; breakPos++) {
- WordContext context = depth == 0 ? COMPOUND_BEGIN : COMPOUND_MIDDLE;
+ WordContext context = prev == null ? COMPOUND_BEGIN : COMPOUND_MIDDLE;
int breakOffset = word.offset + breakPos;
if (mayBreakIntoCompounds(word.chars, word.offset, word.length, breakOffset)) {
CharsRef stem = findStem(word.chars, word.offset, breakPos, originalCase, context);
@@ -187,15 +185,15 @@ public class SpellChecker {
&& word.chars[breakOffset - 1] == word.chars[breakOffset]) {
stem = findStem(word.chars, word.offset, breakPos + 1, originalCase, context);
}
- if (stem != null && checkPatterns.test(stem)) {
- Predicate<CharsRef> nextCheck = checkNextPatterns(word, breakPos, stem);
- if (checkCompoundsAfter(word, breakPos, originalCase, depth, stem, nextCheck)) {
+ if (stem != null && (prev == null || prev.mayCompound(stem, breakPos, originalCase))) {
+ CompoundPart part = new CompoundPart(prev, word, breakPos, stem, null);
+ if (checkCompoundsAfter(originalCase, part)) {
return true;
}
}
}
- if (checkCompoundPatternReplacements(word, breakPos, originalCase, depth)) {
+ if (checkCompoundPatternReplacements(word, breakPos, originalCase, prev)) {
return true;
}
}
@@ -204,17 +202,16 @@ public class SpellChecker {
}
private boolean checkCompoundPatternReplacements(
- CharsRef word, int pos, WordCase originalCase, int depth) {
+ CharsRef word, int pos, WordCase originalCase, CompoundPart prev) {
for (CheckCompoundPattern pattern : dictionary.checkCompoundPatterns) {
CharsRef expanded = pattern.expandReplacement(word, pos);
if (expanded != null) {
- WordContext context = depth == 0 ? COMPOUND_BEGIN : COMPOUND_MIDDLE;
+ WordContext context = prev == null ? COMPOUND_BEGIN : COMPOUND_MIDDLE;
int breakPos = pos + pattern.endLength();
CharsRef stem = findStem(expanded.chars, expanded.offset, breakPos, originalCase, context);
if (stem != null) {
- Predicate<CharsRef> nextCheck =
- next -> pattern.prohibitsCompounding(expanded, breakPos, stem, next);
- if (checkCompoundsAfter(expanded, breakPos, originalCase, depth, stem, nextCheck)) {
+ CompoundPart part = new CompoundPart(prev, expanded, breakPos, stem, pattern);
+ if (checkCompoundsAfter(originalCase, part)) {
return true;
}
}
@@ -223,32 +220,22 @@ public class SpellChecker {
return false;
}
- private Predicate<CharsRef> checkNextPatterns(CharsRef word, int breakPos, CharsRef stems) {
- return nextStem ->
- dictionary.checkCompoundPatterns.stream()
- .noneMatch(p -> p.prohibitsCompounding(word, breakPos, stems, nextStem));
- }
-
- private boolean checkCompoundsAfter(
- CharsRef word,
- int breakPos,
- WordCase originalCase,
- int depth,
- CharsRef prevStem,
- Predicate<CharsRef> checkPatterns) {
+ private boolean checkCompoundsAfter(WordCase originalCase, CompoundPart prev) {
+ CharsRef word = prev.tail;
+ int breakPos = prev.length;
int remainingLength = word.length - breakPos;
int breakOffset = word.offset + breakPos;
CharsRef tailStem =
findStem(word.chars, breakOffset, remainingLength, originalCase, COMPOUND_END);
if (tailStem != null
- && !(dictionary.checkCompoundDup && equalsIgnoreCase(prevStem, tailStem))
+ && !(dictionary.checkCompoundDup && equalsIgnoreCase(prev.stem, tailStem))
&& !hasForceUCaseProblem(word.chars, breakOffset, remainingLength, originalCase)
- && checkPatterns.test(tailStem)) {
+ && prev.mayCompound(tailStem, remainingLength, originalCase)) {
return true;
}
CharsRef tail = new CharsRef(word.chars, breakOffset, remainingLength);
- return checkCompounds(tail, originalCase, depth + 1, checkPatterns);
+ return checkCompounds(tail, originalCase, prev);
}
private boolean hasForceUCaseProblem(
@@ -260,14 +247,64 @@ public class SpellChecker {
return forms != null && dictionary.hasFlag(forms, dictionary.forceUCase, scratch);
}
- private boolean intersectIgnoreCase(List<CharsRef> stems1, List<CharsRef> stems2) {
- return stems1.stream().anyMatch(s1 -> stems2.stream().anyMatch(s2 -> equalsIgnoreCase(s1, s2)));
- }
-
private boolean equalsIgnoreCase(CharsRef cr1, CharsRef cr2) {
return cr1.toString().equalsIgnoreCase(cr2.toString());
}
+ private class CompoundPart {
+ final CompoundPart prev;
+ final int index, length;
+ final CharsRef tail, stem;
+ final CheckCompoundPattern enablingPattern;
+
+ CompoundPart(
+ CompoundPart prev, CharsRef tail, int length, CharsRef stem, CheckCompoundPattern enabler) {
+ this.prev = prev;
+ this.tail = tail;
+ this.length = length;
+ this.stem = stem;
+ index = prev == null ? 1 : prev.index + 1;
+ enablingPattern = enabler;
+ }
+
+ @Override
+ public String toString() {
+ return (prev == null ? "" : prev + "+") + tail.subSequence(0, length);
+ }
+
+ boolean mayCompound(CharsRef nextStem, int nextPartLength, WordCase originalCase) {
+ boolean patternsOk =
+ enablingPattern != null
+ ? enablingPattern.prohibitsCompounding(tail, length, stem, nextStem)
+ : dictionary.checkCompoundPatterns.stream()
+ .noneMatch(p -> p.prohibitsCompounding(tail, length, stem, nextStem));
+ if (!patternsOk) {
+ return false;
+ }
+
+ //noinspection RedundantIfStatement
+ if (dictionary.checkCompoundRep
+ && isMisspelledSimpleWord(length + nextPartLength, originalCase)) {
+ return false;
+ }
+ return true;
+ }
+
+ private boolean isMisspelledSimpleWord(int length, WordCase originalCase) {
+ String word = new String(tail.chars, tail.offset, length);
+ for (RepEntry entry : dictionary.repTable) {
+ if (entry.isMiddle()) {
+ for (String sug : entry.substitute(word)) {
+ if (findStem(sug.toCharArray(), 0, sug.length(), originalCase, SIMPLE_WORD) != null) {
+ return true;
+ }
+ }
+ }
+ }
+ return false;
+ }
+ }
+
private boolean mayBreakIntoCompounds(char[] chars, int offset, int length, int breakPos) {
if (dictionary.checkCompoundCase) {
if (Character.isUpperCase(chars[breakPos - 1]) || Character.isUpperCase(chars[breakPos])) {
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/SpellCheckerTest.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/SpellCheckerTest.java
index dd5ed0c5e74..a2399afcabe 100644
--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/SpellCheckerTest.java
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/SpellCheckerTest.java
@@ -120,6 +120,10 @@ public class SpellCheckerTest extends StemmerTestBase {
doTest("breakoff");
}
+ public void testCheckCompoundRep() throws Exception {
+ doTest("checkcompoundrep");
+ }
+
public void testCompoundrule() throws Exception {
doTest("compoundrule");
}
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/checkcompoundrep.aff b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/checkcompoundrep.aff
new file mode 100644
index 00000000000..20545931c8a
--- /dev/null
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/checkcompoundrep.aff
@@ -0,0 +1,8 @@
+// forbid compound word, if it is also a non-compound word with a REP fault
+// In example: Hungarian `szerv�z' (szer+v�z) compound word is forbidden, because
+// this word is also a dictionary word (szerviz) with typical fault (i->�)
+CHECKCOMPOUNDREP
+COMPOUNDFLAG A
+
+REP 1
+REP � i
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/checkcompoundrep.dic b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/checkcompoundrep.dic
new file mode 100644
index 00000000000..030bda916c3
--- /dev/null
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/checkcompoundrep.dic
@@ -0,0 +1,5 @@
+3
+szer/A
+v�z/A
+szerviz
+kocsi/A
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/checkcompoundrep.good b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/checkcompoundrep.good
new file mode 100644
index 00000000000..80b19b27d3b
--- /dev/null
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/checkcompoundrep.good
@@ -0,0 +1,2 @@
+vízszer
+szerkocsi
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/checkcompoundrep.wrong b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/checkcompoundrep.wrong
new file mode 100644
index 00000000000..13ae6f60e00
--- /dev/null
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/checkcompoundrep.wrong
@@ -0,0 +1,3 @@
+szervíz
+szervízkocsi
+kocsiszervíz