Apache Solr open-source search software
LUCENE-9717: Hunspell: support CHECKCOMPOUNDPATTERN (#2280)
Peter Gromovd0ae2bd2b9c9
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/CheckCompoundPattern.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/CheckCompoundPattern.javanew file mode 100644index 00000000000..44867bc5043--- /dev/null+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/CheckCompoundPattern.java@@ -0,0 +1,141 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.analysis.hunspell;++import java.util.List;+import org.apache.lucene.util.BytesRef;+import org.apache.lucene.util.CharsRef;+import org.apache.lucene.util.IntsRef;++class CheckCompoundPattern {+ private final char[] endChars;+ private final char[] beginChars;+ private final char[] replacement;+ private final char[] endFlags;+ private final char[] beginFlags;+ private final Dictionary dictionary;+ private final BytesRef scratch = new BytesRef();++ CheckCompoundPattern(+ String unparsed, Dictionary.FlagParsingStrategy strategy, Dictionary dictionary) {+ this.dictionary = dictionary;+ String[] parts = unparsed.split("\\s+");+ if (parts.length < 3) {+ throw new IllegalArgumentException("Invalid pattern: " + unparsed);+ }++ int flagSep = parts[1].indexOf("/");+ endChars = (flagSep < 0 ? parts[1] : parts[1].substring(0, flagSep)).toCharArray();+ endFlags = flagSep < 0 ? new char[0] : strategy.parseFlags(parts[1].substring(flagSep + 1));++ flagSep = parts[2].indexOf("/");+ beginChars = (flagSep < 0 ? parts[2] : parts[2].substring(0, flagSep)).toCharArray();+ beginFlags = flagSep < 0 ? new char[0] : strategy.parseFlags(parts[2].substring(flagSep + 1));++ replacement = parts.length == 3 ? null : parts[3].toCharArray();+ }++ @Override+ public String toString() {+ return new String(endChars)+ + " "+ + new String(beginChars)+ + (replacement == null ? "" : " -> " + new String(replacement));+ }++ boolean prohibitsCompounding(+ CharsRef word, int breakPos, List<CharsRef> stemsBefore, List<CharsRef> stemsAfter) {+ if (isNonAffixedPattern(endChars)) {+ if (stemsBefore.stream()+ .noneMatch(stem -> charsMatch(word, breakPos - stem.length, stem.chars))) {+ return false;+ }+ } else if (!charsMatch(word, breakPos - endChars.length, endChars)) {+ return false;+ }++ if (isNonAffixedPattern(beginChars)) {+ if (stemsAfter.stream().noneMatch(stem -> charsMatch(word, breakPos, stem.chars))) {+ return false;+ }+ } else if (!charsMatch(word, breakPos, beginChars)) {+ return false;+ }++ if (endFlags.length > 0 && !hasStemWithFlags(stemsBefore, endFlags)) {+ return false;+ }+ //noinspection RedundantIfStatement+ if (beginFlags.length > 0 && !hasStemWithFlags(stemsAfter, beginFlags)) {+ return false;+ }++ return true;+ }++ private static boolean isNonAffixedPattern(char[] pattern) {+ return pattern.length == 1 && pattern[0] == '0';+ }++ private boolean hasStemWithFlags(List<CharsRef> stems, char[] flags) {+ for (CharsRef stem : stems) {+ IntsRef forms = dictionary.lookupWord(stem.chars, stem.offset, stem.length);+ if (forms != null && hasAllFlags(flags, forms)) {+ return true;+ }+ }+ return false;+ }++ private boolean hasAllFlags(char[] flags, IntsRef forms) {+ for (char flag : flags) {+ if (!dictionary.hasFlag(forms, flag, scratch)) {+ return false;+ }+ }+ return true;+ }++ CharsRef expandReplacement(CharsRef word, int breakPos) {+ if (replacement != null && charsMatch(word, breakPos, replacement)) {+ return new CharsRef(+ word.subSequence(0, breakPos)+ + new String(endChars)+ + new String(beginChars)+ + word.subSequence(breakPos + replacement.length, word.length));+ }+ return null;+ }++ int endLength() {+ return endChars.length;+ }++ private static boolean charsMatch(CharsRef word, int offset, char[] pattern) {+ int len = pattern.length;+ if (word.length - offset < len || offset < 0 || offset > word.length) {+ return false;+ }++ for (int i = 0; i < len; i++) {+ if (word.chars[word.offset + offset + i] != pattern[i]) {+ return false;+ }+ }+ return true;+ }+}diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.javaindex 10340b8acdb..47c57a3bd4c 100644--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java@@ -146,6 +146,7 @@ public class Dictionary {boolean checkCompoundTriple, simplifiedTriple;int compoundMin = 3, compoundMax = Integer.MAX_VALUE;List<CompoundRule> compoundRules; // nullable+ List<CheckCompoundPattern> checkCompoundPatterns = new ArrayList<>();// ignored characters (dictionary, affix, inputs)private char[] ignore;@@ -412,6 +413,12 @@ public class Dictionary {checkCompoundTriple = true;} else if ("SIMPLIFIEDTRIPLE".equals(firstWord)) {simplifiedTriple = true;+ } else if ("CHECKCOMPOUNDPATTERN".equals(firstWord)) {+ int count = Integer.parseInt(singleArgument(reader, line));+ for (int i = 0; i < count; i++) {+ checkCompoundPatterns.add(+ new CheckCompoundPattern(reader.readLine(), flagParsingStrategy, this));+ }}}diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/SpellChecker.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/SpellChecker.javaindex aa0aeac53ec..747b209fa32 100644--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/SpellChecker.java+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/SpellChecker.java@@ -26,6 +26,7 @@ import java.util.ArrayList;import java.util.Collections;import java.util.List;import java.util.Set;+import java.util.function.Predicate;import org.apache.lucene.util.BytesRef;import org.apache.lucene.util.CharsRef;import org.apache.lucene.util.IntsRef;@@ -149,47 +150,94 @@ public class SpellChecker {}if (dictionary.compoundBegin != FLAG_UNSET || dictionary.compoundFlag != FLAG_UNSET) {- return checkCompounds(wordChars, 0, length, originalCase, 0);+ return checkCompounds(new CharsRef(wordChars, 0, length), originalCase, 0, __ -> true);}return false;}private boolean checkCompounds(- char[] chars, int offset, int length, WordCase originalCase, int depth) {+ CharsRef word, WordCase originalCase, int depth, Predicate<List<CharsRef>> checkPatterns) {if (depth > dictionary.compoundMax - 2) return false;- int limit = length - dictionary.compoundMin + 1;+ int limit = word.length - dictionary.compoundMin + 1;for (int breakPos = dictionary.compoundMin; breakPos < limit; breakPos++) {WordContext context = depth == 0 ? COMPOUND_BEGIN : COMPOUND_MIDDLE;- int breakOffset = offset + breakPos;- if (mayBreakIntoCompounds(chars, offset, length, breakOffset)) {- List<CharsRef> stems = stemmer.doStem(chars, offset, breakPos, originalCase, context);+ int breakOffset = word.offset + breakPos;+ if (mayBreakIntoCompounds(word.chars, word.offset, word.length, breakOffset)) {+ List<CharsRef> stems =+ stemmer.doStem(word.chars, word.offset, breakPos, originalCase, context);if (stems.isEmpty()&& dictionary.simplifiedTriple- && chars[breakOffset - 1] == chars[breakOffset]) {- stems = stemmer.doStem(chars, offset, breakPos + 1, originalCase, context);+ && word.chars[breakOffset - 1] == word.chars[breakOffset]) {+ stems = stemmer.doStem(word.chars, word.offset, breakPos + 1, originalCase, context);}- if (stems.isEmpty()) continue;-- int remainingLength = length - breakPos;- List<CharsRef> lastStems =- stemmer.doStem(chars, breakOffset, remainingLength, originalCase, COMPOUND_END);- if (!lastStems.isEmpty()- && !(dictionary.checkCompoundDup && intersectIgnoreCase(stems, lastStems))- && !hasForceUCaseProblem(chars, breakOffset, remainingLength, originalCase)) {- return true;+ if (!stems.isEmpty() && checkPatterns.test(stems)) {+ Predicate<List<CharsRef>> nextCheck = checkNextPatterns(word, breakPos, stems);+ if (checkCompoundsAfter(word, breakPos, originalCase, depth, stems, nextCheck)) {+ return true;+ }}+ }- if (checkCompounds(chars, breakOffset, remainingLength, originalCase, depth + 1)) {- return true;- }+ if (checkCompoundPatternReplacements(word, breakPos, originalCase, depth)) {+ return true;}}return false;}+ private boolean checkCompoundPatternReplacements(+ CharsRef word, int pos, WordCase originalCase, int depth) {+ for (CheckCompoundPattern pattern : dictionary.checkCompoundPatterns) {+ CharsRef expanded = pattern.expandReplacement(word, pos);+ if (expanded != null) {+ WordContext context = depth == 0 ? COMPOUND_BEGIN : COMPOUND_MIDDLE;+ int breakPos = pos + pattern.endLength();+ List<CharsRef> stems =+ stemmer.doStem(expanded.chars, expanded.offset, breakPos, originalCase, context);+ if (!stems.isEmpty()) {+ Predicate<List<CharsRef>> nextCheck =+ next -> pattern.prohibitsCompounding(expanded, breakPos, stems, next);+ if (checkCompoundsAfter(expanded, breakPos, originalCase, depth, stems, nextCheck)) {+ return true;+ }+ }+ }+ }+ return false;+ }++ private Predicate<List<CharsRef>> checkNextPatterns(+ CharsRef word, int breakPos, List<CharsRef> stems) {+ return nextStems ->+ dictionary.checkCompoundPatterns.stream()+ .noneMatch(p -> p.prohibitsCompounding(word, breakPos, stems, nextStems));+ }++ private boolean checkCompoundsAfter(+ CharsRef word,+ int breakPos,+ WordCase originalCase,+ int depth,+ List<CharsRef> prevStems,+ Predicate<List<CharsRef>> checkPatterns) {+ int remainingLength = word.length - breakPos;+ int breakOffset = word.offset + breakPos;+ List<CharsRef> tailStems =+ stemmer.doStem(word.chars, breakOffset, remainingLength, originalCase, COMPOUND_END);+ if (!tailStems.isEmpty()+ && !(dictionary.checkCompoundDup && intersectIgnoreCase(prevStems, tailStems))+ && !hasForceUCaseProblem(word.chars, breakOffset, remainingLength, originalCase)+ && checkPatterns.test(tailStems)) {+ return true;+ }++ CharsRef tail = new CharsRef(word.chars, breakOffset, remainingLength);+ return checkCompounds(tail, originalCase, depth + 1, checkPatterns);+ }+private boolean hasForceUCaseProblem(char[] chars, int offset, int length, WordCase originalCase) {if (dictionary.forceUCase == FLAG_UNSET) return false;diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/SpellCheckerTest.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/SpellCheckerTest.javaindex cb568164a5d..49514ae6d8b 100644--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/SpellCheckerTest.java+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/SpellCheckerTest.java@@ -64,6 +64,18 @@ public class SpellCheckerTest extends StemmerTestBase {doTest("i53643");}+ public void testCheckCompoundPattern() throws Exception {+ doTest("checkcompoundpattern");+ }++ public void testCheckCompoundPattern2() throws Exception {+ doTest("checkcompoundpattern2");+ }++ public void testCheckCompoundPattern3() throws Exception {+ doTest("checkcompoundpattern3");+ }+public void testDotless_i() throws Exception {doTest("dotless_i");}diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/checkcompoundpattern.aff b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/checkcompoundpattern.affnew file mode 100644index 00000000000..dfda51af271--- /dev/null+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/checkcompoundpattern.aff@@ -0,0 +1,5 @@+# forbid compounds with spec. pattern at word bounds+COMPOUNDFLAG A+CHECKCOMPOUNDPATTERN 2+CHECKCOMPOUNDPATTERN nny ny+CHECKCOMPOUNDPATTERN ssz szdiff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/checkcompoundpattern.dic b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/checkcompoundpattern.dicnew file mode 100644index 00000000000..09300f0bcd3--- /dev/null+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/checkcompoundpattern.dic@@ -0,0 +1,5 @@+4+k�nny/A+nyel�s/A+hossz/A+sz�m�t�s/Adiff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/checkcompoundpattern.good b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/checkcompoundpattern.goodnew file mode 100644index 00000000000..f40229e8fdf--- /dev/null+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/checkcompoundpattern.good@@ -0,0 +1,2 @@+könnyszámítás+hossznyelésdiff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/checkcompoundpattern.wrong b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/checkcompoundpattern.wrongnew file mode 100644index 00000000000..378faf940fe--- /dev/null+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/checkcompoundpattern.wrong@@ -0,0 +1,4 @@+könnynyelés+hosszszámítás+hosszkönnynyelés+könnynyeléshosszdiff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/checkcompoundpattern2.aff b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/checkcompoundpattern2.affnew file mode 100644index 00000000000..fdf6560b4f9--- /dev/null+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/checkcompoundpattern2.aff@@ -0,0 +1,7 @@+# forbid compounds with spec. pattern at word bound and allow modificated form+# (for German and Indian languages)+COMPOUNDFLAG A+CHECKCOMPOUNDPATTERN 2+CHECKCOMPOUNDPATTERN o b z+CHECKCOMPOUNDPATTERN oo ba u+COMPOUNDMIN 1diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/checkcompoundpattern2.dic b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/checkcompoundpattern2.dicnew file mode 100644index 00000000000..8ac75f4fc5d--- /dev/null+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/checkcompoundpattern2.dic@@ -0,0 +1,3 @@+2+foo/A+bar/Adiff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/checkcompoundpattern2.good b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/checkcompoundpattern2.goodnew file mode 100644index 00000000000..eaad4f902b9--- /dev/null+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/checkcompoundpattern2.good@@ -0,0 +1,3 @@+barfoo+fozar+furdiff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/checkcompoundpattern2.wrong b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/checkcompoundpattern2.wrongnew file mode 100644index 00000000000..323fae03f46--- /dev/null+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/checkcompoundpattern2.wrong@@ -0,0 +1 @@+foobardiff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/checkcompoundpattern3.aff b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/checkcompoundpattern3.affnew file mode 100644index 00000000000..4feb3db3829--- /dev/null+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/checkcompoundpattern3.aff@@ -0,0 +1,6 @@+# forbid compounds with spec. pattern at word bound and allow modified form+# (for Indian languages)+COMPOUNDFLAG A+CHECKCOMPOUNDPATTERN 1+CHECKCOMPOUNDPATTERN o/X b/Y z+COMPOUNDMIN 1diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/checkcompoundpattern3.dic b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/checkcompoundpattern3.dicnew file mode 100644index 00000000000..6bd1b7fc9e5--- /dev/null+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/checkcompoundpattern3.dic@@ -0,0 +1,5 @@+4+foo/A+boo/AX+bar/A+ban/AYdiff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/checkcompoundpattern3.good b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/checkcompoundpattern3.goodnew file mode 100644index 00000000000..6070eff5c52--- /dev/null+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/checkcompoundpattern3.good@@ -0,0 +1,9 @@+bozan+barfoo+banfoo+banbar+foobar+fooban+foobanbar+boobar+boobarfoodiff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/checkcompoundpattern3.wrong b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/checkcompoundpattern3.wrongnew file mode 100644index 00000000000..41d8d374710--- /dev/null+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/checkcompoundpattern3.wrong@@ -0,0 +1,8 @@+booban+boobanfoo+fozar+fozarfoo+fozan+fozanfoo+bozar+bozarfoo