Apache Solr open-source search software
LUCENE-9709: Hunspell: no special dotted i treatment outside tr/az languages (#2271)
Peter Gromov40e92315aeaf
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.javaindex 4dc76abfd79..5fcbd4f730a 100644--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java@@ -163,7 +163,7 @@ public class Dictionary {// language declaration of the dictionaryString language;// true if case algorithms should use alternate (Turkish/Azeri) mapping- boolean alternateCasing;+ private boolean alternateCasing;/*** Creates a new Dictionary containing the information read from the provided InputStreams to@@ -1027,6 +1027,10 @@ public class Dictionary {}}+ boolean isDotICaseChangeDisallowed(char[] word) {+ return word[0] == 'İ' && !alternateCasing;+ }+private class EntryGrouper {final FSTCompiler<IntsRef> words =new FSTCompiler<>(FST.INPUT_TYPE.BYTE4, IntSequenceOutputs.getSingleton());diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/SpellChecker.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/SpellChecker.javaindex 32c1ab2b534..b0c095ae9b5 100644--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/SpellChecker.java+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/SpellChecker.java@@ -102,6 +102,11 @@ public class SpellChecker {}}}++ if (dictionary.isDotICaseChangeDisallowed(wordChars)) {+ return false;+ }+char[] lower = stemmer.caseFoldLower(caseVariant, wordChars.length);if (checkWord(lower, wordChars.length, wordCase)) {return true;diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Stemmer.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Stemmer.javaindex bcda07322a3..31237156dec 100644--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Stemmer.java+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Stemmer.java@@ -120,6 +120,11 @@ final class Stemmer {list.addAll(doStem(variation, 0, variation.length, null, WordContext.SIMPLE_WORD));}}++ if (dictionary.isDotICaseChangeDisallowed(word)) {+ return;+ }+caseFoldLower(wordCase == WordCase.UPPER ? titleBuffer : word, length);list.addAll(doStem(lowerBuffer, 0, length, wordCase, WordContext.SIMPLE_WORD));if (wordCase == WordCase.UPPER) {diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/SpellCheckerTest.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/SpellCheckerTest.javaindex 4c5601bb7bc..5bcfe05759c 100644--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/SpellCheckerTest.java+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/SpellCheckerTest.java@@ -31,6 +31,11 @@ public class SpellCheckerTest extends StemmerTestBase {doTest("base");}+ @Test+ public void baseUtf() throws Exception {+ doTest("base_utf");+ }+@Testpublic void keepcase() throws Exception {doTest("keepcase");diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestBaseUtf.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestBaseUtf.javanew file mode 100644index 00000000000..c787d3b18b7--- /dev/null+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestBaseUtf.java@@ -0,0 +1,41 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.analysis.hunspell;++import org.junit.BeforeClass;+import org.junit.Test;++public class TestBaseUtf extends StemmerTestBase {++ @BeforeClass+ public static void beforeClass() throws Exception {+ init("base_utf.aff", "base_utf.dic");+ }++ public void good() {+ assertStemsTo("imply", "imply");+ assertStemsTo("Imply", "imply");+ assertStemsTo("IMPLY", "imply");+ assertStemsTo("İZMİR", "İzmir");+ }++ @Test+ public void no_dottedI_decapitalization() {+ assertStemsTo("İmply");+ assertStemsTo("İMPLY");+ }+}diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/base_utf.aff b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/base_utf.affnew file mode 100644index 00000000000..493157b3018--- /dev/null+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/base_utf.aff@@ -0,0 +1,198 @@+# OpenOffice.org’s en_US.aff file+# with Unicode apostrophe: ’++SET UTF-8+TRY esianrtolcdugmphbyfvkwzESIANRTOLCDUGMPHBYFVKWZ'++MAXNGRAMSUGS 1+WORDCHARS .'’++PFX A Y 1+PFX A 0 re .++PFX I Y 1+PFX I 0 in .++PFX U Y 1+PFX U 0 un .++PFX C Y 1+PFX C 0 de .++PFX E Y 1+PFX E 0 dis .++PFX F Y 1+PFX F 0 con .++PFX K Y 1+PFX K 0 pro .++SFX V N 2+SFX V e ive e+SFX V 0 ive [^e]++SFX N Y 3+SFX N e ion e+SFX N y ication y+SFX N 0 en [^ey]++SFX X Y 3+SFX X e ions e+SFX X y ications y+SFX X 0 ens [^ey]++SFX H N 2+SFX H y ieth y+SFX H 0 th [^y]++SFX Y Y 1+SFX Y 0 ly .++SFX G Y 2+SFX G e ing e+SFX G 0 ing [^e]++SFX J Y 2+SFX J e ings e+SFX J 0 ings [^e]++SFX D Y 4+SFX D 0 d e+SFX D y ied [^aeiou]y+SFX D 0 ed [^ey]+SFX D 0 ed [aeiou]y++SFX T N 4+SFX T 0 st e+SFX T y iest [^aeiou]y+SFX T 0 est [aeiou]y+SFX T 0 est [^ey]++SFX R Y 4+SFX R 0 r e+SFX R y ier [^aeiou]y+SFX R 0 er [aeiou]y+SFX R 0 er [^ey]++SFX Z Y 4+SFX Z 0 rs e+SFX Z y iers [^aeiou]y+SFX Z 0 ers [aeiou]y+SFX Z 0 ers [^ey]++SFX S Y 4+SFX S y ies [^aeiou]y+SFX S 0 s [aeiou]y+SFX S 0 es [sxzh]+SFX S 0 s [^sxzhy]++SFX P Y 3+SFX P y iness [^aeiou]y+SFX P 0 ness [aeiou]y+SFX P 0 ness [^y]++SFX M Y 1+SFX M 0 's .++SFX B Y 3+SFX B 0 able [^aeiou]+SFX B 0 able ee+SFX B e able [^aeiou]e++SFX L Y 1+SFX L 0 ment .++REP 88+REP a ei+REP ei a+REP a ey+REP ey a+REP ai ie+REP ie ai+REP are air+REP are ear+REP are eir+REP air are+REP air ere+REP ere air+REP ere ear+REP ere eir+REP ear are+REP ear air+REP ear ere+REP eir are+REP eir ere+REP ch te+REP te ch+REP ch ti+REP ti ch+REP ch tu+REP tu ch+REP ch s+REP s ch+REP ch k+REP k ch+REP f ph+REP ph f+REP gh f+REP f gh+REP i igh+REP igh i+REP i uy+REP uy i+REP i ee+REP ee i+REP j di+REP di j+REP j gg+REP gg j+REP j ge+REP ge j+REP s ti+REP ti s+REP s ci+REP ci s+REP k cc+REP cc k+REP k qu+REP qu k+REP kw qu+REP o eau+REP eau o+REP o ew+REP ew o+REP oo ew+REP ew oo+REP ew ui+REP ui ew+REP oo ui+REP ui oo+REP ew u+REP u ew+REP oo u+REP u oo+REP u oe+REP oe u+REP u ieu+REP ieu u+REP ue ew+REP ew ue+REP uff ough+REP oo ieu+REP ieu oo+REP ier ear+REP ear ier+REP ear air+REP air ear+REP w qu+REP qu w+REP z ss+REP ss z+REP shun tion+REP shun sion+REP shun cion+McDonalds’sá/w+McDonald’sszá/g3) st:McDonaldâs po:noun_prs is:TRANS+McDonald’sszal/g4) st:McDonaldâs po:noun_prs is:INSTR+McDonald’ssal/wdiff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/base_utf.dic b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/base_utf.dicnew file mode 100644index 00000000000..b010030cb55--- /dev/null+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/base_utf.dic@@ -0,0 +1,30 @@+28+created/U+create/XKVNGADS+imply/GNSDX+natural/PUY+like/USPBY+convey/BDGS+look/GZRDS+text+hello+said+sawyer+NASA+rotten+day+tomorrow+seven+FAQ/SM+can’t+doesn’t+etc+won’t+lip+text+horrifying+speech+suggest+uncreate/V+Hunspell+İzmirdiff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/base_utf.good b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/base_utf.goodnew file mode 100644index 00000000000..aed11f2b781--- /dev/null+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/base_utf.good@@ -0,0 +1,33 @@+created+uncreate+uncreated+imply+implied+unnatural+conveyed+sawyer+NASA+FAQs+can’t+doesn’t+won’t+Created+Hello+HELLO+NASA+etc.+etc+HELLO+lip.+text.+NASA.+Text.+TEXT.+Hunspell.+HUNSPELL.+İzmir+İZMİR+İzmir.+İZMİR.+Imply+IMPLYdiff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/base_utf.wrong b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/base_utf.wrongnew file mode 100644index 00000000000..7924982f0dc--- /dev/null+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/base_utf.wrong@@ -0,0 +1,13 @@+loooked+texxt+hlelo+seid+rottenday+tomorow+seeeven+Nasa+horrorfying+peech+sugesst+İmply+İMPLY