Skip to content

armgit / solr

Public Imported from GitHub
0

Apache Solr open-source search software

LUCENE-9709: Hunspell: no special dotted i treatment outside tr/az languages (#2271)

Peter Gromov40e92315aeaf
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java
index 4dc76abfd79..5fcbd4f730a 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java
@@ -163,7 +163,7 @@ public class Dictionary {
// language declaration of the dictionary
String language;
// true if case algorithms should use alternate (Turkish/Azeri) mapping
- boolean alternateCasing;
+ private boolean alternateCasing;
/**
* Creates a new Dictionary containing the information read from the provided InputStreams to
@@ -1027,6 +1027,10 @@ public class Dictionary {
}
}
+ boolean isDotICaseChangeDisallowed(char[] word) {
+ return word[0] == 'İ' && !alternateCasing;
+ }
+
private class EntryGrouper {
final FSTCompiler<IntsRef> words =
new FSTCompiler<>(FST.INPUT_TYPE.BYTE4, IntSequenceOutputs.getSingleton());
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/SpellChecker.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/SpellChecker.java
index 32c1ab2b534..b0c095ae9b5 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/SpellChecker.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/SpellChecker.java
@@ -102,6 +102,11 @@ public class SpellChecker {
}
}
}
+
+ if (dictionary.isDotICaseChangeDisallowed(wordChars)) {
+ return false;
+ }
+
char[] lower = stemmer.caseFoldLower(caseVariant, wordChars.length);
if (checkWord(lower, wordChars.length, wordCase)) {
return true;
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Stemmer.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Stemmer.java
index bcda07322a3..31237156dec 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Stemmer.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Stemmer.java
@@ -120,6 +120,11 @@ final class Stemmer {
list.addAll(doStem(variation, 0, variation.length, null, WordContext.SIMPLE_WORD));
}
}
+
+ if (dictionary.isDotICaseChangeDisallowed(word)) {
+ return;
+ }
+
caseFoldLower(wordCase == WordCase.UPPER ? titleBuffer : word, length);
list.addAll(doStem(lowerBuffer, 0, length, wordCase, WordContext.SIMPLE_WORD));
if (wordCase == WordCase.UPPER) {
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/SpellCheckerTest.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/SpellCheckerTest.java
index 4c5601bb7bc..5bcfe05759c 100644
--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/SpellCheckerTest.java
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/SpellCheckerTest.java
@@ -31,6 +31,11 @@ public class SpellCheckerTest extends StemmerTestBase {
doTest("base");
}
+ @Test
+ public void baseUtf() throws Exception {
+ doTest("base_utf");
+ }
+
@Test
public void keepcase() throws Exception {
doTest("keepcase");
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestBaseUtf.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestBaseUtf.java
new file mode 100644
index 00000000000..c787d3b18b7
--- /dev/null
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestBaseUtf.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.analysis.hunspell;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class TestBaseUtf extends StemmerTestBase {
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ init("base_utf.aff", "base_utf.dic");
+ }
+
+ public void good() {
+ assertStemsTo("imply", "imply");
+ assertStemsTo("Imply", "imply");
+ assertStemsTo("IMPLY", "imply");
+ assertStemsTo("İZMİR", "İzmir");
+ }
+
+ @Test
+ public void no_dottedI_decapitalization() {
+ assertStemsTo("İmply");
+ assertStemsTo("İMPLY");
+ }
+}
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/base_utf.aff b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/base_utf.aff
new file mode 100644
index 00000000000..493157b3018
--- /dev/null
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/base_utf.aff
@@ -0,0 +1,198 @@
+# OpenOffice.org’s en_US.aff file
+# with Unicode apostrophe: ’
+
+SET UTF-8
+TRY esianrtolcdugmphbyfvkwzESIANRTOLCDUGMPHBYFVKWZ'
+
+MAXNGRAMSUGS 1
+WORDCHARS .'’
+
+PFX A Y 1
+PFX A 0 re .
+
+PFX I Y 1
+PFX I 0 in .
+
+PFX U Y 1
+PFX U 0 un .
+
+PFX C Y 1
+PFX C 0 de .
+
+PFX E Y 1
+PFX E 0 dis .
+
+PFX F Y 1
+PFX F 0 con .
+
+PFX K Y 1
+PFX K 0 pro .
+
+SFX V N 2
+SFX V e ive e
+SFX V 0 ive [^e]
+
+SFX N Y 3
+SFX N e ion e
+SFX N y ication y
+SFX N 0 en [^ey]
+
+SFX X Y 3
+SFX X e ions e
+SFX X y ications y
+SFX X 0 ens [^ey]
+
+SFX H N 2
+SFX H y ieth y
+SFX H 0 th [^y]
+
+SFX Y Y 1
+SFX Y 0 ly .
+
+SFX G Y 2
+SFX G e ing e
+SFX G 0 ing [^e]
+
+SFX J Y 2
+SFX J e ings e
+SFX J 0 ings [^e]
+
+SFX D Y 4
+SFX D 0 d e
+SFX D y ied [^aeiou]y
+SFX D 0 ed [^ey]
+SFX D 0 ed [aeiou]y
+
+SFX T N 4
+SFX T 0 st e
+SFX T y iest [^aeiou]y
+SFX T 0 est [aeiou]y
+SFX T 0 est [^ey]
+
+SFX R Y 4
+SFX R 0 r e
+SFX R y ier [^aeiou]y
+SFX R 0 er [aeiou]y
+SFX R 0 er [^ey]
+
+SFX Z Y 4
+SFX Z 0 rs e
+SFX Z y iers [^aeiou]y
+SFX Z 0 ers [aeiou]y
+SFX Z 0 ers [^ey]
+
+SFX S Y 4
+SFX S y ies [^aeiou]y
+SFX S 0 s [aeiou]y
+SFX S 0 es [sxzh]
+SFX S 0 s [^sxzhy]
+
+SFX P Y 3
+SFX P y iness [^aeiou]y
+SFX P 0 ness [aeiou]y
+SFX P 0 ness [^y]
+
+SFX M Y 1
+SFX M 0 's .
+
+SFX B Y 3
+SFX B 0 able [^aeiou]
+SFX B 0 able ee
+SFX B e able [^aeiou]e
+
+SFX L Y 1
+SFX L 0 ment .
+
+REP 88
+REP a ei
+REP ei a
+REP a ey
+REP ey a
+REP ai ie
+REP ie ai
+REP are air
+REP are ear
+REP are eir
+REP air are
+REP air ere
+REP ere air
+REP ere ear
+REP ere eir
+REP ear are
+REP ear air
+REP ear ere
+REP eir are
+REP eir ere
+REP ch te
+REP te ch
+REP ch ti
+REP ti ch
+REP ch tu
+REP tu ch
+REP ch s
+REP s ch
+REP ch k
+REP k ch
+REP f ph
+REP ph f
+REP gh f
+REP f gh
+REP i igh
+REP igh i
+REP i uy
+REP uy i
+REP i ee
+REP ee i
+REP j di
+REP di j
+REP j gg
+REP gg j
+REP j ge
+REP ge j
+REP s ti
+REP ti s
+REP s ci
+REP ci s
+REP k cc
+REP cc k
+REP k qu
+REP qu k
+REP kw qu
+REP o eau
+REP eau o
+REP o ew
+REP ew o
+REP oo ew
+REP ew oo
+REP ew ui
+REP ui ew
+REP oo ui
+REP ui oo
+REP ew u
+REP u ew
+REP oo u
+REP u oo
+REP u oe
+REP oe u
+REP u ieu
+REP ieu u
+REP ue ew
+REP ew ue
+REP uff ough
+REP oo ieu
+REP ieu oo
+REP ier ear
+REP ear ier
+REP ear air
+REP air ear
+REP w qu
+REP qu w
+REP z ss
+REP ss z
+REP shun tion
+REP shun sion
+REP shun cion
+McDonalds’sá/w
+McDonald’sszá/g3) st:McDonald’s po:noun_prs is:TRANS
+McDonald’sszal/g4) st:McDonald’s po:noun_prs is:INSTR
+McDonald’ssal/w
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/base_utf.dic b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/base_utf.dic
new file mode 100644
index 00000000000..b010030cb55
--- /dev/null
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/base_utf.dic
@@ -0,0 +1,30 @@
+28
+created/U
+create/XKVNGADS
+imply/GNSDX
+natural/PUY
+like/USPBY
+convey/BDGS
+look/GZRDS
+text
+hello
+said
+sawyer
+NASA
+rotten
+day
+tomorrow
+seven
+FAQ/SM
+can’t
+doesn’t
+etc
+won’t
+lip
+text
+horrifying
+speech
+suggest
+uncreate/V
+Hunspell
+İzmir
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/base_utf.good b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/base_utf.good
new file mode 100644
index 00000000000..aed11f2b781
--- /dev/null
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/base_utf.good
@@ -0,0 +1,33 @@
+created
+uncreate
+uncreated
+imply
+implied
+unnatural
+conveyed
+sawyer
+NASA
+FAQs
+can’t
+doesn’t
+won’t
+Created
+Hello
+HELLO
+NASA
+etc.
+etc
+HELLO
+lip.
+text.
+NASA.
+Text.
+TEXT.
+Hunspell.
+HUNSPELL.
+İzmir
+İZMİR
+İzmir.
+İZMİR.
+Imply
+IMPLY
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/base_utf.wrong b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/base_utf.wrong
new file mode 100644
index 00000000000..7924982f0dc
--- /dev/null
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/base_utf.wrong
@@ -0,0 +1,13 @@
+loooked
+texxt
+hlelo
+seid
+rottenday
+tomorow
+seeeven
+Nasa
+horrorfying
+peech
+sugesst
+İmply
+İMPLY