Skip to content

armgit / solr

Public Imported from GitHub
0

Apache Solr open-source search software

LUCENE-9690: Hunspell: support special title-case for words with apostrophe (#2235)

Peter Gromovc7e1079da9e8
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/SpellChecker.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/SpellChecker.java
index 66e21a16527..d9ccc919ad4 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/SpellChecker.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/SpellChecker.java
@@ -74,6 +74,10 @@ public class SpellChecker {
if (checkWord(caseVariant, wordChars.length, true)) {
return true;
}
+ char[] aposCase = stemmer.capitalizeAfterApostrophe(caseVariant, wordChars.length);
+ if (aposCase != null && checkWord(aposCase, aposCase.length, true)) {
+ return true;
+ }
}
return checkWord(stemmer.caseFoldLower(caseVariant, wordChars.length), wordChars.length, true);
}
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Stemmer.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Stemmer.java
index 090742b5f7a..cc1c28ba106 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Stemmer.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Stemmer.java
@@ -98,6 +98,10 @@ final class Stemmer {
List<CharsRef> list = doStem(word, length, false);
if (wordCase == WordCase.UPPER) {
caseFoldTitle(word, length);
+ char[] aposCase = capitalizeAfterApostrophe(titleBuffer, length);
+ if (aposCase != null) {
+ list.addAll(doStem(aposCase, length, true));
+ }
list.addAll(doStem(titleBuffer, length, true));
}
if (wordCase == WordCase.UPPER || wordCase == WordCase.TITLE) {
@@ -138,6 +142,23 @@ final class Stemmer {
return lowerBuffer;
}
+ // Special prefix handling for Catalan, French, Italian:
+ // prefixes separated by apostrophe (SANT'ELIA -> Sant'+Elia).
+ char[] capitalizeAfterApostrophe(char[] word, int length) {
+ for (int i = 1; i < length - 1; i++) {
+ if (word[i] == '\'') {
+ char next = word[i + 1];
+ char upper = Character.toUpperCase(next);
+ if (upper != next) {
+ char[] copy = ArrayUtil.copyOfSubArray(word, 0, length);
+ copy[i + 1] = Character.toUpperCase(upper);
+ return copy;
+ }
+ }
+ }
+ return null;
+ }
+
List<CharsRef> doStem(char[] word, int length, boolean caseVariant) {
List<CharsRef> stems = new ArrayList<>();
IntsRef forms = dictionary.lookupWord(word, 0, length);
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/SpellCheckerTest.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/SpellCheckerTest.java
index cfa1719f18a..dacf22e40ed 100644
--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/SpellCheckerTest.java
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/SpellCheckerTest.java
@@ -26,6 +26,11 @@ import org.apache.lucene.util.IOUtils;
import org.junit.Test;
public class SpellCheckerTest extends StemmerTestBase {
+ @Test
+ public void allcaps() throws Exception {
+ doTest("allcaps");
+ }
+
@Test
public void i53643_numbersWithSeparators() throws Exception {
doTest("i53643");
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestAllCaps.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestAllCaps.java
index 43c67644b3e..33f132fce1c 100644
--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestAllCaps.java
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestAllCaps.java
@@ -28,6 +28,8 @@ public class TestAllCaps extends StemmerTestBase {
public void testGood() {
assertStemsTo("OpenOffice.org", "OpenOffice.org");
assertStemsTo("UNICEF's", "UNICEF");
+ assertStemsTo("L'Afrique", "Afrique");
+ assertStemsTo("L'AFRIQUE", "Afrique");
// Hunspell returns these title-cased stems, so for consistency we do, too
assertStemsTo("OPENOFFICE.ORG", "Openoffice.org");
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/allcaps.aff b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/allcaps.aff
index 57e916bf537..53efbafd2cc 100644
--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/allcaps.aff
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/allcaps.aff
@@ -3,3 +3,6 @@ WORDCHARS '.
SFX S N 1
SFX S 0 's .
+
+PFX L N 1
+PFX L 0 L'
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/allcaps.dic b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/allcaps.dic
index 7d3cdcc0469..4732b954d85 100644
--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/allcaps.dic
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/allcaps.dic
@@ -1,3 +1,4 @@
2
OpenOffice.org
UNICEF/S
+Afrique/L
\ No newline at end of file
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/allcaps.good b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/allcaps.good
new file mode 100644
index 00000000000..fb88cc23dd7
--- /dev/null
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/allcaps.good
@@ -0,0 +1,6 @@
+OpenOffice.org
+OPENOFFICE.ORG
+UNICEF's
+UNICEF'S
+L'AFRIQUE
+L'Afrique
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/allcaps.wrong b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/allcaps.wrong
new file mode 100644
index 00000000000..668194906eb
--- /dev/null
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/allcaps.wrong
@@ -0,0 +1,3 @@
+Openoffice.org
+Unicef
+Unicef's