Apache Solr open-source search software
LUCENE-9690: Hunspell: support special title-case for words with apostrophe (#2235)
Peter Gromovc7e1079da9e8
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/SpellChecker.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/SpellChecker.javaindex 66e21a16527..d9ccc919ad4 100644--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/SpellChecker.java+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/SpellChecker.java@@ -74,6 +74,10 @@ public class SpellChecker {if (checkWord(caseVariant, wordChars.length, true)) {return true;}+ char[] aposCase = stemmer.capitalizeAfterApostrophe(caseVariant, wordChars.length);+ if (aposCase != null && checkWord(aposCase, aposCase.length, true)) {+ return true;+ }}return checkWord(stemmer.caseFoldLower(caseVariant, wordChars.length), wordChars.length, true);}diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Stemmer.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Stemmer.javaindex 090742b5f7a..cc1c28ba106 100644--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Stemmer.java+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Stemmer.java@@ -98,6 +98,10 @@ final class Stemmer {List<CharsRef> list = doStem(word, length, false);if (wordCase == WordCase.UPPER) {caseFoldTitle(word, length);+ char[] aposCase = capitalizeAfterApostrophe(titleBuffer, length);+ if (aposCase != null) {+ list.addAll(doStem(aposCase, length, true));+ }list.addAll(doStem(titleBuffer, length, true));}if (wordCase == WordCase.UPPER || wordCase == WordCase.TITLE) {@@ -138,6 +142,23 @@ final class Stemmer {return lowerBuffer;}+ // Special prefix handling for Catalan, French, Italian:+ // prefixes separated by apostrophe (SANT'ELIA -> Sant'+Elia).+ char[] capitalizeAfterApostrophe(char[] word, int length) {+ for (int i = 1; i < length - 1; i++) {+ if (word[i] == '\'') {+ char next = word[i + 1];+ char upper = Character.toUpperCase(next);+ if (upper != next) {+ char[] copy = ArrayUtil.copyOfSubArray(word, 0, length);+ copy[i + 1] = Character.toUpperCase(upper);+ return copy;+ }+ }+ }+ return null;+ }+List<CharsRef> doStem(char[] word, int length, boolean caseVariant) {List<CharsRef> stems = new ArrayList<>();IntsRef forms = dictionary.lookupWord(word, 0, length);diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/SpellCheckerTest.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/SpellCheckerTest.javaindex cfa1719f18a..dacf22e40ed 100644--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/SpellCheckerTest.java+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/SpellCheckerTest.java@@ -26,6 +26,11 @@ import org.apache.lucene.util.IOUtils;import org.junit.Test;public class SpellCheckerTest extends StemmerTestBase {+ @Test+ public void allcaps() throws Exception {+ doTest("allcaps");+ }+@Testpublic void i53643_numbersWithSeparators() throws Exception {doTest("i53643");diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestAllCaps.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestAllCaps.javaindex 43c67644b3e..33f132fce1c 100644--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestAllCaps.java+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestAllCaps.java@@ -28,6 +28,8 @@ public class TestAllCaps extends StemmerTestBase {public void testGood() {assertStemsTo("OpenOffice.org", "OpenOffice.org");assertStemsTo("UNICEF's", "UNICEF");+ assertStemsTo("L'Afrique", "Afrique");+ assertStemsTo("L'AFRIQUE", "Afrique");// Hunspell returns these title-cased stems, so for consistency we do, tooassertStemsTo("OPENOFFICE.ORG", "Openoffice.org");diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/allcaps.aff b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/allcaps.affindex 57e916bf537..53efbafd2cc 100644--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/allcaps.aff+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/allcaps.aff@@ -3,3 +3,6 @@ WORDCHARS '.SFX S N 1SFX S 0 's .++PFX L N 1+PFX L 0 L'diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/allcaps.dic b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/allcaps.dicindex 7d3cdcc0469..4732b954d85 100644--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/allcaps.dic+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/allcaps.dic@@ -1,3 +1,4 @@2OpenOffice.orgUNICEF/S+Afrique/L\ No newline at end of filediff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/allcaps.good b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/allcaps.goodnew file mode 100644index 00000000000..fb88cc23dd7--- /dev/null+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/allcaps.good@@ -0,0 +1,6 @@+OpenOffice.org+OPENOFFICE.ORG+UNICEF's+UNICEF'S+L'AFRIQUE+L'Afriquediff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/allcaps.wrong b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/allcaps.wrongnew file mode 100644index 00000000000..668194906eb--- /dev/null+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/allcaps.wrong@@ -0,0 +1,3 @@+Openoffice.org+Unicef+Unicef's