Apache Solr open-source search software
LUCENE-9769: Hunspell: KEEPCASE should take precedence over affixed forms (#2374)
Peter Gromov5e834b39eb23
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Hunspell.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Hunspell.javaindex 082076a57c3..ea486da3b34 100644--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Hunspell.java+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Hunspell.java@@ -165,9 +165,11 @@ public class Hunspell {wordChars,offset,length,- originalCase,context,(stem, formID, morphDataId) -> {+ if (!acceptCase(originalCase, formID, stem)) {+ return dictionary.hasFlag(formID, Dictionary.HIDDEN_FLAG);+ }if (acceptsStem(formID)) {result[0] = new Root<>(stem, formID);}@@ -176,6 +178,29 @@ public class Hunspell {return result[0];}+ private boolean acceptCase(WordCase originalCase, int entryId, CharsRef root) {+ boolean keepCase = dictionary.hasFlag(entryId, dictionary.keepcase);+ if (originalCase != null) {+ if (keepCase+ && dictionary.checkSharpS+ && originalCase == WordCase.TITLE+ && containsSharpS(root.chars, root.offset, root.length)) {+ return true;+ }+ return !keepCase;+ }+ return !dictionary.hasFlag(entryId, Dictionary.HIDDEN_FLAG);+ }++ private boolean containsSharpS(char[] word, int offset, int length) {+ for (int i = 0; i < length; i++) {+ if (word[i + offset] == 'ß') {+ return true;+ }+ }+ return false;+ }+boolean acceptsStem(int formID) {return true;}diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Stemmer.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Stemmer.javaindex 012b7641fe7..b84050e4711 100644--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Stemmer.java+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Stemmer.java@@ -101,8 +101,7 @@ final class Stemmer {list.add(newStem(stem, stemException));return true;};-- if (!doStem(word, 0, length, null, WordContext.SIMPLE_WORD, processor)) {+ if (!doStem(word, 0, length, WordContext.SIMPLE_WORD, processor)) {return list;}@@ -110,7 +109,7 @@ final class Stemmer {if (wordCase == WordCase.UPPER || wordCase == WordCase.TITLE) {CaseVariationProcessor variationProcessor =(variant, varLength, originalCase) ->- doStem(variant, 0, varLength, originalCase, WordContext.SIMPLE_WORD, processor);+ doStem(variant, 0, varLength, WordContext.SIMPLE_WORD, processor);varyCase(word, length, wordCase, variationProcessor);}return list;@@ -239,19 +238,11 @@ final class Stemmer {}boolean doStem(- char[] word,- int offset,- int length,- WordCase originalCase,- WordContext context,- RootProcessor processor) {+ char[] word, int offset, int length, WordContext context, RootProcessor processor) {IntsRef forms = dictionary.lookupWord(word, offset, length);if (forms != null) {for (int i = 0; i < forms.length; i += formStep) {int entryId = forms.ints[forms.offset + i];- if (!acceptCase(originalCase, entryId, word, offset, length)) {- continue;- }// we can't add this form, it's a pseudostem requiring an affixif (dictionary.hasFlag(entryId, dictionary.needaffix)) {continue;@@ -277,47 +268,12 @@ final class Stemmer {}try {return stem(- word,- offset,- length,- context,- -1,- Dictionary.FLAG_UNSET,- -1,- 0,- true,- false,- originalCase,- processor);+ word, offset, length, context, -1, Dictionary.FLAG_UNSET, -1, 0, true, false, processor);} catch (IOException bogus) {throw new RuntimeException(bogus);}}- private boolean acceptCase(- WordCase originalCase, int entryId, char[] word, int offset, int length) {- boolean keepCase = dictionary.hasFlag(entryId, dictionary.keepcase);- if (originalCase != null) {- if (keepCase- && dictionary.checkSharpS- && originalCase == WordCase.TITLE- && containsSharpS(word, offset, length)) {- return true;- }- return !keepCase;- }- return !dictionary.hasFlag(entryId, Dictionary.HIDDEN_FLAG);- }-- private boolean containsSharpS(char[] word, int offset, int length) {- for (int i = 0; i < length; i++) {- if (word[i + offset] == 'ß') {- return true;- }- }- return false;- }-/*** Find the unique stem(s) of the provided word*@@ -411,8 +367,6 @@ final class Stemmer {* @param previousWasPrefix true if the previous removal was a prefix: if we are removing a* suffix, and it has no continuation requirements, it's ok. but two prefixes* (COMPLEXPREFIXES) or two suffixes must have continuation requirements to recurse.- * @param originalCase if non-null, represents original word case to disallow case variations of- * word with KEEPCASE flags* @return whether the processing should be continued*/private boolean stem(@@ -426,7 +380,6 @@ final class Stemmer {int recursionDepth,boolean doPrefix,boolean previousWasPrefix,- WordCase originalCase,RootProcessor processor)throws IOException {if (doPrefix && dictionary.prefixes != null) {@@ -473,7 +426,6 @@ final class Stemmer {-1,recursionDepth,true,- originalCase,processor)) {return false;}@@ -527,7 +479,6 @@ final class Stemmer {prefixId,recursionDepth,false,- originalCase,processor)) {return false;}@@ -667,7 +618,6 @@ final class Stemmer {int prefixId,int recursionDepth,boolean prefix,- WordCase originalCase,RootProcessor processor)throws IOException {char flag = dictionary.affixData(affix, Dictionary.AFFIX_FLAG);@@ -690,10 +640,6 @@ final class Stemmer {}}- // we are looking for a case variant, but this word does not allow it- if (!acceptCase(originalCase, entryId, strippedWord, offset, length)) {- continue;- }if (!context.isCompound() && dictionary.hasFlag(entryId, dictionary.onlyincompound)) {continue;}@@ -752,7 +698,6 @@ final class Stemmer {recursionDepth + 1,doPrefix,prefix,- originalCase,processor);}diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestAllCaps.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestAllCaps.javaindex 33f132fce1c..84867aef35f 100644--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestAllCaps.java+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestAllCaps.java@@ -37,8 +37,9 @@ public class TestAllCaps extends StemmerTestBase {}public void testWrong() {- assertStemsTo("Openoffice.org");- assertStemsTo("Unicef");- assertStemsTo("Unicef's");+ // "hunspell -s" still stems them+ assertStemsTo("Openoffice.org", "Openoffice.org");+ assertStemsTo("Unicef", "Unicef");+ assertStemsTo("Unicef's", "Unicef");}}diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestKeepCase.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestKeepCase.javaindex 63f9cc14176..f67133a1065 100644--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestKeepCase.java+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestKeepCase.java@@ -34,17 +34,22 @@ public class TestKeepCase extends StemmerTestBase {assertStemsTo("DRINKS", "drink");assertStemsTo("walk", "walk");assertStemsTo("walks", "walk");- assertStemsTo("Walk");- assertStemsTo("Walks");- assertStemsTo("WALKS");+ assertStemsTo("Walk", "walk");+ assertStemsTo("Walks", "walk");+ assertStemsTo("WALKS", "walk");assertStemsTo("test", "test");- assertStemsTo("Test");- assertStemsTo("TEST");+ assertStemsTo("Test", "test");+ assertStemsTo("TEST", "test");+ // dotted stems differ form "hunspell -s", but in a controversial way,+ // and most importantly stem presence is the sameassertStemsTo("baz.", "baz.");- assertStemsTo("Baz.");+ assertStemsTo("Baz.", "baz.");assertStemsTo("Quux.", "Quux.");- assertStemsTo("QUUX.");+ assertStemsTo("QUUX.", "Quux.");++ assertStemsTo("Ways", "way", "ways");+ assertStemsTo("WAYS", "way", "ways");}}diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/forbiddenword.dic b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/forbiddenword.dicindex cb63592276a..b012cc8a5a0 100644--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/forbiddenword.dic+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/forbiddenword.dic@@ -1,4 +1,4 @@-10+11foo/Sfoo/YXbar/YS@@ -8,4 +8,6 @@ kgKg/XKG/Xcm-Cm/X\ No newline at end of file+Cm/X+SIPS/X+Sip/A\ No newline at end of filediff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/forbiddenword.good b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/forbiddenword.goodindex 7bd112e9ea6..1f73e525bd4 100644--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/forbiddenword.good+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/forbiddenword.good@@ -1,3 +1,3 @@foobar-+Sipsdiff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/forbiddenword.wrong b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/forbiddenword.wrongindex 5752c1e4462..32796267735 100644--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/forbiddenword.wrong+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/forbiddenword.wrong@@ -2,3 +2,5 @@ barsfoosfoobarbarfoo+SIPS+sips\ No newline at end of filediff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/keepcase.dic b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/keepcase.dicindex 48d88a72928..4d497dc5d46 100644--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/keepcase.dic+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/keepcase.dic@@ -1,4 +1,4 @@-7+9drink/Xwalk/XZtest/Z@@ -6,3 +6,5 @@ foo/ZBar/Zbaz./ZQuux./Z+way/X+ways/Zdiff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/keepcase.good b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/keepcase.goodindex e6ff1817d37..795112ed313 100644--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/keepcase.good+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/keepcase.good@@ -2,3 +2,7 @@ fooBarbaz.Quux.+way+Way+WAY+ways\ No newline at end of filediff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/keepcase.sug b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/keepcase.sugindex 69e80dd35b9..e4cb7fad151 100644--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/keepcase.sug+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/keepcase.sug@@ -6,3 +6,4 @@ baz.baz.Quux.Quux.+Way\ No newline at end of filediff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/keepcase.wrong b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/keepcase.wrongindex 3b791429157..1986f6d49f8 100644--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/keepcase.wrong+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/keepcase.wrong@@ -6,3 +6,4 @@ Baz.BAZ.quux.QUUX.+Ways