Skip to content

armgit / solr

Public Imported from GitHub
0

Apache Solr open-source search software

LUCENE-9698: Hunspell: reuse char[] when possible when stripping affix (#2243)

Peter Gromov695e789891ae
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/SpellChecker.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/SpellChecker.java
index d9ccc919ad4..b9f29a397a2 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/SpellChecker.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/SpellChecker.java
@@ -74,7 +74,7 @@ public class SpellChecker {
if (checkWord(caseVariant, wordChars.length, true)) {
return true;
}
- char[] aposCase = stemmer.capitalizeAfterApostrophe(caseVariant, wordChars.length);
+ char[] aposCase = Stemmer.capitalizeAfterApostrophe(caseVariant, wordChars.length);
if (aposCase != null && checkWord(aposCase, aposCase.length, true)) {
return true;
}
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Stemmer.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Stemmer.java
index 6f5ef552cab..4a337fbce57 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Stemmer.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Stemmer.java
@@ -143,7 +143,7 @@ final class Stemmer {
// Special prefix handling for Catalan, French, Italian:
// prefixes separated by apostrophe (SANT'ELIA -> Sant'+Elia).
- char[] capitalizeAfterApostrophe(char[] word, int length) {
+ static char[] capitalizeAfterApostrophe(char[] word, int length) {
for (int i = 1; i < length - 1; i++) {
if (word[i] == '\'') {
char next = word[i + 1];
@@ -175,11 +175,12 @@ final class Stemmer {
if (Dictionary.hasFlag(wordFlags, dictionary.onlyincompound)) {
continue;
}
- stems.add(newStem(word, length, forms, i));
+ stems.add(newStem(word, 0, length, forms, i));
}
}
try {
- stems.addAll(stem(word, length, -1, (char) 0, -1, 0, true, true, false, false, caseVariant));
+ stems.addAll(
+ stem(word, 0, length, -1, (char) 0, -1, 0, true, true, false, false, caseVariant));
} catch (IOException bogus) {
throw new RuntimeException(bogus);
}
@@ -214,7 +215,7 @@ final class Stemmer {
return deduped;
}
- private CharsRef newStem(char[] buffer, int length, IntsRef forms, int formID) {
+ private CharsRef newStem(char[] buffer, int offset, int length, IntsRef forms, int formID) {
final String exception;
if (dictionary.hasStemExceptions) {
int exceptionID = forms.ints[forms.offset + formID + 1];
@@ -232,7 +233,7 @@ final class Stemmer {
if (exception != null) {
scratchSegment.append(exception);
} else {
- scratchSegment.append(buffer, 0, length);
+ scratchSegment.append(buffer, offset, length);
}
try {
Dictionary.applyMappings(dictionary.oconv, scratchSegment);
@@ -246,7 +247,7 @@ final class Stemmer {
if (exception != null) {
return new CharsRef(exception);
} else {
- return new CharsRef(buffer, 0, length);
+ return new CharsRef(buffer, offset, length);
}
}
}
@@ -284,6 +285,7 @@ final class Stemmer {
*/
private List<CharsRef> stem(
char[] word,
+ int offset,
int length,
int previous,
char prevFlag,
@@ -308,7 +310,7 @@ final class Stemmer {
int limit = dictionary.fullStrip ? length + 1 : length;
for (int i = 0; i < limit; i++) {
if (i > 0) {
- int ch = word[i - 1];
+ char ch = word[offset + i - 1];
if (fst.findTargetArc(ch, arc, arc, prefixReader) == null) {
break;
} else if (arc.output() != NO_OUTPUT) {
@@ -327,15 +329,17 @@ final class Stemmer {
}
if (isAffixCompatible(prefix, prevFlag, recursionDepth, false)) {
- char[] strippedWord = stripAffix(word, length, i, prefix, true);
+ char[] strippedWord = stripAffix(word, offset, length, i, prefix, true);
if (strippedWord == null) {
continue;
}
+ boolean pureAffix = strippedWord == word;
stems.addAll(
applyAffix(
strippedWord,
- strippedWord.length,
+ pureAffix ? offset + i : 0,
+ pureAffix ? length - i : strippedWord.length,
prefix,
-1,
recursionDepth,
@@ -356,7 +360,7 @@ final class Stemmer {
int limit = dictionary.fullStrip ? 0 : 1;
for (int i = length; i >= limit; i--) {
if (i < length) {
- int ch = word[i];
+ char ch = word[offset + i];
if (fst.findTargetArc(ch, arc, arc, suffixReader) == null) {
break;
} else if (arc.output() != NO_OUTPUT) {
@@ -375,15 +379,17 @@ final class Stemmer {
}
if (isAffixCompatible(suffix, prevFlag, recursionDepth, previousWasPrefix)) {
- char[] strippedWord = stripAffix(word, length, length - i, suffix, false);
+ char[] strippedWord = stripAffix(word, offset, length, length - i, suffix, false);
if (strippedWord == null) {
continue;
}
+ boolean pureAffix = strippedWord == word;
stems.addAll(
applyAffix(
strippedWord,
- strippedWord.length,
+ pureAffix ? offset : 0,
+ pureAffix ? i : strippedWord.length,
suffix,
prefixId,
recursionDepth,
@@ -398,7 +404,13 @@ final class Stemmer {
return stems;
}
- private char[] stripAffix(char[] word, int length, int affixLen, int affix, boolean isPrefix) {
+ /**
+ * @return null if affix conditions isn't met; a reference to the same char[] if the affix has no
+ * strip data and can thus be simply removed, or a new char[] containing the word affix
+ * removal
+ */
+ private char[] stripAffix(
+ char[] word, int offset, int length, int affixLen, int affix, boolean isPrefix) {
int deAffixedLen = length - affixLen;
int stripOrd = dictionary.affixData(affix, Dictionary.AFFIX_STRIP_ORD);
@@ -409,15 +421,22 @@ final class Stemmer {
char[] stripData = dictionary.stripData;
boolean condition =
isPrefix
- ? checkCondition(affix, stripData, stripStart, stripLen, word, affixLen, deAffixedLen)
- : checkCondition(affix, word, 0, deAffixedLen, stripData, stripStart, stripLen);
+ ? checkCondition(
+ affix, stripData, stripStart, stripLen, word, offset + affixLen, deAffixedLen)
+ : checkCondition(affix, word, offset, deAffixedLen, stripData, stripStart, stripLen);
if (!condition) {
return null;
}
+ if (stripLen == 0) return word;
+
char[] strippedWord = new char[stripLen + deAffixedLen];
System.arraycopy(
- word, isPrefix ? affixLen : 0, strippedWord, isPrefix ? stripLen : 0, deAffixedLen);
+ word,
+ offset + (isPrefix ? affixLen : 0),
+ strippedWord,
+ isPrefix ? stripLen : 0,
+ deAffixedLen);
System.arraycopy(stripData, stripStart, strippedWord, isPrefix ? 0 : deAffixedLen, stripLen);
return strippedWord;
}
@@ -484,6 +503,7 @@ final class Stemmer {
*/
private List<CharsRef> applyAffix(
char[] strippedWord,
+ int offset,
int length,
int affix,
int prefixId,
@@ -496,7 +516,7 @@ final class Stemmer {
List<CharsRef> stems = new ArrayList<>();
- IntsRef forms = dictionary.lookupWord(strippedWord, 0, length);
+ IntsRef forms = dictionary.lookupWord(strippedWord, offset, length);
if (forms != null) {
for (int i = 0; i < forms.length; i += formStep) {
char[] wordFlags = dictionary.decodeFlags(forms.ints[forms.offset + i], scratch);
@@ -530,7 +550,7 @@ final class Stemmer {
if (Dictionary.hasFlag(wordFlags, dictionary.onlyincompound)) {
continue;
}
- stems.add(newStem(strippedWord, length, forms, i));
+ stems.add(newStem(strippedWord, offset, length, forms, i));
}
}
}
@@ -572,6 +592,7 @@ final class Stemmer {
stems.addAll(
stem(
strippedWord,
+ offset,
length,
affix,
flag,