Skip to content

Apache Lucene open-source search software

fix: don't process empty strings in thai user dictionary

Robert Muirb304eeae0243
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiTokenizer.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiTokenizer.java
index 1436981e38b..3d40ffafe3b 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiTokenizer.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiTokenizer.java
@@ -142,7 +142,7 @@ public class ThaiTokenizer extends SegmentingTokenizerBase {
minLen = Math.min(minLen, length);
maxLen = Math.max(maxLen, length);
}
- this.minDictWordLen = minLen;
+ this.minDictWordLen = Math.max(1, minLen); // ignore empty string
this.maxDictWordLen = maxLen;
} else {
this.minDictWordLen = 0;
@@ -187,6 +187,7 @@ public class ThaiTokenizer extends SegmentingTokenizerBase {
if (userDictionary != null && !userDictionary.isEmpty()) {
int remaining = sentenceEnd - (sentenceStart + start);
int maxLen = Math.min(maxDictWordLen, remaining);
+ assert minDictWordLen > 0;
for (int l = maxLen; l >= minDictWordLen; l--) {
if (userDictionary.contains(buffer, sentenceStart + start, l)) {
int customEnd = start + l;
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiTokenizer.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiTokenizer.java
index 56780dac821..5a09df67cee 100644
--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiTokenizer.java
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiTokenizer.java
@@ -46,6 +46,15 @@ public class TestThaiTokenizer extends BaseTokenStreamTestCase {
assertTokenStreamContents(tokenizer, new String[] {"ไป", "พารากอน", "กัน"});
}
+ // empty string in a segmentation dictionary REALLY does not make sense...
+ // but don't crash on it
+ public void testUserDictionaryEmptyString() throws IOException {
+ CharArraySet userDict = new CharArraySet(List.of("", "พารากอน"), false);
+ Tokenizer tokenizer = new ThaiTokenizer(userDict);
+ tokenizer.setReader(new StringReader("ไปพารากอนกัน"));
+ assertTokenStreamContents(tokenizer, new String[] {"ไป", "พารากอน", "กัน"});
+ }
+
public void testMultipleUserDictionaryTermsAndOffsets() throws IOException {
CharArraySet userDict = new CharArraySet(List.of("พารากอน", "คนขับรถ"), false);
Tokenizer tokenizer = new ThaiTokenizer(userDict);