Apache Lucene open-source search software
fix: don't process empty strings in thai user dictionary
Robert Muirb304eeae0243
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiTokenizer.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiTokenizer.javaindex 1436981e38b..3d40ffafe3b 100644--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiTokenizer.java+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiTokenizer.java@@ -142,7 +142,7 @@ public class ThaiTokenizer extends SegmentingTokenizerBase {minLen = Math.min(minLen, length);maxLen = Math.max(maxLen, length);}- this.minDictWordLen = minLen;+ this.minDictWordLen = Math.max(1, minLen); // ignore empty stringthis.maxDictWordLen = maxLen;} else {this.minDictWordLen = 0;@@ -187,6 +187,7 @@ public class ThaiTokenizer extends SegmentingTokenizerBase {if (userDictionary != null && !userDictionary.isEmpty()) {int remaining = sentenceEnd - (sentenceStart + start);int maxLen = Math.min(maxDictWordLen, remaining);+ assert minDictWordLen > 0;for (int l = maxLen; l >= minDictWordLen; l--) {if (userDictionary.contains(buffer, sentenceStart + start, l)) {int customEnd = start + l;diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiTokenizer.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiTokenizer.javaindex 56780dac821..5a09df67cee 100644--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiTokenizer.java+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiTokenizer.java@@ -46,6 +46,15 @@ public class TestThaiTokenizer extends BaseTokenStreamTestCase {assertTokenStreamContents(tokenizer, new String[] {"ไป", "พารากอน", "กัน"});}+ // empty string in a segmentation dictionary REALLY does not make sense...+ // but don't crash on it+ public void testUserDictionaryEmptyString() throws IOException {+ CharArraySet userDict = new CharArraySet(List.of("", "พารากอน"), false);+ Tokenizer tokenizer = new ThaiTokenizer(userDict);+ tokenizer.setReader(new StringReader("ไปพารากอนกัน"));+ assertTokenStreamContents(tokenizer, new String[] {"ไป", "พารากอน", "กัน"});+ }+public void testMultipleUserDictionaryTermsAndOffsets() throws IOException {CharArraySet userDict = new CharArraySet(List.of("พารากอน", "คนขับรถ"), false);Tokenizer tokenizer = new ThaiTokenizer(userDict);