Apache Lucene open-source search software
Support user dictionary in ThaiTokenizer and ThaiTokenizerFactory (#16721) (#16722)
Kamthorn Krairaksa6b44b4c2a3af
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex c77499b2f48..2d746932314 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -123,6 +123,9 @@ API ChangesNew Features---------------------+* GITHUB#16722: Support user dictionary in ThaiTokenizer and ThaiTokenizerFactory.+ (Kamthorn Krairaksa)+* GITHUB#16717: Add ThaiCharFilter, ThaiCharFilterFactory, and ThaiNormalizationFilter forpre-tokenization character normalization and token-level normalization in ThaiAnalyzer.(Kamthorn Krairaksa)diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiTokenizer.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiTokenizer.javaindex 50674f1f657..7e76cfde780 100644--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiTokenizer.java+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiTokenizer.java@@ -18,6 +18,7 @@ package org.apache.lucene.analysis.th;import java.text.BreakIterator;import java.util.Locale;+import org.apache.lucene.analysis.CharArraySet;import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;import org.apache.lucene.analysis.util.CharArrayIterator;@@ -25,7 +26,11 @@ import org.apache.lucene.analysis.util.SegmentingTokenizerBase;import org.apache.lucene.util.AttributeFactory;/**- * Tokenizer that use {@link BreakIterator} to tokenize Thai text.+ * Tokenizer that uses {@link BreakIterator} to tokenize Thai text.+ *+ * <p>Supports an optional user dictionary ({@link CharArraySet}) for custom or domain-specific+ * words. When a user dictionary word matches, it takes precedence over default segmentation+ * boundaries.** <p>WARNING: this tokenizer may not be supported by all JREs. It is known to work with Sun/Oracle* and Harmony JREs. If your application needs to be fully portable, consider using ICUTokenizer@@ -53,6 +58,10 @@ public class ThaiTokenizer extends SegmentingTokenizerBase {private final BreakIterator wordBreaker;private final CharArrayIterator wrapper = CharArrayIterator.newWordInstance();+ private final CharArraySet userDictionary;+ private final int minDictWordLen;+ private final int maxDictWordLen;+int sentenceStart;int sentenceEnd;@@ -61,17 +70,51 @@ public class ThaiTokenizer extends SegmentingTokenizerBase {/** Creates a new ThaiTokenizer */public ThaiTokenizer() {- this(DEFAULT_TOKEN_ATTRIBUTE_FACTORY);+ this(DEFAULT_TOKEN_ATTRIBUTE_FACTORY, null);+ }++ /**+ * Creates a new ThaiTokenizer with a user dictionary.+ *+ * @param userDictionary custom dictionary of words, or null for default dictionary only+ */+ public ThaiTokenizer(CharArraySet userDictionary) {+ this(DEFAULT_TOKEN_ATTRIBUTE_FACTORY, userDictionary);}/** Creates a new ThaiTokenizer, supplying the AttributeFactory */public ThaiTokenizer(AttributeFactory factory) {+ this(factory, null);+ }++ /**+ * Creates a new ThaiTokenizer, supplying the AttributeFactory and user dictionary.+ *+ * @param factory AttributeFactory to use+ * @param userDictionary custom dictionary of words, or null for default dictionary only+ */+ public ThaiTokenizer(AttributeFactory factory, CharArraySet userDictionary) {super(factory, (BreakIterator) sentenceProto.clone());if (!DBBI_AVAILABLE) {throw new UnsupportedOperationException("This JRE does not have support for Thai segmentation");}wordBreaker = (BreakIterator) proto.clone();+ this.userDictionary = userDictionary;+ if (userDictionary != null && !userDictionary.isEmpty()) {+ int minLen = Integer.MAX_VALUE;+ int maxLen = 0;+ for (Object obj : userDictionary) {+ int length = ((char[]) obj).length;+ minLen = Math.min(minLen, length);+ maxLen = Math.max(maxLen, length);+ }+ this.minDictWordLen = minLen;+ this.maxDictWordLen = maxLen;+ } else {+ this.minDictWordLen = 0;+ this.maxDictWordLen = 0;+ }}@Override@@ -102,10 +145,39 @@ public class ThaiTokenizer extends SegmentingTokenizerBase {return false; // BreakIterator exhausted}+ boolean reanchor = false;+ if (userDictionary != null && !userDictionary.isEmpty()) {+ int remaining = sentenceEnd - (sentenceStart + start);+ int maxLen = Math.min(maxDictWordLen, remaining);+ for (int l = maxLen; l >= minDictWordLen; l--) {+ if (userDictionary.contains(buffer, sentenceStart + start, l)) {+ int customEnd = start + l;+ if (customEnd != end) {+ end = customEnd;+ if (end >= sentenceEnd - sentenceStart) {+ wordBreaker.last();+ } else if (wordBreaker.isBoundary(end)) {+ wordBreaker.following(end - 1);+ } else {+ reanchor = true;+ }+ }+ break;+ }+ }+ }+clearAttributes();termAtt.copyBuffer(buffer, sentenceStart + start, end - start);offsetAtt.setOffset(correctOffset(offset + sentenceStart + start), correctOffset(offset + sentenceStart + end));++ if (reanchor) {+ sentenceStart += end;+ wrapper.setText(buffer, sentenceStart, sentenceEnd - sentenceStart);+ wordBreaker.setText(wrapper);+ }+return true;}}diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiTokenizerFactory.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiTokenizerFactory.javaindex ef0969681a3..b03f10b796d 100644--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiTokenizerFactory.java+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiTokenizerFactory.java@@ -16,10 +16,14 @@*/package org.apache.lucene.analysis.th;+import java.io.IOException;import java.util.Map;+import org.apache.lucene.analysis.CharArraySet;import org.apache.lucene.analysis.Tokenizer;import org.apache.lucene.analysis.TokenizerFactory;import org.apache.lucene.util.AttributeFactory;+import org.apache.lucene.util.ResourceLoader;+import org.apache.lucene.util.ResourceLoaderAware;/*** Factory for {@link ThaiTokenizer}.@@ -27,21 +31,25 @@ import org.apache.lucene.util.AttributeFactory;* <pre><code class="language-xml">* <fieldType name="text_thai" class="solr.TextField" positionIncrementGap="100">* <analyzer>- * <tokenizer class="solr.ThaiTokenizerFactory"/>+ * <tokenizer class="solr.ThaiTokenizerFactory" dictionary="custom_words.txt"/>* </analyzer>* </fieldType></code></pre>** @since 4.10.0* @lucene.spi {@value #NAME}*/-public class ThaiTokenizerFactory extends TokenizerFactory {+public class ThaiTokenizerFactory extends TokenizerFactory implements ResourceLoaderAware {/** SPI name */public static final String NAME = "thai";+ private final String dictFile;+ private CharArraySet dictionary;+/** Creates a new ThaiTokenizerFactory */public ThaiTokenizerFactory(Map<String, String> args) {super(args);+ dictFile = get(args, "dictionary");if (!args.isEmpty()) {throw new IllegalArgumentException("Unknown parameters: " + args);}@@ -52,8 +60,15 @@ public class ThaiTokenizerFactory extends TokenizerFactory {throw defaultCtorException();}+ @Override+ public void inform(ResourceLoader loader) throws IOException {+ if (dictFile != null) {+ dictionary = getWordSet(loader, dictFile, false);+ }+ }+@Overridepublic Tokenizer create(AttributeFactory factory) {- return new ThaiTokenizer(factory);+ return new ThaiTokenizer(factory, dictionary);}}diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiTokenizer.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiTokenizer.javanew file mode 100644index 00000000000..264aee2598d--- /dev/null+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiTokenizer.java@@ -0,0 +1,66 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.analysis.th;++import java.io.IOException;+import java.io.StringReader;+import java.util.List;+import org.apache.lucene.analysis.CharArraySet;+import org.apache.lucene.analysis.Tokenizer;+import org.apache.lucene.tests.analysis.BaseTokenStreamTestCase;++/** Test case for {@link ThaiTokenizer}. */+public class TestThaiTokenizer extends BaseTokenStreamTestCase {++ @Override+ public void setUp() throws Exception {+ super.setUp();+ assumeTrue(+ "JRE does not support Thai dictionary-based BreakIterator", ThaiTokenizer.DBBI_AVAILABLE);+ }++ public void testDefaultSegmentation() throws IOException {+ Tokenizer tokenizer = new ThaiTokenizer();+ tokenizer.setReader(new StringReader("ภาษาไทย"));+ assertTokenStreamContents(tokenizer, new String[] {"ภาษา", "ไทย"});+ }++ public void testUserDictionary() throws IOException {+ CharArraySet userDict = new CharArraySet(List.of("พารากอน", "คนขับรถ"), false);+ Tokenizer tokenizer = new ThaiTokenizer(userDict);+ tokenizer.setReader(new StringReader("ไปพารากอนกัน"));+ assertTokenStreamContents(tokenizer, new String[] {"ไป", "พารากอน", "กัน"});+ }++ public void testMultipleUserDictionaryTermsAndOffsets() throws IOException {+ CharArraySet userDict = new CharArraySet(List.of("พารากอน", "คนขับรถ"), false);+ Tokenizer tokenizer = new ThaiTokenizer(userDict);+ tokenizer.setReader(new StringReader("เขาเป็นคนขับรถไปพารากอน"));+ assertTokenStreamContents(+ tokenizer,+ new String[] {"เขา", "เป็น", "คนขับรถ", "ไป", "พารากอน"},+ new int[] {0, 3, 7, 14, 16},+ new int[] {3, 7, 14, 16, 23});+ }++ public void testEmptyUserDictionary() throws IOException {+ CharArraySet userDict = new CharArraySet(0, false);+ Tokenizer tokenizer = new ThaiTokenizer(userDict);+ tokenizer.setReader(new StringReader("ภาษาไทย"));+ assertTokenStreamContents(tokenizer, new String[] {"ภาษา", "ไทย"});+ }+}diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiTokenizerFactory.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiTokenizerFactory.javaindex e77a749564f..21c13c7d36b 100644--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiTokenizerFactory.java+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiTokenizerFactory.java@@ -32,6 +32,16 @@ public class TestThaiTokenizerFactory extends BaseTokenStreamFactoryTestCase {tokenizer, new String[] {"การ", "ที่", "ได้", "ต้อง", "แสดง", "ว่า", "งาน", "ดี"});}+ public void testCustomDictionary() throws Exception {+ assumeTrue(+ "JRE does not support Thai dictionary-based BreakIterator", ThaiTokenizer.DBBI_AVAILABLE);+ Tokenizer tokenizer =+ tokenizerFactory("Thai", "dictionary", "customThaiDictionary.txt")+ .create(newAttributeFactory());+ tokenizer.setReader(new StringReader("ไปพารากอน"));+ assertTokenStreamContents(tokenizer, new String[] {"ไป", "พารากอน"});+ }+/** Test that bogus arguments result in exception */public void testBogusArguments() throws Exception {assumeTrue(diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/customThaiDictionary.txt b/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/customThaiDictionary.txtnew file mode 100644index 00000000000..d042b89e41e--- /dev/null+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/customThaiDictionary.txt@@ -0,0 +1,3 @@+# Custom Thai Dictionary+พารากอน+คนขับรถ