Apache Lucene open-source search software
Add ThaiCharFilter, ThaiNormalizationFilter, and ThaiNormalizer (#16716) (#16717)
Kamthorn Krairaksae64ae25dcd89
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex 1df64189326..7e267cfbb04 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -123,6 +123,10 @@ API ChangesNew Features---------------------+* GITHUB#16717: Add ThaiCharFilter, ThaiCharFilterFactory, and ThaiNormalizationFilter for+ pre-tokenization character normalization and token-level normalization in ThaiAnalyzer.+ (Kamthorn Krairaksa)+* GITHUB#16241: Add HangulCompositionCharFilter to analysis-nori for opt-in composition ofmodern Hangul conjoining jamo before KoreanTokenizer. (Mingi Jeong)diff --git a/lucene/analysis/common/src/java/module-info.java b/lucene/analysis/common/src/java/module-info.javaindex 025ea08c6df..ac8be52b084 100644--- a/lucene/analysis/common/src/java/module-info.java+++ b/lucene/analysis/common/src/java/module-info.java@@ -150,7 +150,8 @@ module org.apache.lucene.analysis.common {org.apache.lucene.analysis.charfilter.MappingCharFilterFactory,org.apache.lucene.analysis.cjk.CJKWidthCharFilterFactory,org.apache.lucene.analysis.fa.PersianCharFilterFactory,- org.apache.lucene.analysis.pattern.PatternReplaceCharFilterFactory;+ org.apache.lucene.analysis.pattern.PatternReplaceCharFilterFactory,+ org.apache.lucene.analysis.th.ThaiCharFilterFactory;provides org.apache.lucene.analysis.TokenFilterFactory withorg.apache.lucene.analysis.tr.ApostropheFilterFactory,org.apache.lucene.analysis.ar.ArabicNormalizationFilterFactory,@@ -263,6 +264,7 @@ module org.apache.lucene.analysis.common {org.apache.lucene.analysis.core.FlattenGraphFilterFactory,org.apache.lucene.analysis.te.TeluguNormalizationFilterFactory,org.apache.lucene.analysis.te.TeluguStemFilterFactory,+ org.apache.lucene.analysis.th.ThaiNormalizationFilterFactory,org.apache.lucene.analysis.tr.TurkishLowerCaseFilterFactory,org.apache.lucene.analysis.util.ElisionFilterFactory;provides org.apache.lucene.analysis.TokenizerFactory withdiff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiAnalyzer.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiAnalyzer.javaindex 12595982f9a..715237810af 100644--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiAnalyzer.java+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiAnalyzer.java@@ -94,15 +94,15 @@ public final class ThaiAnalyzer extends StopwordAnalyzerBase {* the text in the provided {@link Reader}.** @return {@link org.apache.lucene.analysis.Analyzer.TokenStreamComponents} built from a {@link- * ThaiTokenizer} filtered with {@link LowerCaseFilter}, {@link DecimalDigitFilter} and {@link- * StopFilter}+ * ThaiTokenizer} filtered with {@link LowerCaseFilter}, {@link DecimalDigitFilter}, {@link+ * ThaiNormalizationFilter} and {@link StopFilter}*/@Overrideprotected TokenStreamComponents createComponents(String fieldName) {final Tokenizer source = new ThaiTokenizer();TokenStream result = new LowerCaseFilter(source);result = new DecimalDigitFilter(result);-+ result = new ThaiNormalizationFilter(result);result = new StopFilter(result, stopwords);return new TokenStreamComponents(source, result);}@@ -111,6 +111,18 @@ public final class ThaiAnalyzer extends StopwordAnalyzerBase {protected TokenStream normalize(String fieldName, TokenStream in) {TokenStream result = new LowerCaseFilter(in);result = new DecimalDigitFilter(result);+ result = new ThaiNormalizationFilter(result);return result;}++ /** Wraps the Reader with {@link ThaiCharFilter} */+ @Override+ protected Reader initReader(String fieldName, Reader reader) {+ return new ThaiCharFilter(reader);+ }++ @Override+ protected Reader initReaderForNormalization(String fieldName, Reader reader) {+ return new ThaiCharFilter(reader);+ }}diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiCharFilter.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiCharFilter.javanew file mode 100644index 00000000000..6577a6b1d3e--- /dev/null+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiCharFilter.java@@ -0,0 +1,211 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.analysis.th;++import java.io.IOException;+import java.io.Reader;+import org.apache.lucene.analysis.charfilter.BaseCharFilter;++/**+ * A {@link org.apache.lucene.analysis.CharFilter} that normalizes Thai text at the character stream+ * level before tokenization.+ *+ * <p>Applying normalization before tokenization is critical for Thai, because typographic anomalies+ * (such as two consecutive Sara E characters instead of Sara Ae, or zero-width characters) prevent+ * the dictionary-based {@link java.text.BreakIterator} from locating word boundaries, causing+ * multiple adjacent words to be mistakenly merged into one token.+ *+ * <p>Normalizations performed:+ *+ * <ul>+ * <li>Removes Zero-Width characters (U+200B ZWSP, U+200C ZWNJ, U+200D ZWJ, U+FEFF BOM)+ * <li>Replaces double Sara E (เ + เ, U+0E40 U+0E40) with Sara Ae (แ, U+0E41)+ * <li>Recomposes decomposed Sara Am (U+0E4D + U+0E32 → U+0E33, and U+0E4D + tone + U+0E32+ * → tone + U+0E33)+ * <li>Deduplicates consecutive repeated diacritics / tone marks (e.g. ดีี → ดี)+ * <li>Reorders misplaced tone mark before an above/below vowel to canonical order+ * <li>Replaces Lakkhangyao (ๅ, U+0E45) with Sara Aa (า, U+0E32) unless preceded by Ru (ฤ) or Lu+ * (ฦ)+ * </ul>+ *+ * <p>All character deletions and contractions correctly update the character offset map using+ * {@link #addOffCorrectMap(int, int)}.+ *+ * @since 11.0.0+ */+public class ThaiCharFilter extends BaseCharFilter {++ private final char[] inputBuffer = new char[8];+ private int inBufLen = 0;+ private boolean eof = false;++ private int outputOff = 0;+ private int cumulativeDiff = 0;+ private char prevChar = 0;+ private char pendingOutputChar = 0;++ /**+ * Creates a new ThaiCharFilter wrapping the given {@link Reader}.+ *+ * @param in the input reader+ */+ public ThaiCharFilter(Reader in) {+ super(in);+ }++ @Override+ public int read() throws IOException {+ if (pendingOutputChar != 0) {+ char c = pendingOutputChar;+ pendingOutputChar = 0;+ prevChar = c;+ outputOff++;+ return c;+ }++ while (true) {+ fillBuffer();++ if (inBufLen == 0) {+ return -1;+ }++ char c0 = inputBuffer[0];++ // 1. Zero-width character removal (U+200B, U+200C, U+200D, U+FEFF)+ if (c0 == '\u200B' || c0 == '\u200C' || c0 == '\u200D' || c0 == '\uFEFF') {+ shiftBuffer(1);+ cumulativeDiff++;+ addOffCorrectMap(outputOff, cumulativeDiff);+ continue;+ }++ // 2. Deduplicate consecutive identical diacritics / tone marks+ if (isThaiDiacritic(c0) && c0 == prevChar) {+ shiftBuffer(1);+ cumulativeDiff++;+ addOffCorrectMap(outputOff, cumulativeDiff);+ continue;+ }++ // 3. Double Sara E (เ + เ -> แ)+ if (c0 == '\u0E40' && inBufLen >= 2 && inputBuffer[1] == '\u0E40') {+ shiftBuffer(2);+ cumulativeDiff++;+ addOffCorrectMap(outputOff + 1, cumulativeDiff);+ prevChar = '\u0E41';+ outputOff++;+ return '\u0E41';+ }++ // 4. Sara Am recomposition+ if (c0 == '\u0E4D') {+ // Subcase 4A: Nikhahit + Sara Aa (ํ + า -> ำ)+ if (inBufLen >= 2 && inputBuffer[1] == '\u0E32') {+ shiftBuffer(2);+ cumulativeDiff++;+ addOffCorrectMap(outputOff + 1, cumulativeDiff);+ prevChar = '\u0E33';+ outputOff++;+ return '\u0E33';+ }+ // Subcase 4B: Nikhahit + Tone + Sara Aa (ํ + tone + า -> tone + ำ)+ if (inBufLen >= 3 && isThaiTone(inputBuffer[1]) && inputBuffer[2] == '\u0E32') {+ char tone = inputBuffer[1];+ shiftBuffer(3);+ pendingOutputChar = '\u0E33';+ cumulativeDiff++;+ addOffCorrectMap(outputOff + 2, cumulativeDiff);+ prevChar = tone;+ outputOff++;+ return tone;+ }+ }++ // 5. Misplaced tone mark before above/below vowel: swap to canonical order+ if (isThaiTone(c0) && inBufLen >= 2 && isThaiAboveBelowVowel(inputBuffer[1])) {+ char vowel = inputBuffer[1];+ char tone = c0;+ inputBuffer[0] = vowel;+ inputBuffer[1] = tone;+ // Continue loop to process the swapped vowel at index 0+ continue;+ }++ // 6. Lakkhangyao (ๅ, U+0E45) -> Sara Aa (า, U+0E32) unless preceded by Ru (ฤ) or Lu (ฦ)+ if (c0 == '\u0E45') {+ if (prevChar != '\u0E24' && prevChar != '\u0E26') {+ c0 = '\u0E32';+ }+ }++ // Default: emit character c0+ shiftBuffer(1);+ prevChar = c0;+ outputOff++;+ return c0;+ }+ }++ @Override+ public int read(char[] cbuf, int off, int len) throws IOException {+ if (len <= 0) {+ return 0;+ }+ int numRead = 0;+ for (int i = off; i < off + len; i++) {+ int c = read();+ if (c == -1) {+ break;+ }+ cbuf[i] = (char) c;+ numRead++;+ }+ return numRead == 0 ? -1 : numRead;+ }++ private void fillBuffer() throws IOException {+ while (!eof && inBufLen < 4) {+ int ch = input.read();+ if (ch == -1) {+ eof = true;+ break;+ }+ inputBuffer[inBufLen++] = (char) ch;+ }+ }++ private void shiftBuffer(int count) {+ int remaining = inBufLen - count;+ if (remaining > 0) {+ System.arraycopy(inputBuffer, count, inputBuffer, 0, remaining);+ }+ inBufLen = remaining;+ }++ private static boolean isThaiDiacritic(char c) {+ return c == '\u0E31' || (c >= '\u0E34' && c <= '\u0E3A') || (c >= '\u0E47' && c <= '\u0E4E');+ }++ private static boolean isThaiTone(char c) {+ return (c >= '\u0E48' && c <= '\u0E4B') || c == '\u0E4C';+ }++ private static boolean isThaiAboveBelowVowel(char c) {+ return c == '\u0E31' || (c >= '\u0E34' && c <= '\u0E3A') || c == '\u0E47';+ }+}diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiCharFilterFactory.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiCharFilterFactory.javanew file mode 100644index 00000000000..dac127d4311--- /dev/null+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiCharFilterFactory.java@@ -0,0 +1,64 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.analysis.th;++import java.io.Reader;+import java.util.Map;+import org.apache.lucene.analysis.CharFilterFactory;++/**+ * Factory for {@link ThaiCharFilter}.+ *+ * <pre class="prettyprint">+ * <fieldType name="text_th" class="solr.TextField" positionIncrementGap="100">+ * <analyzer>+ * <charFilter name="thai"/>+ * <tokenizer name="thai"/>+ * </analyzer>+ * </fieldType></pre>+ *+ * @since 11.0.0+ * @lucene.spi {@value #NAME}+ */+public class ThaiCharFilterFactory extends CharFilterFactory {++ /** SPI name */+ public static final String NAME = "thai";++ /** Default constructor for SPI */+ public ThaiCharFilterFactory(Map<String, String> args) {+ super(args);+ if (!args.isEmpty()) {+ throw new IllegalArgumentException("Unknown parameters: " + args);+ }+ }++ /** Default constructor */+ public ThaiCharFilterFactory() {+ this(Map.of());+ }++ @Override+ public Reader create(Reader input) {+ return new ThaiCharFilter(input);+ }++ @Override+ public Reader normalize(Reader input) {+ return create(input);+ }+}diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiNormalizationFilter.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiNormalizationFilter.javanew file mode 100644index 00000000000..53f6c3dd471--- /dev/null+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiNormalizationFilter.java@@ -0,0 +1,50 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.analysis.th;++import java.io.IOException;+import org.apache.lucene.analysis.TokenFilter;+import org.apache.lucene.analysis.TokenStream;+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;+import org.apache.lucene.analysis.tokenattributes.KeywordAttribute;++/**+ * A {@link TokenFilter} that applies {@link ThaiNormalizer} to normalize Thai characters.+ *+ * @see ThaiNormalizer+ */+public final class ThaiNormalizationFilter extends TokenFilter {++ private final ThaiNormalizer normalizer = new ThaiNormalizer();+ private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);+ private final KeywordAttribute keywordAtt = addAttribute(KeywordAttribute.class);++ public ThaiNormalizationFilter(TokenStream input) {+ super(input);+ }++ @Override+ public boolean incrementToken() throws IOException {+ if (input.incrementToken()) {+ if (!keywordAtt.isKeyword()) {+ termAtt.setLength(normalizer.normalize(termAtt.buffer(), termAtt.length()));+ }+ return true;+ }+ return false;+ }+}diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiNormalizationFilterFactory.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiNormalizationFilterFactory.javanew file mode 100644index 00000000000..a5ab4a0cd29--- /dev/null+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiNormalizationFilterFactory.java@@ -0,0 +1,64 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.analysis.th;++import java.util.Map;+import org.apache.lucene.analysis.TokenFilterFactory;+import org.apache.lucene.analysis.TokenStream;++/**+ * Factory for {@link ThaiNormalizationFilter}.+ *+ * <pre><code class="language-xml">+ * <fieldType name="text_thnormal" class="solr.TextField" positionIncrementGap="100">+ * <analyzer>+ * <tokenizer class="solr.ThaiTokenizerFactory"/>+ * <filter class="solr.ThaiNormalizationFilterFactory"/>+ * </analyzer>+ * </fieldType></code></pre>+ *+ * @since 11.0.0+ * @lucene.spi {@value #NAME}+ */+public class ThaiNormalizationFilterFactory extends TokenFilterFactory {++ /** SPI name */+ public static final String NAME = "thaiNormalization";++ /** Creates a new ThaiNormalizationFilterFactory */+ public ThaiNormalizationFilterFactory(Map<String, String> args) {+ super(args);+ if (!args.isEmpty()) {+ throw new IllegalArgumentException("Unknown parameters: " + args);+ }+ }++ /** Default ctor for compatibility with SPI */+ public ThaiNormalizationFilterFactory() {+ throw defaultCtorException();+ }++ @Override+ public TokenStream create(TokenStream input) {+ return new ThaiNormalizationFilter(input);+ }++ @Override+ public TokenStream normalize(TokenStream input) {+ return create(input);+ }+}diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiNormalizer.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiNormalizer.javanew file mode 100644index 00000000000..5ce1c08fa7b--- /dev/null+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiNormalizer.java@@ -0,0 +1,175 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.analysis.th;++import static org.apache.lucene.analysis.util.StemmerUtil.delete;++/**+ * Normalizer for Thai.+ *+ * <p>Normalizes Thai text to handle common orthographic variations, typos, and encoding anomalies:+ *+ * <ul>+ * <li>Removes zero-width characters (ZWSP U+200B, ZWNJ U+200C).+ * <li>Replaces double Sara E (U+0E40 U+0E40) with Sara Ae (U+0E41).+ * <li>Recomposes decomposed Sara Am (Nikhahit U+0E4D + optional tone mark + Sara Aa U+0E32) into+ * canonical Sara Am (U+0E33).+ * <li>Normalizes Lakkhangyao (U+0E45) to Sara Aa (U+0E32) unless preceded by Ru (U+0E24) or Lu+ * (U+0E26).+ * <li>Reorders misplaced tone marks (U+0E48-U+0E4B) or Thanthakhat (U+0E4C) with above/below+ * vowels (U+0E31, U+0E34-U+0E3A, U+0E47) to canonical Unicode order (vowel before tone mark).+ * <li>Reorders follow vowels (e.g. Sara Aa U+0E32, Sara Am U+0E33) and misplaced trailing tone+ * marks.+ * <li>Deduplicates repeated vowels, tone marks, and diacritics.+ * <li>Removes dangling combining marks at the start of a token.+ * </ul>+ */+public class ThaiNormalizer {++ /**+ * Normalize an input buffer of Thai text.+ *+ * @param s input buffer+ * @param len length of input buffer+ * @return length of input buffer after normalization+ */+ public int normalize(char[] s, int len) {+ if (len == 0) {+ return 0;+ }++ // Step 1: Remove zero-width characters+ for (int i = 0; i < len; i++) {+ if (s[i] == '\u200B' || s[i] == '\u200C') {+ len = delete(s, i, len);+ i--;+ }+ }++ // Remove leading dangling non-base marks at token start+ while (len > 0 && isDanglingMark(s[0])) {+ len = delete(s, 0, len);+ }++ // Step 2: Handle compositions (Double Sara E -> Sara Ae, Sara Am recomposition, Lakkhangyao)+ for (int i = 0; i < len; i++) {+ char c = s[i];++ // Double Sara E -> Sara Ae+ if (c == '\u0E40' && i + 1 < len && s[i + 1] == '\u0E40') {+ s[i] = '\u0E41';+ len = delete(s, i + 1, len);+ continue;+ }++ // Nikhahit (U+0E4D) + optional tone mark + Sara Aa (U+0E32) -> optional tone mark + Sara Am+ // (U+0E33)+ if (c == '\u0E4D') {+ if (i + 1 < len && s[i + 1] == '\u0E32') {+ s[i] = '\u0E33';+ len = delete(s, i + 1, len);+ continue;+ } else if (i + 2 < len && isToneMark(s[i + 1]) && s[i + 2] == '\u0E32') {+ // Nikhahit + Tone + Sara Aa -> Tone + Sara Am+ char tone = s[i + 1];+ s[i] = tone;+ s[i + 1] = '\u0E33';+ len = delete(s, i + 2, len);+ continue;+ }+ }++ // Tone mark + Nikhahit + Sara Aa -> Tone + Sara Am+ if (isToneMark(c) && i + 2 < len && s[i + 1] == '\u0E4D' && s[i + 2] == '\u0E32') {+ s[i + 1] = '\u0E33';+ len = delete(s, i + 2, len);+ continue;+ }++ // Lakkhangyao (U+0E45) -> Sara Aa (U+0E32) unless preceded by Ru (U+0E24) or Lu (U+0E26)+ if (c == '\u0E45') {+ if (i == 0 || (s[i - 1] != '\u0E24' && s[i - 1] != '\u0E26')) {+ s[i] = '\u0E32';+ }+ }+ }++ // Step 3: Canonical reordering+ // Standard order: Consonant -> Above/Below Vowel -> Tone mark / Thanthakhat+ // Also: Follow vowel (Sara Aa / Sara Am) + Tone mark -> Tone mark + Follow vowel+ boolean changed = true;+ while (changed) {+ changed = false;+ for (int i = 0; i < len - 1; i++) {+ // Tone mark / Thanthakhat followed by Above/Below Vowel+ if (isToneMarkOrThanthakhat(s[i]) && isAboveOrBelowVowel(s[i + 1])) {+ char tmp = s[i];+ s[i] = s[i + 1];+ s[i + 1] = tmp;+ changed = true;+ }+ // Follow vowel followed by Tone mark (e.g., Sara Am + Tone -> Tone + Sara Am)+ else if (isFollowVowel(s[i]) && isToneMark(s[i + 1])) {+ char tmp = s[i];+ s[i] = s[i + 1];+ s[i + 1] = tmp;+ changed = true;+ }+ }+ }++ // Step 4: Deduplicate repeated vowels, tone marks, and diacritics+ for (int i = 0; i < len - 1; i++) {+ // Identical consecutive combining mark or vowel+ if (isDeduplicable(s[i]) && s[i] == s[i + 1]) {+ len = delete(s, i + 1, len);+ i--;+ }+ // If two different consecutive tone marks appear, keep the last one+ else if (isToneMark(s[i]) && isToneMark(s[i + 1])) {+ len = delete(s, i, len);+ i--;+ }+ }++ return len;+ }++ private static boolean isToneMark(char c) {+ return c >= '\u0E48' && c <= '\u0E4B';+ }++ private static boolean isToneMarkOrThanthakhat(char c) {+ return isToneMark(c) || c == '\u0E4C';+ }++ private static boolean isAboveOrBelowVowel(char c) {+ return c == '\u0E31' || (c >= '\u0E34' && c <= '\u0E3A') || c == '\u0E47';+ }++ private static boolean isFollowVowel(char c) {+ return c == '\u0E30' || c == '\u0E32' || c == '\u0E33' || c == '\u0E45';+ }++ private static boolean isDanglingMark(char c) {+ return isAboveOrBelowVowel(c) || isToneMarkOrThanthakhat(c) || c == '\u0E4D' || c == '\u0E4E';+ }++ private static boolean isDeduplicable(char c) {+ return (c >= '\u0E30' && c <= '\u0E3A') || c == '\u0E45' || (c >= '\u0E47' && c <= '\u0E4E');+ }+}diff --git a/lucene/analysis/common/src/resources/META-INF/services/org.apache.lucene.analysis.CharFilterFactory b/lucene/analysis/common/src/resources/META-INF/services/org.apache.lucene.analysis.CharFilterFactoryindex c9f43ec2df7..2851b7f152a 100644--- a/lucene/analysis/common/src/resources/META-INF/services/org.apache.lucene.analysis.CharFilterFactory+++ b/lucene/analysis/common/src/resources/META-INF/services/org.apache.lucene.analysis.CharFilterFactory@@ -18,3 +18,4 @@ org.apache.lucene.analysis.charfilter.MappingCharFilterFactoryorg.apache.lucene.analysis.cjk.CJKWidthCharFilterFactoryorg.apache.lucene.analysis.fa.PersianCharFilterFactoryorg.apache.lucene.analysis.pattern.PatternReplaceCharFilterFactory+org.apache.lucene.analysis.th.ThaiCharFilterFactorydiff --git a/lucene/analysis/common/src/resources/META-INF/services/org.apache.lucene.analysis.TokenFilterFactory b/lucene/analysis/common/src/resources/META-INF/services/org.apache.lucene.analysis.TokenFilterFactoryindex 1b33f2603b3..b5b276803bf 100644--- a/lucene/analysis/common/src/resources/META-INF/services/org.apache.lucene.analysis.TokenFilterFactory+++ b/lucene/analysis/common/src/resources/META-INF/services/org.apache.lucene.analysis.TokenFilterFactory@@ -124,5 +124,6 @@ org.apache.lucene.analysis.synonym.word2vec.Word2VecSynonymFilterFactoryorg.apache.lucene.analysis.core.FlattenGraphFilterFactoryorg.apache.lucene.analysis.te.TeluguNormalizationFilterFactoryorg.apache.lucene.analysis.te.TeluguStemFilterFactory+org.apache.lucene.analysis.th.ThaiNormalizationFilterFactoryorg.apache.lucene.analysis.tr.TurkishLowerCaseFilterFactoryorg.apache.lucene.analysis.util.ElisionFilterFactorydiff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiAnalyzer.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiAnalyzer.javaindex bd8c9f8440a..9bca83c8f7b 100644--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiAnalyzer.java+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiAnalyzer.java@@ -146,4 +146,23 @@ public class TestThaiAnalyzer extends BaseTokenStreamTestCase {new int[] {4, 7, 9, 14, 19, 22, 25, 29, 33, 36, 39, 41});analyzer.close();}++ public void testNormalization() throws Exception {+ Analyzer analyzer = new ThaiAnalyzer(CharArraySet.EMPTY_SET);+ // Double Sara E -> Sara Ae+ assertAnalyzesTo(+ analyzer, "\u0E40\u0E40\u0E1B\u0E25\u0E01", new String[] {"\u0E41\u0E1B\u0E25\u0E01"});+ // Decomposed Sara Am -> Sara Am (pre-tokenization allows BreakIterator to properly segment ทำ+ // and งาน)+ assertAnalyzesTo(+ analyzer,+ "\u0E17\u0E4D\u0E32\u0E07\u0E32\u0E19",+ new String[] {"\u0E17\u0E33", "\u0E07\u0E32\u0E19"});+ // Sentence with double Sara E properly segmented into individual words thanks to ThaiCharFilter+ assertAnalyzesTo(+ analyzer, "ฉันรัก\u0E40\u0E40มวมาก", new String[] {"ฉัน", "รัก", "แมว", "มาก"});+ // Duplicate vowels+ assertAnalyzesTo(analyzer, "\u0E14\u0E35\u0E35", new String[] {"\u0E14\u0E35"});+ analyzer.close();+ }}diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiCharFilter.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiCharFilter.javanew file mode 100644index 00000000000..c8901bf6b34--- /dev/null+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiCharFilter.java@@ -0,0 +1,133 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.analysis.th;++import java.io.IOException;+import java.io.Reader;+import java.io.StringReader;+import org.apache.lucene.analysis.Analyzer;+import org.apache.lucene.analysis.CharFilter;+import org.apache.lucene.analysis.TokenStream;+import org.apache.lucene.analysis.Tokenizer;+import org.apache.lucene.tests.analysis.BaseTokenStreamTestCase;+import org.apache.lucene.tests.analysis.MockTokenizer;++/** Test case for {@link ThaiCharFilter}. */+public class TestThaiCharFilter extends BaseTokenStreamTestCase {++ /** Double Sara E (เ + เ) normalized to Sara Ae (แ) with offset correction */+ public void testDoubleSaraE() throws IOException {+ CharFilter reader = new ThaiCharFilter(new StringReader("เเมว"));+ TokenStream ts = whitespaceMockTokenizer(reader);+ assertTokenStreamContents(ts, new String[] {"แมว"}, new int[] {0}, new int[] {4}, 4);+ }++ /** Sara Am recomposition (ํ + า -> ำ, and ํ + tone + า -> tone + ำ) */+ public void testSaraAmRecomposition() throws IOException {+ // น + ํ + ้ + า (4 chars) -> น้ำ (3 chars)+ CharFilter reader = new ThaiCharFilter(new StringReader("น\u0E4D\u0E49\u0E32"));+ TokenStream ts = whitespaceMockTokenizer(reader);+ assertTokenStreamContents(ts, new String[] {"น้ำ"}, new int[] {0}, new int[] {4}, 4);++ // ท + ํ + า (3 chars) -> ทำ (2 chars)+ reader = new ThaiCharFilter(new StringReader("ท\u0E4D\u0E32"));+ ts = whitespaceMockTokenizer(reader);+ assertTokenStreamContents(ts, new String[] {"ทำ"}, new int[] {0}, new int[] {3}, 3);+ }++ /** Consecutive repeated diacritics / tone marks deduplication */+ public void testDuplicateDiacritics() throws IOException {+ // ด + ี + ี (3 chars) -> ดี (2 chars)+ CharFilter reader = new ThaiCharFilter(new StringReader("ดีี"));+ TokenStream ts = whitespaceMockTokenizer(reader);+ assertTokenStreamContents(ts, new String[] {"ดี"}, new int[] {0}, new int[] {3}, 3);++ // ไ + ม + ้ + ้ (4 chars) -> ไม้ (3 chars)+ reader = new ThaiCharFilter(new StringReader("ไม้้"));+ ts = whitespaceMockTokenizer(reader);+ assertTokenStreamContents(ts, new String[] {"ไม้"}, new int[] {0}, new int[] {4}, 4);+ }++ /** Zero-width space and joiner removal */+ public void testZeroWidth() throws IOException {+ // สวัส + \u200B + ดี (7 chars in input) -> สวัสดี (6 chars in output, endOffset 7)+ CharFilter reader = new ThaiCharFilter(new StringReader("สวัส\u200Bดี"));+ TokenStream ts = whitespaceMockTokenizer(reader);+ assertTokenStreamContents(ts, new String[] {"สวัสดี"}, new int[] {0}, new int[] {7}, 7);+ }++ /** Misplaced tone mark before above/below vowel swapped to canonical order */+ public void testToneReordering() throws IOException {+ // บ + ่ + ั + น -> บ + ั + ่ + น (บั่น)+ CharFilter reader = new ThaiCharFilter(new StringReader("บ\u0E48\u0E31น"));+ TokenStream ts = whitespaceMockTokenizer(reader);+ assertTokenStreamContents(ts, new String[] {"บั่น"}, new int[] {0}, new int[] {4}, 4);+ }++ /** Lakkhangyao (ๅ) converted to Sara Aa (า) except after Ru (ฤ) or Lu (ฦ) */+ public void testLakkhangyao() throws IOException {+ CharFilter reader = new ThaiCharFilter(new StringReader("ฤๅ ฦๅ กๅ"));+ TokenStream ts = whitespaceMockTokenizer(reader);+ assertTokenStreamContents(+ ts, new String[] {"ฤๅ", "ฦๅ", "กา"}, new int[] {0, 3, 6}, new int[] {2, 5, 8}, 8);+ }++ /**+ * Demonstrates that pre-tokenization char filtering allows BreakIterator to segment words+ * correctly that would otherwise be mistakenly merged into one huge token.+ */+ public void testBreakIteratorSegmentationWithCharFilter() throws IOException {+ assumeTrue(+ "JRE does not support Thai dictionary-based BreakIterator", ThaiTokenizer.DBBI_AVAILABLE);+ // "ฉันรักเเมวมาก" (contains double Sara E in เเมว)+ // Without ThaiCharFilter, BreakIterator lumps "รักเเมวมาก" into 1 token.+ // With ThaiCharFilter, it normalizes to "ฉันรักแมวมาก" and segments all 4 words properly.+ Reader reader = new ThaiCharFilter(new StringReader("ฉันรักเเมวมาก"));+ Tokenizer tokenizer = new ThaiTokenizer();+ tokenizer.setReader(reader);+ assertTokenStreamContents(+ tokenizer,+ new String[] {"ฉัน", "รัก", "แมว", "มาก"},+ new int[] {0, 3, 6, 10},+ new int[] {3, 6, 10, 13},+ 13);+ }++ public void testEmpty() throws IOException {+ CharFilter reader = new ThaiCharFilter(new StringReader(""));+ TokenStream ts = whitespaceMockTokenizer(reader);+ assertTokenStreamContents(ts, new String[] {});+ }++ public void testRandom() throws IOException {+ Analyzer analyzer =+ new Analyzer() {+ @Override+ protected TokenStreamComponents createComponents(String fieldName) {+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);+ return new TokenStreamComponents(tokenizer, tokenizer);+ }++ @Override+ protected Reader initReader(String fieldName, Reader reader) {+ return new ThaiCharFilter(reader);+ }+ };+ checkRandomData(random(), analyzer, 200 * RANDOM_MULTIPLIER);+ analyzer.close();+ }+}diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiCharFilterFactory.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiCharFilterFactory.javanew file mode 100644index 00000000000..83974e93775--- /dev/null+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiCharFilterFactory.java@@ -0,0 +1,43 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.analysis.th;++import java.io.Reader;+import java.io.StringReader;+import org.apache.lucene.analysis.TokenStream;+import org.apache.lucene.tests.analysis.BaseTokenStreamFactoryTestCase;++/** Simple tests to ensure the Thai char filter factory is working. */+public class TestThaiCharFilterFactory extends BaseTokenStreamFactoryTestCase {++ public void testCharFilter() throws Exception {+ Reader reader = charFilterFactory("Thai").create(new StringReader("เเมว"));+ TokenStream ts = whitespaceMockTokenizer(reader);+ assertTokenStreamContents(ts, new String[] {"แมว"});+ }++ /** Test that bogus arguments result in exception */+ public void testBogusArguments() throws Exception {+ IllegalArgumentException expected =+ expectThrows(+ IllegalArgumentException.class,+ () -> {+ charFilterFactory("Thai", "bogusArg", "bogusValue");+ });+ assertTrue(expected.getMessage().contains("Unknown parameters"));+ }+}diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiNormalizationFilterFactory.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiNormalizationFilterFactory.javanew file mode 100644index 00000000000..594f3dd0500--- /dev/null+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiNormalizationFilterFactory.java@@ -0,0 +1,49 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.analysis.th;++import java.io.Reader;+import java.io.StringReader;+import org.apache.lucene.analysis.TokenStream;+import org.apache.lucene.analysis.Tokenizer;+import org.apache.lucene.tests.analysis.BaseTokenStreamFactoryTestCase;+import org.apache.lucene.tests.analysis.MockTokenizer;++/** Simple tests to ensure the Thai normalization factory is working. */+public class TestThaiNormalizationFilterFactory extends BaseTokenStreamFactoryTestCase {++ public void testNormalization() throws Exception {+ Reader reader =+ new StringReader("\u0E40\u0E40\u0E1B\u0E25\u0E01 \u0E17\u0E4D\u0E32\u0E07\u0E32\u0E19");+ TokenStream stream = new MockTokenizer(MockTokenizer.WHITESPACE, false);+ ((Tokenizer) stream).setReader(reader);+ stream = tokenFilterFactory("ThaiNormalization").create(stream);+ assertTokenStreamContents(+ stream, new String[] {"\u0E41\u0E1B\u0E25\u0E01", "\u0E17\u0E33\u0E07\u0E32\u0E19"});+ }++ /** Test that bogus arguments result in exception */+ public void testBogusArguments() throws Exception {+ IllegalArgumentException expected =+ expectThrows(+ IllegalArgumentException.class,+ () -> {+ tokenFilterFactory("ThaiNormalization", "bogusArg", "bogusValue");+ });+ assertTrue(expected.getMessage().contains("Unknown parameters"));+ }+}diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiNormalizer.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiNormalizer.javanew file mode 100644index 00000000000..8b760582446--- /dev/null+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiNormalizer.java@@ -0,0 +1,110 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.analysis.th;++import java.io.IOException;+import org.apache.lucene.analysis.Analyzer;+import org.apache.lucene.analysis.TokenFilter;+import org.apache.lucene.analysis.Tokenizer;+import org.apache.lucene.analysis.core.KeywordTokenizer;+import org.apache.lucene.tests.analysis.BaseTokenStreamTestCase;++/** Test case for {@link ThaiNormalizer} and {@link ThaiNormalizationFilter}. */+public class TestThaiNormalizer extends BaseTokenStreamTestCase {++ public void testDoubleSaraE() throws IOException {+ // Two Sara E (U+0E40 U+0E40) should become Sara Ae (U+0E41)+ check("\u0E40\u0E40\u0E1B\u0E25\u0E01", "\u0E41\u0E1B\u0E25\u0E01"); // เเปลก -> แปลก+ }++ public void testSaraAmDecomposition() throws IOException {+ // Nikhahit (U+0E4D) + Sara Aa (U+0E32) -> Sara Am (U+0E33)+ check(+ "\u0E17\u0E4D\u0E32\u0E07\u0E32\u0E19",+ "\u0E17\u0E33\u0E07\u0E32\u0E19"); // ทํางาน -> ทำงาน++ // Nikhahit + Tone (U+0E49) + Sara Aa -> Tone + Sara Am+ check("\u0E19\u0E4D\u0E49\u0E32", "\u0E19\u0E49\u0E33"); // นํ้า -> น้ำ++ // Tone + Nikhahit + Sara Aa -> Tone + Sara Am+ check("\u0E19\u0E49\u0E4D\u0E32", "\u0E19\u0E49\u0E33"); // น้ำ+ }++ public void testSaraAmWithFollowedTone() throws IOException {+ // Sara Am followed by Tone mark -> Tone mark + Sara Am+ check("\u0E19\u0E33\u0E49", "\u0E19\u0E49\u0E33");+ }++ public void testReorderToneAndAboveVowel() throws IOException {+ // Tone mark before Above Vowel (Mai Tho before Sara I) -> Sara I before Mai Tho+ check("\u0E01\u0E49\u0E34", "\u0E01\u0E34\u0E49"); // ก้ิ -> กิ้+ }++ public void testDuplicateVowelsAndTones() throws IOException {+ // Duplicate Sara Ii+ check("\u0E14\u0E35\u0E35", "\u0E14\u0E35"); // ดีี -> ดี+ // Duplicate Mai Tho+ check("\u0E41\u0E21\u0E49\u0E49", "\u0E41\u0E21\u0E49"); // แม้้ -> แม้+ // Duplicate Sara Aa+ check("\u0E19\u0E32\u0E32", "\u0E19\u0E32"); // นาา -> นา+ }++ public void testConsecutiveDifferentTones() throws IOException {+ // When multiple different tone marks are typed, keep the last one+ check("\u0E44\u0E21\u0E48\u0E49", "\u0E44\u0E21\u0E49"); // ไม่้ -> ไม้+ }++ public void testZeroWidthCharacters() throws IOException {+ check(+ "\u0E20\u0E32\u0E29\u0E32\u200B\u0E44\u0E17\u0E22",+ "\u0E20\u0E32\u0E29\u0E32\u0E44\u0E17\u0E22");+ check(+ "\u0E20\u0E32\u0E29\u0E32\u200C\u0E44\u0E17\u0E22",+ "\u0E20\u0E32\u0E29\u0E32\u0E44\u0E17\u0E22");+ }++ public void testLakkhangyao() throws IOException {+ // Preceded by Ru: preserve+ check("\u0E24\u0E45", "\u0E24\u0E45");+ // Not preceded by Ru or Lu: normalize to Sara Aa+ check("\u0E01\u0E45", "\u0E01\u0E32");+ }++ public void testDanglingMark() throws IOException {+ // Leading dangling tone mark without consonant+ check("\u0E49\u0E01\u0E32", "\u0E01\u0E32");+ }++ public void testEmptyTerm() throws IOException {+ Analyzer a =+ new Analyzer() {+ @Override+ protected TokenStreamComponents createComponents(String fieldName) {+ Tokenizer tokenizer = new KeywordTokenizer();+ return new TokenStreamComponents(tokenizer, new ThaiNormalizationFilter(tokenizer));+ }+ };+ checkOneTerm(a, "", "");+ a.close();+ }++ private void check(String input, String output) throws IOException {+ Tokenizer tokenizer = whitespaceMockTokenizer(input);+ TokenFilter tf = new ThaiNormalizationFilter(tokenizer);+ assertTokenStreamContents(tf, new String[] {output});+ }+}