Skip to content

Apache Lucene open-source search software

Add Hangul composition char filter (#16242)

Mingi Jeong401c65501c0c
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 4494f28094d..79b27e9d405 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -100,6 +100,9 @@ API Changes
New Features
---------------------
+* GITHUB#16241: Add HangulCompositionCharFilter to analysis-nori for opt-in composition of
+ modern Hangul conjoining jamo before KoreanTokenizer. (Mingi Jeong)
+
* GITHUB#15505: Upgrade snowball to 2d2e312df56f2ede014a4ffb3e91e6dea43c24be. New stemmer: PolishStemmer (and
PolishSnowballAnalyzer in the stempel package) (Justas Sakalauskas, Dawid Weiss)
diff --git a/lucene/analysis/nori/src/java/module-info.java b/lucene/analysis/nori/src/java/module-info.java
index 49edd98a7ff..fb61d2748e9 100644
--- a/lucene/analysis/nori/src/java/module-info.java
+++ b/lucene/analysis/nori/src/java/module-info.java
@@ -24,6 +24,8 @@ module org.apache.lucene.analysis.nori {
exports org.apache.lucene.analysis.ko.dict;
exports org.apache.lucene.analysis.ko.tokenattributes;
+ provides org.apache.lucene.analysis.CharFilterFactory with
+ org.apache.lucene.analysis.ko.HangulCompositionCharFilterFactory;
provides org.apache.lucene.analysis.TokenizerFactory with
org.apache.lucene.analysis.ko.KoreanTokenizerFactory;
provides org.apache.lucene.analysis.TokenFilterFactory with
diff --git a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/HangulCompositionCharFilter.java b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/HangulCompositionCharFilter.java
new file mode 100644
index 00000000000..60b296d180f
--- /dev/null
+++ b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/HangulCompositionCharFilter.java
@@ -0,0 +1,152 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.analysis.ko;
+
+import java.io.IOException;
+import java.io.Reader;
+import org.apache.lucene.analysis.charfilter.BaseCharFilter;
+
+/**
+ * A {@link org.apache.lucene.analysis.CharFilter} that composes modern Hangul conjoining jamo into
+ * precomposed Hangul syllables.
+ *
+ * <p>This filter only handles canonical modern Hangul syllable sequences:
+ *
+ * <ul>
+ * <li>leading consonants U+1100..U+1112
+ * <li>vowels U+1161..U+1175
+ * <li>optional trailing consonants U+11A8..U+11C2
+ * </ul>
+ *
+ * <p>Compatibility jamo and archaic jamo are left unchanged. This is intended as an opt-in
+ * normalization step before {@link KoreanTokenizer}, whose dictionary is built for precomposed
+ * Hangul syllables.
+ *
+ * <p>This filter composes only full conjoining-jamo sequences (L, V, optional T). A precomposed LV
+ * syllable followed by a conjoining trailing consonant (e.g., U+D558 하 + U+11AB), which full NFC
+ * would compose into 한, is intentionally left unchanged; fully decomposed (NFD) input never
+ * contains that shape.
+ */
+public final class HangulCompositionCharFilter extends BaseCharFilter {
+ private static final int S_BASE = 0xAC00;
+ private static final int L_BASE = 0x1100;
+ private static final int V_BASE = 0x1161;
+ private static final int T_BASE = 0x11A7;
+
+ private static final int L_COUNT = 19;
+ private static final int V_COUNT = 21;
+ private static final int T_COUNT = 28;
+ private static final int N_COUNT = V_COUNT * T_COUNT;
+
+ private int pushbackChar = -1;
+ private int outputOffset = 0;
+
+ /** Default constructor that takes a {@link Reader}. */
+ public HangulCompositionCharFilter(Reader input) {
+ super(input);
+ }
+
+ @Override
+ public int read() throws IOException {
+ int ch = nextInputChar();
+ if (ch == -1) {
+ return -1;
+ }
+
+ int lIndex = leadingIndex(ch);
+ if (lIndex < 0) {
+ return emit(ch);
+ }
+
+ int v = nextInputChar();
+ if (v == -1) {
+ return emit(ch);
+ }
+
+ int vIndex = vowelIndex(v);
+ if (vIndex < 0) {
+ pushBack(v);
+ return emit(ch);
+ }
+
+ int consumedChars = 2;
+ int tIndex = 0;
+ int t = nextInputChar();
+ if (t != -1) {
+ tIndex = trailingIndex(t);
+ if (tIndex > 0) {
+ consumedChars = 3;
+ } else {
+ pushBack(t);
+ tIndex = 0;
+ }
+ }
+
+ int syllable = S_BASE + (lIndex * N_COUNT) + (vIndex * T_COUNT) + tIndex;
+ int cumulativeDiff = getLastCumulativeDiff() + consumedChars - 1;
+ addOffCorrectMap(outputOffset + 1, cumulativeDiff);
+ return emit(syllable);
+ }
+
+ @Override
+ public int read(char[] cbuf, int off, int len) throws IOException {
+ int numRead = 0;
+ for (int i = off; i < off + len; i++) {
+ int ch = read();
+ if (ch == -1) {
+ break;
+ }
+ cbuf[i] = (char) ch;
+ numRead++;
+ }
+ return numRead == 0 ? -1 : numRead;
+ }
+
+ private int nextInputChar() throws IOException {
+ if (pushbackChar != -1) {
+ int ch = pushbackChar;
+ pushbackChar = -1;
+ return ch;
+ }
+ return input.read();
+ }
+
+ private void pushBack(int ch) {
+ assert pushbackChar == -1;
+ pushbackChar = ch;
+ }
+
+ private int emit(int ch) {
+ outputOffset++;
+ return ch;
+ }
+
+ private static int leadingIndex(int ch) {
+ int index = ch - L_BASE;
+ return index >= 0 && index < L_COUNT ? index : -1;
+ }
+
+ private static int vowelIndex(int ch) {
+ int index = ch - V_BASE;
+ return index >= 0 && index < V_COUNT ? index : -1;
+ }
+
+ private static int trailingIndex(int ch) {
+ int index = ch - T_BASE;
+ return index > 0 && index < T_COUNT ? index : -1;
+ }
+}
diff --git a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/HangulCompositionCharFilterFactory.java b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/HangulCompositionCharFilterFactory.java
new file mode 100644
index 00000000000..f69052712cd
--- /dev/null
+++ b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/HangulCompositionCharFilterFactory.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.analysis.ko;
+
+import java.io.Reader;
+import java.util.Map;
+import org.apache.lucene.analysis.CharFilterFactory;
+
+/**
+ * Factory for {@link HangulCompositionCharFilter}.
+ *
+ * @lucene.spi {@value #NAME}
+ */
+public class HangulCompositionCharFilterFactory extends CharFilterFactory {
+
+ /** SPI name */
+ public static final String NAME = "hangulComposition";
+
+ /** Creates a new HangulCompositionCharFilterFactory */
+ public HangulCompositionCharFilterFactory(Map<String, String> args) {
+ super(args);
+ if (!args.isEmpty()) {
+ throw new IllegalArgumentException("Unknown parameters: " + args);
+ }
+ }
+
+ /** Default ctor for compatibility with SPI */
+ public HangulCompositionCharFilterFactory() {
+ throw defaultCtorException();
+ }
+
+ @Override
+ public Reader create(Reader input) {
+ return new HangulCompositionCharFilter(input);
+ }
+
+ @Override
+ public Reader normalize(Reader input) {
+ return create(input);
+ }
+}
diff --git a/lucene/analysis/nori/src/resources/META-INF/services/org.apache.lucene.analysis.CharFilterFactory b/lucene/analysis/nori/src/resources/META-INF/services/org.apache.lucene.analysis.CharFilterFactory
new file mode 100644
index 00000000000..c5c9ab6a82d
--- /dev/null
+++ b/lucene/analysis/nori/src/resources/META-INF/services/org.apache.lucene.analysis.CharFilterFactory
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+org.apache.lucene.analysis.ko.HangulCompositionCharFilterFactory
diff --git a/lucene/analysis/nori/src/test/org/apache/lucene/analysis/ko/TestHangulCompositionCharFilter.java b/lucene/analysis/nori/src/test/org/apache/lucene/analysis/ko/TestHangulCompositionCharFilter.java
new file mode 100644
index 00000000000..1fc21746616
--- /dev/null
+++ b/lucene/analysis/nori/src/test/org/apache/lucene/analysis/ko/TestHangulCompositionCharFilter.java
@@ -0,0 +1,210 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.analysis.ko;
+
+import java.io.IOException;
+import java.io.Reader;
+import java.io.StringReader;
+import java.text.Normalizer;
+import java.text.Normalizer.Form;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.CharFilter;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.ko.KoreanTokenizer.DecompoundMode;
+import org.apache.lucene.analysis.ko.tokenattributes.PartOfSpeechAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.tests.analysis.BaseTokenStreamFactoryTestCase;
+import org.apache.lucene.tests.analysis.MockTokenizer;
+
+public class TestHangulCompositionCharFilter extends BaseTokenStreamFactoryTestCase {
+
+ public void testComposeModernConjoiningJamo() throws Exception {
+ assertEquals(
+ "가 각 한글",
+ readFully(
+ new HangulCompositionCharFilter(
+ new StringReader(Normalizer.normalize("가 각 한글", Form.NFD)))));
+ }
+
+ public void testLeavesNonModernSequencesUnchanged() throws Exception {
+ String input = "ㄱㅏ ᄀᄀ ᅡ 가ᅡ";
+ assertEquals("ㄱㅏ ᄀᄀ ᅡ 가ᅡ", readFully(new HangulCompositionCharFilter(new StringReader(input))));
+ }
+
+ public void testNoOpInputs() throws Exception {
+ assertEquals("", readFully(new HangulCompositionCharFilter(new StringReader(""))));
+ assertEquals("ᄀ", readFully(new HangulCompositionCharFilter(new StringReader("ᄀ"))));
+ assertEquals(
+ "가나다 ABC", readFully(new HangulCompositionCharFilter(new StringReader("가나다 ABC"))));
+ }
+
+ public void testCorrectOffsets() throws Exception {
+ CharFilter reader =
+ new HangulCompositionCharFilter(
+ new StringReader(Normalizer.normalize("가 난", Normalizer.Form.NFD)));
+ assertEquals("가 난", readFully(reader));
+ assertEquals(0, reader.correctOffset(0));
+ assertEquals(2, reader.correctOffset(1));
+ assertEquals(3, reader.correctOffset(2));
+ assertEquals(6, reader.correctOffset(3));
+ }
+
+ public void testLeavesPrecomposedLvPlusTrailingJamoUnchanged() throws Exception {
+ // Precomposed LV plus trailing jamo is outside the full conjoining-jamo sequence scope.
+ String input = "하\u11AB";
+ assertEquals(input, readFully(new HangulCompositionCharFilter(new StringReader(input))));
+ }
+
+ public void testNoriTermsMatchNfcForNfdInput() throws Exception {
+ String nfc = "한국어 형태소를 분석합니다";
+ String nfd = Normalizer.normalize(nfc, Form.NFD);
+
+ try (Analyzer analyzer = koreanAnalyzer(false);
+ Analyzer analyzerWithCharFilter = koreanAnalyzer(true)) {
+ assertNotEquals(collectTermData(analyzer, nfc), collectTermData(analyzer, nfd));
+ assertEquals(collectTermData(analyzer, nfc), collectTermData(analyzerWithCharFilter, nfd));
+ }
+ }
+
+ public void testNoriOffsetsReferToOriginalNfdInput() throws Exception {
+ String nfd = Normalizer.normalize("한국어 형태소를 분석합니다", Form.NFD);
+
+ try (Analyzer analyzer = koreanAnalyzer(true)) {
+ assertAnalyzesTo(
+ analyzer,
+ nfd,
+ new String[] {"한국어", "한국", "어", "형태소", "형태", "소", "를", "분석", "합니다", "하", "ᄇ니다"},
+ new int[] {0, 0, 6, 9, 9, 14, 16, 20, 26, 26, 26},
+ new int[] {8, 6, 8, 16, 14, 16, 19, 26, 33, 33, 33});
+ }
+ }
+
+ public void testFactory() throws Exception {
+ Reader reader =
+ charFilterFactory("hangulComposition")
+ .create(new StringReader(Normalizer.normalize("한국어 처리합니다", Form.NFD)));
+ TokenStream stream = whitespaceMockTokenizer(reader);
+ assertTokenStreamContents(stream, new String[] {"한국어", "처리합니다"});
+ }
+
+ public void testBogusFactoryArguments() {
+ IllegalArgumentException expected =
+ expectThrows(
+ IllegalArgumentException.class,
+ () -> charFilterFactory("hangulComposition", "bogusArg", "bogusValue"));
+ assertTrue(expected.getMessage().contains("Unknown parameters"));
+ }
+
+ public void testRandomModernHangulCompositionMatchesNfc() throws Exception {
+ int iterations = atLeast(100);
+ for (int i = 0; i < iterations; i++) {
+ String text = randomModernHangulText();
+ assertEquals(
+ text,
+ readFully(
+ new HangulCompositionCharFilter(
+ new StringReader(Normalizer.normalize(text, Form.NFD)))));
+ }
+ }
+
+ public void testRandomData() throws Exception {
+ Analyzer analyzer =
+ new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ return new TokenStreamComponents(tokenizer, tokenizer);
+ }
+
+ @Override
+ protected Reader initReader(String fieldName, Reader reader) {
+ return new HangulCompositionCharFilter(reader);
+ }
+ };
+ checkRandomData(random(), analyzer, RANDOM_MULTIPLIER * 1000);
+ analyzer.close();
+ }
+
+ private Analyzer koreanAnalyzer(boolean useHangulCompositionCharFilter) {
+ return new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer =
+ new KoreanTokenizer(newAttributeFactory(), null, DecompoundMode.MIXED, false);
+ return new TokenStreamComponents(tokenizer);
+ }
+
+ @Override
+ protected Reader initReader(String fieldName, Reader reader) {
+ if (useHangulCompositionCharFilter) {
+ return new HangulCompositionCharFilter(reader);
+ }
+ return reader;
+ }
+ };
+ }
+
+ private List<String> collectTermData(Analyzer analyzer, String input) throws IOException {
+ List<String> result = new ArrayList<>();
+ try (TokenStream ts = analyzer.tokenStream("ignored", input)) {
+ CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
+ PartOfSpeechAttribute partOfSpeechAtt = ts.addAttribute(PartOfSpeechAttribute.class);
+ ts.reset();
+ while (ts.incrementToken()) {
+ result.add(
+ termAtt
+ + "/"
+ + partOfSpeechAtt.getPOSType()
+ + "/"
+ + partOfSpeechAtt.getLeftPOS()
+ + "/"
+ + partOfSpeechAtt.getRightPOS());
+ }
+ ts.end();
+ }
+ return result;
+ }
+
+ private String randomModernHangulText() {
+ StringBuilder builder = new StringBuilder();
+ int length = random().nextInt(20);
+ for (int i = 0; i < length; i++) {
+ int choice = random().nextInt(8);
+ if (choice == 0) {
+ builder.append(' ');
+ } else if (choice == 1) {
+ builder.append((char) ('a' + random().nextInt(26)));
+ } else {
+ builder.append((char) (0xAC00 + random().nextInt(0xD7A4 - 0xAC00)));
+ }
+ }
+ return builder.toString();
+ }
+
+ private static String readFully(Reader reader) throws IOException {
+ StringBuilder builder = new StringBuilder();
+ char[] buffer = new char[1024];
+ int count;
+ while ((count = reader.read(buffer)) != -1) {
+ builder.append(buffer, 0, count);
+ }
+ return builder.toString();
+ }
+}