Apache Lucene open-source search software
Add Hangul composition char filter (#16242)
Mingi Jeong401c65501c0c
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex 4494f28094d..79b27e9d405 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -100,6 +100,9 @@ API ChangesNew Features---------------------+* GITHUB#16241: Add HangulCompositionCharFilter to analysis-nori for opt-in composition of+ modern Hangul conjoining jamo before KoreanTokenizer. (Mingi Jeong)+* GITHUB#15505: Upgrade snowball to 2d2e312df56f2ede014a4ffb3e91e6dea43c24be. New stemmer: PolishStemmer (andPolishSnowballAnalyzer in the stempel package) (Justas Sakalauskas, Dawid Weiss)diff --git a/lucene/analysis/nori/src/java/module-info.java b/lucene/analysis/nori/src/java/module-info.javaindex 49edd98a7ff..fb61d2748e9 100644--- a/lucene/analysis/nori/src/java/module-info.java+++ b/lucene/analysis/nori/src/java/module-info.java@@ -24,6 +24,8 @@ module org.apache.lucene.analysis.nori {exports org.apache.lucene.analysis.ko.dict;exports org.apache.lucene.analysis.ko.tokenattributes;+ provides org.apache.lucene.analysis.CharFilterFactory with+ org.apache.lucene.analysis.ko.HangulCompositionCharFilterFactory;provides org.apache.lucene.analysis.TokenizerFactory withorg.apache.lucene.analysis.ko.KoreanTokenizerFactory;provides org.apache.lucene.analysis.TokenFilterFactory withdiff --git a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/HangulCompositionCharFilter.java b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/HangulCompositionCharFilter.javanew file mode 100644index 00000000000..60b296d180f--- /dev/null+++ b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/HangulCompositionCharFilter.java@@ -0,0 +1,152 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.analysis.ko;++import java.io.IOException;+import java.io.Reader;+import org.apache.lucene.analysis.charfilter.BaseCharFilter;++/**+ * A {@link org.apache.lucene.analysis.CharFilter} that composes modern Hangul conjoining jamo into+ * precomposed Hangul syllables.+ *+ * <p>This filter only handles canonical modern Hangul syllable sequences:+ *+ * <ul>+ * <li>leading consonants U+1100..U+1112+ * <li>vowels U+1161..U+1175+ * <li>optional trailing consonants U+11A8..U+11C2+ * </ul>+ *+ * <p>Compatibility jamo and archaic jamo are left unchanged. This is intended as an opt-in+ * normalization step before {@link KoreanTokenizer}, whose dictionary is built for precomposed+ * Hangul syllables.+ *+ * <p>This filter composes only full conjoining-jamo sequences (L, V, optional T). A precomposed LV+ * syllable followed by a conjoining trailing consonant (e.g., U+D558 하 + U+11AB), which full NFC+ * would compose into 한, is intentionally left unchanged; fully decomposed (NFD) input never+ * contains that shape.+ */+public final class HangulCompositionCharFilter extends BaseCharFilter {+ private static final int S_BASE = 0xAC00;+ private static final int L_BASE = 0x1100;+ private static final int V_BASE = 0x1161;+ private static final int T_BASE = 0x11A7;++ private static final int L_COUNT = 19;+ private static final int V_COUNT = 21;+ private static final int T_COUNT = 28;+ private static final int N_COUNT = V_COUNT * T_COUNT;++ private int pushbackChar = -1;+ private int outputOffset = 0;++ /** Default constructor that takes a {@link Reader}. */+ public HangulCompositionCharFilter(Reader input) {+ super(input);+ }++ @Override+ public int read() throws IOException {+ int ch = nextInputChar();+ if (ch == -1) {+ return -1;+ }++ int lIndex = leadingIndex(ch);+ if (lIndex < 0) {+ return emit(ch);+ }++ int v = nextInputChar();+ if (v == -1) {+ return emit(ch);+ }++ int vIndex = vowelIndex(v);+ if (vIndex < 0) {+ pushBack(v);+ return emit(ch);+ }++ int consumedChars = 2;+ int tIndex = 0;+ int t = nextInputChar();+ if (t != -1) {+ tIndex = trailingIndex(t);+ if (tIndex > 0) {+ consumedChars = 3;+ } else {+ pushBack(t);+ tIndex = 0;+ }+ }++ int syllable = S_BASE + (lIndex * N_COUNT) + (vIndex * T_COUNT) + tIndex;+ int cumulativeDiff = getLastCumulativeDiff() + consumedChars - 1;+ addOffCorrectMap(outputOffset + 1, cumulativeDiff);+ return emit(syllable);+ }++ @Override+ public int read(char[] cbuf, int off, int len) throws IOException {+ int numRead = 0;+ for (int i = off; i < off + len; i++) {+ int ch = read();+ if (ch == -1) {+ break;+ }+ cbuf[i] = (char) ch;+ numRead++;+ }+ return numRead == 0 ? -1 : numRead;+ }++ private int nextInputChar() throws IOException {+ if (pushbackChar != -1) {+ int ch = pushbackChar;+ pushbackChar = -1;+ return ch;+ }+ return input.read();+ }++ private void pushBack(int ch) {+ assert pushbackChar == -1;+ pushbackChar = ch;+ }++ private int emit(int ch) {+ outputOffset++;+ return ch;+ }++ private static int leadingIndex(int ch) {+ int index = ch - L_BASE;+ return index >= 0 && index < L_COUNT ? index : -1;+ }++ private static int vowelIndex(int ch) {+ int index = ch - V_BASE;+ return index >= 0 && index < V_COUNT ? index : -1;+ }++ private static int trailingIndex(int ch) {+ int index = ch - T_BASE;+ return index > 0 && index < T_COUNT ? index : -1;+ }+}diff --git a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/HangulCompositionCharFilterFactory.java b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/HangulCompositionCharFilterFactory.javanew file mode 100644index 00000000000..f69052712cd--- /dev/null+++ b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/HangulCompositionCharFilterFactory.java@@ -0,0 +1,55 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.analysis.ko;++import java.io.Reader;+import java.util.Map;+import org.apache.lucene.analysis.CharFilterFactory;++/**+ * Factory for {@link HangulCompositionCharFilter}.+ *+ * @lucene.spi {@value #NAME}+ */+public class HangulCompositionCharFilterFactory extends CharFilterFactory {++ /** SPI name */+ public static final String NAME = "hangulComposition";++ /** Creates a new HangulCompositionCharFilterFactory */+ public HangulCompositionCharFilterFactory(Map<String, String> args) {+ super(args);+ if (!args.isEmpty()) {+ throw new IllegalArgumentException("Unknown parameters: " + args);+ }+ }++ /** Default ctor for compatibility with SPI */+ public HangulCompositionCharFilterFactory() {+ throw defaultCtorException();+ }++ @Override+ public Reader create(Reader input) {+ return new HangulCompositionCharFilter(input);+ }++ @Override+ public Reader normalize(Reader input) {+ return create(input);+ }+}diff --git a/lucene/analysis/nori/src/resources/META-INF/services/org.apache.lucene.analysis.CharFilterFactory b/lucene/analysis/nori/src/resources/META-INF/services/org.apache.lucene.analysis.CharFilterFactorynew file mode 100644index 00000000000..c5c9ab6a82d--- /dev/null+++ b/lucene/analysis/nori/src/resources/META-INF/services/org.apache.lucene.analysis.CharFilterFactory@@ -0,0 +1,16 @@+# Licensed to the Apache Software Foundation (ASF) under one or more+# contributor license agreements. See the NOTICE file distributed with+# this work for additional information regarding copyright ownership.+# The ASF licenses this file to You under the Apache License, Version 2.0+# (the "License"); you may not use this file except in compliance with+# the License. You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++org.apache.lucene.analysis.ko.HangulCompositionCharFilterFactorydiff --git a/lucene/analysis/nori/src/test/org/apache/lucene/analysis/ko/TestHangulCompositionCharFilter.java b/lucene/analysis/nori/src/test/org/apache/lucene/analysis/ko/TestHangulCompositionCharFilter.javanew file mode 100644index 00000000000..1fc21746616--- /dev/null+++ b/lucene/analysis/nori/src/test/org/apache/lucene/analysis/ko/TestHangulCompositionCharFilter.java@@ -0,0 +1,210 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.analysis.ko;++import java.io.IOException;+import java.io.Reader;+import java.io.StringReader;+import java.text.Normalizer;+import java.text.Normalizer.Form;+import java.util.ArrayList;+import java.util.List;+import org.apache.lucene.analysis.Analyzer;+import org.apache.lucene.analysis.CharFilter;+import org.apache.lucene.analysis.TokenStream;+import org.apache.lucene.analysis.Tokenizer;+import org.apache.lucene.analysis.ko.KoreanTokenizer.DecompoundMode;+import org.apache.lucene.analysis.ko.tokenattributes.PartOfSpeechAttribute;+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;+import org.apache.lucene.tests.analysis.BaseTokenStreamFactoryTestCase;+import org.apache.lucene.tests.analysis.MockTokenizer;++public class TestHangulCompositionCharFilter extends BaseTokenStreamFactoryTestCase {++ public void testComposeModernConjoiningJamo() throws Exception {+ assertEquals(+ "가 각 한글",+ readFully(+ new HangulCompositionCharFilter(+ new StringReader(Normalizer.normalize("가 각 한글", Form.NFD)))));+ }++ public void testLeavesNonModernSequencesUnchanged() throws Exception {+ String input = "ㄱㅏ ᄀᄀ ᅡ 가ᅡ";+ assertEquals("ㄱㅏ ᄀᄀ ᅡ 가ᅡ", readFully(new HangulCompositionCharFilter(new StringReader(input))));+ }++ public void testNoOpInputs() throws Exception {+ assertEquals("", readFully(new HangulCompositionCharFilter(new StringReader(""))));+ assertEquals("ᄀ", readFully(new HangulCompositionCharFilter(new StringReader("ᄀ"))));+ assertEquals(+ "가나다 ABC", readFully(new HangulCompositionCharFilter(new StringReader("가나다 ABC"))));+ }++ public void testCorrectOffsets() throws Exception {+ CharFilter reader =+ new HangulCompositionCharFilter(+ new StringReader(Normalizer.normalize("가 난", Normalizer.Form.NFD)));+ assertEquals("가 난", readFully(reader));+ assertEquals(0, reader.correctOffset(0));+ assertEquals(2, reader.correctOffset(1));+ assertEquals(3, reader.correctOffset(2));+ assertEquals(6, reader.correctOffset(3));+ }++ public void testLeavesPrecomposedLvPlusTrailingJamoUnchanged() throws Exception {+ // Precomposed LV plus trailing jamo is outside the full conjoining-jamo sequence scope.+ String input = "하\u11AB";+ assertEquals(input, readFully(new HangulCompositionCharFilter(new StringReader(input))));+ }++ public void testNoriTermsMatchNfcForNfdInput() throws Exception {+ String nfc = "한국어 형태소를 분석합니다";+ String nfd = Normalizer.normalize(nfc, Form.NFD);++ try (Analyzer analyzer = koreanAnalyzer(false);+ Analyzer analyzerWithCharFilter = koreanAnalyzer(true)) {+ assertNotEquals(collectTermData(analyzer, nfc), collectTermData(analyzer, nfd));+ assertEquals(collectTermData(analyzer, nfc), collectTermData(analyzerWithCharFilter, nfd));+ }+ }++ public void testNoriOffsetsReferToOriginalNfdInput() throws Exception {+ String nfd = Normalizer.normalize("한국어 형태소를 분석합니다", Form.NFD);++ try (Analyzer analyzer = koreanAnalyzer(true)) {+ assertAnalyzesTo(+ analyzer,+ nfd,+ new String[] {"한국어", "한국", "어", "형태소", "형태", "소", "를", "분석", "합니다", "하", "ᄇ니다"},+ new int[] {0, 0, 6, 9, 9, 14, 16, 20, 26, 26, 26},+ new int[] {8, 6, 8, 16, 14, 16, 19, 26, 33, 33, 33});+ }+ }++ public void testFactory() throws Exception {+ Reader reader =+ charFilterFactory("hangulComposition")+ .create(new StringReader(Normalizer.normalize("한국어 처리합니다", Form.NFD)));+ TokenStream stream = whitespaceMockTokenizer(reader);+ assertTokenStreamContents(stream, new String[] {"한국어", "처리합니다"});+ }++ public void testBogusFactoryArguments() {+ IllegalArgumentException expected =+ expectThrows(+ IllegalArgumentException.class,+ () -> charFilterFactory("hangulComposition", "bogusArg", "bogusValue"));+ assertTrue(expected.getMessage().contains("Unknown parameters"));+ }++ public void testRandomModernHangulCompositionMatchesNfc() throws Exception {+ int iterations = atLeast(100);+ for (int i = 0; i < iterations; i++) {+ String text = randomModernHangulText();+ assertEquals(+ text,+ readFully(+ new HangulCompositionCharFilter(+ new StringReader(Normalizer.normalize(text, Form.NFD)))));+ }+ }++ public void testRandomData() throws Exception {+ Analyzer analyzer =+ new Analyzer() {+ @Override+ protected TokenStreamComponents createComponents(String fieldName) {+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);+ return new TokenStreamComponents(tokenizer, tokenizer);+ }++ @Override+ protected Reader initReader(String fieldName, Reader reader) {+ return new HangulCompositionCharFilter(reader);+ }+ };+ checkRandomData(random(), analyzer, RANDOM_MULTIPLIER * 1000);+ analyzer.close();+ }++ private Analyzer koreanAnalyzer(boolean useHangulCompositionCharFilter) {+ return new Analyzer() {+ @Override+ protected TokenStreamComponents createComponents(String fieldName) {+ Tokenizer tokenizer =+ new KoreanTokenizer(newAttributeFactory(), null, DecompoundMode.MIXED, false);+ return new TokenStreamComponents(tokenizer);+ }++ @Override+ protected Reader initReader(String fieldName, Reader reader) {+ if (useHangulCompositionCharFilter) {+ return new HangulCompositionCharFilter(reader);+ }+ return reader;+ }+ };+ }++ private List<String> collectTermData(Analyzer analyzer, String input) throws IOException {+ List<String> result = new ArrayList<>();+ try (TokenStream ts = analyzer.tokenStream("ignored", input)) {+ CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);+ PartOfSpeechAttribute partOfSpeechAtt = ts.addAttribute(PartOfSpeechAttribute.class);+ ts.reset();+ while (ts.incrementToken()) {+ result.add(+ termAtt+ + "/"+ + partOfSpeechAtt.getPOSType()+ + "/"+ + partOfSpeechAtt.getLeftPOS()+ + "/"+ + partOfSpeechAtt.getRightPOS());+ }+ ts.end();+ }+ return result;+ }++ private String randomModernHangulText() {+ StringBuilder builder = new StringBuilder();+ int length = random().nextInt(20);+ for (int i = 0; i < length; i++) {+ int choice = random().nextInt(8);+ if (choice == 0) {+ builder.append(' ');+ } else if (choice == 1) {+ builder.append((char) ('a' + random().nextInt(26)));+ } else {+ builder.append((char) (0xAC00 + random().nextInt(0xD7A4 - 0xAC00)));+ }+ }+ return builder.toString();+ }++ private static String readFully(Reader reader) throws IOException {+ StringBuilder builder = new StringBuilder();+ char[] buffer = new char[1024];+ int count;+ while ((count = reader.read(buffer)) != -1) {+ builder.append(buffer, 0, count);+ }+ return builder.toString();+ }+}