Apache Lucene open-source search software
LUCENE-10416: Update Korean Dictionary to mecab-ko-dic-2.1.1-20180720 for Nori
Tomoko Uchida76c9fd4e38af
diff --git a/NOTICE.txt b/NOTICE.txtindex c8396cb01e8..1626d8e1dea 100644--- a/NOTICE.txt+++ b/NOTICE.txt@@ -202,11 +202,11 @@ Nori Korean Morphological Analyzer - Apache Lucene IntegrationThis software includes a binary and/or source version of data from- mecab-ko-dic-2.0.3-20170922+ mecab-ko-dic-2.1.1-20180720which can be obtained from- https://bitbucket.org/eunjeon/mecab-ko-dic/downloads/mecab-ko-dic-2.0.3-20170922.tar.gz+ https://bitbucket.org/eunjeon/mecab-ko-dic/downloads/mecab-ko-dic-2.1.1-20180720.tar.gzThe floating point precision conversion in NumericUtils.Float16Converter is derived from work byJeroen van der Zijp, granted for use under the Apache license.diff --git a/gradle/generation/nori.gradle b/gradle/generation/nori.gradleindex 4fadb11d6f7..fc10f0f561e 100644--- a/gradle/generation/nori.gradle+++ b/gradle/generation/nori.gradle@@ -54,7 +54,7 @@ configure(project(":lucene:analysis:nori")) {dependsOn deleteDictionaryDatadependsOn sourceSets.main.runtimeClasspath- def dictionaryName = "mecab-ko-dic-2.0.3-20170922"+ def dictionaryName = "mecab-ko-dic-2.1.1-20180720"def dictionarySource = "https://bitbucket.org/eunjeon/mecab-ko-dic/downloads/${dictionaryName}.tar.gz"def dictionaryFile = file("${buildDir}/generate/${dictionaryName}.tar.gz")def unpackedDir = file("${buildDir}/generate/${dictionaryName}")diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex cd9fd0ac90e..7ddd7dd64f7 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -188,6 +188,9 @@ Improvements* LUCENE-10371: Make IndexRearranger able to arrange segment in a determined order.(Patrick Zhai)++* LUCENE-10416: Update Korean Dictionary to mecab-ko-dic-2.1.1-20180720 for Nori.+ (Uihyun Kim)Optimizations---------------------diff --git a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/util/UnknownDictionaryBuilder.java b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/util/UnknownDictionaryBuilder.javaindex cbf3119719a..634c2a599ee 100644--- a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/util/UnknownDictionaryBuilder.java+++ b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/util/UnknownDictionaryBuilder.java@@ -28,7 +28,7 @@ import java.util.List;import org.apache.lucene.analysis.ko.dict.CharacterDefinition;class UnknownDictionaryBuilder {- private static final String NGRAM_DICTIONARY_ENTRY = "NGRAM,1798,3559,3677,SY,*,*,*,*,*,*,*";+ private static final String NGRAM_DICTIONARY_ENTRY = "NGRAM,1801,3559,3677,SY,*,*,*,*,*,*,*";private String encoding;diff --git a/lucene/analysis/nori/src/resources/org/apache/lucene/analysis/ko/dict/ConnectionCosts.dat b/lucene/analysis/nori/src/resources/org/apache/lucene/analysis/ko/dict/ConnectionCosts.datindex 7fad91e727d..3b92dcff709 100644Binary files a/lucene/analysis/nori/src/resources/org/apache/lucene/analysis/ko/dict/ConnectionCosts.dat and b/lucene/analysis/nori/src/resources/org/apache/lucene/analysis/ko/dict/ConnectionCosts.dat differdiff --git a/lucene/analysis/nori/src/resources/org/apache/lucene/analysis/ko/dict/TokenInfoDictionary$buffer.dat b/lucene/analysis/nori/src/resources/org/apache/lucene/analysis/ko/dict/TokenInfoDictionary$buffer.datindex d7cc866ca34..5d951670592 100644Binary files a/lucene/analysis/nori/src/resources/org/apache/lucene/analysis/ko/dict/TokenInfoDictionary$buffer.dat and b/lucene/analysis/nori/src/resources/org/apache/lucene/analysis/ko/dict/TokenInfoDictionary$buffer.dat differdiff --git a/lucene/analysis/nori/src/resources/org/apache/lucene/analysis/ko/dict/TokenInfoDictionary$fst.dat b/lucene/analysis/nori/src/resources/org/apache/lucene/analysis/ko/dict/TokenInfoDictionary$fst.datindex 91668d24a78..53082e33a37 100644Binary files a/lucene/analysis/nori/src/resources/org/apache/lucene/analysis/ko/dict/TokenInfoDictionary$fst.dat and b/lucene/analysis/nori/src/resources/org/apache/lucene/analysis/ko/dict/TokenInfoDictionary$fst.dat differdiff --git a/lucene/analysis/nori/src/resources/org/apache/lucene/analysis/ko/dict/TokenInfoDictionary$posDict.dat b/lucene/analysis/nori/src/resources/org/apache/lucene/analysis/ko/dict/TokenInfoDictionary$posDict.datindex c4db79866c5..851e9d5df56 100644Binary files a/lucene/analysis/nori/src/resources/org/apache/lucene/analysis/ko/dict/TokenInfoDictionary$posDict.dat and b/lucene/analysis/nori/src/resources/org/apache/lucene/analysis/ko/dict/TokenInfoDictionary$posDict.dat differdiff --git a/lucene/analysis/nori/src/resources/org/apache/lucene/analysis/ko/dict/TokenInfoDictionary$targetMap.dat b/lucene/analysis/nori/src/resources/org/apache/lucene/analysis/ko/dict/TokenInfoDictionary$targetMap.datindex 4661bf8f68e..85cd99141ce 100644Binary files a/lucene/analysis/nori/src/resources/org/apache/lucene/analysis/ko/dict/TokenInfoDictionary$targetMap.dat and b/lucene/analysis/nori/src/resources/org/apache/lucene/analysis/ko/dict/TokenInfoDictionary$targetMap.dat differdiff --git a/lucene/analysis/nori/src/resources/org/apache/lucene/analysis/ko/dict/UnknownDictionary$buffer.dat b/lucene/analysis/nori/src/resources/org/apache/lucene/analysis/ko/dict/UnknownDictionary$buffer.datindex c9a18eef34c..27b2d343151 100644Binary files a/lucene/analysis/nori/src/resources/org/apache/lucene/analysis/ko/dict/UnknownDictionary$buffer.dat and b/lucene/analysis/nori/src/resources/org/apache/lucene/analysis/ko/dict/UnknownDictionary$buffer.dat differdiff --git a/lucene/analysis/nori/src/resources/org/apache/lucene/analysis/ko/dict/UnknownDictionary$posDict.dat b/lucene/analysis/nori/src/resources/org/apache/lucene/analysis/ko/dict/UnknownDictionary$posDict.datindex f1ceebd734f..75479935289 100644Binary files a/lucene/analysis/nori/src/resources/org/apache/lucene/analysis/ko/dict/UnknownDictionary$posDict.dat and b/lucene/analysis/nori/src/resources/org/apache/lucene/analysis/ko/dict/UnknownDictionary$posDict.dat differ