Apache Lucene open-source search software
LUCENE-10393: Unify binary dictionary and dictionary writer in kuromoji and nori (#740)
Tomoko Uchidabd22f199dec5
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex 435434aaf30..cda9ad881cc 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -49,6 +49,9 @@ Other* LUCENE-10253: The @BadApple annotation has been removed from the testframework. (Adrien Grand)+* LUCENE-10393: Unify binary dictionary and dictionary writer in Kuromoji and Nori.+ (Tomoko Uchida, Robert Muir)+======================= Lucene 9.2.0 =======================API Changes---------------------diff --git a/lucene/analysis/common/src/java/module-info.java b/lucene/analysis/common/src/java/module-info.javaindex ec837952e7b..f8e7bf08554 100644--- a/lucene/analysis/common/src/java/module-info.java+++ b/lucene/analysis/common/src/java/module-info.java@@ -60,6 +60,7 @@ module org.apache.lucene.analysis.common {exports org.apache.lucene.analysis.lv;exports org.apache.lucene.analysis.minhash;exports org.apache.lucene.analysis.miscellaneous;+ exports org.apache.lucene.analysis.morph;exports org.apache.lucene.analysis.ne;exports org.apache.lucene.analysis.ngram;exports org.apache.lucene.analysis.nl;diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/BinaryDictionary.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/BinaryDictionary.javanew file mode 100644index 00000000000..f72ed913d35--- /dev/null+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/BinaryDictionary.java@@ -0,0 +1,100 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.analysis.morph;++import java.io.BufferedInputStream;+import java.io.EOFException;+import java.io.IOException;+import java.io.InputStream;+import java.nio.ByteBuffer;+import java.nio.channels.Channels;+import java.nio.channels.ReadableByteChannel;+import org.apache.lucene.codecs.CodecUtil;+import org.apache.lucene.store.DataInput;+import org.apache.lucene.store.InputStreamDataInput;+import org.apache.lucene.util.IOSupplier;+import org.apache.lucene.util.IntsRef;++/** Abstract dictionary base class. */+public abstract class BinaryDictionary<T extends MorphData> implements Dictionary<T> {+ public static final String DICT_FILENAME_SUFFIX = "$buffer.dat";+ public static final String TARGETMAP_FILENAME_SUFFIX = "$targetMap.dat";+ public static final String POSDICT_FILENAME_SUFFIX = "$posDict.dat";++ private final int[] targetMapOffsets, targetMap;+ protected final ByteBuffer buffer;++ protected BinaryDictionary(+ IOSupplier<InputStream> targetMapResource,+ IOSupplier<InputStream> dictResource,+ String targetMapCodecHeader,+ String dictCodecHeader,+ int dictCodecVersion)+ throws IOException {+ try (InputStream mapIS = new BufferedInputStream(targetMapResource.get())) {+ final DataInput in = new InputStreamDataInput(mapIS);+ CodecUtil.checkHeader(in, targetMapCodecHeader, dictCodecVersion, dictCodecVersion);+ this.targetMap = new int[in.readVInt()];+ this.targetMapOffsets = new int[in.readVInt()];+ populateTargetMap(in, this.targetMap, this.targetMapOffsets);+ }++ // no buffering here, as we load in one large buffer+ try (InputStream dictIS = dictResource.get()) {+ final DataInput in = new InputStreamDataInput(dictIS);+ CodecUtil.checkHeader(in, dictCodecHeader, dictCodecVersion, dictCodecVersion);+ final int size = in.readVInt();+ final ByteBuffer tmpBuffer = ByteBuffer.allocateDirect(size);+ final ReadableByteChannel channel = Channels.newChannel(dictIS);+ final int read = channel.read(tmpBuffer);+ if (read != size) {+ throw new EOFException("Cannot read whole dictionary");+ }+ this.buffer = tmpBuffer.asReadOnlyBuffer();+ }+ }++ private static void populateTargetMap(DataInput in, int[] targetMap, int[] targetMapOffsets)+ throws IOException {+ int accum = 0, sourceId = 0;+ for (int ofs = 0; ofs < targetMap.length; ofs++) {+ final int val = in.readVInt();+ if ((val & 0x01) != 0) {+ targetMapOffsets[sourceId] = ofs;+ sourceId++;+ }+ accum += val >>> 1;+ targetMap[ofs] = accum;+ }+ if (sourceId + 1 != targetMapOffsets.length)+ throw new IOException(+ "targetMap file format broken; targetMap.length="+ + targetMap.length+ + ", targetMapOffsets.length="+ + targetMapOffsets.length+ + ", sourceId="+ + sourceId);+ targetMapOffsets[sourceId] = targetMap.length;+ }++ public void lookupWordIds(int sourceId, IntsRef ref) {+ ref.ints = targetMap;+ ref.offset = targetMapOffsets[sourceId];+ // targetMapOffsets always has one more entry pointing behind last:+ ref.length = targetMapOffsets[sourceId + 1] - ref.offset;+ }+}diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/BinaryDictionaryWriter.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/BinaryDictionaryWriter.javanew file mode 100644index 00000000000..bb0a5c096b3--- /dev/null+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/BinaryDictionaryWriter.java@@ -0,0 +1,148 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.analysis.morph;++import java.io.BufferedOutputStream;+import java.io.IOException;+import java.io.OutputStream;+import java.nio.file.Files;+import java.nio.file.Path;+import org.apache.lucene.codecs.CodecUtil;+import org.apache.lucene.store.DataOutput;+import org.apache.lucene.store.OutputStreamDataOutput;+import org.apache.lucene.util.ArrayUtil;++/** Abstract base dictionary writer class. */+public abstract class BinaryDictionaryWriter<T extends BinaryDictionary<? extends MorphData>> {+ private final Class<T> implClazz;+ private int targetMapEndOffset = 0, lastWordId = -1, lastSourceId = -1;+ private int[] targetMap = new int[8192];+ private int[] targetMapOffsets = new int[8192];+ protected final DictionaryEntryWriter entryWriter;++ protected BinaryDictionaryWriter(Class<T> implClazz, DictionaryEntryWriter entryWriter) {+ this.implClazz = implClazz;+ this.entryWriter = entryWriter;+ }++ /**+ * put the entry in map+ *+ * @return current position of buffer, which will be wordId of next entry+ */+ public int put(String[] entry) {+ return entryWriter.putEntry(entry);+ }++ /**+ * Write whole dictionary in a directory.+ *+ * @throws IOException if an I/O error occurs writing the dictionary files+ */+ public abstract void write(Path baseDir) throws IOException;++ protected void addMapping(int sourceId, int wordId) {+ if (wordId <= lastWordId) {+ throw new IllegalStateException(+ "words out of order: " + wordId + " vs lastID: " + lastWordId);+ }++ if (sourceId > lastSourceId) {+ targetMapOffsets = ArrayUtil.grow(targetMapOffsets, sourceId + 1);+ for (int i = lastSourceId + 1; i <= sourceId; i++) {+ targetMapOffsets[i] = targetMapEndOffset;+ }+ } else if (sourceId != lastSourceId) {+ throw new IllegalStateException(+ "source ids not in increasing order: lastSourceId="+ + lastSourceId+ + " vs sourceId="+ + sourceId);+ }++ targetMap = ArrayUtil.grow(targetMap, targetMapEndOffset + 1);+ targetMap[targetMapEndOffset] = wordId;+ targetMapEndOffset++;++ lastSourceId = sourceId;+ lastWordId = wordId;+ }++ /**+ * Write dictionary in file Dictionary format is: [Size of dictionary(int)], [entry:{left+ * id(short)}{right id(short)}{word cost(short)}{length of pos info(short)}{pos info(char)}],+ * [entry...], [entry...].....+ *+ * @throws IOException if an I/O error occurs writing the dictionary files+ */+ protected void write(+ Path baseDir,+ String targetMapCodecHeader,+ String posDictCodecHeader,+ String dictCodecHeader,+ int dictCodecVersion)+ throws IOException {+ final String baseName = getBaseFileName();+ entryWriter.writeDictionary(+ baseDir.resolve(baseName + BinaryDictionary.DICT_FILENAME_SUFFIX),+ dictCodecHeader,+ dictCodecVersion);+ entryWriter.writePosDict(+ baseDir.resolve(baseName + BinaryDictionary.POSDICT_FILENAME_SUFFIX),+ posDictCodecHeader,+ dictCodecVersion);+ writeTargetMap(+ baseDir.resolve(baseName + BinaryDictionary.TARGETMAP_FILENAME_SUFFIX),+ targetMapCodecHeader,+ dictCodecVersion);+ }++ protected final String getBaseFileName() {+ return implClazz.getName().replace('.', '/');+ }++ // TODO: maybe this int[] should instead be the output to the FST...+ private void writeTargetMap(Path path, String targetMapCodecHeader, int dictCodecVersion)+ throws IOException {+ Files.createDirectories(path.getParent());+ try (OutputStream os = Files.newOutputStream(path);+ OutputStream bos = new BufferedOutputStream(os)) {+ final DataOutput out = new OutputStreamDataOutput(bos);+ CodecUtil.writeHeader(out, targetMapCodecHeader, dictCodecVersion);++ final int numSourceIds = lastSourceId + 1;+ out.writeVInt(targetMapEndOffset); // <-- size of main array+ out.writeVInt(numSourceIds + 1); // <-- size of offset array (+ 1 more entry)+ int prev = 0, sourceId = 0;+ for (int ofs = 0; ofs < targetMapEndOffset; ofs++) {+ final int val = targetMap[ofs], delta = val - prev;+ assert delta >= 0;+ if (ofs == targetMapOffsets[sourceId]) {+ out.writeVInt((delta << 1) | 0x01);+ sourceId++;+ } else {+ out.writeVInt((delta << 1));+ }+ prev += delta;+ }+ if (sourceId != numSourceIds) {+ throw new IllegalStateException(+ "sourceId:" + sourceId + " != numSourceIds:" + numSourceIds);+ }+ }+ }+}diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/CharacterDefinition.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/CharacterDefinition.javanew file mode 100644index 00000000000..29c1b012213--- /dev/null+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/CharacterDefinition.java@@ -0,0 +1,73 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.analysis.morph;++import java.io.IOException;+import java.io.InputStream;+import org.apache.lucene.codecs.CodecUtil;+import org.apache.lucene.store.DataInput;+import org.apache.lucene.store.InputStreamDataInput;+import org.apache.lucene.util.IOSupplier;++/** Character category data. */+public abstract class CharacterDefinition {++ public static final String FILENAME_SUFFIX = ".dat";++ protected final byte[] characterCategoryMap = new byte[0x10000];+ private final boolean[] invokeMap;+ private final boolean[] groupMap;++ protected CharacterDefinition(+ IOSupplier<InputStream> charDefResource,+ String charDefCodecHeader,+ int charDefCodecVersion,+ int classCount)+ throws IOException {+ try (InputStream is = charDefResource.get()) {+ final DataInput in = new InputStreamDataInput(is);+ CodecUtil.checkHeader(in, charDefCodecHeader, charDefCodecVersion, charDefCodecVersion);+ in.readBytes(characterCategoryMap, 0, characterCategoryMap.length);+ this.invokeMap = new boolean[classCount];+ this.groupMap = new boolean[classCount];+ for (int i = 0; i < classCount; i++) {+ final byte b = in.readByte();+ invokeMap[i] = (b & 0x01) != 0;+ groupMap[i] = (b & 0x02) != 0;+ }+ }+ }++ public byte getCharacterClass(char c) {+ return characterCategoryMap[c];+ }++ public boolean isInvoke(char c) {+ return invokeMap[characterCategoryMap[c]];+ }++ public boolean isGroup(char c) {+ return groupMap[characterCategoryMap[c]];+ }++ /** Functional interface to lookup character class */+ @FunctionalInterface+ public interface LookupCharacterClass {+ /** looks up character class for given class name */+ byte lookupCharacterClass(String characterClassName);+ }+}diff --git a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/util/CharacterDefinitionWriter.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/CharacterDefinitionWriter.javasimilarity index 57%rename from lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/util/CharacterDefinitionWriter.javarename to lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/CharacterDefinitionWriter.javaindex cbd3f768685..1f4fc3d13a9 100644--- a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/util/CharacterDefinitionWriter.java+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/CharacterDefinitionWriter.java@@ -14,7 +14,7 @@* See the License for the specific language governing permissions and* limitations under the License.*/-package org.apache.lucene.analysis.ko.util;+package org.apache.lucene.analysis.morph;import java.io.BufferedOutputStream;import java.io.IOException;@@ -22,21 +22,33 @@ import java.io.OutputStream;import java.nio.file.Files;import java.nio.file.Path;import java.util.Arrays;-import org.apache.lucene.analysis.ko.dict.CharacterDefinition;import org.apache.lucene.codecs.CodecUtil;import org.apache.lucene.store.DataOutput;import org.apache.lucene.store.OutputStreamDataOutput;-final class CharacterDefinitionWriter {+/** Writes character definition file */+public final class CharacterDefinitionWriter<T extends CharacterDefinition> {- private final byte[] characterCategoryMap = new byte[0x10000];+ private final Class<T> implClazz;- private final boolean[] invokeMap = new boolean[CharacterDefinition.CLASS_COUNT];- private final boolean[] groupMap = new boolean[CharacterDefinition.CLASS_COUNT];+ private final byte[] characterCategoryMap = new byte[0x10000];+ private final int classCount;+ private final boolean[] invokeMap;+ private final boolean[] groupMap;+ private final CharacterDefinition.LookupCharacterClass lookupCharClass;/** Constructor for building. TODO: remove write access */- CharacterDefinitionWriter() {- Arrays.fill(characterCategoryMap, CharacterDefinition.DEFAULT);+ public CharacterDefinitionWriter(+ Class<T> implClazz,+ byte defaultValue,+ int classCount,+ CharacterDefinition.LookupCharacterClass lookupCharClass) {+ this.implClazz = implClazz;+ Arrays.fill(characterCategoryMap, defaultValue);+ this.invokeMap = new boolean[classCount];+ this.groupMap = new boolean[classCount];+ this.classCount = classCount;+ this.lookupCharClass = lookupCharClass;}/**@@ -45,7 +57,7 @@ final class CharacterDefinitionWriter {* @param codePoint code point* @param characterClassName character class name*/- void putCharacterCategory(int codePoint, String characterClassName) {+ public void putCharacterCategory(int codePoint, String characterClassName) {characterClassName = characterClassName.split(" ")[0]; // use first// category// class@@ -54,27 +66,29 @@ final class CharacterDefinitionWriter {if (codePoint == 0x30FB) {characterClassName = "SYMBOL";}- characterCategoryMap[codePoint] = CharacterDefinition.lookupCharacterClass(characterClassName);+ characterCategoryMap[codePoint] = lookupCharClass.lookupCharacterClass(characterClassName);}- void putInvokeDefinition(String characterClassName, int invoke, int group, int length) {- final byte characterClass = CharacterDefinition.lookupCharacterClass(characterClassName);+ public void putInvokeDefinition(String characterClassName, int invoke, int group, int length) {+ final byte characterClass = lookupCharClass.lookupCharacterClass(characterClassName);invokeMap[characterClass] = invoke == 1;groupMap[characterClass] = group == 1;// TODO: length def ignored}- public void write(Path baseDir) throws IOException {- Path path =- baseDir.resolve(- CharacterDefinition.class.getName().replace('.', '/')- + CharacterDefinition.FILENAME_SUFFIX);+ private String getBaseFileName() {+ return implClazz.getName().replace('.', '/');+ }++ public void write(Path baseDir, String charDefCodecHeader, int charDefCodecVersion)+ throws IOException {+ Path path = baseDir.resolve(getBaseFileName() + CharacterDefinition.FILENAME_SUFFIX);Files.createDirectories(path.getParent());try (OutputStream os = new BufferedOutputStream(Files.newOutputStream(path))) {final DataOutput out = new OutputStreamDataOutput(os);- CodecUtil.writeHeader(out, CharacterDefinition.HEADER, CharacterDefinition.VERSION);+ CodecUtil.writeHeader(out, charDefCodecHeader, charDefCodecVersion);out.writeBytes(characterCategoryMap, 0, characterCategoryMap.length);- for (int i = 0; i < CharacterDefinition.CLASS_COUNT; i++) {+ for (int i = 0; i < classCount; i++) {final byte b = (byte) ((invokeMap[i] ? 0x01 : 0x00) | (groupMap[i] ? 0x02 : 0x00));out.writeByte(b);}diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/ConnectionCosts.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/ConnectionCosts.javanew file mode 100644index 00000000000..70c58161196--- /dev/null+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/ConnectionCosts.java@@ -0,0 +1,66 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.analysis.morph;++import java.io.BufferedInputStream;+import java.io.IOException;+import java.io.InputStream;+import java.nio.ByteBuffer;+import org.apache.lucene.codecs.CodecUtil;+import org.apache.lucene.store.DataInput;+import org.apache.lucene.store.InputStreamDataInput;+import org.apache.lucene.util.IOSupplier;++/** n-gram connection cost data */+public abstract class ConnectionCosts {++ public static final String FILENAME_SUFFIX = ".dat";++ private final ByteBuffer buffer;+ private final int forwardSize;++ protected ConnectionCosts(+ IOSupplier<InputStream> connectionCostResource,+ String connectionCostsCodecHeader,+ int dictCodecVersion)+ throws IOException {+ try (InputStream is = new BufferedInputStream(connectionCostResource.get())) {+ final DataInput in = new InputStreamDataInput(is);+ CodecUtil.checkHeader(in, connectionCostsCodecHeader, dictCodecVersion, dictCodecVersion);+ forwardSize = in.readVInt();+ int backwardSize = in.readVInt();+ int size = forwardSize * backwardSize;++ // copy the matrix into a direct byte buffer+ final ByteBuffer tmpBuffer = ByteBuffer.allocateDirect(size * 2);+ int accum = 0;+ for (int j = 0; j < backwardSize; j++) {+ for (int i = 0; i < forwardSize; i++) {+ accum += in.readZInt();+ tmpBuffer.putShort((short) accum);+ }+ }+ buffer = tmpBuffer.asReadOnlyBuffer();+ }+ }++ public int get(int forwardId, int backwardId) {+ // map 2d matrix into a single dimension short array+ int offset = (backwardId * forwardSize + forwardId) * 2;+ return buffer.getShort(offset);+ }+}diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/util/ConnectionCostsWriter.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/ConnectionCostsWriter.javasimilarity index 77%rename from lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/util/ConnectionCostsWriter.javarename to lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/ConnectionCostsWriter.javaindex 8d081f77bfc..f4f0a51c8a8 100644--- a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/util/ConnectionCostsWriter.java+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/ConnectionCostsWriter.java@@ -14,7 +14,7 @@* See the License for the specific language governing permissions and* limitations under the License.*/-package org.apache.lucene.analysis.ja.util;+package org.apache.lucene.analysis.morph;import java.io.BufferedOutputStream;import java.io.IOException;@@ -22,20 +22,23 @@ import java.io.OutputStream;import java.nio.ByteBuffer;import java.nio.file.Files;import java.nio.file.Path;-import org.apache.lucene.analysis.ja.dict.ConnectionCosts;import org.apache.lucene.codecs.CodecUtil;import org.apache.lucene.store.DataOutput;import org.apache.lucene.store.OutputStreamDataOutput;-final class ConnectionCostsWriter {+/** Writes connection costs */+public final class ConnectionCostsWriter<T extends ConnectionCosts> {+ private final Class<T> implClazz;private final ByteBuffercosts; // array is backward IDs first since get is called using the same backward ID// consecutively. maybe doesn't matter.private final int forwardSize;private final int backwardSize;+/** Constructor for building. TODO: remove write access */- ConnectionCostsWriter(int forwardSize, int backwardSize) {+ public ConnectionCostsWriter(Class<T> implClazz, int forwardSize, int backwardSize) {+ this.implClazz = implClazz;this.forwardSize = forwardSize;this.backwardSize = backwardSize;this.costs = ByteBuffer.allocateDirect(2 * backwardSize * forwardSize);@@ -46,14 +49,18 @@ final class ConnectionCostsWriter {costs.putShort(offset, (short) cost);}- public void write(Path baseDir) throws IOException {+ private String getBaseFileName() {+ return implClazz.getName().replace('.', '/');+ }++ public void write(Path baseDir, String connectionCostsCodecHeader, int dictCodecVersion)+ throws IOException {Files.createDirectories(baseDir);- String fileName =- ConnectionCosts.class.getName().replace('.', '/') + ConnectionCosts.FILENAME_SUFFIX;+ String fileName = getBaseFileName() + ConnectionCosts.FILENAME_SUFFIX;try (OutputStream os = Files.newOutputStream(baseDir.resolve(fileName));OutputStream bos = new BufferedOutputStream(os)) {final DataOutput out = new OutputStreamDataOutput(bos);- CodecUtil.writeHeader(out, ConnectionCosts.HEADER, ConnectionCosts.VERSION);+ CodecUtil.writeHeader(out, connectionCostsCodecHeader, dictCodecVersion);out.writeVInt(forwardSize);out.writeVInt(backwardSize);int last = 0;diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/Dictionary.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/Dictionary.javanew file mode 100644index 00000000000..5ff714c63f7--- /dev/null+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/Dictionary.java@@ -0,0 +1,49 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.analysis.morph;++/** High-level dictionary interface for morphological analyzers. */+public interface Dictionary<T extends MorphData> {+ /**+ * Get left id of specified word+ *+ * @return left id+ */+ default int getLeftId(int morphId) {+ return getMorphAttributes().getLeftId(morphId);+ }++ /**+ * Get right id of specified word+ *+ * @return right id+ */+ default int getRightId(int morphId) {+ return getMorphAttributes().getRightId(morphId);+ }++ /**+ * Get word cost of specified word+ *+ * @return word's cost+ */+ default int getWordCost(int morphId) {+ return getMorphAttributes().getWordCost(morphId);+ }++ T getMorphAttributes();+}diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/DictionaryEntryWriter.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/DictionaryEntryWriter.javanew file mode 100644index 00000000000..76fb44ea603--- /dev/null+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/DictionaryEntryWriter.java@@ -0,0 +1,79 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.analysis.morph;++import java.io.BufferedOutputStream;+import java.io.IOException;+import java.io.OutputStream;+import java.nio.ByteBuffer;+import java.nio.channels.Channels;+import java.nio.channels.WritableByteChannel;+import java.nio.file.Files;+import java.nio.file.Path;+import java.util.ArrayList;+import java.util.List;+import org.apache.lucene.codecs.CodecUtil;+import org.apache.lucene.store.DataOutput;+import org.apache.lucene.store.OutputStreamDataOutput;++/** Abstract writer class to write dictionary entries. */+public abstract class DictionaryEntryWriter {++ protected ByteBuffer buffer;+ protected final List<String> posDict;++ protected DictionaryEntryWriter(int size) {+ this.buffer = ByteBuffer.allocate(size);+ this.posDict = new ArrayList<>();+ }++ /** Writes an entry. */+ protected abstract int putEntry(String[] entry);++ /** Flush POS dictionary data. */+ protected abstract void writePosDict(OutputStream bos, DataOutput out) throws IOException;++ void writePosDict(Path path, String posDictCodecHeader, int dictCodecVersion) throws IOException {+ Files.createDirectories(path.getParent());+ try (OutputStream os = Files.newOutputStream(path);+ OutputStream bos = new BufferedOutputStream(os)) {+ final DataOutput out = new OutputStreamDataOutput(bos);+ CodecUtil.writeHeader(out, posDictCodecHeader, dictCodecVersion);+ writePosDict(bos, out);+ }+ }++ void writeDictionary(Path path, String dictCodecHeader, int dictCodecVersion) throws IOException {+ Files.createDirectories(path.getParent());+ try (OutputStream os = Files.newOutputStream(path);+ OutputStream bos = new BufferedOutputStream(os)) {+ final DataOutput out = new OutputStreamDataOutput(bos);+ CodecUtil.writeHeader(out, dictCodecHeader, dictCodecVersion);+ out.writeVInt(buffer.position());+ final WritableByteChannel channel = Channels.newChannel(bos);+ // Write Buffer+ buffer.flip(); // set position to 0, set limit to current position+ channel.write(buffer);+ assert buffer.remaining() == 0L;+ }+ }++ /** Returns current word id. */+ public int currentPosition() {+ return buffer.position();+ }+}diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/MorphData.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/MorphData.javanew file mode 100644index 00000000000..1cafe9af71a--- /dev/null+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/MorphData.java@@ -0,0 +1,41 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.analysis.morph;++/** High-level interface that represents morphological information in a dictionary */+public interface MorphData {+ /**+ * Get left id of specified word+ *+ * @return left id+ */+ int getLeftId(int morphId);++ /**+ * Get right id of specified word+ *+ * @return right id+ */+ int getRightId(int morphId);++ /**+ * Get word cost of specified word+ *+ * @return word's cost+ */+ int getWordCost(int morphId);+}diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/package-info.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/package-info.javanew file mode 100644index 00000000000..7e53020588f--- /dev/null+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/package-info.java@@ -0,0 +1,19 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++/** Abstract classes for morphological analyzers. */+package org.apache.lucene.analysis.morph;diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/GraphvizFormatter.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/GraphvizFormatter.javaindex de3748b2666..4538daa362f 100644--- a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/GraphvizFormatter.java+++ b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/GraphvizFormatter.java@@ -21,7 +21,8 @@ import java.util.Map;import org.apache.lucene.analysis.ja.JapaneseTokenizer.Position;import org.apache.lucene.analysis.ja.JapaneseTokenizer.WrappedPositionArray;import org.apache.lucene.analysis.ja.dict.ConnectionCosts;-import org.apache.lucene.analysis.ja.dict.Dictionary;+import org.apache.lucene.analysis.ja.dict.JaMorphData;+import org.apache.lucene.analysis.morph.Dictionary;// TODO: would be nice to show 2nd best path in a diff't// color...@@ -140,7 +141,7 @@ public class GraphvizFormatter {attrs = "";}- final Dictionary dict = tok.getDict(posData.backType[idx]);+ final Dictionary<? extends JaMorphData> dict = tok.getDict(posData.backType[idx]);final int wordCost = dict.getWordCost(posData.backID[idx]);final int bgCost =costs.get(diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizer.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizer.javaindex 47cb8d19297..a933525b2f0 100644--- a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizer.java+++ b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizer.java@@ -28,12 +28,13 @@ import java.util.List;import org.apache.lucene.analysis.Tokenizer;import org.apache.lucene.analysis.ja.dict.CharacterDefinition;import org.apache.lucene.analysis.ja.dict.ConnectionCosts;-import org.apache.lucene.analysis.ja.dict.Dictionary;+import org.apache.lucene.analysis.ja.dict.JaMorphData;import org.apache.lucene.analysis.ja.dict.TokenInfoDictionary;import org.apache.lucene.analysis.ja.dict.TokenInfoFST;import org.apache.lucene.analysis.ja.dict.UnknownDictionary;import org.apache.lucene.analysis.ja.dict.UserDictionary;import org.apache.lucene.analysis.ja.tokenattributes.*;+import org.apache.lucene.analysis.morph.Dictionary;import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;@@ -116,7 +117,8 @@ public final class JapaneseTokenizer extends Tokenizer {private static final int MAX_UNKNOWN_WORD_LENGTH = 1024;private static final int MAX_BACKTRACE_GAP = 1024;- private final EnumMap<Type, Dictionary> dictionaryMap = new EnumMap<>(Type.class);+ private final EnumMap<Type, Dictionary<? extends JaMorphData>> dictionaryMap =+ new EnumMap<>(Type.class);private final TokenInfoFST fst;private final TokenInfoDictionary dictionary;@@ -494,10 +496,15 @@ public final class JapaneseTokenizer extends Tokenizer {}private void add(- Dictionary dict, Position fromPosData, int endPos, int wordID, Type type, boolean addPenalty)+ JaMorphData morphAtts,+ Position fromPosData,+ int endPos,+ int wordID,+ Type type,+ boolean addPenalty)throws IOException {- final int wordCost = dict.getWordCost(wordID);- final int leftID = dict.getLeftId(wordID);+ final int wordCost = morphAtts.getWordCost(wordID);+ final int leftID = morphAtts.getLeftId(wordID);int leastCost = Integer.MAX_VALUE;int leastIDX = -1;assert fromPosData.count > 0;@@ -560,7 +567,7 @@ public final class JapaneseTokenizer extends Tokenizer {// positions.get(endPos).add(leastCost, dict.getRightId(wordID), fromPosData.pos, leastIDX,// wordID, type);- assert leftID == dict.getRightId(wordID);+ assert leftID == morphAtts.getRightId(wordID);positions.get(endPos).add(leastCost, leftID, fromPosData.pos, leastIDX, wordID, type);}@@ -895,7 +902,7 @@ public final class JapaneseTokenizer extends Tokenizer {+ (posAhead + 1));}add(- userDictionary,+ userDictionary.getMorphAttributes(),posData,posAhead + 1,output + arc.nextFinalOutput().intValue(),@@ -948,7 +955,7 @@ public final class JapaneseTokenizer extends Tokenizer {}for (int ofs = 0; ofs < wordIdRef.length; ofs++) {add(- dictionary,+ dictionary.getMorphAttributes(),posData,posAhead + 1,wordIdRef.ints[wordIdRef.offset + ofs],@@ -1004,7 +1011,7 @@ public final class JapaneseTokenizer extends Tokenizer {}for (int ofs = 0; ofs < wordIdRef.length; ofs++) {add(- unkDictionary,+ unkDictionary.getMorphAttributes(),posData,posData.pos + unknownWordLength,wordIdRef.ints[wordIdRef.offset + ofs],@@ -1126,7 +1133,7 @@ public final class JapaneseTokenizer extends Tokenizer {final int pathCost = posData.costs[bestStartIDX];for (int forwardArcIDX = 0; forwardArcIDX < posData.forwardCount; forwardArcIDX++) {final Type forwardType = posData.forwardType[forwardArcIDX];- final Dictionary dict2 = getDict(forwardType);+ final Dictionary<? extends JaMorphData> dict2 = getDict(forwardType);final int wordID = posData.forwardID[forwardArcIDX];final int toPos = posData.forwardPos[forwardArcIDX];final int newCost =@@ -1169,7 +1176,7 @@ public final class JapaneseTokenizer extends Tokenizer {+ toPos);}add(- getDict(forwardType),+ getDict(forwardType).getMorphAttributes(),posData,toPos,posData.forwardID[forwardArcIDX],@@ -1184,7 +1191,7 @@ public final class JapaneseTokenizer extends Tokenizer {// yet another lattice data structureprivate static final class Lattice {char[] fragment;- EnumMap<Type, Dictionary> dictionaryMap;+ EnumMap<Type, Dictionary<? extends JaMorphData>> dictionaryMap;boolean useEOS;int rootCapacity = 0;@@ -1296,7 +1303,7 @@ public final class JapaneseTokenizer extends Tokenizer {nodeLeftID[node] = 0;nodeRightID[node] = 0;} else {- Dictionary dic = dictionaryMap.get(dicType);+ Dictionary<? extends JaMorphData> dic = dictionaryMap.get(dicType);nodeWordCost[node] = dic.getWordCost(wordID);nodeLeftID[node] = dic.getLeftId(wordID);nodeRightID[node] = dic.getRightId(wordID);@@ -1338,7 +1345,7 @@ public final class JapaneseTokenizer extends Tokenizer {void setup(char[] fragment,- EnumMap<Type, Dictionary> dictionaryMap,+ EnumMap<Type, Dictionary<? extends JaMorphData>> dictionaryMap,WrappedPositionArray positions,int prevOffset,int endOffset,@@ -1565,7 +1572,7 @@ public final class JapaneseTokenizer extends Tokenizer {right - left,Type.USER,lattice.rootBase + left,- userDictionary));+ userDictionary.getMorphAttributes()));// Output compoundint current = 0;for (int j = 1; j < wordIDAndLength.length; j++) {@@ -1579,7 +1586,7 @@ public final class JapaneseTokenizer extends Tokenizer {len,Type.USER,lattice.rootBase + current + left,- userDictionary));+ userDictionary.getMorphAttributes()));}current += len;}@@ -1592,7 +1599,7 @@ public final class JapaneseTokenizer extends Tokenizer {right - left,type,lattice.rootBase + left,- getDict(type)));+ getDict(type).getMorphAttributes()));}}}@@ -1921,7 +1928,7 @@ public final class JapaneseTokenizer extends Tokenizer {length,backType,backPos,- getDict(backType));+ getDict(backType).getMorphAttributes());// Redirect our backtrace to 2nd best:bestIDX = leastIDX;@@ -1980,7 +1987,7 @@ public final class JapaneseTokenizer extends Tokenizer {altToken = null;}- final Dictionary dict = getDict(backType);+ final Dictionary<? extends JaMorphData> dict = getDict(backType);if (backType == Type.USER) {@@ -2000,7 +2007,7 @@ public final class JapaneseTokenizer extends Tokenizer {len,Type.USER,current + backPos,- dict));+ dict.getMorphAttributes()));if (VERBOSE) {System.out.println(" add USER token=" + pending.get(pending.size() - 1));}@@ -2037,14 +2044,16 @@ public final class JapaneseTokenizer extends Tokenizer {charLen,Type.UNKNOWN,backPos + i,- unkDictionary));+ unkDictionary.getMorphAttributes()));unigramTokenCount++;}}backCount += unigramTokenCount;} else if (!discardPunctuation || length == 0 || !isPunctuation(fragment[offset])) {- pending.add(new Token(backID, fragment, offset, length, backType, backPos, dict));+ pending.add(+ new Token(+ backID, fragment, offset, length, backType, backPos, dict.getMorphAttributes()));if (VERBOSE) {System.out.println(" add token=" + pending.get(pending.size() - 1));}@@ -2073,7 +2082,7 @@ public final class JapaneseTokenizer extends Tokenizer {positions.freeBefore(endPos);}- Dictionary getDict(Type type) {+ Dictionary<? extends JaMorphData> getDict(Type type) {return dictionaryMap.get(type);}diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/Token.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/Token.javaindex 5a10b50817b..e9ae9490125 100644--- a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/Token.java+++ b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/Token.java@@ -17,11 +17,11 @@package org.apache.lucene.analysis.ja;import org.apache.lucene.analysis.ja.JapaneseTokenizer.Type;-import org.apache.lucene.analysis.ja.dict.Dictionary;+import org.apache.lucene.analysis.ja.dict.JaMorphData;/** Analyzed token with morphological data from its dictionary. */public class Token {- private final Dictionary dictionary;+ private final JaMorphData morphAtts;private final int wordId;@@ -41,14 +41,14 @@ public class Token {int length,Type type,int position,- Dictionary dictionary) {+ JaMorphData morphAtts) {this.wordId = wordId;this.surfaceForm = surfaceForm;this.offset = offset;this.length = length;this.type = type;this.position = position;- this.dictionary = dictionary;+ this.morphAtts = morphAtts;}@Override@@ -66,7 +66,7 @@ public class Token {+ " wordId="+ wordId+ " leftID="- + dictionary.getLeftId(wordId)+ + morphAtts.getLeftId(wordId)+ ")";}@@ -92,32 +92,32 @@ public class Token {/** @return reading. null if token doesn't have reading. */public String getReading() {- return dictionary.getReading(wordId, surfaceForm, offset, length);+ return morphAtts.getReading(wordId, surfaceForm, offset, length);}/** @return pronunciation. null if token doesn't have pronunciation. */public String getPronunciation() {- return dictionary.getPronunciation(wordId, surfaceForm, offset, length);+ return morphAtts.getPronunciation(wordId, surfaceForm, offset, length);}/** @return part of speech. */public String getPartOfSpeech() {- return dictionary.getPartOfSpeech(wordId);+ return morphAtts.getPartOfSpeech(wordId);}/** @return inflection type or null */public String getInflectionType() {- return dictionary.getInflectionType(wordId);+ return morphAtts.getInflectionType(wordId);}/** @return inflection form or null */public String getInflectionForm() {- return dictionary.getInflectionForm(wordId);+ return morphAtts.getInflectionForm(wordId);}/** @return base form or null if token is not inflected */public String getBaseForm() {- return dictionary.getBaseForm(wordId, surfaceForm, offset, length);+ return morphAtts.getBaseForm(wordId, surfaceForm, offset, length);}/**diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/CharacterDefinition.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/CharacterDefinition.javaindex 36bb825bc2d..be29ebbe367 100644--- a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/CharacterDefinition.java+++ b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/CharacterDefinition.java@@ -16,20 +16,13 @@*/package org.apache.lucene.analysis.ja.dict;-import java.io.BufferedInputStream;import java.io.IOException;import java.io.InputStream;-import org.apache.lucene.codecs.CodecUtil;-import org.apache.lucene.store.DataInput;-import org.apache.lucene.store.InputStreamDataInput;import org.apache.lucene.util.IOUtils;/** Character category data. */-public final class CharacterDefinition {-- public static final String FILENAME_SUFFIX = ".dat";- public static final String HEADER = "kuromoji_cd";- public static final int VERSION = 1;+public final class CharacterDefinition+ extends org.apache.lucene.analysis.morph.CharacterDefinition {public static final int CLASS_COUNT = CharacterClass.values().length;@@ -49,11 +42,6 @@ public final class CharacterDefinition {KANJINUMERIC;}- private final byte[] characterCategoryMap = new byte[0x10000];-- private final boolean[] invokeMap = new boolean[CLASS_COUNT];- private final boolean[] groupMap = new boolean[CLASS_COUNT];-// the classes:public static final byte NGRAM = (byte) CharacterClass.NGRAM.ordinal();public static final byte DEFAULT = (byte) CharacterClass.DEFAULT.ordinal();@@ -69,16 +57,11 @@ public final class CharacterDefinition {public static final byte KANJINUMERIC = (byte) CharacterClass.KANJINUMERIC.ordinal();private CharacterDefinition() throws IOException {- try (InputStream is = new BufferedInputStream(getClassResource())) {- final DataInput in = new InputStreamDataInput(is);- CodecUtil.checkHeader(in, HEADER, VERSION, VERSION);- in.readBytes(characterCategoryMap, 0, characterCategoryMap.length);- for (int i = 0; i < CLASS_COUNT; i++) {- final byte b = in.readByte();- invokeMap[i] = (b & 0x01) != 0;- groupMap[i] = (b & 0x02) != 0;- }- }+ super(+ CharacterDefinition::getClassResource,+ DictionaryConstants.CHARDEF_HEADER,+ DictionaryConstants.VERSION,+ CharacterClass.values().length);}private static InputStream getClassResource() throws IOException {@@ -87,18 +70,6 @@ public final class CharacterDefinition {CharacterDefinition.class.getResourceAsStream(resourcePath), resourcePath);}- public byte getCharacterClass(char c) {- return characterCategoryMap[c];- }-- public boolean isInvoke(char c) {- return invokeMap[characterCategoryMap[c]];- }-- public boolean isGroup(char c) {- return groupMap[characterCategoryMap[c]];- }-public boolean isKanji(char c) {final byte characterClass = characterCategoryMap[c];return characterClass == KANJI || characterClass == KANJINUMERIC;diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/ConnectionCosts.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/ConnectionCosts.javaindex dc5fabe6c54..c11b9ee716a 100644--- a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/ConnectionCosts.java+++ b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/ConnectionCosts.java@@ -16,27 +16,15 @@*/package org.apache.lucene.analysis.ja.dict;-import java.io.BufferedInputStream;import java.io.IOException;import java.io.InputStream;-import java.nio.ByteBuffer;import java.nio.file.Files;import java.nio.file.Path;-import org.apache.lucene.codecs.CodecUtil;-import org.apache.lucene.store.DataInput;-import org.apache.lucene.store.InputStreamDataInput;import org.apache.lucene.util.IOSupplier;import org.apache.lucene.util.IOUtils;/** n-gram connection cost data */-public final class ConnectionCosts {-- public static final String FILENAME_SUFFIX = ".dat";- public static final String HEADER = "kuromoji_cc";- public static final int VERSION = 1;-- private final ByteBuffer buffer;- private final int forwardSize;+public final class ConnectionCosts extends org.apache.lucene.analysis.morph.ConnectionCosts {/*** Create a {@link ConnectionCosts} from an external resource path.@@ -53,24 +41,8 @@ public final class ConnectionCosts {}private ConnectionCosts(IOSupplier<InputStream> connectionCostResource) throws IOException {- try (InputStream is = new BufferedInputStream(connectionCostResource.get())) {- final DataInput in = new InputStreamDataInput(is);- CodecUtil.checkHeader(in, HEADER, VERSION, VERSION);- forwardSize = in.readVInt();- int backwardSize = in.readVInt();- int size = forwardSize * backwardSize;-- // copy the matrix into a direct byte buffer- final ByteBuffer tmpBuffer = ByteBuffer.allocateDirect(size * 2);- int accum = 0;- for (int j = 0; j < backwardSize; j++) {- for (int i = 0; i < forwardSize; i++) {- accum += in.readZInt();- tmpBuffer.putShort((short) accum);- }- }- buffer = tmpBuffer.asReadOnlyBuffer();- }+ super(+ connectionCostResource, DictionaryConstants.CONN_COSTS_HEADER, DictionaryConstants.VERSION);}private static InputStream getClassResource() throws IOException {@@ -79,12 +51,6 @@ public final class ConnectionCosts {ConnectionCosts.class.getResourceAsStream(resourcePath), resourcePath);}- public int get(int forwardId, int backwardId) {- // map 2d matrix into a single dimension short array- int offset = (backwardId * forwardSize + forwardId) * 2;- return buffer.getShort(offset);- }-public static ConnectionCosts getInstance() {return SingletonHolder.INSTANCE;}diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/DictionaryConstants.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/DictionaryConstants.javanew file mode 100644index 00000000000..3046beef3e6--- /dev/null+++ b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/DictionaryConstants.java@@ -0,0 +1,33 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.analysis.ja.dict;++/** Dictionary constants */+public final class DictionaryConstants {+ /** Codec header of the dictionary file. */+ public static final String DICT_HEADER = "kuromoji_dict";+ /** Codec header of the dictionary mapping file. */+ public static final String TARGETMAP_HEADER = "kuromoji_dict_map";+ /** Codec header of the POS dictionary file. */+ public static final String POSDICT_HEADER = "kuromoji_dict_pos";+ /** Codec header of the connection costs. */+ public static final String CONN_COSTS_HEADER = "kuromoji_cc";+ /** Codec header of the character definition file. */+ public static final String CHARDEF_HEADER = "kuromoji_cd";+ /** Codec version of the binary dictionary */+ public static final int VERSION = 1;+}diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/Dictionary.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/JaMorphData.javasimilarity index 60%rename from lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/Dictionary.javarename to lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/JaMorphData.javaindex cfe11b30cd9..8865f6ad09f 100644--- a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/Dictionary.java+++ b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/JaMorphData.java@@ -16,71 +16,49 @@*/package org.apache.lucene.analysis.ja.dict;-/** Dictionary interface for retrieving morphological data by id. */-public interface Dictionary {-- public static final String INTERNAL_SEPARATOR = "\u0000";-- /**- * Get left id of specified word- *- * @return left id- */- public int getLeftId(int wordId);-- /**- * Get right id of specified word- *- * @return right id- */- public int getRightId(int wordId);-- /**- * Get word cost of specified word- *- * @return word's cost- */- public int getWordCost(int wordId);+import org.apache.lucene.analysis.morph.MorphData;+/** Represents Japanese morphological information. */+public interface JaMorphData extends MorphData {/*** Get Part-Of-Speech of tokens*- * @param wordId word ID of token+ * @param morphId word ID of token* @return Part-Of-Speech of the token*/- public String getPartOfSpeech(int wordId);+ String getPartOfSpeech(int morphId);/*** Get reading of tokens*- * @param wordId word ID of token+ * @param morphId word ID of token* @return Reading of the token*/- public String getReading(int wordId, char[] surface, int off, int len);+ String getReading(int morphId, char[] surface, int off, int len);/*** Get base form of word*- * @param wordId word ID of token+ * @param morphId word ID of token* @return Base form (only different for inflected words, otherwise null)*/- public String getBaseForm(int wordId, char[] surface, int off, int len);+ String getBaseForm(int morphId, char[] surface, int off, int len);/*** Get pronunciation of tokens*- * @param wordId word ID of token+ * @param morphId word ID of token* @return Pronunciation of the token*/- public String getPronunciation(int wordId, char[] surface, int off, int len);+ String getPronunciation(int morphId, char[] surface, int off, int len);/*** Get inflection type of tokens*- * @param wordId word ID of token+ * @param morphId word ID of token* @return inflection type, or null*/- public String getInflectionType(int wordId);+ String getInflectionType(int morphId);/*** Get inflection form of tokens@@ -88,7 +66,7 @@ public interface Dictionary {* @param wordId word ID of token* @return inflection form, or null*/- public String getInflectionForm(int wordId);+ String getInflectionForm(int wordId);// TODO: maybe we should have a optimal method, a non-typesafe// 'getAdditionalData' if other dictionaries like unidic have additional data}diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/TokenInfoDictionary.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/TokenInfoDictionary.javaindex d9e50f86a75..c769587829b 100644--- a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/TokenInfoDictionary.java+++ b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/TokenInfoDictionary.java@@ -21,6 +21,7 @@ import java.io.IOException;import java.io.InputStream;import java.nio.file.Files;import java.nio.file.Path;+import org.apache.lucene.analysis.morph.BinaryDictionary;import org.apache.lucene.store.DataInput;import org.apache.lucene.store.InputStreamDataInput;import org.apache.lucene.util.IOSupplier;@@ -32,11 +33,12 @@ import org.apache.lucene.util.fst.PositiveIntOutputs;* Binary dictionary implementation for a known-word dictionary model: Words are encoded into an FST* mapping to a list of wordIDs.*/-public final class TokenInfoDictionary extends BinaryDictionary {+public final class TokenInfoDictionary extends BinaryDictionary<TokenInfoMorphData> {public static final String FST_FILENAME_SUFFIX = "$fst.dat";private final TokenInfoFST fst;+ private final TokenInfoMorphData morphAtts;/*** Create a {@link TokenInfoDictionary} from an external resource path.@@ -70,7 +72,14 @@ public final class TokenInfoDictionary extends BinaryDictionary {IOSupplier<InputStream> dictResource,IOSupplier<InputStream> fstResource)throws IOException {- super(targetMapResource, posResource, dictResource);+ super(+ targetMapResource,+ dictResource,+ DictionaryConstants.TARGETMAP_HEADER,+ DictionaryConstants.DICT_HEADER,+ DictionaryConstants.VERSION);+ this.morphAtts = new TokenInfoMorphData(buffer, posResource);+FST<Long> fst;try (InputStream is = new BufferedInputStream(fstResource.get())) {DataInput in = new InputStreamDataInput(is);@@ -86,6 +95,11 @@ public final class TokenInfoDictionary extends BinaryDictionary {TokenInfoDictionary.class.getResourceAsStream(resourcePath), resourcePath);}+ @Override+ public TokenInfoMorphData getMorphAttributes() {+ return morphAtts;+ }+public TokenInfoFST getFST() {return fst;}diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/BinaryDictionary.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/TokenInfoMorphData.javasimilarity index 56%rename from lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/BinaryDictionary.javarename to lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/TokenInfoMorphData.javaindex 9ad0d813778..d743a7faca6 100644--- a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/BinaryDictionary.java+++ b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/TokenInfoMorphData.java@@ -17,95 +17,37 @@package org.apache.lucene.analysis.ja.dict;import java.io.BufferedInputStream;-import java.io.EOFException;import java.io.IOException;import java.io.InputStream;import java.nio.ByteBuffer;-import java.nio.channels.Channels;-import java.nio.channels.ReadableByteChannel;import org.apache.lucene.codecs.CodecUtil;import org.apache.lucene.store.DataInput;import org.apache.lucene.store.InputStreamDataInput;import org.apache.lucene.util.IOSupplier;-import org.apache.lucene.util.IntsRef;-/** Base class for a binary-encoded in-memory dictionary. */-public abstract class BinaryDictionary implements Dictionary {-- public static final String DICT_FILENAME_SUFFIX = "$buffer.dat";- public static final String TARGETMAP_FILENAME_SUFFIX = "$targetMap.dat";- public static final String POSDICT_FILENAME_SUFFIX = "$posDict.dat";-- public static final String DICT_HEADER = "kuromoji_dict";- public static final String TARGETMAP_HEADER = "kuromoji_dict_map";- public static final String POSDICT_HEADER = "kuromoji_dict_pos";- public static final int VERSION = 1;+/** Morphological information for system dictionary. */+public class TokenInfoMorphData implements JaMorphData {private final ByteBuffer buffer;- private final int[] targetMapOffsets, targetMap;private final String[] posDict;private final String[] inflTypeDict;private final String[] inflFormDict;- protected BinaryDictionary(- IOSupplier<InputStream> targetMapResource,- IOSupplier<InputStream> posResource,- IOSupplier<InputStream> dictResource)- throws IOException {- try (InputStream mapIS = new BufferedInputStream(targetMapResource.get())) {- final DataInput in = new InputStreamDataInput(mapIS);- CodecUtil.checkHeader(in, TARGETMAP_HEADER, VERSION, VERSION);- this.targetMap = new int[in.readVInt()];- this.targetMapOffsets = new int[in.readVInt()];- populateTargetMap(in, this.targetMap, this.targetMapOffsets);- }-+ TokenInfoMorphData(ByteBuffer buffer, IOSupplier<InputStream> posResource) throws IOException {+ this.buffer = buffer;try (InputStream posIS = new BufferedInputStream(posResource.get())) {final DataInput in = new InputStreamDataInput(posIS);- CodecUtil.checkHeader(in, POSDICT_HEADER, VERSION, VERSION);+ CodecUtil.checkHeader(+ in,+ DictionaryConstants.POSDICT_HEADER,+ DictionaryConstants.VERSION,+ DictionaryConstants.VERSION);final int posSize = in.readVInt();this.posDict = new String[posSize];this.inflTypeDict = new String[posSize];this.inflFormDict = new String[posSize];- populatePosDict(in, posSize, this.posDict, this.inflTypeDict, this.inflFormDict);- }-- // no buffering here, as we load in one large buffer- try (InputStream dictIS = dictResource.get()) {- final DataInput in = new InputStreamDataInput(dictIS);- CodecUtil.checkHeader(in, DICT_HEADER, VERSION, VERSION);- final int size = in.readVInt();- final ByteBuffer tmpBuffer = ByteBuffer.allocateDirect(size);- final ReadableByteChannel channel = Channels.newChannel(dictIS);- final int read = channel.read(tmpBuffer);- if (read != size) {- throw new EOFException("Cannot read whole dictionary");- }- this.buffer = tmpBuffer.asReadOnlyBuffer();- }- }-- private static void populateTargetMap(DataInput in, int[] targetMap, int[] targetMapOffsets)- throws IOException {- int accum = 0, sourceId = 0;- for (int ofs = 0; ofs < targetMap.length; ofs++) {- final int val = in.readVInt();- if ((val & 0x01) != 0) {- targetMapOffsets[sourceId] = ofs;- sourceId++;- }- accum += val >>> 1;- targetMap[ofs] = accum;+ populatePosDict(in, posSize, posDict, inflTypeDict, inflFormDict);}- if (sourceId + 1 != targetMapOffsets.length)- throw new IOException(- "targetMap file format broken; targetMap.length="- + targetMap.length- + ", targetMapOffsets.length="- + targetMapOffsets.length- + ", sourceId="- + sourceId);- targetMapOffsets[sourceId] = targetMap.length;}private static void populatePosDict(@@ -125,32 +67,25 @@ public abstract class BinaryDictionary implements Dictionary {}}- public void lookupWordIds(int sourceId, IntsRef ref) {- ref.ints = targetMap;- ref.offset = targetMapOffsets[sourceId];- // targetMapOffsets always has one more entry pointing behind last:- ref.length = targetMapOffsets[sourceId + 1] - ref.offset;- }-@Override- public int getLeftId(int wordId) {- return (buffer.getShort(wordId) & 0xffff) >>> 3;+ public int getLeftId(int morphId) {+ return (buffer.getShort(morphId) & 0xffff) >>> 3;}@Override- public int getRightId(int wordId) {- return (buffer.getShort(wordId) & 0xffff) >>> 3;+ public int getRightId(int morphId) {+ return (buffer.getShort(morphId) & 0xffff) >>> 3;}@Override- public int getWordCost(int wordId) {- return buffer.getShort(wordId + 2); // Skip id+ public int getWordCost(int morphId) {+ return buffer.getShort(morphId + 2); // Skip id}@Override- public String getBaseForm(int wordId, char[] surfaceForm, int off, int len) {- if (hasBaseFormData(wordId)) {- int offset = baseFormOffset(wordId);+ public String getBaseForm(int morphId, char[] surfaceForm, int off, int len) {+ if (hasBaseFormData(morphId)) {+ int offset = baseFormOffset(morphId);int data = buffer.get(offset++) & 0xff;int prefix = data >>> 4;int suffix = data & 0xF;@@ -166,9 +101,9 @@ public abstract class BinaryDictionary implements Dictionary {}@Override- public String getReading(int wordId, char[] surface, int off, int len) {- if (hasReadingData(wordId)) {- int offset = readingOffset(wordId);+ public String getReading(int morphId, char[] surface, int off, int len) {+ if (hasReadingData(morphId)) {+ int offset = readingOffset(morphId);int readingData = buffer.get(offset++) & 0xff;return readString(offset, readingData >>> 1, (readingData & 1) == 1);} else {@@ -187,24 +122,24 @@ public abstract class BinaryDictionary implements Dictionary {}@Override- public String getPartOfSpeech(int wordId) {- return posDict[getLeftId(wordId)];+ public String getPartOfSpeech(int morphId) {+ return posDict[getLeftId(morphId)];}@Override- public String getPronunciation(int wordId, char[] surface, int off, int len) {- if (hasPronunciationData(wordId)) {- int offset = pronunciationOffset(wordId);+ public String getPronunciation(int morphId, char[] surface, int off, int len) {+ if (hasPronunciationData(morphId)) {+ int offset = pronunciationOffset(morphId);int pronunciationData = buffer.get(offset++) & 0xff;return readString(offset, pronunciationData >>> 1, (pronunciationData & 1) == 1);} else {- return getReading(wordId, surface, off, len); // same as the reading+ return getReading(morphId, surface, off, len); // same as the reading}}@Override- public String getInflectionType(int wordId) {- return inflTypeDict[getLeftId(wordId)];+ public String getInflectionType(int morphId) {+ return inflTypeDict[getLeftId(morphId)];}@Override@@ -212,10 +147,6 @@ public abstract class BinaryDictionary implements Dictionary {return inflFormDict[getLeftId(wordId)];}- private static int baseFormOffset(int wordId) {- return wordId + 4;- }-private int readingOffset(int wordId) {int offset = baseFormOffset(wordId);if (hasBaseFormData(wordId)) {@@ -242,6 +173,10 @@ public abstract class BinaryDictionary implements Dictionary {}}+ private static int baseFormOffset(int wordId) {+ return wordId + 4;+ }+private boolean hasBaseFormData(int wordId) {return (buffer.getShort(wordId) & HAS_BASEFORM) != 0;}diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/UnknownDictionary.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/UnknownDictionary.javaindex f9bdc62f49e..b4a9012e3ee 100644--- a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/UnknownDictionary.java+++ b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/UnknownDictionary.java@@ -20,12 +20,15 @@ import java.io.IOException;import java.io.InputStream;import java.nio.file.Files;import java.nio.file.Path;+import org.apache.lucene.analysis.morph.BinaryDictionary;+import org.apache.lucene.util.IOSupplier;import org.apache.lucene.util.IOUtils;/** Dictionary for unknown-word handling. */-public final class UnknownDictionary extends BinaryDictionary {+public final class UnknownDictionary extends BinaryDictionary<UnknownMorphData> {private final CharacterDefinition characterDefinition = CharacterDefinition.getInstance();+ private final UnknownMorphData morphAtts;/*** Create a {@link UnknownDictionary} from an external resource path.@@ -36,25 +39,44 @@ public final class UnknownDictionary extends BinaryDictionary {* @throws IOException if resource was not found or broken*/public UnknownDictionary(Path targetMapFile, Path posDictFile, Path dictFile) throws IOException {- super(+ this(() -> Files.newInputStream(targetMapFile),() -> Files.newInputStream(posDictFile),() -> Files.newInputStream(dictFile));}private UnknownDictionary() throws IOException {- super(+ this(() -> getClassResource(TARGETMAP_FILENAME_SUFFIX),() -> getClassResource(POSDICT_FILENAME_SUFFIX),() -> getClassResource(DICT_FILENAME_SUFFIX));}+ private UnknownDictionary(+ IOSupplier<InputStream> targetMapResource,+ IOSupplier<InputStream> posResource,+ IOSupplier<InputStream> dictResource)+ throws IOException {+ super(+ targetMapResource,+ dictResource,+ DictionaryConstants.TARGETMAP_HEADER,+ DictionaryConstants.DICT_HEADER,+ DictionaryConstants.VERSION);+ this.morphAtts = new UnknownMorphData(buffer, posResource);+ }+private static InputStream getClassResource(String suffix) throws IOException {final String resourcePath = UnknownDictionary.class.getSimpleName() + suffix;return IOUtils.requireResourceNonNull(UnknownDictionary.class.getResourceAsStream(resourcePath), resourcePath);}+ @Override+ public UnknownMorphData getMorphAttributes() {+ return morphAtts;+ }+public int lookup(char[] text, int offset, int len) {if (!characterDefinition.isGroup(text[offset])) {return 1;@@ -79,21 +101,6 @@ public final class UnknownDictionary extends BinaryDictionary {return characterDefinition;}- @Override- public String getReading(int wordId, char[] surface, int off, int len) {- return null;- }-- @Override- public String getInflectionType(int wordId) {- return null;- }-- @Override- public String getInflectionForm(int wordId) {- return null;- }-public static UnknownDictionary getInstance() {return SingletonHolder.INSTANCE;}diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/UnknownMorphData.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/UnknownMorphData.javanew file mode 100644index 00000000000..0810346e218--- /dev/null+++ b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/UnknownMorphData.java@@ -0,0 +1,44 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.analysis.ja.dict;++import java.io.IOException;+import java.io.InputStream;+import java.nio.ByteBuffer;+import org.apache.lucene.util.IOSupplier;++/** Morphological information for unk dictionary. */+final class UnknownMorphData extends TokenInfoMorphData {+ UnknownMorphData(ByteBuffer buffer, IOSupplier<InputStream> posResource) throws IOException {+ super(buffer, posResource);+ }++ @Override+ public String getReading(int morphId, char[] surface, int off, int len) {+ return null;+ }++ @Override+ public String getInflectionType(int morphId) {+ return null;+ }++ @Override+ public String getInflectionForm(int wordId) {+ return null;+ }+}diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/UserDictionary.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/UserDictionary.javaindex 859806e5ef0..12dd27d9584 100644--- a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/UserDictionary.java+++ b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/UserDictionary.java@@ -26,13 +26,16 @@ import java.util.List;import java.util.Map;import java.util.TreeMap;import org.apache.lucene.analysis.ja.util.CSVUtil;+import org.apache.lucene.analysis.morph.Dictionary;import org.apache.lucene.util.IntsRefBuilder;import org.apache.lucene.util.fst.FST;import org.apache.lucene.util.fst.FSTCompiler;import org.apache.lucene.util.fst.PositiveIntOutputs;/** Class for building a User Dictionary. This class allows for custom segmentation of phrases. */-public final class UserDictionary implements Dictionary {+public final class UserDictionary implements Dictionary<UserMorphData> {++ public static final String INTERNAL_SEPARATOR = "\u0000";// phrase text -> phrase IDprivate final TokenInfoFST fst;@@ -41,15 +44,9 @@ public final class UserDictionary implements Dictionary {private final int[][] segmentations;// holds readings and POS, indexed by wordid- private final String[] data;-- private static final int CUSTOM_DICTIONARY_WORD_ID_OFFSET = 100000000;-- public static final int WORD_COST = -100000;+ private final UserMorphData morphAtts;- public static final int LEFT_ID = 5;-- public static final int RIGHT_ID = 5;+ static final int CUSTOM_DICTIONARY_WORD_ID_OFFSET = 100000000;public static UserDictionary open(Reader reader) throws IOException {@@ -150,10 +147,15 @@ public final class UserDictionary implements Dictionary {ord++;}this.fst = new TokenInfoFST(fstCompiler.compile(), false);- this.data = data.toArray(new String[data.size()]);+ this.morphAtts = new UserMorphData(data.toArray(new String[0]));this.segmentations = segmentations.toArray(new int[segmentations.size()][]);}+ @Override+ public UserMorphData getMorphAttributes() {+ return morphAtts;+ }+/*** Lookup words in text*@@ -222,78 +224,4 @@ public final class UserDictionary implements Dictionary {public int[] lookupSegmentation(int phraseID) {return segmentations[phraseID];}-- @Override- public int getLeftId(int wordId) {- return LEFT_ID;- }-- @Override- public int getRightId(int wordId) {- return RIGHT_ID;- }-- @Override- public int getWordCost(int wordId) {- return WORD_COST;- }-- @Override- public String getReading(int wordId, char[] surface, int off, int len) {- return getFeature(wordId, 0);- }-- @Override- public String getPartOfSpeech(int wordId) {- return getFeature(wordId, 1);- }-- @Override- public String getBaseForm(int wordId, char[] surface, int off, int len) {- return null; // TODO: add support?- }-- @Override- public String getPronunciation(int wordId, char[] surface, int off, int len) {- return null; // TODO: add support?- }-- @Override- public String getInflectionType(int wordId) {- return null; // TODO: add support?- }-- @Override- public String getInflectionForm(int wordId) {- return null; // TODO: add support?- }-- private String[] getAllFeaturesArray(int wordId) {- String allFeatures = data[wordId - CUSTOM_DICTIONARY_WORD_ID_OFFSET];- if (allFeatures == null) {- return null;- }-- return allFeatures.split(INTERNAL_SEPARATOR);- }-- private String getFeature(int wordId, int... fields) {- String[] allFeatures = getAllFeaturesArray(wordId);- if (allFeatures == null) {- return null;- }- StringBuilder sb = new StringBuilder();- if (fields.length == 0) { // All features- for (String feature : allFeatures) {- sb.append(CSVUtil.quoteEscape(feature)).append(",");- }- } else if (fields.length == 1) { // One feature doesn't need to escape value- sb.append(allFeatures[fields[0]]).append(",");- } else {- for (int field : fields) {- sb.append(CSVUtil.quoteEscape(allFeatures[field])).append(",");- }- }- return sb.deleteCharAt(sb.length() - 1).toString();- }}diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/UserMorphData.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/UserMorphData.javanew file mode 100644index 00000000000..1f6907052e0--- /dev/null+++ b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/UserMorphData.java@@ -0,0 +1,110 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.analysis.ja.dict;++import static org.apache.lucene.analysis.ja.dict.UserDictionary.CUSTOM_DICTIONARY_WORD_ID_OFFSET;+import static org.apache.lucene.analysis.ja.dict.UserDictionary.INTERNAL_SEPARATOR;++import org.apache.lucene.analysis.ja.util.CSVUtil;++/** Morphological information for user dictionary. */+final class UserMorphData implements JaMorphData {+ public static final int WORD_COST = -100000;+ public static final int LEFT_ID = 5;+ public static final int RIGHT_ID = 5;++ // holds readings and POS, indexed by wordid+ private final String[] data;++ UserMorphData(String[] data) {+ this.data = data;+ }++ @Override+ public int getLeftId(int wordId) {+ return LEFT_ID;+ }++ @Override+ public int getRightId(int wordId) {+ return RIGHT_ID;+ }++ @Override+ public int getWordCost(int wordId) {+ return WORD_COST;+ }++ @Override+ public String getReading(int morphId, char[] surface, int off, int len) {+ return getFeature(morphId, 0);+ }++ @Override+ public String getPartOfSpeech(int morphId) {+ return getFeature(morphId, 1);+ }++ @Override+ public String getBaseForm(int morphId, char[] surface, int off, int len) {+ return null; // TODO: add support?+ }++ @Override+ public String getPronunciation(int morphId, char[] surface, int off, int len) {+ return null; // TODO: add support?+ }++ @Override+ public String getInflectionType(int morphId) {+ return null; // TODO: add support?+ }++ @Override+ public String getInflectionForm(int wordId) {+ return null; // TODO: add support?+ }++ private String[] getAllFeaturesArray(int wordId) {+ String allFeatures = data[wordId - CUSTOM_DICTIONARY_WORD_ID_OFFSET];+ if (allFeatures == null) {+ return null;+ }++ return allFeatures.split(INTERNAL_SEPARATOR);+ }++ private String getFeature(int wordId, int... fields) {+ String[] allFeatures = getAllFeaturesArray(wordId);+ if (allFeatures == null) {+ return null;+ }+ StringBuilder sb = new StringBuilder();+ if (fields.length == 0) { // All features+ for (String feature : allFeatures) {+ sb.append(CSVUtil.quoteEscape(feature)).append(",");+ }+ } else if (fields.length == 1) { // One feature doesn't need to escape value+ sb.append(allFeatures[fields[0]]).append(",");+ } else {+ for (int field : fields) {+ sb.append(CSVUtil.quoteEscape(allFeatures[field])).append(",");+ }+ }+ return sb.deleteCharAt(sb.length() - 1).toString();+ }+}diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/util/BinaryDictionaryWriter.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/util/BinaryDictionaryWriter.javadeleted file mode 100644index bf157b9abb2..00000000000--- a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/util/BinaryDictionaryWriter.java+++ /dev/null@@ -1,334 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.analysis.ja.util;--import java.io.BufferedOutputStream;-import java.io.IOException;-import java.io.OutputStream;-import java.nio.ByteBuffer;-import java.nio.channels.Channels;-import java.nio.channels.WritableByteChannel;-import java.nio.file.Files;-import java.nio.file.Path;-import java.util.ArrayList;-import org.apache.lucene.analysis.ja.dict.BinaryDictionary;-import org.apache.lucene.codecs.CodecUtil;-import org.apache.lucene.store.DataOutput;-import org.apache.lucene.store.OutputStreamDataOutput;-import org.apache.lucene.util.ArrayUtil;--abstract class BinaryDictionaryWriter {- private static final int ID_LIMIT = 8192;-- private final Class<? extends BinaryDictionary> implClazz;- protected ByteBuffer buffer;- private int targetMapEndOffset = 0, lastWordId = -1, lastSourceId = -1;- private int[] targetMap = new int[8192];- private int[] targetMapOffsets = new int[8192];- private final ArrayList<String> posDict = new ArrayList<>();-- BinaryDictionaryWriter(Class<? extends BinaryDictionary> implClazz, int size) {- this.implClazz = implClazz;- buffer = ByteBuffer.allocateDirect(size);- }-- /**- * put the entry in map- *- * @return current position of buffer, which will be wordId of next entry- */- public int put(String[] entry) {- short leftId = Short.parseShort(entry[1]);- short rightId = Short.parseShort(entry[2]);- short wordCost = Short.parseShort(entry[3]);-- StringBuilder sb = new StringBuilder();-- // build up the POS string- for (int i = 4; i < 8; i++) {- String part = entry[i];- assert part.length() > 0;- if (!"*".equals(part)) {- if (sb.length() > 0) {- sb.append('-');- }- sb.append(part);- }- }-- String posData = sb.toString();- if (posData.isEmpty()) {- throw new IllegalArgumentException("POS fields are empty");- }- sb.setLength(0);- sb.append(CSVUtil.quoteEscape(posData));- sb.append(',');- if (!"*".equals(entry[8])) {- sb.append(CSVUtil.quoteEscape(entry[8]));- }- sb.append(',');- if (!"*".equals(entry[9])) {- sb.append(CSVUtil.quoteEscape(entry[9]));- }- String fullPOSData = sb.toString();-- String baseForm = entry[10];- String reading = entry[11];- String pronunciation = entry[12];-- // extend buffer if necessary- int left = buffer.remaining();- // worst case: two short, 3 bytes, and features (all as utf-16)- int worstCase = 4 + 3 + 2 * (baseForm.length() + reading.length() + pronunciation.length());- if (worstCase > left) {- ByteBuffer newBuffer =- ByteBuffer.allocateDirect(ArrayUtil.oversize(buffer.limit() + worstCase - left, 1));- buffer.flip();- newBuffer.put(buffer);- buffer = newBuffer;- }-- int flags = 0;- if (baseForm.isEmpty()) {- throw new IllegalArgumentException("base form is empty");- }- if (!("*".equals(baseForm) || baseForm.equals(entry[0]))) {- flags |= BinaryDictionary.HAS_BASEFORM;- }- if (!reading.equals(toKatakana(entry[0]))) {- flags |= BinaryDictionary.HAS_READING;- }- if (!pronunciation.equals(reading)) {- flags |= BinaryDictionary.HAS_PRONUNCIATION;- }-- if (leftId != rightId) {- throw new IllegalArgumentException("rightId != leftId: " + rightId + " " + leftId);- }- if (leftId >= ID_LIMIT) {- throw new IllegalArgumentException("leftId >= " + ID_LIMIT + ": " + leftId);- }- // add pos mapping- int toFill = 1 + leftId - posDict.size();- for (int i = 0; i < toFill; i++) {- posDict.add(null);- }-- String existing = posDict.get(leftId);- if (existing != null && existing.equals(fullPOSData) == false) {- // TODO: test me- throw new IllegalArgumentException("Multiple entries found for leftID=" + leftId);- }- posDict.set(leftId, fullPOSData);-- buffer.putShort((short) (leftId << 3 | flags));- buffer.putShort(wordCost);-- if ((flags & BinaryDictionary.HAS_BASEFORM) != 0) {- if (baseForm.length() >= 16) {- throw new IllegalArgumentException("Length of base form " + baseForm + " is >= 16");- }- int shared = sharedPrefix(entry[0], baseForm);- int suffix = baseForm.length() - shared;- buffer.put((byte) (shared << 4 | suffix));- for (int i = shared; i < baseForm.length(); i++) {- buffer.putChar(baseForm.charAt(i));- }- }-- if ((flags & BinaryDictionary.HAS_READING) != 0) {- if (isKatakana(reading)) {- buffer.put((byte) (reading.length() << 1 | 1));- writeKatakana(reading);- } else {- buffer.put((byte) (reading.length() << 1));- for (int i = 0; i < reading.length(); i++) {- buffer.putChar(reading.charAt(i));- }- }- }-- if ((flags & BinaryDictionary.HAS_PRONUNCIATION) != 0) {- // we can save 150KB here, but it makes the reader a little complicated.- // int shared = sharedPrefix(reading, pronunciation);- // buffer.put((byte) shared);- // pronunciation = pronunciation.substring(shared);- if (isKatakana(pronunciation)) {- buffer.put((byte) (pronunciation.length() << 1 | 1));- writeKatakana(pronunciation);- } else {- buffer.put((byte) (pronunciation.length() << 1));- for (int i = 0; i < pronunciation.length(); i++) {- buffer.putChar(pronunciation.charAt(i));- }- }- }-- return buffer.position();- }-- private boolean isKatakana(String s) {- for (int i = 0; i < s.length(); i++) {- char ch = s.charAt(i);- if (ch < 0x30A0 || ch > 0x30FF) {- return false;- }- }- return true;- }-- private void writeKatakana(String s) {- for (int i = 0; i < s.length(); i++) {- buffer.put((byte) (s.charAt(i) - 0x30A0));- }- }-- private String toKatakana(String s) {- char[] text = new char[s.length()];- for (int i = 0; i < s.length(); i++) {- char ch = s.charAt(i);- if (ch > 0x3040 && ch < 0x3097) {- text[i] = (char) (ch + 0x60);- } else {- text[i] = ch;- }- }- return new String(text);- }-- private static int sharedPrefix(String left, String right) {- int len = left.length() < right.length() ? left.length() : right.length();- for (int i = 0; i < len; i++) if (left.charAt(i) != right.charAt(i)) return i;- return len;- }-- void addMapping(int sourceId, int wordId) {- if (wordId <= lastWordId) {- throw new IllegalStateException(- "words out of order: " + wordId + " vs lastID: " + lastWordId);- }-- if (sourceId > lastSourceId) {- targetMapOffsets = ArrayUtil.grow(targetMapOffsets, sourceId + 1);- for (int i = lastSourceId + 1; i <= sourceId; i++) {- targetMapOffsets[i] = targetMapEndOffset;- }- } else if (sourceId != lastSourceId) {- throw new IllegalStateException(- "source ids not in increasing order: lastSourceId="- + lastSourceId- + " vs sourceId="- + sourceId);- }-- targetMap = ArrayUtil.grow(targetMap, targetMapEndOffset + 1);- targetMap[targetMapEndOffset] = wordId;- targetMapEndOffset++;-- lastSourceId = sourceId;- lastWordId = wordId;- }-- final String getBaseFileName() {- return implClazz.getName().replace('.', '/');- }-- /**- * Write dictionary in file Dictionary format is: [Size of dictionary(int)], [entry:{left- * id(short)}{right id(short)}{word cost(short)}{length of pos info(short)}{pos info(char)}],- * [entry...], [entry...].....- *- * @throws IOException if an I/O error occurs writing the dictionary files- */- public void write(Path baseDir) throws IOException {- final String baseName = getBaseFileName();- writeDictionary(baseDir.resolve(baseName + BinaryDictionary.DICT_FILENAME_SUFFIX));- writeTargetMap(baseDir.resolve(baseName + BinaryDictionary.TARGETMAP_FILENAME_SUFFIX));- writePosDict(baseDir.resolve(baseName + BinaryDictionary.POSDICT_FILENAME_SUFFIX));- }-- // TODO: maybe this int[] should instead be the output to the FST...- private void writeTargetMap(Path path) throws IOException {- Files.createDirectories(path.getParent());- try (OutputStream os = Files.newOutputStream(path);- OutputStream bos = new BufferedOutputStream(os)) {- final DataOutput out = new OutputStreamDataOutput(bos);- CodecUtil.writeHeader(out, BinaryDictionary.TARGETMAP_HEADER, BinaryDictionary.VERSION);-- final int numSourceIds = lastSourceId + 1;- out.writeVInt(targetMapEndOffset); // <-- size of main array- out.writeVInt(numSourceIds + 1); // <-- size of offset array (+ 1 more entry)- int prev = 0, sourceId = 0;- for (int ofs = 0; ofs < targetMapEndOffset; ofs++) {- final int val = targetMap[ofs], delta = val - prev;- assert delta >= 0;- if (ofs == targetMapOffsets[sourceId]) {- out.writeVInt((delta << 1) | 0x01);- sourceId++;- } else {- out.writeVInt((delta << 1));- }- prev += delta;- }- if (sourceId != numSourceIds) {- throw new IllegalStateException(- "sourceId:" + sourceId + " != numSourceIds:" + numSourceIds);- }- }- }-- private void writePosDict(Path path) throws IOException {- Files.createDirectories(path.getParent());- try (OutputStream os = Files.newOutputStream(path);- OutputStream bos = new BufferedOutputStream(os)) {- final DataOutput out = new OutputStreamDataOutput(bos);- CodecUtil.writeHeader(out, BinaryDictionary.POSDICT_HEADER, BinaryDictionary.VERSION);- out.writeVInt(posDict.size());- for (String s : posDict) {- if (s == null) {- out.writeByte((byte) 0);- out.writeByte((byte) 0);- out.writeByte((byte) 0);- } else {- String[] data = CSVUtil.parse(s);- if (data.length != 3) {- throw new IllegalArgumentException(- "Malformed pos/inflection: " + s + "; expected 3 characters");- }- out.writeString(data[0]);- out.writeString(data[1]);- out.writeString(data[2]);- }- }- }- }-- private void writeDictionary(Path path) throws IOException {- Files.createDirectories(path.getParent());- try (OutputStream os = Files.newOutputStream(path);- OutputStream bos = new BufferedOutputStream(os)) {- final DataOutput out = new OutputStreamDataOutput(bos);- CodecUtil.writeHeader(out, BinaryDictionary.DICT_HEADER, BinaryDictionary.VERSION);- out.writeVInt(buffer.position());- final WritableByteChannel channel = Channels.newChannel(bos);- // Write Buffer- buffer.flip(); // set position to 0, set limit to current position- channel.write(buffer);- assert buffer.remaining() == 0L;- }- }-}diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/util/CharacterDefinitionWriter.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/util/CharacterDefinitionWriter.javadeleted file mode 100644index 0afadeda83d..00000000000--- a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/util/CharacterDefinitionWriter.java+++ /dev/null@@ -1,83 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.analysis.ja.util;--import java.io.BufferedOutputStream;-import java.io.IOException;-import java.io.OutputStream;-import java.nio.file.Files;-import java.nio.file.Path;-import java.util.Arrays;-import org.apache.lucene.analysis.ja.dict.CharacterDefinition;-import org.apache.lucene.codecs.CodecUtil;-import org.apache.lucene.store.DataOutput;-import org.apache.lucene.store.OutputStreamDataOutput;--final class CharacterDefinitionWriter {-- private final byte[] characterCategoryMap = new byte[0x10000];-- private final boolean[] invokeMap = new boolean[CharacterDefinition.CLASS_COUNT];- private final boolean[] groupMap = new boolean[CharacterDefinition.CLASS_COUNT];-- /** Constructor for building. TODO: remove write access */- CharacterDefinitionWriter() {- Arrays.fill(characterCategoryMap, CharacterDefinition.DEFAULT);- }-- /**- * Put mapping from unicode code point to character class.- *- * @param codePoint code point- * @param characterClassName character class name- */- void putCharacterCategory(int codePoint, String characterClassName) {- characterClassName = characterClassName.split(" ")[0]; // use first- // category- // class-- // Override Nakaguro- if (codePoint == 0x30FB) {- characterClassName = "SYMBOL";- }- characterCategoryMap[codePoint] = CharacterDefinition.lookupCharacterClass(characterClassName);- }-- void putInvokeDefinition(String characterClassName, int invoke, int group, int length) {- final byte characterClass = CharacterDefinition.lookupCharacterClass(characterClassName);- invokeMap[characterClass] = invoke == 1;- groupMap[characterClass] = group == 1;- // TODO: length def ignored- }-- public void write(Path baseDir) throws IOException {- Path path =- baseDir.resolve(- CharacterDefinition.class.getName().replace('.', '/')- + CharacterDefinition.FILENAME_SUFFIX);- Files.createDirectories(path.getParent());- try (OutputStream os = new BufferedOutputStream(Files.newOutputStream(path))) {- final DataOutput out = new OutputStreamDataOutput(os);- CodecUtil.writeHeader(out, CharacterDefinition.HEADER, CharacterDefinition.VERSION);- out.writeBytes(characterCategoryMap, 0, characterCategoryMap.length);- for (int i = 0; i < CharacterDefinition.CLASS_COUNT; i++) {- final byte b = (byte) ((invokeMap[i] ? 0x01 : 0x00) | (groupMap[i] ? 0x02 : 0x00));- out.writeByte(b);- }- }- }-}diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/util/ConnectionCostsBuilder.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/util/ConnectionCostsBuilder.javaindex 8b518ee991a..bfd8dd0d17c 100644--- a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/util/ConnectionCostsBuilder.java+++ b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/util/ConnectionCostsBuilder.java@@ -22,12 +22,14 @@ import java.io.Reader;import java.nio.charset.StandardCharsets;import java.nio.file.Files;import java.nio.file.Path;+import org.apache.lucene.analysis.ja.dict.ConnectionCosts;+import org.apache.lucene.analysis.morph.ConnectionCostsWriter;class ConnectionCostsBuilder {private ConnectionCostsBuilder() {}- public static ConnectionCostsWriter build(Path path) throws IOException {+ public static ConnectionCostsWriter<ConnectionCosts> build(Path path) throws IOException {try (Reader reader = Files.newBufferedReader(path, StandardCharsets.US_ASCII);LineNumberReader lineReader = new LineNumberReader(reader)) {@@ -41,7 +43,8 @@ class ConnectionCostsBuilder {assert forwardSize > 0 && backwardSize > 0;- ConnectionCostsWriter costs = new ConnectionCostsWriter(forwardSize, backwardSize);+ ConnectionCostsWriter<ConnectionCosts> costs =+ new ConnectionCostsWriter<>(ConnectionCosts.class, forwardSize, backwardSize);while ((line = lineReader.readLine()) != null) {String[] fields = line.split("\\s+");diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/util/DictionaryBuilder.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/util/DictionaryBuilder.javaindex 72920baf984..d547dcb122d 100644--- a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/util/DictionaryBuilder.java+++ b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/util/DictionaryBuilder.java@@ -20,6 +20,7 @@ import java.io.IOException;import java.nio.file.Path;import java.nio.file.Paths;import java.util.Locale;+import org.apache.lucene.analysis.ja.dict.DictionaryConstants;/*** Tool to build dictionaries. Usage:@@ -68,7 +69,8 @@ public class DictionaryBuilder {new UnknownDictionaryBuilder(encoding).build(inputDir).write(outputDir);- ConnectionCostsBuilder.build(inputDir.resolve("matrix.def")).write(outputDir);+ ConnectionCostsBuilder.build(inputDir.resolve("matrix.def"))+ .write(outputDir, DictionaryConstants.CONN_COSTS_HEADER, DictionaryConstants.VERSION);}public static void main(String[] args) throws IOException {diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/util/TokenInfoDictionaryEntryWriter.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/util/TokenInfoDictionaryEntryWriter.javanew file mode 100644index 00000000000..618047825f2--- /dev/null+++ b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/util/TokenInfoDictionaryEntryWriter.java@@ -0,0 +1,237 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.analysis.ja.util;++import java.io.IOException;+import java.io.OutputStream;+import java.nio.ByteBuffer;+import org.apache.lucene.analysis.ja.dict.TokenInfoMorphData;+import org.apache.lucene.analysis.morph.DictionaryEntryWriter;+import org.apache.lucene.store.DataOutput;+import org.apache.lucene.util.ArrayUtil;++/** Writes system dictionary entries */+class TokenInfoDictionaryEntryWriter extends DictionaryEntryWriter {+ private static final int ID_LIMIT = 8192;++ TokenInfoDictionaryEntryWriter(int size) {+ super(size);+ }++ /**+ * put the entry in map+ *+ * <p>mecab-ipadic features+ *+ * <pre>+ * 0 - surface+ * 1 - left cost+ * 2 - right cost+ * 3 - word cost+ * 4-9 - pos+ * 10 - base form+ * 11 - reading+ * 12 - pronounciation+ * </pre>+ */+ @Override+ protected int putEntry(String[] entry) {+ short leftId = Short.parseShort(entry[1]);+ short rightId = Short.parseShort(entry[2]);+ short wordCost = Short.parseShort(entry[3]);++ StringBuilder sb = new StringBuilder();++ // build up the POS string+ for (int i = 4; i < 8; i++) {+ String part = entry[i];+ assert part.length() > 0;+ if (!"*".equals(part)) {+ if (sb.length() > 0) {+ sb.append('-');+ }+ sb.append(part);+ }+ }++ String posData = sb.toString();+ if (posData.isEmpty()) {+ throw new IllegalArgumentException("POS fields are empty");+ }+ sb.setLength(0);+ sb.append(CSVUtil.quoteEscape(posData));+ sb.append(',');+ if (!"*".equals(entry[8])) {+ sb.append(CSVUtil.quoteEscape(entry[8]));+ }+ sb.append(',');+ if (!"*".equals(entry[9])) {+ sb.append(CSVUtil.quoteEscape(entry[9]));+ }+ String fullPOSData = sb.toString();++ String baseForm = entry[10];+ String reading = entry[11];+ String pronunciation = entry[12];++ // extend buffer if necessary+ int left = buffer.remaining();+ // worst case: two short, 3 bytes, and features (all as utf-16)+ int worstCase = 4 + 3 + 2 * (baseForm.length() + reading.length() + pronunciation.length());+ if (worstCase > left) {+ ByteBuffer newBuffer =+ ByteBuffer.allocateDirect(ArrayUtil.oversize(buffer.limit() + worstCase - left, 1));+ buffer.flip();+ newBuffer.put(buffer);+ buffer = newBuffer;+ }++ int flags = 0;+ if (baseForm.isEmpty()) {+ throw new IllegalArgumentException("base form is empty");+ }+ if (!("*".equals(baseForm) || baseForm.equals(entry[0]))) {+ flags |= TokenInfoMorphData.HAS_BASEFORM;+ }+ if (!reading.equals(toKatakana(entry[0]))) {+ flags |= TokenInfoMorphData.HAS_READING;+ }+ if (!pronunciation.equals(reading)) {+ flags |= TokenInfoMorphData.HAS_PRONUNCIATION;+ }++ if (leftId != rightId) {+ throw new IllegalArgumentException("rightId != leftId: " + rightId + " " + leftId);+ }+ if (leftId >= ID_LIMIT) {+ throw new IllegalArgumentException("leftId >= " + ID_LIMIT + ": " + leftId);+ }+ // add pos mapping+ int toFill = 1 + leftId - posDict.size();+ for (int i = 0; i < toFill; i++) {+ posDict.add(null);+ }++ String existing = posDict.get(leftId);+ if (existing != null && existing.equals(fullPOSData) == false) {+ // TODO: test me+ throw new IllegalArgumentException("Multiple entries found for leftID=" + leftId);+ }+ posDict.set(leftId, fullPOSData);++ buffer.putShort((short) (leftId << 3 | flags));+ buffer.putShort(wordCost);++ if ((flags & TokenInfoMorphData.HAS_BASEFORM) != 0) {+ if (baseForm.length() >= 16) {+ throw new IllegalArgumentException("Length of base form " + baseForm + " is >= 16");+ }+ int shared = sharedPrefix(entry[0], baseForm);+ int suffix = baseForm.length() - shared;+ buffer.put((byte) (shared << 4 | suffix));+ for (int i = shared; i < baseForm.length(); i++) {+ buffer.putChar(baseForm.charAt(i));+ }+ }++ if ((flags & TokenInfoMorphData.HAS_READING) != 0) {+ if (isKatakana(reading)) {+ buffer.put((byte) (reading.length() << 1 | 1));+ writeKatakana(reading, buffer);+ } else {+ buffer.put((byte) (reading.length() << 1));+ for (int i = 0; i < reading.length(); i++) {+ buffer.putChar(reading.charAt(i));+ }+ }+ }++ if ((flags & TokenInfoMorphData.HAS_PRONUNCIATION) != 0) {+ // we can save 150KB here, but it makes the reader a little complicated.+ // int shared = sharedPrefix(reading, pronunciation);+ // buffer.put((byte) shared);+ // pronunciation = pronunciation.substring(shared);+ if (isKatakana(pronunciation)) {+ buffer.put((byte) (pronunciation.length() << 1 | 1));+ writeKatakana(pronunciation, buffer);+ } else {+ buffer.put((byte) (pronunciation.length() << 1));+ for (int i = 0; i < pronunciation.length(); i++) {+ buffer.putChar(pronunciation.charAt(i));+ }+ }+ }++ return buffer.position();+ }++ private boolean isKatakana(String s) {+ for (int i = 0; i < s.length(); i++) {+ char ch = s.charAt(i);+ if (ch < 0x30A0 || ch > 0x30FF) {+ return false;+ }+ }+ return true;+ }++ private void writeKatakana(String s, ByteBuffer buffer) {+ for (int i = 0; i < s.length(); i++) {+ buffer.put((byte) (s.charAt(i) - 0x30A0));+ }+ }++ private String toKatakana(String s) {+ char[] text = new char[s.length()];+ for (int i = 0; i < s.length(); i++) {+ char ch = s.charAt(i);+ if (ch > 0x3040 && ch < 0x3097) {+ text[i] = (char) (ch + 0x60);+ } else {+ text[i] = ch;+ }+ }+ return new String(text);+ }++ private static int sharedPrefix(String left, String right) {+ int len = left.length() < right.length() ? left.length() : right.length();+ for (int i = 0; i < len; i++) if (left.charAt(i) != right.charAt(i)) return i;+ return len;+ }++ @Override+ protected void writePosDict(OutputStream bos, DataOutput out) throws IOException {+ out.writeVInt(posDict.size());+ for (String s : posDict) {+ if (s == null) {+ out.writeByte((byte) 0);+ out.writeByte((byte) 0);+ out.writeByte((byte) 0);+ } else {+ String[] data = CSVUtil.parse(s);+ if (data.length != 3) {+ throw new IllegalArgumentException(+ "Malformed pos/inflection: " + s + "; expected 3 characters");+ }+ out.writeString(data[0]);+ out.writeString(data[1]);+ out.writeString(data[2]);+ }+ }+ }+}diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/util/TokenInfoDictionaryWriter.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/util/TokenInfoDictionaryWriter.javaindex 400c834cadf..fbdf65f34ce 100644--- a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/util/TokenInfoDictionaryWriter.java+++ b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/util/TokenInfoDictionaryWriter.java@@ -20,14 +20,16 @@ import java.io.IOException;import java.nio.file.Files;import java.nio.file.Path;import java.util.Objects;+import org.apache.lucene.analysis.ja.dict.DictionaryConstants;import org.apache.lucene.analysis.ja.dict.TokenInfoDictionary;import org.apache.lucene.util.fst.FST;-class TokenInfoDictionaryWriter extends BinaryDictionaryWriter {+class TokenInfoDictionaryWriter+ extends org.apache.lucene.analysis.morph.BinaryDictionaryWriter<TokenInfoDictionary> {private FST<Long> fst;TokenInfoDictionaryWriter(int size) {- super(TokenInfoDictionary.class, size);+ super(TokenInfoDictionary.class, new TokenInfoDictionaryEntryWriter(size));}public void setFST(FST<Long> fst) {@@ -35,9 +37,19 @@ class TokenInfoDictionaryWriter extends BinaryDictionaryWriter {this.fst = fst;}+ @Override+ protected void addMapping(int sourceId, int wordId) {+ super.addMapping(sourceId, wordId);+ }+@Overridepublic void write(Path baseDir) throws IOException {- super.write(baseDir);+ super.write(+ baseDir,+ DictionaryConstants.TARGETMAP_HEADER,+ DictionaryConstants.POSDICT_HEADER,+ DictionaryConstants.DICT_HEADER,+ DictionaryConstants.VERSION);writeFST(baseDir.resolve(getBaseFileName() + TokenInfoDictionary.FST_FILENAME_SUFFIX));}diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/util/UnknownDictionaryWriter.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/util/UnknownDictionaryWriter.javaindex 6d80f513fec..84000fdf0de 100644--- a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/util/UnknownDictionaryWriter.java+++ b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/util/UnknownDictionaryWriter.java@@ -19,19 +19,28 @@ package org.apache.lucene.analysis.ja.util;import java.io.IOException;import java.nio.file.Path;import org.apache.lucene.analysis.ja.dict.CharacterDefinition;+import org.apache.lucene.analysis.ja.dict.DictionaryConstants;import org.apache.lucene.analysis.ja.dict.UnknownDictionary;+import org.apache.lucene.analysis.morph.BinaryDictionaryWriter;+import org.apache.lucene.analysis.morph.CharacterDefinitionWriter;-class UnknownDictionaryWriter extends BinaryDictionaryWriter {- private final CharacterDefinitionWriter characterDefinition = new CharacterDefinitionWriter();+class UnknownDictionaryWriter extends BinaryDictionaryWriter<UnknownDictionary> {+ private final CharacterDefinitionWriter<CharacterDefinition> characterDefinition =+ new CharacterDefinitionWriter<>(+ CharacterDefinition.class,+ CharacterDefinition.DEFAULT,+ CharacterDefinition.CLASS_COUNT,+ CharacterDefinition::lookupCharacterClass);public UnknownDictionaryWriter(int size) {- super(UnknownDictionary.class, size);+ super(UnknownDictionary.class, new TokenInfoDictionaryEntryWriter(size));}@Overridepublic int put(String[] entry) {// Get wordId of current entry- int wordId = buffer.position();+ // int wordId = buffer.position();+ int wordId = entryWriter.currentPosition();// Put entryint result = super.put(entry);@@ -58,7 +67,13 @@ class UnknownDictionaryWriter extends BinaryDictionaryWriter {@Overridepublic void write(Path baseDir) throws IOException {- super.write(baseDir);- characterDefinition.write(baseDir);+ super.write(+ baseDir,+ DictionaryConstants.TARGETMAP_HEADER,+ DictionaryConstants.POSDICT_HEADER,+ DictionaryConstants.DICT_HEADER,+ DictionaryConstants.VERSION);+ characterDefinition.write(+ baseDir, DictionaryConstants.CHARDEF_HEADER, DictionaryConstants.VERSION);}}diff --git a/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/dict/TestExternalDictionary.java b/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/dict/TestExternalDictionary.javaindex bc44723996a..5423d7a76cb 100644--- a/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/dict/TestExternalDictionary.java+++ b/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/dict/TestExternalDictionary.java@@ -16,10 +16,10 @@*/package org.apache.lucene.analysis.ja.dict;-import static org.apache.lucene.analysis.ja.dict.BinaryDictionary.DICT_FILENAME_SUFFIX;-import static org.apache.lucene.analysis.ja.dict.BinaryDictionary.POSDICT_FILENAME_SUFFIX;-import static org.apache.lucene.analysis.ja.dict.BinaryDictionary.TARGETMAP_FILENAME_SUFFIX;import static org.apache.lucene.analysis.ja.dict.TokenInfoDictionary.FST_FILENAME_SUFFIX;+import static org.apache.lucene.analysis.morph.BinaryDictionary.DICT_FILENAME_SUFFIX;+import static org.apache.lucene.analysis.morph.BinaryDictionary.POSDICT_FILENAME_SUFFIX;+import static org.apache.lucene.analysis.morph.BinaryDictionary.TARGETMAP_FILENAME_SUFFIX;import java.io.BufferedWriter;import java.nio.charset.StandardCharsets;diff --git a/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/dict/TestTokenInfoDictionary.java b/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/dict/TestTokenInfoDictionary.javaindex 11de60c05fb..0320d6e5f13 100644--- a/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/dict/TestTokenInfoDictionary.java+++ b/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/dict/TestTokenInfoDictionary.java@@ -16,10 +16,10 @@*/package org.apache.lucene.analysis.ja.dict;-import static org.apache.lucene.analysis.ja.dict.BinaryDictionary.DICT_FILENAME_SUFFIX;-import static org.apache.lucene.analysis.ja.dict.BinaryDictionary.POSDICT_FILENAME_SUFFIX;-import static org.apache.lucene.analysis.ja.dict.BinaryDictionary.TARGETMAP_FILENAME_SUFFIX;import static org.apache.lucene.analysis.ja.dict.TokenInfoDictionary.FST_FILENAME_SUFFIX;+import static org.apache.lucene.analysis.morph.BinaryDictionary.DICT_FILENAME_SUFFIX;+import static org.apache.lucene.analysis.morph.BinaryDictionary.POSDICT_FILENAME_SUFFIX;+import static org.apache.lucene.analysis.morph.BinaryDictionary.TARGETMAP_FILENAME_SUFFIX;import java.io.OutputStream;import java.io.OutputStreamWriter;@@ -137,17 +137,17 @@ public class TestTokenInfoDictionary extends LuceneTestCase {assertTrue(wordId > lastWordId);lastWordId = wordId;- String baseForm = tid.getBaseForm(wordId, chars, 0, chars.length);+ String baseForm = tid.getMorphAttributes().getBaseForm(wordId, chars, 0, chars.length);assertTrue(baseForm == null || UnicodeUtil.validUTF16String(baseForm));- String inflectionForm = tid.getInflectionForm(wordId);+ String inflectionForm = tid.getMorphAttributes().getInflectionForm(wordId);assertTrue(inflectionForm == null || UnicodeUtil.validUTF16String(inflectionForm));if (inflectionForm != null) {// check that it's actually an ipadic inflection formassertNotNull(ToStringUtil.getInflectedFormTranslation(inflectionForm));}- String inflectionType = tid.getInflectionType(wordId);+ String inflectionType = tid.getMorphAttributes().getInflectionType(wordId);assertTrue(inflectionType == null || UnicodeUtil.validUTF16String(inflectionType));if (inflectionType != null) {// check that it's actually an ipadic inflection type@@ -161,17 +161,18 @@ public class TestTokenInfoDictionary extends LuceneTestCase {tid.getWordCost(wordId);- String pos = tid.getPartOfSpeech(wordId);+ String pos = tid.getMorphAttributes().getPartOfSpeech(wordId);assertNotNull(pos);assertTrue(UnicodeUtil.validUTF16String(pos));// check that it's actually an ipadic pos tagassertNotNull(ToStringUtil.getPOSTranslation(pos));- String pronunciation = tid.getPronunciation(wordId, chars, 0, chars.length);+ String pronunciation =+ tid.getMorphAttributes().getPronunciation(wordId, chars, 0, chars.length);assertNotNull(pronunciation);assertTrue(UnicodeUtil.validUTF16String(pronunciation));- String reading = tid.getReading(wordId, chars, 0, chars.length);+ String reading = tid.getMorphAttributes().getReading(wordId, chars, 0, chars.length);assertNotNull(reading);assertTrue(UnicodeUtil.validUTF16String(reading));}diff --git a/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/dict/TestUserDictionary.java b/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/dict/TestUserDictionary.javaindex b08d75344f0..537d621016e 100644--- a/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/dict/TestUserDictionary.java+++ b/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/dict/TestUserDictionary.java@@ -54,12 +54,15 @@ public class TestUserDictionary extends LuceneTestCase {int[][] result = dictionary.lookup("日本経済新聞".toCharArray(), 0, 6);assertEquals(3, result.length);int wordIdNihon = result[0][0]; // wordId of 日本 in 日本経済新聞- assertEquals("ニホン", dictionary.getReading(wordIdNihon, "日本".toCharArray(), 0, 2));+ assertEquals(+ "ニホン", dictionary.getMorphAttributes().getReading(wordIdNihon, "日本".toCharArray(), 0, 2));result = dictionary.lookup("朝青龍".toCharArray(), 0, 3);assertEquals(1, result.length);int wordIdAsashoryu = result[0][0]; // wordId for 朝青龍- assertEquals("アサショウリュウ", dictionary.getReading(wordIdAsashoryu, "朝青龍".toCharArray(), 0, 3));+ assertEquals(+ "アサショウリュウ",+ dictionary.getMorphAttributes().getReading(wordIdAsashoryu, "朝青龍".toCharArray(), 0, 3));}@Test@@ -68,7 +71,7 @@ public class TestUserDictionary extends LuceneTestCase {int[][] result = dictionary.lookup("日本経済新聞".toCharArray(), 0, 6);assertEquals(3, result.length);int wordIdKeizai = result[1][0]; // wordId of 経済 in 日本経済新聞- assertEquals("カスタム名詞", dictionary.getPartOfSpeech(wordIdKeizai));+ assertEquals("カスタム名詞", dictionary.getMorphAttributes().getPartOfSpeech(wordIdKeizai));}@Test@@ -109,7 +112,7 @@ public class TestUserDictionary extends LuceneTestCase {for (String input : inputs) {System.out.println(input);int[][] result = dictionary.lookup(input.toCharArray(), 0, input.length());- assertEquals("カスタム名刺", dictionary.getPartOfSpeech(result[0][0]));+ assertEquals("カスタム名刺", dictionary.getMorphAttributes().getPartOfSpeech(result[0][0]));}}}diff --git a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/DecompoundToken.java b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/DecompoundToken.javaindex 731378c9c4a..a6af55f482e 100644--- a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/DecompoundToken.java+++ b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/DecompoundToken.java@@ -16,7 +16,7 @@*/package org.apache.lucene.analysis.ko;-import org.apache.lucene.analysis.ko.dict.Dictionary;+import org.apache.lucene.analysis.ko.dict.KoMorphData;/** A token that was generated from a compound. */public class DecompoundToken extends Token {@@ -71,7 +71,7 @@ public class DecompoundToken extends Token {}@Override- public Dictionary.Morpheme[] getMorphemes() {+ public KoMorphData.Morpheme[] getMorphemes() {return null;}}diff --git a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/DictionaryToken.java b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/DictionaryToken.javaindex 82fe981c24d..f548f8007d3 100644--- a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/DictionaryToken.java+++ b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/DictionaryToken.java@@ -16,17 +16,17 @@*/package org.apache.lucene.analysis.ko;-import org.apache.lucene.analysis.ko.dict.Dictionary;+import org.apache.lucene.analysis.ko.dict.KoMorphData;-/** A token stored in a {@link Dictionary}. */+/** A token stored in a {@link KoMorphData}. */public class DictionaryToken extends Token {private final int wordId;private final KoreanTokenizer.Type type;- private final Dictionary dictionary;+ private final KoMorphData morphAtts;public DictionaryToken(KoreanTokenizer.Type type,- Dictionary dictionary,+ KoMorphData morphAtts,int wordId,char[] surfaceForm,int offset,@@ -35,7 +35,7 @@ public class DictionaryToken extends Token {int endOffset) {super(surfaceForm, offset, length, startOffset, endOffset);this.type = type;- this.dictionary = dictionary;+ this.morphAtts = morphAtts;this.wordId = wordId;}@@ -54,7 +54,7 @@ public class DictionaryToken extends Token {+ " wordId="+ wordId+ " leftID="- + dictionary.getLeftId(wordId)+ + morphAtts.getLeftId(wordId)+ ")";}@@ -96,26 +96,26 @@ public class DictionaryToken extends Token {@Overridepublic POS.Type getPOSType() {- return dictionary.getPOSType(wordId);+ return morphAtts.getPOSType(wordId);}@Overridepublic POS.Tag getLeftPOS() {- return dictionary.getLeftPOS(wordId);+ return morphAtts.getLeftPOS(wordId);}@Overridepublic POS.Tag getRightPOS() {- return dictionary.getRightPOS(wordId);+ return morphAtts.getRightPOS(wordId);}@Overridepublic String getReading() {- return dictionary.getReading(wordId);+ return morphAtts.getReading(wordId);}@Override- public Dictionary.Morpheme[] getMorphemes() {- return dictionary.getMorphemes(wordId, getSurfaceForm(), getOffset(), getLength());+ public KoMorphData.Morpheme[] getMorphemes() {+ return morphAtts.getMorphemes(wordId, getSurfaceForm(), getOffset(), getLength());}}diff --git a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/GraphvizFormatter.java b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/GraphvizFormatter.javaindex d8e3f7a9faf..9beaf2e5b9b 100644--- a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/GraphvizFormatter.java+++ b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/GraphvizFormatter.java@@ -21,7 +21,8 @@ import java.util.Map;import org.apache.lucene.analysis.ko.KoreanTokenizer.Position;import org.apache.lucene.analysis.ko.KoreanTokenizer.WrappedPositionArray;import org.apache.lucene.analysis.ko.dict.ConnectionCosts;-import org.apache.lucene.analysis.ko.dict.Dictionary;+import org.apache.lucene.analysis.ko.dict.KoMorphData;+import org.apache.lucene.analysis.morph.Dictionary;// TODO: would be nice to show 2nd best path in a diff't// color...@@ -140,7 +141,7 @@ public class GraphvizFormatter {attrs = "";}- final Dictionary dict = tok.getDict(posData.backType[idx]);+ final Dictionary<? extends KoMorphData> dict = tok.getDict(posData.backType[idx]);final int wordCost = dict.getWordCost(posData.backID[idx]);final int bgCost =costs.get(diff --git a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/KoreanTokenizer.java b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/KoreanTokenizer.javaindex 325fae710b9..028d9dd54d1 100644--- a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/KoreanTokenizer.java+++ b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/KoreanTokenizer.java@@ -26,13 +26,14 @@ import java.util.List;import org.apache.lucene.analysis.Tokenizer;import org.apache.lucene.analysis.ko.dict.CharacterDefinition;import org.apache.lucene.analysis.ko.dict.ConnectionCosts;-import org.apache.lucene.analysis.ko.dict.Dictionary;+import org.apache.lucene.analysis.ko.dict.KoMorphData;import org.apache.lucene.analysis.ko.dict.TokenInfoDictionary;import org.apache.lucene.analysis.ko.dict.TokenInfoFST;import org.apache.lucene.analysis.ko.dict.UnknownDictionary;import org.apache.lucene.analysis.ko.dict.UserDictionary;import org.apache.lucene.analysis.ko.tokenattributes.PartOfSpeechAttribute;import org.apache.lucene.analysis.ko.tokenattributes.ReadingAttribute;+import org.apache.lucene.analysis.morph.Dictionary;import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;@@ -97,7 +98,8 @@ public final class KoreanTokenizer extends Tokenizer {private static final int MAX_UNKNOWN_WORD_LENGTH = 1024;private static final int MAX_BACKTRACE_GAP = 1024;- private final EnumMap<Type, Dictionary> dictionaryMap = new EnumMap<>(Type.class);+ private final EnumMap<Type, Dictionary<? extends KoMorphData>> dictionaryMap =+ new EnumMap<>(Type.class);private final TokenInfoFST fst;private final TokenInfoDictionary dictionary;@@ -406,10 +408,10 @@ public final class KoreanTokenizer extends Tokenizer {}private void add(- Dictionary dict, Position fromPosData, int wordPos, int endPos, int wordID, Type type) {- final POS.Tag leftPOS = dict.getLeftPOS(wordID);- final int wordCost = dict.getWordCost(wordID);- final int leftID = dict.getLeftId(wordID);+ KoMorphData morphAtts, Position fromPosData, int wordPos, int endPos, int wordID, Type type) {+ final POS.Tag leftPOS = morphAtts.getLeftPOS(wordID);+ final int wordCost = morphAtts.getWordCost(wordID);+ final int leftID = morphAtts.getLeftId(wordID);int leastCost = Integer.MAX_VALUE;int leastIDX = -1;assert fromPosData.count > 0;@@ -472,7 +474,14 @@ public final class KoreanTokenizer extends Tokenizer {positions.get(endPos)- .add(leastCost, dict.getRightId(wordID), fromPosData.pos, wordPos, leastIDX, wordID, type);+ .add(+ leastCost,+ morphAtts.getRightId(wordID),+ fromPosData.pos,+ wordPos,+ leastIDX,+ wordID,+ type);}@Override@@ -796,7 +805,7 @@ public final class KoreanTokenizer extends Tokenizer {+ (maxPosAhead + 1));}add(- userDictionary,+ userDictionary.getMorphAttributes(),posData,pos,maxPosAhead + 1,@@ -848,7 +857,7 @@ public final class KoreanTokenizer extends Tokenizer {}for (int ofs = 0; ofs < wordIdRef.length; ofs++) {add(- dictionary,+ dictionary.getMorphAttributes(),posData,pos,posAhead + 1,@@ -922,7 +931,7 @@ public final class KoreanTokenizer extends Tokenizer {}for (int ofs = 0; ofs < wordIdRef.length; ofs++) {add(- unkDictionary,+ unkDictionary.getMorphAttributes(),posData,pos,pos + unknownWordLength,@@ -1016,7 +1025,7 @@ public final class KoreanTokenizer extends Tokenizer {final int fragmentOffset = backWordPos - lastBackTracePos;assert fragmentOffset >= 0;- final Dictionary dict = getDict(backType);+ final Dictionary<? extends KoMorphData> dict = getDict(backType);if (outputUnknownUnigrams && backType == Type.UNKNOWN) {// outputUnknownUnigrams converts unknown word into unigrams:@@ -1029,7 +1038,7 @@ public final class KoreanTokenizer extends Tokenizer {final DictionaryToken token =new DictionaryToken(Type.UNKNOWN,- unkDictionary,+ unkDictionary.getMorphAttributes(),CharacterDefinition.NGRAM,fragment,fragmentOffset + i,@@ -1045,7 +1054,7 @@ public final class KoreanTokenizer extends Tokenizer {final DictionaryToken token =new DictionaryToken(backType,- dict,+ dict.getMorphAttributes(),backID,fragment,fragmentOffset,@@ -1060,7 +1069,7 @@ public final class KoreanTokenizer extends Tokenizer {}}} else {- Dictionary.Morpheme[] morphemes = token.getMorphemes();+ KoMorphData.Morpheme[] morphemes = token.getMorphemes();if (morphemes == null) {pending.add(token);if (VERBOSE) {@@ -1071,7 +1080,7 @@ public final class KoreanTokenizer extends Tokenizer {int posLen = 0;// decompose the compoundfor (int i = morphemes.length - 1; i >= 0; i--) {- final Dictionary.Morpheme morpheme = morphemes[i];+ final KoMorphData.Morpheme morpheme = morphemes[i];final Token compoundToken;if (token.getPOSType() == POS.Type.COMPOUND) {assert endOffset - morpheme.surfaceForm.length() >= 0;@@ -1119,7 +1128,7 @@ public final class KoreanTokenizer extends Tokenizer {DictionaryToken spaceToken =new DictionaryToken(Type.UNKNOWN,- unkDictionary,+ unkDictionary.getMorphAttributes(),wordIdRef.ints[wordIdRef.offset],fragment,offset,@@ -1144,7 +1153,7 @@ public final class KoreanTokenizer extends Tokenizer {positions.freeBefore(endPos);}- Dictionary getDict(Type type) {+ Dictionary<? extends KoMorphData> getDict(Type type) {return dictionaryMap.get(type);}diff --git a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/Token.java b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/Token.javaindex 730262907d8..ed2d7ff6ba7 100644--- a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/Token.java+++ b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/Token.java@@ -16,7 +16,7 @@*/package org.apache.lucene.analysis.ko;-import org.apache.lucene.analysis.ko.dict.Dictionary.Morpheme;+import org.apache.lucene.analysis.ko.dict.KoMorphData;/** Analyzed token with morphological data. */public abstract class Token {@@ -70,8 +70,11 @@ public abstract class Token {/** Get the reading of the token. */public abstract String getReading();- /** Get the {@link Morpheme} decomposition of the token. */- public abstract Morpheme[] getMorphemes();+ /**+ * Get the {@link org.apache.lucene.analysis.ko.dict.KoMorphData.Morpheme} decomposition of the+ * token.+ */+ public abstract KoMorphData.Morpheme[] getMorphemes();/** Get the start offset of the term in the analyzed text. */public int getStartOffset() {diff --git a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/BinaryDictionary.java b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/BinaryDictionary.javadeleted file mode 100644index 4d10e520055..00000000000--- a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/BinaryDictionary.java+++ /dev/null@@ -1,220 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.analysis.ko.dict;--import java.io.BufferedInputStream;-import java.io.EOFException;-import java.io.IOException;-import java.io.InputStream;-import java.nio.ByteBuffer;-import java.nio.channels.Channels;-import java.nio.channels.ReadableByteChannel;-import org.apache.lucene.analysis.ko.POS;-import org.apache.lucene.codecs.CodecUtil;-import org.apache.lucene.store.DataInput;-import org.apache.lucene.store.InputStreamDataInput;-import org.apache.lucene.util.IOSupplier;-import org.apache.lucene.util.IntsRef;--/** Base class for a binary-encoded in-memory dictionary. */-public abstract class BinaryDictionary implements Dictionary {-- public static final String TARGETMAP_FILENAME_SUFFIX = "$targetMap.dat";- public static final String DICT_FILENAME_SUFFIX = "$buffer.dat";- public static final String POSDICT_FILENAME_SUFFIX = "$posDict.dat";-- public static final String DICT_HEADER = "ko_dict";- public static final String TARGETMAP_HEADER = "ko_dict_map";- public static final String POSDICT_HEADER = "ko_dict_pos";- public static final int VERSION = 1;-- private final ByteBuffer buffer;- private final int[] targetMapOffsets, targetMap;- private final POS.Tag[] posDict;-- protected BinaryDictionary(- IOSupplier<InputStream> targetMapResource,- IOSupplier<InputStream> posResource,- IOSupplier<InputStream> dictResource)- throws IOException {- try (InputStream mapIS = new BufferedInputStream(targetMapResource.get())) {- DataInput in = new InputStreamDataInput(mapIS);- CodecUtil.checkHeader(in, TARGETMAP_HEADER, VERSION, VERSION);- this.targetMap = new int[in.readVInt()];- this.targetMapOffsets = new int[in.readVInt()];- populateTargetMap(in, this.targetMap, this.targetMapOffsets);- }-- try (InputStream posIS = new BufferedInputStream(posResource.get())) {- DataInput in = new InputStreamDataInput(posIS);- CodecUtil.checkHeader(in, POSDICT_HEADER, VERSION, VERSION);- int posSize = in.readVInt();- this.posDict = new POS.Tag[posSize];- for (int j = 0; j < posSize; j++) {- posDict[j] = POS.resolveTag(in.readByte());- }- }-- // no buffering here, as we load in one large buffer- try (InputStream dictIS = dictResource.get()) {- DataInput in = new InputStreamDataInput(dictIS);- CodecUtil.checkHeader(in, DICT_HEADER, VERSION, VERSION);- final int size = in.readVInt();- final ByteBuffer tmpBuffer = ByteBuffer.allocateDirect(size);- final ReadableByteChannel channel = Channels.newChannel(dictIS);- final int read = channel.read(tmpBuffer);- if (read != size) {- throw new EOFException("Cannot read whole dictionary");- }- this.buffer = tmpBuffer.asReadOnlyBuffer();- }- }-- private static void populateTargetMap(DataInput in, int[] targetMap, int[] targetMapOffsets)- throws IOException {- int accum = 0, sourceId = 0;- for (int ofs = 0; ofs < targetMap.length; ofs++) {- final int val = in.readVInt();- if ((val & 0x01) != 0) {- targetMapOffsets[sourceId] = ofs;- sourceId++;- }- accum += val >>> 1;- targetMap[ofs] = accum;- }- if (sourceId + 1 != targetMapOffsets.length)- throw new IOException(- "targetMap file format broken; targetMap.length="- + targetMap.length- + ", targetMapOffsets.length="- + targetMapOffsets.length- + ", sourceId="- + sourceId);- targetMapOffsets[sourceId] = targetMap.length;- }-- public void lookupWordIds(int sourceId, IntsRef ref) {- ref.ints = targetMap;- ref.offset = targetMapOffsets[sourceId];- // targetMapOffsets always has one more entry pointing behind last:- ref.length = targetMapOffsets[sourceId + 1] - ref.offset;- }-- @Override- public int getLeftId(int wordId) {- return buffer.getShort(wordId) >>> 2;- }-- @Override- public int getRightId(int wordId) {- return buffer.getShort(wordId + 2) >>> 2; // Skip left id- }-- @Override- public int getWordCost(int wordId) {- return buffer.getShort(wordId + 4); // Skip left and right id- }-- @Override- public POS.Type getPOSType(int wordId) {- byte value = (byte) (buffer.getShort(wordId) & 3);- return POS.resolveType(value);- }-- @Override- public POS.Tag getLeftPOS(int wordId) {- return posDict[getLeftId(wordId)];- }-- @Override- public POS.Tag getRightPOS(int wordId) {- POS.Type type = getPOSType(wordId);- if (type == POS.Type.MORPHEME || type == POS.Type.COMPOUND || hasSinglePOS(wordId)) {- return getLeftPOS(wordId);- } else {- byte value = buffer.get(wordId + 6);- return POS.resolveTag(value);- }- }-- @Override- public String getReading(int wordId) {- if (hasReadingData(wordId)) {- int offset = wordId + 6;- return readString(offset);- }- return null;- }-- @Override- public Morpheme[] getMorphemes(int wordId, char[] surfaceForm, int off, int len) {- POS.Type posType = getPOSType(wordId);- if (posType == POS.Type.MORPHEME) {- return null;- }- int offset = wordId + 6;- boolean hasSinglePos = hasSinglePOS(wordId);- if (hasSinglePos == false) {- offset++; // skip rightPOS- }- int length = buffer.get(offset++);- if (length == 0) {- return null;- }- Morpheme[] morphemes = new Morpheme[length];- int surfaceOffset = 0;- final POS.Tag leftPOS = getLeftPOS(wordId);- for (int i = 0; i < length; i++) {- final String form;- final POS.Tag tag = hasSinglePos ? leftPOS : POS.resolveTag(buffer.get(offset++));- if (posType == POS.Type.INFLECT) {- form = readString(offset);- offset += form.length() * 2 + 1;- } else {- int formLen = buffer.get(offset++);- form = new String(surfaceForm, off + surfaceOffset, formLen);- surfaceOffset += formLen;- }- morphemes[i] = new Morpheme(tag, form);- }- return morphemes;- }-- private String readString(int offset) {- int strOffset = offset;- int len = buffer.get(strOffset++);- char[] text = new char[len];- for (int i = 0; i < len; i++) {- text[i] = buffer.getChar(strOffset + (i << 1));- }- return new String(text);- }-- private boolean hasSinglePOS(int wordId) {- return (buffer.getShort(wordId + 2) & HAS_SINGLE_POS) != 0;- }-- private boolean hasReadingData(int wordId) {- return (buffer.getShort(wordId + 2) & HAS_READING) != 0;- }-- /** flag that the entry has a single part of speech (leftPOS) */- public static final int HAS_SINGLE_POS = 1;-- /** flag that the entry has reading data. otherwise reading is surface form */- public static final int HAS_READING = 2;-}diff --git a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/CharacterDefinition.java b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/CharacterDefinition.javaindex 5e2e48932ba..bcdcfde2f06 100644--- a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/CharacterDefinition.java+++ b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/CharacterDefinition.java@@ -16,20 +16,13 @@*/package org.apache.lucene.analysis.ko.dict;-import java.io.BufferedInputStream;import java.io.IOException;import java.io.InputStream;-import org.apache.lucene.codecs.CodecUtil;-import org.apache.lucene.store.DataInput;-import org.apache.lucene.store.InputStreamDataInput;import org.apache.lucene.util.IOUtils;/** Character category data. */-public final class CharacterDefinition {-- public static final String FILENAME_SUFFIX = ".dat";- public static final String HEADER = "ko_cd";- public static final int VERSION = 1;+public final class CharacterDefinition+ extends org.apache.lucene.analysis.morph.CharacterDefinition {public static final int CLASS_COUNT = CharacterClass.values().length;@@ -51,11 +44,6 @@ public final class CharacterDefinition {HANJANUMERIC}- private final byte[] characterCategoryMap = new byte[0x10000];-- private final boolean[] invokeMap = new boolean[CLASS_COUNT];- private final boolean[] groupMap = new boolean[CLASS_COUNT];-// the classes:public static final byte NGRAM = (byte) CharacterClass.NGRAM.ordinal();public static final byte DEFAULT = (byte) CharacterClass.DEFAULT.ordinal();@@ -73,16 +61,11 @@ public final class CharacterDefinition {public static final byte HANJANUMERIC = (byte) CharacterClass.HANJANUMERIC.ordinal();private CharacterDefinition() throws IOException {- try (InputStream is = new BufferedInputStream(getClassResource())) {- final DataInput in = new InputStreamDataInput(is);- CodecUtil.checkHeader(in, HEADER, VERSION, VERSION);- in.readBytes(characterCategoryMap, 0, characterCategoryMap.length);- for (int i = 0; i < CLASS_COUNT; i++) {- final byte b = in.readByte();- invokeMap[i] = (b & 0x01) != 0;- groupMap[i] = (b & 0x02) != 0;- }- }+ super(+ CharacterDefinition::getClassResource,+ DictionaryConstants.CHARDEF_HEADER,+ DictionaryConstants.VERSION,+ CharacterDefinition.CLASS_COUNT);}private static InputStream getClassResource() throws IOException {@@ -91,18 +74,6 @@ public final class CharacterDefinition {CharacterDefinition.class.getResourceAsStream(resourcePath), resourcePath);}- public byte getCharacterClass(char c) {- return characterCategoryMap[c];- }-- public boolean isInvoke(char c) {- return invokeMap[characterCategoryMap[c]];- }-- public boolean isGroup(char c) {- return groupMap[characterCategoryMap[c]];- }-public boolean isHanja(char c) {final byte characterClass = getCharacterClass(c);return characterClass == HANJA || characterClass == HANJANUMERIC;diff --git a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/ConnectionCosts.java b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/ConnectionCosts.javaindex 61579aef845..3b13a86b90a 100644--- a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/ConnectionCosts.java+++ b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/ConnectionCosts.java@@ -16,27 +16,15 @@*/package org.apache.lucene.analysis.ko.dict;-import java.io.BufferedInputStream;import java.io.IOException;import java.io.InputStream;-import java.nio.ByteBuffer;import java.nio.file.Files;import java.nio.file.Path;-import org.apache.lucene.codecs.CodecUtil;-import org.apache.lucene.store.DataInput;-import org.apache.lucene.store.InputStreamDataInput;import org.apache.lucene.util.IOSupplier;import org.apache.lucene.util.IOUtils;/** n-gram connection cost data */-public final class ConnectionCosts {-- public static final String FILENAME_SUFFIX = ".dat";- public static final String HEADER = "ko_cc";- public static final int VERSION = 1;-- private final ByteBuffer buffer;- private final int forwardSize;+public final class ConnectionCosts extends org.apache.lucene.analysis.morph.ConnectionCosts {/*** Create a {@link ConnectionCosts} from an external resource path.@@ -53,24 +41,8 @@ public final class ConnectionCosts {}private ConnectionCosts(IOSupplier<InputStream> connectionCostResource) throws IOException {- try (InputStream is = new BufferedInputStream(connectionCostResource.get())) {- final DataInput in = new InputStreamDataInput(is);- CodecUtil.checkHeader(in, HEADER, VERSION, VERSION);- this.forwardSize = in.readVInt();- int backwardSize = in.readVInt();- int size = forwardSize * backwardSize;-- // copy the matrix into a direct byte buffer- final ByteBuffer tmpBuffer = ByteBuffer.allocateDirect(size * 2);- int accum = 0;- for (int j = 0; j < backwardSize; j++) {- for (int i = 0; i < forwardSize; i++) {- accum += in.readZInt();- tmpBuffer.putShort((short) accum);- }- }- buffer = tmpBuffer.asReadOnlyBuffer();- }+ super(+ connectionCostResource, DictionaryConstants.CONN_COSTS_HEADER, DictionaryConstants.VERSION);}private static InputStream getClassResource() throws IOException {@@ -79,12 +51,6 @@ public final class ConnectionCosts {ConnectionCosts.class.getResourceAsStream(resourcePath), resourcePath);}- public int get(int forwardId, int backwardId) {- // map 2d matrix into a single dimension short array- int offset = (backwardId * forwardSize + forwardId) * 2;- return buffer.getShort(offset);- }-public static ConnectionCosts getInstance() {return SingletonHolder.INSTANCE;}diff --git a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/DictionaryConstants.java b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/DictionaryConstants.javanew file mode 100644index 00000000000..4f8761e59e7--- /dev/null+++ b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/DictionaryConstants.java@@ -0,0 +1,33 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.analysis.ko.dict;++/** Dictionary constants */+public final class DictionaryConstants {+ /** Codec header of the dictionary file. */+ public static final String DICT_HEADER = "ko_dict";+ /** Codec header of the dictionary mapping file. */+ public static final String TARGETMAP_HEADER = "ko_dict_map";+ /** Codec header of the POS dictionary file. */+ public static final String POSDICT_HEADER = "ko_dict_pos";+ /** Codec header of the connection costs file. */+ public static final String CONN_COSTS_HEADER = "ko_cc";+ /** Codec header of the character definition file */+ public static final String CHARDEF_HEADER = "ko_cd";+ /** Codec version of the binary dictionary */+ public static final int VERSION = 1;+}diff --git a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/Dictionary.java b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/KoMorphData.javasimilarity index 52%rename from lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/Dictionary.javarename to lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/KoMorphData.javaindex b25e1b5ff5e..0887a7f0c42 100644--- a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/Dictionary.java+++ b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/KoMorphData.java@@ -16,51 +16,47 @@*/package org.apache.lucene.analysis.ko.dict;-import org.apache.lucene.analysis.ko.POS.Tag;-import org.apache.lucene.analysis.ko.POS.Type;+import org.apache.lucene.analysis.ko.POS;+import org.apache.lucene.analysis.morph.MorphData;-/** Dictionary interface for retrieving morphological data by id. */-public interface Dictionary {+/** Represents Korean morphological information. */+public interface KoMorphData extends MorphData {/** A morpheme extracted from a compound token. */class Morpheme {- public final Tag posTag;+ public final POS.Tag posTag;public final String surfaceForm;- public Morpheme(Tag posTag, String surfaceForm) {+ public Morpheme(POS.Tag posTag, String surfaceForm) {this.posTag = posTag;this.surfaceForm = surfaceForm;}}- /** Get left id of specified word */- int getLeftId(int wordId);-- /** Get right id of specified word */- int getRightId(int wordId);-- /** Get word cost of specified word */- int getWordCost(int wordId);-- /** Get the {@link Type} of specified word (morpheme, compound, inflect or pre-analysis) */- Type getPOSType(int wordId);+ /**+ * Get the {@link org.apache.lucene.analysis.ko.POS.Type} of specified word (morpheme, compound,+ * inflect or pre-analysis)+ */+ POS.Type getPOSType(int morphId);/**- * Get the left {@link Tag} of specfied word.+ * Get the left {@link org.apache.lucene.analysis.ko.POS.Tag} of specfied word.*- * <p>For {@link Type#MORPHEME} and {@link Type#COMPOUND} the left and right POS are the same.+ * <p>For {@link org.apache.lucene.analysis.ko.POS.Type#MORPHEME} and {@link+ * org.apache.lucene.analysis.ko.POS.Type#COMPOUND} the left and right POS are the same.*/- Tag getLeftPOS(int wordId);+ POS.Tag getLeftPOS(int morphId);/**- * Get the right {@link Tag} of specfied word.+ * Get the right {@link org.apache.lucene.analysis.ko.POS.Tag} of specfied word.*- * <p>For {@link Type#MORPHEME} and {@link Type#COMPOUND} the left and right POS are the same.+ * <p>For {@link org.apache.lucene.analysis.ko.POS.Type#MORPHEME} and {@link+ * org.apache.lucene.analysis.ko.POS.Type#COMPOUND} the left and right POS are the same.*/- Tag getRightPOS(int wordId);+ POS.Tag getRightPOS(int morphId);/** Get the reading of specified word (mainly used for Hanja to Hangul conversion). */- String getReading(int wordId);+ String getReading(int morphId);/** Get the morphemes of specified word (e.g. 가깝으나: 가깝 + 으나). */- Morpheme[] getMorphemes(int wordId, char[] surfaceForm, int off, int len);+ Morpheme[] getMorphemes(int morphId, char[] surfaceForm, int off, int len);}diff --git a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/TokenInfoDictionary.java b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/TokenInfoDictionary.javaindex 1f936b44724..b8132cea61b 100644--- a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/TokenInfoDictionary.java+++ b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/TokenInfoDictionary.java@@ -21,6 +21,7 @@ import java.io.IOException;import java.io.InputStream;import java.nio.file.Files;import java.nio.file.Path;+import org.apache.lucene.analysis.morph.BinaryDictionary;import org.apache.lucene.store.DataInput;import org.apache.lucene.store.InputStreamDataInput;import org.apache.lucene.util.IOSupplier;@@ -32,11 +33,12 @@ import org.apache.lucene.util.fst.PositiveIntOutputs;* Binary dictionary implementation for a known-word dictionary model: Words are encoded into an FST* mapping to a list of wordIDs.*/-public final class TokenInfoDictionary extends BinaryDictionary {+public final class TokenInfoDictionary extends BinaryDictionary<TokenInfoMorphData> {public static final String FST_FILENAME_SUFFIX = "$fst.dat";private final TokenInfoFST fst;+ private final TokenInfoMorphData morphAtts;private TokenInfoDictionary() throws IOException {this(@@ -70,7 +72,13 @@ public final class TokenInfoDictionary extends BinaryDictionary {IOSupplier<InputStream> dictResource,IOSupplier<InputStream> fstResource)throws IOException {- super(targetMapResource, posResource, dictResource);+ super(+ targetMapResource,+ dictResource,+ DictionaryConstants.TARGETMAP_HEADER,+ DictionaryConstants.DICT_HEADER,+ DictionaryConstants.VERSION);+ this.morphAtts = new TokenInfoMorphData(buffer, posResource);FST<Long> fst;try (InputStream is = new BufferedInputStream(fstResource.get())) {DataInput in = new InputStreamDataInput(is);@@ -93,6 +101,11 @@ public final class TokenInfoDictionary extends BinaryDictionary {return SingletonHolder.INSTANCE;}+ @Override+ public TokenInfoMorphData getMorphAttributes() {+ return morphAtts;+ }+private static class SingletonHolder {static final TokenInfoDictionary INSTANCE;diff --git a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/TokenInfoMorphData.java b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/TokenInfoMorphData.javanew file mode 100644index 00000000000..c8046b2e604--- /dev/null+++ b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/TokenInfoMorphData.java@@ -0,0 +1,155 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.analysis.ko.dict;++import java.io.BufferedInputStream;+import java.io.IOException;+import java.io.InputStream;+import java.nio.ByteBuffer;+import org.apache.lucene.analysis.ko.POS;+import org.apache.lucene.codecs.CodecUtil;+import org.apache.lucene.store.DataInput;+import org.apache.lucene.store.InputStreamDataInput;+import org.apache.lucene.util.IOSupplier;++/** Morphological information for system dictionary. */+public class TokenInfoMorphData implements KoMorphData {++ private final ByteBuffer buffer;+ private final POS.Tag[] posDict;++ TokenInfoMorphData(ByteBuffer buffer, IOSupplier<InputStream> posResource) throws IOException {+ this.buffer = buffer;+ try (InputStream posIS = new BufferedInputStream(posResource.get())) {+ DataInput in = new InputStreamDataInput(posIS);+ CodecUtil.checkHeader(+ in,+ DictionaryConstants.POSDICT_HEADER,+ DictionaryConstants.VERSION,+ DictionaryConstants.VERSION);+ int posSize = in.readVInt();+ this.posDict = new POS.Tag[posSize];+ for (int j = 0; j < posSize; j++) {+ posDict[j] = POS.resolveTag(in.readByte());+ }+ }+ }++ @Override+ public int getLeftId(int morphId) {+ return buffer.getShort(morphId) >>> 2;+ }++ @Override+ public int getRightId(int morphId) {+ return buffer.getShort(morphId + 2) >>> 2; // Skip left id+ }++ @Override+ public int getWordCost(int morphId) {+ return buffer.getShort(morphId + 4); // Skip left and right id+ }++ @Override+ public POS.Type getPOSType(int morphId) {+ byte value = (byte) (buffer.getShort(morphId) & 3);+ return POS.resolveType(value);+ }++ @Override+ public POS.Tag getLeftPOS(int morphId) {+ return posDict[getLeftId(morphId)];+ }++ @Override+ public POS.Tag getRightPOS(int morphId) {+ POS.Type type = getPOSType(morphId);+ if (type == POS.Type.MORPHEME || type == POS.Type.COMPOUND || hasSinglePOS(morphId)) {+ return getLeftPOS(morphId);+ } else {+ byte value = buffer.get(morphId + 6);+ return POS.resolveTag(value);+ }+ }++ @Override+ public String getReading(int morphId) {+ if (hasReadingData(morphId)) {+ int offset = morphId + 6;+ return readString(offset);+ }+ return null;+ }++ @Override+ public Morpheme[] getMorphemes(int morphId, char[] surfaceForm, int off, int len) {+ POS.Type posType = getPOSType(morphId);+ if (posType == POS.Type.MORPHEME) {+ return null;+ }+ int offset = morphId + 6;+ boolean hasSinglePos = hasSinglePOS(morphId);+ if (hasSinglePos == false) {+ offset++; // skip rightPOS+ }+ int length = buffer.get(offset++);+ if (length == 0) {+ return null;+ }+ Morpheme[] morphemes = new Morpheme[length];+ int surfaceOffset = 0;+ final POS.Tag leftPOS = getLeftPOS(morphId);+ for (int i = 0; i < length; i++) {+ final String form;+ final POS.Tag tag = hasSinglePos ? leftPOS : POS.resolveTag(buffer.get(offset++));+ if (posType == POS.Type.INFLECT) {+ form = readString(offset);+ offset += form.length() * 2 + 1;+ } else {+ int formLen = buffer.get(offset++);+ form = new String(surfaceForm, off + surfaceOffset, formLen);+ surfaceOffset += formLen;+ }+ morphemes[i] = new Morpheme(tag, form);+ }+ return morphemes;+ }++ private String readString(int offset) {+ int strOffset = offset;+ int len = buffer.get(strOffset++);+ char[] text = new char[len];+ for (int i = 0; i < len; i++) {+ text[i] = buffer.getChar(strOffset + (i << 1));+ }+ return new String(text);+ }++ private boolean hasSinglePOS(int wordId) {+ return (buffer.getShort(wordId + 2) & HAS_SINGLE_POS) != 0;+ }++ private boolean hasReadingData(int wordId) {+ return (buffer.getShort(wordId + 2) & HAS_READING) != 0;+ }++ /** flag that the entry has a single part of speech (leftPOS) */+ public static final int HAS_SINGLE_POS = 1;++ /** flag that the entry has reading data. otherwise reading is surface form */+ public static final int HAS_READING = 2;+}diff --git a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/UnknownDictionary.java b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/UnknownDictionary.javaindex f2aa89ee8c5..4b45fd33258 100644--- a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/UnknownDictionary.java+++ b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/UnknownDictionary.java@@ -20,11 +20,14 @@ import java.io.IOException;import java.io.InputStream;import java.nio.file.Files;import java.nio.file.Path;+import org.apache.lucene.analysis.morph.BinaryDictionary;+import org.apache.lucene.util.IOSupplier;import org.apache.lucene.util.IOUtils;/** Dictionary for unknown-word handling. */-public final class UnknownDictionary extends BinaryDictionary {+public final class UnknownDictionary extends BinaryDictionary<UnknownMorphData> {private final CharacterDefinition characterDefinition = CharacterDefinition.getInstance();+ private final UnknownMorphData morphAtts;/*** Create a {@link UnknownDictionary} from an external resource path.@@ -35,19 +38,33 @@ public final class UnknownDictionary extends BinaryDictionary {* @throws IOException if resource was not found or broken*/public UnknownDictionary(Path targetMapFile, Path posDictFile, Path dictFile) throws IOException {- super(+ this(() -> Files.newInputStream(targetMapFile),() -> Files.newInputStream(posDictFile),() -> Files.newInputStream(dictFile));}private UnknownDictionary() throws IOException {- super(+ this(() -> getClassResource(TARGETMAP_FILENAME_SUFFIX),() -> getClassResource(POSDICT_FILENAME_SUFFIX),() -> getClassResource(DICT_FILENAME_SUFFIX));}+ private UnknownDictionary(+ IOSupplier<InputStream> targetMapResource,+ IOSupplier<InputStream> posResource,+ IOSupplier<InputStream> dictResource)+ throws IOException {+ super(+ targetMapResource,+ dictResource,+ DictionaryConstants.TARGETMAP_HEADER,+ DictionaryConstants.DICT_HEADER,+ DictionaryConstants.VERSION);+ this.morphAtts = new UnknownMorphData(buffer, posResource);+ }+private static InputStream getClassResource(String suffix) throws IOException {final String resourcePath = UnknownDictionary.class.getSimpleName() + suffix;return IOUtils.requireResourceNonNull(@@ -63,13 +80,8 @@ public final class UnknownDictionary extends BinaryDictionary {}@Override- public String getReading(int wordId) {- return null;- }-- @Override- public Morpheme[] getMorphemes(int wordId, char[] surfaceForm, int off, int len) {- return null;+ public UnknownMorphData getMorphAttributes() {+ return morphAtts;}private static class SingletonHolder {diff --git a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/UnknownMorphData.java b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/UnknownMorphData.javanew file mode 100644index 00000000000..dafb174f8d5--- /dev/null+++ b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/UnknownMorphData.java@@ -0,0 +1,39 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.analysis.ko.dict;++import java.io.IOException;+import java.io.InputStream;+import java.nio.ByteBuffer;+import org.apache.lucene.util.IOSupplier;++/** Morphological information for unk dictionary. */+final class UnknownMorphData extends TokenInfoMorphData {+ UnknownMorphData(ByteBuffer buffer, IOSupplier<InputStream> posResource) throws IOException {+ super(buffer, posResource);+ }++ @Override+ public String getReading(int wordId) {+ return null;+ }++ @Override+ public Morpheme[] getMorphemes(int wordId, char[] surfaceForm, int off, int len) {+ return null;+ }+}diff --git a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/UserDictionary.java b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/UserDictionary.javaindex f8fcefdd295..58a233112aa 100644--- a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/UserDictionary.java+++ b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/UserDictionary.java@@ -22,7 +22,7 @@ import java.io.Reader;import java.util.ArrayList;import java.util.Comparator;import java.util.List;-import org.apache.lucene.analysis.ko.POS;+import org.apache.lucene.analysis.morph.Dictionary;import org.apache.lucene.util.IntsRefBuilder;import org.apache.lucene.util.fst.FST;import org.apache.lucene.util.fst.FSTCompiler;@@ -32,15 +32,10 @@ import org.apache.lucene.util.fst.PositiveIntOutputs;* Class for building a User Dictionary. This class allows for adding custom nouns (세종) or compounds* (세종시 세종 시).*/-public final class UserDictionary implements Dictionary {+public final class UserDictionary implements Dictionary<UserMorphData> {// text -> wordIDprivate final TokenInfoFST fst;- private static final int WORD_COST = -100000;-- // NNG left- private static final short LEFT_ID = 1781;-// NNG rightprivate static final short RIGHT_ID = 3533;// NNG right with hangul and a coda on the last char@@ -48,9 +43,7 @@ public final class UserDictionary implements Dictionary {// NNG right with hangul and no coda on the last charprivate static final short RIGHT_ID_F = 3534;- // length, length... indexed by compound ID or null for simple noun- private final int[][] segmentations;- private final short[] rightIds;+ private UserMorphData morphAtts;public static UserDictionary open(Reader reader) throws IOException {@@ -86,8 +79,8 @@ public final class UserDictionary implements Dictionary {IntsRefBuilder scratch = new IntsRefBuilder();String lastToken = null;- List<int[]> segmentations = new ArrayList<>(entries.size());- List<Short> rightIds = new ArrayList<>(entries.size());+ List<int[]> _segmentations = new ArrayList<>(entries.size());+ List<Short> _rightIds = new ArrayList<>(entries.size());long ord = 0;for (String entry : entries) {String[] splits = entry.split("\\s+");@@ -98,16 +91,16 @@ public final class UserDictionary implements Dictionary {char lastChar = entry.charAt(entry.length() - 1);if (charDef.isHangul(lastChar)) {if (charDef.hasCoda(lastChar)) {- rightIds.add(RIGHT_ID_T);+ _rightIds.add(RIGHT_ID_T);} else {- rightIds.add(RIGHT_ID_F);+ _rightIds.add(RIGHT_ID_F);}} else {- rightIds.add(RIGHT_ID);+ _rightIds.add(RIGHT_ID);}if (splits.length == 1) {- segmentations.add(null);+ _segmentations.add(null);} else {int[] length = new int[splits.length - 1];int offset = 0;@@ -123,7 +116,7 @@ public final class UserDictionary implements Dictionary {+ token+ ")");}- segmentations.add(length);+ _segmentations.add(length);}// add mapping to FST@@ -137,11 +130,12 @@ public final class UserDictionary implements Dictionary {ord++;}this.fst = new TokenInfoFST(fstCompiler.compile());- this.segmentations = segmentations.toArray(new int[segmentations.size()][]);- this.rightIds = new short[rightIds.size()];- for (int i = 0; i < rightIds.size(); i++) {- this.rightIds[i] = rightIds.get(i);+ int[][] segmentations = _segmentations.toArray(new int[_segmentations.size()][]);+ short[] rightIds = new short[_rightIds.size()];+ for (int i = 0; i < _rightIds.size(); i++) {+ rightIds[i] = _rightIds.get(i);}+ this.morphAtts = new UserMorphData(segmentations, rightIds);}public TokenInfoFST getFST() {@@ -149,57 +143,8 @@ public final class UserDictionary implements Dictionary {}@Override- public int getLeftId(int wordId) {- return LEFT_ID;- }-- @Override- public int getRightId(int wordId) {- return rightIds[wordId];- }-- @Override- public int getWordCost(int wordId) {- return WORD_COST;- }-- @Override- public POS.Type getPOSType(int wordId) {- if (segmentations[wordId] == null) {- return POS.Type.MORPHEME;- } else {- return POS.Type.COMPOUND;- }- }-- @Override- public POS.Tag getLeftPOS(int wordId) {- return POS.Tag.NNG;- }-- @Override- public POS.Tag getRightPOS(int wordId) {- return POS.Tag.NNG;- }-- @Override- public String getReading(int wordId) {- return null;- }-- @Override- public Morpheme[] getMorphemes(int wordId, char[] surfaceForm, int off, int len) {- int[] segs = segmentations[wordId];- if (segs == null) {- return null;- }- int offset = 0;- Morpheme[] morphemes = new Morpheme[segs.length];- for (int i = 0; i < segs.length; i++) {- morphemes[i] = new Morpheme(POS.Tag.NNG, new String(surfaceForm, off + offset, segs[i]));- offset += segs[i];- }- return morphemes;+ public UserMorphData getMorphAttributes() {+ return morphAtts;}/**diff --git a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/UserMorphData.java b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/UserMorphData.javanew file mode 100644index 00000000000..2056bd0829c--- /dev/null+++ b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/UserMorphData.java@@ -0,0 +1,90 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.analysis.ko.dict;++import org.apache.lucene.analysis.ko.POS;++/** Morphological information for user dictionary. */+final class UserMorphData implements KoMorphData {+ private static final int WORD_COST = -100000;++ // NNG left+ private static final short LEFT_ID = 1781;++ // length, length... indexed by compound ID or null for simple noun+ private final int[][] segmentations;+ private final short[] rightIds;++ UserMorphData(int[][] segmentations, short[] rightIds) {+ this.segmentations = segmentations;+ this.rightIds = rightIds;+ }++ @Override+ public int getLeftId(int morphId) {+ return LEFT_ID;+ }++ @Override+ public int getRightId(int morphId) {+ return rightIds[morphId];+ }++ @Override+ public int getWordCost(int morphId) {+ return WORD_COST;+ }++ @Override+ public POS.Type getPOSType(int morphId) {+ if (segmentations[morphId] == null) {+ return POS.Type.MORPHEME;+ } else {+ return POS.Type.COMPOUND;+ }+ }++ @Override+ public POS.Tag getLeftPOS(int morphId) {+ return POS.Tag.NNG;+ }++ @Override+ public POS.Tag getRightPOS(int morphId) {+ return POS.Tag.NNG;+ }++ @Override+ public String getReading(int morphId) {+ return null;+ }++ @Override+ public Morpheme[] getMorphemes(int morphId, char[] surfaceForm, int off, int len) {+ int[] segs = segmentations[morphId];+ if (segs == null) {+ return null;+ }+ int offset = 0;+ Morpheme[] morphemes = new Morpheme[segs.length];+ for (int i = 0; i < segs.length; i++) {+ morphemes[i] = new Morpheme(POS.Tag.NNG, new String(surfaceForm, off + offset, segs[i]));+ offset += segs[i];+ }+ return morphemes;+ }+}diff --git a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/tokenattributes/PartOfSpeechAttribute.java b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/tokenattributes/PartOfSpeechAttribute.javaindex c9fb33b17c2..1e28155e667 100644--- a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/tokenattributes/PartOfSpeechAttribute.java+++ b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/tokenattributes/PartOfSpeechAttribute.java@@ -19,7 +19,7 @@ package org.apache.lucene.analysis.ko.tokenattributes;import org.apache.lucene.analysis.ko.POS.Tag;import org.apache.lucene.analysis.ko.POS.Type;import org.apache.lucene.analysis.ko.Token;-import org.apache.lucene.analysis.ko.dict.Dictionary.Morpheme;+import org.apache.lucene.analysis.ko.dict.KoMorphData;import org.apache.lucene.util.Attribute;/**@@ -37,8 +37,11 @@ public interface PartOfSpeechAttribute extends Attribute {/** Get the right part of speech of the token. */Tag getRightPOS();- /** Get the {@link Morpheme} decomposition of the token. */- Morpheme[] getMorphemes();+ /**+ * Get the {@link org.apache.lucene.analysis.ko.dict.KoMorphData.Morpheme} decomposition of the+ * token.+ */+ KoMorphData.Morpheme[] getMorphemes();/** Set the current token. */void setToken(Token token);diff --git a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/tokenattributes/PartOfSpeechAttributeImpl.java b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/tokenattributes/PartOfSpeechAttributeImpl.javaindex a1b04cb7f05..a59999db557 100644--- a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/tokenattributes/PartOfSpeechAttributeImpl.java+++ b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/tokenattributes/PartOfSpeechAttributeImpl.java@@ -19,7 +19,7 @@ package org.apache.lucene.analysis.ko.tokenattributes;import org.apache.lucene.analysis.ko.POS.Tag;import org.apache.lucene.analysis.ko.POS.Type;import org.apache.lucene.analysis.ko.Token;-import org.apache.lucene.analysis.ko.dict.Dictionary.Morpheme;+import org.apache.lucene.analysis.ko.dict.KoMorphData;import org.apache.lucene.util.AttributeImpl;import org.apache.lucene.util.AttributeReflector;@@ -47,7 +47,7 @@ public class PartOfSpeechAttributeImpl extends AttributeImpl implements PartOfSp}@Override- public Morpheme[] getMorphemes() {+ public KoMorphData.Morpheme[] getMorphemes() {return token == null ? null : token.getMorphemes();}@@ -76,12 +76,12 @@ public class PartOfSpeechAttributeImpl extends AttributeImpl implements PartOfSpreflector.reflect(PartOfSpeechAttribute.class, "morphemes", displayMorphemes(getMorphemes()));}- private String displayMorphemes(Morpheme[] morphemes) {+ private String displayMorphemes(KoMorphData.Morpheme[] morphemes) {if (morphemes == null) {return null;}StringBuilder builder = new StringBuilder();- for (Morpheme morpheme : morphemes) {+ for (KoMorphData.Morpheme morpheme : morphemes) {if (builder.length() > 0) {builder.append("+");}diff --git a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/util/ConnectionCostsBuilder.java b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/util/ConnectionCostsBuilder.javaindex 4a6fd6d353c..31505fb45c8 100644--- a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/util/ConnectionCostsBuilder.java+++ b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/util/ConnectionCostsBuilder.java@@ -22,12 +22,14 @@ import java.io.Reader;import java.nio.charset.StandardCharsets;import java.nio.file.Files;import java.nio.file.Path;+import org.apache.lucene.analysis.ko.dict.ConnectionCosts;+import org.apache.lucene.analysis.morph.ConnectionCostsWriter;class ConnectionCostsBuilder {private ConnectionCostsBuilder() {}- public static ConnectionCostsWriter build(Path path) throws IOException {+ public static ConnectionCostsWriter<ConnectionCosts> build(Path path) throws IOException {try (Reader reader = Files.newBufferedReader(path, StandardCharsets.US_ASCII);LineNumberReader lineReader = new LineNumberReader(reader)) {@@ -41,7 +43,8 @@ class ConnectionCostsBuilder {assert forwardSize > 0 && backwardSize > 0;- ConnectionCostsWriter costs = new ConnectionCostsWriter(forwardSize, backwardSize);+ ConnectionCostsWriter<ConnectionCosts> costs =+ new ConnectionCostsWriter<>(ConnectionCosts.class, forwardSize, backwardSize);while ((line = lineReader.readLine()) != null) {String[] fields = line.split("\\s+");diff --git a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/util/ConnectionCostsWriter.java b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/util/ConnectionCostsWriter.javadeleted file mode 100644index 7265ef84dbb..00000000000--- a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/util/ConnectionCostsWriter.java+++ /dev/null@@ -1,68 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.analysis.ko.util;--import java.io.BufferedOutputStream;-import java.io.IOException;-import java.io.OutputStream;-import java.nio.ByteBuffer;-import java.nio.file.Files;-import java.nio.file.Path;-import org.apache.lucene.analysis.ko.dict.ConnectionCosts;-import org.apache.lucene.codecs.CodecUtil;-import org.apache.lucene.store.DataOutput;-import org.apache.lucene.store.OutputStreamDataOutput;--final class ConnectionCostsWriter {-- private final ByteBuffer- costs; // array is backward IDs first since get is called using the same backward ID- // consecutively. maybe doesn't matter.- private final int forwardSize;- private final int backwardSize;- /** Constructor for building. TODO: remove write access */- ConnectionCostsWriter(int forwardSize, int backwardSize) {- this.forwardSize = forwardSize;- this.backwardSize = backwardSize;- this.costs = ByteBuffer.allocateDirect(2 * backwardSize * forwardSize);- }-- public void add(int forwardId, int backwardId, int cost) {- int offset = (backwardId * forwardSize + forwardId) * 2;- costs.putShort(offset, (short) cost);- }-- public void write(Path baseDir) throws IOException {- Files.createDirectories(baseDir);- String fileName =- ConnectionCosts.class.getName().replace('.', '/') + ConnectionCosts.FILENAME_SUFFIX;- try (OutputStream os = Files.newOutputStream(baseDir.resolve(fileName));- OutputStream bos = new BufferedOutputStream(os)) {- final DataOutput out = new OutputStreamDataOutput(bos);- CodecUtil.writeHeader(out, ConnectionCosts.HEADER, ConnectionCosts.VERSION);- out.writeVInt(forwardSize);- out.writeVInt(backwardSize);- int last = 0;- for (int i = 0; i < costs.limit() / 2; i++) {- short cost = costs.getShort(i * 2);- int delta = (int) cost - last;- out.writeZInt(delta);- last = cost;- }- }- }-}diff --git a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/util/DictionaryBuilder.java b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/util/DictionaryBuilder.javaindex e4c3b20c9b6..0e8f11f0cf8 100644--- a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/util/DictionaryBuilder.java+++ b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/util/DictionaryBuilder.java@@ -19,6 +19,7 @@ package org.apache.lucene.analysis.ko.util;import java.io.IOException;import java.nio.file.Path;import java.nio.file.Paths;+import org.apache.lucene.analysis.ko.dict.DictionaryConstants;/** Tool to build dictionaries. */public class DictionaryBuilder {@@ -34,7 +35,8 @@ public class DictionaryBuilder {new UnknownDictionaryBuilder(encoding).build(inputDir).write(outputDir);// Build Connection Cost- ConnectionCostsBuilder.build(inputDir.resolve("matrix.def")).write(outputDir);+ ConnectionCostsBuilder.build(inputDir.resolve("matrix.def"))+ .write(outputDir, DictionaryConstants.CONN_COSTS_HEADER, DictionaryConstants.VERSION);}public static void main(String[] args) throws IOException {diff --git a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/util/BinaryDictionaryWriter.java b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/util/TokenInfoDictionaryEntryWriter.javasimilarity index 50%rename from lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/util/BinaryDictionaryWriter.javarename to lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/util/TokenInfoDictionaryEntryWriter.javaindex 37ed157f8c8..e18cae357d3 100644--- a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/util/BinaryDictionaryWriter.java+++ b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/util/TokenInfoDictionaryEntryWriter.java@@ -16,38 +16,25 @@*/package org.apache.lucene.analysis.ko.util;-import java.io.BufferedOutputStream;import java.io.IOException;import java.io.OutputStream;import java.nio.ByteBuffer;-import java.nio.channels.Channels;-import java.nio.channels.WritableByteChannel;-import java.nio.file.Files;-import java.nio.file.Path;import java.util.ArrayList;import java.util.Arrays;import java.util.List;import org.apache.lucene.analysis.ko.POS;-import org.apache.lucene.analysis.ko.dict.BinaryDictionary;-import org.apache.lucene.analysis.ko.dict.Dictionary;-import org.apache.lucene.codecs.CodecUtil;+import org.apache.lucene.analysis.ko.dict.KoMorphData;+import org.apache.lucene.analysis.ko.dict.TokenInfoMorphData;+import org.apache.lucene.analysis.morph.DictionaryEntryWriter;import org.apache.lucene.store.DataOutput;-import org.apache.lucene.store.OutputStreamDataOutput;import org.apache.lucene.util.ArrayUtil;-abstract class BinaryDictionaryWriter {+/** Writes system dictionary entries. */+class TokenInfoDictionaryEntryWriter extends DictionaryEntryWriter {private static final int ID_LIMIT = 8192;- private final Class<? extends BinaryDictionary> implClazz;- protected ByteBuffer buffer;- private int targetMapEndOffset = 0, lastWordId = -1, lastSourceId = -1;- private int[] targetMap = new int[8192];- private int[] targetMapOffsets = new int[8192];- private final ArrayList<String> posDict = new ArrayList<>();-- BinaryDictionaryWriter(Class<? extends BinaryDictionary> implClazz, int size) {- this.implClazz = implClazz;- buffer = ByteBuffer.allocateDirect(size);+ TokenInfoDictionaryEntryWriter(int size) {+ super(size);}/**@@ -72,7 +59,8 @@ abstract class BinaryDictionaryWriter {** @return current position of buffer, which will be wordId of next entry*/- public int put(String[] entry) {+ @Override+ protected int putEntry(String[] entry) {short leftId = Short.parseShort(entry[1]);short rightId = Short.parseShort(entry[2]);short wordCost = Short.parseShort(entry[3]);@@ -113,7 +101,7 @@ abstract class BinaryDictionaryWriter {assert existing == null || existing.equals(fullPOSData);posDict.set(leftId, fullPOSData);- final List<Dictionary.Morpheme> morphemes = new ArrayList<>();+ final List<KoMorphData.Morpheme> morphemes = new ArrayList<>();// true if the POS and decompounds of the token are all the same.boolean hasSinglePOS = (leftPOS == rightPOS);if (posType != POS.Type.MORPHEME && expression.length() > 0) {@@ -124,7 +112,7 @@ abstract class BinaryDictionaryWriter {String surfaceForm = tokenSplit[0].trim();if (surfaceForm.isEmpty() == false) {POS.Tag exprTag = POS.resolveTag(tokenSplit[1]);- morphemes.add(new Dictionary.Morpheme(exprTag, tokenSplit[0]));+ morphemes.add(new KoMorphData.Morpheme(exprTag, tokenSplit[0]));if (leftPOS != exprTag) {hasSinglePOS = false;}@@ -134,10 +122,10 @@ abstract class BinaryDictionaryWriter {int flags = 0;if (hasSinglePOS) {- flags |= BinaryDictionary.HAS_SINGLE_POS;+ flags |= TokenInfoMorphData.HAS_SINGLE_POS;}if (posType == POS.Type.MORPHEME && reading.length() > 0) {- flags |= BinaryDictionary.HAS_READING;+ flags |= TokenInfoMorphData.HAS_READING;}if (leftId >= ID_LIMIT) {@@ -161,7 +149,7 @@ abstract class BinaryDictionaryWriter {}buffer.put((byte) morphemes.size());int compoundOffset = 0;- for (Dictionary.Morpheme morpheme : morphemes) {+ for (KoMorphData.Morpheme morpheme : morphemes) {if (hasSinglePOS == false) {buffer.put((byte) morpheme.posTag.ordinal());}@@ -184,112 +172,20 @@ abstract class BinaryDictionaryWriter {}}- void addMapping(int sourceId, int wordId) {- if (wordId <= lastWordId) {- throw new IllegalStateException(- "words out of order: " + wordId + " vs lastID: " + lastWordId);- }-- if (sourceId > lastSourceId) {- targetMapOffsets = ArrayUtil.grow(targetMapOffsets, sourceId + 1);- for (int i = lastSourceId + 1; i <= sourceId; i++) {- targetMapOffsets[i] = targetMapEndOffset;- }- } else if (sourceId != lastSourceId) {- throw new IllegalStateException(- "source ids not in increasing order: lastSourceId="- + lastSourceId- + " vs sourceId="- + sourceId);- }-- targetMap = ArrayUtil.grow(targetMap, targetMapEndOffset + 1);- targetMap[targetMapEndOffset] = wordId;- targetMapEndOffset++;-- lastSourceId = sourceId;- lastWordId = wordId;- }-- final String getBaseFileName() {- return implClazz.getName().replace('.', '/');- }-- /**- * Write dictionary in file- *- * @throws IOException if an I/O error occurs writing the dictionary files- */- public void write(Path baseDir) throws IOException {- final String baseName = getBaseFileName();- writeDictionary(baseDir.resolve(baseName + BinaryDictionary.DICT_FILENAME_SUFFIX));- writeTargetMap(baseDir.resolve(baseName + BinaryDictionary.TARGETMAP_FILENAME_SUFFIX));- writePosDict(baseDir.resolve(baseName + BinaryDictionary.POSDICT_FILENAME_SUFFIX));- }-- private void writeTargetMap(Path path) throws IOException {- Files.createDirectories(path.getParent());- try (OutputStream os = Files.newOutputStream(path);- OutputStream bos = new BufferedOutputStream(os)) {- final DataOutput out = new OutputStreamDataOutput(bos);- CodecUtil.writeHeader(out, BinaryDictionary.TARGETMAP_HEADER, BinaryDictionary.VERSION);-- final int numSourceIds = lastSourceId + 1;- out.writeVInt(targetMapEndOffset); // <-- size of main array- out.writeVInt(numSourceIds + 1); // <-- size of offset array (+ 1 more entry)- int prev = 0, sourceId = 0;- for (int ofs = 0; ofs < targetMapEndOffset; ofs++) {- final int val = targetMap[ofs], delta = val - prev;- assert delta >= 0;- if (ofs == targetMapOffsets[sourceId]) {- out.writeVInt((delta << 1) | 0x01);- sourceId++;- } else {- out.writeVInt((delta << 1));- }- prev += delta;- }- if (sourceId != numSourceIds) {- throw new IllegalStateException(- "sourceId:" + sourceId + " != numSourceIds:" + numSourceIds);- }- }- }-- private void writePosDict(Path path) throws IOException {- Files.createDirectories(path.getParent());- try (OutputStream os = Files.newOutputStream(path);- OutputStream bos = new BufferedOutputStream(os)) {- final DataOutput out = new OutputStreamDataOutput(bos);- CodecUtil.writeHeader(out, BinaryDictionary.POSDICT_HEADER, BinaryDictionary.VERSION);- out.writeVInt(posDict.size());- for (String s : posDict) {- if (s == null) {- out.writeByte((byte) POS.Tag.UNKNOWN.ordinal());- } else {- String[] data = CSVUtil.parse(s);- if (data.length != 2) {- throw new IllegalArgumentException(- "Malformed pos/inflection: " + s + "; expected 2 characters");- }- out.writeByte((byte) POS.Tag.valueOf(data[0]).ordinal());+ @Override+ protected void writePosDict(OutputStream bos, DataOutput out) throws IOException {+ out.writeVInt(posDict.size());+ for (String s : posDict) {+ if (s == null) {+ out.writeByte((byte) POS.Tag.UNKNOWN.ordinal());+ } else {+ String[] data = CSVUtil.parse(s);+ if (data.length != 2) {+ throw new IllegalArgumentException(+ "Malformed pos/inflection: " + s + "; expected 2 characters");}+ out.writeByte((byte) POS.Tag.valueOf(data[0]).ordinal());}}}-- private void writeDictionary(Path path) throws IOException {- Files.createDirectories(path.getParent());- try (OutputStream os = Files.newOutputStream(path);- OutputStream bos = new BufferedOutputStream(os)) {- final DataOutput out = new OutputStreamDataOutput(bos);- CodecUtil.writeHeader(out, BinaryDictionary.DICT_HEADER, BinaryDictionary.VERSION);- out.writeVInt(buffer.position());- final WritableByteChannel channel = Channels.newChannel(bos);- // Write Buffer- buffer.flip(); // set position to 0, set limit to current position- channel.write(buffer);- assert buffer.remaining() == 0L;- }- }}diff --git a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/util/TokenInfoDictionaryWriter.java b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/util/TokenInfoDictionaryWriter.javaindex 316e1b6232a..78a9acaff8a 100644--- a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/util/TokenInfoDictionaryWriter.java+++ b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/util/TokenInfoDictionaryWriter.java@@ -20,14 +20,16 @@ import java.io.IOException;import java.nio.file.Files;import java.nio.file.Path;import java.util.Objects;+import org.apache.lucene.analysis.ko.dict.DictionaryConstants;import org.apache.lucene.analysis.ko.dict.TokenInfoDictionary;+import org.apache.lucene.analysis.morph.BinaryDictionaryWriter;import org.apache.lucene.util.fst.FST;-class TokenInfoDictionaryWriter extends BinaryDictionaryWriter {+class TokenInfoDictionaryWriter extends BinaryDictionaryWriter<TokenInfoDictionary> {private FST<Long> fst;TokenInfoDictionaryWriter(int size) {- super(TokenInfoDictionary.class, size);+ super(TokenInfoDictionary.class, new TokenInfoDictionaryEntryWriter(size));}public void setFST(FST<Long> fst) {@@ -35,9 +37,19 @@ class TokenInfoDictionaryWriter extends BinaryDictionaryWriter {this.fst = fst;}+ @Override+ protected void addMapping(int sourceId, int wordId) {+ super.addMapping(sourceId, wordId);+ }+@Overridepublic void write(Path baseDir) throws IOException {- super.write(baseDir);+ super.write(+ baseDir,+ DictionaryConstants.TARGETMAP_HEADER,+ DictionaryConstants.POSDICT_HEADER,+ DictionaryConstants.DICT_HEADER,+ DictionaryConstants.VERSION);writeFST(baseDir.resolve(getBaseFileName() + TokenInfoDictionary.FST_FILENAME_SUFFIX));}diff --git a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/util/UnknownDictionaryWriter.java b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/util/UnknownDictionaryWriter.javaindex 97b525d60f0..82285e1c78b 100644--- a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/util/UnknownDictionaryWriter.java+++ b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/util/UnknownDictionaryWriter.java@@ -19,20 +19,29 @@ package org.apache.lucene.analysis.ko.util;import java.io.IOException;import java.nio.file.Path;import org.apache.lucene.analysis.ko.dict.CharacterDefinition;+import org.apache.lucene.analysis.ko.dict.DictionaryConstants;import org.apache.lucene.analysis.ko.dict.UnknownDictionary;+import org.apache.lucene.analysis.morph.BinaryDictionaryWriter;+import org.apache.lucene.analysis.morph.CharacterDefinitionWriter;-class UnknownDictionaryWriter extends BinaryDictionaryWriter {+class UnknownDictionaryWriter extends BinaryDictionaryWriter<UnknownDictionary> {- private final CharacterDefinitionWriter characterDefinition = new CharacterDefinitionWriter();+ private final org.apache.lucene.analysis.morph.CharacterDefinitionWriter<CharacterDefinition>+ characterDefinition =+ new CharacterDefinitionWriter<>(+ CharacterDefinition.class,+ CharacterDefinition.DEFAULT,+ CharacterDefinition.CLASS_COUNT,+ CharacterDefinition::lookupCharacterClass);public UnknownDictionaryWriter(int size) {- super(UnknownDictionary.class, size);+ super(UnknownDictionary.class, new TokenInfoDictionaryEntryWriter(size));}@Overridepublic int put(String[] entry) {// Get wordId of current entry- int wordId = buffer.position();+ int wordId = entryWriter.currentPosition();// Put entryint result = super.put(entry);@@ -59,7 +68,13 @@ class UnknownDictionaryWriter extends BinaryDictionaryWriter {@Overridepublic void write(Path baseDir) throws IOException {- super.write(baseDir);- characterDefinition.write(baseDir);+ super.write(+ baseDir,+ DictionaryConstants.TARGETMAP_HEADER,+ DictionaryConstants.POSDICT_HEADER,+ DictionaryConstants.DICT_HEADER,+ DictionaryConstants.VERSION);+ characterDefinition.write(+ baseDir, DictionaryConstants.CHARDEF_HEADER, DictionaryConstants.VERSION);}}diff --git a/lucene/analysis/nori/src/test/org/apache/lucene/analysis/ko/dict/TestExternalDictionary.java b/lucene/analysis/nori/src/test/org/apache/lucene/analysis/ko/dict/TestExternalDictionary.javaindex 5f8edab8934..2ba7cee6a37 100644--- a/lucene/analysis/nori/src/test/org/apache/lucene/analysis/ko/dict/TestExternalDictionary.java+++ b/lucene/analysis/nori/src/test/org/apache/lucene/analysis/ko/dict/TestExternalDictionary.java@@ -16,10 +16,10 @@*/package org.apache.lucene.analysis.ko.dict;-import static org.apache.lucene.analysis.ko.dict.BinaryDictionary.DICT_FILENAME_SUFFIX;-import static org.apache.lucene.analysis.ko.dict.BinaryDictionary.POSDICT_FILENAME_SUFFIX;-import static org.apache.lucene.analysis.ko.dict.BinaryDictionary.TARGETMAP_FILENAME_SUFFIX;import static org.apache.lucene.analysis.ko.dict.TokenInfoDictionary.FST_FILENAME_SUFFIX;+import static org.apache.lucene.analysis.morph.BinaryDictionary.DICT_FILENAME_SUFFIX;+import static org.apache.lucene.analysis.morph.BinaryDictionary.POSDICT_FILENAME_SUFFIX;+import static org.apache.lucene.analysis.morph.BinaryDictionary.TARGETMAP_FILENAME_SUFFIX;import java.io.BufferedWriter;import java.nio.charset.StandardCharsets;diff --git a/lucene/analysis/nori/src/test/org/apache/lucene/analysis/ko/dict/TestTokenInfoDictionary.java b/lucene/analysis/nori/src/test/org/apache/lucene/analysis/ko/dict/TestTokenInfoDictionary.javaindex 39fc55065d9..6948c4aa851 100644--- a/lucene/analysis/nori/src/test/org/apache/lucene/analysis/ko/dict/TestTokenInfoDictionary.java+++ b/lucene/analysis/nori/src/test/org/apache/lucene/analysis/ko/dict/TestTokenInfoDictionary.java@@ -16,10 +16,10 @@*/package org.apache.lucene.analysis.ko.dict;-import static org.apache.lucene.analysis.ko.dict.BinaryDictionary.DICT_FILENAME_SUFFIX;-import static org.apache.lucene.analysis.ko.dict.BinaryDictionary.POSDICT_FILENAME_SUFFIX;-import static org.apache.lucene.analysis.ko.dict.BinaryDictionary.TARGETMAP_FILENAME_SUFFIX;import static org.apache.lucene.analysis.ko.dict.TokenInfoDictionary.FST_FILENAME_SUFFIX;+import static org.apache.lucene.analysis.morph.BinaryDictionary.DICT_FILENAME_SUFFIX;+import static org.apache.lucene.analysis.morph.BinaryDictionary.POSDICT_FILENAME_SUFFIX;+import static org.apache.lucene.analysis.morph.BinaryDictionary.TARGETMAP_FILENAME_SUFFIX;import java.io.OutputStream;import java.io.OutputStreamWriter;@@ -141,13 +141,13 @@ public class TestTokenInfoDictionary extends LuceneTestCase {tid.getWordCost(wordId);- POS.Type type = tid.getPOSType(wordId);- POS.Tag leftPOS = tid.getLeftPOS(wordId);- POS.Tag rightPOS = tid.getRightPOS(wordId);+ POS.Type type = tid.getMorphAttributes().getPOSType(wordId);+ POS.Tag leftPOS = tid.getMorphAttributes().getLeftPOS(wordId);+ POS.Tag rightPOS = tid.getMorphAttributes().getRightPOS(wordId);if (type == POS.Type.MORPHEME) {assertSame(leftPOS, rightPOS);- String reading = tid.getReading(wordId);+ String reading = tid.getMorphAttributes().getReading(wordId);boolean isHanja = charDef.isHanja(surfaceForm.charAt(0));if (isHanja) {assertNotNull(reading);@@ -163,10 +163,11 @@ public class TestTokenInfoDictionary extends LuceneTestCase {assertSame(leftPOS, rightPOS);assertTrue(leftPOS == POS.Tag.NNG || rightPOS == POS.Tag.NNP);}- Dictionary.Morpheme[] decompound = tid.getMorphemes(wordId, chars, 0, chars.length);+ KoMorphData.Morpheme[] decompound =+ tid.getMorphAttributes().getMorphemes(wordId, chars, 0, chars.length);if (decompound != null) {int offset = 0;- for (Dictionary.Morpheme morph : decompound) {+ for (KoMorphData.Morpheme morph : decompound) {assertTrue(UnicodeUtil.validUTF16String(morph.surfaceForm));assertFalse(morph.surfaceForm.isEmpty());assertEquals(morph.surfaceForm.trim(), morph.surfaceForm);diff --git a/lucene/analysis/nori/src/test/org/apache/lucene/analysis/ko/dict/TestUserDictionary.java b/lucene/analysis/nori/src/test/org/apache/lucene/analysis/ko/dict/TestUserDictionary.javaindex 7f97d0520e8..8fb2827a336 100644--- a/lucene/analysis/nori/src/test/org/apache/lucene/analysis/ko/dict/TestUserDictionary.java+++ b/lucene/analysis/nori/src/test/org/apache/lucene/analysis/ko/dict/TestUserDictionary.java@@ -31,16 +31,16 @@ public class TestUserDictionary extends LuceneTestCase {char[] sArray = s.toCharArray();List<Integer> wordIds = dictionary.lookup(sArray, 0, s.length());assertEquals(1, wordIds.size());- assertNull(dictionary.getMorphemes(wordIds.get(0), sArray, 0, s.length()));+ assertNull(dictionary.getMorphAttributes().getMorphemes(wordIds.get(0), sArray, 0, s.length()));s = "세종시";sArray = s.toCharArray();wordIds = dictionary.lookup(sArray, 0, s.length());assertEquals(2, wordIds.size());- assertNull(dictionary.getMorphemes(wordIds.get(0), sArray, 0, s.length()));+ assertNull(dictionary.getMorphAttributes().getMorphemes(wordIds.get(0), sArray, 0, s.length()));- Dictionary.Morpheme[] decompound =- dictionary.getMorphemes(wordIds.get(1), sArray, 0, s.length());+ KoMorphData.Morpheme[] decompound =+ dictionary.getMorphAttributes().getMorphemes(wordIds.get(1), sArray, 0, s.length());assertNotNull(decompound);assertEquals(2, decompound.length);assertEquals(decompound[0].posTag, POS.Tag.NNG);@@ -52,7 +52,7 @@ public class TestUserDictionary extends LuceneTestCase {sArray = s.toCharArray();wordIds = dictionary.lookup(sArray, 0, s.length());assertEquals(1, wordIds.size());- assertNull(dictionary.getMorphemes(wordIds.get(0), sArray, 0, s.length()));+ assertNull(dictionary.getMorphAttributes().getMorphemes(wordIds.get(0), sArray, 0, s.length()));}@Test