Skip to content

Apache Lucene open-source search software

LUCENE-10393: Unify binary dictionary and dictionary writer in kuromoji and nori (#740)

Tomoko Uchidabd22f199dec5
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 435434aaf30..cda9ad881cc 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -49,6 +49,9 @@ Other
* LUCENE-10253: The @BadApple annotation has been removed from the test
framework. (Adrien Grand)
+* LUCENE-10393: Unify binary dictionary and dictionary writer in Kuromoji and Nori.
+ (Tomoko Uchida, Robert Muir)
+
======================= Lucene 9.2.0 =======================
API Changes
---------------------
diff --git a/lucene/analysis/common/src/java/module-info.java b/lucene/analysis/common/src/java/module-info.java
index ec837952e7b..f8e7bf08554 100644
--- a/lucene/analysis/common/src/java/module-info.java
+++ b/lucene/analysis/common/src/java/module-info.java
@@ -60,6 +60,7 @@ module org.apache.lucene.analysis.common {
exports org.apache.lucene.analysis.lv;
exports org.apache.lucene.analysis.minhash;
exports org.apache.lucene.analysis.miscellaneous;
+ exports org.apache.lucene.analysis.morph;
exports org.apache.lucene.analysis.ne;
exports org.apache.lucene.analysis.ngram;
exports org.apache.lucene.analysis.nl;
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/BinaryDictionary.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/BinaryDictionary.java
new file mode 100644
index 00000000000..f72ed913d35
--- /dev/null
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/BinaryDictionary.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.analysis.morph;
+
+import java.io.BufferedInputStream;
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+import java.nio.channels.Channels;
+import java.nio.channels.ReadableByteChannel;
+import org.apache.lucene.codecs.CodecUtil;
+import org.apache.lucene.store.DataInput;
+import org.apache.lucene.store.InputStreamDataInput;
+import org.apache.lucene.util.IOSupplier;
+import org.apache.lucene.util.IntsRef;
+
+/** Abstract dictionary base class. */
+public abstract class BinaryDictionary<T extends MorphData> implements Dictionary<T> {
+ public static final String DICT_FILENAME_SUFFIX = "$buffer.dat";
+ public static final String TARGETMAP_FILENAME_SUFFIX = "$targetMap.dat";
+ public static final String POSDICT_FILENAME_SUFFIX = "$posDict.dat";
+
+ private final int[] targetMapOffsets, targetMap;
+ protected final ByteBuffer buffer;
+
+ protected BinaryDictionary(
+ IOSupplier<InputStream> targetMapResource,
+ IOSupplier<InputStream> dictResource,
+ String targetMapCodecHeader,
+ String dictCodecHeader,
+ int dictCodecVersion)
+ throws IOException {
+ try (InputStream mapIS = new BufferedInputStream(targetMapResource.get())) {
+ final DataInput in = new InputStreamDataInput(mapIS);
+ CodecUtil.checkHeader(in, targetMapCodecHeader, dictCodecVersion, dictCodecVersion);
+ this.targetMap = new int[in.readVInt()];
+ this.targetMapOffsets = new int[in.readVInt()];
+ populateTargetMap(in, this.targetMap, this.targetMapOffsets);
+ }
+
+ // no buffering here, as we load in one large buffer
+ try (InputStream dictIS = dictResource.get()) {
+ final DataInput in = new InputStreamDataInput(dictIS);
+ CodecUtil.checkHeader(in, dictCodecHeader, dictCodecVersion, dictCodecVersion);
+ final int size = in.readVInt();
+ final ByteBuffer tmpBuffer = ByteBuffer.allocateDirect(size);
+ final ReadableByteChannel channel = Channels.newChannel(dictIS);
+ final int read = channel.read(tmpBuffer);
+ if (read != size) {
+ throw new EOFException("Cannot read whole dictionary");
+ }
+ this.buffer = tmpBuffer.asReadOnlyBuffer();
+ }
+ }
+
+ private static void populateTargetMap(DataInput in, int[] targetMap, int[] targetMapOffsets)
+ throws IOException {
+ int accum = 0, sourceId = 0;
+ for (int ofs = 0; ofs < targetMap.length; ofs++) {
+ final int val = in.readVInt();
+ if ((val & 0x01) != 0) {
+ targetMapOffsets[sourceId] = ofs;
+ sourceId++;
+ }
+ accum += val >>> 1;
+ targetMap[ofs] = accum;
+ }
+ if (sourceId + 1 != targetMapOffsets.length)
+ throw new IOException(
+ "targetMap file format broken; targetMap.length="
+ + targetMap.length
+ + ", targetMapOffsets.length="
+ + targetMapOffsets.length
+ + ", sourceId="
+ + sourceId);
+ targetMapOffsets[sourceId] = targetMap.length;
+ }
+
+ public void lookupWordIds(int sourceId, IntsRef ref) {
+ ref.ints = targetMap;
+ ref.offset = targetMapOffsets[sourceId];
+ // targetMapOffsets always has one more entry pointing behind last:
+ ref.length = targetMapOffsets[sourceId + 1] - ref.offset;
+ }
+}
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/BinaryDictionaryWriter.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/BinaryDictionaryWriter.java
new file mode 100644
index 00000000000..bb0a5c096b3
--- /dev/null
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/BinaryDictionaryWriter.java
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.analysis.morph;
+
+import java.io.BufferedOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import org.apache.lucene.codecs.CodecUtil;
+import org.apache.lucene.store.DataOutput;
+import org.apache.lucene.store.OutputStreamDataOutput;
+import org.apache.lucene.util.ArrayUtil;
+
+/** Abstract base dictionary writer class. */
+public abstract class BinaryDictionaryWriter<T extends BinaryDictionary<? extends MorphData>> {
+ private final Class<T> implClazz;
+ private int targetMapEndOffset = 0, lastWordId = -1, lastSourceId = -1;
+ private int[] targetMap = new int[8192];
+ private int[] targetMapOffsets = new int[8192];
+ protected final DictionaryEntryWriter entryWriter;
+
+ protected BinaryDictionaryWriter(Class<T> implClazz, DictionaryEntryWriter entryWriter) {
+ this.implClazz = implClazz;
+ this.entryWriter = entryWriter;
+ }
+
+ /**
+ * put the entry in map
+ *
+ * @return current position of buffer, which will be wordId of next entry
+ */
+ public int put(String[] entry) {
+ return entryWriter.putEntry(entry);
+ }
+
+ /**
+ * Write whole dictionary in a directory.
+ *
+ * @throws IOException if an I/O error occurs writing the dictionary files
+ */
+ public abstract void write(Path baseDir) throws IOException;
+
+ protected void addMapping(int sourceId, int wordId) {
+ if (wordId <= lastWordId) {
+ throw new IllegalStateException(
+ "words out of order: " + wordId + " vs lastID: " + lastWordId);
+ }
+
+ if (sourceId > lastSourceId) {
+ targetMapOffsets = ArrayUtil.grow(targetMapOffsets, sourceId + 1);
+ for (int i = lastSourceId + 1; i <= sourceId; i++) {
+ targetMapOffsets[i] = targetMapEndOffset;
+ }
+ } else if (sourceId != lastSourceId) {
+ throw new IllegalStateException(
+ "source ids not in increasing order: lastSourceId="
+ + lastSourceId
+ + " vs sourceId="
+ + sourceId);
+ }
+
+ targetMap = ArrayUtil.grow(targetMap, targetMapEndOffset + 1);
+ targetMap[targetMapEndOffset] = wordId;
+ targetMapEndOffset++;
+
+ lastSourceId = sourceId;
+ lastWordId = wordId;
+ }
+
+ /**
+ * Write dictionary in file Dictionary format is: [Size of dictionary(int)], [entry:{left
+ * id(short)}{right id(short)}{word cost(short)}{length of pos info(short)}{pos info(char)}],
+ * [entry...], [entry...].....
+ *
+ * @throws IOException if an I/O error occurs writing the dictionary files
+ */
+ protected void write(
+ Path baseDir,
+ String targetMapCodecHeader,
+ String posDictCodecHeader,
+ String dictCodecHeader,
+ int dictCodecVersion)
+ throws IOException {
+ final String baseName = getBaseFileName();
+ entryWriter.writeDictionary(
+ baseDir.resolve(baseName + BinaryDictionary.DICT_FILENAME_SUFFIX),
+ dictCodecHeader,
+ dictCodecVersion);
+ entryWriter.writePosDict(
+ baseDir.resolve(baseName + BinaryDictionary.POSDICT_FILENAME_SUFFIX),
+ posDictCodecHeader,
+ dictCodecVersion);
+ writeTargetMap(
+ baseDir.resolve(baseName + BinaryDictionary.TARGETMAP_FILENAME_SUFFIX),
+ targetMapCodecHeader,
+ dictCodecVersion);
+ }
+
+ protected final String getBaseFileName() {
+ return implClazz.getName().replace('.', '/');
+ }
+
+ // TODO: maybe this int[] should instead be the output to the FST...
+ private void writeTargetMap(Path path, String targetMapCodecHeader, int dictCodecVersion)
+ throws IOException {
+ Files.createDirectories(path.getParent());
+ try (OutputStream os = Files.newOutputStream(path);
+ OutputStream bos = new BufferedOutputStream(os)) {
+ final DataOutput out = new OutputStreamDataOutput(bos);
+ CodecUtil.writeHeader(out, targetMapCodecHeader, dictCodecVersion);
+
+ final int numSourceIds = lastSourceId + 1;
+ out.writeVInt(targetMapEndOffset); // <-- size of main array
+ out.writeVInt(numSourceIds + 1); // <-- size of offset array (+ 1 more entry)
+ int prev = 0, sourceId = 0;
+ for (int ofs = 0; ofs < targetMapEndOffset; ofs++) {
+ final int val = targetMap[ofs], delta = val - prev;
+ assert delta >= 0;
+ if (ofs == targetMapOffsets[sourceId]) {
+ out.writeVInt((delta << 1) | 0x01);
+ sourceId++;
+ } else {
+ out.writeVInt((delta << 1));
+ }
+ prev += delta;
+ }
+ if (sourceId != numSourceIds) {
+ throw new IllegalStateException(
+ "sourceId:" + sourceId + " != numSourceIds:" + numSourceIds);
+ }
+ }
+ }
+}
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/CharacterDefinition.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/CharacterDefinition.java
new file mode 100644
index 00000000000..29c1b012213
--- /dev/null
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/CharacterDefinition.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.analysis.morph;
+
+import java.io.IOException;
+import java.io.InputStream;
+import org.apache.lucene.codecs.CodecUtil;
+import org.apache.lucene.store.DataInput;
+import org.apache.lucene.store.InputStreamDataInput;
+import org.apache.lucene.util.IOSupplier;
+
+/** Character category data. */
+public abstract class CharacterDefinition {
+
+ public static final String FILENAME_SUFFIX = ".dat";
+
+ protected final byte[] characterCategoryMap = new byte[0x10000];
+ private final boolean[] invokeMap;
+ private final boolean[] groupMap;
+
+ protected CharacterDefinition(
+ IOSupplier<InputStream> charDefResource,
+ String charDefCodecHeader,
+ int charDefCodecVersion,
+ int classCount)
+ throws IOException {
+ try (InputStream is = charDefResource.get()) {
+ final DataInput in = new InputStreamDataInput(is);
+ CodecUtil.checkHeader(in, charDefCodecHeader, charDefCodecVersion, charDefCodecVersion);
+ in.readBytes(characterCategoryMap, 0, characterCategoryMap.length);
+ this.invokeMap = new boolean[classCount];
+ this.groupMap = new boolean[classCount];
+ for (int i = 0; i < classCount; i++) {
+ final byte b = in.readByte();
+ invokeMap[i] = (b & 0x01) != 0;
+ groupMap[i] = (b & 0x02) != 0;
+ }
+ }
+ }
+
+ public byte getCharacterClass(char c) {
+ return characterCategoryMap[c];
+ }
+
+ public boolean isInvoke(char c) {
+ return invokeMap[characterCategoryMap[c]];
+ }
+
+ public boolean isGroup(char c) {
+ return groupMap[characterCategoryMap[c]];
+ }
+
+ /** Functional interface to lookup character class */
+ @FunctionalInterface
+ public interface LookupCharacterClass {
+ /** looks up character class for given class name */
+ byte lookupCharacterClass(String characterClassName);
+ }
+}
diff --git a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/util/CharacterDefinitionWriter.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/CharacterDefinitionWriter.java
similarity index 57%
rename from lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/util/CharacterDefinitionWriter.java
rename to lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/CharacterDefinitionWriter.java
index cbd3f768685..1f4fc3d13a9 100644
--- a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/util/CharacterDefinitionWriter.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/CharacterDefinitionWriter.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.analysis.ko.util;
+package org.apache.lucene.analysis.morph;
import java.io.BufferedOutputStream;
import java.io.IOException;
@@ -22,21 +22,33 @@ import java.io.OutputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Arrays;
-import org.apache.lucene.analysis.ko.dict.CharacterDefinition;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.store.DataOutput;
import org.apache.lucene.store.OutputStreamDataOutput;
-final class CharacterDefinitionWriter {
+/** Writes character definition file */
+public final class CharacterDefinitionWriter<T extends CharacterDefinition> {
- private final byte[] characterCategoryMap = new byte[0x10000];
+ private final Class<T> implClazz;
- private final boolean[] invokeMap = new boolean[CharacterDefinition.CLASS_COUNT];
- private final boolean[] groupMap = new boolean[CharacterDefinition.CLASS_COUNT];
+ private final byte[] characterCategoryMap = new byte[0x10000];
+ private final int classCount;
+ private final boolean[] invokeMap;
+ private final boolean[] groupMap;
+ private final CharacterDefinition.LookupCharacterClass lookupCharClass;
/** Constructor for building. TODO: remove write access */
- CharacterDefinitionWriter() {
- Arrays.fill(characterCategoryMap, CharacterDefinition.DEFAULT);
+ public CharacterDefinitionWriter(
+ Class<T> implClazz,
+ byte defaultValue,
+ int classCount,
+ CharacterDefinition.LookupCharacterClass lookupCharClass) {
+ this.implClazz = implClazz;
+ Arrays.fill(characterCategoryMap, defaultValue);
+ this.invokeMap = new boolean[classCount];
+ this.groupMap = new boolean[classCount];
+ this.classCount = classCount;
+ this.lookupCharClass = lookupCharClass;
}
/**
@@ -45,7 +57,7 @@ final class CharacterDefinitionWriter {
* @param codePoint code point
* @param characterClassName character class name
*/
- void putCharacterCategory(int codePoint, String characterClassName) {
+ public void putCharacterCategory(int codePoint, String characterClassName) {
characterClassName = characterClassName.split(" ")[0]; // use first
// category
// class
@@ -54,27 +66,29 @@ final class CharacterDefinitionWriter {
if (codePoint == 0x30FB) {
characterClassName = "SYMBOL";
}
- characterCategoryMap[codePoint] = CharacterDefinition.lookupCharacterClass(characterClassName);
+ characterCategoryMap[codePoint] = lookupCharClass.lookupCharacterClass(characterClassName);
}
- void putInvokeDefinition(String characterClassName, int invoke, int group, int length) {
- final byte characterClass = CharacterDefinition.lookupCharacterClass(characterClassName);
+ public void putInvokeDefinition(String characterClassName, int invoke, int group, int length) {
+ final byte characterClass = lookupCharClass.lookupCharacterClass(characterClassName);
invokeMap[characterClass] = invoke == 1;
groupMap[characterClass] = group == 1;
// TODO: length def ignored
}
- public void write(Path baseDir) throws IOException {
- Path path =
- baseDir.resolve(
- CharacterDefinition.class.getName().replace('.', '/')
- + CharacterDefinition.FILENAME_SUFFIX);
+ private String getBaseFileName() {
+ return implClazz.getName().replace('.', '/');
+ }
+
+ public void write(Path baseDir, String charDefCodecHeader, int charDefCodecVersion)
+ throws IOException {
+ Path path = baseDir.resolve(getBaseFileName() + CharacterDefinition.FILENAME_SUFFIX);
Files.createDirectories(path.getParent());
try (OutputStream os = new BufferedOutputStream(Files.newOutputStream(path))) {
final DataOutput out = new OutputStreamDataOutput(os);
- CodecUtil.writeHeader(out, CharacterDefinition.HEADER, CharacterDefinition.VERSION);
+ CodecUtil.writeHeader(out, charDefCodecHeader, charDefCodecVersion);
out.writeBytes(characterCategoryMap, 0, characterCategoryMap.length);
- for (int i = 0; i < CharacterDefinition.CLASS_COUNT; i++) {
+ for (int i = 0; i < classCount; i++) {
final byte b = (byte) ((invokeMap[i] ? 0x01 : 0x00) | (groupMap[i] ? 0x02 : 0x00));
out.writeByte(b);
}
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/ConnectionCosts.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/ConnectionCosts.java
new file mode 100644
index 00000000000..70c58161196
--- /dev/null
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/ConnectionCosts.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.analysis.morph;
+
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+import org.apache.lucene.codecs.CodecUtil;
+import org.apache.lucene.store.DataInput;
+import org.apache.lucene.store.InputStreamDataInput;
+import org.apache.lucene.util.IOSupplier;
+
+/** n-gram connection cost data */
+public abstract class ConnectionCosts {
+
+ public static final String FILENAME_SUFFIX = ".dat";
+
+ private final ByteBuffer buffer;
+ private final int forwardSize;
+
+ protected ConnectionCosts(
+ IOSupplier<InputStream> connectionCostResource,
+ String connectionCostsCodecHeader,
+ int dictCodecVersion)
+ throws IOException {
+ try (InputStream is = new BufferedInputStream(connectionCostResource.get())) {
+ final DataInput in = new InputStreamDataInput(is);
+ CodecUtil.checkHeader(in, connectionCostsCodecHeader, dictCodecVersion, dictCodecVersion);
+ forwardSize = in.readVInt();
+ int backwardSize = in.readVInt();
+ int size = forwardSize * backwardSize;
+
+ // copy the matrix into a direct byte buffer
+ final ByteBuffer tmpBuffer = ByteBuffer.allocateDirect(size * 2);
+ int accum = 0;
+ for (int j = 0; j < backwardSize; j++) {
+ for (int i = 0; i < forwardSize; i++) {
+ accum += in.readZInt();
+ tmpBuffer.putShort((short) accum);
+ }
+ }
+ buffer = tmpBuffer.asReadOnlyBuffer();
+ }
+ }
+
+ public int get(int forwardId, int backwardId) {
+ // map 2d matrix into a single dimension short array
+ int offset = (backwardId * forwardSize + forwardId) * 2;
+ return buffer.getShort(offset);
+ }
+}
diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/util/ConnectionCostsWriter.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/ConnectionCostsWriter.java
similarity index 77%
rename from lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/util/ConnectionCostsWriter.java
rename to lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/ConnectionCostsWriter.java
index 8d081f77bfc..f4f0a51c8a8 100644
--- a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/util/ConnectionCostsWriter.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/ConnectionCostsWriter.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.analysis.ja.util;
+package org.apache.lucene.analysis.morph;
import java.io.BufferedOutputStream;
import java.io.IOException;
@@ -22,20 +22,23 @@ import java.io.OutputStream;
import java.nio.ByteBuffer;
import java.nio.file.Files;
import java.nio.file.Path;
-import org.apache.lucene.analysis.ja.dict.ConnectionCosts;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.store.DataOutput;
import org.apache.lucene.store.OutputStreamDataOutput;
-final class ConnectionCostsWriter {
+/** Writes connection costs */
+public final class ConnectionCostsWriter<T extends ConnectionCosts> {
+ private final Class<T> implClazz;
private final ByteBuffer
costs; // array is backward IDs first since get is called using the same backward ID
// consecutively. maybe doesn't matter.
private final int forwardSize;
private final int backwardSize;
+
/** Constructor for building. TODO: remove write access */
- ConnectionCostsWriter(int forwardSize, int backwardSize) {
+ public ConnectionCostsWriter(Class<T> implClazz, int forwardSize, int backwardSize) {
+ this.implClazz = implClazz;
this.forwardSize = forwardSize;
this.backwardSize = backwardSize;
this.costs = ByteBuffer.allocateDirect(2 * backwardSize * forwardSize);
@@ -46,14 +49,18 @@ final class ConnectionCostsWriter {
costs.putShort(offset, (short) cost);
}
- public void write(Path baseDir) throws IOException {
+ private String getBaseFileName() {
+ return implClazz.getName().replace('.', '/');
+ }
+
+ public void write(Path baseDir, String connectionCostsCodecHeader, int dictCodecVersion)
+ throws IOException {
Files.createDirectories(baseDir);
- String fileName =
- ConnectionCosts.class.getName().replace('.', '/') + ConnectionCosts.FILENAME_SUFFIX;
+ String fileName = getBaseFileName() + ConnectionCosts.FILENAME_SUFFIX;
try (OutputStream os = Files.newOutputStream(baseDir.resolve(fileName));
OutputStream bos = new BufferedOutputStream(os)) {
final DataOutput out = new OutputStreamDataOutput(bos);
- CodecUtil.writeHeader(out, ConnectionCosts.HEADER, ConnectionCosts.VERSION);
+ CodecUtil.writeHeader(out, connectionCostsCodecHeader, dictCodecVersion);
out.writeVInt(forwardSize);
out.writeVInt(backwardSize);
int last = 0;
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/Dictionary.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/Dictionary.java
new file mode 100644
index 00000000000..5ff714c63f7
--- /dev/null
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/Dictionary.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.analysis.morph;
+
+/** High-level dictionary interface for morphological analyzers. */
+public interface Dictionary<T extends MorphData> {
+ /**
+ * Get left id of specified word
+ *
+ * @return left id
+ */
+ default int getLeftId(int morphId) {
+ return getMorphAttributes().getLeftId(morphId);
+ }
+
+ /**
+ * Get right id of specified word
+ *
+ * @return right id
+ */
+ default int getRightId(int morphId) {
+ return getMorphAttributes().getRightId(morphId);
+ }
+
+ /**
+ * Get word cost of specified word
+ *
+ * @return word's cost
+ */
+ default int getWordCost(int morphId) {
+ return getMorphAttributes().getWordCost(morphId);
+ }
+
+ T getMorphAttributes();
+}
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/DictionaryEntryWriter.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/DictionaryEntryWriter.java
new file mode 100644
index 00000000000..76fb44ea603
--- /dev/null
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/DictionaryEntryWriter.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.analysis.morph;
+
+import java.io.BufferedOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.nio.channels.Channels;
+import java.nio.channels.WritableByteChannel;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.lucene.codecs.CodecUtil;
+import org.apache.lucene.store.DataOutput;
+import org.apache.lucene.store.OutputStreamDataOutput;
+
+/** Abstract writer class to write dictionary entries. */
+public abstract class DictionaryEntryWriter {
+
+ protected ByteBuffer buffer;
+ protected final List<String> posDict;
+
+ protected DictionaryEntryWriter(int size) {
+ this.buffer = ByteBuffer.allocate(size);
+ this.posDict = new ArrayList<>();
+ }
+
+ /** Writes an entry. */
+ protected abstract int putEntry(String[] entry);
+
+ /** Flush POS dictionary data. */
+ protected abstract void writePosDict(OutputStream bos, DataOutput out) throws IOException;
+
+ void writePosDict(Path path, String posDictCodecHeader, int dictCodecVersion) throws IOException {
+ Files.createDirectories(path.getParent());
+ try (OutputStream os = Files.newOutputStream(path);
+ OutputStream bos = new BufferedOutputStream(os)) {
+ final DataOutput out = new OutputStreamDataOutput(bos);
+ CodecUtil.writeHeader(out, posDictCodecHeader, dictCodecVersion);
+ writePosDict(bos, out);
+ }
+ }
+
+ void writeDictionary(Path path, String dictCodecHeader, int dictCodecVersion) throws IOException {
+ Files.createDirectories(path.getParent());
+ try (OutputStream os = Files.newOutputStream(path);
+ OutputStream bos = new BufferedOutputStream(os)) {
+ final DataOutput out = new OutputStreamDataOutput(bos);
+ CodecUtil.writeHeader(out, dictCodecHeader, dictCodecVersion);
+ out.writeVInt(buffer.position());
+ final WritableByteChannel channel = Channels.newChannel(bos);
+ // Write Buffer
+ buffer.flip(); // set position to 0, set limit to current position
+ channel.write(buffer);
+ assert buffer.remaining() == 0L;
+ }
+ }
+
+ /** Returns current word id. */
+ public int currentPosition() {
+ return buffer.position();
+ }
+}
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/MorphData.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/MorphData.java
new file mode 100644
index 00000000000..1cafe9af71a
--- /dev/null
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/MorphData.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.analysis.morph;
+
+/** High-level interface that represents morphological information in a dictionary */
+public interface MorphData {
+ /**
+ * Get left id of specified word
+ *
+ * @return left id
+ */
+ int getLeftId(int morphId);
+
+ /**
+ * Get right id of specified word
+ *
+ * @return right id
+ */
+ int getRightId(int morphId);
+
+ /**
+ * Get word cost of specified word
+ *
+ * @return word's cost
+ */
+ int getWordCost(int morphId);
+}
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/package-info.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/package-info.java
new file mode 100644
index 00000000000..7e53020588f
--- /dev/null
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/package-info.java
@@ -0,0 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/** Abstract classes for morphological analyzers. */
+package org.apache.lucene.analysis.morph;
diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/GraphvizFormatter.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/GraphvizFormatter.java
index de3748b2666..4538daa362f 100644
--- a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/GraphvizFormatter.java
+++ b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/GraphvizFormatter.java
@@ -21,7 +21,8 @@ import java.util.Map;
import org.apache.lucene.analysis.ja.JapaneseTokenizer.Position;
import org.apache.lucene.analysis.ja.JapaneseTokenizer.WrappedPositionArray;
import org.apache.lucene.analysis.ja.dict.ConnectionCosts;
-import org.apache.lucene.analysis.ja.dict.Dictionary;
+import org.apache.lucene.analysis.ja.dict.JaMorphData;
+import org.apache.lucene.analysis.morph.Dictionary;
// TODO: would be nice to show 2nd best path in a diff't
// color...
@@ -140,7 +141,7 @@ public class GraphvizFormatter {
attrs = "";
}
- final Dictionary dict = tok.getDict(posData.backType[idx]);
+ final Dictionary<? extends JaMorphData> dict = tok.getDict(posData.backType[idx]);
final int wordCost = dict.getWordCost(posData.backID[idx]);
final int bgCost =
costs.get(
diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizer.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizer.java
index 47cb8d19297..a933525b2f0 100644
--- a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizer.java
+++ b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizer.java
@@ -28,12 +28,13 @@ import java.util.List;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.ja.dict.CharacterDefinition;
import org.apache.lucene.analysis.ja.dict.ConnectionCosts;
-import org.apache.lucene.analysis.ja.dict.Dictionary;
+import org.apache.lucene.analysis.ja.dict.JaMorphData;
import org.apache.lucene.analysis.ja.dict.TokenInfoDictionary;
import org.apache.lucene.analysis.ja.dict.TokenInfoFST;
import org.apache.lucene.analysis.ja.dict.UnknownDictionary;
import org.apache.lucene.analysis.ja.dict.UserDictionary;
import org.apache.lucene.analysis.ja.tokenattributes.*;
+import org.apache.lucene.analysis.morph.Dictionary;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
@@ -116,7 +117,8 @@ public final class JapaneseTokenizer extends Tokenizer {
private static final int MAX_UNKNOWN_WORD_LENGTH = 1024;
private static final int MAX_BACKTRACE_GAP = 1024;
- private final EnumMap<Type, Dictionary> dictionaryMap = new EnumMap<>(Type.class);
+ private final EnumMap<Type, Dictionary<? extends JaMorphData>> dictionaryMap =
+ new EnumMap<>(Type.class);
private final TokenInfoFST fst;
private final TokenInfoDictionary dictionary;
@@ -494,10 +496,15 @@ public final class JapaneseTokenizer extends Tokenizer {
}
private void add(
- Dictionary dict, Position fromPosData, int endPos, int wordID, Type type, boolean addPenalty)
+ JaMorphData morphAtts,
+ Position fromPosData,
+ int endPos,
+ int wordID,
+ Type type,
+ boolean addPenalty)
throws IOException {
- final int wordCost = dict.getWordCost(wordID);
- final int leftID = dict.getLeftId(wordID);
+ final int wordCost = morphAtts.getWordCost(wordID);
+ final int leftID = morphAtts.getLeftId(wordID);
int leastCost = Integer.MAX_VALUE;
int leastIDX = -1;
assert fromPosData.count > 0;
@@ -560,7 +567,7 @@ public final class JapaneseTokenizer extends Tokenizer {
// positions.get(endPos).add(leastCost, dict.getRightId(wordID), fromPosData.pos, leastIDX,
// wordID, type);
- assert leftID == dict.getRightId(wordID);
+ assert leftID == morphAtts.getRightId(wordID);
positions.get(endPos).add(leastCost, leftID, fromPosData.pos, leastIDX, wordID, type);
}
@@ -895,7 +902,7 @@ public final class JapaneseTokenizer extends Tokenizer {
+ (posAhead + 1));
}
add(
- userDictionary,
+ userDictionary.getMorphAttributes(),
posData,
posAhead + 1,
output + arc.nextFinalOutput().intValue(),
@@ -948,7 +955,7 @@ public final class JapaneseTokenizer extends Tokenizer {
}
for (int ofs = 0; ofs < wordIdRef.length; ofs++) {
add(
- dictionary,
+ dictionary.getMorphAttributes(),
posData,
posAhead + 1,
wordIdRef.ints[wordIdRef.offset + ofs],
@@ -1004,7 +1011,7 @@ public final class JapaneseTokenizer extends Tokenizer {
}
for (int ofs = 0; ofs < wordIdRef.length; ofs++) {
add(
- unkDictionary,
+ unkDictionary.getMorphAttributes(),
posData,
posData.pos + unknownWordLength,
wordIdRef.ints[wordIdRef.offset + ofs],
@@ -1126,7 +1133,7 @@ public final class JapaneseTokenizer extends Tokenizer {
final int pathCost = posData.costs[bestStartIDX];
for (int forwardArcIDX = 0; forwardArcIDX < posData.forwardCount; forwardArcIDX++) {
final Type forwardType = posData.forwardType[forwardArcIDX];
- final Dictionary dict2 = getDict(forwardType);
+ final Dictionary<? extends JaMorphData> dict2 = getDict(forwardType);
final int wordID = posData.forwardID[forwardArcIDX];
final int toPos = posData.forwardPos[forwardArcIDX];
final int newCost =
@@ -1169,7 +1176,7 @@ public final class JapaneseTokenizer extends Tokenizer {
+ toPos);
}
add(
- getDict(forwardType),
+ getDict(forwardType).getMorphAttributes(),
posData,
toPos,
posData.forwardID[forwardArcIDX],
@@ -1184,7 +1191,7 @@ public final class JapaneseTokenizer extends Tokenizer {
// yet another lattice data structure
private static final class Lattice {
char[] fragment;
- EnumMap<Type, Dictionary> dictionaryMap;
+ EnumMap<Type, Dictionary<? extends JaMorphData>> dictionaryMap;
boolean useEOS;
int rootCapacity = 0;
@@ -1296,7 +1303,7 @@ public final class JapaneseTokenizer extends Tokenizer {
nodeLeftID[node] = 0;
nodeRightID[node] = 0;
} else {
- Dictionary dic = dictionaryMap.get(dicType);
+ Dictionary<? extends JaMorphData> dic = dictionaryMap.get(dicType);
nodeWordCost[node] = dic.getWordCost(wordID);
nodeLeftID[node] = dic.getLeftId(wordID);
nodeRightID[node] = dic.getRightId(wordID);
@@ -1338,7 +1345,7 @@ public final class JapaneseTokenizer extends Tokenizer {
void setup(
char[] fragment,
- EnumMap<Type, Dictionary> dictionaryMap,
+ EnumMap<Type, Dictionary<? extends JaMorphData>> dictionaryMap,
WrappedPositionArray positions,
int prevOffset,
int endOffset,
@@ -1565,7 +1572,7 @@ public final class JapaneseTokenizer extends Tokenizer {
right - left,
Type.USER,
lattice.rootBase + left,
- userDictionary));
+ userDictionary.getMorphAttributes()));
// Output compound
int current = 0;
for (int j = 1; j < wordIDAndLength.length; j++) {
@@ -1579,7 +1586,7 @@ public final class JapaneseTokenizer extends Tokenizer {
len,
Type.USER,
lattice.rootBase + current + left,
- userDictionary));
+ userDictionary.getMorphAttributes()));
}
current += len;
}
@@ -1592,7 +1599,7 @@ public final class JapaneseTokenizer extends Tokenizer {
right - left,
type,
lattice.rootBase + left,
- getDict(type)));
+ getDict(type).getMorphAttributes()));
}
}
}
@@ -1921,7 +1928,7 @@ public final class JapaneseTokenizer extends Tokenizer {
length,
backType,
backPos,
- getDict(backType));
+ getDict(backType).getMorphAttributes());
// Redirect our backtrace to 2nd best:
bestIDX = leastIDX;
@@ -1980,7 +1987,7 @@ public final class JapaneseTokenizer extends Tokenizer {
altToken = null;
}
- final Dictionary dict = getDict(backType);
+ final Dictionary<? extends JaMorphData> dict = getDict(backType);
if (backType == Type.USER) {
@@ -2000,7 +2007,7 @@ public final class JapaneseTokenizer extends Tokenizer {
len,
Type.USER,
current + backPos,
- dict));
+ dict.getMorphAttributes()));
if (VERBOSE) {
System.out.println(" add USER token=" + pending.get(pending.size() - 1));
}
@@ -2037,14 +2044,16 @@ public final class JapaneseTokenizer extends Tokenizer {
charLen,
Type.UNKNOWN,
backPos + i,
- unkDictionary));
+ unkDictionary.getMorphAttributes()));
unigramTokenCount++;
}
}
backCount += unigramTokenCount;
} else if (!discardPunctuation || length == 0 || !isPunctuation(fragment[offset])) {
- pending.add(new Token(backID, fragment, offset, length, backType, backPos, dict));
+ pending.add(
+ new Token(
+ backID, fragment, offset, length, backType, backPos, dict.getMorphAttributes()));
if (VERBOSE) {
System.out.println(" add token=" + pending.get(pending.size() - 1));
}
@@ -2073,7 +2082,7 @@ public final class JapaneseTokenizer extends Tokenizer {
positions.freeBefore(endPos);
}
- Dictionary getDict(Type type) {
+ Dictionary<? extends JaMorphData> getDict(Type type) {
return dictionaryMap.get(type);
}
diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/Token.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/Token.java
index 5a10b50817b..e9ae9490125 100644
--- a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/Token.java
+++ b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/Token.java
@@ -17,11 +17,11 @@
package org.apache.lucene.analysis.ja;
import org.apache.lucene.analysis.ja.JapaneseTokenizer.Type;
-import org.apache.lucene.analysis.ja.dict.Dictionary;
+import org.apache.lucene.analysis.ja.dict.JaMorphData;
/** Analyzed token with morphological data from its dictionary. */
public class Token {
- private final Dictionary dictionary;
+ private final JaMorphData morphAtts;
private final int wordId;
@@ -41,14 +41,14 @@ public class Token {
int length,
Type type,
int position,
- Dictionary dictionary) {
+ JaMorphData morphAtts) {
this.wordId = wordId;
this.surfaceForm = surfaceForm;
this.offset = offset;
this.length = length;
this.type = type;
this.position = position;
- this.dictionary = dictionary;
+ this.morphAtts = morphAtts;
}
@Override
@@ -66,7 +66,7 @@ public class Token {
+ " wordId="
+ wordId
+ " leftID="
- + dictionary.getLeftId(wordId)
+ + morphAtts.getLeftId(wordId)
+ ")";
}
@@ -92,32 +92,32 @@ public class Token {
/** @return reading. null if token doesn't have reading. */
public String getReading() {
- return dictionary.getReading(wordId, surfaceForm, offset, length);
+ return morphAtts.getReading(wordId, surfaceForm, offset, length);
}
/** @return pronunciation. null if token doesn't have pronunciation. */
public String getPronunciation() {
- return dictionary.getPronunciation(wordId, surfaceForm, offset, length);
+ return morphAtts.getPronunciation(wordId, surfaceForm, offset, length);
}
/** @return part of speech. */
public String getPartOfSpeech() {
- return dictionary.getPartOfSpeech(wordId);
+ return morphAtts.getPartOfSpeech(wordId);
}
/** @return inflection type or null */
public String getInflectionType() {
- return dictionary.getInflectionType(wordId);
+ return morphAtts.getInflectionType(wordId);
}
/** @return inflection form or null */
public String getInflectionForm() {
- return dictionary.getInflectionForm(wordId);
+ return morphAtts.getInflectionForm(wordId);
}
/** @return base form or null if token is not inflected */
public String getBaseForm() {
- return dictionary.getBaseForm(wordId, surfaceForm, offset, length);
+ return morphAtts.getBaseForm(wordId, surfaceForm, offset, length);
}
/**
diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/CharacterDefinition.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/CharacterDefinition.java
index 36bb825bc2d..be29ebbe367 100644
--- a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/CharacterDefinition.java
+++ b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/CharacterDefinition.java
@@ -16,20 +16,13 @@
*/
package org.apache.lucene.analysis.ja.dict;
-import java.io.BufferedInputStream;
import java.io.IOException;
import java.io.InputStream;
-import org.apache.lucene.codecs.CodecUtil;
-import org.apache.lucene.store.DataInput;
-import org.apache.lucene.store.InputStreamDataInput;
import org.apache.lucene.util.IOUtils;
/** Character category data. */
-public final class CharacterDefinition {
-
- public static final String FILENAME_SUFFIX = ".dat";
- public static final String HEADER = "kuromoji_cd";
- public static final int VERSION = 1;
+public final class CharacterDefinition
+ extends org.apache.lucene.analysis.morph.CharacterDefinition {
public static final int CLASS_COUNT = CharacterClass.values().length;
@@ -49,11 +42,6 @@ public final class CharacterDefinition {
KANJINUMERIC;
}
- private final byte[] characterCategoryMap = new byte[0x10000];
-
- private final boolean[] invokeMap = new boolean[CLASS_COUNT];
- private final boolean[] groupMap = new boolean[CLASS_COUNT];
-
// the classes:
public static final byte NGRAM = (byte) CharacterClass.NGRAM.ordinal();
public static final byte DEFAULT = (byte) CharacterClass.DEFAULT.ordinal();
@@ -69,16 +57,11 @@ public final class CharacterDefinition {
public static final byte KANJINUMERIC = (byte) CharacterClass.KANJINUMERIC.ordinal();
private CharacterDefinition() throws IOException {
- try (InputStream is = new BufferedInputStream(getClassResource())) {
- final DataInput in = new InputStreamDataInput(is);
- CodecUtil.checkHeader(in, HEADER, VERSION, VERSION);
- in.readBytes(characterCategoryMap, 0, characterCategoryMap.length);
- for (int i = 0; i < CLASS_COUNT; i++) {
- final byte b = in.readByte();
- invokeMap[i] = (b & 0x01) != 0;
- groupMap[i] = (b & 0x02) != 0;
- }
- }
+ super(
+ CharacterDefinition::getClassResource,
+ DictionaryConstants.CHARDEF_HEADER,
+ DictionaryConstants.VERSION,
+ CharacterClass.values().length);
}
private static InputStream getClassResource() throws IOException {
@@ -87,18 +70,6 @@ public final class CharacterDefinition {
CharacterDefinition.class.getResourceAsStream(resourcePath), resourcePath);
}
- public byte getCharacterClass(char c) {
- return characterCategoryMap[c];
- }
-
- public boolean isInvoke(char c) {
- return invokeMap[characterCategoryMap[c]];
- }
-
- public boolean isGroup(char c) {
- return groupMap[characterCategoryMap[c]];
- }
-
public boolean isKanji(char c) {
final byte characterClass = characterCategoryMap[c];
return characterClass == KANJI || characterClass == KANJINUMERIC;
diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/ConnectionCosts.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/ConnectionCosts.java
index dc5fabe6c54..c11b9ee716a 100644
--- a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/ConnectionCosts.java
+++ b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/ConnectionCosts.java
@@ -16,27 +16,15 @@
*/
package org.apache.lucene.analysis.ja.dict;
-import java.io.BufferedInputStream;
import java.io.IOException;
import java.io.InputStream;
-import java.nio.ByteBuffer;
import java.nio.file.Files;
import java.nio.file.Path;
-import org.apache.lucene.codecs.CodecUtil;
-import org.apache.lucene.store.DataInput;
-import org.apache.lucene.store.InputStreamDataInput;
import org.apache.lucene.util.IOSupplier;
import org.apache.lucene.util.IOUtils;
/** n-gram connection cost data */
-public final class ConnectionCosts {
-
- public static final String FILENAME_SUFFIX = ".dat";
- public static final String HEADER = "kuromoji_cc";
- public static final int VERSION = 1;
-
- private final ByteBuffer buffer;
- private final int forwardSize;
+public final class ConnectionCosts extends org.apache.lucene.analysis.morph.ConnectionCosts {
/**
* Create a {@link ConnectionCosts} from an external resource path.
@@ -53,24 +41,8 @@ public final class ConnectionCosts {
}
private ConnectionCosts(IOSupplier<InputStream> connectionCostResource) throws IOException {
- try (InputStream is = new BufferedInputStream(connectionCostResource.get())) {
- final DataInput in = new InputStreamDataInput(is);
- CodecUtil.checkHeader(in, HEADER, VERSION, VERSION);
- forwardSize = in.readVInt();
- int backwardSize = in.readVInt();
- int size = forwardSize * backwardSize;
-
- // copy the matrix into a direct byte buffer
- final ByteBuffer tmpBuffer = ByteBuffer.allocateDirect(size * 2);
- int accum = 0;
- for (int j = 0; j < backwardSize; j++) {
- for (int i = 0; i < forwardSize; i++) {
- accum += in.readZInt();
- tmpBuffer.putShort((short) accum);
- }
- }
- buffer = tmpBuffer.asReadOnlyBuffer();
- }
+ super(
+ connectionCostResource, DictionaryConstants.CONN_COSTS_HEADER, DictionaryConstants.VERSION);
}
private static InputStream getClassResource() throws IOException {
@@ -79,12 +51,6 @@ public final class ConnectionCosts {
ConnectionCosts.class.getResourceAsStream(resourcePath), resourcePath);
}
- public int get(int forwardId, int backwardId) {
- // map 2d matrix into a single dimension short array
- int offset = (backwardId * forwardSize + forwardId) * 2;
- return buffer.getShort(offset);
- }
-
public static ConnectionCosts getInstance() {
return SingletonHolder.INSTANCE;
}
diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/DictionaryConstants.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/DictionaryConstants.java
new file mode 100644
index 00000000000..3046beef3e6
--- /dev/null
+++ b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/DictionaryConstants.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.analysis.ja.dict;
+
+/** Dictionary constants */
+public final class DictionaryConstants {
+ /** Codec header of the dictionary file. */
+ public static final String DICT_HEADER = "kuromoji_dict";
+ /** Codec header of the dictionary mapping file. */
+ public static final String TARGETMAP_HEADER = "kuromoji_dict_map";
+ /** Codec header of the POS dictionary file. */
+ public static final String POSDICT_HEADER = "kuromoji_dict_pos";
+ /** Codec header of the connection costs. */
+ public static final String CONN_COSTS_HEADER = "kuromoji_cc";
+ /** Codec header of the character definition file. */
+ public static final String CHARDEF_HEADER = "kuromoji_cd";
+ /** Codec version of the binary dictionary */
+ public static final int VERSION = 1;
+}
diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/Dictionary.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/JaMorphData.java
similarity index 60%
rename from lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/Dictionary.java
rename to lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/JaMorphData.java
index cfe11b30cd9..8865f6ad09f 100644
--- a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/Dictionary.java
+++ b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/JaMorphData.java
@@ -16,71 +16,49 @@
*/
package org.apache.lucene.analysis.ja.dict;
-/** Dictionary interface for retrieving morphological data by id. */
-public interface Dictionary {
-
- public static final String INTERNAL_SEPARATOR = "\u0000";
-
- /**
- * Get left id of specified word
- *
- * @return left id
- */
- public int getLeftId(int wordId);
-
- /**
- * Get right id of specified word
- *
- * @return right id
- */
- public int getRightId(int wordId);
-
- /**
- * Get word cost of specified word
- *
- * @return word's cost
- */
- public int getWordCost(int wordId);
+import org.apache.lucene.analysis.morph.MorphData;
+/** Represents Japanese morphological information. */
+public interface JaMorphData extends MorphData {
/**
* Get Part-Of-Speech of tokens
*
- * @param wordId word ID of token
+ * @param morphId word ID of token
* @return Part-Of-Speech of the token
*/
- public String getPartOfSpeech(int wordId);
+ String getPartOfSpeech(int morphId);
/**
* Get reading of tokens
*
- * @param wordId word ID of token
+ * @param morphId word ID of token
* @return Reading of the token
*/
- public String getReading(int wordId, char[] surface, int off, int len);
+ String getReading(int morphId, char[] surface, int off, int len);
/**
* Get base form of word
*
- * @param wordId word ID of token
+ * @param morphId word ID of token
* @return Base form (only different for inflected words, otherwise null)
*/
- public String getBaseForm(int wordId, char[] surface, int off, int len);
+ String getBaseForm(int morphId, char[] surface, int off, int len);
/**
* Get pronunciation of tokens
*
- * @param wordId word ID of token
+ * @param morphId word ID of token
* @return Pronunciation of the token
*/
- public String getPronunciation(int wordId, char[] surface, int off, int len);
+ String getPronunciation(int morphId, char[] surface, int off, int len);
/**
* Get inflection type of tokens
*
- * @param wordId word ID of token
+ * @param morphId word ID of token
* @return inflection type, or null
*/
- public String getInflectionType(int wordId);
+ String getInflectionType(int morphId);
/**
* Get inflection form of tokens
@@ -88,7 +66,7 @@ public interface Dictionary {
* @param wordId word ID of token
* @return inflection form, or null
*/
- public String getInflectionForm(int wordId);
+ String getInflectionForm(int wordId);
// TODO: maybe we should have a optimal method, a non-typesafe
// 'getAdditionalData' if other dictionaries like unidic have additional data
}
diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/TokenInfoDictionary.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/TokenInfoDictionary.java
index d9e50f86a75..c769587829b 100644
--- a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/TokenInfoDictionary.java
+++ b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/TokenInfoDictionary.java
@@ -21,6 +21,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Path;
+import org.apache.lucene.analysis.morph.BinaryDictionary;
import org.apache.lucene.store.DataInput;
import org.apache.lucene.store.InputStreamDataInput;
import org.apache.lucene.util.IOSupplier;
@@ -32,11 +33,12 @@ import org.apache.lucene.util.fst.PositiveIntOutputs;
* Binary dictionary implementation for a known-word dictionary model: Words are encoded into an FST
* mapping to a list of wordIDs.
*/
-public final class TokenInfoDictionary extends BinaryDictionary {
+public final class TokenInfoDictionary extends BinaryDictionary<TokenInfoMorphData> {
public static final String FST_FILENAME_SUFFIX = "$fst.dat";
private final TokenInfoFST fst;
+ private final TokenInfoMorphData morphAtts;
/**
* Create a {@link TokenInfoDictionary} from an external resource path.
@@ -70,7 +72,14 @@ public final class TokenInfoDictionary extends BinaryDictionary {
IOSupplier<InputStream> dictResource,
IOSupplier<InputStream> fstResource)
throws IOException {
- super(targetMapResource, posResource, dictResource);
+ super(
+ targetMapResource,
+ dictResource,
+ DictionaryConstants.TARGETMAP_HEADER,
+ DictionaryConstants.DICT_HEADER,
+ DictionaryConstants.VERSION);
+ this.morphAtts = new TokenInfoMorphData(buffer, posResource);
+
FST<Long> fst;
try (InputStream is = new BufferedInputStream(fstResource.get())) {
DataInput in = new InputStreamDataInput(is);
@@ -86,6 +95,11 @@ public final class TokenInfoDictionary extends BinaryDictionary {
TokenInfoDictionary.class.getResourceAsStream(resourcePath), resourcePath);
}
+ @Override
+ public TokenInfoMorphData getMorphAttributes() {
+ return morphAtts;
+ }
+
public TokenInfoFST getFST() {
return fst;
}
diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/BinaryDictionary.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/TokenInfoMorphData.java
similarity index 56%
rename from lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/BinaryDictionary.java
rename to lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/TokenInfoMorphData.java
index 9ad0d813778..d743a7faca6 100644
--- a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/BinaryDictionary.java
+++ b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/TokenInfoMorphData.java
@@ -17,95 +17,37 @@
package org.apache.lucene.analysis.ja.dict;
import java.io.BufferedInputStream;
-import java.io.EOFException;
import java.io.IOException;
import java.io.InputStream;
import java.nio.ByteBuffer;
-import java.nio.channels.Channels;
-import java.nio.channels.ReadableByteChannel;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.store.DataInput;
import org.apache.lucene.store.InputStreamDataInput;
import org.apache.lucene.util.IOSupplier;
-import org.apache.lucene.util.IntsRef;
-/** Base class for a binary-encoded in-memory dictionary. */
-public abstract class BinaryDictionary implements Dictionary {
-
- public static final String DICT_FILENAME_SUFFIX = "$buffer.dat";
- public static final String TARGETMAP_FILENAME_SUFFIX = "$targetMap.dat";
- public static final String POSDICT_FILENAME_SUFFIX = "$posDict.dat";
-
- public static final String DICT_HEADER = "kuromoji_dict";
- public static final String TARGETMAP_HEADER = "kuromoji_dict_map";
- public static final String POSDICT_HEADER = "kuromoji_dict_pos";
- public static final int VERSION = 1;
+/** Morphological information for system dictionary. */
+public class TokenInfoMorphData implements JaMorphData {
private final ByteBuffer buffer;
- private final int[] targetMapOffsets, targetMap;
private final String[] posDict;
private final String[] inflTypeDict;
private final String[] inflFormDict;
- protected BinaryDictionary(
- IOSupplier<InputStream> targetMapResource,
- IOSupplier<InputStream> posResource,
- IOSupplier<InputStream> dictResource)
- throws IOException {
- try (InputStream mapIS = new BufferedInputStream(targetMapResource.get())) {
- final DataInput in = new InputStreamDataInput(mapIS);
- CodecUtil.checkHeader(in, TARGETMAP_HEADER, VERSION, VERSION);
- this.targetMap = new int[in.readVInt()];
- this.targetMapOffsets = new int[in.readVInt()];
- populateTargetMap(in, this.targetMap, this.targetMapOffsets);
- }
-
+ TokenInfoMorphData(ByteBuffer buffer, IOSupplier<InputStream> posResource) throws IOException {
+ this.buffer = buffer;
try (InputStream posIS = new BufferedInputStream(posResource.get())) {
final DataInput in = new InputStreamDataInput(posIS);
- CodecUtil.checkHeader(in, POSDICT_HEADER, VERSION, VERSION);
+ CodecUtil.checkHeader(
+ in,
+ DictionaryConstants.POSDICT_HEADER,
+ DictionaryConstants.VERSION,
+ DictionaryConstants.VERSION);
final int posSize = in.readVInt();
this.posDict = new String[posSize];
this.inflTypeDict = new String[posSize];
this.inflFormDict = new String[posSize];
- populatePosDict(in, posSize, this.posDict, this.inflTypeDict, this.inflFormDict);
- }
-
- // no buffering here, as we load in one large buffer
- try (InputStream dictIS = dictResource.get()) {
- final DataInput in = new InputStreamDataInput(dictIS);
- CodecUtil.checkHeader(in, DICT_HEADER, VERSION, VERSION);
- final int size = in.readVInt();
- final ByteBuffer tmpBuffer = ByteBuffer.allocateDirect(size);
- final ReadableByteChannel channel = Channels.newChannel(dictIS);
- final int read = channel.read(tmpBuffer);
- if (read != size) {
- throw new EOFException("Cannot read whole dictionary");
- }
- this.buffer = tmpBuffer.asReadOnlyBuffer();
- }
- }
-
- private static void populateTargetMap(DataInput in, int[] targetMap, int[] targetMapOffsets)
- throws IOException {
- int accum = 0, sourceId = 0;
- for (int ofs = 0; ofs < targetMap.length; ofs++) {
- final int val = in.readVInt();
- if ((val & 0x01) != 0) {
- targetMapOffsets[sourceId] = ofs;
- sourceId++;
- }
- accum += val >>> 1;
- targetMap[ofs] = accum;
+ populatePosDict(in, posSize, posDict, inflTypeDict, inflFormDict);
}
- if (sourceId + 1 != targetMapOffsets.length)
- throw new IOException(
- "targetMap file format broken; targetMap.length="
- + targetMap.length
- + ", targetMapOffsets.length="
- + targetMapOffsets.length
- + ", sourceId="
- + sourceId);
- targetMapOffsets[sourceId] = targetMap.length;
}
private static void populatePosDict(
@@ -125,32 +67,25 @@ public abstract class BinaryDictionary implements Dictionary {
}
}
- public void lookupWordIds(int sourceId, IntsRef ref) {
- ref.ints = targetMap;
- ref.offset = targetMapOffsets[sourceId];
- // targetMapOffsets always has one more entry pointing behind last:
- ref.length = targetMapOffsets[sourceId + 1] - ref.offset;
- }
-
@Override
- public int getLeftId(int wordId) {
- return (buffer.getShort(wordId) & 0xffff) >>> 3;
+ public int getLeftId(int morphId) {
+ return (buffer.getShort(morphId) & 0xffff) >>> 3;
}
@Override
- public int getRightId(int wordId) {
- return (buffer.getShort(wordId) & 0xffff) >>> 3;
+ public int getRightId(int morphId) {
+ return (buffer.getShort(morphId) & 0xffff) >>> 3;
}
@Override
- public int getWordCost(int wordId) {
- return buffer.getShort(wordId + 2); // Skip id
+ public int getWordCost(int morphId) {
+ return buffer.getShort(morphId + 2); // Skip id
}
@Override
- public String getBaseForm(int wordId, char[] surfaceForm, int off, int len) {
- if (hasBaseFormData(wordId)) {
- int offset = baseFormOffset(wordId);
+ public String getBaseForm(int morphId, char[] surfaceForm, int off, int len) {
+ if (hasBaseFormData(morphId)) {
+ int offset = baseFormOffset(morphId);
int data = buffer.get(offset++) & 0xff;
int prefix = data >>> 4;
int suffix = data & 0xF;
@@ -166,9 +101,9 @@ public abstract class BinaryDictionary implements Dictionary {
}
@Override
- public String getReading(int wordId, char[] surface, int off, int len) {
- if (hasReadingData(wordId)) {
- int offset = readingOffset(wordId);
+ public String getReading(int morphId, char[] surface, int off, int len) {
+ if (hasReadingData(morphId)) {
+ int offset = readingOffset(morphId);
int readingData = buffer.get(offset++) & 0xff;
return readString(offset, readingData >>> 1, (readingData & 1) == 1);
} else {
@@ -187,24 +122,24 @@ public abstract class BinaryDictionary implements Dictionary {
}
@Override
- public String getPartOfSpeech(int wordId) {
- return posDict[getLeftId(wordId)];
+ public String getPartOfSpeech(int morphId) {
+ return posDict[getLeftId(morphId)];
}
@Override
- public String getPronunciation(int wordId, char[] surface, int off, int len) {
- if (hasPronunciationData(wordId)) {
- int offset = pronunciationOffset(wordId);
+ public String getPronunciation(int morphId, char[] surface, int off, int len) {
+ if (hasPronunciationData(morphId)) {
+ int offset = pronunciationOffset(morphId);
int pronunciationData = buffer.get(offset++) & 0xff;
return readString(offset, pronunciationData >>> 1, (pronunciationData & 1) == 1);
} else {
- return getReading(wordId, surface, off, len); // same as the reading
+ return getReading(morphId, surface, off, len); // same as the reading
}
}
@Override
- public String getInflectionType(int wordId) {
- return inflTypeDict[getLeftId(wordId)];
+ public String getInflectionType(int morphId) {
+ return inflTypeDict[getLeftId(morphId)];
}
@Override
@@ -212,10 +147,6 @@ public abstract class BinaryDictionary implements Dictionary {
return inflFormDict[getLeftId(wordId)];
}
- private static int baseFormOffset(int wordId) {
- return wordId + 4;
- }
-
private int readingOffset(int wordId) {
int offset = baseFormOffset(wordId);
if (hasBaseFormData(wordId)) {
@@ -242,6 +173,10 @@ public abstract class BinaryDictionary implements Dictionary {
}
}
+ private static int baseFormOffset(int wordId) {
+ return wordId + 4;
+ }
+
private boolean hasBaseFormData(int wordId) {
return (buffer.getShort(wordId) & HAS_BASEFORM) != 0;
}
diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/UnknownDictionary.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/UnknownDictionary.java
index f9bdc62f49e..b4a9012e3ee 100644
--- a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/UnknownDictionary.java
+++ b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/UnknownDictionary.java
@@ -20,12 +20,15 @@ import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Path;
+import org.apache.lucene.analysis.morph.BinaryDictionary;
+import org.apache.lucene.util.IOSupplier;
import org.apache.lucene.util.IOUtils;
/** Dictionary for unknown-word handling. */
-public final class UnknownDictionary extends BinaryDictionary {
+public final class UnknownDictionary extends BinaryDictionary<UnknownMorphData> {
private final CharacterDefinition characterDefinition = CharacterDefinition.getInstance();
+ private final UnknownMorphData morphAtts;
/**
* Create a {@link UnknownDictionary} from an external resource path.
@@ -36,25 +39,44 @@ public final class UnknownDictionary extends BinaryDictionary {
* @throws IOException if resource was not found or broken
*/
public UnknownDictionary(Path targetMapFile, Path posDictFile, Path dictFile) throws IOException {
- super(
+ this(
() -> Files.newInputStream(targetMapFile),
() -> Files.newInputStream(posDictFile),
() -> Files.newInputStream(dictFile));
}
private UnknownDictionary() throws IOException {
- super(
+ this(
() -> getClassResource(TARGETMAP_FILENAME_SUFFIX),
() -> getClassResource(POSDICT_FILENAME_SUFFIX),
() -> getClassResource(DICT_FILENAME_SUFFIX));
}
+ private UnknownDictionary(
+ IOSupplier<InputStream> targetMapResource,
+ IOSupplier<InputStream> posResource,
+ IOSupplier<InputStream> dictResource)
+ throws IOException {
+ super(
+ targetMapResource,
+ dictResource,
+ DictionaryConstants.TARGETMAP_HEADER,
+ DictionaryConstants.DICT_HEADER,
+ DictionaryConstants.VERSION);
+ this.morphAtts = new UnknownMorphData(buffer, posResource);
+ }
+
private static InputStream getClassResource(String suffix) throws IOException {
final String resourcePath = UnknownDictionary.class.getSimpleName() + suffix;
return IOUtils.requireResourceNonNull(
UnknownDictionary.class.getResourceAsStream(resourcePath), resourcePath);
}
+ @Override
+ public UnknownMorphData getMorphAttributes() {
+ return morphAtts;
+ }
+
public int lookup(char[] text, int offset, int len) {
if (!characterDefinition.isGroup(text[offset])) {
return 1;
@@ -79,21 +101,6 @@ public final class UnknownDictionary extends BinaryDictionary {
return characterDefinition;
}
- @Override
- public String getReading(int wordId, char[] surface, int off, int len) {
- return null;
- }
-
- @Override
- public String getInflectionType(int wordId) {
- return null;
- }
-
- @Override
- public String getInflectionForm(int wordId) {
- return null;
- }
-
public static UnknownDictionary getInstance() {
return SingletonHolder.INSTANCE;
}
diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/UnknownMorphData.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/UnknownMorphData.java
new file mode 100644
index 00000000000..0810346e218
--- /dev/null
+++ b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/UnknownMorphData.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.analysis.ja.dict;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+import org.apache.lucene.util.IOSupplier;
+
+/** Morphological information for unk dictionary. */
+final class UnknownMorphData extends TokenInfoMorphData {
+ UnknownMorphData(ByteBuffer buffer, IOSupplier<InputStream> posResource) throws IOException {
+ super(buffer, posResource);
+ }
+
+ @Override
+ public String getReading(int morphId, char[] surface, int off, int len) {
+ return null;
+ }
+
+ @Override
+ public String getInflectionType(int morphId) {
+ return null;
+ }
+
+ @Override
+ public String getInflectionForm(int wordId) {
+ return null;
+ }
+}
diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/UserDictionary.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/UserDictionary.java
index 859806e5ef0..12dd27d9584 100644
--- a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/UserDictionary.java
+++ b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/UserDictionary.java
@@ -26,13 +26,16 @@ import java.util.List;
import java.util.Map;
import java.util.TreeMap;
import org.apache.lucene.analysis.ja.util.CSVUtil;
+import org.apache.lucene.analysis.morph.Dictionary;
import org.apache.lucene.util.IntsRefBuilder;
import org.apache.lucene.util.fst.FST;
import org.apache.lucene.util.fst.FSTCompiler;
import org.apache.lucene.util.fst.PositiveIntOutputs;
/** Class for building a User Dictionary. This class allows for custom segmentation of phrases. */
-public final class UserDictionary implements Dictionary {
+public final class UserDictionary implements Dictionary<UserMorphData> {
+
+ public static final String INTERNAL_SEPARATOR = "\u0000";
// phrase text -> phrase ID
private final TokenInfoFST fst;
@@ -41,15 +44,9 @@ public final class UserDictionary implements Dictionary {
private final int[][] segmentations;
// holds readings and POS, indexed by wordid
- private final String[] data;
-
- private static final int CUSTOM_DICTIONARY_WORD_ID_OFFSET = 100000000;
-
- public static final int WORD_COST = -100000;
+ private final UserMorphData morphAtts;
- public static final int LEFT_ID = 5;
-
- public static final int RIGHT_ID = 5;
+ static final int CUSTOM_DICTIONARY_WORD_ID_OFFSET = 100000000;
public static UserDictionary open(Reader reader) throws IOException {
@@ -150,10 +147,15 @@ public final class UserDictionary implements Dictionary {
ord++;
}
this.fst = new TokenInfoFST(fstCompiler.compile(), false);
- this.data = data.toArray(new String[data.size()]);
+ this.morphAtts = new UserMorphData(data.toArray(new String[0]));
this.segmentations = segmentations.toArray(new int[segmentations.size()][]);
}
+ @Override
+ public UserMorphData getMorphAttributes() {
+ return morphAtts;
+ }
+
/**
* Lookup words in text
*
@@ -222,78 +224,4 @@ public final class UserDictionary implements Dictionary {
public int[] lookupSegmentation(int phraseID) {
return segmentations[phraseID];
}
-
- @Override
- public int getLeftId(int wordId) {
- return LEFT_ID;
- }
-
- @Override
- public int getRightId(int wordId) {
- return RIGHT_ID;
- }
-
- @Override
- public int getWordCost(int wordId) {
- return WORD_COST;
- }
-
- @Override
- public String getReading(int wordId, char[] surface, int off, int len) {
- return getFeature(wordId, 0);
- }
-
- @Override
- public String getPartOfSpeech(int wordId) {
- return getFeature(wordId, 1);
- }
-
- @Override
- public String getBaseForm(int wordId, char[] surface, int off, int len) {
- return null; // TODO: add support?
- }
-
- @Override
- public String getPronunciation(int wordId, char[] surface, int off, int len) {
- return null; // TODO: add support?
- }
-
- @Override
- public String getInflectionType(int wordId) {
- return null; // TODO: add support?
- }
-
- @Override
- public String getInflectionForm(int wordId) {
- return null; // TODO: add support?
- }
-
- private String[] getAllFeaturesArray(int wordId) {
- String allFeatures = data[wordId - CUSTOM_DICTIONARY_WORD_ID_OFFSET];
- if (allFeatures == null) {
- return null;
- }
-
- return allFeatures.split(INTERNAL_SEPARATOR);
- }
-
- private String getFeature(int wordId, int... fields) {
- String[] allFeatures = getAllFeaturesArray(wordId);
- if (allFeatures == null) {
- return null;
- }
- StringBuilder sb = new StringBuilder();
- if (fields.length == 0) { // All features
- for (String feature : allFeatures) {
- sb.append(CSVUtil.quoteEscape(feature)).append(",");
- }
- } else if (fields.length == 1) { // One feature doesn't need to escape value
- sb.append(allFeatures[fields[0]]).append(",");
- } else {
- for (int field : fields) {
- sb.append(CSVUtil.quoteEscape(allFeatures[field])).append(",");
- }
- }
- return sb.deleteCharAt(sb.length() - 1).toString();
- }
}
diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/UserMorphData.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/UserMorphData.java
new file mode 100644
index 00000000000..1f6907052e0
--- /dev/null
+++ b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/UserMorphData.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.analysis.ja.dict;
+
+import static org.apache.lucene.analysis.ja.dict.UserDictionary.CUSTOM_DICTIONARY_WORD_ID_OFFSET;
+import static org.apache.lucene.analysis.ja.dict.UserDictionary.INTERNAL_SEPARATOR;
+
+import org.apache.lucene.analysis.ja.util.CSVUtil;
+
+/** Morphological information for user dictionary. */
+final class UserMorphData implements JaMorphData {
+ public static final int WORD_COST = -100000;
+ public static final int LEFT_ID = 5;
+ public static final int RIGHT_ID = 5;
+
+ // holds readings and POS, indexed by wordid
+ private final String[] data;
+
+ UserMorphData(String[] data) {
+ this.data = data;
+ }
+
+ @Override
+ public int getLeftId(int wordId) {
+ return LEFT_ID;
+ }
+
+ @Override
+ public int getRightId(int wordId) {
+ return RIGHT_ID;
+ }
+
+ @Override
+ public int getWordCost(int wordId) {
+ return WORD_COST;
+ }
+
+ @Override
+ public String getReading(int morphId, char[] surface, int off, int len) {
+ return getFeature(morphId, 0);
+ }
+
+ @Override
+ public String getPartOfSpeech(int morphId) {
+ return getFeature(morphId, 1);
+ }
+
+ @Override
+ public String getBaseForm(int morphId, char[] surface, int off, int len) {
+ return null; // TODO: add support?
+ }
+
+ @Override
+ public String getPronunciation(int morphId, char[] surface, int off, int len) {
+ return null; // TODO: add support?
+ }
+
+ @Override
+ public String getInflectionType(int morphId) {
+ return null; // TODO: add support?
+ }
+
+ @Override
+ public String getInflectionForm(int wordId) {
+ return null; // TODO: add support?
+ }
+
+ private String[] getAllFeaturesArray(int wordId) {
+ String allFeatures = data[wordId - CUSTOM_DICTIONARY_WORD_ID_OFFSET];
+ if (allFeatures == null) {
+ return null;
+ }
+
+ return allFeatures.split(INTERNAL_SEPARATOR);
+ }
+
+ private String getFeature(int wordId, int... fields) {
+ String[] allFeatures = getAllFeaturesArray(wordId);
+ if (allFeatures == null) {
+ return null;
+ }
+ StringBuilder sb = new StringBuilder();
+ if (fields.length == 0) { // All features
+ for (String feature : allFeatures) {
+ sb.append(CSVUtil.quoteEscape(feature)).append(",");
+ }
+ } else if (fields.length == 1) { // One feature doesn't need to escape value
+ sb.append(allFeatures[fields[0]]).append(",");
+ } else {
+ for (int field : fields) {
+ sb.append(CSVUtil.quoteEscape(allFeatures[field])).append(",");
+ }
+ }
+ return sb.deleteCharAt(sb.length() - 1).toString();
+ }
+}
diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/util/BinaryDictionaryWriter.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/util/BinaryDictionaryWriter.java
deleted file mode 100644
index bf157b9abb2..00000000000
--- a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/util/BinaryDictionaryWriter.java
+++ /dev/null
@@ -1,334 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.analysis.ja.util;
-
-import java.io.BufferedOutputStream;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.nio.ByteBuffer;
-import java.nio.channels.Channels;
-import java.nio.channels.WritableByteChannel;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.ArrayList;
-import org.apache.lucene.analysis.ja.dict.BinaryDictionary;
-import org.apache.lucene.codecs.CodecUtil;
-import org.apache.lucene.store.DataOutput;
-import org.apache.lucene.store.OutputStreamDataOutput;
-import org.apache.lucene.util.ArrayUtil;
-
-abstract class BinaryDictionaryWriter {
- private static final int ID_LIMIT = 8192;
-
- private final Class<? extends BinaryDictionary> implClazz;
- protected ByteBuffer buffer;
- private int targetMapEndOffset = 0, lastWordId = -1, lastSourceId = -1;
- private int[] targetMap = new int[8192];
- private int[] targetMapOffsets = new int[8192];
- private final ArrayList<String> posDict = new ArrayList<>();
-
- BinaryDictionaryWriter(Class<? extends BinaryDictionary> implClazz, int size) {
- this.implClazz = implClazz;
- buffer = ByteBuffer.allocateDirect(size);
- }
-
- /**
- * put the entry in map
- *
- * @return current position of buffer, which will be wordId of next entry
- */
- public int put(String[] entry) {
- short leftId = Short.parseShort(entry[1]);
- short rightId = Short.parseShort(entry[2]);
- short wordCost = Short.parseShort(entry[3]);
-
- StringBuilder sb = new StringBuilder();
-
- // build up the POS string
- for (int i = 4; i < 8; i++) {
- String part = entry[i];
- assert part.length() > 0;
- if (!"*".equals(part)) {
- if (sb.length() > 0) {
- sb.append('-');
- }
- sb.append(part);
- }
- }
-
- String posData = sb.toString();
- if (posData.isEmpty()) {
- throw new IllegalArgumentException("POS fields are empty");
- }
- sb.setLength(0);
- sb.append(CSVUtil.quoteEscape(posData));
- sb.append(',');
- if (!"*".equals(entry[8])) {
- sb.append(CSVUtil.quoteEscape(entry[8]));
- }
- sb.append(',');
- if (!"*".equals(entry[9])) {
- sb.append(CSVUtil.quoteEscape(entry[9]));
- }
- String fullPOSData = sb.toString();
-
- String baseForm = entry[10];
- String reading = entry[11];
- String pronunciation = entry[12];
-
- // extend buffer if necessary
- int left = buffer.remaining();
- // worst case: two short, 3 bytes, and features (all as utf-16)
- int worstCase = 4 + 3 + 2 * (baseForm.length() + reading.length() + pronunciation.length());
- if (worstCase > left) {
- ByteBuffer newBuffer =
- ByteBuffer.allocateDirect(ArrayUtil.oversize(buffer.limit() + worstCase - left, 1));
- buffer.flip();
- newBuffer.put(buffer);
- buffer = newBuffer;
- }
-
- int flags = 0;
- if (baseForm.isEmpty()) {
- throw new IllegalArgumentException("base form is empty");
- }
- if (!("*".equals(baseForm) || baseForm.equals(entry[0]))) {
- flags |= BinaryDictionary.HAS_BASEFORM;
- }
- if (!reading.equals(toKatakana(entry[0]))) {
- flags |= BinaryDictionary.HAS_READING;
- }
- if (!pronunciation.equals(reading)) {
- flags |= BinaryDictionary.HAS_PRONUNCIATION;
- }
-
- if (leftId != rightId) {
- throw new IllegalArgumentException("rightId != leftId: " + rightId + " " + leftId);
- }
- if (leftId >= ID_LIMIT) {
- throw new IllegalArgumentException("leftId >= " + ID_LIMIT + ": " + leftId);
- }
- // add pos mapping
- int toFill = 1 + leftId - posDict.size();
- for (int i = 0; i < toFill; i++) {
- posDict.add(null);
- }
-
- String existing = posDict.get(leftId);
- if (existing != null && existing.equals(fullPOSData) == false) {
- // TODO: test me
- throw new IllegalArgumentException("Multiple entries found for leftID=" + leftId);
- }
- posDict.set(leftId, fullPOSData);
-
- buffer.putShort((short) (leftId << 3 | flags));
- buffer.putShort(wordCost);
-
- if ((flags & BinaryDictionary.HAS_BASEFORM) != 0) {
- if (baseForm.length() >= 16) {
- throw new IllegalArgumentException("Length of base form " + baseForm + " is >= 16");
- }
- int shared = sharedPrefix(entry[0], baseForm);
- int suffix = baseForm.length() - shared;
- buffer.put((byte) (shared << 4 | suffix));
- for (int i = shared; i < baseForm.length(); i++) {
- buffer.putChar(baseForm.charAt(i));
- }
- }
-
- if ((flags & BinaryDictionary.HAS_READING) != 0) {
- if (isKatakana(reading)) {
- buffer.put((byte) (reading.length() << 1 | 1));
- writeKatakana(reading);
- } else {
- buffer.put((byte) (reading.length() << 1));
- for (int i = 0; i < reading.length(); i++) {
- buffer.putChar(reading.charAt(i));
- }
- }
- }
-
- if ((flags & BinaryDictionary.HAS_PRONUNCIATION) != 0) {
- // we can save 150KB here, but it makes the reader a little complicated.
- // int shared = sharedPrefix(reading, pronunciation);
- // buffer.put((byte) shared);
- // pronunciation = pronunciation.substring(shared);
- if (isKatakana(pronunciation)) {
- buffer.put((byte) (pronunciation.length() << 1 | 1));
- writeKatakana(pronunciation);
- } else {
- buffer.put((byte) (pronunciation.length() << 1));
- for (int i = 0; i < pronunciation.length(); i++) {
- buffer.putChar(pronunciation.charAt(i));
- }
- }
- }
-
- return buffer.position();
- }
-
- private boolean isKatakana(String s) {
- for (int i = 0; i < s.length(); i++) {
- char ch = s.charAt(i);
- if (ch < 0x30A0 || ch > 0x30FF) {
- return false;
- }
- }
- return true;
- }
-
- private void writeKatakana(String s) {
- for (int i = 0; i < s.length(); i++) {
- buffer.put((byte) (s.charAt(i) - 0x30A0));
- }
- }
-
- private String toKatakana(String s) {
- char[] text = new char[s.length()];
- for (int i = 0; i < s.length(); i++) {
- char ch = s.charAt(i);
- if (ch > 0x3040 && ch < 0x3097) {
- text[i] = (char) (ch + 0x60);
- } else {
- text[i] = ch;
- }
- }
- return new String(text);
- }
-
- private static int sharedPrefix(String left, String right) {
- int len = left.length() < right.length() ? left.length() : right.length();
- for (int i = 0; i < len; i++) if (left.charAt(i) != right.charAt(i)) return i;
- return len;
- }
-
- void addMapping(int sourceId, int wordId) {
- if (wordId <= lastWordId) {
- throw new IllegalStateException(
- "words out of order: " + wordId + " vs lastID: " + lastWordId);
- }
-
- if (sourceId > lastSourceId) {
- targetMapOffsets = ArrayUtil.grow(targetMapOffsets, sourceId + 1);
- for (int i = lastSourceId + 1; i <= sourceId; i++) {
- targetMapOffsets[i] = targetMapEndOffset;
- }
- } else if (sourceId != lastSourceId) {
- throw new IllegalStateException(
- "source ids not in increasing order: lastSourceId="
- + lastSourceId
- + " vs sourceId="
- + sourceId);
- }
-
- targetMap = ArrayUtil.grow(targetMap, targetMapEndOffset + 1);
- targetMap[targetMapEndOffset] = wordId;
- targetMapEndOffset++;
-
- lastSourceId = sourceId;
- lastWordId = wordId;
- }
-
- final String getBaseFileName() {
- return implClazz.getName().replace('.', '/');
- }
-
- /**
- * Write dictionary in file Dictionary format is: [Size of dictionary(int)], [entry:{left
- * id(short)}{right id(short)}{word cost(short)}{length of pos info(short)}{pos info(char)}],
- * [entry...], [entry...].....
- *
- * @throws IOException if an I/O error occurs writing the dictionary files
- */
- public void write(Path baseDir) throws IOException {
- final String baseName = getBaseFileName();
- writeDictionary(baseDir.resolve(baseName + BinaryDictionary.DICT_FILENAME_SUFFIX));
- writeTargetMap(baseDir.resolve(baseName + BinaryDictionary.TARGETMAP_FILENAME_SUFFIX));
- writePosDict(baseDir.resolve(baseName + BinaryDictionary.POSDICT_FILENAME_SUFFIX));
- }
-
- // TODO: maybe this int[] should instead be the output to the FST...
- private void writeTargetMap(Path path) throws IOException {
- Files.createDirectories(path.getParent());
- try (OutputStream os = Files.newOutputStream(path);
- OutputStream bos = new BufferedOutputStream(os)) {
- final DataOutput out = new OutputStreamDataOutput(bos);
- CodecUtil.writeHeader(out, BinaryDictionary.TARGETMAP_HEADER, BinaryDictionary.VERSION);
-
- final int numSourceIds = lastSourceId + 1;
- out.writeVInt(targetMapEndOffset); // <-- size of main array
- out.writeVInt(numSourceIds + 1); // <-- size of offset array (+ 1 more entry)
- int prev = 0, sourceId = 0;
- for (int ofs = 0; ofs < targetMapEndOffset; ofs++) {
- final int val = targetMap[ofs], delta = val - prev;
- assert delta >= 0;
- if (ofs == targetMapOffsets[sourceId]) {
- out.writeVInt((delta << 1) | 0x01);
- sourceId++;
- } else {
- out.writeVInt((delta << 1));
- }
- prev += delta;
- }
- if (sourceId != numSourceIds) {
- throw new IllegalStateException(
- "sourceId:" + sourceId + " != numSourceIds:" + numSourceIds);
- }
- }
- }
-
- private void writePosDict(Path path) throws IOException {
- Files.createDirectories(path.getParent());
- try (OutputStream os = Files.newOutputStream(path);
- OutputStream bos = new BufferedOutputStream(os)) {
- final DataOutput out = new OutputStreamDataOutput(bos);
- CodecUtil.writeHeader(out, BinaryDictionary.POSDICT_HEADER, BinaryDictionary.VERSION);
- out.writeVInt(posDict.size());
- for (String s : posDict) {
- if (s == null) {
- out.writeByte((byte) 0);
- out.writeByte((byte) 0);
- out.writeByte((byte) 0);
- } else {
- String[] data = CSVUtil.parse(s);
- if (data.length != 3) {
- throw new IllegalArgumentException(
- "Malformed pos/inflection: " + s + "; expected 3 characters");
- }
- out.writeString(data[0]);
- out.writeString(data[1]);
- out.writeString(data[2]);
- }
- }
- }
- }
-
- private void writeDictionary(Path path) throws IOException {
- Files.createDirectories(path.getParent());
- try (OutputStream os = Files.newOutputStream(path);
- OutputStream bos = new BufferedOutputStream(os)) {
- final DataOutput out = new OutputStreamDataOutput(bos);
- CodecUtil.writeHeader(out, BinaryDictionary.DICT_HEADER, BinaryDictionary.VERSION);
- out.writeVInt(buffer.position());
- final WritableByteChannel channel = Channels.newChannel(bos);
- // Write Buffer
- buffer.flip(); // set position to 0, set limit to current position
- channel.write(buffer);
- assert buffer.remaining() == 0L;
- }
- }
-}
diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/util/CharacterDefinitionWriter.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/util/CharacterDefinitionWriter.java
deleted file mode 100644
index 0afadeda83d..00000000000
--- a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/util/CharacterDefinitionWriter.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.analysis.ja.util;
-
-import java.io.BufferedOutputStream;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.Arrays;
-import org.apache.lucene.analysis.ja.dict.CharacterDefinition;
-import org.apache.lucene.codecs.CodecUtil;
-import org.apache.lucene.store.DataOutput;
-import org.apache.lucene.store.OutputStreamDataOutput;
-
-final class CharacterDefinitionWriter {
-
- private final byte[] characterCategoryMap = new byte[0x10000];
-
- private final boolean[] invokeMap = new boolean[CharacterDefinition.CLASS_COUNT];
- private final boolean[] groupMap = new boolean[CharacterDefinition.CLASS_COUNT];
-
- /** Constructor for building. TODO: remove write access */
- CharacterDefinitionWriter() {
- Arrays.fill(characterCategoryMap, CharacterDefinition.DEFAULT);
- }
-
- /**
- * Put mapping from unicode code point to character class.
- *
- * @param codePoint code point
- * @param characterClassName character class name
- */
- void putCharacterCategory(int codePoint, String characterClassName) {
- characterClassName = characterClassName.split(" ")[0]; // use first
- // category
- // class
-
- // Override Nakaguro
- if (codePoint == 0x30FB) {
- characterClassName = "SYMBOL";
- }
- characterCategoryMap[codePoint] = CharacterDefinition.lookupCharacterClass(characterClassName);
- }
-
- void putInvokeDefinition(String characterClassName, int invoke, int group, int length) {
- final byte characterClass = CharacterDefinition.lookupCharacterClass(characterClassName);
- invokeMap[characterClass] = invoke == 1;
- groupMap[characterClass] = group == 1;
- // TODO: length def ignored
- }
-
- public void write(Path baseDir) throws IOException {
- Path path =
- baseDir.resolve(
- CharacterDefinition.class.getName().replace('.', '/')
- + CharacterDefinition.FILENAME_SUFFIX);
- Files.createDirectories(path.getParent());
- try (OutputStream os = new BufferedOutputStream(Files.newOutputStream(path))) {
- final DataOutput out = new OutputStreamDataOutput(os);
- CodecUtil.writeHeader(out, CharacterDefinition.HEADER, CharacterDefinition.VERSION);
- out.writeBytes(characterCategoryMap, 0, characterCategoryMap.length);
- for (int i = 0; i < CharacterDefinition.CLASS_COUNT; i++) {
- final byte b = (byte) ((invokeMap[i] ? 0x01 : 0x00) | (groupMap[i] ? 0x02 : 0x00));
- out.writeByte(b);
- }
- }
- }
-}
diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/util/ConnectionCostsBuilder.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/util/ConnectionCostsBuilder.java
index 8b518ee991a..bfd8dd0d17c 100644
--- a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/util/ConnectionCostsBuilder.java
+++ b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/util/ConnectionCostsBuilder.java
@@ -22,12 +22,14 @@ import java.io.Reader;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
+import org.apache.lucene.analysis.ja.dict.ConnectionCosts;
+import org.apache.lucene.analysis.morph.ConnectionCostsWriter;
class ConnectionCostsBuilder {
private ConnectionCostsBuilder() {}
- public static ConnectionCostsWriter build(Path path) throws IOException {
+ public static ConnectionCostsWriter<ConnectionCosts> build(Path path) throws IOException {
try (Reader reader = Files.newBufferedReader(path, StandardCharsets.US_ASCII);
LineNumberReader lineReader = new LineNumberReader(reader)) {
@@ -41,7 +43,8 @@ class ConnectionCostsBuilder {
assert forwardSize > 0 && backwardSize > 0;
- ConnectionCostsWriter costs = new ConnectionCostsWriter(forwardSize, backwardSize);
+ ConnectionCostsWriter<ConnectionCosts> costs =
+ new ConnectionCostsWriter<>(ConnectionCosts.class, forwardSize, backwardSize);
while ((line = lineReader.readLine()) != null) {
String[] fields = line.split("\\s+");
diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/util/DictionaryBuilder.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/util/DictionaryBuilder.java
index 72920baf984..d547dcb122d 100644
--- a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/util/DictionaryBuilder.java
+++ b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/util/DictionaryBuilder.java
@@ -20,6 +20,7 @@ import java.io.IOException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Locale;
+import org.apache.lucene.analysis.ja.dict.DictionaryConstants;
/**
* Tool to build dictionaries. Usage:
@@ -68,7 +69,8 @@ public class DictionaryBuilder {
new UnknownDictionaryBuilder(encoding).build(inputDir).write(outputDir);
- ConnectionCostsBuilder.build(inputDir.resolve("matrix.def")).write(outputDir);
+ ConnectionCostsBuilder.build(inputDir.resolve("matrix.def"))
+ .write(outputDir, DictionaryConstants.CONN_COSTS_HEADER, DictionaryConstants.VERSION);
}
public static void main(String[] args) throws IOException {
diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/util/TokenInfoDictionaryEntryWriter.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/util/TokenInfoDictionaryEntryWriter.java
new file mode 100644
index 00000000000..618047825f2
--- /dev/null
+++ b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/util/TokenInfoDictionaryEntryWriter.java
@@ -0,0 +1,237 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.analysis.ja.util;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import org.apache.lucene.analysis.ja.dict.TokenInfoMorphData;
+import org.apache.lucene.analysis.morph.DictionaryEntryWriter;
+import org.apache.lucene.store.DataOutput;
+import org.apache.lucene.util.ArrayUtil;
+
+/** Writes system dictionary entries */
+class TokenInfoDictionaryEntryWriter extends DictionaryEntryWriter {
+ private static final int ID_LIMIT = 8192;
+
+ TokenInfoDictionaryEntryWriter(int size) {
+ super(size);
+ }
+
+ /**
+ * put the entry in map
+ *
+ * <p>mecab-ipadic features
+ *
+ * <pre>
+ * 0 - surface
+ * 1 - left cost
+ * 2 - right cost
+ * 3 - word cost
+ * 4-9 - pos
+ * 10 - base form
+ * 11 - reading
+ * 12 - pronounciation
+ * </pre>
+ */
+ @Override
+ protected int putEntry(String[] entry) {
+ short leftId = Short.parseShort(entry[1]);
+ short rightId = Short.parseShort(entry[2]);
+ short wordCost = Short.parseShort(entry[3]);
+
+ StringBuilder sb = new StringBuilder();
+
+ // build up the POS string
+ for (int i = 4; i < 8; i++) {
+ String part = entry[i];
+ assert part.length() > 0;
+ if (!"*".equals(part)) {
+ if (sb.length() > 0) {
+ sb.append('-');
+ }
+ sb.append(part);
+ }
+ }
+
+ String posData = sb.toString();
+ if (posData.isEmpty()) {
+ throw new IllegalArgumentException("POS fields are empty");
+ }
+ sb.setLength(0);
+ sb.append(CSVUtil.quoteEscape(posData));
+ sb.append(',');
+ if (!"*".equals(entry[8])) {
+ sb.append(CSVUtil.quoteEscape(entry[8]));
+ }
+ sb.append(',');
+ if (!"*".equals(entry[9])) {
+ sb.append(CSVUtil.quoteEscape(entry[9]));
+ }
+ String fullPOSData = sb.toString();
+
+ String baseForm = entry[10];
+ String reading = entry[11];
+ String pronunciation = entry[12];
+
+ // extend buffer if necessary
+ int left = buffer.remaining();
+ // worst case: two short, 3 bytes, and features (all as utf-16)
+ int worstCase = 4 + 3 + 2 * (baseForm.length() + reading.length() + pronunciation.length());
+ if (worstCase > left) {
+ ByteBuffer newBuffer =
+ ByteBuffer.allocateDirect(ArrayUtil.oversize(buffer.limit() + worstCase - left, 1));
+ buffer.flip();
+ newBuffer.put(buffer);
+ buffer = newBuffer;
+ }
+
+ int flags = 0;
+ if (baseForm.isEmpty()) {
+ throw new IllegalArgumentException("base form is empty");
+ }
+ if (!("*".equals(baseForm) || baseForm.equals(entry[0]))) {
+ flags |= TokenInfoMorphData.HAS_BASEFORM;
+ }
+ if (!reading.equals(toKatakana(entry[0]))) {
+ flags |= TokenInfoMorphData.HAS_READING;
+ }
+ if (!pronunciation.equals(reading)) {
+ flags |= TokenInfoMorphData.HAS_PRONUNCIATION;
+ }
+
+ if (leftId != rightId) {
+ throw new IllegalArgumentException("rightId != leftId: " + rightId + " " + leftId);
+ }
+ if (leftId >= ID_LIMIT) {
+ throw new IllegalArgumentException("leftId >= " + ID_LIMIT + ": " + leftId);
+ }
+ // add pos mapping
+ int toFill = 1 + leftId - posDict.size();
+ for (int i = 0; i < toFill; i++) {
+ posDict.add(null);
+ }
+
+ String existing = posDict.get(leftId);
+ if (existing != null && existing.equals(fullPOSData) == false) {
+ // TODO: test me
+ throw new IllegalArgumentException("Multiple entries found for leftID=" + leftId);
+ }
+ posDict.set(leftId, fullPOSData);
+
+ buffer.putShort((short) (leftId << 3 | flags));
+ buffer.putShort(wordCost);
+
+ if ((flags & TokenInfoMorphData.HAS_BASEFORM) != 0) {
+ if (baseForm.length() >= 16) {
+ throw new IllegalArgumentException("Length of base form " + baseForm + " is >= 16");
+ }
+ int shared = sharedPrefix(entry[0], baseForm);
+ int suffix = baseForm.length() - shared;
+ buffer.put((byte) (shared << 4 | suffix));
+ for (int i = shared; i < baseForm.length(); i++) {
+ buffer.putChar(baseForm.charAt(i));
+ }
+ }
+
+ if ((flags & TokenInfoMorphData.HAS_READING) != 0) {
+ if (isKatakana(reading)) {
+ buffer.put((byte) (reading.length() << 1 | 1));
+ writeKatakana(reading, buffer);
+ } else {
+ buffer.put((byte) (reading.length() << 1));
+ for (int i = 0; i < reading.length(); i++) {
+ buffer.putChar(reading.charAt(i));
+ }
+ }
+ }
+
+ if ((flags & TokenInfoMorphData.HAS_PRONUNCIATION) != 0) {
+ // we can save 150KB here, but it makes the reader a little complicated.
+ // int shared = sharedPrefix(reading, pronunciation);
+ // buffer.put((byte) shared);
+ // pronunciation = pronunciation.substring(shared);
+ if (isKatakana(pronunciation)) {
+ buffer.put((byte) (pronunciation.length() << 1 | 1));
+ writeKatakana(pronunciation, buffer);
+ } else {
+ buffer.put((byte) (pronunciation.length() << 1));
+ for (int i = 0; i < pronunciation.length(); i++) {
+ buffer.putChar(pronunciation.charAt(i));
+ }
+ }
+ }
+
+ return buffer.position();
+ }
+
+ private boolean isKatakana(String s) {
+ for (int i = 0; i < s.length(); i++) {
+ char ch = s.charAt(i);
+ if (ch < 0x30A0 || ch > 0x30FF) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ private void writeKatakana(String s, ByteBuffer buffer) {
+ for (int i = 0; i < s.length(); i++) {
+ buffer.put((byte) (s.charAt(i) - 0x30A0));
+ }
+ }
+
+ private String toKatakana(String s) {
+ char[] text = new char[s.length()];
+ for (int i = 0; i < s.length(); i++) {
+ char ch = s.charAt(i);
+ if (ch > 0x3040 && ch < 0x3097) {
+ text[i] = (char) (ch + 0x60);
+ } else {
+ text[i] = ch;
+ }
+ }
+ return new String(text);
+ }
+
+ private static int sharedPrefix(String left, String right) {
+ int len = left.length() < right.length() ? left.length() : right.length();
+ for (int i = 0; i < len; i++) if (left.charAt(i) != right.charAt(i)) return i;
+ return len;
+ }
+
+ @Override
+ protected void writePosDict(OutputStream bos, DataOutput out) throws IOException {
+ out.writeVInt(posDict.size());
+ for (String s : posDict) {
+ if (s == null) {
+ out.writeByte((byte) 0);
+ out.writeByte((byte) 0);
+ out.writeByte((byte) 0);
+ } else {
+ String[] data = CSVUtil.parse(s);
+ if (data.length != 3) {
+ throw new IllegalArgumentException(
+ "Malformed pos/inflection: " + s + "; expected 3 characters");
+ }
+ out.writeString(data[0]);
+ out.writeString(data[1]);
+ out.writeString(data[2]);
+ }
+ }
+ }
+}
diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/util/TokenInfoDictionaryWriter.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/util/TokenInfoDictionaryWriter.java
index 400c834cadf..fbdf65f34ce 100644
--- a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/util/TokenInfoDictionaryWriter.java
+++ b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/util/TokenInfoDictionaryWriter.java
@@ -20,14 +20,16 @@ import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Objects;
+import org.apache.lucene.analysis.ja.dict.DictionaryConstants;
import org.apache.lucene.analysis.ja.dict.TokenInfoDictionary;
import org.apache.lucene.util.fst.FST;
-class TokenInfoDictionaryWriter extends BinaryDictionaryWriter {
+class TokenInfoDictionaryWriter
+ extends org.apache.lucene.analysis.morph.BinaryDictionaryWriter<TokenInfoDictionary> {
private FST<Long> fst;
TokenInfoDictionaryWriter(int size) {
- super(TokenInfoDictionary.class, size);
+ super(TokenInfoDictionary.class, new TokenInfoDictionaryEntryWriter(size));
}
public void setFST(FST<Long> fst) {
@@ -35,9 +37,19 @@ class TokenInfoDictionaryWriter extends BinaryDictionaryWriter {
this.fst = fst;
}
+ @Override
+ protected void addMapping(int sourceId, int wordId) {
+ super.addMapping(sourceId, wordId);
+ }
+
@Override
public void write(Path baseDir) throws IOException {
- super.write(baseDir);
+ super.write(
+ baseDir,
+ DictionaryConstants.TARGETMAP_HEADER,
+ DictionaryConstants.POSDICT_HEADER,
+ DictionaryConstants.DICT_HEADER,
+ DictionaryConstants.VERSION);
writeFST(baseDir.resolve(getBaseFileName() + TokenInfoDictionary.FST_FILENAME_SUFFIX));
}
diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/util/UnknownDictionaryWriter.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/util/UnknownDictionaryWriter.java
index 6d80f513fec..84000fdf0de 100644
--- a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/util/UnknownDictionaryWriter.java
+++ b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/util/UnknownDictionaryWriter.java
@@ -19,19 +19,28 @@ package org.apache.lucene.analysis.ja.util;
import java.io.IOException;
import java.nio.file.Path;
import org.apache.lucene.analysis.ja.dict.CharacterDefinition;
+import org.apache.lucene.analysis.ja.dict.DictionaryConstants;
import org.apache.lucene.analysis.ja.dict.UnknownDictionary;
+import org.apache.lucene.analysis.morph.BinaryDictionaryWriter;
+import org.apache.lucene.analysis.morph.CharacterDefinitionWriter;
-class UnknownDictionaryWriter extends BinaryDictionaryWriter {
- private final CharacterDefinitionWriter characterDefinition = new CharacterDefinitionWriter();
+class UnknownDictionaryWriter extends BinaryDictionaryWriter<UnknownDictionary> {
+ private final CharacterDefinitionWriter<CharacterDefinition> characterDefinition =
+ new CharacterDefinitionWriter<>(
+ CharacterDefinition.class,
+ CharacterDefinition.DEFAULT,
+ CharacterDefinition.CLASS_COUNT,
+ CharacterDefinition::lookupCharacterClass);
public UnknownDictionaryWriter(int size) {
- super(UnknownDictionary.class, size);
+ super(UnknownDictionary.class, new TokenInfoDictionaryEntryWriter(size));
}
@Override
public int put(String[] entry) {
// Get wordId of current entry
- int wordId = buffer.position();
+ // int wordId = buffer.position();
+ int wordId = entryWriter.currentPosition();
// Put entry
int result = super.put(entry);
@@ -58,7 +67,13 @@ class UnknownDictionaryWriter extends BinaryDictionaryWriter {
@Override
public void write(Path baseDir) throws IOException {
- super.write(baseDir);
- characterDefinition.write(baseDir);
+ super.write(
+ baseDir,
+ DictionaryConstants.TARGETMAP_HEADER,
+ DictionaryConstants.POSDICT_HEADER,
+ DictionaryConstants.DICT_HEADER,
+ DictionaryConstants.VERSION);
+ characterDefinition.write(
+ baseDir, DictionaryConstants.CHARDEF_HEADER, DictionaryConstants.VERSION);
}
}
diff --git a/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/dict/TestExternalDictionary.java b/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/dict/TestExternalDictionary.java
index bc44723996a..5423d7a76cb 100644
--- a/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/dict/TestExternalDictionary.java
+++ b/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/dict/TestExternalDictionary.java
@@ -16,10 +16,10 @@
*/
package org.apache.lucene.analysis.ja.dict;
-import static org.apache.lucene.analysis.ja.dict.BinaryDictionary.DICT_FILENAME_SUFFIX;
-import static org.apache.lucene.analysis.ja.dict.BinaryDictionary.POSDICT_FILENAME_SUFFIX;
-import static org.apache.lucene.analysis.ja.dict.BinaryDictionary.TARGETMAP_FILENAME_SUFFIX;
import static org.apache.lucene.analysis.ja.dict.TokenInfoDictionary.FST_FILENAME_SUFFIX;
+import static org.apache.lucene.analysis.morph.BinaryDictionary.DICT_FILENAME_SUFFIX;
+import static org.apache.lucene.analysis.morph.BinaryDictionary.POSDICT_FILENAME_SUFFIX;
+import static org.apache.lucene.analysis.morph.BinaryDictionary.TARGETMAP_FILENAME_SUFFIX;
import java.io.BufferedWriter;
import java.nio.charset.StandardCharsets;
diff --git a/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/dict/TestTokenInfoDictionary.java b/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/dict/TestTokenInfoDictionary.java
index 11de60c05fb..0320d6e5f13 100644
--- a/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/dict/TestTokenInfoDictionary.java
+++ b/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/dict/TestTokenInfoDictionary.java
@@ -16,10 +16,10 @@
*/
package org.apache.lucene.analysis.ja.dict;
-import static org.apache.lucene.analysis.ja.dict.BinaryDictionary.DICT_FILENAME_SUFFIX;
-import static org.apache.lucene.analysis.ja.dict.BinaryDictionary.POSDICT_FILENAME_SUFFIX;
-import static org.apache.lucene.analysis.ja.dict.BinaryDictionary.TARGETMAP_FILENAME_SUFFIX;
import static org.apache.lucene.analysis.ja.dict.TokenInfoDictionary.FST_FILENAME_SUFFIX;
+import static org.apache.lucene.analysis.morph.BinaryDictionary.DICT_FILENAME_SUFFIX;
+import static org.apache.lucene.analysis.morph.BinaryDictionary.POSDICT_FILENAME_SUFFIX;
+import static org.apache.lucene.analysis.morph.BinaryDictionary.TARGETMAP_FILENAME_SUFFIX;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
@@ -137,17 +137,17 @@ public class TestTokenInfoDictionary extends LuceneTestCase {
assertTrue(wordId > lastWordId);
lastWordId = wordId;
- String baseForm = tid.getBaseForm(wordId, chars, 0, chars.length);
+ String baseForm = tid.getMorphAttributes().getBaseForm(wordId, chars, 0, chars.length);
assertTrue(baseForm == null || UnicodeUtil.validUTF16String(baseForm));
- String inflectionForm = tid.getInflectionForm(wordId);
+ String inflectionForm = tid.getMorphAttributes().getInflectionForm(wordId);
assertTrue(inflectionForm == null || UnicodeUtil.validUTF16String(inflectionForm));
if (inflectionForm != null) {
// check that it's actually an ipadic inflection form
assertNotNull(ToStringUtil.getInflectedFormTranslation(inflectionForm));
}
- String inflectionType = tid.getInflectionType(wordId);
+ String inflectionType = tid.getMorphAttributes().getInflectionType(wordId);
assertTrue(inflectionType == null || UnicodeUtil.validUTF16String(inflectionType));
if (inflectionType != null) {
// check that it's actually an ipadic inflection type
@@ -161,17 +161,18 @@ public class TestTokenInfoDictionary extends LuceneTestCase {
tid.getWordCost(wordId);
- String pos = tid.getPartOfSpeech(wordId);
+ String pos = tid.getMorphAttributes().getPartOfSpeech(wordId);
assertNotNull(pos);
assertTrue(UnicodeUtil.validUTF16String(pos));
// check that it's actually an ipadic pos tag
assertNotNull(ToStringUtil.getPOSTranslation(pos));
- String pronunciation = tid.getPronunciation(wordId, chars, 0, chars.length);
+ String pronunciation =
+ tid.getMorphAttributes().getPronunciation(wordId, chars, 0, chars.length);
assertNotNull(pronunciation);
assertTrue(UnicodeUtil.validUTF16String(pronunciation));
- String reading = tid.getReading(wordId, chars, 0, chars.length);
+ String reading = tid.getMorphAttributes().getReading(wordId, chars, 0, chars.length);
assertNotNull(reading);
assertTrue(UnicodeUtil.validUTF16String(reading));
}
diff --git a/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/dict/TestUserDictionary.java b/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/dict/TestUserDictionary.java
index b08d75344f0..537d621016e 100644
--- a/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/dict/TestUserDictionary.java
+++ b/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/dict/TestUserDictionary.java
@@ -54,12 +54,15 @@ public class TestUserDictionary extends LuceneTestCase {
int[][] result = dictionary.lookup("日本経済新聞".toCharArray(), 0, 6);
assertEquals(3, result.length);
int wordIdNihon = result[0][0]; // wordId of 日本 in 日本経済新聞
- assertEquals("ニホン", dictionary.getReading(wordIdNihon, "日本".toCharArray(), 0, 2));
+ assertEquals(
+ "ニホン", dictionary.getMorphAttributes().getReading(wordIdNihon, "日本".toCharArray(), 0, 2));
result = dictionary.lookup("朝青龍".toCharArray(), 0, 3);
assertEquals(1, result.length);
int wordIdAsashoryu = result[0][0]; // wordId for 朝青龍
- assertEquals("アサショウリュウ", dictionary.getReading(wordIdAsashoryu, "朝青龍".toCharArray(), 0, 3));
+ assertEquals(
+ "アサショウリュウ",
+ dictionary.getMorphAttributes().getReading(wordIdAsashoryu, "朝青龍".toCharArray(), 0, 3));
}
@Test
@@ -68,7 +71,7 @@ public class TestUserDictionary extends LuceneTestCase {
int[][] result = dictionary.lookup("日本経済新聞".toCharArray(), 0, 6);
assertEquals(3, result.length);
int wordIdKeizai = result[1][0]; // wordId of 経済 in 日本経済新聞
- assertEquals("カスタム名詞", dictionary.getPartOfSpeech(wordIdKeizai));
+ assertEquals("カスタム名詞", dictionary.getMorphAttributes().getPartOfSpeech(wordIdKeizai));
}
@Test
@@ -109,7 +112,7 @@ public class TestUserDictionary extends LuceneTestCase {
for (String input : inputs) {
System.out.println(input);
int[][] result = dictionary.lookup(input.toCharArray(), 0, input.length());
- assertEquals("カスタム名刺", dictionary.getPartOfSpeech(result[0][0]));
+ assertEquals("カスタム名刺", dictionary.getMorphAttributes().getPartOfSpeech(result[0][0]));
}
}
}
diff --git a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/DecompoundToken.java b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/DecompoundToken.java
index 731378c9c4a..a6af55f482e 100644
--- a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/DecompoundToken.java
+++ b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/DecompoundToken.java
@@ -16,7 +16,7 @@
*/
package org.apache.lucene.analysis.ko;
-import org.apache.lucene.analysis.ko.dict.Dictionary;
+import org.apache.lucene.analysis.ko.dict.KoMorphData;
/** A token that was generated from a compound. */
public class DecompoundToken extends Token {
@@ -71,7 +71,7 @@ public class DecompoundToken extends Token {
}
@Override
- public Dictionary.Morpheme[] getMorphemes() {
+ public KoMorphData.Morpheme[] getMorphemes() {
return null;
}
}
diff --git a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/DictionaryToken.java b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/DictionaryToken.java
index 82fe981c24d..f548f8007d3 100644
--- a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/DictionaryToken.java
+++ b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/DictionaryToken.java
@@ -16,17 +16,17 @@
*/
package org.apache.lucene.analysis.ko;
-import org.apache.lucene.analysis.ko.dict.Dictionary;
+import org.apache.lucene.analysis.ko.dict.KoMorphData;
-/** A token stored in a {@link Dictionary}. */
+/** A token stored in a {@link KoMorphData}. */
public class DictionaryToken extends Token {
private final int wordId;
private final KoreanTokenizer.Type type;
- private final Dictionary dictionary;
+ private final KoMorphData morphAtts;
public DictionaryToken(
KoreanTokenizer.Type type,
- Dictionary dictionary,
+ KoMorphData morphAtts,
int wordId,
char[] surfaceForm,
int offset,
@@ -35,7 +35,7 @@ public class DictionaryToken extends Token {
int endOffset) {
super(surfaceForm, offset, length, startOffset, endOffset);
this.type = type;
- this.dictionary = dictionary;
+ this.morphAtts = morphAtts;
this.wordId = wordId;
}
@@ -54,7 +54,7 @@ public class DictionaryToken extends Token {
+ " wordId="
+ wordId
+ " leftID="
- + dictionary.getLeftId(wordId)
+ + morphAtts.getLeftId(wordId)
+ ")";
}
@@ -96,26 +96,26 @@ public class DictionaryToken extends Token {
@Override
public POS.Type getPOSType() {
- return dictionary.getPOSType(wordId);
+ return morphAtts.getPOSType(wordId);
}
@Override
public POS.Tag getLeftPOS() {
- return dictionary.getLeftPOS(wordId);
+ return morphAtts.getLeftPOS(wordId);
}
@Override
public POS.Tag getRightPOS() {
- return dictionary.getRightPOS(wordId);
+ return morphAtts.getRightPOS(wordId);
}
@Override
public String getReading() {
- return dictionary.getReading(wordId);
+ return morphAtts.getReading(wordId);
}
@Override
- public Dictionary.Morpheme[] getMorphemes() {
- return dictionary.getMorphemes(wordId, getSurfaceForm(), getOffset(), getLength());
+ public KoMorphData.Morpheme[] getMorphemes() {
+ return morphAtts.getMorphemes(wordId, getSurfaceForm(), getOffset(), getLength());
}
}
diff --git a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/GraphvizFormatter.java b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/GraphvizFormatter.java
index d8e3f7a9faf..9beaf2e5b9b 100644
--- a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/GraphvizFormatter.java
+++ b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/GraphvizFormatter.java
@@ -21,7 +21,8 @@ import java.util.Map;
import org.apache.lucene.analysis.ko.KoreanTokenizer.Position;
import org.apache.lucene.analysis.ko.KoreanTokenizer.WrappedPositionArray;
import org.apache.lucene.analysis.ko.dict.ConnectionCosts;
-import org.apache.lucene.analysis.ko.dict.Dictionary;
+import org.apache.lucene.analysis.ko.dict.KoMorphData;
+import org.apache.lucene.analysis.morph.Dictionary;
// TODO: would be nice to show 2nd best path in a diff't
// color...
@@ -140,7 +141,7 @@ public class GraphvizFormatter {
attrs = "";
}
- final Dictionary dict = tok.getDict(posData.backType[idx]);
+ final Dictionary<? extends KoMorphData> dict = tok.getDict(posData.backType[idx]);
final int wordCost = dict.getWordCost(posData.backID[idx]);
final int bgCost =
costs.get(
diff --git a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/KoreanTokenizer.java b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/KoreanTokenizer.java
index 325fae710b9..028d9dd54d1 100644
--- a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/KoreanTokenizer.java
+++ b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/KoreanTokenizer.java
@@ -26,13 +26,14 @@ import java.util.List;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.ko.dict.CharacterDefinition;
import org.apache.lucene.analysis.ko.dict.ConnectionCosts;
-import org.apache.lucene.analysis.ko.dict.Dictionary;
+import org.apache.lucene.analysis.ko.dict.KoMorphData;
import org.apache.lucene.analysis.ko.dict.TokenInfoDictionary;
import org.apache.lucene.analysis.ko.dict.TokenInfoFST;
import org.apache.lucene.analysis.ko.dict.UnknownDictionary;
import org.apache.lucene.analysis.ko.dict.UserDictionary;
import org.apache.lucene.analysis.ko.tokenattributes.PartOfSpeechAttribute;
import org.apache.lucene.analysis.ko.tokenattributes.ReadingAttribute;
+import org.apache.lucene.analysis.morph.Dictionary;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
@@ -97,7 +98,8 @@ public final class KoreanTokenizer extends Tokenizer {
private static final int MAX_UNKNOWN_WORD_LENGTH = 1024;
private static final int MAX_BACKTRACE_GAP = 1024;
- private final EnumMap<Type, Dictionary> dictionaryMap = new EnumMap<>(Type.class);
+ private final EnumMap<Type, Dictionary<? extends KoMorphData>> dictionaryMap =
+ new EnumMap<>(Type.class);
private final TokenInfoFST fst;
private final TokenInfoDictionary dictionary;
@@ -406,10 +408,10 @@ public final class KoreanTokenizer extends Tokenizer {
}
private void add(
- Dictionary dict, Position fromPosData, int wordPos, int endPos, int wordID, Type type) {
- final POS.Tag leftPOS = dict.getLeftPOS(wordID);
- final int wordCost = dict.getWordCost(wordID);
- final int leftID = dict.getLeftId(wordID);
+ KoMorphData morphAtts, Position fromPosData, int wordPos, int endPos, int wordID, Type type) {
+ final POS.Tag leftPOS = morphAtts.getLeftPOS(wordID);
+ final int wordCost = morphAtts.getWordCost(wordID);
+ final int leftID = morphAtts.getLeftId(wordID);
int leastCost = Integer.MAX_VALUE;
int leastIDX = -1;
assert fromPosData.count > 0;
@@ -472,7 +474,14 @@ public final class KoreanTokenizer extends Tokenizer {
positions
.get(endPos)
- .add(leastCost, dict.getRightId(wordID), fromPosData.pos, wordPos, leastIDX, wordID, type);
+ .add(
+ leastCost,
+ morphAtts.getRightId(wordID),
+ fromPosData.pos,
+ wordPos,
+ leastIDX,
+ wordID,
+ type);
}
@Override
@@ -796,7 +805,7 @@ public final class KoreanTokenizer extends Tokenizer {
+ (maxPosAhead + 1));
}
add(
- userDictionary,
+ userDictionary.getMorphAttributes(),
posData,
pos,
maxPosAhead + 1,
@@ -848,7 +857,7 @@ public final class KoreanTokenizer extends Tokenizer {
}
for (int ofs = 0; ofs < wordIdRef.length; ofs++) {
add(
- dictionary,
+ dictionary.getMorphAttributes(),
posData,
pos,
posAhead + 1,
@@ -922,7 +931,7 @@ public final class KoreanTokenizer extends Tokenizer {
}
for (int ofs = 0; ofs < wordIdRef.length; ofs++) {
add(
- unkDictionary,
+ unkDictionary.getMorphAttributes(),
posData,
pos,
pos + unknownWordLength,
@@ -1016,7 +1025,7 @@ public final class KoreanTokenizer extends Tokenizer {
final int fragmentOffset = backWordPos - lastBackTracePos;
assert fragmentOffset >= 0;
- final Dictionary dict = getDict(backType);
+ final Dictionary<? extends KoMorphData> dict = getDict(backType);
if (outputUnknownUnigrams && backType == Type.UNKNOWN) {
// outputUnknownUnigrams converts unknown word into unigrams:
@@ -1029,7 +1038,7 @@ public final class KoreanTokenizer extends Tokenizer {
final DictionaryToken token =
new DictionaryToken(
Type.UNKNOWN,
- unkDictionary,
+ unkDictionary.getMorphAttributes(),
CharacterDefinition.NGRAM,
fragment,
fragmentOffset + i,
@@ -1045,7 +1054,7 @@ public final class KoreanTokenizer extends Tokenizer {
final DictionaryToken token =
new DictionaryToken(
backType,
- dict,
+ dict.getMorphAttributes(),
backID,
fragment,
fragmentOffset,
@@ -1060,7 +1069,7 @@ public final class KoreanTokenizer extends Tokenizer {
}
}
} else {
- Dictionary.Morpheme[] morphemes = token.getMorphemes();
+ KoMorphData.Morpheme[] morphemes = token.getMorphemes();
if (morphemes == null) {
pending.add(token);
if (VERBOSE) {
@@ -1071,7 +1080,7 @@ public final class KoreanTokenizer extends Tokenizer {
int posLen = 0;
// decompose the compound
for (int i = morphemes.length - 1; i >= 0; i--) {
- final Dictionary.Morpheme morpheme = morphemes[i];
+ final KoMorphData.Morpheme morpheme = morphemes[i];
final Token compoundToken;
if (token.getPOSType() == POS.Type.COMPOUND) {
assert endOffset - morpheme.surfaceForm.length() >= 0;
@@ -1119,7 +1128,7 @@ public final class KoreanTokenizer extends Tokenizer {
DictionaryToken spaceToken =
new DictionaryToken(
Type.UNKNOWN,
- unkDictionary,
+ unkDictionary.getMorphAttributes(),
wordIdRef.ints[wordIdRef.offset],
fragment,
offset,
@@ -1144,7 +1153,7 @@ public final class KoreanTokenizer extends Tokenizer {
positions.freeBefore(endPos);
}
- Dictionary getDict(Type type) {
+ Dictionary<? extends KoMorphData> getDict(Type type) {
return dictionaryMap.get(type);
}
diff --git a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/Token.java b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/Token.java
index 730262907d8..ed2d7ff6ba7 100644
--- a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/Token.java
+++ b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/Token.java
@@ -16,7 +16,7 @@
*/
package org.apache.lucene.analysis.ko;
-import org.apache.lucene.analysis.ko.dict.Dictionary.Morpheme;
+import org.apache.lucene.analysis.ko.dict.KoMorphData;
/** Analyzed token with morphological data. */
public abstract class Token {
@@ -70,8 +70,11 @@ public abstract class Token {
/** Get the reading of the token. */
public abstract String getReading();
- /** Get the {@link Morpheme} decomposition of the token. */
- public abstract Morpheme[] getMorphemes();
+ /**
+ * Get the {@link org.apache.lucene.analysis.ko.dict.KoMorphData.Morpheme} decomposition of the
+ * token.
+ */
+ public abstract KoMorphData.Morpheme[] getMorphemes();
/** Get the start offset of the term in the analyzed text. */
public int getStartOffset() {
diff --git a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/BinaryDictionary.java b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/BinaryDictionary.java
deleted file mode 100644
index 4d10e520055..00000000000
--- a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/BinaryDictionary.java
+++ /dev/null
@@ -1,220 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.analysis.ko.dict;
-
-import java.io.BufferedInputStream;
-import java.io.EOFException;
-import java.io.IOException;
-import java.io.InputStream;
-import java.nio.ByteBuffer;
-import java.nio.channels.Channels;
-import java.nio.channels.ReadableByteChannel;
-import org.apache.lucene.analysis.ko.POS;
-import org.apache.lucene.codecs.CodecUtil;
-import org.apache.lucene.store.DataInput;
-import org.apache.lucene.store.InputStreamDataInput;
-import org.apache.lucene.util.IOSupplier;
-import org.apache.lucene.util.IntsRef;
-
-/** Base class for a binary-encoded in-memory dictionary. */
-public abstract class BinaryDictionary implements Dictionary {
-
- public static final String TARGETMAP_FILENAME_SUFFIX = "$targetMap.dat";
- public static final String DICT_FILENAME_SUFFIX = "$buffer.dat";
- public static final String POSDICT_FILENAME_SUFFIX = "$posDict.dat";
-
- public static final String DICT_HEADER = "ko_dict";
- public static final String TARGETMAP_HEADER = "ko_dict_map";
- public static final String POSDICT_HEADER = "ko_dict_pos";
- public static final int VERSION = 1;
-
- private final ByteBuffer buffer;
- private final int[] targetMapOffsets, targetMap;
- private final POS.Tag[] posDict;
-
- protected BinaryDictionary(
- IOSupplier<InputStream> targetMapResource,
- IOSupplier<InputStream> posResource,
- IOSupplier<InputStream> dictResource)
- throws IOException {
- try (InputStream mapIS = new BufferedInputStream(targetMapResource.get())) {
- DataInput in = new InputStreamDataInput(mapIS);
- CodecUtil.checkHeader(in, TARGETMAP_HEADER, VERSION, VERSION);
- this.targetMap = new int[in.readVInt()];
- this.targetMapOffsets = new int[in.readVInt()];
- populateTargetMap(in, this.targetMap, this.targetMapOffsets);
- }
-
- try (InputStream posIS = new BufferedInputStream(posResource.get())) {
- DataInput in = new InputStreamDataInput(posIS);
- CodecUtil.checkHeader(in, POSDICT_HEADER, VERSION, VERSION);
- int posSize = in.readVInt();
- this.posDict = new POS.Tag[posSize];
- for (int j = 0; j < posSize; j++) {
- posDict[j] = POS.resolveTag(in.readByte());
- }
- }
-
- // no buffering here, as we load in one large buffer
- try (InputStream dictIS = dictResource.get()) {
- DataInput in = new InputStreamDataInput(dictIS);
- CodecUtil.checkHeader(in, DICT_HEADER, VERSION, VERSION);
- final int size = in.readVInt();
- final ByteBuffer tmpBuffer = ByteBuffer.allocateDirect(size);
- final ReadableByteChannel channel = Channels.newChannel(dictIS);
- final int read = channel.read(tmpBuffer);
- if (read != size) {
- throw new EOFException("Cannot read whole dictionary");
- }
- this.buffer = tmpBuffer.asReadOnlyBuffer();
- }
- }
-
- private static void populateTargetMap(DataInput in, int[] targetMap, int[] targetMapOffsets)
- throws IOException {
- int accum = 0, sourceId = 0;
- for (int ofs = 0; ofs < targetMap.length; ofs++) {
- final int val = in.readVInt();
- if ((val & 0x01) != 0) {
- targetMapOffsets[sourceId] = ofs;
- sourceId++;
- }
- accum += val >>> 1;
- targetMap[ofs] = accum;
- }
- if (sourceId + 1 != targetMapOffsets.length)
- throw new IOException(
- "targetMap file format broken; targetMap.length="
- + targetMap.length
- + ", targetMapOffsets.length="
- + targetMapOffsets.length
- + ", sourceId="
- + sourceId);
- targetMapOffsets[sourceId] = targetMap.length;
- }
-
- public void lookupWordIds(int sourceId, IntsRef ref) {
- ref.ints = targetMap;
- ref.offset = targetMapOffsets[sourceId];
- // targetMapOffsets always has one more entry pointing behind last:
- ref.length = targetMapOffsets[sourceId + 1] - ref.offset;
- }
-
- @Override
- public int getLeftId(int wordId) {
- return buffer.getShort(wordId) >>> 2;
- }
-
- @Override
- public int getRightId(int wordId) {
- return buffer.getShort(wordId + 2) >>> 2; // Skip left id
- }
-
- @Override
- public int getWordCost(int wordId) {
- return buffer.getShort(wordId + 4); // Skip left and right id
- }
-
- @Override
- public POS.Type getPOSType(int wordId) {
- byte value = (byte) (buffer.getShort(wordId) & 3);
- return POS.resolveType(value);
- }
-
- @Override
- public POS.Tag getLeftPOS(int wordId) {
- return posDict[getLeftId(wordId)];
- }
-
- @Override
- public POS.Tag getRightPOS(int wordId) {
- POS.Type type = getPOSType(wordId);
- if (type == POS.Type.MORPHEME || type == POS.Type.COMPOUND || hasSinglePOS(wordId)) {
- return getLeftPOS(wordId);
- } else {
- byte value = buffer.get(wordId + 6);
- return POS.resolveTag(value);
- }
- }
-
- @Override
- public String getReading(int wordId) {
- if (hasReadingData(wordId)) {
- int offset = wordId + 6;
- return readString(offset);
- }
- return null;
- }
-
- @Override
- public Morpheme[] getMorphemes(int wordId, char[] surfaceForm, int off, int len) {
- POS.Type posType = getPOSType(wordId);
- if (posType == POS.Type.MORPHEME) {
- return null;
- }
- int offset = wordId + 6;
- boolean hasSinglePos = hasSinglePOS(wordId);
- if (hasSinglePos == false) {
- offset++; // skip rightPOS
- }
- int length = buffer.get(offset++);
- if (length == 0) {
- return null;
- }
- Morpheme[] morphemes = new Morpheme[length];
- int surfaceOffset = 0;
- final POS.Tag leftPOS = getLeftPOS(wordId);
- for (int i = 0; i < length; i++) {
- final String form;
- final POS.Tag tag = hasSinglePos ? leftPOS : POS.resolveTag(buffer.get(offset++));
- if (posType == POS.Type.INFLECT) {
- form = readString(offset);
- offset += form.length() * 2 + 1;
- } else {
- int formLen = buffer.get(offset++);
- form = new String(surfaceForm, off + surfaceOffset, formLen);
- surfaceOffset += formLen;
- }
- morphemes[i] = new Morpheme(tag, form);
- }
- return morphemes;
- }
-
- private String readString(int offset) {
- int strOffset = offset;
- int len = buffer.get(strOffset++);
- char[] text = new char[len];
- for (int i = 0; i < len; i++) {
- text[i] = buffer.getChar(strOffset + (i << 1));
- }
- return new String(text);
- }
-
- private boolean hasSinglePOS(int wordId) {
- return (buffer.getShort(wordId + 2) & HAS_SINGLE_POS) != 0;
- }
-
- private boolean hasReadingData(int wordId) {
- return (buffer.getShort(wordId + 2) & HAS_READING) != 0;
- }
-
- /** flag that the entry has a single part of speech (leftPOS) */
- public static final int HAS_SINGLE_POS = 1;
-
- /** flag that the entry has reading data. otherwise reading is surface form */
- public static final int HAS_READING = 2;
-}
diff --git a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/CharacterDefinition.java b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/CharacterDefinition.java
index 5e2e48932ba..bcdcfde2f06 100644
--- a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/CharacterDefinition.java
+++ b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/CharacterDefinition.java
@@ -16,20 +16,13 @@
*/
package org.apache.lucene.analysis.ko.dict;
-import java.io.BufferedInputStream;
import java.io.IOException;
import java.io.InputStream;
-import org.apache.lucene.codecs.CodecUtil;
-import org.apache.lucene.store.DataInput;
-import org.apache.lucene.store.InputStreamDataInput;
import org.apache.lucene.util.IOUtils;
/** Character category data. */
-public final class CharacterDefinition {
-
- public static final String FILENAME_SUFFIX = ".dat";
- public static final String HEADER = "ko_cd";
- public static final int VERSION = 1;
+public final class CharacterDefinition
+ extends org.apache.lucene.analysis.morph.CharacterDefinition {
public static final int CLASS_COUNT = CharacterClass.values().length;
@@ -51,11 +44,6 @@ public final class CharacterDefinition {
HANJANUMERIC
}
- private final byte[] characterCategoryMap = new byte[0x10000];
-
- private final boolean[] invokeMap = new boolean[CLASS_COUNT];
- private final boolean[] groupMap = new boolean[CLASS_COUNT];
-
// the classes:
public static final byte NGRAM = (byte) CharacterClass.NGRAM.ordinal();
public static final byte DEFAULT = (byte) CharacterClass.DEFAULT.ordinal();
@@ -73,16 +61,11 @@ public final class CharacterDefinition {
public static final byte HANJANUMERIC = (byte) CharacterClass.HANJANUMERIC.ordinal();
private CharacterDefinition() throws IOException {
- try (InputStream is = new BufferedInputStream(getClassResource())) {
- final DataInput in = new InputStreamDataInput(is);
- CodecUtil.checkHeader(in, HEADER, VERSION, VERSION);
- in.readBytes(characterCategoryMap, 0, characterCategoryMap.length);
- for (int i = 0; i < CLASS_COUNT; i++) {
- final byte b = in.readByte();
- invokeMap[i] = (b & 0x01) != 0;
- groupMap[i] = (b & 0x02) != 0;
- }
- }
+ super(
+ CharacterDefinition::getClassResource,
+ DictionaryConstants.CHARDEF_HEADER,
+ DictionaryConstants.VERSION,
+ CharacterDefinition.CLASS_COUNT);
}
private static InputStream getClassResource() throws IOException {
@@ -91,18 +74,6 @@ public final class CharacterDefinition {
CharacterDefinition.class.getResourceAsStream(resourcePath), resourcePath);
}
- public byte getCharacterClass(char c) {
- return characterCategoryMap[c];
- }
-
- public boolean isInvoke(char c) {
- return invokeMap[characterCategoryMap[c]];
- }
-
- public boolean isGroup(char c) {
- return groupMap[characterCategoryMap[c]];
- }
-
public boolean isHanja(char c) {
final byte characterClass = getCharacterClass(c);
return characterClass == HANJA || characterClass == HANJANUMERIC;
diff --git a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/ConnectionCosts.java b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/ConnectionCosts.java
index 61579aef845..3b13a86b90a 100644
--- a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/ConnectionCosts.java
+++ b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/ConnectionCosts.java
@@ -16,27 +16,15 @@
*/
package org.apache.lucene.analysis.ko.dict;
-import java.io.BufferedInputStream;
import java.io.IOException;
import java.io.InputStream;
-import java.nio.ByteBuffer;
import java.nio.file.Files;
import java.nio.file.Path;
-import org.apache.lucene.codecs.CodecUtil;
-import org.apache.lucene.store.DataInput;
-import org.apache.lucene.store.InputStreamDataInput;
import org.apache.lucene.util.IOSupplier;
import org.apache.lucene.util.IOUtils;
/** n-gram connection cost data */
-public final class ConnectionCosts {
-
- public static final String FILENAME_SUFFIX = ".dat";
- public static final String HEADER = "ko_cc";
- public static final int VERSION = 1;
-
- private final ByteBuffer buffer;
- private final int forwardSize;
+public final class ConnectionCosts extends org.apache.lucene.analysis.morph.ConnectionCosts {
/**
* Create a {@link ConnectionCosts} from an external resource path.
@@ -53,24 +41,8 @@ public final class ConnectionCosts {
}
private ConnectionCosts(IOSupplier<InputStream> connectionCostResource) throws IOException {
- try (InputStream is = new BufferedInputStream(connectionCostResource.get())) {
- final DataInput in = new InputStreamDataInput(is);
- CodecUtil.checkHeader(in, HEADER, VERSION, VERSION);
- this.forwardSize = in.readVInt();
- int backwardSize = in.readVInt();
- int size = forwardSize * backwardSize;
-
- // copy the matrix into a direct byte buffer
- final ByteBuffer tmpBuffer = ByteBuffer.allocateDirect(size * 2);
- int accum = 0;
- for (int j = 0; j < backwardSize; j++) {
- for (int i = 0; i < forwardSize; i++) {
- accum += in.readZInt();
- tmpBuffer.putShort((short) accum);
- }
- }
- buffer = tmpBuffer.asReadOnlyBuffer();
- }
+ super(
+ connectionCostResource, DictionaryConstants.CONN_COSTS_HEADER, DictionaryConstants.VERSION);
}
private static InputStream getClassResource() throws IOException {
@@ -79,12 +51,6 @@ public final class ConnectionCosts {
ConnectionCosts.class.getResourceAsStream(resourcePath), resourcePath);
}
- public int get(int forwardId, int backwardId) {
- // map 2d matrix into a single dimension short array
- int offset = (backwardId * forwardSize + forwardId) * 2;
- return buffer.getShort(offset);
- }
-
public static ConnectionCosts getInstance() {
return SingletonHolder.INSTANCE;
}
diff --git a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/DictionaryConstants.java b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/DictionaryConstants.java
new file mode 100644
index 00000000000..4f8761e59e7
--- /dev/null
+++ b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/DictionaryConstants.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.analysis.ko.dict;
+
+/** Dictionary constants */
+public final class DictionaryConstants {
+ /** Codec header of the dictionary file. */
+ public static final String DICT_HEADER = "ko_dict";
+ /** Codec header of the dictionary mapping file. */
+ public static final String TARGETMAP_HEADER = "ko_dict_map";
+ /** Codec header of the POS dictionary file. */
+ public static final String POSDICT_HEADER = "ko_dict_pos";
+ /** Codec header of the connection costs file. */
+ public static final String CONN_COSTS_HEADER = "ko_cc";
+ /** Codec header of the character definition file */
+ public static final String CHARDEF_HEADER = "ko_cd";
+ /** Codec version of the binary dictionary */
+ public static final int VERSION = 1;
+}
diff --git a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/Dictionary.java b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/KoMorphData.java
similarity index 52%
rename from lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/Dictionary.java
rename to lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/KoMorphData.java
index b25e1b5ff5e..0887a7f0c42 100644
--- a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/Dictionary.java
+++ b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/KoMorphData.java
@@ -16,51 +16,47 @@
*/
package org.apache.lucene.analysis.ko.dict;
-import org.apache.lucene.analysis.ko.POS.Tag;
-import org.apache.lucene.analysis.ko.POS.Type;
+import org.apache.lucene.analysis.ko.POS;
+import org.apache.lucene.analysis.morph.MorphData;
-/** Dictionary interface for retrieving morphological data by id. */
-public interface Dictionary {
+/** Represents Korean morphological information. */
+public interface KoMorphData extends MorphData {
/** A morpheme extracted from a compound token. */
class Morpheme {
- public final Tag posTag;
+ public final POS.Tag posTag;
public final String surfaceForm;
- public Morpheme(Tag posTag, String surfaceForm) {
+ public Morpheme(POS.Tag posTag, String surfaceForm) {
this.posTag = posTag;
this.surfaceForm = surfaceForm;
}
}
- /** Get left id of specified word */
- int getLeftId(int wordId);
-
- /** Get right id of specified word */
- int getRightId(int wordId);
-
- /** Get word cost of specified word */
- int getWordCost(int wordId);
-
- /** Get the {@link Type} of specified word (morpheme, compound, inflect or pre-analysis) */
- Type getPOSType(int wordId);
+ /**
+ * Get the {@link org.apache.lucene.analysis.ko.POS.Type} of specified word (morpheme, compound,
+ * inflect or pre-analysis)
+ */
+ POS.Type getPOSType(int morphId);
/**
- * Get the left {@link Tag} of specfied word.
+ * Get the left {@link org.apache.lucene.analysis.ko.POS.Tag} of specfied word.
*
- * <p>For {@link Type#MORPHEME} and {@link Type#COMPOUND} the left and right POS are the same.
+ * <p>For {@link org.apache.lucene.analysis.ko.POS.Type#MORPHEME} and {@link
+ * org.apache.lucene.analysis.ko.POS.Type#COMPOUND} the left and right POS are the same.
*/
- Tag getLeftPOS(int wordId);
+ POS.Tag getLeftPOS(int morphId);
/**
- * Get the right {@link Tag} of specfied word.
+ * Get the right {@link org.apache.lucene.analysis.ko.POS.Tag} of specfied word.
*
- * <p>For {@link Type#MORPHEME} and {@link Type#COMPOUND} the left and right POS are the same.
+ * <p>For {@link org.apache.lucene.analysis.ko.POS.Type#MORPHEME} and {@link
+ * org.apache.lucene.analysis.ko.POS.Type#COMPOUND} the left and right POS are the same.
*/
- Tag getRightPOS(int wordId);
+ POS.Tag getRightPOS(int morphId);
/** Get the reading of specified word (mainly used for Hanja to Hangul conversion). */
- String getReading(int wordId);
+ String getReading(int morphId);
/** Get the morphemes of specified word (e.g. 가깝으나: 가깝 + 으나). */
- Morpheme[] getMorphemes(int wordId, char[] surfaceForm, int off, int len);
+ Morpheme[] getMorphemes(int morphId, char[] surfaceForm, int off, int len);
}
diff --git a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/TokenInfoDictionary.java b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/TokenInfoDictionary.java
index 1f936b44724..b8132cea61b 100644
--- a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/TokenInfoDictionary.java
+++ b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/TokenInfoDictionary.java
@@ -21,6 +21,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Path;
+import org.apache.lucene.analysis.morph.BinaryDictionary;
import org.apache.lucene.store.DataInput;
import org.apache.lucene.store.InputStreamDataInput;
import org.apache.lucene.util.IOSupplier;
@@ -32,11 +33,12 @@ import org.apache.lucene.util.fst.PositiveIntOutputs;
* Binary dictionary implementation for a known-word dictionary model: Words are encoded into an FST
* mapping to a list of wordIDs.
*/
-public final class TokenInfoDictionary extends BinaryDictionary {
+public final class TokenInfoDictionary extends BinaryDictionary<TokenInfoMorphData> {
public static final String FST_FILENAME_SUFFIX = "$fst.dat";
private final TokenInfoFST fst;
+ private final TokenInfoMorphData morphAtts;
private TokenInfoDictionary() throws IOException {
this(
@@ -70,7 +72,13 @@ public final class TokenInfoDictionary extends BinaryDictionary {
IOSupplier<InputStream> dictResource,
IOSupplier<InputStream> fstResource)
throws IOException {
- super(targetMapResource, posResource, dictResource);
+ super(
+ targetMapResource,
+ dictResource,
+ DictionaryConstants.TARGETMAP_HEADER,
+ DictionaryConstants.DICT_HEADER,
+ DictionaryConstants.VERSION);
+ this.morphAtts = new TokenInfoMorphData(buffer, posResource);
FST<Long> fst;
try (InputStream is = new BufferedInputStream(fstResource.get())) {
DataInput in = new InputStreamDataInput(is);
@@ -93,6 +101,11 @@ public final class TokenInfoDictionary extends BinaryDictionary {
return SingletonHolder.INSTANCE;
}
+ @Override
+ public TokenInfoMorphData getMorphAttributes() {
+ return morphAtts;
+ }
+
private static class SingletonHolder {
static final TokenInfoDictionary INSTANCE;
diff --git a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/TokenInfoMorphData.java b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/TokenInfoMorphData.java
new file mode 100644
index 00000000000..c8046b2e604
--- /dev/null
+++ b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/TokenInfoMorphData.java
@@ -0,0 +1,155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.analysis.ko.dict;
+
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+import org.apache.lucene.analysis.ko.POS;
+import org.apache.lucene.codecs.CodecUtil;
+import org.apache.lucene.store.DataInput;
+import org.apache.lucene.store.InputStreamDataInput;
+import org.apache.lucene.util.IOSupplier;
+
+/** Morphological information for system dictionary. */
+public class TokenInfoMorphData implements KoMorphData {
+
+ private final ByteBuffer buffer;
+ private final POS.Tag[] posDict;
+
+ TokenInfoMorphData(ByteBuffer buffer, IOSupplier<InputStream> posResource) throws IOException {
+ this.buffer = buffer;
+ try (InputStream posIS = new BufferedInputStream(posResource.get())) {
+ DataInput in = new InputStreamDataInput(posIS);
+ CodecUtil.checkHeader(
+ in,
+ DictionaryConstants.POSDICT_HEADER,
+ DictionaryConstants.VERSION,
+ DictionaryConstants.VERSION);
+ int posSize = in.readVInt();
+ this.posDict = new POS.Tag[posSize];
+ for (int j = 0; j < posSize; j++) {
+ posDict[j] = POS.resolveTag(in.readByte());
+ }
+ }
+ }
+
+ @Override
+ public int getLeftId(int morphId) {
+ return buffer.getShort(morphId) >>> 2;
+ }
+
+ @Override
+ public int getRightId(int morphId) {
+ return buffer.getShort(morphId + 2) >>> 2; // Skip left id
+ }
+
+ @Override
+ public int getWordCost(int morphId) {
+ return buffer.getShort(morphId + 4); // Skip left and right id
+ }
+
+ @Override
+ public POS.Type getPOSType(int morphId) {
+ byte value = (byte) (buffer.getShort(morphId) & 3);
+ return POS.resolveType(value);
+ }
+
+ @Override
+ public POS.Tag getLeftPOS(int morphId) {
+ return posDict[getLeftId(morphId)];
+ }
+
+ @Override
+ public POS.Tag getRightPOS(int morphId) {
+ POS.Type type = getPOSType(morphId);
+ if (type == POS.Type.MORPHEME || type == POS.Type.COMPOUND || hasSinglePOS(morphId)) {
+ return getLeftPOS(morphId);
+ } else {
+ byte value = buffer.get(morphId + 6);
+ return POS.resolveTag(value);
+ }
+ }
+
+ @Override
+ public String getReading(int morphId) {
+ if (hasReadingData(morphId)) {
+ int offset = morphId + 6;
+ return readString(offset);
+ }
+ return null;
+ }
+
+ @Override
+ public Morpheme[] getMorphemes(int morphId, char[] surfaceForm, int off, int len) {
+ POS.Type posType = getPOSType(morphId);
+ if (posType == POS.Type.MORPHEME) {
+ return null;
+ }
+ int offset = morphId + 6;
+ boolean hasSinglePos = hasSinglePOS(morphId);
+ if (hasSinglePos == false) {
+ offset++; // skip rightPOS
+ }
+ int length = buffer.get(offset++);
+ if (length == 0) {
+ return null;
+ }
+ Morpheme[] morphemes = new Morpheme[length];
+ int surfaceOffset = 0;
+ final POS.Tag leftPOS = getLeftPOS(morphId);
+ for (int i = 0; i < length; i++) {
+ final String form;
+ final POS.Tag tag = hasSinglePos ? leftPOS : POS.resolveTag(buffer.get(offset++));
+ if (posType == POS.Type.INFLECT) {
+ form = readString(offset);
+ offset += form.length() * 2 + 1;
+ } else {
+ int formLen = buffer.get(offset++);
+ form = new String(surfaceForm, off + surfaceOffset, formLen);
+ surfaceOffset += formLen;
+ }
+ morphemes[i] = new Morpheme(tag, form);
+ }
+ return morphemes;
+ }
+
+ private String readString(int offset) {
+ int strOffset = offset;
+ int len = buffer.get(strOffset++);
+ char[] text = new char[len];
+ for (int i = 0; i < len; i++) {
+ text[i] = buffer.getChar(strOffset + (i << 1));
+ }
+ return new String(text);
+ }
+
+ private boolean hasSinglePOS(int wordId) {
+ return (buffer.getShort(wordId + 2) & HAS_SINGLE_POS) != 0;
+ }
+
+ private boolean hasReadingData(int wordId) {
+ return (buffer.getShort(wordId + 2) & HAS_READING) != 0;
+ }
+
+ /** flag that the entry has a single part of speech (leftPOS) */
+ public static final int HAS_SINGLE_POS = 1;
+
+ /** flag that the entry has reading data. otherwise reading is surface form */
+ public static final int HAS_READING = 2;
+}
diff --git a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/UnknownDictionary.java b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/UnknownDictionary.java
index f2aa89ee8c5..4b45fd33258 100644
--- a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/UnknownDictionary.java
+++ b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/UnknownDictionary.java
@@ -20,11 +20,14 @@ import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Path;
+import org.apache.lucene.analysis.morph.BinaryDictionary;
+import org.apache.lucene.util.IOSupplier;
import org.apache.lucene.util.IOUtils;
/** Dictionary for unknown-word handling. */
-public final class UnknownDictionary extends BinaryDictionary {
+public final class UnknownDictionary extends BinaryDictionary<UnknownMorphData> {
private final CharacterDefinition characterDefinition = CharacterDefinition.getInstance();
+ private final UnknownMorphData morphAtts;
/**
* Create a {@link UnknownDictionary} from an external resource path.
@@ -35,19 +38,33 @@ public final class UnknownDictionary extends BinaryDictionary {
* @throws IOException if resource was not found or broken
*/
public UnknownDictionary(Path targetMapFile, Path posDictFile, Path dictFile) throws IOException {
- super(
+ this(
() -> Files.newInputStream(targetMapFile),
() -> Files.newInputStream(posDictFile),
() -> Files.newInputStream(dictFile));
}
private UnknownDictionary() throws IOException {
- super(
+ this(
() -> getClassResource(TARGETMAP_FILENAME_SUFFIX),
() -> getClassResource(POSDICT_FILENAME_SUFFIX),
() -> getClassResource(DICT_FILENAME_SUFFIX));
}
+ private UnknownDictionary(
+ IOSupplier<InputStream> targetMapResource,
+ IOSupplier<InputStream> posResource,
+ IOSupplier<InputStream> dictResource)
+ throws IOException {
+ super(
+ targetMapResource,
+ dictResource,
+ DictionaryConstants.TARGETMAP_HEADER,
+ DictionaryConstants.DICT_HEADER,
+ DictionaryConstants.VERSION);
+ this.morphAtts = new UnknownMorphData(buffer, posResource);
+ }
+
private static InputStream getClassResource(String suffix) throws IOException {
final String resourcePath = UnknownDictionary.class.getSimpleName() + suffix;
return IOUtils.requireResourceNonNull(
@@ -63,13 +80,8 @@ public final class UnknownDictionary extends BinaryDictionary {
}
@Override
- public String getReading(int wordId) {
- return null;
- }
-
- @Override
- public Morpheme[] getMorphemes(int wordId, char[] surfaceForm, int off, int len) {
- return null;
+ public UnknownMorphData getMorphAttributes() {
+ return morphAtts;
}
private static class SingletonHolder {
diff --git a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/UnknownMorphData.java b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/UnknownMorphData.java
new file mode 100644
index 00000000000..dafb174f8d5
--- /dev/null
+++ b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/UnknownMorphData.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.analysis.ko.dict;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+import org.apache.lucene.util.IOSupplier;
+
+/** Morphological information for unk dictionary. */
+final class UnknownMorphData extends TokenInfoMorphData {
+ UnknownMorphData(ByteBuffer buffer, IOSupplier<InputStream> posResource) throws IOException {
+ super(buffer, posResource);
+ }
+
+ @Override
+ public String getReading(int wordId) {
+ return null;
+ }
+
+ @Override
+ public Morpheme[] getMorphemes(int wordId, char[] surfaceForm, int off, int len) {
+ return null;
+ }
+}
diff --git a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/UserDictionary.java b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/UserDictionary.java
index f8fcefdd295..58a233112aa 100644
--- a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/UserDictionary.java
+++ b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/UserDictionary.java
@@ -22,7 +22,7 @@ import java.io.Reader;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
-import org.apache.lucene.analysis.ko.POS;
+import org.apache.lucene.analysis.morph.Dictionary;
import org.apache.lucene.util.IntsRefBuilder;
import org.apache.lucene.util.fst.FST;
import org.apache.lucene.util.fst.FSTCompiler;
@@ -32,15 +32,10 @@ import org.apache.lucene.util.fst.PositiveIntOutputs;
* Class for building a User Dictionary. This class allows for adding custom nouns (세종) or compounds
* (세종시 세종 시).
*/
-public final class UserDictionary implements Dictionary {
+public final class UserDictionary implements Dictionary<UserMorphData> {
// text -> wordID
private final TokenInfoFST fst;
- private static final int WORD_COST = -100000;
-
- // NNG left
- private static final short LEFT_ID = 1781;
-
// NNG right
private static final short RIGHT_ID = 3533;
// NNG right with hangul and a coda on the last char
@@ -48,9 +43,7 @@ public final class UserDictionary implements Dictionary {
// NNG right with hangul and no coda on the last char
private static final short RIGHT_ID_F = 3534;
- // length, length... indexed by compound ID or null for simple noun
- private final int[][] segmentations;
- private final short[] rightIds;
+ private UserMorphData morphAtts;
public static UserDictionary open(Reader reader) throws IOException {
@@ -86,8 +79,8 @@ public final class UserDictionary implements Dictionary {
IntsRefBuilder scratch = new IntsRefBuilder();
String lastToken = null;
- List<int[]> segmentations = new ArrayList<>(entries.size());
- List<Short> rightIds = new ArrayList<>(entries.size());
+ List<int[]> _segmentations = new ArrayList<>(entries.size());
+ List<Short> _rightIds = new ArrayList<>(entries.size());
long ord = 0;
for (String entry : entries) {
String[] splits = entry.split("\\s+");
@@ -98,16 +91,16 @@ public final class UserDictionary implements Dictionary {
char lastChar = entry.charAt(entry.length() - 1);
if (charDef.isHangul(lastChar)) {
if (charDef.hasCoda(lastChar)) {
- rightIds.add(RIGHT_ID_T);
+ _rightIds.add(RIGHT_ID_T);
} else {
- rightIds.add(RIGHT_ID_F);
+ _rightIds.add(RIGHT_ID_F);
}
} else {
- rightIds.add(RIGHT_ID);
+ _rightIds.add(RIGHT_ID);
}
if (splits.length == 1) {
- segmentations.add(null);
+ _segmentations.add(null);
} else {
int[] length = new int[splits.length - 1];
int offset = 0;
@@ -123,7 +116,7 @@ public final class UserDictionary implements Dictionary {
+ token
+ ")");
}
- segmentations.add(length);
+ _segmentations.add(length);
}
// add mapping to FST
@@ -137,11 +130,12 @@ public final class UserDictionary implements Dictionary {
ord++;
}
this.fst = new TokenInfoFST(fstCompiler.compile());
- this.segmentations = segmentations.toArray(new int[segmentations.size()][]);
- this.rightIds = new short[rightIds.size()];
- for (int i = 0; i < rightIds.size(); i++) {
- this.rightIds[i] = rightIds.get(i);
+ int[][] segmentations = _segmentations.toArray(new int[_segmentations.size()][]);
+ short[] rightIds = new short[_rightIds.size()];
+ for (int i = 0; i < _rightIds.size(); i++) {
+ rightIds[i] = _rightIds.get(i);
}
+ this.morphAtts = new UserMorphData(segmentations, rightIds);
}
public TokenInfoFST getFST() {
@@ -149,57 +143,8 @@ public final class UserDictionary implements Dictionary {
}
@Override
- public int getLeftId(int wordId) {
- return LEFT_ID;
- }
-
- @Override
- public int getRightId(int wordId) {
- return rightIds[wordId];
- }
-
- @Override
- public int getWordCost(int wordId) {
- return WORD_COST;
- }
-
- @Override
- public POS.Type getPOSType(int wordId) {
- if (segmentations[wordId] == null) {
- return POS.Type.MORPHEME;
- } else {
- return POS.Type.COMPOUND;
- }
- }
-
- @Override
- public POS.Tag getLeftPOS(int wordId) {
- return POS.Tag.NNG;
- }
-
- @Override
- public POS.Tag getRightPOS(int wordId) {
- return POS.Tag.NNG;
- }
-
- @Override
- public String getReading(int wordId) {
- return null;
- }
-
- @Override
- public Morpheme[] getMorphemes(int wordId, char[] surfaceForm, int off, int len) {
- int[] segs = segmentations[wordId];
- if (segs == null) {
- return null;
- }
- int offset = 0;
- Morpheme[] morphemes = new Morpheme[segs.length];
- for (int i = 0; i < segs.length; i++) {
- morphemes[i] = new Morpheme(POS.Tag.NNG, new String(surfaceForm, off + offset, segs[i]));
- offset += segs[i];
- }
- return morphemes;
+ public UserMorphData getMorphAttributes() {
+ return morphAtts;
}
/**
diff --git a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/UserMorphData.java b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/UserMorphData.java
new file mode 100644
index 00000000000..2056bd0829c
--- /dev/null
+++ b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/UserMorphData.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.analysis.ko.dict;
+
+import org.apache.lucene.analysis.ko.POS;
+
+/** Morphological information for user dictionary. */
+final class UserMorphData implements KoMorphData {
+ private static final int WORD_COST = -100000;
+
+ // NNG left
+ private static final short LEFT_ID = 1781;
+
+ // length, length... indexed by compound ID or null for simple noun
+ private final int[][] segmentations;
+ private final short[] rightIds;
+
+ UserMorphData(int[][] segmentations, short[] rightIds) {
+ this.segmentations = segmentations;
+ this.rightIds = rightIds;
+ }
+
+ @Override
+ public int getLeftId(int morphId) {
+ return LEFT_ID;
+ }
+
+ @Override
+ public int getRightId(int morphId) {
+ return rightIds[morphId];
+ }
+
+ @Override
+ public int getWordCost(int morphId) {
+ return WORD_COST;
+ }
+
+ @Override
+ public POS.Type getPOSType(int morphId) {
+ if (segmentations[morphId] == null) {
+ return POS.Type.MORPHEME;
+ } else {
+ return POS.Type.COMPOUND;
+ }
+ }
+
+ @Override
+ public POS.Tag getLeftPOS(int morphId) {
+ return POS.Tag.NNG;
+ }
+
+ @Override
+ public POS.Tag getRightPOS(int morphId) {
+ return POS.Tag.NNG;
+ }
+
+ @Override
+ public String getReading(int morphId) {
+ return null;
+ }
+
+ @Override
+ public Morpheme[] getMorphemes(int morphId, char[] surfaceForm, int off, int len) {
+ int[] segs = segmentations[morphId];
+ if (segs == null) {
+ return null;
+ }
+ int offset = 0;
+ Morpheme[] morphemes = new Morpheme[segs.length];
+ for (int i = 0; i < segs.length; i++) {
+ morphemes[i] = new Morpheme(POS.Tag.NNG, new String(surfaceForm, off + offset, segs[i]));
+ offset += segs[i];
+ }
+ return morphemes;
+ }
+}
diff --git a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/tokenattributes/PartOfSpeechAttribute.java b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/tokenattributes/PartOfSpeechAttribute.java
index c9fb33b17c2..1e28155e667 100644
--- a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/tokenattributes/PartOfSpeechAttribute.java
+++ b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/tokenattributes/PartOfSpeechAttribute.java
@@ -19,7 +19,7 @@ package org.apache.lucene.analysis.ko.tokenattributes;
import org.apache.lucene.analysis.ko.POS.Tag;
import org.apache.lucene.analysis.ko.POS.Type;
import org.apache.lucene.analysis.ko.Token;
-import org.apache.lucene.analysis.ko.dict.Dictionary.Morpheme;
+import org.apache.lucene.analysis.ko.dict.KoMorphData;
import org.apache.lucene.util.Attribute;
/**
@@ -37,8 +37,11 @@ public interface PartOfSpeechAttribute extends Attribute {
/** Get the right part of speech of the token. */
Tag getRightPOS();
- /** Get the {@link Morpheme} decomposition of the token. */
- Morpheme[] getMorphemes();
+ /**
+ * Get the {@link org.apache.lucene.analysis.ko.dict.KoMorphData.Morpheme} decomposition of the
+ * token.
+ */
+ KoMorphData.Morpheme[] getMorphemes();
/** Set the current token. */
void setToken(Token token);
diff --git a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/tokenattributes/PartOfSpeechAttributeImpl.java b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/tokenattributes/PartOfSpeechAttributeImpl.java
index a1b04cb7f05..a59999db557 100644
--- a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/tokenattributes/PartOfSpeechAttributeImpl.java
+++ b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/tokenattributes/PartOfSpeechAttributeImpl.java
@@ -19,7 +19,7 @@ package org.apache.lucene.analysis.ko.tokenattributes;
import org.apache.lucene.analysis.ko.POS.Tag;
import org.apache.lucene.analysis.ko.POS.Type;
import org.apache.lucene.analysis.ko.Token;
-import org.apache.lucene.analysis.ko.dict.Dictionary.Morpheme;
+import org.apache.lucene.analysis.ko.dict.KoMorphData;
import org.apache.lucene.util.AttributeImpl;
import org.apache.lucene.util.AttributeReflector;
@@ -47,7 +47,7 @@ public class PartOfSpeechAttributeImpl extends AttributeImpl implements PartOfSp
}
@Override
- public Morpheme[] getMorphemes() {
+ public KoMorphData.Morpheme[] getMorphemes() {
return token == null ? null : token.getMorphemes();
}
@@ -76,12 +76,12 @@ public class PartOfSpeechAttributeImpl extends AttributeImpl implements PartOfSp
reflector.reflect(PartOfSpeechAttribute.class, "morphemes", displayMorphemes(getMorphemes()));
}
- private String displayMorphemes(Morpheme[] morphemes) {
+ private String displayMorphemes(KoMorphData.Morpheme[] morphemes) {
if (morphemes == null) {
return null;
}
StringBuilder builder = new StringBuilder();
- for (Morpheme morpheme : morphemes) {
+ for (KoMorphData.Morpheme morpheme : morphemes) {
if (builder.length() > 0) {
builder.append("+");
}
diff --git a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/util/ConnectionCostsBuilder.java b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/util/ConnectionCostsBuilder.java
index 4a6fd6d353c..31505fb45c8 100644
--- a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/util/ConnectionCostsBuilder.java
+++ b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/util/ConnectionCostsBuilder.java
@@ -22,12 +22,14 @@ import java.io.Reader;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
+import org.apache.lucene.analysis.ko.dict.ConnectionCosts;
+import org.apache.lucene.analysis.morph.ConnectionCostsWriter;
class ConnectionCostsBuilder {
private ConnectionCostsBuilder() {}
- public static ConnectionCostsWriter build(Path path) throws IOException {
+ public static ConnectionCostsWriter<ConnectionCosts> build(Path path) throws IOException {
try (Reader reader = Files.newBufferedReader(path, StandardCharsets.US_ASCII);
LineNumberReader lineReader = new LineNumberReader(reader)) {
@@ -41,7 +43,8 @@ class ConnectionCostsBuilder {
assert forwardSize > 0 && backwardSize > 0;
- ConnectionCostsWriter costs = new ConnectionCostsWriter(forwardSize, backwardSize);
+ ConnectionCostsWriter<ConnectionCosts> costs =
+ new ConnectionCostsWriter<>(ConnectionCosts.class, forwardSize, backwardSize);
while ((line = lineReader.readLine()) != null) {
String[] fields = line.split("\\s+");
diff --git a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/util/ConnectionCostsWriter.java b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/util/ConnectionCostsWriter.java
deleted file mode 100644
index 7265ef84dbb..00000000000
--- a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/util/ConnectionCostsWriter.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.analysis.ko.util;
-
-import java.io.BufferedOutputStream;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.nio.ByteBuffer;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import org.apache.lucene.analysis.ko.dict.ConnectionCosts;
-import org.apache.lucene.codecs.CodecUtil;
-import org.apache.lucene.store.DataOutput;
-import org.apache.lucene.store.OutputStreamDataOutput;
-
-final class ConnectionCostsWriter {
-
- private final ByteBuffer
- costs; // array is backward IDs first since get is called using the same backward ID
- // consecutively. maybe doesn't matter.
- private final int forwardSize;
- private final int backwardSize;
- /** Constructor for building. TODO: remove write access */
- ConnectionCostsWriter(int forwardSize, int backwardSize) {
- this.forwardSize = forwardSize;
- this.backwardSize = backwardSize;
- this.costs = ByteBuffer.allocateDirect(2 * backwardSize * forwardSize);
- }
-
- public void add(int forwardId, int backwardId, int cost) {
- int offset = (backwardId * forwardSize + forwardId) * 2;
- costs.putShort(offset, (short) cost);
- }
-
- public void write(Path baseDir) throws IOException {
- Files.createDirectories(baseDir);
- String fileName =
- ConnectionCosts.class.getName().replace('.', '/') + ConnectionCosts.FILENAME_SUFFIX;
- try (OutputStream os = Files.newOutputStream(baseDir.resolve(fileName));
- OutputStream bos = new BufferedOutputStream(os)) {
- final DataOutput out = new OutputStreamDataOutput(bos);
- CodecUtil.writeHeader(out, ConnectionCosts.HEADER, ConnectionCosts.VERSION);
- out.writeVInt(forwardSize);
- out.writeVInt(backwardSize);
- int last = 0;
- for (int i = 0; i < costs.limit() / 2; i++) {
- short cost = costs.getShort(i * 2);
- int delta = (int) cost - last;
- out.writeZInt(delta);
- last = cost;
- }
- }
- }
-}
diff --git a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/util/DictionaryBuilder.java b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/util/DictionaryBuilder.java
index e4c3b20c9b6..0e8f11f0cf8 100644
--- a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/util/DictionaryBuilder.java
+++ b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/util/DictionaryBuilder.java
@@ -19,6 +19,7 @@ package org.apache.lucene.analysis.ko.util;
import java.io.IOException;
import java.nio.file.Path;
import java.nio.file.Paths;
+import org.apache.lucene.analysis.ko.dict.DictionaryConstants;
/** Tool to build dictionaries. */
public class DictionaryBuilder {
@@ -34,7 +35,8 @@ public class DictionaryBuilder {
new UnknownDictionaryBuilder(encoding).build(inputDir).write(outputDir);
// Build Connection Cost
- ConnectionCostsBuilder.build(inputDir.resolve("matrix.def")).write(outputDir);
+ ConnectionCostsBuilder.build(inputDir.resolve("matrix.def"))
+ .write(outputDir, DictionaryConstants.CONN_COSTS_HEADER, DictionaryConstants.VERSION);
}
public static void main(String[] args) throws IOException {
diff --git a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/util/BinaryDictionaryWriter.java b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/util/TokenInfoDictionaryEntryWriter.java
similarity index 50%
rename from lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/util/BinaryDictionaryWriter.java
rename to lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/util/TokenInfoDictionaryEntryWriter.java
index 37ed157f8c8..e18cae357d3 100644
--- a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/util/BinaryDictionaryWriter.java
+++ b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/util/TokenInfoDictionaryEntryWriter.java
@@ -16,38 +16,25 @@
*/
package org.apache.lucene.analysis.ko.util;
-import java.io.BufferedOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.ByteBuffer;
-import java.nio.channels.Channels;
-import java.nio.channels.WritableByteChannel;
-import java.nio.file.Files;
-import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import org.apache.lucene.analysis.ko.POS;
-import org.apache.lucene.analysis.ko.dict.BinaryDictionary;
-import org.apache.lucene.analysis.ko.dict.Dictionary;
-import org.apache.lucene.codecs.CodecUtil;
+import org.apache.lucene.analysis.ko.dict.KoMorphData;
+import org.apache.lucene.analysis.ko.dict.TokenInfoMorphData;
+import org.apache.lucene.analysis.morph.DictionaryEntryWriter;
import org.apache.lucene.store.DataOutput;
-import org.apache.lucene.store.OutputStreamDataOutput;
import org.apache.lucene.util.ArrayUtil;
-abstract class BinaryDictionaryWriter {
+/** Writes system dictionary entries. */
+class TokenInfoDictionaryEntryWriter extends DictionaryEntryWriter {
private static final int ID_LIMIT = 8192;
- private final Class<? extends BinaryDictionary> implClazz;
- protected ByteBuffer buffer;
- private int targetMapEndOffset = 0, lastWordId = -1, lastSourceId = -1;
- private int[] targetMap = new int[8192];
- private int[] targetMapOffsets = new int[8192];
- private final ArrayList<String> posDict = new ArrayList<>();
-
- BinaryDictionaryWriter(Class<? extends BinaryDictionary> implClazz, int size) {
- this.implClazz = implClazz;
- buffer = ByteBuffer.allocateDirect(size);
+ TokenInfoDictionaryEntryWriter(int size) {
+ super(size);
}
/**
@@ -72,7 +59,8 @@ abstract class BinaryDictionaryWriter {
*
* @return current position of buffer, which will be wordId of next entry
*/
- public int put(String[] entry) {
+ @Override
+ protected int putEntry(String[] entry) {
short leftId = Short.parseShort(entry[1]);
short rightId = Short.parseShort(entry[2]);
short wordCost = Short.parseShort(entry[3]);
@@ -113,7 +101,7 @@ abstract class BinaryDictionaryWriter {
assert existing == null || existing.equals(fullPOSData);
posDict.set(leftId, fullPOSData);
- final List<Dictionary.Morpheme> morphemes = new ArrayList<>();
+ final List<KoMorphData.Morpheme> morphemes = new ArrayList<>();
// true if the POS and decompounds of the token are all the same.
boolean hasSinglePOS = (leftPOS == rightPOS);
if (posType != POS.Type.MORPHEME && expression.length() > 0) {
@@ -124,7 +112,7 @@ abstract class BinaryDictionaryWriter {
String surfaceForm = tokenSplit[0].trim();
if (surfaceForm.isEmpty() == false) {
POS.Tag exprTag = POS.resolveTag(tokenSplit[1]);
- morphemes.add(new Dictionary.Morpheme(exprTag, tokenSplit[0]));
+ morphemes.add(new KoMorphData.Morpheme(exprTag, tokenSplit[0]));
if (leftPOS != exprTag) {
hasSinglePOS = false;
}
@@ -134,10 +122,10 @@ abstract class BinaryDictionaryWriter {
int flags = 0;
if (hasSinglePOS) {
- flags |= BinaryDictionary.HAS_SINGLE_POS;
+ flags |= TokenInfoMorphData.HAS_SINGLE_POS;
}
if (posType == POS.Type.MORPHEME && reading.length() > 0) {
- flags |= BinaryDictionary.HAS_READING;
+ flags |= TokenInfoMorphData.HAS_READING;
}
if (leftId >= ID_LIMIT) {
@@ -161,7 +149,7 @@ abstract class BinaryDictionaryWriter {
}
buffer.put((byte) morphemes.size());
int compoundOffset = 0;
- for (Dictionary.Morpheme morpheme : morphemes) {
+ for (KoMorphData.Morpheme morpheme : morphemes) {
if (hasSinglePOS == false) {
buffer.put((byte) morpheme.posTag.ordinal());
}
@@ -184,112 +172,20 @@ abstract class BinaryDictionaryWriter {
}
}
- void addMapping(int sourceId, int wordId) {
- if (wordId <= lastWordId) {
- throw new IllegalStateException(
- "words out of order: " + wordId + " vs lastID: " + lastWordId);
- }
-
- if (sourceId > lastSourceId) {
- targetMapOffsets = ArrayUtil.grow(targetMapOffsets, sourceId + 1);
- for (int i = lastSourceId + 1; i <= sourceId; i++) {
- targetMapOffsets[i] = targetMapEndOffset;
- }
- } else if (sourceId != lastSourceId) {
- throw new IllegalStateException(
- "source ids not in increasing order: lastSourceId="
- + lastSourceId
- + " vs sourceId="
- + sourceId);
- }
-
- targetMap = ArrayUtil.grow(targetMap, targetMapEndOffset + 1);
- targetMap[targetMapEndOffset] = wordId;
- targetMapEndOffset++;
-
- lastSourceId = sourceId;
- lastWordId = wordId;
- }
-
- final String getBaseFileName() {
- return implClazz.getName().replace('.', '/');
- }
-
- /**
- * Write dictionary in file
- *
- * @throws IOException if an I/O error occurs writing the dictionary files
- */
- public void write(Path baseDir) throws IOException {
- final String baseName = getBaseFileName();
- writeDictionary(baseDir.resolve(baseName + BinaryDictionary.DICT_FILENAME_SUFFIX));
- writeTargetMap(baseDir.resolve(baseName + BinaryDictionary.TARGETMAP_FILENAME_SUFFIX));
- writePosDict(baseDir.resolve(baseName + BinaryDictionary.POSDICT_FILENAME_SUFFIX));
- }
-
- private void writeTargetMap(Path path) throws IOException {
- Files.createDirectories(path.getParent());
- try (OutputStream os = Files.newOutputStream(path);
- OutputStream bos = new BufferedOutputStream(os)) {
- final DataOutput out = new OutputStreamDataOutput(bos);
- CodecUtil.writeHeader(out, BinaryDictionary.TARGETMAP_HEADER, BinaryDictionary.VERSION);
-
- final int numSourceIds = lastSourceId + 1;
- out.writeVInt(targetMapEndOffset); // <-- size of main array
- out.writeVInt(numSourceIds + 1); // <-- size of offset array (+ 1 more entry)
- int prev = 0, sourceId = 0;
- for (int ofs = 0; ofs < targetMapEndOffset; ofs++) {
- final int val = targetMap[ofs], delta = val - prev;
- assert delta >= 0;
- if (ofs == targetMapOffsets[sourceId]) {
- out.writeVInt((delta << 1) | 0x01);
- sourceId++;
- } else {
- out.writeVInt((delta << 1));
- }
- prev += delta;
- }
- if (sourceId != numSourceIds) {
- throw new IllegalStateException(
- "sourceId:" + sourceId + " != numSourceIds:" + numSourceIds);
- }
- }
- }
-
- private void writePosDict(Path path) throws IOException {
- Files.createDirectories(path.getParent());
- try (OutputStream os = Files.newOutputStream(path);
- OutputStream bos = new BufferedOutputStream(os)) {
- final DataOutput out = new OutputStreamDataOutput(bos);
- CodecUtil.writeHeader(out, BinaryDictionary.POSDICT_HEADER, BinaryDictionary.VERSION);
- out.writeVInt(posDict.size());
- for (String s : posDict) {
- if (s == null) {
- out.writeByte((byte) POS.Tag.UNKNOWN.ordinal());
- } else {
- String[] data = CSVUtil.parse(s);
- if (data.length != 2) {
- throw new IllegalArgumentException(
- "Malformed pos/inflection: " + s + "; expected 2 characters");
- }
- out.writeByte((byte) POS.Tag.valueOf(data[0]).ordinal());
+ @Override
+ protected void writePosDict(OutputStream bos, DataOutput out) throws IOException {
+ out.writeVInt(posDict.size());
+ for (String s : posDict) {
+ if (s == null) {
+ out.writeByte((byte) POS.Tag.UNKNOWN.ordinal());
+ } else {
+ String[] data = CSVUtil.parse(s);
+ if (data.length != 2) {
+ throw new IllegalArgumentException(
+ "Malformed pos/inflection: " + s + "; expected 2 characters");
}
+ out.writeByte((byte) POS.Tag.valueOf(data[0]).ordinal());
}
}
}
-
- private void writeDictionary(Path path) throws IOException {
- Files.createDirectories(path.getParent());
- try (OutputStream os = Files.newOutputStream(path);
- OutputStream bos = new BufferedOutputStream(os)) {
- final DataOutput out = new OutputStreamDataOutput(bos);
- CodecUtil.writeHeader(out, BinaryDictionary.DICT_HEADER, BinaryDictionary.VERSION);
- out.writeVInt(buffer.position());
- final WritableByteChannel channel = Channels.newChannel(bos);
- // Write Buffer
- buffer.flip(); // set position to 0, set limit to current position
- channel.write(buffer);
- assert buffer.remaining() == 0L;
- }
- }
}
diff --git a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/util/TokenInfoDictionaryWriter.java b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/util/TokenInfoDictionaryWriter.java
index 316e1b6232a..78a9acaff8a 100644
--- a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/util/TokenInfoDictionaryWriter.java
+++ b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/util/TokenInfoDictionaryWriter.java
@@ -20,14 +20,16 @@ import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Objects;
+import org.apache.lucene.analysis.ko.dict.DictionaryConstants;
import org.apache.lucene.analysis.ko.dict.TokenInfoDictionary;
+import org.apache.lucene.analysis.morph.BinaryDictionaryWriter;
import org.apache.lucene.util.fst.FST;
-class TokenInfoDictionaryWriter extends BinaryDictionaryWriter {
+class TokenInfoDictionaryWriter extends BinaryDictionaryWriter<TokenInfoDictionary> {
private FST<Long> fst;
TokenInfoDictionaryWriter(int size) {
- super(TokenInfoDictionary.class, size);
+ super(TokenInfoDictionary.class, new TokenInfoDictionaryEntryWriter(size));
}
public void setFST(FST<Long> fst) {
@@ -35,9 +37,19 @@ class TokenInfoDictionaryWriter extends BinaryDictionaryWriter {
this.fst = fst;
}
+ @Override
+ protected void addMapping(int sourceId, int wordId) {
+ super.addMapping(sourceId, wordId);
+ }
+
@Override
public void write(Path baseDir) throws IOException {
- super.write(baseDir);
+ super.write(
+ baseDir,
+ DictionaryConstants.TARGETMAP_HEADER,
+ DictionaryConstants.POSDICT_HEADER,
+ DictionaryConstants.DICT_HEADER,
+ DictionaryConstants.VERSION);
writeFST(baseDir.resolve(getBaseFileName() + TokenInfoDictionary.FST_FILENAME_SUFFIX));
}
diff --git a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/util/UnknownDictionaryWriter.java b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/util/UnknownDictionaryWriter.java
index 97b525d60f0..82285e1c78b 100644
--- a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/util/UnknownDictionaryWriter.java
+++ b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/util/UnknownDictionaryWriter.java
@@ -19,20 +19,29 @@ package org.apache.lucene.analysis.ko.util;
import java.io.IOException;
import java.nio.file.Path;
import org.apache.lucene.analysis.ko.dict.CharacterDefinition;
+import org.apache.lucene.analysis.ko.dict.DictionaryConstants;
import org.apache.lucene.analysis.ko.dict.UnknownDictionary;
+import org.apache.lucene.analysis.morph.BinaryDictionaryWriter;
+import org.apache.lucene.analysis.morph.CharacterDefinitionWriter;
-class UnknownDictionaryWriter extends BinaryDictionaryWriter {
+class UnknownDictionaryWriter extends BinaryDictionaryWriter<UnknownDictionary> {
- private final CharacterDefinitionWriter characterDefinition = new CharacterDefinitionWriter();
+ private final org.apache.lucene.analysis.morph.CharacterDefinitionWriter<CharacterDefinition>
+ characterDefinition =
+ new CharacterDefinitionWriter<>(
+ CharacterDefinition.class,
+ CharacterDefinition.DEFAULT,
+ CharacterDefinition.CLASS_COUNT,
+ CharacterDefinition::lookupCharacterClass);
public UnknownDictionaryWriter(int size) {
- super(UnknownDictionary.class, size);
+ super(UnknownDictionary.class, new TokenInfoDictionaryEntryWriter(size));
}
@Override
public int put(String[] entry) {
// Get wordId of current entry
- int wordId = buffer.position();
+ int wordId = entryWriter.currentPosition();
// Put entry
int result = super.put(entry);
@@ -59,7 +68,13 @@ class UnknownDictionaryWriter extends BinaryDictionaryWriter {
@Override
public void write(Path baseDir) throws IOException {
- super.write(baseDir);
- characterDefinition.write(baseDir);
+ super.write(
+ baseDir,
+ DictionaryConstants.TARGETMAP_HEADER,
+ DictionaryConstants.POSDICT_HEADER,
+ DictionaryConstants.DICT_HEADER,
+ DictionaryConstants.VERSION);
+ characterDefinition.write(
+ baseDir, DictionaryConstants.CHARDEF_HEADER, DictionaryConstants.VERSION);
}
}
diff --git a/lucene/analysis/nori/src/test/org/apache/lucene/analysis/ko/dict/TestExternalDictionary.java b/lucene/analysis/nori/src/test/org/apache/lucene/analysis/ko/dict/TestExternalDictionary.java
index 5f8edab8934..2ba7cee6a37 100644
--- a/lucene/analysis/nori/src/test/org/apache/lucene/analysis/ko/dict/TestExternalDictionary.java
+++ b/lucene/analysis/nori/src/test/org/apache/lucene/analysis/ko/dict/TestExternalDictionary.java
@@ -16,10 +16,10 @@
*/
package org.apache.lucene.analysis.ko.dict;
-import static org.apache.lucene.analysis.ko.dict.BinaryDictionary.DICT_FILENAME_SUFFIX;
-import static org.apache.lucene.analysis.ko.dict.BinaryDictionary.POSDICT_FILENAME_SUFFIX;
-import static org.apache.lucene.analysis.ko.dict.BinaryDictionary.TARGETMAP_FILENAME_SUFFIX;
import static org.apache.lucene.analysis.ko.dict.TokenInfoDictionary.FST_FILENAME_SUFFIX;
+import static org.apache.lucene.analysis.morph.BinaryDictionary.DICT_FILENAME_SUFFIX;
+import static org.apache.lucene.analysis.morph.BinaryDictionary.POSDICT_FILENAME_SUFFIX;
+import static org.apache.lucene.analysis.morph.BinaryDictionary.TARGETMAP_FILENAME_SUFFIX;
import java.io.BufferedWriter;
import java.nio.charset.StandardCharsets;
diff --git a/lucene/analysis/nori/src/test/org/apache/lucene/analysis/ko/dict/TestTokenInfoDictionary.java b/lucene/analysis/nori/src/test/org/apache/lucene/analysis/ko/dict/TestTokenInfoDictionary.java
index 39fc55065d9..6948c4aa851 100644
--- a/lucene/analysis/nori/src/test/org/apache/lucene/analysis/ko/dict/TestTokenInfoDictionary.java
+++ b/lucene/analysis/nori/src/test/org/apache/lucene/analysis/ko/dict/TestTokenInfoDictionary.java
@@ -16,10 +16,10 @@
*/
package org.apache.lucene.analysis.ko.dict;
-import static org.apache.lucene.analysis.ko.dict.BinaryDictionary.DICT_FILENAME_SUFFIX;
-import static org.apache.lucene.analysis.ko.dict.BinaryDictionary.POSDICT_FILENAME_SUFFIX;
-import static org.apache.lucene.analysis.ko.dict.BinaryDictionary.TARGETMAP_FILENAME_SUFFIX;
import static org.apache.lucene.analysis.ko.dict.TokenInfoDictionary.FST_FILENAME_SUFFIX;
+import static org.apache.lucene.analysis.morph.BinaryDictionary.DICT_FILENAME_SUFFIX;
+import static org.apache.lucene.analysis.morph.BinaryDictionary.POSDICT_FILENAME_SUFFIX;
+import static org.apache.lucene.analysis.morph.BinaryDictionary.TARGETMAP_FILENAME_SUFFIX;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
@@ -141,13 +141,13 @@ public class TestTokenInfoDictionary extends LuceneTestCase {
tid.getWordCost(wordId);
- POS.Type type = tid.getPOSType(wordId);
- POS.Tag leftPOS = tid.getLeftPOS(wordId);
- POS.Tag rightPOS = tid.getRightPOS(wordId);
+ POS.Type type = tid.getMorphAttributes().getPOSType(wordId);
+ POS.Tag leftPOS = tid.getMorphAttributes().getLeftPOS(wordId);
+ POS.Tag rightPOS = tid.getMorphAttributes().getRightPOS(wordId);
if (type == POS.Type.MORPHEME) {
assertSame(leftPOS, rightPOS);
- String reading = tid.getReading(wordId);
+ String reading = tid.getMorphAttributes().getReading(wordId);
boolean isHanja = charDef.isHanja(surfaceForm.charAt(0));
if (isHanja) {
assertNotNull(reading);
@@ -163,10 +163,11 @@ public class TestTokenInfoDictionary extends LuceneTestCase {
assertSame(leftPOS, rightPOS);
assertTrue(leftPOS == POS.Tag.NNG || rightPOS == POS.Tag.NNP);
}
- Dictionary.Morpheme[] decompound = tid.getMorphemes(wordId, chars, 0, chars.length);
+ KoMorphData.Morpheme[] decompound =
+ tid.getMorphAttributes().getMorphemes(wordId, chars, 0, chars.length);
if (decompound != null) {
int offset = 0;
- for (Dictionary.Morpheme morph : decompound) {
+ for (KoMorphData.Morpheme morph : decompound) {
assertTrue(UnicodeUtil.validUTF16String(morph.surfaceForm));
assertFalse(morph.surfaceForm.isEmpty());
assertEquals(morph.surfaceForm.trim(), morph.surfaceForm);
diff --git a/lucene/analysis/nori/src/test/org/apache/lucene/analysis/ko/dict/TestUserDictionary.java b/lucene/analysis/nori/src/test/org/apache/lucene/analysis/ko/dict/TestUserDictionary.java
index 7f97d0520e8..8fb2827a336 100644
--- a/lucene/analysis/nori/src/test/org/apache/lucene/analysis/ko/dict/TestUserDictionary.java
+++ b/lucene/analysis/nori/src/test/org/apache/lucene/analysis/ko/dict/TestUserDictionary.java
@@ -31,16 +31,16 @@ public class TestUserDictionary extends LuceneTestCase {
char[] sArray = s.toCharArray();
List<Integer> wordIds = dictionary.lookup(sArray, 0, s.length());
assertEquals(1, wordIds.size());
- assertNull(dictionary.getMorphemes(wordIds.get(0), sArray, 0, s.length()));
+ assertNull(dictionary.getMorphAttributes().getMorphemes(wordIds.get(0), sArray, 0, s.length()));
s = "세종시";
sArray = s.toCharArray();
wordIds = dictionary.lookup(sArray, 0, s.length());
assertEquals(2, wordIds.size());
- assertNull(dictionary.getMorphemes(wordIds.get(0), sArray, 0, s.length()));
+ assertNull(dictionary.getMorphAttributes().getMorphemes(wordIds.get(0), sArray, 0, s.length()));
- Dictionary.Morpheme[] decompound =
- dictionary.getMorphemes(wordIds.get(1), sArray, 0, s.length());
+ KoMorphData.Morpheme[] decompound =
+ dictionary.getMorphAttributes().getMorphemes(wordIds.get(1), sArray, 0, s.length());
assertNotNull(decompound);
assertEquals(2, decompound.length);
assertEquals(decompound[0].posTag, POS.Tag.NNG);
@@ -52,7 +52,7 @@ public class TestUserDictionary extends LuceneTestCase {
sArray = s.toCharArray();
wordIds = dictionary.lookup(sArray, 0, s.length());
assertEquals(1, wordIds.size());
- assertNull(dictionary.getMorphemes(wordIds.get(0), sArray, 0, s.length()));
+ assertNull(dictionary.getMorphAttributes().getMorphemes(wordIds.get(0), sArray, 0, s.length()));
}
@Test