Skip to content

Apache Lucene open-source search software

LUCENE-10493: factor out Viterbi algorithm and share it between kuromoji and nori (#805)

Tomoko Uchidac89f8a7ea1e7
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 76304b3eee9..8e1bfcc9cbd 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -60,6 +60,8 @@ Other
All classes in `org.apache.lucene.analysis.[ja|ko].util` was moved to `org.apache.lucene.analysis.[ja|ko].dict`.
(Tomoko Uchida)
+* LUCENE-10493: Factor out Viterbi algorithm in Kuromoji and Nori to analysis-common. (Tomoko Uchida)
+
======================= Lucene 9.2.0 =======================
API Changes
diff --git a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/GraphvizFormatter.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/GraphvizFormatter.java
similarity index 67%
rename from lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/GraphvizFormatter.java
rename to lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/GraphvizFormatter.java
index 9beaf2e5b9b..8279f0131d8 100644
--- a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/GraphvizFormatter.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/GraphvizFormatter.java
@@ -14,22 +14,16 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.analysis.ko;
+package org.apache.lucene.analysis.morph;
import java.util.HashMap;
import java.util.Map;
-import org.apache.lucene.analysis.ko.KoreanTokenizer.Position;
-import org.apache.lucene.analysis.ko.KoreanTokenizer.WrappedPositionArray;
-import org.apache.lucene.analysis.ko.dict.ConnectionCosts;
-import org.apache.lucene.analysis.ko.dict.KoMorphData;
-import org.apache.lucene.analysis.morph.Dictionary;
// TODO: would be nice to show 2nd best path in a diff't
// color...
/** Outputs the dot (graphviz) string for the viterbi lattice. */
-public class GraphvizFormatter {
-
+public class GraphvizFormatter<T extends MorphData> {
private static final String BOS_LABEL = "BOS";
private static final String EOS_LABEL = "EOS";
@@ -56,36 +50,39 @@ public class GraphvizFormatter {
}
// Backtraces another incremental fragment:
- void onBacktrace(
- KoreanTokenizer tok,
- WrappedPositionArray positions,
+ public void onBacktrace(
+ DictionaryProvider<T> dictProvider,
+ Viterbi.WrappedPositionArray<? extends Viterbi.Position> positions,
int lastBackTracePos,
- Position endPosData,
+ Viterbi.Position endPosData,
int fromIDX,
char[] fragment,
boolean isEnd) {
setBestPathMap(positions, lastBackTracePos, endPosData, fromIDX);
- sb.append(formatNodes(tok, positions, lastBackTracePos, endPosData, fragment));
+ sb.append(formatNodes(dictProvider, positions, lastBackTracePos, endPosData, fragment));
if (isEnd) {
sb.append(" fini [style=invis]\n");
sb.append(" ");
- sb.append(getNodeID(endPosData.pos, fromIDX));
+ sb.append(getNodeID(endPosData.getPos(), fromIDX));
sb.append(" -> fini [label=\"" + EOS_LABEL + "\"]");
}
}
// Records which arcs make up the best bath:
private void setBestPathMap(
- WrappedPositionArray positions, int startPos, Position endPosData, int fromIDX) {
+ Viterbi.WrappedPositionArray<? extends Viterbi.Position> positions,
+ int startPos,
+ Viterbi.Position endPosData,
+ int fromIDX) {
bestPathMap.clear();
- int pos = endPosData.pos;
+ int pos = endPosData.getPos();
int bestIDX = fromIDX;
while (pos > startPos) {
- final Position posData = positions.get(pos);
+ final Viterbi.Position posData = positions.get(pos);
- final int backPos = posData.backPos[bestIDX];
- final int backIDX = posData.backIndex[bestIDX];
+ final int backPos = posData.getBackPos(bestIDX);
+ final int backIDX = posData.getBackIndex(bestIDX);
final String toNodeID = getNodeID(pos, bestIDX);
final String fromNodeID = getNodeID(backPos, backIDX);
@@ -99,34 +96,34 @@ public class GraphvizFormatter {
}
private String formatNodes(
- KoreanTokenizer tok,
- WrappedPositionArray positions,
+ DictionaryProvider<T> dictProvider,
+ Viterbi.WrappedPositionArray<? extends Viterbi.Position> positions,
int startPos,
- Position endPosData,
+ Viterbi.Position endPosData,
char[] fragment) {
StringBuilder sb = new StringBuilder();
// Output nodes
- for (int pos = startPos + 1; pos <= endPosData.pos; pos++) {
- final Position posData = positions.get(pos);
- for (int idx = 0; idx < posData.count; idx++) {
+ for (int pos = startPos + 1; pos <= endPosData.getPos(); pos++) {
+ final Viterbi.Position posData = positions.get(pos);
+ for (int idx = 0; idx < posData.getCount(); idx++) {
sb.append(" ");
sb.append(getNodeID(pos, idx));
sb.append(" [label=\"");
sb.append(pos);
sb.append(": ");
- sb.append(posData.lastRightID[idx]);
+ sb.append(posData.getLastRightID(idx));
sb.append("\"]\n");
}
}
// Output arcs
- for (int pos = endPosData.pos; pos > startPos; pos--) {
- final Position posData = positions.get(pos);
- for (int idx = 0; idx < posData.count; idx++) {
- final Position backPosData = positions.get(posData.backPos[idx]);
+ for (int pos = endPosData.getPos(); pos > startPos; pos--) {
+ final Viterbi.Position posData = positions.get(pos);
+ for (int idx = 0; idx < posData.getCount(); idx++) {
+ final Viterbi.Position backPosData = positions.get(posData.getBackPos(idx));
final String toNodeID = getNodeID(pos, idx);
- final String fromNodeID = getNodeID(posData.backPos[idx], posData.backIndex[idx]);
+ final String fromNodeID = getNodeID(posData.getBackPos(idx), posData.getBackIndex(idx));
sb.append(" ");
sb.append(fromNodeID);
@@ -141,15 +138,15 @@ public class GraphvizFormatter {
attrs = "";
}
- final Dictionary<? extends KoMorphData> dict = tok.getDict(posData.backType[idx]);
- final int wordCost = dict.getWordCost(posData.backID[idx]);
+ final Dictionary<? extends T> dict = dictProvider.get(posData.getBackType(idx));
+ final int wordCost = dict.getWordCost(posData.getBackID(idx));
final int bgCost =
costs.get(
- backPosData.lastRightID[posData.backIndex[idx]],
- dict.getLeftId(posData.backID[idx]));
+ backPosData.getLastRightID(posData.getBackIndex(idx)),
+ dict.getLeftId(posData.getBackID(idx)));
final String surfaceForm =
- new String(fragment, posData.backPos[idx] - startPos, pos - posData.backPos[idx]);
+ new String(fragment, posData.getBackPos(idx) - startPos, pos - posData.getBackPos(idx));
sb.append(" [label=\"");
sb.append(surfaceForm);
@@ -190,4 +187,10 @@ public class GraphvizFormatter {
private String getNodeID(int pos, int idx) {
return pos + "." + idx;
}
+
+ /** {@link Dictionary} provider */
+ @FunctionalInterface
+ public interface DictionaryProvider<T extends MorphData> {
+ Dictionary<? extends T> get(TokenType type);
+ }
}
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/Viterbi.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/Viterbi.java
new file mode 100644
index 00000000000..f7533b93007
--- /dev/null
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/Viterbi.java
@@ -0,0 +1,815 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.analysis.morph;
+
+import java.io.IOException;
+import java.io.Reader;
+import java.lang.reflect.Array;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import org.apache.lucene.analysis.util.RollingCharBuffer;
+import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.IntsRef;
+import org.apache.lucene.util.RamUsageEstimator;
+import org.apache.lucene.util.fst.FST;
+
+/**
+ * Performs <a href="https://en.wikipedia.org/wiki/Viterbi_algorithm">Viterbi algorithm</a> for
+ * morphological Tokenizers, which split texts by Hidden Markov Model or Conditional Random Fields.
+ *
+ * @param <T> output token class
+ * @param <U> position class
+ */
+public abstract class Viterbi<T extends Token, U extends Viterbi.Position> {
+ protected static final boolean VERBOSE = false;
+
+ // For safety:
+ protected static final int MAX_UNKNOWN_WORD_LENGTH = 1024;
+ private static final int MAX_BACKTRACE_GAP = 1024;
+
+ private final TokenInfoFST fst;
+ private final BinaryDictionary<? extends MorphData> dictionary;
+ private final Dictionary<? extends MorphData> userDictionary;
+ protected final ConnectionCosts costs;
+
+ private final FST.Arc<Long> arc = new FST.Arc<>();
+ private final FST.BytesReader fstReader;
+ protected final IntsRef wordIdRef = new IntsRef();
+
+ private final FST.BytesReader userFSTReader;
+ private final TokenInfoFST userFST;
+
+ protected final RollingCharBuffer buffer = new RollingCharBuffer();
+
+ protected final WrappedPositionArray<U> positions;
+
+ // True once we've hit the EOF from the input reader:
+ protected boolean end;
+
+ // Last absolute position we backtraced from:
+ protected int lastBackTracePos;
+
+ // Next absolute position to process:
+ protected int pos;
+
+ // Already parsed, but not yet passed to caller, tokens:
+ protected final List<T> pending = new ArrayList<>();
+
+ protected boolean outputNBest = false;
+
+ protected boolean enableSpacePenaltyFactor = false;
+
+ protected boolean outputLongestUserEntryOnly = false;
+
+ protected Viterbi(
+ TokenInfoFST fst,
+ FST.BytesReader fstReader,
+ BinaryDictionary<? extends MorphData> dictionary,
+ TokenInfoFST userFST,
+ FST.BytesReader userFSTReader,
+ Dictionary<? extends MorphData> userDictionary,
+ ConnectionCosts costs,
+ Class<U> positionImpl) {
+ this.fst = fst;
+ this.fstReader = fstReader;
+ this.dictionary = dictionary;
+ this.userFST = userFST;
+ this.userFSTReader = userFSTReader;
+ this.userDictionary = userDictionary;
+ this.costs = costs;
+ this.positions = new WrappedPositionArray<>(positionImpl);
+ }
+
+ /**
+ * Incrementally parse some more characters. This runs the viterbi search forwards "enough" so
+ * that we generate some more tokens. How much forward depends on the chars coming in, since some
+ * chars could cause longer-lasting ambiguity in the parsing. Once the ambiguity is resolved, then
+ * we back trace, produce the pending tokens, and return.
+ */
+ public final void forward() throws IOException {
+ if (VERBOSE) {
+ System.out.println("\nPARSE");
+ }
+
+ // Index of the last character of unknown word:
+ int unknownWordEndIndex = -1;
+
+ // Maximum posAhead of user word in the entire input
+ int userWordMaxPosAhead = -1;
+
+ // Advances over each position (character):
+ while (buffer.get(pos) != -1) {
+ final Position posData = positions.get(pos);
+ final boolean isFrontier = positions.getNextPos() == pos + 1;
+
+ if (posData.count == 0) {
+ // No arcs arrive here; move to next position:
+ if (VERBOSE) {
+ System.out.println(" no arcs in; skip pos=" + pos);
+ }
+ pos++;
+ continue;
+ }
+
+ if (pos > lastBackTracePos && posData.count == 1 && isFrontier) {
+ // We are at a "frontier", and only one node is
+ // alive, so whatever the eventual best path is must
+ // come through this node. So we can safely commit
+ // to the prefix of the best path at this point:
+ if (outputNBest) {
+ backtraceNBest(posData, false);
+ }
+ backtrace(posData, 0);
+ if (outputNBest) {
+ fixupPendingList();
+ }
+
+ // Re-base cost so we don't risk int overflow:
+ posData.costs[0] = 0;
+ if (pending.size() > 0) {
+ return;
+ } else {
+ // This means the backtrace only produced
+ // punctuation tokens, so we must keep parsing.
+ }
+ }
+
+ if (pos - lastBackTracePos >= MAX_BACKTRACE_GAP) {
+ // Safety: if we've buffered too much, force a
+ // backtrace now. We find the least-cost partial
+ // path, across all paths, backtrace from it, and
+ // then prune all others. Note that this, in
+ // general, can produce the wrong result, if the
+ // total best path did not in fact back trace
+ // through this partial best path. But it's the
+ // best we can do... (short of not having a
+ // safety!).
+
+ // First pass: find least cost partial path so far,
+ // including ending at future positions:
+ int leastIDX = -1;
+ int leastCost = Integer.MAX_VALUE;
+ Position leastPosData = null;
+ for (int pos2 = pos; pos2 < positions.getNextPos(); pos2++) {
+ final Position posData2 = positions.get(pos2);
+ for (int idx = 0; idx < posData2.count; idx++) {
+ // System.out.println(" idx=" + idx + " cost=" + cost);
+ final int cost = posData2.costs[idx];
+ if (cost < leastCost) {
+ leastCost = cost;
+ leastIDX = idx;
+ leastPosData = posData2;
+ }
+ }
+ }
+
+ // We will always have at least one live path:
+ assert leastIDX != -1;
+
+ if (outputNBest) {
+ backtraceNBest(leastPosData, false);
+ }
+
+ // Second pass: prune all but the best path:
+ for (int pos2 = pos; pos2 < positions.getNextPos(); pos2++) {
+ final Position posData2 = positions.get(pos2);
+ if (posData2 != leastPosData) {
+ posData2.reset();
+ } else {
+ if (leastIDX != 0) {
+ posData2.costs[0] = posData2.costs[leastIDX];
+ posData2.lastRightID[0] = posData2.lastRightID[leastIDX];
+ posData2.backPos[0] = posData2.backPos[leastIDX];
+ posData2.backWordPos[0] = posData2.backWordPos[leastIDX];
+ posData2.backIndex[0] = posData2.backIndex[leastIDX];
+ posData2.backID[0] = posData2.backID[leastIDX];
+ posData2.backType[0] = posData2.backType[leastIDX];
+ }
+ posData2.count = 1;
+ }
+ }
+
+ backtrace(leastPosData, 0);
+ if (outputNBest) {
+ fixupPendingList();
+ }
+
+ // Re-base cost so we don't risk int overflow:
+ Arrays.fill(leastPosData.costs, 0, leastPosData.count, 0);
+
+ if (pos != leastPosData.pos) {
+ // We jumped into a future position:
+ assert pos < leastPosData.pos;
+ pos = leastPosData.pos;
+ }
+ if (pending.size() > 0) {
+ return;
+ } else {
+ // This means the backtrace only produced
+ // punctuation tokens, so we must keep parsing.
+ continue;
+ }
+ }
+
+ if (VERBOSE) {
+ System.out.println(
+ "\n extend @ pos="
+ + pos
+ + " char="
+ + (char) buffer.get(pos)
+ + " hex="
+ + Integer.toHexString(buffer.get(pos)));
+ }
+
+ if (VERBOSE) {
+ System.out.println(" " + posData.count + " arcs in");
+ }
+
+ if (enableSpacePenaltyFactor
+ && Character.getType(buffer.get(pos)) == Character.SPACE_SEPARATOR) {
+ // We add single space separator as prefixes of the terms that we extract.
+ // This information is needed to compute the space penalty factor of each term.
+ // These whitespace prefixes are removed when the final tokens are generated, or
+ // added as separated tokens when discardPunctuation is unset.
+ if (buffer.get(++pos) == -1) {
+ pos = posData.pos;
+ }
+ }
+
+ boolean anyMatches = false;
+
+ // First try user dict:
+ if (userFST != null) {
+ userFST.getFirstArc(arc);
+ int output = 0;
+ int maxPosAhead = 0;
+ int outputMaxPosAhead = 0;
+ int arcFinalOutMaxPosAhead = 0;
+
+ for (int posAhead = pos; ; posAhead++) {
+ final int ch = buffer.get(posAhead);
+ if (ch == -1) {
+ break;
+ }
+ if (userFST.findTargetArc(ch, arc, arc, posAhead == pos, userFSTReader) == null) {
+ break;
+ }
+ output += arc.output().intValue();
+ if (arc.isFinal()) {
+ maxPosAhead = posAhead;
+ outputMaxPosAhead = output;
+ arcFinalOutMaxPosAhead = arc.nextFinalOutput().intValue();
+ anyMatches = true;
+ if (!outputLongestUserEntryOnly) {
+ // add all matched user entries.
+ add(
+ userDictionary.getMorphAttributes(),
+ posData,
+ pos,
+ posAhead + 1,
+ output + arc.nextFinalOutput().intValue(),
+ TokenType.USER,
+ false);
+ }
+ }
+ }
+
+ // Longest matching for user word
+ if (anyMatches && maxPosAhead > userWordMaxPosAhead) {
+ if (outputLongestUserEntryOnly) {
+ if (VERBOSE) {
+ System.out.println(
+ " USER word "
+ + new String(buffer.get(pos, maxPosAhead + 1))
+ + " toPos="
+ + (maxPosAhead + 1));
+ }
+ add(
+ userDictionary.getMorphAttributes(),
+ posData,
+ pos,
+ maxPosAhead + 1,
+ outputMaxPosAhead + arcFinalOutMaxPosAhead,
+ TokenType.USER,
+ false);
+ }
+ userWordMaxPosAhead = Math.max(userWordMaxPosAhead, maxPosAhead);
+ }
+ }
+
+ // TODO: we can be more aggressive about user
+ // matches? if we are "under" a user match then don't
+ // extend KNOWN/UNKNOWN paths?
+
+ if (!anyMatches) {
+ // Next, try known dictionary matches
+ fst.getFirstArc(arc);
+ int output = 0;
+
+ for (int posAhead = pos; ; posAhead++) {
+ final int ch = buffer.get(posAhead);
+ if (ch == -1) {
+ break;
+ }
+ // System.out.println(" match " + (char) ch + " posAhead=" + posAhead);
+
+ if (fst.findTargetArc(ch, arc, arc, posAhead == pos, fstReader) == null) {
+ break;
+ }
+
+ output += arc.output().intValue();
+
+ // Optimization: for known words that are too-long
+ // (compound), we should pre-compute the 2nd
+ // best segmentation and store it in the
+ // dictionary instead of recomputing it each time a
+ // match is found.
+
+ if (arc.isFinal()) {
+ dictionary.lookupWordIds(output + arc.nextFinalOutput().intValue(), wordIdRef);
+ if (VERBOSE) {
+ System.out.println(
+ " KNOWN word "
+ + new String(buffer.get(pos, posAhead - pos + 1))
+ + " toPos="
+ + (posAhead + 1)
+ + " "
+ + wordIdRef.length
+ + " wordIDs");
+ }
+ for (int ofs = 0; ofs < wordIdRef.length; ofs++) {
+ add(
+ dictionary.getMorphAttributes(),
+ posData,
+ pos,
+ posAhead + 1,
+ wordIdRef.ints[wordIdRef.offset + ofs],
+ TokenType.KNOWN,
+ false);
+ anyMatches = true;
+ }
+ }
+ }
+ }
+
+ if (!shouldSkipProcessUnknownWord(unknownWordEndIndex, posData)) {
+ int unknownWordLength = processUnknownWord(anyMatches, posData);
+ unknownWordEndIndex = posData.pos + unknownWordLength;
+ }
+ pos++;
+ }
+
+ end = true;
+
+ if (pos > 0) {
+
+ final Position endPosData = positions.get(pos);
+ int leastCost = Integer.MAX_VALUE;
+ int leastIDX = -1;
+ if (VERBOSE) {
+ System.out.println(" end: " + endPosData.count + " nodes");
+ }
+ for (int idx = 0; idx < endPosData.count; idx++) {
+ // Add EOS cost:
+ final int cost = endPosData.costs[idx] + costs.get(endPosData.lastRightID[idx], 0);
+ // System.out.println(" idx=" + idx + " cost=" + cost + " (pathCost=" +
+ // endPosData.costs[idx] + " bgCost=" + costs.get(endPosData.lastRightID[idx], 0) + ")
+ // backPos=" + endPosData.backPos[idx]);
+ if (cost < leastCost) {
+ leastCost = cost;
+ leastIDX = idx;
+ }
+ }
+
+ if (outputNBest) {
+ backtraceNBest(endPosData, true);
+ }
+ backtrace(endPosData, leastIDX);
+ if (outputNBest) {
+ fixupPendingList();
+ }
+ } else {
+ // No characters in the input string; return no tokens!
+ }
+ }
+
+ protected boolean shouldSkipProcessUnknownWord(int unknownWordEndIndex, Position posData) {
+ return unknownWordEndIndex > posData.pos;
+ }
+
+ /**
+ * Add unknown words to the position graph.
+ *
+ * @return word length
+ */
+ protected abstract int processUnknownWord(boolean anyMatches, Position posData)
+ throws IOException;
+
+ /**
+ * Backtrace from the provided position, back to the last time we back-traced, accumulating the
+ * resulting tokens to the pending list. The pending list is then in-reverse (last token should be
+ * returned first).
+ */
+ protected abstract void backtrace(final Position endPosData, final int fromIDX)
+ throws IOException;
+
+ /**
+ * Backtrace the n-best path. Subclasses that support n-best paths should implement this method.
+ */
+ protected void backtraceNBest(final Position endPosData, final boolean useEOS)
+ throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ /**
+ * Remove duplicated tokens from the pending list; this is needed because {@link
+ * #backtrace(Position, int)} and {@link #backtraceNBest(Position, boolean)} can add same tokens
+ * to the list. Subclasses that support n-best paths should implement this method.
+ */
+ protected void fixupPendingList() {
+ throw new UnsupportedOperationException();
+ }
+
+ /** Add a token on the minimum cost path to the pending token list. */
+ protected final void add(
+ MorphData morphData,
+ Position fromPosData,
+ int wordPos,
+ int endPos,
+ int wordID,
+ TokenType type,
+ boolean addPenalty)
+ throws IOException {
+ final int wordCost = morphData.getWordCost(wordID);
+ final int leftID = morphData.getLeftId(wordID);
+ int leastCost = Integer.MAX_VALUE;
+ int leastIDX = -1;
+ assert fromPosData.count > 0;
+ for (int idx = 0; idx < fromPosData.count; idx++) {
+ // The number of spaces before the term
+ int numSpaces = wordPos - fromPosData.pos;
+
+ // Cost is path cost so far, plus word cost (added at
+ // end of loop), plus bigram cost and space penalty cost.
+ final int cost =
+ fromPosData.costs[idx]
+ + costs.get(fromPosData.lastRightID[idx], leftID)
+ + computeSpacePenalty(morphData, wordID, numSpaces);
+ if (VERBOSE) {
+ System.out.println(
+ " fromIDX="
+ + idx
+ + ": cost="
+ + cost
+ + " (prevCost="
+ + fromPosData.costs[idx]
+ + " wordCost="
+ + wordCost
+ + " bgCost="
+ + costs.get(fromPosData.lastRightID[idx], leftID)
+ + " spacePenalty="
+ + computeSpacePenalty(morphData, wordID, numSpaces)
+ + ") leftID="
+ + leftID
+ // + " leftPOS="
+ // + leftPOS.name()
+ + ")");
+ }
+ if (cost < leastCost) {
+ leastCost = cost;
+ leastIDX = idx;
+ if (VERBOSE) {
+ System.out.println(" **");
+ }
+ }
+ }
+
+ leastCost += wordCost;
+
+ if (VERBOSE) {
+ System.out.println(
+ " + cost="
+ + leastCost
+ + " wordID="
+ + wordID
+ + " leftID="
+ + leftID
+ + " leastIDX="
+ + leastIDX
+ + " toPos="
+ + endPos
+ + " toPos.idx="
+ + positions.get(endPos).count);
+ }
+
+ if (addPenalty && type != TokenType.USER) {
+ final int penalty = computePenalty(fromPosData.pos, endPos - fromPosData.pos);
+ if (VERBOSE) {
+ if (penalty > 0) {
+ System.out.println(" + penalty=" + penalty + " cost=" + (leastCost + penalty));
+ }
+ }
+ leastCost += penalty;
+ }
+
+ positions
+ .get(endPos)
+ .add(
+ leastCost,
+ morphData.getRightId(wordID),
+ fromPosData.pos,
+ wordPos,
+ leastIDX,
+ wordID,
+ type);
+ }
+
+ /** Returns the space penalty. */
+ protected int computeSpacePenalty(MorphData morphData, int wordID, int numSpaces) {
+ return 0;
+ }
+
+ /** Returns the penalty for a specific input region */
+ protected int computePenalty(int pos, int length) throws IOException {
+ return 0;
+ }
+
+ public int getPos() {
+ return pos;
+ }
+
+ public boolean isEnd() {
+ return end;
+ }
+
+ public List<T> getPending() {
+ return pending;
+ }
+
+ public boolean isOutputNBest() {
+ return outputNBest;
+ }
+
+ public void resetBuffer(Reader reader) {
+ buffer.reset(reader);
+ }
+
+ public void resetState() {
+ positions.reset();
+ pos = 0;
+ end = false;
+ lastBackTracePos = 0;
+ pending.clear();
+
+ // Add BOS:
+ positions.get(0).add(0, 0, -1, -1, -1, -1, TokenType.KNOWN);
+ }
+
+ /**
+ * Holds all back pointers arriving to this position.
+ *
+ * <p>NOTE: This and subclasses must have no-arg constructor. See {@link WrappedPositionArray}.
+ */
+ public static class Position {
+
+ int pos;
+
+ int count;
+
+ // maybe single int array * 5?
+ int[] costs = new int[8];
+ int[] lastRightID = new int[8];
+ int[] backPos = new int[8];
+ int[] backWordPos = new int[8];
+ int[] backIndex = new int[8];
+ int[] backID = new int[8];
+ TokenType[] backType = new TokenType[8];
+
+ private void grow() {
+ costs = ArrayUtil.grow(costs, 1 + count);
+ lastRightID = ArrayUtil.grow(lastRightID, 1 + count);
+ backPos = ArrayUtil.grow(backPos, 1 + count);
+ backWordPos = ArrayUtil.grow(backWordPos, 1 + count);
+ backIndex = ArrayUtil.grow(backIndex, 1 + count);
+ backID = ArrayUtil.grow(backID, 1 + count);
+
+ // NOTE: sneaky: grow separately because
+ // ArrayUtil.grow will otherwise pick a different
+ // length than the int[]s we just grew:
+ final TokenType[] newBackType = new TokenType[backID.length];
+ System.arraycopy(backType, 0, newBackType, 0, backType.length);
+ backType = newBackType;
+ }
+
+ public void add(
+ int cost,
+ int lastRightID,
+ int backPos,
+ int backRPos,
+ int backIndex,
+ int backID,
+ TokenType backType) {
+ // NOTE: this isn't quite a true Viterbi search,
+ // because we should check if lastRightID is
+ // already present here, and only update if the new
+ // cost is less than the current cost, instead of
+ // simply appending. However, that will likely hurt
+ // performance (usually we add a lastRightID only once),
+ // and it means we actually create the full graph
+ // intersection instead of a "normal" Viterbi lattice:
+ if (count == costs.length) {
+ grow();
+ }
+ this.costs[count] = cost;
+ this.lastRightID[count] = lastRightID;
+ this.backPos[count] = backPos;
+ this.backWordPos[count] = backRPos;
+ this.backIndex[count] = backIndex;
+ this.backID[count] = backID;
+ this.backType[count] = backType;
+ count++;
+ }
+
+ public void reset() {
+ count = 0;
+ }
+
+ public int getPos() {
+ return pos;
+ }
+
+ public int getCount() {
+ return count;
+ }
+
+ public void setCount(int count) {
+ this.count = count;
+ }
+
+ public int getCost(int index) {
+ return costs[index];
+ }
+
+ public int getBackPos(int index) {
+ return backPos[index];
+ }
+
+ public int getBackWordPos(int index) {
+ return backWordPos[index];
+ }
+
+ public int getBackID(int index) {
+ return backID[index];
+ }
+
+ public int getBackIndex(int index) {
+ return backIndex[index];
+ }
+
+ public TokenType getBackType(int index) {
+ return backType[index];
+ }
+
+ public int getLastRightID(int index) {
+ return lastRightID[index];
+ }
+ }
+
+ /** Holds partial graph (array of positions) for calculating the minimum cost path */
+ public static final class WrappedPositionArray<U extends Position> {
+ private U[] positions;
+ private final Class<U> clazz;
+
+ @SuppressWarnings("unchecked")
+ WrappedPositionArray(Class<U> clazz) {
+ this.clazz = clazz;
+ positions = (U[]) Array.newInstance(clazz, 8);
+ for (int i = 0; i < positions.length; i++) {
+ try {
+ positions[i] = clazz.getConstructor().newInstance();
+ } catch (ReflectiveOperationException e) {
+ // shouldn't happen; Position class should have no-arg constructor.
+ throw new IllegalStateException(e);
+ }
+ }
+ }
+
+ // Next array index to write to in positions:
+ private int nextWrite;
+
+ // Next position to write:
+ private int nextPos;
+
+ // How many valid Position instances are held in the
+ // positions array:
+ private int count;
+
+ void reset() {
+ nextWrite--;
+ while (count > 0) {
+ if (nextWrite == -1) {
+ nextWrite = positions.length - 1;
+ }
+ positions[nextWrite--].reset();
+ count--;
+ }
+ nextWrite = 0;
+ nextPos = 0;
+ count = 0;
+ }
+
+ /**
+ * Get Position instance for this absolute position; this is allowed to be arbitrarily far "in
+ * the future" but cannot be before the last freeBefore.
+ */
+ @SuppressWarnings("unchecked")
+ public U get(int pos) {
+ while (pos >= nextPos) {
+ // System.out.println("count=" + count + " vs len=" + positions.length);
+ if (count == positions.length) {
+ // Position[] newPositions =
+ // new Position[ArrayUtil.oversize(1 + count, RamUsageEstimator.NUM_BYTES_OBJECT_REF)];
+ U[] newPositions =
+ (U[])
+ Array.newInstance(
+ clazz, ArrayUtil.oversize(1 + count, RamUsageEstimator.NUM_BYTES_OBJECT_REF));
+ // System.out.println("grow positions " + newPositions.length);
+ System.arraycopy(positions, nextWrite, newPositions, 0, positions.length - nextWrite);
+ System.arraycopy(positions, 0, newPositions, positions.length - nextWrite, nextWrite);
+ for (int i = positions.length; i < newPositions.length; i++) {
+ try {
+ newPositions[i] = clazz.getConstructor().newInstance();
+ } catch (ReflectiveOperationException e) {
+ // shouldn't happen
+ throw new IllegalStateException(e);
+ }
+ }
+ nextWrite = positions.length;
+ positions = newPositions;
+ }
+ if (nextWrite == positions.length) {
+ nextWrite = 0;
+ }
+ // Should have already been reset:
+ assert positions[nextWrite].count == 0;
+ positions[nextWrite++].pos = nextPos++;
+ count++;
+ }
+ assert inBounds(pos);
+ final int index = getIndex(pos);
+ assert positions[index].pos == pos;
+ return positions[index];
+ }
+
+ int getNextPos() {
+ return nextPos;
+ }
+
+ // For assert:
+ private boolean inBounds(int pos) {
+ return pos < nextPos && pos >= nextPos - count;
+ }
+
+ private int getIndex(int pos) {
+ int index = nextWrite - (nextPos - pos);
+ if (index < 0) {
+ index += positions.length;
+ }
+ return index;
+ }
+
+ public void freeBefore(int pos) {
+ final int toFree = count - (nextPos - pos);
+ assert toFree >= 0;
+ assert toFree <= count;
+ int index = nextWrite - count;
+ if (index < 0) {
+ index += positions.length;
+ }
+ for (int i = 0; i < toFree; i++) {
+ if (index == positions.length) {
+ index = 0;
+ }
+ // System.out.println(" fb idx=" + index);
+ positions[index].reset();
+ index++;
+ }
+ count -= toFree;
+ }
+ }
+}
diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/GraphvizFormatter.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/GraphvizFormatter.java
deleted file mode 100644
index 4538daa362f..00000000000
--- a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/GraphvizFormatter.java
+++ /dev/null
@@ -1,195 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.analysis.ja;
-
-import java.util.HashMap;
-import java.util.Map;
-import org.apache.lucene.analysis.ja.JapaneseTokenizer.Position;
-import org.apache.lucene.analysis.ja.JapaneseTokenizer.WrappedPositionArray;
-import org.apache.lucene.analysis.ja.dict.ConnectionCosts;
-import org.apache.lucene.analysis.ja.dict.JaMorphData;
-import org.apache.lucene.analysis.morph.Dictionary;
-
-// TODO: would be nice to show 2nd best path in a diff't
-// color...
-
-/** Outputs the dot (graphviz) string for the viterbi lattice. */
-public class GraphvizFormatter {
-
- private static final String BOS_LABEL = "BOS";
-
- private static final String EOS_LABEL = "EOS";
-
- private static final String FONT_NAME = "Helvetica";
-
- private final ConnectionCosts costs;
-
- private final Map<String, String> bestPathMap;
-
- private final StringBuilder sb = new StringBuilder();
-
- public GraphvizFormatter(ConnectionCosts costs) {
- this.costs = costs;
- this.bestPathMap = new HashMap<>();
- sb.append(formatHeader());
- sb.append(" init [style=invis]\n");
- sb.append(" init -> 0.0 [label=\"" + BOS_LABEL + "\"]\n");
- }
-
- public String finish() {
- sb.append(formatTrailer());
- return sb.toString();
- }
-
- // Backtraces another incremental fragment:
- void onBacktrace(
- JapaneseTokenizer tok,
- WrappedPositionArray positions,
- int lastBackTracePos,
- Position endPosData,
- int fromIDX,
- char[] fragment,
- boolean isEnd) {
- setBestPathMap(positions, lastBackTracePos, endPosData, fromIDX);
- sb.append(formatNodes(tok, positions, lastBackTracePos, endPosData, fragment));
- if (isEnd) {
- sb.append(" fini [style=invis]\n");
- sb.append(" ");
- sb.append(getNodeID(endPosData.pos, fromIDX));
- sb.append(" -> fini [label=\"" + EOS_LABEL + "\"]");
- }
- }
-
- // Records which arcs make up the best bath:
- private void setBestPathMap(
- WrappedPositionArray positions, int startPos, Position endPosData, int fromIDX) {
- bestPathMap.clear();
-
- int pos = endPosData.pos;
- int bestIDX = fromIDX;
- while (pos > startPos) {
- final Position posData = positions.get(pos);
-
- final int backPos = posData.backPos[bestIDX];
- final int backIDX = posData.backIndex[bestIDX];
-
- final String toNodeID = getNodeID(pos, bestIDX);
- final String fromNodeID = getNodeID(backPos, backIDX);
-
- assert !bestPathMap.containsKey(fromNodeID);
- assert !bestPathMap.containsValue(toNodeID);
- bestPathMap.put(fromNodeID, toNodeID);
- pos = backPos;
- bestIDX = backIDX;
- }
- }
-
- private String formatNodes(
- JapaneseTokenizer tok,
- WrappedPositionArray positions,
- int startPos,
- Position endPosData,
- char[] fragment) {
-
- StringBuilder sb = new StringBuilder();
- // Output nodes
- for (int pos = startPos + 1; pos <= endPosData.pos; pos++) {
- final Position posData = positions.get(pos);
- for (int idx = 0; idx < posData.count; idx++) {
- sb.append(" ");
- sb.append(getNodeID(pos, idx));
- sb.append(" [label=\"");
- sb.append(pos);
- sb.append(": ");
- sb.append(posData.lastRightID[idx]);
- sb.append("\"]\n");
- }
- }
-
- // Output arcs
- for (int pos = endPosData.pos; pos > startPos; pos--) {
- final Position posData = positions.get(pos);
- for (int idx = 0; idx < posData.count; idx++) {
- final Position backPosData = positions.get(posData.backPos[idx]);
- final String toNodeID = getNodeID(pos, idx);
- final String fromNodeID = getNodeID(posData.backPos[idx], posData.backIndex[idx]);
-
- sb.append(" ");
- sb.append(fromNodeID);
- sb.append(" -> ");
- sb.append(toNodeID);
-
- final String attrs;
- if (toNodeID.equals(bestPathMap.get(fromNodeID))) {
- // This arc is on best path
- attrs = " color=\"#40e050\" fontcolor=\"#40a050\" penwidth=3 fontsize=20";
- } else {
- attrs = "";
- }
-
- final Dictionary<? extends JaMorphData> dict = tok.getDict(posData.backType[idx]);
- final int wordCost = dict.getWordCost(posData.backID[idx]);
- final int bgCost =
- costs.get(
- backPosData.lastRightID[posData.backIndex[idx]],
- dict.getLeftId(posData.backID[idx]));
-
- final String surfaceForm =
- new String(fragment, posData.backPos[idx] - startPos, pos - posData.backPos[idx]);
-
- sb.append(" [label=\"");
- sb.append(surfaceForm);
- sb.append(' ');
- sb.append(wordCost);
- if (bgCost >= 0) {
- sb.append('+');
- }
- sb.append(bgCost);
- sb.append("\"");
- sb.append(attrs);
- sb.append("]\n");
- }
- }
- return sb.toString();
- }
-
- private String formatHeader() {
- StringBuilder sb = new StringBuilder();
- sb.append("digraph viterbi {\n");
- sb.append(
- " graph [ fontsize=30 labelloc=\"t\" label=\"\" splines=true overlap=false rankdir = \"LR\"];\n");
- // sb.append(" // A2 paper size\n");
- // sb.append(" size = \"34.4,16.5\";\n");
- // sb.append(" // try to fill paper\n");
- // sb.append(" ratio = fill;\n");
- sb.append(" edge [ fontname=\"" + FONT_NAME + "\" fontcolor=\"red\" color=\"#606060\" ]\n");
- sb.append(
- " node [ style=\"filled\" fillcolor=\"#e8e8f0\" shape=\"Mrecord\" fontname=\""
- + FONT_NAME
- + "\" ]\n");
-
- return sb.toString();
- }
-
- private String formatTrailer() {
- return "}";
- }
-
- private String getNodeID(int pos, int idx) {
- return pos + "." + idx;
- }
-}
diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizer.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizer.java
index b06b19fc359..71d19f02e1f 100644
--- a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizer.java
+++ b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizer.java
@@ -18,13 +18,6 @@ package org.apache.lucene.analysis.ja;
import java.io.IOException;
import java.io.StringReader;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.EnumMap;
-import java.util.HashMap;
-import java.util.List;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.ja.dict.CharacterDefinition;
import org.apache.lucene.analysis.ja.dict.ConnectionCosts;
@@ -34,23 +27,15 @@ import org.apache.lucene.analysis.ja.dict.TokenInfoFST;
import org.apache.lucene.analysis.ja.dict.UnknownDictionary;
import org.apache.lucene.analysis.ja.dict.UserDictionary;
import org.apache.lucene.analysis.ja.tokenattributes.*;
-import org.apache.lucene.analysis.morph.Dictionary;
-import org.apache.lucene.analysis.morph.TokenType;
+import org.apache.lucene.analysis.morph.GraphvizFormatter;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionLengthAttribute;
-import org.apache.lucene.analysis.util.RollingCharBuffer;
-import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.AttributeFactory;
import org.apache.lucene.util.IgnoreRandomChains;
-import org.apache.lucene.util.IntsRef;
-import org.apache.lucene.util.RamUsageEstimator;
import org.apache.lucene.util.fst.FST;
-// TODO: somehow factor out a reusable viterbi search here,
-// so other decompounders/tokenizers can reuse...
-
/**
* Tokenizer for Japanese that uses morphological analysis.
*
@@ -96,63 +81,11 @@ public final class JapaneseTokenizer extends Tokenizer {
private static final boolean VERBOSE = false;
- private static final int SEARCH_MODE_KANJI_LENGTH = 2;
-
- private static final int SEARCH_MODE_OTHER_LENGTH = 7; // Must be >= SEARCH_MODE_KANJI_LENGTH
-
- private static final int SEARCH_MODE_KANJI_PENALTY = 3000;
-
- private static final int SEARCH_MODE_OTHER_PENALTY = 1700;
-
- // For safety:
- private static final int MAX_UNKNOWN_WORD_LENGTH = 1024;
- private static final int MAX_BACKTRACE_GAP = 1024;
-
- private final EnumMap<TokenType, Dictionary<? extends JaMorphData>> dictionaryMap =
- new EnumMap<>(TokenType.class);
-
- private final TokenInfoFST fst;
- private final TokenInfoDictionary dictionary;
- private final UnknownDictionary unkDictionary;
- private final ConnectionCosts costs;
- private final UserDictionary userDictionary;
- private final CharacterDefinition characterDefinition;
-
- private final FST.Arc<Long> arc = new FST.Arc<>();
- private final FST.BytesReader fstReader;
- private final IntsRef wordIdRef = new IntsRef();
-
- private final FST.BytesReader userFSTReader;
- private final TokenInfoFST userFST;
-
- private final RollingCharBuffer buffer = new RollingCharBuffer();
-
- private final WrappedPositionArray positions = new WrappedPositionArray();
-
- private final boolean discardPunctuation;
- private final boolean searchMode;
- private final boolean extendedMode;
- private final boolean outputCompounds;
- private boolean outputNBest = false;
-
- // Allowable cost difference for N-best output:
- private int nBestCost = 0;
-
- // True once we've hit the EOF from the input reader:
- private boolean end;
-
- // Last absolute position we backtraced from:
- private int lastBackTracePos;
-
// Position of last token we returned; we use this to
// figure out whether to set posIncr to 0 or 1:
private int lastTokenPos;
- // Next absolute position to process:
- private int pos;
-
- // Already parsed, but not yet passed to caller, tokens:
- private final List<Token> pending = new ArrayList<>();
+ private final ViterbiNBest viterbi;
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
@@ -280,21 +213,17 @@ public final class JapaneseTokenizer extends Tokenizer {
boolean discardCompoundToken,
Mode mode) {
super(factory);
- this.dictionary = systemDictionary;
- this.fst = dictionary.getFST();
- this.unkDictionary = unkDictionary;
- this.characterDefinition = unkDictionary.getCharacterDefinition();
- this.userDictionary = userDictionary;
- this.costs = connectionCosts;
- fstReader = fst.getBytesReader();
+ TokenInfoFST fst = systemDictionary.getFST();
+ FST.BytesReader fstReader = fst.getBytesReader();
+ TokenInfoFST userFST = null;
+ FST.BytesReader userFSTReader = null;
if (userDictionary != null) {
userFST = userDictionary.getFST();
userFSTReader = userFST.getBytesReader();
- } else {
- userFST = null;
- userFSTReader = null;
}
- this.discardPunctuation = discardPunctuation;
+ boolean searchMode;
+ boolean extendedMode;
+ boolean outputCompounds;
switch (mode) {
case SEARCH:
searchMode = true;
@@ -313,282 +242,70 @@ public final class JapaneseTokenizer extends Tokenizer {
outputCompounds = false;
break;
}
- buffer.reset(this.input);
-
- resetState();
-
- dictionaryMap.put(TokenType.KNOWN, dictionary);
- dictionaryMap.put(TokenType.UNKNOWN, unkDictionary);
- dictionaryMap.put(TokenType.USER, userDictionary);
+ CharacterDefinition characterDefinition = unkDictionary.getCharacterDefinition();
+ this.viterbi =
+ new ViterbiNBest(
+ fst,
+ fstReader,
+ systemDictionary,
+ userFST,
+ userFSTReader,
+ userDictionary,
+ connectionCosts,
+ ViterbiNBest.PositionNBest.class,
+ unkDictionary,
+ characterDefinition,
+ discardPunctuation,
+ searchMode,
+ extendedMode,
+ outputCompounds);
+ viterbi.resetBuffer(this.input);
+ viterbi.resetState();
}
- private GraphvizFormatter dotOut;
-
/** Expert: set this to produce graphviz (dot) output of the Viterbi lattice */
- public void setGraphvizFormatter(GraphvizFormatter dotOut) {
- this.dotOut = dotOut;
+ public void setGraphvizFormatter(GraphvizFormatter<JaMorphData> dotOut) {
+ viterbi.setGraphvizFormatter(dotOut);
}
@Override
public void close() throws IOException {
super.close();
- buffer.reset(input);
+ viterbi.resetBuffer(input);
}
@Override
public void reset() throws IOException {
super.reset();
- buffer.reset(input);
- resetState();
- }
-
- private void resetState() {
- positions.reset();
- pos = 0;
- end = false;
- lastBackTracePos = 0;
+ viterbi.resetBuffer(input);
+ viterbi.resetState();
lastTokenPos = -1;
- pending.clear();
-
- // Add BOS:
- positions.get(0).add(0, 0, -1, -1, -1, -1, TokenType.KNOWN);
}
@Override
public void end() throws IOException {
super.end();
// Set final offset
- int finalOffset = correctOffset(pos);
+ int finalOffset = correctOffset(viterbi.getPos());
offsetAtt.setOffset(finalOffset, finalOffset);
}
- // Returns the added cost that a 2nd best segmentation is
- // allowed to have. Ie, if we see path with cost X,
- // ending in a compound word, and this method returns
- // threshold > 0, then we will also find the 2nd best
- // segmentation and if its path score is within this
- // threshold of X, we'll include it in the output:
- private int computeSecondBestThreshold(int pos, int length) throws IOException {
- // TODO: maybe we do something else here, instead of just
- // using the penalty...? EG we can be more aggressive on
- // when to also test for 2nd best path
- return computePenalty(pos, length);
- }
-
- private int computePenalty(int pos, int length) throws IOException {
- if (length > SEARCH_MODE_KANJI_LENGTH) {
- boolean allKanji = true;
- // check if node consists of only kanji
- final int endPos = pos + length;
- for (int pos2 = pos; pos2 < endPos; pos2++) {
- if (!characterDefinition.isKanji((char) buffer.get(pos2))) {
- allKanji = false;
- break;
- }
- }
- if (allKanji) { // Process only Kanji keywords
- return (length - SEARCH_MODE_KANJI_LENGTH) * SEARCH_MODE_KANJI_PENALTY;
- } else if (length > SEARCH_MODE_OTHER_LENGTH) {
- return (length - SEARCH_MODE_OTHER_LENGTH) * SEARCH_MODE_OTHER_PENALTY;
- }
- }
- return 0;
- }
-
- // Holds all back pointers arriving to this position:
- static final class Position {
-
- int pos;
-
- int count;
-
- // maybe single int array * 5?
- int[] costs = new int[8];
- int[] lastRightID = new int[8];
- int[] backPos = new int[8];
- int[] backWordPos = new int[8];
- int[] backIndex = new int[8];
- int[] backID = new int[8];
- TokenType[] backType = new TokenType[8];
-
- // Only used when finding 2nd best segmentation under a
- // too-long token:
- int forwardCount;
- int[] forwardPos = new int[8];
- int[] forwardID = new int[8];
- int[] forwardIndex = new int[8];
- TokenType[] forwardType = new TokenType[8];
-
- public void grow() {
- costs = ArrayUtil.grow(costs, 1 + count);
- lastRightID = ArrayUtil.grow(lastRightID, 1 + count);
- backPos = ArrayUtil.grow(backPos, 1 + count);
- backWordPos = ArrayUtil.grow(backWordPos, 1 + count);
- backIndex = ArrayUtil.grow(backIndex, 1 + count);
- backID = ArrayUtil.grow(backID, 1 + count);
-
- // NOTE: sneaky: grow separately because
- // ArrayUtil.grow will otherwise pick a different
- // length than the int[]s we just grew:
- final TokenType[] newBackType = new TokenType[backID.length];
- System.arraycopy(backType, 0, newBackType, 0, backType.length);
- backType = newBackType;
- }
-
- public void growForward() {
- forwardPos = ArrayUtil.grow(forwardPos, 1 + forwardCount);
- forwardID = ArrayUtil.grow(forwardID, 1 + forwardCount);
- forwardIndex = ArrayUtil.grow(forwardIndex, 1 + forwardCount);
-
- // NOTE: sneaky: grow separately because
- // ArrayUtil.grow will otherwise pick a different
- // length than the int[]s we just grew:
- final TokenType[] newForwardType = new TokenType[forwardPos.length];
- System.arraycopy(forwardType, 0, newForwardType, 0, forwardType.length);
- forwardType = newForwardType;
- }
-
- public void add(
- int cost,
- int lastRightID,
- int backPos,
- int backRPos,
- int backIndex,
- int backID,
- TokenType backType) {
- // NOTE: this isn't quite a true Viterbi search,
- // because we should check if lastRightID is
- // already present here, and only update if the new
- // cost is less than the current cost, instead of
- // simply appending. However, that will likely hurt
- // performance (usually we add a lastRightID only once),
- // and it means we actually create the full graph
- // intersection instead of a "normal" Viterbi lattice:
- if (count == costs.length) {
- grow();
- }
- this.costs[count] = cost;
- this.lastRightID[count] = lastRightID;
- this.backPos[count] = backPos;
- this.backWordPos[count] = backRPos;
- this.backIndex[count] = backIndex;
- this.backID[count] = backID;
- this.backType[count] = backType;
- count++;
- }
-
- public void addForward(int forwardPos, int forwardIndex, int forwardID, TokenType forwardType) {
- if (forwardCount == this.forwardID.length) {
- growForward();
- }
- this.forwardPos[forwardCount] = forwardPos;
- this.forwardIndex[forwardCount] = forwardIndex;
- this.forwardID[forwardCount] = forwardID;
- this.forwardType[forwardCount] = forwardType;
- forwardCount++;
- }
-
- public void reset() {
- count = 0;
- // forwardCount naturally resets after it runs:
- assert forwardCount == 0 : "pos=" + pos + " forwardCount=" + forwardCount;
- }
- }
-
- private void add(
- JaMorphData morphAtts,
- Position fromPosData,
- int wordPos,
- int endPos,
- int wordID,
- TokenType type,
- boolean addPenalty)
- throws IOException {
- final int wordCost = morphAtts.getWordCost(wordID);
- final int leftID = morphAtts.getLeftId(wordID);
- int leastCost = Integer.MAX_VALUE;
- int leastIDX = -1;
- assert fromPosData.count > 0;
- for (int idx = 0; idx < fromPosData.count; idx++) {
- // Cost is path cost so far, plus word cost (added at
- // end of loop), plus bigram cost:
- final int cost = fromPosData.costs[idx] + costs.get(fromPosData.lastRightID[idx], leftID);
- if (VERBOSE) {
- System.out.println(
- " fromIDX="
- + idx
- + ": cost="
- + cost
- + " (prevCost="
- + fromPosData.costs[idx]
- + " wordCost="
- + wordCost
- + " bgCost="
- + costs.get(fromPosData.lastRightID[idx], leftID)
- + " leftID="
- + leftID
- + ")");
- }
- if (cost < leastCost) {
- leastCost = cost;
- leastIDX = idx;
- if (VERBOSE) {
- System.out.println(" **");
- }
- }
- }
-
- leastCost += wordCost;
-
- if (VERBOSE) {
- System.out.println(
- " + cost="
- + leastCost
- + " wordID="
- + wordID
- + " leftID="
- + leftID
- + " leastIDX="
- + leastIDX
- + " toPos="
- + endPos
- + " toPos.idx="
- + positions.get(endPos).count);
- }
-
- if (addPenalty && type != TokenType.USER) {
- final int penalty = computePenalty(fromPosData.pos, endPos - fromPosData.pos);
- if (VERBOSE) {
- if (penalty > 0) {
- System.out.println(" + penalty=" + penalty + " cost=" + (leastCost + penalty));
- }
- }
- leastCost += penalty;
- }
-
- // positions.get(endPos).add(leastCost, dict.getRightId(wordID), fromPosData.pos, leastIDX,
- // wordID, type);
- assert leftID == morphAtts.getRightId(wordID);
- positions.get(endPos).add(leastCost, leftID, fromPosData.pos, wordPos, leastIDX, wordID, type);
- }
-
@Override
public boolean incrementToken() throws IOException {
- // parse() is able to return w/o producing any new
+ // forward() can return w/o producing any new
// tokens, when the tokens it had produced were entirely
// punctuation. So we loop here until we get a real
// token or we end:
- while (pending.size() == 0) {
- if (end) {
+ while (viterbi.getPending().size() == 0) {
+ if (viterbi.isEnd()) {
return false;
}
-
// Push Viterbi forward some more:
- parse();
+ viterbi.forward();
}
- final Token token = pending.remove(pending.size() - 1);
+ final Token token = viterbi.getPending().remove(viterbi.getPending().size() - 1);
int length = token.getLength();
clearAttributes();
@@ -604,7 +321,7 @@ public final class JapaneseTokenizer extends Tokenizer {
if (token.getStartOffset() == lastTokenPos) {
posIncAtt.setPositionIncrement(0);
posLengthAtt.setPositionLength(token.getPositionLength());
- } else if (outputNBest) {
+ } else if (viterbi.isOutputNBest()) {
// The position length is always calculated if outputNBest is true.
assert token.getStartOffset() > lastTokenPos;
posIncAtt.setPositionIncrement(1);
@@ -618,1061 +335,8 @@ public final class JapaneseTokenizer extends Tokenizer {
System.out.println(Thread.currentThread().getName() + ": incToken: return token=" + token);
}
lastTokenPos = token.getStartOffset();
- return true;
- }
-
- // TODO: make generic'd version of this "circular array"?
- // It's a bit tricky because we do things to the Position
- // (eg, set .pos = N on reuse)...
- static final class WrappedPositionArray {
- private Position[] positions = new Position[8];
-
- public WrappedPositionArray() {
- for (int i = 0; i < positions.length; i++) {
- positions[i] = new Position();
- }
- }
-
- // Next array index to write to in positions:
- private int nextWrite;
-
- // Next position to write:
- private int nextPos;
-
- // How many valid Position instances are held in the
- // positions array:
- private int count;
-
- public void reset() {
- nextWrite--;
- while (count > 0) {
- if (nextWrite == -1) {
- nextWrite = positions.length - 1;
- }
- positions[nextWrite--].reset();
- count--;
- }
- nextWrite = 0;
- nextPos = 0;
- count = 0;
- }
-
- /**
- * Get Position instance for this absolute position; this is allowed to be arbitrarily far "in
- * the future" but cannot be before the last freeBefore.
- */
- public Position get(int pos) {
- while (pos >= nextPos) {
- // System.out.println("count=" + count + " vs len=" + positions.length);
- if (count == positions.length) {
- Position[] newPositions =
- new Position[ArrayUtil.oversize(1 + count, RamUsageEstimator.NUM_BYTES_OBJECT_REF)];
- // System.out.println("grow positions " + newPositions.length);
- System.arraycopy(positions, nextWrite, newPositions, 0, positions.length - nextWrite);
- System.arraycopy(positions, 0, newPositions, positions.length - nextWrite, nextWrite);
- for (int i = positions.length; i < newPositions.length; i++) {
- newPositions[i] = new Position();
- }
- nextWrite = positions.length;
- positions = newPositions;
- }
- if (nextWrite == positions.length) {
- nextWrite = 0;
- }
- // Should have already been reset:
- assert positions[nextWrite].count == 0;
- positions[nextWrite++].pos = nextPos++;
- count++;
- }
- assert inBounds(pos);
- final int index = getIndex(pos);
- assert positions[index].pos == pos;
- return positions[index];
- }
-
- public int getNextPos() {
- return nextPos;
- }
-
- // For assert:
- private boolean inBounds(int pos) {
- return pos < nextPos && pos >= nextPos - count;
- }
-
- private int getIndex(int pos) {
- int index = nextWrite - (nextPos - pos);
- if (index < 0) {
- index += positions.length;
- }
- return index;
- }
-
- public void freeBefore(int pos) {
- final int toFree = count - (nextPos - pos);
- assert toFree >= 0;
- assert toFree <= count;
- int index = nextWrite - count;
- if (index < 0) {
- index += positions.length;
- }
- for (int i = 0; i < toFree; i++) {
- if (index == positions.length) {
- index = 0;
- }
- // System.out.println(" fb idx=" + index);
- positions[index].reset();
- index++;
- }
- count -= toFree;
- }
- }
-
- /* Incrementally parse some more characters. This runs
- * the viterbi search forwards "enough" so that we
- * generate some more tokens. How much forward depends on
- * the chars coming in, since some chars could cause
- * longer-lasting ambiguity in the parsing. Once the
- * ambiguity is resolved, then we back trace, produce
- * the pending tokens, and return. */
- private void parse() throws IOException {
- if (VERBOSE) {
- System.out.println("\nPARSE");
- }
-
- // Index of the last character of unknown word:
- int unknownWordEndIndex = -1;
-
- // Advances over each position (character):
- while (true) {
-
- if (buffer.get(pos) == -1) {
- // End
- break;
- }
-
- final Position posData = positions.get(pos);
- final boolean isFrontier = positions.getNextPos() == pos + 1;
-
- if (posData.count == 0) {
- // No arcs arrive here; move to next position:
- if (VERBOSE) {
- System.out.println(" no arcs in; skip pos=" + pos);
- }
- pos++;
- continue;
- }
-
- if (pos > lastBackTracePos && posData.count == 1 && isFrontier) {
- // if (pos > lastBackTracePos && posData.count == 1 && isFrontier) {
- // We are at a "frontier", and only one node is
- // alive, so whatever the eventual best path is must
- // come through this node. So we can safely commit
- // to the prefix of the best path at this point:
- if (outputNBest) {
- backtraceNBest(posData, false);
- }
- backtrace(posData, 0);
- if (outputNBest) {
- fixupPendingList();
- }
-
- // Re-base cost so we don't risk int overflow:
- posData.costs[0] = 0;
-
- if (pending.size() != 0) {
- return;
- } else {
- // This means the backtrace only produced
- // punctuation tokens, so we must keep parsing.
- }
- }
-
- if (pos - lastBackTracePos >= MAX_BACKTRACE_GAP) {
- // Safety: if we've buffered too much, force a
- // backtrace now. We find the least-cost partial
- // path, across all paths, backtrace from it, and
- // then prune all others. Note that this, in
- // general, can produce the wrong result, if the
- // total best path did not in fact back trace
- // through this partial best path. But it's the
- // best we can do... (short of not having a
- // safety!).
-
- // First pass: find least cost partial path so far,
- // including ending at future positions:
- int leastIDX = -1;
- int leastCost = Integer.MAX_VALUE;
- Position leastPosData = null;
- for (int pos2 = pos; pos2 < positions.getNextPos(); pos2++) {
- final Position posData2 = positions.get(pos2);
- for (int idx = 0; idx < posData2.count; idx++) {
- // System.out.println(" idx=" + idx + " cost=" + cost);
- final int cost = posData2.costs[idx];
- if (cost < leastCost) {
- leastCost = cost;
- leastIDX = idx;
- leastPosData = posData2;
- }
- }
- }
-
- // We will always have at least one live path:
- assert leastIDX != -1;
-
- if (outputNBest) {
- backtraceNBest(leastPosData, false);
- }
-
- // Second pass: prune all but the best path:
- for (int pos2 = pos; pos2 < positions.getNextPos(); pos2++) {
- final Position posData2 = positions.get(pos2);
- if (posData2 != leastPosData) {
- posData2.reset();
- } else {
- if (leastIDX != 0) {
- posData2.costs[0] = posData2.costs[leastIDX];
- posData2.lastRightID[0] = posData2.lastRightID[leastIDX];
- posData2.backPos[0] = posData2.backPos[leastIDX];
- posData2.backIndex[0] = posData2.backIndex[leastIDX];
- posData2.backID[0] = posData2.backID[leastIDX];
- posData2.backType[0] = posData2.backType[leastIDX];
- }
- posData2.count = 1;
- }
- }
-
- backtrace(leastPosData, 0);
- if (outputNBest) {
- fixupPendingList();
- }
-
- // Re-base cost so we don't risk int overflow:
- Arrays.fill(leastPosData.costs, 0, leastPosData.count, 0);
-
- if (pos != leastPosData.pos) {
- // We jumped into a future position:
- assert pos < leastPosData.pos;
- pos = leastPosData.pos;
- }
-
- if (pending.size() != 0) {
- return;
- } else {
- // This means the backtrace only produced
- // punctuation tokens, so we must keep parsing.
- continue;
- }
- }
-
- if (VERBOSE) {
- System.out.println(
- "\n extend @ pos="
- + pos
- + " char="
- + (char) buffer.get(pos)
- + " hex="
- + Integer.toHexString(buffer.get(pos)));
- }
-
- if (VERBOSE) {
- System.out.println(" " + posData.count + " arcs in");
- }
-
- boolean anyMatches = false;
-
- // First try user dict:
- if (userFST != null) {
- userFST.getFirstArc(arc);
- int output = 0;
- for (int posAhead = posData.pos; ; posAhead++) {
- final int ch = buffer.get(posAhead);
- if (ch == -1) {
- break;
- }
- if (userFST.findTargetArc(ch, arc, arc, posAhead == posData.pos, userFSTReader) == null) {
- break;
- }
- output += arc.output().intValue();
- if (arc.isFinal()) {
- if (VERBOSE) {
- System.out.println(
- " USER word "
- + new String(buffer.get(pos, posAhead - pos + 1))
- + " toPos="
- + (posAhead + 1));
- }
- add(
- userDictionary.getMorphAttributes(),
- posData,
- pos,
- posAhead + 1,
- output + arc.nextFinalOutput().intValue(),
- TokenType.USER,
- false);
- anyMatches = true;
- }
- }
- }
-
- // TODO: we can be more aggressive about user
- // matches? if we are "under" a user match then don't
- // extend KNOWN/UNKNOWN paths?
-
- if (!anyMatches) {
- // Next, try known dictionary matches
- fst.getFirstArc(arc);
- int output = 0;
-
- for (int posAhead = posData.pos; ; posAhead++) {
- final int ch = buffer.get(posAhead);
- if (ch == -1) {
- break;
- }
- // System.out.println(" match " + (char) ch + " posAhead=" + posAhead);
-
- if (fst.findTargetArc(ch, arc, arc, posAhead == posData.pos, fstReader) == null) {
- break;
- }
-
- output += arc.output().intValue();
-
- // Optimization: for known words that are too-long
- // (compound), we should pre-compute the 2nd
- // best segmentation and store it in the
- // dictionary instead of recomputing it each time a
- // match is found.
-
- if (arc.isFinal()) {
- dictionary.lookupWordIds(output + arc.nextFinalOutput().intValue(), wordIdRef);
- if (VERBOSE) {
- System.out.println(
- " KNOWN word "
- + new String(buffer.get(pos, posAhead - pos + 1))
- + " toPos="
- + (posAhead + 1)
- + " "
- + wordIdRef.length
- + " wordIDs");
- }
- for (int ofs = 0; ofs < wordIdRef.length; ofs++) {
- add(
- dictionary.getMorphAttributes(),
- posData,
- pos,
- posAhead + 1,
- wordIdRef.ints[wordIdRef.offset + ofs],
- TokenType.KNOWN,
- false);
- anyMatches = true;
- }
- }
- }
- }
-
- // In the case of normal mode, it doesn't process unknown word greedily.
-
- if (!searchMode && unknownWordEndIndex > posData.pos) {
- pos++;
- continue;
- }
-
- final char firstCharacter = (char) buffer.get(pos);
- if (!anyMatches || characterDefinition.isInvoke(firstCharacter)) {
-
- // Find unknown match:
- final int characterId = characterDefinition.getCharacterClass(firstCharacter);
- final boolean isPunct = isPunctuation(firstCharacter);
-
- // NOTE: copied from UnknownDictionary.lookup:
- int unknownWordLength;
- if (!characterDefinition.isGroup(firstCharacter)) {
- unknownWordLength = 1;
- } else {
- // Extract unknown word. Characters with the same character class are considered to be
- // part of unknown word
- unknownWordLength = 1;
- for (int posAhead = pos + 1; unknownWordLength < MAX_UNKNOWN_WORD_LENGTH; posAhead++) {
- final int ch = buffer.get(posAhead);
- if (ch == -1) {
- break;
- }
- if (characterId == characterDefinition.getCharacterClass((char) ch)
- && isPunctuation((char) ch) == isPunct) {
- unknownWordLength++;
- } else {
- break;
- }
- }
- }
-
- unkDictionary.lookupWordIds(
- characterId, wordIdRef); // characters in input text are supposed to be the same
- if (VERBOSE) {
- System.out.println(
- " UNKNOWN word len=" + unknownWordLength + " " + wordIdRef.length + " wordIDs");
- }
- for (int ofs = 0; ofs < wordIdRef.length; ofs++) {
- add(
- unkDictionary.getMorphAttributes(),
- posData,
- pos,
- posData.pos + unknownWordLength,
- wordIdRef.ints[wordIdRef.offset + ofs],
- TokenType.UNKNOWN,
- false);
- }
-
- unknownWordEndIndex = posData.pos + unknownWordLength;
- }
-
- pos++;
- }
-
- end = true;
-
- if (pos > 0) {
-
- final Position endPosData = positions.get(pos);
- int leastCost = Integer.MAX_VALUE;
- int leastIDX = -1;
- if (VERBOSE) {
- System.out.println(" end: " + endPosData.count + " nodes");
- }
- for (int idx = 0; idx < endPosData.count; idx++) {
- // Add EOS cost:
- final int cost = endPosData.costs[idx] + costs.get(endPosData.lastRightID[idx], 0);
- // System.out.println(" idx=" + idx + " cost=" + cost + " (pathCost=" +
- // endPosData.costs[idx] + " bgCost=" + costs.get(endPosData.lastRightID[idx], 0) + ")
- // backPos=" + endPosData.backPos[idx]);
- if (cost < leastCost) {
- leastCost = cost;
- leastIDX = idx;
- }
- }
-
- if (outputNBest) {
- backtraceNBest(endPosData, true);
- }
- backtrace(endPosData, leastIDX);
- if (outputNBest) {
- fixupPendingList();
- }
- } else {
- // No characters in the input string; return no tokens!
- }
- }
-
- // Eliminates arcs from the lattice that are compound
- // tokens (have a penalty) or are not congruent with the
- // compound token we've matched (ie, span across the
- // startPos). This should be fairly efficient, because we
- // just keep the already intersected structure of the
- // graph, eg we don't have to consult the FSTs again:
-
- private void pruneAndRescore(int startPos, int endPos, int bestStartIDX) throws IOException {
- if (VERBOSE) {
- System.out.println(
- " pruneAndRescore startPos="
- + startPos
- + " endPos="
- + endPos
- + " bestStartIDX="
- + bestStartIDX);
- }
-
- // First pass: walk backwards, building up the forward
- // arcs and pruning inadmissible arcs:
- for (int pos = endPos; pos > startPos; pos--) {
- final Position posData = positions.get(pos);
- if (VERBOSE) {
- System.out.println(" back pos=" + pos);
- }
- for (int arcIDX = 0; arcIDX < posData.count; arcIDX++) {
- final int backPos = posData.backPos[arcIDX];
- if (backPos >= startPos) {
- // Keep this arc:
- // System.out.println(" keep backPos=" + backPos);
- positions
- .get(backPos)
- .addForward(pos, arcIDX, posData.backID[arcIDX], posData.backType[arcIDX]);
- } else {
- if (VERBOSE) {
- System.out.println(" prune");
- }
- }
- }
- if (pos != startPos) {
- posData.count = 0;
- }
- }
-
- // Second pass: walk forward, re-scoring:
- for (int pos = startPos; pos < endPos; pos++) {
- final Position posData = positions.get(pos);
- if (VERBOSE) {
- System.out.println(" forward pos=" + pos + " count=" + posData.forwardCount);
- }
- if (posData.count == 0) {
- // No arcs arrive here...
- if (VERBOSE) {
- System.out.println(" skip");
- }
- posData.forwardCount = 0;
- continue;
- }
-
- if (pos == startPos) {
- // On the initial position, only consider the best
- // path so we "force congruence": the
- // sub-segmentation is "in context" of what the best
- // path (compound token) had matched:
- final int rightID;
- if (startPos == 0) {
- rightID = 0;
- } else {
- rightID =
- getDict(posData.backType[bestStartIDX]).getRightId(posData.backID[bestStartIDX]);
- }
- final int pathCost = posData.costs[bestStartIDX];
- for (int forwardArcIDX = 0; forwardArcIDX < posData.forwardCount; forwardArcIDX++) {
- final TokenType forwardType = posData.forwardType[forwardArcIDX];
- final Dictionary<? extends JaMorphData> dict2 = getDict(forwardType);
- final int wordID = posData.forwardID[forwardArcIDX];
- final int toPos = posData.forwardPos[forwardArcIDX];
- final int newCost =
- pathCost
- + dict2.getWordCost(wordID)
- + costs.get(rightID, dict2.getLeftId(wordID))
- + computePenalty(pos, toPos - pos);
- if (VERBOSE) {
- System.out.println(
- " + "
- + forwardType
- + " word "
- + new String(buffer.get(pos, toPos - pos))
- + " toPos="
- + toPos
- + " cost="
- + newCost
- + " penalty="
- + computePenalty(pos, toPos - pos)
- + " toPos.idx="
- + positions.get(toPos).count);
- }
- positions
- .get(toPos)
- .add(newCost, dict2.getRightId(wordID), pos, -1, bestStartIDX, wordID, forwardType);
- }
- } else {
- // On non-initial positions, we maximize score
- // across all arriving lastRightIDs:
- for (int forwardArcIDX = 0; forwardArcIDX < posData.forwardCount; forwardArcIDX++) {
- final TokenType forwardType = posData.forwardType[forwardArcIDX];
- final int toPos = posData.forwardPos[forwardArcIDX];
- if (VERBOSE) {
- System.out.println(
- " + "
- + forwardType
- + " word "
- + new String(buffer.get(pos, toPos - pos))
- + " toPos="
- + toPos);
- }
- add(
- getDict(forwardType).getMorphAttributes(),
- posData,
- pos,
- toPos,
- posData.forwardID[forwardArcIDX],
- forwardType,
- true);
- }
- }
- posData.forwardCount = 0;
- }
- }
-
- // yet another lattice data structure
- private static final class Lattice {
- char[] fragment;
- EnumMap<TokenType, Dictionary<? extends JaMorphData>> dictionaryMap;
- boolean useEOS;
-
- int rootCapacity = 0;
- int rootSize = 0;
- int rootBase = 0;
-
- // root pointers of node chain by leftChain_ that have same start offset.
- int[] lRoot;
- // root pointers of node chain by rightChain_ that have same end offset.
- int[] rRoot;
-
- int capacity = 0;
- int nodeCount = 0;
-
- // The variables below are elements of lattice node that indexed by node number.
- TokenType[] nodeDicType;
- int[] nodeWordID;
- // nodeMark - -1:excluded, 0:unused, 1:bestpath, 2:2-best-path, ... N:N-best-path
- int[] nodeMark;
- int[] nodeLeftID;
- int[] nodeRightID;
- int[] nodeWordCost;
- int[] nodeLeftCost;
- int[] nodeRightCost;
- // nodeLeftNode, nodeRightNode - are left/right node number with minimum cost path.
- int[] nodeLeftNode;
- int[] nodeRightNode;
- // nodeLeft, nodeRight - start/end offset
- int[] nodeLeft;
- int[] nodeRight;
- int[] nodeLeftChain;
- int[] nodeRightChain;
-
- private void setupRoot(int baseOffset, int lastOffset) {
- assert baseOffset <= lastOffset;
- int size = lastOffset - baseOffset + 1;
- if (rootCapacity < size) {
- int oversize = ArrayUtil.oversize(size, Integer.BYTES);
- lRoot = new int[oversize];
- rRoot = new int[oversize];
- rootCapacity = oversize;
- }
- Arrays.fill(lRoot, 0, size, -1);
- Arrays.fill(rRoot, 0, size, -1);
- rootSize = size;
- rootBase = baseOffset;
- }
-
- // Reserve at least N nodes.
- private void reserve(int n) {
- if (capacity < n) {
- int oversize = ArrayUtil.oversize(n, Integer.BYTES);
- nodeDicType = new TokenType[oversize];
- nodeWordID = new int[oversize];
- nodeMark = new int[oversize];
- nodeLeftID = new int[oversize];
- nodeRightID = new int[oversize];
- nodeWordCost = new int[oversize];
- nodeLeftCost = new int[oversize];
- nodeRightCost = new int[oversize];
- nodeLeftNode = new int[oversize];
- nodeRightNode = new int[oversize];
- nodeLeft = new int[oversize];
- nodeRight = new int[oversize];
- nodeLeftChain = new int[oversize];
- nodeRightChain = new int[oversize];
- capacity = oversize;
- }
- }
-
- private void setupNodePool(int n) {
- reserve(n);
- nodeCount = 0;
- if (VERBOSE) {
- System.out.printf("DEBUG: setupNodePool: n = %d\n", n);
- System.out.printf("DEBUG: setupNodePool: lattice.capacity = %d\n", capacity);
- }
- }
-
- private int addNode(TokenType dicType, int wordID, int left, int right) {
- if (VERBOSE) {
- System.out.printf(
- "DEBUG: addNode: dicType=%s, wordID=%d, left=%d, right=%d, str=%s\n",
- dicType.toString(),
- wordID,
- left,
- right,
- left == -1 ? "BOS" : right == -1 ? "EOS" : new String(fragment, left, right - left));
- }
- assert nodeCount < capacity;
- assert left == -1 || right == -1 || left < right;
- assert left == -1 || (0 <= left && left < rootSize);
- assert right == -1 || (0 <= right && right < rootSize);
-
- int node = nodeCount++;
-
- if (VERBOSE) {
- System.out.printf("DEBUG: addNode: node=%d\n", node);
- }
-
- nodeDicType[node] = dicType;
- nodeWordID[node] = wordID;
- nodeMark[node] = 0;
-
- if (wordID < 0) {
- nodeWordCost[node] = 0;
- nodeLeftCost[node] = 0;
- nodeRightCost[node] = 0;
- nodeLeftID[node] = 0;
- nodeRightID[node] = 0;
- } else {
- Dictionary<? extends JaMorphData> dic = dictionaryMap.get(dicType);
- nodeWordCost[node] = dic.getWordCost(wordID);
- nodeLeftID[node] = dic.getLeftId(wordID);
- nodeRightID[node] = dic.getRightId(wordID);
- }
-
- if (VERBOSE) {
- System.out.printf(
- "DEBUG: addNode: wordCost=%d, leftID=%d, rightID=%d\n",
- nodeWordCost[node], nodeLeftID[node], nodeRightID[node]);
- }
-
- nodeLeft[node] = left;
- nodeRight[node] = right;
- if (0 <= left) {
- nodeLeftChain[node] = lRoot[left];
- lRoot[left] = node;
- } else {
- nodeLeftChain[node] = -1;
- }
- if (0 <= right) {
- nodeRightChain[node] = rRoot[right];
- rRoot[right] = node;
- } else {
- nodeRightChain[node] = -1;
- }
- return node;
- }
-
- // Sum of positions.get(i).count in [beg, end) range.
- // using stream:
- // return IntStream.range(beg, end).map(i -> positions.get(i).count).sum();
- private int positionCount(WrappedPositionArray positions, int beg, int end) {
- int count = 0;
- for (int i = beg; i < end; ++i) {
- count += positions.get(i).count;
- }
- return count;
- }
-
- void setup(
- char[] fragment,
- EnumMap<TokenType, Dictionary<? extends JaMorphData>> dictionaryMap,
- WrappedPositionArray positions,
- int prevOffset,
- int endOffset,
- boolean useEOS) {
- assert positions.get(prevOffset).count == 1;
- if (VERBOSE) {
- System.out.printf("DEBUG: setup: prevOffset=%d, endOffset=%d\n", prevOffset, endOffset);
- }
-
- this.fragment = fragment;
- this.dictionaryMap = dictionaryMap;
- this.useEOS = useEOS;
-
- // Initialize lRoot and rRoot.
- setupRoot(prevOffset, endOffset);
-
- // "+ 2" for first/last record.
- setupNodePool(positionCount(positions, prevOffset + 1, endOffset + 1) + 2);
-
- // substitute for BOS = 0
- Position first = positions.get(prevOffset);
- if (addNode(first.backType[0], first.backID[0], -1, 0) != 0) {
- assert false;
- }
-
- // EOS = 1
- if (addNode(TokenType.KNOWN, -1, endOffset - rootBase, -1) != 1) {
- assert false;
- }
-
- for (int offset = endOffset; prevOffset < offset; --offset) {
- int right = offset - rootBase;
- // optimize: exclude disconnected nodes.
- if (0 <= lRoot[right]) {
- Position pos = positions.get(offset);
- for (int i = 0; i < pos.count; ++i) {
- addNode(pos.backType[i], pos.backID[i], pos.backPos[i] - rootBase, right);
- }
- }
- }
- }
-
- // set mark = -1 for unreachable nodes.
- void markUnreachable() {
- for (int index = 1; index < rootSize - 1; ++index) {
- if (rRoot[index] < 0) {
- for (int node = lRoot[index]; 0 <= node; node = nodeLeftChain[node]) {
- if (VERBOSE) {
- System.out.printf("DEBUG: markUnreachable: node=%d\n", node);
- }
- nodeMark[node] = -1;
- }
- }
- }
- }
-
- int connectionCost(ConnectionCosts costs, int left, int right) {
- int leftID = nodeLeftID[right];
- return ((leftID == 0 && !useEOS) ? 0 : costs.get(nodeRightID[left], leftID));
- }
-
- void calcLeftCost(ConnectionCosts costs) {
- for (int index = 0; index < rootSize; ++index) {
- for (int node = lRoot[index]; 0 <= node; node = nodeLeftChain[node]) {
- if (0 <= nodeMark[node]) {
- int leastNode = -1;
- int leastCost = Integer.MAX_VALUE;
- for (int leftNode = rRoot[index]; 0 <= leftNode; leftNode = nodeRightChain[leftNode]) {
- if (0 <= nodeMark[leftNode]) {
- int cost =
- nodeLeftCost[leftNode]
- + nodeWordCost[leftNode]
- + connectionCost(costs, leftNode, node);
- if (cost < leastCost) {
- leastCost = cost;
- leastNode = leftNode;
- }
- }
- }
- assert 0 <= leastNode;
- nodeLeftNode[node] = leastNode;
- nodeLeftCost[node] = leastCost;
- if (VERBOSE) {
- System.out.printf(
- "DEBUG: calcLeftCost: node=%d, leftNode=%d, leftCost=%d\n",
- node, nodeLeftNode[node], nodeLeftCost[node]);
- }
- }
- }
- }
- }
-
- void calcRightCost(ConnectionCosts costs) {
- for (int index = rootSize - 1; 0 <= index; --index) {
- for (int node = rRoot[index]; 0 <= node; node = nodeRightChain[node]) {
- if (0 <= nodeMark[node]) {
- int leastNode = -1;
- int leastCost = Integer.MAX_VALUE;
- for (int rightNode = lRoot[index];
- 0 <= rightNode;
- rightNode = nodeLeftChain[rightNode]) {
- if (0 <= nodeMark[rightNode]) {
- int cost =
- nodeRightCost[rightNode]
- + nodeWordCost[rightNode]
- + connectionCost(costs, node, rightNode);
- if (cost < leastCost) {
- leastCost = cost;
- leastNode = rightNode;
- }
- }
- }
- assert 0 <= leastNode;
- nodeRightNode[node] = leastNode;
- nodeRightCost[node] = leastCost;
- if (VERBOSE) {
- System.out.printf(
- "DEBUG: calcRightCost: node=%d, rightNode=%d, rightCost=%d\n",
- node, nodeRightNode[node], nodeRightCost[node]);
- }
- }
- }
- }
- }
-
- // Mark all nodes that have same text and different par-of-speech or reading.
- void markSameSpanNode(int refNode, int value) {
- int left = nodeLeft[refNode];
- int right = nodeRight[refNode];
- for (int node = lRoot[left]; 0 <= node; node = nodeLeftChain[node]) {
- if (nodeRight[node] == right) {
- nodeMark[node] = value;
- }
- }
- }
-
- List<Integer> bestPathNodeList() {
- List<Integer> list = new ArrayList<>();
- for (int node = nodeRightNode[0]; node != 1; node = nodeRightNode[node]) {
- list.add(node);
- markSameSpanNode(node, 1);
- }
- return list;
- }
-
- private int cost(int node) {
- return nodeLeftCost[node] + nodeWordCost[node] + nodeRightCost[node];
- }
-
- List<Integer> nBestNodeList(int N) {
- List<Integer> list = new ArrayList<>();
- int leastCost = Integer.MAX_VALUE;
- int leastLeft = -1;
- int leastRight = -1;
- for (int node = 2; node < nodeCount; ++node) {
- if (nodeMark[node] == 0) {
- int cost = cost(node);
- if (cost < leastCost) {
- leastCost = cost;
- leastLeft = nodeLeft[node];
- leastRight = nodeRight[node];
- list.clear();
- list.add(node);
- } else if (cost == leastCost
- && (nodeLeft[node] != leastLeft || nodeRight[node] != leastRight)) {
- list.add(node);
- }
- }
- }
- for (int node : list) {
- markSameSpanNode(node, N);
- }
- return list;
- }
-
- int bestCost() {
- return nodeLeftCost[1];
- }
-
- int probeDelta(int start, int end) {
- int left = start - rootBase;
- int right = end - rootBase;
- if (left < 0 || rootSize < right) {
- return Integer.MAX_VALUE;
- }
- int probedCost = Integer.MAX_VALUE;
- for (int node = lRoot[left]; 0 <= node; node = nodeLeftChain[node]) {
- if (nodeRight[node] == right) {
- probedCost = Math.min(probedCost, cost(node));
- }
- }
- return probedCost - bestCost();
- }
-
- void debugPrint() {
- if (VERBOSE) {
- for (int node = 0; node < nodeCount; ++node) {
- System.out.printf(
- "DEBUG NODE: node=%d, mark=%d, cost=%d, left=%d, right=%d\n",
- node, nodeMark[node], cost(node), nodeLeft[node], nodeRight[node]);
- }
- }
- }
- }
-
- private Lattice lattice = null;
-
- private void registerNode(int node, char[] fragment) {
- int left = lattice.nodeLeft[node];
- int right = lattice.nodeRight[node];
- TokenType type = lattice.nodeDicType[node];
- if (!discardPunctuation || !isPunctuation(fragment[left])) {
- if (type == TokenType.USER) {
- // The code below are based on backtrace().
- //
- // Expand the phraseID we recorded into the actual segmentation:
- final int[] wordIDAndLength = userDictionary.lookupSegmentation(lattice.nodeWordID[node]);
- int wordID = wordIDAndLength[0];
- pending.add(
- new Token(
- fragment,
- left,
- right - left,
- lattice.rootBase + left,
- lattice.rootBase + right,
- wordID,
- TokenType.USER,
- userDictionary.getMorphAttributes()));
- // Output compound
- int current = 0;
- for (int j = 1; j < wordIDAndLength.length; j++) {
- final int len = wordIDAndLength[j];
- if (len < right - left) {
- int startOffset = lattice.rootBase + current + left;
- pending.add(
- new Token(
- fragment,
- current + left,
- len,
- startOffset,
- startOffset + len,
- wordID + j - 1,
- TokenType.USER,
- userDictionary.getMorphAttributes()));
- }
- current += len;
- }
- } else {
- pending.add(
- new Token(
- fragment,
- left,
- right - left,
- lattice.rootBase + left,
- lattice.rootBase + right,
- lattice.nodeWordID[node],
- type,
- getDict(type).getMorphAttributes()));
- }
- }
- }
-
- // Sort pending tokens, and set position increment values.
- private void fixupPendingList() {
- // Sort for removing same tokens.
- // USER token should be ahead from normal one.
- Collections.sort(
- pending,
- new Comparator<Token>() {
- @Override
- public int compare(Token a, Token b) {
- int aOff = a.getOffset();
- int bOff = b.getOffset();
- if (aOff != bOff) {
- return aOff - bOff;
- }
- int aLen = a.getLength();
- int bLen = b.getLength();
- if (aLen != bLen) {
- return aLen - bLen;
- }
- // order of Type is KNOWN, UNKNOWN, USER,
- // so we use reversed comparison here.
- return b.getType().ordinal() - a.getType().ordinal();
- }
- });
-
- // Remove same token.
- for (int i = 1; i < pending.size(); ++i) {
- Token a = pending.get(i - 1);
- Token b = pending.get(i);
- if (a.getOffset() == b.getOffset() && a.getLength() == b.getLength()) {
- pending.remove(i);
- // It is important to decrement "i" here, because a next may be removed.
- --i;
- }
- }
-
- // offset=>position map
- HashMap<Integer, Integer> map = new HashMap<>();
- for (Token t : pending) {
- map.put(t.getOffset(), 0);
- map.put(t.getOffset() + t.getLength(), 0);
- }
-
- // Get uniqe and sorted list of all edge position of tokens.
- Integer[] offsets = map.keySet().toArray(new Integer[0]);
- Arrays.sort(offsets);
- // setup all value of map. It specify N-th position from begin.
- for (int i = 0; i < offsets.length; ++i) {
- map.put(offsets[i], i);
- }
-
- // We got all position length now.
- for (Token t : pending) {
- t.setPositionLength(map.get(t.getOffset() + t.getLength()) - map.get(t.getOffset()));
- }
-
- // Make PENDING to be reversed order to fit its usage.
- // If you would like to speedup, you can try reversed order sort
- // at first of this function.
- Collections.reverse(pending);
+ return true;
}
private int probeDelta(String inText, String requiredToken) throws IOException {
@@ -1683,16 +347,17 @@ public final class JapaneseTokenizer extends Tokenizer {
}
int delta = Integer.MAX_VALUE;
- int saveNBestCost = nBestCost;
+ int saveNBestCost = viterbi.getNBestCost();
setReader(new StringReader(inText));
reset();
try {
setNBestCost(1);
int prevRootBase = -1;
while (incrementToken()) {
- if (lattice.rootBase != prevRootBase) {
- prevRootBase = lattice.rootBase;
- delta = Math.min(delta, lattice.probeDelta(start, start + requiredToken.length()));
+ if (viterbi.lattice.rootBase != prevRootBase) {
+ prevRootBase = viterbi.lattice.rootBase;
+ delta =
+ Math.min(delta, viterbi.lattice.probeDelta(start, start + requiredToken.length()));
}
}
} finally {
@@ -1730,404 +395,6 @@ public final class JapaneseTokenizer extends Tokenizer {
}
public void setNBestCost(int value) {
- nBestCost = value;
- outputNBest = 0 < nBestCost;
- }
-
- private void backtraceNBest(final Position endPosData, final boolean useEOS) throws IOException {
- if (lattice == null) {
- lattice = new Lattice();
- }
-
- final int endPos = endPosData.pos;
- char[] fragment = buffer.get(lastBackTracePos, endPos - lastBackTracePos);
- lattice.setup(fragment, dictionaryMap, positions, lastBackTracePos, endPos, useEOS);
- lattice.markUnreachable();
- lattice.calcLeftCost(costs);
- lattice.calcRightCost(costs);
-
- int bestCost = lattice.bestCost();
- if (VERBOSE) {
- System.out.printf("DEBUG: 1-BEST COST: %d\n", bestCost);
- }
- for (int node : lattice.bestPathNodeList()) {
- registerNode(node, fragment);
- }
-
- for (int n = 2; ; ++n) {
- List<Integer> nbest = lattice.nBestNodeList(n);
- if (nbest.isEmpty()) {
- break;
- }
- int cost = lattice.cost(nbest.get(0));
- if (VERBOSE) {
- System.out.printf("DEBUG: %d-BEST COST: %d\n", n, cost);
- }
- if (bestCost + nBestCost < cost) {
- break;
- }
- for (int node : nbest) {
- registerNode(node, fragment);
- }
- }
- if (VERBOSE) {
- lattice.debugPrint();
- }
- }
-
- // Backtrace from the provided position, back to the last
- // time we back-traced, accumulating the resulting tokens to
- // the pending list. The pending list is then in-reverse
- // (last token should be returned first).
- private void backtrace(final Position endPosData, final int fromIDX) throws IOException {
- final int endPos = endPosData.pos;
-
- /**
- * LUCENE-10059: If the endPos is the same as lastBackTracePos, we don't want to backtrace to
- * avoid an assertion error {@link RollingCharBuffer#get(int)} when it tries to generate an
- * empty buffer
- */
- if (endPos == lastBackTracePos) {
- return;
- }
-
- if (VERBOSE) {
- System.out.println(
- "\n backtrace: endPos="
- + endPos
- + " pos="
- + pos
- + "; "
- + (pos - lastBackTracePos)
- + " characters; last="
- + lastBackTracePos
- + " cost="
- + endPosData.costs[fromIDX]);
- }
-
- final char[] fragment = buffer.get(lastBackTracePos, endPos - lastBackTracePos);
-
- if (dotOut != null) {
- dotOut.onBacktrace(this, positions, lastBackTracePos, endPosData, fromIDX, fragment, end);
- }
-
- int pos = endPos;
- int bestIDX = fromIDX;
- Token altToken = null;
-
- // We trace backwards, so this will be the leftWordID of
- // the token after the one we are now on:
- int lastLeftWordID = -1;
-
- int backCount = 0;
-
- // TODO: sort of silly to make Token instances here; the
- // back trace has all info needed to generate the
- // token. So, we could just directly set the attrs,
- // from the backtrace, in incrementToken w/o ever
- // creating Token; we'd have to defer calling freeBefore
- // until after the backtrace was fully "consumed" by
- // incrementToken.
-
- while (pos > lastBackTracePos) {
- // System.out.println("BT: back pos=" + pos + " bestIDX=" + bestIDX);
- final Position posData = positions.get(pos);
- assert bestIDX < posData.count;
-
- int backPos = posData.backPos[bestIDX];
- assert backPos >= lastBackTracePos
- : "backPos=" + backPos + " vs lastBackTracePos=" + lastBackTracePos;
- int length = pos - backPos;
- TokenType backType = posData.backType[bestIDX];
- int backID = posData.backID[bestIDX];
- int nextBestIDX = posData.backIndex[bestIDX];
-
- if (searchMode && altToken == null && backType != TokenType.USER) {
-
- // In searchMode, if best path had picked a too-long
- // token, we use the "penalty" to compute the allowed
- // max cost of an alternate back-trace. If we find an
- // alternate back trace with cost below that
- // threshold, we pursue it instead (but also output
- // the long token).
- // System.out.println(" 2nd best backPos=" + backPos + " pos=" + pos);
-
- final int penalty = computeSecondBestThreshold(backPos, pos - backPos);
-
- if (penalty > 0) {
- if (VERBOSE) {
- System.out.println(
- " compound="
- + new String(buffer.get(backPos, pos - backPos))
- + " backPos="
- + backPos
- + " pos="
- + pos
- + " penalty="
- + penalty
- + " cost="
- + posData.costs[bestIDX]
- + " bestIDX="
- + bestIDX
- + " lastLeftID="
- + lastLeftWordID);
- }
-
- // Use the penalty to set maxCost on the 2nd best
- // segmentation:
- int maxCost = posData.costs[bestIDX] + penalty;
- if (lastLeftWordID != -1) {
- maxCost += costs.get(getDict(backType).getRightId(backID), lastLeftWordID);
- }
-
- // Now, prune all too-long tokens from the graph:
- pruneAndRescore(backPos, pos, posData.backIndex[bestIDX]);
-
- // Finally, find 2nd best back-trace and resume
- // backtrace there:
- int leastCost = Integer.MAX_VALUE;
- int leastIDX = -1;
- for (int idx = 0; idx < posData.count; idx++) {
- int cost = posData.costs[idx];
- // System.out.println(" idx=" + idx + " prevCost=" + cost);
-
- if (lastLeftWordID != -1) {
- cost +=
- costs.get(
- getDict(posData.backType[idx]).getRightId(posData.backID[idx]),
- lastLeftWordID);
- // System.out.println(" += bgCost=" +
- // costs.get(getDict(posData.backType[idx]).getRightId(posData.backID[idx]),
- // lastLeftWordID) + " -> " + cost);
- }
- // System.out.println("penalty " + posData.backPos[idx] + " to " + pos);
- // cost += computePenalty(posData.backPos[idx], pos - posData.backPos[idx]);
- if (cost < leastCost) {
- // System.out.println(" ** ");
- leastCost = cost;
- leastIDX = idx;
- }
- }
- // System.out.println(" leastIDX=" + leastIDX);
-
- if (VERBOSE) {
- System.out.println(
- " afterPrune: "
- + posData.count
- + " arcs arriving; leastCost="
- + leastCost
- + " vs threshold="
- + maxCost
- + " lastLeftWordID="
- + lastLeftWordID);
- }
-
- if (leastIDX != -1 && leastCost <= maxCost && posData.backPos[leastIDX] != backPos) {
- // We should have pruned the altToken from the graph:
- assert posData.backPos[leastIDX] != backPos;
-
- // Save the current compound token, to output when
- // this alternate path joins back:
- altToken =
- new Token(
- fragment,
- backPos - lastBackTracePos,
- length,
- backPos,
- backPos + length,
- backID,
- backType,
- getDict(backType).getMorphAttributes());
-
- // Redirect our backtrace to 2nd best:
- bestIDX = leastIDX;
- nextBestIDX = posData.backIndex[bestIDX];
-
- backPos = posData.backPos[bestIDX];
- length = pos - backPos;
- backType = posData.backType[bestIDX];
- backID = posData.backID[bestIDX];
- backCount = 0;
- // System.out.println(" do alt token!");
-
- } else {
- // I think in theory it's possible there is no
- // 2nd best path, which is fine; in this case we
- // only output the compound token:
- // System.out.println(" no alt token! bestIDX=" + bestIDX);
- }
- }
- }
-
- final int offset = backPos - lastBackTracePos;
- assert offset >= 0;
-
- if (altToken != null && altToken.getStartOffset() >= backPos) {
- if (outputCompounds) {
- // We've backtraced to the position where the
- // compound token starts; add it now:
-
- // The pruning we did when we created the altToken
- // ensures that the back trace will align back with
- // the start of the altToken:
- assert altToken.getStartOffset() == backPos
- : altToken.getStartOffset() + " vs " + backPos;
-
- // NOTE: not quite right: the compound token may
- // have had all punctuation back traced so far, but
- // then the decompounded token at this position is
- // not punctuation. In this case backCount is 0,
- // but we should maybe add the altToken anyway...?
-
- if (backCount > 0) {
- backCount++;
- altToken.setPositionLength(backCount);
- if (VERBOSE) {
- System.out.println(" add altToken=" + altToken);
- }
- pending.add(altToken);
- } else {
- // This means alt token was all punct tokens:
- if (VERBOSE) {
- System.out.println(" discard all-punctuation altToken=" + altToken);
- }
- assert discardPunctuation;
- }
- }
- altToken = null;
- }
-
- final Dictionary<? extends JaMorphData> dict = getDict(backType);
-
- if (backType == TokenType.USER) {
-
- // Expand the phraseID we recorded into the actual
- // segmentation:
- final int[] wordIDAndLength = userDictionary.lookupSegmentation(backID);
- int wordID = wordIDAndLength[0];
- int current = 0;
- for (int j = 1; j < wordIDAndLength.length; j++) {
- final int len = wordIDAndLength[j];
- // System.out.println(" add user: len=" + len);
- int startOffset = current + backPos;
- pending.add(
- new Token(
- fragment,
- current + offset,
- len,
- startOffset,
- startOffset + len,
- wordID + j - 1,
- TokenType.USER,
- dict.getMorphAttributes()));
- if (VERBOSE) {
- System.out.println(" add USER token=" + pending.get(pending.size() - 1));
- }
- current += len;
- }
-
- // Reverse the tokens we just added, because when we
- // serve them up from incrementToken we serve in
- // reverse:
- Collections.reverse(
- pending.subList(pending.size() - (wordIDAndLength.length - 1), pending.size()));
-
- backCount += wordIDAndLength.length - 1;
- } else {
-
- if (extendedMode && backType == TokenType.UNKNOWN) {
- // In EXTENDED mode we convert unknown word into
- // unigrams:
- int unigramTokenCount = 0;
- for (int i = length - 1; i >= 0; i--) {
- int charLen = 1;
- if (i > 0 && Character.isLowSurrogate(fragment[offset + i])) {
- i--;
- charLen = 2;
- }
- // System.out.println(" extended tok offset="
- // + (offset + i));
- if (!discardPunctuation || !isPunctuation(fragment[offset + i])) {
- int startOffset = backPos + i;
- pending.add(
- new Token(
- fragment,
- offset + i,
- charLen,
- startOffset,
- startOffset + charLen,
- CharacterDefinition.NGRAM,
- TokenType.UNKNOWN,
- unkDictionary.getMorphAttributes()));
- unigramTokenCount++;
- }
- }
- backCount += unigramTokenCount;
-
- } else if (!discardPunctuation || length == 0 || !isPunctuation(fragment[offset])) {
- pending.add(
- new Token(
- fragment,
- offset,
- length,
- backPos,
- backPos + length,
- backID,
- backType,
- dict.getMorphAttributes()));
- if (VERBOSE) {
- System.out.println(" add token=" + pending.get(pending.size() - 1));
- }
- backCount++;
- } else {
- if (VERBOSE) {
- System.out.println(
- " skip punctuation token=" + new String(fragment, offset, length));
- }
- }
- }
-
- lastLeftWordID = dict.getLeftId(backID);
- pos = backPos;
- bestIDX = nextBestIDX;
- }
-
- lastBackTracePos = endPos;
-
- if (VERBOSE) {
- System.out.println(" freeBefore pos=" + endPos);
- }
- // Notify the circular buffers that we are done with
- // these positions:
- buffer.freeBefore(endPos);
- positions.freeBefore(endPos);
- }
-
- Dictionary<? extends JaMorphData> getDict(TokenType type) {
- return dictionaryMap.get(type);
- }
-
- private static boolean isPunctuation(char ch) {
- switch (Character.getType(ch)) {
- case Character.SPACE_SEPARATOR:
- case Character.LINE_SEPARATOR:
- case Character.PARAGRAPH_SEPARATOR:
- case Character.CONTROL:
- case Character.FORMAT:
- case Character.DASH_PUNCTUATION:
- case Character.START_PUNCTUATION:
- case Character.END_PUNCTUATION:
- case Character.CONNECTOR_PUNCTUATION:
- case Character.OTHER_PUNCTUATION:
- case Character.MATH_SYMBOL:
- case Character.CURRENCY_SYMBOL:
- case Character.MODIFIER_SYMBOL:
- case Character.OTHER_SYMBOL:
- case Character.INITIAL_QUOTE_PUNCTUATION:
- case Character.FINAL_QUOTE_PUNCTUATION:
- return true;
- default:
- return false;
- }
+ viterbi.setNBestCost(value);
}
}
diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/ViterbiNBest.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/ViterbiNBest.java
new file mode 100644
index 00000000000..d789947e784
--- /dev/null
+++ b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/ViterbiNBest.java
@@ -0,0 +1,1253 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.analysis.ja;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.EnumMap;
+import java.util.HashMap;
+import java.util.List;
+import org.apache.lucene.analysis.ja.dict.CharacterDefinition;
+import org.apache.lucene.analysis.ja.dict.JaMorphData;
+import org.apache.lucene.analysis.ja.dict.TokenInfoDictionary;
+import org.apache.lucene.analysis.ja.dict.UnknownDictionary;
+import org.apache.lucene.analysis.ja.dict.UserDictionary;
+import org.apache.lucene.analysis.morph.ConnectionCosts;
+import org.apache.lucene.analysis.morph.Dictionary;
+import org.apache.lucene.analysis.morph.GraphvizFormatter;
+import org.apache.lucene.analysis.morph.TokenInfoFST;
+import org.apache.lucene.analysis.morph.TokenType;
+import org.apache.lucene.analysis.morph.Viterbi;
+import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.fst.FST;
+
+/**
+ * {@link org.apache.lucene.analysis.morph.Viterbi} subclass for Japanese morphological analysis.
+ * This also performs n-best path calculation
+ */
+final class ViterbiNBest
+ extends org.apache.lucene.analysis.morph.Viterbi<Token, ViterbiNBest.PositionNBest> {
+
+ private final EnumMap<TokenType, Dictionary<? extends JaMorphData>> dictionaryMap =
+ new EnumMap<>(TokenType.class);
+
+ private final UnknownDictionary unkDictionary;
+ private final CharacterDefinition characterDefinition;
+ private final UserDictionary userDictionary;
+
+ private final boolean discardPunctuation;
+ private final boolean searchMode;
+ private final boolean extendedMode;
+ private final boolean outputCompounds;
+
+ // Allowable cost difference for N-best output:
+ private int nBestCost = 0;
+
+ Lattice lattice = null;
+
+ private GraphvizFormatter<JaMorphData> dotOut;
+
+ ViterbiNBest(
+ TokenInfoFST fst,
+ FST.BytesReader fstReader,
+ TokenInfoDictionary dictionary,
+ TokenInfoFST userFST,
+ FST.BytesReader userFSTReader,
+ UserDictionary userDictionary,
+ ConnectionCosts costs,
+ Class<PositionNBest> positionImpl,
+ UnknownDictionary unkDictionary,
+ CharacterDefinition characterDefinition,
+ boolean discardPunctuation,
+ boolean searchMode,
+ boolean extendedMode,
+ boolean outputCompounds) {
+ super(fst, fstReader, dictionary, userFST, userFSTReader, userDictionary, costs, positionImpl);
+ this.unkDictionary = unkDictionary;
+ this.characterDefinition = characterDefinition;
+ this.userDictionary = userDictionary;
+ this.discardPunctuation = discardPunctuation;
+ this.searchMode = searchMode;
+ this.extendedMode = extendedMode;
+ this.outputCompounds = outputCompounds;
+ dictionaryMap.put(TokenType.KNOWN, dictionary);
+ dictionaryMap.put(TokenType.UNKNOWN, unkDictionary);
+ dictionaryMap.put(TokenType.USER, userDictionary);
+ }
+
+ @Override
+ protected boolean shouldSkipProcessUnknownWord(int unknownWordEndIndex, Position posData) {
+ return !searchMode && super.shouldSkipProcessUnknownWord(unknownWordEndIndex, posData);
+ }
+
+ private static final int SEARCH_MODE_KANJI_LENGTH = 2;
+ private static final int SEARCH_MODE_OTHER_LENGTH = 7; // Must be >= SEARCH_MODE_KANJI_LENGTH
+ private static final int SEARCH_MODE_KANJI_PENALTY = 3000;
+ private static final int SEARCH_MODE_OTHER_PENALTY = 1700;
+
+ @Override
+ protected int computePenalty(int pos, int length) throws IOException {
+ if (length > SEARCH_MODE_KANJI_LENGTH) {
+ boolean allKanji = true;
+ // check if node consists of only kanji
+ final int endPos = pos + length;
+ for (int pos2 = pos; pos2 < endPos; pos2++) {
+ if (!characterDefinition.isKanji((char) buffer.get(pos2))) {
+ allKanji = false;
+ break;
+ }
+ }
+ if (allKanji) { // Process only Kanji keywords
+ return (length - SEARCH_MODE_KANJI_LENGTH) * SEARCH_MODE_KANJI_PENALTY;
+ } else if (length > SEARCH_MODE_OTHER_LENGTH) {
+ return (length - SEARCH_MODE_OTHER_LENGTH) * SEARCH_MODE_OTHER_PENALTY;
+ }
+ }
+ return 0;
+ }
+
+ // Returns the added cost that a 2nd best segmentation is
+ // allowed to have. Ie, if we see path with cost X,
+ // ending in a compound word, and this method returns
+ // threshold > 0, then we will also find the 2nd best
+ // segmentation and if its path score is within this
+ // threshold of X, we'll include it in the output:
+ private int computeSecondBestThreshold(int pos, int length) throws IOException {
+ // TODO: maybe we do something else here, instead of just
+ // using the penalty...? EG we can be more aggressive on
+ // when to also test for 2nd best path
+ return computePenalty(pos, length);
+ }
+
+ @Override
+ protected int processUnknownWord(boolean anyMatches, Position posData) throws IOException {
+ final char firstCharacter = (char) buffer.get(pos);
+ if (!anyMatches || characterDefinition.isInvoke(firstCharacter)) {
+
+ // Find unknown match:
+ final int characterId = characterDefinition.getCharacterClass(firstCharacter);
+ final boolean isPunct = isPunctuation(firstCharacter);
+
+ // NOTE: copied from UnknownDictionary.lookup:
+ int unknownWordLength;
+ if (!characterDefinition.isGroup(firstCharacter)) {
+ unknownWordLength = 1;
+ } else {
+ // Extract unknown word. Characters with the same character class are considered to be
+ // part of unknown word
+ unknownWordLength = 1;
+ for (int posAhead = pos + 1; unknownWordLength < MAX_UNKNOWN_WORD_LENGTH; posAhead++) {
+ final int ch = buffer.get(posAhead);
+ if (ch == -1) {
+ break;
+ }
+ if (characterId == characterDefinition.getCharacterClass((char) ch)
+ && isPunctuation((char) ch) == isPunct) {
+ unknownWordLength++;
+ } else {
+ break;
+ }
+ }
+ }
+
+ unkDictionary.lookupWordIds(
+ characterId, wordIdRef); // characters in input text are supposed to be the same
+ if (VERBOSE) {
+ System.out.println(
+ " UNKNOWN word len=" + unknownWordLength + " " + wordIdRef.length + " wordIDs");
+ }
+ for (int ofs = 0; ofs < wordIdRef.length; ofs++) {
+ add(
+ unkDictionary.getMorphAttributes(),
+ posData,
+ pos,
+ posData.getPos() + unknownWordLength,
+ wordIdRef.ints[wordIdRef.offset + ofs],
+ TokenType.UNKNOWN,
+ false);
+ }
+
+ return unknownWordLength;
+ }
+ return 0;
+ }
+
+ void setGraphvizFormatter(GraphvizFormatter<JaMorphData> dotOut) {
+ this.dotOut = dotOut;
+ }
+
+ @Override
+ protected void backtrace(Position endPosData, int fromIDX) throws IOException {
+ final int endPos = endPosData.getPos();
+
+ /**
+ * LUCENE-10059: If the endPos is the same as lastBackTracePos, we don't want to backtrace to
+ * avoid an assertion error {@link RollingCharBuffer#get(int)} when it tries to generate an
+ * empty buffer
+ */
+ if (endPos == lastBackTracePos) {
+ return;
+ }
+
+ if (VERBOSE) {
+ System.out.println(
+ "\n backtrace: endPos="
+ + endPos
+ + " pos="
+ + pos
+ + "; "
+ + (pos - lastBackTracePos)
+ + " characters; last="
+ + lastBackTracePos
+ + " cost="
+ + endPosData.getCost(fromIDX));
+ }
+
+ final char[] fragment = buffer.get(lastBackTracePos, endPos - lastBackTracePos);
+
+ if (dotOut != null) {
+ dotOut.onBacktrace(
+ this::getDict, positions, lastBackTracePos, endPosData, fromIDX, fragment, end);
+ }
+
+ int pos = endPos;
+ int bestIDX = fromIDX;
+ Token altToken = null;
+
+ // We trace backwards, so this will be the leftWordID of
+ // the token after the one we are now on:
+ int lastLeftWordID = -1;
+
+ int backCount = 0;
+
+ // TODO: sort of silly to make Token instances here; the
+ // back trace has all info needed to generate the
+ // token. So, we could just directly set the attrs,
+ // from the backtrace, in incrementToken w/o ever
+ // creating Token; we'd have to defer calling freeBefore
+ // until after the backtrace was fully "consumed" by
+ // incrementToken.
+
+ while (pos > lastBackTracePos) {
+ // System.out.println("BT: back pos=" + pos + " bestIDX=" + bestIDX);
+ final Position posData = positions.get(pos);
+ assert bestIDX < posData.getCount();
+
+ int backPos = posData.getBackPos(bestIDX);
+ assert backPos >= lastBackTracePos
+ : "backPos=" + backPos + " vs lastBackTracePos=" + lastBackTracePos;
+ int length = pos - backPos;
+ TokenType backType = posData.getBackType(bestIDX);
+ int backID = posData.getBackID(bestIDX);
+ int nextBestIDX = posData.getBackIndex(bestIDX);
+
+ if (searchMode && altToken == null && backType != TokenType.USER) {
+
+ // In searchMode, if best path had picked a too-long
+ // token, we use the "penalty" to compute the allowed
+ // max cost of an alternate back-trace. If we find an
+ // alternate back trace with cost below that
+ // threshold, we pursue it instead (but also output
+ // the long token).
+ // System.out.println(" 2nd best backPos=" + backPos + " pos=" + pos);
+
+ final int penalty = computeSecondBestThreshold(backPos, pos - backPos);
+
+ if (penalty > 0) {
+ if (VERBOSE) {
+ System.out.println(
+ " compound="
+ + new String(buffer.get(backPos, pos - backPos))
+ + " backPos="
+ + backPos
+ + " pos="
+ + pos
+ + " penalty="
+ + penalty
+ + " cost="
+ + posData.getCost(bestIDX)
+ + " bestIDX="
+ + bestIDX
+ + " lastLeftID="
+ + lastLeftWordID);
+ }
+
+ // Use the penalty to set maxCost on the 2nd best
+ // segmentation:
+ int maxCost = posData.getCost(bestIDX) + penalty;
+ if (lastLeftWordID != -1) {
+ maxCost += costs.get(getDict(backType).getRightId(backID), lastLeftWordID);
+ }
+
+ // Now, prune all too-long tokens from the graph:
+ pruneAndRescore(backPos, pos, posData.getBackIndex(bestIDX));
+
+ // Finally, find 2nd best back-trace and resume
+ // backtrace there:
+ int leastCost = Integer.MAX_VALUE;
+ int leastIDX = -1;
+ for (int idx = 0; idx < posData.getCount(); idx++) {
+ int cost = posData.getCost(idx);
+ // System.out.println(" idx=" + idx + " prevCost=" + cost);
+
+ if (lastLeftWordID != -1) {
+ cost +=
+ costs.get(
+ getDict(posData.getBackType(idx)).getRightId(posData.getBackID(idx)),
+ lastLeftWordID);
+ // System.out.println(" += bgCost=" +
+ // costs.get(getDict(posData.backType[idx]).getRightId(posData.backID[idx]),
+ // lastLeftWordID) + " -> " + cost);
+ }
+ // System.out.println("penalty " + posData.backPos[idx] + " to " + pos);
+ // cost += computePenalty(posData.backPos[idx], pos - posData.backPos[idx]);
+ if (cost < leastCost) {
+ // System.out.println(" ** ");
+ leastCost = cost;
+ leastIDX = idx;
+ }
+ }
+ // System.out.println(" leastIDX=" + leastIDX);
+
+ if (VERBOSE) {
+ System.out.println(
+ " afterPrune: "
+ + posData.getCount()
+ + " arcs arriving; leastCost="
+ + leastCost
+ + " vs threshold="
+ + maxCost
+ + " lastLeftWordID="
+ + lastLeftWordID);
+ }
+
+ if (leastIDX != -1 && leastCost <= maxCost && posData.getBackPos(leastIDX) != backPos) {
+ // We should have pruned the altToken from the graph:
+ assert posData.getBackPos(leastIDX) != backPos;
+
+ // Save the current compound token, to output when
+ // this alternate path joins back:
+ altToken =
+ new Token(
+ fragment,
+ backPos - lastBackTracePos,
+ length,
+ backPos,
+ backPos + length,
+ backID,
+ backType,
+ getDict(backType).getMorphAttributes());
+
+ // Redirect our backtrace to 2nd best:
+ bestIDX = leastIDX;
+ nextBestIDX = posData.getBackIndex(bestIDX);
+
+ backPos = posData.getBackPos(bestIDX);
+ length = pos - backPos;
+ backType = posData.getBackType(bestIDX);
+ backID = posData.getBackID(bestIDX);
+ backCount = 0;
+ // System.out.println(" do alt token!");
+
+ } else {
+ // I think in theory it's possible there is no
+ // 2nd best path, which is fine; in this case we
+ // only output the compound token:
+ // System.out.println(" no alt token! bestIDX=" + bestIDX);
+ }
+ }
+ }
+
+ final int offset = backPos - lastBackTracePos;
+ assert offset >= 0;
+
+ if (altToken != null && altToken.getStartOffset() >= backPos) {
+ if (outputCompounds) {
+ // We've backtraced to the position where the
+ // compound token starts; add it now:
+
+ // The pruning we did when we created the altToken
+ // ensures that the back trace will align back with
+ // the start of the altToken:
+ assert altToken.getStartOffset() == backPos
+ : altToken.getStartOffset() + " vs " + backPos;
+
+ // NOTE: not quite right: the compound token may
+ // have had all punctuation back traced so far, but
+ // then the decompounded token at this position is
+ // not punctuation. In this case backCount is 0,
+ // but we should maybe add the altToken anyway...?
+
+ if (backCount > 0) {
+ backCount++;
+ altToken.setPositionLength(backCount);
+ if (VERBOSE) {
+ System.out.println(" add altToken=" + altToken);
+ }
+ pending.add(altToken);
+ } else {
+ // This means alt token was all punct tokens:
+ if (VERBOSE) {
+ System.out.println(" discard all-punctuation altToken=" + altToken);
+ }
+ assert discardPunctuation;
+ }
+ }
+ altToken = null;
+ }
+
+ final Dictionary<? extends JaMorphData> dict = getDict(backType);
+
+ if (backType == TokenType.USER) {
+
+ // Expand the phraseID we recorded into the actual
+ // segmentation:
+ final int[] wordIDAndLength = userDictionary.lookupSegmentation(backID);
+ int wordID = wordIDAndLength[0];
+ int current = 0;
+ for (int j = 1; j < wordIDAndLength.length; j++) {
+ final int len = wordIDAndLength[j];
+ // System.out.println(" add user: len=" + len);
+ int startOffset = current + backPos;
+ pending.add(
+ new Token(
+ fragment,
+ current + offset,
+ len,
+ startOffset,
+ startOffset + len,
+ wordID + j - 1,
+ TokenType.USER,
+ dict.getMorphAttributes()));
+ if (VERBOSE) {
+ System.out.println(" add USER token=" + pending.get(pending.size() - 1));
+ }
+ current += len;
+ }
+
+ // Reverse the tokens we just added, because when we
+ // serve them up from incrementToken we serve in
+ // reverse:
+ Collections.reverse(
+ pending.subList(pending.size() - (wordIDAndLength.length - 1), pending.size()));
+
+ backCount += wordIDAndLength.length - 1;
+ } else {
+
+ if (extendedMode && backType == TokenType.UNKNOWN) {
+ // In EXTENDED mode we convert unknown word into
+ // unigrams:
+ int unigramTokenCount = 0;
+ for (int i = length - 1; i >= 0; i--) {
+ int charLen = 1;
+ if (i > 0 && Character.isLowSurrogate(fragment[offset + i])) {
+ i--;
+ charLen = 2;
+ }
+ // System.out.println(" extended tok offset="
+ // + (offset + i));
+ if (!discardPunctuation || !isPunctuation(fragment[offset + i])) {
+ int startOffset = backPos + i;
+ pending.add(
+ new Token(
+ fragment,
+ offset + i,
+ charLen,
+ startOffset,
+ startOffset + charLen,
+ CharacterDefinition.NGRAM,
+ TokenType.UNKNOWN,
+ unkDictionary.getMorphAttributes()));
+ unigramTokenCount++;
+ }
+ }
+ backCount += unigramTokenCount;
+
+ } else if (!discardPunctuation || length == 0 || !isPunctuation(fragment[offset])) {
+ pending.add(
+ new Token(
+ fragment,
+ offset,
+ length,
+ backPos,
+ backPos + length,
+ backID,
+ backType,
+ dict.getMorphAttributes()));
+ if (VERBOSE) {
+ System.out.println(" add token=" + pending.get(pending.size() - 1));
+ }
+ backCount++;
+ } else {
+ if (VERBOSE) {
+ System.out.println(
+ " skip punctuation token=" + new String(fragment, offset, length));
+ }
+ }
+ }
+
+ lastLeftWordID = dict.getLeftId(backID);
+ pos = backPos;
+ bestIDX = nextBestIDX;
+ }
+
+ lastBackTracePos = endPos;
+
+ if (VERBOSE) {
+ System.out.println(" freeBefore pos=" + endPos);
+ }
+ // Notify the circular buffers that we are done with
+ // these positions:
+ buffer.freeBefore(endPos);
+ positions.freeBefore(endPos);
+ }
+
+ // Eliminates arcs from the lattice that are compound
+ // tokens (have a penalty) or are not congruent with the
+ // compound token we've matched (ie, span across the
+ // startPos). This should be fairly efficient, because we
+ // just keep the already intersected structure of the
+ // graph, eg we don't have to consult the FSTs again:
+ private void pruneAndRescore(int startPos, int endPos, int bestStartIDX) throws IOException {
+ if (VERBOSE) {
+ System.out.println(
+ " pruneAndRescore startPos="
+ + startPos
+ + " endPos="
+ + endPos
+ + " bestStartIDX="
+ + bestStartIDX);
+ }
+
+ // First pass: walk backwards, building up the forward
+ // arcs and pruning inadmissible arcs:
+ for (int pos = endPos; pos > startPos; pos--) {
+ final Position posData = positions.get(pos);
+ if (VERBOSE) {
+ System.out.println(" back pos=" + pos);
+ }
+ for (int arcIDX = 0; arcIDX < posData.getCount(); arcIDX++) {
+ final int backPos = posData.getBackPos(arcIDX);
+ if (backPos >= startPos) {
+ // Keep this arc:
+ // System.out.println(" keep backPos=" + backPos);
+ positions
+ .get(backPos)
+ .addForward(pos, arcIDX, posData.getBackID(arcIDX), posData.getBackType(arcIDX));
+ } else {
+ if (VERBOSE) {
+ System.out.println(" prune");
+ }
+ }
+ }
+ if (pos != startPos) {
+ posData.setCount(0);
+ }
+ }
+
+ // Second pass: walk forward, re-scoring:
+ for (int pos = startPos; pos < endPos; pos++) {
+ final PositionNBest posData = positions.get(pos);
+ if (VERBOSE) {
+ System.out.println(" forward pos=" + pos + " count=" + posData.forwardCount);
+ }
+ if (posData.getCount() == 0) {
+ // No arcs arrive here...
+ if (VERBOSE) {
+ System.out.println(" skip");
+ }
+ posData.forwardCount = 0;
+ continue;
+ }
+
+ if (pos == startPos) {
+ // On the initial position, only consider the best
+ // path so we "force congruence": the
+ // sub-segmentation is "in context" of what the best
+ // path (compound token) had matched:
+ final int rightID;
+ if (startPos == 0) {
+ rightID = 0;
+ } else {
+ rightID =
+ getDict(posData.getBackType(bestStartIDX))
+ .getRightId(posData.getBackID(bestStartIDX));
+ }
+ final int pathCost = posData.getCost(bestStartIDX);
+ for (int forwardArcIDX = 0; forwardArcIDX < posData.forwardCount; forwardArcIDX++) {
+ final TokenType forwardType = posData.forwardType[forwardArcIDX];
+ final Dictionary<? extends JaMorphData> dict2 = getDict(forwardType);
+ final int wordID = posData.forwardID[forwardArcIDX];
+ final int toPos = posData.forwardPos[forwardArcIDX];
+ final int newCost =
+ pathCost
+ + dict2.getWordCost(wordID)
+ + costs.get(rightID, dict2.getLeftId(wordID))
+ + computePenalty(pos, toPos - pos);
+ if (VERBOSE) {
+ System.out.println(
+ " + "
+ + forwardType
+ + " word "
+ + new String(buffer.get(pos, toPos - pos))
+ + " toPos="
+ + toPos
+ + " cost="
+ + newCost
+ + " penalty="
+ + computePenalty(pos, toPos - pos)
+ + " toPos.idx="
+ + positions.get(toPos).getCount());
+ }
+ positions
+ .get(toPos)
+ .add(newCost, dict2.getRightId(wordID), pos, -1, bestStartIDX, wordID, forwardType);
+ }
+ } else {
+ // On non-initial positions, we maximize score
+ // across all arriving lastRightIDs:
+ for (int forwardArcIDX = 0; forwardArcIDX < posData.forwardCount; forwardArcIDX++) {
+ final TokenType forwardType = posData.forwardType[forwardArcIDX];
+ final int toPos = posData.forwardPos[forwardArcIDX];
+ if (VERBOSE) {
+ System.out.println(
+ " + "
+ + forwardType
+ + " word "
+ + new String(buffer.get(pos, toPos - pos))
+ + " toPos="
+ + toPos);
+ }
+ add(
+ getDict(forwardType).getMorphAttributes(),
+ posData,
+ pos,
+ toPos,
+ posData.forwardID[forwardArcIDX],
+ forwardType,
+ true);
+ }
+ }
+ posData.forwardCount = 0;
+ }
+ }
+
+ @Override
+ protected void backtraceNBest(final Position endPosData, final boolean useEOS)
+ throws IOException {
+ if (lattice == null) {
+ lattice = new Lattice();
+ }
+
+ final int endPos = endPosData.getPos();
+ char[] fragment = buffer.get(lastBackTracePos, endPos - lastBackTracePos);
+ lattice.setup(fragment, dictionaryMap, positions, lastBackTracePos, endPos, useEOS);
+ lattice.markUnreachable();
+ lattice.calcLeftCost(costs);
+ lattice.calcRightCost(costs);
+
+ int bestCost = lattice.bestCost();
+ if (VERBOSE) {
+ System.out.printf("DEBUG: 1-BEST COST: %d\n", bestCost);
+ }
+ for (int node : lattice.bestPathNodeList()) {
+ registerNode(node, fragment);
+ }
+
+ for (int n = 2; ; ++n) {
+ List<Integer> nbest = lattice.nBestNodeList(n);
+ if (nbest.isEmpty()) {
+ break;
+ }
+ int cost = lattice.cost(nbest.get(0));
+ if (VERBOSE) {
+ System.out.printf("DEBUG: %d-BEST COST: %d\n", n, cost);
+ }
+ if (bestCost + nBestCost < cost) {
+ break;
+ }
+ for (int node : nbest) {
+ registerNode(node, fragment);
+ }
+ }
+ if (VERBOSE) {
+ lattice.debugPrint();
+ }
+ }
+
+ private void registerNode(int node, char[] fragment) {
+ int left = lattice.nodeLeft[node];
+ int right = lattice.nodeRight[node];
+ TokenType type = lattice.nodeDicType[node];
+ if (!discardPunctuation || !isPunctuation(fragment[left])) {
+ if (type == TokenType.USER) {
+ // The code below are based on backtrace().
+ //
+ // Expand the phraseID we recorded into the actual segmentation:
+ final int[] wordIDAndLength = userDictionary.lookupSegmentation(lattice.nodeWordID[node]);
+ int wordID = wordIDAndLength[0];
+ pending.add(
+ new Token(
+ fragment,
+ left,
+ right - left,
+ lattice.rootBase + left,
+ lattice.rootBase + right,
+ wordID,
+ TokenType.USER,
+ userDictionary.getMorphAttributes()));
+ // Output compound
+ int current = 0;
+ for (int j = 1; j < wordIDAndLength.length; j++) {
+ final int len = wordIDAndLength[j];
+ if (len < right - left) {
+ int startOffset = lattice.rootBase + current + left;
+ pending.add(
+ new Token(
+ fragment,
+ current + left,
+ len,
+ startOffset,
+ startOffset + len,
+ wordID + j - 1,
+ TokenType.USER,
+ userDictionary.getMorphAttributes()));
+ }
+ current += len;
+ }
+ } else {
+ pending.add(
+ new Token(
+ fragment,
+ left,
+ right - left,
+ lattice.rootBase + left,
+ lattice.rootBase + right,
+ lattice.nodeWordID[node],
+ type,
+ getDict(type).getMorphAttributes()));
+ }
+ }
+ }
+
+ @Override
+ protected void fixupPendingList() {
+ // Sort for removing same tokens.
+ // USER token should be ahead from normal one.
+ Collections.sort(
+ pending,
+ (a, b) -> {
+ int aOff = a.getOffset();
+ int bOff = b.getOffset();
+ if (aOff != bOff) {
+ return aOff - bOff;
+ }
+ int aLen = a.getLength();
+ int bLen = b.getLength();
+ if (aLen != bLen) {
+ return aLen - bLen;
+ }
+ // order of Type is KNOWN, UNKNOWN, USER,
+ // so we use reversed comparison here.
+ return b.getType().ordinal() - a.getType().ordinal();
+ });
+
+ // Remove same token.
+ for (int i = 1; i < pending.size(); ++i) {
+ Token a = pending.get(i - 1);
+ Token b = pending.get(i);
+ if (a.getOffset() == b.getOffset() && a.getLength() == b.getLength()) {
+ pending.remove(i);
+ // It is important to decrement "i" here, because a next may be removed.
+ --i;
+ }
+ }
+
+ // offset=>position map
+ HashMap<Integer, Integer> map = new HashMap<>();
+ for (Token t : pending) {
+ map.put(t.getOffset(), 0);
+ map.put(t.getOffset() + t.getLength(), 0);
+ }
+
+ // Get unique and sorted list of all edge position of tokens.
+ Integer[] offsets = map.keySet().toArray(new Integer[0]);
+ Arrays.sort(offsets);
+
+ // setup all value of map. It specifies N-th position from begin.
+ for (int i = 0; i < offsets.length; ++i) {
+ map.put(offsets[i], i);
+ }
+
+ // We got all position length now.
+ for (Token t : pending) {
+ t.setPositionLength(map.get(t.getOffset() + t.getLength()) - map.get(t.getOffset()));
+ }
+
+ // Make PENDING to be reversed order to fit its usage.
+ // If you would like to speedup, you can try reversed order sort
+ // at first of this function.
+ Collections.reverse(pending);
+ }
+
+ Dictionary<? extends JaMorphData> getDict(TokenType type) {
+ return dictionaryMap.get(type);
+ }
+
+ void setNBestCost(int value) {
+ nBestCost = value;
+ outputNBest = 0 < nBestCost;
+ }
+
+ int getNBestCost() {
+ return nBestCost;
+ }
+
+ private static boolean isPunctuation(char ch) {
+ switch (Character.getType(ch)) {
+ case Character.SPACE_SEPARATOR:
+ case Character.LINE_SEPARATOR:
+ case Character.PARAGRAPH_SEPARATOR:
+ case Character.CONTROL:
+ case Character.FORMAT:
+ case Character.DASH_PUNCTUATION:
+ case Character.START_PUNCTUATION:
+ case Character.END_PUNCTUATION:
+ case Character.CONNECTOR_PUNCTUATION:
+ case Character.OTHER_PUNCTUATION:
+ case Character.MATH_SYMBOL:
+ case Character.CURRENCY_SYMBOL:
+ case Character.MODIFIER_SYMBOL:
+ case Character.OTHER_SYMBOL:
+ case Character.INITIAL_QUOTE_PUNCTUATION:
+ case Character.FINAL_QUOTE_PUNCTUATION:
+ return true;
+ default:
+ return false;
+ }
+ }
+
+ /**
+ * {@link org.apache.lucene.analysis.morph.Viterbi.Position} extension; this holds all forward
+ * pointers to calculate n-best path.
+ */
+ public static class PositionNBest extends Viterbi.Position {
+ // Only used when finding 2nd best segmentation under a
+ // too-long token:
+ int forwardCount;
+ int[] forwardPos = new int[8];
+ int[] forwardID = new int[8];
+ int[] forwardIndex = new int[8];
+ TokenType[] forwardType = new TokenType[8];
+
+ private void growForward() {
+ forwardPos = ArrayUtil.grow(forwardPos, 1 + forwardCount);
+ forwardID = ArrayUtil.grow(forwardID, 1 + forwardCount);
+ forwardIndex = ArrayUtil.grow(forwardIndex, 1 + forwardCount);
+
+ // NOTE: sneaky: grow separately because
+ // ArrayUtil.grow will otherwise pick a different
+ // length than the int[]s we just grew:
+ final TokenType[] newForwardType = new TokenType[forwardPos.length];
+ System.arraycopy(forwardType, 0, newForwardType, 0, forwardType.length);
+ forwardType = newForwardType;
+ }
+
+ void addForward(int forwardPos, int forwardIndex, int forwardID, TokenType forwardType) {
+ if (forwardCount == this.forwardID.length) {
+ growForward();
+ }
+ this.forwardPos[forwardCount] = forwardPos;
+ this.forwardIndex[forwardCount] = forwardIndex;
+ this.forwardID[forwardCount] = forwardID;
+ this.forwardType[forwardCount] = forwardType;
+ forwardCount++;
+ }
+
+ @Override
+ public void reset() {
+ super.reset();
+ // forwardCount naturally resets after it runs:
+ assert forwardCount == 0 : "pos=" + getPos() + " forwardCount=" + forwardCount;
+ }
+ }
+
+ // yet another lattice data structure
+ static final class Lattice {
+ char[] fragment;
+ EnumMap<TokenType, Dictionary<? extends JaMorphData>> dictionaryMap;
+ boolean useEOS;
+
+ int rootCapacity = 0;
+ int rootSize = 0;
+ int rootBase = 0;
+
+ // root pointers of node chain by leftChain_ that have same start offset.
+ int[] lRoot;
+ // root pointers of node chain by rightChain_ that have same end offset.
+ int[] rRoot;
+
+ int capacity = 0;
+ int nodeCount = 0;
+
+ // The variables below are elements of lattice node that indexed by node number.
+ TokenType[] nodeDicType;
+ int[] nodeWordID;
+ // nodeMark - -1:excluded, 0:unused, 1:bestpath, 2:2-best-path, ... N:N-best-path
+ int[] nodeMark;
+ int[] nodeLeftID;
+ int[] nodeRightID;
+ int[] nodeWordCost;
+ int[] nodeLeftCost;
+ int[] nodeRightCost;
+ // nodeLeftNode, nodeRightNode - are left/right node number with minimum cost path.
+ int[] nodeLeftNode;
+ int[] nodeRightNode;
+ // nodeLeft, nodeRight - start/end offset
+ int[] nodeLeft;
+ int[] nodeRight;
+ int[] nodeLeftChain;
+ int[] nodeRightChain;
+
+ private void setupRoot(int baseOffset, int lastOffset) {
+ assert baseOffset <= lastOffset;
+ int size = lastOffset - baseOffset + 1;
+ if (rootCapacity < size) {
+ int oversize = ArrayUtil.oversize(size, Integer.BYTES);
+ lRoot = new int[oversize];
+ rRoot = new int[oversize];
+ rootCapacity = oversize;
+ }
+ Arrays.fill(lRoot, 0, size, -1);
+ Arrays.fill(rRoot, 0, size, -1);
+ rootSize = size;
+ rootBase = baseOffset;
+ }
+
+ // Reserve at least N nodes.
+ private void reserve(int n) {
+ if (capacity < n) {
+ int oversize = ArrayUtil.oversize(n, Integer.BYTES);
+ nodeDicType = new TokenType[oversize];
+ nodeWordID = new int[oversize];
+ nodeMark = new int[oversize];
+ nodeLeftID = new int[oversize];
+ nodeRightID = new int[oversize];
+ nodeWordCost = new int[oversize];
+ nodeLeftCost = new int[oversize];
+ nodeRightCost = new int[oversize];
+ nodeLeftNode = new int[oversize];
+ nodeRightNode = new int[oversize];
+ nodeLeft = new int[oversize];
+ nodeRight = new int[oversize];
+ nodeLeftChain = new int[oversize];
+ nodeRightChain = new int[oversize];
+ capacity = oversize;
+ }
+ }
+
+ private void setupNodePool(int n) {
+ reserve(n);
+ nodeCount = 0;
+ if (VERBOSE) {
+ System.out.printf("DEBUG: setupNodePool: n = %d\n", n);
+ System.out.printf("DEBUG: setupNodePool: lattice.capacity = %d\n", capacity);
+ }
+ }
+
+ private int addNode(TokenType dicType, int wordID, int left, int right) {
+ if (VERBOSE) {
+ System.out.printf(
+ "DEBUG: addNode: dicType=%s, wordID=%d, left=%d, right=%d, str=%s\n",
+ dicType.toString(),
+ wordID,
+ left,
+ right,
+ left == -1 ? "BOS" : right == -1 ? "EOS" : new String(fragment, left, right - left));
+ }
+ assert nodeCount < capacity;
+ assert left == -1 || right == -1 || left < right;
+ assert left == -1 || (0 <= left && left < rootSize);
+ assert right == -1 || (0 <= right && right < rootSize);
+
+ int node = nodeCount++;
+
+ if (VERBOSE) {
+ System.out.printf("DEBUG: addNode: node=%d\n", node);
+ }
+
+ nodeDicType[node] = dicType;
+ nodeWordID[node] = wordID;
+ nodeMark[node] = 0;
+
+ if (wordID < 0) {
+ nodeWordCost[node] = 0;
+ nodeLeftCost[node] = 0;
+ nodeRightCost[node] = 0;
+ nodeLeftID[node] = 0;
+ nodeRightID[node] = 0;
+ } else {
+ Dictionary<? extends JaMorphData> dic = dictionaryMap.get(dicType);
+ nodeWordCost[node] = dic.getWordCost(wordID);
+ nodeLeftID[node] = dic.getLeftId(wordID);
+ nodeRightID[node] = dic.getRightId(wordID);
+ }
+
+ if (VERBOSE) {
+ System.out.printf(
+ "DEBUG: addNode: wordCost=%d, leftID=%d, rightID=%d\n",
+ nodeWordCost[node], nodeLeftID[node], nodeRightID[node]);
+ }
+
+ nodeLeft[node] = left;
+ nodeRight[node] = right;
+ if (0 <= left) {
+ nodeLeftChain[node] = lRoot[left];
+ lRoot[left] = node;
+ } else {
+ nodeLeftChain[node] = -1;
+ }
+ if (0 <= right) {
+ nodeRightChain[node] = rRoot[right];
+ rRoot[right] = node;
+ } else {
+ nodeRightChain[node] = -1;
+ }
+ return node;
+ }
+
+ // Sum of positions.get(i).count in [beg, end) range.
+ // using stream:
+ // return IntStream.range(beg, end).map(i -> positions.get(i).count).sum();
+ private int positionCount(WrappedPositionArray<PositionNBest> positions, int beg, int end) {
+ int count = 0;
+ for (int i = beg; i < end; ++i) {
+ count += positions.get(i).getCount();
+ }
+ return count;
+ }
+
+ void setup(
+ char[] fragment,
+ EnumMap<TokenType, Dictionary<? extends JaMorphData>> dictionaryMap,
+ WrappedPositionArray<PositionNBest> positions,
+ int prevOffset,
+ int endOffset,
+ boolean useEOS) {
+ assert positions.get(prevOffset).getCount() == 1;
+ if (VERBOSE) {
+ System.out.printf("DEBUG: setup: prevOffset=%d, endOffset=%d\n", prevOffset, endOffset);
+ }
+
+ this.fragment = fragment;
+ this.dictionaryMap = dictionaryMap;
+ this.useEOS = useEOS;
+
+ // Initialize lRoot and rRoot.
+ setupRoot(prevOffset, endOffset);
+
+ // "+ 2" for first/last record.
+ setupNodePool(positionCount(positions, prevOffset + 1, endOffset + 1) + 2);
+
+ // substitute for BOS = 0
+ Position first = positions.get(prevOffset);
+ if (addNode(first.getBackType(0), first.getBackID(0), -1, 0) != 0) {
+ assert false;
+ }
+
+ // EOS = 1
+ if (addNode(TokenType.KNOWN, -1, endOffset - rootBase, -1) != 1) {
+ assert false;
+ }
+
+ for (int offset = endOffset; prevOffset < offset; --offset) {
+ int right = offset - rootBase;
+ // optimize: exclude disconnected nodes.
+ if (0 <= lRoot[right]) {
+ Position pos = positions.get(offset);
+ for (int i = 0; i < pos.getCount(); ++i) {
+ addNode(pos.getBackType(i), pos.getBackID(i), pos.getBackPos(i) - rootBase, right);
+ }
+ }
+ }
+ }
+
+ // set mark = -1 for unreachable nodes.
+ void markUnreachable() {
+ for (int index = 1; index < rootSize - 1; ++index) {
+ if (rRoot[index] < 0) {
+ for (int node = lRoot[index]; 0 <= node; node = nodeLeftChain[node]) {
+ if (VERBOSE) {
+ System.out.printf("DEBUG: markUnreachable: node=%d\n", node);
+ }
+ nodeMark[node] = -1;
+ }
+ }
+ }
+ }
+
+ private int connectionCost(ConnectionCosts costs, int left, int right) {
+ int leftID = nodeLeftID[right];
+ return ((leftID == 0 && !useEOS) ? 0 : costs.get(nodeRightID[left], leftID));
+ }
+
+ void calcLeftCost(ConnectionCosts costs) {
+ for (int index = 0; index < rootSize; ++index) {
+ for (int node = lRoot[index]; 0 <= node; node = nodeLeftChain[node]) {
+ if (0 <= nodeMark[node]) {
+ int leastNode = -1;
+ int leastCost = Integer.MAX_VALUE;
+ for (int leftNode = rRoot[index]; 0 <= leftNode; leftNode = nodeRightChain[leftNode]) {
+ if (0 <= nodeMark[leftNode]) {
+ int cost =
+ nodeLeftCost[leftNode]
+ + nodeWordCost[leftNode]
+ + connectionCost(costs, leftNode, node);
+ if (cost < leastCost) {
+ leastCost = cost;
+ leastNode = leftNode;
+ }
+ }
+ }
+ assert 0 <= leastNode;
+ nodeLeftNode[node] = leastNode;
+ nodeLeftCost[node] = leastCost;
+ if (VERBOSE) {
+ System.out.printf(
+ "DEBUG: calcLeftCost: node=%d, leftNode=%d, leftCost=%d\n",
+ node, nodeLeftNode[node], nodeLeftCost[node]);
+ }
+ }
+ }
+ }
+ }
+
+ void calcRightCost(ConnectionCosts costs) {
+ for (int index = rootSize - 1; 0 <= index; --index) {
+ for (int node = rRoot[index]; 0 <= node; node = nodeRightChain[node]) {
+ if (0 <= nodeMark[node]) {
+ int leastNode = -1;
+ int leastCost = Integer.MAX_VALUE;
+ for (int rightNode = lRoot[index];
+ 0 <= rightNode;
+ rightNode = nodeLeftChain[rightNode]) {
+ if (0 <= nodeMark[rightNode]) {
+ int cost =
+ nodeRightCost[rightNode]
+ + nodeWordCost[rightNode]
+ + connectionCost(costs, node, rightNode);
+ if (cost < leastCost) {
+ leastCost = cost;
+ leastNode = rightNode;
+ }
+ }
+ }
+ assert 0 <= leastNode;
+ nodeRightNode[node] = leastNode;
+ nodeRightCost[node] = leastCost;
+ if (VERBOSE) {
+ System.out.printf(
+ "DEBUG: calcRightCost: node=%d, rightNode=%d, rightCost=%d\n",
+ node, nodeRightNode[node], nodeRightCost[node]);
+ }
+ }
+ }
+ }
+ }
+
+ // Mark all nodes that have same text and different par-of-speech or reading.
+ void markSameSpanNode(int refNode, int value) {
+ int left = nodeLeft[refNode];
+ int right = nodeRight[refNode];
+ for (int node = lRoot[left]; 0 <= node; node = nodeLeftChain[node]) {
+ if (nodeRight[node] == right) {
+ nodeMark[node] = value;
+ }
+ }
+ }
+
+ List<Integer> bestPathNodeList() {
+ List<Integer> list = new ArrayList<>();
+ for (int node = nodeRightNode[0]; node != 1; node = nodeRightNode[node]) {
+ list.add(node);
+ markSameSpanNode(node, 1);
+ }
+ return list;
+ }
+
+ private int cost(int node) {
+ return nodeLeftCost[node] + nodeWordCost[node] + nodeRightCost[node];
+ }
+
+ List<Integer> nBestNodeList(int N) {
+ List<Integer> list = new ArrayList<>();
+ int leastCost = Integer.MAX_VALUE;
+ int leastLeft = -1;
+ int leastRight = -1;
+ for (int node = 2; node < nodeCount; ++node) {
+ if (nodeMark[node] == 0) {
+ int cost = cost(node);
+ if (cost < leastCost) {
+ leastCost = cost;
+ leastLeft = nodeLeft[node];
+ leastRight = nodeRight[node];
+ list.clear();
+ list.add(node);
+ } else if (cost == leastCost
+ && (nodeLeft[node] != leastLeft || nodeRight[node] != leastRight)) {
+ list.add(node);
+ }
+ }
+ }
+ for (int node : list) {
+ markSameSpanNode(node, N);
+ }
+ return list;
+ }
+
+ int bestCost() {
+ return nodeLeftCost[1];
+ }
+
+ int probeDelta(int start, int end) {
+ int left = start - rootBase;
+ int right = end - rootBase;
+ if (left < 0 || rootSize < right) {
+ return Integer.MAX_VALUE;
+ }
+ int probedCost = Integer.MAX_VALUE;
+ for (int node = lRoot[left]; 0 <= node; node = nodeLeftChain[node]) {
+ if (nodeRight[node] == right) {
+ probedCost = Math.min(probedCost, cost(node));
+ }
+ }
+ return probedCost - bestCost();
+ }
+
+ void debugPrint() {
+ if (VERBOSE) {
+ for (int node = 0; node < nodeCount; ++node) {
+ System.out.printf(
+ "DEBUG NODE: node=%d, mark=%d, cost=%d, left=%d, right=%d\n",
+ node, nodeMark[node], cost(node), nodeLeft[node], nodeRight[node]);
+ }
+ }
+ }
+ }
+}
diff --git a/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseTokenizer.java b/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseTokenizer.java
index 6e3cf94002e..9e55d59c8c3 100644
--- a/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseTokenizer.java
+++ b/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseTokenizer.java
@@ -31,8 +31,10 @@ import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.ja.JapaneseTokenizer.Mode;
import org.apache.lucene.analysis.ja.dict.ConnectionCosts;
+import org.apache.lucene.analysis.ja.dict.JaMorphData;
import org.apache.lucene.analysis.ja.dict.UserDictionary;
import org.apache.lucene.analysis.ja.tokenattributes.*;
+import org.apache.lucene.analysis.morph.GraphvizFormatter;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.tests.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.tests.analysis.MockGraphTokenFilter;
@@ -518,7 +520,8 @@ public class TestJapaneseTokenizer extends BaseTokenStreamTestCase {
}
public void testLatticeToDot() throws Exception {
- final GraphvizFormatter gv2 = new GraphvizFormatter(ConnectionCosts.getInstance());
+ final GraphvizFormatter<JaMorphData> gv2 =
+ new GraphvizFormatter<>(ConnectionCosts.getInstance());
final Analyzer analyzer =
new Analyzer() {
@Override
diff --git a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/KoreanTokenizer.java b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/KoreanTokenizer.java
index 76a95047e1d..15bdb3422ad 100644
--- a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/KoreanTokenizer.java
+++ b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/KoreanTokenizer.java
@@ -16,15 +16,8 @@
*/
package org.apache.lucene.analysis.ko;
-import static java.lang.Character.UnicodeScript;
-
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.EnumMap;
-import java.util.List;
import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.ko.dict.CharacterDefinition;
import org.apache.lucene.analysis.ko.dict.ConnectionCosts;
import org.apache.lucene.analysis.ko.dict.DictionaryBuilder;
import org.apache.lucene.analysis.ko.dict.KoMorphData;
@@ -34,18 +27,13 @@ import org.apache.lucene.analysis.ko.dict.UnknownDictionary;
import org.apache.lucene.analysis.ko.dict.UserDictionary;
import org.apache.lucene.analysis.ko.tokenattributes.PartOfSpeechAttribute;
import org.apache.lucene.analysis.ko.tokenattributes.ReadingAttribute;
-import org.apache.lucene.analysis.morph.Dictionary;
-import org.apache.lucene.analysis.morph.TokenType;
+import org.apache.lucene.analysis.morph.GraphvizFormatter;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionLengthAttribute;
-import org.apache.lucene.analysis.util.RollingCharBuffer;
-import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.AttributeFactory;
import org.apache.lucene.util.IgnoreRandomChains;
-import org.apache.lucene.util.IntsRef;
-import org.apache.lucene.util.RamUsageEstimator;
import org.apache.lucene.util.fst.FST;
/**
@@ -86,46 +74,7 @@ public final class KoreanTokenizer extends Tokenizer {
private static final boolean VERBOSE = false;
- // For safety:
- private static final int MAX_UNKNOWN_WORD_LENGTH = 1024;
- private static final int MAX_BACKTRACE_GAP = 1024;
-
- private final EnumMap<TokenType, Dictionary<? extends KoMorphData>> dictionaryMap =
- new EnumMap<>(TokenType.class);
-
- private final TokenInfoFST fst;
- private final TokenInfoDictionary dictionary;
- private final UnknownDictionary unkDictionary;
- private final ConnectionCosts costs;
- private final UserDictionary userDictionary;
- private final CharacterDefinition characterDefinition;
-
- private final FST.Arc<Long> arc = new FST.Arc<>();
- private final FST.BytesReader fstReader;
- private final IntsRef wordIdRef = new IntsRef();
-
- private final FST.BytesReader userFSTReader;
- private final TokenInfoFST userFST;
-
- private final boolean discardPunctuation;
- private final DecompoundMode mode;
- private final boolean outputUnknownUnigrams;
-
- private final RollingCharBuffer buffer = new RollingCharBuffer();
-
- private final WrappedPositionArray positions = new WrappedPositionArray();
-
- // True once we've hit the EOF from the input reader:
- private boolean end;
-
- // Last absolute position we backtraced from:
- private int lastBackTracePos;
-
- // Next absolute position to process:
- private int pos;
-
- // Already parsed, but not yet passed to caller, tokens:
- private final List<Token> pending = new ArrayList<>();
+ private final Viterbi viterbi;
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
@@ -212,275 +161,54 @@ public final class KoreanTokenizer extends Tokenizer {
boolean outputUnknownUnigrams,
boolean discardPunctuation) {
super(factory);
- this.dictionary = systemDictionary;
- this.fst = dictionary.getFST();
- this.unkDictionary = unkDictionary;
- this.characterDefinition = unkDictionary.getCharacterDefinition();
- this.costs = connectionCosts;
- this.userDictionary = userDictionary;
- fstReader = fst.getBytesReader();
+ TokenInfoFST fst = systemDictionary.getFST();
+ FST.BytesReader fstReader = fst.getBytesReader();
+ TokenInfoFST userFST = null;
+ FST.BytesReader userFSTReader = null;
if (userDictionary != null) {
userFST = userDictionary.getFST();
userFSTReader = userFST.getBytesReader();
- } else {
- userFST = null;
- userFSTReader = null;
}
- this.mode = mode;
- this.outputUnknownUnigrams = outputUnknownUnigrams;
- this.discardPunctuation = discardPunctuation;
- buffer.reset(this.input);
-
- resetState();
-
- dictionaryMap.put(TokenType.KNOWN, dictionary);
- dictionaryMap.put(TokenType.UNKNOWN, unkDictionary);
- dictionaryMap.put(TokenType.USER, userDictionary);
- }
-
- private GraphvizFormatter dotOut;
- /** Expert: set this to produce graphviz (dot) output of the Viterbi lattice */
- public void setGraphvizFormatter(GraphvizFormatter dotOut) {
- this.dotOut = dotOut;
+ viterbi =
+ new Viterbi(
+ fst,
+ fstReader,
+ systemDictionary,
+ userFST,
+ userFSTReader,
+ userDictionary,
+ connectionCosts,
+ unkDictionary,
+ unkDictionary.getCharacterDefinition(),
+ discardPunctuation,
+ mode,
+ outputUnknownUnigrams);
+ viterbi.resetBuffer(input);
+ viterbi.resetState();
}
@Override
public void close() throws IOException {
super.close();
- buffer.reset(input);
+ viterbi.resetBuffer(input);
}
@Override
public void reset() throws IOException {
super.reset();
- buffer.reset(input);
- resetState();
- }
-
- private void resetState() {
- positions.reset();
- pos = 0;
- end = false;
- lastBackTracePos = 0;
- pending.clear();
-
- // Add BOS:
- positions.get(0).add(0, 0, -1, -1, -1, -1, TokenType.KNOWN);
+ viterbi.resetBuffer(input);
+ viterbi.resetState();
}
@Override
public void end() throws IOException {
super.end();
// Set final offset
- int finalOffset = correctOffset(pos);
+ int finalOffset = correctOffset(viterbi.getPos());
offsetAtt.setOffset(finalOffset, finalOffset);
}
- // Holds all back pointers arriving to this position:
- static final class Position {
-
- int pos;
-
- int count;
-
- // maybe single int array * 5?
- int[] costs = new int[8];
- int[] lastRightID = new int[8];
- int[] backPos = new int[8];
- int[] backWordPos = new int[8];
- int[] backIndex = new int[8];
- int[] backID = new int[8];
- TokenType[] backType = new TokenType[8];
-
- public void grow() {
- costs = ArrayUtil.grow(costs, 1 + count);
- lastRightID = ArrayUtil.grow(lastRightID, 1 + count);
- backPos = ArrayUtil.grow(backPos, 1 + count);
- backWordPos = ArrayUtil.grow(backWordPos, 1 + count);
- backIndex = ArrayUtil.grow(backIndex, 1 + count);
- backID = ArrayUtil.grow(backID, 1 + count);
-
- // NOTE: sneaky: grow separately because
- // ArrayUtil.grow will otherwise pick a different
- // length than the int[]s we just grew:
- final TokenType[] newBackType = new TokenType[backID.length];
- System.arraycopy(backType, 0, newBackType, 0, backType.length);
- backType = newBackType;
- }
-
- public void add(
- int cost,
- int lastRightID,
- int backPos,
- int backRPos,
- int backIndex,
- int backID,
- TokenType backType) {
- // NOTE: this isn't quite a true Viterbi search,
- // because we should check if lastRightID is
- // already present here, and only update if the new
- // cost is less than the current cost, instead of
- // simply appending. However, that will likely hurt
- // performance (usually we add a lastRightID only once),
- // and it means we actually create the full graph
- // intersection instead of a "normal" Viterbi lattice:
- if (count == costs.length) {
- grow();
- }
- this.costs[count] = cost;
- this.lastRightID[count] = lastRightID;
- this.backPos[count] = backPos;
- this.backWordPos[count] = backRPos;
- this.backIndex[count] = backIndex;
- this.backID[count] = backID;
- this.backType[count] = backType;
- count++;
- }
-
- public void reset() {
- count = 0;
- }
- }
-
- /**
- * Returns the space penalty associated with the provided {@link POS.Tag}.
- *
- * @param leftPOS the left part of speech of the current token.
- * @param numSpaces the number of spaces before the current token.
- */
- private int computeSpacePenalty(POS.Tag leftPOS, int numSpaces) {
- int spacePenalty = 0;
- if (numSpaces > 0) {
- // TODO we should extract the penalty (left-space-penalty-factor) from the dicrc file.
- switch (leftPOS) {
- case E:
- case J:
- case VCP:
- case XSA:
- case XSN:
- case XSV:
- spacePenalty = 3000;
- break;
- case IC:
- case MAG:
- case MAJ:
- case MM:
- case NA:
- case NNB:
- case NNBC:
- case NNG:
- case NNP:
- case NP:
- case NR:
- case SC:
- case SE:
- case SF:
- case SH:
- case SL:
- case SN:
- case SP:
- case SSC:
- case SSO:
- case SY:
- case UNA:
- case UNKNOWN:
- case VA:
- case VCN:
- case VSV:
- case VV:
- case VX:
- case XPN:
- case XR:
- default:
- break;
- }
- }
- return spacePenalty;
- }
-
- private void add(
- KoMorphData morphAtts,
- Position fromPosData,
- int wordPos,
- int endPos,
- int wordID,
- TokenType type) {
- final POS.Tag leftPOS = morphAtts.getLeftPOS(wordID);
- final int wordCost = morphAtts.getWordCost(wordID);
- final int leftID = morphAtts.getLeftId(wordID);
- int leastCost = Integer.MAX_VALUE;
- int leastIDX = -1;
- assert fromPosData.count > 0;
- for (int idx = 0; idx < fromPosData.count; idx++) {
- // The number of spaces before the term
- int numSpaces = wordPos - fromPosData.pos;
-
- // Cost is path cost so far, plus word cost (added at
- // end of loop), plus bigram cost and space penalty cost.
- final int cost =
- fromPosData.costs[idx]
- + costs.get(fromPosData.lastRightID[idx], leftID)
- + computeSpacePenalty(leftPOS, numSpaces);
- if (VERBOSE) {
- System.out.println(
- " fromIDX="
- + idx
- + ": cost="
- + cost
- + " (prevCost="
- + fromPosData.costs[idx]
- + " wordCost="
- + wordCost
- + " bgCost="
- + costs.get(fromPosData.lastRightID[idx], leftID)
- + " spacePenalty="
- + computeSpacePenalty(leftPOS, numSpaces)
- + ") leftID="
- + leftID
- + " leftPOS="
- + leftPOS.name()
- + ")");
- }
- if (cost < leastCost) {
- leastCost = cost;
- leastIDX = idx;
- if (VERBOSE) {
- System.out.println(" **");
- }
- }
- }
-
- leastCost += wordCost;
-
- if (VERBOSE) {
- System.out.println(
- " + cost="
- + leastCost
- + " wordID="
- + wordID
- + " leftID="
- + leftID
- + " leastIDX="
- + leastIDX
- + " toPos="
- + endPos
- + " toPos.idx="
- + positions.get(endPos).count);
- }
-
- positions
- .get(endPos)
- .add(
- leastCost,
- morphAtts.getRightId(wordID),
- fromPosData.pos,
- wordPos,
- leastIDX,
- wordID,
- type);
- }
-
@Override
public boolean incrementToken() throws IOException {
@@ -488,16 +216,16 @@ public final class KoreanTokenizer extends Tokenizer {
// tokens, when the tokens it had produced were entirely
// punctuation. So we loop here until we get a real
// token or we end:
- while (pending.size() == 0) {
- if (end) {
+ while (viterbi.getPending().size() == 0) {
+ if (viterbi.isEnd()) {
return false;
}
// Push Viterbi forward some more:
- parse();
+ viterbi.forward();
}
- final Token token = pending.remove(pending.size() - 1);
+ final Token token = viterbi.getPending().remove(viterbi.getPending().size() - 1);
int length = token.getLength();
clearAttributes();
@@ -516,690 +244,8 @@ public final class KoreanTokenizer extends Tokenizer {
return true;
}
- // TODO: make generic'd version of this "circular array"?
- // It's a bit tricky because we do things to the Position
- // (eg, set .pos = N on reuse)...
- static final class WrappedPositionArray {
- private Position[] positions = new Position[8];
-
- public WrappedPositionArray() {
- for (int i = 0; i < positions.length; i++) {
- positions[i] = new Position();
- }
- }
-
- // Next array index to write to in positions:
- private int nextWrite;
-
- // Next position to write:
- private int nextPos;
-
- // How many valid Position instances are held in the
- // positions array:
- private int count;
-
- public void reset() {
- nextWrite--;
- while (count > 0) {
- if (nextWrite == -1) {
- nextWrite = positions.length - 1;
- }
- positions[nextWrite--].reset();
- count--;
- }
- nextWrite = 0;
- nextPos = 0;
- count = 0;
- }
-
- /**
- * Get Position instance for this absolute position; this is allowed to be arbitrarily far "in
- * the future" but cannot be before the last freeBefore.
- */
- public Position get(int pos) {
- while (pos >= nextPos) {
- // System.out.println("count=" + count + " vs len=" + positions.length);
- if (count == positions.length) {
- Position[] newPositions =
- new Position[ArrayUtil.oversize(1 + count, RamUsageEstimator.NUM_BYTES_OBJECT_REF)];
- // System.out.println("grow positions " + newPositions.length);
- System.arraycopy(positions, nextWrite, newPositions, 0, positions.length - nextWrite);
- System.arraycopy(positions, 0, newPositions, positions.length - nextWrite, nextWrite);
- for (int i = positions.length; i < newPositions.length; i++) {
- newPositions[i] = new Position();
- }
- nextWrite = positions.length;
- positions = newPositions;
- }
- if (nextWrite == positions.length) {
- nextWrite = 0;
- }
- // Should have already been reset:
- assert positions[nextWrite].count == 0;
- positions[nextWrite++].pos = nextPos++;
- count++;
- }
- assert inBounds(pos);
- final int index = getIndex(pos);
- assert positions[index].pos == pos;
- return positions[index];
- }
-
- public int getNextPos() {
- return nextPos;
- }
-
- // For assert:
- private boolean inBounds(int pos) {
- return pos < nextPos && pos >= nextPos - count;
- }
-
- private int getIndex(int pos) {
- int index = nextWrite - (nextPos - pos);
- if (index < 0) {
- index += positions.length;
- }
- return index;
- }
-
- public void freeBefore(int pos) {
- final int toFree = count - (nextPos - pos);
- assert toFree >= 0;
- assert toFree <= count;
- int index = nextWrite - count;
- if (index < 0) {
- index += positions.length;
- }
- for (int i = 0; i < toFree; i++) {
- if (index == positions.length) {
- index = 0;
- }
- // System.out.println(" fb idx=" + index);
- positions[index].reset();
- index++;
- }
- count -= toFree;
- }
- }
-
- /* Incrementally parse some more characters. This runs
- * the viterbi search forwards "enough" so that we
- * generate some more tokens. How much forward depends on
- * the chars coming in, since some chars could cause
- * longer-lasting ambiguity in the parsing. Once the
- * ambiguity is resolved, then we back trace, produce
- * the pending tokens, and return. */
- private void parse() throws IOException {
- if (VERBOSE) {
- System.out.println("\nPARSE");
- }
-
- // Index of the last character of unknown word:
- int unknownWordEndIndex = -1;
-
- // Maximum posAhead of user word in the entire input
- int userWordMaxPosAhead = -1;
-
- // Advances over each position (character):
- while (buffer.get(pos) != -1) {
- final Position posData = positions.get(pos);
- final boolean isFrontier = positions.getNextPos() == pos + 1;
-
- if (posData.count == 0) {
- // No arcs arrive here; move to next position:
- if (VERBOSE) {
- System.out.println(" no arcs in; skip pos=" + pos);
- }
- pos++;
- continue;
- }
-
- if (pos > lastBackTracePos && posData.count == 1 && isFrontier) {
- // We are at a "frontier", and only one node is
- // alive, so whatever the eventual best path is must
- // come through this node. So we can safely commit
- // to the prefix of the best path at this point:
- backtrace(posData, 0);
-
- // Re-base cost so we don't risk int overflow:
- posData.costs[0] = 0;
- if (pending.size() > 0) {
- return;
- } else {
- // This means the backtrace only produced
- // punctuation tokens, so we must keep parsing.
- }
- }
-
- if (pos - lastBackTracePos >= MAX_BACKTRACE_GAP) {
- // Safety: if we've buffered too much, force a
- // backtrace now. We find the least-cost partial
- // path, across all paths, backtrace from it, and
- // then prune all others. Note that this, in
- // general, can produce the wrong result, if the
- // total best path did not in fact back trace
- // through this partial best path. But it's the
- // best we can do... (short of not having a
- // safety!).
-
- // First pass: find least cost partial path so far,
- // including ending at future positions:
- int leastIDX = -1;
- int leastCost = Integer.MAX_VALUE;
- Position leastPosData = null;
- for (int pos2 = pos; pos2 < positions.getNextPos(); pos2++) {
- final Position posData2 = positions.get(pos2);
- for (int idx = 0; idx < posData2.count; idx++) {
- // System.out.println(" idx=" + idx + " cost=" + cost);
- final int cost = posData2.costs[idx];
- if (cost < leastCost) {
- leastCost = cost;
- leastIDX = idx;
- leastPosData = posData2;
- }
- }
- }
-
- // We will always have at least one live path:
- assert leastIDX != -1;
-
- // Second pass: prune all but the best path:
- for (int pos2 = pos; pos2 < positions.getNextPos(); pos2++) {
- final Position posData2 = positions.get(pos2);
- if (posData2 != leastPosData) {
- posData2.reset();
- } else {
- if (leastIDX != 0) {
- posData2.costs[0] = posData2.costs[leastIDX];
- posData2.lastRightID[0] = posData2.lastRightID[leastIDX];
- posData2.backPos[0] = posData2.backPos[leastIDX];
- posData2.backWordPos[0] = posData2.backWordPos[leastIDX];
- posData2.backIndex[0] = posData2.backIndex[leastIDX];
- posData2.backID[0] = posData2.backID[leastIDX];
- posData2.backType[0] = posData2.backType[leastIDX];
- }
- posData2.count = 1;
- }
- }
-
- backtrace(leastPosData, 0);
-
- // Re-base cost so we don't risk int overflow:
- Arrays.fill(leastPosData.costs, 0, leastPosData.count, 0);
-
- if (pos != leastPosData.pos) {
- // We jumped into a future position:
- assert pos < leastPosData.pos;
- pos = leastPosData.pos;
- }
- if (pending.size() > 0) {
- return;
- } else {
- // This means the backtrace only produced
- // punctuation tokens, so we must keep parsing.
- continue;
- }
- }
-
- if (VERBOSE) {
- System.out.println(
- "\n extend @ pos="
- + pos
- + " char="
- + (char) buffer.get(pos)
- + " hex="
- + Integer.toHexString(buffer.get(pos)));
- }
-
- if (VERBOSE) {
- System.out.println(" " + posData.count + " arcs in");
- }
-
- // We add single space separator as prefixes of the terms that we extract.
- // This information is needed to compute the space penalty factor of each term.
- // These whitespace prefixes are removed when the final tokens are generated, or
- // added as separated tokens when discardPunctuation is unset.
- if (Character.getType(buffer.get(pos)) == Character.SPACE_SEPARATOR) {
- if (buffer.get(++pos) == -1) {
- pos = posData.pos;
- }
- }
-
- boolean anyMatches = false;
-
- // First try user dict:
- if (userFST != null) {
- userFST.getFirstArc(arc);
- int output = 0;
- int maxPosAhead = 0;
- int outputMaxPosAhead = 0;
- int arcFinalOutMaxPosAhead = 0;
-
- for (int posAhead = pos; ; posAhead++) {
- final int ch = buffer.get(posAhead);
- if (ch == -1) {
- break;
- }
- if (userFST.findTargetArc(ch, arc, arc, posAhead == pos, userFSTReader) == null) {
- break;
- }
- output += arc.output().intValue();
- if (arc.isFinal()) {
- maxPosAhead = posAhead;
- outputMaxPosAhead = output;
- arcFinalOutMaxPosAhead = arc.nextFinalOutput().intValue();
- anyMatches = true;
- }
- }
-
- // Longest matching for user word
- if (anyMatches && maxPosAhead > userWordMaxPosAhead) {
- if (VERBOSE) {
- System.out.println(
- " USER word "
- + new String(buffer.get(pos, maxPosAhead + 1))
- + " toPos="
- + (maxPosAhead + 1));
- }
- add(
- userDictionary.getMorphAttributes(),
- posData,
- pos,
- maxPosAhead + 1,
- outputMaxPosAhead + arcFinalOutMaxPosAhead,
- TokenType.USER);
- userWordMaxPosAhead = Math.max(userWordMaxPosAhead, maxPosAhead);
- }
- }
-
- // TODO: we can be more aggressive about user
- // matches? if we are "under" a user match then don't
- // extend KNOWN/UNKNOWN paths?
-
- if (!anyMatches) {
- // Next, try known dictionary matches
- fst.getFirstArc(arc);
- int output = 0;
-
- for (int posAhead = pos; ; posAhead++) {
- final int ch = buffer.get(posAhead);
- if (ch == -1) {
- break;
- }
- // System.out.println(" match " + (char) ch + " posAhead=" + posAhead);
-
- if (fst.findTargetArc(ch, arc, arc, posAhead == pos, fstReader) == null) {
- break;
- }
-
- output += arc.output().intValue();
-
- // Optimization: for known words that are too-long
- // (compound), we should pre-compute the 2nd
- // best segmentation and store it in the
- // dictionary instead of recomputing it each time a
- // match is found.
-
- if (arc.isFinal()) {
- dictionary.lookupWordIds(output + arc.nextFinalOutput().intValue(), wordIdRef);
- if (VERBOSE) {
- System.out.println(
- " KNOWN word "
- + new String(buffer.get(pos, posAhead - pos + 1))
- + " toPos="
- + (posAhead + 1)
- + " "
- + wordIdRef.length
- + " wordIDs");
- }
- for (int ofs = 0; ofs < wordIdRef.length; ofs++) {
- add(
- dictionary.getMorphAttributes(),
- posData,
- pos,
- posAhead + 1,
- wordIdRef.ints[wordIdRef.offset + ofs],
- TokenType.KNOWN);
- anyMatches = true;
- }
- }
- }
- }
-
- if (unknownWordEndIndex > posData.pos) {
- pos++;
- continue;
- }
-
- final char firstCharacter = (char) buffer.get(pos);
- if (!anyMatches || characterDefinition.isInvoke(firstCharacter)) {
-
- // Find unknown match:
- int characterId = characterDefinition.getCharacterClass(firstCharacter);
- // NOTE: copied from UnknownDictionary.lookup:
- int unknownWordLength;
- if (!characterDefinition.isGroup(firstCharacter)) {
- unknownWordLength = 1;
- } else {
- // Extract unknown word. Characters with the same script are considered to be part of
- // unknown word
- unknownWordLength = 1;
- UnicodeScript scriptCode = UnicodeScript.of(firstCharacter);
- final boolean isPunct = isPunctuation(firstCharacter);
- final boolean isDigit = Character.isDigit(firstCharacter);
- for (int posAhead = pos + 1; unknownWordLength < MAX_UNKNOWN_WORD_LENGTH; posAhead++) {
- int next = buffer.get(posAhead);
- if (next == -1) {
- break;
- }
- char ch = (char) next;
- int chType = Character.getType(ch);
- UnicodeScript sc = UnicodeScript.of(next);
- boolean sameScript =
- isSameScript(scriptCode, sc)
- // Non-spacing marks inherit the script of their base character,
- // following recommendations from UTR #24.
- || chType == Character.NON_SPACING_MARK;
-
- if (sameScript
- // split on punctuation
- && isPunctuation(ch, chType) == isPunct
- // split on digit
- && Character.isDigit(ch) == isDigit
- && characterDefinition.isGroup(ch)) {
- unknownWordLength++;
- } else {
- break;
- }
- // Update the script code and character class if the original script
- // is Inherited or Common.
- if (isCommonOrInherited(scriptCode) && isCommonOrInherited(sc) == false) {
- scriptCode = sc;
- characterId = characterDefinition.getCharacterClass(ch);
- }
- }
- }
-
- unkDictionary.lookupWordIds(
- characterId, wordIdRef); // characters in input text are supposed to be the same
- if (VERBOSE) {
- System.out.println(
- " UNKNOWN word len=" + unknownWordLength + " " + wordIdRef.length + " wordIDs");
- }
- for (int ofs = 0; ofs < wordIdRef.length; ofs++) {
- add(
- unkDictionary.getMorphAttributes(),
- posData,
- pos,
- pos + unknownWordLength,
- wordIdRef.ints[wordIdRef.offset + ofs],
- TokenType.UNKNOWN);
- }
- }
-
- pos++;
- }
-
- end = true;
-
- if (pos > 0) {
-
- final Position endPosData = positions.get(pos);
- int leastCost = Integer.MAX_VALUE;
- int leastIDX = -1;
- if (VERBOSE) {
- System.out.println(" end: " + endPosData.count + " nodes");
- }
- for (int idx = 0; idx < endPosData.count; idx++) {
- // Add EOS cost:
- final int cost = endPosData.costs[idx] + costs.get(endPosData.lastRightID[idx], 0);
- // System.out.println(" idx=" + idx + " cost=" + cost + " (pathCost=" +
- // endPosData.costs[idx] + " bgCost=" + costs.get(endPosData.lastRightID[idx], 0) + ")
- // backPos=" + endPosData.backPos[idx]);
- if (cost < leastCost) {
- leastCost = cost;
- leastIDX = idx;
- }
- }
-
- backtrace(endPosData, leastIDX);
- } else {
- // No characters in the input string; return no tokens!
- }
- }
-
- // the pending list. The pending list is then in-reverse
- // (last token should be returned first).
- private void backtrace(final Position endPosData, final int fromIDX) {
- final int endPos = endPosData.pos;
-
- if (VERBOSE) {
- System.out.println(
- "\n backtrace: endPos="
- + endPos
- + " pos="
- + pos
- + "; "
- + (pos - lastBackTracePos)
- + " characters; last="
- + lastBackTracePos
- + " cost="
- + endPosData.costs[fromIDX]);
- }
-
- final char[] fragment = buffer.get(lastBackTracePos, endPos - lastBackTracePos);
-
- if (dotOut != null) {
- dotOut.onBacktrace(this, positions, lastBackTracePos, endPosData, fromIDX, fragment, end);
- }
-
- int pos = endPos;
- int bestIDX = fromIDX;
-
- // TODO: sort of silly to make Token instances here; the
- // back trace has all info needed to generate the
- // token. So, we could just directly set the attrs,
- // from the backtrace, in incrementToken w/o ever
- // creating Token; we'd have to defer calling freeBefore
- // until after the backtrace was fully "consumed" by
- // incrementToken.
-
- while (pos > lastBackTracePos) {
- // System.out.println("BT: back pos=" + pos + " bestIDX=" + bestIDX);
- final Position posData = positions.get(pos);
- assert bestIDX < posData.count;
-
- int backPos = posData.backPos[bestIDX];
- int backWordPos = posData.backWordPos[bestIDX];
- assert backPos >= lastBackTracePos
- : "backPos=" + backPos + " vs lastBackTracePos=" + lastBackTracePos;
- // the length of the word without the whitespaces at the beginning.
- int length = pos - backWordPos;
- TokenType backType = posData.backType[bestIDX];
- int backID = posData.backID[bestIDX];
- int nextBestIDX = posData.backIndex[bestIDX];
- // the start of the word after the whitespace at the beginning.
- final int fragmentOffset = backWordPos - lastBackTracePos;
- assert fragmentOffset >= 0;
-
- final Dictionary<? extends KoMorphData> dict = getDict(backType);
-
- if (outputUnknownUnigrams && backType == TokenType.UNKNOWN) {
- // outputUnknownUnigrams converts unknown word into unigrams:
- for (int i = length - 1; i >= 0; i--) {
- int charLen = 1;
- if (i > 0 && Character.isLowSurrogate(fragment[fragmentOffset + i])) {
- i--;
- charLen = 2;
- }
- final DictionaryToken token =
- new DictionaryToken(
- TokenType.UNKNOWN,
- unkDictionary.getMorphAttributes(),
- CharacterDefinition.NGRAM,
- fragment,
- fragmentOffset + i,
- charLen,
- backWordPos + i,
- backWordPos + i + charLen);
- pending.add(token);
- if (VERBOSE) {
- System.out.println(" add token=" + pending.get(pending.size() - 1));
- }
- }
- } else {
- final DictionaryToken token =
- new DictionaryToken(
- backType,
- dict.getMorphAttributes(),
- backID,
- fragment,
- fragmentOffset,
- length,
- backWordPos,
- backWordPos + length);
- if (token.getPOSType() == POS.Type.MORPHEME || mode == DecompoundMode.NONE) {
- if (shouldFilterToken(token) == false) {
- pending.add(token);
- if (VERBOSE) {
- System.out.println(" add token=" + pending.get(pending.size() - 1));
- }
- }
- } else {
- KoMorphData.Morpheme[] morphemes = token.getMorphemes();
- if (morphemes == null) {
- pending.add(token);
- if (VERBOSE) {
- System.out.println(" add token=" + pending.get(pending.size() - 1));
- }
- } else {
- int endOffset = backWordPos + length;
- int posLen = 0;
- // decompose the compound
- for (int i = morphemes.length - 1; i >= 0; i--) {
- final KoMorphData.Morpheme morpheme = morphemes[i];
- final Token compoundToken;
- if (token.getPOSType() == POS.Type.COMPOUND) {
- assert endOffset - morpheme.surfaceForm.length() >= 0;
- compoundToken =
- new DecompoundToken(
- morpheme.posTag,
- morpheme.surfaceForm,
- endOffset - morpheme.surfaceForm.length(),
- endOffset,
- backType);
- } else {
- compoundToken =
- new DecompoundToken(
- morpheme.posTag,
- morpheme.surfaceForm,
- token.getStartOffset(),
- token.getEndOffset(),
- backType);
- }
- if (i == 0 && mode == DecompoundMode.MIXED) {
- compoundToken.setPositionIncrement(0);
- }
- ++posLen;
- endOffset -= morpheme.surfaceForm.length();
- pending.add(compoundToken);
- if (VERBOSE) {
- System.out.println(" add token=" + pending.get(pending.size() - 1));
- }
- }
- if (mode == DecompoundMode.MIXED) {
- token.setPositionLength(Math.max(1, posLen));
- pending.add(token);
- if (VERBOSE) {
- System.out.println(" add token=" + pending.get(pending.size() - 1));
- }
- }
- }
- }
- }
- if (discardPunctuation == false && backWordPos != backPos) {
- // Add a token for whitespaces between terms
- int offset = backPos - lastBackTracePos;
- int len = backWordPos - backPos;
- // System.out.println(offset + " " + fragmentOffset + " " + len + " " + backWordPos + " " +
- // backPos);
- unkDictionary.lookupWordIds(characterDefinition.getCharacterClass(' '), wordIdRef);
- DictionaryToken spaceToken =
- new DictionaryToken(
- TokenType.UNKNOWN,
- unkDictionary.getMorphAttributes(),
- wordIdRef.ints[wordIdRef.offset],
- fragment,
- offset,
- len,
- backPos,
- backPos + len);
- pending.add(spaceToken);
- }
-
- pos = backPos;
- bestIDX = nextBestIDX;
- }
-
- lastBackTracePos = endPos;
-
- if (VERBOSE) {
- System.out.println(" freeBefore pos=" + endPos);
- }
- // Notify the circular buffers that we are done with
- // these positions:
- buffer.freeBefore(endPos);
- positions.freeBefore(endPos);
- }
-
- Dictionary<? extends KoMorphData> getDict(TokenType type) {
- return dictionaryMap.get(type);
- }
-
- private boolean shouldFilterToken(Token token) {
- return discardPunctuation && isPunctuation(token.getSurfaceForm()[token.getOffset()]);
- }
-
- private static boolean isPunctuation(char ch) {
- return isPunctuation(ch, Character.getType(ch));
- }
-
- private static boolean isPunctuation(char ch, int cid) {
- // special case for Hangul Letter Araea (interpunct)
- if (ch == 0x318D) {
- return true;
- }
- switch (cid) {
- case Character.SPACE_SEPARATOR:
- case Character.LINE_SEPARATOR:
- case Character.PARAGRAPH_SEPARATOR:
- case Character.CONTROL:
- case Character.FORMAT:
- case Character.DASH_PUNCTUATION:
- case Character.START_PUNCTUATION:
- case Character.END_PUNCTUATION:
- case Character.CONNECTOR_PUNCTUATION:
- case Character.OTHER_PUNCTUATION:
- case Character.MATH_SYMBOL:
- case Character.CURRENCY_SYMBOL:
- case Character.MODIFIER_SYMBOL:
- case Character.OTHER_SYMBOL:
- case Character.INITIAL_QUOTE_PUNCTUATION:
- case Character.FINAL_QUOTE_PUNCTUATION:
- return true;
- default:
- return false;
- }
- }
-
- private static boolean isCommonOrInherited(UnicodeScript script) {
- return script == UnicodeScript.INHERITED || script == UnicodeScript.COMMON;
- }
-
- /** Determine if two scripts are compatible. */
- private static boolean isSameScript(UnicodeScript scriptOne, UnicodeScript scriptTwo) {
- return scriptOne == scriptTwo
- || isCommonOrInherited(scriptOne)
- || isCommonOrInherited(scriptTwo);
+ /** Expert: set this to produce graphviz (dot) output of the Viterbi lattice */
+ public void setGraphvizFormatter(GraphvizFormatter<KoMorphData> dotOut) {
+ viterbi.setGraphvizFormatter(dotOut);
}
}
diff --git a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/Viterbi.java b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/Viterbi.java
new file mode 100644
index 00000000000..2a8adbfcb62
--- /dev/null
+++ b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/Viterbi.java
@@ -0,0 +1,447 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.analysis.ko;
+
+import java.io.IOException;
+import java.util.EnumMap;
+import org.apache.lucene.analysis.ko.dict.CharacterDefinition;
+import org.apache.lucene.analysis.ko.dict.KoMorphData;
+import org.apache.lucene.analysis.ko.dict.TokenInfoDictionary;
+import org.apache.lucene.analysis.ko.dict.UnknownDictionary;
+import org.apache.lucene.analysis.ko.dict.UserDictionary;
+import org.apache.lucene.analysis.morph.ConnectionCosts;
+import org.apache.lucene.analysis.morph.Dictionary;
+import org.apache.lucene.analysis.morph.GraphvizFormatter;
+import org.apache.lucene.analysis.morph.MorphData;
+import org.apache.lucene.analysis.morph.TokenInfoFST;
+import org.apache.lucene.analysis.morph.TokenType;
+import org.apache.lucene.util.fst.FST;
+
+/** {@link org.apache.lucene.analysis.morph.Viterbi} subclass for Korean morphological analysis. */
+final class Viterbi
+ extends org.apache.lucene.analysis.morph.Viterbi<
+ Token, org.apache.lucene.analysis.morph.Viterbi.Position> {
+
+ private final EnumMap<TokenType, Dictionary<? extends KoMorphData>> dictionaryMap =
+ new EnumMap<>(TokenType.class);
+
+ private final UnknownDictionary unkDictionary;
+ private final CharacterDefinition characterDefinition;
+
+ private final boolean discardPunctuation;
+ private final KoreanTokenizer.DecompoundMode mode;
+ private final boolean outputUnknownUnigrams;
+
+ private GraphvizFormatter<KoMorphData> dotOut;
+
+ Viterbi(
+ TokenInfoFST fst,
+ FST.BytesReader fstReader,
+ TokenInfoDictionary dictionary,
+ TokenInfoFST userFST,
+ FST.BytesReader userFSTReader,
+ UserDictionary userDictionary,
+ ConnectionCosts costs,
+ UnknownDictionary unkDictionary,
+ CharacterDefinition characterDefinition,
+ boolean discardPunctuation,
+ KoreanTokenizer.DecompoundMode mode,
+ boolean outputUnknownUnigrams) {
+ super(
+ fst, fstReader, dictionary, userFST, userFSTReader, userDictionary, costs, Position.class);
+ this.unkDictionary = unkDictionary;
+ this.characterDefinition = characterDefinition;
+ this.discardPunctuation = discardPunctuation;
+ this.mode = mode;
+ this.outputUnknownUnigrams = outputUnknownUnigrams;
+ this.enableSpacePenaltyFactor = true;
+ this.outputLongestUserEntryOnly = true;
+ dictionaryMap.put(TokenType.KNOWN, dictionary);
+ dictionaryMap.put(TokenType.UNKNOWN, unkDictionary);
+ dictionaryMap.put(TokenType.USER, userDictionary);
+ }
+
+ @Override
+ protected int processUnknownWord(boolean anyMatches, Position posData) throws IOException {
+ final char firstCharacter = (char) buffer.get(pos);
+ if (!anyMatches || characterDefinition.isInvoke(firstCharacter)) {
+
+ // Find unknown match:
+ int characterId = characterDefinition.getCharacterClass(firstCharacter);
+ // NOTE: copied from UnknownDictionary.lookup:
+ int unknownWordLength;
+ if (!characterDefinition.isGroup(firstCharacter)) {
+ unknownWordLength = 1;
+ } else {
+ // Extract unknown word. Characters with the same script are considered to be part of
+ // unknown word
+ unknownWordLength = 1;
+ Character.UnicodeScript scriptCode = Character.UnicodeScript.of(firstCharacter);
+ final boolean isPunct = isPunctuation(firstCharacter);
+ final boolean isDigit = Character.isDigit(firstCharacter);
+ for (int posAhead = pos + 1; unknownWordLength < MAX_UNKNOWN_WORD_LENGTH; posAhead++) {
+ int next = buffer.get(posAhead);
+ if (next == -1) {
+ break;
+ }
+ char ch = (char) next;
+ int chType = Character.getType(ch);
+ Character.UnicodeScript sc = Character.UnicodeScript.of(next);
+ boolean sameScript =
+ isSameScript(scriptCode, sc)
+ // Non-spacing marks inherit the script of their base character,
+ // following recommendations from UTR #24.
+ || chType == Character.NON_SPACING_MARK;
+
+ if (sameScript
+ // split on punctuation
+ && isPunctuation(ch, chType) == isPunct
+ // split on digit
+ && Character.isDigit(ch) == isDigit
+ && characterDefinition.isGroup(ch)) {
+ unknownWordLength++;
+ } else {
+ break;
+ }
+ // Update the script code and character class if the original script
+ // is Inherited or Common.
+ if (isCommonOrInherited(scriptCode) && isCommonOrInherited(sc) == false) {
+ scriptCode = sc;
+ characterId = characterDefinition.getCharacterClass(ch);
+ }
+ }
+ }
+
+ unkDictionary.lookupWordIds(
+ characterId, wordIdRef); // characters in input text are supposed to be the same
+ if (VERBOSE) {
+ System.out.println(
+ " UNKNOWN word len=" + unknownWordLength + " " + wordIdRef.length + " wordIDs");
+ }
+ for (int ofs = 0; ofs < wordIdRef.length; ofs++) {
+ add(
+ unkDictionary.getMorphAttributes(),
+ posData,
+ pos,
+ pos + unknownWordLength,
+ wordIdRef.ints[wordIdRef.offset + ofs],
+ TokenType.UNKNOWN,
+ false);
+ }
+ }
+ // TODO: should return meaningful value?
+ return 0;
+ }
+
+ void setGraphvizFormatter(GraphvizFormatter<KoMorphData> dotOut) {
+ this.dotOut = dotOut;
+ }
+
+ @Override
+ protected void backtrace(Position endPosData, int fromIDX) {
+ final int endPos = endPosData.getPos();
+
+ if (endPos == lastBackTracePos) {
+ return;
+ }
+
+ if (VERBOSE) {
+ System.out.println(
+ "\n backtrace: endPos="
+ + endPos
+ + " pos="
+ + pos
+ + "; "
+ + (pos - lastBackTracePos)
+ + " characters; last="
+ + lastBackTracePos
+ + " cost="
+ + endPosData.getCost(fromIDX));
+ }
+
+ final char[] fragment = buffer.get(lastBackTracePos, endPos - lastBackTracePos);
+
+ if (dotOut != null) {
+ dotOut.onBacktrace(
+ this::getDict, positions, lastBackTracePos, endPosData, fromIDX, fragment, end);
+ }
+
+ int pos = endPos;
+ int bestIDX = fromIDX;
+
+ // TODO: sort of silly to make Token instances here; the
+ // back trace has all info needed to generate the
+ // token. So, we could just directly set the attrs,
+ // from the backtrace, in incrementToken w/o ever
+ // creating Token; we'd have to defer calling freeBefore
+ // until after the backtrace was fully "consumed" by
+ // incrementToken.
+
+ while (pos > lastBackTracePos) {
+ // System.out.println("BT: back pos=" + pos + " bestIDX=" + bestIDX);
+ final Position posData = positions.get(pos);
+ assert bestIDX < posData.getCount();
+
+ int backPos = posData.getBackPos(bestIDX);
+ int backWordPos = posData.getBackWordPos(bestIDX);
+ assert backPos >= lastBackTracePos
+ : "backPos=" + backPos + " vs lastBackTracePos=" + lastBackTracePos;
+ // the length of the word without the whitespaces at the beginning.
+ int length = pos - backWordPos;
+ TokenType backType = posData.getBackType(bestIDX);
+ int backID = posData.getBackID(bestIDX);
+ int nextBestIDX = posData.getBackIndex(bestIDX);
+ // the start of the word after the whitespace at the beginning.
+ final int fragmentOffset = backWordPos - lastBackTracePos;
+ assert fragmentOffset >= 0;
+
+ final Dictionary<? extends KoMorphData> dict = getDict(backType);
+
+ if (outputUnknownUnigrams && backType == TokenType.UNKNOWN) {
+ // outputUnknownUnigrams converts unknown word into unigrams:
+ for (int i = length - 1; i >= 0; i--) {
+ int charLen = 1;
+ if (i > 0 && Character.isLowSurrogate(fragment[fragmentOffset + i])) {
+ i--;
+ charLen = 2;
+ }
+ final DictionaryToken token =
+ new DictionaryToken(
+ TokenType.UNKNOWN,
+ unkDictionary.getMorphAttributes(),
+ CharacterDefinition.NGRAM,
+ fragment,
+ fragmentOffset + i,
+ charLen,
+ backWordPos + i,
+ backWordPos + i + charLen);
+ pending.add(token);
+ if (VERBOSE) {
+ System.out.println(" add token=" + pending.get(pending.size() - 1));
+ }
+ }
+ } else {
+ final DictionaryToken token =
+ new DictionaryToken(
+ backType,
+ dict.getMorphAttributes(),
+ backID,
+ fragment,
+ fragmentOffset,
+ length,
+ backWordPos,
+ backWordPos + length);
+ if (token.getPOSType() == POS.Type.MORPHEME
+ || mode == KoreanTokenizer.DecompoundMode.NONE) {
+ if (shouldFilterToken(token) == false) {
+ pending.add(token);
+ if (VERBOSE) {
+ System.out.println(" add token=" + pending.get(pending.size() - 1));
+ }
+ }
+ } else {
+ KoMorphData.Morpheme[] morphemes = token.getMorphemes();
+ if (morphemes == null) {
+ pending.add(token);
+ if (VERBOSE) {
+ System.out.println(" add token=" + pending.get(pending.size() - 1));
+ }
+ } else {
+ int endOffset = backWordPos + length;
+ int posLen = 0;
+ // decompose the compound
+ for (int i = morphemes.length - 1; i >= 0; i--) {
+ final KoMorphData.Morpheme morpheme = morphemes[i];
+ final Token compoundToken;
+ if (token.getPOSType() == POS.Type.COMPOUND) {
+ assert endOffset - morpheme.surfaceForm.length() >= 0;
+ compoundToken =
+ new DecompoundToken(
+ morpheme.posTag,
+ morpheme.surfaceForm,
+ endOffset - morpheme.surfaceForm.length(),
+ endOffset,
+ backType);
+ } else {
+ compoundToken =
+ new DecompoundToken(
+ morpheme.posTag,
+ morpheme.surfaceForm,
+ token.getStartOffset(),
+ token.getEndOffset(),
+ backType);
+ }
+ if (i == 0 && mode == KoreanTokenizer.DecompoundMode.MIXED) {
+ compoundToken.setPositionIncrement(0);
+ }
+ ++posLen;
+ endOffset -= morpheme.surfaceForm.length();
+ pending.add(compoundToken);
+ if (VERBOSE) {
+ System.out.println(" add token=" + pending.get(pending.size() - 1));
+ }
+ }
+ if (mode == KoreanTokenizer.DecompoundMode.MIXED) {
+ token.setPositionLength(Math.max(1, posLen));
+ pending.add(token);
+ if (VERBOSE) {
+ System.out.println(" add token=" + pending.get(pending.size() - 1));
+ }
+ }
+ }
+ }
+ }
+ if (discardPunctuation == false && backWordPos != backPos) {
+ // Add a token for whitespaces between terms
+ int offset = backPos - lastBackTracePos;
+ int len = backWordPos - backPos;
+ // System.out.println(offset + " " + fragmentOffset + " " + len + " " + backWordPos + " " +
+ // backPos);
+ unkDictionary.lookupWordIds(characterDefinition.getCharacterClass(' '), wordIdRef);
+ DictionaryToken spaceToken =
+ new DictionaryToken(
+ TokenType.UNKNOWN,
+ unkDictionary.getMorphAttributes(),
+ wordIdRef.ints[wordIdRef.offset],
+ fragment,
+ offset,
+ len,
+ backPos,
+ backPos + len);
+ pending.add(spaceToken);
+ }
+
+ pos = backPos;
+ bestIDX = nextBestIDX;
+ }
+
+ lastBackTracePos = endPos;
+
+ if (VERBOSE) {
+ System.out.println(" freeBefore pos=" + endPos);
+ }
+ // Notify the circular buffers that we are done with
+ // these positions:
+ buffer.freeBefore(endPos);
+ positions.freeBefore(endPos);
+ }
+
+ /** Returns the space penalty associated with the provided {@link POS.Tag}. */
+ @Override
+ protected int computeSpacePenalty(MorphData morphData, int wordID, int numSpaces) {
+ final POS.Tag leftPOS = ((KoMorphData) morphData).getLeftPOS(wordID);
+ int spacePenalty = 0;
+ if (numSpaces > 0) {
+ // TODO we should extract the penalty (left-space-penalty-factor) from the dicrc file.
+ switch (leftPOS) {
+ case E:
+ case J:
+ case VCP:
+ case XSA:
+ case XSN:
+ case XSV:
+ spacePenalty = 3000;
+ break;
+ case IC:
+ case MAG:
+ case MAJ:
+ case MM:
+ case NA:
+ case NNB:
+ case NNBC:
+ case NNG:
+ case NNP:
+ case NP:
+ case NR:
+ case SC:
+ case SE:
+ case SF:
+ case SH:
+ case SL:
+ case SN:
+ case SP:
+ case SSC:
+ case SSO:
+ case SY:
+ case UNA:
+ case UNKNOWN:
+ case VA:
+ case VCN:
+ case VSV:
+ case VV:
+ case VX:
+ case XPN:
+ case XR:
+ default:
+ break;
+ }
+ }
+ return spacePenalty;
+ }
+
+ Dictionary<? extends KoMorphData> getDict(TokenType type) {
+ return dictionaryMap.get(type);
+ }
+
+ private boolean shouldFilterToken(Token token) {
+ return discardPunctuation && isPunctuation(token.getSurfaceForm()[token.getOffset()]);
+ }
+
+ private static boolean isPunctuation(char ch) {
+ return isPunctuation(ch, Character.getType(ch));
+ }
+
+ private static boolean isPunctuation(char ch, int cid) {
+ // special case for Hangul Letter Araea (interpunct)
+ if (ch == 0x318D) {
+ return true;
+ }
+ switch (cid) {
+ case Character.SPACE_SEPARATOR:
+ case Character.LINE_SEPARATOR:
+ case Character.PARAGRAPH_SEPARATOR:
+ case Character.CONTROL:
+ case Character.FORMAT:
+ case Character.DASH_PUNCTUATION:
+ case Character.START_PUNCTUATION:
+ case Character.END_PUNCTUATION:
+ case Character.CONNECTOR_PUNCTUATION:
+ case Character.OTHER_PUNCTUATION:
+ case Character.MATH_SYMBOL:
+ case Character.CURRENCY_SYMBOL:
+ case Character.MODIFIER_SYMBOL:
+ case Character.OTHER_SYMBOL:
+ case Character.INITIAL_QUOTE_PUNCTUATION:
+ case Character.FINAL_QUOTE_PUNCTUATION:
+ return true;
+ default:
+ return false;
+ }
+ }
+
+ private static boolean isCommonOrInherited(Character.UnicodeScript script) {
+ return script == Character.UnicodeScript.INHERITED || script == Character.UnicodeScript.COMMON;
+ }
+
+ /** Determine if two scripts are compatible. */
+ private static boolean isSameScript(
+ Character.UnicodeScript scriptOne, Character.UnicodeScript scriptTwo) {
+ return scriptOne == scriptTwo
+ || isCommonOrInherited(scriptOne)
+ || isCommonOrInherited(scriptTwo);
+ }
+}