Apache Lucene open-source search software
LUCENE-10493: factor out Viterbi algorithm and share it between kuromoji and nori (#805)
Tomoko Uchidac89f8a7ea1e7
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex 76304b3eee9..8e1bfcc9cbd 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -60,6 +60,8 @@ OtherAll classes in `org.apache.lucene.analysis.[ja|ko].util` was moved to `org.apache.lucene.analysis.[ja|ko].dict`.(Tomoko Uchida)+* LUCENE-10493: Factor out Viterbi algorithm in Kuromoji and Nori to analysis-common. (Tomoko Uchida)+======================= Lucene 9.2.0 =======================API Changesdiff --git a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/GraphvizFormatter.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/GraphvizFormatter.javasimilarity index 67%rename from lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/GraphvizFormatter.javarename to lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/GraphvizFormatter.javaindex 9beaf2e5b9b..8279f0131d8 100644--- a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/GraphvizFormatter.java+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/GraphvizFormatter.java@@ -14,22 +14,16 @@* See the License for the specific language governing permissions and* limitations under the License.*/-package org.apache.lucene.analysis.ko;+package org.apache.lucene.analysis.morph;import java.util.HashMap;import java.util.Map;-import org.apache.lucene.analysis.ko.KoreanTokenizer.Position;-import org.apache.lucene.analysis.ko.KoreanTokenizer.WrappedPositionArray;-import org.apache.lucene.analysis.ko.dict.ConnectionCosts;-import org.apache.lucene.analysis.ko.dict.KoMorphData;-import org.apache.lucene.analysis.morph.Dictionary;// TODO: would be nice to show 2nd best path in a diff't// color.../** Outputs the dot (graphviz) string for the viterbi lattice. */-public class GraphvizFormatter {-+public class GraphvizFormatter<T extends MorphData> {private static final String BOS_LABEL = "BOS";private static final String EOS_LABEL = "EOS";@@ -56,36 +50,39 @@ public class GraphvizFormatter {}// Backtraces another incremental fragment:- void onBacktrace(- KoreanTokenizer tok,- WrappedPositionArray positions,+ public void onBacktrace(+ DictionaryProvider<T> dictProvider,+ Viterbi.WrappedPositionArray<? extends Viterbi.Position> positions,int lastBackTracePos,- Position endPosData,+ Viterbi.Position endPosData,int fromIDX,char[] fragment,boolean isEnd) {setBestPathMap(positions, lastBackTracePos, endPosData, fromIDX);- sb.append(formatNodes(tok, positions, lastBackTracePos, endPosData, fragment));+ sb.append(formatNodes(dictProvider, positions, lastBackTracePos, endPosData, fragment));if (isEnd) {sb.append(" fini [style=invis]\n");sb.append(" ");- sb.append(getNodeID(endPosData.pos, fromIDX));+ sb.append(getNodeID(endPosData.getPos(), fromIDX));sb.append(" -> fini [label=\"" + EOS_LABEL + "\"]");}}// Records which arcs make up the best bath:private void setBestPathMap(- WrappedPositionArray positions, int startPos, Position endPosData, int fromIDX) {+ Viterbi.WrappedPositionArray<? extends Viterbi.Position> positions,+ int startPos,+ Viterbi.Position endPosData,+ int fromIDX) {bestPathMap.clear();- int pos = endPosData.pos;+ int pos = endPosData.getPos();int bestIDX = fromIDX;while (pos > startPos) {- final Position posData = positions.get(pos);+ final Viterbi.Position posData = positions.get(pos);- final int backPos = posData.backPos[bestIDX];- final int backIDX = posData.backIndex[bestIDX];+ final int backPos = posData.getBackPos(bestIDX);+ final int backIDX = posData.getBackIndex(bestIDX);final String toNodeID = getNodeID(pos, bestIDX);final String fromNodeID = getNodeID(backPos, backIDX);@@ -99,34 +96,34 @@ public class GraphvizFormatter {}private String formatNodes(- KoreanTokenizer tok,- WrappedPositionArray positions,+ DictionaryProvider<T> dictProvider,+ Viterbi.WrappedPositionArray<? extends Viterbi.Position> positions,int startPos,- Position endPosData,+ Viterbi.Position endPosData,char[] fragment) {StringBuilder sb = new StringBuilder();// Output nodes- for (int pos = startPos + 1; pos <= endPosData.pos; pos++) {- final Position posData = positions.get(pos);- for (int idx = 0; idx < posData.count; idx++) {+ for (int pos = startPos + 1; pos <= endPosData.getPos(); pos++) {+ final Viterbi.Position posData = positions.get(pos);+ for (int idx = 0; idx < posData.getCount(); idx++) {sb.append(" ");sb.append(getNodeID(pos, idx));sb.append(" [label=\"");sb.append(pos);sb.append(": ");- sb.append(posData.lastRightID[idx]);+ sb.append(posData.getLastRightID(idx));sb.append("\"]\n");}}// Output arcs- for (int pos = endPosData.pos; pos > startPos; pos--) {- final Position posData = positions.get(pos);- for (int idx = 0; idx < posData.count; idx++) {- final Position backPosData = positions.get(posData.backPos[idx]);+ for (int pos = endPosData.getPos(); pos > startPos; pos--) {+ final Viterbi.Position posData = positions.get(pos);+ for (int idx = 0; idx < posData.getCount(); idx++) {+ final Viterbi.Position backPosData = positions.get(posData.getBackPos(idx));final String toNodeID = getNodeID(pos, idx);- final String fromNodeID = getNodeID(posData.backPos[idx], posData.backIndex[idx]);+ final String fromNodeID = getNodeID(posData.getBackPos(idx), posData.getBackIndex(idx));sb.append(" ");sb.append(fromNodeID);@@ -141,15 +138,15 @@ public class GraphvizFormatter {attrs = "";}- final Dictionary<? extends KoMorphData> dict = tok.getDict(posData.backType[idx]);- final int wordCost = dict.getWordCost(posData.backID[idx]);+ final Dictionary<? extends T> dict = dictProvider.get(posData.getBackType(idx));+ final int wordCost = dict.getWordCost(posData.getBackID(idx));final int bgCost =costs.get(- backPosData.lastRightID[posData.backIndex[idx]],- dict.getLeftId(posData.backID[idx]));+ backPosData.getLastRightID(posData.getBackIndex(idx)),+ dict.getLeftId(posData.getBackID(idx)));final String surfaceForm =- new String(fragment, posData.backPos[idx] - startPos, pos - posData.backPos[idx]);+ new String(fragment, posData.getBackPos(idx) - startPos, pos - posData.getBackPos(idx));sb.append(" [label=\"");sb.append(surfaceForm);@@ -190,4 +187,10 @@ public class GraphvizFormatter {private String getNodeID(int pos, int idx) {return pos + "." + idx;}++ /** {@link Dictionary} provider */+ @FunctionalInterface+ public interface DictionaryProvider<T extends MorphData> {+ Dictionary<? extends T> get(TokenType type);+ }}diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/Viterbi.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/Viterbi.javanew file mode 100644index 00000000000..f7533b93007--- /dev/null+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/Viterbi.java@@ -0,0 +1,815 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.analysis.morph;++import java.io.IOException;+import java.io.Reader;+import java.lang.reflect.Array;+import java.util.ArrayList;+import java.util.Arrays;+import java.util.List;+import org.apache.lucene.analysis.util.RollingCharBuffer;+import org.apache.lucene.util.ArrayUtil;+import org.apache.lucene.util.IntsRef;+import org.apache.lucene.util.RamUsageEstimator;+import org.apache.lucene.util.fst.FST;++/**+ * Performs <a href="https://en.wikipedia.org/wiki/Viterbi_algorithm">Viterbi algorithm</a> for+ * morphological Tokenizers, which split texts by Hidden Markov Model or Conditional Random Fields.+ *+ * @param <T> output token class+ * @param <U> position class+ */+public abstract class Viterbi<T extends Token, U extends Viterbi.Position> {+ protected static final boolean VERBOSE = false;++ // For safety:+ protected static final int MAX_UNKNOWN_WORD_LENGTH = 1024;+ private static final int MAX_BACKTRACE_GAP = 1024;++ private final TokenInfoFST fst;+ private final BinaryDictionary<? extends MorphData> dictionary;+ private final Dictionary<? extends MorphData> userDictionary;+ protected final ConnectionCosts costs;++ private final FST.Arc<Long> arc = new FST.Arc<>();+ private final FST.BytesReader fstReader;+ protected final IntsRef wordIdRef = new IntsRef();++ private final FST.BytesReader userFSTReader;+ private final TokenInfoFST userFST;++ protected final RollingCharBuffer buffer = new RollingCharBuffer();++ protected final WrappedPositionArray<U> positions;++ // True once we've hit the EOF from the input reader:+ protected boolean end;++ // Last absolute position we backtraced from:+ protected int lastBackTracePos;++ // Next absolute position to process:+ protected int pos;++ // Already parsed, but not yet passed to caller, tokens:+ protected final List<T> pending = new ArrayList<>();++ protected boolean outputNBest = false;++ protected boolean enableSpacePenaltyFactor = false;++ protected boolean outputLongestUserEntryOnly = false;++ protected Viterbi(+ TokenInfoFST fst,+ FST.BytesReader fstReader,+ BinaryDictionary<? extends MorphData> dictionary,+ TokenInfoFST userFST,+ FST.BytesReader userFSTReader,+ Dictionary<? extends MorphData> userDictionary,+ ConnectionCosts costs,+ Class<U> positionImpl) {+ this.fst = fst;+ this.fstReader = fstReader;+ this.dictionary = dictionary;+ this.userFST = userFST;+ this.userFSTReader = userFSTReader;+ this.userDictionary = userDictionary;+ this.costs = costs;+ this.positions = new WrappedPositionArray<>(positionImpl);+ }++ /**+ * Incrementally parse some more characters. This runs the viterbi search forwards "enough" so+ * that we generate some more tokens. How much forward depends on the chars coming in, since some+ * chars could cause longer-lasting ambiguity in the parsing. Once the ambiguity is resolved, then+ * we back trace, produce the pending tokens, and return.+ */+ public final void forward() throws IOException {+ if (VERBOSE) {+ System.out.println("\nPARSE");+ }++ // Index of the last character of unknown word:+ int unknownWordEndIndex = -1;++ // Maximum posAhead of user word in the entire input+ int userWordMaxPosAhead = -1;++ // Advances over each position (character):+ while (buffer.get(pos) != -1) {+ final Position posData = positions.get(pos);+ final boolean isFrontier = positions.getNextPos() == pos + 1;++ if (posData.count == 0) {+ // No arcs arrive here; move to next position:+ if (VERBOSE) {+ System.out.println(" no arcs in; skip pos=" + pos);+ }+ pos++;+ continue;+ }++ if (pos > lastBackTracePos && posData.count == 1 && isFrontier) {+ // We are at a "frontier", and only one node is+ // alive, so whatever the eventual best path is must+ // come through this node. So we can safely commit+ // to the prefix of the best path at this point:+ if (outputNBest) {+ backtraceNBest(posData, false);+ }+ backtrace(posData, 0);+ if (outputNBest) {+ fixupPendingList();+ }++ // Re-base cost so we don't risk int overflow:+ posData.costs[0] = 0;+ if (pending.size() > 0) {+ return;+ } else {+ // This means the backtrace only produced+ // punctuation tokens, so we must keep parsing.+ }+ }++ if (pos - lastBackTracePos >= MAX_BACKTRACE_GAP) {+ // Safety: if we've buffered too much, force a+ // backtrace now. We find the least-cost partial+ // path, across all paths, backtrace from it, and+ // then prune all others. Note that this, in+ // general, can produce the wrong result, if the+ // total best path did not in fact back trace+ // through this partial best path. But it's the+ // best we can do... (short of not having a+ // safety!).++ // First pass: find least cost partial path so far,+ // including ending at future positions:+ int leastIDX = -1;+ int leastCost = Integer.MAX_VALUE;+ Position leastPosData = null;+ for (int pos2 = pos; pos2 < positions.getNextPos(); pos2++) {+ final Position posData2 = positions.get(pos2);+ for (int idx = 0; idx < posData2.count; idx++) {+ // System.out.println(" idx=" + idx + " cost=" + cost);+ final int cost = posData2.costs[idx];+ if (cost < leastCost) {+ leastCost = cost;+ leastIDX = idx;+ leastPosData = posData2;+ }+ }+ }++ // We will always have at least one live path:+ assert leastIDX != -1;++ if (outputNBest) {+ backtraceNBest(leastPosData, false);+ }++ // Second pass: prune all but the best path:+ for (int pos2 = pos; pos2 < positions.getNextPos(); pos2++) {+ final Position posData2 = positions.get(pos2);+ if (posData2 != leastPosData) {+ posData2.reset();+ } else {+ if (leastIDX != 0) {+ posData2.costs[0] = posData2.costs[leastIDX];+ posData2.lastRightID[0] = posData2.lastRightID[leastIDX];+ posData2.backPos[0] = posData2.backPos[leastIDX];+ posData2.backWordPos[0] = posData2.backWordPos[leastIDX];+ posData2.backIndex[0] = posData2.backIndex[leastIDX];+ posData2.backID[0] = posData2.backID[leastIDX];+ posData2.backType[0] = posData2.backType[leastIDX];+ }+ posData2.count = 1;+ }+ }++ backtrace(leastPosData, 0);+ if (outputNBest) {+ fixupPendingList();+ }++ // Re-base cost so we don't risk int overflow:+ Arrays.fill(leastPosData.costs, 0, leastPosData.count, 0);++ if (pos != leastPosData.pos) {+ // We jumped into a future position:+ assert pos < leastPosData.pos;+ pos = leastPosData.pos;+ }+ if (pending.size() > 0) {+ return;+ } else {+ // This means the backtrace only produced+ // punctuation tokens, so we must keep parsing.+ continue;+ }+ }++ if (VERBOSE) {+ System.out.println(+ "\n extend @ pos="+ + pos+ + " char="+ + (char) buffer.get(pos)+ + " hex="+ + Integer.toHexString(buffer.get(pos)));+ }++ if (VERBOSE) {+ System.out.println(" " + posData.count + " arcs in");+ }++ if (enableSpacePenaltyFactor+ && Character.getType(buffer.get(pos)) == Character.SPACE_SEPARATOR) {+ // We add single space separator as prefixes of the terms that we extract.+ // This information is needed to compute the space penalty factor of each term.+ // These whitespace prefixes are removed when the final tokens are generated, or+ // added as separated tokens when discardPunctuation is unset.+ if (buffer.get(++pos) == -1) {+ pos = posData.pos;+ }+ }++ boolean anyMatches = false;++ // First try user dict:+ if (userFST != null) {+ userFST.getFirstArc(arc);+ int output = 0;+ int maxPosAhead = 0;+ int outputMaxPosAhead = 0;+ int arcFinalOutMaxPosAhead = 0;++ for (int posAhead = pos; ; posAhead++) {+ final int ch = buffer.get(posAhead);+ if (ch == -1) {+ break;+ }+ if (userFST.findTargetArc(ch, arc, arc, posAhead == pos, userFSTReader) == null) {+ break;+ }+ output += arc.output().intValue();+ if (arc.isFinal()) {+ maxPosAhead = posAhead;+ outputMaxPosAhead = output;+ arcFinalOutMaxPosAhead = arc.nextFinalOutput().intValue();+ anyMatches = true;+ if (!outputLongestUserEntryOnly) {+ // add all matched user entries.+ add(+ userDictionary.getMorphAttributes(),+ posData,+ pos,+ posAhead + 1,+ output + arc.nextFinalOutput().intValue(),+ TokenType.USER,+ false);+ }+ }+ }++ // Longest matching for user word+ if (anyMatches && maxPosAhead > userWordMaxPosAhead) {+ if (outputLongestUserEntryOnly) {+ if (VERBOSE) {+ System.out.println(+ " USER word "+ + new String(buffer.get(pos, maxPosAhead + 1))+ + " toPos="+ + (maxPosAhead + 1));+ }+ add(+ userDictionary.getMorphAttributes(),+ posData,+ pos,+ maxPosAhead + 1,+ outputMaxPosAhead + arcFinalOutMaxPosAhead,+ TokenType.USER,+ false);+ }+ userWordMaxPosAhead = Math.max(userWordMaxPosAhead, maxPosAhead);+ }+ }++ // TODO: we can be more aggressive about user+ // matches? if we are "under" a user match then don't+ // extend KNOWN/UNKNOWN paths?++ if (!anyMatches) {+ // Next, try known dictionary matches+ fst.getFirstArc(arc);+ int output = 0;++ for (int posAhead = pos; ; posAhead++) {+ final int ch = buffer.get(posAhead);+ if (ch == -1) {+ break;+ }+ // System.out.println(" match " + (char) ch + " posAhead=" + posAhead);++ if (fst.findTargetArc(ch, arc, arc, posAhead == pos, fstReader) == null) {+ break;+ }++ output += arc.output().intValue();++ // Optimization: for known words that are too-long+ // (compound), we should pre-compute the 2nd+ // best segmentation and store it in the+ // dictionary instead of recomputing it each time a+ // match is found.++ if (arc.isFinal()) {+ dictionary.lookupWordIds(output + arc.nextFinalOutput().intValue(), wordIdRef);+ if (VERBOSE) {+ System.out.println(+ " KNOWN word "+ + new String(buffer.get(pos, posAhead - pos + 1))+ + " toPos="+ + (posAhead + 1)+ + " "+ + wordIdRef.length+ + " wordIDs");+ }+ for (int ofs = 0; ofs < wordIdRef.length; ofs++) {+ add(+ dictionary.getMorphAttributes(),+ posData,+ pos,+ posAhead + 1,+ wordIdRef.ints[wordIdRef.offset + ofs],+ TokenType.KNOWN,+ false);+ anyMatches = true;+ }+ }+ }+ }++ if (!shouldSkipProcessUnknownWord(unknownWordEndIndex, posData)) {+ int unknownWordLength = processUnknownWord(anyMatches, posData);+ unknownWordEndIndex = posData.pos + unknownWordLength;+ }+ pos++;+ }++ end = true;++ if (pos > 0) {++ final Position endPosData = positions.get(pos);+ int leastCost = Integer.MAX_VALUE;+ int leastIDX = -1;+ if (VERBOSE) {+ System.out.println(" end: " + endPosData.count + " nodes");+ }+ for (int idx = 0; idx < endPosData.count; idx++) {+ // Add EOS cost:+ final int cost = endPosData.costs[idx] + costs.get(endPosData.lastRightID[idx], 0);+ // System.out.println(" idx=" + idx + " cost=" + cost + " (pathCost=" ++ // endPosData.costs[idx] + " bgCost=" + costs.get(endPosData.lastRightID[idx], 0) + ")+ // backPos=" + endPosData.backPos[idx]);+ if (cost < leastCost) {+ leastCost = cost;+ leastIDX = idx;+ }+ }++ if (outputNBest) {+ backtraceNBest(endPosData, true);+ }+ backtrace(endPosData, leastIDX);+ if (outputNBest) {+ fixupPendingList();+ }+ } else {+ // No characters in the input string; return no tokens!+ }+ }++ protected boolean shouldSkipProcessUnknownWord(int unknownWordEndIndex, Position posData) {+ return unknownWordEndIndex > posData.pos;+ }++ /**+ * Add unknown words to the position graph.+ *+ * @return word length+ */+ protected abstract int processUnknownWord(boolean anyMatches, Position posData)+ throws IOException;++ /**+ * Backtrace from the provided position, back to the last time we back-traced, accumulating the+ * resulting tokens to the pending list. The pending list is then in-reverse (last token should be+ * returned first).+ */+ protected abstract void backtrace(final Position endPosData, final int fromIDX)+ throws IOException;++ /**+ * Backtrace the n-best path. Subclasses that support n-best paths should implement this method.+ */+ protected void backtraceNBest(final Position endPosData, final boolean useEOS)+ throws IOException {+ throw new UnsupportedOperationException();+ }++ /**+ * Remove duplicated tokens from the pending list; this is needed because {@link+ * #backtrace(Position, int)} and {@link #backtraceNBest(Position, boolean)} can add same tokens+ * to the list. Subclasses that support n-best paths should implement this method.+ */+ protected void fixupPendingList() {+ throw new UnsupportedOperationException();+ }++ /** Add a token on the minimum cost path to the pending token list. */+ protected final void add(+ MorphData morphData,+ Position fromPosData,+ int wordPos,+ int endPos,+ int wordID,+ TokenType type,+ boolean addPenalty)+ throws IOException {+ final int wordCost = morphData.getWordCost(wordID);+ final int leftID = morphData.getLeftId(wordID);+ int leastCost = Integer.MAX_VALUE;+ int leastIDX = -1;+ assert fromPosData.count > 0;+ for (int idx = 0; idx < fromPosData.count; idx++) {+ // The number of spaces before the term+ int numSpaces = wordPos - fromPosData.pos;++ // Cost is path cost so far, plus word cost (added at+ // end of loop), plus bigram cost and space penalty cost.+ final int cost =+ fromPosData.costs[idx]+ + costs.get(fromPosData.lastRightID[idx], leftID)+ + computeSpacePenalty(morphData, wordID, numSpaces);+ if (VERBOSE) {+ System.out.println(+ " fromIDX="+ + idx+ + ": cost="+ + cost+ + " (prevCost="+ + fromPosData.costs[idx]+ + " wordCost="+ + wordCost+ + " bgCost="+ + costs.get(fromPosData.lastRightID[idx], leftID)+ + " spacePenalty="+ + computeSpacePenalty(morphData, wordID, numSpaces)+ + ") leftID="+ + leftID+ // + " leftPOS="+ // + leftPOS.name()+ + ")");+ }+ if (cost < leastCost) {+ leastCost = cost;+ leastIDX = idx;+ if (VERBOSE) {+ System.out.println(" **");+ }+ }+ }++ leastCost += wordCost;++ if (VERBOSE) {+ System.out.println(+ " + cost="+ + leastCost+ + " wordID="+ + wordID+ + " leftID="+ + leftID+ + " leastIDX="+ + leastIDX+ + " toPos="+ + endPos+ + " toPos.idx="+ + positions.get(endPos).count);+ }++ if (addPenalty && type != TokenType.USER) {+ final int penalty = computePenalty(fromPosData.pos, endPos - fromPosData.pos);+ if (VERBOSE) {+ if (penalty > 0) {+ System.out.println(" + penalty=" + penalty + " cost=" + (leastCost + penalty));+ }+ }+ leastCost += penalty;+ }++ positions+ .get(endPos)+ .add(+ leastCost,+ morphData.getRightId(wordID),+ fromPosData.pos,+ wordPos,+ leastIDX,+ wordID,+ type);+ }++ /** Returns the space penalty. */+ protected int computeSpacePenalty(MorphData morphData, int wordID, int numSpaces) {+ return 0;+ }++ /** Returns the penalty for a specific input region */+ protected int computePenalty(int pos, int length) throws IOException {+ return 0;+ }++ public int getPos() {+ return pos;+ }++ public boolean isEnd() {+ return end;+ }++ public List<T> getPending() {+ return pending;+ }++ public boolean isOutputNBest() {+ return outputNBest;+ }++ public void resetBuffer(Reader reader) {+ buffer.reset(reader);+ }++ public void resetState() {+ positions.reset();+ pos = 0;+ end = false;+ lastBackTracePos = 0;+ pending.clear();++ // Add BOS:+ positions.get(0).add(0, 0, -1, -1, -1, -1, TokenType.KNOWN);+ }++ /**+ * Holds all back pointers arriving to this position.+ *+ * <p>NOTE: This and subclasses must have no-arg constructor. See {@link WrappedPositionArray}.+ */+ public static class Position {++ int pos;++ int count;++ // maybe single int array * 5?+ int[] costs = new int[8];+ int[] lastRightID = new int[8];+ int[] backPos = new int[8];+ int[] backWordPos = new int[8];+ int[] backIndex = new int[8];+ int[] backID = new int[8];+ TokenType[] backType = new TokenType[8];++ private void grow() {+ costs = ArrayUtil.grow(costs, 1 + count);+ lastRightID = ArrayUtil.grow(lastRightID, 1 + count);+ backPos = ArrayUtil.grow(backPos, 1 + count);+ backWordPos = ArrayUtil.grow(backWordPos, 1 + count);+ backIndex = ArrayUtil.grow(backIndex, 1 + count);+ backID = ArrayUtil.grow(backID, 1 + count);++ // NOTE: sneaky: grow separately because+ // ArrayUtil.grow will otherwise pick a different+ // length than the int[]s we just grew:+ final TokenType[] newBackType = new TokenType[backID.length];+ System.arraycopy(backType, 0, newBackType, 0, backType.length);+ backType = newBackType;+ }++ public void add(+ int cost,+ int lastRightID,+ int backPos,+ int backRPos,+ int backIndex,+ int backID,+ TokenType backType) {+ // NOTE: this isn't quite a true Viterbi search,+ // because we should check if lastRightID is+ // already present here, and only update if the new+ // cost is less than the current cost, instead of+ // simply appending. However, that will likely hurt+ // performance (usually we add a lastRightID only once),+ // and it means we actually create the full graph+ // intersection instead of a "normal" Viterbi lattice:+ if (count == costs.length) {+ grow();+ }+ this.costs[count] = cost;+ this.lastRightID[count] = lastRightID;+ this.backPos[count] = backPos;+ this.backWordPos[count] = backRPos;+ this.backIndex[count] = backIndex;+ this.backID[count] = backID;+ this.backType[count] = backType;+ count++;+ }++ public void reset() {+ count = 0;+ }++ public int getPos() {+ return pos;+ }++ public int getCount() {+ return count;+ }++ public void setCount(int count) {+ this.count = count;+ }++ public int getCost(int index) {+ return costs[index];+ }++ public int getBackPos(int index) {+ return backPos[index];+ }++ public int getBackWordPos(int index) {+ return backWordPos[index];+ }++ public int getBackID(int index) {+ return backID[index];+ }++ public int getBackIndex(int index) {+ return backIndex[index];+ }++ public TokenType getBackType(int index) {+ return backType[index];+ }++ public int getLastRightID(int index) {+ return lastRightID[index];+ }+ }++ /** Holds partial graph (array of positions) for calculating the minimum cost path */+ public static final class WrappedPositionArray<U extends Position> {+ private U[] positions;+ private final Class<U> clazz;++ @SuppressWarnings("unchecked")+ WrappedPositionArray(Class<U> clazz) {+ this.clazz = clazz;+ positions = (U[]) Array.newInstance(clazz, 8);+ for (int i = 0; i < positions.length; i++) {+ try {+ positions[i] = clazz.getConstructor().newInstance();+ } catch (ReflectiveOperationException e) {+ // shouldn't happen; Position class should have no-arg constructor.+ throw new IllegalStateException(e);+ }+ }+ }++ // Next array index to write to in positions:+ private int nextWrite;++ // Next position to write:+ private int nextPos;++ // How many valid Position instances are held in the+ // positions array:+ private int count;++ void reset() {+ nextWrite--;+ while (count > 0) {+ if (nextWrite == -1) {+ nextWrite = positions.length - 1;+ }+ positions[nextWrite--].reset();+ count--;+ }+ nextWrite = 0;+ nextPos = 0;+ count = 0;+ }++ /**+ * Get Position instance for this absolute position; this is allowed to be arbitrarily far "in+ * the future" but cannot be before the last freeBefore.+ */+ @SuppressWarnings("unchecked")+ public U get(int pos) {+ while (pos >= nextPos) {+ // System.out.println("count=" + count + " vs len=" + positions.length);+ if (count == positions.length) {+ // Position[] newPositions =+ // new Position[ArrayUtil.oversize(1 + count, RamUsageEstimator.NUM_BYTES_OBJECT_REF)];+ U[] newPositions =+ (U[])+ Array.newInstance(+ clazz, ArrayUtil.oversize(1 + count, RamUsageEstimator.NUM_BYTES_OBJECT_REF));+ // System.out.println("grow positions " + newPositions.length);+ System.arraycopy(positions, nextWrite, newPositions, 0, positions.length - nextWrite);+ System.arraycopy(positions, 0, newPositions, positions.length - nextWrite, nextWrite);+ for (int i = positions.length; i < newPositions.length; i++) {+ try {+ newPositions[i] = clazz.getConstructor().newInstance();+ } catch (ReflectiveOperationException e) {+ // shouldn't happen+ throw new IllegalStateException(e);+ }+ }+ nextWrite = positions.length;+ positions = newPositions;+ }+ if (nextWrite == positions.length) {+ nextWrite = 0;+ }+ // Should have already been reset:+ assert positions[nextWrite].count == 0;+ positions[nextWrite++].pos = nextPos++;+ count++;+ }+ assert inBounds(pos);+ final int index = getIndex(pos);+ assert positions[index].pos == pos;+ return positions[index];+ }++ int getNextPos() {+ return nextPos;+ }++ // For assert:+ private boolean inBounds(int pos) {+ return pos < nextPos && pos >= nextPos - count;+ }++ private int getIndex(int pos) {+ int index = nextWrite - (nextPos - pos);+ if (index < 0) {+ index += positions.length;+ }+ return index;+ }++ public void freeBefore(int pos) {+ final int toFree = count - (nextPos - pos);+ assert toFree >= 0;+ assert toFree <= count;+ int index = nextWrite - count;+ if (index < 0) {+ index += positions.length;+ }+ for (int i = 0; i < toFree; i++) {+ if (index == positions.length) {+ index = 0;+ }+ // System.out.println(" fb idx=" + index);+ positions[index].reset();+ index++;+ }+ count -= toFree;+ }+ }+}diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/GraphvizFormatter.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/GraphvizFormatter.javadeleted file mode 100644index 4538daa362f..00000000000--- a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/GraphvizFormatter.java+++ /dev/null@@ -1,195 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.analysis.ja;--import java.util.HashMap;-import java.util.Map;-import org.apache.lucene.analysis.ja.JapaneseTokenizer.Position;-import org.apache.lucene.analysis.ja.JapaneseTokenizer.WrappedPositionArray;-import org.apache.lucene.analysis.ja.dict.ConnectionCosts;-import org.apache.lucene.analysis.ja.dict.JaMorphData;-import org.apache.lucene.analysis.morph.Dictionary;--// TODO: would be nice to show 2nd best path in a diff't-// color...--/** Outputs the dot (graphviz) string for the viterbi lattice. */-public class GraphvizFormatter {-- private static final String BOS_LABEL = "BOS";-- private static final String EOS_LABEL = "EOS";-- private static final String FONT_NAME = "Helvetica";-- private final ConnectionCosts costs;-- private final Map<String, String> bestPathMap;-- private final StringBuilder sb = new StringBuilder();-- public GraphvizFormatter(ConnectionCosts costs) {- this.costs = costs;- this.bestPathMap = new HashMap<>();- sb.append(formatHeader());- sb.append(" init [style=invis]\n");- sb.append(" init -> 0.0 [label=\"" + BOS_LABEL + "\"]\n");- }-- public String finish() {- sb.append(formatTrailer());- return sb.toString();- }-- // Backtraces another incremental fragment:- void onBacktrace(- JapaneseTokenizer tok,- WrappedPositionArray positions,- int lastBackTracePos,- Position endPosData,- int fromIDX,- char[] fragment,- boolean isEnd) {- setBestPathMap(positions, lastBackTracePos, endPosData, fromIDX);- sb.append(formatNodes(tok, positions, lastBackTracePos, endPosData, fragment));- if (isEnd) {- sb.append(" fini [style=invis]\n");- sb.append(" ");- sb.append(getNodeID(endPosData.pos, fromIDX));- sb.append(" -> fini [label=\"" + EOS_LABEL + "\"]");- }- }-- // Records which arcs make up the best bath:- private void setBestPathMap(- WrappedPositionArray positions, int startPos, Position endPosData, int fromIDX) {- bestPathMap.clear();-- int pos = endPosData.pos;- int bestIDX = fromIDX;- while (pos > startPos) {- final Position posData = positions.get(pos);-- final int backPos = posData.backPos[bestIDX];- final int backIDX = posData.backIndex[bestIDX];-- final String toNodeID = getNodeID(pos, bestIDX);- final String fromNodeID = getNodeID(backPos, backIDX);-- assert !bestPathMap.containsKey(fromNodeID);- assert !bestPathMap.containsValue(toNodeID);- bestPathMap.put(fromNodeID, toNodeID);- pos = backPos;- bestIDX = backIDX;- }- }-- private String formatNodes(- JapaneseTokenizer tok,- WrappedPositionArray positions,- int startPos,- Position endPosData,- char[] fragment) {-- StringBuilder sb = new StringBuilder();- // Output nodes- for (int pos = startPos + 1; pos <= endPosData.pos; pos++) {- final Position posData = positions.get(pos);- for (int idx = 0; idx < posData.count; idx++) {- sb.append(" ");- sb.append(getNodeID(pos, idx));- sb.append(" [label=\"");- sb.append(pos);- sb.append(": ");- sb.append(posData.lastRightID[idx]);- sb.append("\"]\n");- }- }-- // Output arcs- for (int pos = endPosData.pos; pos > startPos; pos--) {- final Position posData = positions.get(pos);- for (int idx = 0; idx < posData.count; idx++) {- final Position backPosData = positions.get(posData.backPos[idx]);- final String toNodeID = getNodeID(pos, idx);- final String fromNodeID = getNodeID(posData.backPos[idx], posData.backIndex[idx]);-- sb.append(" ");- sb.append(fromNodeID);- sb.append(" -> ");- sb.append(toNodeID);-- final String attrs;- if (toNodeID.equals(bestPathMap.get(fromNodeID))) {- // This arc is on best path- attrs = " color=\"#40e050\" fontcolor=\"#40a050\" penwidth=3 fontsize=20";- } else {- attrs = "";- }-- final Dictionary<? extends JaMorphData> dict = tok.getDict(posData.backType[idx]);- final int wordCost = dict.getWordCost(posData.backID[idx]);- final int bgCost =- costs.get(- backPosData.lastRightID[posData.backIndex[idx]],- dict.getLeftId(posData.backID[idx]));-- final String surfaceForm =- new String(fragment, posData.backPos[idx] - startPos, pos - posData.backPos[idx]);-- sb.append(" [label=\"");- sb.append(surfaceForm);- sb.append(' ');- sb.append(wordCost);- if (bgCost >= 0) {- sb.append('+');- }- sb.append(bgCost);- sb.append("\"");- sb.append(attrs);- sb.append("]\n");- }- }- return sb.toString();- }-- private String formatHeader() {- StringBuilder sb = new StringBuilder();- sb.append("digraph viterbi {\n");- sb.append(- " graph [ fontsize=30 labelloc=\"t\" label=\"\" splines=true overlap=false rankdir = \"LR\"];\n");- // sb.append(" // A2 paper size\n");- // sb.append(" size = \"34.4,16.5\";\n");- // sb.append(" // try to fill paper\n");- // sb.append(" ratio = fill;\n");- sb.append(" edge [ fontname=\"" + FONT_NAME + "\" fontcolor=\"red\" color=\"#606060\" ]\n");- sb.append(- " node [ style=\"filled\" fillcolor=\"#e8e8f0\" shape=\"Mrecord\" fontname=\""- + FONT_NAME- + "\" ]\n");-- return sb.toString();- }-- private String formatTrailer() {- return "}";- }-- private String getNodeID(int pos, int idx) {- return pos + "." + idx;- }-}diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizer.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizer.javaindex b06b19fc359..71d19f02e1f 100644--- a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizer.java+++ b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizer.java@@ -18,13 +18,6 @@ package org.apache.lucene.analysis.ja;import java.io.IOException;import java.io.StringReader;-import java.util.ArrayList;-import java.util.Arrays;-import java.util.Collections;-import java.util.Comparator;-import java.util.EnumMap;-import java.util.HashMap;-import java.util.List;import org.apache.lucene.analysis.Tokenizer;import org.apache.lucene.analysis.ja.dict.CharacterDefinition;import org.apache.lucene.analysis.ja.dict.ConnectionCosts;@@ -34,23 +27,15 @@ import org.apache.lucene.analysis.ja.dict.TokenInfoFST;import org.apache.lucene.analysis.ja.dict.UnknownDictionary;import org.apache.lucene.analysis.ja.dict.UserDictionary;import org.apache.lucene.analysis.ja.tokenattributes.*;-import org.apache.lucene.analysis.morph.Dictionary;-import org.apache.lucene.analysis.morph.TokenType;+import org.apache.lucene.analysis.morph.GraphvizFormatter;import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;import org.apache.lucene.analysis.tokenattributes.PositionLengthAttribute;-import org.apache.lucene.analysis.util.RollingCharBuffer;-import org.apache.lucene.util.ArrayUtil;import org.apache.lucene.util.AttributeFactory;import org.apache.lucene.util.IgnoreRandomChains;-import org.apache.lucene.util.IntsRef;-import org.apache.lucene.util.RamUsageEstimator;import org.apache.lucene.util.fst.FST;-// TODO: somehow factor out a reusable viterbi search here,-// so other decompounders/tokenizers can reuse...-/*** Tokenizer for Japanese that uses morphological analysis.*@@ -96,63 +81,11 @@ public final class JapaneseTokenizer extends Tokenizer {private static final boolean VERBOSE = false;- private static final int SEARCH_MODE_KANJI_LENGTH = 2;-- private static final int SEARCH_MODE_OTHER_LENGTH = 7; // Must be >= SEARCH_MODE_KANJI_LENGTH-- private static final int SEARCH_MODE_KANJI_PENALTY = 3000;-- private static final int SEARCH_MODE_OTHER_PENALTY = 1700;-- // For safety:- private static final int MAX_UNKNOWN_WORD_LENGTH = 1024;- private static final int MAX_BACKTRACE_GAP = 1024;-- private final EnumMap<TokenType, Dictionary<? extends JaMorphData>> dictionaryMap =- new EnumMap<>(TokenType.class);-- private final TokenInfoFST fst;- private final TokenInfoDictionary dictionary;- private final UnknownDictionary unkDictionary;- private final ConnectionCosts costs;- private final UserDictionary userDictionary;- private final CharacterDefinition characterDefinition;-- private final FST.Arc<Long> arc = new FST.Arc<>();- private final FST.BytesReader fstReader;- private final IntsRef wordIdRef = new IntsRef();-- private final FST.BytesReader userFSTReader;- private final TokenInfoFST userFST;-- private final RollingCharBuffer buffer = new RollingCharBuffer();-- private final WrappedPositionArray positions = new WrappedPositionArray();-- private final boolean discardPunctuation;- private final boolean searchMode;- private final boolean extendedMode;- private final boolean outputCompounds;- private boolean outputNBest = false;-- // Allowable cost difference for N-best output:- private int nBestCost = 0;-- // True once we've hit the EOF from the input reader:- private boolean end;-- // Last absolute position we backtraced from:- private int lastBackTracePos;-// Position of last token we returned; we use this to// figure out whether to set posIncr to 0 or 1:private int lastTokenPos;- // Next absolute position to process:- private int pos;-- // Already parsed, but not yet passed to caller, tokens:- private final List<Token> pending = new ArrayList<>();+ private final ViterbiNBest viterbi;private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);@@ -280,21 +213,17 @@ public final class JapaneseTokenizer extends Tokenizer {boolean discardCompoundToken,Mode mode) {super(factory);- this.dictionary = systemDictionary;- this.fst = dictionary.getFST();- this.unkDictionary = unkDictionary;- this.characterDefinition = unkDictionary.getCharacterDefinition();- this.userDictionary = userDictionary;- this.costs = connectionCosts;- fstReader = fst.getBytesReader();+ TokenInfoFST fst = systemDictionary.getFST();+ FST.BytesReader fstReader = fst.getBytesReader();+ TokenInfoFST userFST = null;+ FST.BytesReader userFSTReader = null;if (userDictionary != null) {userFST = userDictionary.getFST();userFSTReader = userFST.getBytesReader();- } else {- userFST = null;- userFSTReader = null;}- this.discardPunctuation = discardPunctuation;+ boolean searchMode;+ boolean extendedMode;+ boolean outputCompounds;switch (mode) {case SEARCH:searchMode = true;@@ -313,282 +242,70 @@ public final class JapaneseTokenizer extends Tokenizer {outputCompounds = false;break;}- buffer.reset(this.input);-- resetState();-- dictionaryMap.put(TokenType.KNOWN, dictionary);- dictionaryMap.put(TokenType.UNKNOWN, unkDictionary);- dictionaryMap.put(TokenType.USER, userDictionary);+ CharacterDefinition characterDefinition = unkDictionary.getCharacterDefinition();+ this.viterbi =+ new ViterbiNBest(+ fst,+ fstReader,+ systemDictionary,+ userFST,+ userFSTReader,+ userDictionary,+ connectionCosts,+ ViterbiNBest.PositionNBest.class,+ unkDictionary,+ characterDefinition,+ discardPunctuation,+ searchMode,+ extendedMode,+ outputCompounds);+ viterbi.resetBuffer(this.input);+ viterbi.resetState();}- private GraphvizFormatter dotOut;-/** Expert: set this to produce graphviz (dot) output of the Viterbi lattice */- public void setGraphvizFormatter(GraphvizFormatter dotOut) {- this.dotOut = dotOut;+ public void setGraphvizFormatter(GraphvizFormatter<JaMorphData> dotOut) {+ viterbi.setGraphvizFormatter(dotOut);}@Overridepublic void close() throws IOException {super.close();- buffer.reset(input);+ viterbi.resetBuffer(input);}@Overridepublic void reset() throws IOException {super.reset();- buffer.reset(input);- resetState();- }-- private void resetState() {- positions.reset();- pos = 0;- end = false;- lastBackTracePos = 0;+ viterbi.resetBuffer(input);+ viterbi.resetState();lastTokenPos = -1;- pending.clear();-- // Add BOS:- positions.get(0).add(0, 0, -1, -1, -1, -1, TokenType.KNOWN);}@Overridepublic void end() throws IOException {super.end();// Set final offset- int finalOffset = correctOffset(pos);+ int finalOffset = correctOffset(viterbi.getPos());offsetAtt.setOffset(finalOffset, finalOffset);}- // Returns the added cost that a 2nd best segmentation is- // allowed to have. Ie, if we see path with cost X,- // ending in a compound word, and this method returns- // threshold > 0, then we will also find the 2nd best- // segmentation and if its path score is within this- // threshold of X, we'll include it in the output:- private int computeSecondBestThreshold(int pos, int length) throws IOException {- // TODO: maybe we do something else here, instead of just- // using the penalty...? EG we can be more aggressive on- // when to also test for 2nd best path- return computePenalty(pos, length);- }-- private int computePenalty(int pos, int length) throws IOException {- if (length > SEARCH_MODE_KANJI_LENGTH) {- boolean allKanji = true;- // check if node consists of only kanji- final int endPos = pos + length;- for (int pos2 = pos; pos2 < endPos; pos2++) {- if (!characterDefinition.isKanji((char) buffer.get(pos2))) {- allKanji = false;- break;- }- }- if (allKanji) { // Process only Kanji keywords- return (length - SEARCH_MODE_KANJI_LENGTH) * SEARCH_MODE_KANJI_PENALTY;- } else if (length > SEARCH_MODE_OTHER_LENGTH) {- return (length - SEARCH_MODE_OTHER_LENGTH) * SEARCH_MODE_OTHER_PENALTY;- }- }- return 0;- }-- // Holds all back pointers arriving to this position:- static final class Position {-- int pos;-- int count;-- // maybe single int array * 5?- int[] costs = new int[8];- int[] lastRightID = new int[8];- int[] backPos = new int[8];- int[] backWordPos = new int[8];- int[] backIndex = new int[8];- int[] backID = new int[8];- TokenType[] backType = new TokenType[8];-- // Only used when finding 2nd best segmentation under a- // too-long token:- int forwardCount;- int[] forwardPos = new int[8];- int[] forwardID = new int[8];- int[] forwardIndex = new int[8];- TokenType[] forwardType = new TokenType[8];-- public void grow() {- costs = ArrayUtil.grow(costs, 1 + count);- lastRightID = ArrayUtil.grow(lastRightID, 1 + count);- backPos = ArrayUtil.grow(backPos, 1 + count);- backWordPos = ArrayUtil.grow(backWordPos, 1 + count);- backIndex = ArrayUtil.grow(backIndex, 1 + count);- backID = ArrayUtil.grow(backID, 1 + count);-- // NOTE: sneaky: grow separately because- // ArrayUtil.grow will otherwise pick a different- // length than the int[]s we just grew:- final TokenType[] newBackType = new TokenType[backID.length];- System.arraycopy(backType, 0, newBackType, 0, backType.length);- backType = newBackType;- }-- public void growForward() {- forwardPos = ArrayUtil.grow(forwardPos, 1 + forwardCount);- forwardID = ArrayUtil.grow(forwardID, 1 + forwardCount);- forwardIndex = ArrayUtil.grow(forwardIndex, 1 + forwardCount);-- // NOTE: sneaky: grow separately because- // ArrayUtil.grow will otherwise pick a different- // length than the int[]s we just grew:- final TokenType[] newForwardType = new TokenType[forwardPos.length];- System.arraycopy(forwardType, 0, newForwardType, 0, forwardType.length);- forwardType = newForwardType;- }-- public void add(- int cost,- int lastRightID,- int backPos,- int backRPos,- int backIndex,- int backID,- TokenType backType) {- // NOTE: this isn't quite a true Viterbi search,- // because we should check if lastRightID is- // already present here, and only update if the new- // cost is less than the current cost, instead of- // simply appending. However, that will likely hurt- // performance (usually we add a lastRightID only once),- // and it means we actually create the full graph- // intersection instead of a "normal" Viterbi lattice:- if (count == costs.length) {- grow();- }- this.costs[count] = cost;- this.lastRightID[count] = lastRightID;- this.backPos[count] = backPos;- this.backWordPos[count] = backRPos;- this.backIndex[count] = backIndex;- this.backID[count] = backID;- this.backType[count] = backType;- count++;- }-- public void addForward(int forwardPos, int forwardIndex, int forwardID, TokenType forwardType) {- if (forwardCount == this.forwardID.length) {- growForward();- }- this.forwardPos[forwardCount] = forwardPos;- this.forwardIndex[forwardCount] = forwardIndex;- this.forwardID[forwardCount] = forwardID;- this.forwardType[forwardCount] = forwardType;- forwardCount++;- }-- public void reset() {- count = 0;- // forwardCount naturally resets after it runs:- assert forwardCount == 0 : "pos=" + pos + " forwardCount=" + forwardCount;- }- }-- private void add(- JaMorphData morphAtts,- Position fromPosData,- int wordPos,- int endPos,- int wordID,- TokenType type,- boolean addPenalty)- throws IOException {- final int wordCost = morphAtts.getWordCost(wordID);- final int leftID = morphAtts.getLeftId(wordID);- int leastCost = Integer.MAX_VALUE;- int leastIDX = -1;- assert fromPosData.count > 0;- for (int idx = 0; idx < fromPosData.count; idx++) {- // Cost is path cost so far, plus word cost (added at- // end of loop), plus bigram cost:- final int cost = fromPosData.costs[idx] + costs.get(fromPosData.lastRightID[idx], leftID);- if (VERBOSE) {- System.out.println(- " fromIDX="- + idx- + ": cost="- + cost- + " (prevCost="- + fromPosData.costs[idx]- + " wordCost="- + wordCost- + " bgCost="- + costs.get(fromPosData.lastRightID[idx], leftID)- + " leftID="- + leftID- + ")");- }- if (cost < leastCost) {- leastCost = cost;- leastIDX = idx;- if (VERBOSE) {- System.out.println(" **");- }- }- }-- leastCost += wordCost;-- if (VERBOSE) {- System.out.println(- " + cost="- + leastCost- + " wordID="- + wordID- + " leftID="- + leftID- + " leastIDX="- + leastIDX- + " toPos="- + endPos- + " toPos.idx="- + positions.get(endPos).count);- }-- if (addPenalty && type != TokenType.USER) {- final int penalty = computePenalty(fromPosData.pos, endPos - fromPosData.pos);- if (VERBOSE) {- if (penalty > 0) {- System.out.println(" + penalty=" + penalty + " cost=" + (leastCost + penalty));- }- }- leastCost += penalty;- }-- // positions.get(endPos).add(leastCost, dict.getRightId(wordID), fromPosData.pos, leastIDX,- // wordID, type);- assert leftID == morphAtts.getRightId(wordID);- positions.get(endPos).add(leastCost, leftID, fromPosData.pos, wordPos, leastIDX, wordID, type);- }-@Overridepublic boolean incrementToken() throws IOException {- // parse() is able to return w/o producing any new+ // forward() can return w/o producing any new// tokens, when the tokens it had produced were entirely// punctuation. So we loop here until we get a real// token or we end:- while (pending.size() == 0) {- if (end) {+ while (viterbi.getPending().size() == 0) {+ if (viterbi.isEnd()) {return false;}-// Push Viterbi forward some more:- parse();+ viterbi.forward();}- final Token token = pending.remove(pending.size() - 1);+ final Token token = viterbi.getPending().remove(viterbi.getPending().size() - 1);int length = token.getLength();clearAttributes();@@ -604,7 +321,7 @@ public final class JapaneseTokenizer extends Tokenizer {if (token.getStartOffset() == lastTokenPos) {posIncAtt.setPositionIncrement(0);posLengthAtt.setPositionLength(token.getPositionLength());- } else if (outputNBest) {+ } else if (viterbi.isOutputNBest()) {// The position length is always calculated if outputNBest is true.assert token.getStartOffset() > lastTokenPos;posIncAtt.setPositionIncrement(1);@@ -618,1061 +335,8 @@ public final class JapaneseTokenizer extends Tokenizer {System.out.println(Thread.currentThread().getName() + ": incToken: return token=" + token);}lastTokenPos = token.getStartOffset();- return true;- }-- // TODO: make generic'd version of this "circular array"?- // It's a bit tricky because we do things to the Position- // (eg, set .pos = N on reuse)...- static final class WrappedPositionArray {- private Position[] positions = new Position[8];-- public WrappedPositionArray() {- for (int i = 0; i < positions.length; i++) {- positions[i] = new Position();- }- }-- // Next array index to write to in positions:- private int nextWrite;-- // Next position to write:- private int nextPos;-- // How many valid Position instances are held in the- // positions array:- private int count;-- public void reset() {- nextWrite--;- while (count > 0) {- if (nextWrite == -1) {- nextWrite = positions.length - 1;- }- positions[nextWrite--].reset();- count--;- }- nextWrite = 0;- nextPos = 0;- count = 0;- }-- /**- * Get Position instance for this absolute position; this is allowed to be arbitrarily far "in- * the future" but cannot be before the last freeBefore.- */- public Position get(int pos) {- while (pos >= nextPos) {- // System.out.println("count=" + count + " vs len=" + positions.length);- if (count == positions.length) {- Position[] newPositions =- new Position[ArrayUtil.oversize(1 + count, RamUsageEstimator.NUM_BYTES_OBJECT_REF)];- // System.out.println("grow positions " + newPositions.length);- System.arraycopy(positions, nextWrite, newPositions, 0, positions.length - nextWrite);- System.arraycopy(positions, 0, newPositions, positions.length - nextWrite, nextWrite);- for (int i = positions.length; i < newPositions.length; i++) {- newPositions[i] = new Position();- }- nextWrite = positions.length;- positions = newPositions;- }- if (nextWrite == positions.length) {- nextWrite = 0;- }- // Should have already been reset:- assert positions[nextWrite].count == 0;- positions[nextWrite++].pos = nextPos++;- count++;- }- assert inBounds(pos);- final int index = getIndex(pos);- assert positions[index].pos == pos;- return positions[index];- }-- public int getNextPos() {- return nextPos;- }-- // For assert:- private boolean inBounds(int pos) {- return pos < nextPos && pos >= nextPos - count;- }-- private int getIndex(int pos) {- int index = nextWrite - (nextPos - pos);- if (index < 0) {- index += positions.length;- }- return index;- }-- public void freeBefore(int pos) {- final int toFree = count - (nextPos - pos);- assert toFree >= 0;- assert toFree <= count;- int index = nextWrite - count;- if (index < 0) {- index += positions.length;- }- for (int i = 0; i < toFree; i++) {- if (index == positions.length) {- index = 0;- }- // System.out.println(" fb idx=" + index);- positions[index].reset();- index++;- }- count -= toFree;- }- }-- /* Incrementally parse some more characters. This runs- * the viterbi search forwards "enough" so that we- * generate some more tokens. How much forward depends on- * the chars coming in, since some chars could cause- * longer-lasting ambiguity in the parsing. Once the- * ambiguity is resolved, then we back trace, produce- * the pending tokens, and return. */- private void parse() throws IOException {- if (VERBOSE) {- System.out.println("\nPARSE");- }-- // Index of the last character of unknown word:- int unknownWordEndIndex = -1;-- // Advances over each position (character):- while (true) {-- if (buffer.get(pos) == -1) {- // End- break;- }-- final Position posData = positions.get(pos);- final boolean isFrontier = positions.getNextPos() == pos + 1;-- if (posData.count == 0) {- // No arcs arrive here; move to next position:- if (VERBOSE) {- System.out.println(" no arcs in; skip pos=" + pos);- }- pos++;- continue;- }-- if (pos > lastBackTracePos && posData.count == 1 && isFrontier) {- // if (pos > lastBackTracePos && posData.count == 1 && isFrontier) {- // We are at a "frontier", and only one node is- // alive, so whatever the eventual best path is must- // come through this node. So we can safely commit- // to the prefix of the best path at this point:- if (outputNBest) {- backtraceNBest(posData, false);- }- backtrace(posData, 0);- if (outputNBest) {- fixupPendingList();- }-- // Re-base cost so we don't risk int overflow:- posData.costs[0] = 0;-- if (pending.size() != 0) {- return;- } else {- // This means the backtrace only produced- // punctuation tokens, so we must keep parsing.- }- }-- if (pos - lastBackTracePos >= MAX_BACKTRACE_GAP) {- // Safety: if we've buffered too much, force a- // backtrace now. We find the least-cost partial- // path, across all paths, backtrace from it, and- // then prune all others. Note that this, in- // general, can produce the wrong result, if the- // total best path did not in fact back trace- // through this partial best path. But it's the- // best we can do... (short of not having a- // safety!).-- // First pass: find least cost partial path so far,- // including ending at future positions:- int leastIDX = -1;- int leastCost = Integer.MAX_VALUE;- Position leastPosData = null;- for (int pos2 = pos; pos2 < positions.getNextPos(); pos2++) {- final Position posData2 = positions.get(pos2);- for (int idx = 0; idx < posData2.count; idx++) {- // System.out.println(" idx=" + idx + " cost=" + cost);- final int cost = posData2.costs[idx];- if (cost < leastCost) {- leastCost = cost;- leastIDX = idx;- leastPosData = posData2;- }- }- }-- // We will always have at least one live path:- assert leastIDX != -1;-- if (outputNBest) {- backtraceNBest(leastPosData, false);- }-- // Second pass: prune all but the best path:- for (int pos2 = pos; pos2 < positions.getNextPos(); pos2++) {- final Position posData2 = positions.get(pos2);- if (posData2 != leastPosData) {- posData2.reset();- } else {- if (leastIDX != 0) {- posData2.costs[0] = posData2.costs[leastIDX];- posData2.lastRightID[0] = posData2.lastRightID[leastIDX];- posData2.backPos[0] = posData2.backPos[leastIDX];- posData2.backIndex[0] = posData2.backIndex[leastIDX];- posData2.backID[0] = posData2.backID[leastIDX];- posData2.backType[0] = posData2.backType[leastIDX];- }- posData2.count = 1;- }- }-- backtrace(leastPosData, 0);- if (outputNBest) {- fixupPendingList();- }-- // Re-base cost so we don't risk int overflow:- Arrays.fill(leastPosData.costs, 0, leastPosData.count, 0);-- if (pos != leastPosData.pos) {- // We jumped into a future position:- assert pos < leastPosData.pos;- pos = leastPosData.pos;- }-- if (pending.size() != 0) {- return;- } else {- // This means the backtrace only produced- // punctuation tokens, so we must keep parsing.- continue;- }- }-- if (VERBOSE) {- System.out.println(- "\n extend @ pos="- + pos- + " char="- + (char) buffer.get(pos)- + " hex="- + Integer.toHexString(buffer.get(pos)));- }-- if (VERBOSE) {- System.out.println(" " + posData.count + " arcs in");- }-- boolean anyMatches = false;-- // First try user dict:- if (userFST != null) {- userFST.getFirstArc(arc);- int output = 0;- for (int posAhead = posData.pos; ; posAhead++) {- final int ch = buffer.get(posAhead);- if (ch == -1) {- break;- }- if (userFST.findTargetArc(ch, arc, arc, posAhead == posData.pos, userFSTReader) == null) {- break;- }- output += arc.output().intValue();- if (arc.isFinal()) {- if (VERBOSE) {- System.out.println(- " USER word "- + new String(buffer.get(pos, posAhead - pos + 1))- + " toPos="- + (posAhead + 1));- }- add(- userDictionary.getMorphAttributes(),- posData,- pos,- posAhead + 1,- output + arc.nextFinalOutput().intValue(),- TokenType.USER,- false);- anyMatches = true;- }- }- }-- // TODO: we can be more aggressive about user- // matches? if we are "under" a user match then don't- // extend KNOWN/UNKNOWN paths?-- if (!anyMatches) {- // Next, try known dictionary matches- fst.getFirstArc(arc);- int output = 0;-- for (int posAhead = posData.pos; ; posAhead++) {- final int ch = buffer.get(posAhead);- if (ch == -1) {- break;- }- // System.out.println(" match " + (char) ch + " posAhead=" + posAhead);-- if (fst.findTargetArc(ch, arc, arc, posAhead == posData.pos, fstReader) == null) {- break;- }-- output += arc.output().intValue();-- // Optimization: for known words that are too-long- // (compound), we should pre-compute the 2nd- // best segmentation and store it in the- // dictionary instead of recomputing it each time a- // match is found.-- if (arc.isFinal()) {- dictionary.lookupWordIds(output + arc.nextFinalOutput().intValue(), wordIdRef);- if (VERBOSE) {- System.out.println(- " KNOWN word "- + new String(buffer.get(pos, posAhead - pos + 1))- + " toPos="- + (posAhead + 1)- + " "- + wordIdRef.length- + " wordIDs");- }- for (int ofs = 0; ofs < wordIdRef.length; ofs++) {- add(- dictionary.getMorphAttributes(),- posData,- pos,- posAhead + 1,- wordIdRef.ints[wordIdRef.offset + ofs],- TokenType.KNOWN,- false);- anyMatches = true;- }- }- }- }-- // In the case of normal mode, it doesn't process unknown word greedily.-- if (!searchMode && unknownWordEndIndex > posData.pos) {- pos++;- continue;- }-- final char firstCharacter = (char) buffer.get(pos);- if (!anyMatches || characterDefinition.isInvoke(firstCharacter)) {-- // Find unknown match:- final int characterId = characterDefinition.getCharacterClass(firstCharacter);- final boolean isPunct = isPunctuation(firstCharacter);-- // NOTE: copied from UnknownDictionary.lookup:- int unknownWordLength;- if (!characterDefinition.isGroup(firstCharacter)) {- unknownWordLength = 1;- } else {- // Extract unknown word. Characters with the same character class are considered to be- // part of unknown word- unknownWordLength = 1;- for (int posAhead = pos + 1; unknownWordLength < MAX_UNKNOWN_WORD_LENGTH; posAhead++) {- final int ch = buffer.get(posAhead);- if (ch == -1) {- break;- }- if (characterId == characterDefinition.getCharacterClass((char) ch)- && isPunctuation((char) ch) == isPunct) {- unknownWordLength++;- } else {- break;- }- }- }-- unkDictionary.lookupWordIds(- characterId, wordIdRef); // characters in input text are supposed to be the same- if (VERBOSE) {- System.out.println(- " UNKNOWN word len=" + unknownWordLength + " " + wordIdRef.length + " wordIDs");- }- for (int ofs = 0; ofs < wordIdRef.length; ofs++) {- add(- unkDictionary.getMorphAttributes(),- posData,- pos,- posData.pos + unknownWordLength,- wordIdRef.ints[wordIdRef.offset + ofs],- TokenType.UNKNOWN,- false);- }-- unknownWordEndIndex = posData.pos + unknownWordLength;- }-- pos++;- }-- end = true;-- if (pos > 0) {-- final Position endPosData = positions.get(pos);- int leastCost = Integer.MAX_VALUE;- int leastIDX = -1;- if (VERBOSE) {- System.out.println(" end: " + endPosData.count + " nodes");- }- for (int idx = 0; idx < endPosData.count; idx++) {- // Add EOS cost:- final int cost = endPosData.costs[idx] + costs.get(endPosData.lastRightID[idx], 0);- // System.out.println(" idx=" + idx + " cost=" + cost + " (pathCost=" +- // endPosData.costs[idx] + " bgCost=" + costs.get(endPosData.lastRightID[idx], 0) + ")- // backPos=" + endPosData.backPos[idx]);- if (cost < leastCost) {- leastCost = cost;- leastIDX = idx;- }- }-- if (outputNBest) {- backtraceNBest(endPosData, true);- }- backtrace(endPosData, leastIDX);- if (outputNBest) {- fixupPendingList();- }- } else {- // No characters in the input string; return no tokens!- }- }-- // Eliminates arcs from the lattice that are compound- // tokens (have a penalty) or are not congruent with the- // compound token we've matched (ie, span across the- // startPos). This should be fairly efficient, because we- // just keep the already intersected structure of the- // graph, eg we don't have to consult the FSTs again:-- private void pruneAndRescore(int startPos, int endPos, int bestStartIDX) throws IOException {- if (VERBOSE) {- System.out.println(- " pruneAndRescore startPos="- + startPos- + " endPos="- + endPos- + " bestStartIDX="- + bestStartIDX);- }-- // First pass: walk backwards, building up the forward- // arcs and pruning inadmissible arcs:- for (int pos = endPos; pos > startPos; pos--) {- final Position posData = positions.get(pos);- if (VERBOSE) {- System.out.println(" back pos=" + pos);- }- for (int arcIDX = 0; arcIDX < posData.count; arcIDX++) {- final int backPos = posData.backPos[arcIDX];- if (backPos >= startPos) {- // Keep this arc:- // System.out.println(" keep backPos=" + backPos);- positions- .get(backPos)- .addForward(pos, arcIDX, posData.backID[arcIDX], posData.backType[arcIDX]);- } else {- if (VERBOSE) {- System.out.println(" prune");- }- }- }- if (pos != startPos) {- posData.count = 0;- }- }-- // Second pass: walk forward, re-scoring:- for (int pos = startPos; pos < endPos; pos++) {- final Position posData = positions.get(pos);- if (VERBOSE) {- System.out.println(" forward pos=" + pos + " count=" + posData.forwardCount);- }- if (posData.count == 0) {- // No arcs arrive here...- if (VERBOSE) {- System.out.println(" skip");- }- posData.forwardCount = 0;- continue;- }-- if (pos == startPos) {- // On the initial position, only consider the best- // path so we "force congruence": the- // sub-segmentation is "in context" of what the best- // path (compound token) had matched:- final int rightID;- if (startPos == 0) {- rightID = 0;- } else {- rightID =- getDict(posData.backType[bestStartIDX]).getRightId(posData.backID[bestStartIDX]);- }- final int pathCost = posData.costs[bestStartIDX];- for (int forwardArcIDX = 0; forwardArcIDX < posData.forwardCount; forwardArcIDX++) {- final TokenType forwardType = posData.forwardType[forwardArcIDX];- final Dictionary<? extends JaMorphData> dict2 = getDict(forwardType);- final int wordID = posData.forwardID[forwardArcIDX];- final int toPos = posData.forwardPos[forwardArcIDX];- final int newCost =- pathCost- + dict2.getWordCost(wordID)- + costs.get(rightID, dict2.getLeftId(wordID))- + computePenalty(pos, toPos - pos);- if (VERBOSE) {- System.out.println(- " + "- + forwardType- + " word "- + new String(buffer.get(pos, toPos - pos))- + " toPos="- + toPos- + " cost="- + newCost- + " penalty="- + computePenalty(pos, toPos - pos)- + " toPos.idx="- + positions.get(toPos).count);- }- positions- .get(toPos)- .add(newCost, dict2.getRightId(wordID), pos, -1, bestStartIDX, wordID, forwardType);- }- } else {- // On non-initial positions, we maximize score- // across all arriving lastRightIDs:- for (int forwardArcIDX = 0; forwardArcIDX < posData.forwardCount; forwardArcIDX++) {- final TokenType forwardType = posData.forwardType[forwardArcIDX];- final int toPos = posData.forwardPos[forwardArcIDX];- if (VERBOSE) {- System.out.println(- " + "- + forwardType- + " word "- + new String(buffer.get(pos, toPos - pos))- + " toPos="- + toPos);- }- add(- getDict(forwardType).getMorphAttributes(),- posData,- pos,- toPos,- posData.forwardID[forwardArcIDX],- forwardType,- true);- }- }- posData.forwardCount = 0;- }- }-- // yet another lattice data structure- private static final class Lattice {- char[] fragment;- EnumMap<TokenType, Dictionary<? extends JaMorphData>> dictionaryMap;- boolean useEOS;-- int rootCapacity = 0;- int rootSize = 0;- int rootBase = 0;-- // root pointers of node chain by leftChain_ that have same start offset.- int[] lRoot;- // root pointers of node chain by rightChain_ that have same end offset.- int[] rRoot;-- int capacity = 0;- int nodeCount = 0;-- // The variables below are elements of lattice node that indexed by node number.- TokenType[] nodeDicType;- int[] nodeWordID;- // nodeMark - -1:excluded, 0:unused, 1:bestpath, 2:2-best-path, ... N:N-best-path- int[] nodeMark;- int[] nodeLeftID;- int[] nodeRightID;- int[] nodeWordCost;- int[] nodeLeftCost;- int[] nodeRightCost;- // nodeLeftNode, nodeRightNode - are left/right node number with minimum cost path.- int[] nodeLeftNode;- int[] nodeRightNode;- // nodeLeft, nodeRight - start/end offset- int[] nodeLeft;- int[] nodeRight;- int[] nodeLeftChain;- int[] nodeRightChain;-- private void setupRoot(int baseOffset, int lastOffset) {- assert baseOffset <= lastOffset;- int size = lastOffset - baseOffset + 1;- if (rootCapacity < size) {- int oversize = ArrayUtil.oversize(size, Integer.BYTES);- lRoot = new int[oversize];- rRoot = new int[oversize];- rootCapacity = oversize;- }- Arrays.fill(lRoot, 0, size, -1);- Arrays.fill(rRoot, 0, size, -1);- rootSize = size;- rootBase = baseOffset;- }-- // Reserve at least N nodes.- private void reserve(int n) {- if (capacity < n) {- int oversize = ArrayUtil.oversize(n, Integer.BYTES);- nodeDicType = new TokenType[oversize];- nodeWordID = new int[oversize];- nodeMark = new int[oversize];- nodeLeftID = new int[oversize];- nodeRightID = new int[oversize];- nodeWordCost = new int[oversize];- nodeLeftCost = new int[oversize];- nodeRightCost = new int[oversize];- nodeLeftNode = new int[oversize];- nodeRightNode = new int[oversize];- nodeLeft = new int[oversize];- nodeRight = new int[oversize];- nodeLeftChain = new int[oversize];- nodeRightChain = new int[oversize];- capacity = oversize;- }- }-- private void setupNodePool(int n) {- reserve(n);- nodeCount = 0;- if (VERBOSE) {- System.out.printf("DEBUG: setupNodePool: n = %d\n", n);- System.out.printf("DEBUG: setupNodePool: lattice.capacity = %d\n", capacity);- }- }-- private int addNode(TokenType dicType, int wordID, int left, int right) {- if (VERBOSE) {- System.out.printf(- "DEBUG: addNode: dicType=%s, wordID=%d, left=%d, right=%d, str=%s\n",- dicType.toString(),- wordID,- left,- right,- left == -1 ? "BOS" : right == -1 ? "EOS" : new String(fragment, left, right - left));- }- assert nodeCount < capacity;- assert left == -1 || right == -1 || left < right;- assert left == -1 || (0 <= left && left < rootSize);- assert right == -1 || (0 <= right && right < rootSize);-- int node = nodeCount++;-- if (VERBOSE) {- System.out.printf("DEBUG: addNode: node=%d\n", node);- }-- nodeDicType[node] = dicType;- nodeWordID[node] = wordID;- nodeMark[node] = 0;-- if (wordID < 0) {- nodeWordCost[node] = 0;- nodeLeftCost[node] = 0;- nodeRightCost[node] = 0;- nodeLeftID[node] = 0;- nodeRightID[node] = 0;- } else {- Dictionary<? extends JaMorphData> dic = dictionaryMap.get(dicType);- nodeWordCost[node] = dic.getWordCost(wordID);- nodeLeftID[node] = dic.getLeftId(wordID);- nodeRightID[node] = dic.getRightId(wordID);- }-- if (VERBOSE) {- System.out.printf(- "DEBUG: addNode: wordCost=%d, leftID=%d, rightID=%d\n",- nodeWordCost[node], nodeLeftID[node], nodeRightID[node]);- }-- nodeLeft[node] = left;- nodeRight[node] = right;- if (0 <= left) {- nodeLeftChain[node] = lRoot[left];- lRoot[left] = node;- } else {- nodeLeftChain[node] = -1;- }- if (0 <= right) {- nodeRightChain[node] = rRoot[right];- rRoot[right] = node;- } else {- nodeRightChain[node] = -1;- }- return node;- }-- // Sum of positions.get(i).count in [beg, end) range.- // using stream:- // return IntStream.range(beg, end).map(i -> positions.get(i).count).sum();- private int positionCount(WrappedPositionArray positions, int beg, int end) {- int count = 0;- for (int i = beg; i < end; ++i) {- count += positions.get(i).count;- }- return count;- }-- void setup(- char[] fragment,- EnumMap<TokenType, Dictionary<? extends JaMorphData>> dictionaryMap,- WrappedPositionArray positions,- int prevOffset,- int endOffset,- boolean useEOS) {- assert positions.get(prevOffset).count == 1;- if (VERBOSE) {- System.out.printf("DEBUG: setup: prevOffset=%d, endOffset=%d\n", prevOffset, endOffset);- }-- this.fragment = fragment;- this.dictionaryMap = dictionaryMap;- this.useEOS = useEOS;-- // Initialize lRoot and rRoot.- setupRoot(prevOffset, endOffset);-- // "+ 2" for first/last record.- setupNodePool(positionCount(positions, prevOffset + 1, endOffset + 1) + 2);-- // substitute for BOS = 0- Position first = positions.get(prevOffset);- if (addNode(first.backType[0], first.backID[0], -1, 0) != 0) {- assert false;- }-- // EOS = 1- if (addNode(TokenType.KNOWN, -1, endOffset - rootBase, -1) != 1) {- assert false;- }-- for (int offset = endOffset; prevOffset < offset; --offset) {- int right = offset - rootBase;- // optimize: exclude disconnected nodes.- if (0 <= lRoot[right]) {- Position pos = positions.get(offset);- for (int i = 0; i < pos.count; ++i) {- addNode(pos.backType[i], pos.backID[i], pos.backPos[i] - rootBase, right);- }- }- }- }-- // set mark = -1 for unreachable nodes.- void markUnreachable() {- for (int index = 1; index < rootSize - 1; ++index) {- if (rRoot[index] < 0) {- for (int node = lRoot[index]; 0 <= node; node = nodeLeftChain[node]) {- if (VERBOSE) {- System.out.printf("DEBUG: markUnreachable: node=%d\n", node);- }- nodeMark[node] = -1;- }- }- }- }-- int connectionCost(ConnectionCosts costs, int left, int right) {- int leftID = nodeLeftID[right];- return ((leftID == 0 && !useEOS) ? 0 : costs.get(nodeRightID[left], leftID));- }-- void calcLeftCost(ConnectionCosts costs) {- for (int index = 0; index < rootSize; ++index) {- for (int node = lRoot[index]; 0 <= node; node = nodeLeftChain[node]) {- if (0 <= nodeMark[node]) {- int leastNode = -1;- int leastCost = Integer.MAX_VALUE;- for (int leftNode = rRoot[index]; 0 <= leftNode; leftNode = nodeRightChain[leftNode]) {- if (0 <= nodeMark[leftNode]) {- int cost =- nodeLeftCost[leftNode]- + nodeWordCost[leftNode]- + connectionCost(costs, leftNode, node);- if (cost < leastCost) {- leastCost = cost;- leastNode = leftNode;- }- }- }- assert 0 <= leastNode;- nodeLeftNode[node] = leastNode;- nodeLeftCost[node] = leastCost;- if (VERBOSE) {- System.out.printf(- "DEBUG: calcLeftCost: node=%d, leftNode=%d, leftCost=%d\n",- node, nodeLeftNode[node], nodeLeftCost[node]);- }- }- }- }- }-- void calcRightCost(ConnectionCosts costs) {- for (int index = rootSize - 1; 0 <= index; --index) {- for (int node = rRoot[index]; 0 <= node; node = nodeRightChain[node]) {- if (0 <= nodeMark[node]) {- int leastNode = -1;- int leastCost = Integer.MAX_VALUE;- for (int rightNode = lRoot[index];- 0 <= rightNode;- rightNode = nodeLeftChain[rightNode]) {- if (0 <= nodeMark[rightNode]) {- int cost =- nodeRightCost[rightNode]- + nodeWordCost[rightNode]- + connectionCost(costs, node, rightNode);- if (cost < leastCost) {- leastCost = cost;- leastNode = rightNode;- }- }- }- assert 0 <= leastNode;- nodeRightNode[node] = leastNode;- nodeRightCost[node] = leastCost;- if (VERBOSE) {- System.out.printf(- "DEBUG: calcRightCost: node=%d, rightNode=%d, rightCost=%d\n",- node, nodeRightNode[node], nodeRightCost[node]);- }- }- }- }- }-- // Mark all nodes that have same text and different par-of-speech or reading.- void markSameSpanNode(int refNode, int value) {- int left = nodeLeft[refNode];- int right = nodeRight[refNode];- for (int node = lRoot[left]; 0 <= node; node = nodeLeftChain[node]) {- if (nodeRight[node] == right) {- nodeMark[node] = value;- }- }- }-- List<Integer> bestPathNodeList() {- List<Integer> list = new ArrayList<>();- for (int node = nodeRightNode[0]; node != 1; node = nodeRightNode[node]) {- list.add(node);- markSameSpanNode(node, 1);- }- return list;- }-- private int cost(int node) {- return nodeLeftCost[node] + nodeWordCost[node] + nodeRightCost[node];- }-- List<Integer> nBestNodeList(int N) {- List<Integer> list = new ArrayList<>();- int leastCost = Integer.MAX_VALUE;- int leastLeft = -1;- int leastRight = -1;- for (int node = 2; node < nodeCount; ++node) {- if (nodeMark[node] == 0) {- int cost = cost(node);- if (cost < leastCost) {- leastCost = cost;- leastLeft = nodeLeft[node];- leastRight = nodeRight[node];- list.clear();- list.add(node);- } else if (cost == leastCost- && (nodeLeft[node] != leastLeft || nodeRight[node] != leastRight)) {- list.add(node);- }- }- }- for (int node : list) {- markSameSpanNode(node, N);- }- return list;- }-- int bestCost() {- return nodeLeftCost[1];- }-- int probeDelta(int start, int end) {- int left = start - rootBase;- int right = end - rootBase;- if (left < 0 || rootSize < right) {- return Integer.MAX_VALUE;- }- int probedCost = Integer.MAX_VALUE;- for (int node = lRoot[left]; 0 <= node; node = nodeLeftChain[node]) {- if (nodeRight[node] == right) {- probedCost = Math.min(probedCost, cost(node));- }- }- return probedCost - bestCost();- }-- void debugPrint() {- if (VERBOSE) {- for (int node = 0; node < nodeCount; ++node) {- System.out.printf(- "DEBUG NODE: node=%d, mark=%d, cost=%d, left=%d, right=%d\n",- node, nodeMark[node], cost(node), nodeLeft[node], nodeRight[node]);- }- }- }- }-- private Lattice lattice = null;-- private void registerNode(int node, char[] fragment) {- int left = lattice.nodeLeft[node];- int right = lattice.nodeRight[node];- TokenType type = lattice.nodeDicType[node];- if (!discardPunctuation || !isPunctuation(fragment[left])) {- if (type == TokenType.USER) {- // The code below are based on backtrace().- //- // Expand the phraseID we recorded into the actual segmentation:- final int[] wordIDAndLength = userDictionary.lookupSegmentation(lattice.nodeWordID[node]);- int wordID = wordIDAndLength[0];- pending.add(- new Token(- fragment,- left,- right - left,- lattice.rootBase + left,- lattice.rootBase + right,- wordID,- TokenType.USER,- userDictionary.getMorphAttributes()));- // Output compound- int current = 0;- for (int j = 1; j < wordIDAndLength.length; j++) {- final int len = wordIDAndLength[j];- if (len < right - left) {- int startOffset = lattice.rootBase + current + left;- pending.add(- new Token(- fragment,- current + left,- len,- startOffset,- startOffset + len,- wordID + j - 1,- TokenType.USER,- userDictionary.getMorphAttributes()));- }- current += len;- }- } else {- pending.add(- new Token(- fragment,- left,- right - left,- lattice.rootBase + left,- lattice.rootBase + right,- lattice.nodeWordID[node],- type,- getDict(type).getMorphAttributes()));- }- }- }-- // Sort pending tokens, and set position increment values.- private void fixupPendingList() {- // Sort for removing same tokens.- // USER token should be ahead from normal one.- Collections.sort(- pending,- new Comparator<Token>() {- @Override- public int compare(Token a, Token b) {- int aOff = a.getOffset();- int bOff = b.getOffset();- if (aOff != bOff) {- return aOff - bOff;- }- int aLen = a.getLength();- int bLen = b.getLength();- if (aLen != bLen) {- return aLen - bLen;- }- // order of Type is KNOWN, UNKNOWN, USER,- // so we use reversed comparison here.- return b.getType().ordinal() - a.getType().ordinal();- }- });-- // Remove same token.- for (int i = 1; i < pending.size(); ++i) {- Token a = pending.get(i - 1);- Token b = pending.get(i);- if (a.getOffset() == b.getOffset() && a.getLength() == b.getLength()) {- pending.remove(i);- // It is important to decrement "i" here, because a next may be removed.- --i;- }- }-- // offset=>position map- HashMap<Integer, Integer> map = new HashMap<>();- for (Token t : pending) {- map.put(t.getOffset(), 0);- map.put(t.getOffset() + t.getLength(), 0);- }-- // Get uniqe and sorted list of all edge position of tokens.- Integer[] offsets = map.keySet().toArray(new Integer[0]);- Arrays.sort(offsets);- // setup all value of map. It specify N-th position from begin.- for (int i = 0; i < offsets.length; ++i) {- map.put(offsets[i], i);- }-- // We got all position length now.- for (Token t : pending) {- t.setPositionLength(map.get(t.getOffset() + t.getLength()) - map.get(t.getOffset()));- }-- // Make PENDING to be reversed order to fit its usage.- // If you would like to speedup, you can try reversed order sort- // at first of this function.- Collections.reverse(pending);+ return true;}private int probeDelta(String inText, String requiredToken) throws IOException {@@ -1683,16 +347,17 @@ public final class JapaneseTokenizer extends Tokenizer {}int delta = Integer.MAX_VALUE;- int saveNBestCost = nBestCost;+ int saveNBestCost = viterbi.getNBestCost();setReader(new StringReader(inText));reset();try {setNBestCost(1);int prevRootBase = -1;while (incrementToken()) {- if (lattice.rootBase != prevRootBase) {- prevRootBase = lattice.rootBase;- delta = Math.min(delta, lattice.probeDelta(start, start + requiredToken.length()));+ if (viterbi.lattice.rootBase != prevRootBase) {+ prevRootBase = viterbi.lattice.rootBase;+ delta =+ Math.min(delta, viterbi.lattice.probeDelta(start, start + requiredToken.length()));}}} finally {@@ -1730,404 +395,6 @@ public final class JapaneseTokenizer extends Tokenizer {}public void setNBestCost(int value) {- nBestCost = value;- outputNBest = 0 < nBestCost;- }-- private void backtraceNBest(final Position endPosData, final boolean useEOS) throws IOException {- if (lattice == null) {- lattice = new Lattice();- }-- final int endPos = endPosData.pos;- char[] fragment = buffer.get(lastBackTracePos, endPos - lastBackTracePos);- lattice.setup(fragment, dictionaryMap, positions, lastBackTracePos, endPos, useEOS);- lattice.markUnreachable();- lattice.calcLeftCost(costs);- lattice.calcRightCost(costs);-- int bestCost = lattice.bestCost();- if (VERBOSE) {- System.out.printf("DEBUG: 1-BEST COST: %d\n", bestCost);- }- for (int node : lattice.bestPathNodeList()) {- registerNode(node, fragment);- }-- for (int n = 2; ; ++n) {- List<Integer> nbest = lattice.nBestNodeList(n);- if (nbest.isEmpty()) {- break;- }- int cost = lattice.cost(nbest.get(0));- if (VERBOSE) {- System.out.printf("DEBUG: %d-BEST COST: %d\n", n, cost);- }- if (bestCost + nBestCost < cost) {- break;- }- for (int node : nbest) {- registerNode(node, fragment);- }- }- if (VERBOSE) {- lattice.debugPrint();- }- }-- // Backtrace from the provided position, back to the last- // time we back-traced, accumulating the resulting tokens to- // the pending list. The pending list is then in-reverse- // (last token should be returned first).- private void backtrace(final Position endPosData, final int fromIDX) throws IOException {- final int endPos = endPosData.pos;-- /**- * LUCENE-10059: If the endPos is the same as lastBackTracePos, we don't want to backtrace to- * avoid an assertion error {@link RollingCharBuffer#get(int)} when it tries to generate an- * empty buffer- */- if (endPos == lastBackTracePos) {- return;- }-- if (VERBOSE) {- System.out.println(- "\n backtrace: endPos="- + endPos- + " pos="- + pos- + "; "- + (pos - lastBackTracePos)- + " characters; last="- + lastBackTracePos- + " cost="- + endPosData.costs[fromIDX]);- }-- final char[] fragment = buffer.get(lastBackTracePos, endPos - lastBackTracePos);-- if (dotOut != null) {- dotOut.onBacktrace(this, positions, lastBackTracePos, endPosData, fromIDX, fragment, end);- }-- int pos = endPos;- int bestIDX = fromIDX;- Token altToken = null;-- // We trace backwards, so this will be the leftWordID of- // the token after the one we are now on:- int lastLeftWordID = -1;-- int backCount = 0;-- // TODO: sort of silly to make Token instances here; the- // back trace has all info needed to generate the- // token. So, we could just directly set the attrs,- // from the backtrace, in incrementToken w/o ever- // creating Token; we'd have to defer calling freeBefore- // until after the backtrace was fully "consumed" by- // incrementToken.-- while (pos > lastBackTracePos) {- // System.out.println("BT: back pos=" + pos + " bestIDX=" + bestIDX);- final Position posData = positions.get(pos);- assert bestIDX < posData.count;-- int backPos = posData.backPos[bestIDX];- assert backPos >= lastBackTracePos- : "backPos=" + backPos + " vs lastBackTracePos=" + lastBackTracePos;- int length = pos - backPos;- TokenType backType = posData.backType[bestIDX];- int backID = posData.backID[bestIDX];- int nextBestIDX = posData.backIndex[bestIDX];-- if (searchMode && altToken == null && backType != TokenType.USER) {-- // In searchMode, if best path had picked a too-long- // token, we use the "penalty" to compute the allowed- // max cost of an alternate back-trace. If we find an- // alternate back trace with cost below that- // threshold, we pursue it instead (but also output- // the long token).- // System.out.println(" 2nd best backPos=" + backPos + " pos=" + pos);-- final int penalty = computeSecondBestThreshold(backPos, pos - backPos);-- if (penalty > 0) {- if (VERBOSE) {- System.out.println(- " compound="- + new String(buffer.get(backPos, pos - backPos))- + " backPos="- + backPos- + " pos="- + pos- + " penalty="- + penalty- + " cost="- + posData.costs[bestIDX]- + " bestIDX="- + bestIDX- + " lastLeftID="- + lastLeftWordID);- }-- // Use the penalty to set maxCost on the 2nd best- // segmentation:- int maxCost = posData.costs[bestIDX] + penalty;- if (lastLeftWordID != -1) {- maxCost += costs.get(getDict(backType).getRightId(backID), lastLeftWordID);- }-- // Now, prune all too-long tokens from the graph:- pruneAndRescore(backPos, pos, posData.backIndex[bestIDX]);-- // Finally, find 2nd best back-trace and resume- // backtrace there:- int leastCost = Integer.MAX_VALUE;- int leastIDX = -1;- for (int idx = 0; idx < posData.count; idx++) {- int cost = posData.costs[idx];- // System.out.println(" idx=" + idx + " prevCost=" + cost);-- if (lastLeftWordID != -1) {- cost +=- costs.get(- getDict(posData.backType[idx]).getRightId(posData.backID[idx]),- lastLeftWordID);- // System.out.println(" += bgCost=" +- // costs.get(getDict(posData.backType[idx]).getRightId(posData.backID[idx]),- // lastLeftWordID) + " -> " + cost);- }- // System.out.println("penalty " + posData.backPos[idx] + " to " + pos);- // cost += computePenalty(posData.backPos[idx], pos - posData.backPos[idx]);- if (cost < leastCost) {- // System.out.println(" ** ");- leastCost = cost;- leastIDX = idx;- }- }- // System.out.println(" leastIDX=" + leastIDX);-- if (VERBOSE) {- System.out.println(- " afterPrune: "- + posData.count- + " arcs arriving; leastCost="- + leastCost- + " vs threshold="- + maxCost- + " lastLeftWordID="- + lastLeftWordID);- }-- if (leastIDX != -1 && leastCost <= maxCost && posData.backPos[leastIDX] != backPos) {- // We should have pruned the altToken from the graph:- assert posData.backPos[leastIDX] != backPos;-- // Save the current compound token, to output when- // this alternate path joins back:- altToken =- new Token(- fragment,- backPos - lastBackTracePos,- length,- backPos,- backPos + length,- backID,- backType,- getDict(backType).getMorphAttributes());-- // Redirect our backtrace to 2nd best:- bestIDX = leastIDX;- nextBestIDX = posData.backIndex[bestIDX];-- backPos = posData.backPos[bestIDX];- length = pos - backPos;- backType = posData.backType[bestIDX];- backID = posData.backID[bestIDX];- backCount = 0;- // System.out.println(" do alt token!");-- } else {- // I think in theory it's possible there is no- // 2nd best path, which is fine; in this case we- // only output the compound token:- // System.out.println(" no alt token! bestIDX=" + bestIDX);- }- }- }-- final int offset = backPos - lastBackTracePos;- assert offset >= 0;-- if (altToken != null && altToken.getStartOffset() >= backPos) {- if (outputCompounds) {- // We've backtraced to the position where the- // compound token starts; add it now:-- // The pruning we did when we created the altToken- // ensures that the back trace will align back with- // the start of the altToken:- assert altToken.getStartOffset() == backPos- : altToken.getStartOffset() + " vs " + backPos;-- // NOTE: not quite right: the compound token may- // have had all punctuation back traced so far, but- // then the decompounded token at this position is- // not punctuation. In this case backCount is 0,- // but we should maybe add the altToken anyway...?-- if (backCount > 0) {- backCount++;- altToken.setPositionLength(backCount);- if (VERBOSE) {- System.out.println(" add altToken=" + altToken);- }- pending.add(altToken);- } else {- // This means alt token was all punct tokens:- if (VERBOSE) {- System.out.println(" discard all-punctuation altToken=" + altToken);- }- assert discardPunctuation;- }- }- altToken = null;- }-- final Dictionary<? extends JaMorphData> dict = getDict(backType);-- if (backType == TokenType.USER) {-- // Expand the phraseID we recorded into the actual- // segmentation:- final int[] wordIDAndLength = userDictionary.lookupSegmentation(backID);- int wordID = wordIDAndLength[0];- int current = 0;- for (int j = 1; j < wordIDAndLength.length; j++) {- final int len = wordIDAndLength[j];- // System.out.println(" add user: len=" + len);- int startOffset = current + backPos;- pending.add(- new Token(- fragment,- current + offset,- len,- startOffset,- startOffset + len,- wordID + j - 1,- TokenType.USER,- dict.getMorphAttributes()));- if (VERBOSE) {- System.out.println(" add USER token=" + pending.get(pending.size() - 1));- }- current += len;- }-- // Reverse the tokens we just added, because when we- // serve them up from incrementToken we serve in- // reverse:- Collections.reverse(- pending.subList(pending.size() - (wordIDAndLength.length - 1), pending.size()));-- backCount += wordIDAndLength.length - 1;- } else {-- if (extendedMode && backType == TokenType.UNKNOWN) {- // In EXTENDED mode we convert unknown word into- // unigrams:- int unigramTokenCount = 0;- for (int i = length - 1; i >= 0; i--) {- int charLen = 1;- if (i > 0 && Character.isLowSurrogate(fragment[offset + i])) {- i--;- charLen = 2;- }- // System.out.println(" extended tok offset="- // + (offset + i));- if (!discardPunctuation || !isPunctuation(fragment[offset + i])) {- int startOffset = backPos + i;- pending.add(- new Token(- fragment,- offset + i,- charLen,- startOffset,- startOffset + charLen,- CharacterDefinition.NGRAM,- TokenType.UNKNOWN,- unkDictionary.getMorphAttributes()));- unigramTokenCount++;- }- }- backCount += unigramTokenCount;-- } else if (!discardPunctuation || length == 0 || !isPunctuation(fragment[offset])) {- pending.add(- new Token(- fragment,- offset,- length,- backPos,- backPos + length,- backID,- backType,- dict.getMorphAttributes()));- if (VERBOSE) {- System.out.println(" add token=" + pending.get(pending.size() - 1));- }- backCount++;- } else {- if (VERBOSE) {- System.out.println(- " skip punctuation token=" + new String(fragment, offset, length));- }- }- }-- lastLeftWordID = dict.getLeftId(backID);- pos = backPos;- bestIDX = nextBestIDX;- }-- lastBackTracePos = endPos;-- if (VERBOSE) {- System.out.println(" freeBefore pos=" + endPos);- }- // Notify the circular buffers that we are done with- // these positions:- buffer.freeBefore(endPos);- positions.freeBefore(endPos);- }-- Dictionary<? extends JaMorphData> getDict(TokenType type) {- return dictionaryMap.get(type);- }-- private static boolean isPunctuation(char ch) {- switch (Character.getType(ch)) {- case Character.SPACE_SEPARATOR:- case Character.LINE_SEPARATOR:- case Character.PARAGRAPH_SEPARATOR:- case Character.CONTROL:- case Character.FORMAT:- case Character.DASH_PUNCTUATION:- case Character.START_PUNCTUATION:- case Character.END_PUNCTUATION:- case Character.CONNECTOR_PUNCTUATION:- case Character.OTHER_PUNCTUATION:- case Character.MATH_SYMBOL:- case Character.CURRENCY_SYMBOL:- case Character.MODIFIER_SYMBOL:- case Character.OTHER_SYMBOL:- case Character.INITIAL_QUOTE_PUNCTUATION:- case Character.FINAL_QUOTE_PUNCTUATION:- return true;- default:- return false;- }+ viterbi.setNBestCost(value);}}diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/ViterbiNBest.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/ViterbiNBest.javanew file mode 100644index 00000000000..d789947e784--- /dev/null+++ b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/ViterbiNBest.java@@ -0,0 +1,1253 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.analysis.ja;++import java.io.IOException;+import java.util.ArrayList;+import java.util.Arrays;+import java.util.Collections;+import java.util.EnumMap;+import java.util.HashMap;+import java.util.List;+import org.apache.lucene.analysis.ja.dict.CharacterDefinition;+import org.apache.lucene.analysis.ja.dict.JaMorphData;+import org.apache.lucene.analysis.ja.dict.TokenInfoDictionary;+import org.apache.lucene.analysis.ja.dict.UnknownDictionary;+import org.apache.lucene.analysis.ja.dict.UserDictionary;+import org.apache.lucene.analysis.morph.ConnectionCosts;+import org.apache.lucene.analysis.morph.Dictionary;+import org.apache.lucene.analysis.morph.GraphvizFormatter;+import org.apache.lucene.analysis.morph.TokenInfoFST;+import org.apache.lucene.analysis.morph.TokenType;+import org.apache.lucene.analysis.morph.Viterbi;+import org.apache.lucene.util.ArrayUtil;+import org.apache.lucene.util.fst.FST;++/**+ * {@link org.apache.lucene.analysis.morph.Viterbi} subclass for Japanese morphological analysis.+ * This also performs n-best path calculation+ */+final class ViterbiNBest+ extends org.apache.lucene.analysis.morph.Viterbi<Token, ViterbiNBest.PositionNBest> {++ private final EnumMap<TokenType, Dictionary<? extends JaMorphData>> dictionaryMap =+ new EnumMap<>(TokenType.class);++ private final UnknownDictionary unkDictionary;+ private final CharacterDefinition characterDefinition;+ private final UserDictionary userDictionary;++ private final boolean discardPunctuation;+ private final boolean searchMode;+ private final boolean extendedMode;+ private final boolean outputCompounds;++ // Allowable cost difference for N-best output:+ private int nBestCost = 0;++ Lattice lattice = null;++ private GraphvizFormatter<JaMorphData> dotOut;++ ViterbiNBest(+ TokenInfoFST fst,+ FST.BytesReader fstReader,+ TokenInfoDictionary dictionary,+ TokenInfoFST userFST,+ FST.BytesReader userFSTReader,+ UserDictionary userDictionary,+ ConnectionCosts costs,+ Class<PositionNBest> positionImpl,+ UnknownDictionary unkDictionary,+ CharacterDefinition characterDefinition,+ boolean discardPunctuation,+ boolean searchMode,+ boolean extendedMode,+ boolean outputCompounds) {+ super(fst, fstReader, dictionary, userFST, userFSTReader, userDictionary, costs, positionImpl);+ this.unkDictionary = unkDictionary;+ this.characterDefinition = characterDefinition;+ this.userDictionary = userDictionary;+ this.discardPunctuation = discardPunctuation;+ this.searchMode = searchMode;+ this.extendedMode = extendedMode;+ this.outputCompounds = outputCompounds;+ dictionaryMap.put(TokenType.KNOWN, dictionary);+ dictionaryMap.put(TokenType.UNKNOWN, unkDictionary);+ dictionaryMap.put(TokenType.USER, userDictionary);+ }++ @Override+ protected boolean shouldSkipProcessUnknownWord(int unknownWordEndIndex, Position posData) {+ return !searchMode && super.shouldSkipProcessUnknownWord(unknownWordEndIndex, posData);+ }++ private static final int SEARCH_MODE_KANJI_LENGTH = 2;+ private static final int SEARCH_MODE_OTHER_LENGTH = 7; // Must be >= SEARCH_MODE_KANJI_LENGTH+ private static final int SEARCH_MODE_KANJI_PENALTY = 3000;+ private static final int SEARCH_MODE_OTHER_PENALTY = 1700;++ @Override+ protected int computePenalty(int pos, int length) throws IOException {+ if (length > SEARCH_MODE_KANJI_LENGTH) {+ boolean allKanji = true;+ // check if node consists of only kanji+ final int endPos = pos + length;+ for (int pos2 = pos; pos2 < endPos; pos2++) {+ if (!characterDefinition.isKanji((char) buffer.get(pos2))) {+ allKanji = false;+ break;+ }+ }+ if (allKanji) { // Process only Kanji keywords+ return (length - SEARCH_MODE_KANJI_LENGTH) * SEARCH_MODE_KANJI_PENALTY;+ } else if (length > SEARCH_MODE_OTHER_LENGTH) {+ return (length - SEARCH_MODE_OTHER_LENGTH) * SEARCH_MODE_OTHER_PENALTY;+ }+ }+ return 0;+ }++ // Returns the added cost that a 2nd best segmentation is+ // allowed to have. Ie, if we see path with cost X,+ // ending in a compound word, and this method returns+ // threshold > 0, then we will also find the 2nd best+ // segmentation and if its path score is within this+ // threshold of X, we'll include it in the output:+ private int computeSecondBestThreshold(int pos, int length) throws IOException {+ // TODO: maybe we do something else here, instead of just+ // using the penalty...? EG we can be more aggressive on+ // when to also test for 2nd best path+ return computePenalty(pos, length);+ }++ @Override+ protected int processUnknownWord(boolean anyMatches, Position posData) throws IOException {+ final char firstCharacter = (char) buffer.get(pos);+ if (!anyMatches || characterDefinition.isInvoke(firstCharacter)) {++ // Find unknown match:+ final int characterId = characterDefinition.getCharacterClass(firstCharacter);+ final boolean isPunct = isPunctuation(firstCharacter);++ // NOTE: copied from UnknownDictionary.lookup:+ int unknownWordLength;+ if (!characterDefinition.isGroup(firstCharacter)) {+ unknownWordLength = 1;+ } else {+ // Extract unknown word. Characters with the same character class are considered to be+ // part of unknown word+ unknownWordLength = 1;+ for (int posAhead = pos + 1; unknownWordLength < MAX_UNKNOWN_WORD_LENGTH; posAhead++) {+ final int ch = buffer.get(posAhead);+ if (ch == -1) {+ break;+ }+ if (characterId == characterDefinition.getCharacterClass((char) ch)+ && isPunctuation((char) ch) == isPunct) {+ unknownWordLength++;+ } else {+ break;+ }+ }+ }++ unkDictionary.lookupWordIds(+ characterId, wordIdRef); // characters in input text are supposed to be the same+ if (VERBOSE) {+ System.out.println(+ " UNKNOWN word len=" + unknownWordLength + " " + wordIdRef.length + " wordIDs");+ }+ for (int ofs = 0; ofs < wordIdRef.length; ofs++) {+ add(+ unkDictionary.getMorphAttributes(),+ posData,+ pos,+ posData.getPos() + unknownWordLength,+ wordIdRef.ints[wordIdRef.offset + ofs],+ TokenType.UNKNOWN,+ false);+ }++ return unknownWordLength;+ }+ return 0;+ }++ void setGraphvizFormatter(GraphvizFormatter<JaMorphData> dotOut) {+ this.dotOut = dotOut;+ }++ @Override+ protected void backtrace(Position endPosData, int fromIDX) throws IOException {+ final int endPos = endPosData.getPos();++ /**+ * LUCENE-10059: If the endPos is the same as lastBackTracePos, we don't want to backtrace to+ * avoid an assertion error {@link RollingCharBuffer#get(int)} when it tries to generate an+ * empty buffer+ */+ if (endPos == lastBackTracePos) {+ return;+ }++ if (VERBOSE) {+ System.out.println(+ "\n backtrace: endPos="+ + endPos+ + " pos="+ + pos+ + "; "+ + (pos - lastBackTracePos)+ + " characters; last="+ + lastBackTracePos+ + " cost="+ + endPosData.getCost(fromIDX));+ }++ final char[] fragment = buffer.get(lastBackTracePos, endPos - lastBackTracePos);++ if (dotOut != null) {+ dotOut.onBacktrace(+ this::getDict, positions, lastBackTracePos, endPosData, fromIDX, fragment, end);+ }++ int pos = endPos;+ int bestIDX = fromIDX;+ Token altToken = null;++ // We trace backwards, so this will be the leftWordID of+ // the token after the one we are now on:+ int lastLeftWordID = -1;++ int backCount = 0;++ // TODO: sort of silly to make Token instances here; the+ // back trace has all info needed to generate the+ // token. So, we could just directly set the attrs,+ // from the backtrace, in incrementToken w/o ever+ // creating Token; we'd have to defer calling freeBefore+ // until after the backtrace was fully "consumed" by+ // incrementToken.++ while (pos > lastBackTracePos) {+ // System.out.println("BT: back pos=" + pos + " bestIDX=" + bestIDX);+ final Position posData = positions.get(pos);+ assert bestIDX < posData.getCount();++ int backPos = posData.getBackPos(bestIDX);+ assert backPos >= lastBackTracePos+ : "backPos=" + backPos + " vs lastBackTracePos=" + lastBackTracePos;+ int length = pos - backPos;+ TokenType backType = posData.getBackType(bestIDX);+ int backID = posData.getBackID(bestIDX);+ int nextBestIDX = posData.getBackIndex(bestIDX);++ if (searchMode && altToken == null && backType != TokenType.USER) {++ // In searchMode, if best path had picked a too-long+ // token, we use the "penalty" to compute the allowed+ // max cost of an alternate back-trace. If we find an+ // alternate back trace with cost below that+ // threshold, we pursue it instead (but also output+ // the long token).+ // System.out.println(" 2nd best backPos=" + backPos + " pos=" + pos);++ final int penalty = computeSecondBestThreshold(backPos, pos - backPos);++ if (penalty > 0) {+ if (VERBOSE) {+ System.out.println(+ " compound="+ + new String(buffer.get(backPos, pos - backPos))+ + " backPos="+ + backPos+ + " pos="+ + pos+ + " penalty="+ + penalty+ + " cost="+ + posData.getCost(bestIDX)+ + " bestIDX="+ + bestIDX+ + " lastLeftID="+ + lastLeftWordID);+ }++ // Use the penalty to set maxCost on the 2nd best+ // segmentation:+ int maxCost = posData.getCost(bestIDX) + penalty;+ if (lastLeftWordID != -1) {+ maxCost += costs.get(getDict(backType).getRightId(backID), lastLeftWordID);+ }++ // Now, prune all too-long tokens from the graph:+ pruneAndRescore(backPos, pos, posData.getBackIndex(bestIDX));++ // Finally, find 2nd best back-trace and resume+ // backtrace there:+ int leastCost = Integer.MAX_VALUE;+ int leastIDX = -1;+ for (int idx = 0; idx < posData.getCount(); idx++) {+ int cost = posData.getCost(idx);+ // System.out.println(" idx=" + idx + " prevCost=" + cost);++ if (lastLeftWordID != -1) {+ cost +=+ costs.get(+ getDict(posData.getBackType(idx)).getRightId(posData.getBackID(idx)),+ lastLeftWordID);+ // System.out.println(" += bgCost=" ++ // costs.get(getDict(posData.backType[idx]).getRightId(posData.backID[idx]),+ // lastLeftWordID) + " -> " + cost);+ }+ // System.out.println("penalty " + posData.backPos[idx] + " to " + pos);+ // cost += computePenalty(posData.backPos[idx], pos - posData.backPos[idx]);+ if (cost < leastCost) {+ // System.out.println(" ** ");+ leastCost = cost;+ leastIDX = idx;+ }+ }+ // System.out.println(" leastIDX=" + leastIDX);++ if (VERBOSE) {+ System.out.println(+ " afterPrune: "+ + posData.getCount()+ + " arcs arriving; leastCost="+ + leastCost+ + " vs threshold="+ + maxCost+ + " lastLeftWordID="+ + lastLeftWordID);+ }++ if (leastIDX != -1 && leastCost <= maxCost && posData.getBackPos(leastIDX) != backPos) {+ // We should have pruned the altToken from the graph:+ assert posData.getBackPos(leastIDX) != backPos;++ // Save the current compound token, to output when+ // this alternate path joins back:+ altToken =+ new Token(+ fragment,+ backPos - lastBackTracePos,+ length,+ backPos,+ backPos + length,+ backID,+ backType,+ getDict(backType).getMorphAttributes());++ // Redirect our backtrace to 2nd best:+ bestIDX = leastIDX;+ nextBestIDX = posData.getBackIndex(bestIDX);++ backPos = posData.getBackPos(bestIDX);+ length = pos - backPos;+ backType = posData.getBackType(bestIDX);+ backID = posData.getBackID(bestIDX);+ backCount = 0;+ // System.out.println(" do alt token!");++ } else {+ // I think in theory it's possible there is no+ // 2nd best path, which is fine; in this case we+ // only output the compound token:+ // System.out.println(" no alt token! bestIDX=" + bestIDX);+ }+ }+ }++ final int offset = backPos - lastBackTracePos;+ assert offset >= 0;++ if (altToken != null && altToken.getStartOffset() >= backPos) {+ if (outputCompounds) {+ // We've backtraced to the position where the+ // compound token starts; add it now:++ // The pruning we did when we created the altToken+ // ensures that the back trace will align back with+ // the start of the altToken:+ assert altToken.getStartOffset() == backPos+ : altToken.getStartOffset() + " vs " + backPos;++ // NOTE: not quite right: the compound token may+ // have had all punctuation back traced so far, but+ // then the decompounded token at this position is+ // not punctuation. In this case backCount is 0,+ // but we should maybe add the altToken anyway...?++ if (backCount > 0) {+ backCount++;+ altToken.setPositionLength(backCount);+ if (VERBOSE) {+ System.out.println(" add altToken=" + altToken);+ }+ pending.add(altToken);+ } else {+ // This means alt token was all punct tokens:+ if (VERBOSE) {+ System.out.println(" discard all-punctuation altToken=" + altToken);+ }+ assert discardPunctuation;+ }+ }+ altToken = null;+ }++ final Dictionary<? extends JaMorphData> dict = getDict(backType);++ if (backType == TokenType.USER) {++ // Expand the phraseID we recorded into the actual+ // segmentation:+ final int[] wordIDAndLength = userDictionary.lookupSegmentation(backID);+ int wordID = wordIDAndLength[0];+ int current = 0;+ for (int j = 1; j < wordIDAndLength.length; j++) {+ final int len = wordIDAndLength[j];+ // System.out.println(" add user: len=" + len);+ int startOffset = current + backPos;+ pending.add(+ new Token(+ fragment,+ current + offset,+ len,+ startOffset,+ startOffset + len,+ wordID + j - 1,+ TokenType.USER,+ dict.getMorphAttributes()));+ if (VERBOSE) {+ System.out.println(" add USER token=" + pending.get(pending.size() - 1));+ }+ current += len;+ }++ // Reverse the tokens we just added, because when we+ // serve them up from incrementToken we serve in+ // reverse:+ Collections.reverse(+ pending.subList(pending.size() - (wordIDAndLength.length - 1), pending.size()));++ backCount += wordIDAndLength.length - 1;+ } else {++ if (extendedMode && backType == TokenType.UNKNOWN) {+ // In EXTENDED mode we convert unknown word into+ // unigrams:+ int unigramTokenCount = 0;+ for (int i = length - 1; i >= 0; i--) {+ int charLen = 1;+ if (i > 0 && Character.isLowSurrogate(fragment[offset + i])) {+ i--;+ charLen = 2;+ }+ // System.out.println(" extended tok offset="+ // + (offset + i));+ if (!discardPunctuation || !isPunctuation(fragment[offset + i])) {+ int startOffset = backPos + i;+ pending.add(+ new Token(+ fragment,+ offset + i,+ charLen,+ startOffset,+ startOffset + charLen,+ CharacterDefinition.NGRAM,+ TokenType.UNKNOWN,+ unkDictionary.getMorphAttributes()));+ unigramTokenCount++;+ }+ }+ backCount += unigramTokenCount;++ } else if (!discardPunctuation || length == 0 || !isPunctuation(fragment[offset])) {+ pending.add(+ new Token(+ fragment,+ offset,+ length,+ backPos,+ backPos + length,+ backID,+ backType,+ dict.getMorphAttributes()));+ if (VERBOSE) {+ System.out.println(" add token=" + pending.get(pending.size() - 1));+ }+ backCount++;+ } else {+ if (VERBOSE) {+ System.out.println(+ " skip punctuation token=" + new String(fragment, offset, length));+ }+ }+ }++ lastLeftWordID = dict.getLeftId(backID);+ pos = backPos;+ bestIDX = nextBestIDX;+ }++ lastBackTracePos = endPos;++ if (VERBOSE) {+ System.out.println(" freeBefore pos=" + endPos);+ }+ // Notify the circular buffers that we are done with+ // these positions:+ buffer.freeBefore(endPos);+ positions.freeBefore(endPos);+ }++ // Eliminates arcs from the lattice that are compound+ // tokens (have a penalty) or are not congruent with the+ // compound token we've matched (ie, span across the+ // startPos). This should be fairly efficient, because we+ // just keep the already intersected structure of the+ // graph, eg we don't have to consult the FSTs again:+ private void pruneAndRescore(int startPos, int endPos, int bestStartIDX) throws IOException {+ if (VERBOSE) {+ System.out.println(+ " pruneAndRescore startPos="+ + startPos+ + " endPos="+ + endPos+ + " bestStartIDX="+ + bestStartIDX);+ }++ // First pass: walk backwards, building up the forward+ // arcs and pruning inadmissible arcs:+ for (int pos = endPos; pos > startPos; pos--) {+ final Position posData = positions.get(pos);+ if (VERBOSE) {+ System.out.println(" back pos=" + pos);+ }+ for (int arcIDX = 0; arcIDX < posData.getCount(); arcIDX++) {+ final int backPos = posData.getBackPos(arcIDX);+ if (backPos >= startPos) {+ // Keep this arc:+ // System.out.println(" keep backPos=" + backPos);+ positions+ .get(backPos)+ .addForward(pos, arcIDX, posData.getBackID(arcIDX), posData.getBackType(arcIDX));+ } else {+ if (VERBOSE) {+ System.out.println(" prune");+ }+ }+ }+ if (pos != startPos) {+ posData.setCount(0);+ }+ }++ // Second pass: walk forward, re-scoring:+ for (int pos = startPos; pos < endPos; pos++) {+ final PositionNBest posData = positions.get(pos);+ if (VERBOSE) {+ System.out.println(" forward pos=" + pos + " count=" + posData.forwardCount);+ }+ if (posData.getCount() == 0) {+ // No arcs arrive here...+ if (VERBOSE) {+ System.out.println(" skip");+ }+ posData.forwardCount = 0;+ continue;+ }++ if (pos == startPos) {+ // On the initial position, only consider the best+ // path so we "force congruence": the+ // sub-segmentation is "in context" of what the best+ // path (compound token) had matched:+ final int rightID;+ if (startPos == 0) {+ rightID = 0;+ } else {+ rightID =+ getDict(posData.getBackType(bestStartIDX))+ .getRightId(posData.getBackID(bestStartIDX));+ }+ final int pathCost = posData.getCost(bestStartIDX);+ for (int forwardArcIDX = 0; forwardArcIDX < posData.forwardCount; forwardArcIDX++) {+ final TokenType forwardType = posData.forwardType[forwardArcIDX];+ final Dictionary<? extends JaMorphData> dict2 = getDict(forwardType);+ final int wordID = posData.forwardID[forwardArcIDX];+ final int toPos = posData.forwardPos[forwardArcIDX];+ final int newCost =+ pathCost+ + dict2.getWordCost(wordID)+ + costs.get(rightID, dict2.getLeftId(wordID))+ + computePenalty(pos, toPos - pos);+ if (VERBOSE) {+ System.out.println(+ " + "+ + forwardType+ + " word "+ + new String(buffer.get(pos, toPos - pos))+ + " toPos="+ + toPos+ + " cost="+ + newCost+ + " penalty="+ + computePenalty(pos, toPos - pos)+ + " toPos.idx="+ + positions.get(toPos).getCount());+ }+ positions+ .get(toPos)+ .add(newCost, dict2.getRightId(wordID), pos, -1, bestStartIDX, wordID, forwardType);+ }+ } else {+ // On non-initial positions, we maximize score+ // across all arriving lastRightIDs:+ for (int forwardArcIDX = 0; forwardArcIDX < posData.forwardCount; forwardArcIDX++) {+ final TokenType forwardType = posData.forwardType[forwardArcIDX];+ final int toPos = posData.forwardPos[forwardArcIDX];+ if (VERBOSE) {+ System.out.println(+ " + "+ + forwardType+ + " word "+ + new String(buffer.get(pos, toPos - pos))+ + " toPos="+ + toPos);+ }+ add(+ getDict(forwardType).getMorphAttributes(),+ posData,+ pos,+ toPos,+ posData.forwardID[forwardArcIDX],+ forwardType,+ true);+ }+ }+ posData.forwardCount = 0;+ }+ }++ @Override+ protected void backtraceNBest(final Position endPosData, final boolean useEOS)+ throws IOException {+ if (lattice == null) {+ lattice = new Lattice();+ }++ final int endPos = endPosData.getPos();+ char[] fragment = buffer.get(lastBackTracePos, endPos - lastBackTracePos);+ lattice.setup(fragment, dictionaryMap, positions, lastBackTracePos, endPos, useEOS);+ lattice.markUnreachable();+ lattice.calcLeftCost(costs);+ lattice.calcRightCost(costs);++ int bestCost = lattice.bestCost();+ if (VERBOSE) {+ System.out.printf("DEBUG: 1-BEST COST: %d\n", bestCost);+ }+ for (int node : lattice.bestPathNodeList()) {+ registerNode(node, fragment);+ }++ for (int n = 2; ; ++n) {+ List<Integer> nbest = lattice.nBestNodeList(n);+ if (nbest.isEmpty()) {+ break;+ }+ int cost = lattice.cost(nbest.get(0));+ if (VERBOSE) {+ System.out.printf("DEBUG: %d-BEST COST: %d\n", n, cost);+ }+ if (bestCost + nBestCost < cost) {+ break;+ }+ for (int node : nbest) {+ registerNode(node, fragment);+ }+ }+ if (VERBOSE) {+ lattice.debugPrint();+ }+ }++ private void registerNode(int node, char[] fragment) {+ int left = lattice.nodeLeft[node];+ int right = lattice.nodeRight[node];+ TokenType type = lattice.nodeDicType[node];+ if (!discardPunctuation || !isPunctuation(fragment[left])) {+ if (type == TokenType.USER) {+ // The code below are based on backtrace().+ //+ // Expand the phraseID we recorded into the actual segmentation:+ final int[] wordIDAndLength = userDictionary.lookupSegmentation(lattice.nodeWordID[node]);+ int wordID = wordIDAndLength[0];+ pending.add(+ new Token(+ fragment,+ left,+ right - left,+ lattice.rootBase + left,+ lattice.rootBase + right,+ wordID,+ TokenType.USER,+ userDictionary.getMorphAttributes()));+ // Output compound+ int current = 0;+ for (int j = 1; j < wordIDAndLength.length; j++) {+ final int len = wordIDAndLength[j];+ if (len < right - left) {+ int startOffset = lattice.rootBase + current + left;+ pending.add(+ new Token(+ fragment,+ current + left,+ len,+ startOffset,+ startOffset + len,+ wordID + j - 1,+ TokenType.USER,+ userDictionary.getMorphAttributes()));+ }+ current += len;+ }+ } else {+ pending.add(+ new Token(+ fragment,+ left,+ right - left,+ lattice.rootBase + left,+ lattice.rootBase + right,+ lattice.nodeWordID[node],+ type,+ getDict(type).getMorphAttributes()));+ }+ }+ }++ @Override+ protected void fixupPendingList() {+ // Sort for removing same tokens.+ // USER token should be ahead from normal one.+ Collections.sort(+ pending,+ (a, b) -> {+ int aOff = a.getOffset();+ int bOff = b.getOffset();+ if (aOff != bOff) {+ return aOff - bOff;+ }+ int aLen = a.getLength();+ int bLen = b.getLength();+ if (aLen != bLen) {+ return aLen - bLen;+ }+ // order of Type is KNOWN, UNKNOWN, USER,+ // so we use reversed comparison here.+ return b.getType().ordinal() - a.getType().ordinal();+ });++ // Remove same token.+ for (int i = 1; i < pending.size(); ++i) {+ Token a = pending.get(i - 1);+ Token b = pending.get(i);+ if (a.getOffset() == b.getOffset() && a.getLength() == b.getLength()) {+ pending.remove(i);+ // It is important to decrement "i" here, because a next may be removed.+ --i;+ }+ }++ // offset=>position map+ HashMap<Integer, Integer> map = new HashMap<>();+ for (Token t : pending) {+ map.put(t.getOffset(), 0);+ map.put(t.getOffset() + t.getLength(), 0);+ }++ // Get unique and sorted list of all edge position of tokens.+ Integer[] offsets = map.keySet().toArray(new Integer[0]);+ Arrays.sort(offsets);++ // setup all value of map. It specifies N-th position from begin.+ for (int i = 0; i < offsets.length; ++i) {+ map.put(offsets[i], i);+ }++ // We got all position length now.+ for (Token t : pending) {+ t.setPositionLength(map.get(t.getOffset() + t.getLength()) - map.get(t.getOffset()));+ }++ // Make PENDING to be reversed order to fit its usage.+ // If you would like to speedup, you can try reversed order sort+ // at first of this function.+ Collections.reverse(pending);+ }++ Dictionary<? extends JaMorphData> getDict(TokenType type) {+ return dictionaryMap.get(type);+ }++ void setNBestCost(int value) {+ nBestCost = value;+ outputNBest = 0 < nBestCost;+ }++ int getNBestCost() {+ return nBestCost;+ }++ private static boolean isPunctuation(char ch) {+ switch (Character.getType(ch)) {+ case Character.SPACE_SEPARATOR:+ case Character.LINE_SEPARATOR:+ case Character.PARAGRAPH_SEPARATOR:+ case Character.CONTROL:+ case Character.FORMAT:+ case Character.DASH_PUNCTUATION:+ case Character.START_PUNCTUATION:+ case Character.END_PUNCTUATION:+ case Character.CONNECTOR_PUNCTUATION:+ case Character.OTHER_PUNCTUATION:+ case Character.MATH_SYMBOL:+ case Character.CURRENCY_SYMBOL:+ case Character.MODIFIER_SYMBOL:+ case Character.OTHER_SYMBOL:+ case Character.INITIAL_QUOTE_PUNCTUATION:+ case Character.FINAL_QUOTE_PUNCTUATION:+ return true;+ default:+ return false;+ }+ }++ /**+ * {@link org.apache.lucene.analysis.morph.Viterbi.Position} extension; this holds all forward+ * pointers to calculate n-best path.+ */+ public static class PositionNBest extends Viterbi.Position {+ // Only used when finding 2nd best segmentation under a+ // too-long token:+ int forwardCount;+ int[] forwardPos = new int[8];+ int[] forwardID = new int[8];+ int[] forwardIndex = new int[8];+ TokenType[] forwardType = new TokenType[8];++ private void growForward() {+ forwardPos = ArrayUtil.grow(forwardPos, 1 + forwardCount);+ forwardID = ArrayUtil.grow(forwardID, 1 + forwardCount);+ forwardIndex = ArrayUtil.grow(forwardIndex, 1 + forwardCount);++ // NOTE: sneaky: grow separately because+ // ArrayUtil.grow will otherwise pick a different+ // length than the int[]s we just grew:+ final TokenType[] newForwardType = new TokenType[forwardPos.length];+ System.arraycopy(forwardType, 0, newForwardType, 0, forwardType.length);+ forwardType = newForwardType;+ }++ void addForward(int forwardPos, int forwardIndex, int forwardID, TokenType forwardType) {+ if (forwardCount == this.forwardID.length) {+ growForward();+ }+ this.forwardPos[forwardCount] = forwardPos;+ this.forwardIndex[forwardCount] = forwardIndex;+ this.forwardID[forwardCount] = forwardID;+ this.forwardType[forwardCount] = forwardType;+ forwardCount++;+ }++ @Override+ public void reset() {+ super.reset();+ // forwardCount naturally resets after it runs:+ assert forwardCount == 0 : "pos=" + getPos() + " forwardCount=" + forwardCount;+ }+ }++ // yet another lattice data structure+ static final class Lattice {+ char[] fragment;+ EnumMap<TokenType, Dictionary<? extends JaMorphData>> dictionaryMap;+ boolean useEOS;++ int rootCapacity = 0;+ int rootSize = 0;+ int rootBase = 0;++ // root pointers of node chain by leftChain_ that have same start offset.+ int[] lRoot;+ // root pointers of node chain by rightChain_ that have same end offset.+ int[] rRoot;++ int capacity = 0;+ int nodeCount = 0;++ // The variables below are elements of lattice node that indexed by node number.+ TokenType[] nodeDicType;+ int[] nodeWordID;+ // nodeMark - -1:excluded, 0:unused, 1:bestpath, 2:2-best-path, ... N:N-best-path+ int[] nodeMark;+ int[] nodeLeftID;+ int[] nodeRightID;+ int[] nodeWordCost;+ int[] nodeLeftCost;+ int[] nodeRightCost;+ // nodeLeftNode, nodeRightNode - are left/right node number with minimum cost path.+ int[] nodeLeftNode;+ int[] nodeRightNode;+ // nodeLeft, nodeRight - start/end offset+ int[] nodeLeft;+ int[] nodeRight;+ int[] nodeLeftChain;+ int[] nodeRightChain;++ private void setupRoot(int baseOffset, int lastOffset) {+ assert baseOffset <= lastOffset;+ int size = lastOffset - baseOffset + 1;+ if (rootCapacity < size) {+ int oversize = ArrayUtil.oversize(size, Integer.BYTES);+ lRoot = new int[oversize];+ rRoot = new int[oversize];+ rootCapacity = oversize;+ }+ Arrays.fill(lRoot, 0, size, -1);+ Arrays.fill(rRoot, 0, size, -1);+ rootSize = size;+ rootBase = baseOffset;+ }++ // Reserve at least N nodes.+ private void reserve(int n) {+ if (capacity < n) {+ int oversize = ArrayUtil.oversize(n, Integer.BYTES);+ nodeDicType = new TokenType[oversize];+ nodeWordID = new int[oversize];+ nodeMark = new int[oversize];+ nodeLeftID = new int[oversize];+ nodeRightID = new int[oversize];+ nodeWordCost = new int[oversize];+ nodeLeftCost = new int[oversize];+ nodeRightCost = new int[oversize];+ nodeLeftNode = new int[oversize];+ nodeRightNode = new int[oversize];+ nodeLeft = new int[oversize];+ nodeRight = new int[oversize];+ nodeLeftChain = new int[oversize];+ nodeRightChain = new int[oversize];+ capacity = oversize;+ }+ }++ private void setupNodePool(int n) {+ reserve(n);+ nodeCount = 0;+ if (VERBOSE) {+ System.out.printf("DEBUG: setupNodePool: n = %d\n", n);+ System.out.printf("DEBUG: setupNodePool: lattice.capacity = %d\n", capacity);+ }+ }++ private int addNode(TokenType dicType, int wordID, int left, int right) {+ if (VERBOSE) {+ System.out.printf(+ "DEBUG: addNode: dicType=%s, wordID=%d, left=%d, right=%d, str=%s\n",+ dicType.toString(),+ wordID,+ left,+ right,+ left == -1 ? "BOS" : right == -1 ? "EOS" : new String(fragment, left, right - left));+ }+ assert nodeCount < capacity;+ assert left == -1 || right == -1 || left < right;+ assert left == -1 || (0 <= left && left < rootSize);+ assert right == -1 || (0 <= right && right < rootSize);++ int node = nodeCount++;++ if (VERBOSE) {+ System.out.printf("DEBUG: addNode: node=%d\n", node);+ }++ nodeDicType[node] = dicType;+ nodeWordID[node] = wordID;+ nodeMark[node] = 0;++ if (wordID < 0) {+ nodeWordCost[node] = 0;+ nodeLeftCost[node] = 0;+ nodeRightCost[node] = 0;+ nodeLeftID[node] = 0;+ nodeRightID[node] = 0;+ } else {+ Dictionary<? extends JaMorphData> dic = dictionaryMap.get(dicType);+ nodeWordCost[node] = dic.getWordCost(wordID);+ nodeLeftID[node] = dic.getLeftId(wordID);+ nodeRightID[node] = dic.getRightId(wordID);+ }++ if (VERBOSE) {+ System.out.printf(+ "DEBUG: addNode: wordCost=%d, leftID=%d, rightID=%d\n",+ nodeWordCost[node], nodeLeftID[node], nodeRightID[node]);+ }++ nodeLeft[node] = left;+ nodeRight[node] = right;+ if (0 <= left) {+ nodeLeftChain[node] = lRoot[left];+ lRoot[left] = node;+ } else {+ nodeLeftChain[node] = -1;+ }+ if (0 <= right) {+ nodeRightChain[node] = rRoot[right];+ rRoot[right] = node;+ } else {+ nodeRightChain[node] = -1;+ }+ return node;+ }++ // Sum of positions.get(i).count in [beg, end) range.+ // using stream:+ // return IntStream.range(beg, end).map(i -> positions.get(i).count).sum();+ private int positionCount(WrappedPositionArray<PositionNBest> positions, int beg, int end) {+ int count = 0;+ for (int i = beg; i < end; ++i) {+ count += positions.get(i).getCount();+ }+ return count;+ }++ void setup(+ char[] fragment,+ EnumMap<TokenType, Dictionary<? extends JaMorphData>> dictionaryMap,+ WrappedPositionArray<PositionNBest> positions,+ int prevOffset,+ int endOffset,+ boolean useEOS) {+ assert positions.get(prevOffset).getCount() == 1;+ if (VERBOSE) {+ System.out.printf("DEBUG: setup: prevOffset=%d, endOffset=%d\n", prevOffset, endOffset);+ }++ this.fragment = fragment;+ this.dictionaryMap = dictionaryMap;+ this.useEOS = useEOS;++ // Initialize lRoot and rRoot.+ setupRoot(prevOffset, endOffset);++ // "+ 2" for first/last record.+ setupNodePool(positionCount(positions, prevOffset + 1, endOffset + 1) + 2);++ // substitute for BOS = 0+ Position first = positions.get(prevOffset);+ if (addNode(first.getBackType(0), first.getBackID(0), -1, 0) != 0) {+ assert false;+ }++ // EOS = 1+ if (addNode(TokenType.KNOWN, -1, endOffset - rootBase, -1) != 1) {+ assert false;+ }++ for (int offset = endOffset; prevOffset < offset; --offset) {+ int right = offset - rootBase;+ // optimize: exclude disconnected nodes.+ if (0 <= lRoot[right]) {+ Position pos = positions.get(offset);+ for (int i = 0; i < pos.getCount(); ++i) {+ addNode(pos.getBackType(i), pos.getBackID(i), pos.getBackPos(i) - rootBase, right);+ }+ }+ }+ }++ // set mark = -1 for unreachable nodes.+ void markUnreachable() {+ for (int index = 1; index < rootSize - 1; ++index) {+ if (rRoot[index] < 0) {+ for (int node = lRoot[index]; 0 <= node; node = nodeLeftChain[node]) {+ if (VERBOSE) {+ System.out.printf("DEBUG: markUnreachable: node=%d\n", node);+ }+ nodeMark[node] = -1;+ }+ }+ }+ }++ private int connectionCost(ConnectionCosts costs, int left, int right) {+ int leftID = nodeLeftID[right];+ return ((leftID == 0 && !useEOS) ? 0 : costs.get(nodeRightID[left], leftID));+ }++ void calcLeftCost(ConnectionCosts costs) {+ for (int index = 0; index < rootSize; ++index) {+ for (int node = lRoot[index]; 0 <= node; node = nodeLeftChain[node]) {+ if (0 <= nodeMark[node]) {+ int leastNode = -1;+ int leastCost = Integer.MAX_VALUE;+ for (int leftNode = rRoot[index]; 0 <= leftNode; leftNode = nodeRightChain[leftNode]) {+ if (0 <= nodeMark[leftNode]) {+ int cost =+ nodeLeftCost[leftNode]+ + nodeWordCost[leftNode]+ + connectionCost(costs, leftNode, node);+ if (cost < leastCost) {+ leastCost = cost;+ leastNode = leftNode;+ }+ }+ }+ assert 0 <= leastNode;+ nodeLeftNode[node] = leastNode;+ nodeLeftCost[node] = leastCost;+ if (VERBOSE) {+ System.out.printf(+ "DEBUG: calcLeftCost: node=%d, leftNode=%d, leftCost=%d\n",+ node, nodeLeftNode[node], nodeLeftCost[node]);+ }+ }+ }+ }+ }++ void calcRightCost(ConnectionCosts costs) {+ for (int index = rootSize - 1; 0 <= index; --index) {+ for (int node = rRoot[index]; 0 <= node; node = nodeRightChain[node]) {+ if (0 <= nodeMark[node]) {+ int leastNode = -1;+ int leastCost = Integer.MAX_VALUE;+ for (int rightNode = lRoot[index];+ 0 <= rightNode;+ rightNode = nodeLeftChain[rightNode]) {+ if (0 <= nodeMark[rightNode]) {+ int cost =+ nodeRightCost[rightNode]+ + nodeWordCost[rightNode]+ + connectionCost(costs, node, rightNode);+ if (cost < leastCost) {+ leastCost = cost;+ leastNode = rightNode;+ }+ }+ }+ assert 0 <= leastNode;+ nodeRightNode[node] = leastNode;+ nodeRightCost[node] = leastCost;+ if (VERBOSE) {+ System.out.printf(+ "DEBUG: calcRightCost: node=%d, rightNode=%d, rightCost=%d\n",+ node, nodeRightNode[node], nodeRightCost[node]);+ }+ }+ }+ }+ }++ // Mark all nodes that have same text and different par-of-speech or reading.+ void markSameSpanNode(int refNode, int value) {+ int left = nodeLeft[refNode];+ int right = nodeRight[refNode];+ for (int node = lRoot[left]; 0 <= node; node = nodeLeftChain[node]) {+ if (nodeRight[node] == right) {+ nodeMark[node] = value;+ }+ }+ }++ List<Integer> bestPathNodeList() {+ List<Integer> list = new ArrayList<>();+ for (int node = nodeRightNode[0]; node != 1; node = nodeRightNode[node]) {+ list.add(node);+ markSameSpanNode(node, 1);+ }+ return list;+ }++ private int cost(int node) {+ return nodeLeftCost[node] + nodeWordCost[node] + nodeRightCost[node];+ }++ List<Integer> nBestNodeList(int N) {+ List<Integer> list = new ArrayList<>();+ int leastCost = Integer.MAX_VALUE;+ int leastLeft = -1;+ int leastRight = -1;+ for (int node = 2; node < nodeCount; ++node) {+ if (nodeMark[node] == 0) {+ int cost = cost(node);+ if (cost < leastCost) {+ leastCost = cost;+ leastLeft = nodeLeft[node];+ leastRight = nodeRight[node];+ list.clear();+ list.add(node);+ } else if (cost == leastCost+ && (nodeLeft[node] != leastLeft || nodeRight[node] != leastRight)) {+ list.add(node);+ }+ }+ }+ for (int node : list) {+ markSameSpanNode(node, N);+ }+ return list;+ }++ int bestCost() {+ return nodeLeftCost[1];+ }++ int probeDelta(int start, int end) {+ int left = start - rootBase;+ int right = end - rootBase;+ if (left < 0 || rootSize < right) {+ return Integer.MAX_VALUE;+ }+ int probedCost = Integer.MAX_VALUE;+ for (int node = lRoot[left]; 0 <= node; node = nodeLeftChain[node]) {+ if (nodeRight[node] == right) {+ probedCost = Math.min(probedCost, cost(node));+ }+ }+ return probedCost - bestCost();+ }++ void debugPrint() {+ if (VERBOSE) {+ for (int node = 0; node < nodeCount; ++node) {+ System.out.printf(+ "DEBUG NODE: node=%d, mark=%d, cost=%d, left=%d, right=%d\n",+ node, nodeMark[node], cost(node), nodeLeft[node], nodeRight[node]);+ }+ }+ }+ }+}diff --git a/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseTokenizer.java b/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseTokenizer.javaindex 6e3cf94002e..9e55d59c8c3 100644--- a/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseTokenizer.java+++ b/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseTokenizer.java@@ -31,8 +31,10 @@ import org.apache.lucene.analysis.TokenStream;import org.apache.lucene.analysis.Tokenizer;import org.apache.lucene.analysis.ja.JapaneseTokenizer.Mode;import org.apache.lucene.analysis.ja.dict.ConnectionCosts;+import org.apache.lucene.analysis.ja.dict.JaMorphData;import org.apache.lucene.analysis.ja.dict.UserDictionary;import org.apache.lucene.analysis.ja.tokenattributes.*;+import org.apache.lucene.analysis.morph.GraphvizFormatter;import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;import org.apache.lucene.tests.analysis.BaseTokenStreamTestCase;import org.apache.lucene.tests.analysis.MockGraphTokenFilter;@@ -518,7 +520,8 @@ public class TestJapaneseTokenizer extends BaseTokenStreamTestCase {}public void testLatticeToDot() throws Exception {- final GraphvizFormatter gv2 = new GraphvizFormatter(ConnectionCosts.getInstance());+ final GraphvizFormatter<JaMorphData> gv2 =+ new GraphvizFormatter<>(ConnectionCosts.getInstance());final Analyzer analyzer =new Analyzer() {@Overridediff --git a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/KoreanTokenizer.java b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/KoreanTokenizer.javaindex 76a95047e1d..15bdb3422ad 100644--- a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/KoreanTokenizer.java+++ b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/KoreanTokenizer.java@@ -16,15 +16,8 @@*/package org.apache.lucene.analysis.ko;-import static java.lang.Character.UnicodeScript;-import java.io.IOException;-import java.util.ArrayList;-import java.util.Arrays;-import java.util.EnumMap;-import java.util.List;import org.apache.lucene.analysis.Tokenizer;-import org.apache.lucene.analysis.ko.dict.CharacterDefinition;import org.apache.lucene.analysis.ko.dict.ConnectionCosts;import org.apache.lucene.analysis.ko.dict.DictionaryBuilder;import org.apache.lucene.analysis.ko.dict.KoMorphData;@@ -34,18 +27,13 @@ import org.apache.lucene.analysis.ko.dict.UnknownDictionary;import org.apache.lucene.analysis.ko.dict.UserDictionary;import org.apache.lucene.analysis.ko.tokenattributes.PartOfSpeechAttribute;import org.apache.lucene.analysis.ko.tokenattributes.ReadingAttribute;-import org.apache.lucene.analysis.morph.Dictionary;-import org.apache.lucene.analysis.morph.TokenType;+import org.apache.lucene.analysis.morph.GraphvizFormatter;import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;import org.apache.lucene.analysis.tokenattributes.PositionLengthAttribute;-import org.apache.lucene.analysis.util.RollingCharBuffer;-import org.apache.lucene.util.ArrayUtil;import org.apache.lucene.util.AttributeFactory;import org.apache.lucene.util.IgnoreRandomChains;-import org.apache.lucene.util.IntsRef;-import org.apache.lucene.util.RamUsageEstimator;import org.apache.lucene.util.fst.FST;/**@@ -86,46 +74,7 @@ public final class KoreanTokenizer extends Tokenizer {private static final boolean VERBOSE = false;- // For safety:- private static final int MAX_UNKNOWN_WORD_LENGTH = 1024;- private static final int MAX_BACKTRACE_GAP = 1024;-- private final EnumMap<TokenType, Dictionary<? extends KoMorphData>> dictionaryMap =- new EnumMap<>(TokenType.class);-- private final TokenInfoFST fst;- private final TokenInfoDictionary dictionary;- private final UnknownDictionary unkDictionary;- private final ConnectionCosts costs;- private final UserDictionary userDictionary;- private final CharacterDefinition characterDefinition;-- private final FST.Arc<Long> arc = new FST.Arc<>();- private final FST.BytesReader fstReader;- private final IntsRef wordIdRef = new IntsRef();-- private final FST.BytesReader userFSTReader;- private final TokenInfoFST userFST;-- private final boolean discardPunctuation;- private final DecompoundMode mode;- private final boolean outputUnknownUnigrams;-- private final RollingCharBuffer buffer = new RollingCharBuffer();-- private final WrappedPositionArray positions = new WrappedPositionArray();-- // True once we've hit the EOF from the input reader:- private boolean end;-- // Last absolute position we backtraced from:- private int lastBackTracePos;-- // Next absolute position to process:- private int pos;-- // Already parsed, but not yet passed to caller, tokens:- private final List<Token> pending = new ArrayList<>();+ private final Viterbi viterbi;private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);@@ -212,275 +161,54 @@ public final class KoreanTokenizer extends Tokenizer {boolean outputUnknownUnigrams,boolean discardPunctuation) {super(factory);- this.dictionary = systemDictionary;- this.fst = dictionary.getFST();- this.unkDictionary = unkDictionary;- this.characterDefinition = unkDictionary.getCharacterDefinition();- this.costs = connectionCosts;- this.userDictionary = userDictionary;- fstReader = fst.getBytesReader();+ TokenInfoFST fst = systemDictionary.getFST();+ FST.BytesReader fstReader = fst.getBytesReader();+ TokenInfoFST userFST = null;+ FST.BytesReader userFSTReader = null;if (userDictionary != null) {userFST = userDictionary.getFST();userFSTReader = userFST.getBytesReader();- } else {- userFST = null;- userFSTReader = null;}- this.mode = mode;- this.outputUnknownUnigrams = outputUnknownUnigrams;- this.discardPunctuation = discardPunctuation;- buffer.reset(this.input);-- resetState();-- dictionaryMap.put(TokenType.KNOWN, dictionary);- dictionaryMap.put(TokenType.UNKNOWN, unkDictionary);- dictionaryMap.put(TokenType.USER, userDictionary);- }-- private GraphvizFormatter dotOut;- /** Expert: set this to produce graphviz (dot) output of the Viterbi lattice */- public void setGraphvizFormatter(GraphvizFormatter dotOut) {- this.dotOut = dotOut;+ viterbi =+ new Viterbi(+ fst,+ fstReader,+ systemDictionary,+ userFST,+ userFSTReader,+ userDictionary,+ connectionCosts,+ unkDictionary,+ unkDictionary.getCharacterDefinition(),+ discardPunctuation,+ mode,+ outputUnknownUnigrams);+ viterbi.resetBuffer(input);+ viterbi.resetState();}@Overridepublic void close() throws IOException {super.close();- buffer.reset(input);+ viterbi.resetBuffer(input);}@Overridepublic void reset() throws IOException {super.reset();- buffer.reset(input);- resetState();- }-- private void resetState() {- positions.reset();- pos = 0;- end = false;- lastBackTracePos = 0;- pending.clear();-- // Add BOS:- positions.get(0).add(0, 0, -1, -1, -1, -1, TokenType.KNOWN);+ viterbi.resetBuffer(input);+ viterbi.resetState();}@Overridepublic void end() throws IOException {super.end();// Set final offset- int finalOffset = correctOffset(pos);+ int finalOffset = correctOffset(viterbi.getPos());offsetAtt.setOffset(finalOffset, finalOffset);}- // Holds all back pointers arriving to this position:- static final class Position {-- int pos;-- int count;-- // maybe single int array * 5?- int[] costs = new int[8];- int[] lastRightID = new int[8];- int[] backPos = new int[8];- int[] backWordPos = new int[8];- int[] backIndex = new int[8];- int[] backID = new int[8];- TokenType[] backType = new TokenType[8];-- public void grow() {- costs = ArrayUtil.grow(costs, 1 + count);- lastRightID = ArrayUtil.grow(lastRightID, 1 + count);- backPos = ArrayUtil.grow(backPos, 1 + count);- backWordPos = ArrayUtil.grow(backWordPos, 1 + count);- backIndex = ArrayUtil.grow(backIndex, 1 + count);- backID = ArrayUtil.grow(backID, 1 + count);-- // NOTE: sneaky: grow separately because- // ArrayUtil.grow will otherwise pick a different- // length than the int[]s we just grew:- final TokenType[] newBackType = new TokenType[backID.length];- System.arraycopy(backType, 0, newBackType, 0, backType.length);- backType = newBackType;- }-- public void add(- int cost,- int lastRightID,- int backPos,- int backRPos,- int backIndex,- int backID,- TokenType backType) {- // NOTE: this isn't quite a true Viterbi search,- // because we should check if lastRightID is- // already present here, and only update if the new- // cost is less than the current cost, instead of- // simply appending. However, that will likely hurt- // performance (usually we add a lastRightID only once),- // and it means we actually create the full graph- // intersection instead of a "normal" Viterbi lattice:- if (count == costs.length) {- grow();- }- this.costs[count] = cost;- this.lastRightID[count] = lastRightID;- this.backPos[count] = backPos;- this.backWordPos[count] = backRPos;- this.backIndex[count] = backIndex;- this.backID[count] = backID;- this.backType[count] = backType;- count++;- }-- public void reset() {- count = 0;- }- }-- /**- * Returns the space penalty associated with the provided {@link POS.Tag}.- *- * @param leftPOS the left part of speech of the current token.- * @param numSpaces the number of spaces before the current token.- */- private int computeSpacePenalty(POS.Tag leftPOS, int numSpaces) {- int spacePenalty = 0;- if (numSpaces > 0) {- // TODO we should extract the penalty (left-space-penalty-factor) from the dicrc file.- switch (leftPOS) {- case E:- case J:- case VCP:- case XSA:- case XSN:- case XSV:- spacePenalty = 3000;- break;- case IC:- case MAG:- case MAJ:- case MM:- case NA:- case NNB:- case NNBC:- case NNG:- case NNP:- case NP:- case NR:- case SC:- case SE:- case SF:- case SH:- case SL:- case SN:- case SP:- case SSC:- case SSO:- case SY:- case UNA:- case UNKNOWN:- case VA:- case VCN:- case VSV:- case VV:- case VX:- case XPN:- case XR:- default:- break;- }- }- return spacePenalty;- }-- private void add(- KoMorphData morphAtts,- Position fromPosData,- int wordPos,- int endPos,- int wordID,- TokenType type) {- final POS.Tag leftPOS = morphAtts.getLeftPOS(wordID);- final int wordCost = morphAtts.getWordCost(wordID);- final int leftID = morphAtts.getLeftId(wordID);- int leastCost = Integer.MAX_VALUE;- int leastIDX = -1;- assert fromPosData.count > 0;- for (int idx = 0; idx < fromPosData.count; idx++) {- // The number of spaces before the term- int numSpaces = wordPos - fromPosData.pos;-- // Cost is path cost so far, plus word cost (added at- // end of loop), plus bigram cost and space penalty cost.- final int cost =- fromPosData.costs[idx]- + costs.get(fromPosData.lastRightID[idx], leftID)- + computeSpacePenalty(leftPOS, numSpaces);- if (VERBOSE) {- System.out.println(- " fromIDX="- + idx- + ": cost="- + cost- + " (prevCost="- + fromPosData.costs[idx]- + " wordCost="- + wordCost- + " bgCost="- + costs.get(fromPosData.lastRightID[idx], leftID)- + " spacePenalty="- + computeSpacePenalty(leftPOS, numSpaces)- + ") leftID="- + leftID- + " leftPOS="- + leftPOS.name()- + ")");- }- if (cost < leastCost) {- leastCost = cost;- leastIDX = idx;- if (VERBOSE) {- System.out.println(" **");- }- }- }-- leastCost += wordCost;-- if (VERBOSE) {- System.out.println(- " + cost="- + leastCost- + " wordID="- + wordID- + " leftID="- + leftID- + " leastIDX="- + leastIDX- + " toPos="- + endPos- + " toPos.idx="- + positions.get(endPos).count);- }-- positions- .get(endPos)- .add(- leastCost,- morphAtts.getRightId(wordID),- fromPosData.pos,- wordPos,- leastIDX,- wordID,- type);- }-@Overridepublic boolean incrementToken() throws IOException {@@ -488,16 +216,16 @@ public final class KoreanTokenizer extends Tokenizer {// tokens, when the tokens it had produced were entirely// punctuation. So we loop here until we get a real// token or we end:- while (pending.size() == 0) {- if (end) {+ while (viterbi.getPending().size() == 0) {+ if (viterbi.isEnd()) {return false;}// Push Viterbi forward some more:- parse();+ viterbi.forward();}- final Token token = pending.remove(pending.size() - 1);+ final Token token = viterbi.getPending().remove(viterbi.getPending().size() - 1);int length = token.getLength();clearAttributes();@@ -516,690 +244,8 @@ public final class KoreanTokenizer extends Tokenizer {return true;}- // TODO: make generic'd version of this "circular array"?- // It's a bit tricky because we do things to the Position- // (eg, set .pos = N on reuse)...- static final class WrappedPositionArray {- private Position[] positions = new Position[8];-- public WrappedPositionArray() {- for (int i = 0; i < positions.length; i++) {- positions[i] = new Position();- }- }-- // Next array index to write to in positions:- private int nextWrite;-- // Next position to write:- private int nextPos;-- // How many valid Position instances are held in the- // positions array:- private int count;-- public void reset() {- nextWrite--;- while (count > 0) {- if (nextWrite == -1) {- nextWrite = positions.length - 1;- }- positions[nextWrite--].reset();- count--;- }- nextWrite = 0;- nextPos = 0;- count = 0;- }-- /**- * Get Position instance for this absolute position; this is allowed to be arbitrarily far "in- * the future" but cannot be before the last freeBefore.- */- public Position get(int pos) {- while (pos >= nextPos) {- // System.out.println("count=" + count + " vs len=" + positions.length);- if (count == positions.length) {- Position[] newPositions =- new Position[ArrayUtil.oversize(1 + count, RamUsageEstimator.NUM_BYTES_OBJECT_REF)];- // System.out.println("grow positions " + newPositions.length);- System.arraycopy(positions, nextWrite, newPositions, 0, positions.length - nextWrite);- System.arraycopy(positions, 0, newPositions, positions.length - nextWrite, nextWrite);- for (int i = positions.length; i < newPositions.length; i++) {- newPositions[i] = new Position();- }- nextWrite = positions.length;- positions = newPositions;- }- if (nextWrite == positions.length) {- nextWrite = 0;- }- // Should have already been reset:- assert positions[nextWrite].count == 0;- positions[nextWrite++].pos = nextPos++;- count++;- }- assert inBounds(pos);- final int index = getIndex(pos);- assert positions[index].pos == pos;- return positions[index];- }-- public int getNextPos() {- return nextPos;- }-- // For assert:- private boolean inBounds(int pos) {- return pos < nextPos && pos >= nextPos - count;- }-- private int getIndex(int pos) {- int index = nextWrite - (nextPos - pos);- if (index < 0) {- index += positions.length;- }- return index;- }-- public void freeBefore(int pos) {- final int toFree = count - (nextPos - pos);- assert toFree >= 0;- assert toFree <= count;- int index = nextWrite - count;- if (index < 0) {- index += positions.length;- }- for (int i = 0; i < toFree; i++) {- if (index == positions.length) {- index = 0;- }- // System.out.println(" fb idx=" + index);- positions[index].reset();- index++;- }- count -= toFree;- }- }-- /* Incrementally parse some more characters. This runs- * the viterbi search forwards "enough" so that we- * generate some more tokens. How much forward depends on- * the chars coming in, since some chars could cause- * longer-lasting ambiguity in the parsing. Once the- * ambiguity is resolved, then we back trace, produce- * the pending tokens, and return. */- private void parse() throws IOException {- if (VERBOSE) {- System.out.println("\nPARSE");- }-- // Index of the last character of unknown word:- int unknownWordEndIndex = -1;-- // Maximum posAhead of user word in the entire input- int userWordMaxPosAhead = -1;-- // Advances over each position (character):- while (buffer.get(pos) != -1) {- final Position posData = positions.get(pos);- final boolean isFrontier = positions.getNextPos() == pos + 1;-- if (posData.count == 0) {- // No arcs arrive here; move to next position:- if (VERBOSE) {- System.out.println(" no arcs in; skip pos=" + pos);- }- pos++;- continue;- }-- if (pos > lastBackTracePos && posData.count == 1 && isFrontier) {- // We are at a "frontier", and only one node is- // alive, so whatever the eventual best path is must- // come through this node. So we can safely commit- // to the prefix of the best path at this point:- backtrace(posData, 0);-- // Re-base cost so we don't risk int overflow:- posData.costs[0] = 0;- if (pending.size() > 0) {- return;- } else {- // This means the backtrace only produced- // punctuation tokens, so we must keep parsing.- }- }-- if (pos - lastBackTracePos >= MAX_BACKTRACE_GAP) {- // Safety: if we've buffered too much, force a- // backtrace now. We find the least-cost partial- // path, across all paths, backtrace from it, and- // then prune all others. Note that this, in- // general, can produce the wrong result, if the- // total best path did not in fact back trace- // through this partial best path. But it's the- // best we can do... (short of not having a- // safety!).-- // First pass: find least cost partial path so far,- // including ending at future positions:- int leastIDX = -1;- int leastCost = Integer.MAX_VALUE;- Position leastPosData = null;- for (int pos2 = pos; pos2 < positions.getNextPos(); pos2++) {- final Position posData2 = positions.get(pos2);- for (int idx = 0; idx < posData2.count; idx++) {- // System.out.println(" idx=" + idx + " cost=" + cost);- final int cost = posData2.costs[idx];- if (cost < leastCost) {- leastCost = cost;- leastIDX = idx;- leastPosData = posData2;- }- }- }-- // We will always have at least one live path:- assert leastIDX != -1;-- // Second pass: prune all but the best path:- for (int pos2 = pos; pos2 < positions.getNextPos(); pos2++) {- final Position posData2 = positions.get(pos2);- if (posData2 != leastPosData) {- posData2.reset();- } else {- if (leastIDX != 0) {- posData2.costs[0] = posData2.costs[leastIDX];- posData2.lastRightID[0] = posData2.lastRightID[leastIDX];- posData2.backPos[0] = posData2.backPos[leastIDX];- posData2.backWordPos[0] = posData2.backWordPos[leastIDX];- posData2.backIndex[0] = posData2.backIndex[leastIDX];- posData2.backID[0] = posData2.backID[leastIDX];- posData2.backType[0] = posData2.backType[leastIDX];- }- posData2.count = 1;- }- }-- backtrace(leastPosData, 0);-- // Re-base cost so we don't risk int overflow:- Arrays.fill(leastPosData.costs, 0, leastPosData.count, 0);-- if (pos != leastPosData.pos) {- // We jumped into a future position:- assert pos < leastPosData.pos;- pos = leastPosData.pos;- }- if (pending.size() > 0) {- return;- } else {- // This means the backtrace only produced- // punctuation tokens, so we must keep parsing.- continue;- }- }-- if (VERBOSE) {- System.out.println(- "\n extend @ pos="- + pos- + " char="- + (char) buffer.get(pos)- + " hex="- + Integer.toHexString(buffer.get(pos)));- }-- if (VERBOSE) {- System.out.println(" " + posData.count + " arcs in");- }-- // We add single space separator as prefixes of the terms that we extract.- // This information is needed to compute the space penalty factor of each term.- // These whitespace prefixes are removed when the final tokens are generated, or- // added as separated tokens when discardPunctuation is unset.- if (Character.getType(buffer.get(pos)) == Character.SPACE_SEPARATOR) {- if (buffer.get(++pos) == -1) {- pos = posData.pos;- }- }-- boolean anyMatches = false;-- // First try user dict:- if (userFST != null) {- userFST.getFirstArc(arc);- int output = 0;- int maxPosAhead = 0;- int outputMaxPosAhead = 0;- int arcFinalOutMaxPosAhead = 0;-- for (int posAhead = pos; ; posAhead++) {- final int ch = buffer.get(posAhead);- if (ch == -1) {- break;- }- if (userFST.findTargetArc(ch, arc, arc, posAhead == pos, userFSTReader) == null) {- break;- }- output += arc.output().intValue();- if (arc.isFinal()) {- maxPosAhead = posAhead;- outputMaxPosAhead = output;- arcFinalOutMaxPosAhead = arc.nextFinalOutput().intValue();- anyMatches = true;- }- }-- // Longest matching for user word- if (anyMatches && maxPosAhead > userWordMaxPosAhead) {- if (VERBOSE) {- System.out.println(- " USER word "- + new String(buffer.get(pos, maxPosAhead + 1))- + " toPos="- + (maxPosAhead + 1));- }- add(- userDictionary.getMorphAttributes(),- posData,- pos,- maxPosAhead + 1,- outputMaxPosAhead + arcFinalOutMaxPosAhead,- TokenType.USER);- userWordMaxPosAhead = Math.max(userWordMaxPosAhead, maxPosAhead);- }- }-- // TODO: we can be more aggressive about user- // matches? if we are "under" a user match then don't- // extend KNOWN/UNKNOWN paths?-- if (!anyMatches) {- // Next, try known dictionary matches- fst.getFirstArc(arc);- int output = 0;-- for (int posAhead = pos; ; posAhead++) {- final int ch = buffer.get(posAhead);- if (ch == -1) {- break;- }- // System.out.println(" match " + (char) ch + " posAhead=" + posAhead);-- if (fst.findTargetArc(ch, arc, arc, posAhead == pos, fstReader) == null) {- break;- }-- output += arc.output().intValue();-- // Optimization: for known words that are too-long- // (compound), we should pre-compute the 2nd- // best segmentation and store it in the- // dictionary instead of recomputing it each time a- // match is found.-- if (arc.isFinal()) {- dictionary.lookupWordIds(output + arc.nextFinalOutput().intValue(), wordIdRef);- if (VERBOSE) {- System.out.println(- " KNOWN word "- + new String(buffer.get(pos, posAhead - pos + 1))- + " toPos="- + (posAhead + 1)- + " "- + wordIdRef.length- + " wordIDs");- }- for (int ofs = 0; ofs < wordIdRef.length; ofs++) {- add(- dictionary.getMorphAttributes(),- posData,- pos,- posAhead + 1,- wordIdRef.ints[wordIdRef.offset + ofs],- TokenType.KNOWN);- anyMatches = true;- }- }- }- }-- if (unknownWordEndIndex > posData.pos) {- pos++;- continue;- }-- final char firstCharacter = (char) buffer.get(pos);- if (!anyMatches || characterDefinition.isInvoke(firstCharacter)) {-- // Find unknown match:- int characterId = characterDefinition.getCharacterClass(firstCharacter);- // NOTE: copied from UnknownDictionary.lookup:- int unknownWordLength;- if (!characterDefinition.isGroup(firstCharacter)) {- unknownWordLength = 1;- } else {- // Extract unknown word. Characters with the same script are considered to be part of- // unknown word- unknownWordLength = 1;- UnicodeScript scriptCode = UnicodeScript.of(firstCharacter);- final boolean isPunct = isPunctuation(firstCharacter);- final boolean isDigit = Character.isDigit(firstCharacter);- for (int posAhead = pos + 1; unknownWordLength < MAX_UNKNOWN_WORD_LENGTH; posAhead++) {- int next = buffer.get(posAhead);- if (next == -1) {- break;- }- char ch = (char) next;- int chType = Character.getType(ch);- UnicodeScript sc = UnicodeScript.of(next);- boolean sameScript =- isSameScript(scriptCode, sc)- // Non-spacing marks inherit the script of their base character,- // following recommendations from UTR #24.- || chType == Character.NON_SPACING_MARK;-- if (sameScript- // split on punctuation- && isPunctuation(ch, chType) == isPunct- // split on digit- && Character.isDigit(ch) == isDigit- && characterDefinition.isGroup(ch)) {- unknownWordLength++;- } else {- break;- }- // Update the script code and character class if the original script- // is Inherited or Common.- if (isCommonOrInherited(scriptCode) && isCommonOrInherited(sc) == false) {- scriptCode = sc;- characterId = characterDefinition.getCharacterClass(ch);- }- }- }-- unkDictionary.lookupWordIds(- characterId, wordIdRef); // characters in input text are supposed to be the same- if (VERBOSE) {- System.out.println(- " UNKNOWN word len=" + unknownWordLength + " " + wordIdRef.length + " wordIDs");- }- for (int ofs = 0; ofs < wordIdRef.length; ofs++) {- add(- unkDictionary.getMorphAttributes(),- posData,- pos,- pos + unknownWordLength,- wordIdRef.ints[wordIdRef.offset + ofs],- TokenType.UNKNOWN);- }- }-- pos++;- }-- end = true;-- if (pos > 0) {-- final Position endPosData = positions.get(pos);- int leastCost = Integer.MAX_VALUE;- int leastIDX = -1;- if (VERBOSE) {- System.out.println(" end: " + endPosData.count + " nodes");- }- for (int idx = 0; idx < endPosData.count; idx++) {- // Add EOS cost:- final int cost = endPosData.costs[idx] + costs.get(endPosData.lastRightID[idx], 0);- // System.out.println(" idx=" + idx + " cost=" + cost + " (pathCost=" +- // endPosData.costs[idx] + " bgCost=" + costs.get(endPosData.lastRightID[idx], 0) + ")- // backPos=" + endPosData.backPos[idx]);- if (cost < leastCost) {- leastCost = cost;- leastIDX = idx;- }- }-- backtrace(endPosData, leastIDX);- } else {- // No characters in the input string; return no tokens!- }- }-- // the pending list. The pending list is then in-reverse- // (last token should be returned first).- private void backtrace(final Position endPosData, final int fromIDX) {- final int endPos = endPosData.pos;-- if (VERBOSE) {- System.out.println(- "\n backtrace: endPos="- + endPos- + " pos="- + pos- + "; "- + (pos - lastBackTracePos)- + " characters; last="- + lastBackTracePos- + " cost="- + endPosData.costs[fromIDX]);- }-- final char[] fragment = buffer.get(lastBackTracePos, endPos - lastBackTracePos);-- if (dotOut != null) {- dotOut.onBacktrace(this, positions, lastBackTracePos, endPosData, fromIDX, fragment, end);- }-- int pos = endPos;- int bestIDX = fromIDX;-- // TODO: sort of silly to make Token instances here; the- // back trace has all info needed to generate the- // token. So, we could just directly set the attrs,- // from the backtrace, in incrementToken w/o ever- // creating Token; we'd have to defer calling freeBefore- // until after the backtrace was fully "consumed" by- // incrementToken.-- while (pos > lastBackTracePos) {- // System.out.println("BT: back pos=" + pos + " bestIDX=" + bestIDX);- final Position posData = positions.get(pos);- assert bestIDX < posData.count;-- int backPos = posData.backPos[bestIDX];- int backWordPos = posData.backWordPos[bestIDX];- assert backPos >= lastBackTracePos- : "backPos=" + backPos + " vs lastBackTracePos=" + lastBackTracePos;- // the length of the word without the whitespaces at the beginning.- int length = pos - backWordPos;- TokenType backType = posData.backType[bestIDX];- int backID = posData.backID[bestIDX];- int nextBestIDX = posData.backIndex[bestIDX];- // the start of the word after the whitespace at the beginning.- final int fragmentOffset = backWordPos - lastBackTracePos;- assert fragmentOffset >= 0;-- final Dictionary<? extends KoMorphData> dict = getDict(backType);-- if (outputUnknownUnigrams && backType == TokenType.UNKNOWN) {- // outputUnknownUnigrams converts unknown word into unigrams:- for (int i = length - 1; i >= 0; i--) {- int charLen = 1;- if (i > 0 && Character.isLowSurrogate(fragment[fragmentOffset + i])) {- i--;- charLen = 2;- }- final DictionaryToken token =- new DictionaryToken(- TokenType.UNKNOWN,- unkDictionary.getMorphAttributes(),- CharacterDefinition.NGRAM,- fragment,- fragmentOffset + i,- charLen,- backWordPos + i,- backWordPos + i + charLen);- pending.add(token);- if (VERBOSE) {- System.out.println(" add token=" + pending.get(pending.size() - 1));- }- }- } else {- final DictionaryToken token =- new DictionaryToken(- backType,- dict.getMorphAttributes(),- backID,- fragment,- fragmentOffset,- length,- backWordPos,- backWordPos + length);- if (token.getPOSType() == POS.Type.MORPHEME || mode == DecompoundMode.NONE) {- if (shouldFilterToken(token) == false) {- pending.add(token);- if (VERBOSE) {- System.out.println(" add token=" + pending.get(pending.size() - 1));- }- }- } else {- KoMorphData.Morpheme[] morphemes = token.getMorphemes();- if (morphemes == null) {- pending.add(token);- if (VERBOSE) {- System.out.println(" add token=" + pending.get(pending.size() - 1));- }- } else {- int endOffset = backWordPos + length;- int posLen = 0;- // decompose the compound- for (int i = morphemes.length - 1; i >= 0; i--) {- final KoMorphData.Morpheme morpheme = morphemes[i];- final Token compoundToken;- if (token.getPOSType() == POS.Type.COMPOUND) {- assert endOffset - morpheme.surfaceForm.length() >= 0;- compoundToken =- new DecompoundToken(- morpheme.posTag,- morpheme.surfaceForm,- endOffset - morpheme.surfaceForm.length(),- endOffset,- backType);- } else {- compoundToken =- new DecompoundToken(- morpheme.posTag,- morpheme.surfaceForm,- token.getStartOffset(),- token.getEndOffset(),- backType);- }- if (i == 0 && mode == DecompoundMode.MIXED) {- compoundToken.setPositionIncrement(0);- }- ++posLen;- endOffset -= morpheme.surfaceForm.length();- pending.add(compoundToken);- if (VERBOSE) {- System.out.println(" add token=" + pending.get(pending.size() - 1));- }- }- if (mode == DecompoundMode.MIXED) {- token.setPositionLength(Math.max(1, posLen));- pending.add(token);- if (VERBOSE) {- System.out.println(" add token=" + pending.get(pending.size() - 1));- }- }- }- }- }- if (discardPunctuation == false && backWordPos != backPos) {- // Add a token for whitespaces between terms- int offset = backPos - lastBackTracePos;- int len = backWordPos - backPos;- // System.out.println(offset + " " + fragmentOffset + " " + len + " " + backWordPos + " " +- // backPos);- unkDictionary.lookupWordIds(characterDefinition.getCharacterClass(' '), wordIdRef);- DictionaryToken spaceToken =- new DictionaryToken(- TokenType.UNKNOWN,- unkDictionary.getMorphAttributes(),- wordIdRef.ints[wordIdRef.offset],- fragment,- offset,- len,- backPos,- backPos + len);- pending.add(spaceToken);- }-- pos = backPos;- bestIDX = nextBestIDX;- }-- lastBackTracePos = endPos;-- if (VERBOSE) {- System.out.println(" freeBefore pos=" + endPos);- }- // Notify the circular buffers that we are done with- // these positions:- buffer.freeBefore(endPos);- positions.freeBefore(endPos);- }-- Dictionary<? extends KoMorphData> getDict(TokenType type) {- return dictionaryMap.get(type);- }-- private boolean shouldFilterToken(Token token) {- return discardPunctuation && isPunctuation(token.getSurfaceForm()[token.getOffset()]);- }-- private static boolean isPunctuation(char ch) {- return isPunctuation(ch, Character.getType(ch));- }-- private static boolean isPunctuation(char ch, int cid) {- // special case for Hangul Letter Araea (interpunct)- if (ch == 0x318D) {- return true;- }- switch (cid) {- case Character.SPACE_SEPARATOR:- case Character.LINE_SEPARATOR:- case Character.PARAGRAPH_SEPARATOR:- case Character.CONTROL:- case Character.FORMAT:- case Character.DASH_PUNCTUATION:- case Character.START_PUNCTUATION:- case Character.END_PUNCTUATION:- case Character.CONNECTOR_PUNCTUATION:- case Character.OTHER_PUNCTUATION:- case Character.MATH_SYMBOL:- case Character.CURRENCY_SYMBOL:- case Character.MODIFIER_SYMBOL:- case Character.OTHER_SYMBOL:- case Character.INITIAL_QUOTE_PUNCTUATION:- case Character.FINAL_QUOTE_PUNCTUATION:- return true;- default:- return false;- }- }-- private static boolean isCommonOrInherited(UnicodeScript script) {- return script == UnicodeScript.INHERITED || script == UnicodeScript.COMMON;- }-- /** Determine if two scripts are compatible. */- private static boolean isSameScript(UnicodeScript scriptOne, UnicodeScript scriptTwo) {- return scriptOne == scriptTwo- || isCommonOrInherited(scriptOne)- || isCommonOrInherited(scriptTwo);+ /** Expert: set this to produce graphviz (dot) output of the Viterbi lattice */+ public void setGraphvizFormatter(GraphvizFormatter<KoMorphData> dotOut) {+ viterbi.setGraphvizFormatter(dotOut);}}diff --git a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/Viterbi.java b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/Viterbi.javanew file mode 100644index 00000000000..2a8adbfcb62--- /dev/null+++ b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/Viterbi.java@@ -0,0 +1,447 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.analysis.ko;++import java.io.IOException;+import java.util.EnumMap;+import org.apache.lucene.analysis.ko.dict.CharacterDefinition;+import org.apache.lucene.analysis.ko.dict.KoMorphData;+import org.apache.lucene.analysis.ko.dict.TokenInfoDictionary;+import org.apache.lucene.analysis.ko.dict.UnknownDictionary;+import org.apache.lucene.analysis.ko.dict.UserDictionary;+import org.apache.lucene.analysis.morph.ConnectionCosts;+import org.apache.lucene.analysis.morph.Dictionary;+import org.apache.lucene.analysis.morph.GraphvizFormatter;+import org.apache.lucene.analysis.morph.MorphData;+import org.apache.lucene.analysis.morph.TokenInfoFST;+import org.apache.lucene.analysis.morph.TokenType;+import org.apache.lucene.util.fst.FST;++/** {@link org.apache.lucene.analysis.morph.Viterbi} subclass for Korean morphological analysis. */+final class Viterbi+ extends org.apache.lucene.analysis.morph.Viterbi<+ Token, org.apache.lucene.analysis.morph.Viterbi.Position> {++ private final EnumMap<TokenType, Dictionary<? extends KoMorphData>> dictionaryMap =+ new EnumMap<>(TokenType.class);++ private final UnknownDictionary unkDictionary;+ private final CharacterDefinition characterDefinition;++ private final boolean discardPunctuation;+ private final KoreanTokenizer.DecompoundMode mode;+ private final boolean outputUnknownUnigrams;++ private GraphvizFormatter<KoMorphData> dotOut;++ Viterbi(+ TokenInfoFST fst,+ FST.BytesReader fstReader,+ TokenInfoDictionary dictionary,+ TokenInfoFST userFST,+ FST.BytesReader userFSTReader,+ UserDictionary userDictionary,+ ConnectionCosts costs,+ UnknownDictionary unkDictionary,+ CharacterDefinition characterDefinition,+ boolean discardPunctuation,+ KoreanTokenizer.DecompoundMode mode,+ boolean outputUnknownUnigrams) {+ super(+ fst, fstReader, dictionary, userFST, userFSTReader, userDictionary, costs, Position.class);+ this.unkDictionary = unkDictionary;+ this.characterDefinition = characterDefinition;+ this.discardPunctuation = discardPunctuation;+ this.mode = mode;+ this.outputUnknownUnigrams = outputUnknownUnigrams;+ this.enableSpacePenaltyFactor = true;+ this.outputLongestUserEntryOnly = true;+ dictionaryMap.put(TokenType.KNOWN, dictionary);+ dictionaryMap.put(TokenType.UNKNOWN, unkDictionary);+ dictionaryMap.put(TokenType.USER, userDictionary);+ }++ @Override+ protected int processUnknownWord(boolean anyMatches, Position posData) throws IOException {+ final char firstCharacter = (char) buffer.get(pos);+ if (!anyMatches || characterDefinition.isInvoke(firstCharacter)) {++ // Find unknown match:+ int characterId = characterDefinition.getCharacterClass(firstCharacter);+ // NOTE: copied from UnknownDictionary.lookup:+ int unknownWordLength;+ if (!characterDefinition.isGroup(firstCharacter)) {+ unknownWordLength = 1;+ } else {+ // Extract unknown word. Characters with the same script are considered to be part of+ // unknown word+ unknownWordLength = 1;+ Character.UnicodeScript scriptCode = Character.UnicodeScript.of(firstCharacter);+ final boolean isPunct = isPunctuation(firstCharacter);+ final boolean isDigit = Character.isDigit(firstCharacter);+ for (int posAhead = pos + 1; unknownWordLength < MAX_UNKNOWN_WORD_LENGTH; posAhead++) {+ int next = buffer.get(posAhead);+ if (next == -1) {+ break;+ }+ char ch = (char) next;+ int chType = Character.getType(ch);+ Character.UnicodeScript sc = Character.UnicodeScript.of(next);+ boolean sameScript =+ isSameScript(scriptCode, sc)+ // Non-spacing marks inherit the script of their base character,+ // following recommendations from UTR #24.+ || chType == Character.NON_SPACING_MARK;++ if (sameScript+ // split on punctuation+ && isPunctuation(ch, chType) == isPunct+ // split on digit+ && Character.isDigit(ch) == isDigit+ && characterDefinition.isGroup(ch)) {+ unknownWordLength++;+ } else {+ break;+ }+ // Update the script code and character class if the original script+ // is Inherited or Common.+ if (isCommonOrInherited(scriptCode) && isCommonOrInherited(sc) == false) {+ scriptCode = sc;+ characterId = characterDefinition.getCharacterClass(ch);+ }+ }+ }++ unkDictionary.lookupWordIds(+ characterId, wordIdRef); // characters in input text are supposed to be the same+ if (VERBOSE) {+ System.out.println(+ " UNKNOWN word len=" + unknownWordLength + " " + wordIdRef.length + " wordIDs");+ }+ for (int ofs = 0; ofs < wordIdRef.length; ofs++) {+ add(+ unkDictionary.getMorphAttributes(),+ posData,+ pos,+ pos + unknownWordLength,+ wordIdRef.ints[wordIdRef.offset + ofs],+ TokenType.UNKNOWN,+ false);+ }+ }+ // TODO: should return meaningful value?+ return 0;+ }++ void setGraphvizFormatter(GraphvizFormatter<KoMorphData> dotOut) {+ this.dotOut = dotOut;+ }++ @Override+ protected void backtrace(Position endPosData, int fromIDX) {+ final int endPos = endPosData.getPos();++ if (endPos == lastBackTracePos) {+ return;+ }++ if (VERBOSE) {+ System.out.println(+ "\n backtrace: endPos="+ + endPos+ + " pos="+ + pos+ + "; "+ + (pos - lastBackTracePos)+ + " characters; last="+ + lastBackTracePos+ + " cost="+ + endPosData.getCost(fromIDX));+ }++ final char[] fragment = buffer.get(lastBackTracePos, endPos - lastBackTracePos);++ if (dotOut != null) {+ dotOut.onBacktrace(+ this::getDict, positions, lastBackTracePos, endPosData, fromIDX, fragment, end);+ }++ int pos = endPos;+ int bestIDX = fromIDX;++ // TODO: sort of silly to make Token instances here; the+ // back trace has all info needed to generate the+ // token. So, we could just directly set the attrs,+ // from the backtrace, in incrementToken w/o ever+ // creating Token; we'd have to defer calling freeBefore+ // until after the backtrace was fully "consumed" by+ // incrementToken.++ while (pos > lastBackTracePos) {+ // System.out.println("BT: back pos=" + pos + " bestIDX=" + bestIDX);+ final Position posData = positions.get(pos);+ assert bestIDX < posData.getCount();++ int backPos = posData.getBackPos(bestIDX);+ int backWordPos = posData.getBackWordPos(bestIDX);+ assert backPos >= lastBackTracePos+ : "backPos=" + backPos + " vs lastBackTracePos=" + lastBackTracePos;+ // the length of the word without the whitespaces at the beginning.+ int length = pos - backWordPos;+ TokenType backType = posData.getBackType(bestIDX);+ int backID = posData.getBackID(bestIDX);+ int nextBestIDX = posData.getBackIndex(bestIDX);+ // the start of the word after the whitespace at the beginning.+ final int fragmentOffset = backWordPos - lastBackTracePos;+ assert fragmentOffset >= 0;++ final Dictionary<? extends KoMorphData> dict = getDict(backType);++ if (outputUnknownUnigrams && backType == TokenType.UNKNOWN) {+ // outputUnknownUnigrams converts unknown word into unigrams:+ for (int i = length - 1; i >= 0; i--) {+ int charLen = 1;+ if (i > 0 && Character.isLowSurrogate(fragment[fragmentOffset + i])) {+ i--;+ charLen = 2;+ }+ final DictionaryToken token =+ new DictionaryToken(+ TokenType.UNKNOWN,+ unkDictionary.getMorphAttributes(),+ CharacterDefinition.NGRAM,+ fragment,+ fragmentOffset + i,+ charLen,+ backWordPos + i,+ backWordPos + i + charLen);+ pending.add(token);+ if (VERBOSE) {+ System.out.println(" add token=" + pending.get(pending.size() - 1));+ }+ }+ } else {+ final DictionaryToken token =+ new DictionaryToken(+ backType,+ dict.getMorphAttributes(),+ backID,+ fragment,+ fragmentOffset,+ length,+ backWordPos,+ backWordPos + length);+ if (token.getPOSType() == POS.Type.MORPHEME+ || mode == KoreanTokenizer.DecompoundMode.NONE) {+ if (shouldFilterToken(token) == false) {+ pending.add(token);+ if (VERBOSE) {+ System.out.println(" add token=" + pending.get(pending.size() - 1));+ }+ }+ } else {+ KoMorphData.Morpheme[] morphemes = token.getMorphemes();+ if (morphemes == null) {+ pending.add(token);+ if (VERBOSE) {+ System.out.println(" add token=" + pending.get(pending.size() - 1));+ }+ } else {+ int endOffset = backWordPos + length;+ int posLen = 0;+ // decompose the compound+ for (int i = morphemes.length - 1; i >= 0; i--) {+ final KoMorphData.Morpheme morpheme = morphemes[i];+ final Token compoundToken;+ if (token.getPOSType() == POS.Type.COMPOUND) {+ assert endOffset - morpheme.surfaceForm.length() >= 0;+ compoundToken =+ new DecompoundToken(+ morpheme.posTag,+ morpheme.surfaceForm,+ endOffset - morpheme.surfaceForm.length(),+ endOffset,+ backType);+ } else {+ compoundToken =+ new DecompoundToken(+ morpheme.posTag,+ morpheme.surfaceForm,+ token.getStartOffset(),+ token.getEndOffset(),+ backType);+ }+ if (i == 0 && mode == KoreanTokenizer.DecompoundMode.MIXED) {+ compoundToken.setPositionIncrement(0);+ }+ ++posLen;+ endOffset -= morpheme.surfaceForm.length();+ pending.add(compoundToken);+ if (VERBOSE) {+ System.out.println(" add token=" + pending.get(pending.size() - 1));+ }+ }+ if (mode == KoreanTokenizer.DecompoundMode.MIXED) {+ token.setPositionLength(Math.max(1, posLen));+ pending.add(token);+ if (VERBOSE) {+ System.out.println(" add token=" + pending.get(pending.size() - 1));+ }+ }+ }+ }+ }+ if (discardPunctuation == false && backWordPos != backPos) {+ // Add a token for whitespaces between terms+ int offset = backPos - lastBackTracePos;+ int len = backWordPos - backPos;+ // System.out.println(offset + " " + fragmentOffset + " " + len + " " + backWordPos + " " ++ // backPos);+ unkDictionary.lookupWordIds(characterDefinition.getCharacterClass(' '), wordIdRef);+ DictionaryToken spaceToken =+ new DictionaryToken(+ TokenType.UNKNOWN,+ unkDictionary.getMorphAttributes(),+ wordIdRef.ints[wordIdRef.offset],+ fragment,+ offset,+ len,+ backPos,+ backPos + len);+ pending.add(spaceToken);+ }++ pos = backPos;+ bestIDX = nextBestIDX;+ }++ lastBackTracePos = endPos;++ if (VERBOSE) {+ System.out.println(" freeBefore pos=" + endPos);+ }+ // Notify the circular buffers that we are done with+ // these positions:+ buffer.freeBefore(endPos);+ positions.freeBefore(endPos);+ }++ /** Returns the space penalty associated with the provided {@link POS.Tag}. */+ @Override+ protected int computeSpacePenalty(MorphData morphData, int wordID, int numSpaces) {+ final POS.Tag leftPOS = ((KoMorphData) morphData).getLeftPOS(wordID);+ int spacePenalty = 0;+ if (numSpaces > 0) {+ // TODO we should extract the penalty (left-space-penalty-factor) from the dicrc file.+ switch (leftPOS) {+ case E:+ case J:+ case VCP:+ case XSA:+ case XSN:+ case XSV:+ spacePenalty = 3000;+ break;+ case IC:+ case MAG:+ case MAJ:+ case MM:+ case NA:+ case NNB:+ case NNBC:+ case NNG:+ case NNP:+ case NP:+ case NR:+ case SC:+ case SE:+ case SF:+ case SH:+ case SL:+ case SN:+ case SP:+ case SSC:+ case SSO:+ case SY:+ case UNA:+ case UNKNOWN:+ case VA:+ case VCN:+ case VSV:+ case VV:+ case VX:+ case XPN:+ case XR:+ default:+ break;+ }+ }+ return spacePenalty;+ }++ Dictionary<? extends KoMorphData> getDict(TokenType type) {+ return dictionaryMap.get(type);+ }++ private boolean shouldFilterToken(Token token) {+ return discardPunctuation && isPunctuation(token.getSurfaceForm()[token.getOffset()]);+ }++ private static boolean isPunctuation(char ch) {+ return isPunctuation(ch, Character.getType(ch));+ }++ private static boolean isPunctuation(char ch, int cid) {+ // special case for Hangul Letter Araea (interpunct)+ if (ch == 0x318D) {+ return true;+ }+ switch (cid) {+ case Character.SPACE_SEPARATOR:+ case Character.LINE_SEPARATOR:+ case Character.PARAGRAPH_SEPARATOR:+ case Character.CONTROL:+ case Character.FORMAT:+ case Character.DASH_PUNCTUATION:+ case Character.START_PUNCTUATION:+ case Character.END_PUNCTUATION:+ case Character.CONNECTOR_PUNCTUATION:+ case Character.OTHER_PUNCTUATION:+ case Character.MATH_SYMBOL:+ case Character.CURRENCY_SYMBOL:+ case Character.MODIFIER_SYMBOL:+ case Character.OTHER_SYMBOL:+ case Character.INITIAL_QUOTE_PUNCTUATION:+ case Character.FINAL_QUOTE_PUNCTUATION:+ return true;+ default:+ return false;+ }+ }++ private static boolean isCommonOrInherited(Character.UnicodeScript script) {+ return script == Character.UnicodeScript.INHERITED || script == Character.UnicodeScript.COMMON;+ }++ /** Determine if two scripts are compatible. */+ private static boolean isSameScript(+ Character.UnicodeScript scriptOne, Character.UnicodeScript scriptTwo) {+ return scriptOne == scriptTwo+ || isCommonOrInherited(scriptOne)+ || isCommonOrInherited(scriptTwo);+ }+}