Skip to content

Apache Lucene open-source search software

LUCENE-10493: move n-best logic to analysis-common (#846)

Tomoko Uchidac28f575b6db1
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/ViterbiNBest.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/ViterbiNBest.java
new file mode 100644
index 00000000000..8c4d06ff158
--- /dev/null
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/ViterbiNBest.java
@@ -0,0 +1,629 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.analysis.morph;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.EnumMap;
+import java.util.HashMap;
+import java.util.List;
+import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.fst.FST;
+
+/** {@link Viterbi} subclass for n-best path calculation. */
+public abstract class ViterbiNBest<T extends Token, U extends MorphData>
+ extends Viterbi<T, ViterbiNBest.PositionNBest> {
+
+ protected final EnumMap<TokenType, Dictionary<? extends U>> dictionaryMap =
+ new EnumMap<>(TokenType.class);
+
+ // Allowable cost difference for N-best output:
+ private int nBestCost = 0;
+
+ protected Lattice<U> lattice = null;
+
+ protected ViterbiNBest(
+ TokenInfoFST fst,
+ FST.BytesReader fstReader,
+ BinaryDictionary<? extends MorphData> dictionary,
+ TokenInfoFST userFST,
+ FST.BytesReader userFSTReader,
+ Dictionary<? extends MorphData> userDictionary,
+ ConnectionCosts costs) {
+ super(
+ fst,
+ fstReader,
+ dictionary,
+ userFST,
+ userFSTReader,
+ userDictionary,
+ costs,
+ ViterbiNBest.PositionNBest.class);
+ }
+
+ @Override
+ protected final void backtraceNBest(final Position endPosData, final boolean useEOS)
+ throws IOException {
+ if (lattice == null) {
+ lattice = new Lattice<U>();
+ }
+
+ final int endPos = endPosData.getPos();
+ char[] fragment = buffer.get(lastBackTracePos, endPos - lastBackTracePos);
+ lattice.setup(fragment, dictionaryMap, positions, lastBackTracePos, endPos, useEOS);
+ lattice.markUnreachable();
+ lattice.calcLeftCost(costs);
+ lattice.calcRightCost(costs);
+
+ int bestCost = lattice.bestCost();
+ if (VERBOSE) {
+ System.out.printf("DEBUG: 1-BEST COST: %d\n", bestCost);
+ }
+ for (int node : lattice.bestPathNodeList()) {
+ registerNode(node, fragment);
+ }
+
+ for (int n = 2; ; ++n) {
+ List<Integer> nbest = lattice.nBestNodeList(n);
+ if (nbest.isEmpty()) {
+ break;
+ }
+ int cost = lattice.cost(nbest.get(0));
+ if (VERBOSE) {
+ System.out.printf("DEBUG: %d-BEST COST: %d\n", n, cost);
+ }
+ if (bestCost + nBestCost < cost) {
+ break;
+ }
+ for (int node : nbest) {
+ registerNode(node, fragment);
+ }
+ }
+ if (VERBOSE) {
+ lattice.debugPrint();
+ }
+ }
+
+ /** Add n-best tokens to the pending list. */
+ protected abstract void registerNode(int node, char[] fragment);
+
+ @Override
+ protected final void fixupPendingList() {
+ // Sort for removing same tokens.
+ // USER token should be ahead from normal one.
+ Collections.sort(
+ pending,
+ (a, b) -> {
+ int aOff = a.getOffset();
+ int bOff = b.getOffset();
+ if (aOff != bOff) {
+ return aOff - bOff;
+ }
+ int aLen = a.getLength();
+ int bLen = b.getLength();
+ if (aLen != bLen) {
+ return aLen - bLen;
+ }
+ // order of Type is KNOWN, UNKNOWN, USER,
+ // so we use reversed comparison here.
+ return b.getType().ordinal() - a.getType().ordinal();
+ });
+
+ // Remove same token.
+ for (int i = 1; i < pending.size(); ++i) {
+ Token a = pending.get(i - 1);
+ Token b = pending.get(i);
+ if (a.getOffset() == b.getOffset() && a.getLength() == b.getLength()) {
+ pending.remove(i);
+ // It is important to decrement "i" here, because a next may be removed.
+ --i;
+ }
+ }
+
+ // offset=>position map
+ HashMap<Integer, Integer> map = new HashMap<>();
+ for (Token t : pending) {
+ map.put(t.getOffset(), 0);
+ map.put(t.getOffset() + t.getLength(), 0);
+ }
+
+ // Get unique and sorted list of all edge position of tokens.
+ Integer[] offsets = map.keySet().toArray(new Integer[0]);
+ Arrays.sort(offsets);
+
+ // setup all value of map. It specifies N-th position from begin.
+ for (int i = 0; i < offsets.length; ++i) {
+ map.put(offsets[i], i);
+ }
+
+ // We got all position length now.
+ for (Token t : pending) {
+ t.setPositionLength(map.get(t.getOffset() + t.getLength()) - map.get(t.getOffset()));
+ }
+
+ // Make PENDING to be reversed order to fit its usage.
+ // If you would like to speedup, you can try reversed order sort
+ // at first of this function.
+ Collections.reverse(pending);
+ }
+
+ protected void setNBestCost(int value) {
+ nBestCost = value;
+ outputNBest = 0 < nBestCost;
+ }
+
+ protected int getNBestCost() {
+ return nBestCost;
+ }
+
+ public int getLatticeRootBase() {
+ return lattice.getRootBase();
+ }
+
+ public int probeDelta(int start, int end) {
+ return lattice.probeDelta(start, end);
+ }
+
+ /**
+ * {@link Viterbi.Position} extension; this holds all forward pointers to calculate n-best path.
+ */
+ public static final class PositionNBest extends Viterbi.Position {
+ // Only used when finding 2nd best segmentation under a
+ // too-long token:
+ int forwardCount;
+ int[] forwardPos = new int[8];
+ int[] forwardID = new int[8];
+ int[] forwardIndex = new int[8];
+ TokenType[] forwardType = new TokenType[8];
+
+ private void growForward() {
+ forwardPos = ArrayUtil.grow(forwardPos, 1 + forwardCount);
+ forwardID = ArrayUtil.grow(forwardID, 1 + forwardCount);
+ forwardIndex = ArrayUtil.grow(forwardIndex, 1 + forwardCount);
+
+ // NOTE: sneaky: grow separately because
+ // ArrayUtil.grow will otherwise pick a different
+ // length than the int[]s we just grew:
+ final TokenType[] newForwardType = new TokenType[forwardPos.length];
+ System.arraycopy(forwardType, 0, newForwardType, 0, forwardType.length);
+ forwardType = newForwardType;
+ }
+
+ public void addForward(int forwardPos, int forwardIndex, int forwardID, TokenType forwardType) {
+ if (forwardCount == this.forwardID.length) {
+ growForward();
+ }
+ this.forwardPos[forwardCount] = forwardPos;
+ this.forwardIndex[forwardCount] = forwardIndex;
+ this.forwardID[forwardCount] = forwardID;
+ this.forwardType[forwardCount] = forwardType;
+ forwardCount++;
+ }
+
+ @Override
+ public void reset() {
+ super.reset();
+ // forwardCount naturally resets after it runs:
+ assert forwardCount == 0 : "pos=" + getPos() + " forwardCount=" + forwardCount;
+ }
+
+ public int getForwardCount() {
+ return forwardCount;
+ }
+
+ public void setForwardCount(int forwardCount) {
+ this.forwardCount = forwardCount;
+ }
+
+ public TokenType getForwardType(int index) {
+ return forwardType[index];
+ }
+
+ public int getForwardID(int index) {
+ return forwardID[index];
+ }
+
+ public int getForwardPos(int index) {
+ return forwardPos[index];
+ }
+ }
+
+ /** Yet another lattice data structure for keeping n-best path. */
+ protected static final class Lattice<U extends MorphData> {
+ private char[] fragment;
+ private EnumMap<TokenType, Dictionary<? extends U>> dictionaryMap;
+ private boolean useEOS;
+
+ private int rootCapacity = 0;
+ private int rootSize = 0;
+ private int rootBase = 0;
+
+ // root pointers of node chain by leftChain_ that have same start offset.
+ private int[] lRoot;
+ // root pointers of node chain by rightChain_ that have same end offset.
+ private int[] rRoot;
+
+ private int capacity = 0;
+ private int nodeCount = 0;
+
+ // The variables below are elements of lattice node that indexed by node number.
+ private TokenType[] nodeDicType;
+ private int[] nodeWordID;
+ // nodeMark - -1:excluded, 0:unused, 1:bestpath, 2:2-best-path, ... N:N-best-path
+ private int[] nodeMark;
+ private int[] nodeLeftID;
+ private int[] nodeRightID;
+ private int[] nodeWordCost;
+ private int[] nodeLeftCost;
+ private int[] nodeRightCost;
+ // nodeLeftNode, nodeRightNode - are left/right node number with minimum cost path.
+ private int[] nodeLeftNode;
+ private int[] nodeRightNode;
+ // nodeLeft, nodeRight - start/end offset
+ private int[] nodeLeft;
+ private int[] nodeRight;
+ private int[] nodeLeftChain;
+ private int[] nodeRightChain;
+
+ public int getNodeLeft(int node) {
+ return nodeLeft[node];
+ }
+
+ public int getNodeRight(int node) {
+ return nodeRight[node];
+ }
+
+ public TokenType getNodeDicType(int node) {
+ return nodeDicType[node];
+ }
+
+ public int getNodeWordID(int node) {
+ return nodeWordID[node];
+ }
+
+ public int getRootBase() {
+ return rootBase;
+ }
+
+ private void setupRoot(int baseOffset, int lastOffset) {
+ assert baseOffset <= lastOffset;
+ int size = lastOffset - baseOffset + 1;
+ if (rootCapacity < size) {
+ int oversize = ArrayUtil.oversize(size, Integer.BYTES);
+ lRoot = new int[oversize];
+ rRoot = new int[oversize];
+ rootCapacity = oversize;
+ }
+ Arrays.fill(lRoot, 0, size, -1);
+ Arrays.fill(rRoot, 0, size, -1);
+ rootSize = size;
+ rootBase = baseOffset;
+ }
+
+ // Reserve at least N nodes.
+ private void reserve(int n) {
+ if (capacity < n) {
+ int oversize = ArrayUtil.oversize(n, Integer.BYTES);
+ nodeDicType = new TokenType[oversize];
+ nodeWordID = new int[oversize];
+ nodeMark = new int[oversize];
+ nodeLeftID = new int[oversize];
+ nodeRightID = new int[oversize];
+ nodeWordCost = new int[oversize];
+ nodeLeftCost = new int[oversize];
+ nodeRightCost = new int[oversize];
+ nodeLeftNode = new int[oversize];
+ nodeRightNode = new int[oversize];
+ nodeLeft = new int[oversize];
+ nodeRight = new int[oversize];
+ nodeLeftChain = new int[oversize];
+ nodeRightChain = new int[oversize];
+ capacity = oversize;
+ }
+ }
+
+ private void setupNodePool(int n) {
+ reserve(n);
+ nodeCount = 0;
+ if (VERBOSE) {
+ System.out.printf("DEBUG: setupNodePool: n = %d\n", n);
+ System.out.printf("DEBUG: setupNodePool: lattice.capacity = %d\n", capacity);
+ }
+ }
+
+ private int addNode(TokenType dicType, int wordID, int left, int right) {
+ if (VERBOSE) {
+ System.out.printf(
+ "DEBUG: addNode: dicType=%s, wordID=%d, left=%d, right=%d, str=%s\n",
+ dicType.toString(),
+ wordID,
+ left,
+ right,
+ left == -1 ? "BOS" : right == -1 ? "EOS" : new String(fragment, left, right - left));
+ }
+ assert nodeCount < capacity;
+ assert left == -1 || right == -1 || left < right;
+ assert left == -1 || (0 <= left && left < rootSize);
+ assert right == -1 || (0 <= right && right < rootSize);
+
+ int node = nodeCount++;
+
+ if (VERBOSE) {
+ System.out.printf("DEBUG: addNode: node=%d\n", node);
+ }
+
+ nodeDicType[node] = dicType;
+ nodeWordID[node] = wordID;
+ nodeMark[node] = 0;
+
+ if (wordID < 0) {
+ nodeWordCost[node] = 0;
+ nodeLeftCost[node] = 0;
+ nodeRightCost[node] = 0;
+ nodeLeftID[node] = 0;
+ nodeRightID[node] = 0;
+ } else {
+ Dictionary<? extends MorphData> dic = dictionaryMap.get(dicType);
+ nodeWordCost[node] = dic.getWordCost(wordID);
+ nodeLeftID[node] = dic.getLeftId(wordID);
+ nodeRightID[node] = dic.getRightId(wordID);
+ }
+
+ if (VERBOSE) {
+ System.out.printf(
+ "DEBUG: addNode: wordCost=%d, leftID=%d, rightID=%d\n",
+ nodeWordCost[node], nodeLeftID[node], nodeRightID[node]);
+ }
+
+ nodeLeft[node] = left;
+ nodeRight[node] = right;
+ if (0 <= left) {
+ nodeLeftChain[node] = lRoot[left];
+ lRoot[left] = node;
+ } else {
+ nodeLeftChain[node] = -1;
+ }
+ if (0 <= right) {
+ nodeRightChain[node] = rRoot[right];
+ rRoot[right] = node;
+ } else {
+ nodeRightChain[node] = -1;
+ }
+ return node;
+ }
+
+ // Sum of positions.get(i).count in [beg, end) range.
+ // using stream:
+ // return IntStream.range(beg, end).map(i -> positions.get(i).count).sum();
+ private int positionCount(WrappedPositionArray<PositionNBest> positions, int beg, int end) {
+ int count = 0;
+ for (int i = beg; i < end; ++i) {
+ count += positions.get(i).getCount();
+ }
+ return count;
+ }
+
+ void setup(
+ char[] fragment,
+ EnumMap<TokenType, Dictionary<? extends U>> dictionaryMap,
+ WrappedPositionArray<PositionNBest> positions,
+ int prevOffset,
+ int endOffset,
+ boolean useEOS) {
+ assert positions.get(prevOffset).getCount() == 1;
+ if (VERBOSE) {
+ System.out.printf("DEBUG: setup: prevOffset=%d, endOffset=%d\n", prevOffset, endOffset);
+ }
+
+ this.fragment = fragment;
+ this.dictionaryMap = dictionaryMap;
+ this.useEOS = useEOS;
+
+ // Initialize lRoot and rRoot.
+ setupRoot(prevOffset, endOffset);
+
+ // "+ 2" for first/last record.
+ setupNodePool(positionCount(positions, prevOffset + 1, endOffset + 1) + 2);
+
+ // substitute for BOS = 0
+ Position first = positions.get(prevOffset);
+ if (addNode(first.getBackType(0), first.getBackID(0), -1, 0) != 0) {
+ assert false;
+ }
+
+ // EOS = 1
+ if (addNode(TokenType.KNOWN, -1, endOffset - rootBase, -1) != 1) {
+ assert false;
+ }
+
+ for (int offset = endOffset; prevOffset < offset; --offset) {
+ int right = offset - rootBase;
+ // optimize: exclude disconnected nodes.
+ if (0 <= lRoot[right]) {
+ Position pos = positions.get(offset);
+ for (int i = 0; i < pos.getCount(); ++i) {
+ addNode(pos.getBackType(i), pos.getBackID(i), pos.getBackPos(i) - rootBase, right);
+ }
+ }
+ }
+ }
+
+ // set mark = -1 for unreachable nodes.
+ void markUnreachable() {
+ for (int index = 1; index < rootSize - 1; ++index) {
+ if (rRoot[index] < 0) {
+ for (int node = lRoot[index]; 0 <= node; node = nodeLeftChain[node]) {
+ if (VERBOSE) {
+ System.out.printf("DEBUG: markUnreachable: node=%d\n", node);
+ }
+ nodeMark[node] = -1;
+ }
+ }
+ }
+ }
+
+ private int connectionCost(ConnectionCosts costs, int left, int right) {
+ int leftID = nodeLeftID[right];
+ return ((leftID == 0 && !useEOS) ? 0 : costs.get(nodeRightID[left], leftID));
+ }
+
+ void calcLeftCost(ConnectionCosts costs) {
+ for (int index = 0; index < rootSize; ++index) {
+ for (int node = lRoot[index]; 0 <= node; node = nodeLeftChain[node]) {
+ if (0 <= nodeMark[node]) {
+ int leastNode = -1;
+ int leastCost = Integer.MAX_VALUE;
+ for (int leftNode = rRoot[index]; 0 <= leftNode; leftNode = nodeRightChain[leftNode]) {
+ if (0 <= nodeMark[leftNode]) {
+ int cost =
+ nodeLeftCost[leftNode]
+ + nodeWordCost[leftNode]
+ + connectionCost(costs, leftNode, node);
+ if (cost < leastCost) {
+ leastCost = cost;
+ leastNode = leftNode;
+ }
+ }
+ }
+ assert 0 <= leastNode;
+ nodeLeftNode[node] = leastNode;
+ nodeLeftCost[node] = leastCost;
+ if (VERBOSE) {
+ System.out.printf(
+ "DEBUG: calcLeftCost: node=%d, leftNode=%d, leftCost=%d\n",
+ node, nodeLeftNode[node], nodeLeftCost[node]);
+ }
+ }
+ }
+ }
+ }
+
+ void calcRightCost(ConnectionCosts costs) {
+ for (int index = rootSize - 1; 0 <= index; --index) {
+ for (int node = rRoot[index]; 0 <= node; node = nodeRightChain[node]) {
+ if (0 <= nodeMark[node]) {
+ int leastNode = -1;
+ int leastCost = Integer.MAX_VALUE;
+ for (int rightNode = lRoot[index];
+ 0 <= rightNode;
+ rightNode = nodeLeftChain[rightNode]) {
+ if (0 <= nodeMark[rightNode]) {
+ int cost =
+ nodeRightCost[rightNode]
+ + nodeWordCost[rightNode]
+ + connectionCost(costs, node, rightNode);
+ if (cost < leastCost) {
+ leastCost = cost;
+ leastNode = rightNode;
+ }
+ }
+ }
+ assert 0 <= leastNode;
+ nodeRightNode[node] = leastNode;
+ nodeRightCost[node] = leastCost;
+ if (VERBOSE) {
+ System.out.printf(
+ "DEBUG: calcRightCost: node=%d, rightNode=%d, rightCost=%d\n",
+ node, nodeRightNode[node], nodeRightCost[node]);
+ }
+ }
+ }
+ }
+ }
+
+ // Mark all nodes that have same text and different par-of-speech or reading.
+ void markSameSpanNode(int refNode, int value) {
+ int left = nodeLeft[refNode];
+ int right = nodeRight[refNode];
+ for (int node = lRoot[left]; 0 <= node; node = nodeLeftChain[node]) {
+ if (nodeRight[node] == right) {
+ nodeMark[node] = value;
+ }
+ }
+ }
+
+ List<Integer> bestPathNodeList() {
+ List<Integer> list = new ArrayList<>();
+ for (int node = nodeRightNode[0]; node != 1; node = nodeRightNode[node]) {
+ list.add(node);
+ markSameSpanNode(node, 1);
+ }
+ return list;
+ }
+
+ private int cost(int node) {
+ return nodeLeftCost[node] + nodeWordCost[node] + nodeRightCost[node];
+ }
+
+ List<Integer> nBestNodeList(int N) {
+ List<Integer> list = new ArrayList<>();
+ int leastCost = Integer.MAX_VALUE;
+ int leastLeft = -1;
+ int leastRight = -1;
+ for (int node = 2; node < nodeCount; ++node) {
+ if (nodeMark[node] == 0) {
+ int cost = cost(node);
+ if (cost < leastCost) {
+ leastCost = cost;
+ leastLeft = nodeLeft[node];
+ leastRight = nodeRight[node];
+ list.clear();
+ list.add(node);
+ } else if (cost == leastCost
+ && (nodeLeft[node] != leastLeft || nodeRight[node] != leastRight)) {
+ list.add(node);
+ }
+ }
+ }
+ for (int node : list) {
+ markSameSpanNode(node, N);
+ }
+ return list;
+ }
+
+ int bestCost() {
+ return nodeLeftCost[1];
+ }
+
+ int probeDelta(int start, int end) {
+ int left = start - rootBase;
+ int right = end - rootBase;
+ if (left < 0 || rootSize < right) {
+ return Integer.MAX_VALUE;
+ }
+ int probedCost = Integer.MAX_VALUE;
+ for (int node = lRoot[left]; 0 <= node; node = nodeLeftChain[node]) {
+ if (nodeRight[node] == right) {
+ probedCost = Math.min(probedCost, cost(node));
+ }
+ }
+ return probedCost - bestCost();
+ }
+
+ void debugPrint() {
+ if (VERBOSE) {
+ for (int node = 0; node < nodeCount; ++node) {
+ System.out.printf(
+ "DEBUG NODE: node=%d, mark=%d, cost=%d, left=%d, right=%d\n",
+ node, nodeMark[node], cost(node), nodeLeft[node], nodeRight[node]);
+ }
+ }
+ }
+ }
+}
diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizer.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizer.java
index 71d19f02e1f..1232c5b800f 100644
--- a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizer.java
+++ b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizer.java
@@ -252,7 +252,6 @@ public final class JapaneseTokenizer extends Tokenizer {
userFSTReader,
userDictionary,
connectionCosts,
- ViterbiNBest.PositionNBest.class,
unkDictionary,
characterDefinition,
discardPunctuation,
@@ -354,10 +353,9 @@ public final class JapaneseTokenizer extends Tokenizer {
setNBestCost(1);
int prevRootBase = -1;
while (incrementToken()) {
- if (viterbi.lattice.rootBase != prevRootBase) {
- prevRootBase = viterbi.lattice.rootBase;
- delta =
- Math.min(delta, viterbi.lattice.probeDelta(start, start + requiredToken.length()));
+ if (viterbi.getLatticeRootBase() != prevRootBase) {
+ prevRootBase = viterbi.getLatticeRootBase();
+ delta = Math.min(delta, viterbi.probeDelta(start, start + requiredToken.length()));
}
}
} finally {
diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/ViterbiNBest.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/ViterbiNBest.java
index d789947e784..207feabe6b7 100644
--- a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/ViterbiNBest.java
+++ b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/ViterbiNBest.java
@@ -17,12 +17,7 @@
package org.apache.lucene.analysis.ja;
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
import java.util.Collections;
-import java.util.EnumMap;
-import java.util.HashMap;
-import java.util.List;
import org.apache.lucene.analysis.ja.dict.CharacterDefinition;
import org.apache.lucene.analysis.ja.dict.JaMorphData;
import org.apache.lucene.analysis.ja.dict.TokenInfoDictionary;
@@ -33,19 +28,13 @@ import org.apache.lucene.analysis.morph.Dictionary;
import org.apache.lucene.analysis.morph.GraphvizFormatter;
import org.apache.lucene.analysis.morph.TokenInfoFST;
import org.apache.lucene.analysis.morph.TokenType;
-import org.apache.lucene.analysis.morph.Viterbi;
-import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.fst.FST;
/**
* {@link org.apache.lucene.analysis.morph.Viterbi} subclass for Japanese morphological analysis.
* This also performs n-best path calculation
*/
-final class ViterbiNBest
- extends org.apache.lucene.analysis.morph.Viterbi<Token, ViterbiNBest.PositionNBest> {
-
- private final EnumMap<TokenType, Dictionary<? extends JaMorphData>> dictionaryMap =
- new EnumMap<>(TokenType.class);
+final class ViterbiNBest extends org.apache.lucene.analysis.morph.ViterbiNBest<Token, JaMorphData> {
private final UnknownDictionary unkDictionary;
private final CharacterDefinition characterDefinition;
@@ -56,11 +45,6 @@ final class ViterbiNBest
private final boolean extendedMode;
private final boolean outputCompounds;
- // Allowable cost difference for N-best output:
- private int nBestCost = 0;
-
- Lattice lattice = null;
-
private GraphvizFormatter<JaMorphData> dotOut;
ViterbiNBest(
@@ -71,14 +55,13 @@ final class ViterbiNBest
FST.BytesReader userFSTReader,
UserDictionary userDictionary,
ConnectionCosts costs,
- Class<PositionNBest> positionImpl,
UnknownDictionary unkDictionary,
CharacterDefinition characterDefinition,
boolean discardPunctuation,
boolean searchMode,
boolean extendedMode,
boolean outputCompounds) {
- super(fst, fstReader, dictionary, userFST, userFSTReader, userDictionary, costs, positionImpl);
+ super(fst, fstReader, dictionary, userFST, userFSTReader, userDictionary, costs);
this.unkDictionary = unkDictionary;
this.characterDefinition = characterDefinition;
this.userDictionary = userDictionary;
@@ -565,14 +548,14 @@ final class ViterbiNBest
for (int pos = startPos; pos < endPos; pos++) {
final PositionNBest posData = positions.get(pos);
if (VERBOSE) {
- System.out.println(" forward pos=" + pos + " count=" + posData.forwardCount);
+ System.out.println(" forward pos=" + pos + " count=" + posData.getForwardCount());
}
if (posData.getCount() == 0) {
// No arcs arrive here...
if (VERBOSE) {
System.out.println(" skip");
}
- posData.forwardCount = 0;
+ posData.setForwardCount(0);
continue;
}
@@ -590,11 +573,11 @@ final class ViterbiNBest
.getRightId(posData.getBackID(bestStartIDX));
}
final int pathCost = posData.getCost(bestStartIDX);
- for (int forwardArcIDX = 0; forwardArcIDX < posData.forwardCount; forwardArcIDX++) {
- final TokenType forwardType = posData.forwardType[forwardArcIDX];
+ for (int forwardArcIDX = 0; forwardArcIDX < posData.getForwardCount(); forwardArcIDX++) {
+ final TokenType forwardType = posData.getForwardType(forwardArcIDX);
final Dictionary<? extends JaMorphData> dict2 = getDict(forwardType);
- final int wordID = posData.forwardID[forwardArcIDX];
- final int toPos = posData.forwardPos[forwardArcIDX];
+ final int wordID = posData.getForwardID(forwardArcIDX);
+ final int toPos = posData.getForwardPos(forwardArcIDX);
final int newCost =
pathCost
+ dict2.getWordCost(wordID)
@@ -622,9 +605,9 @@ final class ViterbiNBest
} else {
// On non-initial positions, we maximize score
// across all arriving lastRightIDs:
- for (int forwardArcIDX = 0; forwardArcIDX < posData.forwardCount; forwardArcIDX++) {
- final TokenType forwardType = posData.forwardType[forwardArcIDX];
- final int toPos = posData.forwardPos[forwardArcIDX];
+ for (int forwardArcIDX = 0; forwardArcIDX < posData.getForwardCount(); forwardArcIDX++) {
+ final TokenType forwardType = posData.getForwardType(forwardArcIDX);
+ final int toPos = posData.getForwardPos(forwardArcIDX);
if (VERBOSE) {
System.out.println(
" + "
@@ -639,76 +622,35 @@ final class ViterbiNBest
posData,
pos,
toPos,
- posData.forwardID[forwardArcIDX],
+ posData.getForwardID(forwardArcIDX),
forwardType,
true);
}
}
- posData.forwardCount = 0;
+ posData.setForwardCount(0);
}
}
@Override
- protected void backtraceNBest(final Position endPosData, final boolean useEOS)
- throws IOException {
- if (lattice == null) {
- lattice = new Lattice();
- }
-
- final int endPos = endPosData.getPos();
- char[] fragment = buffer.get(lastBackTracePos, endPos - lastBackTracePos);
- lattice.setup(fragment, dictionaryMap, positions, lastBackTracePos, endPos, useEOS);
- lattice.markUnreachable();
- lattice.calcLeftCost(costs);
- lattice.calcRightCost(costs);
-
- int bestCost = lattice.bestCost();
- if (VERBOSE) {
- System.out.printf("DEBUG: 1-BEST COST: %d\n", bestCost);
- }
- for (int node : lattice.bestPathNodeList()) {
- registerNode(node, fragment);
- }
-
- for (int n = 2; ; ++n) {
- List<Integer> nbest = lattice.nBestNodeList(n);
- if (nbest.isEmpty()) {
- break;
- }
- int cost = lattice.cost(nbest.get(0));
- if (VERBOSE) {
- System.out.printf("DEBUG: %d-BEST COST: %d\n", n, cost);
- }
- if (bestCost + nBestCost < cost) {
- break;
- }
- for (int node : nbest) {
- registerNode(node, fragment);
- }
- }
- if (VERBOSE) {
- lattice.debugPrint();
- }
- }
-
- private void registerNode(int node, char[] fragment) {
- int left = lattice.nodeLeft[node];
- int right = lattice.nodeRight[node];
- TokenType type = lattice.nodeDicType[node];
+ protected void registerNode(int node, char[] fragment) {
+ int left = lattice.getNodeLeft(node);
+ int right = lattice.getNodeRight(node);
+ TokenType type = lattice.getNodeDicType(node);
if (!discardPunctuation || !isPunctuation(fragment[left])) {
if (type == TokenType.USER) {
// The code below are based on backtrace().
//
// Expand the phraseID we recorded into the actual segmentation:
- final int[] wordIDAndLength = userDictionary.lookupSegmentation(lattice.nodeWordID[node]);
+ final int[] wordIDAndLength =
+ userDictionary.lookupSegmentation(lattice.getNodeWordID(node));
int wordID = wordIDAndLength[0];
pending.add(
new Token(
fragment,
left,
right - left,
- lattice.rootBase + left,
- lattice.rootBase + right,
+ lattice.getRootBase() + left,
+ lattice.getRootBase() + right,
wordID,
TokenType.USER,
userDictionary.getMorphAttributes()));
@@ -717,7 +659,7 @@ final class ViterbiNBest
for (int j = 1; j < wordIDAndLength.length; j++) {
final int len = wordIDAndLength[j];
if (len < right - left) {
- int startOffset = lattice.rootBase + current + left;
+ int startOffset = lattice.getRootBase() + current + left;
pending.add(
new Token(
fragment,
@@ -737,86 +679,27 @@ final class ViterbiNBest
fragment,
left,
right - left,
- lattice.rootBase + left,
- lattice.rootBase + right,
- lattice.nodeWordID[node],
+ lattice.getRootBase() + left,
+ lattice.getRootBase() + right,
+ lattice.getNodeWordID(node),
type,
getDict(type).getMorphAttributes()));
}
}
}
- @Override
- protected void fixupPendingList() {
- // Sort for removing same tokens.
- // USER token should be ahead from normal one.
- Collections.sort(
- pending,
- (a, b) -> {
- int aOff = a.getOffset();
- int bOff = b.getOffset();
- if (aOff != bOff) {
- return aOff - bOff;
- }
- int aLen = a.getLength();
- int bLen = b.getLength();
- if (aLen != bLen) {
- return aLen - bLen;
- }
- // order of Type is KNOWN, UNKNOWN, USER,
- // so we use reversed comparison here.
- return b.getType().ordinal() - a.getType().ordinal();
- });
-
- // Remove same token.
- for (int i = 1; i < pending.size(); ++i) {
- Token a = pending.get(i - 1);
- Token b = pending.get(i);
- if (a.getOffset() == b.getOffset() && a.getLength() == b.getLength()) {
- pending.remove(i);
- // It is important to decrement "i" here, because a next may be removed.
- --i;
- }
- }
-
- // offset=>position map
- HashMap<Integer, Integer> map = new HashMap<>();
- for (Token t : pending) {
- map.put(t.getOffset(), 0);
- map.put(t.getOffset() + t.getLength(), 0);
- }
-
- // Get unique and sorted list of all edge position of tokens.
- Integer[] offsets = map.keySet().toArray(new Integer[0]);
- Arrays.sort(offsets);
-
- // setup all value of map. It specifies N-th position from begin.
- for (int i = 0; i < offsets.length; ++i) {
- map.put(offsets[i], i);
- }
-
- // We got all position length now.
- for (Token t : pending) {
- t.setPositionLength(map.get(t.getOffset() + t.getLength()) - map.get(t.getOffset()));
- }
-
- // Make PENDING to be reversed order to fit its usage.
- // If you would like to speedup, you can try reversed order sort
- // at first of this function.
- Collections.reverse(pending);
- }
-
Dictionary<? extends JaMorphData> getDict(TokenType type) {
return dictionaryMap.get(type);
}
- void setNBestCost(int value) {
- nBestCost = value;
- outputNBest = 0 < nBestCost;
+ @Override
+ protected void setNBestCost(int value) {
+ super.setNBestCost(value);
}
- int getNBestCost() {
- return nBestCost;
+ @Override
+ protected int getNBestCost() {
+ return super.getNBestCost();
}
private static boolean isPunctuation(char ch) {
@@ -842,412 +725,4 @@ final class ViterbiNBest
return false;
}
}
-
- /**
- * {@link org.apache.lucene.analysis.morph.Viterbi.Position} extension; this holds all forward
- * pointers to calculate n-best path.
- */
- public static class PositionNBest extends Viterbi.Position {
- // Only used when finding 2nd best segmentation under a
- // too-long token:
- int forwardCount;
- int[] forwardPos = new int[8];
- int[] forwardID = new int[8];
- int[] forwardIndex = new int[8];
- TokenType[] forwardType = new TokenType[8];
-
- private void growForward() {
- forwardPos = ArrayUtil.grow(forwardPos, 1 + forwardCount);
- forwardID = ArrayUtil.grow(forwardID, 1 + forwardCount);
- forwardIndex = ArrayUtil.grow(forwardIndex, 1 + forwardCount);
-
- // NOTE: sneaky: grow separately because
- // ArrayUtil.grow will otherwise pick a different
- // length than the int[]s we just grew:
- final TokenType[] newForwardType = new TokenType[forwardPos.length];
- System.arraycopy(forwardType, 0, newForwardType, 0, forwardType.length);
- forwardType = newForwardType;
- }
-
- void addForward(int forwardPos, int forwardIndex, int forwardID, TokenType forwardType) {
- if (forwardCount == this.forwardID.length) {
- growForward();
- }
- this.forwardPos[forwardCount] = forwardPos;
- this.forwardIndex[forwardCount] = forwardIndex;
- this.forwardID[forwardCount] = forwardID;
- this.forwardType[forwardCount] = forwardType;
- forwardCount++;
- }
-
- @Override
- public void reset() {
- super.reset();
- // forwardCount naturally resets after it runs:
- assert forwardCount == 0 : "pos=" + getPos() + " forwardCount=" + forwardCount;
- }
- }
-
- // yet another lattice data structure
- static final class Lattice {
- char[] fragment;
- EnumMap<TokenType, Dictionary<? extends JaMorphData>> dictionaryMap;
- boolean useEOS;
-
- int rootCapacity = 0;
- int rootSize = 0;
- int rootBase = 0;
-
- // root pointers of node chain by leftChain_ that have same start offset.
- int[] lRoot;
- // root pointers of node chain by rightChain_ that have same end offset.
- int[] rRoot;
-
- int capacity = 0;
- int nodeCount = 0;
-
- // The variables below are elements of lattice node that indexed by node number.
- TokenType[] nodeDicType;
- int[] nodeWordID;
- // nodeMark - -1:excluded, 0:unused, 1:bestpath, 2:2-best-path, ... N:N-best-path
- int[] nodeMark;
- int[] nodeLeftID;
- int[] nodeRightID;
- int[] nodeWordCost;
- int[] nodeLeftCost;
- int[] nodeRightCost;
- // nodeLeftNode, nodeRightNode - are left/right node number with minimum cost path.
- int[] nodeLeftNode;
- int[] nodeRightNode;
- // nodeLeft, nodeRight - start/end offset
- int[] nodeLeft;
- int[] nodeRight;
- int[] nodeLeftChain;
- int[] nodeRightChain;
-
- private void setupRoot(int baseOffset, int lastOffset) {
- assert baseOffset <= lastOffset;
- int size = lastOffset - baseOffset + 1;
- if (rootCapacity < size) {
- int oversize = ArrayUtil.oversize(size, Integer.BYTES);
- lRoot = new int[oversize];
- rRoot = new int[oversize];
- rootCapacity = oversize;
- }
- Arrays.fill(lRoot, 0, size, -1);
- Arrays.fill(rRoot, 0, size, -1);
- rootSize = size;
- rootBase = baseOffset;
- }
-
- // Reserve at least N nodes.
- private void reserve(int n) {
- if (capacity < n) {
- int oversize = ArrayUtil.oversize(n, Integer.BYTES);
- nodeDicType = new TokenType[oversize];
- nodeWordID = new int[oversize];
- nodeMark = new int[oversize];
- nodeLeftID = new int[oversize];
- nodeRightID = new int[oversize];
- nodeWordCost = new int[oversize];
- nodeLeftCost = new int[oversize];
- nodeRightCost = new int[oversize];
- nodeLeftNode = new int[oversize];
- nodeRightNode = new int[oversize];
- nodeLeft = new int[oversize];
- nodeRight = new int[oversize];
- nodeLeftChain = new int[oversize];
- nodeRightChain = new int[oversize];
- capacity = oversize;
- }
- }
-
- private void setupNodePool(int n) {
- reserve(n);
- nodeCount = 0;
- if (VERBOSE) {
- System.out.printf("DEBUG: setupNodePool: n = %d\n", n);
- System.out.printf("DEBUG: setupNodePool: lattice.capacity = %d\n", capacity);
- }
- }
-
- private int addNode(TokenType dicType, int wordID, int left, int right) {
- if (VERBOSE) {
- System.out.printf(
- "DEBUG: addNode: dicType=%s, wordID=%d, left=%d, right=%d, str=%s\n",
- dicType.toString(),
- wordID,
- left,
- right,
- left == -1 ? "BOS" : right == -1 ? "EOS" : new String(fragment, left, right - left));
- }
- assert nodeCount < capacity;
- assert left == -1 || right == -1 || left < right;
- assert left == -1 || (0 <= left && left < rootSize);
- assert right == -1 || (0 <= right && right < rootSize);
-
- int node = nodeCount++;
-
- if (VERBOSE) {
- System.out.printf("DEBUG: addNode: node=%d\n", node);
- }
-
- nodeDicType[node] = dicType;
- nodeWordID[node] = wordID;
- nodeMark[node] = 0;
-
- if (wordID < 0) {
- nodeWordCost[node] = 0;
- nodeLeftCost[node] = 0;
- nodeRightCost[node] = 0;
- nodeLeftID[node] = 0;
- nodeRightID[node] = 0;
- } else {
- Dictionary<? extends JaMorphData> dic = dictionaryMap.get(dicType);
- nodeWordCost[node] = dic.getWordCost(wordID);
- nodeLeftID[node] = dic.getLeftId(wordID);
- nodeRightID[node] = dic.getRightId(wordID);
- }
-
- if (VERBOSE) {
- System.out.printf(
- "DEBUG: addNode: wordCost=%d, leftID=%d, rightID=%d\n",
- nodeWordCost[node], nodeLeftID[node], nodeRightID[node]);
- }
-
- nodeLeft[node] = left;
- nodeRight[node] = right;
- if (0 <= left) {
- nodeLeftChain[node] = lRoot[left];
- lRoot[left] = node;
- } else {
- nodeLeftChain[node] = -1;
- }
- if (0 <= right) {
- nodeRightChain[node] = rRoot[right];
- rRoot[right] = node;
- } else {
- nodeRightChain[node] = -1;
- }
- return node;
- }
-
- // Sum of positions.get(i).count in [beg, end) range.
- // using stream:
- // return IntStream.range(beg, end).map(i -> positions.get(i).count).sum();
- private int positionCount(WrappedPositionArray<PositionNBest> positions, int beg, int end) {
- int count = 0;
- for (int i = beg; i < end; ++i) {
- count += positions.get(i).getCount();
- }
- return count;
- }
-
- void setup(
- char[] fragment,
- EnumMap<TokenType, Dictionary<? extends JaMorphData>> dictionaryMap,
- WrappedPositionArray<PositionNBest> positions,
- int prevOffset,
- int endOffset,
- boolean useEOS) {
- assert positions.get(prevOffset).getCount() == 1;
- if (VERBOSE) {
- System.out.printf("DEBUG: setup: prevOffset=%d, endOffset=%d\n", prevOffset, endOffset);
- }
-
- this.fragment = fragment;
- this.dictionaryMap = dictionaryMap;
- this.useEOS = useEOS;
-
- // Initialize lRoot and rRoot.
- setupRoot(prevOffset, endOffset);
-
- // "+ 2" for first/last record.
- setupNodePool(positionCount(positions, prevOffset + 1, endOffset + 1) + 2);
-
- // substitute for BOS = 0
- Position first = positions.get(prevOffset);
- if (addNode(first.getBackType(0), first.getBackID(0), -1, 0) != 0) {
- assert false;
- }
-
- // EOS = 1
- if (addNode(TokenType.KNOWN, -1, endOffset - rootBase, -1) != 1) {
- assert false;
- }
-
- for (int offset = endOffset; prevOffset < offset; --offset) {
- int right = offset - rootBase;
- // optimize: exclude disconnected nodes.
- if (0 <= lRoot[right]) {
- Position pos = positions.get(offset);
- for (int i = 0; i < pos.getCount(); ++i) {
- addNode(pos.getBackType(i), pos.getBackID(i), pos.getBackPos(i) - rootBase, right);
- }
- }
- }
- }
-
- // set mark = -1 for unreachable nodes.
- void markUnreachable() {
- for (int index = 1; index < rootSize - 1; ++index) {
- if (rRoot[index] < 0) {
- for (int node = lRoot[index]; 0 <= node; node = nodeLeftChain[node]) {
- if (VERBOSE) {
- System.out.printf("DEBUG: markUnreachable: node=%d\n", node);
- }
- nodeMark[node] = -1;
- }
- }
- }
- }
-
- private int connectionCost(ConnectionCosts costs, int left, int right) {
- int leftID = nodeLeftID[right];
- return ((leftID == 0 && !useEOS) ? 0 : costs.get(nodeRightID[left], leftID));
- }
-
- void calcLeftCost(ConnectionCosts costs) {
- for (int index = 0; index < rootSize; ++index) {
- for (int node = lRoot[index]; 0 <= node; node = nodeLeftChain[node]) {
- if (0 <= nodeMark[node]) {
- int leastNode = -1;
- int leastCost = Integer.MAX_VALUE;
- for (int leftNode = rRoot[index]; 0 <= leftNode; leftNode = nodeRightChain[leftNode]) {
- if (0 <= nodeMark[leftNode]) {
- int cost =
- nodeLeftCost[leftNode]
- + nodeWordCost[leftNode]
- + connectionCost(costs, leftNode, node);
- if (cost < leastCost) {
- leastCost = cost;
- leastNode = leftNode;
- }
- }
- }
- assert 0 <= leastNode;
- nodeLeftNode[node] = leastNode;
- nodeLeftCost[node] = leastCost;
- if (VERBOSE) {
- System.out.printf(
- "DEBUG: calcLeftCost: node=%d, leftNode=%d, leftCost=%d\n",
- node, nodeLeftNode[node], nodeLeftCost[node]);
- }
- }
- }
- }
- }
-
- void calcRightCost(ConnectionCosts costs) {
- for (int index = rootSize - 1; 0 <= index; --index) {
- for (int node = rRoot[index]; 0 <= node; node = nodeRightChain[node]) {
- if (0 <= nodeMark[node]) {
- int leastNode = -1;
- int leastCost = Integer.MAX_VALUE;
- for (int rightNode = lRoot[index];
- 0 <= rightNode;
- rightNode = nodeLeftChain[rightNode]) {
- if (0 <= nodeMark[rightNode]) {
- int cost =
- nodeRightCost[rightNode]
- + nodeWordCost[rightNode]
- + connectionCost(costs, node, rightNode);
- if (cost < leastCost) {
- leastCost = cost;
- leastNode = rightNode;
- }
- }
- }
- assert 0 <= leastNode;
- nodeRightNode[node] = leastNode;
- nodeRightCost[node] = leastCost;
- if (VERBOSE) {
- System.out.printf(
- "DEBUG: calcRightCost: node=%d, rightNode=%d, rightCost=%d\n",
- node, nodeRightNode[node], nodeRightCost[node]);
- }
- }
- }
- }
- }
-
- // Mark all nodes that have same text and different par-of-speech or reading.
- void markSameSpanNode(int refNode, int value) {
- int left = nodeLeft[refNode];
- int right = nodeRight[refNode];
- for (int node = lRoot[left]; 0 <= node; node = nodeLeftChain[node]) {
- if (nodeRight[node] == right) {
- nodeMark[node] = value;
- }
- }
- }
-
- List<Integer> bestPathNodeList() {
- List<Integer> list = new ArrayList<>();
- for (int node = nodeRightNode[0]; node != 1; node = nodeRightNode[node]) {
- list.add(node);
- markSameSpanNode(node, 1);
- }
- return list;
- }
-
- private int cost(int node) {
- return nodeLeftCost[node] + nodeWordCost[node] + nodeRightCost[node];
- }
-
- List<Integer> nBestNodeList(int N) {
- List<Integer> list = new ArrayList<>();
- int leastCost = Integer.MAX_VALUE;
- int leastLeft = -1;
- int leastRight = -1;
- for (int node = 2; node < nodeCount; ++node) {
- if (nodeMark[node] == 0) {
- int cost = cost(node);
- if (cost < leastCost) {
- leastCost = cost;
- leastLeft = nodeLeft[node];
- leastRight = nodeRight[node];
- list.clear();
- list.add(node);
- } else if (cost == leastCost
- && (nodeLeft[node] != leastLeft || nodeRight[node] != leastRight)) {
- list.add(node);
- }
- }
- }
- for (int node : list) {
- markSameSpanNode(node, N);
- }
- return list;
- }
-
- int bestCost() {
- return nodeLeftCost[1];
- }
-
- int probeDelta(int start, int end) {
- int left = start - rootBase;
- int right = end - rootBase;
- if (left < 0 || rootSize < right) {
- return Integer.MAX_VALUE;
- }
- int probedCost = Integer.MAX_VALUE;
- for (int node = lRoot[left]; 0 <= node; node = nodeLeftChain[node]) {
- if (nodeRight[node] == right) {
- probedCost = Math.min(probedCost, cost(node));
- }
- }
- return probedCost - bestCost();
- }
-
- void debugPrint() {
- if (VERBOSE) {
- for (int node = 0; node < nodeCount; ++node) {
- System.out.printf(
- "DEBUG NODE: node=%d, mark=%d, cost=%d, left=%d, right=%d\n",
- node, nodeMark[node], cost(node), nodeLeft[node], nodeRight[node]);
- }
- }
- }
- }
}