Skip to content

Apache Lucene open-source search software

LUCENE-10493: Unify token Type enum in kuromoji and nori (#801)

Tomoko Uchida13630d361e28
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/Token.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/Token.java
index f9f0468d473..7050e4e87e2 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/Token.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/Token.java
@@ -27,12 +27,16 @@ public abstract class Token {
protected int posIncr = 1;
protected int posLen = 1;
- protected Token(char[] surfaceForm, int offset, int length, int startOffset, int endOffset) {
+ protected final TokenType type;
+
+ protected Token(
+ char[] surfaceForm, int offset, int length, int startOffset, int endOffset, TokenType type) {
this.surfaceForm = surfaceForm;
this.offset = offset;
this.length = length;
this.startOffset = startOffset;
this.endOffset = endOffset;
+ this.type = type;
}
/** @return surfaceForm */
@@ -88,4 +92,9 @@ public abstract class Token {
public int getPositionLength() {
return posLen;
}
+
+ /** Returns the type of this token */
+ public TokenType getType() {
+ return type;
+ }
}
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/TokenType.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/TokenType.java
new file mode 100644
index 00000000000..01084af16ce
--- /dev/null
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/TokenType.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.analysis.morph;
+
+/** Token type reflecting the original source of this token */
+public enum TokenType {
+ /** Known words from the system dictionary. */
+ KNOWN,
+ /** Unknown words (heuristically segmented). */
+ UNKNOWN,
+ /** Known words from the user dictionary. */
+ USER
+}
diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizer.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizer.java
index 84464d76730..b06b19fc359 100644
--- a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizer.java
+++ b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizer.java
@@ -35,6 +35,7 @@ import org.apache.lucene.analysis.ja.dict.UnknownDictionary;
import org.apache.lucene.analysis.ja.dict.UserDictionary;
import org.apache.lucene.analysis.ja.tokenattributes.*;
import org.apache.lucene.analysis.morph.Dictionary;
+import org.apache.lucene.analysis.morph.TokenType;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
@@ -93,16 +94,6 @@ public final class JapaneseTokenizer extends Tokenizer {
/** Default tokenization mode. Currently this is {@link Mode#SEARCH}. */
public static final Mode DEFAULT_MODE = Mode.SEARCH;
- /** Token type reflecting the original source of this token */
- public enum Type {
- /** Known words from the system dictionary. */
- KNOWN,
- /** Unknown words (heuristically segmented). */
- UNKNOWN,
- /** Known words from the user dictionary. */
- USER
- }
-
private static final boolean VERBOSE = false;
private static final int SEARCH_MODE_KANJI_LENGTH = 2;
@@ -117,8 +108,8 @@ public final class JapaneseTokenizer extends Tokenizer {
private static final int MAX_UNKNOWN_WORD_LENGTH = 1024;
private static final int MAX_BACKTRACE_GAP = 1024;
- private final EnumMap<Type, Dictionary<? extends JaMorphData>> dictionaryMap =
- new EnumMap<>(Type.class);
+ private final EnumMap<TokenType, Dictionary<? extends JaMorphData>> dictionaryMap =
+ new EnumMap<>(TokenType.class);
private final TokenInfoFST fst;
private final TokenInfoDictionary dictionary;
@@ -326,9 +317,9 @@ public final class JapaneseTokenizer extends Tokenizer {
resetState();
- dictionaryMap.put(Type.KNOWN, dictionary);
- dictionaryMap.put(Type.UNKNOWN, unkDictionary);
- dictionaryMap.put(Type.USER, userDictionary);
+ dictionaryMap.put(TokenType.KNOWN, dictionary);
+ dictionaryMap.put(TokenType.UNKNOWN, unkDictionary);
+ dictionaryMap.put(TokenType.USER, userDictionary);
}
private GraphvizFormatter dotOut;
@@ -360,7 +351,7 @@ public final class JapaneseTokenizer extends Tokenizer {
pending.clear();
// Add BOS:
- positions.get(0).add(0, 0, -1, -1, -1, -1, Type.KNOWN);
+ positions.get(0).add(0, 0, -1, -1, -1, -1, TokenType.KNOWN);
}
@Override
@@ -418,7 +409,7 @@ public final class JapaneseTokenizer extends Tokenizer {
int[] backWordPos = new int[8];
int[] backIndex = new int[8];
int[] backID = new int[8];
- Type[] backType = new Type[8];
+ TokenType[] backType = new TokenType[8];
// Only used when finding 2nd best segmentation under a
// too-long token:
@@ -426,7 +417,7 @@ public final class JapaneseTokenizer extends Tokenizer {
int[] forwardPos = new int[8];
int[] forwardID = new int[8];
int[] forwardIndex = new int[8];
- Type[] forwardType = new Type[8];
+ TokenType[] forwardType = new TokenType[8];
public void grow() {
costs = ArrayUtil.grow(costs, 1 + count);
@@ -439,7 +430,7 @@ public final class JapaneseTokenizer extends Tokenizer {
// NOTE: sneaky: grow separately because
// ArrayUtil.grow will otherwise pick a different
// length than the int[]s we just grew:
- final Type[] newBackType = new Type[backID.length];
+ final TokenType[] newBackType = new TokenType[backID.length];
System.arraycopy(backType, 0, newBackType, 0, backType.length);
backType = newBackType;
}
@@ -452,7 +443,7 @@ public final class JapaneseTokenizer extends Tokenizer {
// NOTE: sneaky: grow separately because
// ArrayUtil.grow will otherwise pick a different
// length than the int[]s we just grew:
- final Type[] newForwardType = new Type[forwardPos.length];
+ final TokenType[] newForwardType = new TokenType[forwardPos.length];
System.arraycopy(forwardType, 0, newForwardType, 0, forwardType.length);
forwardType = newForwardType;
}
@@ -464,7 +455,7 @@ public final class JapaneseTokenizer extends Tokenizer {
int backRPos,
int backIndex,
int backID,
- Type backType) {
+ TokenType backType) {
// NOTE: this isn't quite a true Viterbi search,
// because we should check if lastRightID is
// already present here, and only update if the new
@@ -486,7 +477,7 @@ public final class JapaneseTokenizer extends Tokenizer {
count++;
}
- public void addForward(int forwardPos, int forwardIndex, int forwardID, Type forwardType) {
+ public void addForward(int forwardPos, int forwardIndex, int forwardID, TokenType forwardType) {
if (forwardCount == this.forwardID.length) {
growForward();
}
@@ -510,7 +501,7 @@ public final class JapaneseTokenizer extends Tokenizer {
int wordPos,
int endPos,
int wordID,
- Type type,
+ TokenType type,
boolean addPenalty)
throws IOException {
final int wordCost = morphAtts.getWordCost(wordID);
@@ -565,7 +556,7 @@ public final class JapaneseTokenizer extends Tokenizer {
+ positions.get(endPos).count);
}
- if (addPenalty && type != Type.USER) {
+ if (addPenalty && type != TokenType.USER) {
final int penalty = computePenalty(fromPosData.pos, endPos - fromPosData.pos);
if (VERBOSE) {
if (penalty > 0) {
@@ -916,7 +907,7 @@ public final class JapaneseTokenizer extends Tokenizer {
pos,
posAhead + 1,
output + arc.nextFinalOutput().intValue(),
- Type.USER,
+ TokenType.USER,
false);
anyMatches = true;
}
@@ -970,7 +961,7 @@ public final class JapaneseTokenizer extends Tokenizer {
pos,
posAhead + 1,
wordIdRef.ints[wordIdRef.offset + ofs],
- Type.KNOWN,
+ TokenType.KNOWN,
false);
anyMatches = true;
}
@@ -1027,7 +1018,7 @@ public final class JapaneseTokenizer extends Tokenizer {
pos,
posData.pos + unknownWordLength,
wordIdRef.ints[wordIdRef.offset + ofs],
- Type.UNKNOWN,
+ TokenType.UNKNOWN,
false);
}
@@ -1144,7 +1135,7 @@ public final class JapaneseTokenizer extends Tokenizer {
}
final int pathCost = posData.costs[bestStartIDX];
for (int forwardArcIDX = 0; forwardArcIDX < posData.forwardCount; forwardArcIDX++) {
- final Type forwardType = posData.forwardType[forwardArcIDX];
+ final TokenType forwardType = posData.forwardType[forwardArcIDX];
final Dictionary<? extends JaMorphData> dict2 = getDict(forwardType);
final int wordID = posData.forwardID[forwardArcIDX];
final int toPos = posData.forwardPos[forwardArcIDX];
@@ -1176,7 +1167,7 @@ public final class JapaneseTokenizer extends Tokenizer {
// On non-initial positions, we maximize score
// across all arriving lastRightIDs:
for (int forwardArcIDX = 0; forwardArcIDX < posData.forwardCount; forwardArcIDX++) {
- final Type forwardType = posData.forwardType[forwardArcIDX];
+ final TokenType forwardType = posData.forwardType[forwardArcIDX];
final int toPos = posData.forwardPos[forwardArcIDX];
if (VERBOSE) {
System.out.println(
@@ -1204,7 +1195,7 @@ public final class JapaneseTokenizer extends Tokenizer {
// yet another lattice data structure
private static final class Lattice {
char[] fragment;
- EnumMap<Type, Dictionary<? extends JaMorphData>> dictionaryMap;
+ EnumMap<TokenType, Dictionary<? extends JaMorphData>> dictionaryMap;
boolean useEOS;
int rootCapacity = 0;
@@ -1220,7 +1211,7 @@ public final class JapaneseTokenizer extends Tokenizer {
int nodeCount = 0;
// The variables below are elements of lattice node that indexed by node number.
- Type[] nodeDicType;
+ TokenType[] nodeDicType;
int[] nodeWordID;
// nodeMark - -1:excluded, 0:unused, 1:bestpath, 2:2-best-path, ... N:N-best-path
int[] nodeMark;
@@ -1257,7 +1248,7 @@ public final class JapaneseTokenizer extends Tokenizer {
private void reserve(int n) {
if (capacity < n) {
int oversize = ArrayUtil.oversize(n, Integer.BYTES);
- nodeDicType = new Type[oversize];
+ nodeDicType = new TokenType[oversize];
nodeWordID = new int[oversize];
nodeMark = new int[oversize];
nodeLeftID = new int[oversize];
@@ -1284,7 +1275,7 @@ public final class JapaneseTokenizer extends Tokenizer {
}
}
- private int addNode(Type dicType, int wordID, int left, int right) {
+ private int addNode(TokenType dicType, int wordID, int left, int right) {
if (VERBOSE) {
System.out.printf(
"DEBUG: addNode: dicType=%s, wordID=%d, left=%d, right=%d, str=%s\n",
@@ -1358,7 +1349,7 @@ public final class JapaneseTokenizer extends Tokenizer {
void setup(
char[] fragment,
- EnumMap<Type, Dictionary<? extends JaMorphData>> dictionaryMap,
+ EnumMap<TokenType, Dictionary<? extends JaMorphData>> dictionaryMap,
WrappedPositionArray positions,
int prevOffset,
int endOffset,
@@ -1385,7 +1376,7 @@ public final class JapaneseTokenizer extends Tokenizer {
}
// EOS = 1
- if (addNode(Type.KNOWN, -1, endOffset - rootBase, -1) != 1) {
+ if (addNode(TokenType.KNOWN, -1, endOffset - rootBase, -1) != 1) {
assert false;
}
@@ -1569,9 +1560,9 @@ public final class JapaneseTokenizer extends Tokenizer {
private void registerNode(int node, char[] fragment) {
int left = lattice.nodeLeft[node];
int right = lattice.nodeRight[node];
- Type type = lattice.nodeDicType[node];
+ TokenType type = lattice.nodeDicType[node];
if (!discardPunctuation || !isPunctuation(fragment[left])) {
- if (type == Type.USER) {
+ if (type == TokenType.USER) {
// The code below are based on backtrace().
//
// Expand the phraseID we recorded into the actual segmentation:
@@ -1585,7 +1576,7 @@ public final class JapaneseTokenizer extends Tokenizer {
lattice.rootBase + left,
lattice.rootBase + right,
wordID,
- Type.USER,
+ TokenType.USER,
userDictionary.getMorphAttributes()));
// Output compound
int current = 0;
@@ -1601,7 +1592,7 @@ public final class JapaneseTokenizer extends Tokenizer {
startOffset,
startOffset + len,
wordID + j - 1,
- Type.USER,
+ TokenType.USER,
userDictionary.getMorphAttributes()));
}
current += len;
@@ -1847,11 +1838,11 @@ public final class JapaneseTokenizer extends Tokenizer {
assert backPos >= lastBackTracePos
: "backPos=" + backPos + " vs lastBackTracePos=" + lastBackTracePos;
int length = pos - backPos;
- Type backType = posData.backType[bestIDX];
+ TokenType backType = posData.backType[bestIDX];
int backID = posData.backID[bestIDX];
int nextBestIDX = posData.backIndex[bestIDX];
- if (searchMode && altToken == null && backType != Type.USER) {
+ if (searchMode && altToken == null && backType != TokenType.USER) {
// In searchMode, if best path had picked a too-long
// token, we use the "penalty" to compute the allowed
@@ -2008,7 +1999,7 @@ public final class JapaneseTokenizer extends Tokenizer {
final Dictionary<? extends JaMorphData> dict = getDict(backType);
- if (backType == Type.USER) {
+ if (backType == TokenType.USER) {
// Expand the phraseID we recorded into the actual
// segmentation:
@@ -2027,7 +2018,7 @@ public final class JapaneseTokenizer extends Tokenizer {
startOffset,
startOffset + len,
wordID + j - 1,
- Type.USER,
+ TokenType.USER,
dict.getMorphAttributes()));
if (VERBOSE) {
System.out.println(" add USER token=" + pending.get(pending.size() - 1));
@@ -2044,7 +2035,7 @@ public final class JapaneseTokenizer extends Tokenizer {
backCount += wordIDAndLength.length - 1;
} else {
- if (extendedMode && backType == Type.UNKNOWN) {
+ if (extendedMode && backType == TokenType.UNKNOWN) {
// In EXTENDED mode we convert unknown word into
// unigrams:
int unigramTokenCount = 0;
@@ -2066,7 +2057,7 @@ public final class JapaneseTokenizer extends Tokenizer {
startOffset,
startOffset + charLen,
CharacterDefinition.NGRAM,
- Type.UNKNOWN,
+ TokenType.UNKNOWN,
unkDictionary.getMorphAttributes()));
unigramTokenCount++;
}
@@ -2112,7 +2103,7 @@ public final class JapaneseTokenizer extends Tokenizer {
positions.freeBefore(endPos);
}
- Dictionary<? extends JaMorphData> getDict(Type type) {
+ Dictionary<? extends JaMorphData> getDict(TokenType type) {
return dictionaryMap.get(type);
}
diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/Token.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/Token.java
index c393008fef1..2de9e018329 100644
--- a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/Token.java
+++ b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/Token.java
@@ -16,8 +16,8 @@
*/
package org.apache.lucene.analysis.ja;
-import org.apache.lucene.analysis.ja.JapaneseTokenizer.Type;
import org.apache.lucene.analysis.ja.dict.JaMorphData;
+import org.apache.lucene.analysis.morph.TokenType;
/** Analyzed token with morphological data from its dictionary. */
public class Token extends org.apache.lucene.analysis.morph.Token {
@@ -25,8 +25,6 @@ public class Token extends org.apache.lucene.analysis.morph.Token {
private final int morphId;
- private final Type type;
-
public Token(
char[] surfaceForm,
int offset,
@@ -34,11 +32,10 @@ public class Token extends org.apache.lucene.analysis.morph.Token {
int startOffset,
int endOffset,
int morphId,
- Type type,
+ TokenType type,
JaMorphData morphData) {
- super(surfaceForm, offset, length, startOffset, endOffset);
+ super(surfaceForm, offset, length, startOffset, endOffset, type);
this.morphId = morphId;
- this.type = type;
this.morphData = morphData;
}
@@ -91,22 +88,13 @@ public class Token extends org.apache.lucene.analysis.morph.Token {
return morphData.getBaseForm(morphId, surfaceForm, offset, length);
}
- /**
- * Returns the type of this token
- *
- * @return token type, not null
- */
- public Type getType() {
- return type;
- }
-
/**
* Returns true if this token is known word
*
* @return true if this token is in standard dictionary. false if not.
*/
public boolean isKnown() {
- return type == Type.KNOWN;
+ return type == TokenType.KNOWN;
}
/**
@@ -115,7 +103,7 @@ public class Token extends org.apache.lucene.analysis.morph.Token {
* @return true if this token is unknown word. false if not.
*/
public boolean isUnknown() {
- return type == Type.UNKNOWN;
+ return type == TokenType.UNKNOWN;
}
/**
@@ -124,6 +112,6 @@ public class Token extends org.apache.lucene.analysis.morph.Token {
* @return true if this token is in user dictionary. false if not.
*/
public boolean isUser() {
- return type == Type.USER;
+ return type == TokenType.USER;
}
}
diff --git a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/DecompoundToken.java b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/DecompoundToken.java
index a6af55f482e..bdd30ee2812 100644
--- a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/DecompoundToken.java
+++ b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/DecompoundToken.java
@@ -17,6 +17,7 @@
package org.apache.lucene.analysis.ko;
import org.apache.lucene.analysis.ko.dict.KoMorphData;
+import org.apache.lucene.analysis.morph.TokenType;
/** A token that was generated from a compound. */
public class DecompoundToken extends Token {
@@ -29,9 +30,11 @@ public class DecompoundToken extends Token {
* @param surfaceForm The surface form of the token.
* @param startOffset The start offset of the token in the analyzed text.
* @param endOffset The end offset of the token in the analyzed text.
+ * @param type The type of this token.
*/
- public DecompoundToken(POS.Tag posTag, String surfaceForm, int startOffset, int endOffset) {
- super(surfaceForm.toCharArray(), 0, surfaceForm.length(), startOffset, endOffset);
+ public DecompoundToken(
+ POS.Tag posTag, String surfaceForm, int startOffset, int endOffset, TokenType type) {
+ super(surfaceForm.toCharArray(), 0, surfaceForm.length(), startOffset, endOffset, type);
this.posTag = posTag;
}
diff --git a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/DictionaryToken.java b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/DictionaryToken.java
index f548f8007d3..3d04515f2f4 100644
--- a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/DictionaryToken.java
+++ b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/DictionaryToken.java
@@ -17,15 +17,15 @@
package org.apache.lucene.analysis.ko;
import org.apache.lucene.analysis.ko.dict.KoMorphData;
+import org.apache.lucene.analysis.morph.TokenType;
/** A token stored in a {@link KoMorphData}. */
public class DictionaryToken extends Token {
private final int wordId;
- private final KoreanTokenizer.Type type;
private final KoMorphData morphAtts;
public DictionaryToken(
- KoreanTokenizer.Type type,
+ TokenType type,
KoMorphData morphAtts,
int wordId,
char[] surfaceForm,
@@ -33,8 +33,7 @@ public class DictionaryToken extends Token {
int length,
int startOffset,
int endOffset) {
- super(surfaceForm, offset, length, startOffset, endOffset);
- this.type = type;
+ super(surfaceForm, offset, length, startOffset, endOffset, type);
this.morphAtts = morphAtts;
this.wordId = wordId;
}
@@ -58,22 +57,13 @@ public class DictionaryToken extends Token {
+ ")";
}
- /**
- * Returns the type of this token
- *
- * @return token type, not null
- */
- public KoreanTokenizer.Type getType() {
- return type;
- }
-
/**
* Returns true if this token is known word
*
* @return true if this token is in standard dictionary. false if not.
*/
public boolean isKnown() {
- return type == KoreanTokenizer.Type.KNOWN;
+ return type == TokenType.KNOWN;
}
/**
@@ -82,7 +72,7 @@ public class DictionaryToken extends Token {
* @return true if this token is unknown word. false if not.
*/
public boolean isUnknown() {
- return type == KoreanTokenizer.Type.UNKNOWN;
+ return type == TokenType.UNKNOWN;
}
/**
@@ -91,7 +81,7 @@ public class DictionaryToken extends Token {
* @return true if this token is in user dictionary. false if not.
*/
public boolean isUser() {
- return type == KoreanTokenizer.Type.USER;
+ return type == TokenType.USER;
}
@Override
diff --git a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/KoreanTokenizer.java b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/KoreanTokenizer.java
index ae64076ce43..76a95047e1d 100644
--- a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/KoreanTokenizer.java
+++ b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/KoreanTokenizer.java
@@ -35,6 +35,7 @@ import org.apache.lucene.analysis.ko.dict.UserDictionary;
import org.apache.lucene.analysis.ko.tokenattributes.PartOfSpeechAttribute;
import org.apache.lucene.analysis.ko.tokenattributes.ReadingAttribute;
import org.apache.lucene.analysis.morph.Dictionary;
+import org.apache.lucene.analysis.morph.TokenType;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
@@ -65,16 +66,6 @@ import org.apache.lucene.util.fst.FST;
@IgnoreRandomChains(reason = "LUCENE-10359: fails with incorrect offsets")
public final class KoreanTokenizer extends Tokenizer {
- /** Token type reflecting the original source of this token */
- public enum Type {
- /** Known words from the system dictionary. */
- KNOWN,
- /** Unknown words (heuristically segmented). */
- UNKNOWN,
- /** Known words from the user dictionary. */
- USER
- }
-
/**
* Decompound mode: this determines how the tokenizer handles {@link POS.Type#COMPOUND}, {@link
* POS.Type#INFLECT} and {@link POS.Type#PREANALYSIS} tokens.
@@ -99,8 +90,8 @@ public final class KoreanTokenizer extends Tokenizer {
private static final int MAX_UNKNOWN_WORD_LENGTH = 1024;
private static final int MAX_BACKTRACE_GAP = 1024;
- private final EnumMap<Type, Dictionary<? extends KoMorphData>> dictionaryMap =
- new EnumMap<>(Type.class);
+ private final EnumMap<TokenType, Dictionary<? extends KoMorphData>> dictionaryMap =
+ new EnumMap<>(TokenType.class);
private final TokenInfoFST fst;
private final TokenInfoDictionary dictionary;
@@ -242,9 +233,9 @@ public final class KoreanTokenizer extends Tokenizer {
resetState();
- dictionaryMap.put(Type.KNOWN, dictionary);
- dictionaryMap.put(Type.UNKNOWN, unkDictionary);
- dictionaryMap.put(Type.USER, userDictionary);
+ dictionaryMap.put(TokenType.KNOWN, dictionary);
+ dictionaryMap.put(TokenType.UNKNOWN, unkDictionary);
+ dictionaryMap.put(TokenType.USER, userDictionary);
}
private GraphvizFormatter dotOut;
@@ -275,7 +266,7 @@ public final class KoreanTokenizer extends Tokenizer {
pending.clear();
// Add BOS:
- positions.get(0).add(0, 0, -1, -1, -1, -1, Type.KNOWN);
+ positions.get(0).add(0, 0, -1, -1, -1, -1, TokenType.KNOWN);
}
@Override
@@ -300,7 +291,7 @@ public final class KoreanTokenizer extends Tokenizer {
int[] backWordPos = new int[8];
int[] backIndex = new int[8];
int[] backID = new int[8];
- Type[] backType = new Type[8];
+ TokenType[] backType = new TokenType[8];
public void grow() {
costs = ArrayUtil.grow(costs, 1 + count);
@@ -313,7 +304,7 @@ public final class KoreanTokenizer extends Tokenizer {
// NOTE: sneaky: grow separately because
// ArrayUtil.grow will otherwise pick a different
// length than the int[]s we just grew:
- final Type[] newBackType = new Type[backID.length];
+ final TokenType[] newBackType = new TokenType[backID.length];
System.arraycopy(backType, 0, newBackType, 0, backType.length);
backType = newBackType;
}
@@ -325,7 +316,7 @@ public final class KoreanTokenizer extends Tokenizer {
int backRPos,
int backIndex,
int backID,
- Type backType) {
+ TokenType backType) {
// NOTE: this isn't quite a true Viterbi search,
// because we should check if lastRightID is
// already present here, and only update if the new
@@ -409,7 +400,12 @@ public final class KoreanTokenizer extends Tokenizer {
}
private void add(
- KoMorphData morphAtts, Position fromPosData, int wordPos, int endPos, int wordID, Type type) {
+ KoMorphData morphAtts,
+ Position fromPosData,
+ int wordPos,
+ int endPos,
+ int wordID,
+ TokenType type) {
final POS.Tag leftPOS = morphAtts.getLeftPOS(wordID);
final int wordCost = morphAtts.getWordCost(wordID);
final int leftID = morphAtts.getLeftId(wordID);
@@ -811,7 +807,7 @@ public final class KoreanTokenizer extends Tokenizer {
pos,
maxPosAhead + 1,
outputMaxPosAhead + arcFinalOutMaxPosAhead,
- Type.USER);
+ TokenType.USER);
userWordMaxPosAhead = Math.max(userWordMaxPosAhead, maxPosAhead);
}
}
@@ -863,7 +859,7 @@ public final class KoreanTokenizer extends Tokenizer {
pos,
posAhead + 1,
wordIdRef.ints[wordIdRef.offset + ofs],
- Type.KNOWN);
+ TokenType.KNOWN);
anyMatches = true;
}
}
@@ -937,7 +933,7 @@ public final class KoreanTokenizer extends Tokenizer {
pos,
pos + unknownWordLength,
wordIdRef.ints[wordIdRef.offset + ofs],
- Type.UNKNOWN);
+ TokenType.UNKNOWN);
}
}
@@ -1019,7 +1015,7 @@ public final class KoreanTokenizer extends Tokenizer {
: "backPos=" + backPos + " vs lastBackTracePos=" + lastBackTracePos;
// the length of the word without the whitespaces at the beginning.
int length = pos - backWordPos;
- Type backType = posData.backType[bestIDX];
+ TokenType backType = posData.backType[bestIDX];
int backID = posData.backID[bestIDX];
int nextBestIDX = posData.backIndex[bestIDX];
// the start of the word after the whitespace at the beginning.
@@ -1028,7 +1024,7 @@ public final class KoreanTokenizer extends Tokenizer {
final Dictionary<? extends KoMorphData> dict = getDict(backType);
- if (outputUnknownUnigrams && backType == Type.UNKNOWN) {
+ if (outputUnknownUnigrams && backType == TokenType.UNKNOWN) {
// outputUnknownUnigrams converts unknown word into unigrams:
for (int i = length - 1; i >= 0; i--) {
int charLen = 1;
@@ -1038,7 +1034,7 @@ public final class KoreanTokenizer extends Tokenizer {
}
final DictionaryToken token =
new DictionaryToken(
- Type.UNKNOWN,
+ TokenType.UNKNOWN,
unkDictionary.getMorphAttributes(),
CharacterDefinition.NGRAM,
fragment,
@@ -1090,14 +1086,16 @@ public final class KoreanTokenizer extends Tokenizer {
morpheme.posTag,
morpheme.surfaceForm,
endOffset - morpheme.surfaceForm.length(),
- endOffset);
+ endOffset,
+ backType);
} else {
compoundToken =
new DecompoundToken(
morpheme.posTag,
morpheme.surfaceForm,
token.getStartOffset(),
- token.getEndOffset());
+ token.getEndOffset(),
+ backType);
}
if (i == 0 && mode == DecompoundMode.MIXED) {
compoundToken.setPositionIncrement(0);
@@ -1128,7 +1126,7 @@ public final class KoreanTokenizer extends Tokenizer {
unkDictionary.lookupWordIds(characterDefinition.getCharacterClass(' '), wordIdRef);
DictionaryToken spaceToken =
new DictionaryToken(
- Type.UNKNOWN,
+ TokenType.UNKNOWN,
unkDictionary.getMorphAttributes(),
wordIdRef.ints[wordIdRef.offset],
fragment,
@@ -1154,7 +1152,7 @@ public final class KoreanTokenizer extends Tokenizer {
positions.freeBefore(endPos);
}
- Dictionary<? extends KoMorphData> getDict(Type type) {
+ Dictionary<? extends KoMorphData> getDict(TokenType type) {
return dictionaryMap.get(type);
}
diff --git a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/Token.java b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/Token.java
index 73b6ddab171..480d4abf410 100644
--- a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/Token.java
+++ b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/Token.java
@@ -17,12 +17,14 @@
package org.apache.lucene.analysis.ko;
import org.apache.lucene.analysis.ko.dict.KoMorphData;
+import org.apache.lucene.analysis.morph.TokenType;
/** Analyzed token with morphological data. */
public abstract class Token extends org.apache.lucene.analysis.morph.Token {
- protected Token(char[] surfaceForm, int offset, int length, int startOffset, int endOffset) {
- super(surfaceForm, offset, length, startOffset, endOffset);
+ protected Token(
+ char[] surfaceForm, int offset, int length, int startOffset, int endOffset, TokenType type) {
+ super(surfaceForm, offset, length, startOffset, endOffset, type);
}
/** Get the {@link POS.Type} of the token. */