Apache Lucene open-source search software
LUCENE-10493: Unify token Type enum in kuromoji and nori (#801)
Tomoko Uchida13630d361e28
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/Token.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/Token.javaindex f9f0468d473..7050e4e87e2 100644--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/Token.java+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/Token.java@@ -27,12 +27,16 @@ public abstract class Token {protected int posIncr = 1;protected int posLen = 1;- protected Token(char[] surfaceForm, int offset, int length, int startOffset, int endOffset) {+ protected final TokenType type;++ protected Token(+ char[] surfaceForm, int offset, int length, int startOffset, int endOffset, TokenType type) {this.surfaceForm = surfaceForm;this.offset = offset;this.length = length;this.startOffset = startOffset;this.endOffset = endOffset;+ this.type = type;}/** @return surfaceForm */@@ -88,4 +92,9 @@ public abstract class Token {public int getPositionLength() {return posLen;}++ /** Returns the type of this token */+ public TokenType getType() {+ return type;+ }}diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/TokenType.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/TokenType.javanew file mode 100644index 00000000000..01084af16ce--- /dev/null+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/TokenType.java@@ -0,0 +1,28 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++package org.apache.lucene.analysis.morph;++/** Token type reflecting the original source of this token */+public enum TokenType {+ /** Known words from the system dictionary. */+ KNOWN,+ /** Unknown words (heuristically segmented). */+ UNKNOWN,+ /** Known words from the user dictionary. */+ USER+}diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizer.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizer.javaindex 84464d76730..b06b19fc359 100644--- a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizer.java+++ b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizer.java@@ -35,6 +35,7 @@ import org.apache.lucene.analysis.ja.dict.UnknownDictionary;import org.apache.lucene.analysis.ja.dict.UserDictionary;import org.apache.lucene.analysis.ja.tokenattributes.*;import org.apache.lucene.analysis.morph.Dictionary;+import org.apache.lucene.analysis.morph.TokenType;import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;@@ -93,16 +94,6 @@ public final class JapaneseTokenizer extends Tokenizer {/** Default tokenization mode. Currently this is {@link Mode#SEARCH}. */public static final Mode DEFAULT_MODE = Mode.SEARCH;- /** Token type reflecting the original source of this token */- public enum Type {- /** Known words from the system dictionary. */- KNOWN,- /** Unknown words (heuristically segmented). */- UNKNOWN,- /** Known words from the user dictionary. */- USER- }-private static final boolean VERBOSE = false;private static final int SEARCH_MODE_KANJI_LENGTH = 2;@@ -117,8 +108,8 @@ public final class JapaneseTokenizer extends Tokenizer {private static final int MAX_UNKNOWN_WORD_LENGTH = 1024;private static final int MAX_BACKTRACE_GAP = 1024;- private final EnumMap<Type, Dictionary<? extends JaMorphData>> dictionaryMap =- new EnumMap<>(Type.class);+ private final EnumMap<TokenType, Dictionary<? extends JaMorphData>> dictionaryMap =+ new EnumMap<>(TokenType.class);private final TokenInfoFST fst;private final TokenInfoDictionary dictionary;@@ -326,9 +317,9 @@ public final class JapaneseTokenizer extends Tokenizer {resetState();- dictionaryMap.put(Type.KNOWN, dictionary);- dictionaryMap.put(Type.UNKNOWN, unkDictionary);- dictionaryMap.put(Type.USER, userDictionary);+ dictionaryMap.put(TokenType.KNOWN, dictionary);+ dictionaryMap.put(TokenType.UNKNOWN, unkDictionary);+ dictionaryMap.put(TokenType.USER, userDictionary);}private GraphvizFormatter dotOut;@@ -360,7 +351,7 @@ public final class JapaneseTokenizer extends Tokenizer {pending.clear();// Add BOS:- positions.get(0).add(0, 0, -1, -1, -1, -1, Type.KNOWN);+ positions.get(0).add(0, 0, -1, -1, -1, -1, TokenType.KNOWN);}@Override@@ -418,7 +409,7 @@ public final class JapaneseTokenizer extends Tokenizer {int[] backWordPos = new int[8];int[] backIndex = new int[8];int[] backID = new int[8];- Type[] backType = new Type[8];+ TokenType[] backType = new TokenType[8];// Only used when finding 2nd best segmentation under a// too-long token:@@ -426,7 +417,7 @@ public final class JapaneseTokenizer extends Tokenizer {int[] forwardPos = new int[8];int[] forwardID = new int[8];int[] forwardIndex = new int[8];- Type[] forwardType = new Type[8];+ TokenType[] forwardType = new TokenType[8];public void grow() {costs = ArrayUtil.grow(costs, 1 + count);@@ -439,7 +430,7 @@ public final class JapaneseTokenizer extends Tokenizer {// NOTE: sneaky: grow separately because// ArrayUtil.grow will otherwise pick a different// length than the int[]s we just grew:- final Type[] newBackType = new Type[backID.length];+ final TokenType[] newBackType = new TokenType[backID.length];System.arraycopy(backType, 0, newBackType, 0, backType.length);backType = newBackType;}@@ -452,7 +443,7 @@ public final class JapaneseTokenizer extends Tokenizer {// NOTE: sneaky: grow separately because// ArrayUtil.grow will otherwise pick a different// length than the int[]s we just grew:- final Type[] newForwardType = new Type[forwardPos.length];+ final TokenType[] newForwardType = new TokenType[forwardPos.length];System.arraycopy(forwardType, 0, newForwardType, 0, forwardType.length);forwardType = newForwardType;}@@ -464,7 +455,7 @@ public final class JapaneseTokenizer extends Tokenizer {int backRPos,int backIndex,int backID,- Type backType) {+ TokenType backType) {// NOTE: this isn't quite a true Viterbi search,// because we should check if lastRightID is// already present here, and only update if the new@@ -486,7 +477,7 @@ public final class JapaneseTokenizer extends Tokenizer {count++;}- public void addForward(int forwardPos, int forwardIndex, int forwardID, Type forwardType) {+ public void addForward(int forwardPos, int forwardIndex, int forwardID, TokenType forwardType) {if (forwardCount == this.forwardID.length) {growForward();}@@ -510,7 +501,7 @@ public final class JapaneseTokenizer extends Tokenizer {int wordPos,int endPos,int wordID,- Type type,+ TokenType type,boolean addPenalty)throws IOException {final int wordCost = morphAtts.getWordCost(wordID);@@ -565,7 +556,7 @@ public final class JapaneseTokenizer extends Tokenizer {+ positions.get(endPos).count);}- if (addPenalty && type != Type.USER) {+ if (addPenalty && type != TokenType.USER) {final int penalty = computePenalty(fromPosData.pos, endPos - fromPosData.pos);if (VERBOSE) {if (penalty > 0) {@@ -916,7 +907,7 @@ public final class JapaneseTokenizer extends Tokenizer {pos,posAhead + 1,output + arc.nextFinalOutput().intValue(),- Type.USER,+ TokenType.USER,false);anyMatches = true;}@@ -970,7 +961,7 @@ public final class JapaneseTokenizer extends Tokenizer {pos,posAhead + 1,wordIdRef.ints[wordIdRef.offset + ofs],- Type.KNOWN,+ TokenType.KNOWN,false);anyMatches = true;}@@ -1027,7 +1018,7 @@ public final class JapaneseTokenizer extends Tokenizer {pos,posData.pos + unknownWordLength,wordIdRef.ints[wordIdRef.offset + ofs],- Type.UNKNOWN,+ TokenType.UNKNOWN,false);}@@ -1144,7 +1135,7 @@ public final class JapaneseTokenizer extends Tokenizer {}final int pathCost = posData.costs[bestStartIDX];for (int forwardArcIDX = 0; forwardArcIDX < posData.forwardCount; forwardArcIDX++) {- final Type forwardType = posData.forwardType[forwardArcIDX];+ final TokenType forwardType = posData.forwardType[forwardArcIDX];final Dictionary<? extends JaMorphData> dict2 = getDict(forwardType);final int wordID = posData.forwardID[forwardArcIDX];final int toPos = posData.forwardPos[forwardArcIDX];@@ -1176,7 +1167,7 @@ public final class JapaneseTokenizer extends Tokenizer {// On non-initial positions, we maximize score// across all arriving lastRightIDs:for (int forwardArcIDX = 0; forwardArcIDX < posData.forwardCount; forwardArcIDX++) {- final Type forwardType = posData.forwardType[forwardArcIDX];+ final TokenType forwardType = posData.forwardType[forwardArcIDX];final int toPos = posData.forwardPos[forwardArcIDX];if (VERBOSE) {System.out.println(@@ -1204,7 +1195,7 @@ public final class JapaneseTokenizer extends Tokenizer {// yet another lattice data structureprivate static final class Lattice {char[] fragment;- EnumMap<Type, Dictionary<? extends JaMorphData>> dictionaryMap;+ EnumMap<TokenType, Dictionary<? extends JaMorphData>> dictionaryMap;boolean useEOS;int rootCapacity = 0;@@ -1220,7 +1211,7 @@ public final class JapaneseTokenizer extends Tokenizer {int nodeCount = 0;// The variables below are elements of lattice node that indexed by node number.- Type[] nodeDicType;+ TokenType[] nodeDicType;int[] nodeWordID;// nodeMark - -1:excluded, 0:unused, 1:bestpath, 2:2-best-path, ... N:N-best-pathint[] nodeMark;@@ -1257,7 +1248,7 @@ public final class JapaneseTokenizer extends Tokenizer {private void reserve(int n) {if (capacity < n) {int oversize = ArrayUtil.oversize(n, Integer.BYTES);- nodeDicType = new Type[oversize];+ nodeDicType = new TokenType[oversize];nodeWordID = new int[oversize];nodeMark = new int[oversize];nodeLeftID = new int[oversize];@@ -1284,7 +1275,7 @@ public final class JapaneseTokenizer extends Tokenizer {}}- private int addNode(Type dicType, int wordID, int left, int right) {+ private int addNode(TokenType dicType, int wordID, int left, int right) {if (VERBOSE) {System.out.printf("DEBUG: addNode: dicType=%s, wordID=%d, left=%d, right=%d, str=%s\n",@@ -1358,7 +1349,7 @@ public final class JapaneseTokenizer extends Tokenizer {void setup(char[] fragment,- EnumMap<Type, Dictionary<? extends JaMorphData>> dictionaryMap,+ EnumMap<TokenType, Dictionary<? extends JaMorphData>> dictionaryMap,WrappedPositionArray positions,int prevOffset,int endOffset,@@ -1385,7 +1376,7 @@ public final class JapaneseTokenizer extends Tokenizer {}// EOS = 1- if (addNode(Type.KNOWN, -1, endOffset - rootBase, -1) != 1) {+ if (addNode(TokenType.KNOWN, -1, endOffset - rootBase, -1) != 1) {assert false;}@@ -1569,9 +1560,9 @@ public final class JapaneseTokenizer extends Tokenizer {private void registerNode(int node, char[] fragment) {int left = lattice.nodeLeft[node];int right = lattice.nodeRight[node];- Type type = lattice.nodeDicType[node];+ TokenType type = lattice.nodeDicType[node];if (!discardPunctuation || !isPunctuation(fragment[left])) {- if (type == Type.USER) {+ if (type == TokenType.USER) {// The code below are based on backtrace().//// Expand the phraseID we recorded into the actual segmentation:@@ -1585,7 +1576,7 @@ public final class JapaneseTokenizer extends Tokenizer {lattice.rootBase + left,lattice.rootBase + right,wordID,- Type.USER,+ TokenType.USER,userDictionary.getMorphAttributes()));// Output compoundint current = 0;@@ -1601,7 +1592,7 @@ public final class JapaneseTokenizer extends Tokenizer {startOffset,startOffset + len,wordID + j - 1,- Type.USER,+ TokenType.USER,userDictionary.getMorphAttributes()));}current += len;@@ -1847,11 +1838,11 @@ public final class JapaneseTokenizer extends Tokenizer {assert backPos >= lastBackTracePos: "backPos=" + backPos + " vs lastBackTracePos=" + lastBackTracePos;int length = pos - backPos;- Type backType = posData.backType[bestIDX];+ TokenType backType = posData.backType[bestIDX];int backID = posData.backID[bestIDX];int nextBestIDX = posData.backIndex[bestIDX];- if (searchMode && altToken == null && backType != Type.USER) {+ if (searchMode && altToken == null && backType != TokenType.USER) {// In searchMode, if best path had picked a too-long// token, we use the "penalty" to compute the allowed@@ -2008,7 +1999,7 @@ public final class JapaneseTokenizer extends Tokenizer {final Dictionary<? extends JaMorphData> dict = getDict(backType);- if (backType == Type.USER) {+ if (backType == TokenType.USER) {// Expand the phraseID we recorded into the actual// segmentation:@@ -2027,7 +2018,7 @@ public final class JapaneseTokenizer extends Tokenizer {startOffset,startOffset + len,wordID + j - 1,- Type.USER,+ TokenType.USER,dict.getMorphAttributes()));if (VERBOSE) {System.out.println(" add USER token=" + pending.get(pending.size() - 1));@@ -2044,7 +2035,7 @@ public final class JapaneseTokenizer extends Tokenizer {backCount += wordIDAndLength.length - 1;} else {- if (extendedMode && backType == Type.UNKNOWN) {+ if (extendedMode && backType == TokenType.UNKNOWN) {// In EXTENDED mode we convert unknown word into// unigrams:int unigramTokenCount = 0;@@ -2066,7 +2057,7 @@ public final class JapaneseTokenizer extends Tokenizer {startOffset,startOffset + charLen,CharacterDefinition.NGRAM,- Type.UNKNOWN,+ TokenType.UNKNOWN,unkDictionary.getMorphAttributes()));unigramTokenCount++;}@@ -2112,7 +2103,7 @@ public final class JapaneseTokenizer extends Tokenizer {positions.freeBefore(endPos);}- Dictionary<? extends JaMorphData> getDict(Type type) {+ Dictionary<? extends JaMorphData> getDict(TokenType type) {return dictionaryMap.get(type);}diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/Token.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/Token.javaindex c393008fef1..2de9e018329 100644--- a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/Token.java+++ b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/Token.java@@ -16,8 +16,8 @@*/package org.apache.lucene.analysis.ja;-import org.apache.lucene.analysis.ja.JapaneseTokenizer.Type;import org.apache.lucene.analysis.ja.dict.JaMorphData;+import org.apache.lucene.analysis.morph.TokenType;/** Analyzed token with morphological data from its dictionary. */public class Token extends org.apache.lucene.analysis.morph.Token {@@ -25,8 +25,6 @@ public class Token extends org.apache.lucene.analysis.morph.Token {private final int morphId;- private final Type type;-public Token(char[] surfaceForm,int offset,@@ -34,11 +32,10 @@ public class Token extends org.apache.lucene.analysis.morph.Token {int startOffset,int endOffset,int morphId,- Type type,+ TokenType type,JaMorphData morphData) {- super(surfaceForm, offset, length, startOffset, endOffset);+ super(surfaceForm, offset, length, startOffset, endOffset, type);this.morphId = morphId;- this.type = type;this.morphData = morphData;}@@ -91,22 +88,13 @@ public class Token extends org.apache.lucene.analysis.morph.Token {return morphData.getBaseForm(morphId, surfaceForm, offset, length);}- /**- * Returns the type of this token- *- * @return token type, not null- */- public Type getType() {- return type;- }-/*** Returns true if this token is known word** @return true if this token is in standard dictionary. false if not.*/public boolean isKnown() {- return type == Type.KNOWN;+ return type == TokenType.KNOWN;}/**@@ -115,7 +103,7 @@ public class Token extends org.apache.lucene.analysis.morph.Token {* @return true if this token is unknown word. false if not.*/public boolean isUnknown() {- return type == Type.UNKNOWN;+ return type == TokenType.UNKNOWN;}/**@@ -124,6 +112,6 @@ public class Token extends org.apache.lucene.analysis.morph.Token {* @return true if this token is in user dictionary. false if not.*/public boolean isUser() {- return type == Type.USER;+ return type == TokenType.USER;}}diff --git a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/DecompoundToken.java b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/DecompoundToken.javaindex a6af55f482e..bdd30ee2812 100644--- a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/DecompoundToken.java+++ b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/DecompoundToken.java@@ -17,6 +17,7 @@package org.apache.lucene.analysis.ko;import org.apache.lucene.analysis.ko.dict.KoMorphData;+import org.apache.lucene.analysis.morph.TokenType;/** A token that was generated from a compound. */public class DecompoundToken extends Token {@@ -29,9 +30,11 @@ public class DecompoundToken extends Token {* @param surfaceForm The surface form of the token.* @param startOffset The start offset of the token in the analyzed text.* @param endOffset The end offset of the token in the analyzed text.+ * @param type The type of this token.*/- public DecompoundToken(POS.Tag posTag, String surfaceForm, int startOffset, int endOffset) {- super(surfaceForm.toCharArray(), 0, surfaceForm.length(), startOffset, endOffset);+ public DecompoundToken(+ POS.Tag posTag, String surfaceForm, int startOffset, int endOffset, TokenType type) {+ super(surfaceForm.toCharArray(), 0, surfaceForm.length(), startOffset, endOffset, type);this.posTag = posTag;}diff --git a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/DictionaryToken.java b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/DictionaryToken.javaindex f548f8007d3..3d04515f2f4 100644--- a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/DictionaryToken.java+++ b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/DictionaryToken.java@@ -17,15 +17,15 @@package org.apache.lucene.analysis.ko;import org.apache.lucene.analysis.ko.dict.KoMorphData;+import org.apache.lucene.analysis.morph.TokenType;/** A token stored in a {@link KoMorphData}. */public class DictionaryToken extends Token {private final int wordId;- private final KoreanTokenizer.Type type;private final KoMorphData morphAtts;public DictionaryToken(- KoreanTokenizer.Type type,+ TokenType type,KoMorphData morphAtts,int wordId,char[] surfaceForm,@@ -33,8 +33,7 @@ public class DictionaryToken extends Token {int length,int startOffset,int endOffset) {- super(surfaceForm, offset, length, startOffset, endOffset);- this.type = type;+ super(surfaceForm, offset, length, startOffset, endOffset, type);this.morphAtts = morphAtts;this.wordId = wordId;}@@ -58,22 +57,13 @@ public class DictionaryToken extends Token {+ ")";}- /**- * Returns the type of this token- *- * @return token type, not null- */- public KoreanTokenizer.Type getType() {- return type;- }-/*** Returns true if this token is known word** @return true if this token is in standard dictionary. false if not.*/public boolean isKnown() {- return type == KoreanTokenizer.Type.KNOWN;+ return type == TokenType.KNOWN;}/**@@ -82,7 +72,7 @@ public class DictionaryToken extends Token {* @return true if this token is unknown word. false if not.*/public boolean isUnknown() {- return type == KoreanTokenizer.Type.UNKNOWN;+ return type == TokenType.UNKNOWN;}/**@@ -91,7 +81,7 @@ public class DictionaryToken extends Token {* @return true if this token is in user dictionary. false if not.*/public boolean isUser() {- return type == KoreanTokenizer.Type.USER;+ return type == TokenType.USER;}@Overridediff --git a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/KoreanTokenizer.java b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/KoreanTokenizer.javaindex ae64076ce43..76a95047e1d 100644--- a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/KoreanTokenizer.java+++ b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/KoreanTokenizer.java@@ -35,6 +35,7 @@ import org.apache.lucene.analysis.ko.dict.UserDictionary;import org.apache.lucene.analysis.ko.tokenattributes.PartOfSpeechAttribute;import org.apache.lucene.analysis.ko.tokenattributes.ReadingAttribute;import org.apache.lucene.analysis.morph.Dictionary;+import org.apache.lucene.analysis.morph.TokenType;import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;@@ -65,16 +66,6 @@ import org.apache.lucene.util.fst.FST;@IgnoreRandomChains(reason = "LUCENE-10359: fails with incorrect offsets")public final class KoreanTokenizer extends Tokenizer {- /** Token type reflecting the original source of this token */- public enum Type {- /** Known words from the system dictionary. */- KNOWN,- /** Unknown words (heuristically segmented). */- UNKNOWN,- /** Known words from the user dictionary. */- USER- }-/*** Decompound mode: this determines how the tokenizer handles {@link POS.Type#COMPOUND}, {@link* POS.Type#INFLECT} and {@link POS.Type#PREANALYSIS} tokens.@@ -99,8 +90,8 @@ public final class KoreanTokenizer extends Tokenizer {private static final int MAX_UNKNOWN_WORD_LENGTH = 1024;private static final int MAX_BACKTRACE_GAP = 1024;- private final EnumMap<Type, Dictionary<? extends KoMorphData>> dictionaryMap =- new EnumMap<>(Type.class);+ private final EnumMap<TokenType, Dictionary<? extends KoMorphData>> dictionaryMap =+ new EnumMap<>(TokenType.class);private final TokenInfoFST fst;private final TokenInfoDictionary dictionary;@@ -242,9 +233,9 @@ public final class KoreanTokenizer extends Tokenizer {resetState();- dictionaryMap.put(Type.KNOWN, dictionary);- dictionaryMap.put(Type.UNKNOWN, unkDictionary);- dictionaryMap.put(Type.USER, userDictionary);+ dictionaryMap.put(TokenType.KNOWN, dictionary);+ dictionaryMap.put(TokenType.UNKNOWN, unkDictionary);+ dictionaryMap.put(TokenType.USER, userDictionary);}private GraphvizFormatter dotOut;@@ -275,7 +266,7 @@ public final class KoreanTokenizer extends Tokenizer {pending.clear();// Add BOS:- positions.get(0).add(0, 0, -1, -1, -1, -1, Type.KNOWN);+ positions.get(0).add(0, 0, -1, -1, -1, -1, TokenType.KNOWN);}@Override@@ -300,7 +291,7 @@ public final class KoreanTokenizer extends Tokenizer {int[] backWordPos = new int[8];int[] backIndex = new int[8];int[] backID = new int[8];- Type[] backType = new Type[8];+ TokenType[] backType = new TokenType[8];public void grow() {costs = ArrayUtil.grow(costs, 1 + count);@@ -313,7 +304,7 @@ public final class KoreanTokenizer extends Tokenizer {// NOTE: sneaky: grow separately because// ArrayUtil.grow will otherwise pick a different// length than the int[]s we just grew:- final Type[] newBackType = new Type[backID.length];+ final TokenType[] newBackType = new TokenType[backID.length];System.arraycopy(backType, 0, newBackType, 0, backType.length);backType = newBackType;}@@ -325,7 +316,7 @@ public final class KoreanTokenizer extends Tokenizer {int backRPos,int backIndex,int backID,- Type backType) {+ TokenType backType) {// NOTE: this isn't quite a true Viterbi search,// because we should check if lastRightID is// already present here, and only update if the new@@ -409,7 +400,12 @@ public final class KoreanTokenizer extends Tokenizer {}private void add(- KoMorphData morphAtts, Position fromPosData, int wordPos, int endPos, int wordID, Type type) {+ KoMorphData morphAtts,+ Position fromPosData,+ int wordPos,+ int endPos,+ int wordID,+ TokenType type) {final POS.Tag leftPOS = morphAtts.getLeftPOS(wordID);final int wordCost = morphAtts.getWordCost(wordID);final int leftID = morphAtts.getLeftId(wordID);@@ -811,7 +807,7 @@ public final class KoreanTokenizer extends Tokenizer {pos,maxPosAhead + 1,outputMaxPosAhead + arcFinalOutMaxPosAhead,- Type.USER);+ TokenType.USER);userWordMaxPosAhead = Math.max(userWordMaxPosAhead, maxPosAhead);}}@@ -863,7 +859,7 @@ public final class KoreanTokenizer extends Tokenizer {pos,posAhead + 1,wordIdRef.ints[wordIdRef.offset + ofs],- Type.KNOWN);+ TokenType.KNOWN);anyMatches = true;}}@@ -937,7 +933,7 @@ public final class KoreanTokenizer extends Tokenizer {pos,pos + unknownWordLength,wordIdRef.ints[wordIdRef.offset + ofs],- Type.UNKNOWN);+ TokenType.UNKNOWN);}}@@ -1019,7 +1015,7 @@ public final class KoreanTokenizer extends Tokenizer {: "backPos=" + backPos + " vs lastBackTracePos=" + lastBackTracePos;// the length of the word without the whitespaces at the beginning.int length = pos - backWordPos;- Type backType = posData.backType[bestIDX];+ TokenType backType = posData.backType[bestIDX];int backID = posData.backID[bestIDX];int nextBestIDX = posData.backIndex[bestIDX];// the start of the word after the whitespace at the beginning.@@ -1028,7 +1024,7 @@ public final class KoreanTokenizer extends Tokenizer {final Dictionary<? extends KoMorphData> dict = getDict(backType);- if (outputUnknownUnigrams && backType == Type.UNKNOWN) {+ if (outputUnknownUnigrams && backType == TokenType.UNKNOWN) {// outputUnknownUnigrams converts unknown word into unigrams:for (int i = length - 1; i >= 0; i--) {int charLen = 1;@@ -1038,7 +1034,7 @@ public final class KoreanTokenizer extends Tokenizer {}final DictionaryToken token =new DictionaryToken(- Type.UNKNOWN,+ TokenType.UNKNOWN,unkDictionary.getMorphAttributes(),CharacterDefinition.NGRAM,fragment,@@ -1090,14 +1086,16 @@ public final class KoreanTokenizer extends Tokenizer {morpheme.posTag,morpheme.surfaceForm,endOffset - morpheme.surfaceForm.length(),- endOffset);+ endOffset,+ backType);} else {compoundToken =new DecompoundToken(morpheme.posTag,morpheme.surfaceForm,token.getStartOffset(),- token.getEndOffset());+ token.getEndOffset(),+ backType);}if (i == 0 && mode == DecompoundMode.MIXED) {compoundToken.setPositionIncrement(0);@@ -1128,7 +1126,7 @@ public final class KoreanTokenizer extends Tokenizer {unkDictionary.lookupWordIds(characterDefinition.getCharacterClass(' '), wordIdRef);DictionaryToken spaceToken =new DictionaryToken(- Type.UNKNOWN,+ TokenType.UNKNOWN,unkDictionary.getMorphAttributes(),wordIdRef.ints[wordIdRef.offset],fragment,@@ -1154,7 +1152,7 @@ public final class KoreanTokenizer extends Tokenizer {positions.freeBefore(endPos);}- Dictionary<? extends KoMorphData> getDict(Type type) {+ Dictionary<? extends KoMorphData> getDict(TokenType type) {return dictionaryMap.get(type);}diff --git a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/Token.java b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/Token.javaindex 73b6ddab171..480d4abf410 100644--- a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/Token.java+++ b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/Token.java@@ -17,12 +17,14 @@package org.apache.lucene.analysis.ko;import org.apache.lucene.analysis.ko.dict.KoMorphData;+import org.apache.lucene.analysis.morph.TokenType;/** Analyzed token with morphological data. */public abstract class Token extends org.apache.lucene.analysis.morph.Token {- protected Token(char[] surfaceForm, int offset, int length, int startOffset, int endOffset) {- super(surfaceForm, offset, length, startOffset, endOffset);+ protected Token(+ char[] surfaceForm, int offset, int length, int startOffset, int endOffset, TokenType type) {+ super(surfaceForm, offset, length, startOffset, endOffset, type);}/** Get the {@link POS.Type} of the token. */