Apache Lucene open-source search software
LUCENE-10497: Add a base Token class to analysis-common (for kuromoji and nori) (#783)
Tomoko Uchidabb4a0dc19ba9
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/Token.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/Token.javanew file mode 100644index 00000000000..f9f0468d473--- /dev/null+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/Token.java@@ -0,0 +1,91 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.analysis.morph;++/** Analyzed token with morphological data. */+public abstract class Token {+ protected final char[] surfaceForm;+ protected final int offset;+ protected final int length;++ protected final int startOffset;+ protected final int endOffset;+ protected int posIncr = 1;+ protected int posLen = 1;++ protected Token(char[] surfaceForm, int offset, int length, int startOffset, int endOffset) {+ this.surfaceForm = surfaceForm;+ this.offset = offset;+ this.length = length;+ this.startOffset = startOffset;+ this.endOffset = endOffset;+ }++ /** @return surfaceForm */+ public char[] getSurfaceForm() {+ return surfaceForm;+ }++ /** @return offset into surfaceForm */+ public int getOffset() {+ return offset;+ }++ /** @return length of surfaceForm */+ public int getLength() {+ return length;+ }++ /** @return surfaceForm as a String */+ public String getSurfaceFormString() {+ return new String(surfaceForm, offset, length);+ }++ /** Get the start offset of the term in the analyzed text. */+ public int getStartOffset() {+ return startOffset;+ }++ /** Get the end offset of the term in the analyzed text. */+ public int getEndOffset() {+ return endOffset;+ }++ public void setPositionIncrement(int posIncr) {+ this.posIncr = posIncr;+ }++ public int getPositionIncrement() {+ return posIncr;+ }++ /**+ * Set the position length (in tokens) of this token. For normal tokens this is 1; for compound+ * tokens it's > 1.+ */+ public void setPositionLength(int posLen) {+ this.posLen = posLen;+ }++ /**+ * Get the length (in tokens) of this token. For normal tokens this is 1; for compound tokens it's+ * > 1.+ */+ public int getPositionLength() {+ return posLen;+ }+}diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizer.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizer.javaindex 13d50272d32..a485931d5de 100644--- a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizer.java+++ b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizer.java@@ -589,35 +589,34 @@ public final class JapaneseTokenizer extends Tokenizer {final Token token = pending.remove(pending.size() - 1);- int position = token.getPosition();int length = token.getLength();clearAttributes();assert length > 0;// System.out.println("off=" + token.getOffset() + " len=" + length + " vs " +// token.getSurfaceForm().length);termAtt.copyBuffer(token.getSurfaceForm(), token.getOffset(), length);- offsetAtt.setOffset(correctOffset(position), correctOffset(position + length));+ offsetAtt.setOffset(correctOffset(token.getStartOffset()), correctOffset(token.getEndOffset()));basicFormAtt.setToken(token);posAtt.setToken(token);readingAtt.setToken(token);inflectionAtt.setToken(token);- if (token.getPosition() == lastTokenPos) {+ if (token.getStartOffset() == lastTokenPos) {posIncAtt.setPositionIncrement(0);posLengthAtt.setPositionLength(token.getPositionLength());} else if (outputNBest) {// The position length is always calculated if outputNBest is true.- assert token.getPosition() > lastTokenPos;+ assert token.getStartOffset() > lastTokenPos;posIncAtt.setPositionIncrement(1);posLengthAtt.setPositionLength(token.getPositionLength());} else {- assert token.getPosition() > lastTokenPos;+ assert token.getStartOffset() > lastTokenPos;posIncAtt.setPositionIncrement(1);posLengthAtt.setPositionLength(1);}if (VERBOSE) {System.out.println(Thread.currentThread().getName() + ": incToken: return token=" + token);}- lastTokenPos = token.getPosition();+ lastTokenPos = token.getStartOffset();return true;}@@ -1566,26 +1565,29 @@ public final class JapaneseTokenizer extends Tokenizer {int wordID = wordIDAndLength[0];pending.add(new Token(- wordID,fragment,left,right - left,- Type.USER,lattice.rootBase + left,+ lattice.rootBase + right,+ wordID,+ Type.USER,userDictionary.getMorphAttributes()));// Output compoundint current = 0;for (int j = 1; j < wordIDAndLength.length; j++) {final int len = wordIDAndLength[j];if (len < right - left) {+ int startOffset = lattice.rootBase + current + left;pending.add(new Token(- wordID + j - 1,fragment,current + left,len,+ startOffset,+ startOffset + len,+ wordID + j - 1,Type.USER,- lattice.rootBase + current + left,userDictionary.getMorphAttributes()));}current += len;@@ -1593,12 +1595,13 @@ public final class JapaneseTokenizer extends Tokenizer {} else {pending.add(new Token(- lattice.nodeWordID[node],fragment,left,right - left,- type,lattice.rootBase + left,+ lattice.rootBase + right,+ lattice.nodeWordID[node],+ type,getDict(type).getMorphAttributes()));}}@@ -1922,12 +1925,13 @@ public final class JapaneseTokenizer extends Tokenizer {// this alternate path joins back:altToken =new Token(- backID,fragment,backPos - lastBackTracePos,length,- backType,backPos,+ backPos + length,+ backID,+ backType,getDict(backType).getMorphAttributes());// Redirect our backtrace to 2nd best:@@ -1953,7 +1957,7 @@ public final class JapaneseTokenizer extends Tokenizer {final int offset = backPos - lastBackTracePos;assert offset >= 0;- if (altToken != null && altToken.getPosition() >= backPos) {+ if (altToken != null && altToken.getStartOffset() >= backPos) {if (outputCompounds) {// We've backtraced to the position where the// compound token starts; add it now:@@ -1961,7 +1965,8 @@ public final class JapaneseTokenizer extends Tokenizer {// The pruning we did when we created the altToken// ensures that the back trace will align back with// the start of the altToken:- assert altToken.getPosition() == backPos : altToken.getPosition() + " vs " + backPos;+ assert altToken.getStartOffset() == backPos+ : altToken.getStartOffset() + " vs " + backPos;// NOTE: not quite right: the compound token may// have had all punctuation back traced so far, but@@ -1999,14 +2004,16 @@ public final class JapaneseTokenizer extends Tokenizer {for (int j = 1; j < wordIDAndLength.length; j++) {final int len = wordIDAndLength[j];// System.out.println(" add user: len=" + len);+ int startOffset = current + backPos;pending.add(new Token(- wordID + j - 1,fragment,current + offset,len,+ startOffset,+ startOffset + len,+ wordID + j - 1,Type.USER,- current + backPos,dict.getMorphAttributes()));if (VERBOSE) {System.out.println(" add USER token=" + pending.get(pending.size() - 1));@@ -2036,14 +2043,16 @@ public final class JapaneseTokenizer extends Tokenizer {// System.out.println(" extended tok offset="// + (offset + i));if (!discardPunctuation || !isPunctuation(fragment[offset + i])) {+ int startOffset = backPos + i;pending.add(new Token(- CharacterDefinition.NGRAM,fragment,offset + i,charLen,+ startOffset,+ startOffset + charLen,+ CharacterDefinition.NGRAM,Type.UNKNOWN,- backPos + i,unkDictionary.getMorphAttributes()));unigramTokenCount++;}@@ -2053,7 +2062,14 @@ public final class JapaneseTokenizer extends Tokenizer {} else if (!discardPunctuation || length == 0 || !isPunctuation(fragment[offset])) {pending.add(new Token(- backID, fragment, offset, length, backType, backPos, dict.getMorphAttributes()));+ fragment,+ offset,+ length,+ backPos,+ backPos + length,+ backID,+ backType,+ dict.getMorphAttributes()));if (VERBOSE) {System.out.println(" add token=" + pending.get(pending.size() - 1));}diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/Token.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/Token.javaindex e9ae9490125..c393008fef1 100644--- a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/Token.java+++ b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/Token.java@@ -20,104 +20,75 @@ import org.apache.lucene.analysis.ja.JapaneseTokenizer.Type;import org.apache.lucene.analysis.ja.dict.JaMorphData;/** Analyzed token with morphological data from its dictionary. */-public class Token {- private final JaMorphData morphAtts;+public class Token extends org.apache.lucene.analysis.morph.Token {+ private final JaMorphData morphData;- private final int wordId;-- private final char[] surfaceForm;- private final int offset;- private final int length;-- private final int position;- private int positionLength;+ private final int morphId;private final Type type;public Token(- int wordId,char[] surfaceForm,int offset,int length,+ int startOffset,+ int endOffset,+ int morphId,Type type,- int position,- JaMorphData morphAtts) {- this.wordId = wordId;- this.surfaceForm = surfaceForm;- this.offset = offset;- this.length = length;+ JaMorphData morphData) {+ super(surfaceForm, offset, length, startOffset, endOffset);+ this.morphId = morphId;this.type = type;- this.position = position;- this.morphAtts = morphAtts;+ this.morphData = morphData;}@Overridepublic String toString() {return "Token(\""+ new String(surfaceForm, offset, length)- + "\" pos="- + position+ + "\" offset="+ + startOffset+ " length="+ length+ " posLen="- + positionLength+ + posLen+ " type="+ type- + " wordId="- + wordId+ + " morphId="+ + morphId+ " leftID="- + morphAtts.getLeftId(wordId)+ + morphData.getLeftId(morphId)+ ")";}- /** @return surfaceForm */- public char[] getSurfaceForm() {- return surfaceForm;- }-- /** @return offset into surfaceForm */- public int getOffset() {- return offset;- }-- /** @return length of surfaceForm */- public int getLength() {- return length;- }-- /** @return surfaceForm as a String */- public String getSurfaceFormString() {- return new String(surfaceForm, offset, length);- }-/** @return reading. null if token doesn't have reading. */public String getReading() {- return morphAtts.getReading(wordId, surfaceForm, offset, length);+ return morphData.getReading(morphId, surfaceForm, offset, length);}/** @return pronunciation. null if token doesn't have pronunciation. */public String getPronunciation() {- return morphAtts.getPronunciation(wordId, surfaceForm, offset, length);+ return morphData.getPronunciation(morphId, surfaceForm, offset, length);}/** @return part of speech. */public String getPartOfSpeech() {- return morphAtts.getPartOfSpeech(wordId);+ return morphData.getPartOfSpeech(morphId);}/** @return inflection type or null */public String getInflectionType() {- return morphAtts.getInflectionType(wordId);+ return morphData.getInflectionType(morphId);}/** @return inflection form or null */public String getInflectionForm() {- return morphAtts.getInflectionForm(wordId);+ return morphData.getInflectionForm(morphId);}/** @return base form or null if token is not inflected */public String getBaseForm() {- return morphAtts.getBaseForm(wordId, surfaceForm, offset, length);+ return morphData.getBaseForm(morphId, surfaceForm, offset, length);}/**@@ -155,31 +126,4 @@ public class Token {public boolean isUser() {return type == Type.USER;}-- /**- * Get index of this token in input text- *- * @return position of token- */- public int getPosition() {- return position;- }-- /**- * Set the position length (in tokens) of this token. For normal tokens this is 1; for compound- * tokens it's > 1.- */- public void setPositionLength(int positionLength) {- this.positionLength = positionLength;- }-- /**- * Get the length (in tokens) of this token. For normal tokens this is 1; for compound tokens it's- * > 1.- *- * @return position length of token- */- public int getPositionLength() {- return positionLength;- }}diff --git a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/Token.java b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/Token.javaindex ed2d7ff6ba7..73b6ddab171 100644--- a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/Token.java+++ b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/Token.java@@ -19,43 +19,10 @@ package org.apache.lucene.analysis.ko;import org.apache.lucene.analysis.ko.dict.KoMorphData;/** Analyzed token with morphological data. */-public abstract class Token {- private final char[] surfaceForm;- private final int offset;- private final int length;+public abstract class Token extends org.apache.lucene.analysis.morph.Token {- private final int startOffset;- private final int endOffset;- private int posIncr = 1;- private int posLen = 1;-- public Token(char[] surfaceForm, int offset, int length, int startOffset, int endOffset) {- this.surfaceForm = surfaceForm;- this.offset = offset;- this.length = length;-- this.startOffset = startOffset;- this.endOffset = endOffset;- }-- /** @return surfaceForm */- public char[] getSurfaceForm() {- return surfaceForm;- }-- /** @return offset into surfaceForm */- public int getOffset() {- return offset;- }-- /** @return length of surfaceForm */- public int getLength() {- return length;- }-- /** @return surfaceForm as a String */- public String getSurfaceFormString() {- return new String(surfaceForm, offset, length);+ protected Token(char[] surfaceForm, int offset, int length, int startOffset, int endOffset) {+ super(surfaceForm, offset, length, startOffset, endOffset);}/** Get the {@link POS.Type} of the token. */@@ -75,30 +42,4 @@ public abstract class Token {* token.*/public abstract KoMorphData.Morpheme[] getMorphemes();-- /** Get the start offset of the term in the analyzed text. */- public int getStartOffset() {- return startOffset;- }-- /** Get the end offset of the term in the analyzed text. */- public int getEndOffset() {- return endOffset;- }-- public void setPositionIncrement(int posIncr) {- this.posIncr = posIncr;- }-- public int getPositionIncrement() {- return posIncr;- }-- public void setPositionLength(int posLen) {- this.posLen = posLen;- }-- public int getPositionLength() {- return posLen;- }}