Skip to content

Apache Lucene open-source search software

LUCENE-10497: Add a base Token class to analysis-common (for kuromoji and nori) (#783)

Tomoko Uchidabb4a0dc19ba9
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/Token.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/Token.java
new file mode 100644
index 00000000000..f9f0468d473
--- /dev/null
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/Token.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.analysis.morph;
+
+/** Analyzed token with morphological data. */
+public abstract class Token {
+ protected final char[] surfaceForm;
+ protected final int offset;
+ protected final int length;
+
+ protected final int startOffset;
+ protected final int endOffset;
+ protected int posIncr = 1;
+ protected int posLen = 1;
+
+ protected Token(char[] surfaceForm, int offset, int length, int startOffset, int endOffset) {
+ this.surfaceForm = surfaceForm;
+ this.offset = offset;
+ this.length = length;
+ this.startOffset = startOffset;
+ this.endOffset = endOffset;
+ }
+
+ /** @return surfaceForm */
+ public char[] getSurfaceForm() {
+ return surfaceForm;
+ }
+
+ /** @return offset into surfaceForm */
+ public int getOffset() {
+ return offset;
+ }
+
+ /** @return length of surfaceForm */
+ public int getLength() {
+ return length;
+ }
+
+ /** @return surfaceForm as a String */
+ public String getSurfaceFormString() {
+ return new String(surfaceForm, offset, length);
+ }
+
+ /** Get the start offset of the term in the analyzed text. */
+ public int getStartOffset() {
+ return startOffset;
+ }
+
+ /** Get the end offset of the term in the analyzed text. */
+ public int getEndOffset() {
+ return endOffset;
+ }
+
+ public void setPositionIncrement(int posIncr) {
+ this.posIncr = posIncr;
+ }
+
+ public int getPositionIncrement() {
+ return posIncr;
+ }
+
+ /**
+ * Set the position length (in tokens) of this token. For normal tokens this is 1; for compound
+ * tokens it's > 1.
+ */
+ public void setPositionLength(int posLen) {
+ this.posLen = posLen;
+ }
+
+ /**
+ * Get the length (in tokens) of this token. For normal tokens this is 1; for compound tokens it's
+ * > 1.
+ */
+ public int getPositionLength() {
+ return posLen;
+ }
+}
diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizer.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizer.java
index 13d50272d32..a485931d5de 100644
--- a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizer.java
+++ b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizer.java
@@ -589,35 +589,34 @@ public final class JapaneseTokenizer extends Tokenizer {
final Token token = pending.remove(pending.size() - 1);
- int position = token.getPosition();
int length = token.getLength();
clearAttributes();
assert length > 0;
// System.out.println("off=" + token.getOffset() + " len=" + length + " vs " +
// token.getSurfaceForm().length);
termAtt.copyBuffer(token.getSurfaceForm(), token.getOffset(), length);
- offsetAtt.setOffset(correctOffset(position), correctOffset(position + length));
+ offsetAtt.setOffset(correctOffset(token.getStartOffset()), correctOffset(token.getEndOffset()));
basicFormAtt.setToken(token);
posAtt.setToken(token);
readingAtt.setToken(token);
inflectionAtt.setToken(token);
- if (token.getPosition() == lastTokenPos) {
+ if (token.getStartOffset() == lastTokenPos) {
posIncAtt.setPositionIncrement(0);
posLengthAtt.setPositionLength(token.getPositionLength());
} else if (outputNBest) {
// The position length is always calculated if outputNBest is true.
- assert token.getPosition() > lastTokenPos;
+ assert token.getStartOffset() > lastTokenPos;
posIncAtt.setPositionIncrement(1);
posLengthAtt.setPositionLength(token.getPositionLength());
} else {
- assert token.getPosition() > lastTokenPos;
+ assert token.getStartOffset() > lastTokenPos;
posIncAtt.setPositionIncrement(1);
posLengthAtt.setPositionLength(1);
}
if (VERBOSE) {
System.out.println(Thread.currentThread().getName() + ": incToken: return token=" + token);
}
- lastTokenPos = token.getPosition();
+ lastTokenPos = token.getStartOffset();
return true;
}
@@ -1566,26 +1565,29 @@ public final class JapaneseTokenizer extends Tokenizer {
int wordID = wordIDAndLength[0];
pending.add(
new Token(
- wordID,
fragment,
left,
right - left,
- Type.USER,
lattice.rootBase + left,
+ lattice.rootBase + right,
+ wordID,
+ Type.USER,
userDictionary.getMorphAttributes()));
// Output compound
int current = 0;
for (int j = 1; j < wordIDAndLength.length; j++) {
final int len = wordIDAndLength[j];
if (len < right - left) {
+ int startOffset = lattice.rootBase + current + left;
pending.add(
new Token(
- wordID + j - 1,
fragment,
current + left,
len,
+ startOffset,
+ startOffset + len,
+ wordID + j - 1,
Type.USER,
- lattice.rootBase + current + left,
userDictionary.getMorphAttributes()));
}
current += len;
@@ -1593,12 +1595,13 @@ public final class JapaneseTokenizer extends Tokenizer {
} else {
pending.add(
new Token(
- lattice.nodeWordID[node],
fragment,
left,
right - left,
- type,
lattice.rootBase + left,
+ lattice.rootBase + right,
+ lattice.nodeWordID[node],
+ type,
getDict(type).getMorphAttributes()));
}
}
@@ -1922,12 +1925,13 @@ public final class JapaneseTokenizer extends Tokenizer {
// this alternate path joins back:
altToken =
new Token(
- backID,
fragment,
backPos - lastBackTracePos,
length,
- backType,
backPos,
+ backPos + length,
+ backID,
+ backType,
getDict(backType).getMorphAttributes());
// Redirect our backtrace to 2nd best:
@@ -1953,7 +1957,7 @@ public final class JapaneseTokenizer extends Tokenizer {
final int offset = backPos - lastBackTracePos;
assert offset >= 0;
- if (altToken != null && altToken.getPosition() >= backPos) {
+ if (altToken != null && altToken.getStartOffset() >= backPos) {
if (outputCompounds) {
// We've backtraced to the position where the
// compound token starts; add it now:
@@ -1961,7 +1965,8 @@ public final class JapaneseTokenizer extends Tokenizer {
// The pruning we did when we created the altToken
// ensures that the back trace will align back with
// the start of the altToken:
- assert altToken.getPosition() == backPos : altToken.getPosition() + " vs " + backPos;
+ assert altToken.getStartOffset() == backPos
+ : altToken.getStartOffset() + " vs " + backPos;
// NOTE: not quite right: the compound token may
// have had all punctuation back traced so far, but
@@ -1999,14 +2004,16 @@ public final class JapaneseTokenizer extends Tokenizer {
for (int j = 1; j < wordIDAndLength.length; j++) {
final int len = wordIDAndLength[j];
// System.out.println(" add user: len=" + len);
+ int startOffset = current + backPos;
pending.add(
new Token(
- wordID + j - 1,
fragment,
current + offset,
len,
+ startOffset,
+ startOffset + len,
+ wordID + j - 1,
Type.USER,
- current + backPos,
dict.getMorphAttributes()));
if (VERBOSE) {
System.out.println(" add USER token=" + pending.get(pending.size() - 1));
@@ -2036,14 +2043,16 @@ public final class JapaneseTokenizer extends Tokenizer {
// System.out.println(" extended tok offset="
// + (offset + i));
if (!discardPunctuation || !isPunctuation(fragment[offset + i])) {
+ int startOffset = backPos + i;
pending.add(
new Token(
- CharacterDefinition.NGRAM,
fragment,
offset + i,
charLen,
+ startOffset,
+ startOffset + charLen,
+ CharacterDefinition.NGRAM,
Type.UNKNOWN,
- backPos + i,
unkDictionary.getMorphAttributes()));
unigramTokenCount++;
}
@@ -2053,7 +2062,14 @@ public final class JapaneseTokenizer extends Tokenizer {
} else if (!discardPunctuation || length == 0 || !isPunctuation(fragment[offset])) {
pending.add(
new Token(
- backID, fragment, offset, length, backType, backPos, dict.getMorphAttributes()));
+ fragment,
+ offset,
+ length,
+ backPos,
+ backPos + length,
+ backID,
+ backType,
+ dict.getMorphAttributes()));
if (VERBOSE) {
System.out.println(" add token=" + pending.get(pending.size() - 1));
}
diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/Token.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/Token.java
index e9ae9490125..c393008fef1 100644
--- a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/Token.java
+++ b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/Token.java
@@ -20,104 +20,75 @@ import org.apache.lucene.analysis.ja.JapaneseTokenizer.Type;
import org.apache.lucene.analysis.ja.dict.JaMorphData;
/** Analyzed token with morphological data from its dictionary. */
-public class Token {
- private final JaMorphData morphAtts;
+public class Token extends org.apache.lucene.analysis.morph.Token {
+ private final JaMorphData morphData;
- private final int wordId;
-
- private final char[] surfaceForm;
- private final int offset;
- private final int length;
-
- private final int position;
- private int positionLength;
+ private final int morphId;
private final Type type;
public Token(
- int wordId,
char[] surfaceForm,
int offset,
int length,
+ int startOffset,
+ int endOffset,
+ int morphId,
Type type,
- int position,
- JaMorphData morphAtts) {
- this.wordId = wordId;
- this.surfaceForm = surfaceForm;
- this.offset = offset;
- this.length = length;
+ JaMorphData morphData) {
+ super(surfaceForm, offset, length, startOffset, endOffset);
+ this.morphId = morphId;
this.type = type;
- this.position = position;
- this.morphAtts = morphAtts;
+ this.morphData = morphData;
}
@Override
public String toString() {
return "Token(\""
+ new String(surfaceForm, offset, length)
- + "\" pos="
- + position
+ + "\" offset="
+ + startOffset
+ " length="
+ length
+ " posLen="
- + positionLength
+ + posLen
+ " type="
+ type
- + " wordId="
- + wordId
+ + " morphId="
+ + morphId
+ " leftID="
- + morphAtts.getLeftId(wordId)
+ + morphData.getLeftId(morphId)
+ ")";
}
- /** @return surfaceForm */
- public char[] getSurfaceForm() {
- return surfaceForm;
- }
-
- /** @return offset into surfaceForm */
- public int getOffset() {
- return offset;
- }
-
- /** @return length of surfaceForm */
- public int getLength() {
- return length;
- }
-
- /** @return surfaceForm as a String */
- public String getSurfaceFormString() {
- return new String(surfaceForm, offset, length);
- }
-
/** @return reading. null if token doesn't have reading. */
public String getReading() {
- return morphAtts.getReading(wordId, surfaceForm, offset, length);
+ return morphData.getReading(morphId, surfaceForm, offset, length);
}
/** @return pronunciation. null if token doesn't have pronunciation. */
public String getPronunciation() {
- return morphAtts.getPronunciation(wordId, surfaceForm, offset, length);
+ return morphData.getPronunciation(morphId, surfaceForm, offset, length);
}
/** @return part of speech. */
public String getPartOfSpeech() {
- return morphAtts.getPartOfSpeech(wordId);
+ return morphData.getPartOfSpeech(morphId);
}
/** @return inflection type or null */
public String getInflectionType() {
- return morphAtts.getInflectionType(wordId);
+ return morphData.getInflectionType(morphId);
}
/** @return inflection form or null */
public String getInflectionForm() {
- return morphAtts.getInflectionForm(wordId);
+ return morphData.getInflectionForm(morphId);
}
/** @return base form or null if token is not inflected */
public String getBaseForm() {
- return morphAtts.getBaseForm(wordId, surfaceForm, offset, length);
+ return morphData.getBaseForm(morphId, surfaceForm, offset, length);
}
/**
@@ -155,31 +126,4 @@ public class Token {
public boolean isUser() {
return type == Type.USER;
}
-
- /**
- * Get index of this token in input text
- *
- * @return position of token
- */
- public int getPosition() {
- return position;
- }
-
- /**
- * Set the position length (in tokens) of this token. For normal tokens this is 1; for compound
- * tokens it's &gt; 1.
- */
- public void setPositionLength(int positionLength) {
- this.positionLength = positionLength;
- }
-
- /**
- * Get the length (in tokens) of this token. For normal tokens this is 1; for compound tokens it's
- * &gt; 1.
- *
- * @return position length of token
- */
- public int getPositionLength() {
- return positionLength;
- }
}
diff --git a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/Token.java b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/Token.java
index ed2d7ff6ba7..73b6ddab171 100644
--- a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/Token.java
+++ b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/Token.java
@@ -19,43 +19,10 @@ package org.apache.lucene.analysis.ko;
import org.apache.lucene.analysis.ko.dict.KoMorphData;
/** Analyzed token with morphological data. */
-public abstract class Token {
- private final char[] surfaceForm;
- private final int offset;
- private final int length;
+public abstract class Token extends org.apache.lucene.analysis.morph.Token {
- private final int startOffset;
- private final int endOffset;
- private int posIncr = 1;
- private int posLen = 1;
-
- public Token(char[] surfaceForm, int offset, int length, int startOffset, int endOffset) {
- this.surfaceForm = surfaceForm;
- this.offset = offset;
- this.length = length;
-
- this.startOffset = startOffset;
- this.endOffset = endOffset;
- }
-
- /** @return surfaceForm */
- public char[] getSurfaceForm() {
- return surfaceForm;
- }
-
- /** @return offset into surfaceForm */
- public int getOffset() {
- return offset;
- }
-
- /** @return length of surfaceForm */
- public int getLength() {
- return length;
- }
-
- /** @return surfaceForm as a String */
- public String getSurfaceFormString() {
- return new String(surfaceForm, offset, length);
+ protected Token(char[] surfaceForm, int offset, int length, int startOffset, int endOffset) {
+ super(surfaceForm, offset, length, startOffset, endOffset);
}
/** Get the {@link POS.Type} of the token. */
@@ -75,30 +42,4 @@ public abstract class Token {
* token.
*/
public abstract KoMorphData.Morpheme[] getMorphemes();
-
- /** Get the start offset of the term in the analyzed text. */
- public int getStartOffset() {
- return startOffset;
- }
-
- /** Get the end offset of the term in the analyzed text. */
- public int getEndOffset() {
- return endOffset;
- }
-
- public void setPositionIncrement(int posIncr) {
- this.posIncr = posIncr;
- }
-
- public int getPositionIncrement() {
- return posIncr;
- }
-
- public void setPositionLength(int posLen) {
- this.posLen = posLen;
- }
-
- public int getPositionLength() {
- return posLen;
- }
}