Skip to content

Apache Lucene open-source search software

LUCENE-10493: Unify TokenInfoFST in kuromoji and nori (#795)

Tomoko Uchida9aa8ec9d06a2
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/TokenInfoFST.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/TokenInfoFST.java
new file mode 100644
index 00000000000..460118f712a
--- /dev/null
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/TokenInfoFST.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.analysis.morph;
+
+import java.io.IOException;
+import org.apache.lucene.util.fst.FST;
+import org.apache.lucene.util.fst.FST.Arc;
+
+/**
+ * Thin wrapper around an FST with root-arc caching.
+ *
+ * <p>Root arcs between <code>cacheFloor</code> and <code>cacheFloor</code> are cached.
+ */
+public abstract class TokenInfoFST {
+ protected final FST<Long> fst;
+ private final int cacheCeiling;
+ private final int cacheFloor;
+ private final Arc<Long>[] rootCache;
+
+ public final Long NO_OUTPUT;
+
+ protected TokenInfoFST(FST<Long> fst, int cacheCeiling, int cacheFloor) throws IOException {
+ if (cacheCeiling < cacheFloor) {
+ throw new IllegalArgumentException(
+ "cacheCeiling must be larger than cacheFloor; cacheCeiling="
+ + cacheCeiling
+ + ", cacheFloor="
+ + cacheFloor);
+ }
+ this.fst = fst;
+ this.cacheCeiling = cacheCeiling;
+ this.cacheFloor = cacheFloor;
+ NO_OUTPUT = fst.outputs.getNoOutput();
+ rootCache = cacheRootArcs();
+ }
+
+ @SuppressWarnings({"rawtypes", "unchecked"})
+ private Arc<Long>[] cacheRootArcs() throws IOException {
+ Arc<Long>[] rootCache = new Arc[1 + (cacheCeiling - cacheFloor)];
+ Arc<Long> firstArc = new Arc<>();
+ fst.getFirstArc(firstArc);
+ Arc<Long> arc = new Arc<>();
+ final FST.BytesReader fstReader = fst.getBytesReader();
+ // TODO: jump to cacheFloor, readNextRealArc to ceiling? (just be careful we don't add bugs)
+ for (int i = 0; i < rootCache.length; i++) {
+ if (fst.findTargetArc(cacheFloor + i, firstArc, arc, fstReader) != null) {
+ rootCache[i] = new Arc<Long>().copyFrom(arc);
+ }
+ }
+ return rootCache;
+ }
+
+ public Arc<Long> findTargetArc(
+ int ch, Arc<Long> follow, Arc<Long> arc, boolean useCache, FST.BytesReader fstReader)
+ throws IOException {
+ if (useCache && ch >= cacheFloor && ch <= cacheCeiling) {
+ assert ch != FST.END_LABEL;
+ final Arc<Long> result = rootCache[ch - cacheFloor];
+ if (result == null) {
+ return null;
+ } else {
+ arc.copyFrom(result);
+ return arc;
+ }
+ } else {
+ return fst.findTargetArc(ch, follow, arc, fstReader);
+ }
+ }
+
+ public Arc<Long> getFirstArc(Arc<Long> arc) {
+ return fst.getFirstArc(arc);
+ }
+
+ public FST.BytesReader getBytesReader() {
+ return fst.getBytesReader();
+ }
+}
diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/TokenInfoFST.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/TokenInfoFST.java
index 2c6a5c6ecad..23861585f4e 100644
--- a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/TokenInfoFST.java
+++ b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/TokenInfoFST.java
@@ -18,7 +18,6 @@ package org.apache.lucene.analysis.ja.dict;
import java.io.IOException;
import org.apache.lucene.util.fst.FST;
-import org.apache.lucene.util.fst.FST.Arc;
/**
* Thin wrapper around an FST with root-arc caching for Japanese.
@@ -26,64 +25,13 @@ import org.apache.lucene.util.fst.FST.Arc;
* <p>Depending upon fasterButMoreRam, either just kana (191 arcs), or kana and han (28,607 arcs)
* are cached. The latter offers additional performance at the cost of more RAM.
*/
-public final class TokenInfoFST {
- private final FST<Long> fst;
-
+public final class TokenInfoFST extends org.apache.lucene.analysis.morph.TokenInfoFST {
// depending upon fasterButMoreRam, we cache root arcs for either
// kana (0x3040-0x30FF) or kana + han (0x3040-0x9FFF)
// false: 191 arcs
// true: 28,607 arcs (costs ~1.5MB)
- private final int cacheCeiling;
- private final FST.Arc<Long>[] rootCache;
-
- public final Long NO_OUTPUT;
-
public TokenInfoFST(FST<Long> fst, boolean fasterButMoreRam) throws IOException {
- this.fst = fst;
- this.cacheCeiling = fasterButMoreRam ? 0x9FFF : 0x30FF;
- NO_OUTPUT = fst.outputs.getNoOutput();
- rootCache = cacheRootArcs();
- }
-
- @SuppressWarnings({"rawtypes", "unchecked"})
- private FST.Arc<Long>[] cacheRootArcs() throws IOException {
- FST.Arc<Long>[] rootCache = new FST.Arc[1 + (cacheCeiling - 0x3040)];
- FST.Arc<Long> firstArc = new FST.Arc<>();
- fst.getFirstArc(firstArc);
- FST.Arc<Long> arc = new FST.Arc<>();
- final FST.BytesReader fstReader = fst.getBytesReader();
- // TODO: jump to 3040, readNextRealArc to ceiling? (just be careful we don't add bugs)
- for (int i = 0; i < rootCache.length; i++) {
- if (fst.findTargetArc(0x3040 + i, firstArc, arc, fstReader) != null) {
- rootCache[i] = new FST.Arc<Long>().copyFrom(arc);
- }
- }
- return rootCache;
- }
-
- public FST.Arc<Long> findTargetArc(
- int ch, FST.Arc<Long> follow, FST.Arc<Long> arc, boolean useCache, FST.BytesReader fstReader)
- throws IOException {
- if (useCache && ch >= 0x3040 && ch <= cacheCeiling) {
- assert ch != FST.END_LABEL;
- final Arc<Long> result = rootCache[ch - 0x3040];
- if (result == null) {
- return null;
- } else {
- arc.copyFrom(result);
- return arc;
- }
- } else {
- return fst.findTargetArc(ch, follow, arc, fstReader);
- }
- }
-
- public Arc<Long> getFirstArc(FST.Arc<Long> arc) {
- return fst.getFirstArc(arc);
- }
-
- public FST.BytesReader getBytesReader() {
- return fst.getBytesReader();
+ super(fst, fasterButMoreRam ? 0x9FFF : 0x30FF, 0x3040);
}
/** @lucene.internal for testing only */
diff --git a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/TokenInfoFST.java b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/TokenInfoFST.java
index 6c05d01e224..806f1209772 100644
--- a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/TokenInfoFST.java
+++ b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/TokenInfoFST.java
@@ -18,63 +18,12 @@ package org.apache.lucene.analysis.ko.dict;
import java.io.IOException;
import org.apache.lucene.util.fst.FST;
-import org.apache.lucene.util.fst.FST.Arc;
/** Thin wrapper around an FST with root-arc caching for Hangul syllables (11,172 arcs). */
-public final class TokenInfoFST {
- private final FST<Long> fst;
-
- private final int cacheCeiling;
- private final FST.Arc<Long>[] rootCache;
-
- public final Long NO_OUTPUT;
+public final class TokenInfoFST extends org.apache.lucene.analysis.morph.TokenInfoFST {
public TokenInfoFST(FST<Long> fst) throws IOException {
- this.fst = fst;
- this.cacheCeiling = 0xD7A3;
- NO_OUTPUT = fst.outputs.getNoOutput();
- rootCache = cacheRootArcs();
- }
-
- @SuppressWarnings({"rawtypes", "unchecked"})
- private FST.Arc<Long>[] cacheRootArcs() throws IOException {
- FST.Arc<Long>[] rootCache = new FST.Arc[1 + (cacheCeiling - 0xAC00)];
- FST.Arc<Long> firstArc = new FST.Arc<>();
- fst.getFirstArc(firstArc);
- FST.Arc<Long> arc = new FST.Arc<>();
- final FST.BytesReader fstReader = fst.getBytesReader();
- // TODO: jump to AC00, readNextRealArc to ceiling? (just be careful we don't add bugs)
- for (int i = 0; i < rootCache.length; i++) {
- if (fst.findTargetArc(0xAC00 + i, firstArc, arc, fstReader) != null) {
- rootCache[i] = new FST.Arc<Long>().copyFrom(arc);
- }
- }
- return rootCache;
- }
-
- public FST.Arc<Long> findTargetArc(
- int ch, FST.Arc<Long> follow, FST.Arc<Long> arc, boolean useCache, FST.BytesReader fstReader)
- throws IOException {
- if (useCache && ch >= 0xAC00 && ch <= cacheCeiling) {
- assert ch != FST.END_LABEL;
- final Arc<Long> result = rootCache[ch - 0xAC00];
- if (result == null) {
- return null;
- } else {
- arc.copyFrom(result);
- return arc;
- }
- } else {
- return fst.findTargetArc(ch, follow, arc, fstReader);
- }
- }
-
- public Arc<Long> getFirstArc(FST.Arc<Long> arc) {
- return fst.getFirstArc(arc);
- }
-
- public FST.BytesReader getBytesReader() {
- return fst.getBytesReader();
+ super(fst, 0xD7A3, 0xAC00);
}
/** @lucene.internal for testing only */