Apache Lucene open-source search software
LUCENE-10493: Unify TokenInfoFST in kuromoji and nori (#795)
Tomoko Uchida9aa8ec9d06a2
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/TokenInfoFST.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/TokenInfoFST.javanew file mode 100644index 00000000000..460118f712a--- /dev/null+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/morph/TokenInfoFST.java@@ -0,0 +1,91 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.analysis.morph;++import java.io.IOException;+import org.apache.lucene.util.fst.FST;+import org.apache.lucene.util.fst.FST.Arc;++/**+ * Thin wrapper around an FST with root-arc caching.+ *+ * <p>Root arcs between <code>cacheFloor</code> and <code>cacheFloor</code> are cached.+ */+public abstract class TokenInfoFST {+ protected final FST<Long> fst;+ private final int cacheCeiling;+ private final int cacheFloor;+ private final Arc<Long>[] rootCache;++ public final Long NO_OUTPUT;++ protected TokenInfoFST(FST<Long> fst, int cacheCeiling, int cacheFloor) throws IOException {+ if (cacheCeiling < cacheFloor) {+ throw new IllegalArgumentException(+ "cacheCeiling must be larger than cacheFloor; cacheCeiling="+ + cacheCeiling+ + ", cacheFloor="+ + cacheFloor);+ }+ this.fst = fst;+ this.cacheCeiling = cacheCeiling;+ this.cacheFloor = cacheFloor;+ NO_OUTPUT = fst.outputs.getNoOutput();+ rootCache = cacheRootArcs();+ }++ @SuppressWarnings({"rawtypes", "unchecked"})+ private Arc<Long>[] cacheRootArcs() throws IOException {+ Arc<Long>[] rootCache = new Arc[1 + (cacheCeiling - cacheFloor)];+ Arc<Long> firstArc = new Arc<>();+ fst.getFirstArc(firstArc);+ Arc<Long> arc = new Arc<>();+ final FST.BytesReader fstReader = fst.getBytesReader();+ // TODO: jump to cacheFloor, readNextRealArc to ceiling? (just be careful we don't add bugs)+ for (int i = 0; i < rootCache.length; i++) {+ if (fst.findTargetArc(cacheFloor + i, firstArc, arc, fstReader) != null) {+ rootCache[i] = new Arc<Long>().copyFrom(arc);+ }+ }+ return rootCache;+ }++ public Arc<Long> findTargetArc(+ int ch, Arc<Long> follow, Arc<Long> arc, boolean useCache, FST.BytesReader fstReader)+ throws IOException {+ if (useCache && ch >= cacheFloor && ch <= cacheCeiling) {+ assert ch != FST.END_LABEL;+ final Arc<Long> result = rootCache[ch - cacheFloor];+ if (result == null) {+ return null;+ } else {+ arc.copyFrom(result);+ return arc;+ }+ } else {+ return fst.findTargetArc(ch, follow, arc, fstReader);+ }+ }++ public Arc<Long> getFirstArc(Arc<Long> arc) {+ return fst.getFirstArc(arc);+ }++ public FST.BytesReader getBytesReader() {+ return fst.getBytesReader();+ }+}diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/TokenInfoFST.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/TokenInfoFST.javaindex 2c6a5c6ecad..23861585f4e 100644--- a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/TokenInfoFST.java+++ b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/TokenInfoFST.java@@ -18,7 +18,6 @@ package org.apache.lucene.analysis.ja.dict;import java.io.IOException;import org.apache.lucene.util.fst.FST;-import org.apache.lucene.util.fst.FST.Arc;/*** Thin wrapper around an FST with root-arc caching for Japanese.@@ -26,64 +25,13 @@ import org.apache.lucene.util.fst.FST.Arc;* <p>Depending upon fasterButMoreRam, either just kana (191 arcs), or kana and han (28,607 arcs)* are cached. The latter offers additional performance at the cost of more RAM.*/-public final class TokenInfoFST {- private final FST<Long> fst;-+public final class TokenInfoFST extends org.apache.lucene.analysis.morph.TokenInfoFST {// depending upon fasterButMoreRam, we cache root arcs for either// kana (0x3040-0x30FF) or kana + han (0x3040-0x9FFF)// false: 191 arcs// true: 28,607 arcs (costs ~1.5MB)- private final int cacheCeiling;- private final FST.Arc<Long>[] rootCache;-- public final Long NO_OUTPUT;-public TokenInfoFST(FST<Long> fst, boolean fasterButMoreRam) throws IOException {- this.fst = fst;- this.cacheCeiling = fasterButMoreRam ? 0x9FFF : 0x30FF;- NO_OUTPUT = fst.outputs.getNoOutput();- rootCache = cacheRootArcs();- }-- @SuppressWarnings({"rawtypes", "unchecked"})- private FST.Arc<Long>[] cacheRootArcs() throws IOException {- FST.Arc<Long>[] rootCache = new FST.Arc[1 + (cacheCeiling - 0x3040)];- FST.Arc<Long> firstArc = new FST.Arc<>();- fst.getFirstArc(firstArc);- FST.Arc<Long> arc = new FST.Arc<>();- final FST.BytesReader fstReader = fst.getBytesReader();- // TODO: jump to 3040, readNextRealArc to ceiling? (just be careful we don't add bugs)- for (int i = 0; i < rootCache.length; i++) {- if (fst.findTargetArc(0x3040 + i, firstArc, arc, fstReader) != null) {- rootCache[i] = new FST.Arc<Long>().copyFrom(arc);- }- }- return rootCache;- }-- public FST.Arc<Long> findTargetArc(- int ch, FST.Arc<Long> follow, FST.Arc<Long> arc, boolean useCache, FST.BytesReader fstReader)- throws IOException {- if (useCache && ch >= 0x3040 && ch <= cacheCeiling) {- assert ch != FST.END_LABEL;- final Arc<Long> result = rootCache[ch - 0x3040];- if (result == null) {- return null;- } else {- arc.copyFrom(result);- return arc;- }- } else {- return fst.findTargetArc(ch, follow, arc, fstReader);- }- }-- public Arc<Long> getFirstArc(FST.Arc<Long> arc) {- return fst.getFirstArc(arc);- }-- public FST.BytesReader getBytesReader() {- return fst.getBytesReader();+ super(fst, fasterButMoreRam ? 0x9FFF : 0x30FF, 0x3040);}/** @lucene.internal for testing only */diff --git a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/TokenInfoFST.java b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/TokenInfoFST.javaindex 6c05d01e224..806f1209772 100644--- a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/TokenInfoFST.java+++ b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/TokenInfoFST.java@@ -18,63 +18,12 @@ package org.apache.lucene.analysis.ko.dict;import java.io.IOException;import org.apache.lucene.util.fst.FST;-import org.apache.lucene.util.fst.FST.Arc;/** Thin wrapper around an FST with root-arc caching for Hangul syllables (11,172 arcs). */-public final class TokenInfoFST {- private final FST<Long> fst;-- private final int cacheCeiling;- private final FST.Arc<Long>[] rootCache;-- public final Long NO_OUTPUT;+public final class TokenInfoFST extends org.apache.lucene.analysis.morph.TokenInfoFST {public TokenInfoFST(FST<Long> fst) throws IOException {- this.fst = fst;- this.cacheCeiling = 0xD7A3;- NO_OUTPUT = fst.outputs.getNoOutput();- rootCache = cacheRootArcs();- }-- @SuppressWarnings({"rawtypes", "unchecked"})- private FST.Arc<Long>[] cacheRootArcs() throws IOException {- FST.Arc<Long>[] rootCache = new FST.Arc[1 + (cacheCeiling - 0xAC00)];- FST.Arc<Long> firstArc = new FST.Arc<>();- fst.getFirstArc(firstArc);- FST.Arc<Long> arc = new FST.Arc<>();- final FST.BytesReader fstReader = fst.getBytesReader();- // TODO: jump to AC00, readNextRealArc to ceiling? (just be careful we don't add bugs)- for (int i = 0; i < rootCache.length; i++) {- if (fst.findTargetArc(0xAC00 + i, firstArc, arc, fstReader) != null) {- rootCache[i] = new FST.Arc<Long>().copyFrom(arc);- }- }- return rootCache;- }-- public FST.Arc<Long> findTargetArc(- int ch, FST.Arc<Long> follow, FST.Arc<Long> arc, boolean useCache, FST.BytesReader fstReader)- throws IOException {- if (useCache && ch >= 0xAC00 && ch <= cacheCeiling) {- assert ch != FST.END_LABEL;- final Arc<Long> result = rootCache[ch - 0xAC00];- if (result == null) {- return null;- } else {- arc.copyFrom(result);- return arc;- }- } else {- return fst.findTargetArc(ch, follow, arc, fstReader);- }- }-- public Arc<Long> getFirstArc(FST.Arc<Long> arc) {- return fst.getFirstArc(arc);- }-- public FST.BytesReader getBytesReader() {- return fst.getBytesReader();+ super(fst, 0xD7A3, 0xAC00);}/** @lucene.internal for testing only */