Skip to content

armgit / solr

Public Imported from GitHub
0

Apache Solr open-source search software

LUCENE-9778: Hunspell: speed up input conversion (#2376)

Peter Gromov2ae45cc9859f
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/ConvTable.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/ConvTable.java
new file mode 100644
index 00000000000..6a87167e7e1
--- /dev/null
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/ConvTable.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.analysis.hunspell;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.TreeMap;
+import org.apache.lucene.util.CharsRef;
+import org.apache.lucene.util.FixedBitSet;
+import org.apache.lucene.util.IntsRefBuilder;
+import org.apache.lucene.util.fst.CharSequenceOutputs;
+import org.apache.lucene.util.fst.FST;
+import org.apache.lucene.util.fst.FSTCompiler;
+import org.apache.lucene.util.fst.Outputs;
+import org.apache.lucene.util.fst.Util;
+
+/** ICONV or OCONV replacement table */
+class ConvTable {
+ private final FST<CharsRef> fst;
+ private final FixedBitSet firstCharHashes;
+ private final int mod;
+
+ ConvTable(TreeMap<String, String> mappings) {
+ mod = Math.max(256, Integer.highestOneBit(mappings.size()) << 1);
+ firstCharHashes = new FixedBitSet(mod);
+
+ try {
+ Outputs<CharsRef> outputs = CharSequenceOutputs.getSingleton();
+ FSTCompiler<CharsRef> fstCompiler = new FSTCompiler<>(FST.INPUT_TYPE.BYTE2, outputs);
+ IntsRefBuilder scratchInts = new IntsRefBuilder();
+ for (Map.Entry<String, String> entry : mappings.entrySet()) {
+ String key = entry.getKey();
+ assert key.length() > 0;
+ firstCharHashes.set(key.charAt(0) % mod);
+ Util.toUTF16(key, scratchInts);
+ fstCompiler.add(scratchInts.get(), new CharsRef(entry.getValue()));
+ }
+
+ fst = fstCompiler.compile();
+ } catch (IOException bogus) {
+ throw new RuntimeException(bogus);
+ }
+ }
+
+ void applyMappings(StringBuilder sb) {
+ FST.BytesReader bytesReader = null;
+ FST.Arc<CharsRef> firstArc = null;
+ FST.Arc<CharsRef> arc = null;
+
+ int longestMatch;
+ CharsRef longestOutput;
+
+ for (int i = 0; i < sb.length(); i++) {
+ if (!mightReplaceChar(sb.charAt(i))) {
+ continue;
+ }
+
+ if (firstArc == null) {
+ firstArc = fst.getFirstArc(new FST.Arc<>());
+ bytesReader = fst.getBytesReader();
+ arc = new FST.Arc<>();
+ }
+ arc.copyFrom(firstArc);
+ CharsRef output = fst.outputs.getNoOutput();
+ longestMatch = -1;
+ longestOutput = null;
+
+ for (int j = i; j < sb.length(); j++) {
+ char ch = sb.charAt(j);
+
+ try {
+ if (fst.findTargetArc(ch, arc, arc, bytesReader) == null) {
+ break;
+ }
+ output = fst.outputs.add(output, arc.output());
+ } catch (IOException bogus) {
+ throw new RuntimeException(bogus);
+ }
+ if (arc.isFinal()) {
+ longestOutput = fst.outputs.add(output, arc.nextFinalOutput());
+ longestMatch = j;
+ }
+ }
+
+ if (longestMatch >= 0) {
+ sb.delete(i, longestMatch + 1);
+ sb.insert(i, longestOutput);
+ i += (longestOutput.length - 1);
+ }
+ }
+ }
+
+ boolean mightReplaceChar(char c) {
+ return firstCharHashes.get(c % mod);
+ }
+}
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java
index b65d287621d..d22c6eb8832 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java
@@ -51,7 +51,6 @@ import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.CharsRef;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.IntsRef;
import org.apache.lucene.util.IntsRefBuilder;
@@ -60,11 +59,9 @@ import org.apache.lucene.util.OfflineSorter.ByteSequencesReader;
import org.apache.lucene.util.OfflineSorter.ByteSequencesWriter;
import org.apache.lucene.util.automaton.CharacterRunAutomaton;
import org.apache.lucene.util.automaton.RegExp;
-import org.apache.lucene.util.fst.CharSequenceOutputs;
import org.apache.lucene.util.fst.FST;
import org.apache.lucene.util.fst.FSTCompiler;
import org.apache.lucene.util.fst.IntSequenceOutputs;
-import org.apache.lucene.util.fst.Outputs;
import org.apache.lucene.util.fst.Util;
/** In-memory structure for the dictionary (.dic) and affix (.aff) data of a hunspell dictionary. */
@@ -172,13 +169,7 @@ public class Dictionary {
int maxNGramSuggestions = Integer.MAX_VALUE;
boolean onlyMaxDiff;
char noSuggest, subStandard;
-
- // FSTs used for ICONV/OCONV, output ord pointing to replacement text
- FST<CharsRef> iconv;
- FST<CharsRef> oconv;
-
- boolean needsInputCleaning;
- boolean needsOutputCleaning;
+ ConvTable iconv, oconv;
// true if we can strip suffixes "down to nothing"
boolean fullStrip;
@@ -224,8 +215,6 @@ public class Dictionary {
boolean ignoreCase)
throws IOException, ParseException {
this.ignoreCase = ignoreCase;
- this.needsInputCleaning = ignoreCase;
- this.needsOutputCleaning = false; // set if we have an OCONV
try (BufferedInputStream affixStream =
new BufferedInputStream(affix, MAX_PROLOGUE_SCAN_WINDOW) {
@@ -379,16 +368,13 @@ public class Dictionary {
} else if ("IGNORE".equals(firstWord)) {
ignore = singleArgument(reader, line).toCharArray();
Arrays.sort(ignore);
- needsInputCleaning = true;
} else if ("ICONV".equals(firstWord) || "OCONV".equals(firstWord)) {
int num = parseNum(reader, line);
- FST<CharsRef> res = parseConversions(reader, num);
+ ConvTable res = parseConversions(reader, num);
if (line.startsWith("I")) {
iconv = res;
- needsInputCleaning |= iconv != null;
} else {
oconv = res;
- needsOutputCleaning |= oconv != null;
}
} else if ("FULLSTRIP".equals(firstWord)) {
fullStrip = true;
@@ -803,9 +789,8 @@ public class Dictionary {
affixData[dataStart + AFFIX_CONDITION] = (char) patternOrd;
affixData[dataStart + AFFIX_APPEND] = (char) appendFlagsOrd;
- if (needsInputCleaning) {
- CharSequence cleaned = cleanInput(affixArg, sb);
- affixArg = cleaned.toString();
+ if (needsInputCleaning(affixArg)) {
+ affixArg = cleanInput(affixArg, sb).toString();
}
if (isSuffix) {
@@ -840,9 +825,9 @@ public class Dictionary {
return affixData(affix, AFFIX_CONDITION) >>> 1;
}
- private FST<CharsRef> parseConversions(LineNumberReader reader, int num)
+ private ConvTable parseConversions(LineNumberReader reader, int num)
throws IOException, ParseException {
- Map<String, String> mappings = new TreeMap<>();
+ TreeMap<String, String> mappings = new TreeMap<>();
for (int i = 0; i < num; i++) {
String[] parts = splitBySpace(reader, reader.readLine(), 3);
@@ -851,15 +836,7 @@ public class Dictionary {
}
}
- Outputs<CharsRef> outputs = CharSequenceOutputs.getSingleton();
- FSTCompiler<CharsRef> fstCompiler = new FSTCompiler<>(FST.INPUT_TYPE.BYTE2, outputs);
- IntsRefBuilder scratchInts = new IntsRefBuilder();
- for (Map.Entry<String, String> entry : mappings.entrySet()) {
- Util.toUTF16(entry.getKey(), scratchInts);
- fstCompiler.add(scratchInts.get(), new CharsRef(entry.getValue()));
- }
-
- return fstCompiler.compile();
+ return new ConvTable(mappings);
}
private static final byte[] BOM_UTF8 = {(byte) 0xef, (byte) 0xbb, (byte) 0xbf};
@@ -1085,8 +1062,9 @@ public class Dictionary {
int sep = flagSep < 0 ? morphSep : flagSep;
CharSequence toWrite;
- if (needsInputCleaning) {
- cleanInput(line, sep, reuse);
+ String beforeSep = line.substring(0, sep);
+ if (needsInputCleaning(beforeSep)) {
+ cleanInput(beforeSep, reuse);
reuse.append(line, sep, line.length());
toWrite = reuse;
} else {
@@ -1571,14 +1549,28 @@ public class Dictionary {
return flagLookup.hasFlag(entryId, flag);
}
- CharSequence cleanInput(CharSequence input, StringBuilder reuse) {
- return cleanInput(input, input.length(), reuse);
+ boolean mayNeedInputCleaning() {
+ return ignoreCase || ignore != null || iconv != null;
+ }
+
+ boolean needsInputCleaning(CharSequence input) {
+ if (mayNeedInputCleaning()) {
+ for (int i = 0; i < input.length(); i++) {
+ char ch = input.charAt(i);
+ if (ignore != null && Arrays.binarySearch(ignore, ch) >= 0
+ || ignoreCase && caseFold(ch) != ch
+ || iconv != null && iconv.mightReplaceChar(ch)) {
+ return true;
+ }
+ }
+ }
+ return false;
}
- private CharSequence cleanInput(CharSequence input, int prefixLength, StringBuilder reuse) {
+ CharSequence cleanInput(CharSequence input, StringBuilder reuse) {
reuse.setLength(0);
- for (int i = 0; i < prefixLength; i++) {
+ for (int i = 0; i < input.length(); i++) {
char ch = input.charAt(i);
if (ignore != null && Arrays.binarySearch(ignore, ch) >= 0) {
@@ -1594,11 +1586,7 @@ public class Dictionary {
}
if (iconv != null) {
- try {
- applyMappings(iconv, reuse);
- } catch (IOException bogus) {
- throw new RuntimeException(bogus);
- }
+ iconv.applyMappings(reuse);
if (ignoreCase) {
for (int i = 0; i < reuse.length(); i++) {
reuse.setCharAt(i, caseFold(reuse.charAt(i)));
@@ -1624,44 +1612,6 @@ public class Dictionary {
}
}
- // TODO: this could be more efficient!
- static void applyMappings(FST<CharsRef> fst, StringBuilder sb) throws IOException {
- final FST.BytesReader bytesReader = fst.getBytesReader();
- final FST.Arc<CharsRef> firstArc = fst.getFirstArc(new FST.Arc<>());
- final CharsRef NO_OUTPUT = fst.outputs.getNoOutput();
-
- // temporary stuff
- final FST.Arc<CharsRef> arc = new FST.Arc<>();
- int longestMatch;
- CharsRef longestOutput;
-
- for (int i = 0; i < sb.length(); i++) {
- arc.copyFrom(firstArc);
- CharsRef output = NO_OUTPUT;
- longestMatch = -1;
- longestOutput = null;
-
- for (int j = i; j < sb.length(); j++) {
- char ch = sb.charAt(j);
- if (fst.findTargetArc(ch, arc, arc, bytesReader) == null) {
- break;
- } else {
- output = fst.outputs.add(output, arc.output());
- }
- if (arc.isFinal()) {
- longestOutput = fst.outputs.add(output, arc.nextFinalOutput());
- longestMatch = j;
- }
- }
-
- if (longestMatch >= 0) {
- sb.delete(i, longestMatch + 1);
- sb.insert(i, longestOutput);
- i += (longestOutput.length - 1);
- }
- }
- }
-
/** Returns true if this dictionary was constructed with the {@code ignoreCase} option */
public boolean getIgnoreCase() {
return ignoreCase;
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Hunspell.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Hunspell.java
index e85494b57d9..082076a57c3 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Hunspell.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Hunspell.java
@@ -22,7 +22,6 @@ import static org.apache.lucene.analysis.hunspell.WordContext.COMPOUND_END;
import static org.apache.lucene.analysis.hunspell.WordContext.COMPOUND_MIDDLE;
import static org.apache.lucene.analysis.hunspell.WordContext.SIMPLE_WORD;
-import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.LinkedHashSet;
@@ -72,7 +71,7 @@ public class Hunspell {
checkCanceled.run();
if (word.isEmpty()) return true;
- if (dictionary.needsInputCleaning) {
+ if (dictionary.needsInputCleaning(word)) {
word = dictionary.cleanInput(word, new StringBuilder()).toString();
}
@@ -479,7 +478,7 @@ public class Hunspell {
checkCanceled.run();
if (word.length() >= 100) return Collections.emptyList();
- if (dictionary.needsInputCleaning) {
+ if (dictionary.needsInputCleaning(word)) {
word = dictionary.cleanInput(word, new StringBuilder()).toString();
}
@@ -565,14 +564,10 @@ public class Hunspell {
}
private String cleanOutput(String s) {
- if (!dictionary.needsOutputCleaning) return s;
+ if (dictionary.oconv == null) return s;
- try {
- StringBuilder sb = new StringBuilder(s);
- Dictionary.applyMappings(dictionary.oconv, sb);
- return sb.toString();
- } catch (IOException bogus) {
- throw new RuntimeException(bogus);
- }
+ StringBuilder sb = new StringBuilder(s);
+ dictionary.oconv.applyMappings(sb);
+ return sb.toString();
}
}
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Stemmer.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Stemmer.java
index ceb47b2bd98..012b7641fe7 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Stemmer.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Stemmer.java
@@ -83,14 +83,16 @@ final class Stemmer {
*/
public List<CharsRef> stem(char[] word, int length) {
- if (dictionary.needsInputCleaning) {
+ if (dictionary.mayNeedInputCleaning()) {
scratchSegment.setLength(0);
scratchSegment.append(word, 0, length);
- CharSequence cleaned = dictionary.cleanInput(scratchSegment, segment);
- scratchBuffer = ArrayUtil.grow(scratchBuffer, cleaned.length());
- length = segment.length();
- segment.getChars(0, length, scratchBuffer, 0);
- word = scratchBuffer;
+ if (dictionary.needsInputCleaning(scratchSegment)) {
+ CharSequence cleaned = dictionary.cleanInput(scratchSegment, segment);
+ scratchBuffer = ArrayUtil.grow(scratchBuffer, cleaned.length());
+ length = segment.length();
+ segment.getChars(0, length, scratchBuffer, 0);
+ word = scratchBuffer;
+ }
}
List<CharsRef> list = new ArrayList<>();
@@ -365,18 +367,14 @@ final class Stemmer {
private CharsRef newStem(CharsRef stem, int morphDataId) {
String exception = stemException(morphDataId);
- if (dictionary.needsOutputCleaning) {
+ if (dictionary.oconv != null) {
scratchSegment.setLength(0);
if (exception != null) {
scratchSegment.append(exception);
} else {
scratchSegment.append(stem.chars, stem.offset, stem.length);
}
- try {
- Dictionary.applyMappings(dictionary.oconv, scratchSegment);
- } catch (IOException bogus) {
- throw new RuntimeException(bogus);
- }
+ dictionary.oconv.applyMappings(scratchSegment);
char[] cleaned = new char[scratchSegment.length()];
scratchSegment.getChars(0, cleaned.length, cleaned, 0);
return new CharsRef(cleaned, 0, cleaned.length);
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestDictionary.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestDictionary.java
index 2cc0c8495ac..1b64c3a9d31 100644
--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestDictionary.java
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestDictionary.java
@@ -24,19 +24,13 @@ import java.nio.charset.StandardCharsets;
import java.text.ParseException;
import java.util.Arrays;
import java.util.Collections;
+import java.util.TreeMap;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
import org.apache.lucene.store.ByteBuffersDirectory;
import org.apache.lucene.store.Directory;
-import org.apache.lucene.util.CharsRef;
import org.apache.lucene.util.IntsRef;
-import org.apache.lucene.util.IntsRefBuilder;
import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.util.fst.CharSequenceOutputs;
-import org.apache.lucene.util.fst.FST;
-import org.apache.lucene.util.fst.FSTCompiler;
-import org.apache.lucene.util.fst.Outputs;
-import org.apache.lucene.util.fst.Util;
import org.junit.Test;
public class TestDictionary extends LuceneTestCase {
@@ -166,51 +160,36 @@ public class TestDictionary extends LuceneTestCase {
assertTrue(dictStream.isClosed());
}
- public void testReplacements() throws Exception {
- Outputs<CharsRef> outputs = CharSequenceOutputs.getSingleton();
- FSTCompiler<CharsRef> fstCompiler = new FSTCompiler<>(FST.INPUT_TYPE.BYTE2, outputs);
- IntsRefBuilder scratchInts = new IntsRefBuilder();
-
- // a -> b
- Util.toUTF16("a", scratchInts);
- fstCompiler.add(scratchInts.get(), new CharsRef("b"));
-
- // ab -> c
- Util.toUTF16("ab", scratchInts);
- fstCompiler.add(scratchInts.get(), new CharsRef("c"));
-
- // c -> de
- Util.toUTF16("c", scratchInts);
- fstCompiler.add(scratchInts.get(), new CharsRef("de"));
-
- // def -> gh
- Util.toUTF16("def", scratchInts);
- fstCompiler.add(scratchInts.get(), new CharsRef("gh"));
-
- FST<CharsRef> fst = fstCompiler.compile();
+ public void testReplacements() {
+ TreeMap<String, String> map = new TreeMap<>();
+ map.put("a", "b");
+ map.put("ab", "c");
+ map.put("c", "de");
+ map.put("def", "gh");
+ ConvTable table = new ConvTable(map);
StringBuilder sb = new StringBuilder("atestanother");
- Dictionary.applyMappings(fst, sb);
+ table.applyMappings(sb);
assertEquals("btestbnother", sb.toString());
sb = new StringBuilder("abtestanother");
- Dictionary.applyMappings(fst, sb);
+ table.applyMappings(sb);
assertEquals("ctestbnother", sb.toString());
sb = new StringBuilder("atestabnother");
- Dictionary.applyMappings(fst, sb);
+ table.applyMappings(sb);
assertEquals("btestcnother", sb.toString());
sb = new StringBuilder("abtestabnother");
- Dictionary.applyMappings(fst, sb);
+ table.applyMappings(sb);
assertEquals("ctestcnother", sb.toString());
sb = new StringBuilder("abtestabcnother");
- Dictionary.applyMappings(fst, sb);
+ table.applyMappings(sb);
assertEquals("ctestcdenother", sb.toString());
sb = new StringBuilder("defdefdefc");
- Dictionary.applyMappings(fst, sb);
+ table.applyMappings(sb);
assertEquals("ghghghde", sb.toString());
}