Apache Lucene open-source search software
hunspell: speed up GeneratingSuggester by not deserializing non-suggestible roots (#11859)
Peter Gromov05971b331575
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex 79eef5b39a9..16e7bed965c 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -60,7 +60,7 @@ ImprovementsOptimizations----------------------* GITHUB#11857: Hunspell: improved suggestion performance+* GITHUB#11857, GITHUB#11859: Hunspell: improved suggestion performanceBug Fixes---------------------diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.javaindex 09212794a8b..e94047b67db 100644--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java@@ -49,6 +49,7 @@ import java.util.Map;import java.util.Set;import java.util.TreeMap;import java.util.stream.Collectors;+import java.util.stream.Stream;import org.apache.lucene.codecs.CodecUtil;import org.apache.lucene.store.Directory;import org.apache.lucene.store.IOContext;@@ -1131,7 +1132,8 @@ public class Dictionary {Map<String, Integer> morphIndices = new HashMap<>();- WordStorage.Builder builder = new WordStorage.Builder(wordCount, hasCustomMorphData, flags);+ WordStorage.Builder builder =+ new WordStorage.Builder(wordCount, hasCustomMorphData, flags, allNonSuggestibleFlags());try (ByteSequencesReader reader =new ByteSequencesReader(tempDir.openChecksumInput(sorted, IOContext.READONCE), sorted)) {@@ -1197,6 +1199,13 @@ public class Dictionary {}}+ char[] allNonSuggestibleFlags() {+ return Dictionary.toSortedCharArray(+ Stream.of(HIDDEN_FLAG, noSuggest, forbiddenword, onlyincompound, subStandard)+ .filter(c -> c != FLAG_UNSET)+ .collect(Collectors.toSet()));+ }+private List<String> readMorphFields(String word, String unparsed) {List<String> morphFields = null;for (String datum : splitMorphData(unparsed)) {diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/GeneratingSuggester.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/GeneratingSuggester.javaindex 6d999fffbf5..08b58f925f9 100644--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/GeneratingSuggester.java+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/GeneratingSuggester.java@@ -19,8 +19,6 @@ package org.apache.lucene.analysis.hunspell;import static org.apache.lucene.analysis.hunspell.Dictionary.AFFIX_APPEND;import static org.apache.lucene.analysis.hunspell.Dictionary.AFFIX_FLAG;import static org.apache.lucene.analysis.hunspell.Dictionary.AFFIX_STRIP_ORD;-import static org.apache.lucene.analysis.hunspell.Dictionary.FLAG_UNSET;-import static org.apache.lucene.analysis.hunspell.Dictionary.HIDDEN_FLAG;import java.util.ArrayList;import java.util.Comparator;@@ -31,7 +29,6 @@ import java.util.PriorityQueue;import java.util.Set;import java.util.TreeSet;import java.util.stream.Collectors;-import java.util.stream.Stream;import org.apache.lucene.util.IntsRef;import org.apache.lucene.util.fst.FST;@@ -73,19 +70,10 @@ class GeneratingSuggester {}};- dictionary.words.processAllWords(- Math.max(1, word.length() - 4),- word.length() + 4,+ dictionary.words.processSuggestibleWords(+ Math.max(1, word.length() - MAX_ROOT_LENGTH_DIFF),+ word.length() + MAX_ROOT_LENGTH_DIFF,(rootChars, forms) -> {- assert rootChars.length > 0;- if (Math.abs(rootChars.length - word.length()) > MAX_ROOT_LENGTH_DIFF) {- assert rootChars.length < word.length(); // processAllWords takes care of longer keys- return;- }-- int suitable = filter.findSuitableFormIndex(forms, 0);- if (suitable < 0) return;-if (ignoreTitleCaseRoots&& Character.isUpperCase(rootChars.charAt(0))&& WordCase.caseOf(rootChars) == WordCase.TITLE) {@@ -99,13 +87,14 @@ class GeneratingSuggester {sc += commonPrefix(word, rootChars) - longerWorsePenalty(word.length(), rootChars.length);- if (roots.size() == MAX_ROOTS && sc < roots.peek().score) {+ if (roots.size() == MAX_ROOTS && sc <= roots.peek().score) {return;}speller.checkCanceled.run();String root = rootChars.toString();+ int suitable = filter.findSuitableFormIndex(forms, 0);do {roots.add(new Weighted<>(new Root<>(root, forms.ints[forms.offset + suitable]), sc));suitable = filter.findSuitableFormIndex(forms, suitable + filter.formStep);@@ -126,11 +115,7 @@ class GeneratingSuggester {EntryFilter(Dictionary dic) {formStep = dic.formStep();flagLookup = dic.flagLookup;-- Character[] flags = {HIDDEN_FLAG, dic.noSuggest, dic.forbiddenword, dic.onlyincompound};- excludeFlags =- Dictionary.toSortedCharArray(- Stream.of(flags).filter(c -> c != FLAG_UNSET).collect(Collectors.toSet()));+ excludeFlags = dic.allNonSuggestibleFlags();}int findSuitableFormIndex(IntsRef forms, int start) {diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/WordStorage.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/WordStorage.javaindex 4d457e1f306..b66428010b5 100644--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/WordStorage.java+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/WordStorage.java@@ -52,7 +52,8 @@ class WordStorage {private static final int OFFSET_BITS = 25;private static final int OFFSET_MASK = (1 << OFFSET_BITS) - 1;private static final int COLLISION_MASK = 0x40;- private static final int MAX_STORED_LENGTH = COLLISION_MASK - 1;+ private static final int SUGGESTIBLE_MASK = 0x20;+ private static final int MAX_STORED_LENGTH = SUGGESTIBLE_MASK - 1;/*** A map from word's hash (modulo array's length) into an int containing:@@ -60,9 +61,10 @@ class WordStorage {* <ul>* <li>lower {@link #OFFSET_BITS}: the offset in {@link #wordData} of the last entry with this* hash- * <li>the remaining highest bits: COLLISION+LENGTH info for that entry, i.e. one bit indicating- * whether there are other entries with the same hash, and the length of the entry in chars,- * or {@link #MAX_STORED_LENGTH} if the length exceeds that limit (next highest bits)+ * <li>the remaining highest bits: COLLISION+SUGGESTIBLE+LENGTH info for that entry, i.e. one+ * bit indicating whether there are other entries with the same hash, one bit indicating+ * whether this entry makes sense to be used in suggestions, and the length of the entry in+ * chars, or {@link #MAX_STORED_LENGTH} if the length exceeds that limit (next highest bits)* </ul>*/private final int[] hashTable;@@ -77,8 +79,8 @@ class WordStorage {* single-character entries* <li>(Optional, for hash-colliding entries only)* <ul>- * <li>BYTE: COLLISION+LENGTH info (see {@link #hashTable}) for the previous entry with- * the same hash+ * <li>BYTE: COLLISION+SUGGESTIBLE+LENGTH info (see {@link #hashTable}) for the previous+ * entry with the same hash* <li>VINT: (delta) pointer to the previous entry* </ul>* <li>(Optional, for non-leaf entries only) VINT+: word form data, returned from {@link@@ -140,12 +142,18 @@ class WordStorage {return (mask & COLLISION_MASK) != 0;}+ private static boolean hasSuggestibleEntries(int mask) {+ return (mask & SUGGESTIBLE_MASK) != 0;+ }+/*** Calls the processor for every dictionary entry with length between minLength and maxLength,- * both ends inclusive. Note that the callback arguments (word and forms) are reused, so they can- * be modified in any way, but may not be saved for later by the processor+ * both ends inclusive, and at least one suggestible alternative (without NOSUGGEST, FORBIDDENWORD+ * or ONLYINCOMPOUND flags). Note that the callback arguments (word and forms) are reused, so they+ * can be modified in any way, but may not be saved for later by the processor*/- void processAllWords(int minLength, int maxLength, BiConsumer<CharsRef, IntsRef> processor) {+ void processSuggestibleWords(+ int minLength, int maxLength, BiConsumer<CharsRef, IntsRef> processor) {assert minLength <= maxLength;CharsRef chars = new CharsRef(maxLength);IntsRef forms = new IntsRef();@@ -162,7 +170,8 @@ class WordStorage {int prevPos = pos - in.readVInt();boolean last = !hasCollision(mask);- boolean mightMatch = hasLengthInRange(mask, minLength, maxLength);+ boolean mightMatch =+ hasSuggestibleEntries(mask) && hasLengthInRange(mask, minLength, maxLength);if (!last) {mask = in.readByte();@@ -235,6 +244,7 @@ class WordStorage {private final boolean hasCustomMorphData;private final int[] hashTable;private byte[] wordData;+ private final char[] noSuggestFlags;private final int[] chainLengths;private final IntsRefBuilder currentOrds = new IntsRefBuilder();@@ -253,10 +263,15 @@ class WordStorage {* pre-size the hash table. This argument can be a bit larger than the actual word count,* but not smaller.*/- Builder(int wordCount, boolean hasCustomMorphData, FlagEnumerator flagEnumerator) {+ Builder(+ int wordCount,+ boolean hasCustomMorphData,+ FlagEnumerator flagEnumerator,+ char[] noSuggestFlags) {this.wordCount = wordCount;this.flagEnumerator = flagEnumerator;this.hasCustomMorphData = hasCustomMorphData;+ this.noSuggestFlags = noSuggestFlags;hashTable = new int[wordCount];wordData = new byte[wordCount * 6];@@ -317,7 +332,7 @@ class WordStorage {currentOrds.clear();boolean hasNonHidden = false;for (char[] flags : group) {- if (!hasHiddenFlag(flags)) {+ if (!hasFlag(flags, Dictionary.HIDDEN_FLAG)) {hasNonHidden = true;break;}@@ -325,7 +340,7 @@ class WordStorage {for (int i = 0; i < group.size(); i++) {char[] flags = group.get(i);- if (hasNonHidden && hasHiddenFlag(flags)) {+ if (hasNonHidden && hasFlag(flags, Dictionary.HIDDEN_FLAG)) {continue;}@@ -353,7 +368,9 @@ class WordStorage {int prevCode = hashTable[hash];int mask =- (prevCode == 0 ? 0 : COLLISION_MASK) | Math.min(currentEntry.length(), MAX_STORED_LENGTH);+ (prevCode == 0 ? 0 : COLLISION_MASK)+ | (group.stream().anyMatch(flags -> !hasNoSuggestFlag(flags)) ? SUGGESTIBLE_MASK : 0)+ | Math.min(currentEntry.length(), MAX_STORED_LENGTH);hashTable[hash] = (mask << OFFSET_BITS) | pos;if (++chainLengths[hash] > 20) {@@ -375,9 +392,16 @@ class WordStorage {return pos;}- private static boolean hasHiddenFlag(char[] flags) {+ private boolean hasNoSuggestFlag(char[] flags) {for (char flag : flags) {- if (flag == Dictionary.HIDDEN_FLAG) {+ if (hasFlag(noSuggestFlags, flag)) return true;+ }+ return false;+ }++ private static boolean hasFlag(char[] flags, char flag) {+ for (char f : flags) {+ if (f == flag) {return true;}}diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestDictionary.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestDictionary.javaindex b18b2a4015e..7b704d2e31c 100644--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestDictionary.java+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestDictionary.java@@ -86,11 +86,16 @@ public class TestDictionary extends LuceneTestCase {}}+ public void testProcessSuggestibleWords() throws Exception {+ Dictionary dictionary = loadDictionary("suggestible.aff", "suggestible.dic");++ Set<String> processed = processSuggestibleWords(dictionary, 1, 100);+ assertEquals(Set.of("normal", "ambiguous"), processed);+ }+private void checkProcessWords(Dictionary dictionary, Set<String> allWords, int minLength, int maxLength) {- Set<String> processed = new HashSet<>();- dictionary.words.processAllWords(- minLength, maxLength, (word, __) -> processed.add(word.toString()));+ Set<String> processed = processSuggestibleWords(dictionary, minLength, maxLength);Set<String> filtered =allWords.stream()@@ -100,6 +105,14 @@ public class TestDictionary extends LuceneTestCase {assertEquals("For lengths [" + minLength + "," + maxLength + "]", filtered, processed);}+ private static Set<String> processSuggestibleWords(+ Dictionary dictionary, int minLength, int maxLength) {+ Set<String> processed = new HashSet<>();+ dictionary.words.processSuggestibleWords(+ minLength, maxLength, (word, __) -> processed.add(word.toString()));+ return processed;+ }+public void testCompressedDictionary() throws Exception {Dictionary dictionary = loadDictionary("compressed.aff", "compressed.dic");assertEquals(3, dictionary.lookupSuffix(new char[] {'e'}).length);diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/suggestible.aff b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/suggestible.affnew file mode 100644index 00000000000..c79bafcede5--- /dev/null+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/suggestible.aff@@ -0,0 +1,4 @@+ONLYINCOMPOUND O+NOSUGGEST N+FORBIDDENWORD F+SUBSTANDARD S\ No newline at end of filediff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/suggestible.dic b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/suggestible.dicnew file mode 100644index 00000000000..7c899f7a678--- /dev/null+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/suggestible.dic@@ -0,0 +1,8 @@+1+normal+compound/O+forbidden/F+nosuggest/N+substandard/S+ambiguous+ambiguous/N\ No newline at end of file