Apache Lucene open-source search software
feat: Add API to use custom dictionary suggester (#16363)
Ilia Permiashkin4a724291f047
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex d51fb3955fa..d14c6e7753a 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -101,6 +101,8 @@ API Changes* GITHUB#16281: The checkIntegrity() method of codec Producer APIs gains a OneMerge parameter,which is used to check for aborted merges during checksum validation. (Tanguy Leroux)+* GITHUB#16363: Add API to use a custom `DictionarySuggester` instead of hardcoded `GeneratingSuggester`+New Features---------------------* GITHUB#16241: Add HangulCompositionCharFilter to analysis-nori for opt-in composition ofdiff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/DictionarySuggester.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/DictionarySuggester.javanew file mode 100644index 00000000000..81f005af7ba--- /dev/null+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/DictionarySuggester.java@@ -0,0 +1,41 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.analysis.hunspell;++import java.util.List;+import java.util.Set;++/** Generates suggestions by traversing dictionary entries. */+@FunctionalInterface+public interface DictionarySuggester {++ /**+ * @param hunspell spell checker configured for suggestion generation+ * @param suggestibleEntryCache optional cache from {@link Suggester#withSuggestibleEntryCache()}+ * or {@code null}+ * @param lowerCase the lower-case misspelled word+ * @param wordCase case of the misspelled word before lower-casing+ * @param suggestions suggestions already found by earlier phases+ * @return raw suggestions to add+ */+ List<String> suggest(+ Hunspell hunspell,+ SuggestibleEntryCache suggestibleEntryCache,+ String lowerCase,+ WordCase wordCase,+ Set<Suggestion> suggestions);+}diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/FlyweightEntry.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/FlyweightEntry.javaindex 337a9b5c0c0..6652381f26d 100644--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/FlyweightEntry.java+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/FlyweightEntry.java@@ -19,13 +19,27 @@ package org.apache.lucene.analysis.hunspell;import org.apache.lucene.util.CharsRef;import org.apache.lucene.util.IntsRef;-/** A mutable entry object used when enumerating the dictionary internally */-abstract class FlyweightEntry {- abstract boolean hasTitleCase();+/** A mutable entry object used when enumerating dictionary entries. */+public abstract class FlyweightEntry {+ FlyweightEntry() {}- abstract CharsRef root();+ /**+ * @return whether this entry's root is title-cased+ */+ public abstract boolean hasTitleCase();- abstract CharSequence lowerCaseRoot();+ /**+ * @return the dictionary root+ */+ public abstract CharsRef root();- abstract IntsRef forms();+ /**+ * @return the lower-case dictionary root+ */+ public abstract CharSequence lowerCaseRoot();++ /**+ * @return encoded form data for this entry+ */+ public abstract IntsRef forms();}diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Suggester.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Suggester.javaindex 1edf2888c7c..4b361da8602 100644--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Suggester.java+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Suggester.java@@ -54,20 +54,30 @@ public class Suggester {private final SuggestibleEntryCache suggestibleCache;private final FragmentChecker fragmentChecker;private final boolean proceedPastRep;+ private final DictionarySuggester suggester;public Suggester(Dictionary dictionary) {- this(dictionary, null, FragmentChecker.EVERYTHING_POSSIBLE, false);+ this(+ dictionary,+ null,+ FragmentChecker.EVERYTHING_POSSIBLE,+ false,+ (speller, entryCache, word, originalCase, prevSuggestions) ->+ new GeneratingSuggester(speller, entryCache)+ .suggest(word, originalCase, prevSuggestions));}private Suggester(Dictionary dictionary,SuggestibleEntryCache suggestibleCache,FragmentChecker checker,- boolean proceedPastRep) {+ boolean proceedPastRep,+ DictionarySuggester suggester) {this.dictionary = dictionary;this.suggestibleCache = suggestibleCache;this.fragmentChecker = checker;this.proceedPastRep = proceedPastRep;+ this.suggester = suggester;}/**@@ -77,7 +87,7 @@ public class Suggester {*/public Suggester withSuggestibleEntryCache() {SuggestibleEntryCache cache = SuggestibleEntryCache.buildCache(dictionary.words);- return new Suggester(dictionary, cache, fragmentChecker, proceedPastRep);+ return new Suggester(dictionary, cache, fragmentChecker, proceedPastRep, suggester);}/**@@ -85,7 +95,7 @@ public class Suggester {* the performance of the "Modification" phase performance.*/public Suggester withFragmentChecker(FragmentChecker checker) {- return new Suggester(dictionary, suggestibleCache, checker, proceedPastRep);+ return new Suggester(dictionary, suggestibleCache, checker, proceedPastRep, suggester);}/**@@ -95,7 +105,16 @@ public class Suggester {* not "times", which could also be meant.*/public Suggester proceedPastRep() {- return new Suggester(dictionary, suggestibleCache, fragmentChecker, true);+ return new Suggester(dictionary, suggestibleCache, fragmentChecker, true, suggester);+ }++ /**+ * Returns a copy of this suggester instance that uses custom {@link DictionarySuggester} instead+ * of default {@link GeneratingSuggester}+ */+ public Suggester withDictionarySuggester(DictionarySuggester dictionarySuggester) {+ return new Suggester(+ dictionary, suggestibleCache, fragmentChecker, proceedPastRep, dictionarySuggester);}/**@@ -195,8 +214,12 @@ public class Suggester {if (!hasGoodSuggestions && dictionary.maxNGramSuggestions > 0) {List<String> generated =- new GeneratingSuggester(suggestionSpeller, suggestibleCache)- .suggest(dictionary.toLowerCase(word), wordCase, suggestions);+ this.suggester.suggest(+ suggestionSpeller,+ suggestibleCache,+ dictionary.toLowerCase(word),+ wordCase,+ suggestions);for (String raw : generated) {suggestions.add(new Suggestion(raw, word, wordCase, suggestionSpeller));}diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/SuggestibleEntryCache.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/SuggestibleEntryCache.javaindex 23a594d91f0..01ecb3e5550 100644--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/SuggestibleEntryCache.java+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/SuggestibleEntryCache.java@@ -27,7 +27,7 @@ import org.apache.lucene.util.IntsRef;* used for suggestions. The words and the form data are stored in plain contiguous arrays with no* compression.*/-class SuggestibleEntryCache {+public class SuggestibleEntryCache {private static final short LOWER_CASE = (short) WordCase.LOWER.ordinal();private static final short NEUTRAL_CASE = (short) WordCase.NEUTRAL.ordinal();private static final short TITLE_CASE = (short) WordCase.TITLE.ordinal();@@ -120,7 +120,13 @@ class SuggestibleEntryCache {return rootCase != LOWER_CASE && rootCase != NEUTRAL_CASE;}- void processSuggestibleWords(int minLength, int maxLength, Consumer<FlyweightEntry> processor) {+ /**+ * Calls the processor for every cached dictionary entry with length between {@code minLength} and+ * {@code maxLength}, both ends inclusive. The callback entry is reused and may not be saved for+ * later.+ */+ public void processSuggestibleWords(+ int minLength, int maxLength, Consumer<FlyweightEntry> processor) {maxLength = Math.min(maxLength, sections.length - 1);for (int i = Math.min(minLength, sections.length); i <= maxLength; i++) {Section section = sections[i];@@ -147,22 +153,22 @@ class SuggestibleEntryCache {short wordCase;@Override- CharsRef root() {+ public CharsRef root() {return hasUpperCase(wordCase) ? chars : lowerChars;}@Override- boolean hasTitleCase() {+ public boolean hasTitleCase() {return wordCase == TITLE_CASE;}@Override- CharSequence lowerCaseRoot() {+ public CharSequence lowerCaseRoot() {return lowerChars;}@Override- IntsRef forms() {+ public IntsRef forms() {return forms;}};diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Suggestion.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Suggestion.javaindex 49ecd20c32d..adb8db34518 100644--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Suggestion.java+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Suggestion.java@@ -22,7 +22,8 @@ import java.util.List;import java.util.Locale;import java.util.Objects;-class Suggestion {+/** A spelling suggestion with its raw and final output forms. */+public class Suggestion {final String raw;final String[] result;@@ -39,6 +40,20 @@ class Suggestion {this.result = result.toArray(String[]::new);}+ /**+ * @return the raw suggestion before output conversion and case adjustment+ */+ public String getRaw() {+ return raw;+ }++ /**+ * @return the final spell-check suggestions produced from this raw suggestion+ */+ public List<String> getResult() {+ return List.of(result);+ }+private String adjustSuggestionCase(String candidate, String misspelled, WordCase originalCase) {if (originalCase == WordCase.UPPER) {return candidate.toUpperCase(Locale.ROOT);diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/WordCase.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/WordCase.javaindex 36188a085e3..b018270d0ee 100644--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/WordCase.java+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/WordCase.java@@ -16,7 +16,8 @@*/package org.apache.lucene.analysis.hunspell;-enum WordCase {+/** Case pattern of a word. */+public enum WordCase {/** e.g. WORD */UPPER,diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/WordStorage.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/WordStorage.javaindex b107b5d7c25..f686fcd65a2 100644--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/WordStorage.java+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/WordStorage.java@@ -471,22 +471,22 @@ abstract class WordStorage {}@Override- boolean hasTitleCase() {+ public boolean hasTitleCase() {return Character.isUpperCase(chars.charAt(0)) && WordCase.caseOf(chars) == WordCase.TITLE;}@Override- CharsRef root() {+ public CharsRef root() {return chars;}@Override- CharSequence lowerCaseRoot() {+ public CharSequence lowerCaseRoot() {return lower;}@Override- IntsRef forms() {+ public IntsRef forms() {in.setPosition(dataPos);int entryCount = in.readVInt() / (hasCustomMorphData ? 2 : 1);if (forms.ints.length < entryCount) {