Skip to content

Apache Lucene open-source search software

feat: Add API to use custom dictionary suggester (#16363)

Ilia Permiashkin4a724291f047
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index d51fb3955fa..d14c6e7753a 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -101,6 +101,8 @@ API Changes
* GITHUB#16281: The checkIntegrity() method of codec Producer APIs gains a OneMerge parameter,
which is used to check for aborted merges during checksum validation. (Tanguy Leroux)
+* GITHUB#16363: Add API to use a custom `DictionarySuggester` instead of hardcoded `GeneratingSuggester`
+
New Features
---------------------
* GITHUB#16241: Add HangulCompositionCharFilter to analysis-nori for opt-in composition of
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/DictionarySuggester.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/DictionarySuggester.java
new file mode 100644
index 00000000000..81f005af7ba
--- /dev/null
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/DictionarySuggester.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.analysis.hunspell;
+
+import java.util.List;
+import java.util.Set;
+
+/** Generates suggestions by traversing dictionary entries. */
+@FunctionalInterface
+public interface DictionarySuggester {
+
+ /**
+ * @param hunspell spell checker configured for suggestion generation
+ * @param suggestibleEntryCache optional cache from {@link Suggester#withSuggestibleEntryCache()}
+ * or {@code null}
+ * @param lowerCase the lower-case misspelled word
+ * @param wordCase case of the misspelled word before lower-casing
+ * @param suggestions suggestions already found by earlier phases
+ * @return raw suggestions to add
+ */
+ List<String> suggest(
+ Hunspell hunspell,
+ SuggestibleEntryCache suggestibleEntryCache,
+ String lowerCase,
+ WordCase wordCase,
+ Set<Suggestion> suggestions);
+}
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/FlyweightEntry.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/FlyweightEntry.java
index 337a9b5c0c0..6652381f26d 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/FlyweightEntry.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/FlyweightEntry.java
@@ -19,13 +19,27 @@ package org.apache.lucene.analysis.hunspell;
import org.apache.lucene.util.CharsRef;
import org.apache.lucene.util.IntsRef;
-/** A mutable entry object used when enumerating the dictionary internally */
-abstract class FlyweightEntry {
- abstract boolean hasTitleCase();
+/** A mutable entry object used when enumerating dictionary entries. */
+public abstract class FlyweightEntry {
+ FlyweightEntry() {}
- abstract CharsRef root();
+ /**
+ * @return whether this entry's root is title-cased
+ */
+ public abstract boolean hasTitleCase();
- abstract CharSequence lowerCaseRoot();
+ /**
+ * @return the dictionary root
+ */
+ public abstract CharsRef root();
- abstract IntsRef forms();
+ /**
+ * @return the lower-case dictionary root
+ */
+ public abstract CharSequence lowerCaseRoot();
+
+ /**
+ * @return encoded form data for this entry
+ */
+ public abstract IntsRef forms();
}
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Suggester.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Suggester.java
index 1edf2888c7c..4b361da8602 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Suggester.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Suggester.java
@@ -54,20 +54,30 @@ public class Suggester {
private final SuggestibleEntryCache suggestibleCache;
private final FragmentChecker fragmentChecker;
private final boolean proceedPastRep;
+ private final DictionarySuggester suggester;
public Suggester(Dictionary dictionary) {
- this(dictionary, null, FragmentChecker.EVERYTHING_POSSIBLE, false);
+ this(
+ dictionary,
+ null,
+ FragmentChecker.EVERYTHING_POSSIBLE,
+ false,
+ (speller, entryCache, word, originalCase, prevSuggestions) ->
+ new GeneratingSuggester(speller, entryCache)
+ .suggest(word, originalCase, prevSuggestions));
}
private Suggester(
Dictionary dictionary,
SuggestibleEntryCache suggestibleCache,
FragmentChecker checker,
- boolean proceedPastRep) {
+ boolean proceedPastRep,
+ DictionarySuggester suggester) {
this.dictionary = dictionary;
this.suggestibleCache = suggestibleCache;
this.fragmentChecker = checker;
this.proceedPastRep = proceedPastRep;
+ this.suggester = suggester;
}
/**
@@ -77,7 +87,7 @@ public class Suggester {
*/
public Suggester withSuggestibleEntryCache() {
SuggestibleEntryCache cache = SuggestibleEntryCache.buildCache(dictionary.words);
- return new Suggester(dictionary, cache, fragmentChecker, proceedPastRep);
+ return new Suggester(dictionary, cache, fragmentChecker, proceedPastRep, suggester);
}
/**
@@ -85,7 +95,7 @@ public class Suggester {
* the performance of the "Modification" phase performance.
*/
public Suggester withFragmentChecker(FragmentChecker checker) {
- return new Suggester(dictionary, suggestibleCache, checker, proceedPastRep);
+ return new Suggester(dictionary, suggestibleCache, checker, proceedPastRep, suggester);
}
/**
@@ -95,7 +105,16 @@ public class Suggester {
* not "times", which could also be meant.
*/
public Suggester proceedPastRep() {
- return new Suggester(dictionary, suggestibleCache, fragmentChecker, true);
+ return new Suggester(dictionary, suggestibleCache, fragmentChecker, true, suggester);
+ }
+
+ /**
+ * Returns a copy of this suggester instance that uses custom {@link DictionarySuggester} instead
+ * of default {@link GeneratingSuggester}
+ */
+ public Suggester withDictionarySuggester(DictionarySuggester dictionarySuggester) {
+ return new Suggester(
+ dictionary, suggestibleCache, fragmentChecker, proceedPastRep, dictionarySuggester);
}
/**
@@ -195,8 +214,12 @@ public class Suggester {
if (!hasGoodSuggestions && dictionary.maxNGramSuggestions > 0) {
List<String> generated =
- new GeneratingSuggester(suggestionSpeller, suggestibleCache)
- .suggest(dictionary.toLowerCase(word), wordCase, suggestions);
+ this.suggester.suggest(
+ suggestionSpeller,
+ suggestibleCache,
+ dictionary.toLowerCase(word),
+ wordCase,
+ suggestions);
for (String raw : generated) {
suggestions.add(new Suggestion(raw, word, wordCase, suggestionSpeller));
}
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/SuggestibleEntryCache.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/SuggestibleEntryCache.java
index 23a594d91f0..01ecb3e5550 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/SuggestibleEntryCache.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/SuggestibleEntryCache.java
@@ -27,7 +27,7 @@ import org.apache.lucene.util.IntsRef;
* used for suggestions. The words and the form data are stored in plain contiguous arrays with no
* compression.
*/
-class SuggestibleEntryCache {
+public class SuggestibleEntryCache {
private static final short LOWER_CASE = (short) WordCase.LOWER.ordinal();
private static final short NEUTRAL_CASE = (short) WordCase.NEUTRAL.ordinal();
private static final short TITLE_CASE = (short) WordCase.TITLE.ordinal();
@@ -120,7 +120,13 @@ class SuggestibleEntryCache {
return rootCase != LOWER_CASE && rootCase != NEUTRAL_CASE;
}
- void processSuggestibleWords(int minLength, int maxLength, Consumer<FlyweightEntry> processor) {
+ /**
+ * Calls the processor for every cached dictionary entry with length between {@code minLength} and
+ * {@code maxLength}, both ends inclusive. The callback entry is reused and may not be saved for
+ * later.
+ */
+ public void processSuggestibleWords(
+ int minLength, int maxLength, Consumer<FlyweightEntry> processor) {
maxLength = Math.min(maxLength, sections.length - 1);
for (int i = Math.min(minLength, sections.length); i <= maxLength; i++) {
Section section = sections[i];
@@ -147,22 +153,22 @@ class SuggestibleEntryCache {
short wordCase;
@Override
- CharsRef root() {
+ public CharsRef root() {
return hasUpperCase(wordCase) ? chars : lowerChars;
}
@Override
- boolean hasTitleCase() {
+ public boolean hasTitleCase() {
return wordCase == TITLE_CASE;
}
@Override
- CharSequence lowerCaseRoot() {
+ public CharSequence lowerCaseRoot() {
return lowerChars;
}
@Override
- IntsRef forms() {
+ public IntsRef forms() {
return forms;
}
};
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Suggestion.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Suggestion.java
index 49ecd20c32d..adb8db34518 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Suggestion.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Suggestion.java
@@ -22,7 +22,8 @@ import java.util.List;
import java.util.Locale;
import java.util.Objects;
-class Suggestion {
+/** A spelling suggestion with its raw and final output forms. */
+public class Suggestion {
final String raw;
final String[] result;
@@ -39,6 +40,20 @@ class Suggestion {
this.result = result.toArray(String[]::new);
}
+ /**
+ * @return the raw suggestion before output conversion and case adjustment
+ */
+ public String getRaw() {
+ return raw;
+ }
+
+ /**
+ * @return the final spell-check suggestions produced from this raw suggestion
+ */
+ public List<String> getResult() {
+ return List.of(result);
+ }
+
private String adjustSuggestionCase(String candidate, String misspelled, WordCase originalCase) {
if (originalCase == WordCase.UPPER) {
return candidate.toUpperCase(Locale.ROOT);
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/WordCase.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/WordCase.java
index 36188a085e3..b018270d0ee 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/WordCase.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/WordCase.java
@@ -16,7 +16,8 @@
*/
package org.apache.lucene.analysis.hunspell;
-enum WordCase {
+/** Case pattern of a word. */
+public enum WordCase {
/** e.g. WORD */
UPPER,
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/WordStorage.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/WordStorage.java
index b107b5d7c25..f686fcd65a2 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/WordStorage.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/WordStorage.java
@@ -471,22 +471,22 @@ abstract class WordStorage {
}
@Override
- boolean hasTitleCase() {
+ public boolean hasTitleCase() {
return Character.isUpperCase(chars.charAt(0)) && WordCase.caseOf(chars) == WordCase.TITLE;
}
@Override
- CharsRef root() {
+ public CharsRef root() {
return chars;
}
@Override
- CharSequence lowerCaseRoot() {
+ public CharSequence lowerCaseRoot() {
return lower;
}
@Override
- IntsRef forms() {
+ public IntsRef forms() {
in.setPosition(dataPos);
int entryCount = in.readVInt() / (hasCustomMorphData ? 2 : 1);
if (forms.ints.length < entryCount) {