Skip to content

Apache Lucene open-source search software

WordBreakSpellChecker.suggestWordBreaks now does a breadth first search (#12100)

Chris Hostettercc854f408fd0
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 26a7c06e483..d67318da678 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -282,6 +282,9 @@ Optimizations
* GITHUB#13538: Slightly reduce heap usage for HNSW and scalar quantized vector writers. (Ben Trent)
+* GITHUB#12100: WordBreakSpellChecker.suggestWordBreaks now does a breadth first search, allowing it to return
+ better matches with fewer evaluations (hossman)
+
Changes in runtime behavior
---------------------
diff --git a/lucene/suggest/src/java/org/apache/lucene/search/spell/WordBreakSpellChecker.java b/lucene/suggest/src/java/org/apache/lucene/search/spell/WordBreakSpellChecker.java
index b5572f717d6..b3a84e9aaf8 100644
--- a/lucene/suggest/src/java/org/apache/lucene/search/spell/WordBreakSpellChecker.java
+++ b/lucene/suggest/src/java/org/apache/lucene/search/spell/WordBreakSpellChecker.java
@@ -22,6 +22,9 @@ import java.util.PriorityQueue;
import java.util.Queue;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.Term;
+import org.apache.lucene.util.BitSet;
+import org.apache.lucene.util.BitSetIterator;
+import org.apache.lucene.util.FixedBitSet;
/**
* A spell checker whose sole function is to offer suggestions by combining multiple terms into one
@@ -246,51 +249,80 @@ public class WordBreakSpellChecker {
int totalEvaluations,
BreakSuggestionSortMethod sortMethod)
throws IOException {
- String termText = term.text();
- int termLength = termText.codePointCount(0, termText.length());
- int useMinBreakWordLength = minBreakWordLength;
- if (useMinBreakWordLength < 1) {
- useMinBreakWordLength = 1;
- }
+ final String termText = term.text();
+ final int termLength = termText.codePointCount(0, termText.length());
+ final int useMinBreakWordLength = Math.max(minBreakWordLength, 1);
if (termLength < (useMinBreakWordLength * 2)) {
return totalEvaluations;
}
- for (int i = useMinBreakWordLength; i <= (termLength - useMinBreakWordLength); i++) {
+ // Two phase breadth first search.
+ //
+ // Phase #1: checking for bi-sects of our termText, and recording the
+ // positions of valid leftText splits for later
+
+ final int maxSplitPosition = termLength - useMinBreakWordLength;
+ final BitSet validLeftSplitPositions = new FixedBitSet(termText.length());
+
+ for (int i = useMinBreakWordLength; i <= maxSplitPosition; i++) {
if (totalEvaluations >= maxEvaluations) {
- break;
+ return totalEvaluations;
}
totalEvaluations++;
- int end = termText.offsetByCodePoints(0, i);
- String leftText = termText.substring(0, end);
- String rightText = termText.substring(end);
- SuggestWord leftWord = generateSuggestWord(ir, term.field(), leftText);
+ final int end = termText.offsetByCodePoints(0, i);
+ final String leftText = termText.substring(0, end);
+ final String rightText = termText.substring(end);
+
+ final SuggestWord leftWord = generateSuggestWord(ir, term.field(), leftText);
if (leftWord.freq >= useMinSuggestionFrequency) {
- SuggestWord rightWord = generateSuggestWord(ir, term.field(), rightText);
+ validLeftSplitPositions.set(end);
+ final SuggestWord rightWord = generateSuggestWord(ir, term.field(), rightText);
if (rightWord.freq >= useMinSuggestionFrequency) {
- SuggestWordArrayWrapper suggestion =
- new SuggestWordArrayWrapper(newSuggestion(prefix, leftWord, rightWord));
- suggestions.offer(suggestion);
+ suggestions.offer(
+ new SuggestWordArrayWrapper(newSuggestion(prefix, leftWord, rightWord)));
if (suggestions.size() > maxSuggestions) {
suggestions.poll();
}
}
- int newNumberBreaks = numberBreaks + 1;
- if (newNumberBreaks <= maxChanges) {
- totalEvaluations =
- generateBreakUpSuggestions(
- new Term(term.field(), rightWord.string),
- ir,
- newNumberBreaks,
- maxSuggestions,
- useMinSuggestionFrequency,
- newPrefix(prefix, leftWord),
- suggestions,
- totalEvaluations,
- sortMethod);
- }
+ }
+ }
+
+ // if we are about to exceed our maxChanges *OR* we have a full list of
+ // suggestions, we can return now.
+ //
+ // (because any subsequent suggestions are garunteed to have more changes
+ // then anything currently in the queue, and not be competitive)
+
+ final int newNumberBreaks = numberBreaks + 1;
+ if (totalEvaluations >= maxEvaluations
+ || newNumberBreaks > maxChanges
+ || suggestions.size() >= maxSuggestions) {
+ return totalEvaluations;
+ }
+
+ // Phase #2: recursing on the right side of any valid leftText terms
+ final BitSetIterator leftIter = new BitSetIterator(validLeftSplitPositions, 0);
+ for (int pos = leftIter.nextDoc();
+ pos != BitSetIterator.NO_MORE_DOCS;
+ pos = leftIter.nextDoc()) {
+ final String leftText = termText.substring(0, pos);
+ final String rightText = termText.substring(pos);
+ final SuggestWord leftWord = generateSuggestWord(ir, term.field(), leftText);
+ totalEvaluations =
+ generateBreakUpSuggestions(
+ new Term(term.field(), rightText),
+ ir,
+ newNumberBreaks,
+ maxSuggestions,
+ useMinSuggestionFrequency,
+ newPrefix(prefix, leftWord),
+ suggestions,
+ totalEvaluations,
+ sortMethod);
+ if (totalEvaluations >= maxEvaluations) {
+ break;
}
}
diff --git a/lucene/suggest/src/test/org/apache/lucene/search/spell/TestWordBreakSpellChecker.java b/lucene/suggest/src/test/org/apache/lucene/search/spell/TestWordBreakSpellChecker.java
index 83b7066fa0b..38121ced691 100644
--- a/lucene/suggest/src/test/org/apache/lucene/search/spell/TestWordBreakSpellChecker.java
+++ b/lucene/suggest/src/test/org/apache/lucene/search/spell/TestWordBreakSpellChecker.java
@@ -89,20 +89,57 @@ public class TestWordBreakSpellChecker extends LuceneTestCase {
}
public void testMaxEvaluations() throws Exception {
- final int maxEvals = 100;
+ try (IndexReader ir = DirectoryReader.open(dir)) {
+
+ final String input = "ab".repeat(5);
+ final int maxEvals = 100;
+ final int maxSuggestions = maxEvals * 2; // plenty
+
+ WordBreakSpellChecker wbsp = new WordBreakSpellChecker();
+ wbsp.setMinBreakWordLength(1);
+ wbsp.setMinSuggestionFrequency(1);
+
+ wbsp.setMaxChanges(2 * input.length()); // plenty
+ wbsp.setMaxEvaluations(maxEvals);
+
+ Term term = new Term("abba", input);
+ SuggestWord[][] sw =
+ wbsp.suggestWordBreaks(
+ term,
+ maxSuggestions,
+ ir,
+ SuggestMode.SUGGEST_WHEN_NOT_IN_INDEX,
+ BreakSuggestionSortMethod.NUM_CHANGES_THEN_MAX_FREQUENCY);
+
+ // sanity check that our suggester isn't completely broken
+ MatcherAssert.assertThat(sw.length, greaterThan(0));
+
+ // if maxEvaluations is respected, we can't possibly have more suggestions than that
+ MatcherAssert.assertThat(sw.length, lessThan(maxEvals));
+ }
+ }
+
+ public void testSmallMaxEvaluations() throws Exception {
+ // even using small maxEvals (relative to maxChanges) should produce
+ // good results if possible
try (IndexReader ir = DirectoryReader.open(dir)) {
+
+ final int maxEvals = TestUtil.nextInt(random(), 6, 20);
+ final int maxSuggestions = maxEvals * 10; // plenty
+
WordBreakSpellChecker wbsp = new WordBreakSpellChecker();
- wbsp.setMaxChanges(10);
wbsp.setMinBreakWordLength(1);
wbsp.setMinSuggestionFrequency(1);
- wbsp.setMaxEvaluations(100);
- Term term = new Term("abba", "ab".repeat(5));
+ wbsp.setMaxChanges(maxEvals * 10); // plenty
+ wbsp.setMaxEvaluations(maxEvals);
+
+ Term term = new Term("abba", "ababab");
SuggestWord[][] sw =
wbsp.suggestWordBreaks(
term,
- 500,
+ maxSuggestions,
ir,
SuggestMode.SUGGEST_WHEN_NOT_IN_INDEX,
BreakSuggestionSortMethod.NUM_CHANGES_THEN_MAX_FREQUENCY);
@@ -112,6 +149,12 @@ public class TestWordBreakSpellChecker extends LuceneTestCase {
// if maxEvaluations is respected, we can't possibly have more suggestions than that
MatcherAssert.assertThat(sw.length, lessThan(maxEvals));
+
+ // we should have been able to find this "optimal" (due to fewest num changes)
+ // suggestion before hitting our small maxEvals (and before any suggests with more breaks)
+ assertEquals(2, sw[0].length);
+ assertEquals("aba", sw[0][0].string);
+ assertEquals("bab", sw[0][1].string);
}
}