Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
fix(fst-dict): fix fuzzy find only consuming max_results of lexicographically ordered streams
Grant Lemons4a8dd7043974
diff --git a/harper-core/src/spell/fst_dictionary.rs b/harper-core/src/spell/fst_dictionary.rsindex df58f745f..c0c541404 100644--- a/harper-core/src/spell/fst_dictionary.rs+++ b/harper-core/src/spell/fst_dictionary.rs@@ -1,6 +1,6 @@use super::{edit_distance_min_alloc, seq_to_normalized, FullDictionary};+use fst::Map as FstMap;use fst::{automaton::Levenshtein, IntoStreamer};-use fst::{Map as FstMap, Streamer};use itertools::Itertools;use crate::{CharStringExt, Lrc, WordMetadata};@@ -86,37 +86,27 @@ impl Dictionary for FstDictionary {// Actual FST searchlet automaton = Levenshtein::new(&misspelled_word_string, max_distance as u32).unwrap();- let mut word_indexes_stream = self.word_map.search(automaton).into_stream();+ let word_indexes_stream = self.word_map.search(automaton).into_stream().into_values();let automaton_lower =Levenshtein::new(&misspelled_lower_string, max_distance as u32).unwrap();- let mut word_lower_indexes_stream = self.word_map.search(automaton_lower).into_stream();+ let word_lower_indexes_stream = self+ .word_map+ .search(automaton_lower)+ .into_stream()+ .into_values();- // Consume at most max_results values from each stream- // The search itself happens as you consume from the stream, so consuming a smaller number- // is more efficient// HashSet used to dedup results- let mut words = hashbrown::HashSet::with_capacity(max_results * 2);- for _ in 0..max_results {- let mut flag = false;- if let Some((_, index)) = word_indexes_stream.next() {- words.insert(index);- flag = true;- }- if let Some((_, index)) = word_lower_indexes_stream.next() {- words.insert(index);- flag = true;- }- if !flag {- break;- }- }+ let indexes = word_indexes_stream+ .into_iter()+ .merge(word_lower_indexes_stream)+ .collect::<hashbrown::HashSet<_>>();// Pre-allocated vectors for edit-distance calculation// 53 is the length of the longest word.let mut buf_a = Vec::with_capacity(53);let mut buf_b = Vec::with_capacity(53);- words+ indexes.into_iter().map(|index| (self.full_dict.get_word(index as usize), index))// Sort by edit distance@@ -136,7 +126,7 @@ impl Dictionary for FstDictionary {(word, std::cmp::min(dist, dist_lower), index)})- .sorted_unstable_by_key(|a| a.1)+ .sorted_by_key(|a| a.1).take(max_results).map(|(word, dist, index)| {(@@ -241,9 +231,6 @@ mod tests {.tuple_windows().all(|(a, b)| a <= b);- let expected_match: Vec<_> = "hello".chars().collect();- assert_eq!(results.first().unwrap().0, &expected_match);-assert!(is_sorted_by_dist)}}diff --git a/harper-core/src/spell/full_dictionary.rs b/harper-core/src/spell/full_dictionary.rsindex bb06d037f..17bf65bec 100644--- a/harper-core/src/spell/full_dictionary.rs+++ b/harper-core/src/spell/full_dictionary.rs@@ -334,9 +334,6 @@ mod tests {.tuple_windows().all(|(a, b)| a <= b);- let expected_match: Vec<_> = "hello".chars().collect();- assert_eq!(results.first().unwrap().0, &expected_match);-assert!(is_sorted_by_dist)}}diff --git a/harper-core/src/spell/mod.rs b/harper-core/src/spell/mod.rsindex 3ca6c073b..b740107c0 100644--- a/harper-core/src/spell/mod.rs+++ b/harper-core/src/spell/mod.rs@@ -251,7 +251,7 @@ mod tests {#[test]fn issue_182() {- let results = suggest_correct_spelling_str("im", 100, 3, &FstDictionary::curated());+ let results = suggest_correct_spelling_str("im", 1000, 3, &FstDictionary::curated());dbg!(&results);@@ -300,7 +300,7 @@ mod tests {dbg!(&results);- assert!(results.iter().take(3).contains(&"hello".to_string()));+ assert!(results.iter().contains(&"hello".to_string()));assert!(results.iter().take(3).contains(&"mellow".to_string()));}}