Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
feat(fst-dict): change how fuzzy search handles lowercase
Grant Lemonsfeb9c3b93e3c
diff --git a/harper-core/src/spell/fst_dictionary.rs b/harper-core/src/spell/fst_dictionary.rsindex c0c541404..ea1c539d6 100644--- a/harper-core/src/spell/fst_dictionary.rs+++ b/harper-core/src/spell/fst_dictionary.rs@@ -82,51 +82,33 @@ impl Dictionary for FstDictionary {let misspelled_word_charslice = seq_to_normalized(&chars);let misspelled_lower_charslice = misspelled_word_charslice.to_lower();let misspelled_word_string = misspelled_word_charslice.to_string();- let misspelled_lower_string = misspelled_lower_charslice.to_string();// Actual FST searchlet automaton = Levenshtein::new(&misspelled_word_string, max_distance as u32).unwrap();let word_indexes_stream = self.word_map.search(automaton).into_stream().into_values();- let automaton_lower =- Levenshtein::new(&misspelled_lower_string, max_distance as u32).unwrap();- let word_lower_indexes_stream = self- .word_map- .search(automaton_lower)- .into_stream()- .into_values();-- // HashSet used to dedup results- let indexes = word_indexes_stream- .into_iter()- .merge(word_lower_indexes_stream)- .collect::<hashbrown::HashSet<_>>();// Pre-allocated vectors for edit-distance calculation// 53 is the length of the longest word.let mut buf_a = Vec::with_capacity(53);let mut buf_b = Vec::with_capacity(53);- indexes+ word_indexes_stream.into_iter()+ .sorted_unstable()+ .dedup().map(|index| (self.full_dict.get_word(index as usize), index))// Sort by edit distance.map(|(word, index)| {let dist = edit_distance_min_alloc(- &misspelled_word_charslice,- word,- &mut buf_a,- &mut buf_b,- );- let dist_lower = edit_distance_min_alloc(&misspelled_lower_charslice,- word,+ &word.to_lower(),&mut buf_a,&mut buf_b,);- (word, std::cmp::min(dist, dist_lower), index)+ (word, dist, index)})- .sorted_by_key(|a| a.1)+ .sorted_unstable_by_key(|a| a.1).take(max_results).map(|(word, dist, index)| {(diff --git a/harper-core/src/spell/full_dictionary.rs b/harper-core/src/spell/full_dictionary.rsindex 17bf65bec..6a9bbbbec 100644--- a/harper-core/src/spell/full_dictionary.rs+++ b/harper-core/src/spell/full_dictionary.rs@@ -193,21 +193,21 @@ impl Dictionary for FullDictionary {// 53 is the length of the longest word.let mut buf_a = Vec::with_capacity(53);let mut buf_b = Vec::with_capacity(53);- let pruned_words = words_to_search.filter_map(|word| {- let dist =- edit_distance_min_alloc(&misspelled_normalized, word, &mut buf_a, &mut buf_b);- let dist_lower =- edit_distance_min_alloc(&misspelled_lower, word, &mut buf_a, &mut buf_b);- let smallest_dist = std::cmp::min(dist, dist_lower);-- if smallest_dist <= max_distance {- Some((word, smallest_dist))- } else {- None- }- });-- pruned_words+ words_to_search+ .filter_map(|word| {+ let dist = edit_distance_min_alloc(+ &misspelled_lower,+ &word.to_lower(),+ &mut buf_a,+ &mut buf_b,+ );++ if dist <= max_distance {+ Some((word, dist))+ } else {+ None+ }+ }).sorted_by_key(|a| a.1).take(max_results).map(|(word, dist)| (word, dist, self.get_word_metadata(word)))diff --git a/harper-core/src/spell/mod.rs b/harper-core/src/spell/mod.rsindex b740107c0..492a51a6f 100644--- a/harper-core/src/spell/mod.rs+++ b/harper-core/src/spell/mod.rs@@ -230,32 +230,28 @@ mod tests {}#[test]- fn identical_results_between_dicts() {- let fst_results = suggest_correct_spelling_str("im", 100, 3, &FstDictionary::curated());- let full_results = suggest_correct_spelling_str("im", 100, 3, &FullDictionary::curated());+ fn full_dict_fuzzy_find() {+ let results = suggest_correct_spelling_str("anout", 100, 3, &FullDictionary::curated());- assert_eq!(fst_results, full_results);-- let fst_results = suggest_correct_spelling_str("hvllo", 100, 3, &FstDictionary::curated());- let full_results =- suggest_correct_spelling_str("hvllo", 100, 3, &FullDictionary::curated());-- assert_eq!(fst_results, full_results);-- let fst_results = suggest_correct_spelling_str("aboot", 100, 3, &FstDictionary::curated());- let full_results =- suggest_correct_spelling_str("aboot", 100, 3, &FullDictionary::curated());+ dbg!(&results);- assert_eq!(fst_results, full_results);+ assert!(results.iter().take(3).contains(&"about".to_string()));}#[test]fn issue_182() {- let results = suggest_correct_spelling_str("im", 1000, 3, &FstDictionary::curated());+ let results = suggest_correct_spelling_str("Im", 100, 3, &FstDictionary::curated());+ let lowercase_results =+ suggest_correct_spelling_str("im", 100, 3, &FstDictionary::curated());dbg!(&results);+ dbg!(&lowercase_results);assert!(results.iter().take(3).contains(&"I'm".to_string()));+ assert!(lowercase_results+ .iter()+ .take(3)+ .contains(&"I'm".to_string()));}#[test]@@ -295,12 +291,13 @@ mod tests {}#[test]- fn spellcheck_multiple_choices() {- let results = suggest_correct_spelling_str("mello", 100, 3, &FstDictionary::curated());-- dbg!(&results);-- assert!(results.iter().contains(&"hello".to_string()));- assert!(results.iter().take(3).contains(&"mellow".to_string()));+ fn spellchecking_is_deterministic() {+ let results1 = suggest_correct_spelling_str("hello", 100, 3, &FstDictionary::curated());+ let results2 = suggest_correct_spelling_str("hello", 100, 3, &FstDictionary::curated());+ let results3 = suggest_correct_spelling_str("hello", 100, 3, &FstDictionary::curated());++ assert_eq!(results1, results2);+ assert_eq!(results1, results3);+ assert_eq!(results2, results3);}}