Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
feat(fst-dict): refactor and add more tests
Grant Lemons11d37944f98f
diff --git a/harper-core/src/spell/fst_dictionary.rs b/harper-core/src/spell/fst_dictionary.rsindex 68f2d0b14..df58f745f 100644--- a/harper-core/src/spell/fst_dictionary.rs+++ b/harper-core/src/spell/fst_dictionary.rs@@ -86,10 +86,10 @@ impl Dictionary for FstDictionary {// Actual FST searchlet automaton = Levenshtein::new(&misspelled_word_string, max_distance as u32).unwrap();+ let mut word_indexes_stream = self.word_map.search(automaton).into_stream();let automaton_lower =Levenshtein::new(&misspelled_lower_string, max_distance as u32).unwrap();let mut word_lower_indexes_stream = self.word_map.search(automaton_lower).into_stream();- let mut word_indexes_stream = self.word_map.search(automaton).into_stream();// Consume at most max_results values from each stream// The search itself happens as you consume from the stream, so consuming a smaller number@@ -98,11 +98,11 @@ impl Dictionary for FstDictionary {let mut words = hashbrown::HashSet::with_capacity(max_results * 2);for _ in 0..max_results {let mut flag = false;- if let Some((_, index)) = word_lower_indexes_stream.next() {+ if let Some((_, index)) = word_indexes_stream.next() {words.insert(index);flag = true;}- if let Some((_, index)) = word_indexes_stream.next() {+ if let Some((_, index)) = word_lower_indexes_stream.next() {words.insert(index);flag = true;}@@ -127,8 +127,6 @@ impl Dictionary for FstDictionary {&mut buf_a,&mut buf_b,);- buf_a.clear();- buf_b.clear();let dist_lower = edit_distance_min_alloc(&misspelled_lower_charslice,word,@@ -136,11 +134,11 @@ impl Dictionary for FstDictionary {&mut buf_b,);- (std::cmp::min(dist, dist_lower), word, index)+ (word, std::cmp::min(dist, dist_lower), index)})- .sorted_unstable_by_key(|a| a.0)+ .sorted_unstable_by_key(|a| a.1).take(max_results)- .map(|(dist, word, index)| {+ .map(|(word, dist, index)| {(word.as_slice(),dist,@@ -169,7 +167,7 @@ mod tests {use super::FstDictionary;#[test]- fn fst_contains() {+ fn fst_map_contains_all_in_full_dict() {let dict = FstDictionary::curated();for word in dict.words_iter() {@@ -233,16 +231,19 @@ mod tests {}#[test]- fn fuzzy_hello() {+ fn fuzzy_result_sorted_by_edit_distance() {let dict = FstDictionary::curated();- let matches: Vec<_> = dict- .fuzzy_match_str("hvllo", 3, 100)+ let results = dict.fuzzy_match_str("hello", 3, 100);+ let is_sorted_by_dist = results.iter()- .map(|cs| cs.0.iter().collect::<String>())- .collect();- dbg!(&matches);+ .map(|(_, dist, _)| dist)+ .tuple_windows()+ .all(|(a, b)| a <= b);++ let expected_match: Vec<_> = "hello".chars().collect();+ assert_eq!(results.first().unwrap().0, &expected_match);- assert!(matches.iter().take(10).contains(&"hello".to_string()))+ assert!(is_sorted_by_dist)}}diff --git a/harper-core/src/spell/full_dictionary.rs b/harper-core/src/spell/full_dictionary.rsindex 657a92e21..c969db359 100644--- a/harper-core/src/spell/full_dictionary.rs+++ b/harper-core/src/spell/full_dictionary.rs@@ -332,4 +332,21 @@ mod tests {let dict = FullDictionary::curated();assert!(dict.get_word_metadata_str("I'm").common);}++ #[test]+ fn fuzzy_result_sorted_by_edit_distance() {+ let dict = FullDictionary::curated();++ let results = dict.fuzzy_match_str("hello", 3, 100);+ let is_sorted_by_dist = results+ .iter()+ .map(|(_, dist, _)| dist)+ .tuple_windows()+ .all(|(a, b)| a <= b);++ let expected_match: Vec<_> = "hello".chars().collect();+ assert_eq!(results.first().unwrap().0, &expected_match);++ assert!(is_sorted_by_dist)+ }}diff --git a/harper-core/src/spell/mod.rs b/harper-core/src/spell/mod.rsindex ed4683190..3ca6c073b 100644--- a/harper-core/src/spell/mod.rs+++ b/harper-core/src/spell/mod.rs@@ -15,19 +15,7 @@ mod fst_dictionary;mod full_dictionary;mod merged_dictionary;-/// Get the closest matches in the provided [`Dictionary`] and rank them-/// Implementation is left up to the underlying dictionary.-pub fn suggest_correct_spelling<'a>(- misspelled_word: &[char],- result_limit: usize,- max_edit_dist: u8,- dictionary: &'a impl Dictionary,-) -> Vec<&'a [char]> {- let matches: Vec<(&[char], u8, WordMetadata)> = dictionary- .fuzzy_match(misspelled_word, max_edit_dist, result_limit)- .into_iter()- .collect();-+fn order_suggestions(matches: Vec<(&[char], u8, WordMetadata)>) -> Vec<&[char]> {let mut found: Vec<(&[char], u8, WordMetadata)> = Vec::with_capacity(matches.len());// Often the longest and the shortest words are the most helpful, so lets push// them first.@@ -64,6 +52,21 @@ pub fn suggest_correct_spelling<'a>(found.into_iter().map(|(word, _, _)| word).collect()}+/// Get the closest matches in the provided [`Dictionary`] and rank them+/// Implementation is left up to the underlying dictionary.+pub fn suggest_correct_spelling<'a>(+ misspelled_word: &[char],+ result_limit: usize,+ max_edit_dist: u8,+ dictionary: &'a impl Dictionary,+) -> Vec<&'a [char]> {+ let matches: Vec<(&[char], u8, WordMetadata)> = dictionary+ .fuzzy_match(misspelled_word, max_edit_dist, result_limit)+ .into_iter()+ .collect();+ order_suggestions(matches)+}+/// Convenience function over [`suggest_correct_spelling`] that does conversions/// for you.pub fn suggest_correct_spelling_str(@@ -144,7 +147,11 @@ fn edit_distance(source: &[char], target: &[char]) -> u8 {mod tests {use itertools::Itertools;- use super::{suggest_correct_spelling_str, FstDictionary};+ use crate::spell::edit_distance;++ use super::{+ order_suggestions, suggest_correct_spelling_str, Dictionary, FstDictionary, FullDictionary,+ };#[test]fn produces_no_duplicates() {@@ -155,12 +162,145 @@ mod tests {assert_eq!(results.iter().unique().count(), results.len())}+ #[test]+ fn zero_edit_distance() {+ let source: Vec<_> = "hello".chars().collect();+ let target: Vec<_> = "hello".chars().collect();+ assert_eq!(edit_distance(&source, &target), 0);+ }++ #[test]+ fn one_edit_distance() {+ let source: Vec<_> = "hello".chars().collect();+ let target: Vec<_> = "hellos".chars().collect();+ assert_eq!(edit_distance(&source, &target), 1);+ assert_eq!(edit_distance(&target, &source), 1);++ let target: Vec<_> = "hell".chars().collect();+ assert_eq!(edit_distance(&source, &target), 1);+ assert_eq!(edit_distance(&target, &source), 1);++ let target: Vec<_> = "hell".chars().collect();+ assert_eq!(edit_distance(&source, &target), 1);+ assert_eq!(edit_distance(&target, &source), 1);++ let target: Vec<_> = "hvllo".chars().collect();+ assert_eq!(edit_distance(&source, &target), 1);+ assert_eq!(edit_distance(&target, &source), 1);++ let target: Vec<_> = "Hello".chars().collect();+ assert_eq!(edit_distance(&source, &target), 1);+ assert_eq!(edit_distance(&target, &source), 1);+ }++ #[test]+ fn common_words_first() {+ let dict = FstDictionary::curated();+ let common_words = dict+ .words_iter()+ .filter_map(|word| {+ let metadata = dict.get_word_metadata(word);+ if metadata.common {+ Some((word, 0, metadata))+ } else {+ None+ }+ })+ .take(3);+ let uncommon_words = dict+ .words_iter()+ .filter_map(|word| {+ let metadata = dict.get_word_metadata(word);+ if metadata.common {+ None+ } else {+ Some((word, 0, metadata))+ }+ })+ .take(3);+ let words = common_words.merge(uncommon_words).collect();+ let suggestions = order_suggestions(words);++ let common_first = suggestions+ .into_iter()+ .take(3)+ .all(|word| dict.get_word_metadata(word).common);++ assert!(common_first);+ }++ #[test]+ fn identical_results_between_dicts() {+ let fst_results = suggest_correct_spelling_str("im", 100, 3, &FstDictionary::curated());+ let full_results = suggest_correct_spelling_str("im", 100, 3, &FullDictionary::curated());++ assert_eq!(fst_results, full_results);++ let fst_results = suggest_correct_spelling_str("hvllo", 100, 3, &FstDictionary::curated());+ let full_results =+ suggest_correct_spelling_str("hvllo", 100, 3, &FullDictionary::curated());++ assert_eq!(fst_results, full_results);++ let fst_results = suggest_correct_spelling_str("aboot", 100, 3, &FstDictionary::curated());+ let full_results =+ suggest_correct_spelling_str("aboot", 100, 3, &FullDictionary::curated());++ assert_eq!(fst_results, full_results);+ }+#[test]fn issue_182() {- let results = suggest_correct_spelling_str("im", 100, 3, &super::FullDictionary::curated());+ let results = suggest_correct_spelling_str("im", 100, 3, &FstDictionary::curated());dbg!(&results);assert!(results.iter().take(3).contains(&"I'm".to_string()));}++ #[test]+ fn spellcheck_hvllo() {+ let results = suggest_correct_spelling_str("hvllo", 100, 3, &FstDictionary::curated());++ dbg!(&results);++ assert!(results.iter().take(3).contains(&"hello".to_string()));+ }++ #[test]+ fn spellcheck_common() {+ let results = suggest_correct_spelling_str("aboot", 100, 3, &FstDictionary::curated());++ dbg!(&results);++ assert!(results.iter().take(3).contains(&"about".to_string()));+ }++ #[test]+ fn spellcheck_match() {+ let results = suggest_correct_spelling_str("hello", 100, 3, &FstDictionary::curated());++ dbg!(&results);++ assert!(results.iter().take(3).contains(&"hello".to_string()));+ }++ #[test]+ fn spellcheck_capital() {+ let results = suggest_correct_spelling_str("Hello", 100, 3, &FstDictionary::curated());++ dbg!(&results);++ assert!(results.iter().take(3).contains(&"hello".to_string()));+ }++ #[test]+ fn spellcheck_multiple_choices() {+ let results = suggest_correct_spelling_str("mello", 100, 3, &FstDictionary::curated());++ dbg!(&results);++ assert!(results.iter().take(3).contains(&"hello".to_string()));+ assert!(results.iter().take(3).contains(&"mellow".to_string()));+ }}