Skip to content

Offline, privacy-first grammar checker. Fast, open-source, Rust-powered

feat(fst-dict): refactor and add more tests

Grant Lemons11d37944f98f
diff --git a/harper-core/src/spell/fst_dictionary.rs b/harper-core/src/spell/fst_dictionary.rs
index 68f2d0b14..df58f745f 100644
--- a/harper-core/src/spell/fst_dictionary.rs
+++ b/harper-core/src/spell/fst_dictionary.rs
@@ -86,10 +86,10 @@ impl Dictionary for FstDictionary {
// Actual FST search
let automaton = Levenshtein::new(&misspelled_word_string, max_distance as u32).unwrap();
+ let mut word_indexes_stream = self.word_map.search(automaton).into_stream();
let automaton_lower =
Levenshtein::new(&misspelled_lower_string, max_distance as u32).unwrap();
let mut word_lower_indexes_stream = self.word_map.search(automaton_lower).into_stream();
- let mut word_indexes_stream = self.word_map.search(automaton).into_stream();
// Consume at most max_results values from each stream
// The search itself happens as you consume from the stream, so consuming a smaller number
@@ -98,11 +98,11 @@ impl Dictionary for FstDictionary {
let mut words = hashbrown::HashSet::with_capacity(max_results * 2);
for _ in 0..max_results {
let mut flag = false;
- if let Some((_, index)) = word_lower_indexes_stream.next() {
+ if let Some((_, index)) = word_indexes_stream.next() {
words.insert(index);
flag = true;
}
- if let Some((_, index)) = word_indexes_stream.next() {
+ if let Some((_, index)) = word_lower_indexes_stream.next() {
words.insert(index);
flag = true;
}
@@ -127,8 +127,6 @@ impl Dictionary for FstDictionary {
&mut buf_a,
&mut buf_b,
);
- buf_a.clear();
- buf_b.clear();
let dist_lower = edit_distance_min_alloc(
&misspelled_lower_charslice,
word,
@@ -136,11 +134,11 @@ impl Dictionary for FstDictionary {
&mut buf_b,
);
- (std::cmp::min(dist, dist_lower), word, index)
+ (word, std::cmp::min(dist, dist_lower), index)
})
- .sorted_unstable_by_key(|a| a.0)
+ .sorted_unstable_by_key(|a| a.1)
.take(max_results)
- .map(|(dist, word, index)| {
+ .map(|(word, dist, index)| {
(
word.as_slice(),
dist,
@@ -169,7 +167,7 @@ mod tests {
use super::FstDictionary;
#[test]
- fn fst_contains() {
+ fn fst_map_contains_all_in_full_dict() {
let dict = FstDictionary::curated();
for word in dict.words_iter() {
@@ -233,16 +231,19 @@ mod tests {
}
#[test]
- fn fuzzy_hello() {
+ fn fuzzy_result_sorted_by_edit_distance() {
let dict = FstDictionary::curated();
- let matches: Vec<_> = dict
- .fuzzy_match_str("hvllo", 3, 100)
+ let results = dict.fuzzy_match_str("hello", 3, 100);
+ let is_sorted_by_dist = results
.iter()
- .map(|cs| cs.0.iter().collect::<String>())
- .collect();
- dbg!(&matches);
+ .map(|(_, dist, _)| dist)
+ .tuple_windows()
+ .all(|(a, b)| a <= b);
+
+ let expected_match: Vec<_> = "hello".chars().collect();
+ assert_eq!(results.first().unwrap().0, &expected_match);
- assert!(matches.iter().take(10).contains(&"hello".to_string()))
+ assert!(is_sorted_by_dist)
}
}
diff --git a/harper-core/src/spell/full_dictionary.rs b/harper-core/src/spell/full_dictionary.rs
index 657a92e21..c969db359 100644
--- a/harper-core/src/spell/full_dictionary.rs
+++ b/harper-core/src/spell/full_dictionary.rs
@@ -332,4 +332,21 @@ mod tests {
let dict = FullDictionary::curated();
assert!(dict.get_word_metadata_str("I'm").common);
}
+
+ #[test]
+ fn fuzzy_result_sorted_by_edit_distance() {
+ let dict = FullDictionary::curated();
+
+ let results = dict.fuzzy_match_str("hello", 3, 100);
+ let is_sorted_by_dist = results
+ .iter()
+ .map(|(_, dist, _)| dist)
+ .tuple_windows()
+ .all(|(a, b)| a <= b);
+
+ let expected_match: Vec<_> = "hello".chars().collect();
+ assert_eq!(results.first().unwrap().0, &expected_match);
+
+ assert!(is_sorted_by_dist)
+ }
}
diff --git a/harper-core/src/spell/mod.rs b/harper-core/src/spell/mod.rs
index ed4683190..3ca6c073b 100644
--- a/harper-core/src/spell/mod.rs
+++ b/harper-core/src/spell/mod.rs
@@ -15,19 +15,7 @@ mod fst_dictionary;
mod full_dictionary;
mod merged_dictionary;
-/// Get the closest matches in the provided [`Dictionary`] and rank them
-/// Implementation is left up to the underlying dictionary.
-pub fn suggest_correct_spelling<'a>(
- misspelled_word: &[char],
- result_limit: usize,
- max_edit_dist: u8,
- dictionary: &'a impl Dictionary,
-) -> Vec<&'a [char]> {
- let matches: Vec<(&[char], u8, WordMetadata)> = dictionary
- .fuzzy_match(misspelled_word, max_edit_dist, result_limit)
- .into_iter()
- .collect();
-
+fn order_suggestions(matches: Vec<(&[char], u8, WordMetadata)>) -> Vec<&[char]> {
let mut found: Vec<(&[char], u8, WordMetadata)> = Vec::with_capacity(matches.len());
// Often the longest and the shortest words are the most helpful, so lets push
// them first.
@@ -64,6 +52,21 @@ pub fn suggest_correct_spelling<'a>(
found.into_iter().map(|(word, _, _)| word).collect()
}
+/// Get the closest matches in the provided [`Dictionary`] and rank them
+/// Implementation is left up to the underlying dictionary.
+pub fn suggest_correct_spelling<'a>(
+ misspelled_word: &[char],
+ result_limit: usize,
+ max_edit_dist: u8,
+ dictionary: &'a impl Dictionary,
+) -> Vec<&'a [char]> {
+ let matches: Vec<(&[char], u8, WordMetadata)> = dictionary
+ .fuzzy_match(misspelled_word, max_edit_dist, result_limit)
+ .into_iter()
+ .collect();
+ order_suggestions(matches)
+}
+
/// Convenience function over [`suggest_correct_spelling`] that does conversions
/// for you.
pub fn suggest_correct_spelling_str(
@@ -144,7 +147,11 @@ fn edit_distance(source: &[char], target: &[char]) -> u8 {
mod tests {
use itertools::Itertools;
- use super::{suggest_correct_spelling_str, FstDictionary};
+ use crate::spell::edit_distance;
+
+ use super::{
+ order_suggestions, suggest_correct_spelling_str, Dictionary, FstDictionary, FullDictionary,
+ };
#[test]
fn produces_no_duplicates() {
@@ -155,12 +162,145 @@ mod tests {
assert_eq!(results.iter().unique().count(), results.len())
}
+ #[test]
+ fn zero_edit_distance() {
+ let source: Vec<_> = "hello".chars().collect();
+ let target: Vec<_> = "hello".chars().collect();
+ assert_eq!(edit_distance(&source, &target), 0);
+ }
+
+ #[test]
+ fn one_edit_distance() {
+ let source: Vec<_> = "hello".chars().collect();
+ let target: Vec<_> = "hellos".chars().collect();
+ assert_eq!(edit_distance(&source, &target), 1);
+ assert_eq!(edit_distance(&target, &source), 1);
+
+ let target: Vec<_> = "hell".chars().collect();
+ assert_eq!(edit_distance(&source, &target), 1);
+ assert_eq!(edit_distance(&target, &source), 1);
+
+ let target: Vec<_> = "hell".chars().collect();
+ assert_eq!(edit_distance(&source, &target), 1);
+ assert_eq!(edit_distance(&target, &source), 1);
+
+ let target: Vec<_> = "hvllo".chars().collect();
+ assert_eq!(edit_distance(&source, &target), 1);
+ assert_eq!(edit_distance(&target, &source), 1);
+
+ let target: Vec<_> = "Hello".chars().collect();
+ assert_eq!(edit_distance(&source, &target), 1);
+ assert_eq!(edit_distance(&target, &source), 1);
+ }
+
+ #[test]
+ fn common_words_first() {
+ let dict = FstDictionary::curated();
+ let common_words = dict
+ .words_iter()
+ .filter_map(|word| {
+ let metadata = dict.get_word_metadata(word);
+ if metadata.common {
+ Some((word, 0, metadata))
+ } else {
+ None
+ }
+ })
+ .take(3);
+ let uncommon_words = dict
+ .words_iter()
+ .filter_map(|word| {
+ let metadata = dict.get_word_metadata(word);
+ if metadata.common {
+ None
+ } else {
+ Some((word, 0, metadata))
+ }
+ })
+ .take(3);
+ let words = common_words.merge(uncommon_words).collect();
+ let suggestions = order_suggestions(words);
+
+ let common_first = suggestions
+ .into_iter()
+ .take(3)
+ .all(|word| dict.get_word_metadata(word).common);
+
+ assert!(common_first);
+ }
+
+ #[test]
+ fn identical_results_between_dicts() {
+ let fst_results = suggest_correct_spelling_str("im", 100, 3, &FstDictionary::curated());
+ let full_results = suggest_correct_spelling_str("im", 100, 3, &FullDictionary::curated());
+
+ assert_eq!(fst_results, full_results);
+
+ let fst_results = suggest_correct_spelling_str("hvllo", 100, 3, &FstDictionary::curated());
+ let full_results =
+ suggest_correct_spelling_str("hvllo", 100, 3, &FullDictionary::curated());
+
+ assert_eq!(fst_results, full_results);
+
+ let fst_results = suggest_correct_spelling_str("aboot", 100, 3, &FstDictionary::curated());
+ let full_results =
+ suggest_correct_spelling_str("aboot", 100, 3, &FullDictionary::curated());
+
+ assert_eq!(fst_results, full_results);
+ }
+
#[test]
fn issue_182() {
- let results = suggest_correct_spelling_str("im", 100, 3, &super::FullDictionary::curated());
+ let results = suggest_correct_spelling_str("im", 100, 3, &FstDictionary::curated());
dbg!(&results);
assert!(results.iter().take(3).contains(&"I'm".to_string()));
}
+
+ #[test]
+ fn spellcheck_hvllo() {
+ let results = suggest_correct_spelling_str("hvllo", 100, 3, &FstDictionary::curated());
+
+ dbg!(&results);
+
+ assert!(results.iter().take(3).contains(&"hello".to_string()));
+ }
+
+ #[test]
+ fn spellcheck_common() {
+ let results = suggest_correct_spelling_str("aboot", 100, 3, &FstDictionary::curated());
+
+ dbg!(&results);
+
+ assert!(results.iter().take(3).contains(&"about".to_string()));
+ }
+
+ #[test]
+ fn spellcheck_match() {
+ let results = suggest_correct_spelling_str("hello", 100, 3, &FstDictionary::curated());
+
+ dbg!(&results);
+
+ assert!(results.iter().take(3).contains(&"hello".to_string()));
+ }
+
+ #[test]
+ fn spellcheck_capital() {
+ let results = suggest_correct_spelling_str("Hello", 100, 3, &FstDictionary::curated());
+
+ dbg!(&results);
+
+ assert!(results.iter().take(3).contains(&"hello".to_string()));
+ }
+
+ #[test]
+ fn spellcheck_multiple_choices() {
+ let results = suggest_correct_spelling_str("mello", 100, 3, &FstDictionary::curated());
+
+ dbg!(&results);
+
+ assert!(results.iter().take(3).contains(&"hello".to_string()));
+ assert!(results.iter().take(3).contains(&"mellow".to_string()));
+ }
}