Skip to content

Offline, privacy-first grammar checker. Fast, open-source, Rust-powered

feat(fst-dict): change how fuzzy search handles lowercase

Grant Lemonsfeb9c3b93e3c
diff --git a/harper-core/src/spell/fst_dictionary.rs b/harper-core/src/spell/fst_dictionary.rs
index c0c541404..ea1c539d6 100644
--- a/harper-core/src/spell/fst_dictionary.rs
+++ b/harper-core/src/spell/fst_dictionary.rs
@@ -82,51 +82,33 @@ impl Dictionary for FstDictionary {
let misspelled_word_charslice = seq_to_normalized(&chars);
let misspelled_lower_charslice = misspelled_word_charslice.to_lower();
let misspelled_word_string = misspelled_word_charslice.to_string();
- let misspelled_lower_string = misspelled_lower_charslice.to_string();
// Actual FST search
let automaton = Levenshtein::new(&misspelled_word_string, max_distance as u32).unwrap();
let word_indexes_stream = self.word_map.search(automaton).into_stream().into_values();
- let automaton_lower =
- Levenshtein::new(&misspelled_lower_string, max_distance as u32).unwrap();
- let word_lower_indexes_stream = self
- .word_map
- .search(automaton_lower)
- .into_stream()
- .into_values();
-
- // HashSet used to dedup results
- let indexes = word_indexes_stream
- .into_iter()
- .merge(word_lower_indexes_stream)
- .collect::<hashbrown::HashSet<_>>();
// Pre-allocated vectors for edit-distance calculation
// 53 is the length of the longest word.
let mut buf_a = Vec::with_capacity(53);
let mut buf_b = Vec::with_capacity(53);
- indexes
+ word_indexes_stream
.into_iter()
+ .sorted_unstable()
+ .dedup()
.map(|index| (self.full_dict.get_word(index as usize), index))
// Sort by edit distance
.map(|(word, index)| {
let dist = edit_distance_min_alloc(
- &misspelled_word_charslice,
- word,
- &mut buf_a,
- &mut buf_b,
- );
- let dist_lower = edit_distance_min_alloc(
&misspelled_lower_charslice,
- word,
+ &word.to_lower(),
&mut buf_a,
&mut buf_b,
);
- (word, std::cmp::min(dist, dist_lower), index)
+ (word, dist, index)
})
- .sorted_by_key(|a| a.1)
+ .sorted_unstable_by_key(|a| a.1)
.take(max_results)
.map(|(word, dist, index)| {
(
diff --git a/harper-core/src/spell/full_dictionary.rs b/harper-core/src/spell/full_dictionary.rs
index 17bf65bec..6a9bbbbec 100644
--- a/harper-core/src/spell/full_dictionary.rs
+++ b/harper-core/src/spell/full_dictionary.rs
@@ -193,21 +193,21 @@ impl Dictionary for FullDictionary {
// 53 is the length of the longest word.
let mut buf_a = Vec::with_capacity(53);
let mut buf_b = Vec::with_capacity(53);
- let pruned_words = words_to_search.filter_map(|word| {
- let dist =
- edit_distance_min_alloc(&misspelled_normalized, word, &mut buf_a, &mut buf_b);
- let dist_lower =
- edit_distance_min_alloc(&misspelled_lower, word, &mut buf_a, &mut buf_b);
- let smallest_dist = std::cmp::min(dist, dist_lower);
-
- if smallest_dist <= max_distance {
- Some((word, smallest_dist))
- } else {
- None
- }
- });
-
- pruned_words
+ words_to_search
+ .filter_map(|word| {
+ let dist = edit_distance_min_alloc(
+ &misspelled_lower,
+ &word.to_lower(),
+ &mut buf_a,
+ &mut buf_b,
+ );
+
+ if dist <= max_distance {
+ Some((word, dist))
+ } else {
+ None
+ }
+ })
.sorted_by_key(|a| a.1)
.take(max_results)
.map(|(word, dist)| (word, dist, self.get_word_metadata(word)))
diff --git a/harper-core/src/spell/mod.rs b/harper-core/src/spell/mod.rs
index b740107c0..492a51a6f 100644
--- a/harper-core/src/spell/mod.rs
+++ b/harper-core/src/spell/mod.rs
@@ -230,32 +230,28 @@ mod tests {
}
#[test]
- fn identical_results_between_dicts() {
- let fst_results = suggest_correct_spelling_str("im", 100, 3, &FstDictionary::curated());
- let full_results = suggest_correct_spelling_str("im", 100, 3, &FullDictionary::curated());
+ fn full_dict_fuzzy_find() {
+ let results = suggest_correct_spelling_str("anout", 100, 3, &FullDictionary::curated());
- assert_eq!(fst_results, full_results);
-
- let fst_results = suggest_correct_spelling_str("hvllo", 100, 3, &FstDictionary::curated());
- let full_results =
- suggest_correct_spelling_str("hvllo", 100, 3, &FullDictionary::curated());
-
- assert_eq!(fst_results, full_results);
-
- let fst_results = suggest_correct_spelling_str("aboot", 100, 3, &FstDictionary::curated());
- let full_results =
- suggest_correct_spelling_str("aboot", 100, 3, &FullDictionary::curated());
+ dbg!(&results);
- assert_eq!(fst_results, full_results);
+ assert!(results.iter().take(3).contains(&"about".to_string()));
}
#[test]
fn issue_182() {
- let results = suggest_correct_spelling_str("im", 1000, 3, &FstDictionary::curated());
+ let results = suggest_correct_spelling_str("Im", 100, 3, &FstDictionary::curated());
+ let lowercase_results =
+ suggest_correct_spelling_str("im", 100, 3, &FstDictionary::curated());
dbg!(&results);
+ dbg!(&lowercase_results);
assert!(results.iter().take(3).contains(&"I'm".to_string()));
+ assert!(lowercase_results
+ .iter()
+ .take(3)
+ .contains(&"I'm".to_string()));
}
#[test]
@@ -295,12 +291,13 @@ mod tests {
}
#[test]
- fn spellcheck_multiple_choices() {
- let results = suggest_correct_spelling_str("mello", 100, 3, &FstDictionary::curated());
-
- dbg!(&results);
-
- assert!(results.iter().contains(&"hello".to_string()));
- assert!(results.iter().take(3).contains(&"mellow".to_string()));
+ fn spellchecking_is_deterministic() {
+ let results1 = suggest_correct_spelling_str("hello", 100, 3, &FstDictionary::curated());
+ let results2 = suggest_correct_spelling_str("hello", 100, 3, &FstDictionary::curated());
+ let results3 = suggest_correct_spelling_str("hello", 100, 3, &FstDictionary::curated());
+
+ assert_eq!(results1, results2);
+ assert_eq!(results1, results3);
+ assert_eq!(results2, results3);
}
}