Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
fix(fst-dictionary): lowercase and uppercase distances for correctness
Grant Lemons652e92fe51da
diff --git a/harper-core/src/spell/fst_dictionary.rs b/harper-core/src/spell/fst_dictionary.rsindex 155d64860..559aab8c9 100644--- a/harper-core/src/spell/fst_dictionary.rs+++ b/harper-core/src/spell/fst_dictionary.rs@@ -150,11 +150,22 @@ impl Dictionary for FstDictionary {// Actual FST searchlet dfa = build_dfa(max_distance, &misspelled_word_string);+ let dfa_lowercase = build_dfa(max_distance, &misspelled_word_string.to_lowercase());let mut word_indexes_stream = self.word_map.search_with_state(&dfa).into_stream();+ let mut word_indexes_lowercase_stream = self+ .word_map+ .search_with_state(&dfa_lowercase)+ .into_stream();stream_distances_vec(&mut word_indexes_stream, &dfa).into_iter()+ .merge(+ stream_distances_vec(&mut word_indexes_lowercase_stream, &dfa_lowercase)+ .into_iter(),+ )// Sort by edit distance+ .sorted_unstable_by_key(|a| a.0)+ .dedup_by(|a, b| a.0 == b.0).sorted_unstable_by_key(|a| a.1).take(max_results).map(|(index, edit_distance)| {diff --git a/harper-core/src/spell/full_dictionary.rs b/harper-core/src/spell/full_dictionary.rsindex dc2d26972..3febcc566 100644--- a/harper-core/src/spell/full_dictionary.rs+++ b/harper-core/src/spell/full_dictionary.rs@@ -199,9 +199,16 @@ impl Dictionary for FullDictionary {.filter_map(|word| {let dist =edit_distance_min_alloc(&misspelled_charslice, word, &mut buf_a, &mut buf_b);-- if dist <= max_distance {- Some((word, dist))+ let lowercase_dist = edit_distance_min_alloc(+ &misspelled_charslice.to_lower(),+ word,+ &mut buf_a,+ &mut buf_b,+ );++ let smaller_dist = dist.min(lowercase_dist);+ if smaller_dist <= max_distance {+ Some((word, smaller_dist))} else {None}diff --git a/harper-core/src/spell/mod.rs b/harper-core/src/spell/mod.rsindex 6993f8525..a74a2abc1 100644--- a/harper-core/src/spell/mod.rs+++ b/harper-core/src/spell/mod.rs@@ -287,6 +287,20 @@ mod tests {assert!(common_first);}+ #[test]+ fn this_correction() {+ let results = suggest_correct_spelling_str(+ "Ths",+ RESULT_LIMIT,+ MAX_EDIT_DIST,+ &FstDictionary::curated(),+ );++ dbg!(&results);++ assert!(results.iter().take(3).contains(&"this".to_string()));+ }+// I'm ignoring this one because the sorting algorithm prioritizes shorter words at the same// edit distance that are also common.#[ignore]