Skip to content

Offline, privacy-first grammar checker. Fast, open-source, Rust-powered

fix(fst-dictionary): lowercase and uppercase distances for correctness

Grant Lemons652e92fe51da
diff --git a/harper-core/src/spell/fst_dictionary.rs b/harper-core/src/spell/fst_dictionary.rs
index 155d64860..559aab8c9 100644
--- a/harper-core/src/spell/fst_dictionary.rs
+++ b/harper-core/src/spell/fst_dictionary.rs
@@ -150,11 +150,22 @@ impl Dictionary for FstDictionary {
// Actual FST search
let dfa = build_dfa(max_distance, &misspelled_word_string);
+ let dfa_lowercase = build_dfa(max_distance, &misspelled_word_string.to_lowercase());
let mut word_indexes_stream = self.word_map.search_with_state(&dfa).into_stream();
+ let mut word_indexes_lowercase_stream = self
+ .word_map
+ .search_with_state(&dfa_lowercase)
+ .into_stream();
stream_distances_vec(&mut word_indexes_stream, &dfa)
.into_iter()
+ .merge(
+ stream_distances_vec(&mut word_indexes_lowercase_stream, &dfa_lowercase)
+ .into_iter(),
+ )
// Sort by edit distance
+ .sorted_unstable_by_key(|a| a.0)
+ .dedup_by(|a, b| a.0 == b.0)
.sorted_unstable_by_key(|a| a.1)
.take(max_results)
.map(|(index, edit_distance)| {
diff --git a/harper-core/src/spell/full_dictionary.rs b/harper-core/src/spell/full_dictionary.rs
index dc2d26972..3febcc566 100644
--- a/harper-core/src/spell/full_dictionary.rs
+++ b/harper-core/src/spell/full_dictionary.rs
@@ -199,9 +199,16 @@ impl Dictionary for FullDictionary {
.filter_map(|word| {
let dist =
edit_distance_min_alloc(&misspelled_charslice, word, &mut buf_a, &mut buf_b);
-
- if dist <= max_distance {
- Some((word, dist))
+ let lowercase_dist = edit_distance_min_alloc(
+ &misspelled_charslice.to_lower(),
+ word,
+ &mut buf_a,
+ &mut buf_b,
+ );
+
+ let smaller_dist = dist.min(lowercase_dist);
+ if smaller_dist <= max_distance {
+ Some((word, smaller_dist))
} else {
None
}
diff --git a/harper-core/src/spell/mod.rs b/harper-core/src/spell/mod.rs
index 6993f8525..a74a2abc1 100644
--- a/harper-core/src/spell/mod.rs
+++ b/harper-core/src/spell/mod.rs
@@ -287,6 +287,20 @@ mod tests {
assert!(common_first);
}
+ #[test]
+ fn this_correction() {
+ let results = suggest_correct_spelling_str(
+ "Ths",
+ RESULT_LIMIT,
+ MAX_EDIT_DIST,
+ &FstDictionary::curated(),
+ );
+
+ dbg!(&results);
+
+ assert!(results.iter().take(3).contains(&"this".to_string()));
+ }
+
// I'm ignoring this one because the sorting algorithm prioritizes shorter words at the same
// edit distance that are also common.
#[ignore]