Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
fix(core): deduplicate fuzzy match results after merge
Elijah Potter0d7ee85aaed3
diff --git a/harper-core/src/spell/fst_dictionary.rs b/harper-core/src/spell/fst_dictionary.rsindex d9d688bd8..89c23ef95 100644--- a/harper-core/src/spell/fst_dictionary.rs+++ b/harper-core/src/spell/fst_dictionary.rs@@ -94,7 +94,7 @@ impl FstDictionary {}fn build_dfa(max_distance: u8, query: &str) -> DFA {- // Insert if does not exist+ // Insert if it does not existAUTOMATON_BUILDERS.with_borrow_mut(|v| {if !v.iter().any(|t| t.0 == max_distance) {v.push((@@ -181,6 +181,7 @@ impl Dictionary for FstDictionary {}merged.sort_unstable_by_key(|v| v.word);+ merged.dedup_by_key(|v| v.word);merged.sort_unstable_by_key(|v| v.edit_distance);merged.truncate(max_results);@@ -266,4 +267,11 @@ mod tests {assert!(is_sorted_by_dist)}++ #[test]+ fn curated_contains_no_duplicates() {+ let dict = FstDictionary::curated();++ assert!(dict.words.iter().map(|(word, _)| word).all_unique());+ }}diff --git a/harper-core/src/spell/mod.rs b/harper-core/src/spell/mod.rsindex 68ba37791..b287a7eed 100644--- a/harper-core/src/spell/mod.rs+++ b/harper-core/src/spell/mod.rs@@ -15,7 +15,7 @@ mod full_dictionary;pub mod hunspell;mod merged_dictionary;-#[derive(PartialEq, Debug)]+#[derive(PartialEq, Debug, Hash, Eq)]pub struct FuzzyMatchResult<'a> {word: &'a [char],edit_distance: u8,@@ -85,6 +85,7 @@ pub fn suggest_correct_spelling<'a>(.fuzzy_match(misspelled_word, max_edit_dist, result_limit).into_iter().collect();+order_suggestions(matches)}@@ -145,7 +146,7 @@ mod tests {}#[test]- fn produces_no_duplicates() {+ fn punctation_no_duplicates() {let results = suggest_correct_spelling_str("punctation",RESULT_LIMIT,@@ -153,9 +154,7 @@ mod tests {&FstDictionary::curated(),);- dbg!(&results, results.iter().unique().collect_vec());-- assert_eq!(results.iter().unique().count(), results.len())+ assert!(results.iter().all_unique())}/// Ensures that the suggestions are ordered taking into account commonality@@ -250,6 +249,20 @@ mod tests {assert!(results.iter().take(3).contains(&"need".to_string()));}+ #[test]+ fn issue_624_no_duplicates() {+ let results = suggest_correct_spelling_str(+ "Semantical",+ RESULT_LIMIT,+ MAX_EDIT_DIST,+ &FstDictionary::curated(),+ );++ dbg!(&results);++ assert!(results.iter().all_unique())+ }+#[test]fn issue_182_fst() {let results = suggest_correct_spelling_str(