Skip to content

Offline, privacy-first grammar checker. Fast, open-source, Rust-powered

fix(core): deduplicate fuzzy match results after merge

Elijah Potter0d7ee85aaed3
diff --git a/harper-core/src/spell/fst_dictionary.rs b/harper-core/src/spell/fst_dictionary.rs
index d9d688bd8..89c23ef95 100644
--- a/harper-core/src/spell/fst_dictionary.rs
+++ b/harper-core/src/spell/fst_dictionary.rs
@@ -94,7 +94,7 @@ impl FstDictionary {
}
fn build_dfa(max_distance: u8, query: &str) -> DFA {
- // Insert if does not exist
+ // Insert if it does not exist
AUTOMATON_BUILDERS.with_borrow_mut(|v| {
if !v.iter().any(|t| t.0 == max_distance) {
v.push((
@@ -181,6 +181,7 @@ impl Dictionary for FstDictionary {
}
merged.sort_unstable_by_key(|v| v.word);
+ merged.dedup_by_key(|v| v.word);
merged.sort_unstable_by_key(|v| v.edit_distance);
merged.truncate(max_results);
@@ -266,4 +267,11 @@ mod tests {
assert!(is_sorted_by_dist)
}
+
+ #[test]
+ fn curated_contains_no_duplicates() {
+ let dict = FstDictionary::curated();
+
+ assert!(dict.words.iter().map(|(word, _)| word).all_unique());
+ }
}
diff --git a/harper-core/src/spell/mod.rs b/harper-core/src/spell/mod.rs
index 68ba37791..b287a7eed 100644
--- a/harper-core/src/spell/mod.rs
+++ b/harper-core/src/spell/mod.rs
@@ -15,7 +15,7 @@ mod full_dictionary;
pub mod hunspell;
mod merged_dictionary;
-#[derive(PartialEq, Debug)]
+#[derive(PartialEq, Debug, Hash, Eq)]
pub struct FuzzyMatchResult<'a> {
word: &'a [char],
edit_distance: u8,
@@ -85,6 +85,7 @@ pub fn suggest_correct_spelling<'a>(
.fuzzy_match(misspelled_word, max_edit_dist, result_limit)
.into_iter()
.collect();
+
order_suggestions(matches)
}
@@ -145,7 +146,7 @@ mod tests {
}
#[test]
- fn produces_no_duplicates() {
+ fn punctation_no_duplicates() {
let results = suggest_correct_spelling_str(
"punctation",
RESULT_LIMIT,
@@ -153,9 +154,7 @@ mod tests {
&FstDictionary::curated(),
);
- dbg!(&results, results.iter().unique().collect_vec());
-
- assert_eq!(results.iter().unique().count(), results.len())
+ assert!(results.iter().all_unique())
}
/// Ensures that the suggestions are ordered taking into account commonality
@@ -250,6 +249,20 @@ mod tests {
assert!(results.iter().take(3).contains(&"need".to_string()));
}
+ #[test]
+ fn issue_624_no_duplicates() {
+ let results = suggest_correct_spelling_str(
+ "Semantical",
+ RESULT_LIMIT,
+ MAX_EDIT_DIST,
+ &FstDictionary::curated(),
+ );
+
+ dbg!(&results);
+
+ assert!(results.iter().all_unique())
+ }
+
#[test]
fn issue_182_fst() {
let results = suggest_correct_spelling_str(