Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
fix: duplicate spelling suggestions under special circumstances
Elijah Pottere1e2c50fc8a1
diff --git a/harper-core/src/spell/full_dictionary.rs b/harper-core/src/spell/full_dictionary.rsindex 434b8cdde..db7530b0c 100644--- a/harper-core/src/spell/full_dictionary.rs+++ b/harper-core/src/spell/full_dictionary.rs@@ -14,7 +14,7 @@ pub struct FullDictionary {/// Storing a separate [`Vec`] for iterations speeds up spellchecking by/// ~16% at the cost of additional memory.///- /// This is likely due to increased locality :shrug:.+ /// This is likely due to increased locality 🤷.////// This list is sorted by word length (i.e. the shortest words are first).words: Vec<CharString>,@@ -30,10 +30,14 @@ fn uncached_inner_new() -> FullDictionary {let word_list = parse_default_word_list().unwrap();let attr_list = parse_default_attribute_list().unwrap();- let mut words = Vec::new();+ // There will be at _least_ this number of words+ let mut words = Vec::with_capacity(word_list.len());attr_list.expand_marked_words(word_list, &mut words);+ words.sort();+ words.dedup();+FullDictionary {word_set: HashSet::from_iter(words.iter().cloned()),word_len_starts: FullDictionary::create_len_starts(&mut words),@@ -131,3 +135,16 @@ impl Dictionary for FullDictionary {self.word_set.contains(normalized.as_ref()) || self.word_set.contains(&lowercase)}}++#[cfg(test)]+mod tests {+ use itertools::Itertools;++ use crate::{Dictionary, FullDictionary};++ #[test]+ fn curated_contains_no_duplicates() {+ let dict = FullDictionary::create_from_curated();+ assert!(dict.words_iter().all_unique());+ }+}diff --git a/harper-core/src/spell/hunspell/attributes.rs b/harper-core/src/spell/hunspell/attributes.rsindex fe22bd5fc..f89886798 100644--- a/harper-core/src/spell/hunspell/attributes.rs+++ b/harper-core/src/spell/hunspell/attributes.rs@@ -167,6 +167,9 @@ impl AttributeList {dest.append(&mut split);}+ /// Expand an iterator of marked words into strings.+ /// Note that this does __not__ guarantee that produced words will be+ /// unique.pub fn expand_marked_words(&self,words: impl IntoIterator<Item = MarkedWord>,diff --git a/harper-core/src/spell/mod.rs b/harper-core/src/spell/mod.rsindex 62e40b8a1..6e6835904 100644--- a/harper-core/src/spell/mod.rs+++ b/harper-core/src/spell/mod.rs@@ -61,14 +61,10 @@ pub fn suggest_correct_spelling<'a>(for (word, dist) in pruned_words {if found_dist.len() < result_limit {found_dist.push((word, dist));- found_dist.sort_by_key(|a| a.1);- continue;- }-- if dist < found_dist[result_limit - 1].1 {+ } else if dist < found_dist[result_limit - 1].1 {found_dist[result_limit - 1] = (word, dist);- found_dist.sort_by_key(|a| a.1);}+ found_dist.sort_by_key(|a| a.1);}// Create final, ordered list of suggestions.@@ -79,19 +75,24 @@ pub fn suggest_correct_spelling<'a>(let minmax = found_dist.iter().position_minmax_by_key(|a| a.0.len());if let MinMaxResult::MinMax(a, b) = minmax {if a == b {- found.push(found_dist.remove(a).0);+ found.push(found_dist[a].0);} else {found.push(found_dist[a].0);- found.push(found_dist.remove(b).0);- if a < b {- found_dist.remove(b - 1);- } else {- found_dist.remove(b);- }+ found.push(found_dist[b].0);}- }- found.extend(found_dist.into_iter().map(|v| v.0));+ // Push the rest+ found.extend(+ found_dist+ .into_iter()+ .enumerate()+ .filter(|(i, _)| *i != a && *i != b)+ .map(|v| v.1 .0)+ );+ } else {+ // Push the rest+ found.extend(found_dist.into_iter().map(|v| v.0));+ }// Finally, swap the lowest edit distance word with the shortest.if found.len() >= 3 {@@ -180,7 +181,10 @@ fn char_to_normalized(c: char) -> char {#[cfg(test)]mod tests {- use super::{edit_distance, seq_to_normalized};+ use itertools::Itertools;++ use super::{edit_distance, seq_to_normalized, suggest_correct_spelling_str};+ use crate::FullDictionary;fn assert_edit_dist(source: &str, target: &str, expected: u8) {let source: Vec<_> = source.chars().collect();@@ -207,4 +211,18 @@ mod tests {fn simple2() {assert_edit_dist("saturday", "sunday", 3)}++ #[test]+ fn produces_no_duplicates() {+ let results = suggest_correct_spelling_str(+ "punctation",+ 100,+ 3,+ &FullDictionary::create_from_curated()+ );++ dbg!(&results, results.iter().unique().collect_vec());++ assert_eq!(results.iter().unique().count(), results.len())+ }}