Skip to content

Offline, privacy-first grammar checker. Fast, open-source, Rust-powered

fix: duplicate spelling suggestions under special circumstances

Elijah Pottere1e2c50fc8a1
diff --git a/harper-core/src/spell/full_dictionary.rs b/harper-core/src/spell/full_dictionary.rs
index 434b8cdde..db7530b0c 100644
--- a/harper-core/src/spell/full_dictionary.rs
+++ b/harper-core/src/spell/full_dictionary.rs
@@ -14,7 +14,7 @@ pub struct FullDictionary {
/// Storing a separate [`Vec`] for iterations speeds up spellchecking by
/// ~16% at the cost of additional memory.
///
- /// This is likely due to increased locality :shrug:.
+ /// This is likely due to increased locality 🤷.
///
/// This list is sorted by word length (i.e. the shortest words are first).
words: Vec<CharString>,
@@ -30,10 +30,14 @@ fn uncached_inner_new() -> FullDictionary {
let word_list = parse_default_word_list().unwrap();
let attr_list = parse_default_attribute_list().unwrap();
- let mut words = Vec::new();
+ // There will be at _least_ this number of words
+ let mut words = Vec::with_capacity(word_list.len());
attr_list.expand_marked_words(word_list, &mut words);
+ words.sort();
+ words.dedup();
+
FullDictionary {
word_set: HashSet::from_iter(words.iter().cloned()),
word_len_starts: FullDictionary::create_len_starts(&mut words),
@@ -131,3 +135,16 @@ impl Dictionary for FullDictionary {
self.word_set.contains(normalized.as_ref()) || self.word_set.contains(&lowercase)
}
}
+
+#[cfg(test)]
+mod tests {
+ use itertools::Itertools;
+
+ use crate::{Dictionary, FullDictionary};
+
+ #[test]
+ fn curated_contains_no_duplicates() {
+ let dict = FullDictionary::create_from_curated();
+ assert!(dict.words_iter().all_unique());
+ }
+}
diff --git a/harper-core/src/spell/hunspell/attributes.rs b/harper-core/src/spell/hunspell/attributes.rs
index fe22bd5fc..f89886798 100644
--- a/harper-core/src/spell/hunspell/attributes.rs
+++ b/harper-core/src/spell/hunspell/attributes.rs
@@ -167,6 +167,9 @@ impl AttributeList {
dest.append(&mut split);
}
+ /// Expand an iterator of marked words into strings.
+ /// Note that this does __not__ guarantee that produced words will be
+ /// unique.
pub fn expand_marked_words(
&self,
words: impl IntoIterator<Item = MarkedWord>,
diff --git a/harper-core/src/spell/mod.rs b/harper-core/src/spell/mod.rs
index 62e40b8a1..6e6835904 100644
--- a/harper-core/src/spell/mod.rs
+++ b/harper-core/src/spell/mod.rs
@@ -61,14 +61,10 @@ pub fn suggest_correct_spelling<'a>(
for (word, dist) in pruned_words {
if found_dist.len() < result_limit {
found_dist.push((word, dist));
- found_dist.sort_by_key(|a| a.1);
- continue;
- }
-
- if dist < found_dist[result_limit - 1].1 {
+ } else if dist < found_dist[result_limit - 1].1 {
found_dist[result_limit - 1] = (word, dist);
- found_dist.sort_by_key(|a| a.1);
}
+ found_dist.sort_by_key(|a| a.1);
}
// Create final, ordered list of suggestions.
@@ -79,19 +75,24 @@ pub fn suggest_correct_spelling<'a>(
let minmax = found_dist.iter().position_minmax_by_key(|a| a.0.len());
if let MinMaxResult::MinMax(a, b) = minmax {
if a == b {
- found.push(found_dist.remove(a).0);
+ found.push(found_dist[a].0);
} else {
found.push(found_dist[a].0);
- found.push(found_dist.remove(b).0);
- if a < b {
- found_dist.remove(b - 1);
- } else {
- found_dist.remove(b);
- }
+ found.push(found_dist[b].0);
}
- }
- found.extend(found_dist.into_iter().map(|v| v.0));
+ // Push the rest
+ found.extend(
+ found_dist
+ .into_iter()
+ .enumerate()
+ .filter(|(i, _)| *i != a && *i != b)
+ .map(|v| v.1 .0)
+ );
+ } else {
+ // Push the rest
+ found.extend(found_dist.into_iter().map(|v| v.0));
+ }
// Finally, swap the lowest edit distance word with the shortest.
if found.len() >= 3 {
@@ -180,7 +181,10 @@ fn char_to_normalized(c: char) -> char {
#[cfg(test)]
mod tests {
- use super::{edit_distance, seq_to_normalized};
+ use itertools::Itertools;
+
+ use super::{edit_distance, seq_to_normalized, suggest_correct_spelling_str};
+ use crate::FullDictionary;
fn assert_edit_dist(source: &str, target: &str, expected: u8) {
let source: Vec<_> = source.chars().collect();
@@ -207,4 +211,18 @@ mod tests {
fn simple2() {
assert_edit_dist("saturday", "sunday", 3)
}
+
+ #[test]
+ fn produces_no_duplicates() {
+ let results = suggest_correct_spelling_str(
+ "punctation",
+ 100,
+ 3,
+ &FullDictionary::create_from_curated()
+ );
+
+ dbg!(&results, results.iter().unique().collect_vec());
+
+ assert_eq!(results.iter().unique().count(), results.len())
+ }
}