Skip to content

Offline, privacy-first grammar checker. Fast, open-source, Rust-powered

feat(core): use scoring rather than shuffling for spelling suggestions

Elijah Potter99e77c4fb3f4
diff --git a/harper-core/src/spell/mod.rs b/harper-core/src/spell/mod.rs
index 9baf12850..9721a90b8 100644
--- a/harper-core/src/spell/mod.rs
+++ b/harper-core/src/spell/mod.rs
@@ -1,7 +1,5 @@
use std::borrow::Cow;
-use itertools::{Itertools, MinMaxResult};
-
use crate::{CharString, CharStringExt, WordMetadata};
pub use self::dictionary::Dictionary;
@@ -28,49 +26,42 @@ impl PartialOrd for FuzzyMatchResult<'_> {
}
}
-/// Order the suggestions to be shown to the user.
-fn order_suggestions(matches: Vec<FuzzyMatchResult>) -> Vec<&[char]> {
- let mut found: Vec<&FuzzyMatchResult> = Vec::with_capacity(matches.len());
- // Often the longest and the shortest words are the most helpful, so let's push
- // them first.
- let minmax = matches.iter().position_minmax_by_key(|fmr| fmr.word.len());
- if let MinMaxResult::MinMax(a, b) = minmax {
- if a == b {
- found.push(&matches[a]);
- } else {
- found.push(&matches[a]);
- found.push(&matches[b]);
- }
-
- // Push the rest
- found.extend(
- matches
- .iter()
- .enumerate()
- .filter(|(i, _)| *i != a && *i != b)
- .map(|v| v.1),
- );
- } else {
- found.extend(&matches);
+/// Scores a possible spelling suggestion based on possible relevance to the user.
+///
+/// Lower = better.
+fn score_suggestion(misspelled_word: &[char], sug: &FuzzyMatchResult) -> i32 {
+ if misspelled_word.is_empty() {
+ return i32::MAX;
+ }
+
+ let mut score = sug.edit_distance as i32 * 10;
+
+ // If the original word is plural, the correct one probably is too.
+ if *misspelled_word.last().unwrap() == 's' && *sug.word.last().unwrap() == 's' {
+ score -= 5;
}
- // Swap the lowest edit distance word with the shortest.
- if found.len() >= 3 {
- found.swap(0, 2);
+ // For turning words into contractions.
+ if sug.metadata.common {
+ score -= 5;
}
- if let Some(nominal_index) = found
- .iter()
- .skip(3)
- .position(|i| i.metadata.is_proper_noun())
- {
- found.swap(2, nominal_index + 3);
+ // For turning words into contractions.
+ if sug.word.iter().filter(|c| **c == '\'').count() == 1 {
+ score -= 5;
}
- // Make commonality relevant
- found.sort_by_key(|fmr| if fmr.metadata.common { 0 } else { 1 });
+ score
+}
+
+/// Order the suggestions to be shown to the user.
+fn order_suggestions<'b>(
+ misspelled_word: &[char],
+ mut matches: Vec<FuzzyMatchResult<'b>>,
+) -> Vec<&'b [char]> {
+ matches.sort_by_key(|v| score_suggestion(misspelled_word, v));
- found.into_iter().map(|fmr| fmr.word).collect()
+ matches.into_iter().map(|v| v.word).collect()
}
/// Get the closest matches in the provided [`Dictionary`] and rank them
@@ -86,7 +77,7 @@ pub fn suggest_correct_spelling<'a>(
.into_iter()
.collect();
- order_suggestions(matches)
+ order_suggestions(misspelled_word, matches)
}
/// Convenience function over [`suggest_correct_spelling`] that does conversions
@@ -127,11 +118,8 @@ fn char_to_normalized(c: char) -> char {
mod tests {
use itertools::Itertools;
- use crate::spell::FuzzyMatchResult;
-
use super::{
- Dictionary, FstDictionary, MutableDictionary, order_suggestions, seq_to_normalized,
- suggest_correct_spelling_str,
+ FstDictionary, order_suggestions, seq_to_normalized, suggest_correct_spelling_str,
};
const RESULT_LIMIT: usize = 100;
@@ -157,96 +145,24 @@ mod tests {
assert!(results.iter().all_unique())
}
- /// Ensures that the suggestions are ordered taking into account commonality
#[test]
- fn common_words_first() {
- let dict = FstDictionary::curated();
- // Select three common words
- let common_words = dict
- .words_iter()
- .filter_map(|word| {
- let metadata = dict.get_word_metadata(word).unwrap();
- if metadata.common {
- Some(FuzzyMatchResult {
- word,
- edit_distance: 0,
- metadata,
- })
- } else {
- None
- }
- })
- .take(3);
- // Select three uncommon words
- let uncommon_words = dict
- .words_iter()
- .filter_map(|word| {
- let metadata = dict.get_word_metadata(word).unwrap();
- if metadata.common {
- None
- } else {
- Some(FuzzyMatchResult {
- word,
- edit_distance: 0,
- metadata,
- })
- }
- })
- .take(3);
- // Feed the common and uncommon words into the ordering function, starting with uncommon
- // words
- let words = uncommon_words.merge(common_words).collect();
- let suggestions = order_suggestions(words);
-
- // Asserts that the ordering prioritizes common words
- let common_first = suggestions
- .into_iter()
- .take(3)
- .all(|word| dict.get_word_metadata(word).unwrap().common);
-
- assert!(common_first);
+ fn youre_contraction() {
+ assert_suggests_correction("youre", "you're");
}
#[test]
- fn this_correction() {
- let results = suggest_correct_spelling_str(
- "Ths",
- RESULT_LIMIT,
- MAX_EDIT_DIST,
- &FstDictionary::curated(),
- );
-
- dbg!(&results);
-
- assert!(results.iter().take(3).contains(&"this".to_string()));
+ fn thats_contraction() {
+ assert_suggests_correction("thats", "that's");
}
#[test]
- fn need_correction_full() {
- let results = suggest_correct_spelling_str(
- "ned",
- RESULT_LIMIT,
- MAX_EDIT_DIST,
- &MutableDictionary::curated(),
- );
-
- dbg!(&results);
-
- assert!(results.iter().take(3).contains(&"need".to_string()));
+ fn weve_contraction() {
+ assert_suggests_correction("weve", "we've");
}
#[test]
- fn need_correction_fst() {
- let results = suggest_correct_spelling_str(
- "ned",
- RESULT_LIMIT,
- MAX_EDIT_DIST,
- &FstDictionary::curated(),
- );
-
- dbg!(&results);
-
- assert!(results.iter().take(3).contains(&"need".to_string()));
+ fn this_correction() {
+ assert_suggests_correction("ths", "this");
}
#[test]
@@ -264,31 +180,8 @@ mod tests {
}
#[test]
- fn issue_182_fst() {
- let results = suggest_correct_spelling_str(
- "Im",
- RESULT_LIMIT,
- MAX_EDIT_DIST,
- &FstDictionary::curated(),
- );
-
- dbg!(&results);
-
- assert!(results.iter().take(3).contains(&"I'm".to_string()));
- }
-
- #[test]
- fn issue_182_full() {
- let results = suggest_correct_spelling_str(
- "Im",
- RESULT_LIMIT,
- MAX_EDIT_DIST,
- &MutableDictionary::curated(),
- );
-
- dbg!(&results);
-
- assert!(results.iter().take(3).contains(&"I'm".to_string()));
+ fn issue_182() {
+ assert_suggests_correction("Im", "I'm");
}
#[test]
@@ -305,46 +198,29 @@ mod tests {
assert!(results.iter().take(3).contains(&"hello".to_string()));
}
- #[test]
- fn full_spellcheck_hvllo() {
+ /// Assert that the default suggestion settings result in a specific word
+ /// being in the top three results for a given misspelling.
+ fn assert_suggests_correction(misspelled_word: &str, correct: &str) {
let results = suggest_correct_spelling_str(
- "hvllo",
+ misspelled_word,
RESULT_LIMIT,
MAX_EDIT_DIST,
- &MutableDictionary::curated(),
+ &FstDictionary::curated(),
);
dbg!(&results);
- assert!(results.iter().take(3).contains(&"hello".to_string()));
+ assert!(results.iter().take(3).contains(&correct.to_string()));
}
#[test]
- fn fst_spellcheck_common() {
- let results = suggest_correct_spelling_str(
- "aboot",
- RESULT_LIMIT,
- MAX_EDIT_DIST,
- &FstDictionary::curated(),
- );
-
- dbg!(&results);
-
- assert!(results.iter().take(3).contains(&"about".to_string()));
+ fn spellcheck_hvllo() {
+ assert_suggests_correction("hvllo", "hello");
}
#[test]
- fn full_spellcheck_common() {
- let results = suggest_correct_spelling_str(
- "aboot",
- RESULT_LIMIT,
- MAX_EDIT_DIST,
- &MutableDictionary::curated(),
- );
-
- dbg!(&results);
-
- assert!(results.iter().take(3).contains(&"about".to_string()));
+ fn spellcheck_aout() {
+ assert_suggests_correction("aout", "about");
}
#[test]
@@ -371,4 +247,29 @@ mod tests {
assert_eq!(results1, results2);
assert_eq!(results1, results3);
}
+
+ #[test]
+ fn adviced_correction() {
+ assert_suggests_correction("adviced", "advised");
+ }
+
+ #[test]
+ fn aknowledged_correction() {
+ assert_suggests_correction("aknowledged", "acknowledged");
+ }
+
+ #[test]
+ fn alcaholic_correction() {
+ assert_suggests_correction("alcaholic", "alcoholic");
+ }
+
+ #[test]
+ fn slaves_correction() {
+ assert_suggests_correction("Slaves", "Slavs");
+ }
+
+ #[test]
+ fn conciousness_correction() {
+ assert_suggests_correction("conciousness", "consciousness");
+ }
}