Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
feat(core): use scoring rather than shuffling for spelling suggestions
Elijah Potter99e77c4fb3f4
diff --git a/harper-core/src/spell/mod.rs b/harper-core/src/spell/mod.rsindex 9baf12850..9721a90b8 100644--- a/harper-core/src/spell/mod.rs+++ b/harper-core/src/spell/mod.rs@@ -1,7 +1,5 @@use std::borrow::Cow;-use itertools::{Itertools, MinMaxResult};-use crate::{CharString, CharStringExt, WordMetadata};pub use self::dictionary::Dictionary;@@ -28,49 +26,42 @@ impl PartialOrd for FuzzyMatchResult<'_> {}}-/// Order the suggestions to be shown to the user.-fn order_suggestions(matches: Vec<FuzzyMatchResult>) -> Vec<&[char]> {- let mut found: Vec<&FuzzyMatchResult> = Vec::with_capacity(matches.len());- // Often the longest and the shortest words are the most helpful, so let's push- // them first.- let minmax = matches.iter().position_minmax_by_key(|fmr| fmr.word.len());- if let MinMaxResult::MinMax(a, b) = minmax {- if a == b {- found.push(&matches[a]);- } else {- found.push(&matches[a]);- found.push(&matches[b]);- }-- // Push the rest- found.extend(- matches- .iter()- .enumerate()- .filter(|(i, _)| *i != a && *i != b)- .map(|v| v.1),- );- } else {- found.extend(&matches);+/// Scores a possible spelling suggestion based on possible relevance to the user.+///+/// Lower = better.+fn score_suggestion(misspelled_word: &[char], sug: &FuzzyMatchResult) -> i32 {+ if misspelled_word.is_empty() {+ return i32::MAX;+ }++ let mut score = sug.edit_distance as i32 * 10;++ // If the original word is plural, the correct one probably is too.+ if *misspelled_word.last().unwrap() == 's' && *sug.word.last().unwrap() == 's' {+ score -= 5;}- // Swap the lowest edit distance word with the shortest.- if found.len() >= 3 {- found.swap(0, 2);+ // For turning words into contractions.+ if sug.metadata.common {+ score -= 5;}- if let Some(nominal_index) = found- .iter()- .skip(3)- .position(|i| i.metadata.is_proper_noun())- {- found.swap(2, nominal_index + 3);+ // For turning words into contractions.+ if sug.word.iter().filter(|c| **c == '\'').count() == 1 {+ score -= 5;}- // Make commonality relevant- found.sort_by_key(|fmr| if fmr.metadata.common { 0 } else { 1 });+ score+}++/// Order the suggestions to be shown to the user.+fn order_suggestions<'b>(+ misspelled_word: &[char],+ mut matches: Vec<FuzzyMatchResult<'b>>,+) -> Vec<&'b [char]> {+ matches.sort_by_key(|v| score_suggestion(misspelled_word, v));- found.into_iter().map(|fmr| fmr.word).collect()+ matches.into_iter().map(|v| v.word).collect()}/// Get the closest matches in the provided [`Dictionary`] and rank them@@ -86,7 +77,7 @@ pub fn suggest_correct_spelling<'a>(.into_iter().collect();- order_suggestions(matches)+ order_suggestions(misspelled_word, matches)}/// Convenience function over [`suggest_correct_spelling`] that does conversions@@ -127,11 +118,8 @@ fn char_to_normalized(c: char) -> char {mod tests {use itertools::Itertools;- use crate::spell::FuzzyMatchResult;-use super::{- Dictionary, FstDictionary, MutableDictionary, order_suggestions, seq_to_normalized,- suggest_correct_spelling_str,+ FstDictionary, order_suggestions, seq_to_normalized, suggest_correct_spelling_str,};const RESULT_LIMIT: usize = 100;@@ -157,96 +145,24 @@ mod tests {assert!(results.iter().all_unique())}- /// Ensures that the suggestions are ordered taking into account commonality#[test]- fn common_words_first() {- let dict = FstDictionary::curated();- // Select three common words- let common_words = dict- .words_iter()- .filter_map(|word| {- let metadata = dict.get_word_metadata(word).unwrap();- if metadata.common {- Some(FuzzyMatchResult {- word,- edit_distance: 0,- metadata,- })- } else {- None- }- })- .take(3);- // Select three uncommon words- let uncommon_words = dict- .words_iter()- .filter_map(|word| {- let metadata = dict.get_word_metadata(word).unwrap();- if metadata.common {- None- } else {- Some(FuzzyMatchResult {- word,- edit_distance: 0,- metadata,- })- }- })- .take(3);- // Feed the common and uncommon words into the ordering function, starting with uncommon- // words- let words = uncommon_words.merge(common_words).collect();- let suggestions = order_suggestions(words);-- // Asserts that the ordering prioritizes common words- let common_first = suggestions- .into_iter()- .take(3)- .all(|word| dict.get_word_metadata(word).unwrap().common);-- assert!(common_first);+ fn youre_contraction() {+ assert_suggests_correction("youre", "you're");}#[test]- fn this_correction() {- let results = suggest_correct_spelling_str(- "Ths",- RESULT_LIMIT,- MAX_EDIT_DIST,- &FstDictionary::curated(),- );-- dbg!(&results);-- assert!(results.iter().take(3).contains(&"this".to_string()));+ fn thats_contraction() {+ assert_suggests_correction("thats", "that's");}#[test]- fn need_correction_full() {- let results = suggest_correct_spelling_str(- "ned",- RESULT_LIMIT,- MAX_EDIT_DIST,- &MutableDictionary::curated(),- );-- dbg!(&results);-- assert!(results.iter().take(3).contains(&"need".to_string()));+ fn weve_contraction() {+ assert_suggests_correction("weve", "we've");}#[test]- fn need_correction_fst() {- let results = suggest_correct_spelling_str(- "ned",- RESULT_LIMIT,- MAX_EDIT_DIST,- &FstDictionary::curated(),- );-- dbg!(&results);-- assert!(results.iter().take(3).contains(&"need".to_string()));+ fn this_correction() {+ assert_suggests_correction("ths", "this");}#[test]@@ -264,31 +180,8 @@ mod tests {}#[test]- fn issue_182_fst() {- let results = suggest_correct_spelling_str(- "Im",- RESULT_LIMIT,- MAX_EDIT_DIST,- &FstDictionary::curated(),- );-- dbg!(&results);-- assert!(results.iter().take(3).contains(&"I'm".to_string()));- }-- #[test]- fn issue_182_full() {- let results = suggest_correct_spelling_str(- "Im",- RESULT_LIMIT,- MAX_EDIT_DIST,- &MutableDictionary::curated(),- );-- dbg!(&results);-- assert!(results.iter().take(3).contains(&"I'm".to_string()));+ fn issue_182() {+ assert_suggests_correction("Im", "I'm");}#[test]@@ -305,46 +198,29 @@ mod tests {assert!(results.iter().take(3).contains(&"hello".to_string()));}- #[test]- fn full_spellcheck_hvllo() {+ /// Assert that the default suggestion settings result in a specific word+ /// being in the top three results for a given misspelling.+ fn assert_suggests_correction(misspelled_word: &str, correct: &str) {let results = suggest_correct_spelling_str(- "hvllo",+ misspelled_word,RESULT_LIMIT,MAX_EDIT_DIST,- &MutableDictionary::curated(),+ &FstDictionary::curated(),);dbg!(&results);- assert!(results.iter().take(3).contains(&"hello".to_string()));+ assert!(results.iter().take(3).contains(&correct.to_string()));}#[test]- fn fst_spellcheck_common() {- let results = suggest_correct_spelling_str(- "aboot",- RESULT_LIMIT,- MAX_EDIT_DIST,- &FstDictionary::curated(),- );-- dbg!(&results);-- assert!(results.iter().take(3).contains(&"about".to_string()));+ fn spellcheck_hvllo() {+ assert_suggests_correction("hvllo", "hello");}#[test]- fn full_spellcheck_common() {- let results = suggest_correct_spelling_str(- "aboot",- RESULT_LIMIT,- MAX_EDIT_DIST,- &MutableDictionary::curated(),- );-- dbg!(&results);-- assert!(results.iter().take(3).contains(&"about".to_string()));+ fn spellcheck_aout() {+ assert_suggests_correction("aout", "about");}#[test]@@ -371,4 +247,29 @@ mod tests {assert_eq!(results1, results2);assert_eq!(results1, results3);}++ #[test]+ fn adviced_correction() {+ assert_suggests_correction("adviced", "advised");+ }++ #[test]+ fn aknowledged_correction() {+ assert_suggests_correction("aknowledged", "acknowledged");+ }++ #[test]+ fn alcaholic_correction() {+ assert_suggests_correction("alcaholic", "alcoholic");+ }++ #[test]+ fn slaves_correction() {+ assert_suggests_correction("Slaves", "Slavs");+ }++ #[test]+ fn conciousness_correction() {+ assert_suggests_correction("conciousness", "consciousness");+ }}