Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
fix(test): improved performance and correctness
Elijah Potter62ebf5e66fc6
diff --git a/harper-core/dictionary.dict b/harper-core/dictionary.dictindex f226f088f..a11d7ec5f 100644--- a/harper-core/dictionary.dict+++ b/harper-core/dictionary.dict@@ -33268,7 +33268,7 @@ necrotic/5nectar/14Mnectarine/15MSnee/5-need/14MDSG+need/14MDSG~needed/54Uneedful/51Yneediness/1Mdiff --git a/harper-core/src/spell/fst_dictionary.rs b/harper-core/src/spell/fst_dictionary.rsindex 7702fdb0b..574b11154 100644--- a/harper-core/src/spell/fst_dictionary.rs+++ b/harper-core/src/spell/fst_dictionary.rs@@ -4,7 +4,6 @@ use super::{};use fst::{map::StreamWithState, IntoStreamer, Map as FstMap, Streamer};use hashbrown::HashMap;-use itertools::Itertools;use lazy_static::lazy_static;use levenshtein_automata::{LevenshteinAutomatonBuilder, DFA};use std::{cell::RefCell, sync::Arc};@@ -70,6 +69,7 @@ impl FstDictionary {pub fn new(new_words: HashMap<CharString, WordMetadata>) -> Self {let mut words: Vec<(CharString, WordMetadata)> = new_words.into_iter().collect();words.sort_unstable_by(|(a, _), (b, _)| a.cmp(b));+ words.dedup_by(|(a, _), (b, _)| a == b);let mut builder = fst::MapBuilder::memory();for (index, (word, _)) in words.iter().enumerate() {@@ -113,13 +113,14 @@ fn build_dfa(max_distance: u8, query: &str) -> DFA {})}+/// Consumes a DFA stream and emits the index-edit distance pairs it produces.fn stream_distances_vec(stream: &mut StreamWithState<&DFA>, dfa: &DFA) -> Vec<(u64, u8)> {- let mut word_indexes = Vec::new();+ let mut word_index_pairs = Vec::new();while let Some((_, v, s)) = stream.next() {- word_indexes.push((v, dfa.distance(s).to_u8()));+ word_index_pairs.push((v, dfa.distance(s).to_u8()));}- word_indexes+ word_index_pairs}impl Dictionary for FstDictionary {@@ -157,28 +158,33 @@ impl Dictionary for FstDictionary {.search_with_state(&dfa_lowercase).into_stream();- stream_distances_vec(&mut word_indexes_stream, &dfa)- .into_iter()- .merge(stream_distances_vec(- &mut word_indexes_lowercase_stream,- &dfa_lowercase,- ))- // Remove duplicates caused by checking exact and lowercase- .sorted_unstable_by_key(|a| a.0)- .dedup_by(|a, b| a.0 == b.0)- // Sort by edit distance- .sorted_unstable_by_key(|a| a.1)- .take(max_results)- .map(|(index, edit_distance)| {- let (word, metadata) = &self.words[index as usize];-- FuzzyMatchResult {- word,- edit_distance,- metadata: *metadata,- }+ let upper_dists = stream_distances_vec(&mut word_indexes_stream, &dfa);+ let lower_dists = stream_distances_vec(&mut word_indexes_lowercase_stream, &dfa_lowercase);++ let mut merged = Vec::with_capacity(upper_dists.len());++ // Merge the two results+ for ((i_u, dist_u), (i_l, dist_l)) in upper_dists.into_iter().zip(lower_dists.into_iter()) {+ let (chosen_index, edit_distance) = if dist_u <= dist_l {+ (i_u, dist_u)+ } else {+ (i_l, dist_l)+ };++ let (word, metadata) = &self.words[chosen_index as usize];++ merged.push(FuzzyMatchResult {+ word,+ edit_distance,+ metadata: *metadata,})- .collect()+ }++ merged.sort_unstable_by_key(|v| v.word);+ merged.sort_unstable_by_key(|v| v.edit_distance);+ merged.truncate(max_results);++ merged}fn fuzzy_match_str(diff --git a/harper-core/src/spell/full_dictionary.rs b/harper-core/src/spell/full_dictionary.rsindex 3febcc566..a11cd516b 100644--- a/harper-core/src/spell/full_dictionary.rs+++ b/harper-core/src/spell/full_dictionary.rs@@ -176,6 +176,7 @@ impl Dictionary for FullDictionary {max_results: usize,) -> Vec<FuzzyMatchResult> {let misspelled_charslice = seq_to_normalized(word);+ let misspelled_charslice_lower = misspelled_charslice.to_lower();let shortest_word_len = if misspelled_charslice.len() <= max_distance as usize {1@@ -200,7 +201,7 @@ impl Dictionary for FullDictionary {let dist =edit_distance_min_alloc(&misspelled_charslice, word, &mut buf_a, &mut buf_b);let lowercase_dist = edit_distance_min_alloc(- &misspelled_charslice.to_lower(),+ &misspelled_charslice_lower,word,&mut buf_a,&mut buf_b,diff --git a/harper-core/src/spell/mod.rs b/harper-core/src/spell/mod.rsindex a74a2abc1..7e9f02a11 100644--- a/harper-core/src/spell/mod.rs+++ b/harper-core/src/spell/mod.rs@@ -28,6 +28,7 @@ impl<'a> PartialOrd for FuzzyMatchResult<'a> {}}+/// Order the suggestions to be shown to the user.fn order_suggestions(matches: Vec<FuzzyMatchResult>) -> Vec<&[char]> {let mut found: Vec<&FuzzyMatchResult> = Vec::with_capacity(matches.len());// Often the longest and the shortest words are the most helpful, so lets push@@ -58,8 +59,8 @@ fn order_suggestions(matches: Vec<FuzzyMatchResult>) -> Vec<&[char]> {found.swap(0, 2);}- // Let common words bubble up, but do not prioritize them over all else.- found.sort_by_key(|fmr| fmr.edit_distance + if fmr.metadata.common { 0 } else { 1 });+ // Make commonality relevant+ found.sort_by_key(|fmr| if fmr.metadata.common { 0 } else { 1 });found.into_iter().map(|fmr| fmr.word).collect()}@@ -168,7 +169,7 @@ mod tests {Dictionary, FstDictionary, FullDictionary,};- const RESULT_LIMIT: usize = 100;+ const RESULT_LIMIT: usize = 60;const MAX_EDIT_DIST: u8 = 3;fn assert_edit_dist(source: &str, target: &str, expected: u8) {@@ -301,13 +302,38 @@ mod tests {assert!(results.iter().take(3).contains(&"this".to_string()));}- // I'm ignoring this one because the sorting algorithm prioritizes shorter words at the same- // edit distance that are also common.- #[ignore]#[test]- fn issue_182() {+ fn need_correction_full() {let results = suggest_correct_spelling_str(- "im",+ "ned",+ RESULT_LIMIT,+ MAX_EDIT_DIST,+ &FullDictionary::curated(),+ );++ dbg!(&results);++ assert!(results.iter().take(3).contains(&"need".to_string()));+ }++ #[test]+ fn need_correction_fst() {+ let results = suggest_correct_spelling_str(+ "ned",+ RESULT_LIMIT,+ MAX_EDIT_DIST,+ &FstDictionary::curated(),+ );++ dbg!(&results);++ assert!(results.iter().take(3).contains(&"need".to_string()));+ }++ #[test]+ fn issue_182_fst() {+ let results = suggest_correct_spelling_str(+ "Im",RESULT_LIMIT,MAX_EDIT_DIST,&FstDictionary::curated(),@@ -318,6 +344,20 @@ mod tests {assert!(results.iter().take(3).contains(&"I'm".to_string()));}+ #[test]+ fn issue_182_full() {+ let results = suggest_correct_spelling_str(+ "Im",+ RESULT_LIMIT,+ MAX_EDIT_DIST,+ &FullDictionary::curated(),+ );++ dbg!(&results);++ assert!(results.iter().take(3).contains(&"I'm".to_string()));+ }+#[test]fn fst_spellcheck_hvllo() {let results = suggest_correct_spelling_str(