Skip to content

Offline, privacy-first grammar checker. Fast, open-source, Rust-powered

fix(test): improved performance and correctness

Elijah Potter62ebf5e66fc6
diff --git a/harper-core/dictionary.dict b/harper-core/dictionary.dict
index f226f088f..a11d7ec5f 100644
--- a/harper-core/dictionary.dict
+++ b/harper-core/dictionary.dict
@@ -33268,7 +33268,7 @@ necrotic/5
nectar/14M
nectarine/15MS
nee/5
-need/14MDSG
+need/14MDSG~
needed/54U
needful/51Y
neediness/1M
diff --git a/harper-core/src/spell/fst_dictionary.rs b/harper-core/src/spell/fst_dictionary.rs
index 7702fdb0b..574b11154 100644
--- a/harper-core/src/spell/fst_dictionary.rs
+++ b/harper-core/src/spell/fst_dictionary.rs
@@ -4,7 +4,6 @@ use super::{
};
use fst::{map::StreamWithState, IntoStreamer, Map as FstMap, Streamer};
use hashbrown::HashMap;
-use itertools::Itertools;
use lazy_static::lazy_static;
use levenshtein_automata::{LevenshteinAutomatonBuilder, DFA};
use std::{cell::RefCell, sync::Arc};
@@ -70,6 +69,7 @@ impl FstDictionary {
pub fn new(new_words: HashMap<CharString, WordMetadata>) -> Self {
let mut words: Vec<(CharString, WordMetadata)> = new_words.into_iter().collect();
words.sort_unstable_by(|(a, _), (b, _)| a.cmp(b));
+ words.dedup_by(|(a, _), (b, _)| a == b);
let mut builder = fst::MapBuilder::memory();
for (index, (word, _)) in words.iter().enumerate() {
@@ -113,13 +113,14 @@ fn build_dfa(max_distance: u8, query: &str) -> DFA {
})
}
+/// Consumes a DFA stream and emits the index-edit distance pairs it produces.
fn stream_distances_vec(stream: &mut StreamWithState<&DFA>, dfa: &DFA) -> Vec<(u64, u8)> {
- let mut word_indexes = Vec::new();
+ let mut word_index_pairs = Vec::new();
while let Some((_, v, s)) = stream.next() {
- word_indexes.push((v, dfa.distance(s).to_u8()));
+ word_index_pairs.push((v, dfa.distance(s).to_u8()));
}
- word_indexes
+ word_index_pairs
}
impl Dictionary for FstDictionary {
@@ -157,28 +158,33 @@ impl Dictionary for FstDictionary {
.search_with_state(&dfa_lowercase)
.into_stream();
- stream_distances_vec(&mut word_indexes_stream, &dfa)
- .into_iter()
- .merge(stream_distances_vec(
- &mut word_indexes_lowercase_stream,
- &dfa_lowercase,
- ))
- // Remove duplicates caused by checking exact and lowercase
- .sorted_unstable_by_key(|a| a.0)
- .dedup_by(|a, b| a.0 == b.0)
- // Sort by edit distance
- .sorted_unstable_by_key(|a| a.1)
- .take(max_results)
- .map(|(index, edit_distance)| {
- let (word, metadata) = &self.words[index as usize];
-
- FuzzyMatchResult {
- word,
- edit_distance,
- metadata: *metadata,
- }
+ let upper_dists = stream_distances_vec(&mut word_indexes_stream, &dfa);
+ let lower_dists = stream_distances_vec(&mut word_indexes_lowercase_stream, &dfa_lowercase);
+
+ let mut merged = Vec::with_capacity(upper_dists.len());
+
+ // Merge the two results
+ for ((i_u, dist_u), (i_l, dist_l)) in upper_dists.into_iter().zip(lower_dists.into_iter()) {
+ let (chosen_index, edit_distance) = if dist_u <= dist_l {
+ (i_u, dist_u)
+ } else {
+ (i_l, dist_l)
+ };
+
+ let (word, metadata) = &self.words[chosen_index as usize];
+
+ merged.push(FuzzyMatchResult {
+ word,
+ edit_distance,
+ metadata: *metadata,
})
- .collect()
+ }
+
+ merged.sort_unstable_by_key(|v| v.word);
+ merged.sort_unstable_by_key(|v| v.edit_distance);
+ merged.truncate(max_results);
+
+ merged
}
fn fuzzy_match_str(
diff --git a/harper-core/src/spell/full_dictionary.rs b/harper-core/src/spell/full_dictionary.rs
index 3febcc566..a11cd516b 100644
--- a/harper-core/src/spell/full_dictionary.rs
+++ b/harper-core/src/spell/full_dictionary.rs
@@ -176,6 +176,7 @@ impl Dictionary for FullDictionary {
max_results: usize,
) -> Vec<FuzzyMatchResult> {
let misspelled_charslice = seq_to_normalized(word);
+ let misspelled_charslice_lower = misspelled_charslice.to_lower();
let shortest_word_len = if misspelled_charslice.len() <= max_distance as usize {
1
@@ -200,7 +201,7 @@ impl Dictionary for FullDictionary {
let dist =
edit_distance_min_alloc(&misspelled_charslice, word, &mut buf_a, &mut buf_b);
let lowercase_dist = edit_distance_min_alloc(
- &misspelled_charslice.to_lower(),
+ &misspelled_charslice_lower,
word,
&mut buf_a,
&mut buf_b,
diff --git a/harper-core/src/spell/mod.rs b/harper-core/src/spell/mod.rs
index a74a2abc1..7e9f02a11 100644
--- a/harper-core/src/spell/mod.rs
+++ b/harper-core/src/spell/mod.rs
@@ -28,6 +28,7 @@ impl<'a> PartialOrd for FuzzyMatchResult<'a> {
}
}
+/// Order the suggestions to be shown to the user.
fn order_suggestions(matches: Vec<FuzzyMatchResult>) -> Vec<&[char]> {
let mut found: Vec<&FuzzyMatchResult> = Vec::with_capacity(matches.len());
// Often the longest and the shortest words are the most helpful, so lets push
@@ -58,8 +59,8 @@ fn order_suggestions(matches: Vec<FuzzyMatchResult>) -> Vec<&[char]> {
found.swap(0, 2);
}
- // Let common words bubble up, but do not prioritize them over all else.
- found.sort_by_key(|fmr| fmr.edit_distance + if fmr.metadata.common { 0 } else { 1 });
+ // Make commonality relevant
+ found.sort_by_key(|fmr| if fmr.metadata.common { 0 } else { 1 });
found.into_iter().map(|fmr| fmr.word).collect()
}
@@ -168,7 +169,7 @@ mod tests {
Dictionary, FstDictionary, FullDictionary,
};
- const RESULT_LIMIT: usize = 100;
+ const RESULT_LIMIT: usize = 60;
const MAX_EDIT_DIST: u8 = 3;
fn assert_edit_dist(source: &str, target: &str, expected: u8) {
@@ -301,13 +302,38 @@ mod tests {
assert!(results.iter().take(3).contains(&"this".to_string()));
}
- // I'm ignoring this one because the sorting algorithm prioritizes shorter words at the same
- // edit distance that are also common.
- #[ignore]
#[test]
- fn issue_182() {
+ fn need_correction_full() {
let results = suggest_correct_spelling_str(
- "im",
+ "ned",
+ RESULT_LIMIT,
+ MAX_EDIT_DIST,
+ &FullDictionary::curated(),
+ );
+
+ dbg!(&results);
+
+ assert!(results.iter().take(3).contains(&"need".to_string()));
+ }
+
+ #[test]
+ fn need_correction_fst() {
+ let results = suggest_correct_spelling_str(
+ "ned",
+ RESULT_LIMIT,
+ MAX_EDIT_DIST,
+ &FstDictionary::curated(),
+ );
+
+ dbg!(&results);
+
+ assert!(results.iter().take(3).contains(&"need".to_string()));
+ }
+
+ #[test]
+ fn issue_182_fst() {
+ let results = suggest_correct_spelling_str(
+ "Im",
RESULT_LIMIT,
MAX_EDIT_DIST,
&FstDictionary::curated(),
@@ -318,6 +344,20 @@ mod tests {
assert!(results.iter().take(3).contains(&"I'm".to_string()));
}
+ #[test]
+ fn issue_182_full() {
+ let results = suggest_correct_spelling_str(
+ "Im",
+ RESULT_LIMIT,
+ MAX_EDIT_DIST,
+ &FullDictionary::curated(),
+ );
+
+ dbg!(&results);
+
+ assert!(results.iter().take(3).contains(&"I'm".to_string()));
+ }
+
#[test]
fn fst_spellcheck_hvllo() {
let results = suggest_correct_spelling_str(