Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
fix(core): remove redundant copy of dictionary in `FstDictionary` (#4030)
Elijah Potterc4490ad62d47
diff --git a/harper-core/src/spell/fst_dictionary.rs b/harper-core/src/spell/fst_dictionary.rsindex 70c0fa9c4..9b39e1e84 100644--- a/harper-core/src/spell/fst_dictionary.rs+++ b/harper-core/src/spell/fst_dictionary.rs@@ -18,10 +18,8 @@ use crate::{CharString, CharStringExt, DictWordMetadata};pub struct FstDictionary {/// Underlying [`super::MutableDictionary`] used for everything except fuzzy findingmutable_dict: Arc<MutableDictionary>,- /// Used for fuzzy-finding the index of words or metadata+ /// Used for fuzzy-finding the WordId of words or metadataword_map: FstMap<Vec<u8>>,- /// Used for fuzzy-finding the index of words or metadata- words: Vec<(CharString, DictWordMetadata)>,}const EXPECTED_DISTANCE: u8 = 3;@@ -61,10 +59,10 @@ impl FstDictionary {words.dedup_by(|(a, _), (b, _)| a == b);let mut builder = fst::MapBuilder::memory();- for (index, (word, _)) in words.iter().enumerate() {- let word = word.iter().collect::<String>();+ for (word_chars, _) in words.iter() {+ let word = word_chars.iter().collect::<String>();builder- .insert(word, index as u64)+ .insert(word, WordId::from_word_chars(word_chars).into()).expect("Insertion not in lexicographical order!");}@@ -77,7 +75,6 @@ impl FstDictionary {FstDictionary {mutable_dict: Arc::new(mutable_dict),word_map,- words,}}}@@ -102,24 +99,24 @@ fn build_dfa(max_distance: u8, query: &str) -> DFA {})}-/// Consumes a DFA stream and emits the index-edit distance pairs it produces.+/// Consumes a DFA stream and emits the WordID-edit distance pairs it produces.fn stream_distances_vec(stream: &mut StreamWithState<&DFA>, dfa: &DFA) -> Vec<(u64, u8)> {- let mut word_index_pairs = Vec::new();+ let mut word_id_pairs = Vec::new();while let Some((_, v, s)) = stream.next() {- word_index_pairs.push((v, dfa.distance(s).to_u8()));+ word_id_pairs.push((v, dfa.distance(s).to_u8()));}- word_index_pairs+ word_id_pairs}-/// Merges index-distance pairs, keeping the smallest distance for each word.+/// Merges WordID-distance pairs, keeping the smallest distance for each word.fn merge_best_distances(best_distances: &mut HashMap<u64, u8>,distances: impl IntoIterator<Item = (u64, u8)>,) {- for (idx, dist) in distances {+ for (word_id, dist) in distances {best_distances- .entry(idx)+ .entry(word_id).and_modify(|existing| *existing = (*existing).min(dist)).or_insert(dist);}@@ -155,8 +152,8 @@ impl Dictionary for FstDictionary {// Actual FST searchlet dfa = build_dfa(max_distance, &misspelled_word_string);- let mut word_indexes_stream = self.word_map.search_with_state(&dfa).into_stream();- let upper_dists = stream_distances_vec(&mut word_indexes_stream, &dfa);+ let mut word_ids_stream = self.word_map.search_with_state(&dfa).into_stream();+ let upper_dists = stream_distances_vec(&mut word_ids_stream, &dfa);// Merge the two results, keeping the smallest distance when both DFAs match.// The uppercase and lowercase searches can return different result counts, so@@ -168,23 +165,23 @@ impl Dictionary for FstDictionary {// Only build the lowercase DFA when the query is not already lowercase.if !is_already_lower {let dfa_lowercase = build_dfa(max_distance, &misspelled_lower);- let mut word_indexes_lowercase_stream = self+ let mut word_ids_lowercase_stream = self.word_map.search_with_state(&dfa_lowercase).into_stream();- let lower_dists =- stream_distances_vec(&mut word_indexes_lowercase_stream, &dfa_lowercase);+ let lower_dists = stream_distances_vec(&mut word_ids_lowercase_stream, &dfa_lowercase);merge_best_distances(&mut best_distances, lower_dists);}let mut merged = Vec::with_capacity(best_distances.len());- for (index, edit_distance) in best_distances {- let (word, metadata) = &self.words[index as usize];+ for (word_id, edit_distance) in best_distances {+ let word = self.mutable_dict.get_word_from_id(&word_id.into()).unwrap();+ let metadata = self.mutable_dict.get_word_metadata(word).unwrap();merged.push(FuzzyMatchResult {word,edit_distance,- metadata: Cow::Borrowed(metadata),+ metadata,});}@@ -357,13 +354,6 @@ mod tests {assert!(is_sorted_by_dist)}- #[test]- fn curated_contains_no_duplicates() {- let dict = FstDictionary::curated();-- assert!(dict.words.iter().map(|(word, _)| word).all_unique());- }-#[test]fn contractions_not_derived() {let dict = FstDictionary::curated();diff --git a/harper-core/src/spell/word_id.rs b/harper-core/src/spell/word_id.rsindex 9140a3911..f93fc2087 100644--- a/harper-core/src/spell/word_id.rs+++ b/harper-core/src/spell/word_id.rs@@ -32,3 +32,15 @@ impl WordId {Self::from_word_chars(chars)}}++impl From<WordId> for u64 {+ fn from(val: WordId) -> Self {+ val.hash+ }+}++impl From<u64> for WordId {+ fn from(value: u64) -> Self {+ Self { hash: value }+ }+}