Skip to content

Offline, privacy-first grammar checker. Fast, open-source, Rust-powered

fix(core): remove redundant copy of dictionary in `FstDictionary` (#4030)

Elijah Potterc4490ad62d47
diff --git a/harper-core/src/spell/fst_dictionary.rs b/harper-core/src/spell/fst_dictionary.rs
index 70c0fa9c4..9b39e1e84 100644
--- a/harper-core/src/spell/fst_dictionary.rs
+++ b/harper-core/src/spell/fst_dictionary.rs
@@ -18,10 +18,8 @@ use crate::{CharString, CharStringExt, DictWordMetadata};
pub struct FstDictionary {
/// Underlying [`super::MutableDictionary`] used for everything except fuzzy finding
mutable_dict: Arc<MutableDictionary>,
- /// Used for fuzzy-finding the index of words or metadata
+ /// Used for fuzzy-finding the WordId of words or metadata
word_map: FstMap<Vec<u8>>,
- /// Used for fuzzy-finding the index of words or metadata
- words: Vec<(CharString, DictWordMetadata)>,
}
const EXPECTED_DISTANCE: u8 = 3;
@@ -61,10 +59,10 @@ impl FstDictionary {
words.dedup_by(|(a, _), (b, _)| a == b);
let mut builder = fst::MapBuilder::memory();
- for (index, (word, _)) in words.iter().enumerate() {
- let word = word.iter().collect::<String>();
+ for (word_chars, _) in words.iter() {
+ let word = word_chars.iter().collect::<String>();
builder
- .insert(word, index as u64)
+ .insert(word, WordId::from_word_chars(word_chars).into())
.expect("Insertion not in lexicographical order!");
}
@@ -77,7 +75,6 @@ impl FstDictionary {
FstDictionary {
mutable_dict: Arc::new(mutable_dict),
word_map,
- words,
}
}
}
@@ -102,24 +99,24 @@ fn build_dfa(max_distance: u8, query: &str) -> DFA {
})
}
-/// Consumes a DFA stream and emits the index-edit distance pairs it produces.
+/// Consumes a DFA stream and emits the WordID-edit distance pairs it produces.
fn stream_distances_vec(stream: &mut StreamWithState<&DFA>, dfa: &DFA) -> Vec<(u64, u8)> {
- let mut word_index_pairs = Vec::new();
+ let mut word_id_pairs = Vec::new();
while let Some((_, v, s)) = stream.next() {
- word_index_pairs.push((v, dfa.distance(s).to_u8()));
+ word_id_pairs.push((v, dfa.distance(s).to_u8()));
}
- word_index_pairs
+ word_id_pairs
}
-/// Merges index-distance pairs, keeping the smallest distance for each word.
+/// Merges WordID-distance pairs, keeping the smallest distance for each word.
fn merge_best_distances(
best_distances: &mut HashMap<u64, u8>,
distances: impl IntoIterator<Item = (u64, u8)>,
) {
- for (idx, dist) in distances {
+ for (word_id, dist) in distances {
best_distances
- .entry(idx)
+ .entry(word_id)
.and_modify(|existing| *existing = (*existing).min(dist))
.or_insert(dist);
}
@@ -155,8 +152,8 @@ impl Dictionary for FstDictionary {
// Actual FST search
let dfa = build_dfa(max_distance, &misspelled_word_string);
- let mut word_indexes_stream = self.word_map.search_with_state(&dfa).into_stream();
- let upper_dists = stream_distances_vec(&mut word_indexes_stream, &dfa);
+ let mut word_ids_stream = self.word_map.search_with_state(&dfa).into_stream();
+ let upper_dists = stream_distances_vec(&mut word_ids_stream, &dfa);
// Merge the two results, keeping the smallest distance when both DFAs match.
// The uppercase and lowercase searches can return different result counts, so
@@ -168,23 +165,23 @@ impl Dictionary for FstDictionary {
// Only build the lowercase DFA when the query is not already lowercase.
if !is_already_lower {
let dfa_lowercase = build_dfa(max_distance, &misspelled_lower);
- let mut word_indexes_lowercase_stream = self
+ let mut word_ids_lowercase_stream = self
.word_map
.search_with_state(&dfa_lowercase)
.into_stream();
- let lower_dists =
- stream_distances_vec(&mut word_indexes_lowercase_stream, &dfa_lowercase);
+ let lower_dists = stream_distances_vec(&mut word_ids_lowercase_stream, &dfa_lowercase);
merge_best_distances(&mut best_distances, lower_dists);
}
let mut merged = Vec::with_capacity(best_distances.len());
- for (index, edit_distance) in best_distances {
- let (word, metadata) = &self.words[index as usize];
+ for (word_id, edit_distance) in best_distances {
+ let word = self.mutable_dict.get_word_from_id(&word_id.into()).unwrap();
+ let metadata = self.mutable_dict.get_word_metadata(word).unwrap();
merged.push(FuzzyMatchResult {
word,
edit_distance,
- metadata: Cow::Borrowed(metadata),
+ metadata,
});
}
@@ -357,13 +354,6 @@ mod tests {
assert!(is_sorted_by_dist)
}
- #[test]
- fn curated_contains_no_duplicates() {
- let dict = FstDictionary::curated();
-
- assert!(dict.words.iter().map(|(word, _)| word).all_unique());
- }
-
#[test]
fn contractions_not_derived() {
let dict = FstDictionary::curated();
diff --git a/harper-core/src/spell/word_id.rs b/harper-core/src/spell/word_id.rs
index 9140a3911..f93fc2087 100644
--- a/harper-core/src/spell/word_id.rs
+++ b/harper-core/src/spell/word_id.rs
@@ -32,3 +32,15 @@ impl WordId {
Self::from_word_chars(chars)
}
}
+
+impl From<WordId> for u64 {
+ fn from(val: WordId) -> Self {
+ val.hash
+ }
+}
+
+impl From<u64> for WordId {
+ fn from(value: u64) -> Self {
+ Self { hash: value }
+ }
+}