Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
feat(fst-dict): add new method for FstDictionary that accepts a hashmap
Grant Lemons23bccb923372
diff --git a/harper-core/src/spell/fst_dictionary.rs b/harper-core/src/spell/fst_dictionary.rsindex 2f6d32a99..552e9fbd3 100644--- a/harper-core/src/spell/fst_dictionary.rs+++ b/harper-core/src/spell/fst_dictionary.rs@@ -1,6 +1,8 @@use super::{edit_distance_min_alloc, seq_to_normalized, FullDictionary};use fst::Map as FstMap;use fst::{automaton::Levenshtein, IntoStreamer};+use harper_dictionary_parsing::CharString;+use hashbrown::HashMap;use itertools::Itertools;use std::sync::Arc;@@ -44,6 +46,38 @@ impl FstDictionary {pub fn curated() -> Arc<Self> {DICT.with(|v| v.clone())}++ pub fn new(new_words: HashMap<CharString, WordMetadata>) -> Self {+ let words = new_words+ .into_iter()+ .sorted_by_key(|p| p.0.to_owned())+ .dedup_by(|p1, p2| p1.0 == p2.0)+ .sorted_by_key(|p| p.0.len());++ let mut builder = fst::MapBuilder::memory();+ words+ .clone()+ .enumerate()+ .map(|(i, (w, _))| (i, w))+ .sorted_by_key(|w| w.1.to_owned())+ .for_each(|(i, w)| {+ let word = w.iter().collect::<String>();+ builder+ .insert(word, i as u64)+ .expect("Insertion not in lexicographical order!");+ });++ let mut full_dict = FullDictionary::new();+ full_dict.extend_words(words);++ let fst_bytes = builder.into_inner().unwrap();+ let word_map = FstMap::new(fst_bytes).expect("Unable to build FST map.");++ FstDictionary {+ full_dict: Arc::new(full_dict),+ word_map,+ }+ }}impl Dictionary for FstDictionary {diff --git a/harper-core/src/spell/full_dictionary.rs b/harper-core/src/spell/full_dictionary.rsindex a0b90f870..978b0b9a6 100644--- a/harper-core/src/spell/full_dictionary.rs+++ b/harper-core/src/spell/full_dictionary.rs@@ -84,6 +84,7 @@ impl FullDictionary {.collect();self.words.extend(pairs.iter().map(|(v, _)| v.clone()));+ self.words.sort_by_key(|w| w.len());self.word_len_starts = Self::create_len_starts(&self.words);self.word_map.extend(pairs);}