Skip to content

Offline, privacy-first grammar checker. Fast, open-source, Rust-powered

feat(fst-dict): add new method for FstDictionary that accepts a hashmap

Grant Lemons23bccb923372
diff --git a/harper-core/src/spell/fst_dictionary.rs b/harper-core/src/spell/fst_dictionary.rs
index 2f6d32a99..552e9fbd3 100644
--- a/harper-core/src/spell/fst_dictionary.rs
+++ b/harper-core/src/spell/fst_dictionary.rs
@@ -1,6 +1,8 @@
use super::{edit_distance_min_alloc, seq_to_normalized, FullDictionary};
use fst::Map as FstMap;
use fst::{automaton::Levenshtein, IntoStreamer};
+use harper_dictionary_parsing::CharString;
+use hashbrown::HashMap;
use itertools::Itertools;
use std::sync::Arc;
@@ -44,6 +46,38 @@ impl FstDictionary {
pub fn curated() -> Arc<Self> {
DICT.with(|v| v.clone())
}
+
+ pub fn new(new_words: HashMap<CharString, WordMetadata>) -> Self {
+ let words = new_words
+ .into_iter()
+ .sorted_by_key(|p| p.0.to_owned())
+ .dedup_by(|p1, p2| p1.0 == p2.0)
+ .sorted_by_key(|p| p.0.len());
+
+ let mut builder = fst::MapBuilder::memory();
+ words
+ .clone()
+ .enumerate()
+ .map(|(i, (w, _))| (i, w))
+ .sorted_by_key(|w| w.1.to_owned())
+ .for_each(|(i, w)| {
+ let word = w.iter().collect::<String>();
+ builder
+ .insert(word, i as u64)
+ .expect("Insertion not in lexicographical order!");
+ });
+
+ let mut full_dict = FullDictionary::new();
+ full_dict.extend_words(words);
+
+ let fst_bytes = builder.into_inner().unwrap();
+ let word_map = FstMap::new(fst_bytes).expect("Unable to build FST map.");
+
+ FstDictionary {
+ full_dict: Arc::new(full_dict),
+ word_map,
+ }
+ }
}
impl Dictionary for FstDictionary {
diff --git a/harper-core/src/spell/full_dictionary.rs b/harper-core/src/spell/full_dictionary.rs
index a0b90f870..978b0b9a6 100644
--- a/harper-core/src/spell/full_dictionary.rs
+++ b/harper-core/src/spell/full_dictionary.rs
@@ -84,6 +84,7 @@ impl FullDictionary {
.collect();
self.words.extend(pairs.iter().map(|(v, _)| v.clone()));
+ self.words.sort_by_key(|w| w.len());
self.word_len_starts = Self::create_len_starts(&self.words);
self.word_map.extend(pairs);
}