Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
feat(fst-dict): edit dictionary trait to be less FullDictionary-specific
Grant Lemons55f8794234d7
diff --git a/harper-core/src/spell/dictionary.rs b/harper-core/src/spell/dictionary.rsindex 002f5a90f..71221413e 100644--- a/harper-core/src/spell/dictionary.rs+++ b/harper-core/src/spell/dictionary.rs@@ -1,18 +1,18 @@use blanket::blanket;-use crate::WordMetadata;+use crate::{CharString, WordMetadata};#[cfg(not(feature = "concurrent"))]#[blanket(derive(Rc))]-pub trait Dictionary: Clone {+pub trait Dictionary {/// Check if the dictionary contains a given word.fn contains_word(&self, word: &[char]) -> bool;/// Check if the dictionary contains a given word.fn contains_word_str(&self, word: &str) -> bool;- /// Produce an iterator over all words in the dictionary.- fn words_iter(&self) -> impl Iterator<Item = &'_ [char]>;- /// Iterate over all the words in the dictionary of a given length- fn words_with_len_iter(&self, len: usize) -> Box<dyn Iterator<Item = &'_ [char]> + '_>;+ /// Gets best fuzzy match from dictionary+ fn fuzzy_match(&self, word: &[char], max_distance: u8) -> Vec<(CharString, WordMetadata)>;+ /// Gets best fuzzy match from dictionary+ fn fuzzy_match_str(&self, word: &str, max_distance: u8) -> Vec<(CharString, WordMetadata)>;/// Get the associated [`WordMetadata`] for a given word./// If the word isn't in the dictionary, the resulting metadata will be/// empty.@@ -25,15 +25,15 @@ pub trait Dictionary: Clone {#[cfg(feature = "concurrent")]#[blanket(derive(Arc))]-pub trait Dictionary: Send + Sync + Clone {+pub trait Dictionary: Send + Sync {/// Check if the dictionary contains a given word.fn contains_word(&self, word: &[char]) -> bool;/// Check if the dictionary contains a given word.fn contains_word_str(&self, word: &str) -> bool;- /// Produce an iterator over all words in the dictionary.- fn words_iter(&self) -> impl Iterator<Item = &'_ [char]>;- /// Iterate over all the words in the dictionary of a given length- fn words_with_len_iter(&self, len: usize) -> Box<dyn Iterator<Item = &'_ [char]> + '_>;+ /// Gets best fuzzy match from dictionary+ fn fuzzy_match(&self, word: &[char], max_distance: u8) -> Vec<(CharString, WordMetadata)>;+ /// Gets best fuzzy match from dictionary+ fn fuzzy_match_str(&self, word: &str, max_distance: u8) -> Vec<(CharString, WordMetadata)>;/// Get the associated [`WordMetadata`] for a given word./// If the word isn't in the dictionary, the resulting metadata will be/// empty.diff --git a/harper-core/src/spell/fst_dictionary.rs b/harper-core/src/spell/fst_dictionary.rsindex e3c749f94..f739d9073 100644--- a/harper-core/src/spell/fst_dictionary.rs+++ b/harper-core/src/spell/fst_dictionary.rs@@ -1,6 +1,7 @@use std::fs::File;use fst::Map as FstMap;+use fst::{automaton::Levenshtein, IntoStreamer};use memmap::Mmap;use super::hunspell::{parse_default_attribute_list, parse_default_word_list};@@ -8,16 +9,14 @@ use crate::{CharString, Lrc, WordMetadata};use super::Dictionary;-#[derive(Debug, Clone)]+#[derive(Debug)]pub struct FstDictionary {- /// Copied from [`super::FullDictionary`], not sure if rationale there holds here yet.- /// All English words- words: Vec<CharString>,+ /// Sorted by string in lexicographic ordermetadata: Vec<WordMetadata>,/// Used for fuzzy-finding the index of words or metadata- word_map: FstMap<String>,+ word_map: FstMap<Mmap>,}-///+/// The uncached function that is used to produce the original copy of the/// curated dictionary.fn uncached_inner_new() -> Lrc<FstDictionary> {@@ -34,11 +33,7 @@ fn uncached_inner_new() -> Lrc<FstDictionary> {words.sort();words.dedup();- Lrc::new(FstDictionary {- words,- metadata,- word_map,- })+ Lrc::new(FstDictionary { metadata, word_map })}thread_local! {@@ -54,10 +49,6 @@ impl Dictionary for FstDictionary {self.word_map.contains_key(word)}- fn words_iter(&self) -> impl Iterator<Item = &'_ [char]> {- self.words.iter().map(|v| v.as_slice())- }-fn get_word_metadata(&self, word: &[char]) -> WordMetadata {let index: usize = self.word_map.get(word.iter().collect::<String>()).unwrap() as usize;self.metadata[index]@@ -67,4 +58,32 @@ impl Dictionary for FstDictionary {let index: usize = self.word_map.get(word).unwrap() as usize;self.metadata[index]}++ fn fuzzy_match(&self, word: &[char], max_distance: u8) -> Vec<(CharString, WordMetadata)> {+ let aut = Levenshtein::new(&word.iter().collect::<String>(), max_distance as u32).unwrap();+ let words = self+ .word_map+ .search(aut)+ .into_stream()+ .into_str_vec()+ .unwrap();+ words+ .into_iter()+ .map(|(word, i)| (word.chars().collect(), self.metadata[i as usize]))+ .collect()+ }++ fn fuzzy_match_str(&self, word: &str, max_distance: u8) -> Vec<(CharString, WordMetadata)> {+ let aut = Levenshtein::new(word, max_distance as u32).unwrap();+ let words = self+ .word_map+ .search(aut)+ .into_stream()+ .into_str_vec()+ .unwrap();+ words+ .into_iter()+ .map(|(word, i)| (word.chars().collect(), self.metadata[i as usize]))+ .collect()+ }}diff --git a/harper-core/src/spell/full_dictionary.rs b/harper-core/src/spell/full_dictionary.rsindex 936bba3dc..4f6a4ea60 100644--- a/harper-core/src/spell/full_dictionary.rs+++ b/harper-core/src/spell/full_dictionary.rs@@ -125,26 +125,6 @@ impl Default for FullDictionary {}impl Dictionary for FullDictionary {- fn words_iter(&self) -> impl Iterator<Item = &'_ [char]> {- self.words.iter().map(|v| v.as_slice())- }-- /// Iterate over all the words in the dictionary of a given length- fn words_with_len_iter(&self, len: usize) -> Box<dyn Iterator<Item = &'_ [char]> + '_> {- if len == 0 || len >= self.word_len_starts.len() {- return Box::new(std::iter::empty());- }-- let start = self.word_len_starts[len];- let end = if len + 1 == self.word_len_starts.len() {- self.words.len()- } else {- self.word_len_starts[len + 1]- };-- Box::new(self.words[start..end].iter().map(|v| v.as_slice()))- }-fn get_word_metadata(&self, word: &[char]) -> WordMetadata {let normalized = seq_to_normalized(word);let lowercase: CharString = normalized.iter().flat_map(|c| c.to_lowercase()).collect();@@ -172,6 +152,14 @@ impl Dictionary for FullDictionary {let chars: CharString = word.chars().collect();self.get_word_metadata(&chars)}++ fn fuzzy_match(&self, word: &[char], max_distance: u8) -> Vec<(CharString, WordMetadata)> {+ todo!()+ }++ fn fuzzy_match_str(&self, word: &str, max_distance: u8) -> Vec<(CharString, WordMetadata)> {+ todo!()+ }}#[cfg(test)]@@ -183,7 +171,7 @@ mod tests {#[test]fn curated_contains_no_duplicates() {let dict = FullDictionary::curated();- assert!(dict.words_iter().all_unique());+ assert!(dict.words.iter().all_unique());}#[test]diff --git a/harper-core/src/spell/merged_dictionary.rs b/harper-core/src/spell/merged_dictionary.rsindex 12a137d2e..a3fb5a51a 100644--- a/harper-core/src/spell/merged_dictionary.rs+++ b/harper-core/src/spell/merged_dictionary.rs@@ -75,18 +75,6 @@ wherefound_metadata}- fn words_iter(&self) -> impl Iterator<Item = &'_ [char]> {- self.children.iter().flat_map(|c| c.words_iter())- }-- fn words_with_len_iter(&self, len: usize) -> Box<dyn Iterator<Item = &'_ [char]> + '_> {- Box::new(- self.children- .iter()- .flat_map(move |c| c.words_with_len_iter(len)),- )- }-fn contains_word_str(&self, word: &str) -> bool {let chars: CharString = word.chars().collect();self.contains_word(&chars)@@ -96,4 +84,12 @@ wherelet chars: CharString = word.chars().collect();self.get_word_metadata(&chars)}++ fn fuzzy_match(&self, word: &[char], max_distance: u8) -> Vec<(CharString, WordMetadata)> {+ todo!()+ }++ fn fuzzy_match_str(&self, word: &str, max_distance: u8) -> Vec<(CharString, WordMetadata)> {+ todo!()+ }}