Skip to content

Offline, privacy-first grammar checker. Fast, open-source, Rust-powered

feat(fst-dict): edit dictionary trait to be less FullDictionary-specific

Grant Lemons55f8794234d7
diff --git a/harper-core/src/spell/dictionary.rs b/harper-core/src/spell/dictionary.rs
index 002f5a90f..71221413e 100644
--- a/harper-core/src/spell/dictionary.rs
+++ b/harper-core/src/spell/dictionary.rs
@@ -1,18 +1,18 @@
use blanket::blanket;
-use crate::WordMetadata;
+use crate::{CharString, WordMetadata};
#[cfg(not(feature = "concurrent"))]
#[blanket(derive(Rc))]
-pub trait Dictionary: Clone {
+pub trait Dictionary {
/// Check if the dictionary contains a given word.
fn contains_word(&self, word: &[char]) -> bool;
/// Check if the dictionary contains a given word.
fn contains_word_str(&self, word: &str) -> bool;
- /// Produce an iterator over all words in the dictionary.
- fn words_iter(&self) -> impl Iterator<Item = &'_ [char]>;
- /// Iterate over all the words in the dictionary of a given length
- fn words_with_len_iter(&self, len: usize) -> Box<dyn Iterator<Item = &'_ [char]> + '_>;
+ /// Gets best fuzzy match from dictionary
+ fn fuzzy_match(&self, word: &[char], max_distance: u8) -> Vec<(CharString, WordMetadata)>;
+ /// Gets best fuzzy match from dictionary
+ fn fuzzy_match_str(&self, word: &str, max_distance: u8) -> Vec<(CharString, WordMetadata)>;
/// Get the associated [`WordMetadata`] for a given word.
/// If the word isn't in the dictionary, the resulting metadata will be
/// empty.
@@ -25,15 +25,15 @@ pub trait Dictionary: Clone {
#[cfg(feature = "concurrent")]
#[blanket(derive(Arc))]
-pub trait Dictionary: Send + Sync + Clone {
+pub trait Dictionary: Send + Sync {
/// Check if the dictionary contains a given word.
fn contains_word(&self, word: &[char]) -> bool;
/// Check if the dictionary contains a given word.
fn contains_word_str(&self, word: &str) -> bool;
- /// Produce an iterator over all words in the dictionary.
- fn words_iter(&self) -> impl Iterator<Item = &'_ [char]>;
- /// Iterate over all the words in the dictionary of a given length
- fn words_with_len_iter(&self, len: usize) -> Box<dyn Iterator<Item = &'_ [char]> + '_>;
+ /// Gets best fuzzy match from dictionary
+ fn fuzzy_match(&self, word: &[char], max_distance: u8) -> Vec<(CharString, WordMetadata)>;
+ /// Gets best fuzzy match from dictionary
+ fn fuzzy_match_str(&self, word: &str, max_distance: u8) -> Vec<(CharString, WordMetadata)>;
/// Get the associated [`WordMetadata`] for a given word.
/// If the word isn't in the dictionary, the resulting metadata will be
/// empty.
diff --git a/harper-core/src/spell/fst_dictionary.rs b/harper-core/src/spell/fst_dictionary.rs
index e3c749f94..f739d9073 100644
--- a/harper-core/src/spell/fst_dictionary.rs
+++ b/harper-core/src/spell/fst_dictionary.rs
@@ -1,6 +1,7 @@
use std::fs::File;
use fst::Map as FstMap;
+use fst::{automaton::Levenshtein, IntoStreamer};
use memmap::Mmap;
use super::hunspell::{parse_default_attribute_list, parse_default_word_list};
@@ -8,16 +9,14 @@ use crate::{CharString, Lrc, WordMetadata};
use super::Dictionary;
-#[derive(Debug, Clone)]
+#[derive(Debug)]
pub struct FstDictionary {
- /// Copied from [`super::FullDictionary`], not sure if rationale there holds here yet.
- /// All English words
- words: Vec<CharString>,
+ /// Sorted by string in lexicographic order
metadata: Vec<WordMetadata>,
/// Used for fuzzy-finding the index of words or metadata
- word_map: FstMap<String>,
+ word_map: FstMap<Mmap>,
}
-///
+
/// The uncached function that is used to produce the original copy of the
/// curated dictionary.
fn uncached_inner_new() -> Lrc<FstDictionary> {
@@ -34,11 +33,7 @@ fn uncached_inner_new() -> Lrc<FstDictionary> {
words.sort();
words.dedup();
- Lrc::new(FstDictionary {
- words,
- metadata,
- word_map,
- })
+ Lrc::new(FstDictionary { metadata, word_map })
}
thread_local! {
@@ -54,10 +49,6 @@ impl Dictionary for FstDictionary {
self.word_map.contains_key(word)
}
- fn words_iter(&self) -> impl Iterator<Item = &'_ [char]> {
- self.words.iter().map(|v| v.as_slice())
- }
-
fn get_word_metadata(&self, word: &[char]) -> WordMetadata {
let index: usize = self.word_map.get(word.iter().collect::<String>()).unwrap() as usize;
self.metadata[index]
@@ -67,4 +58,32 @@ impl Dictionary for FstDictionary {
let index: usize = self.word_map.get(word).unwrap() as usize;
self.metadata[index]
}
+
+ fn fuzzy_match(&self, word: &[char], max_distance: u8) -> Vec<(CharString, WordMetadata)> {
+ let aut = Levenshtein::new(&word.iter().collect::<String>(), max_distance as u32).unwrap();
+ let words = self
+ .word_map
+ .search(aut)
+ .into_stream()
+ .into_str_vec()
+ .unwrap();
+ words
+ .into_iter()
+ .map(|(word, i)| (word.chars().collect(), self.metadata[i as usize]))
+ .collect()
+ }
+
+ fn fuzzy_match_str(&self, word: &str, max_distance: u8) -> Vec<(CharString, WordMetadata)> {
+ let aut = Levenshtein::new(word, max_distance as u32).unwrap();
+ let words = self
+ .word_map
+ .search(aut)
+ .into_stream()
+ .into_str_vec()
+ .unwrap();
+ words
+ .into_iter()
+ .map(|(word, i)| (word.chars().collect(), self.metadata[i as usize]))
+ .collect()
+ }
}
diff --git a/harper-core/src/spell/full_dictionary.rs b/harper-core/src/spell/full_dictionary.rs
index 936bba3dc..4f6a4ea60 100644
--- a/harper-core/src/spell/full_dictionary.rs
+++ b/harper-core/src/spell/full_dictionary.rs
@@ -125,26 +125,6 @@ impl Default for FullDictionary {
}
impl Dictionary for FullDictionary {
- fn words_iter(&self) -> impl Iterator<Item = &'_ [char]> {
- self.words.iter().map(|v| v.as_slice())
- }
-
- /// Iterate over all the words in the dictionary of a given length
- fn words_with_len_iter(&self, len: usize) -> Box<dyn Iterator<Item = &'_ [char]> + '_> {
- if len == 0 || len >= self.word_len_starts.len() {
- return Box::new(std::iter::empty());
- }
-
- let start = self.word_len_starts[len];
- let end = if len + 1 == self.word_len_starts.len() {
- self.words.len()
- } else {
- self.word_len_starts[len + 1]
- };
-
- Box::new(self.words[start..end].iter().map(|v| v.as_slice()))
- }
-
fn get_word_metadata(&self, word: &[char]) -> WordMetadata {
let normalized = seq_to_normalized(word);
let lowercase: CharString = normalized.iter().flat_map(|c| c.to_lowercase()).collect();
@@ -172,6 +152,14 @@ impl Dictionary for FullDictionary {
let chars: CharString = word.chars().collect();
self.get_word_metadata(&chars)
}
+
+ fn fuzzy_match(&self, word: &[char], max_distance: u8) -> Vec<(CharString, WordMetadata)> {
+ todo!()
+ }
+
+ fn fuzzy_match_str(&self, word: &str, max_distance: u8) -> Vec<(CharString, WordMetadata)> {
+ todo!()
+ }
}
#[cfg(test)]
@@ -183,7 +171,7 @@ mod tests {
#[test]
fn curated_contains_no_duplicates() {
let dict = FullDictionary::curated();
- assert!(dict.words_iter().all_unique());
+ assert!(dict.words.iter().all_unique());
}
#[test]
diff --git a/harper-core/src/spell/merged_dictionary.rs b/harper-core/src/spell/merged_dictionary.rs
index 12a137d2e..a3fb5a51a 100644
--- a/harper-core/src/spell/merged_dictionary.rs
+++ b/harper-core/src/spell/merged_dictionary.rs
@@ -75,18 +75,6 @@ where
found_metadata
}
- fn words_iter(&self) -> impl Iterator<Item = &'_ [char]> {
- self.children.iter().flat_map(|c| c.words_iter())
- }
-
- fn words_with_len_iter(&self, len: usize) -> Box<dyn Iterator<Item = &'_ [char]> + '_> {
- Box::new(
- self.children
- .iter()
- .flat_map(move |c| c.words_with_len_iter(len)),
- )
- }
-
fn contains_word_str(&self, word: &str) -> bool {
let chars: CharString = word.chars().collect();
self.contains_word(&chars)
@@ -96,4 +84,12 @@ where
let chars: CharString = word.chars().collect();
self.get_word_metadata(&chars)
}
+
+ fn fuzzy_match(&self, word: &[char], max_distance: u8) -> Vec<(CharString, WordMetadata)> {
+ todo!()
+ }
+
+ fn fuzzy_match_str(&self, word: &str, max_distance: u8) -> Vec<(CharString, WordMetadata)> {
+ todo!()
+ }
}