Skip to content

Offline, privacy-first grammar checker. Fast, open-source, Rust-powered

fix(fst-dict): change all dicts to use Arc even when not concurrent to make MergedDictionary work

Grant Lemonsd4ea782d70a3
diff --git a/harper-core/src/parsers/collapse_identifiers.rs b/harper-core/src/parsers/collapse_identifiers.rs
index 18434c372..59fd81a1c 100644
--- a/harper-core/src/parsers/collapse_identifiers.rs
+++ b/harper-core/src/parsers/collapse_identifiers.rs
@@ -1,21 +1,21 @@
-use crate::Lrc;
use std::collections::VecDeque;
+use std::sync::Arc;
use itertools::Itertools;
use super::{Parser, TokenKind};
use crate::patterns::{PatternExt, SequencePattern};
-use crate::{Dictionary, Span, Token, VecExt};
+use crate::{Dictionary, Lrc, Span, Token, VecExt};
/// A parser that wraps any other parser to collapse token strings that match
/// the pattern `word_word` or `word-word`.
pub struct CollapseIdentifiers {
inner: Box<dyn Parser>,
- dict: Lrc<dyn Dictionary>,
+ dict: Arc<dyn Dictionary>,
}
impl CollapseIdentifiers {
- pub fn new(inner: Box<dyn Parser>, dict: Box<Lrc<dyn Dictionary>>) -> Self {
+ pub fn new(inner: Box<dyn Parser>, dict: Box<Arc<dyn Dictionary>>) -> Self {
Self {
inner,
dict: *dict.clone(),
@@ -104,10 +104,10 @@ mod tests {
let mut merged_dict = MergedDictionary::new();
merged_dict.add_dictionary(curated_dictionary);
- merged_dict.add_dictionary(Lrc::new(dict));
+ merged_dict.add_dictionary(Arc::new(dict));
let tokens =
- CollapseIdentifiers::new(Box::new(PlainEnglish), Box::new(Lrc::new(merged_dict)))
+ CollapseIdentifiers::new(Box::new(PlainEnglish), Box::new(Arc::new(merged_dict)))
.parse_str(source);
assert_eq!(tokens.len(), 11);
}
@@ -128,10 +128,10 @@ mod tests {
let mut merged_dict = MergedDictionary::new();
merged_dict.add_dictionary(curated_dictionary);
- merged_dict.add_dictionary(Lrc::new(dict));
+ merged_dict.add_dictionary(Arc::new(dict));
let tokens =
- CollapseIdentifiers::new(Box::new(PlainEnglish), Box::new(Lrc::new(merged_dict)))
+ CollapseIdentifiers::new(Box::new(PlainEnglish), Box::new(Arc::new(merged_dict)))
.parse_str(source);
assert_eq!(tokens.len(), 11);
@@ -152,10 +152,10 @@ mod tests {
let mut merged_dict = MergedDictionary::new();
merged_dict.add_dictionary(curated_dictionary);
- merged_dict.add_dictionary(Lrc::new(dict));
+ merged_dict.add_dictionary(Arc::new(dict));
let tokens =
- CollapseIdentifiers::new(Box::new(PlainEnglish), Box::new(Lrc::new(merged_dict)))
+ CollapseIdentifiers::new(Box::new(PlainEnglish), Box::new(Arc::new(merged_dict)))
.parse_str(source);
assert_eq!(tokens.len(), 11);
}
@@ -175,10 +175,10 @@ mod tests {
let mut merged_dict = MergedDictionary::new();
merged_dict.add_dictionary(curated_dictionary);
- merged_dict.add_dictionary(Lrc::new(dict));
+ merged_dict.add_dictionary(Arc::new(dict));
let tokens =
- CollapseIdentifiers::new(Box::new(PlainEnglish), Box::new(Lrc::new(merged_dict)))
+ CollapseIdentifiers::new(Box::new(PlainEnglish), Box::new(Arc::new(merged_dict)))
.parse_str(source);
assert_eq!(tokens.len(), 13);
}
@@ -199,10 +199,10 @@ mod tests {
let mut merged_dict = MergedDictionary::new();
merged_dict.add_dictionary(curated_dictionary);
- merged_dict.add_dictionary(Lrc::new(dict));
+ merged_dict.add_dictionary(Arc::new(dict));
let tokens =
- CollapseIdentifiers::new(Box::new(PlainEnglish), Box::new(Lrc::new(merged_dict)))
+ CollapseIdentifiers::new(Box::new(PlainEnglish), Box::new(Arc::new(merged_dict)))
.parse_str(source);
assert_eq!(tokens.len(), 15);
}
@@ -223,10 +223,10 @@ mod tests {
let mut merged_dict = MergedDictionary::new();
merged_dict.add_dictionary(curated_dictionary);
- merged_dict.add_dictionary(Lrc::new(dict));
+ merged_dict.add_dictionary(Arc::new(dict));
let tokens =
- CollapseIdentifiers::new(Box::new(PlainEnglish), Box::new(Lrc::new(merged_dict)))
+ CollapseIdentifiers::new(Box::new(PlainEnglish), Box::new(Arc::new(merged_dict)))
.parse_str(source);
assert_eq!(tokens.len(), 11);
}
diff --git a/harper-core/src/spell/dictionary.rs b/harper-core/src/spell/dictionary.rs
index 4d98ac13f..79cacf4e5 100644
--- a/harper-core/src/spell/dictionary.rs
+++ b/harper-core/src/spell/dictionary.rs
@@ -2,44 +2,6 @@ use blanket::blanket;
use crate::WordMetadata;
-#[cfg(not(feature = "concurrent"))]
-#[blanket(derive(Rc))]
-pub trait Dictionary {
- /// Check if the dictionary contains a given word.
- fn contains_word(&self, word: &[char]) -> bool;
- /// Check if the dictionary contains a given word.
- fn contains_word_str(&self, word: &str) -> bool;
- /// Gets best fuzzy match from dictionary
- fn fuzzy_match(
- &self,
- word: &[char],
- max_distance: u8,
- max_results: usize,
- ) -> Vec<(&[char], u8, WordMetadata)>;
- /// Gets best fuzzy match from dictionary
- fn fuzzy_match_str(
- &self,
- word: &str,
- max_distance: u8,
- max_results: usize,
- ) -> Vec<(&[char], u8, WordMetadata)>;
- /// Get the associated [`WordMetadata`] for a given word.
- /// If the word isn't in the dictionary, the resulting metadata will be
- /// empty.
- fn get_word_metadata(&self, word: &[char]) -> WordMetadata;
- /// Get the associated [`WordMetadata`] for a given word.
- /// If the word isn't in the dictionary, the resulting metadata will be
- /// empty.
- fn get_word_metadata_str(&self, word: &str) -> WordMetadata;
-
- /// Iterate over the words in the dictionary.
- fn words_iter(&self) -> Box<dyn Iterator<Item = &'_ [char]> + '_>;
-
- /// Iterate over all the words in the dictionary of a given length
- fn words_with_len_iter(&self, len: usize) -> Box<dyn Iterator<Item = &'_ [char]> + '_>;
-}
-
-#[cfg(feature = "concurrent")]
#[blanket(derive(Arc))]
pub trait Dictionary: Send + Sync {
/// Check if the dictionary contains a given word.
diff --git a/harper-core/src/spell/fst_dictionary.rs b/harper-core/src/spell/fst_dictionary.rs
index ea1c539d6..4b4373315 100644
--- a/harper-core/src/spell/fst_dictionary.rs
+++ b/harper-core/src/spell/fst_dictionary.rs
@@ -2,33 +2,34 @@ use super::{edit_distance_min_alloc, seq_to_normalized, FullDictionary};
use fst::Map as FstMap;
use fst::{automaton::Levenshtein, IntoStreamer};
use itertools::Itertools;
+use std::sync::Arc;
-use crate::{CharStringExt, Lrc, WordMetadata};
+use crate::{CharStringExt, WordMetadata};
use super::Dictionary;
#[derive(Debug)]
pub struct FstDictionary {
/// Underlying FullDictionary used for everything except fuzzy finding
- full_dict: Lrc<FullDictionary>,
+ full_dict: Arc<FullDictionary>,
/// Used for fuzzy-finding the index of words or metadata
word_map: FstMap<Vec<u8>>,
}
/// The uncached function that is used to produce the original copy of the
/// curated dictionary.
-fn uncached_inner_new() -> Lrc<FstDictionary> {
+fn uncached_inner_new() -> Arc<FstDictionary> {
let full_dict = FullDictionary::curated();
let word_map = FstMap::new(include_bytes!("../../dictionary.fst").to_vec()).unwrap();
- Lrc::new(FstDictionary {
+ Arc::new(FstDictionary {
full_dict,
word_map,
})
}
thread_local! {
- static DICT: Lrc<FstDictionary> = uncached_inner_new();
+ static DICT: Arc<FstDictionary> = uncached_inner_new();
}
impl PartialEq for FstDictionary {
@@ -40,7 +41,7 @@ impl PartialEq for FstDictionary {
impl FstDictionary {
/// Create a dictionary from the curated dictionary included
/// in the Harper binary.
- pub fn curated() -> Lrc<Self> {
+ pub fn curated() -> Arc<Self> {
DICT.with(|v| v.clone())
}
}
diff --git a/harper-core/src/spell/full_dictionary.rs b/harper-core/src/spell/full_dictionary.rs
index 6a9bbbbec..d2c56d100 100644
--- a/harper-core/src/spell/full_dictionary.rs
+++ b/harper-core/src/spell/full_dictionary.rs
@@ -2,9 +2,10 @@ use super::{edit_distance_min_alloc, seq_to_normalized};
use hashbrown::HashMap;
use itertools::Itertools;
use smallvec::{SmallVec, ToSmallVec};
+use std::sync::Arc;
use super::dictionary::Dictionary;
-use crate::{CharString, CharStringExt, Lrc, WordMetadata};
+use crate::{CharString, CharStringExt, WordMetadata};
use harper_dictionary_parsing::{parse_default_attribute_list, parse_default_word_list};
/// A full, fat dictionary.
@@ -28,7 +29,7 @@ pub struct FullDictionary {
/// The uncached function that is used to produce the original copy of the
/// curated dictionary.
-fn uncached_inner_new() -> Lrc<FullDictionary> {
+fn uncached_inner_new() -> Arc<FullDictionary> {
let word_list = parse_default_word_list().unwrap();
let attr_list = parse_default_attribute_list();
@@ -41,7 +42,7 @@ fn uncached_inner_new() -> Lrc<FullDictionary> {
words.sort();
words.dedup();
- Lrc::new(FullDictionary {
+ Arc::new(FullDictionary {
word_map,
word_len_starts: FullDictionary::create_len_starts(&words),
words,
@@ -49,7 +50,7 @@ fn uncached_inner_new() -> Lrc<FullDictionary> {
}
thread_local! {
- static DICT: Lrc<FullDictionary> = uncached_inner_new();
+ static DICT: Arc<FullDictionary> = uncached_inner_new();
}
impl FullDictionary {
@@ -63,7 +64,7 @@ impl FullDictionary {
/// Create a dictionary from the curated dictionary included
/// in the Harper binary.
- pub fn curated() -> Lrc<Self> {
+ pub fn curated() -> Arc<Self> {
DICT.with(|v| v.clone())
}
diff --git a/harper-core/src/spell/merged_dictionary.rs b/harper-core/src/spell/merged_dictionary.rs
index 38524dc9d..c26fa0cb8 100644
--- a/harper-core/src/spell/merged_dictionary.rs
+++ b/harper-core/src/spell/merged_dictionary.rs
@@ -1,15 +1,15 @@
-use crate::Lrc;
-
+use harper_dictionary_parsing::CharString;
use itertools::Itertools;
+use std::sync::Arc;
use super::dictionary::Dictionary;
-use crate::{CharString, WordMetadata};
+use crate::WordMetadata;
/// A simple wrapper over [`Dictionary`] that allows
/// one to merge multiple dictionaries without copying.
#[derive(Clone)]
pub struct MergedDictionary {
- children: Vec<Lrc<dyn Dictionary>>,
+ children: Vec<Arc<dyn Dictionary>>,
}
impl MergedDictionary {
@@ -19,8 +19,8 @@ impl MergedDictionary {
}
}
- pub fn add_dictionary(&mut self, dictionary: Lrc<dyn Dictionary>) {
- self.children.push(dictionary.clone());
+ pub fn add_dictionary(&mut self, dictionary: Arc<dyn Dictionary>) {
+ self.children.push(dictionary);
}
}
diff --git a/harper-core/src/spell/mod.rs b/harper-core/src/spell/mod.rs
index 492a51a6f..de20fd2e4 100644
--- a/harper-core/src/spell/mod.rs
+++ b/harper-core/src/spell/mod.rs
@@ -230,7 +230,7 @@ mod tests {
}
#[test]
- fn full_dict_fuzzy_find() {
+ fn full_dict_fuzzy_find_anout() {
let results = suggest_correct_spelling_str("anout", 100, 3, &FullDictionary::curated());
dbg!(&results);
@@ -238,6 +238,24 @@ mod tests {
assert!(results.iter().take(3).contains(&"about".to_string()));
}
+ #[test]
+ fn full_dict_fuzzy_find_ello() {
+ let results = suggest_correct_spelling_str("ello", 100, 3, &FullDictionary::curated());
+
+ dbg!(&results);
+
+ assert!(results.iter().take(3).contains(&"hello".to_string()));
+ }
+
+ #[test]
+ fn full_dict_fuzzy_find_hello_capital() {
+ let results = suggest_correct_spelling_str("Hello", 100, 3, &FullDictionary::curated());
+
+ dbg!(&results);
+
+ assert!(results.iter().take(3).contains(&"hello".to_string()));
+ }
+
#[test]
fn issue_182() {
let results = suggest_correct_spelling_str("Im", 100, 3, &FstDictionary::curated());