Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
fix(fst-dict): change all dicts to use Arc even when not concurrent to make MergedDictionary work
Grant Lemonsd4ea782d70a3
diff --git a/harper-core/src/parsers/collapse_identifiers.rs b/harper-core/src/parsers/collapse_identifiers.rsindex 18434c372..59fd81a1c 100644--- a/harper-core/src/parsers/collapse_identifiers.rs+++ b/harper-core/src/parsers/collapse_identifiers.rs@@ -1,21 +1,21 @@-use crate::Lrc;use std::collections::VecDeque;+use std::sync::Arc;use itertools::Itertools;use super::{Parser, TokenKind};use crate::patterns::{PatternExt, SequencePattern};-use crate::{Dictionary, Span, Token, VecExt};+use crate::{Dictionary, Lrc, Span, Token, VecExt};/// A parser that wraps any other parser to collapse token strings that match/// the pattern `word_word` or `word-word`.pub struct CollapseIdentifiers {inner: Box<dyn Parser>,- dict: Lrc<dyn Dictionary>,+ dict: Arc<dyn Dictionary>,}impl CollapseIdentifiers {- pub fn new(inner: Box<dyn Parser>, dict: Box<Lrc<dyn Dictionary>>) -> Self {+ pub fn new(inner: Box<dyn Parser>, dict: Box<Arc<dyn Dictionary>>) -> Self {Self {inner,dict: *dict.clone(),@@ -104,10 +104,10 @@ mod tests {let mut merged_dict = MergedDictionary::new();merged_dict.add_dictionary(curated_dictionary);- merged_dict.add_dictionary(Lrc::new(dict));+ merged_dict.add_dictionary(Arc::new(dict));let tokens =- CollapseIdentifiers::new(Box::new(PlainEnglish), Box::new(Lrc::new(merged_dict)))+ CollapseIdentifiers::new(Box::new(PlainEnglish), Box::new(Arc::new(merged_dict))).parse_str(source);assert_eq!(tokens.len(), 11);}@@ -128,10 +128,10 @@ mod tests {let mut merged_dict = MergedDictionary::new();merged_dict.add_dictionary(curated_dictionary);- merged_dict.add_dictionary(Lrc::new(dict));+ merged_dict.add_dictionary(Arc::new(dict));let tokens =- CollapseIdentifiers::new(Box::new(PlainEnglish), Box::new(Lrc::new(merged_dict)))+ CollapseIdentifiers::new(Box::new(PlainEnglish), Box::new(Arc::new(merged_dict))).parse_str(source);assert_eq!(tokens.len(), 11);@@ -152,10 +152,10 @@ mod tests {let mut merged_dict = MergedDictionary::new();merged_dict.add_dictionary(curated_dictionary);- merged_dict.add_dictionary(Lrc::new(dict));+ merged_dict.add_dictionary(Arc::new(dict));let tokens =- CollapseIdentifiers::new(Box::new(PlainEnglish), Box::new(Lrc::new(merged_dict)))+ CollapseIdentifiers::new(Box::new(PlainEnglish), Box::new(Arc::new(merged_dict))).parse_str(source);assert_eq!(tokens.len(), 11);}@@ -175,10 +175,10 @@ mod tests {let mut merged_dict = MergedDictionary::new();merged_dict.add_dictionary(curated_dictionary);- merged_dict.add_dictionary(Lrc::new(dict));+ merged_dict.add_dictionary(Arc::new(dict));let tokens =- CollapseIdentifiers::new(Box::new(PlainEnglish), Box::new(Lrc::new(merged_dict)))+ CollapseIdentifiers::new(Box::new(PlainEnglish), Box::new(Arc::new(merged_dict))).parse_str(source);assert_eq!(tokens.len(), 13);}@@ -199,10 +199,10 @@ mod tests {let mut merged_dict = MergedDictionary::new();merged_dict.add_dictionary(curated_dictionary);- merged_dict.add_dictionary(Lrc::new(dict));+ merged_dict.add_dictionary(Arc::new(dict));let tokens =- CollapseIdentifiers::new(Box::new(PlainEnglish), Box::new(Lrc::new(merged_dict)))+ CollapseIdentifiers::new(Box::new(PlainEnglish), Box::new(Arc::new(merged_dict))).parse_str(source);assert_eq!(tokens.len(), 15);}@@ -223,10 +223,10 @@ mod tests {let mut merged_dict = MergedDictionary::new();merged_dict.add_dictionary(curated_dictionary);- merged_dict.add_dictionary(Lrc::new(dict));+ merged_dict.add_dictionary(Arc::new(dict));let tokens =- CollapseIdentifiers::new(Box::new(PlainEnglish), Box::new(Lrc::new(merged_dict)))+ CollapseIdentifiers::new(Box::new(PlainEnglish), Box::new(Arc::new(merged_dict))).parse_str(source);assert_eq!(tokens.len(), 11);}diff --git a/harper-core/src/spell/dictionary.rs b/harper-core/src/spell/dictionary.rsindex 4d98ac13f..79cacf4e5 100644--- a/harper-core/src/spell/dictionary.rs+++ b/harper-core/src/spell/dictionary.rs@@ -2,44 +2,6 @@ use blanket::blanket;use crate::WordMetadata;-#[cfg(not(feature = "concurrent"))]-#[blanket(derive(Rc))]-pub trait Dictionary {- /// Check if the dictionary contains a given word.- fn contains_word(&self, word: &[char]) -> bool;- /// Check if the dictionary contains a given word.- fn contains_word_str(&self, word: &str) -> bool;- /// Gets best fuzzy match from dictionary- fn fuzzy_match(- &self,- word: &[char],- max_distance: u8,- max_results: usize,- ) -> Vec<(&[char], u8, WordMetadata)>;- /// Gets best fuzzy match from dictionary- fn fuzzy_match_str(- &self,- word: &str,- max_distance: u8,- max_results: usize,- ) -> Vec<(&[char], u8, WordMetadata)>;- /// Get the associated [`WordMetadata`] for a given word.- /// If the word isn't in the dictionary, the resulting metadata will be- /// empty.- fn get_word_metadata(&self, word: &[char]) -> WordMetadata;- /// Get the associated [`WordMetadata`] for a given word.- /// If the word isn't in the dictionary, the resulting metadata will be- /// empty.- fn get_word_metadata_str(&self, word: &str) -> WordMetadata;-- /// Iterate over the words in the dictionary.- fn words_iter(&self) -> Box<dyn Iterator<Item = &'_ [char]> + '_>;-- /// Iterate over all the words in the dictionary of a given length- fn words_with_len_iter(&self, len: usize) -> Box<dyn Iterator<Item = &'_ [char]> + '_>;-}--#[cfg(feature = "concurrent")]#[blanket(derive(Arc))]pub trait Dictionary: Send + Sync {/// Check if the dictionary contains a given word.diff --git a/harper-core/src/spell/fst_dictionary.rs b/harper-core/src/spell/fst_dictionary.rsindex ea1c539d6..4b4373315 100644--- a/harper-core/src/spell/fst_dictionary.rs+++ b/harper-core/src/spell/fst_dictionary.rs@@ -2,33 +2,34 @@ use super::{edit_distance_min_alloc, seq_to_normalized, FullDictionary};use fst::Map as FstMap;use fst::{automaton::Levenshtein, IntoStreamer};use itertools::Itertools;+use std::sync::Arc;-use crate::{CharStringExt, Lrc, WordMetadata};+use crate::{CharStringExt, WordMetadata};use super::Dictionary;#[derive(Debug)]pub struct FstDictionary {/// Underlying FullDictionary used for everything except fuzzy finding- full_dict: Lrc<FullDictionary>,+ full_dict: Arc<FullDictionary>,/// Used for fuzzy-finding the index of words or metadataword_map: FstMap<Vec<u8>>,}/// The uncached function that is used to produce the original copy of the/// curated dictionary.-fn uncached_inner_new() -> Lrc<FstDictionary> {+fn uncached_inner_new() -> Arc<FstDictionary> {let full_dict = FullDictionary::curated();let word_map = FstMap::new(include_bytes!("../../dictionary.fst").to_vec()).unwrap();- Lrc::new(FstDictionary {+ Arc::new(FstDictionary {full_dict,word_map,})}thread_local! {- static DICT: Lrc<FstDictionary> = uncached_inner_new();+ static DICT: Arc<FstDictionary> = uncached_inner_new();}impl PartialEq for FstDictionary {@@ -40,7 +41,7 @@ impl PartialEq for FstDictionary {impl FstDictionary {/// Create a dictionary from the curated dictionary included/// in the Harper binary.- pub fn curated() -> Lrc<Self> {+ pub fn curated() -> Arc<Self> {DICT.with(|v| v.clone())}}diff --git a/harper-core/src/spell/full_dictionary.rs b/harper-core/src/spell/full_dictionary.rsindex 6a9bbbbec..d2c56d100 100644--- a/harper-core/src/spell/full_dictionary.rs+++ b/harper-core/src/spell/full_dictionary.rs@@ -2,9 +2,10 @@ use super::{edit_distance_min_alloc, seq_to_normalized};use hashbrown::HashMap;use itertools::Itertools;use smallvec::{SmallVec, ToSmallVec};+use std::sync::Arc;use super::dictionary::Dictionary;-use crate::{CharString, CharStringExt, Lrc, WordMetadata};+use crate::{CharString, CharStringExt, WordMetadata};use harper_dictionary_parsing::{parse_default_attribute_list, parse_default_word_list};/// A full, fat dictionary.@@ -28,7 +29,7 @@ pub struct FullDictionary {/// The uncached function that is used to produce the original copy of the/// curated dictionary.-fn uncached_inner_new() -> Lrc<FullDictionary> {+fn uncached_inner_new() -> Arc<FullDictionary> {let word_list = parse_default_word_list().unwrap();let attr_list = parse_default_attribute_list();@@ -41,7 +42,7 @@ fn uncached_inner_new() -> Lrc<FullDictionary> {words.sort();words.dedup();- Lrc::new(FullDictionary {+ Arc::new(FullDictionary {word_map,word_len_starts: FullDictionary::create_len_starts(&words),words,@@ -49,7 +50,7 @@ fn uncached_inner_new() -> Lrc<FullDictionary> {}thread_local! {- static DICT: Lrc<FullDictionary> = uncached_inner_new();+ static DICT: Arc<FullDictionary> = uncached_inner_new();}impl FullDictionary {@@ -63,7 +64,7 @@ impl FullDictionary {/// Create a dictionary from the curated dictionary included/// in the Harper binary.- pub fn curated() -> Lrc<Self> {+ pub fn curated() -> Arc<Self> {DICT.with(|v| v.clone())}diff --git a/harper-core/src/spell/merged_dictionary.rs b/harper-core/src/spell/merged_dictionary.rsindex 38524dc9d..c26fa0cb8 100644--- a/harper-core/src/spell/merged_dictionary.rs+++ b/harper-core/src/spell/merged_dictionary.rs@@ -1,15 +1,15 @@-use crate::Lrc;-+use harper_dictionary_parsing::CharString;use itertools::Itertools;+use std::sync::Arc;use super::dictionary::Dictionary;-use crate::{CharString, WordMetadata};+use crate::WordMetadata;/// A simple wrapper over [`Dictionary`] that allows/// one to merge multiple dictionaries without copying.#[derive(Clone)]pub struct MergedDictionary {- children: Vec<Lrc<dyn Dictionary>>,+ children: Vec<Arc<dyn Dictionary>>,}impl MergedDictionary {@@ -19,8 +19,8 @@ impl MergedDictionary {}}- pub fn add_dictionary(&mut self, dictionary: Lrc<dyn Dictionary>) {- self.children.push(dictionary.clone());+ pub fn add_dictionary(&mut self, dictionary: Arc<dyn Dictionary>) {+ self.children.push(dictionary);}}diff --git a/harper-core/src/spell/mod.rs b/harper-core/src/spell/mod.rsindex 492a51a6f..de20fd2e4 100644--- a/harper-core/src/spell/mod.rs+++ b/harper-core/src/spell/mod.rs@@ -230,7 +230,7 @@ mod tests {}#[test]- fn full_dict_fuzzy_find() {+ fn full_dict_fuzzy_find_anout() {let results = suggest_correct_spelling_str("anout", 100, 3, &FullDictionary::curated());dbg!(&results);@@ -238,6 +238,24 @@ mod tests {assert!(results.iter().take(3).contains(&"about".to_string()));}+ #[test]+ fn full_dict_fuzzy_find_ello() {+ let results = suggest_correct_spelling_str("ello", 100, 3, &FullDictionary::curated());++ dbg!(&results);++ assert!(results.iter().take(3).contains(&"hello".to_string()));+ }++ #[test]+ fn full_dict_fuzzy_find_hello_capital() {+ let results = suggest_correct_spelling_str("Hello", 100, 3, &FullDictionary::curated());++ dbg!(&results);++ assert!(results.iter().take(3).contains(&"hello".to_string()));+ }+#[test]fn issue_182() {let results = suggest_correct_spelling_str("Im", 100, 3, &FstDictionary::curated());