Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
feat(fst-dict): migrate harper-core to use FstDictionary
Grant Lemons2dca3d86b097
diff --git a/harper-core/benches/parse_demo.rs b/harper-core/benches/parse_demo.rsindex f8c2b2f8f..d76265321 100644--- a/harper-core/benches/parse_demo.rs+++ b/harper-core/benches/parse_demo.rs@@ -1,6 +1,6 @@use criterion::{black_box, criterion_group, criterion_main, Criterion};use harper_core::linting::{LintGroup, LintGroupConfig, Linter};-use harper_core::{Document, FstDictionary, FullDictionary};+use harper_core::{Document, FstDictionary};static DEMO: &str = include_str!("../../demo.md");@@ -11,7 +11,7 @@ fn parse_demo(c: &mut Criterion) {}fn lint_demo(c: &mut Criterion) {- let dictionary = FullDictionary::curated();+ let dictionary = FstDictionary::curated();let mut lint_set = LintGroup::new(Default::default(), dictionary);let document = Document::new_markdown_curated(black_box(DEMO));diff --git a/harper-core/src/document.rs b/harper-core/src/document.rsindex 9ec2b3284..94a414916 100644--- a/harper-core/src/document.rs+++ b/harper-core/src/document.rs@@ -10,7 +10,7 @@ use crate::punctuation::Punctuation;use crate::span::Span;use crate::token::NumberSuffix;use crate::vec_ext::VecExt;-use crate::{Dictionary, FatToken, FullDictionary, Lrc, Token, TokenKind, TokenStringExt};+use crate::{Dictionary, FatToken, FstDictionary, Lrc, Token, TokenKind, TokenStringExt};/// A document containing some amount of lexed and parsed English text.#[derive(Debug, Clone)]@@ -21,7 +21,7 @@ pub struct Document {impl Default for Document {fn default() -> Self {- Self::new("", &mut PlainEnglish, &FullDictionary::curated())+ Self::new("", &mut PlainEnglish, &FstDictionary::curated())}}@@ -39,7 +39,7 @@ impl Document {pub fn new_curated(text: &str, parser: &mut impl Parser) -> Self {let source: Vec<_> = text.chars().collect();- Self::new_from_vec(Lrc::new(source), parser, &FullDictionary::curated())+ Self::new_from_vec(Lrc::new(source), parser, &FstDictionary::curated())}/// Lexes and parses text to produce a document using a provided language@@ -60,7 +60,7 @@ impl Document {/// Parse text to produce a document using the built-in [`PlainEnglish`]/// parser and curated dictionary.pub fn new_plain_english_curated(text: &str) -> Self {- Self::new(text, &mut PlainEnglish, &FullDictionary::curated())+ Self::new(text, &mut PlainEnglish, &FstDictionary::curated())}/// Parse text to produce a document using the built-in [`PlainEnglish`]@@ -72,7 +72,7 @@ impl Document {/// Parse text to produce a document using the built-in [`Markdown`] parser/// and curated dictionary.pub fn new_markdown_curated(text: &str) -> Self {- Self::new(text, &mut Markdown, &FullDictionary::curated())+ Self::new(text, &mut Markdown, &FstDictionary::curated())}/// Parse text to produce a document using the built-in [`PlainEnglish`]diff --git a/harper-core/src/language_detection.rs b/harper-core/src/language_detection.rsindex ac037461e..2c8378501 100644--- a/harper-core/src/language_detection.rs+++ b/harper-core/src/language_detection.rs@@ -51,10 +51,10 @@ pub fn is_likely_english(toks: &[Token], source: &[char], dict: &impl Dictionary#[cfg(test)]mod tests {use super::is_doc_likely_english;- use crate::{Document, FullDictionary};+ use crate::{Document, FstDictionary};fn assert_not_english(source: &'static str) {- let dict = FullDictionary::curated();+ let dict = FstDictionary::curated();let doc = Document::new_plain_english(source, &dict);let is_likely_english = is_doc_likely_english(&doc, &dict);dbg!(source);@@ -62,7 +62,7 @@ mod tests {}fn assert_english(source: &'static str) {- let dict = FullDictionary::curated();+ let dict = FstDictionary::curated();let doc = Document::new_plain_english(source, &dict);let is_likely_english = is_doc_likely_english(&doc, &dict);dbg!(source);diff --git a/harper-core/src/lib.rs b/harper-core/src/lib.rsindex 2aa28baff..f7ae7e8f5 100644--- a/harper-core/src/lib.rs+++ b/harper-core/src/lib.rs@@ -66,7 +66,7 @@ pub fn remove_overlaps(lints: &mut Vec<Lint>) {mod tests {use crate::{linting::{LintGroup, LintGroupConfig, Linter},- remove_overlaps, Document, FullDictionary,+ remove_overlaps, Document, FstDictionary,};#[test]@@ -78,7 +78,7 @@ mod tests {spaces: Some(true),..LintGroupConfig::none()};- let mut linter = LintGroup::new(lint_config, FullDictionary::curated());+ let mut linter = LintGroup::new(lint_config, FstDictionary::curated());let mut lints = linter.lint(&doc);diff --git a/harper-core/src/parsers/collapse_identifiers.rs b/harper-core/src/parsers/collapse_identifiers.rsindex 3b26dc67c..18434c372 100644--- a/harper-core/src/parsers/collapse_identifiers.rs+++ b/harper-core/src/parsers/collapse_identifiers.rs@@ -5,20 +5,20 @@ use itertools::Itertools;use super::{Parser, TokenKind};use crate::patterns::{PatternExt, SequencePattern};-use crate::{Dictionary, FullDictionary, MergedDictionary, Span, Token, VecExt};+use crate::{Dictionary, Span, Token, VecExt};/// A parser that wraps any other parser to collapse token strings that match/// the pattern `word_word` or `word-word`.pub struct CollapseIdentifiers {inner: Box<dyn Parser>,- dict: Lrc<MergedDictionary<FullDictionary>>,+ dict: Lrc<dyn Dictionary>,}impl CollapseIdentifiers {- pub fn new(inner: Box<dyn Parser>, dict: &Lrc<MergedDictionary<FullDictionary>>) -> Self {+ pub fn new(inner: Box<dyn Parser>, dict: Box<Lrc<dyn Dictionary>>) -> Self {Self {inner,- dict: dict.clone(),+ dict: *dict.clone(),}}}@@ -61,7 +61,7 @@ impl Parser for CollapseIdentifiers {mod tests {use crate::{parsers::{PlainEnglish, StrParser},- WordMetadata,+ FstDictionary, FullDictionary, MergedDictionary, WordMetadata,};use super::*;@@ -81,58 +81,58 @@ mod tests {#[test]fn no_collapse() {- let dict = FullDictionary::curated();+ let dict = FstDictionary::curated();let source = "This is a test.";- let tokens = CollapseIdentifiers::new(Box::new(PlainEnglish), &Lrc::new(dict.into()))- .parse_str(source);+ let tokens =+ CollapseIdentifiers::new(Box::new(PlainEnglish), Box::new(dict)).parse_str(source);assert_eq!(tokens.len(), 8);}#[test]fn one_collapse() {let source = "This is a separated_identifier, wow!";- let default_dict = FullDictionary::curated();+ let curated_dictionary = FstDictionary::curated();- let tokens = CollapseIdentifiers::new(- Box::new(PlainEnglish),- &Lrc::new(default_dict.clone().into()),- )- .parse_str(source);+ let tokens =+ CollapseIdentifiers::new(Box::new(PlainEnglish), Box::new(curated_dictionary.clone()))+ .parse_str(source);assert_eq!(tokens.len(), 13);let mut dict = FullDictionary::new();dict.append_word_str("separated_identifier", WordMetadata::default());- let mut merged_dict = MergedDictionary::from(default_dict);+ let mut merged_dict = MergedDictionary::new();+ merged_dict.add_dictionary(curated_dictionary);merged_dict.add_dictionary(Lrc::new(dict));- let tokens = CollapseIdentifiers::new(Box::new(PlainEnglish), &Lrc::new(merged_dict))- .parse_str(source);+ let tokens =+ CollapseIdentifiers::new(Box::new(PlainEnglish), Box::new(Lrc::new(merged_dict)))+ .parse_str(source);assert_eq!(tokens.len(), 11);}#[test]fn kebab_collapse() {let source = "This is a separated-identifier, wow!";- let default_dict = FullDictionary::curated();+ let curated_dictionary = FstDictionary::curated();- let tokens = CollapseIdentifiers::new(- Box::new(PlainEnglish),- &Lrc::new(default_dict.clone().into()),- )- .parse_str(source);+ let tokens =+ CollapseIdentifiers::new(Box::new(PlainEnglish), Box::new(curated_dictionary.clone()))+ .parse_str(source);assert_eq!(tokens.len(), 13);let mut dict = FullDictionary::new();dict.append_word_str("separated-identifier", WordMetadata::default());- let mut merged_dict = MergedDictionary::from(default_dict);+ let mut merged_dict = MergedDictionary::new();+ merged_dict.add_dictionary(curated_dictionary);merged_dict.add_dictionary(Lrc::new(dict));- let tokens = CollapseIdentifiers::new(Box::new(PlainEnglish), &Lrc::new(merged_dict))- .parse_str(source);+ let tokens =+ CollapseIdentifiers::new(Box::new(PlainEnglish), Box::new(Lrc::new(merged_dict)))+ .parse_str(source);assert_eq!(tokens.len(), 11);}@@ -140,94 +140,94 @@ mod tests {#[test]fn double_collapse() {let source = "This is a separated_identifier_token, wow!";- let default_dict = FullDictionary::curated();+ let curated_dictionary = FstDictionary::curated();- let tokens = CollapseIdentifiers::new(- Box::new(PlainEnglish),- &Lrc::new(default_dict.clone().into()),- )- .parse_str(source);+ let tokens =+ CollapseIdentifiers::new(Box::new(PlainEnglish), Box::new(curated_dictionary.clone()))+ .parse_str(source);assert_eq!(tokens.len(), 15);let mut dict = FullDictionary::new();dict.append_word_str("separated_identifier_token", WordMetadata::default());- let mut merged_dict = MergedDictionary::from(default_dict);+ let mut merged_dict = MergedDictionary::new();+ merged_dict.add_dictionary(curated_dictionary);merged_dict.add_dictionary(Lrc::new(dict));- let tokens = CollapseIdentifiers::new(Box::new(PlainEnglish), &Lrc::new(merged_dict))- .parse_str(source);+ let tokens =+ CollapseIdentifiers::new(Box::new(PlainEnglish), Box::new(Lrc::new(merged_dict)))+ .parse_str(source);assert_eq!(tokens.len(), 11);}#[test]fn two_collapses() {let source = "This is a separated_identifier, wow! separated_identifier";- let default_dict = FullDictionary::curated();+ let curated_dictionary = FstDictionary::curated();- let tokens = CollapseIdentifiers::new(- Box::new(PlainEnglish),- &Lrc::new(default_dict.clone().into()),- )- .parse_str(source);+ let tokens =+ CollapseIdentifiers::new(Box::new(PlainEnglish), Box::new(curated_dictionary.clone()))+ .parse_str(source);assert_eq!(tokens.len(), 17);let mut dict = FullDictionary::new();dict.append_word_str("separated_identifier", WordMetadata::default());- let mut merged_dict = MergedDictionary::from(default_dict);+ let mut merged_dict = MergedDictionary::new();+ merged_dict.add_dictionary(curated_dictionary);merged_dict.add_dictionary(Lrc::new(dict));- let tokens = CollapseIdentifiers::new(Box::new(PlainEnglish), &Lrc::new(merged_dict))- .parse_str(source);+ let tokens =+ CollapseIdentifiers::new(Box::new(PlainEnglish), Box::new(Lrc::new(merged_dict)))+ .parse_str(source);assert_eq!(tokens.len(), 13);}#[test]fn overlapping_identifiers() {let source = "This is a separated_identifier_token, wow!";- let default_dict = FullDictionary::curated();+ let curated_dictionary = FstDictionary::curated();- let tokens = CollapseIdentifiers::new(- Box::new(PlainEnglish),- &Lrc::new(default_dict.clone().into()),- )- .parse_str(source);+ let tokens =+ CollapseIdentifiers::new(Box::new(PlainEnglish), Box::new(curated_dictionary.clone()))+ .parse_str(source);assert_eq!(tokens.len(), 15);let mut dict = FullDictionary::new();dict.append_word_str("separated_identifier", WordMetadata::default());dict.append_word_str("identifier_token", WordMetadata::default());- let mut merged_dict = MergedDictionary::from(default_dict);+ let mut merged_dict = MergedDictionary::new();+ merged_dict.add_dictionary(curated_dictionary);merged_dict.add_dictionary(Lrc::new(dict));- let tokens = CollapseIdentifiers::new(Box::new(PlainEnglish), &Lrc::new(merged_dict))- .parse_str(source);+ let tokens =+ CollapseIdentifiers::new(Box::new(PlainEnglish), Box::new(Lrc::new(merged_dict)))+ .parse_str(source);assert_eq!(tokens.len(), 15);}#[test]fn nested_identifiers() {let source = "This is a separated_identifier_token, wow!";- let default_dict = FullDictionary::curated();+ let curated_dictionary = FstDictionary::curated();- let tokens = CollapseIdentifiers::new(- Box::new(PlainEnglish),- &Lrc::new(default_dict.clone().into()),- )- .parse_str(source);+ let tokens =+ CollapseIdentifiers::new(Box::new(PlainEnglish), Box::new(curated_dictionary.clone()))+ .parse_str(source);assert_eq!(tokens.len(), 15);let mut dict = FullDictionary::new();dict.append_word_str("separated_identifier_token", WordMetadata::default());dict.append_word_str("separated_identifier", WordMetadata::default());- let mut merged_dict = MergedDictionary::from(default_dict);+ let mut merged_dict = MergedDictionary::new();+ merged_dict.add_dictionary(curated_dictionary);merged_dict.add_dictionary(Lrc::new(dict));- let tokens = CollapseIdentifiers::new(Box::new(PlainEnglish), &Lrc::new(merged_dict))- .parse_str(source);+ let tokens =+ CollapseIdentifiers::new(Box::new(PlainEnglish), Box::new(Lrc::new(merged_dict)))+ .parse_str(source);assert_eq!(tokens.len(), 11);}}diff --git a/harper-core/src/parsers/isolate_english.rs b/harper-core/src/parsers/isolate_english.rsindex 70fcc69c1..151e5fddd 100644--- a/harper-core/src/parsers/isolate_english.rs+++ b/harper-core/src/parsers/isolate_english.rs@@ -36,13 +36,13 @@ impl<D: Dictionary> Parser for IsolateEnglish<D> {#[cfg(test)]mod tests {- use crate::{parsers::PlainEnglish, Document, FullDictionary, TokenStringExt};+ use crate::{parsers::PlainEnglish, Document, FstDictionary, TokenStringExt};use super::IsolateEnglish;/// Assert that the provided text contains _no_ chunks of valid Englishfn assert_no_english(text: &str) {- let dict = FullDictionary::curated();+ let dict = FstDictionary::curated();let document = Document::new(text,@@ -57,7 +57,7 @@ mod tests {/// Assert that, once stripped of non-English chunks, the resulting document looks like another/// piece of text.fn assert_stripped_english(source: &str, target: &str) {- let dict = FullDictionary::curated();+ let dict = FstDictionary::curated();let document = Document::new(source,diff --git a/harper-core/src/spell/dictionary.rs b/harper-core/src/spell/dictionary.rsindex 44100e32e..c62cabbe0 100644--- a/harper-core/src/spell/dictionary.rs+++ b/harper-core/src/spell/dictionary.rs@@ -33,7 +33,7 @@ pub trait Dictionary {fn get_word_metadata_str(&self, word: &str) -> WordMetadata;/// Iterate over the words in the dictionary.- fn words_iter(&self) -> impl Iterator<Item = &'_ [char]>;+ fn words_iter(&self) -> Box<dyn Iterator<Item = &'_ [char]> + '_>;/// Iterate over all the words in the dictionary of a given lengthfn words_with_len_iter(&self, len: usize) -> Box<dyn Iterator<Item = &'_ [char]> + '_>;@@ -70,7 +70,7 @@ pub trait Dictionary: Send + Sync {fn get_word_metadata_str(&self, word: &str) -> WordMetadata;/// Iterate over the words in the dictionary.- fn words_iter(&self) -> impl Iterator<Item = &'_ [char]>;+ fn words_iter(&self) -> Box<dyn Iterator<Item = &'_ [char]> + '_>;/// Iterate over all the words in the dictionary of a given lengthfn words_with_len_iter(&self, len: usize) -> Box<dyn Iterator<Item = &'_ [char]> + '_>;diff --git a/harper-core/src/spell/fst_dictionary.rs b/harper-core/src/spell/fst_dictionary.rsindex 020ee993c..a5ea79f58 100644--- a/harper-core/src/spell/fst_dictionary.rs+++ b/harper-core/src/spell/fst_dictionary.rs@@ -1,4 +1,4 @@-use super::FullDictionary;+use super::{seq_to_normalized, FullDictionary};use fst::{automaton::Levenshtein, IntoStreamer};use fst::{Map as FstMap, Streamer};@@ -30,6 +30,12 @@ thread_local! {static DICT: Lrc<FstDictionary> = uncached_inner_new();}+impl PartialEq for FstDictionary {+ fn eq(&self, other: &Self) -> bool {+ self.full_dict == other.full_dict+ }+}+impl FstDictionary {/// Create a dictionary from the curated dictionary included/// in the Harper binary.@@ -70,7 +76,14 @@ impl Dictionary for FstDictionary {max_distance: u8,max_results: usize,) -> Vec<(&[char], u8, WordMetadata)> {- let aut = Levenshtein::new(word, max_distance as u32).unwrap();+ let chars: Vec<_> = word.chars().collect();+ let misspelled_word = seq_to_normalized(&chars);+ let misspelled_lower: String = misspelled_word+ .iter()+ .flat_map(|v| v.to_lowercase())+ .collect();++ let aut = Levenshtein::new(&misspelled_lower, max_distance as u32).unwrap();let mut word_indexes_stream = self.word_map.search(aut).into_stream();let mut word_indexes = Vec::with_capacity(max_results);@@ -97,7 +110,7 @@ impl Dictionary for FstDictionary {.collect()}- fn words_iter(&self) -> impl Iterator<Item = &'_ [char]> {+ fn words_iter(&self) -> Box<dyn Iterator<Item = &'_ [char]> + '_> {self.full_dict.words_iter()}@@ -105,3 +118,26 @@ impl Dictionary for FstDictionary {self.full_dict.words_with_len_iter(len)}}++#[cfg(test)]+mod tests {+ use crate::{spell::seq_to_normalized, Dictionary};++ use super::FstDictionary;++ #[test]+ fn fst_contains() {+ let dict = FstDictionary::curated();++ for word in dict.words_iter().step_by(5).take(20) {+ let misspelled_word = seq_to_normalized(word);+ let misspelled_lower: String = misspelled_word+ .iter()+ .flat_map(|v| v.to_lowercase())+ .collect();++ assert!(dict.contains_word(word));+ assert!(dict.word_map.contains_key(misspelled_lower));+ }+ }+}diff --git a/harper-core/src/spell/full_dictionary.rs b/harper-core/src/spell/full_dictionary.rsindex 8cd5a85ad..aee09314c 100644--- a/harper-core/src/spell/full_dictionary.rs+++ b/harper-core/src/spell/full_dictionary.rs@@ -233,8 +233,8 @@ impl Dictionary for FullDictionary {self.fuzzy_match(&word, max_distance, max_results)}- fn words_iter(&self) -> impl Iterator<Item = &'_ [char]> {- self.words.iter().map(|v| v.as_slice())+ fn words_iter(&self) -> Box<dyn Iterator<Item = &'_ [char]> + '_> {+ Box::new(self.words.iter().map(|v| v.as_slice()))}fn words_with_len_iter(&self, len: usize) -> Box<dyn Iterator<Item = &'_ [char]> + '_> {diff --git a/harper-core/src/spell/merged_dictionary.rs b/harper-core/src/spell/merged_dictionary.rsindex 3933e9167..648fed17d 100644--- a/harper-core/src/spell/merged_dictionary.rs+++ b/harper-core/src/spell/merged_dictionary.rs@@ -8,19 +8,13 @@ use crate::{CharString, WordMetadata};/// A simple wrapper over [`Dictionary`] that allows/// one to merge multiple dictionaries without copying.-#[derive(Clone, PartialEq)]-pub struct MergedDictionary<T>-where- T: Dictionary + Clone,-{- children: Vec<Lrc<T>>,+#[derive(Clone)]+pub struct MergedDictionary {+ children: Vec<Lrc<dyn Dictionary>>,merged: HashMap<CharString, WordMetadata>,}-impl<T> MergedDictionary<T>-where- T: Dictionary + Clone,-{+impl MergedDictionary {pub fn new() -> Self {Self {children: Vec::new(),@@ -28,36 +22,24 @@ where}}- pub fn add_dictionary(&mut self, dictionary: Lrc<T>) {+ pub fn add_dictionary(&mut self, dictionary: Lrc<dyn Dictionary>) {self.children.push(dictionary.clone());}}-impl<T> From<Lrc<T>> for MergedDictionary<T>-where- T: Dictionary + Clone,-{- fn from(value: Lrc<T>) -> Self {- Self {- children: vec![value],- ..Default::default()- }+impl PartialEq for MergedDictionary {+ fn eq(&self, other: &Self) -> bool {+ self.merged == other.merged}}-impl<T> Default for MergedDictionary<T>-where- T: Dictionary + Clone,-{+impl Default for MergedDictionary {fn default() -> Self {Self::new()}}-impl<T> Dictionary for MergedDictionary<T>-where- T: Dictionary + Clone,-{+impl Dictionary for MergedDictionary {fn contains_word(&self, word: &[char]) -> bool {for child in &self.children {if child.contains_word(word) {@@ -76,8 +58,8 @@ wherefound_metadata}- fn words_iter(&self) -> impl Iterator<Item = &'_ [char]> {- self.children.iter().flat_map(|c| c.words_iter())+ fn words_iter(&self) -> Box<dyn Iterator<Item = &'_ [char]> + '_> {+ Box::new(self.children.iter().flat_map(|c| c.words_iter()))}fn words_with_len_iter(&self, len: usize) -> Box<dyn Iterator<Item = &'_ [char]> + '_> {diff --git a/harper-core/src/spell/mod.rs b/harper-core/src/spell/mod.rsindex 38b3ecdbb..4516fed37 100644--- a/harper-core/src/spell/mod.rs+++ b/harper-core/src/spell/mod.rs@@ -77,7 +77,7 @@ pub fn suggest_correct_spelling_str(misspelled_word: impl Into<String>,result_limit: usize,max_edit_dist: u8,- dictionary: &FullDictionary,+ dictionary: &impl Dictionary,) -> Vec<String> {let chars: CharString = misspelled_word.into().chars().collect();suggest_correct_spelling(&chars, result_limit, max_edit_dist, dictionary)@@ -107,12 +107,11 @@ pub fn char_to_normalized(c: char) -> char {mod tests {use itertools::Itertools;- use crate::{spell::suggest_correct_spelling_str, FullDictionary};+ use super::{suggest_correct_spelling_str, FstDictionary};#[test]fn produces_no_duplicates() {- let results =- suggest_correct_spelling_str("punctation", 100, 3, &FullDictionary::curated());+ let results = suggest_correct_spelling_str("punctation", 100, 3, &FstDictionary::curated());dbg!(&results, results.iter().unique().collect_vec());@@ -121,7 +120,7 @@ mod tests {#[test]fn issue_182() {- let results = suggest_correct_spelling_str("im", 100, 3, &FullDictionary::curated());+ let results = suggest_correct_spelling_str("im", 100, 3, &FstDictionary::curated());dbg!(&results);