Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
feat: added API to allow merging generic dictionaries
Elijah Potterfde38b4035fa
diff --git a/Cargo.lock b/Cargo.lockindex ccf9383f3..cf5e8c765 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -203,6 +203,17 @@ version = "2.4.2"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "ed570934406eb16438a4e976b1b4500774099c13b8cb96eec99f620f05090ddf"+[[package]]+name = "blanket"+version = "0.3.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e0b121a9fe0df916e362fb3271088d071159cdf11db0e4182d02152850756eff"+dependencies = [+ "proc-macro2",+ "quote",+ "syn 2.0.48",+]+[[package]]name = "bumpalo"version = "3.14.0"@@ -458,6 +469,7 @@ checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253"name = "harper-core"version = "0.5.8"dependencies = [+ "blanket","divan","hashbrown","is-macro",diff --git a/harper-core/Cargo.toml b/harper-core/Cargo.tomlindex 537368e6e..17f3192c3 100644--- a/harper-core/Cargo.toml+++ b/harper-core/Cargo.toml@@ -8,6 +8,7 @@ readme = "README.md"homepage = "https://github.com/chilipepperhott/harper"[dependencies]+blanket = "0.3.0"hashbrown = { version = "0.14.3", features = ["serde"] }is-macro = "0.3.0"itertools = "0.11.0"diff --git a/harper-core/benches/parse_demo.rs b/harper-core/benches/parse_demo.rsindex 3f28cbc29..f8da88dae 100644--- a/harper-core/benches/parse_demo.rs+++ b/harper-core/benches/parse_demo.rs@@ -1,5 +1,5 @@use divan::{black_box, AllocProfiler, Bencher};-use harper_core::{Dictionary, Document, LintSet, Linter};+use harper_core::{Document, FullDictionary, LintSet, Linter};#[global_allocator]static ALLOC: AllocProfiler = AllocProfiler::system();@@ -15,7 +15,7 @@ fn parse_demo(bencher: Bencher) {#[divan::bench]fn create_lint_set(bencher: Bencher) {- let dictionary = Dictionary::create_from_curated();+ let dictionary = FullDictionary::create_from_curated();bencher.bench_local(|| {let _lint_set = LintSet::new().with_standard(dictionary.clone());@@ -24,7 +24,7 @@ fn create_lint_set(bencher: Bencher) {#[divan::bench]fn lint_demo(bencher: Bencher) {- let dictionary = Dictionary::create_from_curated();+ let dictionary = FullDictionary::create_from_curated();let mut lint_set = LintSet::new().with_standard(dictionary);let document = Document::new_markdown(black_box(DEMO));diff --git a/harper-core/src/lib.rs b/harper-core/src/lib.rsindex 9c72889bf..d22c705ff 100644--- a/harper-core/src/lib.rs+++ b/harper-core/src/lib.rs@@ -12,7 +12,7 @@ pub use document::Document;pub use linting::LintSet;pub use linting::{Lint, LintKind, Linter, Suggestion};pub use span::Span;-pub use spell::Dictionary;+pub use spell::{Dictionary, FullDictionary, MergedDictionary};pub use token::{FatToken, Punctuation, Token, TokenKind, TokenStringExt};/// A utility function that removes overlapping lints in a vector,diff --git a/harper-core/src/linting/lint_set.rs b/harper-core/src/linting/lint_set.rsindex f52637ae0..101175c04 100644--- a/harper-core/src/linting/lint_set.rs+++ b/harper-core/src/linting/lint_set.rs@@ -1,4 +1,4 @@-use crate::{Dictionary, Document, Lint};+use crate::{Dictionary, Document, FullDictionary, Lint};use super::{spaces::Spaces, Linter};use paste::paste;@@ -9,11 +9,11 @@ use super::{unclosed_quotes::UnclosedQuotes, wrong_quotes::WrongQuotes,};-pub struct LintSet {- pub(super) linters: Vec<Box<dyn Linter>>,+pub struct LintSet<'a> {+ pub(super) linters: Vec<Box<dyn Linter + 'a>>,}-impl Linter for LintSet {+impl<'a> Linter for LintSet<'a> {fn lint(&mut self, document: &Document) -> Vec<Lint> {let mut lints = Vec::new();@@ -27,14 +27,14 @@ impl Linter for LintSet {}}-impl LintSet {+impl<'a> LintSet<'a> {pub fn new() -> Self {Self {linters: Vec::new(),}}- pub fn add_standard(&mut self, dictionary: Dictionary) -> &mut Self {+ pub fn add_standard(&mut self, dictionary: impl Dictionary + 'a) -> &mut Self {self.add_repeated_words().add_long_sentences().add_unclosed_quotes()@@ -45,23 +45,23 @@ impl LintSet {self}- pub fn with_standard(mut self, dictionary: Dictionary) -> Self {+ pub fn with_standard(mut self, dictionary: impl Dictionary + 'a) -> Self {self.add_standard(dictionary);self}- pub fn add_spell_check(&mut self, dictionary: Dictionary) -> &mut Self {+ pub fn add_spell_check(&mut self, dictionary: impl Dictionary + 'a) -> &mut Self {self.linters.push(Box::new(SpellCheck::new(dictionary)));self}- pub fn with_spell_check(mut self, dictionary: Dictionary) -> Self {+ pub fn with_spell_check(mut self, dictionary: impl Dictionary + 'a) -> Self {self.add_spell_check(dictionary);self}}-impl Default for LintSet {+impl Default for LintSet<'_> {fn default() -> Self {Self::new()}@@ -70,7 +70,7 @@ impl Default for LintSet {/// Create builder methods for the linters that do not take any arguments.macro_rules! create_simple_builder_methods {($($linter:ident),*) => {- impl LintSet {+ impl LintSet<'_> {paste! {$(#[doc = "Modifies self, adding the `" $linter "` linter to the set."]diff --git a/harper-core/src/linting/spell_check.rs b/harper-core/src/linting/spell_check.rsindex 1de4ccc19..63f74dbb9 100644--- a/harper-core/src/linting/spell_check.rs+++ b/harper-core/src/linting/spell_check.rs@@ -1,20 +1,20 @@use hashbrown::HashMap;use super::{Lint, LintKind, Linter};-use crate::{- document::Document,- spell::{suggest_correct_spelling, Dictionary},-};+use crate::{document::Document, spell::suggest_correct_spelling, Dictionary};use super::lint::Suggestion;-pub struct SpellCheck {- dictionary: Dictionary,+pub struct SpellCheck<T>+where+ T: Dictionary,+{+ dictionary: T,word_cache: HashMap<Vec<char>, Vec<Vec<char>>>,}-impl SpellCheck {- pub fn new(dictionary: Dictionary) -> Self {+impl<T: Dictionary> SpellCheck<T> {+ pub fn new(dictionary: T) -> Self {Self {dictionary,word_cache: HashMap::new(),@@ -22,7 +22,7 @@ impl SpellCheck {}}-impl SpellCheck {+impl<T: Dictionary> SpellCheck<T> {fn cached_suggest_correct_spelling(&mut self, word: &[char]) -> Vec<Vec<char>> {let word = word.to_vec();@@ -38,7 +38,7 @@ impl SpellCheck {}}-impl Linter for SpellCheck {+impl<T: Dictionary> Linter for SpellCheck<T> {fn lint(&mut self, document: &Document) -> Vec<Lint> {let mut lints = Vec::new();diff --git a/harper-core/src/spell/dictionary.rs b/harper-core/src/spell/dictionary.rsindex 50cb66269..80f7f0ef5 100644--- a/harper-core/src/spell/dictionary.rs+++ b/harper-core/src/spell/dictionary.rs@@ -1,103 +1,9 @@-use std::borrow::Borrow;--use hashbrown::HashSet;-use once_cell::sync::Lazy;-use smallvec::SmallVec;--use super::{- hunspell::{parse_default_attribute_list, parse_default_word_list},- seq_to_normalized, DictWord,-};--#[derive(Debug, Clone)]-pub struct Dictionary {- /// Storing a separate [`Vec`] for iterations speeds up spellchecking by ~16% at the cost of- /// additional memory.- ///- /// This is likely due to increased locality :shrug:.- ///- /// This list is sorted by word length (i.e. the shortest words are first).- words: Vec<DictWord>,- /// A lookup list for each word length.- /// Each index of this list will return the first index of [`Self::words`] that has a word- /// whose index is that length.- word_len_starts: Vec<usize>,- /// All English words- word_set: HashSet<DictWord>,-}--fn uncached_inner_new() -> Dictionary {- let word_list = parse_default_word_list().unwrap();- let attr_list = parse_default_attribute_list().unwrap();-- let words = attr_list.expand_marked_words(word_list).unwrap();- let mut words: Vec<DictWord> = words.into_iter().collect();-- Dictionary {- word_set: HashSet::from_iter(words.iter().cloned()),- word_len_starts: Dictionary::create_len_starts(&mut words),- words,- }-}--static DICT: Lazy<Dictionary> = Lazy::new(uncached_inner_new);--impl Dictionary {- /// Create a dictionary from the curated Hunspell dictionary included- /// in the Harper binary.- pub fn create_from_curated() -> Self {- DICT.clone()- }-- /// Appends words to the dictionary.- /// It is significantly faster to append many words with one call than many- /// distinct calls to this function.- pub fn append_words(&mut self, words: &mut [DictWord]) {- self.words.extend(words.iter().cloned());- self.word_set.extend(words.iter().cloned());- self.word_len_starts = Self::create_len_starts(&mut self.words);- }-- /// Create a lookup table for finding words of a specific length in a word list.- /// NOTE: This function will sort the original word list by its length.- /// If the word list's order is changed after creating the lookup, it will no longer be valid.- fn create_len_starts(words: &mut [DictWord]) -> Vec<usize> {- words.sort_by_key(|a| a.len());- let mut word_len_starts = vec![0, 0];-- for (index, len) in words.iter().map(SmallVec::len).enumerate() {- if word_len_starts.len() == len {- word_len_starts.push(index);- }- }-- word_len_starts- }+use blanket::blanket;+#[blanket(derive(Arc))]+pub trait Dictionary: Send + Sync + Clone {+ fn contains_word(&self, word: &[char]) -> bool;+ fn words_iter(&self) -> impl Iterator<Item = &'_ [char]>;/// Iterate over all the words in the dictionary of a given length- pub fn words_with_len_iter(&self, len: usize) -> Box<dyn Iterator<Item = &'_ [char]> + '_> {- if len == 0 || len >= self.word_len_starts.len() {- return Box::new(std::iter::empty());- }-- let start = self.word_len_starts[len];- let end = if len + 1 == self.word_len_starts.len() {- self.words.len()- } else {- self.word_len_starts[len + 1]- };-- Box::new(self.words[start..end].iter().map(|v| v.as_slice()))- }-- pub fn words_iter(&self) -> impl Iterator<Item = &'_ [char]> {- self.words.iter().map(|v| v.as_slice())- }-- pub fn contains_word(&self, word: &[char]) -> bool {- let normalized = seq_to_normalized(word);- let lowercase: SmallVec<_> = normalized.iter().flat_map(|c| c.to_lowercase()).collect();-- self.word_set.contains(normalized.as_ref()) || self.word_set.contains(&lowercase)- }+ fn words_with_len_iter(&self, len: usize) -> Box<dyn Iterator<Item = &'_ [char]> + '_>;}diff --git a/harper-core/src/spell/full_dictionary.rs b/harper-core/src/spell/full_dictionary.rsnew file mode 100644index 000000000..585fcb81d--- /dev/null+++ b/harper-core/src/spell/full_dictionary.rs@@ -0,0 +1,106 @@+use hashbrown::HashSet;+use once_cell::sync::Lazy;+use smallvec::SmallVec;++use super::{+ dictionary::Dictionary,+ hunspell::{parse_default_attribute_list, parse_default_word_list},+ seq_to_normalized, DictWord,+};++/// A full, fat dictionary.+/// All of the elements are stored in-memory.+#[derive(Debug, Clone)]+pub struct FullDictionary {+ /// Storing a separate [`Vec`] for iterations speeds up spellchecking by ~16% at the cost of+ /// additional memory.+ ///+ /// This is likely due to increased locality :shrug:.+ ///+ /// This list is sorted by word length (i.e. the shortest words are first).+ words: Vec<DictWord>,+ /// A lookup list for each word length.+ /// Each index of this list will return the first index of [`Self::words`] that has a word+ /// whose index is that length.+ word_len_starts: Vec<usize>,+ /// All English words+ word_set: HashSet<DictWord>,+}++fn uncached_inner_new() -> FullDictionary {+ let word_list = parse_default_word_list().unwrap();+ let attr_list = parse_default_attribute_list().unwrap();++ let words = attr_list.expand_marked_words(word_list).unwrap();+ let mut words: Vec<DictWord> = words.into_iter().collect();++ FullDictionary {+ word_set: HashSet::from_iter(words.iter().cloned()),+ word_len_starts: FullDictionary::create_len_starts(&mut words),+ words,+ }+}++static DICT: Lazy<FullDictionary> = Lazy::new(uncached_inner_new);++impl FullDictionary {+ /// Create a dictionary from the curated Hunspell dictionary included+ /// in the Harper binary.+ pub fn create_from_curated() -> Self {+ DICT.clone()+ }++ /// Appends words to the dictionary.+ /// It is significantly faster to append many words with one call than many+ /// distinct calls to this function.+ pub fn append_words(&mut self, words: &mut [DictWord]) {+ self.words.extend(words.iter().cloned());+ self.word_set.extend(words.iter().cloned());+ self.word_len_starts = Self::create_len_starts(&mut self.words);+ }++ /// Create a lookup table for finding words of a specific length in a word list.+ /// NOTE: This function will sort the original word list by its length.+ /// If the word list's order is changed after creating the lookup, it will no longer be valid.+ fn create_len_starts(words: &mut [DictWord]) -> Vec<usize> {+ words.sort_by_key(|a| a.len());+ let mut word_len_starts = vec![0, 0];++ for (index, len) in words.iter().map(SmallVec::len).enumerate() {+ if word_len_starts.len() == len {+ word_len_starts.push(index);+ }+ }++ word_len_starts+ }+}++impl Dictionary for FullDictionary {+ fn words_iter(&self) -> impl Iterator<Item = &'_ [char]> {+ self.words.iter().map(|v| v.as_slice())+ }++ /// Iterate over all the words in the dictionary of a given length+ fn words_with_len_iter(&self, len: usize) -> Box<dyn Iterator<Item = &'_ [char]> + '_> {+ if len == 0 || len >= self.word_len_starts.len() {+ return Box::new(std::iter::empty());+ }++ let start = self.word_len_starts[len];+ let end = if len + 1 == self.word_len_starts.len() {+ self.words.len()+ } else {+ self.word_len_starts[len + 1]+ };++ Box::new(self.words[start..end].iter().map(|v| v.as_slice()))+ }++ fn contains_word(&self, word: &[char]) -> bool {+ let normalized = seq_to_normalized(word);+ let lowercase: SmallVec<_> = normalized.iter().flat_map(|c| c.to_lowercase()).collect();++ self.word_set.contains(normalized.as_ref()) || self.word_set.contains(&lowercase)+ }+}diff --git a/harper-core/src/spell/merged_dictionary.rs b/harper-core/src/spell/merged_dictionary.rsnew file mode 100644index 000000000..63d66a234--- /dev/null+++ b/harper-core/src/spell/merged_dictionary.rs@@ -0,0 +1,54 @@+use std::sync::Arc;++use super::dictionary::Dictionary;++/// A simple wrapper over [`Dictionary`] that allows+/// one to merge multiple dictionaries without copying.+#[derive(Clone)]+pub struct MergedDictionary<T>+where+ T: Dictionary + Clone,+{+ children: Vec<Arc<T>>,+}++impl<T> MergedDictionary<T>+where+ T: Dictionary + Clone,+{+ pub fn new() -> Self {+ Self {+ children: Vec::new(),+ }+ }++ pub fn add_dictionary(&mut self, dictionary: impl AsRef<Arc<T>>) {+ self.children.push(dictionary.as_ref().clone());+ }+}++impl<T> Dictionary for MergedDictionary<T>+where+ T: Dictionary + Clone,+{+ fn contains_word(&self, word: &[char]) -> bool {+ for child in &self.children {+ if child.contains_word(word) {+ return true;+ }+ }+ return false;+ }++ fn words_iter(&self) -> impl Iterator<Item = &'_ [char]> {+ self.children.iter().flat_map(|c| c.words_iter())+ }++ fn words_with_len_iter(&self, len: usize) -> Box<dyn Iterator<Item = &'_ [char]> + '_> {+ Box::new(+ self.children+ .iter()+ .flat_map(move |c| c.words_with_len_iter(len)),+ )+ }+}diff --git a/harper-core/src/spell/mod.rs b/harper-core/src/spell/mod.rsindex d2f01a4db..531eefabc 100644--- a/harper-core/src/spell/mod.rs+++ b/harper-core/src/spell/mod.rs@@ -4,9 +4,13 @@ use itertools::{Itertools, MinMaxResult};use smallvec::SmallVec;pub use self::dictionary::Dictionary;+pub use self::full_dictionary::FullDictionary;+pub use self::merged_dictionary::MergedDictionary;mod dictionary;+mod full_dictionary;mod hunspell;+mod merged_dictionary;/// A word from a dictionary or other similar structure.pub type DictWord = SmallVec<[char; 6]>;@@ -18,7 +22,7 @@ pub fn suggest_correct_spelling<'a>(misspelled_word: &[char],result_limit: usize,max_edit_dist: u8,- dictionary: &'a Dictionary,+ dictionary: &'a impl Dictionary,) -> Vec<&'a [char]> {let misspelled_word = seq_to_normalized(misspelled_word);@@ -104,7 +108,7 @@ pub fn suggest_correct_spelling_str(misspelled_word: impl AsRef<str>,result_limit: usize,max_edit_dist: u8,- dictionary: &Dictionary,+ dictionary: &FullDictionary,) -> Vec<String> {let chars: Vec<char> = misspelled_word.as_ref().chars().collect();diff --git a/harper-ls/src/backend.rs b/harper-ls/src/backend.rsindex 0b4bf7b19..a46b8e662 100644--- a/harper-ls/src/backend.rs+++ b/harper-ls/src/backend.rs@@ -1,8 +1,8 @@-use std::collections::HashMap;+use std::{collections::HashMap, sync::Arc};use harper_core::{parsers::{Markdown, Parser},- Dictionary, Document, LintSet, Linter,+ Document, FullDictionary, LintSet, Linter,};use tokio::sync::Mutex;use tower_lsp::{@@ -27,7 +27,7 @@ use crate::{pub struct Backend {client: Client,- linter: Mutex<LintSet>,+ global_dictionary: Arc<FullDictionary>,files: Mutex<HashMap<Url, Document>>,}@@ -51,15 +51,18 @@ impl Backend {files.insert(url.clone(), doc);}+ fn create_linter(&self) -> LintSet {+ LintSet::new().with_standard(self.global_dictionary.clone())+ }+async fn generate_code_actions(&self, url: &Url, range: Range) -> Result<Vec<CodeAction>> {let files = self.files.lock().await;let Some(document) = files.get(url) else {return Ok(vec![]);};- let mut linter = self.linter.lock().await;+ let mut linter = self.create_linter();let mut lints = linter.lint(document);-lints.sort_by_key(|l| l.priority);let source_chars = document.get_full_content();@@ -77,12 +80,11 @@ impl Backend {}pub fn new(client: Client) -> Self {- let dictionary = Dictionary::create_from_curated();- let linter = Mutex::new(LintSet::new().with_standard(dictionary));+ let dictionary = FullDictionary::create_from_curated();Self {client,- linter,+ global_dictionary: dictionary.into(),files: Mutex::new(HashMap::new()),}}@@ -94,7 +96,7 @@ impl Backend {return vec![];};- let mut linter = self.linter.lock().await;+ let mut linter = self.create_linter();let lints = linter.lint(document);lints_to_diagnostics(document.get_full_content(), &lints)diff --git a/harper-serve/src/main.rs b/harper-serve/src/main.rsindex 62f8ac23e..76521827c 100644--- a/harper-serve/src/main.rs+++ b/harper-serve/src/main.rs@@ -1,6 +1,6 @@#![allow(dead_code)]-use harper_core::{Dictionary, Document, FatToken, Lint, LintSet, Linter, Span, Suggestion};+use harper_core::{Document, FatToken, FullDictionary, Lint, LintSet, Linter, Span, Suggestion};use std::net::SocketAddr;use tokio::time::Instant;use tracing::{info, Level};@@ -92,7 +92,7 @@ async fn lint(Json(payload): Json<LintRequest>) -> (StatusCode, Json<LintResponslet document = Document::new_markdown(&text);- let dictionary = Dictionary::create_from_curated();+ let dictionary = FullDictionary::create_from_curated();let mut linter = LintSet::new().with_standard(dictionary);let lints = linter.lint(&document);diff --git a/harper-wasm/src/lib.rs b/harper-wasm/src/lib.rsindex 0e3756432..b08f60cfd 100644--- a/harper-wasm/src/lib.rs+++ b/harper-wasm/src/lib.rs@@ -1,12 +1,12 @@use std::sync::Mutex;-use harper_core::{remove_overlaps, Dictionary, Document, LintSet, Linter};+use harper_core::{remove_overlaps, Document, FullDictionary, LintSet, Linter};use once_cell::sync::Lazy;use serde::Serialize;use wasm_bindgen::{prelude::wasm_bindgen, JsValue};static LINTER: Lazy<Mutex<LintSet>> =- Lazy::new(|| Mutex::new(LintSet::new().with_standard(Dictionary::create_from_curated())));+ Lazy::new(|| Mutex::new(LintSet::new().with_standard(FullDictionary::create_from_curated())));/// Create the serializer that preserves types across the JavaScript barrierfn glue_serializer() -> serde_wasm_bindgen::Serializer {