Skip to content

Offline, privacy-first grammar checker. Fast, open-source, Rust-powered

feat: added API to allow merging generic dictionaries

Elijah Potterfde38b4035fa
diff --git a/Cargo.lock b/Cargo.lock
index ccf9383f3..cf5e8c765 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -203,6 +203,17 @@ version = "2.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ed570934406eb16438a4e976b1b4500774099c13b8cb96eec99f620f05090ddf"
+[[package]]
+name = "blanket"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e0b121a9fe0df916e362fb3271088d071159cdf11db0e4182d02152850756eff"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.48",
+]
+
[[package]]
name = "bumpalo"
version = "3.14.0"
@@ -458,6 +469,7 @@ checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253"
name = "harper-core"
version = "0.5.8"
dependencies = [
+ "blanket",
"divan",
"hashbrown",
"is-macro",
diff --git a/harper-core/Cargo.toml b/harper-core/Cargo.toml
index 537368e6e..17f3192c3 100644
--- a/harper-core/Cargo.toml
+++ b/harper-core/Cargo.toml
@@ -8,6 +8,7 @@ readme = "README.md"
homepage = "https://github.com/chilipepperhott/harper"
[dependencies]
+blanket = "0.3.0"
hashbrown = { version = "0.14.3", features = ["serde"] }
is-macro = "0.3.0"
itertools = "0.11.0"
diff --git a/harper-core/benches/parse_demo.rs b/harper-core/benches/parse_demo.rs
index 3f28cbc29..f8da88dae 100644
--- a/harper-core/benches/parse_demo.rs
+++ b/harper-core/benches/parse_demo.rs
@@ -1,5 +1,5 @@
use divan::{black_box, AllocProfiler, Bencher};
-use harper_core::{Dictionary, Document, LintSet, Linter};
+use harper_core::{Document, FullDictionary, LintSet, Linter};
#[global_allocator]
static ALLOC: AllocProfiler = AllocProfiler::system();
@@ -15,7 +15,7 @@ fn parse_demo(bencher: Bencher) {
#[divan::bench]
fn create_lint_set(bencher: Bencher) {
- let dictionary = Dictionary::create_from_curated();
+ let dictionary = FullDictionary::create_from_curated();
bencher.bench_local(|| {
let _lint_set = LintSet::new().with_standard(dictionary.clone());
@@ -24,7 +24,7 @@ fn create_lint_set(bencher: Bencher) {
#[divan::bench]
fn lint_demo(bencher: Bencher) {
- let dictionary = Dictionary::create_from_curated();
+ let dictionary = FullDictionary::create_from_curated();
let mut lint_set = LintSet::new().with_standard(dictionary);
let document = Document::new_markdown(black_box(DEMO));
diff --git a/harper-core/src/lib.rs b/harper-core/src/lib.rs
index 9c72889bf..d22c705ff 100644
--- a/harper-core/src/lib.rs
+++ b/harper-core/src/lib.rs
@@ -12,7 +12,7 @@ pub use document::Document;
pub use linting::LintSet;
pub use linting::{Lint, LintKind, Linter, Suggestion};
pub use span::Span;
-pub use spell::Dictionary;
+pub use spell::{Dictionary, FullDictionary, MergedDictionary};
pub use token::{FatToken, Punctuation, Token, TokenKind, TokenStringExt};
/// A utility function that removes overlapping lints in a vector,
diff --git a/harper-core/src/linting/lint_set.rs b/harper-core/src/linting/lint_set.rs
index f52637ae0..101175c04 100644
--- a/harper-core/src/linting/lint_set.rs
+++ b/harper-core/src/linting/lint_set.rs
@@ -1,4 +1,4 @@
-use crate::{Dictionary, Document, Lint};
+use crate::{Dictionary, Document, FullDictionary, Lint};
use super::{spaces::Spaces, Linter};
use paste::paste;
@@ -9,11 +9,11 @@ use super::{
unclosed_quotes::UnclosedQuotes, wrong_quotes::WrongQuotes,
};
-pub struct LintSet {
- pub(super) linters: Vec<Box<dyn Linter>>,
+pub struct LintSet<'a> {
+ pub(super) linters: Vec<Box<dyn Linter + 'a>>,
}
-impl Linter for LintSet {
+impl<'a> Linter for LintSet<'a> {
fn lint(&mut self, document: &Document) -> Vec<Lint> {
let mut lints = Vec::new();
@@ -27,14 +27,14 @@ impl Linter for LintSet {
}
}
-impl LintSet {
+impl<'a> LintSet<'a> {
pub fn new() -> Self {
Self {
linters: Vec::new(),
}
}
- pub fn add_standard(&mut self, dictionary: Dictionary) -> &mut Self {
+ pub fn add_standard(&mut self, dictionary: impl Dictionary + 'a) -> &mut Self {
self.add_repeated_words()
.add_long_sentences()
.add_unclosed_quotes()
@@ -45,23 +45,23 @@ impl LintSet {
self
}
- pub fn with_standard(mut self, dictionary: Dictionary) -> Self {
+ pub fn with_standard(mut self, dictionary: impl Dictionary + 'a) -> Self {
self.add_standard(dictionary);
self
}
- pub fn add_spell_check(&mut self, dictionary: Dictionary) -> &mut Self {
+ pub fn add_spell_check(&mut self, dictionary: impl Dictionary + 'a) -> &mut Self {
self.linters.push(Box::new(SpellCheck::new(dictionary)));
self
}
- pub fn with_spell_check(mut self, dictionary: Dictionary) -> Self {
+ pub fn with_spell_check(mut self, dictionary: impl Dictionary + 'a) -> Self {
self.add_spell_check(dictionary);
self
}
}
-impl Default for LintSet {
+impl Default for LintSet<'_> {
fn default() -> Self {
Self::new()
}
@@ -70,7 +70,7 @@ impl Default for LintSet {
/// Create builder methods for the linters that do not take any arguments.
macro_rules! create_simple_builder_methods {
($($linter:ident),*) => {
- impl LintSet {
+ impl LintSet<'_> {
paste! {
$(
#[doc = "Modifies self, adding the `" $linter "` linter to the set."]
diff --git a/harper-core/src/linting/spell_check.rs b/harper-core/src/linting/spell_check.rs
index 1de4ccc19..63f74dbb9 100644
--- a/harper-core/src/linting/spell_check.rs
+++ b/harper-core/src/linting/spell_check.rs
@@ -1,20 +1,20 @@
use hashbrown::HashMap;
use super::{Lint, LintKind, Linter};
-use crate::{
- document::Document,
- spell::{suggest_correct_spelling, Dictionary},
-};
+use crate::{document::Document, spell::suggest_correct_spelling, Dictionary};
use super::lint::Suggestion;
-pub struct SpellCheck {
- dictionary: Dictionary,
+pub struct SpellCheck<T>
+where
+ T: Dictionary,
+{
+ dictionary: T,
word_cache: HashMap<Vec<char>, Vec<Vec<char>>>,
}
-impl SpellCheck {
- pub fn new(dictionary: Dictionary) -> Self {
+impl<T: Dictionary> SpellCheck<T> {
+ pub fn new(dictionary: T) -> Self {
Self {
dictionary,
word_cache: HashMap::new(),
@@ -22,7 +22,7 @@ impl SpellCheck {
}
}
-impl SpellCheck {
+impl<T: Dictionary> SpellCheck<T> {
fn cached_suggest_correct_spelling(&mut self, word: &[char]) -> Vec<Vec<char>> {
let word = word.to_vec();
@@ -38,7 +38,7 @@ impl SpellCheck {
}
}
-impl Linter for SpellCheck {
+impl<T: Dictionary> Linter for SpellCheck<T> {
fn lint(&mut self, document: &Document) -> Vec<Lint> {
let mut lints = Vec::new();
diff --git a/harper-core/src/spell/dictionary.rs b/harper-core/src/spell/dictionary.rs
index 50cb66269..80f7f0ef5 100644
--- a/harper-core/src/spell/dictionary.rs
+++ b/harper-core/src/spell/dictionary.rs
@@ -1,103 +1,9 @@
-use std::borrow::Borrow;
-
-use hashbrown::HashSet;
-use once_cell::sync::Lazy;
-use smallvec::SmallVec;
-
-use super::{
- hunspell::{parse_default_attribute_list, parse_default_word_list},
- seq_to_normalized, DictWord,
-};
-
-#[derive(Debug, Clone)]
-pub struct Dictionary {
- /// Storing a separate [`Vec`] for iterations speeds up spellchecking by ~16% at the cost of
- /// additional memory.
- ///
- /// This is likely due to increased locality :shrug:.
- ///
- /// This list is sorted by word length (i.e. the shortest words are first).
- words: Vec<DictWord>,
- /// A lookup list for each word length.
- /// Each index of this list will return the first index of [`Self::words`] that has a word
- /// whose index is that length.
- word_len_starts: Vec<usize>,
- /// All English words
- word_set: HashSet<DictWord>,
-}
-
-fn uncached_inner_new() -> Dictionary {
- let word_list = parse_default_word_list().unwrap();
- let attr_list = parse_default_attribute_list().unwrap();
-
- let words = attr_list.expand_marked_words(word_list).unwrap();
- let mut words: Vec<DictWord> = words.into_iter().collect();
-
- Dictionary {
- word_set: HashSet::from_iter(words.iter().cloned()),
- word_len_starts: Dictionary::create_len_starts(&mut words),
- words,
- }
-}
-
-static DICT: Lazy<Dictionary> = Lazy::new(uncached_inner_new);
-
-impl Dictionary {
- /// Create a dictionary from the curated Hunspell dictionary included
- /// in the Harper binary.
- pub fn create_from_curated() -> Self {
- DICT.clone()
- }
-
- /// Appends words to the dictionary.
- /// It is significantly faster to append many words with one call than many
- /// distinct calls to this function.
- pub fn append_words(&mut self, words: &mut [DictWord]) {
- self.words.extend(words.iter().cloned());
- self.word_set.extend(words.iter().cloned());
- self.word_len_starts = Self::create_len_starts(&mut self.words);
- }
-
- /// Create a lookup table for finding words of a specific length in a word list.
- /// NOTE: This function will sort the original word list by its length.
- /// If the word list's order is changed after creating the lookup, it will no longer be valid.
- fn create_len_starts(words: &mut [DictWord]) -> Vec<usize> {
- words.sort_by_key(|a| a.len());
- let mut word_len_starts = vec![0, 0];
-
- for (index, len) in words.iter().map(SmallVec::len).enumerate() {
- if word_len_starts.len() == len {
- word_len_starts.push(index);
- }
- }
-
- word_len_starts
- }
+use blanket::blanket;
+#[blanket(derive(Arc))]
+pub trait Dictionary: Send + Sync + Clone {
+ fn contains_word(&self, word: &[char]) -> bool;
+ fn words_iter(&self) -> impl Iterator<Item = &'_ [char]>;
/// Iterate over all the words in the dictionary of a given length
- pub fn words_with_len_iter(&self, len: usize) -> Box<dyn Iterator<Item = &'_ [char]> + '_> {
- if len == 0 || len >= self.word_len_starts.len() {
- return Box::new(std::iter::empty());
- }
-
- let start = self.word_len_starts[len];
- let end = if len + 1 == self.word_len_starts.len() {
- self.words.len()
- } else {
- self.word_len_starts[len + 1]
- };
-
- Box::new(self.words[start..end].iter().map(|v| v.as_slice()))
- }
-
- pub fn words_iter(&self) -> impl Iterator<Item = &'_ [char]> {
- self.words.iter().map(|v| v.as_slice())
- }
-
- pub fn contains_word(&self, word: &[char]) -> bool {
- let normalized = seq_to_normalized(word);
- let lowercase: SmallVec<_> = normalized.iter().flat_map(|c| c.to_lowercase()).collect();
-
- self.word_set.contains(normalized.as_ref()) || self.word_set.contains(&lowercase)
- }
+ fn words_with_len_iter(&self, len: usize) -> Box<dyn Iterator<Item = &'_ [char]> + '_>;
}
diff --git a/harper-core/src/spell/full_dictionary.rs b/harper-core/src/spell/full_dictionary.rs
new file mode 100644
index 000000000..585fcb81d
--- /dev/null
+++ b/harper-core/src/spell/full_dictionary.rs
@@ -0,0 +1,106 @@
+use hashbrown::HashSet;
+use once_cell::sync::Lazy;
+use smallvec::SmallVec;
+
+use super::{
+ dictionary::Dictionary,
+ hunspell::{parse_default_attribute_list, parse_default_word_list},
+ seq_to_normalized, DictWord,
+};
+
+/// A full, fat dictionary.
+/// All of the elements are stored in-memory.
+#[derive(Debug, Clone)]
+pub struct FullDictionary {
+ /// Storing a separate [`Vec`] for iterations speeds up spellchecking by ~16% at the cost of
+ /// additional memory.
+ ///
+ /// This is likely due to increased locality :shrug:.
+ ///
+ /// This list is sorted by word length (i.e. the shortest words are first).
+ words: Vec<DictWord>,
+ /// A lookup list for each word length.
+ /// Each index of this list will return the first index of [`Self::words`] that has a word
+ /// whose index is that length.
+ word_len_starts: Vec<usize>,
+ /// All English words
+ word_set: HashSet<DictWord>,
+}
+
+fn uncached_inner_new() -> FullDictionary {
+ let word_list = parse_default_word_list().unwrap();
+ let attr_list = parse_default_attribute_list().unwrap();
+
+ let words = attr_list.expand_marked_words(word_list).unwrap();
+ let mut words: Vec<DictWord> = words.into_iter().collect();
+
+ FullDictionary {
+ word_set: HashSet::from_iter(words.iter().cloned()),
+ word_len_starts: FullDictionary::create_len_starts(&mut words),
+ words,
+ }
+}
+
+static DICT: Lazy<FullDictionary> = Lazy::new(uncached_inner_new);
+
+impl FullDictionary {
+ /// Create a dictionary from the curated Hunspell dictionary included
+ /// in the Harper binary.
+ pub fn create_from_curated() -> Self {
+ DICT.clone()
+ }
+
+ /// Appends words to the dictionary.
+ /// It is significantly faster to append many words with one call than many
+ /// distinct calls to this function.
+ pub fn append_words(&mut self, words: &mut [DictWord]) {
+ self.words.extend(words.iter().cloned());
+ self.word_set.extend(words.iter().cloned());
+ self.word_len_starts = Self::create_len_starts(&mut self.words);
+ }
+
+ /// Create a lookup table for finding words of a specific length in a word list.
+ /// NOTE: This function will sort the original word list by its length.
+ /// If the word list's order is changed after creating the lookup, it will no longer be valid.
+ fn create_len_starts(words: &mut [DictWord]) -> Vec<usize> {
+ words.sort_by_key(|a| a.len());
+ let mut word_len_starts = vec![0, 0];
+
+ for (index, len) in words.iter().map(SmallVec::len).enumerate() {
+ if word_len_starts.len() == len {
+ word_len_starts.push(index);
+ }
+ }
+
+ word_len_starts
+ }
+}
+
+impl Dictionary for FullDictionary {
+ fn words_iter(&self) -> impl Iterator<Item = &'_ [char]> {
+ self.words.iter().map(|v| v.as_slice())
+ }
+
+ /// Iterate over all the words in the dictionary of a given length
+ fn words_with_len_iter(&self, len: usize) -> Box<dyn Iterator<Item = &'_ [char]> + '_> {
+ if len == 0 || len >= self.word_len_starts.len() {
+ return Box::new(std::iter::empty());
+ }
+
+ let start = self.word_len_starts[len];
+ let end = if len + 1 == self.word_len_starts.len() {
+ self.words.len()
+ } else {
+ self.word_len_starts[len + 1]
+ };
+
+ Box::new(self.words[start..end].iter().map(|v| v.as_slice()))
+ }
+
+ fn contains_word(&self, word: &[char]) -> bool {
+ let normalized = seq_to_normalized(word);
+ let lowercase: SmallVec<_> = normalized.iter().flat_map(|c| c.to_lowercase()).collect();
+
+ self.word_set.contains(normalized.as_ref()) || self.word_set.contains(&lowercase)
+ }
+}
diff --git a/harper-core/src/spell/merged_dictionary.rs b/harper-core/src/spell/merged_dictionary.rs
new file mode 100644
index 000000000..63d66a234
--- /dev/null
+++ b/harper-core/src/spell/merged_dictionary.rs
@@ -0,0 +1,54 @@
+use std::sync::Arc;
+
+use super::dictionary::Dictionary;
+
+/// A simple wrapper over [`Dictionary`] that allows
+/// one to merge multiple dictionaries without copying.
+#[derive(Clone)]
+pub struct MergedDictionary<T>
+where
+ T: Dictionary + Clone,
+{
+ children: Vec<Arc<T>>,
+}
+
+impl<T> MergedDictionary<T>
+where
+ T: Dictionary + Clone,
+{
+ pub fn new() -> Self {
+ Self {
+ children: Vec::new(),
+ }
+ }
+
+ pub fn add_dictionary(&mut self, dictionary: impl AsRef<Arc<T>>) {
+ self.children.push(dictionary.as_ref().clone());
+ }
+}
+
+impl<T> Dictionary for MergedDictionary<T>
+where
+ T: Dictionary + Clone,
+{
+ fn contains_word(&self, word: &[char]) -> bool {
+ for child in &self.children {
+ if child.contains_word(word) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ fn words_iter(&self) -> impl Iterator<Item = &'_ [char]> {
+ self.children.iter().flat_map(|c| c.words_iter())
+ }
+
+ fn words_with_len_iter(&self, len: usize) -> Box<dyn Iterator<Item = &'_ [char]> + '_> {
+ Box::new(
+ self.children
+ .iter()
+ .flat_map(move |c| c.words_with_len_iter(len)),
+ )
+ }
+}
diff --git a/harper-core/src/spell/mod.rs b/harper-core/src/spell/mod.rs
index d2f01a4db..531eefabc 100644
--- a/harper-core/src/spell/mod.rs
+++ b/harper-core/src/spell/mod.rs
@@ -4,9 +4,13 @@ use itertools::{Itertools, MinMaxResult};
use smallvec::SmallVec;
pub use self::dictionary::Dictionary;
+pub use self::full_dictionary::FullDictionary;
+pub use self::merged_dictionary::MergedDictionary;
mod dictionary;
+mod full_dictionary;
mod hunspell;
+mod merged_dictionary;
/// A word from a dictionary or other similar structure.
pub type DictWord = SmallVec<[char; 6]>;
@@ -18,7 +22,7 @@ pub fn suggest_correct_spelling<'a>(
misspelled_word: &[char],
result_limit: usize,
max_edit_dist: u8,
- dictionary: &'a Dictionary,
+ dictionary: &'a impl Dictionary,
) -> Vec<&'a [char]> {
let misspelled_word = seq_to_normalized(misspelled_word);
@@ -104,7 +108,7 @@ pub fn suggest_correct_spelling_str(
misspelled_word: impl AsRef<str>,
result_limit: usize,
max_edit_dist: u8,
- dictionary: &Dictionary,
+ dictionary: &FullDictionary,
) -> Vec<String> {
let chars: Vec<char> = misspelled_word.as_ref().chars().collect();
diff --git a/harper-ls/src/backend.rs b/harper-ls/src/backend.rs
index 0b4bf7b19..a46b8e662 100644
--- a/harper-ls/src/backend.rs
+++ b/harper-ls/src/backend.rs
@@ -1,8 +1,8 @@
-use std::collections::HashMap;
+use std::{collections::HashMap, sync::Arc};
use harper_core::{
parsers::{Markdown, Parser},
- Dictionary, Document, LintSet, Linter,
+ Document, FullDictionary, LintSet, Linter,
};
use tokio::sync::Mutex;
use tower_lsp::{
@@ -27,7 +27,7 @@ use crate::{
pub struct Backend {
client: Client,
- linter: Mutex<LintSet>,
+ global_dictionary: Arc<FullDictionary>,
files: Mutex<HashMap<Url, Document>>,
}
@@ -51,15 +51,18 @@ impl Backend {
files.insert(url.clone(), doc);
}
+ fn create_linter(&self) -> LintSet {
+ LintSet::new().with_standard(self.global_dictionary.clone())
+ }
+
async fn generate_code_actions(&self, url: &Url, range: Range) -> Result<Vec<CodeAction>> {
let files = self.files.lock().await;
let Some(document) = files.get(url) else {
return Ok(vec![]);
};
- let mut linter = self.linter.lock().await;
+ let mut linter = self.create_linter();
let mut lints = linter.lint(document);
-
lints.sort_by_key(|l| l.priority);
let source_chars = document.get_full_content();
@@ -77,12 +80,11 @@ impl Backend {
}
pub fn new(client: Client) -> Self {
- let dictionary = Dictionary::create_from_curated();
- let linter = Mutex::new(LintSet::new().with_standard(dictionary));
+ let dictionary = FullDictionary::create_from_curated();
Self {
client,
- linter,
+ global_dictionary: dictionary.into(),
files: Mutex::new(HashMap::new()),
}
}
@@ -94,7 +96,7 @@ impl Backend {
return vec![];
};
- let mut linter = self.linter.lock().await;
+ let mut linter = self.create_linter();
let lints = linter.lint(document);
lints_to_diagnostics(document.get_full_content(), &lints)
diff --git a/harper-serve/src/main.rs b/harper-serve/src/main.rs
index 62f8ac23e..76521827c 100644
--- a/harper-serve/src/main.rs
+++ b/harper-serve/src/main.rs
@@ -1,6 +1,6 @@
#![allow(dead_code)]
-use harper_core::{Dictionary, Document, FatToken, Lint, LintSet, Linter, Span, Suggestion};
+use harper_core::{Document, FatToken, FullDictionary, Lint, LintSet, Linter, Span, Suggestion};
use std::net::SocketAddr;
use tokio::time::Instant;
use tracing::{info, Level};
@@ -92,7 +92,7 @@ async fn lint(Json(payload): Json<LintRequest>) -> (StatusCode, Json<LintRespons
let document = Document::new_markdown(&text);
- let dictionary = Dictionary::create_from_curated();
+ let dictionary = FullDictionary::create_from_curated();
let mut linter = LintSet::new().with_standard(dictionary);
let lints = linter.lint(&document);
diff --git a/harper-wasm/src/lib.rs b/harper-wasm/src/lib.rs
index 0e3756432..b08f60cfd 100644
--- a/harper-wasm/src/lib.rs
+++ b/harper-wasm/src/lib.rs
@@ -1,12 +1,12 @@
use std::sync::Mutex;
-use harper_core::{remove_overlaps, Dictionary, Document, LintSet, Linter};
+use harper_core::{remove_overlaps, Document, FullDictionary, LintSet, Linter};
use once_cell::sync::Lazy;
use serde::Serialize;
use wasm_bindgen::{prelude::wasm_bindgen, JsValue};
static LINTER: Lazy<Mutex<LintSet>> =
- Lazy::new(|| Mutex::new(LintSet::new().with_standard(Dictionary::create_from_curated())));
+ Lazy::new(|| Mutex::new(LintSet::new().with_standard(FullDictionary::create_from_curated())));
/// Create the serializer that preserves types across the JavaScript barrier
fn glue_serializer() -> serde_wasm_bindgen::Serializer {