Skip to content

Offline, privacy-first grammar checker. Fast, open-source, Rust-powered

feat(core): orchestrate caching at a higher level

Elijah Potter2050fbd18de4
diff --git a/Cargo.lock b/Cargo.lock
index eaece4973..0bab533ad 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -795,6 +795,7 @@ dependencies = [
"harper-html",
"harper-literate-haskell",
"harper-typst",
+ "indexmap",
"itertools 0.14.0",
"once_cell",
"open",
diff --git a/harper-core/src/linting/lint_group.rs b/harper-core/src/linting/lint_group.rs
index e06bb0a64..97f1883c4 100644
--- a/harper-core/src/linting/lint_group.rs
+++ b/harper-core/src/linting/lint_group.rs
@@ -1,14 +1,17 @@
use std::collections::BTreeMap;
+use std::hash::Hash;
+use std::hash::{BuildHasher, Hasher};
use std::mem;
+use std::num::NonZero;
use std::sync::Arc;
use cached::proc_macro::cached;
+use foldhash::quality::RandomState;
use hashbrown::HashMap;
use indexmap::IndexMap;
+use lru::LruCache;
use serde::{Deserialize, Serialize};
-use super::Lint;
-use super::PatternLinterCache;
use super::an_a::AnA;
use super::avoid_curses::AvoidCurses;
use super::back_in_the_day::BackInTheDay;
@@ -39,6 +42,7 @@ use super::nobody::Nobody;
use super::number_suffix_capitalization::NumberSuffixCapitalization;
use super::out_of_date::OutOfDate;
use super::oxymorons::Oxymorons;
+use super::pattern_linter::run_on_chunk;
use super::pique_interest::PiqueInterest;
use super::plural_conjugate::PluralConjugate;
use super::possessive_your::PossessiveYour;
@@ -60,8 +64,9 @@ use super::whereas::Whereas;
use super::wordpress_dotcom::WordPressDotcom;
use super::wrong_quotes::WrongQuotes;
use super::{CurrencyPlacement, Linter, NoOxfordComma, OxfordComma};
-use crate::Document;
+use super::{Lint, PatternLinter};
use crate::linting::{closed_compounds, phrase_corrections};
+use crate::{CharString, Document, TokenStringExt};
use crate::{Dictionary, MutableDictionary};
#[derive(Debug, Serialize, Deserialize, Default, Clone)]
@@ -132,28 +137,79 @@ impl LintGroupConfig {
}
}
-#[derive(Default)]
+impl Hash for LintGroupConfig {
+ fn hash<H: Hasher>(&self, hasher: &mut H) {
+ for (key, value) in &self.inner {
+ hasher.write(key.as_bytes());
+ if let Some(value) = value {
+ hasher.write_u8(1);
+ hasher.write_u8(*value as u8);
+ } else {
+ hasher.write_u8(0);
+ hasher.write_u8(0);
+ }
+ }
+ }
+}
+
pub struct LintGroup {
pub config: LintGroupConfig,
/// We use a binary map here so the ordering is stable.
- inner: BTreeMap<String, Box<dyn Linter>>,
+ linters: BTreeMap<String, Box<dyn Linter>>,
+ /// We use a binary map here so the ordering is stable.
+ pattern_linters: BTreeMap<String, Box<dyn PatternLinter>>,
+ /// Since [`PatternLinter`]s operate on a chunk-basis, we can store a
+ /// mapping of `Chunk -> Lint` and only re-run the pattern linters
+ /// when a chunk changes.
+ ///
+ /// Since the pattern linter results also depend on the config, we hash it and pass it as part
+ /// of the key.
+ chunk_pattern_cache: LruCache<(CharString, u64), Vec<Lint>>,
+ hasher_builder: RandomState,
}
impl LintGroup {
pub fn empty() -> Self {
Self {
config: LintGroupConfig::default(),
- inner: BTreeMap::new(),
+ linters: BTreeMap::new(),
+ pattern_linters: BTreeMap::new(),
+ chunk_pattern_cache: LruCache::new(NonZero::new(10000).unwrap()),
+ hasher_builder: RandomState::default(),
}
}
+ /// Check if the group already contains a linter with a given name.
+ pub fn contains_key(&self, name: impl AsRef<str>) -> bool {
+ self.linters.contains_key(name.as_ref()) || self.pattern_linters.contains_key(name.as_ref())
+ }
+
/// Add a [`Linter`] to the group, returning whether the operation was successful.
/// If it returns `false`, it is because a linter with that key already existed in the group.
pub fn add(&mut self, name: impl AsRef<str>, linter: Box<dyn Linter>) -> bool {
- if self.inner.contains_key(name.as_ref()) {
+ if self.contains_key(&name) {
+ false
+ } else {
+ self.linters.insert(name.as_ref().to_string(), linter);
+ true
+ }
+ }
+
+ /// Add a [`PatternLinter`] to the group, returning whether the operation was successful.
+ /// If it returns `false`, it is because a linter with that key already existed in the group.
+ ///
+ /// This function is not significantly different from [`Self::add`], but allows us to take
+ /// advantage of some properties of [`PatterLinter`]s for cache optimization.
+ pub fn add_pattern_linter(
+ &mut self,
+ name: impl AsRef<str>,
+ linter: Box<dyn PatternLinter>,
+ ) -> bool {
+ if self.contains_key(&name) {
false
} else {
- self.inner.insert(name.as_ref().to_string(), linter);
+ self.pattern_linters
+ .insert(name.as_ref().to_string(), linter);
true
}
}
@@ -163,15 +219,26 @@ impl LintGroup {
pub fn merge_from(&mut self, other: &mut LintGroup) {
self.config.merge_from(&mut other.config);
- let other_map = std::mem::take(&mut other.inner);
+ let other_linters = std::mem::take(&mut other.linters);
+ self.linters.extend(other_linters);
- self.inner.extend(other_map);
+ let other_pattern_linters = std::mem::take(&mut other.pattern_linters);
+ self.pattern_linters.extend(other_pattern_linters);
+ }
+
+ pub fn iter_keys(&self) -> impl Iterator<Item = &str> {
+ self.linters
+ .keys()
+ .chain(self.pattern_linters.keys())
+ .map(|v| v.as_str())
}
/// Set all contained rules to a specific value.
/// Passing `None` will unset that rule, allowing it to assume its default state.
pub fn set_all_rules_to(&mut self, enabled: Option<bool>) {
- for key in self.inner.keys() {
+ let keys = self.iter_keys().map(|v| v.to_string()).collect::<Vec<_>>();
+
+ for key in keys {
match enabled {
Some(v) => self.config.set_rule_enabled(key, v),
None => self.config.unset_rule_enabled(key),
@@ -180,9 +247,14 @@ impl LintGroup {
}
pub fn all_descriptions(&self) -> HashMap<&str, &str> {
- self.inner
+ self.linters
.iter()
.map(|(key, value)| (key.as_str(), value.description()))
+ .chain(
+ self.pattern_linters
+ .iter()
+ .map(|(key, value)| (key.as_str(), PatternLinter::description(value))),
+ )
.collect()
}
@@ -205,10 +277,7 @@ impl LintGroup {
macro_rules! insert_pattern_rule {
($rule:ident, $default_config:expr) => {
- out.add(
- stringify!($rule),
- Box::new(PatternLinterCache::new_default_size($rule::default())),
- );
+ out.add_pattern_linter(stringify!($rule), Box::new($rule::default()));
out.config
.set_rule_enabled(stringify!($rule), $default_config);
};
@@ -223,9 +292,9 @@ impl LintGroup {
// Add all the more complex rules to the group.
insert_pattern_rule!(BackInTheDay, true);
insert_struct_rule!(WordPressDotcom, true);
- insert_struct_rule!(OutOfDate, true);
- insert_struct_rule!(ThenThan, true);
- insert_struct_rule!(PiqueInterest, true);
+ insert_pattern_rule!(OutOfDate, true);
+ insert_pattern_rule!(ThenThan, true);
+ insert_pattern_rule!(PiqueInterest, true);
insert_pattern_rule!(WasAloud, true);
insert_pattern_rule!(HyphenateNumberDay, true);
insert_pattern_rule!(LeftRightHand, true);
@@ -247,12 +316,12 @@ impl LintGroup {
insert_struct_rule!(Matcher, true);
insert_struct_rule!(CorrectNumberSuffix, true);
insert_struct_rule!(NumberSuffixCapitalization, true);
- insert_struct_rule!(MultipleSequentialPronouns, true);
+ insert_pattern_rule!(MultipleSequentialPronouns, true);
insert_struct_rule!(LinkingVerbs, false);
insert_struct_rule!(AvoidCurses, true);
- insert_struct_rule!(TerminatingConjunctions, true);
+ insert_pattern_rule!(TerminatingConjunctions, true);
insert_struct_rule!(EllipsisLength, true);
- insert_struct_rule!(DotInitialisms, true);
+ insert_pattern_rule!(DotInitialisms, true);
insert_pattern_rule!(BoringWords, false);
insert_pattern_rule!(UseGenitive, false);
insert_pattern_rule!(ThatWhich, true);
@@ -287,16 +356,50 @@ impl LintGroup {
}
}
+impl Default for LintGroup {
+ fn default() -> Self {
+ Self::empty()
+ }
+}
+
impl Linter for LintGroup {
fn lint(&mut self, document: &Document) -> Vec<Lint> {
let mut results = Vec::new();
- for (key, linter) in &mut self.inner {
+ // Normal linters
+ for (key, linter) in &mut self.linters {
if self.config.is_rule_enabled(key) {
results.extend(linter.lint(document));
}
}
+ // Pattern linters
+ for chunk in document.iter_chunks() {
+ let Some(chunk_span) = chunk.span() else {
+ continue;
+ };
+
+ let chunk_chars = document.get_span_content(chunk_span);
+ let config_hash = self.hasher_builder.hash_one(&self.config);
+ let key = (chunk_chars.into(), config_hash);
+
+ if let Some(hit) = self.chunk_pattern_cache.get(&key) {
+ results.extend(hit.iter().cloned());
+ } else {
+ let mut pattern_lints = Vec::new();
+
+ for (key, linter) in &mut self.pattern_linters {
+ if self.config.is_rule_enabled(key) {
+ pattern_lints.extend(run_on_chunk(linter, chunk, document.get_source()));
+ }
+ }
+
+ self.chunk_pattern_cache.put(key, pattern_lints.clone());
+
+ results.append(&mut pattern_lints);
+ }
+ }
+
results
}
diff --git a/harper-core/src/linting/mod.rs b/harper-core/src/linting/mod.rs
index 1206155ef..649120882 100644
--- a/harper-core/src/linting/mod.rs
+++ b/harper-core/src/linting/mod.rs
@@ -103,7 +103,7 @@ pub use number_suffix_capitalization::NumberSuffixCapitalization;
pub use out_of_date::OutOfDate;
pub use oxford_comma::OxfordComma;
pub use oxymorons::Oxymorons;
-pub use pattern_linter::{PatternLinter, PatternLinterCache};
+pub use pattern_linter::PatternLinter;
pub use pique_interest::PiqueInterest;
pub use plural_conjugate::PluralConjugate;
pub use possessive_your::PossessiveYour;
diff --git a/harper-core/src/linting/pattern_linter.rs b/harper-core/src/linting/pattern_linter.rs
index e99a7c407..da213ac8c 100644
--- a/harper-core/src/linting/pattern_linter.rs
+++ b/harper-core/src/linting/pattern_linter.rs
@@ -1,12 +1,6 @@
-use std::{
- hash::{BuildHasher, Hasher},
- num::{NonZero, NonZeroUsize},
-};
+use blanket::blanket;
-use foldhash::quality::FixedState;
-use lru::LruCache;
-
-use crate::{Document, LSend, Lrc, Token, TokenStringExt, patterns::Pattern};
+use crate::{Document, LSend, Token, TokenStringExt, patterns::Pattern};
use super::{Lint, Linter};
@@ -14,6 +8,7 @@ use super::{Lint, Linter};
///
/// Makes use of [`TokenStringExt::iter_chunks`] to avoid matching across sentence or clause
/// boundaries.
+#[blanket(derive(Box))]
pub trait PatternLinter: LSend {
/// A simple getter for the pattern to be searched for.
fn pattern(&self) -> &dyn Pattern;
@@ -47,53 +42,7 @@ where
}
}
-type ChunkCache = LruCache<u64, Lrc<Vec<Lint>>>;
-
-/// A cache that wraps around a [`PatternLinter`], caching
-/// results by chunk.
-pub struct PatternLinterCache<P: PatternLinter> {
- cache: ChunkCache,
- inner: P,
-}
-
-impl<P: PatternLinter> PatternLinterCache<P> {
- /// Add a cache to a given [`PatternLinter`] with a given cache size.
- pub fn new(inner: P, cache_size: NonZeroUsize) -> Self {
- Self {
- cache: ChunkCache::new(cache_size),
- inner,
- }
- }
-
- /// Add a cache to a given [`PatternLinter`] with a given cache size.
- pub fn new_default_size(inner: P) -> Self {
- Self {
- cache: ChunkCache::new(NonZero::new(100000).unwrap()),
- inner,
- }
- }
-}
-
-impl<P: PatternLinter> Linter for PatternLinterCache<P> {
- fn lint(&mut self, document: &Document) -> Vec<Lint> {
- let mut lints = Vec::new();
- let source = document.get_source();
-
- for chunk in document.iter_chunks() {
- let chunk_lints = run_on_chunk_cached(&self.inner, chunk, source, &mut self.cache);
-
- lints.extend(chunk_lints.as_ref().iter().cloned());
- }
-
- lints
- }
-
- fn description(&self) -> &str {
- self.inner.description()
- }
-}
-
-fn run_on_chunk(linter: &impl PatternLinter, chunk: &[Token], source: &[char]) -> Vec<Lint> {
+pub fn run_on_chunk(linter: &impl PatternLinter, chunk: &[Token], source: &[char]) -> Vec<Lint> {
let mut lints = Vec::new();
let mut tok_cursor = 0;
@@ -116,33 +65,3 @@ fn run_on_chunk(linter: &impl PatternLinter, chunk: &[Token], source: &[char]) -
lints
}
-
-fn run_on_chunk_cached(
- linter: &impl PatternLinter,
- chunk: &[Token],
- source: &[char],
- cache: &mut ChunkCache,
-) -> Lrc<Vec<Lint>> {
- let Some(chunk_span) = chunk.span() else {
- return Vec::new().into();
- };
-
- let chars = chunk_span.get_content(source);
-
- let mut hasher = FixedState::default().build_hasher();
-
- for c in chars {
- hasher.write_u32(*c as u32);
- }
-
- let key = hasher.finish();
-
- if let Some(hit) = cache.get(&key) {
- return hit.clone();
- }
-
- let lints = Lrc::new(run_on_chunk(linter, chunk, source));
- cache.put(key, lints.clone());
-
- lints
-}
diff --git a/harper-core/src/linting/phrase_corrections.rs b/harper-core/src/linting/phrase_corrections.rs
index 0295cf8ac..611f97706 100644
--- a/harper-core/src/linting/phrase_corrections.rs
+++ b/harper-core/src/linting/phrase_corrections.rs
@@ -1,4 +1,4 @@
-use super::{LintGroup, MapPhraseLinter, PatternLinterCache};
+use super::{LintGroup, MapPhraseLinter};
/// Produce a [`LintGroup`] that looks for errors in common phrases.
/// Comes pre-configured with the recommended default settings.
@@ -10,16 +10,16 @@ pub fn lint_group() -> LintGroup {
$($name:expr => ($input:expr, $corrections:expr, $hint:expr, $description:expr)),+ $(,)?
}) => {
$(
- $group.add(
+ $group.add_pattern_linter(
$name,
- Box::new(PatternLinterCache::new_default_size(
+ Box::new(
MapPhraseLinter::new_exact_phrases(
$input,
$corrections,
$hint,
$description
),
- )),
+ ),
);
)+
};
diff --git a/harper-core/src/linting/proper_noun_capitalization_linters.rs b/harper-core/src/linting/proper_noun_capitalization_linters.rs
index bfbc2493d..55438a8d7 100644
--- a/harper-core/src/linting/proper_noun_capitalization_linters.rs
+++ b/harper-core/src/linting/proper_noun_capitalization_linters.rs
@@ -1,7 +1,6 @@
use hashbrown::HashMap;
use serde::{Deserialize, Serialize};
-use super::pattern_linter::PatternLinterCache;
use super::{Lint, LintKind, Suggestion};
use super::{LintGroup, PatternLinter};
use crate::parsers::PlainEnglish;
@@ -117,14 +116,12 @@ fn lint_group_from_json(json: &str, dictionary: Arc<impl Dictionary + 'static>)
let rules: HashMap<String, RuleEntry> = serde_json::from_str(json).unwrap();
for (key, rule) in rules.into_iter() {
- group.add(
+ group.add_pattern_linter(
key,
- Box::new(PatternLinterCache::new_default_size(
- ProperNounCapitalizationLinter::new_strs(
- rule.canonical,
- rule.description,
- dictionary.clone(),
- ),
+ Box::new(ProperNounCapitalizationLinter::new_strs(
+ rule.canonical,
+ rule.description,
+ dictionary.clone(),
)),
);
}
diff --git a/harper-core/src/linting/spell_check.rs b/harper-core/src/linting/spell_check.rs
index 11a7c2194..1940326c9 100644
--- a/harper-core/src/linting/spell_check.rs
+++ b/harper-core/src/linting/spell_check.rs
@@ -1,4 +1,6 @@
-use hashbrown::HashMap;
+use std::num::NonZero;
+
+use lru::LruCache;
use smallvec::ToSmallVec;
use super::Suggestion;
@@ -12,41 +14,40 @@ where
T: Dictionary,
{
dictionary: T,
- word_cache: HashMap<CharString, Vec<CharString>>,
+ word_cache: LruCache<CharString, Vec<CharString>>,
}
impl<T: Dictionary> SpellCheck<T> {
pub fn new(dictionary: T) -> Self {
Self {
dictionary,
- word_cache: HashMap::new(),
+ word_cache: LruCache::new(NonZero::new(10000).unwrap()),
}
}
}
impl<T: Dictionary> SpellCheck<T> {
fn cached_suggest_correct_spelling(&mut self, word: &[char]) -> Vec<CharString> {
- let word = word.to_smallvec();
-
- self.word_cache
- .entry(word.clone())
- .or_insert_with(|| {
- // Back off until we find a match.
- let mut suggestions = Vec::new();
- let mut dist = 2;
-
- while suggestions.is_empty() && dist < 5 {
- suggestions = suggest_correct_spelling(&word, 100, dist, &self.dictionary)
- .into_iter()
- .map(|v| v.to_smallvec())
- .collect();
-
- dist += 1;
- }
+ if let Some(hit) = self.word_cache.get(word) {
+ return hit.clone();
+ }
- suggestions
- })
- .clone()
+ // Back off until we find a match.
+ let mut suggestions = Vec::new();
+ let mut dist = 2;
+
+ while suggestions.is_empty() && dist < 5 {
+ suggestions = suggest_correct_spelling(word, 100, dist, &self.dictionary)
+ .into_iter()
+ .map(|v| v.to_smallvec())
+ .collect();
+
+ dist += 1;
+ }
+
+ self.word_cache.put(word.into(), suggestions.clone());
+
+ suggestions
}
}
diff --git a/harper-core/src/spell/merged_dictionary.rs b/harper-core/src/spell/merged_dictionary.rs
index 315eaf733..2149c385a 100644
--- a/harper-core/src/spell/merged_dictionary.rs
+++ b/harper-core/src/spell/merged_dictionary.rs
@@ -1,9 +1,10 @@
use std::hash::{BuildHasher, Hasher};
use std::sync::Arc;
-use foldhash::fast::FixedState;
+use foldhash::quality::FixedState;
use itertools::Itertools;
+use super::FstDictionary;
use super::{FuzzyMatchResult, dictionary::Dictionary};
use crate::{CharString, WordMetadata};
@@ -12,6 +13,7 @@ use crate::{CharString, WordMetadata};
#[derive(Clone)]
pub struct MergedDictionary {
children: Vec<Arc<dyn Dictionary>>,
+ hasher_builder: FixedState,
child_hashes: Vec<u64>,
}
@@ -19,19 +21,32 @@ impl MergedDictionary {
pub fn new() -> Self {
Self {
children: Vec::new(),
+ hasher_builder: FixedState::default(),
child_hashes: Vec::new(),
}
}
pub fn add_dictionary(&mut self, dictionary: Arc<dyn Dictionary>) {
- let mut hasher = FixedState::default().build_hasher();
+ self.child_hashes.push(self.hash_dictionary(&dictionary));
+ self.children.push(dictionary);
+ }
+
+ fn hash_dictionary(&self, dictionary: &Arc<dyn Dictionary>) -> u64 {
+ // Hashing the curated dictionary isn't super helpful and takes a long time.
+ if Arc::ptr_eq(
+ dictionary,
+ &(FstDictionary::curated() as Arc<dyn Dictionary>),
+ ) {
+ return 1;
+ }
+
+ let mut hasher = self.hasher_builder.build_hasher();
dictionary
.words_iter()
.for_each(|w| w.iter().for_each(|c| hasher.write_u32(*c as u32)));
- self.child_hashes.push(hasher.finish());
- self.children.push(dictionary);
+ hasher.finish()
}
}
diff --git a/harper-ls/Cargo.toml b/harper-ls/Cargo.toml
index 14488142b..31a5656d5 100644
--- a/harper-ls/Cargo.toml
+++ b/harper-ls/Cargo.toml
@@ -27,6 +27,7 @@ resolve-path = "0.1.0"
open = "5.3.0"
futures = "0.3.31"
serde = { version = "1.0.218", features = ["derive"] }
+indexmap = { version = "2.7.1", features = ["serde"] }
[features]
default = []