Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
feat(core): created cache for PatternLinters
Elijah Pottercbdce836f204
diff --git a/Cargo.lock b/Cargo.lockindex 05526fdbc..24d2ac33d 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -742,6 +742,7 @@ dependencies = ["itertools 0.14.0","lazy_static","levenshtein_automata",+ "lru","ordered-float","paste","pulldown-cmark",@@ -1168,6 +1169,15 @@ version = "0.4.25"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "04cbf5b083de1c7e0222a7a51dbfdba1cbe1c6ab0b15e29fff3f6c077fd9cd9f"+[[package]]+name = "lru"+version = "0.13.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "227748d55f2f0ab4735d87fd623798cb6b664512fe979705f829c9f81c934465"+dependencies = [+ "hashbrown 0.15.2",+]+[[package]]name = "lsp-types"version = "0.94.1"diff --git a/harper-core/Cargo.toml b/harper-core/Cargo.tomlindex 524cbd55e..daa5123f4 100644--- a/harper-core/Cargo.toml+++ b/harper-core/Cargo.toml@@ -26,6 +26,7 @@ unicode-script = "0.5.7"unicode-width = "0.2.0"levenshtein_automata = { version = "0.2.1", features = ["fst_automaton"] }cached = "0.54.0"+lru = "0.13.0"[dev-dependencies]criterion = { version = "0.5.1", default-features = false }diff --git a/harper-core/src/lib.rs b/harper-core/src/lib.rsindex e522dd8f3..6ebee5c72 100644--- a/harper-core/src/lib.rs+++ b/harper-core/src/lib.rs@@ -39,7 +39,7 @@ pub use number::{Number, NumberSuffix};pub use punctuation::{Punctuation, Quote};pub use span::Span;pub use spell::{Dictionary, FstDictionary, MergedDictionary, MutableDictionary};-pub use sync::Lrc;+pub use sync::{LSend, Lrc};pub use title_case::{make_title_case, make_title_case_str};pub use token::Token;pub use token_kind::TokenKind;diff --git a/harper-core/src/linting/lint_group.rs b/harper-core/src/linting/lint_group.rsindex 75db4c9d5..07ad82c73 100644--- a/harper-core/src/linting/lint_group.rs+++ b/harper-core/src/linting/lint_group.rs@@ -7,6 +7,7 @@ use hashbrown::HashMap;use serde::{Deserialize, Serialize};use super::Lint;+use super::PatternLinterCache;use super::an_a::AnA;use super::avoid_curses::AvoidCurses;use super::back_in_the_day::BackInTheDay;@@ -61,6 +62,7 @@ use super::{CurrencyPlacement, Linter, NoOxfordComma, OxfordComma};use crate::Document;use crate::linting::{closed_compounds, phrase_corrections};use crate::{Dictionary, MutableDictionary};+use std::num::NonZero;#[derive(Debug, Serialize, Deserialize, Default, Clone)]#[serde(transparent)]@@ -201,6 +203,20 @@ impl LintGroup {};}+ macro_rules! insert_pattern_rule {+ ($rule:ident, $default_config:expr) => {+ out.add(+ stringify!($rule),+ Box::new(PatternLinterCache::new(+ $rule::default(),+ NonZero::new(1000).unwrap(),+ )),+ );+ out.config+ .set_rule_enabled(stringify!($rule), $default_config);+ };+ }+out.merge_from(&mut phrase_corrections::lint_group());out.merge_from(&mut proper_noun_capitalization_linters::lint_group(dictionary.clone(),@@ -208,21 +224,21 @@ impl LintGroup {out.merge_from(&mut closed_compounds::lint_group());// Add all the more complex rules to the group.- insert_struct_rule!(BackInTheDay, true);+ insert_pattern_rule!(BackInTheDay, true);insert_struct_rule!(WordPressDotcom, true);insert_struct_rule!(OutOfDate, true);insert_struct_rule!(ThenThan, true);insert_struct_rule!(PiqueInterest, true);- insert_struct_rule!(WasAloud, true);- insert_struct_rule!(HyphenateNumberDay, true);- insert_struct_rule!(LeftRightHand, true);+ insert_pattern_rule!(WasAloud, true);+ insert_pattern_rule!(HyphenateNumberDay, true);+ insert_pattern_rule!(LeftRightHand, true);insert_struct_rule!(HopHope, true);- insert_struct_rule!(Hereby, true);- insert_struct_rule!(Likewise, true);+ insert_pattern_rule!(Hereby, true);+ insert_pattern_rule!(Likewise, true);insert_struct_rule!(CompoundNouns, true);- insert_struct_rule!(Nobody, true);- insert_struct_rule!(Whereas, true);- insert_struct_rule!(PossessiveYour, true);+ insert_pattern_rule!(Nobody, true);+ insert_pattern_rule!(Whereas, true);+ insert_pattern_rule!(PossessiveYour, true);insert_struct_rule!(SpelledNumbers, false);insert_struct_rule!(AnA, true);insert_struct_rule!(SentenceCapitalization, true);@@ -240,25 +256,25 @@ impl LintGroup {insert_struct_rule!(TerminatingConjunctions, true);insert_struct_rule!(EllipsisLength, true);insert_struct_rule!(DotInitialisms, true);- insert_struct_rule!(BoringWords, false);- insert_struct_rule!(UseGenitive, false);- insert_struct_rule!(ThatWhich, true);+ insert_pattern_rule!(BoringWords, false);+ insert_pattern_rule!(UseGenitive, false);+ insert_pattern_rule!(ThatWhich, true);insert_struct_rule!(CapitalizePersonalPronouns, true);insert_struct_rule!(MergeWords, true);- insert_struct_rule!(PluralConjugate, false);+ insert_pattern_rule!(PluralConjugate, false);insert_struct_rule!(OxfordComma, true);insert_struct_rule!(NoOxfordComma, false);insert_struct_rule!(PronounContraction, true);insert_struct_rule!(CurrencyPlacement, true);- insert_struct_rule!(SomewhatSomething, true);+ insert_pattern_rule!(SomewhatSomething, true);insert_struct_rule!(LetsConfusion, true);- insert_struct_rule!(DespiteOf, true);- insert_struct_rule!(ChockFull, true);- insert_struct_rule!(Confident, true);- insert_struct_rule!(Oxymorons, true);- insert_struct_rule!(Hedging, true);- insert_struct_rule!(ExpandTimeShorthands, true);- insert_struct_rule!(ModalOf, true);+ insert_pattern_rule!(DespiteOf, true);+ insert_pattern_rule!(ChockFull, true);+ insert_pattern_rule!(Confident, true);+ insert_pattern_rule!(Oxymorons, true);+ insert_pattern_rule!(Hedging, true);+ insert_pattern_rule!(ExpandTimeShorthands, true);+ insert_pattern_rule!(ModalOf, true);out.add("SpellCheck", Box::new(SpellCheck::new(dictionary)));out.config.set_rule_enabled("SpellCheck", true);diff --git a/harper-core/src/linting/mod.rs b/harper-core/src/linting/mod.rsindex 89895c846..ff634bc40 100644--- a/harper-core/src/linting/mod.rs+++ b/harper-core/src/linting/mod.rs@@ -103,7 +103,7 @@ pub use number_suffix_capitalization::NumberSuffixCapitalization;pub use out_of_date::OutOfDate;pub use oxford_comma::OxfordComma;pub use oxymorons::Oxymorons;-pub use pattern_linter::PatternLinter;+pub use pattern_linter::{PatternLinter, PatternLinterCache};pub use pique_interest::PiqueInterest;pub use plural_conjugate::PluralConjugate;pub use possessive_your::PossessiveYour;diff --git a/harper-core/src/linting/pattern_linter.rs b/harper-core/src/linting/pattern_linter.rsindex 0113df548..844db6aec 100644--- a/harper-core/src/linting/pattern_linter.rs+++ b/harper-core/src/linting/pattern_linter.rs@@ -1,8 +1,12 @@+use std::num::NonZeroUsize;++use lru::LruCache;++use crate::{CharString, Document, Token, TokenStringExt, patterns::Pattern};+use super::{Lint, Linter};-use crate::patterns::Pattern;-use crate::{Token, TokenStringExt};-/// A trait that searches for [`Pattern`]s in [`Document`](crate::Document)s.+/// A trait that searches for [`Pattern`]s in [`Document`]s.////// Makes use of [`TokenStringExt::iter_chunks`] to avoid matching across sentence or clause/// boundaries.@@ -10,7 +14,7 @@ use crate::{Token, TokenStringExt};pub trait PatternLinter {/// A simple getter for the pattern to be searched for.fn pattern(&self) -> &dyn Pattern;- /// If any portions of a [`Document`](crate::Document) match [`Self::pattern`], they are passed through [`PatternLinter::match_to_lint`] to be+ /// If any portions of a [`Document`] match [`Self::pattern`], they are passed through [`PatternLinter::match_to_lint`] to be/// transformed into a [`Lint`] for editor consumption.////// This function may return `None` to elect _not_ to produce a lint.@@ -20,7 +24,7 @@ pub trait PatternLinter {fn description(&self) -> &str;}-/// A trait that searches for [`Pattern`]s in [`Document`](crate::Document)s.+/// A trait that searches for [`Pattern`]s in [`Document`]s.////// Makes use of [`TokenStringExt::iter_chunks`] to avoid matching across sentence or clause/// boundaries.@@ -28,7 +32,7 @@ pub trait PatternLinter {pub trait PatternLinter: Send + Sync {/// A simple getter for the pattern to be searched for.fn pattern(&self) -> &dyn Pattern;- /// If any portions of a [`Document`](crate::Document) match [`Self::pattern`], they are passed through [`PatternLinter::match_to_lint`] to be+ /// If any portions of a [`Document`] match [`Self::pattern`], they are passed through [`PatternLinter::match_to_lint`] to be/// transformed into a [`Lint`] for editor consumption.////// This function may return `None` to elect _not_ to produce a lint.@@ -42,36 +46,105 @@ impl<L> Linter for LwhereL: PatternLinter,{- fn lint(&mut self, document: &crate::Document) -> Vec<Lint> {+ fn lint(&mut self, document: &Document) -> Vec<Lint> {let mut lints = Vec::new();let source = document.get_source();for chunk in document.iter_chunks() {- let mut tok_cursor = 0;+ lints.extend(run_on_chunk(self, chunk, source));+ }- loop {- if tok_cursor >= chunk.len() {- break;- }+ lints+ }- let match_len = self.pattern().matches(&chunk[tok_cursor..], source);+ fn description(&self) -> &str {+ self.description()+ }+}- if match_len != 0 {- let lint =- self.match_to_lint(&chunk[tok_cursor..tok_cursor + match_len], source);+type ChunkCache = LruCache<CharString, Vec<Lint>>;- lints.extend(lint);- tok_cursor += match_len;- } else {- tok_cursor += 1;- }- }+/// A cache that wraps around a [`PatternLinter`], caching+/// results by chunk.+pub struct PatternLinterCache<P: PatternLinter> {+ cache: ChunkCache,+ inner: P,+}++impl<P: PatternLinter> PatternLinterCache<P> {+ /// Add a cache to a given [`PatternLinter`] with a given cache size.+ /// About a 1000 rows is recommended.+ pub fn new(inner: P, cache_size: NonZeroUsize) -> Self {+ Self {+ cache: ChunkCache::new(cache_size),+ inner,+ }+ }+}++impl<P: PatternLinter> Linter for PatternLinterCache<P> {+ fn lint(&mut self, document: &Document) -> Vec<Lint> {+ let mut lints = Vec::new();+ let source = document.get_source();++ for chunk in document.iter_chunks() {+ lints.extend(run_on_chunk_cached(+ &self.inner,+ chunk,+ source,+ &mut self.cache,+ ));}lints}fn description(&self) -> &str {- self.description()+ self.inner.description()+ }+}++fn run_on_chunk(linter: &impl PatternLinter, chunk: &[Token], source: &[char]) -> Vec<Lint> {+ let mut lints = Vec::new();+ let mut tok_cursor = 0;++ loop {+ if tok_cursor >= chunk.len() {+ break;+ }++ let match_len = linter.pattern().matches(&chunk[tok_cursor..], source);++ if match_len != 0 {+ let lint = linter.match_to_lint(&chunk[tok_cursor..tok_cursor + match_len], source);++ lints.extend(lint);+ tok_cursor += match_len;+ } else {+ tok_cursor += 1;+ }+ }++ lints+}++fn run_on_chunk_cached(+ linter: &impl PatternLinter,+ chunk: &[Token],+ source: &[char],+ cache: &mut ChunkCache,+) -> Vec<Lint> {+ let Some(chunk_span) = chunk.span() else {+ return Vec::new();+ };++ let key = chunk_span.get_content(source);++ if let Some(hit) = cache.get(key) {+ hit.clone()+ } else {+ let lints = run_on_chunk(linter, chunk, source);+ cache.put(key.into(), lints.clone());+ lints}}diff --git a/harper-core/src/linting/phrase_corrections.rs b/harper-core/src/linting/phrase_corrections.rsindex 6900e48bf..6398946ac 100644--- a/harper-core/src/linting/phrase_corrections.rs+++ b/harper-core/src/linting/phrase_corrections.rs@@ -1,4 +1,5 @@-use super::{LintGroup, MapPhraseLinter};+use super::{LintGroup, MapPhraseLinter, PatternLinterCache};+use std::num::NonZero;/// Produce a [`LintGroup`] that looks for errors in common phrases./// Comes pre-configured with the recommended default settings.@@ -12,11 +13,14 @@ pub fn lint_group() -> LintGroup {$($group.add($name,- Box::new(MapPhraseLinter::new_exact_phrases(- $input,- $corrections,- $hint,- $description,+ Box::new(PatternLinterCache::new(+ MapPhraseLinter::new_exact_phrases(+ $input,+ $corrections,+ $hint,+ $description+ ),+ NonZero::new(1000).unwrap())),);)+diff --git a/harper-core/src/linting/proper_noun_capitalization_linters.rs b/harper-core/src/linting/proper_noun_capitalization_linters.rsindex cbe334c5e..eeee428f7 100644--- a/harper-core/src/linting/proper_noun_capitalization_linters.rs+++ b/harper-core/src/linting/proper_noun_capitalization_linters.rs@@ -1,12 +1,14 @@use hashbrown::HashMap;use serde::{Deserialize, Serialize};+use super::pattern_linter::PatternLinterCache;use super::{Lint, LintKind, Suggestion};use super::{LintGroup, PatternLinter};use crate::parsers::PlainEnglish;use crate::patterns::{ExactPhrase, Pattern, PatternMap};use crate::{Dictionary, Document};use crate::{Token, TokenStringExt};+use std::num::NonZero;use std::sync::Arc;/// A linter that corrects the capitalization of multi-word proper nouns.@@ -118,10 +120,13 @@ fn lint_group_from_json(json: &str, dictionary: Arc<impl Dictionary + 'static>)for (key, rule) in rules.into_iter() {group.add(key,- Box::new(ProperNounCapitalizationLinter::new_strs(- rule.canonical,- rule.description,- dictionary.clone(),+ Box::new(PatternLinterCache::new(+ ProperNounCapitalizationLinter::new_strs(+ rule.canonical,+ rule.description,+ dictionary.clone(),+ ),+ NonZero::new(1000).unwrap(),)),);}diff --git a/harper-core/src/patterns/pattern_map.rs b/harper-core/src/patterns/pattern_map.rsindex 89d3fd0bf..98ec9c13b 100644--- a/harper-core/src/patterns/pattern_map.rs+++ b/harper-core/src/patterns/pattern_map.rs@@ -1,3 +1,4 @@+use crate::LSend;use crate::Token;use super::Pattern;@@ -13,14 +14,14 @@ use super::Pattern;/// the corresponding key.pub struct PatternMap<T>where- T: Send + Sync,+ T: LSend,{rows: Vec<Row<T>>,}struct Row<T>where- T: Send + Sync,+ T: LSend,{pub key: Box<dyn Pattern>,pub element: T,@@ -28,7 +29,7 @@ whereimpl<T> Default for PatternMap<T>where- T: Send + Sync,+ T: LSend,{fn default() -> Self {Self {@@ -39,7 +40,7 @@ whereimpl<T> PatternMap<T>where- T: Send + Sync,+ T: LSend,{pub fn insert(&mut self, pattern: impl Pattern + 'static, value: T) {self.rows.push(Row {@@ -64,7 +65,7 @@ whereimpl<T> Pattern for PatternMap<T>where- T: Send + Sync,+ T: LSend,{fn matches(&self, tokens: &[Token], source: &[char]) -> usize {for row in &self.rows {diff --git a/harper-core/src/sync.rs b/harper-core/src/sync.rsindex 8c78a269a..23b77be05 100644--- a/harper-core/src/sync.rs+++ b/harper-core/src/sync.rs@@ -2,3 +2,15 @@pub use std::rc::Rc as Lrc;#[cfg(feature = "concurrent")]pub use std::sync::Arc as Lrc;++#[cfg(not(feature = "concurrent"))]+pub trait LSend {}++#[cfg(not(feature = "concurrent"))]+impl<T> LSend for T {}++#[cfg(feature = "concurrent")]+pub trait LSend: Send + Sync {}++#[cfg(feature = "concurrent")]+impl<T: Send + Sync> LSend for T {}