Skip to content

Offline, privacy-first grammar checker. Fast, open-source, Rust-powered

feat(core): created cache for PatternLinters

Elijah Pottercbdce836f204
diff --git a/Cargo.lock b/Cargo.lock
index 05526fdbc..24d2ac33d 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -742,6 +742,7 @@ dependencies = [
"itertools 0.14.0",
"lazy_static",
"levenshtein_automata",
+ "lru",
"ordered-float",
"paste",
"pulldown-cmark",
@@ -1168,6 +1169,15 @@ version = "0.4.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "04cbf5b083de1c7e0222a7a51dbfdba1cbe1c6ab0b15e29fff3f6c077fd9cd9f"
+[[package]]
+name = "lru"
+version = "0.13.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "227748d55f2f0ab4735d87fd623798cb6b664512fe979705f829c9f81c934465"
+dependencies = [
+ "hashbrown 0.15.2",
+]
+
[[package]]
name = "lsp-types"
version = "0.94.1"
diff --git a/harper-core/Cargo.toml b/harper-core/Cargo.toml
index 524cbd55e..daa5123f4 100644
--- a/harper-core/Cargo.toml
+++ b/harper-core/Cargo.toml
@@ -26,6 +26,7 @@ unicode-script = "0.5.7"
unicode-width = "0.2.0"
levenshtein_automata = { version = "0.2.1", features = ["fst_automaton"] }
cached = "0.54.0"
+lru = "0.13.0"
[dev-dependencies]
criterion = { version = "0.5.1", default-features = false }
diff --git a/harper-core/src/lib.rs b/harper-core/src/lib.rs
index e522dd8f3..6ebee5c72 100644
--- a/harper-core/src/lib.rs
+++ b/harper-core/src/lib.rs
@@ -39,7 +39,7 @@ pub use number::{Number, NumberSuffix};
pub use punctuation::{Punctuation, Quote};
pub use span::Span;
pub use spell::{Dictionary, FstDictionary, MergedDictionary, MutableDictionary};
-pub use sync::Lrc;
+pub use sync::{LSend, Lrc};
pub use title_case::{make_title_case, make_title_case_str};
pub use token::Token;
pub use token_kind::TokenKind;
diff --git a/harper-core/src/linting/lint_group.rs b/harper-core/src/linting/lint_group.rs
index 75db4c9d5..07ad82c73 100644
--- a/harper-core/src/linting/lint_group.rs
+++ b/harper-core/src/linting/lint_group.rs
@@ -7,6 +7,7 @@ use hashbrown::HashMap;
use serde::{Deserialize, Serialize};
use super::Lint;
+use super::PatternLinterCache;
use super::an_a::AnA;
use super::avoid_curses::AvoidCurses;
use super::back_in_the_day::BackInTheDay;
@@ -61,6 +62,7 @@ use super::{CurrencyPlacement, Linter, NoOxfordComma, OxfordComma};
use crate::Document;
use crate::linting::{closed_compounds, phrase_corrections};
use crate::{Dictionary, MutableDictionary};
+use std::num::NonZero;
#[derive(Debug, Serialize, Deserialize, Default, Clone)]
#[serde(transparent)]
@@ -201,6 +203,20 @@ impl LintGroup {
};
}
+ macro_rules! insert_pattern_rule {
+ ($rule:ident, $default_config:expr) => {
+ out.add(
+ stringify!($rule),
+ Box::new(PatternLinterCache::new(
+ $rule::default(),
+ NonZero::new(1000).unwrap(),
+ )),
+ );
+ out.config
+ .set_rule_enabled(stringify!($rule), $default_config);
+ };
+ }
+
out.merge_from(&mut phrase_corrections::lint_group());
out.merge_from(&mut proper_noun_capitalization_linters::lint_group(
dictionary.clone(),
@@ -208,21 +224,21 @@ impl LintGroup {
out.merge_from(&mut closed_compounds::lint_group());
// Add all the more complex rules to the group.
- insert_struct_rule!(BackInTheDay, true);
+ insert_pattern_rule!(BackInTheDay, true);
insert_struct_rule!(WordPressDotcom, true);
insert_struct_rule!(OutOfDate, true);
insert_struct_rule!(ThenThan, true);
insert_struct_rule!(PiqueInterest, true);
- insert_struct_rule!(WasAloud, true);
- insert_struct_rule!(HyphenateNumberDay, true);
- insert_struct_rule!(LeftRightHand, true);
+ insert_pattern_rule!(WasAloud, true);
+ insert_pattern_rule!(HyphenateNumberDay, true);
+ insert_pattern_rule!(LeftRightHand, true);
insert_struct_rule!(HopHope, true);
- insert_struct_rule!(Hereby, true);
- insert_struct_rule!(Likewise, true);
+ insert_pattern_rule!(Hereby, true);
+ insert_pattern_rule!(Likewise, true);
insert_struct_rule!(CompoundNouns, true);
- insert_struct_rule!(Nobody, true);
- insert_struct_rule!(Whereas, true);
- insert_struct_rule!(PossessiveYour, true);
+ insert_pattern_rule!(Nobody, true);
+ insert_pattern_rule!(Whereas, true);
+ insert_pattern_rule!(PossessiveYour, true);
insert_struct_rule!(SpelledNumbers, false);
insert_struct_rule!(AnA, true);
insert_struct_rule!(SentenceCapitalization, true);
@@ -240,25 +256,25 @@ impl LintGroup {
insert_struct_rule!(TerminatingConjunctions, true);
insert_struct_rule!(EllipsisLength, true);
insert_struct_rule!(DotInitialisms, true);
- insert_struct_rule!(BoringWords, false);
- insert_struct_rule!(UseGenitive, false);
- insert_struct_rule!(ThatWhich, true);
+ insert_pattern_rule!(BoringWords, false);
+ insert_pattern_rule!(UseGenitive, false);
+ insert_pattern_rule!(ThatWhich, true);
insert_struct_rule!(CapitalizePersonalPronouns, true);
insert_struct_rule!(MergeWords, true);
- insert_struct_rule!(PluralConjugate, false);
+ insert_pattern_rule!(PluralConjugate, false);
insert_struct_rule!(OxfordComma, true);
insert_struct_rule!(NoOxfordComma, false);
insert_struct_rule!(PronounContraction, true);
insert_struct_rule!(CurrencyPlacement, true);
- insert_struct_rule!(SomewhatSomething, true);
+ insert_pattern_rule!(SomewhatSomething, true);
insert_struct_rule!(LetsConfusion, true);
- insert_struct_rule!(DespiteOf, true);
- insert_struct_rule!(ChockFull, true);
- insert_struct_rule!(Confident, true);
- insert_struct_rule!(Oxymorons, true);
- insert_struct_rule!(Hedging, true);
- insert_struct_rule!(ExpandTimeShorthands, true);
- insert_struct_rule!(ModalOf, true);
+ insert_pattern_rule!(DespiteOf, true);
+ insert_pattern_rule!(ChockFull, true);
+ insert_pattern_rule!(Confident, true);
+ insert_pattern_rule!(Oxymorons, true);
+ insert_pattern_rule!(Hedging, true);
+ insert_pattern_rule!(ExpandTimeShorthands, true);
+ insert_pattern_rule!(ModalOf, true);
out.add("SpellCheck", Box::new(SpellCheck::new(dictionary)));
out.config.set_rule_enabled("SpellCheck", true);
diff --git a/harper-core/src/linting/mod.rs b/harper-core/src/linting/mod.rs
index 89895c846..ff634bc40 100644
--- a/harper-core/src/linting/mod.rs
+++ b/harper-core/src/linting/mod.rs
@@ -103,7 +103,7 @@ pub use number_suffix_capitalization::NumberSuffixCapitalization;
pub use out_of_date::OutOfDate;
pub use oxford_comma::OxfordComma;
pub use oxymorons::Oxymorons;
-pub use pattern_linter::PatternLinter;
+pub use pattern_linter::{PatternLinter, PatternLinterCache};
pub use pique_interest::PiqueInterest;
pub use plural_conjugate::PluralConjugate;
pub use possessive_your::PossessiveYour;
diff --git a/harper-core/src/linting/pattern_linter.rs b/harper-core/src/linting/pattern_linter.rs
index 0113df548..844db6aec 100644
--- a/harper-core/src/linting/pattern_linter.rs
+++ b/harper-core/src/linting/pattern_linter.rs
@@ -1,8 +1,12 @@
+use std::num::NonZeroUsize;
+
+use lru::LruCache;
+
+use crate::{CharString, Document, Token, TokenStringExt, patterns::Pattern};
+
use super::{Lint, Linter};
-use crate::patterns::Pattern;
-use crate::{Token, TokenStringExt};
-/// A trait that searches for [`Pattern`]s in [`Document`](crate::Document)s.
+/// A trait that searches for [`Pattern`]s in [`Document`]s.
///
/// Makes use of [`TokenStringExt::iter_chunks`] to avoid matching across sentence or clause
/// boundaries.
@@ -10,7 +14,7 @@ use crate::{Token, TokenStringExt};
pub trait PatternLinter {
/// A simple getter for the pattern to be searched for.
fn pattern(&self) -> &dyn Pattern;
- /// If any portions of a [`Document`](crate::Document) match [`Self::pattern`], they are passed through [`PatternLinter::match_to_lint`] to be
+ /// If any portions of a [`Document`] match [`Self::pattern`], they are passed through [`PatternLinter::match_to_lint`] to be
/// transformed into a [`Lint`] for editor consumption.
///
/// This function may return `None` to elect _not_ to produce a lint.
@@ -20,7 +24,7 @@ pub trait PatternLinter {
fn description(&self) -> &str;
}
-/// A trait that searches for [`Pattern`]s in [`Document`](crate::Document)s.
+/// A trait that searches for [`Pattern`]s in [`Document`]s.
///
/// Makes use of [`TokenStringExt::iter_chunks`] to avoid matching across sentence or clause
/// boundaries.
@@ -28,7 +32,7 @@ pub trait PatternLinter {
pub trait PatternLinter: Send + Sync {
/// A simple getter for the pattern to be searched for.
fn pattern(&self) -> &dyn Pattern;
- /// If any portions of a [`Document`](crate::Document) match [`Self::pattern`], they are passed through [`PatternLinter::match_to_lint`] to be
+ /// If any portions of a [`Document`] match [`Self::pattern`], they are passed through [`PatternLinter::match_to_lint`] to be
/// transformed into a [`Lint`] for editor consumption.
///
/// This function may return `None` to elect _not_ to produce a lint.
@@ -42,36 +46,105 @@ impl<L> Linter for L
where
L: PatternLinter,
{
- fn lint(&mut self, document: &crate::Document) -> Vec<Lint> {
+ fn lint(&mut self, document: &Document) -> Vec<Lint> {
let mut lints = Vec::new();
let source = document.get_source();
for chunk in document.iter_chunks() {
- let mut tok_cursor = 0;
+ lints.extend(run_on_chunk(self, chunk, source));
+ }
- loop {
- if tok_cursor >= chunk.len() {
- break;
- }
+ lints
+ }
- let match_len = self.pattern().matches(&chunk[tok_cursor..], source);
+ fn description(&self) -> &str {
+ self.description()
+ }
+}
- if match_len != 0 {
- let lint =
- self.match_to_lint(&chunk[tok_cursor..tok_cursor + match_len], source);
+type ChunkCache = LruCache<CharString, Vec<Lint>>;
- lints.extend(lint);
- tok_cursor += match_len;
- } else {
- tok_cursor += 1;
- }
- }
+/// A cache that wraps around a [`PatternLinter`], caching
+/// results by chunk.
+pub struct PatternLinterCache<P: PatternLinter> {
+ cache: ChunkCache,
+ inner: P,
+}
+
+impl<P: PatternLinter> PatternLinterCache<P> {
+ /// Add a cache to a given [`PatternLinter`] with a given cache size.
+ /// About a 1000 rows is recommended.
+ pub fn new(inner: P, cache_size: NonZeroUsize) -> Self {
+ Self {
+ cache: ChunkCache::new(cache_size),
+ inner,
+ }
+ }
+}
+
+impl<P: PatternLinter> Linter for PatternLinterCache<P> {
+ fn lint(&mut self, document: &Document) -> Vec<Lint> {
+ let mut lints = Vec::new();
+ let source = document.get_source();
+
+ for chunk in document.iter_chunks() {
+ lints.extend(run_on_chunk_cached(
+ &self.inner,
+ chunk,
+ source,
+ &mut self.cache,
+ ));
}
lints
}
fn description(&self) -> &str {
- self.description()
+ self.inner.description()
+ }
+}
+
+fn run_on_chunk(linter: &impl PatternLinter, chunk: &[Token], source: &[char]) -> Vec<Lint> {
+ let mut lints = Vec::new();
+ let mut tok_cursor = 0;
+
+ loop {
+ if tok_cursor >= chunk.len() {
+ break;
+ }
+
+ let match_len = linter.pattern().matches(&chunk[tok_cursor..], source);
+
+ if match_len != 0 {
+ let lint = linter.match_to_lint(&chunk[tok_cursor..tok_cursor + match_len], source);
+
+ lints.extend(lint);
+ tok_cursor += match_len;
+ } else {
+ tok_cursor += 1;
+ }
+ }
+
+ lints
+}
+
+fn run_on_chunk_cached(
+ linter: &impl PatternLinter,
+ chunk: &[Token],
+ source: &[char],
+ cache: &mut ChunkCache,
+) -> Vec<Lint> {
+ let Some(chunk_span) = chunk.span() else {
+ return Vec::new();
+ };
+
+ let key = chunk_span.get_content(source);
+
+ if let Some(hit) = cache.get(key) {
+ hit.clone()
+ } else {
+ let lints = run_on_chunk(linter, chunk, source);
+ cache.put(key.into(), lints.clone());
+ lints
}
}
diff --git a/harper-core/src/linting/phrase_corrections.rs b/harper-core/src/linting/phrase_corrections.rs
index 6900e48bf..6398946ac 100644
--- a/harper-core/src/linting/phrase_corrections.rs
+++ b/harper-core/src/linting/phrase_corrections.rs
@@ -1,4 +1,5 @@
-use super::{LintGroup, MapPhraseLinter};
+use super::{LintGroup, MapPhraseLinter, PatternLinterCache};
+use std::num::NonZero;
/// Produce a [`LintGroup`] that looks for errors in common phrases.
/// Comes pre-configured with the recommended default settings.
@@ -12,11 +13,14 @@ pub fn lint_group() -> LintGroup {
$(
$group.add(
$name,
- Box::new(MapPhraseLinter::new_exact_phrases(
- $input,
- $corrections,
- $hint,
- $description,
+ Box::new(PatternLinterCache::new(
+ MapPhraseLinter::new_exact_phrases(
+ $input,
+ $corrections,
+ $hint,
+ $description
+ ),
+ NonZero::new(1000).unwrap()
)),
);
)+
diff --git a/harper-core/src/linting/proper_noun_capitalization_linters.rs b/harper-core/src/linting/proper_noun_capitalization_linters.rs
index cbe334c5e..eeee428f7 100644
--- a/harper-core/src/linting/proper_noun_capitalization_linters.rs
+++ b/harper-core/src/linting/proper_noun_capitalization_linters.rs
@@ -1,12 +1,14 @@
use hashbrown::HashMap;
use serde::{Deserialize, Serialize};
+use super::pattern_linter::PatternLinterCache;
use super::{Lint, LintKind, Suggestion};
use super::{LintGroup, PatternLinter};
use crate::parsers::PlainEnglish;
use crate::patterns::{ExactPhrase, Pattern, PatternMap};
use crate::{Dictionary, Document};
use crate::{Token, TokenStringExt};
+use std::num::NonZero;
use std::sync::Arc;
/// A linter that corrects the capitalization of multi-word proper nouns.
@@ -118,10 +120,13 @@ fn lint_group_from_json(json: &str, dictionary: Arc<impl Dictionary + 'static>)
for (key, rule) in rules.into_iter() {
group.add(
key,
- Box::new(ProperNounCapitalizationLinter::new_strs(
- rule.canonical,
- rule.description,
- dictionary.clone(),
+ Box::new(PatternLinterCache::new(
+ ProperNounCapitalizationLinter::new_strs(
+ rule.canonical,
+ rule.description,
+ dictionary.clone(),
+ ),
+ NonZero::new(1000).unwrap(),
)),
);
}
diff --git a/harper-core/src/patterns/pattern_map.rs b/harper-core/src/patterns/pattern_map.rs
index 89d3fd0bf..98ec9c13b 100644
--- a/harper-core/src/patterns/pattern_map.rs
+++ b/harper-core/src/patterns/pattern_map.rs
@@ -1,3 +1,4 @@
+use crate::LSend;
use crate::Token;
use super::Pattern;
@@ -13,14 +14,14 @@ use super::Pattern;
/// the corresponding key.
pub struct PatternMap<T>
where
- T: Send + Sync,
+ T: LSend,
{
rows: Vec<Row<T>>,
}
struct Row<T>
where
- T: Send + Sync,
+ T: LSend,
{
pub key: Box<dyn Pattern>,
pub element: T,
@@ -28,7 +29,7 @@ where
impl<T> Default for PatternMap<T>
where
- T: Send + Sync,
+ T: LSend,
{
fn default() -> Self {
Self {
@@ -39,7 +40,7 @@ where
impl<T> PatternMap<T>
where
- T: Send + Sync,
+ T: LSend,
{
pub fn insert(&mut self, pattern: impl Pattern + 'static, value: T) {
self.rows.push(Row {
@@ -64,7 +65,7 @@ where
impl<T> Pattern for PatternMap<T>
where
- T: Send + Sync,
+ T: LSend,
{
fn matches(&self, tokens: &[Token], source: &[char]) -> usize {
for row in &self.rows {
diff --git a/harper-core/src/sync.rs b/harper-core/src/sync.rs
index 8c78a269a..23b77be05 100644
--- a/harper-core/src/sync.rs
+++ b/harper-core/src/sync.rs
@@ -2,3 +2,15 @@
pub use std::rc::Rc as Lrc;
#[cfg(feature = "concurrent")]
pub use std::sync::Arc as Lrc;
+
+#[cfg(not(feature = "concurrent"))]
+pub trait LSend {}
+
+#[cfg(not(feature = "concurrent"))]
+impl<T> LSend for T {}
+
+#[cfg(feature = "concurrent")]
+pub trait LSend: Send + Sync {}
+
+#[cfg(feature = "concurrent")]
+impl<T: Send + Sync> LSend for T {}