Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
fix: improved perf with ref-counted pointers
Elijah Potterb73060845ed0
diff --git a/Cargo.lock b/Cargo.lockindex 3d5d775ab..f357a6d7a 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -530,7 +530,7 @@ dependencies = ["divan","hashbrown","is-macro",- "itertools 0.12.1",+ "itertools 0.13.0","once_cell","paste","pulldown-cmark",diff --git a/harper-core/Cargo.toml b/harper-core/Cargo.tomlindex a511dff4c..5c8357b2e 100644--- a/harper-core/Cargo.toml+++ b/harper-core/Cargo.toml@@ -11,7 +11,7 @@ repository = "https://github.com/elijah-potter/harper"blanket = "0.4.0"hashbrown = { version = "0.14.3", features = ["serde"] }is-macro = "0.3.0"-itertools = "0.12.1"+itertools = "0.13.0"once_cell = "1.19.0"paste = "1.0.14"pulldown-cmark = "0.11.0"@@ -28,3 +28,7 @@ rand = "0.8.5"[[bench]]name = "parse_demo"harness = false++[features]+default = []+concurrent = []diff --git a/harper-core/src/document.rs b/harper-core/src/document.rsindex 998f0bb68..ef732befd 100644--- a/harper-core/src/document.rs+++ b/harper-core/src/document.rs@@ -1,3 +1,4 @@+use crate::Lrc;use std::cmp::Ordering;use std::collections::VecDeque;use std::fmt::Display;@@ -13,9 +14,9 @@ use crate::token::NumberSuffix;use crate::{FatToken, Token, TokenKind, TokenStringExt};pub struct Document {- source: Vec<char>,+ source: Lrc<Vec<char>>,tokens: Vec<Token>,- parser: Box<dyn Parser>+ parser: Box<dyn Parser>,}impl Default for Document {@@ -30,14 +31,14 @@ impl Document {pub fn new(text: &str, parser: Box<dyn Parser>) -> Self {let source: Vec<_> = text.chars().collect();- Self::new_from_vec(source, parser)+ Self::new_from_vec(Lrc::new(source), parser)}- pub fn new_from_vec(source: Vec<char>, parser: Box<dyn Parser>) -> Self {+ pub fn new_from_vec(source: Lrc<Vec<char>>, parser: Box<dyn Parser>) -> Self {let mut doc = Self {source,tokens: Vec::new(),- parser+ parser,};doc.parse();@@ -102,7 +103,7 @@ impl Document {&old[indices.last().map(|v| v + stretch_len)- .unwrap_or(indices.len())..]+ .unwrap_or(indices.len())..],);}@@ -242,7 +243,7 @@ impl Document {pub fn get_full_string(&self) -> String {self.get_span_content_str(Span {start: 0,- end: self.source.len()+ end: self.source.len(),})}@@ -251,26 +252,28 @@ impl Document {}pub fn apply_suggestion(&mut self, suggestion: &Suggestion, span: Span) {+ let source = Lrc::make_mut(&mut self.source);+match suggestion {Suggestion::ReplaceWith(chars) => {// Avoid allocation if possibleif chars.len() == span.len() {for (index, c) in chars.iter().enumerate() {- self.source[index + span.start] = *c+ source[index + span.start] = *c}} else {- let popped = self.source.split_off(span.start);+ let popped = source.split_off(span.start);- self.source.extend(chars);- self.source.extend(popped.into_iter().skip(span.len()));+ source.extend(chars);+ source.extend(popped.into_iter().skip(span.len()));}}Suggestion::Remove => {- for i in span.end..self.source.len() {- self.source[i - span.len()] = self.source[i];+ for i in span.end..source.len() {+ source[i - span.len()] = source[i];}- self.source.truncate(self.source.len() - span.len());+ source.truncate(source.len() - span.len());}}@@ -500,7 +503,7 @@ fn is_chunk_terminator(token: &TokenKind) -> bool {match token {TokenKind::Punctuation(punct) => [Punctuation::Comma].contains(punct),- _ => false+ _ => false,}}@@ -509,11 +512,11 @@ fn is_sentence_terminator(token: &TokenKind) -> bool {TokenKind::Punctuation(punct) => [Punctuation::Period,Punctuation::Bang,- Punctuation::Question+ Punctuation::Question,].contains(punct),TokenKind::Newline(count) => *count >= 2,- _ => false+ _ => false,}}@@ -634,7 +637,7 @@ mod tests {assert_token_count("This is the 3rd test", 9);assert_token_count("It works even with weird capitalization like this: 600nD",- 18+ 18,);}diff --git a/harper-core/src/lib.rs b/harper-core/src/lib.rsindex e646e8d73..61ed492c3 100644--- a/harper-core/src/lib.rs+++ b/harper-core/src/lib.rs@@ -7,12 +7,14 @@ mod document;mod language_detection;mod lexing;mod linting;+mod sync;pub mod parsers;mod punctuation;mod span;mod spell;mod token;+pub use sync::Lrc;pub use char_string::{CharString, CharStringExt};pub use document::Document;pub use linting::{Lint, LintGroup, LintGroupConfig, LintKind, Linter, Suggestion};diff --git a/harper-core/src/linting/spell_check.rs b/harper-core/src/linting/spell_check.rsindex 4accfd0bf..f19d99bbc 100644--- a/harper-core/src/linting/spell_check.rs+++ b/harper-core/src/linting/spell_check.rs@@ -1,31 +1,32 @@use hashbrown::HashMap;+use smallvec::ToSmallVec;use super::lint::Suggestion;use super::{Lint, LintKind, Linter};use crate::document::Document;use crate::spell::suggest_correct_spelling;-use crate::{Dictionary, TokenStringExt};+use crate::{CharString, Dictionary, TokenStringExt};pub struct SpellCheck<T>where- T: Dictionary+ T: Dictionary,{dictionary: T,- word_cache: HashMap<Vec<char>, Vec<Vec<char>>>+ word_cache: HashMap<CharString, Vec<CharString>>,}impl<T: Dictionary> SpellCheck<T> {pub fn new(dictionary: T) -> Self {Self {dictionary,- word_cache: HashMap::new()+ word_cache: HashMap::new(),}}}impl<T: Dictionary> SpellCheck<T> {- fn cached_suggest_correct_spelling(&mut self, word: &[char]) -> Vec<Vec<char>> {- let word = word.to_vec();+ fn cached_suggest_correct_spelling(&mut self, word: &[char]) -> Vec<CharString> {+ let word = word.to_smallvec();self.word_cache.entry(word.clone())@@ -37,7 +38,7 @@ impl<T: Dictionary> SpellCheck<T> {while suggestions.is_empty() && dist < 5 {suggestions = suggest_correct_spelling(&word, 100, dist, &self.dictionary).into_iter()- .map(|v| v.to_vec())+ .map(|v| v.to_smallvec()).collect();dist += 1;@@ -86,7 +87,7 @@ impl<T: Dictionary> Linter for SpellCheck<T> {"Did you mean to spell “{}” this way?",document.get_span_content_str(word.span)),- priority: 63+ priority: 63,})}diff --git a/harper-core/src/sync.rs b/harper-core/src/sync.rsnew file mode 100644index 000000000..30a24bc42--- /dev/null+++ b/harper-core/src/sync.rs@@ -0,0 +1,5 @@+#[cfg(feature = "concurrent")]+pub use std::sync::Arc as Lrc;++#[cfg(not(feature = "concurrent"))]+pub use std::rc::Rc as Lrc;diff --git a/harper-ls/Cargo.toml b/harper-ls/Cargo.tomlindex c223d754b..b87d4f4d9 100644--- a/harper-ls/Cargo.toml+++ b/harper-ls/Cargo.toml@@ -8,7 +8,7 @@ readme = "README.md"repository = "https://github.com/elijah-potter/harper"[dependencies]-harper-core = { path = "../harper-core", version = "0.8.0" }+harper-core = { path = "../harper-core", version = "0.8.0", features = ["concurrent"] }harper-comments = { path = "../harper-comments", version = "0.8.0" }tower-lsp = "0.20.0"tokio = { version = "1.36.0", features = ["fs", "rt", "rt-multi-thread", "macros", "io-std", "io-util", "net"] }diff --git a/harper-ls/src/backend.rs b/harper-ls/src/backend.rsindex bc16cf53d..cd57a2d1c 100644--- a/harper-ls/src/backend.rs+++ b/harper-ls/src/backend.rs@@ -5,45 +5,20 @@ use std::sync::Arc;use harper_comments::TreeSitterParser;use harper_core::parsers::{Markdown, PlainEnglish};use harper_core::{- Dictionary,- Document,- FullDictionary,- LintGroup,- Linter,- MergedDictionary,- Token,- TokenKind+ Dictionary, Document, FullDictionary, LintGroup, Linter, MergedDictionary, Token, TokenKind,};use serde_json::Value;use tokio::sync::{Mutex, RwLock};use tower_lsp::jsonrpc::Result;use tower_lsp::lsp_types::notification::PublishDiagnostics;use tower_lsp::lsp_types::{- CodeActionOrCommand,- CodeActionParams,- CodeActionProviderCapability,- CodeActionResponse,- Command,- Diagnostic,- DidChangeConfigurationParams,- DidChangeTextDocumentParams,- DidCloseTextDocumentParams,- DidOpenTextDocumentParams,- DidSaveTextDocumentParams,- ExecuteCommandOptions,- ExecuteCommandParams,- InitializeParams,- InitializeResult,- InitializedParams,- MessageType,- PublishDiagnosticsParams,- Range,- ServerCapabilities,- TextDocumentSyncCapability,- TextDocumentSyncKind,- TextDocumentSyncOptions,- TextDocumentSyncSaveOptions,- Url+ CodeActionOrCommand, CodeActionParams, CodeActionProviderCapability, CodeActionResponse,+ Command, Diagnostic, DidChangeConfigurationParams, DidChangeTextDocumentParams,+ DidCloseTextDocumentParams, DidOpenTextDocumentParams, DidSaveTextDocumentParams,+ ExecuteCommandOptions, ExecuteCommandParams, InitializeParams, InitializeResult,+ InitializedParams, MessageType, PublishDiagnosticsParams, Range, ServerCapabilities,+ TextDocumentSyncCapability, TextDocumentSyncKind, TextDocumentSyncOptions,+ TextDocumentSyncSaveOptions, Url,};use tower_lsp::{Client, LanguageServer};use tracing::{error, info, instrument};@@ -59,7 +34,7 @@ struct DocumentState {document: Document,ident_dict: Arc<FullDictionary>,linter: LintGroup<MergedDictionary<FullDictionary>>,- language_id: Option<String>+ language_id: Option<String>,}/// Deallocate@@ -67,7 +42,7 @@ pub struct Backend {client: Client,static_dictionary: Arc<FullDictionary>,config: RwLock<Config>,- doc_state: Mutex<HashMap<Url, DocumentState>>+ doc_state: Mutex<HashMap<Url, DocumentState>>,}impl Backend {@@ -78,7 +53,7 @@ impl Backend {client,static_dictionary: dictionary.into(),doc_state: Mutex::new(HashMap::new()),- config: RwLock::new(config)+ config: RwLock::new(config),}}@@ -173,7 +148,7 @@ impl Backend {#[instrument(skip(self))]async fn generate_file_dictionary(&self,- url: &Url+ url: &Url,) -> anyhow::Result<MergedDictionary<FullDictionary>> {let (global_dictionary, file_dictionary) = tokio::join!(self.generate_global_dictionary(),@@ -190,11 +165,11 @@ impl Backend {async fn update_document_from_file(&self,url: &Url,- language_id: Option<&str>+ language_id: Option<&str>,) -> anyhow::Result<()> {let content = match tokio::fs::read_to_string(url.to_file_path()- .map_err(|_| anyhow::format_err!("Could not extract file path."))?+ .map_err(|_| anyhow::format_err!("Could not extract file path."))?,).await{@@ -213,7 +188,7 @@ impl Backend {&self,url: &Url,text: &str,- language_id: Option<&str>+ language_id: Option<&str>,) -> anyhow::Result<()> {let mut doc_lock = self.doc_state.lock().await;let config_lock = self.config.read().await;@@ -225,7 +200,7 @@ impl Backend {let mut doc_state = DocumentState {linter: LintGroup::new(config_lock.lint_config,- self.generate_file_dictionary(url).await?+ self.generate_file_dictionary(url).await?,),language_id: language_id.map(|v| v.to_string())@@ -241,6 +216,7 @@ impl Backend {doc_state.document =if let Some(ts_parser) = TreeSitterParser::new_from_language_id(language_id) {let source: Vec<char> = text.chars().collect();+ let source = Arc::new(source);if let Some(new_dict) = ts_parser.create_ident_dict(source.as_slice()) {let new_dict = Arc::new(new_dict);@@ -275,7 +251,7 @@ impl Backend {async fn generate_code_actions(&self,url: &Url,- range: Range+ range: Range,) -> Result<Vec<CodeActionOrCommand>> {let (config, mut doc_states) = tokio::join!(self.config.read(), self.doc_state.lock());let Some(doc_state) = doc_states.get_mut(url) else {@@ -307,7 +283,7 @@ impl Backend {actions.push(CodeActionOrCommand::Command(Command::new("Open URL".to_string(),"HarperOpen".to_string(),- Some(vec![doc_state.document.get_span_content_str(span).into()])+ Some(vec![doc_state.document.get_span_content_str(span).into()]),)))}@@ -327,7 +303,7 @@ impl Backend {lints_to_diagnostics(doc_state.document.get_full_content(),&lints,- config.diagnostic_severity+ config.diagnostic_severity,)}@@ -338,7 +314,7 @@ impl Backend {let result = PublishDiagnosticsParams {uri: url.clone(),diagnostics,- version: None+ version: None,};self.client@@ -368,12 +344,12 @@ impl LanguageServer for Backend {change: Some(TextDocumentSyncKind::FULL),will_save: None,will_save_wait_until: None,- save: Some(TextDocumentSyncSaveOptions::Supported(true))- }+ save: Some(TextDocumentSyncSaveOptions::Supported(true)),+ },)),..Default::default()- }+ },})}@@ -404,7 +380,7 @@ impl LanguageServer for Backend {.update_document(¶ms.text_document.uri,¶ms.text_document.text,- Some(¶ms.text_document.language_id)+ Some(¶ms.text_document.language_id),).await;@@ -493,7 +469,7 @@ impl LanguageServer for Backend {error!("Unable to open URL: {}", err);}},- _ => ()+ _ => (),}Ok(None)@@ -524,7 +500,7 @@ impl LanguageServer for Backend {// Update documents with new configfutures::future::join_all(keys.iter()- .map(|key| self.update_document_from_file(key, None))+ .map(|key| self.update_document_from_file(key, None)),).await;}diff --git a/harper-wasm/Cargo.toml b/harper-wasm/Cargo.tomlindex ae2f04133..eca966b51 100644--- a/harper-wasm/Cargo.toml+++ b/harper-wasm/Cargo.toml@@ -14,5 +14,5 @@ console_error_panic_hook = "0.1.7"tracing = "0.1.40"tracing-wasm = "0.2.1"wasm-bindgen = "0.2.91"-harper-core = { path = "../harper-core" }+harper-core = { path = "../harper-core", version = "0.8.0", features = ["concurrent"] }once_cell = "1.19.0"diff --git a/harper-wasm/src/lib.rs b/harper-wasm/src/lib.rsindex 68ea5f71c..3991c86eb 100644--- a/harper-wasm/src/lib.rs+++ b/harper-wasm/src/lib.rs@@ -1,17 +1,16 @@#![doc = include_str!("../README.md")]-use std::rc::Rc;use std::sync::Mutex;use harper_core::parsers::Markdown;-use harper_core::{remove_overlaps, Document, FullDictionary, LintGroup, Linter};+use harper_core::{remove_overlaps, Document, FullDictionary, LintGroup, Linter, Lrc};use once_cell::sync::Lazy;use wasm_bindgen::prelude::wasm_bindgen;static LINTER: Lazy<Mutex<LintGroup<FullDictionary>>> = Lazy::new(|| {Mutex::new(LintGroup::new(Default::default(),- FullDictionary::create_from_curated()+ FullDictionary::create_from_curated(),))});@@ -38,6 +37,7 @@ pub fn use_spell_check(set: bool) {#[wasm_bindgen]pub fn lint(text: String) -> Vec<Lint> {let source: Vec<_> = text.chars().collect();+ let source = Lrc::new(source);let document = Document::new_from_vec(source.clone(), Box::new(Markdown));@@ -45,11 +45,9 @@ pub fn lint(text: String) -> Vec<Lint> {remove_overlaps(&mut lints);- let text = Rc::new(source);-lints.into_iter()- .map(|l| Lint::new(l, text.clone()))+ .map(|l| Lint::new(l, source.clone())).collect()}@@ -57,7 +55,7 @@ pub fn lint(text: String) -> Vec<Lint> {pub fn apply_suggestion(text: String,span: Span,- suggestion: &Suggestion+ suggestion: &Suggestion,) -> Result<String, String> {let mut document = Document::new_markdown(&text);document.apply_suggestion(&suggestion.inner, span.into());@@ -67,13 +65,13 @@ pub fn apply_suggestion(#[wasm_bindgen]pub struct Suggestion {- inner: harper_core::Suggestion+ inner: harper_core::Suggestion,}#[wasm_bindgen]pub enum SuggestionKind {Replace,- Remove+ Remove,}#[wasm_bindgen]@@ -88,14 +86,14 @@ impl Suggestion {pub fn get_replacement_text(&self) -> String {match &self.inner {harper_core::Suggestion::Remove => "".to_string(),- harper_core::Suggestion::ReplaceWith(chars) => chars.iter().collect()+ harper_core::Suggestion::ReplaceWith(chars) => chars.iter().collect(),}}pub fn kind(&self) -> SuggestionKind {match &self.inner {harper_core::Suggestion::Remove => SuggestionKind::Remove,- harper_core::Suggestion::ReplaceWith(_) => SuggestionKind::Replace+ harper_core::Suggestion::ReplaceWith(_) => SuggestionKind::Replace,}}}@@ -103,12 +101,12 @@ impl Suggestion {#[wasm_bindgen]pub struct Lint {inner: harper_core::Lint,- source: Rc<Vec<char>>+ source: Lrc<Vec<char>>,}#[wasm_bindgen]impl Lint {- pub(crate) fn new(inner: harper_core::Lint, source: Rc<Vec<char>>) -> Self {+ pub(crate) fn new(inner: harper_core::Lint, source: Lrc<Vec<char>>) -> Self {Self { inner, source }}@@ -146,7 +144,7 @@ impl Lint {#[wasm_bindgen]pub struct Span {pub start: usize,- pub end: usize+ pub end: usize,}#[wasm_bindgen]