Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
feat: added global dictionary
Elijah Potter64207292cc91
diff --git a/Cargo.lock b/Cargo.lockindex 191c6cb89..b40308c58 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -98,6 +98,12 @@ dependencies = ["windows-sys 0.52.0",]+[[package]]+name = "anyhow"+version = "1.0.80"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "5ad32ce52e4161730f7098c077cd2ed6229b5804ccf99e5366be1ab72a98b4e1"+[[package]]name = "async-trait"version = "0.1.77"@@ -317,6 +323,27 @@ dependencies = ["parking_lot_core",]+[[package]]+name = "dirs"+version = "5.0.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "44c45a9d03d6676652bcb5e724c7e988de1acad23a711b5217ab9cbecbec2225"+dependencies = [+ "dirs-sys",+]++[[package]]+name = "dirs-sys"+version = "0.4.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "520f05a5cbd335fae5a99ff7a6ab8627577660ee5cfd6a94a6a929b52ff0321c"+dependencies = [+ "libc",+ "option-ext",+ "redox_users",+ "windows-sys 0.48.0",+]+[[package]]name = "divan"version = "0.1.11"@@ -459,6 +486,17 @@ dependencies = ["unicode-width",]+[[package]]+name = "getrandom"+version = "0.2.12"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "190092ea657667030ac6a35e305e62fc4dd69fd98ac98631e5d3a2b1575a12b5"+dependencies = [+ "cfg-if",+ "libc",+ "wasi",+]+[[package]]name = "gimli"version = "0.28.1"@@ -473,7 +511,7 @@ dependencies = ["divan","hashbrown","is-macro",- "itertools",+ "itertools 0.11.0","once_cell","paste","pulldown-cmark",@@ -486,9 +524,13 @@ dependencies = [name = "harper-ls"version = "0.6.3"dependencies = [+ "anyhow","clap",+ "dirs","harper-core",+ "itertools 0.12.1","once_cell",+ "serde_json","tokio","tower-lsp","tree-sitter",@@ -644,6 +686,15 @@ dependencies = ["either",]+[[package]]+name = "itertools"+version = "0.12.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569"+dependencies = [+ "either",+]+[[package]]name = "itoa"version = "1.0.10"@@ -671,6 +722,17 @@ version = "0.2.152"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "13e3bf6590cbc649f4d1a3eefc9d5d6eb746f5200ffb04e5e142700b8faa56e7"+[[package]]+name = "libredox"+version = "0.0.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "85c833ca1e66078851dba29046874e38f08b2c883700aa29a03ddd3b23814ee8"+dependencies = [+ "bitflags 2.4.2",+ "libc",+ "redox_syscall",+]+[[package]]name = "linux-raw-sys"version = "0.4.13"@@ -780,20 +842,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92"[[package]]-name = "overload"-version = "0.1.1"+name = "option-ext"+version = "0.2.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39"+checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d"[[package]]-name = "parking_lot"-version = "0.12.1"+name = "overload"+version = "0.1.1"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f"-dependencies = [- "lock_api",- "parking_lot_core",-]+checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39"[[package]]name = "parking_lot_core"@@ -915,6 +973,17 @@ dependencies = ["bitflags 1.3.2",]+[[package]]+name = "redox_users"+version = "0.4.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a18479200779601e498ada4e8c1e1f50e3ee19deb0259c25825a98b5603b2cb4"+dependencies = [+ "getrandom",+ "libredox",+ "thiserror",+]+[[package]]name = "regex"version = "1.10.3"@@ -1020,9 +1089,9 @@ dependencies = [[[package]]name = "serde_json"-version = "1.0.111"+version = "1.0.114"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "176e46fa42316f18edd598015a5166857fc835ec732f5215eac6b7bdbf0a84f4"+checksum = "c5f09b1bd632ef549eaa9f60a1f8de742bdbc698e6cee2095fc84dde5f549ae0"dependencies = ["itoa","ryu",@@ -1071,15 +1140,6 @@ dependencies = ["lazy_static",]-[[package]]-name = "signal-hook-registry"-version = "1.4.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1"-dependencies = [- "libc",-]-[[package]]name = "slab"version = "0.4.9"@@ -1208,9 +1268,7 @@ dependencies = ["libc","mio","num_cpus",- "parking_lot","pin-project-lite",- "signal-hook-registry","socket2","tokio-macros","windows-sys 0.48.0",diff --git a/harper-core/dictionary.dict b/harper-core/dictionary.dictindex e81061144..d4242ad6d 100644--- a/harper-core/dictionary.dict+++ b/harper-core/dictionary.dict@@ -11689,7 +11689,7 @@ alligator/MSalliterate/DSXGNValliteration/Malliterative/Y-allocate/ADSGN+allocate/ADSGNYallocation/AMallocationsallot/LS@@ -49607,3 +49607,4 @@ LIDARLiDARLADARTODO+structdiff --git a/harper-core/src/lexing/mod.rs b/harper-core/src/lexing/mod.rsindex b6d19d10d..1825d418b 100644--- a/harper-core/src/lexing/mod.rs+++ b/harper-core/src/lexing/mod.rs@@ -136,8 +136,6 @@ fn lex_punctuation(source: &[char]) -> Option<FoundToken> {'-' => Hyphen,'[' => OpenSquare,']' => CloseSquare,- '<' => OpenAngle,- '>' => CloseAngle,'{' => OpenCurly,'}' => CloseCurly,'(' => OpenRound,diff --git a/harper-core/src/spell/full_dictionary.rs b/harper-core/src/spell/full_dictionary.rsindex 72aa9ce3c..b7465dafe 100644--- a/harper-core/src/spell/full_dictionary.rs+++ b/harper-core/src/spell/full_dictionary.rs@@ -61,12 +61,22 @@ impl FullDictionary {/// Appends words to the dictionary./// It is significantly faster to append many words with one call than many/// distinct calls to this function.- pub fn append_words(&mut self, words: &[&[char]]) {- self.words.extend(words.iter().map(|v| v.to_smallvec()));- self.word_set.extend(words.iter().map(|v| v.to_smallvec()));+ pub fn extend_words(&mut self, words: impl IntoIterator<Item = impl AsRef<[char]>>) {+ let init_size = self.words.len();+ self.words+ .extend(words.into_iter().map(|v| v.as_ref().to_smallvec()));+ self.word_set+ .extend(self.words[init_size..].iter().cloned());self.word_len_starts = Self::create_len_starts(&mut self.words);}+ /// Append a single word to the dictionary.+ ///+ /// If you are appending many words, consider using [`Self::extend_words`] instead.+ pub fn append_word(&mut self, word: impl AsRef<[char]>) {+ self.extend_words(std::iter::once(word.as_ref()))+ }+/// Create a lookup table for finding words of a specific length in a word list./// NOTE: This function will sort the original word list by its length./// If the word list's order is changed after creating the lookup, it will no longer be valid.diff --git a/harper-core/src/token.rs b/harper-core/src/token.rsindex e2d574590..b019f0f4a 100644--- a/harper-core/src/token.rs+++ b/harper-core/src/token.rs@@ -100,10 +100,6 @@ pub enum Punctuation {OpenRound,/// `)`CloseRound,- /// `<`- OpenAngle,- /// `>`- CloseAngle,/// `{`OpenCurly,/// `}`diff --git a/harper-ls/Cargo.toml b/harper-ls/Cargo.tomlindex 6e47e0c0a..198b88e53 100644--- a/harper-ls/Cargo.toml+++ b/harper-ls/Cargo.toml@@ -26,3 +26,7 @@ tree-sitter-swift = "0.4.0"tree-sitter-c-sharp = "0.20.0"tree-sitter-toml = "0.20.0"tree-sitter-lua = "0.0.19"+dirs = "5.0.1"+anyhow = "1.0.80"+serde_json = "1.0.114"+itertools = "0.12.1"diff --git a/harper-ls/src/backend.rs b/harper-ls/src/backend.rsindex 3dcd6d408..358c34d6c 100644--- a/harper-ls/src/backend.rs+++ b/harper-ls/src/backend.rs@@ -1,23 +1,30 @@use std::{collections::HashMap, sync::Arc};-use harper_core::{parsers::Markdown, Document, FullDictionary, LintSet, Linter, MergedDictionary};+use harper_core::{+ parsers::Markdown, Dictionary, Document, FullDictionary, LintSet, Linter, MergedDictionary,+};+use itertools::Itertools;+use serde_json::Value;use tokio::sync::Mutex;use tower_lsp::{jsonrpc::Result,lsp_types::{notification::{PublishDiagnostics, ShowMessage},- CodeAction, CodeActionOrCommand, CodeActionParams, CodeActionProviderCapability,- CodeActionResponse, Diagnostic, DidChangeTextDocumentParams, DidCloseTextDocumentParams,- DidOpenTextDocumentParams, DidSaveTextDocumentParams, InitializeParams, InitializeResult,- InitializedParams, MessageType, PublishDiagnosticsParams, Range, ServerCapabilities,- ShowMessageParams, TextDocumentSyncCapability, TextDocumentSyncKind,- TextDocumentSyncOptions, TextDocumentSyncSaveOptions, Url,+ CodeActionOrCommand, CodeActionParams, CodeActionProviderCapability, CodeActionResponse,+ Diagnostic, DidChangeTextDocumentParams, DidCloseTextDocumentParams,+ DidOpenTextDocumentParams, DidSaveTextDocumentParams, ExecuteCommandParams,+ InitializeParams, InitializeResult, InitializedParams, MessageType,+ PublishDiagnosticsParams, Range, ServerCapabilities, ShowMessageParams,+ TextDocumentSyncCapability, TextDocumentSyncKind, TextDocumentSyncOptions,+ TextDocumentSyncSaveOptions, Url,},Client, LanguageServer,};use crate::{+ config::Config,diagnostics::{lint_to_code_actions, lints_to_diagnostics},+ dictionary_io::{load_dict, save_dict},pos_conv::range_to_span,tree_sitter_parser::TreeSitterParser,};@@ -29,20 +36,23 @@ struct DocumentState {linter: LintSet,}+/// Deallocatepub struct Backend {client: Client,- global_dictionary: Arc<FullDictionary>,+ static_dictionary: Arc<FullDictionary>,+ config: Config,doc_state: Mutex<HashMap<Url, DocumentState>>,}impl Backend {- pub fn new(client: Client) -> Self {+ pub fn new(client: Client, config: Config) -> Self {let dictionary = FullDictionary::create_from_curated();Self {client,- global_dictionary: dictionary.into(),+ static_dictionary: dictionary.into(),doc_state: Mutex::new(HashMap::new()),+ config,}}@@ -54,10 +64,26 @@ impl Backend {self.update_document(url, &content).await;}- async fn update_document(&self, url: &Url, text: &str) {+ async fn load_user_dictionary(&self) -> anyhow::Result<FullDictionary> {+ Ok(load_dict(&self.config.user_dict_path).await?)+ }++ async fn save_user_dictionary(&self, dict: impl Dictionary) -> anyhow::Result<()> {+ Ok(save_dict(&self.config.user_dict_path, dict).await?)+ }++ async fn generate_global_dictionary(&self) -> anyhow::Result<MergedDictionary<FullDictionary>> {+ let mut dict = MergedDictionary::new();+ dict.add_dictionary(self.static_dictionary.clone());+ let user_dict = self.load_user_dictionary().await?;+ dict.add_dictionary(Arc::new(user_dict));+ Ok(dict)+ }++ async fn update_document(&self, url: &Url, text: &str) -> anyhow::Result<()> {let mut lock = self.doc_state.lock().await;let doc_state = lock.entry(url.clone()).or_insert(DocumentState {- linter: LintSet::new().with_standard(self.global_dictionary.clone()),+ linter: LintSet::new().with_standard(self.generate_global_dictionary().await?),..Default::default()});@@ -72,9 +98,9 @@ impl Backend {if doc_state.ident_dict != new_dict {doc_state.ident_dict = new_dict.clone();- let mut merged = MergedDictionary::new();+ let mut merged = self.generate_global_dictionary().await?;merged.add_dictionary(new_dict);- merged.add_dictionary(self.global_dictionary.clone());+doc_state.linter = LintSet::new().with_standard(merged);}}@@ -86,9 +112,15 @@ impl Backend {} else {Document::new(text, Box::new(Markdown))};++ Ok(())}- async fn generate_code_actions(&self, url: &Url, range: Range) -> Result<Vec<CodeAction>> {+ async fn generate_code_actions(+ &self,+ url: &Url,+ range: Range,+ ) -> Result<Vec<CodeActionOrCommand>> {let mut doc_states = self.doc_state.lock().await;let Some(doc_state) = doc_states.get_mut(url) else {return Ok(Vec::new());@@ -105,7 +137,7 @@ impl Backend {let actions = lints.into_iter().filter(|lint| lint.span.overlaps_with(span))- .flat_map(|lint| lint_to_code_actions(&lint, url, source_chars).collect::<Vec<_>>())+ .flat_map(|lint| lint_to_code_actions(&lint, url, source_chars)).collect();Ok(actions)@@ -210,7 +242,8 @@ impl LanguageServer for Backend {.await;self.update_document(¶ms.text_document.uri, &last.text)- .await;+ .await+ .unwrap();self.publish_diagnostics(¶ms.text_document.uri).await;}@@ -220,20 +253,37 @@ impl LanguageServer for Backend {.await;}+ async fn execute_command(&self, params: ExecuteCommandParams) -> Result<Option<Value>> {+ match params.command.as_str() {+ "AddWord" => {+ let mut string_args = params+ .arguments+ .into_iter()+ .map(|v| serde_json::from_value::<String>(v).unwrap());++ let Some((first, second)) = string_args.next_tuple() else {+ return Ok(None);+ };++ let mut dict = self.load_user_dictionary().await.unwrap();+ dict.append_word(&first.chars().collect::<Vec<_>>());+ self.save_user_dictionary(dict).await.unwrap();++ self.publish_diagnostics(&Url::parse(&second).unwrap())+ .await;++ Ok(None)+ }++ _ => Ok(None),+ }+ }+async fn code_action(&self, params: CodeActionParams) -> Result<Option<CodeActionResponse>> {let actions = self.generate_code_actions(¶ms.text_document.uri, params.range).await?;- self.client- .log_message(MessageType::INFO, format!("{:?}", actions))- .await;-- Ok(Some(- actions- .into_iter()- .map(CodeActionOrCommand::CodeAction)- .collect(),- ))+ Ok(Some(actions))}}diff --git a/harper-ls/src/config.rs b/harper-ls/src/config.rsnew file mode 100644index 000000000..bdd835097--- /dev/null+++ b/harper-ls/src/config.rs@@ -0,0 +1,16 @@+use std::path::PathBuf;++use dirs::config_dir;++#[derive(Debug, Clone)]+pub struct Config {+ pub user_dict_path: PathBuf,+}++impl Default for Config {+ fn default() -> Self {+ Self {+ user_dict_path: config_dir().unwrap().join("harper-ls/dictionary.txt"),+ }+ }+}diff --git a/harper-ls/src/diagnostics.rs b/harper-ls/src/diagnostics.rsindex ec27e9f38..278b894e1 100644--- a/harper-ls/src/diagnostics.rs+++ b/harper-ls/src/diagnostics.rs@@ -1,6 +1,9 @@use harper_core::{Lint, Suggestion};use std::collections::HashMap;-use tower_lsp::lsp_types::{CodeAction, CodeActionKind, Diagnostic, TextEdit, Url, WorkspaceEdit};+use tower_lsp::lsp_types::{+ CodeAction, CodeActionKind, CodeActionOrCommand, Command, Diagnostic, TextEdit, Url,+ WorkspaceEdit,+};use crate::pos_conv::span_to_range;@@ -15,33 +18,52 @@ pub fn lint_to_code_actions<'a>(lint: &'a Lint,url: &'a Url,source: &'a [char],-) -> impl Iterator<Item = CodeAction> + 'a {- lint.suggestions.iter().flat_map(|suggestion| {- let range = span_to_range(source, lint.span);-- let Suggestion::ReplaceWith(with) = suggestion;- Some(CodeAction {- title: suggestion.to_string(),- kind: Some(CodeActionKind::QUICKFIX),- diagnostics: None,- edit: Some(WorkspaceEdit {- changes: Some(HashMap::from([(- url.clone(),- vec![TextEdit {- range,-- new_text: with.iter().collect(),- }],- )])),- document_changes: None,- change_annotations: None,- }),- command: None,- is_preferred: None,- disabled: None,- data: None,- })- })+) -> Vec<CodeActionOrCommand> {+ let mut results = Vec::new();++ results.extend(+ lint.suggestions+ .iter()+ .flat_map(|suggestion| {+ let range = span_to_range(source, lint.span);++ let Suggestion::ReplaceWith(with) = suggestion;+ Some(CodeAction {+ title: suggestion.to_string(),+ kind: Some(CodeActionKind::QUICKFIX),+ diagnostics: None,+ edit: Some(WorkspaceEdit {+ changes: Some(HashMap::from([(+ url.clone(),+ vec![TextEdit {+ range,++ new_text: with.iter().collect(),+ }],+ )])),+ document_changes: None,+ change_annotations: None,+ }),+ command: None,+ is_preferred: None,+ disabled: None,+ data: None,+ })+ })+ .map(CodeActionOrCommand::CodeAction),+ );++ if lint.lint_kind.is_spelling() {+ let orig = lint.span.get_content_string(source);++ results.push(CodeActionOrCommand::Command(Command::new(+ format!("Add \"{}\" to the global dictionary.", orig),+ "AddWord".to_string(),+ Some(vec![orig.into(), url.to_string().into()]),+ )))+ }++ results}fn lint_to_diagnostic(lint: &Lint, source: &[char]) -> Diagnostic {diff --git a/harper-ls/src/dictionary_io.rs b/harper-ls/src/dictionary_io.rsnew file mode 100644index 000000000..82ad92910--- /dev/null+++ b/harper-ls/src/dictionary_io.rs@@ -0,0 +1,49 @@+use std::path::{Path, PathBuf};++use harper_core::{Dictionary, FullDictionary};+use tokio::fs::File;+use tokio::io::{self, AsyncRead, AsyncReadExt, BufReader};+use tokio::io::{AsyncWrite, AsyncWriteExt};++pub async fn save_dict(path: impl AsRef<Path>, dict: impl Dictionary) -> io::Result<()> {+ let file = File::create(path.as_ref()).await?;+ let write = BufReader::new(file);++ write_word_list(dict, write).await?;++ Ok(())+}++async fn write_word_list(dict: impl Dictionary, mut w: impl AsyncWrite + Unpin) -> io::Result<()> {+ let mut cur_str = String::new();++ for word in dict.words_iter() {+ cur_str.clear();+ cur_str.extend(word);++ w.write_all(cur_str.as_bytes()).await?;+ w.write_all(b"\n").await?;+ }++ Ok(())+}++pub async fn load_dict(path: impl AsRef<Path>) -> io::Result<FullDictionary> {+ let file = File::open(path.as_ref()).await?;+ let read = BufReader::new(file);++ Ok(dict_from_word_list(read).await?)+}++/// This function could definitely be optimized to use less memory.+/// Right now, it isn't an issue.+async fn dict_from_word_list(mut r: impl AsyncRead + Unpin) -> io::Result<FullDictionary> {+ let mut str = String::new();++ r.read_to_string(&mut str).await?;++ let mut dict = FullDictionary::new();+ dict.extend_words(str.lines().map(|l| l.chars().collect::<Vec<char>>()));++ Ok(dict)+}diff --git a/harper-ls/src/main.rs b/harper-ls/src/main.rsindex 7f17f6419..7829306d9 100644--- a/harper-ls/src/main.rs+++ b/harper-ls/src/main.rs@@ -1,6 +1,9 @@-use tokio::net::TcpListener;+use config::Config;+use tokio::{fs, net::TcpListener};mod backend;+mod config;mod diagnostics;+mod dictionary_io;mod pos_conv;mod tree_sitter_parser;@@ -15,10 +18,15 @@ struct Args {}#[tokio::main]-async fn main() {+async fn main() -> anyhow::Result<()> {let args = Args::parse();- let (service, socket) = LspService::new(Backend::new);+ let config = Config::default();++ // Make sure this is available.+ fs::create_dir_all(config.user_dict_path.parent().unwrap()).await?;++ let (service, socket) = LspService::new(|client| Backend::new(client, config));if args.stdio {let stdin = tokio::io::stdin();@@ -32,4 +40,6 @@ async fn main() {let (read, write) = tokio::io::split(stream);Server::new(read, write, socket).serve(service).await;}++ Ok(())}diff --git a/harper-ls/src/tree_sitter_parser.rs b/harper-ls/src/tree_sitter_parser.rsindex 0836f1ee9..d5675e379 100644--- a/harper-ls/src/tree_sitter_parser.rs+++ b/harper-ls/src/tree_sitter_parser.rs@@ -73,7 +73,7 @@ impl TreeSitterParser {let idents: Vec<_> = idents.into_iter().collect();let mut dictionary = FullDictionary::new();- dictionary.append_words(idents.as_slice());+ dictionary.extend_words(idents);Some(dictionary)}diff --git a/harper-serve/Cargo.toml b/harper-serve/Cargo.tomlindex 978a925ed..e002f816e 100644--- a/harper-serve/Cargo.toml+++ b/harper-serve/Cargo.toml@@ -8,7 +8,7 @@ publish = false[dependencies]axum = "0.6.20"serde = { version = "1.0.189", features = ["derive"] }-tokio = { version = "1.33.0", features = ["full"] }+tokio = { version = "1.35.1", features = ["fs", "rt", "rt-multi-thread", "macros"] }tracing = "0.1.40"tracing-subscriber = "0.3.17"harper-core = { path = "../harper-core" }