Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
Merge branch 'master' into local-stats
Elijah Potterfa5d13965643
diff --cc Cargo.lockindex 22989b29a,f9c5cbbb7..f91688539--- a/Cargo.lock+++ b/Cargo.lock@@@ -349,31 -395,45 +395,66 @@@ checksum = "d0a5c400df2834b80a4c3327b3a[[package]]name = "crunchy"- version = "0.2.2"+ version = "0.2.3"source = "registry+https://github.com/rust-lang/crates.io-index"- checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7"+ checksum = "43da5946c66ffcc7745f48db692ffbb10a83bfe0afd96235c5c2a4fb23994929"+[[package]]+name = "csv"+version = "1.3.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "acdc4883a9c96732e4733212c01447ebd805833b7275a73ca3ee080fd77afdaf"+dependencies = [+ "csv-core",+ "itoa",+ "ryu",+ "serde",+]++[[package]]+name = "csv-core"+version = "0.1.11"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "5efa2b3d7902f4b634a20cae3c9c4e6209dc4779feb6863329607560143efa70"+dependencies = [+ "memchr",+]++ [[package]]+ name = "darling"+ version = "0.20.10"+ source = "registry+https://github.com/rust-lang/crates.io-index"+ checksum = "6f63b86c8a8826a49b8c21f08a2d07338eec8d900540f8630dc76284be802989"+ dependencies = [+ "darling_core",+ "darling_macro",+ ]++ [[package]]+ name = "darling_core"+ version = "0.20.10"+ source = "registry+https://github.com/rust-lang/crates.io-index"+ checksum = "95133861a8032aaea082871032f5815eb9e98cef03fa916ab4500513994df9e5"+ dependencies = [+ "fnv",+ "ident_case",+ "proc-macro2",+ "quote",+ "strsim",+ "syn 2.0.96",+ ]++ [[package]]+ name = "darling_macro"+ version = "0.20.10"+ source = "registry+https://github.com/rust-lang/crates.io-index"+ checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806"+ dependencies = [+ "darling_core",+ "quote",+ "syn 2.0.96",+ ]+[[package]]name = "dashmap"version = "5.5.3"@@@ -626,8 -692,8 +713,9 @@@ dependencies ="harper-comments","harper-core","harper-literate-haskell",+ "harper-stats","harper-typst",+ "hashbrown 0.15.2","serde","serde_json",]@@@ -721,8 -796,8 +818,9 @@@ dependencies ="harper-core","harper-html","harper-literate-haskell",+ "harper-stats","harper-typst",+ "indexmap","itertools 0.14.0","once_cell","open",@@@ -735,19 -810,9 +833,19 @@@"tracing-subscriber",]+[[package]]+name = "harper-stats"- version = "0.17.0"++version = "0.25.0"+dependencies = [+ "csv",+ "harper-core",+ "serde",+ "uuid",+]+[[package]]name = "harper-tree-sitter"- version = "0.17.0"+ version = "0.25.1"dependencies = ["harper-core","tree-sitter",@@@ -2231,21 -2383,17 +2416,27 @@@ version = "0.2.2source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"+[[package]]+name = "uuid"+version = "1.12.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "744018581f9a3454a9e15beb8a33b017183f1e7c0cd170232a2d1453b23a51c4"+dependencies = [+ "getrandom",+ "serde",+]+[[package]]name = "valuable"- version = "0.1.0"+ version = "0.1.1"source = "registry+https://github.com/rust-lang/crates.io-index"- checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d"+ checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65"++ [[package]]+ name = "version_check"+ version = "0.9.5"+ source = "registry+https://github.com/rust-lang/crates.io-index"+ checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"[[package]]name = "walkdir"diff --cc harper-cli/Cargo.tomlindex 142360635,b451c05cf..2ccd226c1--- a/harper-cli/Cargo.toml+++ b/harper-cli/Cargo.toml@@@ -7,16 -7,16 +7,17 @@@ publish = falsrepository = "https://github.com/automattic/harper"[dependencies]- anyhow = "1.0.95"+ anyhow = "1.0.97"ariadne = "0.4.1"- clap = { version = "4.5.27", features = ["derive"] }- harper-literate-haskell = { path = "../harper-literate-haskell", version = "0.17.0" }- harper-core = { path = "../harper-core", version = "0.17.0" }- harper-comments = { path = "../harper-comments", version = "0.17.0" }- harper-typst = { path = "../harper-typst", version = "0.17.0" }- harper-stats = { path = "../harper-stats", version = "0.17.0" }- serde = { version = "1.0.214", features = ["derive"] }- serde_json = "1.0.137"+ clap = { version = "4.5.32", features = ["derive"] }++harper-stats = { path = "../harper-stats", version = "0.25.0" }+ harper-literate-haskell = { path = "../harper-literate-haskell", version = "0.25.0" }+ harper-core = { path = "../harper-core", version = "0.25.0" }+ harper-comments = { path = "../harper-comments", version = "0.25.0" }+ harper-typst = { path = "../harper-typst", version = "0.25.0" }+ hashbrown = "0.15.2"+ serde = { version = "1.0.219", features = ["derive"] }+ serde_json = "1.0.140"[features]default = []diff --cc harper-cli/src/main.rsindex fbf1b9df1,c753fa98c..f6f0ccb2e--- a/harper-cli/src/main.rs+++ b/harper-cli/src/main.rs@@@ -1,19 -1,21 +1,23 @@@#![doc = include_str!("../README.md")]-use std::collections::BTreeMap;+use std::collections::{BTreeMap, HashMap};+use std::fs::File;+use std::io::BufReader;use std::path::{Path, PathBuf};+ use std::process;use anyhow::format_err;use ariadne::{Color, Label, Report, ReportKind, Source};use clap::Parser;use harper_comments::CommentParser;- use harper_core::linting::{LintGroup, LintGroupConfig, Linter};+ use harper_core::linting::{LintGroup, Linter};use harper_core::parsers::{Markdown, MarkdownOptions};- use harper_core::{remove_overlaps, Dictionary, Document, FstDictionary, TokenKind};+ use harper_core::{+ remove_overlaps, CharStringExt, Dialect, Dictionary, Document, FstDictionary,+ MutableDictionary, TokenKind, TokenStringExt, WordId,+ };use harper_literate_haskell::LiterateHaskellParser;-use hashbrown::HashMap;+use harper_stats::Stats;use serde::Serialize;/// A debugging tool for the Harper grammar checker.@@@ -44,12 -52,17 +54,19 @@@ enum Args},/// Get the metadata associated with a particular word.Metadata { word: String },+ /// Get all the forms of a word using the affixes.+ Forms { line: String },/// Emit a decompressed, line-separated list of the words in Harper's dictionary.Words,+ /// Summarize a lint record+ SummarizeLintRecord { file: PathBuf },/// Print the default config with descriptions.Config,+ /// Print a list of all the words in a document, sorted by frequency.+ MineWords {+ /// The document to mine words from.+ file: PathBuf,+ },}fn main() -> anyhow::Result<()> {@@@ -174,16 -200,73 +204,83 @@@Ok(())}+ Args::SummarizeLintRecord { file } => {+ let file = File::open(file)?;+ let mut reader = BufReader::new(file);+ let stats = Stats::read_csv(&mut reader)?;++ let summary = stats.summarize_lints_applied();+ println!("{summary}");++ Ok(())+ }+ Args::Forms { line } => {+ let (word, annot) = line_to_parts(&line);++ let curated_word_list = include_str!("../../harper-core/dictionary.dict");+ let dict_lines = curated_word_list.split('\n');++ let mut entry_in_dict = None;++ // Check if the word is contained in the list.+ for dict_line in dict_lines {+ let (dict_word, dict_annot) = line_to_parts(dict_line);++ if dict_word == word {+ entry_in_dict = Some((dict_word, dict_annot));+ break;+ }+ }++ let summary = match &entry_in_dict {+ Some((dict_word, dict_annot)) => {+ let mut status_summary = if dict_annot.is_empty() {+ format!(+ "'{}' is already in the dictionary but not annotated.",+ dict_word+ )+ } else {+ format!(+ "'{}' is already in the dictionary with annotation `{}`.",+ dict_word, dict_annot+ )+ };++ if !annot.is_empty() {+ if annot.as_str() != dict_annot.as_str() {+ status_summary+ .push_str("\n Your annotations differ from the dictionary.\n");+ } else {+ status_summary+ .push_str("\n Your annotations are the same as the dictionary.\n");+ }+ }++ status_summary+ }+ None => format!("'{}' is not in the dictionary yet.", word),+ };++ println!("{summary}");++ if let Some((dict_word, dict_annot)) = &entry_in_dict {+ println!("Old, from the dictionary:");+ print_word_derivations(dict_word, dict_annot, &FstDictionary::curated());+ };++ if !annot.is_empty() {+ let rune_words = format!("1\n{line}");+ let dict = MutableDictionary::from_rune_files(+ &rune_words,+ include_str!("../../harper-core/affixes.json"),+ )?;++ println!("New, from you:");+ print_word_derivations(&word, &annot, &dict);+ }++ Ok(())+ }Args::Config => {#[derive(Serialize)]struct Config {diff --cc harper-core/src/linting/lint_kind.rsindex 000000000,1d7ea1672..177de20abmode 000000,100644..100644--- a/harper-core/src/linting/lint_kind.rs+++ b/harper-core/src/linting/lint_kind.rs@@@ -1,0 -1,58 +1,73 @@@+ use std::fmt::Display;++ use is_macro::Is;+ use serde::{Deserialize, Serialize};++ /// The general category a [`Lint`](super::Lint) falls into.+ /// There's no reason not to add a new item here if you are adding a new rule that doesn't fit+ /// the existing categories.-#[derive(Debug, Clone, Copy, Serialize, Deserialize, Is, Default, Hash, PartialEq)]++#[derive(Debug, Clone, Copy, Serialize, Deserialize, Is, Default, Hash, PartialEq, Eq)]+ pub enum LintKind {+ /// This should only be used by linters doing spellcheck on individual words.+ Spelling,+ Capitalization,+ Style,+ Formatting,+ Repetition,+ Enhancement,+ Readability,+ WordChoice,+ #[default]+ Miscellaneous,+ }++ impl LintKind {++ pub fn new_from_str(s: &str) -> Option<Self> {++ Some(match s {++ "Spelling" => LintKind::Spelling,++ "Capitalization" => LintKind::Capitalization,++ "Formatting" => LintKind::Formatting,++ "Repetition" => LintKind::Repetition,++ "Readability" => LintKind::Readability,++ "Miscellaneous" => LintKind::Miscellaneous,++ "Enhancement" => LintKind::Enhancement,++ "Word Choice" => LintKind::WordChoice,++ "Style" => LintKind::Style,++ _ => return None,++ })++ }+++ /// Produce a string representation, which can be used as keys in a map or CSS variables.+ pub fn to_string_key(&self) -> String {+ match self {+ LintKind::Spelling => "Spelling",+ LintKind::Capitalization => "Capitalization",+ LintKind::Formatting => "Formatting",+ LintKind::Repetition => "Repetition",+ LintKind::Readability => "Readability",+ LintKind::Miscellaneous => "Miscellaneous",+ LintKind::Enhancement => "Enhancement",+ LintKind::WordChoice => "WordChoice",+ LintKind::Style => "Style",+ }+ .to_owned()+ }+ }++ impl Display for LintKind {+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ let s = match self {+ LintKind::Spelling => "Spelling",+ LintKind::Capitalization => "Capitalization",+ LintKind::Formatting => "Formatting",+ LintKind::Repetition => "Repetition",+ LintKind::Readability => "Readability",+ LintKind::Miscellaneous => "Miscellaneous",+ LintKind::Enhancement => "Enhancement",+ LintKind::WordChoice => "Word Choice",+ LintKind::Style => "Style",+ };++ write!(f, "{}", s)+ }+ }diff --cc harper-ls/Cargo.tomlindex bac4ef196,8ae127c5c..7dd7b0fc0--- a/harper-ls/Cargo.toml+++ b/harper-ls/Cargo.toml@@@ -8,19 -8,18 +8,19 @@@ readme = "README.mdrepository = "https://github.com/automattic/harper"[dependencies]- harper-literate-haskell = { path = "../harper-literate-haskell", version = "0.17.0" }- harper-core = { path = "../harper-core", version = "0.17.0", features = ["concurrent"] }- harper-stats = { path = "../harper-stats", version = "0.17.0" }- harper-comments = { path = "../harper-comments", version = "0.17.0" }- harper-typst = { path = "../harper-typst", version = "0.17.0" }- harper-html = { path = "../harper-html", version = "0.17.0" }+ harper-literate-haskell = { path = "../harper-literate-haskell", version = "0.25.0" }++harper-stats = { path = "../harper-stats", version = "0.25.0" }+ harper-core = { path = "../harper-core", version = "0.25.0", features = ["concurrent"] }+ harper-comments = { path = "../harper-comments", version = "0.25.0" }+ harper-typst = { path = "../harper-typst", version = "0.25.0" }+ harper-html = { path = "../harper-html", version = "0.25.0" }tower-lsp = "0.20.0"- tokio = { version = "1.43.0", features = ["fs", "rt", "rt-multi-thread", "macros", "io-std", "io-util", "net"] }- clap = { version = "4.5.27", features = ["derive"] }- once_cell = "1.20.2"+ tokio = { version = "1.44.1", features = ["fs", "rt", "rt-multi-thread", "macros", "io-std", "io-util", "net"] }+ clap = { version = "4.5.32", features = ["derive"] }+ once_cell = "1.20.3"dirs = "6.0.0"- anyhow = "1.0.95"- serde_json = "1.0.137"+ anyhow = "1.0.97"+ serde_json = "1.0.140"itertools = "0.14.0"tracing = "0.1.41"tracing-subscriber = "0.3.19"diff --cc harper-ls/src/backend.rsindex d82f82c4c,20ec7781e..e5fccf96a--- a/harper-ls/src/backend.rs+++ b/harper-ls/src/backend.rs@@@ -1,17 -1,13 +1,16 @@@use std::collections::HashMap;+use std::fs::OpenOptions;+use std::io::{BufWriter, Write};- use std::path::{Component, PathBuf};+ use std::path::PathBuf;use std::sync::Arc;- use anyhow::{anyhow, Context, Result};+ use anyhow::{Context, Result, anyhow};+use futures::future::join;use harper_comments::CommentParser;- use harper_core::linting::{LintGroup, LintKind, Linter};-use harper_core::linting::{LintGroup, LintGroupConfig};++use harper_core::linting::{LintGroup, LintGroupConfig, LintKind};use harper_core::parsers::{CollapseIdentifiers, IsolateEnglish, Markdown, Parser, PlainEnglish};use harper_core::{- Dictionary, Document, FstDictionary, FullDictionary, MergedDictionary, Token, TokenKind,- WordMetadata,+ Dialect, Dictionary, Document, FstDictionary, MergedDictionary, MutableDictionary, WordMetadata,};use harper_html::HtmlParser;use harper_literate_haskell::LiterateHaskellParser;@@@ -35,12 -30,9 +33,10 @@@ use tower_lsp::{Client, LanguageServer}use tracing::{error, info, warn};use crate::config::Config;- use crate::diagnostics::{lint_to_code_actions, lints_to_diagnostics};- use crate::dictionary_io::{load_dict, save_dict};+ use crate::dictionary_io::{file_dict_name, load_dict, save_dict};use crate::document_state::DocumentState;use crate::git_commit_parser::GitCommitParser;- use crate::pos_conv::range_to_span;+use harper_stats::{LintRecord, Stats};pub struct Backend {client: Client,diff --cc harper-ls/src/diagnostics.rsindex 13a8453aa,a7bad6a1b..23ec0e520--- a/harper-ls/src/diagnostics.rs+++ b/harper-ls/src/diagnostics.rs@@@ -1,8 -1,7 +1,8 @@@use std::collections::HashMap;- use harper_core::linting::{Lint, Suggestion};use harper_core::CharStringExt;+ use harper_core::linting::{Lint, Suggestion};+use serde_json::Value;use tower_lsp::lsp_types::{CodeAction, CodeActionKind, CodeActionOrCommand, Command, Diagnostic, TextEdit, Url,WorkspaceEdit,diff --cc harper-stats/Cargo.tomlindex da80752a3,000000000..30d94b5ffmode 100644,000000..100644--- a/harper-stats/Cargo.toml+++ b/harper-stats/Cargo.toml@@@ -1,10 -1,0 +1,10 @@@+[package]+name = "harper-stats"- version = "0.17.0"++version = "0.25.0"+edition = "2021"++[dependencies]+serde = { version = "1.0.217", features = ["derive"] }- harper-core = { path = "../harper-core", version = "0.17.0", features = ["concurrent"] }++harper-core = { path = "../harper-core", version = "0.25.0", features = ["concurrent"] }+uuid = { version = "1.12.0", features = ["serde", "v4"] }+csv = "1.3.1"