Skip to content

Offline, privacy-first grammar checker. Fast, open-source, Rust-powered

Merge branch 'master' into local-stats

Elijah Potterfa5d13965643
diff --cc Cargo.lock
index 22989b29a,f9c5cbbb7..f91688539
--- a/Cargo.lock
+++ b/Cargo.lock
@@@ -349,31 -395,45 +395,66 @@@ checksum = "d0a5c400df2834b80a4c3327b3a
[[package]]
name = "crunchy"
- version = "0.2.2"
+ version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
- checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7"
+ checksum = "43da5946c66ffcc7745f48db692ffbb10a83bfe0afd96235c5c2a4fb23994929"
+[[package]]
+name = "csv"
+version = "1.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "acdc4883a9c96732e4733212c01447ebd805833b7275a73ca3ee080fd77afdaf"
+dependencies = [
+ "csv-core",
+ "itoa",
+ "ryu",
+ "serde",
+]
+
+[[package]]
+name = "csv-core"
+version = "0.1.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5efa2b3d7902f4b634a20cae3c9c4e6209dc4779feb6863329607560143efa70"
+dependencies = [
+ "memchr",
+]
+
+ [[package]]
+ name = "darling"
+ version = "0.20.10"
+ source = "registry+https://github.com/rust-lang/crates.io-index"
+ checksum = "6f63b86c8a8826a49b8c21f08a2d07338eec8d900540f8630dc76284be802989"
+ dependencies = [
+ "darling_core",
+ "darling_macro",
+ ]
+
+ [[package]]
+ name = "darling_core"
+ version = "0.20.10"
+ source = "registry+https://github.com/rust-lang/crates.io-index"
+ checksum = "95133861a8032aaea082871032f5815eb9e98cef03fa916ab4500513994df9e5"
+ dependencies = [
+ "fnv",
+ "ident_case",
+ "proc-macro2",
+ "quote",
+ "strsim",
+ "syn 2.0.96",
+ ]
+
+ [[package]]
+ name = "darling_macro"
+ version = "0.20.10"
+ source = "registry+https://github.com/rust-lang/crates.io-index"
+ checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806"
+ dependencies = [
+ "darling_core",
+ "quote",
+ "syn 2.0.96",
+ ]
+
[[package]]
name = "dashmap"
version = "5.5.3"
@@@ -626,8 -692,8 +713,9 @@@ dependencies =
"harper-comments",
"harper-core",
"harper-literate-haskell",
+ "harper-stats",
"harper-typst",
+ "hashbrown 0.15.2",
"serde",
"serde_json",
]
@@@ -721,8 -796,8 +818,9 @@@ dependencies =
"harper-core",
"harper-html",
"harper-literate-haskell",
+ "harper-stats",
"harper-typst",
+ "indexmap",
"itertools 0.14.0",
"once_cell",
"open",
@@@ -735,19 -810,9 +833,19 @@@
"tracing-subscriber",
]
+[[package]]
+name = "harper-stats"
- version = "0.17.0"
++version = "0.25.0"
+dependencies = [
+ "csv",
+ "harper-core",
+ "serde",
+ "uuid",
+]
+
[[package]]
name = "harper-tree-sitter"
- version = "0.17.0"
+ version = "0.25.1"
dependencies = [
"harper-core",
"tree-sitter",
@@@ -2231,21 -2383,17 +2416,27 @@@ version = "0.2.2
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
+[[package]]
+name = "uuid"
+version = "1.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "744018581f9a3454a9e15beb8a33b017183f1e7c0cd170232a2d1453b23a51c4"
+dependencies = [
+ "getrandom",
+ "serde",
+]
+
[[package]]
name = "valuable"
- version = "0.1.0"
+ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
- checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d"
+ checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65"
+
+ [[package]]
+ name = "version_check"
+ version = "0.9.5"
+ source = "registry+https://github.com/rust-lang/crates.io-index"
+ checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
[[package]]
name = "walkdir"
diff --cc harper-cli/Cargo.toml
index 142360635,b451c05cf..2ccd226c1
--- a/harper-cli/Cargo.toml
+++ b/harper-cli/Cargo.toml
@@@ -7,16 -7,16 +7,17 @@@ publish = fals
repository = "https://github.com/automattic/harper"
[dependencies]
- anyhow = "1.0.95"
+ anyhow = "1.0.97"
ariadne = "0.4.1"
- clap = { version = "4.5.27", features = ["derive"] }
- harper-literate-haskell = { path = "../harper-literate-haskell", version = "0.17.0" }
- harper-core = { path = "../harper-core", version = "0.17.0" }
- harper-comments = { path = "../harper-comments", version = "0.17.0" }
- harper-typst = { path = "../harper-typst", version = "0.17.0" }
- harper-stats = { path = "../harper-stats", version = "0.17.0" }
- serde = { version = "1.0.214", features = ["derive"] }
- serde_json = "1.0.137"
+ clap = { version = "4.5.32", features = ["derive"] }
++harper-stats = { path = "../harper-stats", version = "0.25.0" }
+ harper-literate-haskell = { path = "../harper-literate-haskell", version = "0.25.0" }
+ harper-core = { path = "../harper-core", version = "0.25.0" }
+ harper-comments = { path = "../harper-comments", version = "0.25.0" }
+ harper-typst = { path = "../harper-typst", version = "0.25.0" }
+ hashbrown = "0.15.2"
+ serde = { version = "1.0.219", features = ["derive"] }
+ serde_json = "1.0.140"
[features]
default = []
diff --cc harper-cli/src/main.rs
index fbf1b9df1,c753fa98c..f6f0ccb2e
--- a/harper-cli/src/main.rs
+++ b/harper-cli/src/main.rs
@@@ -1,19 -1,21 +1,23 @@@
#![doc = include_str!("../README.md")]
-use std::collections::BTreeMap;
+use std::collections::{BTreeMap, HashMap};
+use std::fs::File;
+use std::io::BufReader;
use std::path::{Path, PathBuf};
+ use std::process;
use anyhow::format_err;
use ariadne::{Color, Label, Report, ReportKind, Source};
use clap::Parser;
use harper_comments::CommentParser;
- use harper_core::linting::{LintGroup, LintGroupConfig, Linter};
+ use harper_core::linting::{LintGroup, Linter};
use harper_core::parsers::{Markdown, MarkdownOptions};
- use harper_core::{remove_overlaps, Dictionary, Document, FstDictionary, TokenKind};
+ use harper_core::{
+ remove_overlaps, CharStringExt, Dialect, Dictionary, Document, FstDictionary,
+ MutableDictionary, TokenKind, TokenStringExt, WordId,
+ };
use harper_literate_haskell::LiterateHaskellParser;
-use hashbrown::HashMap;
+use harper_stats::Stats;
use serde::Serialize;
/// A debugging tool for the Harper grammar checker.
@@@ -44,12 -52,17 +54,19 @@@ enum Args
},
/// Get the metadata associated with a particular word.
Metadata { word: String },
+ /// Get all the forms of a word using the affixes.
+ Forms { line: String },
/// Emit a decompressed, line-separated list of the words in Harper's dictionary.
Words,
+ /// Summarize a lint record
+ SummarizeLintRecord { file: PathBuf },
/// Print the default config with descriptions.
Config,
+ /// Print a list of all the words in a document, sorted by frequency.
+ MineWords {
+ /// The document to mine words from.
+ file: PathBuf,
+ },
}
fn main() -> anyhow::Result<()> {
@@@ -174,16 -200,73 +204,83 @@@
Ok(())
}
+ Args::SummarizeLintRecord { file } => {
+ let file = File::open(file)?;
+ let mut reader = BufReader::new(file);
+ let stats = Stats::read_csv(&mut reader)?;
+
+ let summary = stats.summarize_lints_applied();
+ println!("{summary}");
+
+ Ok(())
+ }
+ Args::Forms { line } => {
+ let (word, annot) = line_to_parts(&line);
+
+ let curated_word_list = include_str!("../../harper-core/dictionary.dict");
+ let dict_lines = curated_word_list.split('\n');
+
+ let mut entry_in_dict = None;
+
+ // Check if the word is contained in the list.
+ for dict_line in dict_lines {
+ let (dict_word, dict_annot) = line_to_parts(dict_line);
+
+ if dict_word == word {
+ entry_in_dict = Some((dict_word, dict_annot));
+ break;
+ }
+ }
+
+ let summary = match &entry_in_dict {
+ Some((dict_word, dict_annot)) => {
+ let mut status_summary = if dict_annot.is_empty() {
+ format!(
+ "'{}' is already in the dictionary but not annotated.",
+ dict_word
+ )
+ } else {
+ format!(
+ "'{}' is already in the dictionary with annotation `{}`.",
+ dict_word, dict_annot
+ )
+ };
+
+ if !annot.is_empty() {
+ if annot.as_str() != dict_annot.as_str() {
+ status_summary
+ .push_str("\n Your annotations differ from the dictionary.\n");
+ } else {
+ status_summary
+ .push_str("\n Your annotations are the same as the dictionary.\n");
+ }
+ }
+
+ status_summary
+ }
+ None => format!("'{}' is not in the dictionary yet.", word),
+ };
+
+ println!("{summary}");
+
+ if let Some((dict_word, dict_annot)) = &entry_in_dict {
+ println!("Old, from the dictionary:");
+ print_word_derivations(dict_word, dict_annot, &FstDictionary::curated());
+ };
+
+ if !annot.is_empty() {
+ let rune_words = format!("1\n{line}");
+ let dict = MutableDictionary::from_rune_files(
+ &rune_words,
+ include_str!("../../harper-core/affixes.json"),
+ )?;
+
+ println!("New, from you:");
+ print_word_derivations(&word, &annot, &dict);
+ }
+
+ Ok(())
+ }
Args::Config => {
#[derive(Serialize)]
struct Config {
diff --cc harper-core/src/linting/lint_kind.rs
index 000000000,1d7ea1672..177de20ab
mode 000000,100644..100644
--- a/harper-core/src/linting/lint_kind.rs
+++ b/harper-core/src/linting/lint_kind.rs
@@@ -1,0 -1,58 +1,73 @@@
+ use std::fmt::Display;
+
+ use is_macro::Is;
+ use serde::{Deserialize, Serialize};
+
+ /// The general category a [`Lint`](super::Lint) falls into.
+ /// There's no reason not to add a new item here if you are adding a new rule that doesn't fit
+ /// the existing categories.
-#[derive(Debug, Clone, Copy, Serialize, Deserialize, Is, Default, Hash, PartialEq)]
++#[derive(Debug, Clone, Copy, Serialize, Deserialize, Is, Default, Hash, PartialEq, Eq)]
+ pub enum LintKind {
+ /// This should only be used by linters doing spellcheck on individual words.
+ Spelling,
+ Capitalization,
+ Style,
+ Formatting,
+ Repetition,
+ Enhancement,
+ Readability,
+ WordChoice,
+ #[default]
+ Miscellaneous,
+ }
+
+ impl LintKind {
++ pub fn new_from_str(s: &str) -> Option<Self> {
++ Some(match s {
++ "Spelling" => LintKind::Spelling,
++ "Capitalization" => LintKind::Capitalization,
++ "Formatting" => LintKind::Formatting,
++ "Repetition" => LintKind::Repetition,
++ "Readability" => LintKind::Readability,
++ "Miscellaneous" => LintKind::Miscellaneous,
++ "Enhancement" => LintKind::Enhancement,
++ "Word Choice" => LintKind::WordChoice,
++ "Style" => LintKind::Style,
++ _ => return None,
++ })
++ }
++
+ /// Produce a string representation, which can be used as keys in a map or CSS variables.
+ pub fn to_string_key(&self) -> String {
+ match self {
+ LintKind::Spelling => "Spelling",
+ LintKind::Capitalization => "Capitalization",
+ LintKind::Formatting => "Formatting",
+ LintKind::Repetition => "Repetition",
+ LintKind::Readability => "Readability",
+ LintKind::Miscellaneous => "Miscellaneous",
+ LintKind::Enhancement => "Enhancement",
+ LintKind::WordChoice => "WordChoice",
+ LintKind::Style => "Style",
+ }
+ .to_owned()
+ }
+ }
+
+ impl Display for LintKind {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ let s = match self {
+ LintKind::Spelling => "Spelling",
+ LintKind::Capitalization => "Capitalization",
+ LintKind::Formatting => "Formatting",
+ LintKind::Repetition => "Repetition",
+ LintKind::Readability => "Readability",
+ LintKind::Miscellaneous => "Miscellaneous",
+ LintKind::Enhancement => "Enhancement",
+ LintKind::WordChoice => "Word Choice",
+ LintKind::Style => "Style",
+ };
+
+ write!(f, "{}", s)
+ }
+ }
diff --cc harper-ls/Cargo.toml
index bac4ef196,8ae127c5c..7dd7b0fc0
--- a/harper-ls/Cargo.toml
+++ b/harper-ls/Cargo.toml
@@@ -8,19 -8,18 +8,19 @@@ readme = "README.md
repository = "https://github.com/automattic/harper"
[dependencies]
- harper-literate-haskell = { path = "../harper-literate-haskell", version = "0.17.0" }
- harper-core = { path = "../harper-core", version = "0.17.0", features = ["concurrent"] }
- harper-stats = { path = "../harper-stats", version = "0.17.0" }
- harper-comments = { path = "../harper-comments", version = "0.17.0" }
- harper-typst = { path = "../harper-typst", version = "0.17.0" }
- harper-html = { path = "../harper-html", version = "0.17.0" }
+ harper-literate-haskell = { path = "../harper-literate-haskell", version = "0.25.0" }
++harper-stats = { path = "../harper-stats", version = "0.25.0" }
+ harper-core = { path = "../harper-core", version = "0.25.0", features = ["concurrent"] }
+ harper-comments = { path = "../harper-comments", version = "0.25.0" }
+ harper-typst = { path = "../harper-typst", version = "0.25.0" }
+ harper-html = { path = "../harper-html", version = "0.25.0" }
tower-lsp = "0.20.0"
- tokio = { version = "1.43.0", features = ["fs", "rt", "rt-multi-thread", "macros", "io-std", "io-util", "net"] }
- clap = { version = "4.5.27", features = ["derive"] }
- once_cell = "1.20.2"
+ tokio = { version = "1.44.1", features = ["fs", "rt", "rt-multi-thread", "macros", "io-std", "io-util", "net"] }
+ clap = { version = "4.5.32", features = ["derive"] }
+ once_cell = "1.20.3"
dirs = "6.0.0"
- anyhow = "1.0.95"
- serde_json = "1.0.137"
+ anyhow = "1.0.97"
+ serde_json = "1.0.140"
itertools = "0.14.0"
tracing = "0.1.41"
tracing-subscriber = "0.3.19"
diff --cc harper-ls/src/backend.rs
index d82f82c4c,20ec7781e..e5fccf96a
--- a/harper-ls/src/backend.rs
+++ b/harper-ls/src/backend.rs
@@@ -1,17 -1,13 +1,16 @@@
use std::collections::HashMap;
+use std::fs::OpenOptions;
+use std::io::{BufWriter, Write};
- use std::path::{Component, PathBuf};
+ use std::path::PathBuf;
use std::sync::Arc;
- use anyhow::{anyhow, Context, Result};
+ use anyhow::{Context, Result, anyhow};
+use futures::future::join;
use harper_comments::CommentParser;
- use harper_core::linting::{LintGroup, LintKind, Linter};
-use harper_core::linting::{LintGroup, LintGroupConfig};
++use harper_core::linting::{LintGroup, LintGroupConfig, LintKind};
use harper_core::parsers::{CollapseIdentifiers, IsolateEnglish, Markdown, Parser, PlainEnglish};
use harper_core::{
- Dictionary, Document, FstDictionary, FullDictionary, MergedDictionary, Token, TokenKind,
- WordMetadata,
+ Dialect, Dictionary, Document, FstDictionary, MergedDictionary, MutableDictionary, WordMetadata,
};
use harper_html::HtmlParser;
use harper_literate_haskell::LiterateHaskellParser;
@@@ -35,12 -30,9 +33,10 @@@ use tower_lsp::{Client, LanguageServer}
use tracing::{error, info, warn};
use crate::config::Config;
- use crate::diagnostics::{lint_to_code_actions, lints_to_diagnostics};
- use crate::dictionary_io::{load_dict, save_dict};
+ use crate::dictionary_io::{file_dict_name, load_dict, save_dict};
use crate::document_state::DocumentState;
use crate::git_commit_parser::GitCommitParser;
- use crate::pos_conv::range_to_span;
+use harper_stats::{LintRecord, Stats};
pub struct Backend {
client: Client,
diff --cc harper-ls/src/diagnostics.rs
index 13a8453aa,a7bad6a1b..23ec0e520
--- a/harper-ls/src/diagnostics.rs
+++ b/harper-ls/src/diagnostics.rs
@@@ -1,8 -1,7 +1,8 @@@
use std::collections::HashMap;
- use harper_core::linting::{Lint, Suggestion};
use harper_core::CharStringExt;
+ use harper_core::linting::{Lint, Suggestion};
+use serde_json::Value;
use tower_lsp::lsp_types::{
CodeAction, CodeActionKind, CodeActionOrCommand, Command, Diagnostic, TextEdit, Url,
WorkspaceEdit,
diff --cc harper-stats/Cargo.toml
index da80752a3,000000000..30d94b5ff
mode 100644,000000..100644
--- a/harper-stats/Cargo.toml
+++ b/harper-stats/Cargo.toml
@@@ -1,10 -1,0 +1,10 @@@
+[package]
+name = "harper-stats"
- version = "0.17.0"
++version = "0.25.0"
+edition = "2021"
+
+[dependencies]
+serde = { version = "1.0.217", features = ["derive"] }
- harper-core = { path = "../harper-core", version = "0.17.0", features = ["concurrent"] }
++harper-core = { path = "../harper-core", version = "0.25.0", features = ["concurrent"] }
+uuid = { version = "1.12.0", features = ["serde", "v4"] }
+csv = "1.3.1"