Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
feat(stats): now includes lint context + misspelled words
Elijah Potter29f03d86bfde
diff --git a/harper-core/src/document.rs b/harper-core/src/document.rsindex 8ec8916f5..95be7b3f2 100644--- a/harper-core/src/document.rs+++ b/harper-core/src/document.rs@@ -37,6 +37,18 @@ impl Document {.collect()}+ /// Locate all the tokens that intersect a provided span and convert them to [`FatToken`]s.+ ///+ /// Desperately needs optimization.+ pub fn fat_tokens_intersecting(&self, span: Span) -> Vec<FatToken> {+ let indices = self.token_indices_intersecting(span);++ indices+ .into_iter()+ .map(|i| self.tokens[i].to_fat(&self.source))+ .collect()+ }+/// Lexes and parses text to produce a document using a provided language/// parser and dictionary.pub fn new(text: &str, parser: &impl Parser, dictionary: &impl Dictionary) -> Self {diff --git a/harper-core/src/fat_token.rs b/harper-core/src/fat_token.rsindex 9f1fbd830..63bf17a9a 100644--- a/harper-core/src/fat_token.rs+++ b/harper-core/src/fat_token.rs@@ -4,7 +4,7 @@ use crate::TokenKind;/// A [`Token`](crate::Token) that holds its content as a fat [`Vec<char>`] rather than as a/// [`Span`](crate::Span).-#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, PartialOrd, Hash)]+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, PartialOrd, Hash, Eq)]pub struct FatToken {pub content: Vec<char>,pub kind: TokenKind,diff --git a/harper-ls/src/backend.rs b/harper-ls/src/backend.rsindex cddb3a75a..d1dce5211 100644--- a/harper-ls/src/backend.rs+++ b/harper-ls/src/backend.rs@@ -1,4 +1,3 @@-use harper_stats::RecordKind;use std::collections::HashMap;use std::fs::OpenOptions;use std::io::{BufWriter, Write};@@ -8,7 +7,7 @@ use std::sync::Arc;use anyhow::{Context, Result, anyhow};use futures::future::join;use harper_comments::CommentParser;-use harper_core::linting::{LintGroup, LintGroupConfig, LintKind};+use harper_core::linting::{LintGroup, LintGroupConfig};use harper_core::parsers::{CollapseIdentifiers, IsolateEnglish, Markdown, Parser, PlainEnglish};use harper_core::{Dialect, Dictionary, Document, FstDictionary, MergedDictionary, MutableDictionary, WordMetadata,@@ -538,12 +537,12 @@ impl LanguageServer for Backend {match params.command.as_str() {"HarperRecordLint" => {- let Some(kind) = LintKind::new_from_str(&first) else {- error!("Unable to deserialize LintKind.");+ let Ok(kind) = serde_json::from_str(&first) else {+ error!("Unable to deserialize RecordKind.");return Ok(None);};- let record = Record::now(RecordKind::Lint(kind));+ let record = Record::now(kind);let mut stats = self.stats.write().await;stats.records.push(record);diff --git a/harper-ls/src/diagnostics.rs b/harper-ls/src/diagnostics.rsindex 23ec0e520..93296524c 100644--- a/harper-ls/src/diagnostics.rs+++ b/harper-ls/src/diagnostics.rs@@ -1,7 +1,8 @@use std::collections::HashMap;-use harper_core::CharStringExt;use harper_core::linting::{Lint, Suggestion};+use harper_core::{CharStringExt, Document};+use harper_stats::RecordKind;use serde_json::Value;use tower_lsp::lsp_types::{CodeAction, CodeActionKind, CodeActionOrCommand, Command, Diagnostic, TextEdit, Url,@@ -25,10 +26,11 @@ pub fn lints_to_diagnostics(pub fn lint_to_code_actions<'a>(lint: &'a Lint,url: &'a Url,- source: &'a [char],+ document: &Document,config: &CodeActionConfig,) -> Vec<CodeActionOrCommand> {let mut results = Vec::new();+ let source = document.get_source();results.extend(lint.suggestions@@ -64,7 +66,9 @@ pub fn lint_to_code_actions<'a>(command: Some(Command {title: "Record lint statistic".to_owned(),command: "HarperRecordLint".to_owned(),- arguments: Some(vec![Value::String(lint.lint_kind.to_string())]),+ arguments: Some(vec![Value::String(+ serde_json::to_string(&RecordKind::from_lint(lint, document)).unwrap(),+ )]),}),is_preferred: None,disabled: None,diff --git a/harper-ls/src/document_state.rs b/harper-ls/src/document_state.rsindex 1823136ae..7dfa113a3 100644--- a/harper-ls/src/document_state.rs+++ b/harper-ls/src/document_state.rs@@ -62,7 +62,7 @@ impl DocumentState {.into_iter().filter(|lint| lint.span.overlaps_with(span)).flat_map(|lint| {- lint_to_code_actions(&lint, &self.url, source_chars, code_action_config)+ lint_to_code_actions(&lint, &self.url, &self.document, code_action_config)}).collect();diff --git a/harper-stats/src/lib.rs b/harper-stats/src/lib.rsindex 9ffeff42d..f6eeb9ef2 100644--- a/harper-stats/src/lib.rs+++ b/harper-stats/src/lib.rs@@ -4,6 +4,7 @@ mod summary;use std::io::{self, Read, Write};use std::io::{BufRead, BufReader};+use harper_core::{CharStringExt, TokenKind};pub use record::Record;pub use record::RecordKind;use serde::Serialize;@@ -29,7 +30,15 @@ impl Stats {for record in &self.records {match &record.kind {- RecordKind::Lint(lint_kind) => summary.inc_lint_count(*lint_kind),+ RecordKind::Lint { kind, context } => {+ summary.inc_lint_count(*kind);++ for tok in context {+ if let TokenKind::Word(None) = tok.kind {+ summary.inc_misspelled_count(tok.content.to_string());+ }+ }+ }RecordKind::LintConfigUpdate(lint_group_config) => {summary.final_config = lint_group_config.clone();}diff --git a/harper-stats/src/record.rs b/harper-stats/src/record.rsindex eee140a04..fbc9eb3ae 100644--- a/harper-stats/src/record.rs+++ b/harper-stats/src/record.rs@@ -1,4 +1,7 @@-use harper_core::linting::{LintGroupConfig, LintKind};+use harper_core::{+ linting::{Lint, LintGroupConfig, LintKind},+ Document, FatToken,+};use serde::{Deserialize, Serialize};use uuid::Uuid;@@ -23,13 +26,28 @@ impl Record {#[derive(Debug, Deserialize, Serialize, Clone, Eq, PartialEq)]pub enum RecordKind {- Lint(LintKind),+ Lint {+ kind: LintKind,+ context: Vec<FatToken>,+ },LintConfigUpdate(LintGroupConfig),}+impl RecordKind {+ pub fn from_lint(lint: &Lint, doc: &Document) -> Self {+ Self::Lint {+ kind: lint.lint_kind,+ context: doc.fat_tokens_intersecting(lint.span),+ }+ }+}+#[cfg(test)]mod tests {- use harper_core::linting::{LintGroupConfig, LintKind};+ use harper_core::{+ linting::{LintGroupConfig, LintKind},+ Document,+ };use quickcheck::Arbitrary;use super::{Record, RecordKind};@@ -48,20 +66,28 @@ mod tests {fn arbitrary(g: &mut quickcheck::Gen) -> Self {let lcu = Self::LintConfigUpdate(arbitrary_lintconfig(g));- g.choose(&[- Self::Lint(LintKind::Spelling),- Self::Lint(LintKind::Capitalization),- Self::Lint(LintKind::Style),- Self::Lint(LintKind::Formatting),- Self::Lint(LintKind::Repetition),- Self::Lint(LintKind::Enhancement),- Self::Lint(LintKind::Readability),- Self::Lint(LintKind::WordChoice),- Self::Lint(LintKind::Miscellaneous),- lcu,- ])- .unwrap()- .clone()+ let context = Document::new_plain_english_curated(&String::arbitrary(g))+ .fat_tokens()+ .collect();++ let kind = g+ .choose(&[+ LintKind::Spelling,+ LintKind::Capitalization,+ LintKind::Style,+ LintKind::Formatting,+ LintKind::Repetition,+ LintKind::Enhancement,+ LintKind::Readability,+ LintKind::WordChoice,+ LintKind::Miscellaneous,+ ])+ .unwrap()+ .clone();++ g.choose(&[RecordKind::Lint { kind, context }, lcu])+ .unwrap()+ .clone()}}diff --git a/harper-stats/src/summary.rs b/harper-stats/src/summary.rsindex c27aef956..ac2e82393 100644--- a/harper-stats/src/summary.rs+++ b/harper-stats/src/summary.rs@@ -5,6 +5,8 @@ use harper_core::linting::{LintGroupConfig, LintKind};pub struct Summary {pub lint_counts: HashMap<LintKind, u32>,pub final_config: LintGroupConfig,+ // The most common misspelled words.+ pub misspelled: HashMap<String, u32>,}impl Summary {@@ -12,6 +14,7 @@ impl Summary {Self {lint_counts: HashMap::new(),final_config: LintGroupConfig::default(),+ misspelled: HashMap::new(),}}@@ -23,6 +26,15 @@ impl Summary {.or_insert(1);}+ /// Increment the count for a particular misspelled word.+ pub fn inc_misspelled_count(&mut self, word: impl AsRef<str>) {+ if let Some(counter) = self.misspelled.get_mut(word.as_ref()) {+ *counter += 1+ } else {+ self.misspelled.insert(word.as_ref().to_owned(), 1);+ }+ }+/// Get the count for a particular lint kind.pub fn get_count(&self, kind: LintKind) -> u32 {self.lint_counts.get(&kind).copied().unwrap_or(0)@@ -37,10 +49,28 @@ impl Default for Summary {impl Display for Summary {fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ writeln!(f, "`LintKind` counts")?;+ writeln!(f, "=================")?;+for (kind, count) in &self.lint_counts {writeln!(f, "{kind}\t{count}")?;}+ writeln!(f, "Misspelling counts")?;+ writeln!(f, "=================")?;++ let mut misspelled: Vec<_> = self+ .misspelled+ .iter()+ .map(|(a, b)| (a.clone(), *b))+ .collect();++ misspelled.sort_by_key(|(_a, b)| u32::MAX - b);++ for (kind, count) in &misspelled {+ writeln!(f, "{kind}\t{count}")?;+ }+Ok(())}}diff --git a/harper-wasm/src/lib.rs b/harper-wasm/src/lib.rsindex 55afdc6b2..bb9ff15b2 100644--- a/harper-wasm/src/lib.rs+++ b/harper-wasm/src/lib.rs@@ -299,9 +299,15 @@ impl Linter {) -> Result<String, String> {let mut source = lint.source.clone();+ let doc = Document::new_from_vec(+ source.clone().into(),+ &lint.language.create_parser(),+ &self.dictionary,+ );+self.stats.records- .push(Record::now(RecordKind::Lint(lint.inner.lint_kind)));+ .push(Record::now(RecordKind::from_lint(&lint.inner, &doc)));suggestion.inner.apply(lint.inner.span, &mut source);