Skip to content

Offline, privacy-first grammar checker. Fast, open-source, Rust-powered

feat(stats): now includes lint context + misspelled words

Elijah Potter29f03d86bfde
diff --git a/harper-core/src/document.rs b/harper-core/src/document.rs
index 8ec8916f5..95be7b3f2 100644
--- a/harper-core/src/document.rs
+++ b/harper-core/src/document.rs
@@ -37,6 +37,18 @@ impl Document {
.collect()
}
+ /// Locate all the tokens that intersect a provided span and convert them to [`FatToken`]s.
+ ///
+ /// Desperately needs optimization.
+ pub fn fat_tokens_intersecting(&self, span: Span) -> Vec<FatToken> {
+ let indices = self.token_indices_intersecting(span);
+
+ indices
+ .into_iter()
+ .map(|i| self.tokens[i].to_fat(&self.source))
+ .collect()
+ }
+
/// Lexes and parses text to produce a document using a provided language
/// parser and dictionary.
pub fn new(text: &str, parser: &impl Parser, dictionary: &impl Dictionary) -> Self {
diff --git a/harper-core/src/fat_token.rs b/harper-core/src/fat_token.rs
index 9f1fbd830..63bf17a9a 100644
--- a/harper-core/src/fat_token.rs
+++ b/harper-core/src/fat_token.rs
@@ -4,7 +4,7 @@ use crate::TokenKind;
/// A [`Token`](crate::Token) that holds its content as a fat [`Vec<char>`] rather than as a
/// [`Span`](crate::Span).
-#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, PartialOrd, Hash)]
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, PartialOrd, Hash, Eq)]
pub struct FatToken {
pub content: Vec<char>,
pub kind: TokenKind,
diff --git a/harper-ls/src/backend.rs b/harper-ls/src/backend.rs
index cddb3a75a..d1dce5211 100644
--- a/harper-ls/src/backend.rs
+++ b/harper-ls/src/backend.rs
@@ -1,4 +1,3 @@
-use harper_stats::RecordKind;
use std::collections::HashMap;
use std::fs::OpenOptions;
use std::io::{BufWriter, Write};
@@ -8,7 +7,7 @@ use std::sync::Arc;
use anyhow::{Context, Result, anyhow};
use futures::future::join;
use harper_comments::CommentParser;
-use harper_core::linting::{LintGroup, LintGroupConfig, LintKind};
+use harper_core::linting::{LintGroup, LintGroupConfig};
use harper_core::parsers::{CollapseIdentifiers, IsolateEnglish, Markdown, Parser, PlainEnglish};
use harper_core::{
Dialect, Dictionary, Document, FstDictionary, MergedDictionary, MutableDictionary, WordMetadata,
@@ -538,12 +537,12 @@ impl LanguageServer for Backend {
match params.command.as_str() {
"HarperRecordLint" => {
- let Some(kind) = LintKind::new_from_str(&first) else {
- error!("Unable to deserialize LintKind.");
+ let Ok(kind) = serde_json::from_str(&first) else {
+ error!("Unable to deserialize RecordKind.");
return Ok(None);
};
- let record = Record::now(RecordKind::Lint(kind));
+ let record = Record::now(kind);
let mut stats = self.stats.write().await;
stats.records.push(record);
diff --git a/harper-ls/src/diagnostics.rs b/harper-ls/src/diagnostics.rs
index 23ec0e520..93296524c 100644
--- a/harper-ls/src/diagnostics.rs
+++ b/harper-ls/src/diagnostics.rs
@@ -1,7 +1,8 @@
use std::collections::HashMap;
-use harper_core::CharStringExt;
use harper_core::linting::{Lint, Suggestion};
+use harper_core::{CharStringExt, Document};
+use harper_stats::RecordKind;
use serde_json::Value;
use tower_lsp::lsp_types::{
CodeAction, CodeActionKind, CodeActionOrCommand, Command, Diagnostic, TextEdit, Url,
@@ -25,10 +26,11 @@ pub fn lints_to_diagnostics(
pub fn lint_to_code_actions<'a>(
lint: &'a Lint,
url: &'a Url,
- source: &'a [char],
+ document: &Document,
config: &CodeActionConfig,
) -> Vec<CodeActionOrCommand> {
let mut results = Vec::new();
+ let source = document.get_source();
results.extend(
lint.suggestions
@@ -64,7 +66,9 @@ pub fn lint_to_code_actions<'a>(
command: Some(Command {
title: "Record lint statistic".to_owned(),
command: "HarperRecordLint".to_owned(),
- arguments: Some(vec![Value::String(lint.lint_kind.to_string())]),
+ arguments: Some(vec![Value::String(
+ serde_json::to_string(&RecordKind::from_lint(lint, document)).unwrap(),
+ )]),
}),
is_preferred: None,
disabled: None,
diff --git a/harper-ls/src/document_state.rs b/harper-ls/src/document_state.rs
index 1823136ae..7dfa113a3 100644
--- a/harper-ls/src/document_state.rs
+++ b/harper-ls/src/document_state.rs
@@ -62,7 +62,7 @@ impl DocumentState {
.into_iter()
.filter(|lint| lint.span.overlaps_with(span))
.flat_map(|lint| {
- lint_to_code_actions(&lint, &self.url, source_chars, code_action_config)
+ lint_to_code_actions(&lint, &self.url, &self.document, code_action_config)
})
.collect();
diff --git a/harper-stats/src/lib.rs b/harper-stats/src/lib.rs
index 9ffeff42d..f6eeb9ef2 100644
--- a/harper-stats/src/lib.rs
+++ b/harper-stats/src/lib.rs
@@ -4,6 +4,7 @@ mod summary;
use std::io::{self, Read, Write};
use std::io::{BufRead, BufReader};
+use harper_core::{CharStringExt, TokenKind};
pub use record::Record;
pub use record::RecordKind;
use serde::Serialize;
@@ -29,7 +30,15 @@ impl Stats {
for record in &self.records {
match &record.kind {
- RecordKind::Lint(lint_kind) => summary.inc_lint_count(*lint_kind),
+ RecordKind::Lint { kind, context } => {
+ summary.inc_lint_count(*kind);
+
+ for tok in context {
+ if let TokenKind::Word(None) = tok.kind {
+ summary.inc_misspelled_count(tok.content.to_string());
+ }
+ }
+ }
RecordKind::LintConfigUpdate(lint_group_config) => {
summary.final_config = lint_group_config.clone();
}
diff --git a/harper-stats/src/record.rs b/harper-stats/src/record.rs
index eee140a04..fbc9eb3ae 100644
--- a/harper-stats/src/record.rs
+++ b/harper-stats/src/record.rs
@@ -1,4 +1,7 @@
-use harper_core::linting::{LintGroupConfig, LintKind};
+use harper_core::{
+ linting::{Lint, LintGroupConfig, LintKind},
+ Document, FatToken,
+};
use serde::{Deserialize, Serialize};
use uuid::Uuid;
@@ -23,13 +26,28 @@ impl Record {
#[derive(Debug, Deserialize, Serialize, Clone, Eq, PartialEq)]
pub enum RecordKind {
- Lint(LintKind),
+ Lint {
+ kind: LintKind,
+ context: Vec<FatToken>,
+ },
LintConfigUpdate(LintGroupConfig),
}
+impl RecordKind {
+ pub fn from_lint(lint: &Lint, doc: &Document) -> Self {
+ Self::Lint {
+ kind: lint.lint_kind,
+ context: doc.fat_tokens_intersecting(lint.span),
+ }
+ }
+}
+
#[cfg(test)]
mod tests {
- use harper_core::linting::{LintGroupConfig, LintKind};
+ use harper_core::{
+ linting::{LintGroupConfig, LintKind},
+ Document,
+ };
use quickcheck::Arbitrary;
use super::{Record, RecordKind};
@@ -48,20 +66,28 @@ mod tests {
fn arbitrary(g: &mut quickcheck::Gen) -> Self {
let lcu = Self::LintConfigUpdate(arbitrary_lintconfig(g));
- g.choose(&[
- Self::Lint(LintKind::Spelling),
- Self::Lint(LintKind::Capitalization),
- Self::Lint(LintKind::Style),
- Self::Lint(LintKind::Formatting),
- Self::Lint(LintKind::Repetition),
- Self::Lint(LintKind::Enhancement),
- Self::Lint(LintKind::Readability),
- Self::Lint(LintKind::WordChoice),
- Self::Lint(LintKind::Miscellaneous),
- lcu,
- ])
- .unwrap()
- .clone()
+ let context = Document::new_plain_english_curated(&String::arbitrary(g))
+ .fat_tokens()
+ .collect();
+
+ let kind = g
+ .choose(&[
+ LintKind::Spelling,
+ LintKind::Capitalization,
+ LintKind::Style,
+ LintKind::Formatting,
+ LintKind::Repetition,
+ LintKind::Enhancement,
+ LintKind::Readability,
+ LintKind::WordChoice,
+ LintKind::Miscellaneous,
+ ])
+ .unwrap()
+ .clone();
+
+ g.choose(&[RecordKind::Lint { kind, context }, lcu])
+ .unwrap()
+ .clone()
}
}
diff --git a/harper-stats/src/summary.rs b/harper-stats/src/summary.rs
index c27aef956..ac2e82393 100644
--- a/harper-stats/src/summary.rs
+++ b/harper-stats/src/summary.rs
@@ -5,6 +5,8 @@ use harper_core::linting::{LintGroupConfig, LintKind};
pub struct Summary {
pub lint_counts: HashMap<LintKind, u32>,
pub final_config: LintGroupConfig,
+ // The most common misspelled words.
+ pub misspelled: HashMap<String, u32>,
}
impl Summary {
@@ -12,6 +14,7 @@ impl Summary {
Self {
lint_counts: HashMap::new(),
final_config: LintGroupConfig::default(),
+ misspelled: HashMap::new(),
}
}
@@ -23,6 +26,15 @@ impl Summary {
.or_insert(1);
}
+ /// Increment the count for a particular misspelled word.
+ pub fn inc_misspelled_count(&mut self, word: impl AsRef<str>) {
+ if let Some(counter) = self.misspelled.get_mut(word.as_ref()) {
+ *counter += 1
+ } else {
+ self.misspelled.insert(word.as_ref().to_owned(), 1);
+ }
+ }
+
/// Get the count for a particular lint kind.
pub fn get_count(&self, kind: LintKind) -> u32 {
self.lint_counts.get(&kind).copied().unwrap_or(0)
@@ -37,10 +49,28 @@ impl Default for Summary {
impl Display for Summary {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ writeln!(f, "`LintKind` counts")?;
+ writeln!(f, "=================")?;
+
for (kind, count) in &self.lint_counts {
writeln!(f, "{kind}\t{count}")?;
}
+ writeln!(f, "Misspelling counts")?;
+ writeln!(f, "=================")?;
+
+ let mut misspelled: Vec<_> = self
+ .misspelled
+ .iter()
+ .map(|(a, b)| (a.clone(), *b))
+ .collect();
+
+ misspelled.sort_by_key(|(_a, b)| u32::MAX - b);
+
+ for (kind, count) in &misspelled {
+ writeln!(f, "{kind}\t{count}")?;
+ }
+
Ok(())
}
}
diff --git a/harper-wasm/src/lib.rs b/harper-wasm/src/lib.rs
index 55afdc6b2..bb9ff15b2 100644
--- a/harper-wasm/src/lib.rs
+++ b/harper-wasm/src/lib.rs
@@ -299,9 +299,15 @@ impl Linter {
) -> Result<String, String> {
let mut source = lint.source.clone();
+ let doc = Document::new_from_vec(
+ source.clone().into(),
+ &lint.language.create_parser(),
+ &self.dictionary,
+ );
+
self.stats
.records
- .push(Record::now(RecordKind::Lint(lint.inner.lint_kind)));
+ .push(Record::now(RecordKind::from_lint(&lint.inner, &doc)));
suggestion.inner.apply(lint.inner.span, &mut source);