Skip to content

Offline, privacy-first grammar checker. Fast, open-source, Rust-powered

feat(stats): use new `FatStringToken` to make logs more readable

Elijah Potter1a7c860f3926
diff --git a/harper-core/src/document.rs b/harper-core/src/document.rs
index ca5d80c31..9c0d31006 100644
--- a/harper-core/src/document.rs
+++ b/harper-core/src/document.rs
@@ -10,7 +10,9 @@ use crate::patterns::{
};
use crate::punctuation::Punctuation;
use crate::vec_ext::VecExt;
-use crate::{Dictionary, FatToken, FstDictionary, Lrc, Token, TokenKind, TokenStringExt};
+use crate::{
+ Dictionary, FatStringToken, FatToken, FstDictionary, Lrc, Token, TokenKind, TokenStringExt,
+};
use crate::{NumberSuffix, Span};
/// A document containing some amount of lexed and parsed English text.
@@ -257,6 +259,11 @@ impl Document {
self.tokens().map(|token| token.to_fat(&self.source))
}
+ /// Get an iterator over all the tokens contained in the document.
+ pub fn fat_string_tokens(&self) -> impl Iterator<Item = FatStringToken> + '_ {
+ self.fat_tokens().map(|t| t.into())
+ }
+
pub fn get_span_content(&self, span: &Span) -> &[char] {
span.get_content(&self.source)
}
diff --git a/harper-core/src/fat_token.rs b/harper-core/src/fat_token.rs
index 63bf17a9a..00770ac19 100644
--- a/harper-core/src/fat_token.rs
+++ b/harper-core/src/fat_token.rs
@@ -1,6 +1,6 @@
use serde::{Deserialize, Serialize};
-use crate::TokenKind;
+use crate::{CharStringExt, TokenKind};
/// A [`Token`](crate::Token) that holds its content as a fat [`Vec<char>`] rather than as a
/// [`Span`](crate::Span).
@@ -9,3 +9,28 @@ pub struct FatToken {
pub content: Vec<char>,
pub kind: TokenKind,
}
+
+impl From<FatStringToken> for FatToken {
+ fn from(value: FatStringToken) -> Self {
+ Self {
+ content: value.content.chars().collect(),
+ kind: value.kind,
+ }
+ }
+}
+
+/// Similar to a [`FatToken`], but uses a [`String`] as the underlying store.
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, PartialOrd, Hash, Eq)]
+pub struct FatStringToken {
+ pub content: String,
+ pub kind: TokenKind,
+}
+
+impl From<FatToken> for FatStringToken {
+ fn from(value: FatToken) -> Self {
+ Self {
+ content: value.content.to_string(),
+ kind: value.kind,
+ }
+ }
+}
diff --git a/harper-core/src/lib.rs b/harper-core/src/lib.rs
index fccfda191..739dc8296 100644
--- a/harper-core/src/lib.rs
+++ b/harper-core/src/lib.rs
@@ -31,7 +31,7 @@ use std::collections::VecDeque;
pub use char_string::{CharString, CharStringExt};
pub use currency::Currency;
pub use document::Document;
-pub use fat_token::FatToken;
+pub use fat_token::{FatStringToken, FatToken};
pub use ignored_lints::IgnoredLints;
use linting::Lint;
pub use mask::{Mask, Masker};
diff --git a/harper-stats/src/lib.rs b/harper-stats/src/lib.rs
index f6eeb9ef2..739a29264 100644
--- a/harper-stats/src/lib.rs
+++ b/harper-stats/src/lib.rs
@@ -4,7 +4,7 @@ mod summary;
use std::io::{self, Read, Write};
use std::io::{BufRead, BufReader};
-use harper_core::{CharStringExt, TokenKind};
+use harper_core::TokenKind;
pub use record::Record;
pub use record::RecordKind;
use serde::Serialize;
diff --git a/harper-stats/src/record.rs b/harper-stats/src/record.rs
index fbc9eb3ae..7fe097f4f 100644
--- a/harper-stats/src/record.rs
+++ b/harper-stats/src/record.rs
@@ -1,6 +1,6 @@
use harper_core::{
linting::{Lint, LintGroupConfig, LintKind},
- Document, FatToken,
+ Document, FatStringToken,
};
use serde::{Deserialize, Serialize};
use uuid::Uuid;
@@ -28,7 +28,7 @@ impl Record {
pub enum RecordKind {
Lint {
kind: LintKind,
- context: Vec<FatToken>,
+ context: Vec<FatStringToken>,
},
LintConfigUpdate(LintGroupConfig),
}
@@ -37,7 +37,11 @@ impl RecordKind {
pub fn from_lint(lint: &Lint, doc: &Document) -> Self {
Self::Lint {
kind: lint.lint_kind,
- context: doc.fat_tokens_intersecting(lint.span),
+ context: doc
+ .fat_tokens_intersecting(lint.span)
+ .into_iter()
+ .map(|t| t.into())
+ .collect(),
}
}
}
@@ -67,10 +71,10 @@ mod tests {
let lcu = Self::LintConfigUpdate(arbitrary_lintconfig(g));
let context = Document::new_plain_english_curated(&String::arbitrary(g))
- .fat_tokens()
+ .fat_string_tokens()
.collect();
- let kind = g
+ let kind = *g
.choose(&[
LintKind::Spelling,
LintKind::Capitalization,
@@ -82,8 +86,7 @@ mod tests {
LintKind::WordChoice,
LintKind::Miscellaneous,
])
- .unwrap()
- .clone();
+ .unwrap();
g.choose(&[RecordKind::Lint { kind, context }, lcu])
.unwrap()
diff --git a/packages/web/src/routes/docs/contributors/local-stats/+page.md b/packages/web/src/routes/docs/contributors/local-stats/+page.md
index 6d47bd3d7..619c5ac92 100644
--- a/packages/web/src/routes/docs/contributors/local-stats/+page.md
+++ b/packages/web/src/routes/docs/contributors/local-stats/+page.md
@@ -23,12 +23,9 @@ It records specific events, along with some contextual information (like which w
The `stats.txt` file is formatted into lines (so it is easy to open in append-mode), each containing a JSON object.
```
-{"kind":{"Lint":{"kind":"Capitalization","context":[{"content":["i"],"kind":{"kind":"Word","value":{"noun":null,"pronoun":{"is_plural":false,"is_possessive":null,"person":null,"case":null},"verb":null,"adjective":null,"adverb":null,"conjunction":null,"swear":null,"dialect":null,"determiner":false,"preposition":false,"common":true,"derived_from":null}}}]}},"when":1743428384,"uuid":"b095c60b-6ae0-4c82-bcb8-c7de737d6509"}
-{"kind":{"Lint":{"kind":"Spelling","context":[{"content":["m","i","s","p","e","l","l"],"kind":{"kind":"Word","value":null}}]}},"when":1743428388,"uuid":"70850697-a3f1-4eb6-94c8-be0be6d5b16b"}
-{"kind":{"Lint":{"kind":"Spelling","context":[{"content":["t","h","i","g","s"],"kind":{"kind":"Word","value":null}}]}},"when":1743428392,"uuid":"c0ff601a-f285-4f6d-9c47-1536213bee3f"}
-{"kind":{"Lint":{"kind":"Spelling","context":[{"content":["t","h","i","g","s"],"kind":{"kind":"Word","value":null}}]}},"when":1743428397,"uuid":"c49a67c2-4d03-460a-b450-1ae9b20f243c"}
-{"kind":{"Lint":{"kind":"Spelling","context":[{"content":["m","i","s","p","e","l","l","e","d"],"kind":{"kind":"Word","value":null}}]}},"when":1743429129,"uuid":"6ee7634e-dc8d-489b-9e13-f3b7dcd50741"}
-{"kind":{"Lint":{"kind":"Miscellaneous","context":[{"content":["a"],"kind":{"kind":"Word","value":{"noun":null,"pronoun":null,"verb":null,"adjective":null,"adverb":null,"conjunction":null,"swear":null,"dialect":null,"determiner":true,"preposition":true,"common":true,"derived_from":null}}}]}},"when":1743525210,"uuid":"cbb2ca64-2073-4714-a346-628bb31cfc65"}
+{"kind":{"Lint":{"kind":"Spelling","context":[{"content":"mispelled","kind":{"kind":"Word","value":null}}]}},"when":1743696274,"uuid":"39d29bd0-5eb1-4bad-89ee-5a48531a4cbe"}
+{"kind":{"Lint":{"kind":"Spelling","context":[{"content":"isnt","kind":{"kind":"Word","value":null}}]}},"when":1743696281,"uuid":"22e1ca15-e583-49c5-9da3-bc7e625d9682"}
+{"kind":{"Lint":{"kind":"Spelling","context":[{"content":"Teasting","kind":{"kind":"Word","value":null}}]}},"when":1743696288,"uuid":"bd955190-a4d9-4f3e-b7df-d4bf6f12a415"}
```
In `harper-ls` it is written to the Harper `data` directory.