Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
feat(stats): use new `FatStringToken` to make logs more readable
Elijah Potter1a7c860f3926
diff --git a/harper-core/src/document.rs b/harper-core/src/document.rsindex ca5d80c31..9c0d31006 100644--- a/harper-core/src/document.rs+++ b/harper-core/src/document.rs@@ -10,7 +10,9 @@ use crate::patterns::{};use crate::punctuation::Punctuation;use crate::vec_ext::VecExt;-use crate::{Dictionary, FatToken, FstDictionary, Lrc, Token, TokenKind, TokenStringExt};+use crate::{+ Dictionary, FatStringToken, FatToken, FstDictionary, Lrc, Token, TokenKind, TokenStringExt,+};use crate::{NumberSuffix, Span};/// A document containing some amount of lexed and parsed English text.@@ -257,6 +259,11 @@ impl Document {self.tokens().map(|token| token.to_fat(&self.source))}+ /// Get an iterator over all the tokens contained in the document.+ pub fn fat_string_tokens(&self) -> impl Iterator<Item = FatStringToken> + '_ {+ self.fat_tokens().map(|t| t.into())+ }+pub fn get_span_content(&self, span: &Span) -> &[char] {span.get_content(&self.source)}diff --git a/harper-core/src/fat_token.rs b/harper-core/src/fat_token.rsindex 63bf17a9a..00770ac19 100644--- a/harper-core/src/fat_token.rs+++ b/harper-core/src/fat_token.rs@@ -1,6 +1,6 @@use serde::{Deserialize, Serialize};-use crate::TokenKind;+use crate::{CharStringExt, TokenKind};/// A [`Token`](crate::Token) that holds its content as a fat [`Vec<char>`] rather than as a/// [`Span`](crate::Span).@@ -9,3 +9,28 @@ pub struct FatToken {pub content: Vec<char>,pub kind: TokenKind,}++impl From<FatStringToken> for FatToken {+ fn from(value: FatStringToken) -> Self {+ Self {+ content: value.content.chars().collect(),+ kind: value.kind,+ }+ }+}++/// Similar to a [`FatToken`], but uses a [`String`] as the underlying store.+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, PartialOrd, Hash, Eq)]+pub struct FatStringToken {+ pub content: String,+ pub kind: TokenKind,+}++impl From<FatToken> for FatStringToken {+ fn from(value: FatToken) -> Self {+ Self {+ content: value.content.to_string(),+ kind: value.kind,+ }+ }+}diff --git a/harper-core/src/lib.rs b/harper-core/src/lib.rsindex fccfda191..739dc8296 100644--- a/harper-core/src/lib.rs+++ b/harper-core/src/lib.rs@@ -31,7 +31,7 @@ use std::collections::VecDeque;pub use char_string::{CharString, CharStringExt};pub use currency::Currency;pub use document::Document;-pub use fat_token::FatToken;+pub use fat_token::{FatStringToken, FatToken};pub use ignored_lints::IgnoredLints;use linting::Lint;pub use mask::{Mask, Masker};diff --git a/harper-stats/src/lib.rs b/harper-stats/src/lib.rsindex f6eeb9ef2..739a29264 100644--- a/harper-stats/src/lib.rs+++ b/harper-stats/src/lib.rs@@ -4,7 +4,7 @@ mod summary;use std::io::{self, Read, Write};use std::io::{BufRead, BufReader};-use harper_core::{CharStringExt, TokenKind};+use harper_core::TokenKind;pub use record::Record;pub use record::RecordKind;use serde::Serialize;diff --git a/harper-stats/src/record.rs b/harper-stats/src/record.rsindex fbc9eb3ae..7fe097f4f 100644--- a/harper-stats/src/record.rs+++ b/harper-stats/src/record.rs@@ -1,6 +1,6 @@use harper_core::{linting::{Lint, LintGroupConfig, LintKind},- Document, FatToken,+ Document, FatStringToken,};use serde::{Deserialize, Serialize};use uuid::Uuid;@@ -28,7 +28,7 @@ impl Record {pub enum RecordKind {Lint {kind: LintKind,- context: Vec<FatToken>,+ context: Vec<FatStringToken>,},LintConfigUpdate(LintGroupConfig),}@@ -37,7 +37,11 @@ impl RecordKind {pub fn from_lint(lint: &Lint, doc: &Document) -> Self {Self::Lint {kind: lint.lint_kind,- context: doc.fat_tokens_intersecting(lint.span),+ context: doc+ .fat_tokens_intersecting(lint.span)+ .into_iter()+ .map(|t| t.into())+ .collect(),}}}@@ -67,10 +71,10 @@ mod tests {let lcu = Self::LintConfigUpdate(arbitrary_lintconfig(g));let context = Document::new_plain_english_curated(&String::arbitrary(g))- .fat_tokens()+ .fat_string_tokens().collect();- let kind = g+ let kind = *g.choose(&[LintKind::Spelling,LintKind::Capitalization,@@ -82,8 +86,7 @@ mod tests {LintKind::WordChoice,LintKind::Miscellaneous,])- .unwrap()- .clone();+ .unwrap();g.choose(&[RecordKind::Lint { kind, context }, lcu]).unwrap()diff --git a/packages/web/src/routes/docs/contributors/local-stats/+page.md b/packages/web/src/routes/docs/contributors/local-stats/+page.mdindex 6d47bd3d7..619c5ac92 100644--- a/packages/web/src/routes/docs/contributors/local-stats/+page.md+++ b/packages/web/src/routes/docs/contributors/local-stats/+page.md@@ -23,12 +23,9 @@ It records specific events, along with some contextual information (like which wThe `stats.txt` file is formatted into lines (so it is easy to open in append-mode), each containing a JSON object.```-{"kind":{"Lint":{"kind":"Capitalization","context":[{"content":["i"],"kind":{"kind":"Word","value":{"noun":null,"pronoun":{"is_plural":false,"is_possessive":null,"person":null,"case":null},"verb":null,"adjective":null,"adverb":null,"conjunction":null,"swear":null,"dialect":null,"determiner":false,"preposition":false,"common":true,"derived_from":null}}}]}},"when":1743428384,"uuid":"b095c60b-6ae0-4c82-bcb8-c7de737d6509"}-{"kind":{"Lint":{"kind":"Spelling","context":[{"content":["m","i","s","p","e","l","l"],"kind":{"kind":"Word","value":null}}]}},"when":1743428388,"uuid":"70850697-a3f1-4eb6-94c8-be0be6d5b16b"}-{"kind":{"Lint":{"kind":"Spelling","context":[{"content":["t","h","i","g","s"],"kind":{"kind":"Word","value":null}}]}},"when":1743428392,"uuid":"c0ff601a-f285-4f6d-9c47-1536213bee3f"}-{"kind":{"Lint":{"kind":"Spelling","context":[{"content":["t","h","i","g","s"],"kind":{"kind":"Word","value":null}}]}},"when":1743428397,"uuid":"c49a67c2-4d03-460a-b450-1ae9b20f243c"}-{"kind":{"Lint":{"kind":"Spelling","context":[{"content":["m","i","s","p","e","l","l","e","d"],"kind":{"kind":"Word","value":null}}]}},"when":1743429129,"uuid":"6ee7634e-dc8d-489b-9e13-f3b7dcd50741"}-{"kind":{"Lint":{"kind":"Miscellaneous","context":[{"content":["a"],"kind":{"kind":"Word","value":{"noun":null,"pronoun":null,"verb":null,"adjective":null,"adverb":null,"conjunction":null,"swear":null,"dialect":null,"determiner":true,"preposition":true,"common":true,"derived_from":null}}}]}},"when":1743525210,"uuid":"cbb2ca64-2073-4714-a346-628bb31cfc65"}+{"kind":{"Lint":{"kind":"Spelling","context":[{"content":"mispelled","kind":{"kind":"Word","value":null}}]}},"when":1743696274,"uuid":"39d29bd0-5eb1-4bad-89ee-5a48531a4cbe"}+{"kind":{"Lint":{"kind":"Spelling","context":[{"content":"isnt","kind":{"kind":"Word","value":null}}]}},"when":1743696281,"uuid":"22e1ca15-e583-49c5-9da3-bc7e625d9682"}+{"kind":{"Lint":{"kind":"Spelling","context":[{"content":"Teasting","kind":{"kind":"Word","value":null}}]}},"when":1743696288,"uuid":"bd955190-a4d9-4f3e-b7df-d4bf6f12a415"}```In `harper-ls` it is written to the Harper `data` directory.