Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
feat: added metadata to `TokenKind::Word`
Elijah Potter38f5bcd4461d
diff --git a/harper-comments/src/comment_parsers/jsdoc.rs b/harper-comments/src/comment_parsers/jsdoc.rsindex 4c79fd46f..2d7680893 100644--- a/harper-comments/src/comment_parsers/jsdoc.rs+++ b/harper-comments/src/comment_parsers/jsdoc.rs@@ -56,7 +56,7 @@ fn parse_line(source: &[char]) -> Vec<Token> {..},Token {- kind: TokenKind::Word,+ kind: TokenKind::Word(..),..})@@ -120,7 +120,7 @@ fn parse_inline_tag(tokens: &[Token]) -> Option<usize> {..},Token {- kind: TokenKind::Word,+ kind: TokenKind::Word(..),..},..,@@ -166,7 +166,7 @@ mod tests {assert_eq!(document.tokens().map(|t| t.kind).collect::<Vec<_>>(),vec![- TokenKind::Word,+ TokenKind::blank_word(),TokenKind::Space(1),TokenKind::Unlintable,TokenKind::Unlintable,@@ -175,14 +175,14 @@ mod tests {TokenKind::Unlintable,TokenKind::Unlintable,TokenKind::Space(1),- TokenKind::Word,+ TokenKind::blank_word(),TokenKind::Space(1),TokenKind::Punctuation(Punctuation::OpenSquare),- TokenKind::Word,+ TokenKind::blank_word(),TokenKind::Space(1),- TokenKind::Word,+ TokenKind::blank_word(),TokenKind::Space(1),- TokenKind::Word,+ TokenKind::blank_word(),TokenKind::Punctuation(Punctuation::CloseSquare),TokenKind::Unlintable,TokenKind::Unlintable,diff --git a/harper-core/src/document.rs b/harper-core/src/document.rsindex a13ea109e..770a2a9c3 100644--- a/harper-core/src/document.rs+++ b/harper-core/src/document.rs@@ -331,7 +331,7 @@ impl Document {// TODO: Allow spaces between `a` and `b`- if let (TokenKind::Number(..), TokenKind::Word) = (a.kind, b.kind) {+ if let (TokenKind::Number(..), TokenKind::Word(..)) = (a.kind, b.kind) {if let Some(found_suffix) = NumberSuffix::from_chars(self.get_span_content(b.span)){*self.tokens[idx].kind.as_mut_number().unwrap().1 = Some(found_suffix);@@ -485,9 +485,9 @@ impl Document {if matches!((a.kind, b.kind, c.kind),(- TokenKind::Word,+ TokenKind::Word(..),TokenKind::Punctuation(Punctuation::Apostrophe),- TokenKind::Word+ TokenKind::Word(..))) {// Ensure there is no overlapping between replacements@@ -656,7 +656,7 @@ mod tests {assert_eq!(document.get_token_at_char_index(19),Some(Token {- kind: TokenKind::Word,+ kind: TokenKind::blank_word(),span: Span::new(17, 23)}))diff --git a/harper-core/src/language_detection.rs b/harper-core/src/language_detection.rsindex a1a873016..a351d56a7 100644--- a/harper-core/src/language_detection.rs+++ b/harper-core/src/language_detection.rs@@ -9,7 +9,7 @@ pub fn is_likely_english(doc: &Document, dict: &impl Dictionary) -> bool {for token in doc.tokens() {match token.kind {- TokenKind::Word => {+ TokenKind::Word(_) => {total_words += 1;let word_content = doc.get_span_content(token.span);diff --git a/harper-core/src/lexing/mod.rs b/harper-core/src/lexing/mod.rsindex bd6d3c7f0..998b31807 100644--- a/harper-core/src/lexing/mod.rs+++ b/harper-core/src/lexing/mod.rs@@ -8,6 +8,7 @@ use self::email_address::lex_email_address;use crate::char_ext::CharExt;use crate::punctuation::{Punctuation, Quote};use crate::token::TokenKind;+use crate::WordMetadata;#[derive(Debug)]pub struct FoundToken {@@ -50,7 +51,7 @@ fn lex_word(source: &[char]) -> Option<FoundToken> {} else {Some(FoundToken {next_index: end,- token: TokenKind::Word+ token: TokenKind::Word(WordMetadata::default())})}}diff --git a/harper-core/src/linting/matcher.rs b/harper-core/src/linting/matcher.rsindex aebee0716..c506fd88d 100644--- a/harper-core/src/linting/matcher.rs+++ b/harper-core/src/linting/matcher.rs@@ -8,7 +8,8 @@ use crate::{Span,Suggestion,Token,- TokenKind+ TokenKind,+ WordMetadata};#[derive(Debug, PartialEq, PartialOrd, Clone)]@@ -42,7 +43,7 @@ macro_rules! vecword {macro_rules! pt {($str:literal) => {PatternToken {- kind: TokenKind::Word,+ kind: TokenKind::Word(WordMetadata::default()),content: Some($str.chars().collect()),}};diff --git a/harper-core/src/linting/spaces.rs b/harper-core/src/linting/spaces.rsindex 066b8aabf..756e12944 100644--- a/harper-core/src/linting/spaces.rs+++ b/harper-core/src/linting/spaces.rs@@ -34,7 +34,7 @@ impl Linter for Spaces {[..,Token {- kind: TokenKind::Word,+ kind: TokenKind::Word(_),..},Token {diff --git a/harper-core/src/parsers/mod.rs b/harper-core/src/parsers/mod.rsindex 1140b8a48..eb7b7ce7e 100644--- a/harper-core/src/parsers/mod.rs+++ b/harper-core/src/parsers/mod.rs@@ -59,7 +59,7 @@ mod tests {#[test]fn single_letter() {- assert_tokens_eq_plain("a", &[Word])+ assert_tokens_eq_plain("a", &[TokenKind::blank_word()])}#[test]@@ -67,14 +67,14 @@ mod tests {assert_tokens_eq_plain("hello world, my friend",&[- Word,+ TokenKind::blank_word(),Space(1),- Word,+ TokenKind::blank_word(),Punctuation(Punctuation::Comma),Space(1),- Word,+ TokenKind::blank_word(),Space(1),- Word+ TokenKind::blank_word()])}@@ -84,14 +84,14 @@ mod tests {assert_tokens_eq_md("__hello__ world, [my]() friend",&[- Word,+ TokenKind::blank_word(),Space(1),- Word,+ TokenKind::blank_word(),Punctuation(Punctuation::Comma),Space(1),- Word,+ TokenKind::blank_word(),Space(1),- Word+ TokenKind::blank_word()]);}@@ -101,14 +101,14 @@ mod tests {assert_tokens_eq_md("__hello__ world,\n\n[my]() friend",&[- Word,+ TokenKind::blank_word(),Space(1),- Word,+ TokenKind::blank_word(),Punctuation(Punctuation::Comma),Newline(2),- Word,+ TokenKind::blank_word(),Space(1),- Word+ TokenKind::blank_word()]);}@@ -117,7 +117,7 @@ mod tests {/// characters as part of the same word.#[test]fn parses_non_english() {- assert_tokens_eq_plain("Løvetann", &[Word]);- assert_tokens_eq_plain("Naïve", &[Word]);+ assert_tokens_eq_plain("Løvetann", &[TokenKind::blank_word()]);+ assert_tokens_eq_plain("Naïve", &[TokenKind::blank_word()]);}}diff --git a/harper-core/src/spell/full_dictionary.rs b/harper-core/src/spell/full_dictionary.rsindex beb96597f..102dc663a 100644--- a/harper-core/src/spell/full_dictionary.rs+++ b/harper-core/src/spell/full_dictionary.rs@@ -1,4 +1,4 @@-use hashbrown::{HashMap, HashSet};+use hashbrown::HashMap;use once_cell::sync::Lazy;use smallvec::{SmallVec, ToSmallVec};diff --git a/harper-core/src/token.rs b/harper-core/src/token.rsindex e7126742b..9eaa6f63c 100644--- a/harper-core/src/token.rs+++ b/harper-core/src/token.rs@@ -5,7 +5,7 @@ use serde::{Deserialize, Serialize};use crate::punctuation::Punctuation;use crate::span::Span;-use crate::Quote;+use crate::{Quote, WordMetadata};#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Default)]pub struct Token {@@ -40,8 +40,7 @@ pub struct FatToken {#[derive(Debug, Is, Clone, Copy, Serialize, Deserialize, PartialEq, Default, PartialOrd)]#[serde(tag = "kind", content = "value")]pub enum TokenKind {- #[default]- Word,+ Word(WordMetadata),Punctuation(Punctuation),Number(f64, Option<NumberSuffix>),/// A sequence of " " spaces.@@ -53,10 +52,18 @@ pub enum TokenKind {Hostname,/// A special token used for things like inline code blocks that should be/// ignored by all linters.+ #[default]Unlintable,ParagraphBreak}+impl TokenKind {+ /// Construct a [`TokenKind::Word`] with no metadata.+ pub fn blank_word() -> Self {+ Self::Word(WordMetadata::default())+ }+}+#[derive(Debug, Serialize, Deserialize, Default, PartialEq, PartialOrd, Clone, Copy, Is)]pub enum NumberSuffix {#[default]