Skip to content

Offline, privacy-first grammar checker. Fast, open-source, Rust-powered

feat: added metadata to `TokenKind::Word`

Elijah Potter38f5bcd4461d
diff --git a/harper-comments/src/comment_parsers/jsdoc.rs b/harper-comments/src/comment_parsers/jsdoc.rs
index 4c79fd46f..2d7680893 100644
--- a/harper-comments/src/comment_parsers/jsdoc.rs
+++ b/harper-comments/src/comment_parsers/jsdoc.rs
@@ -56,7 +56,7 @@ fn parse_line(source: &[char]) -> Vec<Token> {
..
},
Token {
- kind: TokenKind::Word,
+ kind: TokenKind::Word(..),
..
}
)
@@ -120,7 +120,7 @@ fn parse_inline_tag(tokens: &[Token]) -> Option<usize> {
..
},
Token {
- kind: TokenKind::Word,
+ kind: TokenKind::Word(..),
..
},
..,
@@ -166,7 +166,7 @@ mod tests {
assert_eq!(
document.tokens().map(|t| t.kind).collect::<Vec<_>>(),
vec![
- TokenKind::Word,
+ TokenKind::blank_word(),
TokenKind::Space(1),
TokenKind::Unlintable,
TokenKind::Unlintable,
@@ -175,14 +175,14 @@ mod tests {
TokenKind::Unlintable,
TokenKind::Unlintable,
TokenKind::Space(1),
- TokenKind::Word,
+ TokenKind::blank_word(),
TokenKind::Space(1),
TokenKind::Punctuation(Punctuation::OpenSquare),
- TokenKind::Word,
+ TokenKind::blank_word(),
TokenKind::Space(1),
- TokenKind::Word,
+ TokenKind::blank_word(),
TokenKind::Space(1),
- TokenKind::Word,
+ TokenKind::blank_word(),
TokenKind::Punctuation(Punctuation::CloseSquare),
TokenKind::Unlintable,
TokenKind::Unlintable,
diff --git a/harper-core/src/document.rs b/harper-core/src/document.rs
index a13ea109e..770a2a9c3 100644
--- a/harper-core/src/document.rs
+++ b/harper-core/src/document.rs
@@ -331,7 +331,7 @@ impl Document {
// TODO: Allow spaces between `a` and `b`
- if let (TokenKind::Number(..), TokenKind::Word) = (a.kind, b.kind) {
+ if let (TokenKind::Number(..), TokenKind::Word(..)) = (a.kind, b.kind) {
if let Some(found_suffix) = NumberSuffix::from_chars(self.get_span_content(b.span))
{
*self.tokens[idx].kind.as_mut_number().unwrap().1 = Some(found_suffix);
@@ -485,9 +485,9 @@ impl Document {
if matches!(
(a.kind, b.kind, c.kind),
(
- TokenKind::Word,
+ TokenKind::Word(..),
TokenKind::Punctuation(Punctuation::Apostrophe),
- TokenKind::Word
+ TokenKind::Word(..)
)
) {
// Ensure there is no overlapping between replacements
@@ -656,7 +656,7 @@ mod tests {
assert_eq!(
document.get_token_at_char_index(19),
Some(Token {
- kind: TokenKind::Word,
+ kind: TokenKind::blank_word(),
span: Span::new(17, 23)
})
)
diff --git a/harper-core/src/language_detection.rs b/harper-core/src/language_detection.rs
index a1a873016..a351d56a7 100644
--- a/harper-core/src/language_detection.rs
+++ b/harper-core/src/language_detection.rs
@@ -9,7 +9,7 @@ pub fn is_likely_english(doc: &Document, dict: &impl Dictionary) -> bool {
for token in doc.tokens() {
match token.kind {
- TokenKind::Word => {
+ TokenKind::Word(_) => {
total_words += 1;
let word_content = doc.get_span_content(token.span);
diff --git a/harper-core/src/lexing/mod.rs b/harper-core/src/lexing/mod.rs
index bd6d3c7f0..998b31807 100644
--- a/harper-core/src/lexing/mod.rs
+++ b/harper-core/src/lexing/mod.rs
@@ -8,6 +8,7 @@ use self::email_address::lex_email_address;
use crate::char_ext::CharExt;
use crate::punctuation::{Punctuation, Quote};
use crate::token::TokenKind;
+use crate::WordMetadata;
#[derive(Debug)]
pub struct FoundToken {
@@ -50,7 +51,7 @@ fn lex_word(source: &[char]) -> Option<FoundToken> {
} else {
Some(FoundToken {
next_index: end,
- token: TokenKind::Word
+ token: TokenKind::Word(WordMetadata::default())
})
}
}
diff --git a/harper-core/src/linting/matcher.rs b/harper-core/src/linting/matcher.rs
index aebee0716..c506fd88d 100644
--- a/harper-core/src/linting/matcher.rs
+++ b/harper-core/src/linting/matcher.rs
@@ -8,7 +8,8 @@ use crate::{
Span,
Suggestion,
Token,
- TokenKind
+ TokenKind,
+ WordMetadata
};
#[derive(Debug, PartialEq, PartialOrd, Clone)]
@@ -42,7 +43,7 @@ macro_rules! vecword {
macro_rules! pt {
($str:literal) => {
PatternToken {
- kind: TokenKind::Word,
+ kind: TokenKind::Word(WordMetadata::default()),
content: Some($str.chars().collect()),
}
};
diff --git a/harper-core/src/linting/spaces.rs b/harper-core/src/linting/spaces.rs
index 066b8aabf..756e12944 100644
--- a/harper-core/src/linting/spaces.rs
+++ b/harper-core/src/linting/spaces.rs
@@ -34,7 +34,7 @@ impl Linter for Spaces {
[
..,
Token {
- kind: TokenKind::Word,
+ kind: TokenKind::Word(_),
..
},
Token {
diff --git a/harper-core/src/parsers/mod.rs b/harper-core/src/parsers/mod.rs
index 1140b8a48..eb7b7ce7e 100644
--- a/harper-core/src/parsers/mod.rs
+++ b/harper-core/src/parsers/mod.rs
@@ -59,7 +59,7 @@ mod tests {
#[test]
fn single_letter() {
- assert_tokens_eq_plain("a", &[Word])
+ assert_tokens_eq_plain("a", &[TokenKind::blank_word()])
}
#[test]
@@ -67,14 +67,14 @@ mod tests {
assert_tokens_eq_plain(
"hello world, my friend",
&[
- Word,
+ TokenKind::blank_word(),
Space(1),
- Word,
+ TokenKind::blank_word(),
Punctuation(Punctuation::Comma),
Space(1),
- Word,
+ TokenKind::blank_word(),
Space(1),
- Word
+ TokenKind::blank_word()
]
)
}
@@ -84,14 +84,14 @@ mod tests {
assert_tokens_eq_md(
"__hello__ world, [my]() friend",
&[
- Word,
+ TokenKind::blank_word(),
Space(1),
- Word,
+ TokenKind::blank_word(),
Punctuation(Punctuation::Comma),
Space(1),
- Word,
+ TokenKind::blank_word(),
Space(1),
- Word
+ TokenKind::blank_word()
]
);
}
@@ -101,14 +101,14 @@ mod tests {
assert_tokens_eq_md(
"__hello__ world,\n\n[my]() friend",
&[
- Word,
+ TokenKind::blank_word(),
Space(1),
- Word,
+ TokenKind::blank_word(),
Punctuation(Punctuation::Comma),
Newline(2),
- Word,
+ TokenKind::blank_word(),
Space(1),
- Word
+ TokenKind::blank_word()
]
);
}
@@ -117,7 +117,7 @@ mod tests {
/// characters as part of the same word.
#[test]
fn parses_non_english() {
- assert_tokens_eq_plain("Løvetann", &[Word]);
- assert_tokens_eq_plain("Naïve", &[Word]);
+ assert_tokens_eq_plain("Løvetann", &[TokenKind::blank_word()]);
+ assert_tokens_eq_plain("Naïve", &[TokenKind::blank_word()]);
}
}
diff --git a/harper-core/src/spell/full_dictionary.rs b/harper-core/src/spell/full_dictionary.rs
index beb96597f..102dc663a 100644
--- a/harper-core/src/spell/full_dictionary.rs
+++ b/harper-core/src/spell/full_dictionary.rs
@@ -1,4 +1,4 @@
-use hashbrown::{HashMap, HashSet};
+use hashbrown::HashMap;
use once_cell::sync::Lazy;
use smallvec::{SmallVec, ToSmallVec};
diff --git a/harper-core/src/token.rs b/harper-core/src/token.rs
index e7126742b..9eaa6f63c 100644
--- a/harper-core/src/token.rs
+++ b/harper-core/src/token.rs
@@ -5,7 +5,7 @@ use serde::{Deserialize, Serialize};
use crate::punctuation::Punctuation;
use crate::span::Span;
-use crate::Quote;
+use crate::{Quote, WordMetadata};
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Default)]
pub struct Token {
@@ -40,8 +40,7 @@ pub struct FatToken {
#[derive(Debug, Is, Clone, Copy, Serialize, Deserialize, PartialEq, Default, PartialOrd)]
#[serde(tag = "kind", content = "value")]
pub enum TokenKind {
- #[default]
- Word,
+ Word(WordMetadata),
Punctuation(Punctuation),
Number(f64, Option<NumberSuffix>),
/// A sequence of " " spaces.
@@ -53,10 +52,18 @@ pub enum TokenKind {
Hostname,
/// A special token used for things like inline code blocks that should be
/// ignored by all linters.
+ #[default]
Unlintable,
ParagraphBreak
}
+impl TokenKind {
+ /// Construct a [`TokenKind::Word`] with no metadata.
+ pub fn blank_word() -> Self {
+ Self::Word(WordMetadata::default())
+ }
+}
+
#[derive(Debug, Serialize, Deserialize, Default, PartialEq, PartialOrd, Clone, Copy, Is)]
pub enum NumberSuffix {
#[default]