Skip to content

Offline, privacy-first grammar checker. Fast, open-source, Rust-powered

feat: created Mask pattern

Elijah Potter676527eae9ac
diff --git a/harper-comments/src/better.rs b/harper-comments/src/better.rs
new file mode 100644
index 000000000..8b1378917
--- /dev/null
+++ b/harper-comments/src/better.rs
@@ -0,0 +1 @@
+
diff --git a/harper-comments/src/comment_parsers/jsdoc.rs b/harper-comments/src/comment_parsers/jsdoc.rs
index e4f72a00c..4db6b908a 100644
--- a/harper-comments/src/comment_parsers/jsdoc.rs
+++ b/harper-comments/src/comment_parsers/jsdoc.rs
@@ -1,5 +1,5 @@
use harper_core::parsers::{Markdown, Parser};
-use harper_core::{Punctuation, Token, TokenKind};
+use harper_core::{Punctuation, Span, Token, TokenKind};
use itertools::Itertools;
use super::without_initiators;
@@ -15,6 +15,11 @@ impl Parser for JsDoc {
for line in source.split(|c| *c == '\n') {
let mut new_tokens = parse_line(line);
+ new_tokens.push(Token::new(
+ Span::new_with_len(line.len(), 1),
+ harper_core::TokenKind::Newline(1),
+ ));
+
new_tokens
.iter_mut()
.for_each(|t| t.span.push_by(chars_traversed));
@@ -199,7 +204,7 @@ mod tests {
TokenKind::Unlintable,
TokenKind::Unlintable,
TokenKind::Punctuation(Punctuation::Period),
- TokenKind::Newline(1),
+ TokenKind::Newline(2),
]
);
}
diff --git a/harper-comments/src/comment_parsers/unit.rs b/harper-comments/src/comment_parsers/unit.rs
index 44fb33a04..07eb144f6 100644
--- a/harper-comments/src/comment_parsers/unit.rs
+++ b/harper-comments/src/comment_parsers/unit.rs
@@ -1,5 +1,5 @@
use harper_core::parsers::{Markdown, Parser};
-use harper_core::Token;
+use harper_core::{Span, Token};
use super::without_initiators;
@@ -20,6 +20,11 @@ impl Parser for Unit {
for line in source.split(|c| *c == '\n') {
let mut new_tokens = parse_line(line);
+ new_tokens.push(Token::new(
+ Span::new_with_len(line.len(), 1),
+ harper_core::TokenKind::Newline(1),
+ ));
+
new_tokens
.iter_mut()
.for_each(|t| t.span.push_by(chars_traversed));
diff --git a/harper-comments/src/lib.rs b/harper-comments/src/lib.rs
index ed3eebd4b..f7b014eb2 100644
--- a/harper-comments/src/lib.rs
+++ b/harper-comments/src/lib.rs
@@ -1,6 +1,7 @@
#![doc = include_str!("../README.md")]
+mod better;
mod comment_parsers;
+mod tree_sitter_masker;
mod tree_sitter_parser;
-
pub use tree_sitter_parser::TreeSitterParser;
diff --git a/harper-comments/src/tree_sitter_masker.rs b/harper-comments/src/tree_sitter_masker.rs
new file mode 100644
index 000000000..09c0df740
--- /dev/null
+++ b/harper-comments/src/tree_sitter_masker.rs
@@ -0,0 +1,145 @@
+use std::collections::HashSet;
+
+use harper_core::{FullDictionary, Span};
+use harper_core::{Mask, Masker};
+use tree_sitter::{Language, Node, Tree, TreeCursor};
+
+/// A Harper parser that wraps various comment parsers that
+/// exclusively parses comments in any language supported by [`tree_sitter`].
+pub struct TreeSitterMasker {
+ language: Language,
+}
+
+impl TreeSitterMasker {
+ pub fn new(language: Language) -> Self {
+ Self { language }
+ }
+
+ fn parse_root(&self, text: &str) -> Option<Tree> {
+ let mut parser = tree_sitter::Parser::new();
+ parser.set_language(self.language).unwrap();
+
+ // TODO: Use incremental parsing
+ parser.parse(text, None)
+ }
+
+ pub fn create_ident_dict(&self, source: &[char]) -> Option<FullDictionary> {
+ let text: String = source.iter().collect();
+
+ // Byte-indexed
+ let mut ident_spans = Vec::new();
+
+ let tree = self.parse_root(&text)?;
+ Self::visit_nodes(&mut tree.walk(), &mut |node: &Node| {
+ if node.child_count() == 0 && node.kind().contains("ident") {
+ ident_spans.push(node.byte_range().into())
+ }
+ });
+
+ byte_spans_to_char_spans(&mut ident_spans, &text);
+
+ let mut idents = HashSet::new();
+
+ for span in ident_spans {
+ idents.insert(span.get_content(source));
+ }
+
+ let idents: Vec<_> = idents.into_iter().collect();
+
+ let mut dictionary = FullDictionary::new();
+ dictionary.extend_words(idents);
+
+ Some(dictionary)
+ }
+
+ /// Visits the children of a TreeSitter node, searching for comments.
+ ///
+ /// Returns the BYTE spans of the comment position.
+ fn extract_comments(cursor: &mut TreeCursor, comments: &mut Vec<Span>) {
+ Self::visit_nodes(cursor, &mut |node: &Node| {
+ if node.kind().contains("comment") {
+ comments.push(node.byte_range().into());
+ }
+ });
+ }
+
+ fn visit_nodes(cursor: &mut TreeCursor, visit: &mut impl FnMut(&Node)) {
+ if !cursor.goto_first_child() {
+ return;
+ }
+
+ loop {
+ let node = cursor.node();
+
+ visit(&node);
+
+ Self::visit_nodes(cursor, visit);
+
+ if !cursor.goto_next_sibling() {
+ break;
+ }
+ }
+
+ cursor.goto_parent();
+ }
+}
+
+impl Masker for TreeSitterMasker {
+ fn create_mask(&mut self, source: &[char]) -> Mask {
+ let text: String = source.iter().collect();
+
+ let Some(root) = self.parse_root(&text) else {
+ return Mask::new_blank();
+ };
+
+ let mut comments_spans = Vec::new();
+
+ Self::extract_comments(&mut root.walk(), &mut comments_spans);
+ byte_spans_to_char_spans(&mut comments_spans, &text);
+
+ let mut mask = Mask::new_blank();
+
+ for span in comments_spans {
+ mask.push_allowed(span);
+ }
+
+ mask.merge_whitespace_sep(source);
+
+ mask
+ }
+}
+
+/// Converts a set of byte-indexed [`Span`]s to char-index Spans, in-place.
+/// NOTE: Will sort the given slice by their [`Span::start`].
+///
+/// If any spans overlap, it will remove the second one.
+fn byte_spans_to_char_spans(byte_spans: &mut Vec<Span>, source: &str) {
+ byte_spans.sort_by_key(|s| s.start);
+
+ let cloned = byte_spans.clone();
+
+ let mut i: usize = 0;
+ byte_spans.retain(|cur| {
+ i += 1;
+ if let Some(prev) = cloned.get(i.wrapping_sub(2)) {
+ !cur.overlaps_with(*prev)
+ } else {
+ true
+ }
+ });
+
+ let mut last_byte_pos = 0;
+ let mut last_char_pos = 0;
+
+ byte_spans.iter_mut().for_each(|span| {
+ let byte_span = *span;
+
+ last_char_pos += source[last_byte_pos..byte_span.start].chars().count();
+ span.start = last_char_pos;
+
+ last_char_pos += source[byte_span.start..byte_span.end].chars().count();
+ span.end = last_char_pos;
+
+ last_byte_pos = byte_span.end;
+ })
+}
diff --git a/harper-comments/src/tree_sitter_parser.rs b/harper-comments/src/tree_sitter_parser.rs
index c17d06bbf..d6d1a3964 100644
--- a/harper-comments/src/tree_sitter_parser.rs
+++ b/harper-comments/src/tree_sitter_parser.rs
@@ -1,21 +1,21 @@
-use std::collections::HashSet;
use std::path::Path;
-use harper_core::parsers::Parser;
-use harper_core::{FullDictionary, Span, Token, TokenKind};
-use tree_sitter::{Language, Node, Tree, TreeCursor};
+use crate::{comment_parsers, tree_sitter_masker::TreeSitterMasker};
+use comment_parsers::{Go, JsDoc, Unit};
+use harper_core::{
+ parsers::{self, Parser},
+ FullDictionary, Token,
+};
-use super::comment_parsers::{Go, Unit};
-use crate::comment_parsers::JsDoc;
-
-/// A Harper parser that wraps various comment parsers that
-/// exclusively parses comments in any language supported by [`tree_sitter`].
pub struct TreeSitterParser {
- language: Language,
- comment_parser: Box<dyn Parser>
+ inner: parsers::Mask<TreeSitterMasker, Box<dyn Parser>>,
}
impl TreeSitterParser {
+ pub fn create_ident_dict(&self, source: &[char]) -> Option<FullDictionary> {
+ self.inner.masker.create_ident_dict(source)
+ }
+
pub fn new_from_language_id(language_id: &str) -> Option<Self> {
let language = match language_id {
"rust" => tree_sitter_rust::language(),
@@ -34,18 +34,17 @@ impl TreeSitterParser {
"lua" => tree_sitter_lua::language(),
"sh" => tree_sitter_bash::language(),
"java" => tree_sitter_java::language(),
- _ => return None
+ _ => return None,
};
let comment_parser: Box<dyn Parser> = match language_id {
"javascriptreact" | "typescript" | "typescriptreact" | "javascript" => Box::new(JsDoc),
"go" => Box::new(Go),
- _ => Box::new(Unit)
+ _ => Box::new(Unit),
};
Some(Self {
- language,
- comment_parser
+ inner: parsers::Mask::new(TreeSitterMasker::new(language), comment_parser),
})
}
@@ -78,167 +77,13 @@ impl TreeSitterParser {
"sh" => "sh",
"bash" => "sh",
"java" => "java",
- _ => return None
+ _ => return None,
})
}
-
- fn parse_root(&self, text: &str) -> Option<Tree> {
- let mut parser = tree_sitter::Parser::new();
- parser.set_language(self.language).unwrap();
-
- // TODO: Use incremental parsing
- parser.parse(text, None)
- }
-
- pub fn create_ident_dict(&self, source: &[char]) -> Option<FullDictionary> {
- let text: String = source.iter().collect();
-
- // Byte-indexed
- let mut ident_spans = Vec::new();
-
- let tree = self.parse_root(&text)?;
- Self::visit_nodes(&mut tree.walk(), &mut |node: &Node| {
- if node.child_count() == 0 && node.kind().contains("ident") {
- ident_spans.push(node.byte_range().into())
- }
- });
-
- byte_spans_to_char_spans(&mut ident_spans, &text);
-
- let mut idents = HashSet::new();
-
- for span in ident_spans {
- idents.insert(span.get_content(source));
- }
-
- let idents: Vec<_> = idents.into_iter().collect();
-
- let mut dictionary = FullDictionary::new();
- dictionary.extend_words(idents);
-
- Some(dictionary)
- }
-
- /// Visits the children of a TreeSitter node, searching for comments.
- ///
- /// Returns the BYTE spans of the comment position.
- fn extract_comments(cursor: &mut TreeCursor, comments: &mut Vec<Span>) {
- Self::visit_nodes(cursor, &mut |node: &Node| {
- if node.kind().contains("comment") {
- comments.push(node.byte_range().into());
- }
- });
- }
-
- fn visit_nodes(cursor: &mut TreeCursor, visit: &mut impl FnMut(&Node)) {
- if !cursor.goto_first_child() {
- return;
- }
-
- loop {
- let node = cursor.node();
-
- visit(&node);
-
- Self::visit_nodes(cursor, visit);
-
- if !cursor.goto_next_sibling() {
- break;
- }
- }
-
- cursor.goto_parent();
- }
}
impl Parser for TreeSitterParser {
fn parse(&mut self, source: &[char]) -> Vec<Token> {
- let text: String = source.iter().collect();
-
- let Some(root) = self.parse_root(&text) else {
- return vec![];
- };
-
- let mut comments_spans = Vec::new();
-
- Self::extract_comments(&mut root.walk(), &mut comments_spans);
-
- byte_spans_to_char_spans(&mut comments_spans, &text);
-
- let mut tokens = Vec::new();
-
- let mut span_iter = comments_spans.iter().peekable();
-
- while let Some(span) = span_iter.next() {
- let mut new_tokens = self.comment_parser.parse(span.get_content(source));
-
- new_tokens
- .iter_mut()
- .for_each(|v| v.span.push_by(span.start));
-
- // Same goes for newlines within the comment
- for t in new_tokens.iter_mut() {
- if let TokenKind::Newline(n) = &mut t.kind {
- *n = 1;
- }
- }
-
- tokens.append(&mut new_tokens);
-
- // Insert a newline manually since we didn't pass the last one (if it existed)
- // to the comment parser.
- if let Some(last) = tokens.last() {
- let is_terminating = if let Some(next_span) = span_iter.peek() {
- Span::new(last.span.end, next_span.start)
- .get_content(source)
- .iter()
- .any(|c| !c.is_whitespace())
- } else {
- false
- };
-
- tokens.push(Token::new(
- Span::new_with_len(last.span.end, 1),
- TokenKind::Newline(if is_terminating { 2 } else { 1 })
- ));
- }
- }
-
- tokens
+ self.inner.parse(source)
}
}
-
-/// Converts a set of byte-indexed [`Span`]s to char-index Spans, in-place.
-/// NOTE: Will sort the given slice by their [`Span::start`].
-///
-/// If any spans overlap, it will remove the second one.
-fn byte_spans_to_char_spans(byte_spans: &mut Vec<Span>, source: &str) {
- byte_spans.sort_by_key(|s| s.start);
-
- let cloned = byte_spans.clone();
-
- let mut i: usize = 0;
- byte_spans.retain(|cur| {
- i += 1;
- if let Some(prev) = cloned.get(i.wrapping_sub(2)) {
- !cur.overlaps_with(*prev)
- } else {
- true
- }
- });
-
- let mut last_byte_pos = 0;
- let mut last_char_pos = 0;
-
- byte_spans.iter_mut().for_each(|span| {
- let byte_span = *span;
-
- last_char_pos += source[last_byte_pos..byte_span.start].chars().count();
- span.start = last_char_pos;
-
- last_char_pos += source[byte_span.start..byte_span.end].chars().count();
- span.end = last_char_pos;
-
- last_byte_pos = byte_span.end;
- })
-}
diff --git a/harper-comments/tests/language_support.rs b/harper-comments/tests/language_support.rs
index a2ab82aa6..b25c9c256 100644
--- a/harper-comments/tests/language_support.rs
+++ b/harper-comments/tests/language_support.rs
@@ -40,3 +40,4 @@ create_test!(multiline_comments.ts, 3);
create_test!(clean.rs, 0);
create_test!(jsdoc.ts, 4);
create_test!(issue_96.lua, 0);
+create_test!(merged_lines.ts, 1);
diff --git a/harper-comments/tests/language_support_sources/issue_96.rb b/harper-comments/tests/language_support_sources/issue_96.rb
index d2dbee230..38b904f9c 100644
--- a/harper-comments/tests/language_support_sources/issue_96.rb
+++ b/harper-comments/tests/language_support_sources/issue_96.rb
@@ -10,6 +10,6 @@ alphabet = [
"g", # This is a test
"h", # This is a test
"i", # This is a test
-= "j", # This is a test
-= "k", # This is a test
+ "j", # This is a test
+ "k", # This is a test
]
diff --git a/harper-comments/tests/language_support_sources/jsdoc.ts b/harper-comments/tests/language_support_sources/jsdoc.ts
index cf19f6fc1..f46178f33 100644
--- a/harper-comments/tests/language_support_sources/jsdoc.ts
+++ b/harper-comments/tests/language_support_sources/jsdoc.ts
@@ -19,10 +19,6 @@ class Clazz { }
* @return {string} ignor
*
* This should not be ignor
- *
- * @example
- *
- * foo('hello')
*/
function foo(n, o, d) {
diff --git a/harper-comments/tests/language_support_sources/merged_lines.ts b/harper-comments/tests/language_support_sources/merged_lines.ts
new file mode 100644
index 000000000..32fd0a113
--- /dev/null
+++ b/harper-comments/tests/language_support_sources/merged_lines.ts
@@ -0,0 +1,11 @@
+// This is a test to make sure we don't split up paragraphs on newlines.
+// This should be the same sentence,
+// this should be the same sentence,
+// this should be the same sentence,
+// this should be the same sentence,
+// this should be the same sentence,
+// this should be the same sentence,
+// this should be the same sentence,
+// this should be the same sentence,
+//
+// And this should be a new sentence.
diff --git a/harper-comments/tests/language_support_sources/multiline_comments.ts b/harper-comments/tests/language_support_sources/multiline_comments.ts
index 51d87c35e..07f3a8cfa 100644
--- a/harper-comments/tests/language_support_sources/multiline_comments.ts
+++ b/harper-comments/tests/language_support_sources/multiline_comments.ts
@@ -1,5 +1,5 @@
-/// This is an example of an problematic comment.
-/// It should produce one error.
+// This is an example of an problematic comment
+// It should produce one error
function test() {}
/***
@@ -9,4 +9,5 @@ function test() {}
*/
function arbitrary() {}
-/// Let's aadd a cuple spelling errors for good measure.
+// Let's aadd a cuple spelling errors for good measure.
+
diff --git a/harper-core/src/lib.rs b/harper-core/src/lib.rs
index 61ed492c3..c6f18df42 100644
--- a/harper-core/src/lib.rs
+++ b/harper-core/src/lib.rs
@@ -7,20 +7,22 @@ mod document;
mod language_detection;
mod lexing;
mod linting;
-mod sync;
+mod mask;
pub mod parsers;
mod punctuation;
mod span;
mod spell;
+mod sync;
mod token;
-pub use sync::Lrc;
pub use char_string::{CharString, CharStringExt};
pub use document::Document;
pub use linting::{Lint, LintGroup, LintGroupConfig, LintKind, Linter, Suggestion};
+pub use mask::{Mask, Masker};
pub use punctuation::{Punctuation, Quote};
pub use span::Span;
pub use spell::{Dictionary, FullDictionary, MergedDictionary};
+pub use sync::Lrc;
pub use token::{FatToken, Token, TokenKind, TokenStringExt};
/// A utility function that removes overlapping lints in a vector,
diff --git a/harper-core/src/mask/mod.rs b/harper-core/src/mask/mod.rs
new file mode 100644
index 000000000..2300d8304
--- /dev/null
+++ b/harper-core/src/mask/mod.rs
@@ -0,0 +1,109 @@
+use itertools::Itertools;
+
+use crate::Span;
+
+/// A Masker is a tool that can be composed to eliminate chunks of text from being parsed.
+/// They can be composed to do things like isolate comments from a programming language or disable
+/// linting for languages that have been determined to not be English.
+pub trait Masker: Send + Sync {
+ fn create_mask(&mut self, source: &[char]) -> Mask;
+}
+
+pub struct Mask {
+ // Right now, there aren't any use-cases where we can't treat this as a stack.
+ //
+ // Assumed that no elements overlap and exist in sorted order.
+ pub(self) allowed: Vec<Span>,
+}
+
+impl Mask {
+ /// Create a new Mask for a given piece of text, marking all text as disallowed.
+ pub fn new_blank() -> Self {
+ Self {
+ allowed: Vec::new(),
+ }
+ }
+
+ pub fn iter_allowed<'a>(
+ &'a self,
+ source: &'a [char],
+ ) -> impl Iterator<Item = (Span, &'a [char])> + '_ {
+ self.allowed.iter().map(|s| (*s, s.get_content(source)))
+ }
+
+ /// Mark a span of the text as allowed.
+ pub fn push_allowed(&mut self, allowed: Span) {
+ if let Some(last) = self.allowed.last_mut() {
+ assert!(allowed.start >= last.end);
+
+ if allowed.start == last.end {
+ last.end = allowed.end;
+ return;
+ }
+ }
+
+ self.allowed.push(allowed)
+ }
+
+ /// Merge chunks that are only separated by whitespace.
+ pub fn merge_whitespace_sep(&mut self, source: &[char]) {
+ let mut after = Vec::with_capacity(self.allowed.len());
+
+ let mut iter = 0..self.allowed.len();
+
+ while let Some(i) = iter.next() {
+ let a = self.allowed[i];
+
+ if let Some(b) = self.allowed.get(i + 1) {
+ let sep = Span::new(a.end, b.start);
+ let sep_content = sep.get_content(source);
+
+ if sep_content.iter().all(|c| c.is_whitespace() || *c == '\n') {
+ iter.next();
+ after.push(Span::new(a.start, b.end));
+ continue;
+ }
+ }
+
+ after.push(a);
+ }
+
+ if self.allowed.len() != after.len() {
+ self.allowed = after;
+ self.merge_whitespace_sep(source);
+ } else {
+ self.allowed = after;
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use crate::{Mask, Span};
+
+ #[test]
+ fn bumps_existing() {
+ let mut mask = Mask::new_blank();
+
+ mask.push_allowed(Span::new_with_len(0, 1));
+ mask.push_allowed(Span::new_with_len(1, 2));
+
+ assert_eq!(mask.allowed.len(), 1)
+ }
+
+ #[test]
+ fn merges_whitespace_sep() {
+ let source: Vec<_> = "word word\nword".chars().collect();
+
+ let mut mask = Mask::new_blank();
+ mask.push_allowed(Span::new_with_len(0, 4));
+ mask.push_allowed(Span::new_with_len(5, 4));
+ mask.push_allowed(Span::new_with_len(10, 4));
+
+ assert_eq!(mask.allowed.len(), 3);
+
+ mask.merge_whitespace_sep(&source);
+
+ assert_eq!(mask.allowed.len(), 1);
+ }
+}
diff --git a/harper-core/src/parsers/mask.rs b/harper-core/src/parsers/mask.rs
new file mode 100644
index 000000000..47a8af456
--- /dev/null
+++ b/harper-core/src/parsers/mask.rs
@@ -0,0 +1,55 @@
+use crate::{mask::Masker, Span, Token, TokenKind};
+
+use super::Parser;
+
+/// Composes a Masker and a Parser to parse only masked chunks of text.
+pub struct Mask<M, P>
+where
+ M: Masker,
+ P: Parser,
+{
+ pub masker: M,
+ pub parser: P,
+}
+
+impl<M, P> Mask<M, P>
+where
+ M: Masker,
+ P: Parser,
+{
+ pub fn new(masker: M, parser: P) -> Self {
+ Self { masker, parser }
+ }
+}
+
+impl<M, P> Parser for Mask<M, P>
+where
+ M: Masker,
+ P: Parser,
+{
+ fn parse(&mut self, source: &[char]) -> Vec<Token> {
+ let mask = self.masker.create_mask(source);
+
+ let mut tokens = Vec::new();
+
+ for (span, content) in mask.iter_allowed(source) {
+ let new_tokens = &mut self.parser.parse(content);
+
+ if let Some(last) = new_tokens.last_mut() {
+ if let TokenKind::Newline(n) = &mut last.kind {
+ if *n == 1 {
+ *n = 2;
+ }
+ }
+ }
+
+ for token in new_tokens.iter_mut() {
+ token.span.push_by(span.start);
+ }
+
+ tokens.append(new_tokens);
+ }
+
+ tokens
+ }
+}
diff --git a/harper-core/src/parsers/mod.rs b/harper-core/src/parsers/mod.rs
index 17cbfa3cc..94fd5779d 100644
--- a/harper-core/src/parsers/mod.rs
+++ b/harper-core/src/parsers/mod.rs
@@ -1,11 +1,15 @@
mod markdown;
+mod mask;
mod plain_english;
+use blanket::blanket;
pub use markdown::Markdown;
+pub use mask::Mask;
pub use plain_english::PlainEnglish;
pub use crate::token::{Token, TokenKind, TokenStringExt};
+#[blanket(derive(Box))]
pub trait Parser: Send + Sync {
fn parse(&mut self, source: &[char]) -> Vec<Token>;
}
@@ -16,7 +20,7 @@ pub trait StrParser {
impl<T> StrParser for T
where
- T: Parser
+ T: Parser,
{
fn parse_str(&mut self, source: impl AsRef<str>) -> Vec<Token> {
let source: Vec<_> = source.as_ref().chars().collect();
@@ -33,7 +37,7 @@ mod tests {
fn assert_tokens_eq(
test_str: impl AsRef<str>,
expected: &[TokenKind],
- parser: &mut impl Parser
+ parser: &mut impl Parser,
) {
let chars: Vec<_> = test_str.as_ref().chars().collect();
let tokens = parser.parse(&chars);
@@ -70,8 +74,8 @@ mod tests {
Space(1),
Word,
Space(1),
- Word
- ]
+ Word,
+ ],
)
}
@@ -87,8 +91,8 @@ mod tests {
Space(1),
Word,
Space(1),
- Word
- ]
+ Word,
+ ],
);
}
@@ -104,8 +108,8 @@ mod tests {
Newline(2),
Word,
Space(1),
- Word
- ]
+ Word,
+ ],
);
}