Skip to content

Offline, privacy-first grammar checker. Fast, open-source, Rust-powered

fix(core): don't misplace `ParagraphBreak` tokens (#1239)

Steve Kurch19ebb9915745
diff --git a/harper-core/src/parsers/markdown.rs b/harper-core/src/parsers/markdown.rs
index d16fad959..e1606cf13 100644
--- a/harper-core/src/parsers/markdown.rs
+++ b/harper-core/src/parsers/markdown.rs
@@ -201,7 +201,12 @@ impl Parser for Markdown {
| pulldown_cmark::Event::End(pulldown_cmark::TagEnd::CodeBlock)
| pulldown_cmark::Event::End(pulldown_cmark::TagEnd::TableCell) => {
tokens.push(Token {
- span: Span::new_with_len(traversed_chars, 0),
+ // We cannot use `traversed_chars` here, as it will still point to the
+ // first character of the `Event` at this point. Instead, we use the
+ // position of the previous token's last character. This ensures the
+ // paragraph break is placed at the end of the content, not its beginning.
+ // For more info, see: https://github.com/Automattic/harper/pull/1239.
+ span: Span::new_with_len(tokens.last().map_or(0, |last| last.span.end), 0),
kind: TokenKind::ParagraphBreak,
});
stack.pop();
@@ -535,4 +540,15 @@ Paragraph.
]
))
}
+
+ /// Helps ensure that ending tokens (like `ParagraphBreak`) don't get erroneously placed at
+ /// the beginning of a sentence. This kind of behavior can cause crashes, as seen in
+ /// [#1181](https://github.com/Automattic/harper/issues/1181).
+ #[test]
+ fn no_end_token_incorrectly_ending_at_zero() {
+ let source = "Something\n";
+ let parser = Markdown::new(MarkdownOptions::default());
+ let tokens = parser.parse_str(source);
+ assert_ne!(tokens.last().unwrap().span.end, 0);
+ }
}
diff --git a/harper-core/tests/text/linters/Alice's Adventures in Wonderland.snap.yml b/harper-core/tests/text/linters/Alice's Adventures in Wonderland.snap.yml
index b402688fd..402d3a0f4 100644
--- a/harper-core/tests/text/linters/Alice's Adventures in Wonderland.snap.yml
+++ b/harper-core/tests/text/linters/Alice's Adventures in Wonderland.snap.yml
@@ -523,8 +523,9 @@ Message: |
250 | try and say ‘How doth the little—’” and she crossed her hands on her lap as if
| ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
251 | she were saying lessons, and began to repeat it, but her voice sounded hoarse
- | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ This sentence is 47 words long.
+ | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
252 | and strange, and the words did not come the same as they used to do:—
+ | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ This sentence is 47 words long.
@@ -3373,8 +3374,9 @@ Message: |
2639 | “I’m a poor man, your Majesty,” the Hatter began, in a trembling voice, “—and I
| ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
2640 | hadn’t begun my tea—not above a week or so—and what with the bread-and-butter
- | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ This sentence is 41 words long.
+ | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
2641 | getting so thin—and the twinkling of the tea—”
+ | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ This sentence is 41 words long.
@@ -3448,8 +3450,9 @@ Message: |
2907 | “All right, so far,” said the King, and he went on muttering over the verses to
| ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
2908 | himself: “‘We know it to be true—’ that’s the jury, of course—‘I gave her one,
- | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ This sentence is 48 words long.
+ | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
2909 | they gave him two—’ why, that must be what he did with the tarts, you know—”
+ | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ This sentence is 48 words long.
@@ -3528,8 +3531,9 @@ Message: |
2961 | watching the setting sun, and thinking of little Alice and all her wonderful
| ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
2962 | Adventures, till she too began dreaming after a fashion, and this was her
- | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ This sentence is 43 words long.
+ | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
2963 | dream:—
+ | ~~~~~~~ This sentence is 43 words long.
diff --git a/harper-core/tests/text/linters/The Constitution of the United States.snap.yml b/harper-core/tests/text/linters/The Constitution of the United States.snap.yml
index 009e99261..d685d343a 100644
--- a/harper-core/tests/text/linters/The Constitution of the United States.snap.yml
+++ b/harper-core/tests/text/linters/The Constitution of the United States.snap.yml
@@ -512,8 +512,9 @@ Message: |
203 | Imposts and Excises, to pay the Debts and provide for the common Defence and
| ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
204 | general Welfare of the United States; but all Duties, Imposts and Excises shall
- | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ This sentence is 44 words long.
+ | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
205 | be uniform throughout the United States;
+ | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ This sentence is 44 words long.
@@ -577,8 +578,9 @@ Message: |
247 | States, reserving to the States respectively, the Appointment of the Officers,
| ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
248 | and the Authority of training the Militia according to the discipline
- | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ This sentence is 51 words long.
+ | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
249 | prescribed by Congress;
+ | ~~~~~~~~~~~~~~~~~~~~~~~ This sentence is 51 words long.
@@ -604,8 +606,9 @@ Message: |
254 | States, and to exercise like Authority over all Places purchased by the Consent
| ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
255 | of the Legislature of the State in which the Same shall be, for the Erection of
- | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ This sentence is 76 words long.
+ | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
256 | Forts, Magazines, Arsenals, dock-Yards, and other needful Buildings;—And
+ | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ This sentence is 76 words long.
@@ -1676,8 +1679,9 @@ Message: |
722 | Seventeenth Day of September in the Year of our Lord one thousand seven hundred
| ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
723 | and Eighty seven and of the Independence of the United States of America the
- | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ This sentence is 50 words long.
+ | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
724 | Twelfth In witness whereof We have hereunto subscribed our Names,
+ | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ This sentence is 50 words long.