Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
test: add failing example for markdown comment parser (#1948)
rnxpyke3b1b126dd7c6
diff --git a/harper-comments/src/comment_parser.rs b/harper-comments/src/comment_parser.rsindex a414c2f6a..08493e07e 100644--- a/harper-comments/src/comment_parser.rs+++ b/harper-comments/src/comment_parser.rs@@ -119,3 +119,28 @@ impl Parser for CommentParser {self.inner.parse(source)}}++#[cfg(test)]+mod tests {+ use super::CommentParser;+ use harper_core::parsers::{MarkdownOptions, StrParser};++ #[test]+ fn hang() {+ use std::sync::mpsc::channel;+ use std::thread;+ use std::time::Duration;++ let (tx, rx) = channel::<()>();++ let handle = thread::spawn(move || {+ let opts = MarkdownOptions::default();+ let parser = CommentParser::new_from_language_id("java", opts).unwrap();+ let _res = parser.parse_str("//{@j");+ tx.send(()).expect("send failed");+ });++ rx.recv_timeout(Duration::from_secs(10)).expect("timed out");+ handle.join().expect("failed to join");+ }+}diff --git a/harper-comments/src/comment_parsers/jsdoc.rs b/harper-comments/src/comment_parsers/jsdoc.rsindex 2aefb1286..4dae289bb 100644--- a/harper-comments/src/comment_parsers/jsdoc.rs+++ b/harper-comments/src/comment_parsers/jsdoc.rs@@ -145,15 +145,21 @@ fn parse_inline_tag(tokens: &[Token]) -> Option<usize> {return None;}+ if tokens.len() <= 3 {+ return None;+ }+let mut cursor = 3;- while !matches!(- tokens.get(cursor),- Some(Token {- kind: TokenKind::Punctuation(Punctuation::CloseCurly),- ..- })- ) {+ while cursor < tokens.len()+ && !matches!(+ tokens.get(cursor),+ Some(Token {+ kind: TokenKind::Punctuation(Punctuation::CloseCurly),+ ..+ })+ )+ {cursor += 1;}diff --git a/harper-core/src/parsers/markdown.rs b/harper-core/src/parsers/markdown.rsindex fcdeb9319..320828f81 100644--- a/harper-core/src/parsers/markdown.rs+++ b/harper-core/src/parsers/markdown.rs@@ -161,35 +161,45 @@ impl Parser for Markdown {let mut tokens = Vec::new();- let mut traversed_bytes = 0;- let mut traversed_chars = 0;+ // Build a mapping from the inner parser's byte-based indexing to Harper's char-based+ // indexing+ let mut byte_to_char = vec![0; source_str.len() + 1];+ let mut char_index = 0;+ let mut byte_idx = 0;+ for ch in source_str.chars() {+ let char_len = ch.len_utf8();+ for _ in 0..char_len {+ byte_to_char[byte_idx] = char_index;+ byte_idx += 1;+ }+ char_index += 1;+ }+ byte_to_char[source_str.len()] = char_index;let mut stack = Vec::new();// NOTE: the range spits out __byte__ indices, not char indices.// This is why we keep track above.for (event, range) in md_parser.into_offset_iter() {- if range.start > traversed_bytes {- traversed_chars += source_str[traversed_bytes..range.start].chars().count();- traversed_bytes = range.start;- }+ let span_start = byte_to_char[range.start];+ let span_end = byte_to_char[range.end];match event {pulldown_cmark::Event::SoftBreak => {tokens.push(Token {- span: Span::new_with_len(traversed_chars, 1),+ span: Span::new_with_len(span_start, 1),kind: TokenKind::Newline(1),});}pulldown_cmark::Event::HardBreak => {tokens.push(Token {- span: Span::new_with_len(traversed_chars, 1),+ span: Span::new_with_len(span_start, 1),kind: TokenKind::Newline(2),});}pulldown_cmark::Event::Start(pulldown_cmark::Tag::List(v)) => {tokens.push(Token {- span: Span::new_with_len(traversed_chars, 0),+ span: Span::new_with_len(span_start, 0),kind: TokenKind::Newline(2),});stack.push(pulldown_cmark::Tag::List(v));@@ -201,7 +211,7 @@ impl Parser for Markdown {| pulldown_cmark::Event::End(pulldown_cmark::TagEnd::CodeBlock)| pulldown_cmark::Event::End(pulldown_cmark::TagEnd::TableCell) => {tokens.push(Token {- // We cannot use `traversed_chars` here, as it will still point to the+ // We cannot use `span_start` here, as it will still point to the// first character of the `Event` at this point. Instead, we use the// position of the previous token's last character. This ensures the// paragraph break is placed at the end of the content, not its beginning.@@ -214,38 +224,39 @@ impl Parser for Markdown {pulldown_cmark::Event::End(_) => {stack.pop();}- pulldown_cmark::Event::InlineMath(code)- | pulldown_cmark::Event::DisplayMath(code)- | pulldown_cmark::Event::Code(code) => {- let chunk_len = code.chars().count();+ pulldown_cmark::Event::InlineMath(_)+ | pulldown_cmark::Event::DisplayMath(_)+ | pulldown_cmark::Event::Code(_) => {+ let chunk_len = span_end - span_start;tokens.push(Token {- span: Span::new_with_len(traversed_chars, chunk_len),+ span: Span::new_with_len(span_start, chunk_len),kind: TokenKind::Unlintable,});}- pulldown_cmark::Event::Text(text) => {- let chunk_len = text.chars().count();+ pulldown_cmark::Event::Text(_text) => {+ let chunk_len = span_end - span_start;if let Some(tag) = stack.last() {use pulldown_cmark::Tag;if matches!(tag, Tag::CodeBlock(..)) {tokens.push(Token {- span: Span::new_with_len(traversed_chars, text.chars().count()),+ span: Span::new_with_len(span_start, chunk_len),+kind: TokenKind::Unlintable,});continue;}if matches!(tag, Tag::Link { .. }) && self.options.ignore_link_title {tokens.push(Token {- span: Span::new_with_len(traversed_chars, text.chars().count()),+ span: Span::new_with_len(span_start, chunk_len),kind: TokenKind::Unlintable,});continue;}if !(matches!(tag, Tag::Paragraph)- || matches!(tag, Tag::Link { .. }) && !self.options.ignore_link_title+ || (matches!(tag, Tag::Link { .. }) && !self.options.ignore_link_title)|| matches!(tag, Tag::Heading { .. })|| matches!(tag, Tag::Item)|| matches!(tag, Tag::TableCell)@@ -257,21 +268,19 @@ impl Parser for Markdown {}}- let mut new_tokens =- english_parser.parse(&source[traversed_chars..traversed_chars + chunk_len]);+ let mut new_tokens = english_parser.parse(&source[span_start..span_end]);new_tokens.iter_mut()- .for_each(|token| token.span.push_by(traversed_chars));+ .for_each(|token| token.span.push_by(span_start));tokens.append(&mut new_tokens);}// TODO: Support via `harper-html`- pulldown_cmark::Event::Html(_content)- | pulldown_cmark::Event::InlineHtml(_content) => {- let size = _content.chars().count();+ pulldown_cmark::Event::Html(_) | pulldown_cmark::Event::InlineHtml(_) => {+ let size = span_end - span_start;tokens.push(Token {- span: Span::new_with_len(traversed_chars, size),+ span: Span::new_with_len(span_start, size),kind: TokenKind::Unlintable,});}@@ -551,4 +560,19 @@ Paragraph.let tokens = parser.parse_str(source);assert_ne!(tokens.last().unwrap().span.end, 0);}++ #[test]+ fn hang() {+ let opts = MarkdownOptions::default();+ let parser = Markdown::new(opts);+ let _res = parser.parse_str("[[#|]]:A]");+ }++ #[test]+ fn hang2() {+ // This seems to only be a java specific problem...+ let opts = MarkdownOptions::default();+ let parser = Markdown::new(opts);+ let _res = parser.parse_str("//{@j");+ }}diff --git a/harper-core/tests/text/tagged/Spell.US.md b/harper-core/tests/text/tagged/Spell.US.mdindex e0a0ef6df..f2d3354c2 100644--- a/harper-core/tests/text/tagged/Spell.US.md+++ b/harper-core/tests/text/tagged/Spell.US.md@@ -6,7 +6,7 @@# I/Ddem+ NSg/VB+ V3 D/P NSg/VB P NPl/V3+ VP/J R NPr/J/P I/J/R/Dq NPl P NPr🅪Sg/VB/J+ . NSg/C/P NSg/R/C NPr/J NPr🅪Sg/VB/J+ . I/Ddem+ VL3 VP/J P NSg/VB D+ Nᴹ/Vg/J+ NPl+ IPl+ NSg/VB C/P NSg/I+ NPl/V3+ .>#-> To achieve this , the filename of this file contains `.US , which will tell the snapshot generator to use the American dialect , rather than trying to use an automatically detected dialect .+> To achieve this , the filename of this file contains `.US.` , which will tell the snapshot generator to use the American dialect , rather than trying to use an automatically detected dialect .# P VB I/Ddem+ . D NSg P I/Ddem NSg/VB+ V3 Unlintable . I/C+ NPr/VXB NPr/VB D NSg/VB+ NSg P N🅪Sg/VB D NPr/J NSg+ . NPr/VB/J/R C/P Nᴹ/Vg/J P N🅪Sg/VB D/P R VP/J NSg+ .>#