Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
fix: issue with nested Markdown lists
Elijah Potter080dc407d801
diff --git a/harper-comments/src/comment_parsers/jsdoc.rs b/harper-comments/src/comment_parsers/jsdoc.rsindex f208466fb..dfbd836c0 100644--- a/harper-comments/src/comment_parsers/jsdoc.rs+++ b/harper-comments/src/comment_parsers/jsdoc.rs@@ -8,76 +8,94 @@ pub struct JsDoc;impl Parser for JsDoc {fn parse(&mut self, source: &[char]) -> Vec<Token> {- dbg!();+ let mut tokens = Vec::new();- let actual = without_initiators(source);+ let mut chars_traversed = 0;- if actual.is_empty() {- return Vec::new();+ for line in source.split(|c| *c == '\n') {+ let mut new_tokens = parse_line(line);++ new_tokens+ .iter_mut()+ .for_each(|t| t.span.push_by(chars_traversed));++ chars_traversed += line.len() + 1;+ tokens.append(&mut new_tokens);}- let source = actual.get_content(source);- let mut tokens = Markdown.parse(source);+ tokens+ }+}- let mut cursor = 0;+fn parse_line(source: &[char]) -> Vec<Token> {+ let actual_line = without_initiators(source);- // Handle inline tags- loop {- if cursor >= tokens.len() {- break;- }+ if actual_line.is_empty() {+ return vec![];+ }- if let Some(new_cursor) = &tokens[cursor..]- .iter()- .position(|t| t.kind == TokenKind::Punctuation(Punctuation::OpenCurly))- .map(|i| i + cursor)- {- cursor = *new_cursor;- } else {- break;- }+ let source_line = actual_line.get_content(source);- let parsers = [parse_link, parse_tutorial];+ let mut new_tokens = Markdown.parse(source_line);- for parser in parsers {- if let Some(p) = parser(&tokens[cursor..], source) {- for tok in &mut tokens[cursor..cursor + p] {- tok.kind = TokenKind::Unlintable;- }+ let mut cursor = 0;- cursor += p;- continue;- }- }+ // Handle inline tags+ loop {+ if cursor >= new_tokens.len() {+ break;}- // Handle the block tag, if it exists- if let Some(tag_start) = tokens.iter().tuple_windows().position(|(a, b)| {- matches!(- (a, b),- (- Token {- kind: TokenKind::Punctuation(Punctuation::At),- ..- },- Token {- kind: TokenKind::Word,- ..- }- )- )- }) {- for token in &mut tokens[tag_start..] {- token.kind = TokenKind::Unlintable;+ if let Some(new_cursor) = &new_tokens[cursor..]+ .iter()+ .position(|t| t.kind == TokenKind::Punctuation(Punctuation::OpenCurly))+ .map(|i| i + cursor)+ {+ cursor = *new_cursor;+ } else {+ break;+ }++ let parsers = [parse_link, parse_tutorial];++ for parser in parsers {+ if let Some(p) = parser(&new_tokens[cursor..], source_line) {+ for tok in &mut new_tokens[cursor..cursor + p] {+ tok.kind = TokenKind::Unlintable;+ }++ cursor += p;+ continue;}}+ }- for token in tokens.iter_mut() {- token.span.push_by(actual.start);+ // Handle the block tag, if it exists on the current line.+ if let Some(tag_start) = new_tokens.iter().tuple_windows().position(|(a, b)| {+ matches!(+ (a, b),+ (+ Token {+ kind: TokenKind::Punctuation(Punctuation::At),+ ..+ },+ Token {+ kind: TokenKind::Word,+ ..+ }+ )+ )+ }) {+ for token in &mut new_tokens[tag_start..] {+ token.kind = TokenKind::Unlintable;}+ }- tokens+ for token in new_tokens.iter_mut() {+ token.span.push_by(actual_line.start);}++ new_tokens}fn parse_link(tokens: &[Token], source: &[char]) -> Option<usize> {@@ -112,10 +130,7 @@ fn parse_inline_tag(tag_name: &[char], tokens: &[Token], source: &[char]) -> Optreturn None;}- dbg!(tokens[2].span.get_content(source));-if tokens[2].span.get_content(source) != tag_name {- dbg!();return None;}diff --git a/harper-comments/src/comment_parsers/unit.rs b/harper-comments/src/comment_parsers/unit.rsindex 359bb2653..44fb33a04 100644--- a/harper-comments/src/comment_parsers/unit.rs+++ b/harper-comments/src/comment_parsers/unit.rs@@ -13,22 +13,41 @@ pub struct Unit;impl Parser for Unit {fn parse(&mut self, source: &[char]) -> Vec<Token> {- let actual = without_initiators(source);+ let mut tokens = Vec::new();- if actual.is_empty() {- return Vec::new();- }+ let mut chars_traversed = 0;++ for line in source.split(|c| *c == '\n') {+ let mut new_tokens = parse_line(line);- let source = actual.get_content(source);+ new_tokens+ .iter_mut()+ .for_each(|t| t.span.push_by(chars_traversed));- let mut markdown_parser = Markdown;+ chars_traversed += line.len() + 1;+ tokens.append(&mut new_tokens);+ }- let mut new_tokens = markdown_parser.parse(source);+ tokens+ }+}- new_tokens- .iter_mut()- .for_each(|t| t.span.push_by(actual.start));+fn parse_line(source: &[char]) -> Vec<Token> {+ let actual = without_initiators(source);- new_tokens+ if actual.is_empty() {+ return Vec::new();}++ let source = actual.get_content(source);++ let mut markdown_parser = Markdown;++ let mut new_tokens = markdown_parser.parse(source);++ new_tokens+ .iter_mut()+ .for_each(|t| t.span.push_by(actual.start));++ new_tokens}diff --git a/harper-core/src/char_ext.rs b/harper-core/src/char_ext.rsindex 1c4110456..2d1c7b1ce 100644--- a/harper-core/src/char_ext.rs+++ b/harper-core/src/char_ext.rs@@ -29,7 +29,7 @@ impl CharExt for char {unicode_blocks::EMOTICONS,unicode_blocks::MISCELLANEOUS_SYMBOLS,unicode_blocks::VARIATION_SELECTORS,- unicode_blocks::SUPPLEMENTAL_SYMBOLS_AND_PICTOGRAPHS+ unicode_blocks::SUPPLEMENTAL_SYMBOLS_AND_PICTOGRAPHS,];blocks.contains(&block)diff --git a/harper-core/src/parsers/markdown.rs b/harper-core/src/parsers/markdown.rsindex d36c3ccd0..90e834969 100644--- a/harper-core/src/parsers/markdown.rs+++ b/harper-core/src/parsers/markdown.rs@@ -33,15 +33,29 @@ impl Parser for Markdown {}match event {- pulldown_cmark::Event::SoftBreak | pulldown_cmark::Event::HardBreak => {+ pulldown_cmark::Event::SoftBreak => {tokens.push(Token {span: Span::new_with_len(traversed_chars, 1),kind: TokenKind::Newline(1)});}+ pulldown_cmark::Event::HardBreak => {+ tokens.push(Token {+ span: Span::new_with_len(traversed_chars, 1),+ kind: TokenKind::Newline(2)+ });+ }+ pulldown_cmark::Event::Start(pulldown_cmark::Tag::List(v)) => {+ tokens.push(Token {+ span: Span::new_with_len(traversed_chars, 0),+ kind: TokenKind::Newline(2)+ });+ stack.push(pulldown_cmark::Tag::List(v));+ }pulldown_cmark::Event::Start(tag) => stack.push(tag),pulldown_cmark::Event::End(pulldown_cmark::TagEnd::Paragraph)| pulldown_cmark::Event::End(pulldown_cmark::TagEnd::Item)+ | pulldown_cmark::Event::End(pulldown_cmark::TagEnd::Heading(_))| pulldown_cmark::Event::End(pulldown_cmark::TagEnd::TableCell) => {tokens.push(Token {span: Span::new_with_len(traversed_chars, 0),diff --git a/harper-core/tests/run_tests.rs b/harper-core/tests/run_tests.rsnew file mode 100644index 000000000..63fdb6153--- /dev/null+++ b/harper-core/tests/run_tests.rs@@ -0,0 +1,32 @@+use harper_core::{Document, FullDictionary, LintGroup, LintGroupConfig, Linter};++/// Creates a unit test checking that the linting of a Markdown document (in+/// `tests_sources`) produces the expected number of lints.+macro_rules! create_test {+ ($filename:ident.md, $correct_expected:expr) => {+ paste::paste! {+ #[test]+ fn [<lints_ $filename _correctly>](){+ let source = include_str!(+ concat!(+ "./test_sources/",+ concat!(stringify!($filename), ".md")+ )+ );++ let document = Document::new_markdown(&source);++ let mut linter = LintGroup::new(+ LintGroupConfig::default(),+ FullDictionary::create_from_curated()+ );+ let lints = linter.lint(&document);++ dbg!(&lints);+ assert_eq!(lints.len(), $correct_expected);+ }+ }+ };+}++create_test!(whack_bullets.md, 1);diff --git a/harper-core/tests/test_sources/whack_bullets.md b/harper-core/tests/test_sources/whack_bullets.mdnew file mode 100644index 000000000..e7f886e29--- /dev/null+++ b/harper-core/tests/test_sources/whack_bullets.md@@ -0,0 +1,7 @@+# This is a big heading, with a lot of words++- New here's a list, this part doesn't have as many words+ - But this part does, it has so many words, more words than you could ever dream of+ Just look at all those words+ - So does this part, I might be overwhelmed with all these words+ - This is an test to make sure it isn't crashing