Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
Markdown parser now properly parses lists
Elijah Potter1077ceb73b69
diff --git a/harper-core/dictionary.aff b/harper-core/dictionary.affindex af8eb37e6..5c6acba93 100644--- a/harper-core/dictionary.aff+++ b/harper-core/dictionary.aff@@ -113,7 +113,10 @@ SFX L Y 1SFX L 0 ment .SFX O Y 1-SFX O 0 ful .+SFX O 0 ful .++SFX Q Y 1+SFX Q 0 ally .REP 90REP a eidiff --git a/harper-core/dictionary.dict b/harper-core/dictionary.dictindex dbc6b27eb..0cfe142bc 100644--- a/harper-core/dictionary.dict+++ b/harper-core/dictionary.dict@@ -11632,7 +11632,7 @@ algebra/SMalgebraicalgebraicallyalgorithm/SM-algorithmic+algorithmic/Qalias/GMDSalibi/GMDSalien/BGMDS@@ -49591,3 +49591,4 @@ raytracerviewportbackendfrontend+automatadiff --git a/harper-core/src/document.rs b/harper-core/src/document.rsindex 08be7e687..b9d249c1f 100644--- a/harper-core/src/document.rs+++ b/harper-core/src/document.rs@@ -172,14 +172,8 @@ impl Document {/// Iterate over the locations of the sentence terminators in the document.fn sentence_terminators(&self) -> impl Iterator<Item = usize> + '_ {self.tokens.iter().enumerate().filter_map(|(index, token)| {- if let Token {- kind: TokenKind::Punctuation(punct),- ..- } = token- {- if is_sentence_terminator(punct) {- return Some(index);- }+ if is_sentence_terminator(&token.kind) {+ return Some(index);}None})@@ -192,14 +186,8 @@ impl Document {.enumerate().rev().find_map(|(index, token)| {- if let Token {- kind: TokenKind::Punctuation(punct),- ..- } = token- {- if is_sentence_terminator(punct) {- return Some(index);- }+ if is_sentence_terminator(&token.kind) {+ return Some(index);}None})@@ -287,13 +275,17 @@ impl Display for Document {}}-fn is_sentence_terminator(punctuation: &Punctuation) -> bool {- [- Punctuation::Period,- Punctuation::Bang,- Punctuation::Question,- ]- .contains(punctuation)+fn is_sentence_terminator(token: &TokenKind) -> bool {+ match token {+ TokenKind::Punctuation(punct) => [+ Punctuation::Period,+ Punctuation::Bang,+ Punctuation::Question,+ ]+ .contains(punct),+ TokenKind::Newline(_) => true,+ _ => false,+ }}#[cfg(test)]@@ -313,7 +305,8 @@ mod tests {let mut document = Document::new(text, Box::new(Markdown));document.condense_contractions();- assert_eq!(document.tokens.len(), final_tok_count);+ // We add one because the Markdown parser inserts a newline at end-of-input.+ assert_eq!(document.tokens.len(), final_tok_count + 1);}#[test]diff --git a/harper-core/src/parsers/markdown.rs b/harper-core/src/parsers/markdown.rsindex 29dab5f6c..9fb39a09e 100644--- a/harper-core/src/parsers/markdown.rs+++ b/harper-core/src/parsers/markdown.rs@@ -24,15 +24,28 @@ impl Parser for Markdown {// NOTE: the range spits out __byte__ indices, not char indices.// This is why we keep track above.for (event, range) in md_parser.into_offset_iter() {+ if range.start > traversed_bytes {+ traversed_chars += source_str[traversed_bytes..range.start].chars().count();+ traversed_bytes = range.start;+ }+match event {+ pulldown_cmark::Event::HardBreak => {+ tokens.push(Token {+ span: Span::new_with_len(traversed_chars, 1),+ kind: TokenKind::Newline(1),+ });+ }pulldown_cmark::Event::Start(tag) => stack.push(tag),+ pulldown_cmark::Event::End(pulldown_cmark::Tag::Paragraph)+ | pulldown_cmark::Event::End(pulldown_cmark::Tag::Item) => tokens.push(Token {+ span: Span::new_with_len(traversed_chars, 1),+ kind: TokenKind::Newline(1),+ }),pulldown_cmark::Event::End(_) => {stack.pop();}pulldown_cmark::Event::Code(code) => {- traversed_chars += source_str[traversed_bytes..range.start].chars().count();- traversed_bytes = range.start;-let chunk_len = code.chars().count();tokens.push(Token {@@ -41,9 +54,6 @@ impl Parser for Markdown {});}pulldown_cmark::Event::Text(text) => {- traversed_chars += source_str[traversed_bytes..range.start].chars().count();- traversed_bytes = range.start;-let chunk_len = text.chars().count();if let Some(tag) = stack.last() {diff --git a/harper-core/src/parsers/mod.rs b/harper-core/src/parsers/mod.rsindex 69a3b3dc6..9e608c7cd 100644--- a/harper-core/src/parsers/mod.rs+++ b/harper-core/src/parsers/mod.rs@@ -89,6 +89,25 @@ mod tests {Word,Space(1),Word,+ Newline(1),+ ],+ );+ }++ #[test]+ fn inserts_newlines() {+ assert_tokens_eq_md(+ "__hello__ world,\n\n[my]() friend",+ &[+ Word,+ Space(1),+ Word,+ Punctuation(Punctuation::Comma),+ Newline(1),+ Word,+ Space(1),+ Word,+ Newline(1),],);}diff --git a/harper-core/src/span.rs b/harper-core/src/span.rsindex 0ec7e89fa..1c5ebef6c 100644--- a/harper-core/src/span.rs+++ b/harper-core/src/span.rs@@ -14,6 +14,13 @@ impl Span {Self { start, end }}+ pub fn new_with_len(start: usize, len: usize) -> Self {+ Self {+ start,+ end: start + len,+ }+ }+pub fn len(&self) -> usize {self.end - self.start}