Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
feat: Markdown parser now ignores hidden Wikilink text
Elijah Potter0761702a8417
diff --git a/harper-core/dictionary.dict b/harper-core/dictionary.dictindex 4c9b7499f..c3e32db4a 100644--- a/harper-core/dictionary.dict+++ b/harper-core/dictionary.dict@@ -49650,3 +49650,4 @@ substring/SMRosenthal/SMACM/Mscatterplot/SMG+Wikilink/SMdiff --git a/harper-core/src/document.rs b/harper-core/src/document.rsindex 5cd8959e5..83748503e 100644--- a/harper-core/src/document.rs+++ b/harper-core/src/document.rs@@ -548,6 +548,7 @@ impl TokenStringExt for Document {create_fns_on_doc!(word);create_fns_on_doc!(space);create_fns_on_doc!(apostrophe);+ create_fns_on_doc!(pipe);create_fns_on_doc!(quote);create_fns_on_doc!(number);create_fns_on_doc!(at);diff --git a/harper-core/src/parsers/markdown.rs b/harper-core/src/parsers/markdown.rsindex 34e5225e0..155fdd10e 100644--- a/harper-core/src/parsers/markdown.rs+++ b/harper-core/src/parsers/markdown.rs@@ -1,5 +1,7 @@+use std::collections::VecDeque;+use super::{Parser, PlainEnglish};-use crate::{Span, Token, TokenKind};+use crate::{Span, Token, TokenKind, TokenStringExt, VecExt};/// A parser that wraps the [`PlainEnglish`] parser that allows one to parse/// CommonMark files.@@ -7,6 +9,114 @@ use crate::{Span, Token, TokenKind};/// Will ignore code blocks and tables.pub struct Markdown;+impl Markdown {+ /// Remove hidden Wikilink target text.+ ///+ /// As in, the stuff to the left of the pipe operator:+ ///+ /// ```markdown+ /// [[Target text|Display Text]]+ /// ```+ fn remove_hidden_wikilink_tokens(tokens: &mut Vec<Token>) {+ let mut to_remove = VecDeque::new();++ for pipe_idx in tokens.iter_pipe_indices() {+ // Locate preceding `[[`+ let mut cursor = pipe_idx - 2;+ let mut open_bracket = None;++ loop {+ let Some((a, b)) = tokens.get(cursor).zip(tokens.get(cursor + 1)) else {+ break;+ };++ if a.kind.is_newline() {+ break;+ }++ if a.kind.is_open_square() && b.kind.is_open_square() {+ open_bracket = Some(cursor);+ break;+ } else if cursor == 0 {+ break;+ } else {+ cursor -= 1;+ }+ }++ // Locate succeeding `[[`+ cursor = pipe_idx + 1;+ let mut close_bracket = None;++ loop {+ let Some((a, b)) = tokens.get(cursor).zip(tokens.get(cursor + 1)) else {+ break;+ };++ if a.kind.is_newline() {+ break;+ }++ if a.kind.is_close_square() && b.kind.is_close_square() {+ close_bracket = Some(cursor);+ break;+ } else {+ cursor += 1;+ }+ }++ if let Some(open_bracket_idx) = open_bracket {+ if let Some(close_bracket_idx) = close_bracket {+ to_remove.extend(open_bracket_idx..=pipe_idx);+ to_remove.push_back(close_bracket_idx);+ to_remove.push_back(close_bracket_idx + 1);+ }+ }+ }++ tokens.remove_indices(to_remove);+ }++ /// Remove the brackets from Wikilinks without pipe operators.+ /// For __those__ Wikilinks, see [`Self::remove_hidden_wikilink_tokens`]+ fn remove_wikilink_brackets(tokens: &mut Vec<Token>) {+ let mut to_remove = VecDeque::new();+ let mut open_brackets = None;++ let mut cursor = 0;++ loop {+ let Some((a, b)) = tokens.get(cursor).zip(tokens.get(cursor + 1)) else {+ break;+ };++ if let Some(open_brackets_idx) = open_brackets {+ if a.kind.is_newline() {+ open_brackets = None;+ cursor += 1;+ continue;+ }++ if a.kind.is_close_square() && b.kind.is_close_square() {+ to_remove.push_back(open_brackets_idx);+ to_remove.push_back(open_brackets_idx + 1);++ to_remove.push_back(cursor);+ to_remove.push_back(cursor + 1);++ open_brackets = None;+ }+ } else if a.kind.is_open_square() && b.kind.is_open_square() {+ open_brackets = Some(cursor);+ }++ cursor += 1;+ }++ tokens.remove_indices(to_remove);+ }+}+impl Parser for Markdown {/// This implementation is quite gross to look at, but it works./// If any issues arise, it would likely help to refactor this out first.@@ -127,6 +237,9 @@ impl Parser for Markdown {tokens.pop();}+ Self::remove_hidden_wikilink_tokens(&mut tokens);+ Self::remove_wikilink_brackets(&mut tokens);+tokens}}@@ -174,4 +287,67 @@ mod tests {])}++ #[test]+ fn hidden_wikilink_text() {+ let source = r#"[[this is hidden|this is not]]"#;++ let tokens = Markdown.parse_str(source);++ let token_kinds = tokens.iter().map(|t| t.kind).collect::<Vec<_>>();++ assert!(matches!(+ token_kinds.as_slice(),+ &[+ TokenKind::Word(_),+ TokenKind::Space(1),+ TokenKind::Word(_),+ TokenKind::Space(1),+ TokenKind::Word(_),+ ]+ ))+ }++ #[test]+ fn improper_wikilink_text() {+ let source = r#"this is shown|this is also shown]]"#;++ let tokens = Markdown.parse_str(source);++ let token_kinds = tokens.iter().map(|t| t.kind).collect::<Vec<_>>();++ dbg!(&token_kinds);++ assert!(matches!(+ token_kinds.as_slice(),+ &[+ TokenKind::Word(_),+ TokenKind::Space(1),+ TokenKind::Word(_),+ TokenKind::Space(1),+ TokenKind::Word(_),+ TokenKind::Punctuation(Punctuation::Pipe),+ TokenKind::Word(_),+ TokenKind::Space(1),+ TokenKind::Word(_),+ TokenKind::Space(1),+ TokenKind::Word(_),+ TokenKind::Space(1),+ TokenKind::Word(_),+ TokenKind::Punctuation(Punctuation::CloseSquare),+ TokenKind::Punctuation(Punctuation::CloseSquare),+ ]+ ))+ }++ #[test]+ fn normal_wikilink() {+ let source = r#"[[Wikilink]]"#;+ let tokens = Markdown.parse_str(source);+ let token_kinds = tokens.iter().map(|t| t.kind).collect::<Vec<_>>();++ dbg!(&token_kinds);++ assert!(matches!(token_kinds.as_slice(), &[TokenKind::Word(_)]))+ }}diff --git a/harper-core/src/token.rs b/harper-core/src/token.rsindex 013889633..be760b0ac 100644--- a/harper-core/src/token.rs+++ b/harper-core/src/token.rs@@ -58,6 +58,18 @@ pub enum TokenKind {}impl TokenKind {+ pub fn is_open_square(&self) -> bool {+ matches!(self, TokenKind::Punctuation(Punctuation::OpenSquare))+ }++ pub fn is_close_square(&self) -> bool {+ matches!(self, TokenKind::Punctuation(Punctuation::CloseSquare))+ }++ pub fn is_pipe(&self) -> bool {+ matches!(self, TokenKind::Punctuation(Punctuation::Pipe))+ }+/// Checks that `self` is the same enum variant as `other`, regardless of/// whether the inner metadata is also equal.pub fn matches_variant_of(&self, other: &Self) -> bool {@@ -234,6 +246,7 @@ pub trait TokenStringExt {create_decl_for!(word);create_decl_for!(space);create_decl_for!(apostrophe);+ create_decl_for!(pipe);create_decl_for!(quote);create_decl_for!(number);create_decl_for!(at);@@ -243,6 +256,7 @@ impl TokenStringExt for [Token] {create_fns_for!(word);create_fns_for!(space);create_fns_for!(apostrophe);+ create_fns_for!(pipe);create_fns_for!(quote);create_fns_for!(number);create_fns_for!(at);diff --git a/harper-core/tests/run_tests.rs b/harper-core/tests/run_tests.rsindex 48e179df5..9717ccfbc 100644--- a/harper-core/tests/run_tests.rs+++ b/harper-core/tests/run_tests.rs@@ -36,3 +36,4 @@ create_test!(preexisting.md, 0);create_test!(issue_109.md, 0);create_test!(issue_109_ext.md, 0);create_test!(chinese_lorem_ipsum.md, 2);+create_test!(obsidian_links.md, 2);diff --git a/harper-core/tests/test_sources/obsidian_links.md b/harper-core/tests/test_sources/obsidian_links.mdnew file mode 100644index 000000000..9b5cb15f3--- /dev/null+++ b/harper-core/tests/test_sources/obsidian_links.md@@ -0,0 +1,15 @@+# Example Obsidian Links++Below, you will find a number of example links that Obsidian is able to process.++These should be treated as normal Markdown links.+The stuff inside the square brackets is visible and should be checked by Harper.++[[Three lws of motion]]+[Three las of motion](Three%20laws%20of%20motion.md)+++Wikilinks allow you to replace the visible text with a pipe (|) operator.+The text to the left of the operator should be ignored.++[[lnk tget|Link Text]]