Skip to content

Offline, privacy-first grammar checker. Fast, open-source, Rust-powered

feat: Markdown parser now ignores hidden Wikilink text

Elijah Potter0761702a8417
diff --git a/harper-core/dictionary.dict b/harper-core/dictionary.dict
index 4c9b7499f..c3e32db4a 100644
--- a/harper-core/dictionary.dict
+++ b/harper-core/dictionary.dict
@@ -49650,3 +49650,4 @@ substring/SM
Rosenthal/SM
ACM/M
scatterplot/SMG
+Wikilink/SM
diff --git a/harper-core/src/document.rs b/harper-core/src/document.rs
index 5cd8959e5..83748503e 100644
--- a/harper-core/src/document.rs
+++ b/harper-core/src/document.rs
@@ -548,6 +548,7 @@ impl TokenStringExt for Document {
create_fns_on_doc!(word);
create_fns_on_doc!(space);
create_fns_on_doc!(apostrophe);
+ create_fns_on_doc!(pipe);
create_fns_on_doc!(quote);
create_fns_on_doc!(number);
create_fns_on_doc!(at);
diff --git a/harper-core/src/parsers/markdown.rs b/harper-core/src/parsers/markdown.rs
index 34e5225e0..155fdd10e 100644
--- a/harper-core/src/parsers/markdown.rs
+++ b/harper-core/src/parsers/markdown.rs
@@ -1,5 +1,7 @@
+use std::collections::VecDeque;
+
use super::{Parser, PlainEnglish};
-use crate::{Span, Token, TokenKind};
+use crate::{Span, Token, TokenKind, TokenStringExt, VecExt};
/// A parser that wraps the [`PlainEnglish`] parser that allows one to parse
/// CommonMark files.
@@ -7,6 +9,114 @@ use crate::{Span, Token, TokenKind};
/// Will ignore code blocks and tables.
pub struct Markdown;
+impl Markdown {
+ /// Remove hidden Wikilink target text.
+ ///
+ /// As in, the stuff to the left of the pipe operator:
+ ///
+ /// ```markdown
+ /// [[Target text|Display Text]]
+ /// ```
+ fn remove_hidden_wikilink_tokens(tokens: &mut Vec<Token>) {
+ let mut to_remove = VecDeque::new();
+
+ for pipe_idx in tokens.iter_pipe_indices() {
+ // Locate preceding `[[`
+ let mut cursor = pipe_idx - 2;
+ let mut open_bracket = None;
+
+ loop {
+ let Some((a, b)) = tokens.get(cursor).zip(tokens.get(cursor + 1)) else {
+ break;
+ };
+
+ if a.kind.is_newline() {
+ break;
+ }
+
+ if a.kind.is_open_square() && b.kind.is_open_square() {
+ open_bracket = Some(cursor);
+ break;
+ } else if cursor == 0 {
+ break;
+ } else {
+ cursor -= 1;
+ }
+ }
+
+ // Locate succeeding `[[`
+ cursor = pipe_idx + 1;
+ let mut close_bracket = None;
+
+ loop {
+ let Some((a, b)) = tokens.get(cursor).zip(tokens.get(cursor + 1)) else {
+ break;
+ };
+
+ if a.kind.is_newline() {
+ break;
+ }
+
+ if a.kind.is_close_square() && b.kind.is_close_square() {
+ close_bracket = Some(cursor);
+ break;
+ } else {
+ cursor += 1;
+ }
+ }
+
+ if let Some(open_bracket_idx) = open_bracket {
+ if let Some(close_bracket_idx) = close_bracket {
+ to_remove.extend(open_bracket_idx..=pipe_idx);
+ to_remove.push_back(close_bracket_idx);
+ to_remove.push_back(close_bracket_idx + 1);
+ }
+ }
+ }
+
+ tokens.remove_indices(to_remove);
+ }
+
+ /// Remove the brackets from Wikilinks without pipe operators.
+ /// For __those__ Wikilinks, see [`Self::remove_hidden_wikilink_tokens`]
+ fn remove_wikilink_brackets(tokens: &mut Vec<Token>) {
+ let mut to_remove = VecDeque::new();
+ let mut open_brackets = None;
+
+ let mut cursor = 0;
+
+ loop {
+ let Some((a, b)) = tokens.get(cursor).zip(tokens.get(cursor + 1)) else {
+ break;
+ };
+
+ if let Some(open_brackets_idx) = open_brackets {
+ if a.kind.is_newline() {
+ open_brackets = None;
+ cursor += 1;
+ continue;
+ }
+
+ if a.kind.is_close_square() && b.kind.is_close_square() {
+ to_remove.push_back(open_brackets_idx);
+ to_remove.push_back(open_brackets_idx + 1);
+
+ to_remove.push_back(cursor);
+ to_remove.push_back(cursor + 1);
+
+ open_brackets = None;
+ }
+ } else if a.kind.is_open_square() && b.kind.is_open_square() {
+ open_brackets = Some(cursor);
+ }
+
+ cursor += 1;
+ }
+
+ tokens.remove_indices(to_remove);
+ }
+}
+
impl Parser for Markdown {
/// This implementation is quite gross to look at, but it works.
/// If any issues arise, it would likely help to refactor this out first.
@@ -127,6 +237,9 @@ impl Parser for Markdown {
tokens.pop();
}
+ Self::remove_hidden_wikilink_tokens(&mut tokens);
+ Self::remove_wikilink_brackets(&mut tokens);
+
tokens
}
}
@@ -174,4 +287,67 @@ mod tests {
]
)
}
+
+ #[test]
+ fn hidden_wikilink_text() {
+ let source = r#"[[this is hidden|this is not]]"#;
+
+ let tokens = Markdown.parse_str(source);
+
+ let token_kinds = tokens.iter().map(|t| t.kind).collect::<Vec<_>>();
+
+ assert!(matches!(
+ token_kinds.as_slice(),
+ &[
+ TokenKind::Word(_),
+ TokenKind::Space(1),
+ TokenKind::Word(_),
+ TokenKind::Space(1),
+ TokenKind::Word(_),
+ ]
+ ))
+ }
+
+ #[test]
+ fn improper_wikilink_text() {
+ let source = r#"this is shown|this is also shown]]"#;
+
+ let tokens = Markdown.parse_str(source);
+
+ let token_kinds = tokens.iter().map(|t| t.kind).collect::<Vec<_>>();
+
+ dbg!(&token_kinds);
+
+ assert!(matches!(
+ token_kinds.as_slice(),
+ &[
+ TokenKind::Word(_),
+ TokenKind::Space(1),
+ TokenKind::Word(_),
+ TokenKind::Space(1),
+ TokenKind::Word(_),
+ TokenKind::Punctuation(Punctuation::Pipe),
+ TokenKind::Word(_),
+ TokenKind::Space(1),
+ TokenKind::Word(_),
+ TokenKind::Space(1),
+ TokenKind::Word(_),
+ TokenKind::Space(1),
+ TokenKind::Word(_),
+ TokenKind::Punctuation(Punctuation::CloseSquare),
+ TokenKind::Punctuation(Punctuation::CloseSquare),
+ ]
+ ))
+ }
+
+ #[test]
+ fn normal_wikilink() {
+ let source = r#"[[Wikilink]]"#;
+ let tokens = Markdown.parse_str(source);
+ let token_kinds = tokens.iter().map(|t| t.kind).collect::<Vec<_>>();
+
+ dbg!(&token_kinds);
+
+ assert!(matches!(token_kinds.as_slice(), &[TokenKind::Word(_)]))
+ }
}
diff --git a/harper-core/src/token.rs b/harper-core/src/token.rs
index 013889633..be760b0ac 100644
--- a/harper-core/src/token.rs
+++ b/harper-core/src/token.rs
@@ -58,6 +58,18 @@ pub enum TokenKind {
}
impl TokenKind {
+ pub fn is_open_square(&self) -> bool {
+ matches!(self, TokenKind::Punctuation(Punctuation::OpenSquare))
+ }
+
+ pub fn is_close_square(&self) -> bool {
+ matches!(self, TokenKind::Punctuation(Punctuation::CloseSquare))
+ }
+
+ pub fn is_pipe(&self) -> bool {
+ matches!(self, TokenKind::Punctuation(Punctuation::Pipe))
+ }
+
/// Checks that `self` is the same enum variant as `other`, regardless of
/// whether the inner metadata is also equal.
pub fn matches_variant_of(&self, other: &Self) -> bool {
@@ -234,6 +246,7 @@ pub trait TokenStringExt {
create_decl_for!(word);
create_decl_for!(space);
create_decl_for!(apostrophe);
+ create_decl_for!(pipe);
create_decl_for!(quote);
create_decl_for!(number);
create_decl_for!(at);
@@ -243,6 +256,7 @@ impl TokenStringExt for [Token] {
create_fns_for!(word);
create_fns_for!(space);
create_fns_for!(apostrophe);
+ create_fns_for!(pipe);
create_fns_for!(quote);
create_fns_for!(number);
create_fns_for!(at);
diff --git a/harper-core/tests/run_tests.rs b/harper-core/tests/run_tests.rs
index 48e179df5..9717ccfbc 100644
--- a/harper-core/tests/run_tests.rs
+++ b/harper-core/tests/run_tests.rs
@@ -36,3 +36,4 @@ create_test!(preexisting.md, 0);
create_test!(issue_109.md, 0);
create_test!(issue_109_ext.md, 0);
create_test!(chinese_lorem_ipsum.md, 2);
+create_test!(obsidian_links.md, 2);
diff --git a/harper-core/tests/test_sources/obsidian_links.md b/harper-core/tests/test_sources/obsidian_links.md
new file mode 100644
index 000000000..9b5cb15f3
--- /dev/null
+++ b/harper-core/tests/test_sources/obsidian_links.md
@@ -0,0 +1,15 @@
+# Example Obsidian Links
+
+Below, you will find a number of example links that Obsidian is able to process.
+
+These should be treated as normal Markdown links.
+The stuff inside the square brackets is visible and should be checked by Harper.
+
+[[Three lws of motion]]
+[Three las of motion](Three%20laws%20of%20motion.md)
+
+
+Wikilinks allow you to replace the visible text with a pipe (|) operator.
+The text to the left of the operator should be ignored.
+
+[[lnk tget|Link Text]]