Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
fix(harper-tex): parse triple-hyphens as an em-dash (#2805)
Elijah Potter62870769e9de
diff --git a/harper-tex/src/lib.rs b/harper-tex/src/lib.rsindex d1be80eb7..125ffaf73 100644--- a/harper-tex/src/lib.rs+++ b/harper-tex/src/lib.rs@@ -1,7 +1,7 @@mod masker;-use harper_core::Token;use harper_core::parsers::{Mask, Parser, PlainEnglish};+use harper_core::{Punctuation, Span, Token, TokenKind};use self::masker::Masker;@@ -24,10 +24,62 @@ impl Default for TeX {impl Parser for TeX {fn parse(&self, source: &[char]) -> Vec<Token> {- self.inner.parse(source)+ let tokens = self.inner.parse(source);+ collapse_tex_dashes(tokens)}}+fn collapse_tex_dashes(tokens: Vec<Token>) -> Vec<Token> {+ let mut out = Vec::with_capacity(tokens.len());+ let mut i = 0;++ while i < tokens.len() {+ let is_triple_hyphen = i + 2 < tokens.len()+ && matches!(tokens[i].kind, TokenKind::Punctuation(Punctuation::Hyphen))+ && matches!(+ tokens[i + 1].kind,+ TokenKind::Punctuation(Punctuation::Hyphen)+ )+ && matches!(+ tokens[i + 2].kind,+ TokenKind::Punctuation(Punctuation::Hyphen)+ )+ && tokens[i].span.end == tokens[i + 1].span.start+ && tokens[i + 1].span.end == tokens[i + 2].span.start;++ if is_triple_hyphen {+ out.push(Token::new(+ Span::new(tokens[i].span.start, tokens[i + 2].span.end),+ TokenKind::Punctuation(Punctuation::EmDash),+ ));+ i += 3;+ continue;+ }++ let is_double_hyphen = i + 1 < tokens.len()+ && matches!(tokens[i].kind, TokenKind::Punctuation(Punctuation::Hyphen))+ && matches!(+ tokens[i + 1].kind,+ TokenKind::Punctuation(Punctuation::Hyphen)+ )+ && tokens[i].span.end == tokens[i + 1].span.start;++ if is_double_hyphen {+ out.push(Token::new(+ Span::new(tokens[i].span.start, tokens[i + 1].span.end),+ TokenKind::Punctuation(Punctuation::EnDash),+ ));+ i += 2;+ continue;+ }++ out.push(tokens[i].clone());+ i += 1;+ }++ out+}+#[cfg(test)]mod tests {use harper_core::TokenKind;@@ -66,4 +118,38 @@ mod tests {])}++ #[test]+ fn parses_triple_hyphen_as_em_dash() {+ let source = "A---B";++ let toks = TeX::default().parse_str(source);+ let tok_kinds: Vec<_> = toks.into_iter().map(|t| t.kind).collect();++ assert_eq!(+ tok_kinds,+ vec![+ TokenKind::Word(None),+ TokenKind::Punctuation(harper_core::Punctuation::EmDash),+ TokenKind::Word(None),+ ]+ )+ }++ #[test]+ fn parses_double_hyphen_as_en_dash() {+ let source = "A--B";++ let toks = TeX::default().parse_str(source);+ let tok_kinds: Vec<_> = toks.into_iter().map(|t| t.kind).collect();++ assert_eq!(+ tok_kinds,+ vec![+ TokenKind::Word(None),+ TokenKind::Punctuation(harper_core::Punctuation::EnDash),+ TokenKind::Word(None),+ ]+ )+ }}diff --git a/harper-tex/tests/run_tests.rs b/harper-tex/tests/run_tests.rsindex 24a498795..cfe8f72ac 100644--- a/harper-tex/tests/run_tests.rs+++ b/harper-tex/tests/run_tests.rs@@ -42,3 +42,4 @@ create_test!(title.tex, 1);create_test!(city.tex, 0);create_test!(many_tags.tex, 6);create_test!(many_more_tags.tex, 11);+create_test!(em_dash.tex, 0);diff --git a/harper-tex/tests/test_sources/em_dash.tex b/harper-tex/tests/test_sources/em_dash.texnew file mode 100644index 000000000..51701a3a7--- /dev/null+++ b/harper-tex/tests/test_sources/em_dash.tex@@ -0,0 +1,7 @@+\documentclass{article}++\begin{document}++No one's expecting you to drink yourself under the table---no one's ever expected that---but maybe a tipple is what you need after today.++\end{document}