Skip to content

Offline, privacy-first grammar checker. Fast, open-source, Rust-powered

fix(harper-tex): parse triple-hyphens as an em-dash (#2805)

Elijah Potter62870769e9de
diff --git a/harper-tex/src/lib.rs b/harper-tex/src/lib.rs
index d1be80eb7..125ffaf73 100644
--- a/harper-tex/src/lib.rs
+++ b/harper-tex/src/lib.rs
@@ -1,7 +1,7 @@
mod masker;
-use harper_core::Token;
use harper_core::parsers::{Mask, Parser, PlainEnglish};
+use harper_core::{Punctuation, Span, Token, TokenKind};
use self::masker::Masker;
@@ -24,10 +24,62 @@ impl Default for TeX {
impl Parser for TeX {
fn parse(&self, source: &[char]) -> Vec<Token> {
- self.inner.parse(source)
+ let tokens = self.inner.parse(source);
+ collapse_tex_dashes(tokens)
}
}
+fn collapse_tex_dashes(tokens: Vec<Token>) -> Vec<Token> {
+ let mut out = Vec::with_capacity(tokens.len());
+ let mut i = 0;
+
+ while i < tokens.len() {
+ let is_triple_hyphen = i + 2 < tokens.len()
+ && matches!(tokens[i].kind, TokenKind::Punctuation(Punctuation::Hyphen))
+ && matches!(
+ tokens[i + 1].kind,
+ TokenKind::Punctuation(Punctuation::Hyphen)
+ )
+ && matches!(
+ tokens[i + 2].kind,
+ TokenKind::Punctuation(Punctuation::Hyphen)
+ )
+ && tokens[i].span.end == tokens[i + 1].span.start
+ && tokens[i + 1].span.end == tokens[i + 2].span.start;
+
+ if is_triple_hyphen {
+ out.push(Token::new(
+ Span::new(tokens[i].span.start, tokens[i + 2].span.end),
+ TokenKind::Punctuation(Punctuation::EmDash),
+ ));
+ i += 3;
+ continue;
+ }
+
+ let is_double_hyphen = i + 1 < tokens.len()
+ && matches!(tokens[i].kind, TokenKind::Punctuation(Punctuation::Hyphen))
+ && matches!(
+ tokens[i + 1].kind,
+ TokenKind::Punctuation(Punctuation::Hyphen)
+ )
+ && tokens[i].span.end == tokens[i + 1].span.start;
+
+ if is_double_hyphen {
+ out.push(Token::new(
+ Span::new(tokens[i].span.start, tokens[i + 1].span.end),
+ TokenKind::Punctuation(Punctuation::EnDash),
+ ));
+ i += 2;
+ continue;
+ }
+
+ out.push(tokens[i].clone());
+ i += 1;
+ }
+
+ out
+}
+
#[cfg(test)]
mod tests {
use harper_core::TokenKind;
@@ -66,4 +118,38 @@ mod tests {
]
)
}
+
+ #[test]
+ fn parses_triple_hyphen_as_em_dash() {
+ let source = "A---B";
+
+ let toks = TeX::default().parse_str(source);
+ let tok_kinds: Vec<_> = toks.into_iter().map(|t| t.kind).collect();
+
+ assert_eq!(
+ tok_kinds,
+ vec![
+ TokenKind::Word(None),
+ TokenKind::Punctuation(harper_core::Punctuation::EmDash),
+ TokenKind::Word(None),
+ ]
+ )
+ }
+
+ #[test]
+ fn parses_double_hyphen_as_en_dash() {
+ let source = "A--B";
+
+ let toks = TeX::default().parse_str(source);
+ let tok_kinds: Vec<_> = toks.into_iter().map(|t| t.kind).collect();
+
+ assert_eq!(
+ tok_kinds,
+ vec![
+ TokenKind::Word(None),
+ TokenKind::Punctuation(harper_core::Punctuation::EnDash),
+ TokenKind::Word(None),
+ ]
+ )
+ }
}
diff --git a/harper-tex/tests/run_tests.rs b/harper-tex/tests/run_tests.rs
index 24a498795..cfe8f72ac 100644
--- a/harper-tex/tests/run_tests.rs
+++ b/harper-tex/tests/run_tests.rs
@@ -42,3 +42,4 @@ create_test!(title.tex, 1);
create_test!(city.tex, 0);
create_test!(many_tags.tex, 6);
create_test!(many_more_tags.tex, 11);
+create_test!(em_dash.tex, 0);
diff --git a/harper-tex/tests/test_sources/em_dash.tex b/harper-tex/tests/test_sources/em_dash.tex
new file mode 100644
index 000000000..51701a3a7
--- /dev/null
+++ b/harper-tex/tests/test_sources/em_dash.tex
@@ -0,0 +1,7 @@
+\documentclass{article}
+
+\begin{document}
+
+No one's expecting you to drink yourself under the table---no one's ever expected that---but maybe a tipple is what you need after today.
+
+\end{document}