Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
fix: better handling of spaces and newlines
Grant Lemonsb61c78c4753c
diff --git a/harper-cli/src/main.rs b/harper-cli/src/main.rsindex 3209ec4bb..0478fdf47 100644--- a/harper-cli/src/main.rs+++ b/harper-cli/src/main.rs@@ -30,6 +30,9 @@ enum Args {Spans {/// The file you wish to display the spans.file: PathBuf,+ /// Include newlines in the output+ #[arg(short, long)]+ include_newlines: bool,},/// Emit decompressed, line-separated list of words in Harper's dictionary.Words,@@ -89,7 +92,10 @@ fn main() -> anyhow::Result<()> {Ok(())}- Args::Spans { file } => {+ Args::Spans {+ file,+ include_newlines,+ } => {let (doc, source) = load_file(&file)?;let primary_color = Color::Blue;@@ -103,7 +109,11 @@ fn main() -> anyhow::Result<()> {let mut report_builder =Report::build(ReportKind::Custom("Spans", primary_color), &filename, 0);let mut color = primary_color;- for token in doc.tokens() {++ for token in doc.tokens().filter(|t| {+ include_newlines+ || !matches!(t.kind, TokenKind::Newline(_) | TokenKind::ParagraphBreak)+ }) {report_builder = report_builder.with_label(Label::new((&filename, token.span.into())).with_message(format!("[{}, {})", token.span.start, token.span.end))diff --git a/harper-core/src/document.rs b/harper-core/src/document.rsindex 80e7f2b05..612de9edf 100644--- a/harper-core/src/document.rs+++ b/harper-core/src/document.rs@@ -286,15 +286,15 @@ impl Document {let child_tok = ©[cursor];// Only condense adjacent spans- if start_tok.span.end == child_tok.span.start {- if let TokenKind::Space(n) = child_tok.kind {- *start_count += n;- start_tok.span.end = child_tok.span.end;- remove_these.push_back(cursor);- cursor += 1;- } else {- break;- }+ if start_tok.span.end != child_tok.span.start {+ break;+ }++ if let TokenKind::Space(n) = child_tok.kind {+ *start_count += n;+ start_tok.span.end = child_tok.span.end;+ remove_these.push_back(cursor);+ cursor += 1;} else {break;};diff --git a/harper-core/src/parsers/typst.rs b/harper-core/src/parsers/typst.rsindex a58f89d8c..9264dcf19 100644--- a/harper-core/src/parsers/typst.rs+++ b/harper-core/src/parsers/typst.rs@@ -237,7 +237,21 @@ impl<'a> ParseHelper<'a> {match ex {Expr::Text(text) => self.parse_english(text.get(), offset.push_to_span(text.span())),- Expr::Space(a) => token!(a, TokenKind::Space(1)),+ Expr::Space(a) => {+ let mut chars = self+ .doc+ .get(self.doc.range(a.span()).unwrap())+ .unwrap()+ .chars();+ let first_char = chars.next().unwrap();+ let length = chars.count() + 1;++ if first_char == '\n' {+ token!(a, TokenKind::Newline(1))+ } else {+ token!(a, TokenKind::Space(length))+ }+ }Expr::Linebreak(a) => token!(a, TokenKind::Newline(1)),Expr::Parbreak(a) => token!(a, TokenKind::ParagraphBreak),Expr::SmartQuote(quote) => {@@ -274,7 +288,7 @@ impl<'a> ParseHelper<'a> {}Expr::Str(text) => {let offset = offset.push_to_span(text.span()).char + 1;- let string = text.to_untyped().text().to_string();+ let string = text.to_untyped().text();Some(self.parser@@ -322,7 +336,8 @@ impl<'a> ParseHelper<'a> {}Expr::Set(set_rule) => merge!(recurse!(set_rule.target()),- set_rule.condition().and_then(|expr| recurse!(expr))+ set_rule.condition().and_then(|expr| recurse!(expr)),+ parse_args(&mut set_rule.args().items())),Expr::Show(show_rule) => merge!(recurse!(show_rule.transform()),@@ -565,7 +580,7 @@ mod tests {#[test]fn non_adjacent_spaces_not_condensed() {- let source = r#"#authors_slice.join(", ", last: ", and ") "#;+ let source = r#"#authors_slice.join(", ", last: ", and ") bob"#;let token_kinds = Typst.parse_str(source).iter().map(|t| t.kind).collect_vec();dbg!(&token_kinds);@@ -581,7 +596,8 @@ mod tests {TokenKind::Space(1),TokenKind::Word(_), // andTokenKind::Space(1),- TokenKind::Space(1)+ TokenKind::Space(2),+ TokenKind::Word(_),]))}@@ -601,7 +617,11 @@ mod tests {assert!(matches!(&token_kinds.as_slice(),- &[TokenKind::Word(_), TokenKind::Space(1), TokenKind::Word(_)]+ &[+ TokenKind::Word(_),+ TokenKind::Newline(1),+ TokenKind::Word(_)+ ]))}@@ -638,9 +658,9 @@ mod tests {&token_kinds.as_slice(),&[TokenKind::Word(_),- TokenKind::Space(1),+ TokenKind::Newline(1),TokenKind::Unlintable,- TokenKind::Space(1),+ TokenKind::Newline(1),TokenKind::Word(_),]))@@ -700,7 +720,7 @@ writing"#;}),..}),- TokenKind::Space(1),+ TokenKind::Newline(1),TokenKind::Word(_),]));