Skip to content

Offline, privacy-first grammar checker. Fast, open-source, Rust-powered

fix: better handling of spaces and newlines

Grant Lemonsb61c78c4753c
diff --git a/harper-cli/src/main.rs b/harper-cli/src/main.rs
index 3209ec4bb..0478fdf47 100644
--- a/harper-cli/src/main.rs
+++ b/harper-cli/src/main.rs
@@ -30,6 +30,9 @@ enum Args {
Spans {
/// The file you wish to display the spans.
file: PathBuf,
+ /// Include newlines in the output
+ #[arg(short, long)]
+ include_newlines: bool,
},
/// Emit decompressed, line-separated list of words in Harper's dictionary.
Words,
@@ -89,7 +92,10 @@ fn main() -> anyhow::Result<()> {
Ok(())
}
- Args::Spans { file } => {
+ Args::Spans {
+ file,
+ include_newlines,
+ } => {
let (doc, source) = load_file(&file)?;
let primary_color = Color::Blue;
@@ -103,7 +109,11 @@ fn main() -> anyhow::Result<()> {
let mut report_builder =
Report::build(ReportKind::Custom("Spans", primary_color), &filename, 0);
let mut color = primary_color;
- for token in doc.tokens() {
+
+ for token in doc.tokens().filter(|t| {
+ include_newlines
+ || !matches!(t.kind, TokenKind::Newline(_) | TokenKind::ParagraphBreak)
+ }) {
report_builder = report_builder.with_label(
Label::new((&filename, token.span.into()))
.with_message(format!("[{}, {})", token.span.start, token.span.end))
diff --git a/harper-core/src/document.rs b/harper-core/src/document.rs
index 80e7f2b05..612de9edf 100644
--- a/harper-core/src/document.rs
+++ b/harper-core/src/document.rs
@@ -286,15 +286,15 @@ impl Document {
let child_tok = &copy[cursor];
// Only condense adjacent spans
- if start_tok.span.end == child_tok.span.start {
- if let TokenKind::Space(n) = child_tok.kind {
- *start_count += n;
- start_tok.span.end = child_tok.span.end;
- remove_these.push_back(cursor);
- cursor += 1;
- } else {
- break;
- }
+ if start_tok.span.end != child_tok.span.start {
+ break;
+ }
+
+ if let TokenKind::Space(n) = child_tok.kind {
+ *start_count += n;
+ start_tok.span.end = child_tok.span.end;
+ remove_these.push_back(cursor);
+ cursor += 1;
} else {
break;
};
diff --git a/harper-core/src/parsers/typst.rs b/harper-core/src/parsers/typst.rs
index a58f89d8c..9264dcf19 100644
--- a/harper-core/src/parsers/typst.rs
+++ b/harper-core/src/parsers/typst.rs
@@ -237,7 +237,21 @@ impl<'a> ParseHelper<'a> {
match ex {
Expr::Text(text) => self.parse_english(text.get(), offset.push_to_span(text.span())),
- Expr::Space(a) => token!(a, TokenKind::Space(1)),
+ Expr::Space(a) => {
+ let mut chars = self
+ .doc
+ .get(self.doc.range(a.span()).unwrap())
+ .unwrap()
+ .chars();
+ let first_char = chars.next().unwrap();
+ let length = chars.count() + 1;
+
+ if first_char == '\n' {
+ token!(a, TokenKind::Newline(1))
+ } else {
+ token!(a, TokenKind::Space(length))
+ }
+ }
Expr::Linebreak(a) => token!(a, TokenKind::Newline(1)),
Expr::Parbreak(a) => token!(a, TokenKind::ParagraphBreak),
Expr::SmartQuote(quote) => {
@@ -274,7 +288,7 @@ impl<'a> ParseHelper<'a> {
}
Expr::Str(text) => {
let offset = offset.push_to_span(text.span()).char + 1;
- let string = text.to_untyped().text().to_string();
+ let string = text.to_untyped().text();
Some(
self.parser
@@ -322,7 +336,8 @@ impl<'a> ParseHelper<'a> {
}
Expr::Set(set_rule) => merge!(
recurse!(set_rule.target()),
- set_rule.condition().and_then(|expr| recurse!(expr))
+ set_rule.condition().and_then(|expr| recurse!(expr)),
+ parse_args(&mut set_rule.args().items())
),
Expr::Show(show_rule) => merge!(
recurse!(show_rule.transform()),
@@ -565,7 +580,7 @@ mod tests {
#[test]
fn non_adjacent_spaces_not_condensed() {
- let source = r#"#authors_slice.join(", ", last: ", and ") "#;
+ let source = r#"#authors_slice.join(", ", last: ", and ") bob"#;
let token_kinds = Typst.parse_str(source).iter().map(|t| t.kind).collect_vec();
dbg!(&token_kinds);
@@ -581,7 +596,8 @@ mod tests {
TokenKind::Space(1),
TokenKind::Word(_), // and
TokenKind::Space(1),
- TokenKind::Space(1)
+ TokenKind::Space(2),
+ TokenKind::Word(_),
]
))
}
@@ -601,7 +617,11 @@ mod tests {
assert!(matches!(
&token_kinds.as_slice(),
- &[TokenKind::Word(_), TokenKind::Space(1), TokenKind::Word(_)]
+ &[
+ TokenKind::Word(_),
+ TokenKind::Newline(1),
+ TokenKind::Word(_)
+ ]
))
}
@@ -638,9 +658,9 @@ mod tests {
&token_kinds.as_slice(),
&[
TokenKind::Word(_),
- TokenKind::Space(1),
+ TokenKind::Newline(1),
TokenKind::Unlintable,
- TokenKind::Space(1),
+ TokenKind::Newline(1),
TokenKind::Word(_),
]
))
@@ -700,7 +720,7 @@ writing"#;
}),
..
}),
- TokenKind::Space(1),
+ TokenKind::Newline(1),
TokenKind::Word(_),
]
));