Skip to content

Offline, privacy-first grammar checker. Fast, open-source, Rust-powered

Fixed issue with newlines breaking sentences in Tree-Sitter parser

Elijah Potter9d433403ea2d
diff --git a/Cargo.lock b/Cargo.lock
index 7627b2a54..1dafd0984 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -456,7 +456,7 @@ checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253"
[[package]]
name = "harper-core"
-version = "0.5.7"
+version = "0.5.4"
dependencies = [
"divan",
"hashbrown",
@@ -472,7 +472,7 @@ dependencies = [
[[package]]
name = "harper-ls"
-version = "0.5.3"
+version = "0.5.4"
dependencies = [
"clap",
"harper-core",
diff --git a/harper-core/Cargo.toml b/harper-core/Cargo.toml
index 7f0f81967..148a7f1e3 100644
--- a/harper-core/Cargo.toml
+++ b/harper-core/Cargo.toml
@@ -1,6 +1,6 @@
[package]
name = "harper-core"
-version = "0.5.7"
+version = "0.5.4"
edition = "2021"
description = "The language checker for artists."
license = "MIT OR Apache-2.0"
diff --git a/harper-ls/Cargo.toml b/harper-ls/Cargo.toml
index 603a19b75..5723013fd 100644
--- a/harper-ls/Cargo.toml
+++ b/harper-ls/Cargo.toml
@@ -1,6 +1,6 @@
[package]
name = "harper-ls"
-version = "0.5.3"
+version = "0.5.4"
edition = "2021"
description = "The language checker for artists."
license = "MIT OR Apache-2.0"
diff --git a/harper-ls/src/tree_sitter_parser.rs b/harper-ls/src/tree_sitter_parser.rs
index a2b6f89b2..63a3e569d 100644
--- a/harper-ls/src/tree_sitter_parser.rs
+++ b/harper-ls/src/tree_sitter_parser.rs
@@ -59,7 +59,7 @@ impl Parser for TreeSitterParser {
let mut tokens = Vec::new();
- for span in comments_spans {
+ for (s_index, span) in comments_spans.iter().enumerate() {
// Skip over the comment start characters
let actual_start = source[span.start..span.end]
.iter()
@@ -73,6 +73,14 @@ impl Parser for TreeSitterParser {
let mut new_tokens = markdown_parser.parse(&source[actual_start..span.end]);
+ // The markdown parser will insert a newline at end-of-input.
+ // If the next treesitter chunk is a comment, we want to remove that.
+ if let Some(next_start) = comments_spans.get(s_index + 1).map(|v| v.start) {
+ if is_span_whitespace(Span::new(span.end, next_start), source) {
+ new_tokens.pop();
+ }
+ }
+
new_tokens
.iter_mut()
.for_each(|t| t.span.offset(actual_start));
@@ -84,6 +92,15 @@ impl Parser for TreeSitterParser {
}
}
+/// Check if the contents of a span is just white-space.
+fn is_span_whitespace(span: Span, source: &[char]) -> bool {
+ span.get_content(source)
+ .iter()
+ .filter(|c| !c.is_whitespace())
+ .count()
+ == 0
+}
+
fn is_comment_character(c: char) -> bool {
matches!(c, '#' | '-' | '/')
}