Skip to content

Offline, privacy-first grammar checker. Fast, open-source, Rust-powered

fix: issues with commas in certain linters

Elijah Pottere6871bebbe5d
diff --git a/harper-core/src/document.rs b/harper-core/src/document.rs
index 47df1c006..8bf33a498 100644
--- a/harper-core/src/document.rs
+++ b/harper-core/src/document.rs
@@ -133,6 +133,55 @@ impl Document {
self.tokens().map(|token| token.to_fat(&self.source))
}
+ /// Iterate over the locations of punctuation the separates chunks.
+ fn chunk_terminators(&self) -> impl Iterator<Item = usize> + '_ {
+ self.tokens.iter().enumerate().filter_map(|(index, token)| {
+ if is_chunk_terminator(&token.kind) {
+ return Some(index);
+ }
+ None
+ })
+ }
+
+ /// Get the index of the last chunk terminator.
+ fn last_chunk_terminator(&self) -> Option<usize> {
+ self.tokens
+ .iter()
+ .enumerate()
+ .rev()
+ .find_map(|(index, token)| {
+ if is_chunk_terminator(&token.kind) {
+ return Some(index);
+ }
+ None
+ })
+ }
+
+ /// Iterate over sentence chunks.
+ pub fn chunks(&self) -> impl Iterator<Item = &'_ [Token]> + '_ {
+ let first_sentence = self
+ .chunk_terminators()
+ .next()
+ .map(|first_term| &self.tokens[0..=first_term]);
+
+ let rest = self
+ .chunk_terminators()
+ .tuple_windows()
+ .map(move |(a, b)| &self.tokens[a + 1..=b]);
+
+ let last = if let Some(last_i) = self.last_chunk_terminator() {
+ if last_i + 1 < self.tokens.len() {
+ Some(&self.tokens[last_i + 1..])
+ } else {
+ None
+ }
+ } else {
+ Some(self.tokens.as_slice())
+ };
+
+ first_sentence.into_iter().chain(rest).chain(last)
+ }
+
/// Iterate over the locations of the sentence terminators in the document.
fn sentence_terminators(&self) -> impl Iterator<Item = usize> + '_ {
self.tokens.iter().enumerate().filter_map(|(index, token)| {
@@ -470,6 +519,17 @@ impl TokenStringExt for Document {
}
}
+fn is_chunk_terminator(token: &TokenKind) -> bool {
+ if is_sentence_terminator(token) {
+ return true;
+ }
+
+ match token {
+ TokenKind::Punctuation(punct) => [Punctuation::Comma].contains(punct),
+ _ => false
+ }
+}
+
fn is_sentence_terminator(token: &TokenKind) -> bool {
match token {
TokenKind::Punctuation(punct) => [
diff --git a/harper-core/src/linting/multiple_sequential_pronouns.rs b/harper-core/src/linting/multiple_sequential_pronouns.rs
index 35472a10d..6d400dc46 100644
--- a/harper-core/src/linting/multiple_sequential_pronouns.rs
+++ b/harper-core/src/linting/multiple_sequential_pronouns.rs
@@ -51,8 +51,8 @@ impl Linter for MultipleSequentialPronouns {
found_pronouns.clear();
};
- for sentence in document.sentences() {
- for word in sentence.iter_words() {
+ for chunk in document.chunks() {
+ for word in chunk.iter_words() {
let word_chars = document.get_span_content(word.span);
if self.is_pronoun(word_chars) {
@@ -63,10 +63,12 @@ impl Linter for MultipleSequentialPronouns {
emit_lint(&mut found_pronouns);
}
}
- }
- if found_pronouns.len() > 1 {
- emit_lint(&mut found_pronouns);
+ if found_pronouns.len() > 1 {
+ emit_lint(&mut found_pronouns);
+ }
+
+ found_pronouns.clear();
}
lints
@@ -119,4 +121,9 @@ mod tests {
1
)
}
+
+ #[test]
+ fn comma_separated() {
+ assert_lint_count("To prove it, we...", MultipleSequentialPronouns::new(), 0)
+ }
}
diff --git a/harper-core/src/linting/repeated_words.rs b/harper-core/src/linting/repeated_words.rs
index d7e24118d..e3ff54b45 100644
--- a/harper-core/src/linting/repeated_words.rs
+++ b/harper-core/src/linting/repeated_words.rs
@@ -51,18 +51,15 @@ impl Linter for RepeatedWords {
fn lint(&mut self, document: &Document) -> Vec<Lint> {
let mut lints = Vec::new();
- for sentence in document.sentences() {
- let mut iter = sentence
- .iter_word_indices()
- .zip(sentence.iter_words())
- .peekable();
+ for chunk in document.chunks() {
+ let mut iter = chunk.iter_word_indices().zip(chunk.iter_words()).peekable();
while let (Some((idx_a, tok_a)), Some((idx_b, tok_b))) = (iter.next(), iter.peek()) {
let word_a = document.get_span_content(tok_a.span);
let word_b = document.get_span_content(tok_b.span);
if self.set.contains(word_a) && word_a == word_b {
- let intervening_tokens = &sentence[idx_a + 1..*idx_b];
+ let intervening_tokens = &chunk[idx_a + 1..*idx_b];
if intervening_tokens.iter().any(|t| !t.kind.is_whitespace()) {
continue;