Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
fix: issues with commas in certain linters
Elijah Pottere6871bebbe5d
diff --git a/harper-core/src/document.rs b/harper-core/src/document.rsindex 47df1c006..8bf33a498 100644--- a/harper-core/src/document.rs+++ b/harper-core/src/document.rs@@ -133,6 +133,55 @@ impl Document {self.tokens().map(|token| token.to_fat(&self.source))}+ /// Iterate over the locations of punctuation the separates chunks.+ fn chunk_terminators(&self) -> impl Iterator<Item = usize> + '_ {+ self.tokens.iter().enumerate().filter_map(|(index, token)| {+ if is_chunk_terminator(&token.kind) {+ return Some(index);+ }+ None+ })+ }++ /// Get the index of the last chunk terminator.+ fn last_chunk_terminator(&self) -> Option<usize> {+ self.tokens+ .iter()+ .enumerate()+ .rev()+ .find_map(|(index, token)| {+ if is_chunk_terminator(&token.kind) {+ return Some(index);+ }+ None+ })+ }++ /// Iterate over sentence chunks.+ pub fn chunks(&self) -> impl Iterator<Item = &'_ [Token]> + '_ {+ let first_sentence = self+ .chunk_terminators()+ .next()+ .map(|first_term| &self.tokens[0..=first_term]);++ let rest = self+ .chunk_terminators()+ .tuple_windows()+ .map(move |(a, b)| &self.tokens[a + 1..=b]);++ let last = if let Some(last_i) = self.last_chunk_terminator() {+ if last_i + 1 < self.tokens.len() {+ Some(&self.tokens[last_i + 1..])+ } else {+ None+ }+ } else {+ Some(self.tokens.as_slice())+ };++ first_sentence.into_iter().chain(rest).chain(last)+ }+/// Iterate over the locations of the sentence terminators in the document.fn sentence_terminators(&self) -> impl Iterator<Item = usize> + '_ {self.tokens.iter().enumerate().filter_map(|(index, token)| {@@ -470,6 +519,17 @@ impl TokenStringExt for Document {}}+fn is_chunk_terminator(token: &TokenKind) -> bool {+ if is_sentence_terminator(token) {+ return true;+ }++ match token {+ TokenKind::Punctuation(punct) => [Punctuation::Comma].contains(punct),+ _ => false+ }+}+fn is_sentence_terminator(token: &TokenKind) -> bool {match token {TokenKind::Punctuation(punct) => [diff --git a/harper-core/src/linting/multiple_sequential_pronouns.rs b/harper-core/src/linting/multiple_sequential_pronouns.rsindex 35472a10d..6d400dc46 100644--- a/harper-core/src/linting/multiple_sequential_pronouns.rs+++ b/harper-core/src/linting/multiple_sequential_pronouns.rs@@ -51,8 +51,8 @@ impl Linter for MultipleSequentialPronouns {found_pronouns.clear();};- for sentence in document.sentences() {- for word in sentence.iter_words() {+ for chunk in document.chunks() {+ for word in chunk.iter_words() {let word_chars = document.get_span_content(word.span);if self.is_pronoun(word_chars) {@@ -63,10 +63,12 @@ impl Linter for MultipleSequentialPronouns {emit_lint(&mut found_pronouns);}}- }- if found_pronouns.len() > 1 {- emit_lint(&mut found_pronouns);+ if found_pronouns.len() > 1 {+ emit_lint(&mut found_pronouns);+ }++ found_pronouns.clear();}lints@@ -119,4 +121,9 @@ mod tests {1)}++ #[test]+ fn comma_separated() {+ assert_lint_count("To prove it, we...", MultipleSequentialPronouns::new(), 0)+ }}diff --git a/harper-core/src/linting/repeated_words.rs b/harper-core/src/linting/repeated_words.rsindex d7e24118d..e3ff54b45 100644--- a/harper-core/src/linting/repeated_words.rs+++ b/harper-core/src/linting/repeated_words.rs@@ -51,18 +51,15 @@ impl Linter for RepeatedWords {fn lint(&mut self, document: &Document) -> Vec<Lint> {let mut lints = Vec::new();- for sentence in document.sentences() {- let mut iter = sentence- .iter_word_indices()- .zip(sentence.iter_words())- .peekable();+ for chunk in document.chunks() {+ let mut iter = chunk.iter_word_indices().zip(chunk.iter_words()).peekable();while let (Some((idx_a, tok_a)), Some((idx_b, tok_b))) = (iter.next(), iter.peek()) {let word_a = document.get_span_content(tok_a.span);let word_b = document.get_span_content(tok_b.span);if self.set.contains(word_a) && word_a == word_b {- let intervening_tokens = &sentence[idx_a + 1..*idx_b];+ let intervening_tokens = &chunk[idx_a + 1..*idx_b];if intervening_tokens.iter().any(|t| !t.kind.is_whitespace()) {continue;