Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
feat: now properly parse ellipsis
Elijah Potterdcfb8f7f2565
diff --git a/harper-core/src/document.rs b/harper-core/src/document.rsindex 6d948da55..21ae64d2b 100644--- a/harper-core/src/document.rs+++ b/harper-core/src/document.rs@@ -6,6 +6,7 @@ use itertools::Itertools;use paste::paste;use crate::parsers::{Markdown, Parser, PlainEnglish};+use crate::patterns::{PatternExt, RepeatingPattern, SequencePattern};use crate::punctuation::Punctuation;use crate::span::Span;use crate::token::NumberSuffix;@@ -87,6 +88,7 @@ impl Document {/// Should be run after every change to the underlying [`Self::source`].fn parse(&mut self, dictionary: &impl Dictionary) {self.condense_spaces();+ self.condense_ellipsis();self.condense_newlines();self.newlines_to_breaks();self.condense_contractions();@@ -493,6 +495,40 @@ impl Document {self.tokens.remove_indices(to_remove);}+ fn uncached_ellipsis_pattern() -> Lrc<RepeatingPattern> {+ let period = SequencePattern::default().then_period();+ Lrc::new(RepeatingPattern::new(Box::new(period)))+ }++ thread_local! {+ static ELLIPSIS_PATTERN: Lrc<RepeatingPattern> = Document::uncached_ellipsis_pattern();+ }++ fn condense_ellipsis(&mut self) {+ let found = Self::ELLIPSIS_PATTERN+ .with(|v| v.clone())+ .find_all_matches(&self.tokens, &self.source);+ let mut to_remove = VecDeque::new();++ for found_slice in found {+ if found_slice.len() <= 2 {+ continue;+ }++ let found_toks = &mut self.tokens[found_slice.start..found_slice.end];++ let end_char = found_toks.last().unwrap().span.end;+ let first = found_toks.first_mut().unwrap();+ first.kind = TokenKind::Punctuation(Punctuation::Ellipsis);+ first.span.end = end_char;+ for i in found_slice.start + 1..found_slice.end {+ to_remove.push_back(i)+ }+ }++ self.tokens.remove_indices(to_remove);+ }+/// Searches for contractions and condenses them down into single/// tokens.fn condense_contractions(&mut self) {@@ -734,4 +770,9 @@ mod tests {fn condenses_nsa() {assert_token_count(r#"Condenses words like "i.e.", "e.g." and "N.S.A.""#, 20);}++ #[test]+ fn parses_ellipsis() {+ assert_token_count("...", 1);+ }}diff --git a/harper-core/src/lib.rs b/harper-core/src/lib.rsindex 9e38d713b..1075de15d 100644--- a/harper-core/src/lib.rs+++ b/harper-core/src/lib.rs@@ -18,6 +18,8 @@ mod token;mod vec_ext;mod word_metadata;+use std::collections::VecDeque;+pub use char_string::{CharString, CharStringExt};pub use document::Document;use linting::Lint;@@ -41,7 +43,7 @@ pub fn remove_overlaps(lints: &mut Vec<Lint>) {lints.sort_by_key(|l| l.span.start);- let mut remove_indices = Vec::new();+ let mut remove_indices = VecDeque::new();for i in 0..lints.len() - 1 {let cur = &lints[i];@@ -50,16 +52,12 @@ pub fn remove_overlaps(lints: &mut Vec<Lint>) {if cur.span.overlaps_with(next.span) {// Remember, lower priority means higher importance.if next.priority < cur.priority {- remove_indices.push(i);+ remove_indices.push_back(i);} else {- remove_indices.push(i + 1);+ remove_indices.push_back(i + 1);}}}- let mut index = 0;- lints.retain(|_| {- index += 1;- !remove_indices.contains(&(index - 1))- })+ lints.remove_indices(remove_indices);}diff --git a/harper-core/src/linting/matcher.rs b/harper-core/src/linting/matcher.rsindex f3ab2e389..bf743f4c9 100644--- a/harper-core/src/linting/matcher.rs+++ b/harper-core/src/linting/matcher.rs@@ -289,12 +289,6 @@ impl Matcher {replace_with: vecword!("–")});- // And this ellipsis- triggers.push(Rule {- pattern: vec![pt!(Period), pt!(Period), pt!(Period)],- replace_with: vecword!("…")- });-triggers.push(Rule {pattern: vec![pt!("L"), pt!(Period), pt!("L"), pt!(Period), pt!("M")],replace_with: vecword!("large language model")@@ -382,12 +376,4 @@ mod tests {let lints = matcher.lint(&document);assert_eq!(lints.len(), 1);}-- #[test]- fn matches_ellipsis() {- let document = Document::new_plain_english_curated("...");- let mut matcher = Matcher::new();- let lints = matcher.lint(&document);- assert_eq!(lints.len(), 1);- }}diff --git a/harper-core/src/patterns/mod.rs b/harper-core/src/patterns/mod.rsindex 58cf7cf48..67ab45954 100644--- a/harper-core/src/patterns/mod.rs+++ b/harper-core/src/patterns/mod.rs@@ -1,4 +1,6 @@-use crate::Token;+use std::collections::VecDeque;++use crate::{Span, Token, VecExt};mod any_pattern;mod consumes_remaining_pattern;@@ -28,6 +30,49 @@ pub trait Pattern: Send + Sync {fn matches(&self, tokens: &[Token], source: &[char]) -> usize;}+pub trait PatternExt {+ /// Search through all tokens to locate all non-overlapping pattern matches.+ fn find_all_matches(&self, tokens: &[Token], source: &[char]) -> Vec<Span>;+}++impl<P> PatternExt for P+where+ P: Pattern+{+ fn find_all_matches(&self, tokens: &[Token], source: &[char]) -> Vec<Span> {+ let mut found = Vec::new();++ for i in 0..tokens.len() {+ let len = self.matches(&tokens[i..], source);++ if len > 0 {+ found.push(Span::new_with_len(i, len));+ }+ }++ if found.len() < 2 {+ return found;+ }++ found.sort_by_key(|s| s.start);++ let mut remove_indices = VecDeque::new();++ for i in 0..found.len() - 1 {+ let cur = &found[i];+ let next = &found[i + 1];++ if cur.overlaps_with(*next) {+ remove_indices.push_back(i + 1);+ }+ }++ found.remove_indices(remove_indices);++ found+ }+}+#[cfg(feature = "concurrent")]impl<F> Pattern for Fwherediff --git a/harper-core/src/patterns/sequence_pattern.rs b/harper-core/src/patterns/sequence_pattern.rsindex 5e67b480d..e7e0ec180 100644--- a/harper-core/src/patterns/sequence_pattern.rs+++ b/harper-core/src/patterns/sequence_pattern.rs@@ -33,6 +33,7 @@ impl SequencePattern {gen_then_from_is!(punctuation);gen_then_from_is!(conjunction);gen_then_from_is!(comma);+ gen_then_from_is!(period);pub fn then_exact_word(mut self, word: &'static str) -> Self {self.token_patterns