Skip to content

Offline, privacy-first grammar checker. Fast, open-source, Rust-powered

feat: now properly parse ellipsis

Elijah Potterdcfb8f7f2565
diff --git a/harper-core/src/document.rs b/harper-core/src/document.rs
index 6d948da55..21ae64d2b 100644
--- a/harper-core/src/document.rs
+++ b/harper-core/src/document.rs
@@ -6,6 +6,7 @@ use itertools::Itertools;
use paste::paste;
use crate::parsers::{Markdown, Parser, PlainEnglish};
+use crate::patterns::{PatternExt, RepeatingPattern, SequencePattern};
use crate::punctuation::Punctuation;
use crate::span::Span;
use crate::token::NumberSuffix;
@@ -87,6 +88,7 @@ impl Document {
/// Should be run after every change to the underlying [`Self::source`].
fn parse(&mut self, dictionary: &impl Dictionary) {
self.condense_spaces();
+ self.condense_ellipsis();
self.condense_newlines();
self.newlines_to_breaks();
self.condense_contractions();
@@ -493,6 +495,40 @@ impl Document {
self.tokens.remove_indices(to_remove);
}
+ fn uncached_ellipsis_pattern() -> Lrc<RepeatingPattern> {
+ let period = SequencePattern::default().then_period();
+ Lrc::new(RepeatingPattern::new(Box::new(period)))
+ }
+
+ thread_local! {
+ static ELLIPSIS_PATTERN: Lrc<RepeatingPattern> = Document::uncached_ellipsis_pattern();
+ }
+
+ fn condense_ellipsis(&mut self) {
+ let found = Self::ELLIPSIS_PATTERN
+ .with(|v| v.clone())
+ .find_all_matches(&self.tokens, &self.source);
+ let mut to_remove = VecDeque::new();
+
+ for found_slice in found {
+ if found_slice.len() <= 2 {
+ continue;
+ }
+
+ let found_toks = &mut self.tokens[found_slice.start..found_slice.end];
+
+ let end_char = found_toks.last().unwrap().span.end;
+ let first = found_toks.first_mut().unwrap();
+ first.kind = TokenKind::Punctuation(Punctuation::Ellipsis);
+ first.span.end = end_char;
+ for i in found_slice.start + 1..found_slice.end {
+ to_remove.push_back(i)
+ }
+ }
+
+ self.tokens.remove_indices(to_remove);
+ }
+
/// Searches for contractions and condenses them down into single
/// tokens.
fn condense_contractions(&mut self) {
@@ -734,4 +770,9 @@ mod tests {
fn condenses_nsa() {
assert_token_count(r#"Condenses words like "i.e.", "e.g." and "N.S.A.""#, 20);
}
+
+ #[test]
+ fn parses_ellipsis() {
+ assert_token_count("...", 1);
+ }
}
diff --git a/harper-core/src/lib.rs b/harper-core/src/lib.rs
index 9e38d713b..1075de15d 100644
--- a/harper-core/src/lib.rs
+++ b/harper-core/src/lib.rs
@@ -18,6 +18,8 @@ mod token;
mod vec_ext;
mod word_metadata;
+use std::collections::VecDeque;
+
pub use char_string::{CharString, CharStringExt};
pub use document::Document;
use linting::Lint;
@@ -41,7 +43,7 @@ pub fn remove_overlaps(lints: &mut Vec<Lint>) {
lints.sort_by_key(|l| l.span.start);
- let mut remove_indices = Vec::new();
+ let mut remove_indices = VecDeque::new();
for i in 0..lints.len() - 1 {
let cur = &lints[i];
@@ -50,16 +52,12 @@ pub fn remove_overlaps(lints: &mut Vec<Lint>) {
if cur.span.overlaps_with(next.span) {
// Remember, lower priority means higher importance.
if next.priority < cur.priority {
- remove_indices.push(i);
+ remove_indices.push_back(i);
} else {
- remove_indices.push(i + 1);
+ remove_indices.push_back(i + 1);
}
}
}
- let mut index = 0;
- lints.retain(|_| {
- index += 1;
- !remove_indices.contains(&(index - 1))
- })
+ lints.remove_indices(remove_indices);
}
diff --git a/harper-core/src/linting/matcher.rs b/harper-core/src/linting/matcher.rs
index f3ab2e389..bf743f4c9 100644
--- a/harper-core/src/linting/matcher.rs
+++ b/harper-core/src/linting/matcher.rs
@@ -289,12 +289,6 @@ impl Matcher {
replace_with: vecword!("–")
});
- // And this ellipsis
- triggers.push(Rule {
- pattern: vec![pt!(Period), pt!(Period), pt!(Period)],
- replace_with: vecword!("…")
- });
-
triggers.push(Rule {
pattern: vec![pt!("L"), pt!(Period), pt!("L"), pt!(Period), pt!("M")],
replace_with: vecword!("large language model")
@@ -382,12 +376,4 @@ mod tests {
let lints = matcher.lint(&document);
assert_eq!(lints.len(), 1);
}
-
- #[test]
- fn matches_ellipsis() {
- let document = Document::new_plain_english_curated("...");
- let mut matcher = Matcher::new();
- let lints = matcher.lint(&document);
- assert_eq!(lints.len(), 1);
- }
}
diff --git a/harper-core/src/patterns/mod.rs b/harper-core/src/patterns/mod.rs
index 58cf7cf48..67ab45954 100644
--- a/harper-core/src/patterns/mod.rs
+++ b/harper-core/src/patterns/mod.rs
@@ -1,4 +1,6 @@
-use crate::Token;
+use std::collections::VecDeque;
+
+use crate::{Span, Token, VecExt};
mod any_pattern;
mod consumes_remaining_pattern;
@@ -28,6 +30,49 @@ pub trait Pattern: Send + Sync {
fn matches(&self, tokens: &[Token], source: &[char]) -> usize;
}
+pub trait PatternExt {
+ /// Search through all tokens to locate all non-overlapping pattern matches.
+ fn find_all_matches(&self, tokens: &[Token], source: &[char]) -> Vec<Span>;
+}
+
+impl<P> PatternExt for P
+where
+ P: Pattern
+{
+ fn find_all_matches(&self, tokens: &[Token], source: &[char]) -> Vec<Span> {
+ let mut found = Vec::new();
+
+ for i in 0..tokens.len() {
+ let len = self.matches(&tokens[i..], source);
+
+ if len > 0 {
+ found.push(Span::new_with_len(i, len));
+ }
+ }
+
+ if found.len() < 2 {
+ return found;
+ }
+
+ found.sort_by_key(|s| s.start);
+
+ let mut remove_indices = VecDeque::new();
+
+ for i in 0..found.len() - 1 {
+ let cur = &found[i];
+ let next = &found[i + 1];
+
+ if cur.overlaps_with(*next) {
+ remove_indices.push_back(i + 1);
+ }
+ }
+
+ found.remove_indices(remove_indices);
+
+ found
+ }
+}
+
#[cfg(feature = "concurrent")]
impl<F> Pattern for F
where
diff --git a/harper-core/src/patterns/sequence_pattern.rs b/harper-core/src/patterns/sequence_pattern.rs
index 5e67b480d..e7e0ec180 100644
--- a/harper-core/src/patterns/sequence_pattern.rs
+++ b/harper-core/src/patterns/sequence_pattern.rs
@@ -33,6 +33,7 @@ impl SequencePattern {
gen_then_from_is!(punctuation);
gen_then_from_is!(conjunction);
gen_then_from_is!(comma);
+ gen_then_from_is!(period);
pub fn then_exact_word(mut self, word: &'static str) -> Self {
self.token_patterns