Skip to content

Offline, privacy-first grammar checker. Fast, open-source, Rust-powered

refactor(core): use patterns for more condensation

Elijah Potterf8fa2bfc584a
diff --git a/harper-core/src/document.rs b/harper-core/src/document.rs
index 3ecb6678b..3c1dd8e4a 100644
--- a/harper-core/src/document.rs
+++ b/harper-core/src/document.rs
@@ -107,12 +107,12 @@ impl Document {
/// Should be run after every change to the underlying [`Self::source`].
fn parse(&mut self, dictionary: &impl Dictionary) {
self.condense_spaces();
- self.condense_ellipsis();
self.condense_newlines();
self.newlines_to_breaks();
self.condense_contractions();
self.condense_dotted_initialisms();
self.condense_number_suffixes();
+ self.condense_ellipsis();
self.condense_latin();
self.match_quotes();
@@ -350,21 +350,26 @@ impl Document {
}
/// Assumes that the first matched token is the canonical one to be condensed into.
- fn condense_pattern(&mut self, pattern: &impl Pattern) {
+ /// Takes a callback that can be used to retroactively edit the canonical token afterwards.
+ fn condense_pattern<F>(&mut self, pattern: &impl Pattern, edit: F)
+ where
+ F: Fn(&mut Token),
+ {
let matches = pattern.find_all_matches_in_doc(self);
let mut remove_indices = VecDeque::with_capacity(matches.len());
for m in matches {
remove_indices.extend(m.start + 1..m.end);
- self.tokens[m.start].span.end = self.tokens[m.end - 1].span.end;
+ self.tokens[m.start].span = self.tokens[m.into_iter()].span().unwrap();
+ edit(&mut self.tokens[m.start]);
}
self.tokens.remove_indices(remove_indices);
}
fn condense_latin(&mut self) {
- self.condense_pattern(&Self::LATIN_PATTERN.with(|v| v.clone()))
+ self.condense_pattern(&Self::LATIN_PATTERN.with(|v| v.clone()), |_| {})
}
/// Searches for multiple sequential newline tokens and condenses them down
@@ -455,7 +460,7 @@ impl Document {
fn uncached_ellipsis_pattern() -> Lrc<RepeatingPattern> {
let period = SequencePattern::default().then_period();
- Lrc::new(RepeatingPattern::new(Box::new(period)))
+ Lrc::new(RepeatingPattern::new(Box::new(period), 2))
}
thread_local! {
@@ -463,68 +468,31 @@ impl Document {
}
fn condense_ellipsis(&mut self) {
- let found = Self::ELLIPSIS_PATTERN
- .with(|v| v.clone())
- .find_all_matches(&self.tokens, &self.source);
- let mut to_remove = VecDeque::new();
-
- for found_slice in found {
- if found_slice.len() <= 1 {
- continue;
- }
-
- let found_toks = &mut self.tokens[found_slice.start..found_slice.end];
-
- let end_char = found_toks.last().unwrap().span.end;
- let first = found_toks.first_mut().unwrap();
- first.kind = TokenKind::Punctuation(Punctuation::Ellipsis);
- first.span.end = end_char;
- for i in found_slice.start + 1..found_slice.end {
- to_remove.push_back(i)
- }
- }
+ let pattern = Self::ELLIPSIS_PATTERN.with(|v| v.clone());
+ self.condense_pattern(&pattern, |tok| {
+ tok.kind = TokenKind::Punctuation(Punctuation::Ellipsis)
+ });
+ }
+
+ fn uncached_contraction_pattern() -> Lrc<SequencePattern> {
+ Lrc::new(
+ SequencePattern::default()
+ .then_any_word()
+ .then_apostrophe()
+ .then_any_word(),
+ )
+ }
- self.tokens.remove_indices(to_remove);
+ thread_local! {
+ static CONTRACTION_PATTERN: Lrc<SequencePattern> = Document::uncached_contraction_pattern();
}
/// Searches for contractions and condenses them down into single
/// tokens.
fn condense_contractions(&mut self) {
- if self.tokens.len() < 3 {
- return;
- }
-
- // Indices of the three token stretches we are going to condense.
- let mut replace_starts = Vec::new();
-
- for idx in 0..self.tokens.len() - 2 {
- let a = self.tokens[idx];
- let b = self.tokens[idx + 1];
- let c = self.tokens[idx + 2];
-
- if matches!(
- (a.kind, b.kind, c.kind),
- (
- TokenKind::Word(..),
- TokenKind::Punctuation(Punctuation::Apostrophe),
- TokenKind::Word(..)
- )
- ) {
- // Ensure there is no overlapping between replacements
- let should_replace = if let Some(last_idx) = replace_starts.last() {
- *last_idx < idx - 2
- } else {
- true
- };
-
- if should_replace {
- replace_starts.push(idx);
- self.tokens[idx].span.end = c.span.end;
- }
- }
- }
+ let pattern = Self::CONTRACTION_PATTERN.with(|v| v.clone());
- self.condense_indices(&replace_starts, 3);
+ self.condense_pattern(&pattern, |_| {});
}
}
diff --git a/harper-core/src/patterns/mod.rs b/harper-core/src/patterns/mod.rs
index a5f00a89b..92ed0a542 100644
--- a/harper-core/src/patterns/mod.rs
+++ b/harper-core/src/patterns/mod.rs
@@ -67,8 +67,6 @@ where
return found;
}
- found.sort_by_key(|s| s.start);
-
let mut remove_indices = VecDeque::new();
for i in 0..found.len() - 1 {
diff --git a/harper-core/src/patterns/repeating_pattern.rs b/harper-core/src/patterns/repeating_pattern.rs
index a78c37ee7..a400abacc 100644
--- a/harper-core/src/patterns/repeating_pattern.rs
+++ b/harper-core/src/patterns/repeating_pattern.rs
@@ -6,25 +6,35 @@ use crate::Token;
/// Somewhat reminiscent of the `.*` operator in Regex.
pub struct RepeatingPattern {
inner: Box<dyn Pattern>,
+ required_repetitions: usize,
}
impl RepeatingPattern {
- pub fn new(pattern: Box<dyn Pattern>) -> Self {
- Self { inner: pattern }
+ pub fn new(pattern: Box<dyn Pattern>, required_repetitions: usize) -> Self {
+ Self {
+ inner: pattern,
+ required_repetitions,
+ }
}
}
impl Pattern for RepeatingPattern {
fn matches(&self, tokens: &[Token], source: &[char]) -> usize {
let mut tok_cursor = 0;
+ let mut repetition = 0;
loop {
let match_len = self.inner.matches(&tokens[tok_cursor..], source);
if match_len == 0 {
- return tok_cursor;
+ if repetition >= self.required_repetitions {
+ return tok_cursor;
+ } else {
+ return 0;
+ }
} else {
tok_cursor += match_len;
+ repetition += 1;
}
}
}
@@ -41,11 +51,19 @@ mod tests {
let doc = Document::new_plain_english_curated(
"This matcher will match the entirety of any document!",
);
- let pat = RepeatingPattern::new(Box::new(AnyPattern));
+ let pat = RepeatingPattern::new(Box::new(AnyPattern), 0);
assert_eq!(
pat.matches(doc.get_tokens(), doc.get_source()),
doc.get_tokens().len()
)
}
+
+ #[test]
+ fn does_not_match_short() {
+ let doc = Document::new_plain_english_curated("No match");
+ let pat = RepeatingPattern::new(Box::new(AnyPattern), 4);
+
+ assert_eq!(pat.matches(doc.get_tokens(), doc.get_source()), 0)
+ }
}
diff --git a/harper-core/src/patterns/sequence_pattern.rs b/harper-core/src/patterns/sequence_pattern.rs
index c1eb7ac53..c49ccd0a3 100644
--- a/harper-core/src/patterns/sequence_pattern.rs
+++ b/harper-core/src/patterns/sequence_pattern.rs
@@ -199,7 +199,7 @@ impl SequencePattern {
pub fn then_one_or_more(mut self, pat: Box<dyn Pattern>) -> Self {
self.token_patterns
- .push(Box::new(RepeatingPattern::new(pat)));
+ .push(Box::new(RepeatingPattern::new(pat, 0)));
self
}