Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
refactor(core): use patterns for more condensation
Elijah Potterf8fa2bfc584a
diff --git a/harper-core/src/document.rs b/harper-core/src/document.rsindex 3ecb6678b..3c1dd8e4a 100644--- a/harper-core/src/document.rs+++ b/harper-core/src/document.rs@@ -107,12 +107,12 @@ impl Document {/// Should be run after every change to the underlying [`Self::source`].fn parse(&mut self, dictionary: &impl Dictionary) {self.condense_spaces();- self.condense_ellipsis();self.condense_newlines();self.newlines_to_breaks();self.condense_contractions();self.condense_dotted_initialisms();self.condense_number_suffixes();+ self.condense_ellipsis();self.condense_latin();self.match_quotes();@@ -350,21 +350,26 @@ impl Document {}/// Assumes that the first matched token is the canonical one to be condensed into.- fn condense_pattern(&mut self, pattern: &impl Pattern) {+ /// Takes a callback that can be used to retroactively edit the canonical token afterwards.+ fn condense_pattern<F>(&mut self, pattern: &impl Pattern, edit: F)+ where+ F: Fn(&mut Token),+ {let matches = pattern.find_all_matches_in_doc(self);let mut remove_indices = VecDeque::with_capacity(matches.len());for m in matches {remove_indices.extend(m.start + 1..m.end);- self.tokens[m.start].span.end = self.tokens[m.end - 1].span.end;+ self.tokens[m.start].span = self.tokens[m.into_iter()].span().unwrap();+ edit(&mut self.tokens[m.start]);}self.tokens.remove_indices(remove_indices);}fn condense_latin(&mut self) {- self.condense_pattern(&Self::LATIN_PATTERN.with(|v| v.clone()))+ self.condense_pattern(&Self::LATIN_PATTERN.with(|v| v.clone()), |_| {})}/// Searches for multiple sequential newline tokens and condenses them down@@ -455,7 +460,7 @@ impl Document {fn uncached_ellipsis_pattern() -> Lrc<RepeatingPattern> {let period = SequencePattern::default().then_period();- Lrc::new(RepeatingPattern::new(Box::new(period)))+ Lrc::new(RepeatingPattern::new(Box::new(period), 2))}thread_local! {@@ -463,68 +468,31 @@ impl Document {}fn condense_ellipsis(&mut self) {- let found = Self::ELLIPSIS_PATTERN- .with(|v| v.clone())- .find_all_matches(&self.tokens, &self.source);- let mut to_remove = VecDeque::new();-- for found_slice in found {- if found_slice.len() <= 1 {- continue;- }-- let found_toks = &mut self.tokens[found_slice.start..found_slice.end];-- let end_char = found_toks.last().unwrap().span.end;- let first = found_toks.first_mut().unwrap();- first.kind = TokenKind::Punctuation(Punctuation::Ellipsis);- first.span.end = end_char;- for i in found_slice.start + 1..found_slice.end {- to_remove.push_back(i)- }- }+ let pattern = Self::ELLIPSIS_PATTERN.with(|v| v.clone());+ self.condense_pattern(&pattern, |tok| {+ tok.kind = TokenKind::Punctuation(Punctuation::Ellipsis)+ });+ }++ fn uncached_contraction_pattern() -> Lrc<SequencePattern> {+ Lrc::new(+ SequencePattern::default()+ .then_any_word()+ .then_apostrophe()+ .then_any_word(),+ )+ }- self.tokens.remove_indices(to_remove);+ thread_local! {+ static CONTRACTION_PATTERN: Lrc<SequencePattern> = Document::uncached_contraction_pattern();}/// Searches for contractions and condenses them down into single/// tokens.fn condense_contractions(&mut self) {- if self.tokens.len() < 3 {- return;- }-- // Indices of the three token stretches we are going to condense.- let mut replace_starts = Vec::new();-- for idx in 0..self.tokens.len() - 2 {- let a = self.tokens[idx];- let b = self.tokens[idx + 1];- let c = self.tokens[idx + 2];-- if matches!(- (a.kind, b.kind, c.kind),- (- TokenKind::Word(..),- TokenKind::Punctuation(Punctuation::Apostrophe),- TokenKind::Word(..)- )- ) {- // Ensure there is no overlapping between replacements- let should_replace = if let Some(last_idx) = replace_starts.last() {- *last_idx < idx - 2- } else {- true- };-- if should_replace {- replace_starts.push(idx);- self.tokens[idx].span.end = c.span.end;- }- }- }+ let pattern = Self::CONTRACTION_PATTERN.with(|v| v.clone());- self.condense_indices(&replace_starts, 3);+ self.condense_pattern(&pattern, |_| {});}}diff --git a/harper-core/src/patterns/mod.rs b/harper-core/src/patterns/mod.rsindex a5f00a89b..92ed0a542 100644--- a/harper-core/src/patterns/mod.rs+++ b/harper-core/src/patterns/mod.rs@@ -67,8 +67,6 @@ wherereturn found;}- found.sort_by_key(|s| s.start);-let mut remove_indices = VecDeque::new();for i in 0..found.len() - 1 {diff --git a/harper-core/src/patterns/repeating_pattern.rs b/harper-core/src/patterns/repeating_pattern.rsindex a78c37ee7..a400abacc 100644--- a/harper-core/src/patterns/repeating_pattern.rs+++ b/harper-core/src/patterns/repeating_pattern.rs@@ -6,25 +6,35 @@ use crate::Token;/// Somewhat reminiscent of the `.*` operator in Regex.pub struct RepeatingPattern {inner: Box<dyn Pattern>,+ required_repetitions: usize,}impl RepeatingPattern {- pub fn new(pattern: Box<dyn Pattern>) -> Self {- Self { inner: pattern }+ pub fn new(pattern: Box<dyn Pattern>, required_repetitions: usize) -> Self {+ Self {+ inner: pattern,+ required_repetitions,+ }}}impl Pattern for RepeatingPattern {fn matches(&self, tokens: &[Token], source: &[char]) -> usize {let mut tok_cursor = 0;+ let mut repetition = 0;loop {let match_len = self.inner.matches(&tokens[tok_cursor..], source);if match_len == 0 {- return tok_cursor;+ if repetition >= self.required_repetitions {+ return tok_cursor;+ } else {+ return 0;+ }} else {tok_cursor += match_len;+ repetition += 1;}}}@@ -41,11 +51,19 @@ mod tests {let doc = Document::new_plain_english_curated("This matcher will match the entirety of any document!",);- let pat = RepeatingPattern::new(Box::new(AnyPattern));+ let pat = RepeatingPattern::new(Box::new(AnyPattern), 0);assert_eq!(pat.matches(doc.get_tokens(), doc.get_source()),doc.get_tokens().len())}++ #[test]+ fn does_not_match_short() {+ let doc = Document::new_plain_english_curated("No match");+ let pat = RepeatingPattern::new(Box::new(AnyPattern), 4);++ assert_eq!(pat.matches(doc.get_tokens(), doc.get_source()), 0)+ }}diff --git a/harper-core/src/patterns/sequence_pattern.rs b/harper-core/src/patterns/sequence_pattern.rsindex c1eb7ac53..c49ccd0a3 100644--- a/harper-core/src/patterns/sequence_pattern.rs+++ b/harper-core/src/patterns/sequence_pattern.rs@@ -199,7 +199,7 @@ impl SequencePattern {pub fn then_one_or_more(mut self, pat: Box<dyn Pattern>) -> Self {self.token_patterns- .push(Box::new(RepeatingPattern::new(pat)));+ .push(Box::new(RepeatingPattern::new(pat, 0)));self}