Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
feat: zero-width patterns (#1153)
Michael Schmidtbbb980dac2c9
diff --git a/harper-core/src/patterns/all.rs b/harper-core/src/patterns/all.rsindex 7e5122581..648833aee 100644--- a/harper-core/src/patterns/all.rs+++ b/harper-core/src/patterns/all.rs@@ -1,5 +1,3 @@-use std::num::NonZeroUsize;-use crate::Token;use super::Pattern;@@ -24,14 +22,14 @@ impl All {}impl Pattern for All {- fn matches(&self, tokens: &[Token], source: &[char]) -> Option<NonZeroUsize> {+ fn matches(&self, tokens: &[Token], source: &[char]) -> Option<usize> {let mut max = 0;for pattern in &self.children {let len = pattern.matches(tokens, source)?;- max = max.max(len.get());+ max = max.max(len);}- NonZeroUsize::new(max)+ Some(max)}}diff --git a/harper-core/src/patterns/any_pattern.rs b/harper-core/src/patterns/any_pattern.rsindex 2d68b4516..44063572a 100644--- a/harper-core/src/patterns/any_pattern.rs+++ b/harper-core/src/patterns/any_pattern.rs@@ -1,5 +1,3 @@-use std::num::NonZeroUsize;-use crate::Token;use super::Pattern;@@ -8,7 +6,7 @@ use super::Pattern;pub struct AnyPattern;impl Pattern for AnyPattern {- fn matches(&self, tokens: &[Token], _source: &[char]) -> Option<NonZeroUsize> {- NonZeroUsize::new(if tokens.is_empty() { 0 } else { 1 })+ fn matches(&self, tokens: &[Token], _source: &[char]) -> Option<usize> {+ if tokens.is_empty() { None } else { Some(1) }}}diff --git a/harper-core/src/patterns/either_pattern.rs b/harper-core/src/patterns/either_pattern.rsindex 512818828..d4ad6ec02 100644--- a/harper-core/src/patterns/either_pattern.rs+++ b/harper-core/src/patterns/either_pattern.rs@@ -1,5 +1,3 @@-use std::num::NonZeroUsize;-use crate::Token;use super::Pattern;@@ -21,17 +19,23 @@ impl EitherPattern {}impl Pattern for EitherPattern {- fn matches(&self, tokens: &[Token], source: &[char]) -> Option<NonZeroUsize> {- let mut longest = 0;+ fn matches(&self, tokens: &[Token], source: &[char]) -> Option<usize> {+ let mut longest = None;for pattern in self.patterns.iter() {- let match_len = pattern.matches(tokens, source).map_or(0, NonZeroUsize::get);-- if match_len > longest {- longest = match_len+ let Some(match_len) = pattern.matches(tokens, source) else {+ continue;+ };++ if let Some(longest_len) = longest {+ if match_len > longest_len {+ longest = Some(match_len);+ }+ } else {+ longest = Some(match_len);}}- NonZeroUsize::new(longest)+ longest}}diff --git a/harper-core/src/patterns/exact_phrase.rs b/harper-core/src/patterns/exact_phrase.rsindex eb40061c9..5a78f21a7 100644--- a/harper-core/src/patterns/exact_phrase.rs+++ b/harper-core/src/patterns/exact_phrase.rs@@ -1,5 +1,3 @@-use std::num::NonZeroUsize;-use crate::{Document, Token, TokenKind};use super::{Pattern, SequencePattern, Word};@@ -46,7 +44,7 @@ impl ExactPhrase {}impl Pattern for ExactPhrase {- fn matches(&self, tokens: &[Token], source: &[char]) -> Option<NonZeroUsize> {+ fn matches(&self, tokens: &[Token], source: &[char]) -> Option<usize> {self.inner.matches(tokens, source)}}diff --git a/harper-core/src/patterns/implies_quantity.rs b/harper-core/src/patterns/implies_quantity.rsindex 06a96cf27..2ba59f550 100644--- a/harper-core/src/patterns/implies_quantity.rs+++ b/harper-core/src/patterns/implies_quantity.rs@@ -1,5 +1,3 @@-use std::num::NonZeroUsize;-use crate::{Token, TokenKind};use super::Pattern;@@ -38,12 +36,12 @@ impl ImpliesQuantity {}impl Pattern for ImpliesQuantity {- fn matches(&self, tokens: &[Token], source: &[char]) -> Option<NonZeroUsize> {- NonZeroUsize::new(if Self::implies_plurality(tokens, source).is_some() {- 1+ fn matches(&self, tokens: &[Token], source: &[char]) -> Option<usize> {+ if Self::implies_plurality(tokens, source).is_some() {+ Some(1)} else {- 0- })+ None+ }}}diff --git a/harper-core/src/patterns/indefinite_article.rs b/harper-core/src/patterns/indefinite_article.rsindex 6ef2bb597..498311d30 100644--- a/harper-core/src/patterns/indefinite_article.rs+++ b/harper-core/src/patterns/indefinite_article.rs@@ -1,5 +1,3 @@-use std::num::NonZeroUsize;-use crate::Token;use super::{Pattern, WordSet};@@ -17,7 +15,7 @@ impl Default for IndefiniteArticle {}impl Pattern for IndefiniteArticle {- fn matches(&self, tokens: &[Token], source: &[char]) -> Option<NonZeroUsize> {+ fn matches(&self, tokens: &[Token], source: &[char]) -> Option<usize> {self.inner.matches(tokens, source)}}diff --git a/harper-core/src/patterns/inflection_of_be.rs b/harper-core/src/patterns/inflection_of_be.rsindex 282d51160..f3ae13c2a 100644--- a/harper-core/src/patterns/inflection_of_be.rs+++ b/harper-core/src/patterns/inflection_of_be.rs@@ -1,5 +1,3 @@-use std::num::NonZero;-use super::Pattern;use crate::Token;use crate::patterns::WordSet;@@ -26,7 +24,7 @@ impl InflectionOfBe {}impl Pattern for InflectionOfBe {- fn matches(&self, tokens: &[Token], source: &[char]) -> Option<NonZero<usize>> {+ fn matches(&self, tokens: &[Token], source: &[char]) -> Option<usize> {self.inner.matches(tokens, source)}}diff --git a/harper-core/src/patterns/invert.rs b/harper-core/src/patterns/invert.rsindex b5d4febde..f1d9d80fc 100644--- a/harper-core/src/patterns/invert.rs+++ b/harper-core/src/patterns/invert.rs@@ -1,5 +1,3 @@-use std::num::NonZeroUsize;-use crate::Token;use super::Pattern;@@ -18,11 +16,11 @@ impl Invert {}impl Pattern for Invert {- fn matches(&self, tokens: &[Token], source: &[char]) -> Option<NonZeroUsize> {- NonZeroUsize::new(if self.inner.matches(tokens, source).is_some() {- 0+ fn matches(&self, tokens: &[Token], source: &[char]) -> Option<usize> {+ if self.inner.matches(tokens, source).is_some() {+ None} else {- 1- })+ Some(1)+ }}}diff --git a/harper-core/src/patterns/mod.rs b/harper-core/src/patterns/mod.rsindex cf75e9158..2b857fe01 100644--- a/harper-core/src/patterns/mod.rs+++ b/harper-core/src/patterns/mod.rs@@ -5,8 +5,6 @@//!//! See the page about [`SequencePattern`] for a concrete example of their use.-use std::num::NonZeroUsize;-use crate::{Document, Span, Token};mod all;@@ -57,7 +55,7 @@ pub trait Pattern {/// Check if the pattern matches at the start of the given token slice.////// Returns the length of the match if successful, or `None` if not.- fn matches(&self, tokens: &[Token], source: &[char]) -> Option<NonZeroUsize>;+ fn matches(&self, tokens: &[Token], source: &[char]) -> Option<usize>;}#[cfg(feature = "concurrent")]#[blanket(derive(Arc))]@@ -65,7 +63,7 @@ pub trait Pattern: Send + Sync {/// Check if the pattern matches at the start of the given token slice.////// Returns the length of the match if successful, or `None` if not.- fn matches(&self, tokens: &[Token], source: &[char]) -> Option<NonZeroUsize>;+ fn matches(&self, tokens: &[Token], source: &[char]) -> Option<usize>;}pub trait PatternExt {@@ -81,6 +79,9 @@ impl<P> PatternExt for PwhereP: Pattern + ?Sized,{+ fn find_all_matches(&self, tokens: &[Token], source: &[char]) -> Vec<Span> {+ self.iter_matches(tokens, source).collect()+ }fn iter_matches(&self, tokens: &[Token], source: &[char]) -> impl Iterator<Item = Span> {MatchIter::new(self, tokens, source)}@@ -117,8 +118,8 @@ where.pattern.matches(&self.tokens[self.index..], self.source){- let span = Span::new_with_len(self.index, len.get());- self.index += len.get();+ let span = Span::new_with_len(self.index, len);+ self.index += len.max(1);return Some(span);} else {self.index += 1;@@ -148,8 +149,12 @@ whereF: Fn(&Token, &[char]) -> bool,F: Send + Sync,{- fn matches(&self, tokens: &[Token], source: &[char]) -> Option<NonZeroUsize> {- NonZeroUsize::new(if self(tokens.first()?, source) { 1 } else { 0 })+ fn matches(&self, tokens: &[Token], source: &[char]) -> Option<usize> {+ if self(tokens.first()?, source) {+ Some(1)+ } else {+ None+ }}}@@ -159,7 +164,11 @@ whereF: Fn(&Token, &[char]) -> bool,{fn matches(&self, tokens: &[Token], source: &[char]) -> Option<NonZeroUsize> {- NonZeroUsize::new(if self(tokens.first()?, source) { 1 } else { 0 })+ if self(tokens.first()?, source) {+ Some(1)+ } else {+ None+ }}}diff --git a/harper-core/src/patterns/naive_pattern_group.rs b/harper-core/src/patterns/naive_pattern_group.rsindex 0c08564e6..4520834bb 100644--- a/harper-core/src/patterns/naive_pattern_group.rs+++ b/harper-core/src/patterns/naive_pattern_group.rs@@ -1,5 +1,3 @@-use std::num::NonZeroUsize;-use super::Pattern;use crate::Token;@@ -17,7 +15,7 @@ impl NaivePatternGroup {}impl Pattern for NaivePatternGroup {- fn matches(&self, tokens: &[Token], source: &[char]) -> Option<NonZeroUsize> {+ fn matches(&self, tokens: &[Token], source: &[char]) -> Option<usize> {self.patterns.iter().filter_map(|p| p.matches(tokens, source))diff --git a/harper-core/src/patterns/nominal_phrase.rs b/harper-core/src/patterns/nominal_phrase.rsindex a655f5bef..555c1beee 100644--- a/harper-core/src/patterns/nominal_phrase.rs+++ b/harper-core/src/patterns/nominal_phrase.rs@@ -1,5 +1,3 @@-use std::num::NonZeroUsize;-use crate::Token;use super::Pattern;@@ -8,7 +6,7 @@ use super::Pattern;pub struct NominalPhrase;impl Pattern for NominalPhrase {- fn matches(&self, tokens: &[Token], _source: &[char]) -> Option<NonZeroUsize> {+ fn matches(&self, tokens: &[Token], _source: &[char]) -> Option<usize> {let mut cursor = 0;loop {@@ -26,7 +24,7 @@ impl Pattern for NominalPhrase {}if tok.kind.is_nominal() {- return NonZeroUsize::new(cursor + 1);+ return Some(cursor + 1);}return None;diff --git a/harper-core/src/patterns/pattern_map.rs b/harper-core/src/patterns/pattern_map.rsindex 3a6b19ad1..b4a654896 100644--- a/harper-core/src/patterns/pattern_map.rs+++ b/harper-core/src/patterns/pattern_map.rs@@ -1,5 +1,3 @@-use std::num::NonZeroUsize;-use crate::LSend;use crate::Token;@@ -69,7 +67,7 @@ impl<T> Pattern for PatternMap<T>whereT: LSend,{- fn matches(&self, tokens: &[Token], source: &[char]) -> Option<NonZeroUsize> {+ fn matches(&self, tokens: &[Token], source: &[char]) -> Option<usize> {self.rows.iter().filter_map(|row| row.key.matches(tokens, source))diff --git a/harper-core/src/patterns/repeating_pattern.rs b/harper-core/src/patterns/repeating_pattern.rsindex 514551c4c..b18aa2eef 100644--- a/harper-core/src/patterns/repeating_pattern.rs+++ b/harper-core/src/patterns/repeating_pattern.rs@@ -1,5 +1,3 @@-use std::num::NonZeroUsize;-use super::Pattern;use crate::Token;@@ -21,7 +19,7 @@ impl RepeatingPattern {}impl Pattern for RepeatingPattern {- fn matches(&self, tokens: &[Token], source: &[char]) -> Option<NonZeroUsize> {+ fn matches(&self, tokens: &[Token], source: &[char]) -> Option<usize> {let mut tok_cursor = 0;let mut repetition = 0;@@ -29,10 +27,17 @@ impl Pattern for RepeatingPattern {let match_len = self.inner.matches(&tokens[tok_cursor..], source);if let Some(match_len) = match_len {- tok_cursor += match_len.get();+ if match_len == 0 {+ // If match_len == 0, we won't move forward ever again.+ // This means that we can get infinitely many repetitions,+ // so repetition >= self.required_repetitions is guaranteed.+ return Some(tok_cursor);+ }++ tok_cursor += match_len;repetition += 1;} else if repetition >= self.required_repetitions {- return NonZeroUsize::new(tok_cursor);+ return Some(tok_cursor);} else {return None;}@@ -42,7 +47,6 @@ impl Pattern for RepeatingPattern {#[cfg(test)]mod tests {- use std::num::NonZeroUsize;use super::RepeatingPattern;use crate::Document;@@ -57,7 +61,7 @@ mod tests {assert_eq!(pat.matches(doc.get_tokens(), doc.get_source()),- NonZeroUsize::new(doc.get_tokens().len())+ Some(doc.get_tokens().len()))}diff --git a/harper-core/src/patterns/sequence_pattern.rs b/harper-core/src/patterns/sequence_pattern.rsindex 959fc45e1..14e4d937b 100644--- a/harper-core/src/patterns/sequence_pattern.rs+++ b/harper-core/src/patterns/sequence_pattern.rs@@ -1,5 +1,3 @@-use std::num::NonZeroUsize;-use paste::paste;use super::whitespace_pattern::WhitespacePattern;@@ -147,7 +145,7 @@ impl SequencePattern {pub fn then_one_or_more(mut self, pat: impl Pattern + 'static) -> Self {self.token_patterns- .push(Box::new(RepeatingPattern::new(Box::new(pat), 0)));+ .push(Box::new(RepeatingPattern::new(Box::new(pat), 1)));self}@@ -165,23 +163,21 @@ impl SequencePattern {}impl Pattern for SequencePattern {- fn matches(&self, tokens: &[Token], source: &[char]) -> Option<NonZeroUsize> {+ fn matches(&self, tokens: &[Token], source: &[char]) -> Option<usize> {let mut tok_cursor = 0;for pat in self.token_patterns.iter() {let match_length = pat.matches(&tokens[tok_cursor..], source)?;- tok_cursor += match_length.get();+ tok_cursor += match_length;}- NonZeroUsize::new(tok_cursor)+ Some(tok_cursor)}}#[cfg(test)]mod tests {- use std::num::NonZeroUsize;-use super::SequencePattern;use crate::Document;use crate::patterns::Pattern;@@ -196,7 +192,7 @@ mod tests {assert_eq!(pat.matches(doc.get_tokens(), doc.get_source()),- NonZeroUsize::new(doc.get_tokens().len())+ Some(doc.get_tokens().len()));}@@ -210,7 +206,7 @@ mod tests {assert_eq!(pat.matches(doc.get_tokens(), doc.get_source()),- NonZeroUsize::new(doc.get_tokens().len())+ Some(doc.get_tokens().len()));}@@ -221,7 +217,7 @@ mod tests {assert_eq!(pat.matches(doc.get_tokens(), doc.get_source()),- NonZeroUsize::new(doc.get_tokens().len())+ Some(doc.get_tokens().len()));}}diff --git a/harper-core/src/patterns/similar_to_phrase.rs b/harper-core/src/patterns/similar_to_phrase.rsindex 1577a4388..729854f50 100644--- a/harper-core/src/patterns/similar_to_phrase.rs+++ b/harper-core/src/patterns/similar_to_phrase.rs@@ -1,5 +1,3 @@-use std::num::NonZeroUsize;-use crate::{Document, Token, TokenKind};use super::{Pattern, SequencePattern, Word, within_edit_distance::WithinEditDistance};@@ -59,7 +57,7 @@ impl SimilarToPhrase {}impl Pattern for SimilarToPhrase {- fn matches(&self, tokens: &[Token], source: &[char]) -> Option<NonZeroUsize> {+ fn matches(&self, tokens: &[Token], source: &[char]) -> Option<usize> {if self.phrase.matches(tokens, source).is_some() {return None;}diff --git a/harper-core/src/patterns/split_compound_word.rs b/harper-core/src/patterns/split_compound_word.rsindex 836d788ec..445c9896c 100644--- a/harper-core/src/patterns/split_compound_word.rs+++ b/harper-core/src/patterns/split_compound_word.rs@@ -1,4 +1,4 @@-use std::{num::NonZeroUsize, sync::Arc};+use std::sync::Arc;use crate::{CharString, Dictionary, FstDictionary, Token, WordMetadata};@@ -57,10 +57,10 @@ impl SplitCompoundWord {}impl Pattern for SplitCompoundWord {- fn matches(&self, tokens: &[Token], source: &[char]) -> Option<NonZeroUsize> {+ fn matches(&self, tokens: &[Token], source: &[char]) -> Option<usize> {let inner_match = self.inner.matches(tokens, source)?;- if inner_match.get() != 3 {+ if inner_match != 3 {return None;}diff --git a/harper-core/src/patterns/whitespace_pattern.rs b/harper-core/src/patterns/whitespace_pattern.rsindex 2ead95e6f..69cb92113 100644--- a/harper-core/src/patterns/whitespace_pattern.rs+++ b/harper-core/src/patterns/whitespace_pattern.rs@@ -1,16 +1,14 @@-use std::num::NonZeroUsize;-use super::Pattern;pub struct WhitespacePattern;impl Pattern for WhitespacePattern {- fn matches(&self, tokens: &[crate::Token], _source: &[char]) -> Option<NonZeroUsize> {- NonZeroUsize::new(- tokens- .iter()- .position(|t| !t.kind.is_whitespace())- .unwrap_or(tokens.len()),- )+ fn matches(&self, tokens: &[crate::Token], _source: &[char]) -> Option<usize> {+ let count = tokens+ .iter()+ .position(|t| !t.kind.is_whitespace())+ .unwrap_or(tokens.len());++ if count == 0 { None } else { Some(count) }}}diff --git a/harper-core/src/patterns/within_edit_distance.rs b/harper-core/src/patterns/within_edit_distance.rsindex 0b5fb6b5b..897d7fe6e 100644--- a/harper-core/src/patterns/within_edit_distance.rs+++ b/harper-core/src/patterns/within_edit_distance.rs@@ -1,5 +1,4 @@use std::cell::RefCell;-use std::num::NonZeroUsize;use super::Pattern;use crate::{CharString, CharStringExt, Token};@@ -33,7 +32,7 @@ thread_local! {}impl Pattern for WithinEditDistance {- fn matches(&self, tokens: &[Token], source: &[char]) -> Option<NonZeroUsize> {+ fn matches(&self, tokens: &[Token], source: &[char]) -> Option<usize> {let first = tokens.first()?;if !first.kind.is_word() {return None;@@ -48,7 +47,11 @@ impl Pattern for WithinEditDistance {buffer_a,buffer_b,);- NonZeroUsize::new(if distance <= self.max_edit_dist { 1 } else { 0 })+ if distance <= self.max_edit_dist {+ Some(1)+ } else {+ None+ }})}}diff --git a/harper-core/src/patterns/word.rs b/harper-core/src/patterns/word.rsindex 53a098e77..6ae5b5145 100644--- a/harper-core/src/patterns/word.rs+++ b/harper-core/src/patterns/word.rs@@ -1,5 +1,3 @@-use std::num::NonZeroUsize;-use super::Pattern;use crate::{CharString, Token};@@ -39,7 +37,7 @@ impl Word {}impl Pattern for Word {- fn matches(&self, tokens: &[Token], source: &[char]) -> Option<NonZeroUsize> {+ fn matches(&self, tokens: &[Token], source: &[char]) -> Option<usize> {let tok = tokens.first()?;if !tok.kind.is_word() {return None;@@ -58,7 +56,7 @@ impl Pattern for Word {.all(|(a, b)| a.eq_ignore_ascii_case(b))};- NonZeroUsize::new(if eq { 1 } else { 0 })+ if eq { Some(1) } else { None }}}diff --git a/harper-core/src/patterns/word_pattern_group.rs b/harper-core/src/patterns/word_pattern_group.rsindex 1c3f5399d..5028b592e 100644--- a/harper-core/src/patterns/word_pattern_group.rs+++ b/harper-core/src/patterns/word_pattern_group.rs@@ -1,5 +1,3 @@-use std::num::NonZeroUsize;-use hashbrown::HashMap;use super::naive_pattern_group::NaivePatternGroup;@@ -42,7 +40,7 @@ impl<P> Pattern for WordPatternGroup<P>whereP: Pattern,{- fn matches(&self, tokens: &[crate::Token], source: &[char]) -> Option<NonZeroUsize> {+ fn matches(&self, tokens: &[crate::Token], source: &[char]) -> Option<usize> {let first = tokens.first()?;if !first.kind.is_word() {return None;diff --git a/harper-core/src/patterns/word_set.rs b/harper-core/src/patterns/word_set.rsindex e60961b80..47034224d 100644--- a/harper-core/src/patterns/word_set.rs+++ b/harper-core/src/patterns/word_set.rs@@ -1,5 +1,3 @@-use std::num::NonZeroUsize;-use super::Pattern;use smallvec::SmallVec;@@ -40,7 +38,7 @@ impl WordSet {}impl Pattern for WordSet {- fn matches(&self, tokens: &[Token], source: &[char]) -> Option<NonZeroUsize> {+ fn matches(&self, tokens: &[Token], source: &[char]) -> Option<usize> {let tok = tokens.first()?;if !tok.kind.is_word() {return None;@@ -59,7 +57,7 @@ impl Pattern for WordSet {.all(|(a, b)| a.eq_ignore_ascii_case(b));if partial_match {- return NonZeroUsize::new(1);+ return Some(1);}}