Skip to content

Offline, privacy-first grammar checker. Fast, open-source, Rust-powered

feat: zero-width patterns (#1153)

Michael Schmidtbbb980dac2c9
diff --git a/harper-core/src/patterns/all.rs b/harper-core/src/patterns/all.rs
index 7e5122581..648833aee 100644
--- a/harper-core/src/patterns/all.rs
+++ b/harper-core/src/patterns/all.rs
@@ -1,5 +1,3 @@
-use std::num::NonZeroUsize;
-
use crate::Token;
use super::Pattern;
@@ -24,14 +22,14 @@ impl All {
}
impl Pattern for All {
- fn matches(&self, tokens: &[Token], source: &[char]) -> Option<NonZeroUsize> {
+ fn matches(&self, tokens: &[Token], source: &[char]) -> Option<usize> {
let mut max = 0;
for pattern in &self.children {
let len = pattern.matches(tokens, source)?;
- max = max.max(len.get());
+ max = max.max(len);
}
- NonZeroUsize::new(max)
+ Some(max)
}
}
diff --git a/harper-core/src/patterns/any_pattern.rs b/harper-core/src/patterns/any_pattern.rs
index 2d68b4516..44063572a 100644
--- a/harper-core/src/patterns/any_pattern.rs
+++ b/harper-core/src/patterns/any_pattern.rs
@@ -1,5 +1,3 @@
-use std::num::NonZeroUsize;
-
use crate::Token;
use super::Pattern;
@@ -8,7 +6,7 @@ use super::Pattern;
pub struct AnyPattern;
impl Pattern for AnyPattern {
- fn matches(&self, tokens: &[Token], _source: &[char]) -> Option<NonZeroUsize> {
- NonZeroUsize::new(if tokens.is_empty() { 0 } else { 1 })
+ fn matches(&self, tokens: &[Token], _source: &[char]) -> Option<usize> {
+ if tokens.is_empty() { None } else { Some(1) }
}
}
diff --git a/harper-core/src/patterns/either_pattern.rs b/harper-core/src/patterns/either_pattern.rs
index 512818828..d4ad6ec02 100644
--- a/harper-core/src/patterns/either_pattern.rs
+++ b/harper-core/src/patterns/either_pattern.rs
@@ -1,5 +1,3 @@
-use std::num::NonZeroUsize;
-
use crate::Token;
use super::Pattern;
@@ -21,17 +19,23 @@ impl EitherPattern {
}
impl Pattern for EitherPattern {
- fn matches(&self, tokens: &[Token], source: &[char]) -> Option<NonZeroUsize> {
- let mut longest = 0;
+ fn matches(&self, tokens: &[Token], source: &[char]) -> Option<usize> {
+ let mut longest = None;
for pattern in self.patterns.iter() {
- let match_len = pattern.matches(tokens, source).map_or(0, NonZeroUsize::get);
-
- if match_len > longest {
- longest = match_len
+ let Some(match_len) = pattern.matches(tokens, source) else {
+ continue;
+ };
+
+ if let Some(longest_len) = longest {
+ if match_len > longest_len {
+ longest = Some(match_len);
+ }
+ } else {
+ longest = Some(match_len);
}
}
- NonZeroUsize::new(longest)
+ longest
}
}
diff --git a/harper-core/src/patterns/exact_phrase.rs b/harper-core/src/patterns/exact_phrase.rs
index eb40061c9..5a78f21a7 100644
--- a/harper-core/src/patterns/exact_phrase.rs
+++ b/harper-core/src/patterns/exact_phrase.rs
@@ -1,5 +1,3 @@
-use std::num::NonZeroUsize;
-
use crate::{Document, Token, TokenKind};
use super::{Pattern, SequencePattern, Word};
@@ -46,7 +44,7 @@ impl ExactPhrase {
}
impl Pattern for ExactPhrase {
- fn matches(&self, tokens: &[Token], source: &[char]) -> Option<NonZeroUsize> {
+ fn matches(&self, tokens: &[Token], source: &[char]) -> Option<usize> {
self.inner.matches(tokens, source)
}
}
diff --git a/harper-core/src/patterns/implies_quantity.rs b/harper-core/src/patterns/implies_quantity.rs
index 06a96cf27..2ba59f550 100644
--- a/harper-core/src/patterns/implies_quantity.rs
+++ b/harper-core/src/patterns/implies_quantity.rs
@@ -1,5 +1,3 @@
-use std::num::NonZeroUsize;
-
use crate::{Token, TokenKind};
use super::Pattern;
@@ -38,12 +36,12 @@ impl ImpliesQuantity {
}
impl Pattern for ImpliesQuantity {
- fn matches(&self, tokens: &[Token], source: &[char]) -> Option<NonZeroUsize> {
- NonZeroUsize::new(if Self::implies_plurality(tokens, source).is_some() {
- 1
+ fn matches(&self, tokens: &[Token], source: &[char]) -> Option<usize> {
+ if Self::implies_plurality(tokens, source).is_some() {
+ Some(1)
} else {
- 0
- })
+ None
+ }
}
}
diff --git a/harper-core/src/patterns/indefinite_article.rs b/harper-core/src/patterns/indefinite_article.rs
index 6ef2bb597..498311d30 100644
--- a/harper-core/src/patterns/indefinite_article.rs
+++ b/harper-core/src/patterns/indefinite_article.rs
@@ -1,5 +1,3 @@
-use std::num::NonZeroUsize;
-
use crate::Token;
use super::{Pattern, WordSet};
@@ -17,7 +15,7 @@ impl Default for IndefiniteArticle {
}
impl Pattern for IndefiniteArticle {
- fn matches(&self, tokens: &[Token], source: &[char]) -> Option<NonZeroUsize> {
+ fn matches(&self, tokens: &[Token], source: &[char]) -> Option<usize> {
self.inner.matches(tokens, source)
}
}
diff --git a/harper-core/src/patterns/inflection_of_be.rs b/harper-core/src/patterns/inflection_of_be.rs
index 282d51160..f3ae13c2a 100644
--- a/harper-core/src/patterns/inflection_of_be.rs
+++ b/harper-core/src/patterns/inflection_of_be.rs
@@ -1,5 +1,3 @@
-use std::num::NonZero;
-
use super::Pattern;
use crate::Token;
use crate::patterns::WordSet;
@@ -26,7 +24,7 @@ impl InflectionOfBe {
}
impl Pattern for InflectionOfBe {
- fn matches(&self, tokens: &[Token], source: &[char]) -> Option<NonZero<usize>> {
+ fn matches(&self, tokens: &[Token], source: &[char]) -> Option<usize> {
self.inner.matches(tokens, source)
}
}
diff --git a/harper-core/src/patterns/invert.rs b/harper-core/src/patterns/invert.rs
index b5d4febde..f1d9d80fc 100644
--- a/harper-core/src/patterns/invert.rs
+++ b/harper-core/src/patterns/invert.rs
@@ -1,5 +1,3 @@
-use std::num::NonZeroUsize;
-
use crate::Token;
use super::Pattern;
@@ -18,11 +16,11 @@ impl Invert {
}
impl Pattern for Invert {
- fn matches(&self, tokens: &[Token], source: &[char]) -> Option<NonZeroUsize> {
- NonZeroUsize::new(if self.inner.matches(tokens, source).is_some() {
- 0
+ fn matches(&self, tokens: &[Token], source: &[char]) -> Option<usize> {
+ if self.inner.matches(tokens, source).is_some() {
+ None
} else {
- 1
- })
+ Some(1)
+ }
}
}
diff --git a/harper-core/src/patterns/mod.rs b/harper-core/src/patterns/mod.rs
index cf75e9158..2b857fe01 100644
--- a/harper-core/src/patterns/mod.rs
+++ b/harper-core/src/patterns/mod.rs
@@ -5,8 +5,6 @@
//!
//! See the page about [`SequencePattern`] for a concrete example of their use.
-use std::num::NonZeroUsize;
-
use crate::{Document, Span, Token};
mod all;
@@ -57,7 +55,7 @@ pub trait Pattern {
/// Check if the pattern matches at the start of the given token slice.
///
/// Returns the length of the match if successful, or `None` if not.
- fn matches(&self, tokens: &[Token], source: &[char]) -> Option<NonZeroUsize>;
+ fn matches(&self, tokens: &[Token], source: &[char]) -> Option<usize>;
}
#[cfg(feature = "concurrent")]
#[blanket(derive(Arc))]
@@ -65,7 +63,7 @@ pub trait Pattern: Send + Sync {
/// Check if the pattern matches at the start of the given token slice.
///
/// Returns the length of the match if successful, or `None` if not.
- fn matches(&self, tokens: &[Token], source: &[char]) -> Option<NonZeroUsize>;
+ fn matches(&self, tokens: &[Token], source: &[char]) -> Option<usize>;
}
pub trait PatternExt {
@@ -81,6 +79,9 @@ impl<P> PatternExt for P
where
P: Pattern + ?Sized,
{
+ fn find_all_matches(&self, tokens: &[Token], source: &[char]) -> Vec<Span> {
+ self.iter_matches(tokens, source).collect()
+ }
fn iter_matches(&self, tokens: &[Token], source: &[char]) -> impl Iterator<Item = Span> {
MatchIter::new(self, tokens, source)
}
@@ -117,8 +118,8 @@ where
.pattern
.matches(&self.tokens[self.index..], self.source)
{
- let span = Span::new_with_len(self.index, len.get());
- self.index += len.get();
+ let span = Span::new_with_len(self.index, len);
+ self.index += len.max(1);
return Some(span);
} else {
self.index += 1;
@@ -148,8 +149,12 @@ where
F: Fn(&Token, &[char]) -> bool,
F: Send + Sync,
{
- fn matches(&self, tokens: &[Token], source: &[char]) -> Option<NonZeroUsize> {
- NonZeroUsize::new(if self(tokens.first()?, source) { 1 } else { 0 })
+ fn matches(&self, tokens: &[Token], source: &[char]) -> Option<usize> {
+ if self(tokens.first()?, source) {
+ Some(1)
+ } else {
+ None
+ }
}
}
@@ -159,7 +164,11 @@ where
F: Fn(&Token, &[char]) -> bool,
{
fn matches(&self, tokens: &[Token], source: &[char]) -> Option<NonZeroUsize> {
- NonZeroUsize::new(if self(tokens.first()?, source) { 1 } else { 0 })
+ if self(tokens.first()?, source) {
+ Some(1)
+ } else {
+ None
+ }
}
}
diff --git a/harper-core/src/patterns/naive_pattern_group.rs b/harper-core/src/patterns/naive_pattern_group.rs
index 0c08564e6..4520834bb 100644
--- a/harper-core/src/patterns/naive_pattern_group.rs
+++ b/harper-core/src/patterns/naive_pattern_group.rs
@@ -1,5 +1,3 @@
-use std::num::NonZeroUsize;
-
use super::Pattern;
use crate::Token;
@@ -17,7 +15,7 @@ impl NaivePatternGroup {
}
impl Pattern for NaivePatternGroup {
- fn matches(&self, tokens: &[Token], source: &[char]) -> Option<NonZeroUsize> {
+ fn matches(&self, tokens: &[Token], source: &[char]) -> Option<usize> {
self.patterns
.iter()
.filter_map(|p| p.matches(tokens, source))
diff --git a/harper-core/src/patterns/nominal_phrase.rs b/harper-core/src/patterns/nominal_phrase.rs
index a655f5bef..555c1beee 100644
--- a/harper-core/src/patterns/nominal_phrase.rs
+++ b/harper-core/src/patterns/nominal_phrase.rs
@@ -1,5 +1,3 @@
-use std::num::NonZeroUsize;
-
use crate::Token;
use super::Pattern;
@@ -8,7 +6,7 @@ use super::Pattern;
pub struct NominalPhrase;
impl Pattern for NominalPhrase {
- fn matches(&self, tokens: &[Token], _source: &[char]) -> Option<NonZeroUsize> {
+ fn matches(&self, tokens: &[Token], _source: &[char]) -> Option<usize> {
let mut cursor = 0;
loop {
@@ -26,7 +24,7 @@ impl Pattern for NominalPhrase {
}
if tok.kind.is_nominal() {
- return NonZeroUsize::new(cursor + 1);
+ return Some(cursor + 1);
}
return None;
diff --git a/harper-core/src/patterns/pattern_map.rs b/harper-core/src/patterns/pattern_map.rs
index 3a6b19ad1..b4a654896 100644
--- a/harper-core/src/patterns/pattern_map.rs
+++ b/harper-core/src/patterns/pattern_map.rs
@@ -1,5 +1,3 @@
-use std::num::NonZeroUsize;
-
use crate::LSend;
use crate::Token;
@@ -69,7 +67,7 @@ impl<T> Pattern for PatternMap<T>
where
T: LSend,
{
- fn matches(&self, tokens: &[Token], source: &[char]) -> Option<NonZeroUsize> {
+ fn matches(&self, tokens: &[Token], source: &[char]) -> Option<usize> {
self.rows
.iter()
.filter_map(|row| row.key.matches(tokens, source))
diff --git a/harper-core/src/patterns/repeating_pattern.rs b/harper-core/src/patterns/repeating_pattern.rs
index 514551c4c..b18aa2eef 100644
--- a/harper-core/src/patterns/repeating_pattern.rs
+++ b/harper-core/src/patterns/repeating_pattern.rs
@@ -1,5 +1,3 @@
-use std::num::NonZeroUsize;
-
use super::Pattern;
use crate::Token;
@@ -21,7 +19,7 @@ impl RepeatingPattern {
}
impl Pattern for RepeatingPattern {
- fn matches(&self, tokens: &[Token], source: &[char]) -> Option<NonZeroUsize> {
+ fn matches(&self, tokens: &[Token], source: &[char]) -> Option<usize> {
let mut tok_cursor = 0;
let mut repetition = 0;
@@ -29,10 +27,17 @@ impl Pattern for RepeatingPattern {
let match_len = self.inner.matches(&tokens[tok_cursor..], source);
if let Some(match_len) = match_len {
- tok_cursor += match_len.get();
+ if match_len == 0 {
+ // If match_len == 0, we won't move forward ever again.
+ // This means that we can get infinitely many repetitions,
+ // so repetition >= self.required_repetitions is guaranteed.
+ return Some(tok_cursor);
+ }
+
+ tok_cursor += match_len;
repetition += 1;
} else if repetition >= self.required_repetitions {
- return NonZeroUsize::new(tok_cursor);
+ return Some(tok_cursor);
} else {
return None;
}
@@ -42,7 +47,6 @@ impl Pattern for RepeatingPattern {
#[cfg(test)]
mod tests {
- use std::num::NonZeroUsize;
use super::RepeatingPattern;
use crate::Document;
@@ -57,7 +61,7 @@ mod tests {
assert_eq!(
pat.matches(doc.get_tokens(), doc.get_source()),
- NonZeroUsize::new(doc.get_tokens().len())
+ Some(doc.get_tokens().len())
)
}
diff --git a/harper-core/src/patterns/sequence_pattern.rs b/harper-core/src/patterns/sequence_pattern.rs
index 959fc45e1..14e4d937b 100644
--- a/harper-core/src/patterns/sequence_pattern.rs
+++ b/harper-core/src/patterns/sequence_pattern.rs
@@ -1,5 +1,3 @@
-use std::num::NonZeroUsize;
-
use paste::paste;
use super::whitespace_pattern::WhitespacePattern;
@@ -147,7 +145,7 @@ impl SequencePattern {
pub fn then_one_or_more(mut self, pat: impl Pattern + 'static) -> Self {
self.token_patterns
- .push(Box::new(RepeatingPattern::new(Box::new(pat), 0)));
+ .push(Box::new(RepeatingPattern::new(Box::new(pat), 1)));
self
}
@@ -165,23 +163,21 @@ impl SequencePattern {
}
impl Pattern for SequencePattern {
- fn matches(&self, tokens: &[Token], source: &[char]) -> Option<NonZeroUsize> {
+ fn matches(&self, tokens: &[Token], source: &[char]) -> Option<usize> {
let mut tok_cursor = 0;
for pat in self.token_patterns.iter() {
let match_length = pat.matches(&tokens[tok_cursor..], source)?;
- tok_cursor += match_length.get();
+ tok_cursor += match_length;
}
- NonZeroUsize::new(tok_cursor)
+ Some(tok_cursor)
}
}
#[cfg(test)]
mod tests {
- use std::num::NonZeroUsize;
-
use super::SequencePattern;
use crate::Document;
use crate::patterns::Pattern;
@@ -196,7 +192,7 @@ mod tests {
assert_eq!(
pat.matches(doc.get_tokens(), doc.get_source()),
- NonZeroUsize::new(doc.get_tokens().len())
+ Some(doc.get_tokens().len())
);
}
@@ -210,7 +206,7 @@ mod tests {
assert_eq!(
pat.matches(doc.get_tokens(), doc.get_source()),
- NonZeroUsize::new(doc.get_tokens().len())
+ Some(doc.get_tokens().len())
);
}
@@ -221,7 +217,7 @@ mod tests {
assert_eq!(
pat.matches(doc.get_tokens(), doc.get_source()),
- NonZeroUsize::new(doc.get_tokens().len())
+ Some(doc.get_tokens().len())
);
}
}
diff --git a/harper-core/src/patterns/similar_to_phrase.rs b/harper-core/src/patterns/similar_to_phrase.rs
index 1577a4388..729854f50 100644
--- a/harper-core/src/patterns/similar_to_phrase.rs
+++ b/harper-core/src/patterns/similar_to_phrase.rs
@@ -1,5 +1,3 @@
-use std::num::NonZeroUsize;
-
use crate::{Document, Token, TokenKind};
use super::{Pattern, SequencePattern, Word, within_edit_distance::WithinEditDistance};
@@ -59,7 +57,7 @@ impl SimilarToPhrase {
}
impl Pattern for SimilarToPhrase {
- fn matches(&self, tokens: &[Token], source: &[char]) -> Option<NonZeroUsize> {
+ fn matches(&self, tokens: &[Token], source: &[char]) -> Option<usize> {
if self.phrase.matches(tokens, source).is_some() {
return None;
}
diff --git a/harper-core/src/patterns/split_compound_word.rs b/harper-core/src/patterns/split_compound_word.rs
index 836d788ec..445c9896c 100644
--- a/harper-core/src/patterns/split_compound_word.rs
+++ b/harper-core/src/patterns/split_compound_word.rs
@@ -1,4 +1,4 @@
-use std::{num::NonZeroUsize, sync::Arc};
+use std::sync::Arc;
use crate::{CharString, Dictionary, FstDictionary, Token, WordMetadata};
@@ -57,10 +57,10 @@ impl SplitCompoundWord {
}
impl Pattern for SplitCompoundWord {
- fn matches(&self, tokens: &[Token], source: &[char]) -> Option<NonZeroUsize> {
+ fn matches(&self, tokens: &[Token], source: &[char]) -> Option<usize> {
let inner_match = self.inner.matches(tokens, source)?;
- if inner_match.get() != 3 {
+ if inner_match != 3 {
return None;
}
diff --git a/harper-core/src/patterns/whitespace_pattern.rs b/harper-core/src/patterns/whitespace_pattern.rs
index 2ead95e6f..69cb92113 100644
--- a/harper-core/src/patterns/whitespace_pattern.rs
+++ b/harper-core/src/patterns/whitespace_pattern.rs
@@ -1,16 +1,14 @@
-use std::num::NonZeroUsize;
-
use super::Pattern;
pub struct WhitespacePattern;
impl Pattern for WhitespacePattern {
- fn matches(&self, tokens: &[crate::Token], _source: &[char]) -> Option<NonZeroUsize> {
- NonZeroUsize::new(
- tokens
- .iter()
- .position(|t| !t.kind.is_whitespace())
- .unwrap_or(tokens.len()),
- )
+ fn matches(&self, tokens: &[crate::Token], _source: &[char]) -> Option<usize> {
+ let count = tokens
+ .iter()
+ .position(|t| !t.kind.is_whitespace())
+ .unwrap_or(tokens.len());
+
+ if count == 0 { None } else { Some(count) }
}
}
diff --git a/harper-core/src/patterns/within_edit_distance.rs b/harper-core/src/patterns/within_edit_distance.rs
index 0b5fb6b5b..897d7fe6e 100644
--- a/harper-core/src/patterns/within_edit_distance.rs
+++ b/harper-core/src/patterns/within_edit_distance.rs
@@ -1,5 +1,4 @@
use std::cell::RefCell;
-use std::num::NonZeroUsize;
use super::Pattern;
use crate::{CharString, CharStringExt, Token};
@@ -33,7 +32,7 @@ thread_local! {
}
impl Pattern for WithinEditDistance {
- fn matches(&self, tokens: &[Token], source: &[char]) -> Option<NonZeroUsize> {
+ fn matches(&self, tokens: &[Token], source: &[char]) -> Option<usize> {
let first = tokens.first()?;
if !first.kind.is_word() {
return None;
@@ -48,7 +47,11 @@ impl Pattern for WithinEditDistance {
buffer_a,
buffer_b,
);
- NonZeroUsize::new(if distance <= self.max_edit_dist { 1 } else { 0 })
+ if distance <= self.max_edit_dist {
+ Some(1)
+ } else {
+ None
+ }
})
}
}
diff --git a/harper-core/src/patterns/word.rs b/harper-core/src/patterns/word.rs
index 53a098e77..6ae5b5145 100644
--- a/harper-core/src/patterns/word.rs
+++ b/harper-core/src/patterns/word.rs
@@ -1,5 +1,3 @@
-use std::num::NonZeroUsize;
-
use super::Pattern;
use crate::{CharString, Token};
@@ -39,7 +37,7 @@ impl Word {
}
impl Pattern for Word {
- fn matches(&self, tokens: &[Token], source: &[char]) -> Option<NonZeroUsize> {
+ fn matches(&self, tokens: &[Token], source: &[char]) -> Option<usize> {
let tok = tokens.first()?;
if !tok.kind.is_word() {
return None;
@@ -58,7 +56,7 @@ impl Pattern for Word {
.all(|(a, b)| a.eq_ignore_ascii_case(b))
};
- NonZeroUsize::new(if eq { 1 } else { 0 })
+ if eq { Some(1) } else { None }
}
}
diff --git a/harper-core/src/patterns/word_pattern_group.rs b/harper-core/src/patterns/word_pattern_group.rs
index 1c3f5399d..5028b592e 100644
--- a/harper-core/src/patterns/word_pattern_group.rs
+++ b/harper-core/src/patterns/word_pattern_group.rs
@@ -1,5 +1,3 @@
-use std::num::NonZeroUsize;
-
use hashbrown::HashMap;
use super::naive_pattern_group::NaivePatternGroup;
@@ -42,7 +40,7 @@ impl<P> Pattern for WordPatternGroup<P>
where
P: Pattern,
{
- fn matches(&self, tokens: &[crate::Token], source: &[char]) -> Option<NonZeroUsize> {
+ fn matches(&self, tokens: &[crate::Token], source: &[char]) -> Option<usize> {
let first = tokens.first()?;
if !first.kind.is_word() {
return None;
diff --git a/harper-core/src/patterns/word_set.rs b/harper-core/src/patterns/word_set.rs
index e60961b80..47034224d 100644
--- a/harper-core/src/patterns/word_set.rs
+++ b/harper-core/src/patterns/word_set.rs
@@ -1,5 +1,3 @@
-use std::num::NonZeroUsize;
-
use super::Pattern;
use smallvec::SmallVec;
@@ -40,7 +38,7 @@ impl WordSet {
}
impl Pattern for WordSet {
- fn matches(&self, tokens: &[Token], source: &[char]) -> Option<NonZeroUsize> {
+ fn matches(&self, tokens: &[Token], source: &[char]) -> Option<usize> {
let tok = tokens.first()?;
if !tok.kind.is_word() {
return None;
@@ -59,7 +57,7 @@ impl Pattern for WordSet {
.all(|(a, b)| a.eq_ignore_ascii_case(b));
if partial_match {
- return NonZeroUsize::new(1);
+ return Some(1);
}
}