Skip to content

Offline, privacy-first grammar checker. Fast, open-source, Rust-powered

test: `exact_phrase` and `exact_word` differ in case sensitivity (#1361)

Andrew Dunbar2c3f0c659126
diff --git a/harper-core/src/linting/a_part.rs b/harper-core/src/linting/a_part.rs
index a1ca5d21b..0215f45d8 100644
--- a/harper-core/src/linting/a_part.rs
+++ b/harper-core/src/linting/a_part.rs
@@ -1,7 +1,7 @@
use crate::{
Token, TokenStringExt,
linting::{Lint, LintKind, PatternLinter, Suggestion},
- patterns::{EitherPattern, ExactPhrase, Pattern},
+ patterns::{EitherPattern, FixedPhrase, Pattern},
};
pub struct APart {
@@ -11,10 +11,10 @@ pub struct APart {
impl Default for APart {
fn default() -> Self {
let pattern = EitherPattern::new(vec![
- Box::new(ExactPhrase::from_phrase("a part from")),
- Box::new(ExactPhrase::from_phrase("apart of")),
- Box::new(ExactPhrase::from_phrase("fall a part")),
- Box::new(ExactPhrase::from_phrase("far a part")),
+ Box::new(FixedPhrase::from_phrase("a part from")),
+ Box::new(FixedPhrase::from_phrase("apart of")),
+ Box::new(FixedPhrase::from_phrase("fall a part")),
+ Box::new(FixedPhrase::from_phrase("far a part")),
]);
Self {
diff --git a/harper-core/src/linting/amounts_for.rs b/harper-core/src/linting/amounts_for.rs
index 0fa76f337..0130ff666 100644
--- a/harper-core/src/linting/amounts_for.rs
+++ b/harper-core/src/linting/amounts_for.rs
@@ -1,6 +1,6 @@
use crate::{
Lrc, Token, TokenStringExt,
- patterns::{EitherPattern, ExactPhrase, Pattern, SequencePattern, WordSet},
+ patterns::{EitherPattern, FixedPhrase, Pattern, SequencePattern, WordSet},
};
use super::{Lint, LintKind, PatternLinter, Suggestion};
@@ -16,7 +16,7 @@ impl Default for AmountsFor {
let singular_pattern = SequencePattern::default()
.then(singular_context)
.then_whitespace()
- .then(Lrc::new(ExactPhrase::from_phrase("amounts for")));
+ .then(Lrc::new(FixedPhrase::from_phrase("amounts for")));
let singular_context = WordSet::new(&[
"they", "can", "could", "may", "might", "must", "should", "will", "would",
@@ -25,7 +25,7 @@ impl Default for AmountsFor {
let plural_pattern = SequencePattern::default()
.then(singular_context)
.then_whitespace()
- .then(Lrc::new(ExactPhrase::from_phrase("amount for")));
+ .then(Lrc::new(FixedPhrase::from_phrase("amount for")));
Self {
pattern: Box::new(EitherPattern::new(vec![
diff --git a/harper-core/src/linting/back_in_the_day.rs b/harper-core/src/linting/back_in_the_day.rs
index c28cd5c9d..46af73bfc 100644
--- a/harper-core/src/linting/back_in_the_day.rs
+++ b/harper-core/src/linting/back_in_the_day.rs
@@ -1,6 +1,6 @@
use crate::{
Lrc, Token, TokenStringExt,
- patterns::{ExactPhrase, OwnedPatternExt, Pattern, SequencePattern, WordSet},
+ patterns::{FixedPhrase, OwnedPatternExt, Pattern, SequencePattern, WordSet},
};
use super::{Lint, LintKind, PatternLinter, Suggestion};
@@ -14,7 +14,7 @@ pub struct BackInTheDay {
impl Default for BackInTheDay {
fn default() -> Self {
let exceptions = Lrc::new(WordSet::new(&["before", "of", "when"]));
- let phrase = Lrc::new(ExactPhrase::from_phrase("back in the days"));
+ let phrase = Lrc::new(FixedPhrase::from_phrase("back in the days"));
let pattern = SequencePattern::default()
.then(phrase.clone())
diff --git a/harper-core/src/linting/hedging.rs b/harper-core/src/linting/hedging.rs
index 74faee628..dd8867137 100644
--- a/harper-core/src/linting/hedging.rs
+++ b/harper-core/src/linting/hedging.rs
@@ -1,5 +1,5 @@
use crate::linting::{Lint, LintKind, PatternLinter};
-use crate::patterns::{EitherPattern, ExactPhrase, Pattern};
+use crate::patterns::{EitherPattern, FixedPhrase, Pattern};
use crate::{Token, TokenStringExt};
/// A linter that detects hedging language.
@@ -13,7 +13,7 @@ impl Default for Hedging {
let patterns: Vec<Box<dyn Pattern>> = phrases
.into_iter()
- .map(|s| Box::new(ExactPhrase::from_phrase(s)) as Box<dyn Pattern>)
+ .map(|s| Box::new(FixedPhrase::from_phrase(s)) as Box<dyn Pattern>)
.collect();
let pattern = Box::new(EitherPattern::new(patterns));
diff --git a/harper-core/src/linting/map_phrase_linter.rs b/harper-core/src/linting/map_phrase_linter.rs
index 4bb541513..74d122213 100644
--- a/harper-core/src/linting/map_phrase_linter.rs
+++ b/harper-core/src/linting/map_phrase_linter.rs
@@ -1,6 +1,6 @@
use super::{Lint, LintKind, PatternLinter};
use crate::linting::Suggestion;
-use crate::patterns::{EitherPattern, ExactPhrase, Pattern, SimilarToPhrase};
+use crate::patterns::{EitherPattern, FixedPhrase, Pattern, SimilarToPhrase};
use crate::{Token, TokenStringExt};
pub struct MapPhraseLinter {
@@ -34,7 +34,7 @@ impl MapPhraseLinter {
)
}
- pub fn new_exact_phrases(
+ pub fn new_fixed_phrases(
phrase: impl IntoIterator<Item = impl AsRef<str>>,
correct_forms: impl IntoIterator<Item = impl ToString>,
message: impl ToString,
@@ -44,7 +44,7 @@ impl MapPhraseLinter {
phrase
.into_iter()
.map(|p| {
- let pattern: Box<dyn Pattern> = Box::new(ExactPhrase::from_phrase(p.as_ref()));
+ let pattern: Box<dyn Pattern> = Box::new(FixedPhrase::from_phrase(p.as_ref()));
pattern
})
.collect(),
@@ -53,14 +53,14 @@ impl MapPhraseLinter {
Self::new(Box::new(patterns), correct_forms, message, description)
}
- pub fn new_exact_phrase(
+ pub fn new_fixed_phrase(
phrase: impl AsRef<str>,
correct_forms: impl IntoIterator<Item = impl ToString>,
message: impl ToString,
description: impl ToString,
) -> Self {
Self::new(
- Box::new(ExactPhrase::from_phrase(phrase.as_ref())),
+ Box::new(FixedPhrase::from_phrase(phrase.as_ref())),
correct_forms,
message,
description,
@@ -78,7 +78,7 @@ impl MapPhraseLinter {
correct_form.to_string()
);
- Self::new_exact_phrase(phrase, [correct_form], message, description)
+ Self::new_fixed_phrase(phrase, [correct_form], message, description)
}
}
diff --git a/harper-core/src/linting/one_and_the_same.rs b/harper-core/src/linting/one_and_the_same.rs
index 582f521e1..a1061929d 100644
--- a/harper-core/src/linting/one_and_the_same.rs
+++ b/harper-core/src/linting/one_and_the_same.rs
@@ -1,6 +1,6 @@
use crate::{
Lrc, Token, TokenStringExt,
- patterns::{EitherPattern, ExactPhrase, Pattern, SequencePattern, WordSet},
+ patterns::{EitherPattern, FixedPhrase, Pattern, SequencePattern, WordSet},
};
use super::{Lint, LintKind, PatternLinter, Suggestion};
@@ -11,7 +11,7 @@ pub struct OneAndTheSame {
impl Default for OneAndTheSame {
fn default() -> Self {
- let one_in_the_same = Lrc::new(ExactPhrase::from_phrase("one in the same"));
+ let one_in_the_same = Lrc::new(FixedPhrase::from_phrase("one in the same"));
Self {
pattern: Box::new(EitherPattern::new(vec![
diff --git a/harper-core/src/linting/out_of_date.rs b/harper-core/src/linting/out_of_date.rs
index 00c31fd9c..2cbc412cd 100644
--- a/harper-core/src/linting/out_of_date.rs
+++ b/harper-core/src/linting/out_of_date.rs
@@ -1,6 +1,6 @@
use crate::{
Token, TokenStringExt,
- patterns::{EitherPattern, ExactPhrase, Pattern},
+ patterns::{EitherPattern, FixedPhrase, Pattern},
};
use super::{Lint, LintKind, PatternLinter, Suggestion};
@@ -12,9 +12,9 @@ pub struct OutOfDate {
impl Default for OutOfDate {
fn default() -> Self {
let pattern = EitherPattern::new(vec![
- Box::new(ExactPhrase::from_phrase("out of date")),
- Box::new(ExactPhrase::from_phrase("out-of date")),
- Box::new(ExactPhrase::from_phrase("out of-date")),
+ Box::new(FixedPhrase::from_phrase("out of date")),
+ Box::new(FixedPhrase::from_phrase("out-of date")),
+ Box::new(FixedPhrase::from_phrase("out of-date")),
]);
Self {
diff --git a/harper-core/src/linting/oxymorons.rs b/harper-core/src/linting/oxymorons.rs
index 78e230b53..46dc8f130 100644
--- a/harper-core/src/linting/oxymorons.rs
+++ b/harper-core/src/linting/oxymorons.rs
@@ -1,5 +1,5 @@
use crate::linting::{Lint, LintKind, PatternLinter};
-use crate::patterns::{EitherPattern, ExactPhrase, Pattern};
+use crate::patterns::{EitherPattern, FixedPhrase, Pattern};
use crate::{Token, TokenStringExt};
/// A linter that flags oxymoronic phrases.
@@ -33,7 +33,7 @@ impl Oxymorons {
// Build a vector of exact-match patterns for each oxymoron.
let patterns: Vec<Box<dyn Pattern>> = phrases
.into_iter()
- .map(|s| Box::new(ExactPhrase::from_phrase(s)) as Box<dyn Pattern>)
+ .map(|s| Box::new(FixedPhrase::from_phrase(s)) as Box<dyn Pattern>)
.collect();
let pattern = Box::new(EitherPattern::new(patterns));
diff --git a/harper-core/src/linting/phrase_corrections.rs b/harper-core/src/linting/phrase_corrections.rs
index e7b45f314..5a715d618 100644
--- a/harper-core/src/linting/phrase_corrections.rs
+++ b/harper-core/src/linting/phrase_corrections.rs
@@ -13,7 +13,7 @@ pub fn lint_group() -> LintGroup {
$group.add_pattern_linter(
$name,
Box::new(
- MapPhraseLinter::new_exact_phrases(
+ MapPhraseLinter::new_fixed_phrases(
$input,
$corrections,
$hint,
@@ -28,7 +28,7 @@ pub fn lint_group() -> LintGroup {
add_exact_mappings!(group, {
// The name of the rule
"ChangeTack" => (
- // The exact phrase(s) to look for.
+ // The phrase(s) to look for.
["change tact", "change tacks", "change tacts"],
// The corrections to provide.
["change tack"],
diff --git a/harper-core/src/linting/proper_noun_capitalization_linters.rs b/harper-core/src/linting/proper_noun_capitalization_linters.rs
index 065894c27..92e0e25c0 100644
--- a/harper-core/src/linting/proper_noun_capitalization_linters.rs
+++ b/harper-core/src/linting/proper_noun_capitalization_linters.rs
@@ -4,7 +4,7 @@ use serde::{Deserialize, Serialize};
use super::{Lint, LintKind, Suggestion};
use super::{LintGroup, PatternLinter};
use crate::parsers::PlainEnglish;
-use crate::patterns::{ExactPhrase, Pattern, PatternMap};
+use crate::patterns::{FixedPhrase, Pattern, PatternMap};
use crate::{Dictionary, Document};
use crate::{Token, TokenStringExt};
use std::sync::Arc;
@@ -51,7 +51,7 @@ impl<D: Dictionary + 'static> ProperNounCapitalizationLinter<D> {
&PlainEnglish,
&dictionary,
);
- let pattern = ExactPhrase::from_document(&doc);
+ let pattern = FixedPhrase::from_document(&doc);
pattern_map.insert(pattern, doc);
}
diff --git a/harper-core/src/patterns/exact_phrase.rs b/harper-core/src/patterns/exact_phrase.rs
deleted file mode 100644
index 06659e1ce..000000000
--- a/harper-core/src/patterns/exact_phrase.rs
+++ /dev/null
@@ -1,50 +0,0 @@
-use crate::{Document, Token, TokenKind};
-
-use super::{Pattern, SequencePattern, Word};
-
-pub struct ExactPhrase {
- inner: SequencePattern,
-}
-
-impl ExactPhrase {
- pub fn from_phrase(text: &str) -> Self {
- let document = Document::new_plain_english_curated(text);
- Self::from_document(&document)
- }
-
- pub fn from_document(doc: &Document) -> Self {
- let mut phrase = SequencePattern::default();
-
- for token in doc.fat_tokens() {
- match token.kind {
- TokenKind::Word(_word_metadata) => {
- phrase = phrase.then(Word::from_chars(token.content.as_slice()));
- }
- TokenKind::Space(_) => {
- phrase = phrase.then_whitespace();
- }
- TokenKind::Punctuation(p) => {
- phrase = phrase.then(move |t: &Token, _source: &[char]| {
- t.kind.as_punctuation().cloned() == Some(p)
- })
- }
- TokenKind::ParagraphBreak => {
- phrase = phrase.then_whitespace();
- }
- TokenKind::Number(n) => {
- phrase = phrase
- .then(move |tok: &Token, _source: &[char]| tok.kind == TokenKind::Number(n))
- }
- _ => panic!("Fell out of expected document formats."),
- }
- }
-
- Self { inner: phrase }
- }
-}
-
-impl Pattern for ExactPhrase {
- fn matches(&self, tokens: &[Token], source: &[char]) -> Option<usize> {
- self.inner.matches(tokens, source)
- }
-}
diff --git a/harper-core/src/patterns/fixed_phrase.rs b/harper-core/src/patterns/fixed_phrase.rs
new file mode 100644
index 000000000..de9b70357
--- /dev/null
+++ b/harper-core/src/patterns/fixed_phrase.rs
@@ -0,0 +1,109 @@
+use crate::{Document, Token, TokenKind};
+
+use super::{Pattern, SequencePattern, Word};
+
+/// Matches a fixed sequence of tokens as they appear in the input.
+/// Case-insensitive for words but maintains exact matching for other token types.
+///
+/// # Example
+///
+/// ```rust
+/// use harper_core::patterns::{FixedPhrase, Pattern};
+/// use harper_core::Document;
+///
+/// let doc = Document::new_plain_english_curated("Hello, world!");
+/// let phrase = FixedPhrase::from_phrase("Hello, world!");
+/// assert!(phrase.matches(doc.get_tokens(), doc.get_source()).is_some());
+/// ```
+pub struct FixedPhrase {
+ inner: SequencePattern,
+}
+
+impl FixedPhrase {
+ /// Creates a [FixedPhrase] from a plain text string.
+ /// Uses plain English tokenization rules.
+ pub fn from_phrase(text: &str) -> Self {
+ let document = Document::new_plain_english_curated(text);
+ Self::from_document(&document)
+ }
+
+ /// Creates a [FixedPhrase] from a pre-tokenized document.
+ /// Allows custom tokenization by creating a `Document` first.
+ pub fn from_document(doc: &Document) -> Self {
+ let mut phrase = SequencePattern::default();
+
+ for token in doc.fat_tokens() {
+ match token.kind {
+ TokenKind::Word(_word_metadata) => {
+ phrase = phrase.then(Word::from_chars(token.content.as_slice()));
+ }
+ TokenKind::Space(_) => {
+ phrase = phrase.then_whitespace();
+ }
+ TokenKind::Punctuation(p) => {
+ phrase = phrase.then(move |t: &Token, _source: &[char]| {
+ t.kind.as_punctuation().cloned() == Some(p)
+ })
+ }
+ TokenKind::ParagraphBreak => {
+ phrase = phrase.then_whitespace();
+ }
+ TokenKind::Number(n) => {
+ phrase = phrase
+ .then(move |tok: &Token, _source: &[char]| tok.kind == TokenKind::Number(n))
+ }
+ _ => panic!("Fell out of expected document formats."),
+ }
+ }
+
+ Self { inner: phrase }
+ }
+}
+
+impl Pattern for FixedPhrase {
+ /// Matches this phrase against the start of the token slice.
+ /// Returns `Some(len)` on match (number of tokens consumed), `None` otherwise.
+ fn matches(&self, tokens: &[Token], source: &[char]) -> Option<usize> {
+ self.inner.matches(tokens, source)
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use crate::{
+ Document,
+ patterns::{FixedPhrase, Pattern},
+ };
+
+ #[test]
+ fn test_not_case_sensitive() {
+ let doc_lower = Document::new_plain_english_curated("hello world");
+ let doc_upper = Document::new_plain_english_curated("HELLO WORLD");
+ let doc_title = Document::new_plain_english_curated("Hello World");
+ let phrase = FixedPhrase::from_document(&doc_lower);
+ assert_eq!(
+ phrase.matches(doc_lower.get_tokens(), doc_title.get_source()),
+ Some(3)
+ );
+ assert_eq!(
+ phrase.matches(doc_lower.get_tokens(), doc_upper.get_source()),
+ Some(3)
+ );
+ assert_eq!(
+ phrase.matches(doc_title.get_tokens(), doc_lower.get_source()),
+ Some(3)
+ );
+ assert_eq!(
+ phrase.matches(doc_title.get_tokens(), doc_upper.get_source()),
+ Some(3)
+ );
+ assert_eq!(
+ phrase.matches(doc_upper.get_tokens(), doc_lower.get_source()),
+ Some(3)
+ );
+ assert_eq!(
+ phrase.matches(doc_upper.get_tokens(), doc_title.get_source()),
+ Some(3)
+ );
+ }
+}
diff --git a/harper-core/src/patterns/mod.rs b/harper-core/src/patterns/mod.rs
index 17cdac810..fcdf69756 100644
--- a/harper-core/src/patterns/mod.rs
+++ b/harper-core/src/patterns/mod.rs
@@ -10,7 +10,7 @@ use crate::{Document, LSend, Span, Token};
mod all;
mod any_pattern;
mod either_pattern;
-mod exact_phrase;
+mod fixed_phrase;
mod implies_quantity;
mod indefinite_article;
mod inflection_of_be;
@@ -33,7 +33,7 @@ pub use all::All;
pub use any_pattern::AnyPattern;
use blanket::blanket;
pub use either_pattern::EitherPattern;
-pub use exact_phrase::ExactPhrase;
+pub use fixed_phrase::FixedPhrase;
pub use implies_quantity::ImpliesQuantity;
pub use indefinite_article::IndefiniteArticle;
pub use inflection_of_be::InflectionOfBe;
diff --git a/harper-core/src/patterns/sequence_pattern.rs b/harper-core/src/patterns/sequence_pattern.rs
index 10f9c11d0..e5029b6df 100644
--- a/harper-core/src/patterns/sequence_pattern.rs
+++ b/harper-core/src/patterns/sequence_pattern.rs
@@ -239,7 +239,7 @@ mod tests {
}
#[test]
- fn exact_word_doesnt_match_different_cases() {
+ fn exact_means_case_sensitive() {
let pat = SequencePattern::default().then_exact_word("Foo");
let doc = Document::new_plain_english_curated("foo Foo FOO");
let matches = pat.find_all_matches_in_doc(&doc);