Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
test: `exact_phrase` and `exact_word` differ in case sensitivity (#1361)
Andrew Dunbar2c3f0c659126
diff --git a/harper-core/src/linting/a_part.rs b/harper-core/src/linting/a_part.rsindex a1ca5d21b..0215f45d8 100644--- a/harper-core/src/linting/a_part.rs+++ b/harper-core/src/linting/a_part.rs@@ -1,7 +1,7 @@use crate::{Token, TokenStringExt,linting::{Lint, LintKind, PatternLinter, Suggestion},- patterns::{EitherPattern, ExactPhrase, Pattern},+ patterns::{EitherPattern, FixedPhrase, Pattern},};pub struct APart {@@ -11,10 +11,10 @@ pub struct APart {impl Default for APart {fn default() -> Self {let pattern = EitherPattern::new(vec![- Box::new(ExactPhrase::from_phrase("a part from")),- Box::new(ExactPhrase::from_phrase("apart of")),- Box::new(ExactPhrase::from_phrase("fall a part")),- Box::new(ExactPhrase::from_phrase("far a part")),+ Box::new(FixedPhrase::from_phrase("a part from")),+ Box::new(FixedPhrase::from_phrase("apart of")),+ Box::new(FixedPhrase::from_phrase("fall a part")),+ Box::new(FixedPhrase::from_phrase("far a part")),]);Self {diff --git a/harper-core/src/linting/amounts_for.rs b/harper-core/src/linting/amounts_for.rsindex 0fa76f337..0130ff666 100644--- a/harper-core/src/linting/amounts_for.rs+++ b/harper-core/src/linting/amounts_for.rs@@ -1,6 +1,6 @@use crate::{Lrc, Token, TokenStringExt,- patterns::{EitherPattern, ExactPhrase, Pattern, SequencePattern, WordSet},+ patterns::{EitherPattern, FixedPhrase, Pattern, SequencePattern, WordSet},};use super::{Lint, LintKind, PatternLinter, Suggestion};@@ -16,7 +16,7 @@ impl Default for AmountsFor {let singular_pattern = SequencePattern::default().then(singular_context).then_whitespace()- .then(Lrc::new(ExactPhrase::from_phrase("amounts for")));+ .then(Lrc::new(FixedPhrase::from_phrase("amounts for")));let singular_context = WordSet::new(&["they", "can", "could", "may", "might", "must", "should", "will", "would",@@ -25,7 +25,7 @@ impl Default for AmountsFor {let plural_pattern = SequencePattern::default().then(singular_context).then_whitespace()- .then(Lrc::new(ExactPhrase::from_phrase("amount for")));+ .then(Lrc::new(FixedPhrase::from_phrase("amount for")));Self {pattern: Box::new(EitherPattern::new(vec![diff --git a/harper-core/src/linting/back_in_the_day.rs b/harper-core/src/linting/back_in_the_day.rsindex c28cd5c9d..46af73bfc 100644--- a/harper-core/src/linting/back_in_the_day.rs+++ b/harper-core/src/linting/back_in_the_day.rs@@ -1,6 +1,6 @@use crate::{Lrc, Token, TokenStringExt,- patterns::{ExactPhrase, OwnedPatternExt, Pattern, SequencePattern, WordSet},+ patterns::{FixedPhrase, OwnedPatternExt, Pattern, SequencePattern, WordSet},};use super::{Lint, LintKind, PatternLinter, Suggestion};@@ -14,7 +14,7 @@ pub struct BackInTheDay {impl Default for BackInTheDay {fn default() -> Self {let exceptions = Lrc::new(WordSet::new(&["before", "of", "when"]));- let phrase = Lrc::new(ExactPhrase::from_phrase("back in the days"));+ let phrase = Lrc::new(FixedPhrase::from_phrase("back in the days"));let pattern = SequencePattern::default().then(phrase.clone())diff --git a/harper-core/src/linting/hedging.rs b/harper-core/src/linting/hedging.rsindex 74faee628..dd8867137 100644--- a/harper-core/src/linting/hedging.rs+++ b/harper-core/src/linting/hedging.rs@@ -1,5 +1,5 @@use crate::linting::{Lint, LintKind, PatternLinter};-use crate::patterns::{EitherPattern, ExactPhrase, Pattern};+use crate::patterns::{EitherPattern, FixedPhrase, Pattern};use crate::{Token, TokenStringExt};/// A linter that detects hedging language.@@ -13,7 +13,7 @@ impl Default for Hedging {let patterns: Vec<Box<dyn Pattern>> = phrases.into_iter()- .map(|s| Box::new(ExactPhrase::from_phrase(s)) as Box<dyn Pattern>)+ .map(|s| Box::new(FixedPhrase::from_phrase(s)) as Box<dyn Pattern>).collect();let pattern = Box::new(EitherPattern::new(patterns));diff --git a/harper-core/src/linting/map_phrase_linter.rs b/harper-core/src/linting/map_phrase_linter.rsindex 4bb541513..74d122213 100644--- a/harper-core/src/linting/map_phrase_linter.rs+++ b/harper-core/src/linting/map_phrase_linter.rs@@ -1,6 +1,6 @@use super::{Lint, LintKind, PatternLinter};use crate::linting::Suggestion;-use crate::patterns::{EitherPattern, ExactPhrase, Pattern, SimilarToPhrase};+use crate::patterns::{EitherPattern, FixedPhrase, Pattern, SimilarToPhrase};use crate::{Token, TokenStringExt};pub struct MapPhraseLinter {@@ -34,7 +34,7 @@ impl MapPhraseLinter {)}- pub fn new_exact_phrases(+ pub fn new_fixed_phrases(phrase: impl IntoIterator<Item = impl AsRef<str>>,correct_forms: impl IntoIterator<Item = impl ToString>,message: impl ToString,@@ -44,7 +44,7 @@ impl MapPhraseLinter {phrase.into_iter().map(|p| {- let pattern: Box<dyn Pattern> = Box::new(ExactPhrase::from_phrase(p.as_ref()));+ let pattern: Box<dyn Pattern> = Box::new(FixedPhrase::from_phrase(p.as_ref()));pattern}).collect(),@@ -53,14 +53,14 @@ impl MapPhraseLinter {Self::new(Box::new(patterns), correct_forms, message, description)}- pub fn new_exact_phrase(+ pub fn new_fixed_phrase(phrase: impl AsRef<str>,correct_forms: impl IntoIterator<Item = impl ToString>,message: impl ToString,description: impl ToString,) -> Self {Self::new(- Box::new(ExactPhrase::from_phrase(phrase.as_ref())),+ Box::new(FixedPhrase::from_phrase(phrase.as_ref())),correct_forms,message,description,@@ -78,7 +78,7 @@ impl MapPhraseLinter {correct_form.to_string());- Self::new_exact_phrase(phrase, [correct_form], message, description)+ Self::new_fixed_phrase(phrase, [correct_form], message, description)}}diff --git a/harper-core/src/linting/one_and_the_same.rs b/harper-core/src/linting/one_and_the_same.rsindex 582f521e1..a1061929d 100644--- a/harper-core/src/linting/one_and_the_same.rs+++ b/harper-core/src/linting/one_and_the_same.rs@@ -1,6 +1,6 @@use crate::{Lrc, Token, TokenStringExt,- patterns::{EitherPattern, ExactPhrase, Pattern, SequencePattern, WordSet},+ patterns::{EitherPattern, FixedPhrase, Pattern, SequencePattern, WordSet},};use super::{Lint, LintKind, PatternLinter, Suggestion};@@ -11,7 +11,7 @@ pub struct OneAndTheSame {impl Default for OneAndTheSame {fn default() -> Self {- let one_in_the_same = Lrc::new(ExactPhrase::from_phrase("one in the same"));+ let one_in_the_same = Lrc::new(FixedPhrase::from_phrase("one in the same"));Self {pattern: Box::new(EitherPattern::new(vec![diff --git a/harper-core/src/linting/out_of_date.rs b/harper-core/src/linting/out_of_date.rsindex 00c31fd9c..2cbc412cd 100644--- a/harper-core/src/linting/out_of_date.rs+++ b/harper-core/src/linting/out_of_date.rs@@ -1,6 +1,6 @@use crate::{Token, TokenStringExt,- patterns::{EitherPattern, ExactPhrase, Pattern},+ patterns::{EitherPattern, FixedPhrase, Pattern},};use super::{Lint, LintKind, PatternLinter, Suggestion};@@ -12,9 +12,9 @@ pub struct OutOfDate {impl Default for OutOfDate {fn default() -> Self {let pattern = EitherPattern::new(vec![- Box::new(ExactPhrase::from_phrase("out of date")),- Box::new(ExactPhrase::from_phrase("out-of date")),- Box::new(ExactPhrase::from_phrase("out of-date")),+ Box::new(FixedPhrase::from_phrase("out of date")),+ Box::new(FixedPhrase::from_phrase("out-of date")),+ Box::new(FixedPhrase::from_phrase("out of-date")),]);Self {diff --git a/harper-core/src/linting/oxymorons.rs b/harper-core/src/linting/oxymorons.rsindex 78e230b53..46dc8f130 100644--- a/harper-core/src/linting/oxymorons.rs+++ b/harper-core/src/linting/oxymorons.rs@@ -1,5 +1,5 @@use crate::linting::{Lint, LintKind, PatternLinter};-use crate::patterns::{EitherPattern, ExactPhrase, Pattern};+use crate::patterns::{EitherPattern, FixedPhrase, Pattern};use crate::{Token, TokenStringExt};/// A linter that flags oxymoronic phrases.@@ -33,7 +33,7 @@ impl Oxymorons {// Build a vector of exact-match patterns for each oxymoron.let patterns: Vec<Box<dyn Pattern>> = phrases.into_iter()- .map(|s| Box::new(ExactPhrase::from_phrase(s)) as Box<dyn Pattern>)+ .map(|s| Box::new(FixedPhrase::from_phrase(s)) as Box<dyn Pattern>).collect();let pattern = Box::new(EitherPattern::new(patterns));diff --git a/harper-core/src/linting/phrase_corrections.rs b/harper-core/src/linting/phrase_corrections.rsindex e7b45f314..5a715d618 100644--- a/harper-core/src/linting/phrase_corrections.rs+++ b/harper-core/src/linting/phrase_corrections.rs@@ -13,7 +13,7 @@ pub fn lint_group() -> LintGroup {$group.add_pattern_linter($name,Box::new(- MapPhraseLinter::new_exact_phrases(+ MapPhraseLinter::new_fixed_phrases($input,$corrections,$hint,@@ -28,7 +28,7 @@ pub fn lint_group() -> LintGroup {add_exact_mappings!(group, {// The name of the rule"ChangeTack" => (- // The exact phrase(s) to look for.+ // The phrase(s) to look for.["change tact", "change tacks", "change tacts"],// The corrections to provide.["change tack"],diff --git a/harper-core/src/linting/proper_noun_capitalization_linters.rs b/harper-core/src/linting/proper_noun_capitalization_linters.rsindex 065894c27..92e0e25c0 100644--- a/harper-core/src/linting/proper_noun_capitalization_linters.rs+++ b/harper-core/src/linting/proper_noun_capitalization_linters.rs@@ -4,7 +4,7 @@ use serde::{Deserialize, Serialize};use super::{Lint, LintKind, Suggestion};use super::{LintGroup, PatternLinter};use crate::parsers::PlainEnglish;-use crate::patterns::{ExactPhrase, Pattern, PatternMap};+use crate::patterns::{FixedPhrase, Pattern, PatternMap};use crate::{Dictionary, Document};use crate::{Token, TokenStringExt};use std::sync::Arc;@@ -51,7 +51,7 @@ impl<D: Dictionary + 'static> ProperNounCapitalizationLinter<D> {&PlainEnglish,&dictionary,);- let pattern = ExactPhrase::from_document(&doc);+ let pattern = FixedPhrase::from_document(&doc);pattern_map.insert(pattern, doc);}diff --git a/harper-core/src/patterns/exact_phrase.rs b/harper-core/src/patterns/exact_phrase.rsdeleted file mode 100644index 06659e1ce..000000000--- a/harper-core/src/patterns/exact_phrase.rs+++ /dev/null@@ -1,50 +0,0 @@-use crate::{Document, Token, TokenKind};--use super::{Pattern, SequencePattern, Word};--pub struct ExactPhrase {- inner: SequencePattern,-}--impl ExactPhrase {- pub fn from_phrase(text: &str) -> Self {- let document = Document::new_plain_english_curated(text);- Self::from_document(&document)- }-- pub fn from_document(doc: &Document) -> Self {- let mut phrase = SequencePattern::default();-- for token in doc.fat_tokens() {- match token.kind {- TokenKind::Word(_word_metadata) => {- phrase = phrase.then(Word::from_chars(token.content.as_slice()));- }- TokenKind::Space(_) => {- phrase = phrase.then_whitespace();- }- TokenKind::Punctuation(p) => {- phrase = phrase.then(move |t: &Token, _source: &[char]| {- t.kind.as_punctuation().cloned() == Some(p)- })- }- TokenKind::ParagraphBreak => {- phrase = phrase.then_whitespace();- }- TokenKind::Number(n) => {- phrase = phrase- .then(move |tok: &Token, _source: &[char]| tok.kind == TokenKind::Number(n))- }- _ => panic!("Fell out of expected document formats."),- }- }-- Self { inner: phrase }- }-}--impl Pattern for ExactPhrase {- fn matches(&self, tokens: &[Token], source: &[char]) -> Option<usize> {- self.inner.matches(tokens, source)- }-}diff --git a/harper-core/src/patterns/fixed_phrase.rs b/harper-core/src/patterns/fixed_phrase.rsnew file mode 100644index 000000000..de9b70357--- /dev/null+++ b/harper-core/src/patterns/fixed_phrase.rs@@ -0,0 +1,109 @@+use crate::{Document, Token, TokenKind};++use super::{Pattern, SequencePattern, Word};++/// Matches a fixed sequence of tokens as they appear in the input.+/// Case-insensitive for words but maintains exact matching for other token types.+///+/// # Example+///+/// ```rust+/// use harper_core::patterns::{FixedPhrase, Pattern};+/// use harper_core::Document;+///+/// let doc = Document::new_plain_english_curated("Hello, world!");+/// let phrase = FixedPhrase::from_phrase("Hello, world!");+/// assert!(phrase.matches(doc.get_tokens(), doc.get_source()).is_some());+/// ```+pub struct FixedPhrase {+ inner: SequencePattern,+}++impl FixedPhrase {+ /// Creates a [FixedPhrase] from a plain text string.+ /// Uses plain English tokenization rules.+ pub fn from_phrase(text: &str) -> Self {+ let document = Document::new_plain_english_curated(text);+ Self::from_document(&document)+ }++ /// Creates a [FixedPhrase] from a pre-tokenized document.+ /// Allows custom tokenization by creating a `Document` first.+ pub fn from_document(doc: &Document) -> Self {+ let mut phrase = SequencePattern::default();++ for token in doc.fat_tokens() {+ match token.kind {+ TokenKind::Word(_word_metadata) => {+ phrase = phrase.then(Word::from_chars(token.content.as_slice()));+ }+ TokenKind::Space(_) => {+ phrase = phrase.then_whitespace();+ }+ TokenKind::Punctuation(p) => {+ phrase = phrase.then(move |t: &Token, _source: &[char]| {+ t.kind.as_punctuation().cloned() == Some(p)+ })+ }+ TokenKind::ParagraphBreak => {+ phrase = phrase.then_whitespace();+ }+ TokenKind::Number(n) => {+ phrase = phrase+ .then(move |tok: &Token, _source: &[char]| tok.kind == TokenKind::Number(n))+ }+ _ => panic!("Fell out of expected document formats."),+ }+ }++ Self { inner: phrase }+ }+}++impl Pattern for FixedPhrase {+ /// Matches this phrase against the start of the token slice.+ /// Returns `Some(len)` on match (number of tokens consumed), `None` otherwise.+ fn matches(&self, tokens: &[Token], source: &[char]) -> Option<usize> {+ self.inner.matches(tokens, source)+ }+}++#[cfg(test)]+mod tests {+ use crate::{+ Document,+ patterns::{FixedPhrase, Pattern},+ };++ #[test]+ fn test_not_case_sensitive() {+ let doc_lower = Document::new_plain_english_curated("hello world");+ let doc_upper = Document::new_plain_english_curated("HELLO WORLD");+ let doc_title = Document::new_plain_english_curated("Hello World");+ let phrase = FixedPhrase::from_document(&doc_lower);+ assert_eq!(+ phrase.matches(doc_lower.get_tokens(), doc_title.get_source()),+ Some(3)+ );+ assert_eq!(+ phrase.matches(doc_lower.get_tokens(), doc_upper.get_source()),+ Some(3)+ );+ assert_eq!(+ phrase.matches(doc_title.get_tokens(), doc_lower.get_source()),+ Some(3)+ );+ assert_eq!(+ phrase.matches(doc_title.get_tokens(), doc_upper.get_source()),+ Some(3)+ );+ assert_eq!(+ phrase.matches(doc_upper.get_tokens(), doc_lower.get_source()),+ Some(3)+ );+ assert_eq!(+ phrase.matches(doc_upper.get_tokens(), doc_title.get_source()),+ Some(3)+ );+ }+}diff --git a/harper-core/src/patterns/mod.rs b/harper-core/src/patterns/mod.rsindex 17cdac810..fcdf69756 100644--- a/harper-core/src/patterns/mod.rs+++ b/harper-core/src/patterns/mod.rs@@ -10,7 +10,7 @@ use crate::{Document, LSend, Span, Token};mod all;mod any_pattern;mod either_pattern;-mod exact_phrase;+mod fixed_phrase;mod implies_quantity;mod indefinite_article;mod inflection_of_be;@@ -33,7 +33,7 @@ pub use all::All;pub use any_pattern::AnyPattern;use blanket::blanket;pub use either_pattern::EitherPattern;-pub use exact_phrase::ExactPhrase;+pub use fixed_phrase::FixedPhrase;pub use implies_quantity::ImpliesQuantity;pub use indefinite_article::IndefiniteArticle;pub use inflection_of_be::InflectionOfBe;diff --git a/harper-core/src/patterns/sequence_pattern.rs b/harper-core/src/patterns/sequence_pattern.rsindex 10f9c11d0..e5029b6df 100644--- a/harper-core/src/patterns/sequence_pattern.rs+++ b/harper-core/src/patterns/sequence_pattern.rs@@ -239,7 +239,7 @@ mod tests {}#[test]- fn exact_word_doesnt_match_different_cases() {+ fn exact_means_case_sensitive() {let pat = SequencePattern::default().then_exact_word("Foo");let doc = Document::new_plain_english_curated("foo Foo FOO");let matches = pat.find_all_matches_in_doc(&doc);