Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
feat: spelled-number pattern & since duration linter (#1325)
Andrew Dunbarbde8720d05eb
diff --git a/harper-core/src/linting/lint_group.rs b/harper-core/src/linting/lint_group.rsindex 02f7076ba..72b364b3c 100644--- a/harper-core/src/linting/lint_group.rs+++ b/harper-core/src/linting/lint_group.rs@@ -67,6 +67,7 @@ use super::proper_noun_capitalization_linters;use super::repeated_words::RepeatedWords;use super::save_to_safe::SaveToSafe;use super::sentence_capitalization::SentenceCapitalization;+use super::since_duration::SinceDuration;use super::somewhat_something::SomewhatSomething;use super::spaces::Spaces;use super::spell_check::SpellCheck;@@ -391,6 +392,7 @@ impl LintGroup {insert_struct_rule!(PronounKnew, true);insert_struct_rule!(RepeatedWords, true);insert_struct_rule!(SaveToSafe, true);+ insert_pattern_rule!(SinceDuration, true);insert_pattern_rule!(SomewhatSomething, true);insert_struct_rule!(Spaces, true);insert_struct_rule!(SpelledNumbers, false);diff --git a/harper-core/src/linting/mod.rs b/harper-core/src/linting/mod.rsindex 373542a9d..42cc7a568 100644--- a/harper-core/src/linting/mod.rs+++ b/harper-core/src/linting/mod.rs@@ -72,6 +72,7 @@ mod proper_noun_capitalization_linters;mod repeated_words;mod save_to_safe;mod sentence_capitalization;+mod since_duration;mod somewhat_something;mod spaces;mod spell_check;@@ -148,6 +149,7 @@ pub use pronoun_contraction::PronounContraction;pub use repeated_words::RepeatedWords;pub use save_to_safe::SaveToSafe;pub use sentence_capitalization::SentenceCapitalization;+pub use since_duration::SinceDuration;pub use somewhat_something::SomewhatSomething;pub use spaces::Spaces;pub use spell_check::SpellCheck;diff --git a/harper-core/src/linting/since_duration.rs b/harper-core/src/linting/since_duration.rsnew file mode 100644index 000000000..f7e8f7bca--- /dev/null+++ b/harper-core/src/linting/since_duration.rs@@ -0,0 +1,299 @@+use crate::{+ Lrc, Token, TokenStringExt,+ patterns::{EitherPattern, Pattern, SequencePattern, SpelledNumberPattern, WordSet},+};++use super::{Lint, LintKind, PatternLinter, Suggestion};++const AGO_VARIANTS: [&[char]; 3] = [&['a', 'g', 'o'], &['A', 'g', 'o'], &['A', 'G', 'O']];+const FOR_VARIANTS: [&[char]; 3] = [&['f', 'o', 'r'], &['F', 'o', 'r'], &['F', 'O', 'R']];++fn match_case_string<'a>(template: &[char], variants: [&'a [char]; 3]) -> &'a [char] {+ let c1 = template.first().copied().unwrap();+ let c2 = template.get(1).copied().unwrap_or(' ');+ if c1.is_uppercase() && c2.is_uppercase() {+ variants[2]+ } else if c1.is_uppercase() {+ variants[1]+ } else {+ variants[0]+ }+}++pub struct SinceDuration {+ pattern: Box<dyn Pattern>,+}++impl Default for SinceDuration {+ fn default() -> Self {+ let units = WordSet::new(&[+ "minute", "minutes", "hour", "hours", "day", "days", "week", "weeks", "month",+ "months", "year", "years",+ ]);++ let pattern_without_ago = Lrc::new(+ SequencePattern::default()+ .then_any_capitalization_of("since")+ .then_whitespace()+ .then(EitherPattern::new(vec![+ Box::new(SpelledNumberPattern),+ Box::new(SequencePattern::default().then_number()),+ ]))+ .then_whitespace()+ .then(units),+ );++ let pattern_with_ago = SequencePattern::default()+ .then(pattern_without_ago.clone())+ .then_whitespace()+ .then_any_capitalization_of("ago");++ Self {+ pattern: Box::new(EitherPattern::new(vec![+ Box::new(pattern_without_ago),+ Box::new(pattern_with_ago),+ ])),+ }+ }+}++impl PatternLinter for SinceDuration {+ fn pattern(&self) -> &dyn Pattern {+ self.pattern.as_ref()+ }++ fn match_to_lint(&self, toks: &[Token], src: &[char]) -> Option<Lint> {+ let last = toks.last()?;+ if last.span.get_content_string(src).to_lowercase() == "ago" {+ return None;+ }++ let since_duration_span = toks.span()?;++ let mut since_point_in_time = since_duration_span.get_content(src).to_vec();+ since_point_in_time.push(' ');+ let unit_template = toks.last()?.span.get_content(src);+ since_point_in_time.extend(+ match_case_string(unit_template, AGO_VARIANTS)+ .iter()+ .copied(),+ );+ let ago_suggestion = Suggestion::ReplaceWith(since_point_in_time);++ let duration = toks[1..].span()?.get_content(src);+ let since_template = toks.first()?.span.get_content(src);+ let mut for_duration = match_case_string(since_template, FOR_VARIANTS).to_vec();+ for_duration.extend(duration);+ let for_suggestion = Suggestion::ReplaceWith(for_duration);++ Some(Lint {+ span: since_duration_span,+ lint_kind: LintKind::Miscellaneous,+ suggestions: vec![for_suggestion, ago_suggestion],+ message: "For a duration, use 'for' instead of 'since'. Or for a point in time, add 'ago' at the end.".to_string(),+ priority: 50,+ })+ }++ fn description(&self) -> &str {+ "Detects the use of 'since' with a duration instead of a point in time."+ }+}++#[cfg(test)]+mod tests {+ use crate::linting::SinceDuration;+ use crate::linting::tests::{assert_lint_count, assert_top3_suggestion_result};++ #[test]+ fn catches_spelled() {+ assert_lint_count(+ "I have been waiting since two hours.",+ SinceDuration::default(),+ 1,+ );+ }++ #[test]+ fn permits_spelled_with_ago() {+ assert_lint_count(+ "I have been waiting since two hours ago.",+ SinceDuration::default(),+ 0,+ );+ }++ #[test]+ fn catches_numerals() {+ assert_lint_count(+ "I have been waiting since 2 hours.",+ SinceDuration::default(),+ 1,+ );+ }++ #[test]+ fn permits_numerals_with_ago() {+ assert_lint_count(+ "I have been waiting since 2 hours ago.",+ SinceDuration::default(),+ 0,+ );+ }++ #[test]+ fn correct_without_issues() {+ assert_top3_suggestion_result(+ "I'm running v2.2.1 on bare metal (no docker, vm) since two weeks without issues.",+ SinceDuration::default(),+ "I'm running v2.2.1 on bare metal (no docker, vm) for two weeks without issues.",+ );+ }++ #[test]+ fn correct_anything_back() {+ assert_top3_suggestion_result(+ "I have not heard anything back since three months.",+ SinceDuration::default(),+ "I have not heard anything back for three months.",+ );+ }++ #[test]+ fn correct_get_done() {+ assert_top3_suggestion_result(+ "I am trying to get this done since two days, someone please help.",+ SinceDuration::default(),+ "I am trying to get this done for two days, someone please help.",+ );+ }++ #[test]+ fn correct_deprecated() {+ assert_top3_suggestion_result(+ "This project is now officially deprecated, since I worked with virtualabs on the next version of Mirage since three years now: an ecosystem of tools named WHAD.",+ SinceDuration::default(),+ "This project is now officially deprecated, since I worked with virtualabs on the next version of Mirage for three years now: an ecosystem of tools named WHAD.",+ );+ }++ #[test]+ fn correct_same() {+ assert_top3_suggestion_result(+ "Same! Since two days.",+ SinceDuration::default(),+ "Same! For two days.",+ );+ }++ #[test]+ fn correct_what_changed() {+ assert_top3_suggestion_result(+ "What changed since two weeks?",+ SinceDuration::default(),+ "What changed since two weeks ago?",+ );+ }++ #[test]+ fn correct_with_period() {+ assert_top3_suggestion_result(+ "I have been waiting since two hours.",+ SinceDuration::default(),+ "I have been waiting since two hours ago.",+ );+ }++ #[test]+ fn correct_with_exclamation() {+ assert_top3_suggestion_result(+ "I have been waiting since two hours!",+ SinceDuration::default(),+ "I have been waiting since two hours ago!",+ );+ }++ #[test]+ fn correct_with_question_mark() {+ assert_top3_suggestion_result(+ "Have you been waiting since two hours?",+ SinceDuration::default(),+ "Have you been waiting for two hours?",+ );+ }++ #[test]+ fn correct_with_comma() {+ assert_top3_suggestion_result(+ "Since two days, I have been trying to get this done.",+ SinceDuration::default(),+ "For two days, I have been trying to get this done.",+ );+ }++ #[test]+ fn correct_for_title_case() {+ assert_top3_suggestion_result(+ "Since 45 Minutes I See The Following Picture In The Terminal.",+ SinceDuration::default(),+ "For 45 Minutes I See The Following Picture In The Terminal.",+ );+ }++ #[test]+ fn correct_for_all_caps() {+ assert_top3_suggestion_result(+ "STOPPED SINCE 12 HOURS WITH EXIT CODE 0",+ SinceDuration::default(),+ "STOPPED FOR 12 HOURS WITH EXIT CODE 0",+ );+ }++ #[test]+ fn correct_ago_title_case() {+ assert_top3_suggestion_result(+ "It Is In Development Since Two Years.",+ SinceDuration::default(),+ "It Is In Development Since Two Years Ago.",+ );+ }++ #[test]+ fn correct_ago_all_caps() {+ assert_top3_suggestion_result(+ "BUG: SINCE 6 MONTHS UNLOAD CHECKPOINT",+ SinceDuration::default(),+ "BUG: SINCE 6 MONTHS AGO UNLOAD CHECKPOINT",+ );+ }++ #[test]+ #[ignore = "We can't yet handle modifiers like 'over'. Plus it doesn't work with 'ago'."]+ fn not_yet_handled() {+ assert_top3_suggestion_result(+ "It's an asked feature since over 9 years",+ SinceDuration::default(),+ "It's an asked feature for over 9 years.",+ );+ }++ #[test]+ #[ignore = "We can't yet handle modifiers like 'more than'. Plus it doesn't work with 'ago'."]+ fn not_yet_handled_2() {+ assert_top3_suggestion_result(+ "It's an asked feature since more than 9 years",+ SinceDuration::default(),+ "It's an asked feature for more than 9 years.",+ );+ }++ #[test]+ #[ignore = "We can't yet handle indefinite numbers."]+ fn not_yet_handled_3() {+ assert_top3_suggestion_result(+ "I use a Wacom Cintiq 27QHDT since several years on Linux",+ SinceDuration::default(),+ "I use a Wacom Cintiq 27QHDT for several years on Linux",+ );+ }+}diff --git a/harper-core/src/patterns/mod.rs b/harper-core/src/patterns/mod.rsindex 73842d97d..a5d12e413 100644--- a/harper-core/src/patterns/mod.rs+++ b/harper-core/src/patterns/mod.rs@@ -21,6 +21,7 @@ mod pattern_map;mod repeating_pattern;mod sequence_pattern;mod similar_to_phrase;+mod spelled_number_pattern;mod split_compound_word;mod whitespace_pattern;mod within_edit_distance;@@ -43,6 +44,7 @@ pub use pattern_map::PatternMap;pub use repeating_pattern::RepeatingPattern;pub use sequence_pattern::SequencePattern;pub use similar_to_phrase::SimilarToPhrase;+pub use spelled_number_pattern::SpelledNumberPattern;pub use split_compound_word::SplitCompoundWord;pub use whitespace_pattern::WhitespacePattern;pub use word::Word;diff --git a/harper-core/src/patterns/sequence_pattern.rs b/harper-core/src/patterns/sequence_pattern.rsindex dd40da95b..10f9c11d0 100644--- a/harper-core/src/patterns/sequence_pattern.rs+++ b/harper-core/src/patterns/sequence_pattern.rs@@ -91,6 +91,7 @@ impl SequencePattern {self.then(IndefiniteArticle::default())}+ /// Match examples of `word` case-sensitively.pub fn then_exact_word(mut self, word: &'static str) -> Self {self.token_patterns.push(Box::new(Word::new_exact(word)));self@@ -185,7 +186,7 @@ mod tests {use super::SequencePattern;use crate::Document;- use crate::patterns::Pattern;+ use crate::patterns::{DocPattern, Pattern};#[test]fn matches_n_whitespace_tokens() {@@ -225,4 +226,31 @@ mod tests {Some(doc.get_tokens().len()));}++ #[test]+ fn exact_word_matches_title_case() {+ let pat = SequencePattern::default().then_exact_word("Foo");+ let doc = Document::new_plain_english_curated("Foo");++ assert_eq!(+ pat.matches(doc.get_tokens(), doc.get_source()),+ Some(doc.get_tokens().len())+ );+ }++ #[test]+ fn exact_word_doesnt_match_different_cases() {+ let pat = SequencePattern::default().then_exact_word("Foo");+ let doc = Document::new_plain_english_curated("foo Foo FOO");+ let matches = pat.find_all_matches_in_doc(&doc);+ assert_eq!(matches.len(), 1); // Only "Foo" should match+ }++ #[test]+ fn any_capitalization_of_matches_different_cases() {+ let pat = SequencePattern::aco("foo");+ let doc = Document::new_plain_english_curated("foo Foo FOO");+ let matches = pat.find_all_matches_in_doc(&doc);+ assert_eq!(matches.len(), 3); // All three should match+ }}diff --git a/harper-core/src/patterns/spelled_number_pattern.rs b/harper-core/src/patterns/spelled_number_pattern.rsnew file mode 100644index 000000000..89117ee2c--- /dev/null+++ b/harper-core/src/patterns/spelled_number_pattern.rs@@ -0,0 +1,197 @@+use super::{EitherPattern, Pattern, SequencePattern, WhitespacePattern, WordSet};+use crate::Token;++/// Matches spelled-out numbers from one to ninety-nine+#[derive(Default)]+pub struct SpelledNumberPattern;++impl Pattern for SpelledNumberPattern {+ fn matches(&self, tokens: &[Token], source: &[char]) -> Option<usize> {+ if tokens.is_empty() {+ return None;+ }++ // The numbers that can be in the 2nd position of a compound number.+ // A subset of the standalone numbers since we can't say "twenty zero" or "twenty eleven"+ // "Zero" and "ten" don't belong: twenty-one ✅ twenty-zero ❌ twenty-ten ❌+ let units = &[+ "one", "two", "three", "four", "five", "six", "seven", "eight", "nine",+ ];++ // These can't make a compound with `tens` but they can stand alone+ let teens = &[+ "ten",+ "eleven",+ "twelve",+ "thirteen",+ "fourteen",+ "fifteen",+ "sixteen",+ "seventeen",+ "eighteen",+ "nineteen",+ ];++ // These can make a compound with the part_2 standalones above.+ // "Ten" and "hundred" don't belong: twenty-one ✅ ten-one ❌ hundred-one ❌+ let tens = &[+ "twenty", "thirty", "forty", "fifty", "sixty", "seventy", "eighty", "ninety",+ ];++ let single_words = WordSet::new(+ &units+ .iter()+ .chain(teens.iter())+ .chain(tens.iter())+ .copied()+ .chain(std::iter::once("zero"))+ .collect::<Vec<&str>>(),+ );++ let tens_units_compounds = SequencePattern::default()+ .then(WordSet::new(tens))+ .then(EitherPattern::new(vec![+ Box::new(|t: &Token, _s: &[char]| t.kind.is_hyphen()),+ Box::new(WhitespacePattern),+ ]))+ .then(WordSet::new(units));++ let pat = EitherPattern::new(vec![Box::new(single_words), Box::new(tens_units_compounds)]);++ pat.matches(tokens, source)+ }+}++#[cfg(test)]+mod tests {+ use super::super::DocPattern;+ use super::SpelledNumberPattern;+ use crate::{Document, Span};++ trait SpanVecExt {+ fn to_strings(&self, doc: &Document) -> Vec<String>;+ }++ impl SpanVecExt for Vec<Span> {+ fn to_strings(&self, doc: &Document) -> Vec<String> {+ self.iter()+ .map(|sp| {+ doc.get_tokens()[sp.start..sp.end]+ .iter()+ .map(|tok| doc.get_span_content_str(&tok.span))+ .collect::<String>()+ })+ .collect()+ }+ }++ #[test]+ fn matches_single_digit() {+ let doc = Document::new_markdown_default_curated("one two three");+ let matches = SpelledNumberPattern.find_all_matches_in_doc(&doc);+ assert_eq!(matches.len(), 3);+ }++ #[test]+ fn matches_teens() {+ let doc = Document::new_markdown_default_curated("ten eleven twelve");+ let matches = SpelledNumberPattern.find_all_matches_in_doc(&doc);+ assert_eq!(matches.len(), 3);+ }++ #[test]+ fn matches_tens() {+ let doc = Document::new_markdown_default_curated("twenty thirty forty");+ let matches = SpelledNumberPattern.find_all_matches_in_doc(&doc);+ assert_eq!(matches.len(), 3);+ }++ #[test]+ fn matches_compound_numbers() {+ let doc = Document::new_markdown_default_curated("twenty-one thirty-two");+ let matches = SpelledNumberPattern.find_all_matches_in_doc(&doc);++ // Debug output+ println!("Found {} matches:", matches.len());+ for m in &matches {+ let text: String = doc.get_tokens()[m.start..m.end]+ .iter()+ .map(|t| doc.get_span_content_str(&t.span))+ .collect();+ println!("- '{}' (span: {:?})", text, m);+ }++ assert_eq!(matches.len(), 2);+ }++ #[test]+ fn deep_thought() {+ let doc = Document::new_markdown_default_curated(+ "the answer to the ultimate question of life, the universe, and everything is forty-two",+ );+ let matches = SpelledNumberPattern.find_all_matches_in_doc(&doc);++ dbg!(&matches);+ dbg!(matches.to_strings(&doc));++ assert_eq!(matches.to_strings(&doc), vec!["forty-two"]);+ }++ #[test]+ fn jacksons() {+ let doc = Document::new_markdown_default_curated(+ "A, B, C It's easy as one, two, three. Or simple as Do-Re-Mi",+ );+ let matches = SpelledNumberPattern.find_all_matches_in_doc(&doc);++ assert_eq!(matches.to_strings(&doc), vec!["one", "two", "three"]);+ }++ #[test]+ fn orwell() {+ let doc = Document::new_markdown_default_curated("Nineteen Eighty-Four");+ let matches = SpelledNumberPattern.find_all_matches_in_doc(&doc);++ assert_eq!(matches.to_strings(&doc), vec!["Nineteen", "Eighty-Four"]);+ }++ #[test]+ fn get_smart() {+ let doc = Document::new_markdown_default_curated(+ "Maxwell Smart was Agent Eighty-Six, but who was Agent Ninety-Nine?",+ );+ let matches = SpelledNumberPattern.find_all_matches_in_doc(&doc);++ assert_eq!(matches.to_strings(&doc), vec!["Eighty-Six", "Ninety-Nine"]);+ }++ #[test]+ fn hyphens_or_spaces() {+ let doc = Document::new_markdown_default_curated(+ "twenty-one, thirty two, forty-three, fifty four, sixty-five, seventy six, eighty-seven, ninety eight",+ );+ let matches = SpelledNumberPattern.find_all_matches_in_doc(&doc);++ assert_eq!(+ matches.to_strings(&doc),+ vec![+ "twenty-one",+ "thirty two",+ "forty-three",+ "fifty four",+ "sixty-five",+ "seventy six",+ "eighty-seven",+ "ninety eight",+ ]+ );+ }++ #[test]+ fn waiting_since() {+ let doc = Document::new_markdown_default_curated("I have been waiting since two hours.");+ let matches = SpelledNumberPattern.find_all_matches_in_doc(&doc);++ assert_eq!(matches.to_strings(&doc), vec!["two"]);+ }+}