Skip to content

Offline, privacy-first grammar checker. Fast, open-source, Rust-powered

feat: spelled-number pattern & since duration linter (#1325)

Andrew Dunbarbde8720d05eb
diff --git a/harper-core/src/linting/lint_group.rs b/harper-core/src/linting/lint_group.rs
index 02f7076ba..72b364b3c 100644
--- a/harper-core/src/linting/lint_group.rs
+++ b/harper-core/src/linting/lint_group.rs
@@ -67,6 +67,7 @@ use super::proper_noun_capitalization_linters;
use super::repeated_words::RepeatedWords;
use super::save_to_safe::SaveToSafe;
use super::sentence_capitalization::SentenceCapitalization;
+use super::since_duration::SinceDuration;
use super::somewhat_something::SomewhatSomething;
use super::spaces::Spaces;
use super::spell_check::SpellCheck;
@@ -391,6 +392,7 @@ impl LintGroup {
insert_struct_rule!(PronounKnew, true);
insert_struct_rule!(RepeatedWords, true);
insert_struct_rule!(SaveToSafe, true);
+ insert_pattern_rule!(SinceDuration, true);
insert_pattern_rule!(SomewhatSomething, true);
insert_struct_rule!(Spaces, true);
insert_struct_rule!(SpelledNumbers, false);
diff --git a/harper-core/src/linting/mod.rs b/harper-core/src/linting/mod.rs
index 373542a9d..42cc7a568 100644
--- a/harper-core/src/linting/mod.rs
+++ b/harper-core/src/linting/mod.rs
@@ -72,6 +72,7 @@ mod proper_noun_capitalization_linters;
mod repeated_words;
mod save_to_safe;
mod sentence_capitalization;
+mod since_duration;
mod somewhat_something;
mod spaces;
mod spell_check;
@@ -148,6 +149,7 @@ pub use pronoun_contraction::PronounContraction;
pub use repeated_words::RepeatedWords;
pub use save_to_safe::SaveToSafe;
pub use sentence_capitalization::SentenceCapitalization;
+pub use since_duration::SinceDuration;
pub use somewhat_something::SomewhatSomething;
pub use spaces::Spaces;
pub use spell_check::SpellCheck;
diff --git a/harper-core/src/linting/since_duration.rs b/harper-core/src/linting/since_duration.rs
new file mode 100644
index 000000000..f7e8f7bca
--- /dev/null
+++ b/harper-core/src/linting/since_duration.rs
@@ -0,0 +1,299 @@
+use crate::{
+ Lrc, Token, TokenStringExt,
+ patterns::{EitherPattern, Pattern, SequencePattern, SpelledNumberPattern, WordSet},
+};
+
+use super::{Lint, LintKind, PatternLinter, Suggestion};
+
+const AGO_VARIANTS: [&[char]; 3] = [&['a', 'g', 'o'], &['A', 'g', 'o'], &['A', 'G', 'O']];
+const FOR_VARIANTS: [&[char]; 3] = [&['f', 'o', 'r'], &['F', 'o', 'r'], &['F', 'O', 'R']];
+
+fn match_case_string<'a>(template: &[char], variants: [&'a [char]; 3]) -> &'a [char] {
+ let c1 = template.first().copied().unwrap();
+ let c2 = template.get(1).copied().unwrap_or(' ');
+ if c1.is_uppercase() && c2.is_uppercase() {
+ variants[2]
+ } else if c1.is_uppercase() {
+ variants[1]
+ } else {
+ variants[0]
+ }
+}
+
+pub struct SinceDuration {
+ pattern: Box<dyn Pattern>,
+}
+
+impl Default for SinceDuration {
+ fn default() -> Self {
+ let units = WordSet::new(&[
+ "minute", "minutes", "hour", "hours", "day", "days", "week", "weeks", "month",
+ "months", "year", "years",
+ ]);
+
+ let pattern_without_ago = Lrc::new(
+ SequencePattern::default()
+ .then_any_capitalization_of("since")
+ .then_whitespace()
+ .then(EitherPattern::new(vec![
+ Box::new(SpelledNumberPattern),
+ Box::new(SequencePattern::default().then_number()),
+ ]))
+ .then_whitespace()
+ .then(units),
+ );
+
+ let pattern_with_ago = SequencePattern::default()
+ .then(pattern_without_ago.clone())
+ .then_whitespace()
+ .then_any_capitalization_of("ago");
+
+ Self {
+ pattern: Box::new(EitherPattern::new(vec![
+ Box::new(pattern_without_ago),
+ Box::new(pattern_with_ago),
+ ])),
+ }
+ }
+}
+
+impl PatternLinter for SinceDuration {
+ fn pattern(&self) -> &dyn Pattern {
+ self.pattern.as_ref()
+ }
+
+ fn match_to_lint(&self, toks: &[Token], src: &[char]) -> Option<Lint> {
+ let last = toks.last()?;
+ if last.span.get_content_string(src).to_lowercase() == "ago" {
+ return None;
+ }
+
+ let since_duration_span = toks.span()?;
+
+ let mut since_point_in_time = since_duration_span.get_content(src).to_vec();
+ since_point_in_time.push(' ');
+ let unit_template = toks.last()?.span.get_content(src);
+ since_point_in_time.extend(
+ match_case_string(unit_template, AGO_VARIANTS)
+ .iter()
+ .copied(),
+ );
+ let ago_suggestion = Suggestion::ReplaceWith(since_point_in_time);
+
+ let duration = toks[1..].span()?.get_content(src);
+ let since_template = toks.first()?.span.get_content(src);
+ let mut for_duration = match_case_string(since_template, FOR_VARIANTS).to_vec();
+ for_duration.extend(duration);
+ let for_suggestion = Suggestion::ReplaceWith(for_duration);
+
+ Some(Lint {
+ span: since_duration_span,
+ lint_kind: LintKind::Miscellaneous,
+ suggestions: vec![for_suggestion, ago_suggestion],
+ message: "For a duration, use 'for' instead of 'since'. Or for a point in time, add 'ago' at the end.".to_string(),
+ priority: 50,
+ })
+ }
+
+ fn description(&self) -> &str {
+ "Detects the use of 'since' with a duration instead of a point in time."
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use crate::linting::SinceDuration;
+ use crate::linting::tests::{assert_lint_count, assert_top3_suggestion_result};
+
+ #[test]
+ fn catches_spelled() {
+ assert_lint_count(
+ "I have been waiting since two hours.",
+ SinceDuration::default(),
+ 1,
+ );
+ }
+
+ #[test]
+ fn permits_spelled_with_ago() {
+ assert_lint_count(
+ "I have been waiting since two hours ago.",
+ SinceDuration::default(),
+ 0,
+ );
+ }
+
+ #[test]
+ fn catches_numerals() {
+ assert_lint_count(
+ "I have been waiting since 2 hours.",
+ SinceDuration::default(),
+ 1,
+ );
+ }
+
+ #[test]
+ fn permits_numerals_with_ago() {
+ assert_lint_count(
+ "I have been waiting since 2 hours ago.",
+ SinceDuration::default(),
+ 0,
+ );
+ }
+
+ #[test]
+ fn correct_without_issues() {
+ assert_top3_suggestion_result(
+ "I'm running v2.2.1 on bare metal (no docker, vm) since two weeks without issues.",
+ SinceDuration::default(),
+ "I'm running v2.2.1 on bare metal (no docker, vm) for two weeks without issues.",
+ );
+ }
+
+ #[test]
+ fn correct_anything_back() {
+ assert_top3_suggestion_result(
+ "I have not heard anything back since three months.",
+ SinceDuration::default(),
+ "I have not heard anything back for three months.",
+ );
+ }
+
+ #[test]
+ fn correct_get_done() {
+ assert_top3_suggestion_result(
+ "I am trying to get this done since two days, someone please help.",
+ SinceDuration::default(),
+ "I am trying to get this done for two days, someone please help.",
+ );
+ }
+
+ #[test]
+ fn correct_deprecated() {
+ assert_top3_suggestion_result(
+ "This project is now officially deprecated, since I worked with virtualabs on the next version of Mirage since three years now: an ecosystem of tools named WHAD.",
+ SinceDuration::default(),
+ "This project is now officially deprecated, since I worked with virtualabs on the next version of Mirage for three years now: an ecosystem of tools named WHAD.",
+ );
+ }
+
+ #[test]
+ fn correct_same() {
+ assert_top3_suggestion_result(
+ "Same! Since two days.",
+ SinceDuration::default(),
+ "Same! For two days.",
+ );
+ }
+
+ #[test]
+ fn correct_what_changed() {
+ assert_top3_suggestion_result(
+ "What changed since two weeks?",
+ SinceDuration::default(),
+ "What changed since two weeks ago?",
+ );
+ }
+
+ #[test]
+ fn correct_with_period() {
+ assert_top3_suggestion_result(
+ "I have been waiting since two hours.",
+ SinceDuration::default(),
+ "I have been waiting since two hours ago.",
+ );
+ }
+
+ #[test]
+ fn correct_with_exclamation() {
+ assert_top3_suggestion_result(
+ "I have been waiting since two hours!",
+ SinceDuration::default(),
+ "I have been waiting since two hours ago!",
+ );
+ }
+
+ #[test]
+ fn correct_with_question_mark() {
+ assert_top3_suggestion_result(
+ "Have you been waiting since two hours?",
+ SinceDuration::default(),
+ "Have you been waiting for two hours?",
+ );
+ }
+
+ #[test]
+ fn correct_with_comma() {
+ assert_top3_suggestion_result(
+ "Since two days, I have been trying to get this done.",
+ SinceDuration::default(),
+ "For two days, I have been trying to get this done.",
+ );
+ }
+
+ #[test]
+ fn correct_for_title_case() {
+ assert_top3_suggestion_result(
+ "Since 45 Minutes I See The Following Picture In The Terminal.",
+ SinceDuration::default(),
+ "For 45 Minutes I See The Following Picture In The Terminal.",
+ );
+ }
+
+ #[test]
+ fn correct_for_all_caps() {
+ assert_top3_suggestion_result(
+ "STOPPED SINCE 12 HOURS WITH EXIT CODE 0",
+ SinceDuration::default(),
+ "STOPPED FOR 12 HOURS WITH EXIT CODE 0",
+ );
+ }
+
+ #[test]
+ fn correct_ago_title_case() {
+ assert_top3_suggestion_result(
+ "It Is In Development Since Two Years.",
+ SinceDuration::default(),
+ "It Is In Development Since Two Years Ago.",
+ );
+ }
+
+ #[test]
+ fn correct_ago_all_caps() {
+ assert_top3_suggestion_result(
+ "BUG: SINCE 6 MONTHS UNLOAD CHECKPOINT",
+ SinceDuration::default(),
+ "BUG: SINCE 6 MONTHS AGO UNLOAD CHECKPOINT",
+ );
+ }
+
+ #[test]
+ #[ignore = "We can't yet handle modifiers like 'over'. Plus it doesn't work with 'ago'."]
+ fn not_yet_handled() {
+ assert_top3_suggestion_result(
+ "It's an asked feature since over 9 years",
+ SinceDuration::default(),
+ "It's an asked feature for over 9 years.",
+ );
+ }
+
+ #[test]
+ #[ignore = "We can't yet handle modifiers like 'more than'. Plus it doesn't work with 'ago'."]
+ fn not_yet_handled_2() {
+ assert_top3_suggestion_result(
+ "It's an asked feature since more than 9 years",
+ SinceDuration::default(),
+ "It's an asked feature for more than 9 years.",
+ );
+ }
+
+ #[test]
+ #[ignore = "We can't yet handle indefinite numbers."]
+ fn not_yet_handled_3() {
+ assert_top3_suggestion_result(
+ "I use a Wacom Cintiq 27QHDT since several years on Linux",
+ SinceDuration::default(),
+ "I use a Wacom Cintiq 27QHDT for several years on Linux",
+ );
+ }
+}
diff --git a/harper-core/src/patterns/mod.rs b/harper-core/src/patterns/mod.rs
index 73842d97d..a5d12e413 100644
--- a/harper-core/src/patterns/mod.rs
+++ b/harper-core/src/patterns/mod.rs
@@ -21,6 +21,7 @@ mod pattern_map;
mod repeating_pattern;
mod sequence_pattern;
mod similar_to_phrase;
+mod spelled_number_pattern;
mod split_compound_word;
mod whitespace_pattern;
mod within_edit_distance;
@@ -43,6 +44,7 @@ pub use pattern_map::PatternMap;
pub use repeating_pattern::RepeatingPattern;
pub use sequence_pattern::SequencePattern;
pub use similar_to_phrase::SimilarToPhrase;
+pub use spelled_number_pattern::SpelledNumberPattern;
pub use split_compound_word::SplitCompoundWord;
pub use whitespace_pattern::WhitespacePattern;
pub use word::Word;
diff --git a/harper-core/src/patterns/sequence_pattern.rs b/harper-core/src/patterns/sequence_pattern.rs
index dd40da95b..10f9c11d0 100644
--- a/harper-core/src/patterns/sequence_pattern.rs
+++ b/harper-core/src/patterns/sequence_pattern.rs
@@ -91,6 +91,7 @@ impl SequencePattern {
self.then(IndefiniteArticle::default())
}
+ /// Match examples of `word` case-sensitively.
pub fn then_exact_word(mut self, word: &'static str) -> Self {
self.token_patterns.push(Box::new(Word::new_exact(word)));
self
@@ -185,7 +186,7 @@ mod tests {
use super::SequencePattern;
use crate::Document;
- use crate::patterns::Pattern;
+ use crate::patterns::{DocPattern, Pattern};
#[test]
fn matches_n_whitespace_tokens() {
@@ -225,4 +226,31 @@ mod tests {
Some(doc.get_tokens().len())
);
}
+
+ #[test]
+ fn exact_word_matches_title_case() {
+ let pat = SequencePattern::default().then_exact_word("Foo");
+ let doc = Document::new_plain_english_curated("Foo");
+
+ assert_eq!(
+ pat.matches(doc.get_tokens(), doc.get_source()),
+ Some(doc.get_tokens().len())
+ );
+ }
+
+ #[test]
+ fn exact_word_doesnt_match_different_cases() {
+ let pat = SequencePattern::default().then_exact_word("Foo");
+ let doc = Document::new_plain_english_curated("foo Foo FOO");
+ let matches = pat.find_all_matches_in_doc(&doc);
+ assert_eq!(matches.len(), 1); // Only "Foo" should match
+ }
+
+ #[test]
+ fn any_capitalization_of_matches_different_cases() {
+ let pat = SequencePattern::aco("foo");
+ let doc = Document::new_plain_english_curated("foo Foo FOO");
+ let matches = pat.find_all_matches_in_doc(&doc);
+ assert_eq!(matches.len(), 3); // All three should match
+ }
}
diff --git a/harper-core/src/patterns/spelled_number_pattern.rs b/harper-core/src/patterns/spelled_number_pattern.rs
new file mode 100644
index 000000000..89117ee2c
--- /dev/null
+++ b/harper-core/src/patterns/spelled_number_pattern.rs
@@ -0,0 +1,197 @@
+use super::{EitherPattern, Pattern, SequencePattern, WhitespacePattern, WordSet};
+use crate::Token;
+
+/// Matches spelled-out numbers from one to ninety-nine
+#[derive(Default)]
+pub struct SpelledNumberPattern;
+
+impl Pattern for SpelledNumberPattern {
+ fn matches(&self, tokens: &[Token], source: &[char]) -> Option<usize> {
+ if tokens.is_empty() {
+ return None;
+ }
+
+ // The numbers that can be in the 2nd position of a compound number.
+ // A subset of the standalone numbers since we can't say "twenty zero" or "twenty eleven"
+ // "Zero" and "ten" don't belong: twenty-one ✅ twenty-zero ❌ twenty-ten ❌
+ let units = &[
+ "one", "two", "three", "four", "five", "six", "seven", "eight", "nine",
+ ];
+
+ // These can't make a compound with `tens` but they can stand alone
+ let teens = &[
+ "ten",
+ "eleven",
+ "twelve",
+ "thirteen",
+ "fourteen",
+ "fifteen",
+ "sixteen",
+ "seventeen",
+ "eighteen",
+ "nineteen",
+ ];
+
+ // These can make a compound with the part_2 standalones above.
+ // "Ten" and "hundred" don't belong: twenty-one ✅ ten-one ❌ hundred-one ❌
+ let tens = &[
+ "twenty", "thirty", "forty", "fifty", "sixty", "seventy", "eighty", "ninety",
+ ];
+
+ let single_words = WordSet::new(
+ &units
+ .iter()
+ .chain(teens.iter())
+ .chain(tens.iter())
+ .copied()
+ .chain(std::iter::once("zero"))
+ .collect::<Vec<&str>>(),
+ );
+
+ let tens_units_compounds = SequencePattern::default()
+ .then(WordSet::new(tens))
+ .then(EitherPattern::new(vec![
+ Box::new(|t: &Token, _s: &[char]| t.kind.is_hyphen()),
+ Box::new(WhitespacePattern),
+ ]))
+ .then(WordSet::new(units));
+
+ let pat = EitherPattern::new(vec![Box::new(single_words), Box::new(tens_units_compounds)]);
+
+ pat.matches(tokens, source)
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::super::DocPattern;
+ use super::SpelledNumberPattern;
+ use crate::{Document, Span};
+
+ trait SpanVecExt {
+ fn to_strings(&self, doc: &Document) -> Vec<String>;
+ }
+
+ impl SpanVecExt for Vec<Span> {
+ fn to_strings(&self, doc: &Document) -> Vec<String> {
+ self.iter()
+ .map(|sp| {
+ doc.get_tokens()[sp.start..sp.end]
+ .iter()
+ .map(|tok| doc.get_span_content_str(&tok.span))
+ .collect::<String>()
+ })
+ .collect()
+ }
+ }
+
+ #[test]
+ fn matches_single_digit() {
+ let doc = Document::new_markdown_default_curated("one two three");
+ let matches = SpelledNumberPattern.find_all_matches_in_doc(&doc);
+ assert_eq!(matches.len(), 3);
+ }
+
+ #[test]
+ fn matches_teens() {
+ let doc = Document::new_markdown_default_curated("ten eleven twelve");
+ let matches = SpelledNumberPattern.find_all_matches_in_doc(&doc);
+ assert_eq!(matches.len(), 3);
+ }
+
+ #[test]
+ fn matches_tens() {
+ let doc = Document::new_markdown_default_curated("twenty thirty forty");
+ let matches = SpelledNumberPattern.find_all_matches_in_doc(&doc);
+ assert_eq!(matches.len(), 3);
+ }
+
+ #[test]
+ fn matches_compound_numbers() {
+ let doc = Document::new_markdown_default_curated("twenty-one thirty-two");
+ let matches = SpelledNumberPattern.find_all_matches_in_doc(&doc);
+
+ // Debug output
+ println!("Found {} matches:", matches.len());
+ for m in &matches {
+ let text: String = doc.get_tokens()[m.start..m.end]
+ .iter()
+ .map(|t| doc.get_span_content_str(&t.span))
+ .collect();
+ println!("- '{}' (span: {:?})", text, m);
+ }
+
+ assert_eq!(matches.len(), 2);
+ }
+
+ #[test]
+ fn deep_thought() {
+ let doc = Document::new_markdown_default_curated(
+ "the answer to the ultimate question of life, the universe, and everything is forty-two",
+ );
+ let matches = SpelledNumberPattern.find_all_matches_in_doc(&doc);
+
+ dbg!(&matches);
+ dbg!(matches.to_strings(&doc));
+
+ assert_eq!(matches.to_strings(&doc), vec!["forty-two"]);
+ }
+
+ #[test]
+ fn jacksons() {
+ let doc = Document::new_markdown_default_curated(
+ "A, B, C It's easy as one, two, three. Or simple as Do-Re-Mi",
+ );
+ let matches = SpelledNumberPattern.find_all_matches_in_doc(&doc);
+
+ assert_eq!(matches.to_strings(&doc), vec!["one", "two", "three"]);
+ }
+
+ #[test]
+ fn orwell() {
+ let doc = Document::new_markdown_default_curated("Nineteen Eighty-Four");
+ let matches = SpelledNumberPattern.find_all_matches_in_doc(&doc);
+
+ assert_eq!(matches.to_strings(&doc), vec!["Nineteen", "Eighty-Four"]);
+ }
+
+ #[test]
+ fn get_smart() {
+ let doc = Document::new_markdown_default_curated(
+ "Maxwell Smart was Agent Eighty-Six, but who was Agent Ninety-Nine?",
+ );
+ let matches = SpelledNumberPattern.find_all_matches_in_doc(&doc);
+
+ assert_eq!(matches.to_strings(&doc), vec!["Eighty-Six", "Ninety-Nine"]);
+ }
+
+ #[test]
+ fn hyphens_or_spaces() {
+ let doc = Document::new_markdown_default_curated(
+ "twenty-one, thirty two, forty-three, fifty four, sixty-five, seventy six, eighty-seven, ninety eight",
+ );
+ let matches = SpelledNumberPattern.find_all_matches_in_doc(&doc);
+
+ assert_eq!(
+ matches.to_strings(&doc),
+ vec![
+ "twenty-one",
+ "thirty two",
+ "forty-three",
+ "fifty four",
+ "sixty-five",
+ "seventy six",
+ "eighty-seven",
+ "ninety eight",
+ ]
+ );
+ }
+
+ #[test]
+ fn waiting_since() {
+ let doc = Document::new_markdown_default_curated("I have been waiting since two hours.");
+ let matches = SpelledNumberPattern.find_all_matches_in_doc(&doc);
+
+ assert_eq!(matches.to_strings(&doc), vec!["two"]);
+ }
+}