Skip to content

Offline, privacy-first grammar checker. Fast, open-source, Rust-powered

fix(core): address recent false positives and negatives (#3090)

Elijah Potterf579328f6cbd
diff --git a/harper-core/src/linting/and_in.rs b/harper-core/src/linting/and_in.rs
deleted file mode 100644
index bc10e7cfe..000000000
--- a/harper-core/src/linting/and_in.rs
+++ /dev/null
@@ -1,93 +0,0 @@
-use crate::Token;
-use crate::expr::{Expr, SequenceExpr};
-use crate::linting::expr_linter::Chunk;
-use crate::linting::{ExprLinter, Lint, LintKind, Suggestion};
-
-pub struct AndIn {
- expr: SequenceExpr,
-}
-
-impl Default for AndIn {
- fn default() -> Self {
- Self {
- expr: SequenceExpr::fixed_phrase("an in").then_optional_hyphen(),
- }
- }
-}
-
-impl ExprLinter for AndIn {
- type Unit = Chunk;
-
- fn expr(&self) -> &dyn Expr {
- &self.expr
- }
-
- fn match_to_lint(&self, toks: &[Token], src: &[char]) -> Option<Lint> {
- if toks.len() != 3 {
- return None;
- }
-
- Some(Lint {
- span: toks[0].span,
- lint_kind: LintKind::Typo,
- message: "Did you mean `and in`?".to_string(),
- suggestions: vec![Suggestion::replace_with_match_case(
- ['a', 'n', 'd'].to_vec(),
- toks[2].get_ch(src),
- )],
- ..Default::default()
- })
- }
-
- fn description(&self) -> &str {
- "Fixes the incorrect phrase `an in` to `and in` for proper conjunction usage."
- }
-}
-
-#[cfg(test)]
-mod tests {
- use super::AndIn;
- use crate::linting::tests::{assert_no_lints, assert_suggestion_result};
-
- #[test]
- fn dont_flag_an_in_house() {
- assert_no_lints(
- "for several years as an in-house engine, used to ...",
- AndIn::default(),
- );
- }
-
- #[test]
- fn dont_flag_an_in_memory() {
- assert_no_lints(
- "including an in-memory real-time Vector Index,",
- AndIn::default(),
- );
- }
-
- #[test]
- fn dont_flag_an_in_the_moment() {
- assert_no_lints(
- "His words serve as an in-the-moment explanation for what had happened.",
- AndIn::default(),
- );
- }
-
- #[test]
- fn fix_an_in_to_and_in() {
- assert_suggestion_result(
- "This is an expensive operation, so try to only do it at startup an in tests.",
- AndIn::default(),
- "This is an expensive operation, so try to only do it at startup and in tests.",
- );
- }
-
- #[test]
- #[ignore = "This is a known false positive - `an in` can be valid in some contexts"]
- fn dont_flag_an_in_with_company() {
- assert_no_lints(
- "His parents got him an in with the company.",
- AndIn::default(),
- );
- }
-}
diff --git a/harper-core/src/linting/closed_compounds.rs b/harper-core/src/linting/closed_compounds.rs
index 2778248d4..55bccde6d 100644
--- a/harper-core/src/linting/closed_compounds.rs
+++ b/harper-core/src/linting/closed_compounds.rs
@@ -44,6 +44,7 @@ pub fn lint_group() -> LintGroup {
"Keystrokes" => (&["key stokes", "key strokes"][..], "keystrokes"),
"Laptop" => (&["lap top"][..], "laptop"),
"Middleware" => (&["middle ware"][..], "middleware"),
+ "Meanwhile" => (&["mean while"][..], "meanwhile"),
"Misunderstand" => (&["miss understand"][..], "misunderstand"),
"Misunderstood" => (&["miss understood"][..], "misunderstood"),
"Misuse" => (&["miss use"][..], "misuse"),
@@ -135,6 +136,13 @@ mod tests {
assert_suggestion_result(test_sentence, lint_group(), expected);
}
+ #[test]
+ fn mean_while() {
+ let test_sentence = "Mean while, the team kept working.";
+ let expected = "Meanwhile, the team kept working.";
+ assert_suggestion_result(test_sentence, lint_group(), expected);
+ }
+
#[test]
fn world_wide() {
let test_sentence = "The world wide impact was significant.";
diff --git a/harper-core/src/linting/how_to.rs b/harper-core/src/linting/how_to.rs
index 73b9cb2bd..082929136 100644
--- a/harper-core/src/linting/how_to.rs
+++ b/harper-core/src/linting/how_to.rs
@@ -2,10 +2,10 @@ use harper_brill::UPOS;
use crate::linting::expr_linter::Chunk;
use crate::{
- Token, TokenKind, TokenStringExt,
+ Token, TokenKind,
expr::{All, Expr, OwnedExprExt, SequenceExpr},
linting::{ExprLinter, Lint, LintKind, Suggestion},
- patterns::{InflectionOfBe, UPOSSet},
+ patterns::{InflectionOfBe, NominalPhrase, UPOSSet},
};
pub struct HowTo {
@@ -23,20 +23,57 @@ impl Default for HowTo {
.then_verb_lemma();
pattern.add(pos_pattern);
+ let finite_clause_verb = SequenceExpr::whitespace().then_kind_any(&[
+ TokenKind::is_auxiliary_verb,
+ TokenKind::is_verb_third_person_singular_present_form,
+ TokenKind::is_verb_simple_past_form,
+ ] as &[_]);
+
+ let noun_led_clause = SequenceExpr::default()
+ .then_kind_either(TokenKind::is_nominal, TokenKind::is_proper_noun)
+ .then_any_of(vec![
+ Box::new(finite_clause_verb),
+ Box::new(
+ SequenceExpr::whitespace()
+ .then_kind_any(&[
+ TokenKind::is_noun,
+ TokenKind::is_proper_noun,
+ TokenKind::is_verb_progressive_form,
+ ] as &[_])
+ .then_seq(SequenceExpr::whitespace().then(
+ InflectionOfBe::new().or(SequenceExpr::default().then_auxiliary_verb()),
+ )),
+ ),
+ Box::new(
+ SequenceExpr::whitespace()
+ .t_aco("to")
+ .then_whitespace()
+ .then(NominalPhrase)
+ .then_seq(SequenceExpr::whitespace().then_kind_any(&[
+ TokenKind::is_auxiliary_verb,
+ TokenKind::is_verb_third_person_singular_present_form,
+ TokenKind::is_verb_simple_past_form,
+ ]
+ as &[_])),
+ ),
+ ]);
+
let exceptions = SequenceExpr::unless(UPOSSet::new(&[UPOS::PART]))
.then_anything()
.then_unless(|tok: &Token, _: &[char]| tok.kind.is_np_member())
.then_anything()
.then_unless(
- InflectionOfBe::new().or(SequenceExpr::default().then_kind_any_or_words(
- &[
- TokenKind::is_auxiliary_verb,
- TokenKind::is_adjective,
- TokenKind::is_conjunction,
- TokenKind::is_proper_noun,
- ] as &[_],
- &["did", "come", "does"],
- )),
+ InflectionOfBe::new()
+ .or(SequenceExpr::default().then_kind_any_or_words(
+ &[
+ TokenKind::is_auxiliary_verb,
+ TokenKind::is_adjective,
+ TokenKind::is_conjunction,
+ TokenKind::is_proper_noun,
+ ] as &[_],
+ &["did", "come", "does"],
+ ))
+ .or(noun_led_clause),
);
pattern.add(exceptions);
@@ -53,11 +90,10 @@ impl ExprLinter for HowTo {
}
fn match_to_lint(&self, toks: &[Token], _src: &[char]) -> Option<Lint> {
- let span = toks[2..4].span()?;
- let fix: Vec<char> = "to ".chars().collect();
+ let fix: Vec<char> = " to".chars().collect();
Some(Lint {
- span,
+ span: toks[2].span,
lint_kind: LintKind::WordChoice,
suggestions: vec![Suggestion::InsertAfter(fix)],
message: "Insert `to` after `how` (e.g., `how to clone`).".into(),
@@ -296,6 +332,35 @@ mod tests {
);
}
+ #[test]
+ fn dont_flag_how_proper_noun_handles() {
+ assert_no_lints("rewrites how Wine handles file locking", HowTo::default());
+ }
+
+ #[test]
+ fn dont_flag_how_work_is_structured() {
+ assert_no_lints(
+ "They need to rethink how work is structured and valued.",
+ HowTo::default(),
+ );
+ }
+
+ #[test]
+ fn dont_flag_how_form_submissions_are_delivered() {
+ assert_no_lints(
+ "The Mail tab dictates how form submissions are delivered to you.",
+ HowTo::default(),
+ );
+ }
+
+ #[test]
+ fn dont_flag_how_access_to_food_shaped_revolutions() {
+ assert_no_lints(
+ "We analyze how access to food shaped political ideologies.",
+ HowTo::default(),
+ );
+ }
+
#[test]
fn issue_2124() {
assert_no_lints(
diff --git a/harper-core/src/linting/lint_group.rs b/harper-core/src/linting/lint_group.rs
index 3c05c12cf..896caa968 100644
--- a/harper-core/src/linting/lint_group.rs
+++ b/harper-core/src/linting/lint_group.rs
@@ -23,7 +23,6 @@ use super::allow_to::AllowTo;
use super::am_in_the_morning::AmInTheMorning;
use super::amounts_for::AmountsFor;
use super::an_a::AnA;
-use super::and_in::AndIn;
use super::and_the_like::AndTheLike;
use super::another_thing_coming::AnotherThingComing;
use super::another_think_coming::AnotherThinkComing;
@@ -614,7 +613,6 @@ impl LintGroup {
insert_expr_rule!(AmInTheMorning, true);
insert_expr_rule!(AmountsFor, true);
insert_struct_rule_with_dialect!(AnA, true);
- insert_expr_rule!(AndIn, true);
insert_expr_rule!(AndTheLike, true);
insert_expr_rule!(AnotherThingComing, true);
insert_expr_rule!(AnotherThinkComing, false);
diff --git a/harper-core/src/linting/mod.rs b/harper-core/src/linting/mod.rs
index d496d14b2..293ae0629 100644
--- a/harper-core/src/linting/mod.rs
+++ b/harper-core/src/linting/mod.rs
@@ -14,7 +14,6 @@ mod allow_to;
mod am_in_the_morning;
mod amounts_for;
mod an_a;
-mod and_in;
mod and_the_like;
mod another_thing_coming;
mod another_think_coming;
diff --git a/harper-core/src/linting/weir_rules/AndIn.weir b/harper-core/src/linting/weir_rules/AndIn.weir
new file mode 100644
index 000000000..957eadfca
--- /dev/null
+++ b/harper-core/src/linting/weir_rules/AndIn.weir
@@ -0,0 +1,27 @@
+expr relPronouns [which, that, who, whom, whose]
+expr main <((an in) ![-, with, (NOUN @relPronouns), (PROPN @relPronouns)]), an>
+
+let message "Did you mean `and in`?"
+let description "Fixes the typo `an in` when it stands in for the conjunction `and in`, while avoiding common `in-...` noun phrases and a few attested exceptions."
+let kind "Typo"
+let becomes "and"
+let strategy "MatchCase"
+
+test "This is an expensive operation, so try to only do it at startup an in tests." "This is an expensive operation, so try to only do it at startup and in tests."
+test "We compare the results locally an in CI." "We compare the results locally and in CI."
+test "The feature works in Chrome an in Firefox." "The feature works in Chrome and in Firefox."
+test "She was quiet at home an in public." "She was quiet at home and in public."
+test "The package is available on Linux an in macOS." "The package is available on Linux and in macOS."
+test "Please validate it in staging an in production." "Please validate it in staging and in production."
+test "They publish updates weekly an in emergencies." "They publish updates weekly and in emergencies."
+test "AN IN tests, this should still be fixed." "AND IN tests, this should still be fixed."
+
+allows "for several years as an in-house engine, used to ..."
+allows "including an in-memory real-time Vector Index,"
+allows "His words serve as an in-the-moment explanation for what had happened."
+allows "His parents got him an in with the company."
+allows "Fascism requires an in group which the law protects."
+allows "The article describes an in group that outsiders rarely notice."
+allows "They studied an in group who controlled admissions."
+allows "The reform strengthened an in group whose leaders shaped policy."
+allows "The board met and in doing so approved the budget."
diff --git a/harper-core/src/linting/weir_rules/EnMasse.weir b/harper-core/src/linting/weir_rules/EnMasse.weir
index ae58c90d4..7d0167c19 100644
--- a/harper-core/src/linting/weir_rules/EnMasse.weir
+++ b/harper-core/src/linting/weir_rules/EnMasse.weir
@@ -1,8 +1,12 @@
-expr main [(on mass), (on masse), (in mass)]
+expr main [<((in mass) !production), (in mass)>, (on mass), (on masse)]
let message "Did you mean `en masse`?"
let description "Detects variants like `on mass` or `in mass` and suggests `en masse`."
let kind "Eggcorn"
let becomes "en masse"
-test "in mass" "en masse"
+test "They moved in mass yesterday." "They moved en masse yesterday."
+test "on mass" "en masse"
+test "on masse" "en masse"
+test "It's in mass production now." "It's in mass production now."
+test "The product entered mass production last year." "The product entered mass production last year."
diff --git a/harper-core/src/linting/weir_rules/Yourself.weir b/harper-core/src/linting/weir_rules/Yourself.weir
new file mode 100644
index 000000000..5bd30f434
--- /dev/null
+++ b/harper-core/src/linting/weir_rules/Yourself.weir
@@ -0,0 +1,25 @@
+expr main <((your self) !-), (your self)>
+
+let message "Did you mean the closed compound `yourself`?"
+let description "Looks for incorrect spacing inside the closed compound `yourself`, while skipping hyphenated `self-...` compounds."
+let kind "Miscellaneous"
+let becomes "yourself"
+let strategy "MatchCase"
+
+test "You can look up the definition your self on Google." "You can look up the definition yourself on Google."
+test "Be honest with your self before the interview." "Be honest with yourself before the interview."
+test "Remind your self to submit the form." "Remind yourself to submit the form."
+test "Ask your self whether the premise holds." "Ask yourself whether the premise holds."
+test "YOUR SELF deserves a break." "YOURSELF deserves a break."
+test "Please pace your self during the first mile." "Please pace yourself during the first mile."
+test "Trust your self a little more here." "Trust yourself a little more here."
+test "You should forgive your self after that mistake." "You should forgive yourself after that mistake."
+
+allows "It's natural to question your self-worth after feedback like that."
+allows "We can reframe your self-care as part of recovery."
+allows "You may notice your self-doubt getting louder."
+allows "The workbook tracks your self-esteem over time."
+allows "A coach can help with your self-image."
+allows "Yourself is the right spelling there."
+allows "Keep your self - worth separate from the result."
+allows "They focused on your self - care plan first."
diff --git a/harper-core/tests/text/linters/Difficult sentences.snap.yml b/harper-core/tests/text/linters/Difficult sentences.snap.yml
index 187a7c390..57da14a59 100644
--- a/harper-core/tests/text/linters/Difficult sentences.snap.yml
+++ b/harper-core/tests/text/linters/Difficult sentences.snap.yml
@@ -106,15 +106,6 @@ Suggest:
-Lint: Typo (127 priority)
-Message: |
- 243 | His parents got him an in with the company.
- | ^~ Did you mean `and in`?
-Suggest:
- - Replace with: “and”
-
-
-
Lint: Spelling (63 priority)
Message: |
249 | The bullet is about five centimetres in.