Skip to content

Offline, privacy-first grammar checker. Fast, open-source, Rust-powered

refactor: reduce false positive blacklist with new logic (#1282)

Andrew Dunbarec475af7ea1d
diff --git a/harper-core/src/linting/adjective_of_a.rs b/harper-core/src/linting/adjective_of_a.rs
index 6458a8446..00063bc2a 100644
--- a/harper-core/src/linting/adjective_of_a.rs
+++ b/harper-core/src/linting/adjective_of_a.rs
@@ -5,64 +5,52 @@ use crate::{Document, Span, TokenStringExt};
#[derive(Debug, Clone, Copy, Default)]
pub struct AdjectiveOfA;
-const FALSE_POSITIVES: &[&str] = &[
- // Different valid constructions.
- "all",
- "emblematic",
- "equivalent",
- "full",
- "fun",
- "illustrative",
- "inside",
- "more",
- "much",
- "off",
- "out",
- "shy",
- "up",
- // The word is used more as a noun in this context.
- // (using .kind.is_likely_homograph() here is too strict)
- "back",
- "bastard",
- "bit",
- "borderline",
- "bottom",
- "chance",
- "clockwork",
- "derivative",
- "dream",
- "eighth",
- "front",
- "half",
- "head",
- "kind",
- "left",
- "light",
- "meaning",
- "middle",
- "multiple",
- "one",
- "part",
- "perspective",
- "potential",
- "precision",
- "red",
- // for "rid" I removed the `5` flag in `dictionary.dict``
- "shadow",
- "side",
- "short",
- "slack",
- "something",
- "sound",
- "top",
- "waste",
+const ADJECTIVE_WHITELIST: &[&str] = &["bad", "big", "good", "large", "long", "vague"];
+
+const CONTEXT_WORDS: &[&str] = &[
+ "as", "how", // but "how much of a"
+ "that", "this", "too",
];
-fn is_false_positive(chars: &[char]) -> bool {
- let word = chars.iter().collect::<String>();
- FALSE_POSITIVES
+const ADJECTIVE_BLACKLIST: &[&str] = &["much", "part"];
+
+fn has_context_word(document: &Document, adj_idx: usize) -> bool {
+ if adj_idx < 2 {
+ // Need at least 2 tokens before the adjective (word + space)
+ return false;
+ }
+
+ // Get the token before the adjective (should be a space)
+ if let Some(space_token) = document.get_token(adj_idx - 1) {
+ if !space_token.kind.is_whitespace() {
+ return false;
+ }
+
+ // Get the token before the space (should be our context word)
+ if let Some(word_token) = document.get_token(adj_idx - 2) {
+ if !word_token.kind.is_word() {
+ return false;
+ }
+
+ let word = document.get_span_content_str(&word_token.span);
+
+ return CONTEXT_WORDS.iter().any(|&w| w.eq_ignore_ascii_case(&word));
+ }
+ }
+
+ false
+}
+
+fn is_good_adjective(word: &str) -> bool {
+ ADJECTIVE_WHITELIST
+ .iter()
+ .any(|&adj| word.eq_ignore_ascii_case(adj))
+}
+
+fn is_bad_adjective(word: &str) -> bool {
+ ADJECTIVE_BLACKLIST
.iter()
- .any(|&false_pos| word.eq_ignore_ascii_case(false_pos))
+ .any(|&adj| word.eq_ignore_ascii_case(adj))
}
impl Linter for AdjectiveOfA {
@@ -75,10 +63,17 @@ impl Linter for AdjectiveOfA {
let word_of = document.get_token(i + 2);
let space_2 = document.get_token(i + 3);
let a_or_an = document.get_token(i + 4);
- let adj_chars: &[char] = document.get_span_content(&adjective.span);
+ let adj_str = document
+ .get_span_content_str(&adjective.span)
+ .to_lowercase();
- // Rule out false positives
- if is_false_positive(adj_chars) {
+ // Only flag adjectives known to use this construction
+ // Unless we have a clearer context
+ if !is_good_adjective(&adj_str) && !has_context_word(document, i) {
+ continue;
+ }
+ // Some adjectives still create false positives even with the extra context
+ if is_bad_adjective(&adj_str) {
continue;
}
@@ -93,15 +88,13 @@ impl Linter for AdjectiveOfA {
// "see if you can give us a little better of an answer"
// "hopefully it won't be too much worse of a problem"
// "seems far worse of a result to me"
- if adj_chars.ends_with(&['e', 'r']) || adj_chars.ends_with(&['s', 't']) {
+ if adj_str.ends_with("er") || adj_str.ends_with("st") {
continue;
}
// Rule out present participles (e.g. "beginning of a")
// The -ing form of a verb acts as an adjective called a present participle
// and also acts as a noun called a gerund.
- if adj_chars.ends_with(&['i', 'n', 'g'])
- && (adjective.kind.is_noun() || adjective.kind.is_verb())
- {
+ if adj_str.ends_with("ing") && (adjective.kind.is_noun() || adjective.kind.is_verb()) {
continue;
}
@@ -116,8 +109,8 @@ impl Linter for AdjectiveOfA {
if !word_of.kind.is_word() {
continue;
}
- let word_of = document.get_span_content(&word_of.span);
- if word_of != ['o', 'f'] {
+ let word_of = document.get_span_content_str(&word_of.span).to_lowercase();
+ if word_of != "of" {
continue;
}
let space_2 = space_2.unwrap();
@@ -128,8 +121,8 @@ impl Linter for AdjectiveOfA {
if !a_or_an.kind.is_word() {
continue;
}
- let a_or_an_chars = document.get_span_content(&a_or_an.span);
- if a_or_an_chars != ['a'] && a_or_an_chars != ['a', 'n'] {
+ let a_or_an_str = document.get_span_content_str(&a_or_an.span).to_lowercase();
+ if a_or_an_str != "a" && a_or_an_str != "an" {
continue;
}
@@ -464,15 +457,6 @@ mod tests {
);
}
- #[test]
- fn dont_flag_clockwork() {
- assert_lint_count(
- "so something's wrong in this clockwork of a thing and I'm not going to bother taking this apart",
- AdjectiveOfA,
- 0,
- );
- }
-
#[test]
fn dont_flag_up() {
assert_lint_count(
@@ -579,4 +563,112 @@ mod tests {
0,
);
}
+
+ #[test]
+ fn correct_too_large_of_a() {
+ assert_suggestion_result(
+ "Warn users if setting too large of a session object",
+ AdjectiveOfA,
+ "Warn users if setting too large a session object",
+ )
+ }
+
+ #[test]
+ fn correct_too_long_of_a() {
+ assert_suggestion_result(
+ "An Org Role with Too Long of a Name Hides Delete Option",
+ AdjectiveOfA,
+ "An Org Role with Too Long a Name Hides Delete Option",
+ )
+ }
+
+ #[test]
+ fn correct_too_big_of_a() {
+ assert_suggestion_result(
+ "StepButton has too big of a space to click",
+ AdjectiveOfA,
+ "StepButton has too big a space to click",
+ )
+ }
+
+ #[test]
+ fn correct_too_vague_of_a() {
+ assert_suggestion_result(
+ "\"No Speech provider is registered.\" is too vague of an error",
+ AdjectiveOfA,
+ "\"No Speech provider is registered.\" is too vague an error",
+ )
+ }
+
+ #[test]
+ fn correct_too_dumb_of_a() {
+ assert_suggestion_result(
+ "Hopefully this isn't too dumb of a question.",
+ AdjectiveOfA,
+ "Hopefully this isn't too dumb a question.",
+ )
+ }
+
+ #[test]
+ fn correct_how_important_of_a() {
+ assert_suggestion_result(
+ "This should tell us how important of a use case that is and how often writing a type literal in a case is deliberate.",
+ AdjectiveOfA,
+ "This should tell us how important a use case that is and how often writing a type literal in a case is deliberate.",
+ )
+ }
+
+ #[test]
+ fn correct_that_rare_of_an() {
+ assert_suggestion_result(
+ "so making changes isn't that rare of an occurrence for me.",
+ AdjectiveOfA,
+ "so making changes isn't that rare an occurrence for me.",
+ )
+ }
+
+ #[test]
+ fn correct_as_important_of_a() {
+ assert_suggestion_result(
+ "Might be nice to have it draggable from other places as well, but not as important of a bug anymore.",
+ AdjectiveOfA,
+ "Might be nice to have it draggable from other places as well, but not as important a bug anymore.",
+ )
+ }
+
+ #[test]
+ fn correct_too_short_of_a() {
+ assert_suggestion_result(
+ "I login infrequently as well and 6 months is too short of a time.",
+ AdjectiveOfA,
+ "I login infrequently as well and 6 months is too short a time.",
+ )
+ }
+
+ #[test]
+ fn correct_that_common_of_a() {
+ assert_suggestion_result(
+ "that common of a name for a cluster role its hard to rule out",
+ AdjectiveOfA,
+ "that common a name for a cluster role its hard to rule out",
+ )
+ }
+
+ #[test]
+ fn correct_as_great_of_an() {
+ assert_suggestion_result(
+ "the w factor into the u factor to as great of an extent as possible.",
+ AdjectiveOfA,
+ "the w factor into the u factor to as great an extent as possible.",
+ )
+ }
+
+ #[test]
+ fn correct_too_uncommon_of_a() {
+ assert_suggestion_result(
+ "but this is probably too uncommon of a practice to be the default",
+ AdjectiveOfA,
+ "but this is probably too uncommon a practice to be the default",
+ )
+ }
}
diff --git a/harper-core/tests/text/linters/Alice's Adventures in Wonderland.snap.yml b/harper-core/tests/text/linters/Alice's Adventures in Wonderland.snap.yml
index a766feb0e..2bcca03ba 100644
--- a/harper-core/tests/text/linters/Alice's Adventures in Wonderland.snap.yml
+++ b/harper-core/tests/text/linters/Alice's Adventures in Wonderland.snap.yml
@@ -338,16 +338,6 @@ Message: |
-Lint: Style (63 priority)
-Message: |
- 149 | altogether, like a candle. I wonder what I should be like then?” And she tried
- 150 | to fancy what the flame of a candle is like after the candle is blown out, for
- | ^~~~~~~~~~ The word `of` is not needed here.
-Suggest:
- - Replace with: “flame a”
-
-
-
Lint: Capitalization (31 priority)
Message: |
154 | garden at once; but, alas for poor Alice! when she got to the door, she found
diff --git a/harper-core/tests/text/linters/Computer science.snap.yml b/harper-core/tests/text/linters/Computer science.snap.yml
index d44a0e010..612986e5b 100644
--- a/harper-core/tests/text/linters/Computer science.snap.yml
+++ b/harper-core/tests/text/linters/Computer science.snap.yml
@@ -243,16 +243,6 @@ Suggest:
-Lint: Style (63 priority)
-Message: |
- 108 | management science, the subject is applied and interdisciplinary in nature,
- 109 | while having the characteristics typical of an academic discipline. His efforts,
- | ^~~~~~~~~~~~~ The word `of` is not needed here.
-Suggest:
- - Replace with: “typical an”
-
-
-
Lint: Spelling (63 priority)
Message: |
110 | and those of others such as numerical analyst George Forsythe, were rewarded:
diff --git a/harper-core/tests/text/linters/Difficult sentences.snap.yml b/harper-core/tests/text/linters/Difficult sentences.snap.yml
index ab752976f..c37f5ea7e 100644
--- a/harper-core/tests/text/linters/Difficult sentences.snap.yml
+++ b/harper-core/tests/text/linters/Difficult sentences.snap.yml
@@ -131,15 +131,6 @@ Suggest:
-Lint: Style (63 priority)
-Message: |
- 277 | It is said that she died of a broken heart.
- | ^~~~~~~~~ The word `of` is not needed here.
-Suggest:
- - Replace with: “died a”
-
-
-
Lint: Spelling (63 priority)
Message: |
280 | Welcome to the historic town of Harwich.
diff --git a/harper-core/tests/text/linters/Part-of-speech tagging.snap.yml b/harper-core/tests/text/linters/Part-of-speech tagging.snap.yml
index df821e99f..9ffa220ef 100644
--- a/harper-core/tests/text/linters/Part-of-speech tagging.snap.yml
+++ b/harper-core/tests/text/linters/Part-of-speech tagging.snap.yml
@@ -232,16 +232,6 @@ Suggest:
-Lint: Style (63 priority)
-Message: |
- 94 | many years. A first approximation was done with a program by Greene and Rubin,
- 95 | which consisted of a huge handmade list of what categories could co-occur at
- | ^~~~~~~~~~~~~~ The word `of` is not needed here.
-Suggest:
- - Replace with: “consisted a”
-
-
-
Lint: Spelling (63 priority)
Message: |
98 | and corrected by hand, and later users sent in errata so that by the late 70s
diff --git a/harper-core/tests/text/linters/The Great Gatsby.snap.yml b/harper-core/tests/text/linters/The Great Gatsby.snap.yml
index a07d3ff00..c252cb2f7 100644
--- a/harper-core/tests/text/linters/The Great Gatsby.snap.yml
+++ b/harper-core/tests/text/linters/The Great Gatsby.snap.yml
@@ -573,16 +573,6 @@ Suggest:
-Lint: Style (63 priority)
-Message: |
- 345 | do it. That’s what I get for marrying a brute of a man, a great, big, hulking
- | ^~~~~~~~~~ The word `of` is not needed here.
- 346 | physical specimen of a—”’
-Suggest:
- - Replace with: “brute a”
-
-
-
Lint: Spelling (63 priority)
Message: |
352 | Sometimes she and Miss Baker talked at once, unobtrusively and with a bantering
@@ -1018,6 +1008,16 @@ Message: |
+Lint: Style (63 priority)
+Message: |
+ 703 | dust-covered wreck of a Ford which crouched in a dim corner. It had occurred to
+ 704 | me that this shadow of a garage must be a blind, and that sumptuous and romantic
+ | ^~~~~~~~~~~ The word `of` is not needed here.
+Suggest:
+ - Replace with: “shadow a”
+
+
+
Lint: Spelling (63 priority)
Message: |
706 | door of an office, wiping his hands on a piece of waste. He was a blond,
@@ -4408,19 +4408,6 @@ Suggest:
-Lint: Style (63 priority)
-Message: |
- 3263 | “Perhaps you know that lady,” Gatsby indicated a gorgeous, scarcely human orchid
- | ^~~~~~~
- 3264 | of a woman who sat in state under a white-plum tree. Tom and Daisy stared, with
- | ~~~~ The word `of` is not needed here.
-Suggest:
- - Replace with: “orchid
-a”
- - Replace with: “orchid a”
-
-
-
Lint: Formatting (63 priority)
Message: |
3274 | “Mrs. Buchanan . . . and Mr. Buchanan—” After an instant’s hesitation he added:
@@ -4968,15 +4955,6 @@ Suggest:
-Lint: Style (63 priority)
-Message: |
- 3923 | The coupé flashed by us with a flurry of dust and the flash of a waving hand.
- | ^~~~~~~~~~ The word `of` is not needed here.
-Suggest:
- - Replace with: “flash a”
-
-
-
Lint: Readability (127 priority)
Message: |
3937 | world, and the shock had made him physically sick. I stared at him and then at
@@ -5425,15 +5403,6 @@ Suggest:
-Lint: Style (63 priority)
-Message: |
- 4375 | I was thirty. Before me stretched the portentous, menacing road of a new decade.
- | ^~~~~~~~~ The word `of` is not needed here.
-Suggest:
- - Replace with: “road a”
-
-
-
Lint: Spelling (63 priority)
Message: |
4377 | It was seven o’clock when we got into the coupé with him and started for Long
@@ -5567,15 +5536,6 @@ Suggest:
-Lint: Style (63 priority)
-Message: |
- 4450 | I became aware now of a hollow, wailing sound which issued incessantly from the
- | ^~~~~~~~ The word `of` is not needed here.
-Suggest:
- - Replace with: “now a”
-
-
-
Lint: Spelling (63 priority)
Message: |
4451 | garage, a sound which as we got out of the coupé and walked toward the door
@@ -7180,16 +7140,6 @@ Message: |
-Lint: Style (63 priority)
-Message: |
- 5713 | illustration, her chin raised a little jauntily, her hair the color of an autumn
- | ^~~~~~~~~~~ The word `of` is not needed here.
- 5714 | leaf, her face the same brown tint as the fingerless glove on her knee. When I
-Suggest:
- - Replace with: “color an”
-
-
-
Lint: Spelling (63 priority)
Message: |
5714 | leaf, her face the same brown tint as the fingerless glove on her knee. When I