Skip to content

Offline, privacy-first grammar checker. Fast, open-source, Rust-powered

fix(core): major issues with `SplitWords` (#3857)

Elijah Potter0724f5901ffa
diff --git a/harper-core/dictionary.dict b/harper-core/dictionary.dict
index 2bf42ca14..7d0c54116 100644
--- a/harper-core/dictionary.dict
+++ b/harper-core/dictionary.dict
@@ -13788,7 +13788,7 @@ azalea/NSg
azimuth/~N0g
azimuths/N9
azure/~NSgJV
-b/~J^VdK
+b/J^VdK
baa/~NSgVdG
babble/VGd>SNmgZ
babbler/~Ng
@@ -15950,7 +15950,7 @@ brute force/NwgSVdG
brutish/JYp
brutishness/Nmg
brutus/
-bu/~N
+bu/N
bub/NSgV
bubble/~NSgVdG>
bubble tea/NwgS
@@ -16267,7 +16267,7 @@ bytecode/NwgS
byway/~NSg
byword/NSg
byzantine/~JN
-c/~NSViE
+c/NSViE
ca/P
cab/~NSgV>Z
cab sav/Nmg†_ # Australia slang = Cabernet Sauvignon
@@ -21216,6 +21216,7 @@ desert/~NwSgJ>VdGZ
deserter/NgS
desertification/Nmg
desertion/~NwSg
+deserve/~V>SZ
deserved/~JYVU
deserving/~JNVU
desi/NgS₹
@@ -21680,7 +21681,7 @@ dirtball/NS
dirtily/Ry
dirtiness/Nmg
dirty/~J^>pVdSG
-dis/~VNgI(
+dis/VNgI(
disable/~VdSGJL
disablement/Ng
disambiguate/~VSdGn
@@ -22711,7 +22712,7 @@ dystopi
dystopia/NwgS
dystopian/~J
dz/NV
-e/~NSId^W
+e/NSId^W
e-book/NgS
e-commerce/Nmg
e'en/
@@ -22720,7 +22721,6 @@ e.g./~N
eBay/OgV
eMusic/g
eSIM/NgS
-ea/~N
each/~Dq
each other/I
eager/~J^Y>pVN
@@ -22853,7 +22853,7 @@ ecumenical/~JY
ecumenicism/Ng
ecumenism/Nmg
eczema/Nwg
-ed/~NSgre
+ed/NSgre
edamame/N
eddy/~NSgVdG
edelweiss/Ng
@@ -24916,7 +24916,6 @@ fewness/Ng
fey/~JN
fez/~Ng
fezzes/N
-ff/~V
fiance/NgVe
fiancee/NgS
fiances/N
@@ -26346,7 +26345,7 @@ fuzziness/Ng
fuzzy/~J^>pNS
fwd/~JNV
fwy/N
-g/~NSnXvB
+g/NSnXvB
gab/~NSgV
gabardine/NSg
gabbed/VtT
@@ -27764,7 +27763,7 @@ gyrofrequency/NgS
gyroscope/NgS
gyroscopic/J
gyve/NgSVGd
-h/~NJGnXZvz # removed `>` 'her' is not comparative of 'h'
+h/NJGnXZvz # removed `>` 'her' is not comparative of 'h'
h'm/
ha/~
hah/~
@@ -30206,7 +30205,7 @@ inexorability/N
inexorable/J
inexorably/Ry
inexpedient/J
-inexperience/Nmg
+inexperience/~Nmgd
inexpert/JYN
inexpiable/J
inexplicably/~Ry
@@ -31076,7 +31075,7 @@ iv/~U
ivory/~NwSgJ
ivy/~NSgd
ix/~
-j/~NW
+j/NW
jab/NSgV
jabbed/VtTJ
jabber/Vd>GSNgZ
@@ -37947,7 +37946,7 @@ oxymoronic/J
oyster/~NSgJV
oz/~N
ozone/~NgV
-p/~PNV>G^nXz
+p/PNV>G^nXz
p.m./
pH/
pa/~NSgH
@@ -41443,7 +41442,7 @@ pyruvate/~N
python/~NSg
pyx/NgSV
pzazz/N
-q/~N
+q/N
qr/~N
qt/~NS
qty/N
@@ -46930,7 +46929,7 @@ squish/NgSVGd
squishy/J>^N
sriracha/N
ssh/~VO
-st/~N
+st/N
stab/~NgSVJY
stabbed/~VtT
stabber/NgS
@@ -49611,7 +49610,7 @@ tinder/NgV
tinderbox/NgS
tine/NgSJV
tinfoil/NmgV
-ting/~NgVdGY
+ting/NgVdGY
tinge/NSgV
tingeing/V
tingle/VdGSNgz
@@ -53566,7 +53565,7 @@ wurst/NSg
wuss/NgSV
wussy/J>^NSg
wyvern/NSg
-x/~ # removed `5` and `7` adjective and `conjunction`
+x/ # removed `5` and `7` adjective and `conjunction`
x-axis/Ng
x64/Ng
x86/~Ng
@@ -54717,3 +54716,4 @@ vicontiel/J # As above, alternative version
viscomital/J # As above, alternative version
WordCamp/Og
worldbuild/V>G
+playthrough/gNS
diff --git a/harper-core/src/linting/lint_group/mod.rs b/harper-core/src/linting/lint_group/mod.rs
index f5a614aca..90f8abbd6 100644
--- a/harper-core/src/linting/lint_group/mod.rs
+++ b/harper-core/src/linting/lint_group/mod.rs
@@ -1191,6 +1191,52 @@ mod tests {
);
}
+ #[test]
+ fn fix_stoped() {
+ assert_suggestion_result("He stoped to think.", test_group(), "He stopped to think.");
+ }
+
+ #[test]
+ fn allow_playthrough() {
+ assert_no_lints("I recently did a playthrough.", test_group());
+ }
+
+ #[test]
+ fn fixes_politicans() {
+ assert_suggestion_result(
+ "I guess politicans like to complain.",
+ test_group(),
+ "I guess politicans like to complain.",
+ );
+ }
+
+ #[test]
+ fn fixes_benefitting() {
+ assert_suggestion_result(
+ "Who is really benefitting from it?",
+ test_group(),
+ "Who is really benefiting from it?",
+ );
+ }
+
+ #[test]
+ fn fixes_easir() {
+ assert_suggestion_result(
+ "It makes it easir to select it.",
+ test_group(),
+ "It makes it easier to select it.",
+ );
+ }
+
+ #[test]
+ fn fixes_buget() {
+ assert_suggestion_result(
+ "It was in their buget range.",
+ test_group(),
+ "It was in their budget range.",
+ );
+ }
+
/// Tests that no linters' descriptions contain errors handled by other linters.
///
/// This test verifies that the description of each linter (which is written in natural language)
diff --git a/harper-core/src/linting/split_words.rs b/harper-core/src/linting/split_words.rs
index 98d73ba62..ade0b1794 100644
--- a/harper-core/src/linting/split_words.rs
+++ b/harper-core/src/linting/split_words.rs
@@ -208,10 +208,14 @@ fn should_defer_to_spellcheck(
#[cfg(test)]
mod tests {
+ use itertools::Itertools;
+
+ use crate::Document;
use crate::linting::tests::{
assert_good_and_bad_suggestions, assert_lint_message, assert_no_lints,
assert_suggestion_result,
};
+ use crate::linting::{Linter, Suggestion};
use super::SplitWords;
@@ -365,4 +369,51 @@ mod tests {
assert_no_lints(source, SplitWords::default());
}
}
+
+ #[test]
+ fn does_not_split_iff() {
+ assert_no_lints("iff", SplitWords::default());
+ }
+
+ /// Checks for a condition where the SplitWords rule would correct a word to be composed of a
+ /// single letter (which is not a word), followed by a valid word.
+ ///
+ /// For example, `comitted` -> `c omitted`.
+ #[test]
+ fn never_corrects_to_invalid_single_letter_words() {
+ let triggers = [
+ "comitted", "testc", "testh", "testb", "testq", "testx", "testg", "teste", "testj",
+ "shes",
+ ];
+ let relevant_letters = ['c', 'd', 't', 'h', 'b', 'x', 'e', 'j', 's'];
+
+ for trigger in triggers {
+ let mut rule = SplitWords::default();
+
+ let doc = Document::new_plain_english_curated(trigger);
+ let lints = rule.lint(&doc);
+
+ for lint in lints {
+ dbg!(&lint);
+
+ for sug in lint.suggestions {
+ match sug {
+ Suggestion::ReplaceWith(items) => {
+ // Words created by the rule.
+ let created_words = items.split(|c| c == &' ');
+
+ for word in created_words {
+ if word.len() == 1
+ && relevant_letters.iter().contains(&word.first().unwrap())
+ {
+ panic!("Encountered bad output {word:?}")
+ }
+ }
+ }
+ _ => (),
+ }
+ }
+ }
+ }
+ }
}
diff --git a/harper-core/src/spell/mutable_dictionary.rs b/harper-core/src/spell/mutable_dictionary.rs
index 03e6abaec..bbbaa1e3a 100644
--- a/harper-core/src/spell/mutable_dictionary.rs
+++ b/harper-core/src/spell/mutable_dictionary.rs
@@ -406,6 +406,11 @@ mod tests {
)
}
+ #[test]
+ fn contains_inexperienced() {
+ assert!(MutableDictionary::curated().contains_word_str("inexperienced"))
+ }
+
#[test]
fn has_is_not_a_nominal() {
let dict = MutableDictionary::curated();
diff --git a/harper-core/tests/text/linters/Part-of-speech tagging.snap.yml b/harper-core/tests/text/linters/Part-of-speech tagging.snap.yml
index b7610cf8a..3a3b4f31a 100644
--- a/harper-core/tests/text/linters/Part-of-speech tagging.snap.yml
+++ b/harper-core/tests/text/linters/Part-of-speech tagging.snap.yml
@@ -47,9 +47,9 @@ Message: |
18 | two distinctive groups: rule-based and stochastic. E. Brill's tagger, one of the
| ^~ Did you mean to spell `E.` this way?
Suggest:
- - Replace with: “E”
- - Replace with: “Ea”
- Replace with: “Eh”
+ - Replace with: “Em”
+ - Replace with: “Ex”
diff --git a/harper-core/tests/text/linters/The Great Gatsby.snap.yml b/harper-core/tests/text/linters/The Great Gatsby.snap.yml
index e32d0dfe2..a95f5d588 100644
--- a/harper-core/tests/text/linters/The Great Gatsby.snap.yml
+++ b/harper-core/tests/text/linters/The Great Gatsby.snap.yml
@@ -5,7 +5,7 @@ Message: |
Suggest:
- Replace with: “Fa”
- Replace with: “Fe”
- - Replace with: “Ff”
+ - Replace with: “Ft”
@@ -782,8 +782,8 @@ Message: |
| ^~ Did you mean to spell `J.` this way?
Suggest:
- Replace with: “Jo”
- - Replace with: “J”
- Replace with: “Jg”
+ - Replace with: “Jr”
@@ -816,8 +816,8 @@ Message: |
| ^~ Did you mean to spell `J.` this way?
Suggest:
- Replace with: “Jo”
- - Replace with: “J”
- Replace with: “Jg”
+ - Replace with: “Jr”
@@ -893,7 +893,7 @@ Message: |
Suggest:
- Replace with: “Be”
- Replace with: “Bi”
- - Replace with: “Bu”
+ - Replace with: “BO”
@@ -1144,7 +1144,7 @@ Message: |
Suggest:
- Replace with: “Be”
- Replace with: “Bi”
- - Replace with: “Bu”
+ - Replace with: “BO”
@@ -1703,15 +1703,6 @@ Suggest:
-Lint: Typo (31 priority)
-Message: |
- 1413 | trembling a little to the stiff, tinny drip of the banjoes on the lawn.
- | ^~~~~~~ `banjoes` should probably be written as `banjo es`.
-Suggest:
- - Replace with: “banjo es”
-
-
-
Lint: Enhancement (31 priority)
Message: |
1457 | “I thought you knew, old sport. I’m afraid I’m not a very good host.”
@@ -2379,7 +2370,7 @@ Message: |
Suggest:
- Replace with: “Be”
- Replace with: “Bi”
- - Replace with: “Bu”
+ - Replace with: “BO”
@@ -2607,7 +2598,7 @@ Message: |
Suggest:
- Replace with: “Be”
- Replace with: “Bi”
- - Replace with: “Bu”
+ - Replace with: “BO”
@@ -3566,11 +3557,10 @@ Suggest:
Lint: Typo (31 priority)
Message: |
2319 | That was nineteen-seventeen. By the next year I had a few beaux myself, and I
- | ^~~~~ `beaux` has a missing space between words.
+ | ^~~~~ `beaux` should probably be written as `be aux`.
2320 | began to play in tournaments, so I didn’t see Daisy very often. She went with a
Suggest:
- Replace with: “be aux”
- - Replace with: “beau x”
@@ -3645,15 +3635,6 @@ Suggest:
-Lint: Typo (31 priority)
-Message: |
- 2349 | “Here, deares’.” She groped around in a wastebasket she had with her on the bed
- | ^~~~~~ `deares` should probably be written as `dear es`.
-Suggest:
- - Replace with: “dear es”
-
-
-
Lint: Formatting (255 priority)
Message: |
2352 | change’ her mine!’”
@@ -4997,15 +4978,6 @@ Suggest:
-Lint: Typo (31 priority)
-Message: |
- 3646 | “The bles-sed pre-cious! Did mother get powder on your old yellowy hair? Stand
- | ^~~~ `bles` should probably be written as `bl es`.
-Suggest:
- - Replace with: “bl es”
-
-
-
Lint: Spelling (63 priority)
Message: |
3646 | “The bles-sed pre-cious! Did mother get powder on your old yellowy hair? Stand
@@ -5154,8 +5126,8 @@ Message: |
| ^~ Did you mean to spell `J.` this way?
Suggest:
- Replace with: “Jo”
- - Replace with: “J”
- Replace with: “Jg”
+ - Replace with: “Jr”
@@ -5232,8 +5204,8 @@ Message: |
| ^~ Did you mean to spell `J.` this way?
Suggest:
- Replace with: “Jo”
- - Replace with: “J”
- Replace with: “Jg”
+ - Replace with: “Jr”
@@ -5429,6 +5401,28 @@ Suggest:
+Lint: Spelling (63 priority)
+Message: |
+ 4070 | the window, followed by intermittent cries of “Yea—ea—ea!” and finally by a
+ | ^~ Did you mean to spell `ea` this way?
+Suggest:
+ - Replace with: “ear”
+ - Replace with: “eat”
+ - Replace with: “eh”
+
+
+
+Lint: Spelling (63 priority)
+Message: |
+ 4070 | the window, followed by intermittent cries of “Yea—ea—ea!” and finally by a
+ | ^~ Did you mean to spell `ea` this way?
+Suggest:
+ - Replace with: “ear”
+ - Replace with: “eat”
+ - Replace with: “eh”
+
+
+
Lint: Spelling (63 priority)
Message: |
4073 | “We're getting old,” said Daisy. “lf we were young we’d rise and dance.”
@@ -5947,7 +5941,7 @@ Message: |
Suggest:
- Replace with: “re”
- Replace with: “a”
- - Replace with: “e”
+ - Replace with: “i”
@@ -5958,7 +5952,7 @@ Message: |
Suggest:
- Replace with: “re”
- Replace with: “a”
- - Replace with: “e”
+ - Replace with: “i”
@@ -5989,7 +5983,7 @@ Message: |
Suggest:
- Replace with: “re”
- Replace with: “a”
- - Replace with: “e”
+ - Replace with: “i”
@@ -6057,9 +6051,8 @@ Suggest:
Lint: Typo (31 priority)
Message: |
4503 | “She ran out ina road. Son-of-a-bitch didn’t even stopus car.”
- | ^~~~~~ `stopus` has a missing space between words.
+ | ^~~~~~ `stopus` should probably be written as `stop us`.
Suggest:
- - Replace with: “st opus”
- Replace with: “stop us”
@@ -6783,8 +6776,8 @@ Message: |
| ^~ Did you mean to spell `J.` this way?
Suggest:
- Replace with: “Jo”
- - Replace with: “J”
- Replace with: “Jg”
+ - Replace with: “Jr”
@@ -7285,8 +7278,8 @@ Message: |
| ^~ Did you mean to spell `J.` this way?
Suggest:
- Replace with: “Jo”
- - Replace with: “J”
- Replace with: “Jg”
+ - Replace with: “Jr”
@@ -7317,7 +7310,7 @@ Message: |
Suggest:
- Replace with: “Be”
- Replace with: “Bi”
- - Replace with: “Bu”
+ - Replace with: “BO”
@@ -7328,7 +7321,7 @@ Message: |
Suggest:
- Replace with: “Fa”
- Replace with: “Fe”
- - Replace with: “Ff”
+ - Replace with: “Ft”
@@ -7574,9 +7567,9 @@ Message: |
5613 | great for that. He told me I et like a hog once, and I beat him for it.”
| ^~ Did you mean to spell `et` this way?
Suggest:
- - Replace with: “e”
- - Replace with: “ea”
- Replace with: “eat”
+ - Replace with: “eh”
+ - Replace with: “em”
@@ -7842,8 +7835,8 @@ Message: |
| ^~ Did you mean to spell `El` this way?
Suggest:
- Replace with: “Ell”
- - Replace with: “E”
- - Replace with: “Ea”
+ - Replace with: “Eel”
+ - Replace with: “Eh”
diff --git a/harper-core/tests/text/tagged/The Great Gatsby.md b/harper-core/tests/text/tagged/The Great Gatsby.md
index 9e985e16e..6f127cc7e 100644
--- a/harper-core/tests/text/tagged/The Great Gatsby.md
+++ b/harper-core/tests/text/tagged/The Great Gatsby.md
@@ -8136,8 +8136,8 @@
#
> The music had died down as the ceremony began and now a long cheer floated in at
# D+ N🅪Sg/VB/J+ VP VP/J N🅪Sg/VB/J/P R/C/P D+ N🅪Sg+ VPt VB/C NSg/J/R/C D/P+ NPr/VB/J+ NSg/VB+ VP/J NPr/J/R/P NSg/P
-> the window , followed by intermittent cries of “ Yea — ea — ea ! ” and finally by a
-# D+ NSg/VB+ . VP/J NSg/P NSg/J NPl/V3 P . NSg/C . NSg . NSg . . VB/C R NSg/P D/P
+> the window , followed by intermittent cries of “ Yea — ea — ea ! ” and finally by a
+# D+ NSg/VB+ . VP/J NSg/P NSg/J NPl/V3 P . NSg/C . ? . ? . . VB/C R NSg/P D/P
> burst of jazz as the dancing began .
# NSg/VB P NSg/VB+ R/C/P D Nᴹ/Vg/J VPt .
>