Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
fix(core): major issues with `SplitWords` (#3857)
Elijah Potter0724f5901ffa
diff --git a/harper-core/dictionary.dict b/harper-core/dictionary.dictindex 2bf42ca14..7d0c54116 100644--- a/harper-core/dictionary.dict+++ b/harper-core/dictionary.dict@@ -13788,7 +13788,7 @@ azalea/NSgazimuth/~N0gazimuths/N9azure/~NSgJV-b/~J^VdK+b/J^VdKbaa/~NSgVdGbabble/VGd>SNmgZbabbler/~Ng@@ -15950,7 +15950,7 @@ brute force/NwgSVdGbrutish/JYpbrutishness/Nmgbrutus/-bu/~N+bu/Nbub/NSgVbubble/~NSgVdG>bubble tea/NwgS@@ -16267,7 +16267,7 @@ bytecode/NwgSbyway/~NSgbyword/NSgbyzantine/~JN-c/~NSViE+c/NSViEca/Pcab/~NSgV>Zcab sav/Nmg†_ # Australia slang = Cabernet Sauvignon@@ -21216,6 +21216,7 @@ desert/~NwSgJ>VdGZdeserter/NgSdesertification/Nmgdesertion/~NwSg+deserve/~V>SZdeserved/~JYVUdeserving/~JNVUdesi/NgS₹@@ -21680,7 +21681,7 @@ dirtball/NSdirtily/Rydirtiness/Nmgdirty/~J^>pVdSG-dis/~VNgI(+dis/VNgI(disable/~VdSGJLdisablement/Ngdisambiguate/~VSdGn@@ -22711,7 +22712,7 @@ dystopidystopia/NwgSdystopian/~Jdz/NV-e/~NSId^W+e/NSId^We-book/NgSe-commerce/Nmge'en/@@ -22720,7 +22721,6 @@ e.g./~NeBay/OgVeMusic/geSIM/NgS-ea/~Neach/~Dqeach other/Ieager/~J^Y>pVN@@ -22853,7 +22853,7 @@ ecumenical/~JYecumenicism/Ngecumenism/Nmgeczema/Nwg-ed/~NSgre+ed/NSgreedamame/Neddy/~NSgVdGedelweiss/Ng@@ -24916,7 +24916,6 @@ fewness/Ngfey/~JNfez/~Ngfezzes/N-ff/~Vfiance/NgVefiancee/NgSfiances/N@@ -26346,7 +26345,7 @@ fuzziness/Ngfuzzy/~J^>pNSfwd/~JNVfwy/N-g/~NSnXvB+g/NSnXvBgab/~NSgVgabardine/NSggabbed/VtT@@ -27764,7 +27763,7 @@ gyrofrequency/NgSgyroscope/NgSgyroscopic/Jgyve/NgSVGd-h/~NJGnXZvz # removed `>` 'her' is not comparative of 'h'+h/NJGnXZvz # removed `>` 'her' is not comparative of 'h'h'm/ha/~hah/~@@ -30206,7 +30205,7 @@ inexorability/Ninexorable/Jinexorably/Ryinexpedient/J-inexperience/Nmg+inexperience/~Nmgdinexpert/JYNinexpiable/Jinexplicably/~Ry@@ -31076,7 +31075,7 @@ iv/~Uivory/~NwSgJivy/~NSgdix/~-j/~NW+j/NWjab/NSgVjabbed/VtTJjabber/Vd>GSNgZ@@ -37947,7 +37946,7 @@ oxymoronic/Joyster/~NSgJVoz/~Nozone/~NgV-p/~PNV>G^nXz+p/PNV>G^nXzp.m./pH/pa/~NSgH@@ -41443,7 +41442,7 @@ pyruvate/~Npython/~NSgpyx/NgSVpzazz/N-q/~N+q/Nqr/~Nqt/~NSqty/N@@ -46930,7 +46929,7 @@ squish/NgSVGdsquishy/J>^Nsriracha/Nssh/~VO-st/~N+st/Nstab/~NgSVJYstabbed/~VtTstabber/NgS@@ -49611,7 +49610,7 @@ tinder/NgVtinderbox/NgStine/NgSJVtinfoil/NmgV-ting/~NgVdGY+ting/NgVdGYtinge/NSgVtingeing/Vtingle/VdGSNgz@@ -53566,7 +53565,7 @@ wurst/NSgwuss/NgSVwussy/J>^NSgwyvern/NSg-x/~ # removed `5` and `7` adjective and `conjunction`+x/ # removed `5` and `7` adjective and `conjunction`x-axis/Ngx64/Ngx86/~Ng@@ -54717,3 +54716,4 @@ vicontiel/J # As above, alternative versionviscomital/J # As above, alternative versionWordCamp/Ogworldbuild/V>G+playthrough/gNSdiff --git a/harper-core/src/linting/lint_group/mod.rs b/harper-core/src/linting/lint_group/mod.rsindex f5a614aca..90f8abbd6 100644--- a/harper-core/src/linting/lint_group/mod.rs+++ b/harper-core/src/linting/lint_group/mod.rs@@ -1191,6 +1191,52 @@ mod tests {);}+ #[test]+ fn fix_stoped() {+ assert_suggestion_result("He stoped to think.", test_group(), "He stopped to think.");+ }++ #[test]+ fn allow_playthrough() {+ assert_no_lints("I recently did a playthrough.", test_group());+ }++ #[test]+ fn fixes_politicans() {+ assert_suggestion_result(+ "I guess politicans like to complain.",+ test_group(),+ "I guess politicans like to complain.",+ );+ }++ #[test]+ fn fixes_benefitting() {+ assert_suggestion_result(+ "Who is really benefitting from it?",+ test_group(),+ "Who is really benefiting from it?",+ );+ }++ #[test]+ fn fixes_easir() {+ assert_suggestion_result(+ "It makes it easir to select it.",+ test_group(),+ "It makes it easier to select it.",+ );+ }++ #[test]+ fn fixes_buget() {+ assert_suggestion_result(+ "It was in their buget range.",+ test_group(),+ "It was in their budget range.",+ );+ }+/// Tests that no linters' descriptions contain errors handled by other linters.////// This test verifies that the description of each linter (which is written in natural language)diff --git a/harper-core/src/linting/split_words.rs b/harper-core/src/linting/split_words.rsindex 98d73ba62..ade0b1794 100644--- a/harper-core/src/linting/split_words.rs+++ b/harper-core/src/linting/split_words.rs@@ -208,10 +208,14 @@ fn should_defer_to_spellcheck(#[cfg(test)]mod tests {+ use itertools::Itertools;++ use crate::Document;use crate::linting::tests::{assert_good_and_bad_suggestions, assert_lint_message, assert_no_lints,assert_suggestion_result,};+ use crate::linting::{Linter, Suggestion};use super::SplitWords;@@ -365,4 +369,51 @@ mod tests {assert_no_lints(source, SplitWords::default());}}++ #[test]+ fn does_not_split_iff() {+ assert_no_lints("iff", SplitWords::default());+ }++ /// Checks for a condition where the SplitWords rule would correct a word to be composed of a+ /// single letter (which is not a word), followed by a valid word.+ ///+ /// For example, `comitted` -> `c omitted`.+ #[test]+ fn never_corrects_to_invalid_single_letter_words() {+ let triggers = [+ "comitted", "testc", "testh", "testb", "testq", "testx", "testg", "teste", "testj",+ "shes",+ ];+ let relevant_letters = ['c', 'd', 't', 'h', 'b', 'x', 'e', 'j', 's'];++ for trigger in triggers {+ let mut rule = SplitWords::default();++ let doc = Document::new_plain_english_curated(trigger);+ let lints = rule.lint(&doc);++ for lint in lints {+ dbg!(&lint);++ for sug in lint.suggestions {+ match sug {+ Suggestion::ReplaceWith(items) => {+ // Words created by the rule.+ let created_words = items.split(|c| c == &' ');++ for word in created_words {+ if word.len() == 1+ && relevant_letters.iter().contains(&word.first().unwrap())+ {+ panic!("Encountered bad output {word:?}")+ }+ }+ }+ _ => (),+ }+ }+ }+ }+ }}diff --git a/harper-core/src/spell/mutable_dictionary.rs b/harper-core/src/spell/mutable_dictionary.rsindex 03e6abaec..bbbaa1e3a 100644--- a/harper-core/src/spell/mutable_dictionary.rs+++ b/harper-core/src/spell/mutable_dictionary.rs@@ -406,6 +406,11 @@ mod tests {)}+ #[test]+ fn contains_inexperienced() {+ assert!(MutableDictionary::curated().contains_word_str("inexperienced"))+ }+#[test]fn has_is_not_a_nominal() {let dict = MutableDictionary::curated();diff --git a/harper-core/tests/text/linters/Part-of-speech tagging.snap.yml b/harper-core/tests/text/linters/Part-of-speech tagging.snap.ymlindex b7610cf8a..3a3b4f31a 100644--- a/harper-core/tests/text/linters/Part-of-speech tagging.snap.yml+++ b/harper-core/tests/text/linters/Part-of-speech tagging.snap.yml@@ -47,9 +47,9 @@ Message: |18 | two distinctive groups: rule-based and stochastic. E. Brill's tagger, one of the| ^~ Did you mean to spell `E.` this way?Suggest:- - Replace with: “E”- - Replace with: “Ea”- Replace with: “Eh”+ - Replace with: “Em”+ - Replace with: “Ex”diff --git a/harper-core/tests/text/linters/The Great Gatsby.snap.yml b/harper-core/tests/text/linters/The Great Gatsby.snap.ymlindex e32d0dfe2..a95f5d588 100644--- a/harper-core/tests/text/linters/The Great Gatsby.snap.yml+++ b/harper-core/tests/text/linters/The Great Gatsby.snap.yml@@ -5,7 +5,7 @@ Message: |Suggest:- Replace with: “Fa”- Replace with: “Fe”- - Replace with: “Ff”+ - Replace with: “Ft”@@ -782,8 +782,8 @@ Message: || ^~ Did you mean to spell `J.` this way?Suggest:- Replace with: “Jo”- - Replace with: “J”- Replace with: “Jg”+ - Replace with: “Jr”@@ -816,8 +816,8 @@ Message: || ^~ Did you mean to spell `J.` this way?Suggest:- Replace with: “Jo”- - Replace with: “J”- Replace with: “Jg”+ - Replace with: “Jr”@@ -893,7 +893,7 @@ Message: |Suggest:- Replace with: “Be”- Replace with: “Bi”- - Replace with: “Bu”+ - Replace with: “BO”@@ -1144,7 +1144,7 @@ Message: |Suggest:- Replace with: “Be”- Replace with: “Bi”- - Replace with: “Bu”+ - Replace with: “BO”@@ -1703,15 +1703,6 @@ Suggest:-Lint: Typo (31 priority)-Message: |- 1413 | trembling a little to the stiff, tinny drip of the banjoes on the lawn.- | ^~~~~~~ `banjoes` should probably be written as `banjo es`.-Suggest:- - Replace with: “banjo es”---Lint: Enhancement (31 priority)Message: |1457 | “I thought you knew, old sport. I’m afraid I’m not a very good host.”@@ -2379,7 +2370,7 @@ Message: |Suggest:- Replace with: “Be”- Replace with: “Bi”- - Replace with: “Bu”+ - Replace with: “BO”@@ -2607,7 +2598,7 @@ Message: |Suggest:- Replace with: “Be”- Replace with: “Bi”- - Replace with: “Bu”+ - Replace with: “BO”@@ -3566,11 +3557,10 @@ Suggest:Lint: Typo (31 priority)Message: |2319 | That was nineteen-seventeen. By the next year I had a few beaux myself, and I- | ^~~~~ `beaux` has a missing space between words.+ | ^~~~~ `beaux` should probably be written as `be aux`.2320 | began to play in tournaments, so I didn’t see Daisy very often. She went with aSuggest:- Replace with: “be aux”- - Replace with: “beau x”@@ -3645,15 +3635,6 @@ Suggest:-Lint: Typo (31 priority)-Message: |- 2349 | “Here, deares’.” She groped around in a wastebasket she had with her on the bed- | ^~~~~~ `deares` should probably be written as `dear es`.-Suggest:- - Replace with: “dear es”---Lint: Formatting (255 priority)Message: |2352 | change’ her mine!’”@@ -4997,15 +4978,6 @@ Suggest:-Lint: Typo (31 priority)-Message: |- 3646 | “The bles-sed pre-cious! Did mother get powder on your old yellowy hair? Stand- | ^~~~ `bles` should probably be written as `bl es`.-Suggest:- - Replace with: “bl es”---Lint: Spelling (63 priority)Message: |3646 | “The bles-sed pre-cious! Did mother get powder on your old yellowy hair? Stand@@ -5154,8 +5126,8 @@ Message: || ^~ Did you mean to spell `J.` this way?Suggest:- Replace with: “Jo”- - Replace with: “J”- Replace with: “Jg”+ - Replace with: “Jr”@@ -5232,8 +5204,8 @@ Message: || ^~ Did you mean to spell `J.` this way?Suggest:- Replace with: “Jo”- - Replace with: “J”- Replace with: “Jg”+ - Replace with: “Jr”@@ -5429,6 +5401,28 @@ Suggest:+Lint: Spelling (63 priority)+Message: |+ 4070 | the window, followed by intermittent cries of “Yea—ea—ea!” and finally by a+ | ^~ Did you mean to spell `ea` this way?+Suggest:+ - Replace with: “ear”+ - Replace with: “eat”+ - Replace with: “eh”++++Lint: Spelling (63 priority)+Message: |+ 4070 | the window, followed by intermittent cries of “Yea—ea—ea!” and finally by a+ | ^~ Did you mean to spell `ea` this way?+Suggest:+ - Replace with: “ear”+ - Replace with: “eat”+ - Replace with: “eh”+++Lint: Spelling (63 priority)Message: |4073 | “We're getting old,” said Daisy. “lf we were young we’d rise and dance.”@@ -5947,7 +5941,7 @@ Message: |Suggest:- Replace with: “re”- Replace with: “a”- - Replace with: “e”+ - Replace with: “i”@@ -5958,7 +5952,7 @@ Message: |Suggest:- Replace with: “re”- Replace with: “a”- - Replace with: “e”+ - Replace with: “i”@@ -5989,7 +5983,7 @@ Message: |Suggest:- Replace with: “re”- Replace with: “a”- - Replace with: “e”+ - Replace with: “i”@@ -6057,9 +6051,8 @@ Suggest:Lint: Typo (31 priority)Message: |4503 | “She ran out ina road. Son-of-a-bitch didn’t even stopus car.”- | ^~~~~~ `stopus` has a missing space between words.+ | ^~~~~~ `stopus` should probably be written as `stop us`.Suggest:- - Replace with: “st opus”- Replace with: “stop us”@@ -6783,8 +6776,8 @@ Message: || ^~ Did you mean to spell `J.` this way?Suggest:- Replace with: “Jo”- - Replace with: “J”- Replace with: “Jg”+ - Replace with: “Jr”@@ -7285,8 +7278,8 @@ Message: || ^~ Did you mean to spell `J.` this way?Suggest:- Replace with: “Jo”- - Replace with: “J”- Replace with: “Jg”+ - Replace with: “Jr”@@ -7317,7 +7310,7 @@ Message: |Suggest:- Replace with: “Be”- Replace with: “Bi”- - Replace with: “Bu”+ - Replace with: “BO”@@ -7328,7 +7321,7 @@ Message: |Suggest:- Replace with: “Fa”- Replace with: “Fe”- - Replace with: “Ff”+ - Replace with: “Ft”@@ -7574,9 +7567,9 @@ Message: |5613 | great for that. He told me I et like a hog once, and I beat him for it.”| ^~ Did you mean to spell `et` this way?Suggest:- - Replace with: “e”- - Replace with: “ea”- Replace with: “eat”+ - Replace with: “eh”+ - Replace with: “em”@@ -7842,8 +7835,8 @@ Message: || ^~ Did you mean to spell `El` this way?Suggest:- Replace with: “Ell”- - Replace with: “E”- - Replace with: “Ea”+ - Replace with: “Eel”+ - Replace with: “Eh”diff --git a/harper-core/tests/text/tagged/The Great Gatsby.md b/harper-core/tests/text/tagged/The Great Gatsby.mdindex 9e985e16e..6f127cc7e 100644--- a/harper-core/tests/text/tagged/The Great Gatsby.md+++ b/harper-core/tests/text/tagged/The Great Gatsby.md@@ -8136,8 +8136,8 @@#> The music had died down as the ceremony began and now a long cheer floated in at# D+ N🅪Sg/VB/J+ VP VP/J N🅪Sg/VB/J/P R/C/P D+ N🅪Sg+ VPt VB/C NSg/J/R/C D/P+ NPr/VB/J+ NSg/VB+ VP/J NPr/J/R/P NSg/P-> the window , followed by intermittent cries of “ Yea — ea — ea ! ” and finally by a-# D+ NSg/VB+ . VP/J NSg/P NSg/J NPl/V3 P . NSg/C . NSg . NSg . . VB/C R NSg/P D/P+> the window , followed by intermittent cries of “ Yea — ea — ea ! ” and finally by a+# D+ NSg/VB+ . VP/J NSg/P NSg/J NPl/V3 P . NSg/C . ? . ? . . VB/C R NSg/P D/P> burst of jazz as the dancing began .# NSg/VB P NSg/VB+ R/C/P D Nᴹ/Vg/J VPt .>