Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
feat: spellchecking now prioritizes especially common words
Elijah Potter7ffcb6f32a4b
diff --git a/harper-core/affixes.json b/harper-core/affixes.jsonindex b8ddd8a1d..5c257dbba 100644--- a/harper-core/affixes.json+++ b/harper-core/affixes.json@@ -553,9 +553,19 @@"adds_metadata": {},"gifts_metadata": {"noun": {- "is_pronoun": true+ "is_pronoun": true,+ "is_common": true}}+ },+ "~": {+ "suffix": true,+ "cross_product": true,+ "replacements": [],+ "adds_metadata": {},+ "gifts_metadata": {+ "common": true+ }}}}diff --git a/harper-core/dictionary.dict b/harper-core/dictionary.dictindex 7762e0e9f..3fbb4ef79 100644--- a/harper-core/dictionary.dict+++ b/harper-core/dictionary.dict@@ -4609,9 +4609,9 @@ Hyundai/1MHz/MI'd/I'll/-I'm/+I'm/~I've/-I/81M+I/81M~IA/21IBM/21MICBM/1SM@@ -10782,7 +10782,7 @@ Zworykin/MZyrtec/MZyuganov/MZzz-a/14857S+a/14857S~aah/14aardvark/1SMab/14SDY@@ -10872,7 +10872,7 @@ abortion/1MSabortionist/1MSabortive/514Yabound/4DSG-about/5+about/5~above/51Maboveboard/5abracadabra/1M@@ -11662,7 +11662,7 @@ alkalinity/1Malkalize/4DSGalkaloid/15SMalkyd/1MS-all/8175M+all/8175M~allay/41GDSallegation/1MSallege/4GDS@@ -11748,7 +11748,7 @@ alphanumerical/51Yalpine/51Salready/alright/5-also/+also/~alt/15Saltar/1MSaltarpiece/1SM@@ -11918,7 +11918,7 @@ amusing/45Yamygdala/1amylase/1Mamyloid/15-an/71CS+an/71CS~anabolism/1Manachronism/1SManachronistic/5@@ -11979,7 +11979,7 @@ anchovy/1SMancient/51SPMRYTancientness/1Mancillary/51SM-and/714+and/714~andante/15SMandiron/1SMandrogen/1M@@ -12258,7 +12258,7 @@ anvil/14MSanxiety/1SManxious/5YPanxiousness/1M-any/8+any/8~anybody/8SManyhow/anymore/@@ -12829,7 +12829,7 @@ asymptomatic/51asymptotic/5asymptotically/asynchronous/5Y-at/148+at/148~atavism/1Matavist/1SMatavistic/5@@ -13161,7 +13161,7 @@ bachelorhood/1Mbacillary/5bacilli/1bacillus/1M-back/514SJZGMDR+back/514SJZGMDR~backache/1MSbackbench/51Sbackbit/4@@ -13616,7 +13616,7 @@ bazillion/1Sbazooka/14SMbbl/1bdrm-be/41+be/41~beach/14MDSGbeachcomber/1SMbeachfront/51@@ -13676,7 +13676,7 @@ beaver/14SGMDbebop/14MSbecalm/4GSDbecame/4-because/7+because/7~beck/14SMbeckon/41SGDbecloud/4GDS@@ -15367,7 +15367,7 @@ busy/514DRSTGPbusybody/14SMbusyness/1Mbusywork/1M-but/714ACS+but/714ACS~butane/1Mbutch/514MRSZbutcher/145MDG@@ -15408,7 +15408,7 @@ buzzkill/1SMbuzzword/1SMbxbxs-by/51M+by/51M~bye/15SMbygone/51SMbylaw/1SM@@ -17444,7 +17444,7 @@ combust/451SGVDcombustibility/1Mcombustible/51MScombustion/1M-come/41IMZGRS+come/41IMZGRS~comeback/14MScomedian/1MScomedic/5@@ -18490,7 +18490,7 @@ couchette/1Scougar/1SMcough/41MDGcoughs/14-could/41+could/41~could've/4couldn't/4coulee/1SM@@ -19392,7 +19392,7 @@ davit/1MSdawdle/41ZGDRSdawdler/1Mdawn/41SGMD-day/14SM+day/14SM~daybed/1MSdaybreak/1Mdaycare/1M@@ -20667,7 +20667,7 @@ dizzily/dizziness/1Mdizzy/541DRSPTGdjellaba/1MS-do/41SJMRHZG+do/41SJMRHZG~doable/51dob/41Sdobbed/4@@ -22471,7 +22471,7 @@ evasion/1SMevasive/5YPevasiveness/1Meve/1ASM-even/541MDRYTGSJP+even/541MDRYTGSJP~evenhanded/5Yevening/14Mevenness/1UM@@ -23566,7 +23566,7 @@ firm/154MDRYPTGSfirmament/1SMfirmness/1Mfirmware/1M-first/514SMY+first/514SMY~firstborn/15SMfirsthand/5firth/1M@@ -24039,7 +24039,7 @@ fop/1SMfoppery/1Mfoppish/5Pfoppishness/1M-for/7H+for/7H~fora/1forage/14DRSMZGforager/1M@@ -24477,7 +24477,7 @@ frolicked/4frolicker/1SMfrolicking/41frolicsome/5-from/+from/~frond/1SMfront'sfront/154FSDG@@ -25074,7 +25074,7 @@ gesticulation/1Mgestural/5gesture/14MGDSgesundheit/-get/41S+get/41S~getaway/15SMgetting/41getup/1M@@ -25175,7 +25175,7 @@ girths/1gist/14Mgit/142Sgite/1S-give/41ZGJRS+give/41ZGJRS~giveaway/15MSgiveback/1MSgiven/415SM@@ -25335,7 +25335,7 @@ gnome/1SMgnomic/5gnomish/5gnu/1SM-go/415JMRHZG+go/415JMRHZG~goad/14MDGSgoal/14MSgoalie/14SM@@ -25420,7 +25420,7 @@ gonorrheal/5gonzo/51goo/14Mgoober/14SM-good/514MYSP+good/514MYSP~goodbye/14MSgoodhearted/5goodish/5@@ -26353,7 +26353,7 @@ haunt/41SMDRZGhaunter/1Mhaunting/451Yhauteur/1M-have/41MGS+have/41MGS~haven't/4haven/14SMhaversack/1SM@@ -26390,7 +26390,7 @@ hazy/5RTPhdqrs/1he'd/he'll/-he/81M+he/81M~head/154MDRZGJSheadache/1MSheadband/1MS@@ -27252,7 +27252,7 @@ hoverboard/14MShovercraft/1MShow'd/how're/-how/71SM+how/71SM~howbeit/7howdah/1Mhowdahs/1@@ -27985,7 +27985,7 @@ impure/54RYTimpurity/1SMimputation/1SMimpute/4BDSG-in/415ASM+in/415ASM~inaccuracy/1Sinaction/1Minadequacy/1S@@ -29006,7 +29006,7 @@ isthmian/51isthmus/1MSit'd/it'll/-it/815USM+it/815USM~ital/15italic/51SMitalicization/1M@@ -30683,7 +30683,7 @@ longways/loo/14loofah/14Mloofahs/14-look/41MDRSZG+look/41MDRSZG~lookalike/1MSlooker/1Mlookout/1MS@@ -31538,7 +31538,7 @@ mayst/4maze/14MSmazurka/1MSmdse-me/81DSH+me/81DSH~mead/1Mmeadow/14MSmeadowlark/1MS@@ -32725,7 +32725,7 @@ mosquitoes/1moss/14MSmossback/1SMmossy/51TR-most/581MY+most/581MY~mot/1SMmote'smote/14KCXSVN@@ -33069,7 +33069,7 @@ muzak/14muzzily/muzzle/14DSMGmuzzy/51P-my/+my/~mycologist/1SMmycology/1Mmyelitis/1M@@ -33433,7 +33433,7 @@ nevermore/nevertheless/nevi/1nevus/1M-new/514STMRYP+new/514STMRYP~newbie/1MSnewborn/51SMnewcomer/1SM@@ -33977,7 +33977,7 @@ nostalgically/nostril/1MSnostrum/1MSnosy/514RPT-not/71B+not/71B~notability/1SMnotable/51SMnotably/@@ -34031,7 +34031,7 @@ novena/1MSnovene/5novice/1MSnovitiate/1MS-now/5714M+now/5714M~nowadays/Mnoway/Snowhere/51M@@ -34340,7 +34340,7 @@ oedipal/5oenology/1Moenophile/1SMoeuvre/1MS-of/4+of/4~off/541SZGDRJoffal/1Moffbeat/15MS@@ -34457,14 +34457,14 @@ omniscient/51omnivore/1MSomnivorous/5PYomnivorousness/1M-on/541Y+on/541Y~onboard/54once/71Moncogene/1SMoncologist/1SMoncology/1Moncoming/154-one/1854SXMNP+one/1854SXMNP~oneness/1Monerous/5PYonerousness/1M@@ -34581,7 +34581,7 @@ optometry/1Mopulence/1Mopulent/5Yopus/1MS-or/715+or/715~oracle/14SMoracular/5oral/51MYS@@ -34726,7 +34726,7 @@ ostler/1Sostracism/1Mostracize/4GDSostrich/1MS-other/514MSP+other/514MSP~otherwise/5otherworldly/5otiose/5@@ -34737,11 +34737,11 @@ ouch/14ought/481oughtn't/4ounce/1MS-our/4S+our/4S~ourselves/8oust/4ZGDRSouster/14M-out/145SJGMDR+out/145SJGMDR~outage/1SMoutargue/4GDSoutback/154MS@@ -34897,7 +34897,7 @@ oven/14MSovenbird/1SMovenproof/5ovenware/1-over/514MYS+over/514MYS~overabundance/1Moverabundant/5overachieve/4ZGDRS@@ -35968,7 +35968,7 @@ penury/1Mpeon/1MSpeonage/1Mpeony/1SM-people/14MGDS+people/14MGDS~pep/41SMpepped/4pepper/14GMDS@@ -40602,7 +40602,7 @@ saxifrage/1SMsaxophone/14MSsaxophonist/1SMsay's-say/41USG+say/41USG~saying/41SMscab/14MSscabbard/14MS@@ -41103,7 +41103,7 @@ seductive/5YPseductiveness/1Mseductress/1MSsedulous/5Y-see/41RSMZ+see/41RSMZ~seed'sseed/14AGDSseedbed/1MS@@ -41550,7 +41550,7 @@ shawl/14MSshay/1MSshe'd/she'll/-she/81DSM+she/81DSM~sheaf/14Mshear/415MDRSZGshearer/1M@@ -42616,7 +42616,7 @@ snuggest/5snugging/4snuggle/14MGDSsnugness/1M-so/7581+so/7581~soak/41MDGSJsoaking/415Msoap/14MDGS@@ -42756,7 +42756,7 @@ somatosensory/5somber/54PYsomberness/1Msombrero/1MS-some/8+some/8~somebody/81SMsomeday/somehow/@@ -44912,7 +44912,7 @@ tailspin/14SMtailwind/14SMtaint/14MDSGtainted/54U-take/41AIMS+take/41AIMS~takeaway/51Staken/54Atakeoff/1MS@@ -45448,7 +45448,7 @@ thalami/1thalamus/1Mthalidomide/1Mthallium/1M-than/7+than/7~thane/1SMthank/14SDGthankful/5YP@@ -45458,12 +45458,12 @@ thanklessness/1Mthanksgiving/1SMthat'd/that'll/-that/781M+that/781M~thatch/14MDRSZGthatcher/1Mthatching/14Mthaw/41MDGS-the/81JG+the/81JG~theater/1SMtheatergoer/1SMtheatrical/51YS@@ -45472,16 +45472,16 @@ theatricals/1Mtheatrics/1Mthee/841Stheft/1SM-their/S+their/S~theism/1Mtheist/1SMtheistic/5-them/8+them/8~thematic/51thematically/theme/14DSMthemselves/8-then/517M+then/517M~thence/thenceforth/thenceforward/@@ -45507,7 +45507,7 @@ therapeutically/therapeutics/1Mtherapist/1SMtherapy/14SM-there/18M+there/18M~thereabout/Sthereafter/1thereat/@@ -45540,12 +45540,12 @@ thermostatic/5thermostatically/thesauri/1thesaurus/1MS-these/8S+these/8S~thesis/1Mthespian/51SMtheta/1SMthew/14MS-they/8+they/8~they'd/they'll/they're/@@ -45578,7 +45578,7 @@ thingamajig/1SMthingumabob/1Sthingummy/1Sthingy/158S-think/41SRBZG+think/41SRBZG~thinkable/5Uthinker/1Mthinking's@@ -45598,7 +45598,7 @@ thirteenths/1thirtieth/51Mthirtieths/1thirty/1HSM-this/81+this/81~thistle/1MSthistledown/1Mthither/5@@ -45911,7 +45911,7 @@ tizz/1tizzy/1SMtn/1tnpk/1-to/IU+to/IU~toad/14MStoadstool/1MStoady/145DSMG@@ -47447,7 +47447,7 @@ unworthy/51Tunwound/4unwrapping/41unyielding/5-up/514S+up/514S~upbeat/51MSupbraid/41SGDupbringing/1MS@@ -47560,7 +47560,7 @@ urticaria/1Musability/1Musable/5UAusage/1SM-use/14AEDSMG+use/14AEDSMG~used/45Uuseful/5PYusefulness/1M@@ -48336,7 +48336,7 @@ wannabee/1Swanner/5wanness/1Mwannest/5-want/41MDGS+want/41MDGS~wanted/54Uwanton/514MDYSPGwantonness/1M@@ -48496,7 +48496,7 @@ waxiness/1Mwaxwing/1SMwaxwork/1SMwaxy/51RPT-way/145SM+way/145SM~waybill/14SMwayfarer/1MSwayfaring/514SM@@ -48507,7 +48507,7 @@ wayside/15SMwayward/5PYwaywardness/1Mwazoo/1S-we/8+we/8~we'd/we'll/we're/@@ -48618,7 +48618,7 @@ weld/14MDRBSZGwelder/1Mwelfare/14Mwelkin/1M-well/514MDPSG+well/514MDPSG~wellhead/1SMwellie/1wellington/1MS@@ -48725,7 +48725,7 @@ whetted/54whetting/41whew/4whey/1M-which/8+which/8~whichever/8whiff/145SMDGwhiffletree/1MS@@ -48805,7 +48805,7 @@ who'd/who'll/who're/who've/-who/81M+who/81M~whoa/4whodunit/1MSwhoever/8@@ -48903,7 +48903,7 @@ wildness/1Mwilds/14Mwile/14MGDSwiliness/1M-will/41MDS+will/41MDS~willful/5PYwillfulness/1Mwillies/1M@@ -49025,7 +49025,7 @@ wit/14SMwitch/14MDSGwitchcraft/1Mwitchery/1M-with/1+with/1~withal/withdraw/41SGwithdrawal/1MS@@ -49170,7 +49170,7 @@ wordsmiths/14wordy/5TPRwore/4work's-work/14ADJSG+work/14ADJSG~workable/5Uworkaday/51workaholic/15SM@@ -49249,7 +49249,7 @@ worthy/514UPRTwot/48wotcha/would've/-would/41S+would/41S~wouldn't/4wouldst/4wound/14SGMDR@@ -49418,7 +49418,7 @@ ye/81RSTyea/71SMyeah/1Myeahs/1-year/1MYS+year/1MYS~yearbook/1MSyearling/1MSyearlong/5@@ -49479,11 +49479,11 @@ you'd/you'll/you're/you've/-you/841SMH+you/841SMH~young/514TMRyoungish/5youngster/1MS-your/S+your/S~yourself/8yourselves/8youth/1Mdiff --git a/harper-core/src/spell/full_dictionary.rs b/harper-core/src/spell/full_dictionary.rsindex 30e117406..818ff1b85 100644--- a/harper-core/src/spell/full_dictionary.rs+++ b/harper-core/src/spell/full_dictionary.rs@@ -211,4 +211,10 @@ mod tests {let dict = FullDictionary::curated();assert!(dict.contains_word_str("natively"));}++ #[test]+ fn im_is_common() {+ let dict = FullDictionary::curated();+ assert!(dict.get_word_metadata_str("I'm").common);+ }}diff --git a/harper-core/src/spell/mod.rs b/harper-core/src/spell/mod.rsindex 929f3a8e7..ed495b806 100644--- a/harper-core/src/spell/mod.rs+++ b/harper-core/src/spell/mod.rs@@ -94,11 +94,20 @@ pub fn suggest_correct_spelling<'a>(found.extend(found_dist.into_iter().map(|v| v.0));}- // Finally, swap the lowest edit distance word with the shortest.+ // Swap the lowest edit distance word with the shortest.if found.len() >= 3 {found.swap(0, 2);}+ // Let common words bubble up.+ found.sort_by_key(|v| {+ if dictionary.get_word_metadata(v).common {+ 0+ } else {+ 1+ }+ });+found}@@ -221,4 +230,13 @@ mod tests {assert_eq!(results.iter().unique().count(), results.len())}++ #[test]+ fn issue_182() {+ let results = suggest_correct_spelling_str("im", 100, 3, &FullDictionary::curated());++ dbg!(&results);++ assert!(results.iter().take(3).contains(&"I'm".to_string()));+ }}diff --git a/harper-core/src/word_metadata.rs b/harper-core/src/word_metadata.rsindex bee6741df..7a37ffaa6 100644--- a/harper-core/src/word_metadata.rs+++ b/harper-core/src/word_metadata.rs@@ -8,7 +8,14 @@ pub struct WordMetadata {pub adjective: Option<AdjectiveData>,pub adverb: Option<AdverbData>,pub conjunction: Option<ConjunctionData>,- pub swear: Option<bool>+ pub swear: Option<bool>,+ /// Whether the word is considered especially common.+ #[serde(default = "default_common")]+ pub common: bool+}++fn default_common() -> bool {+ false}impl WordMetadata {@@ -31,7 +38,8 @@ impl WordMetadata {adjective: merge!(self.adjective, other.adjective),adverb: merge!(self.adverb, other.adverb),conjunction: merge!(self.conjunction, other.conjunction),- swear: self.swear.or(other.swear)+ swear: self.swear.or(other.swear),+ common: self.common || other.common}}