Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
fix: false positive spell-checking with contractions with typographic apostrophes
Elijah Potter2c17de5dfa73
diff --git a/demo.md b/demo.mdindex 83fbe3b49..3dc005f18 100644--- a/demo.md+++ b/demo.md@@ -1,9 +1,8 @@-Harper is a language checker for artists. it can detect-improper capitalization and misspelled words. There are-some cases, where the the standard grammar checkers-don't cut it.+There are some cases where the the standard grammar checkers+don't cut it. That s where Harper comes in handy.-That s where Harper comes in handy.+Harper is a language checker for developers. it can detect+improper capitalization and mispelled words.Harper works everywhere, even offline. Since you r datanever leaves your device, you don't ned to worry aout usdiff --git a/harper-core/dictionary.dict b/harper-core/dictionary.dictindex 018c22c40..f1ca2371a 100644--- a/harper-core/dictionary.dict+++ b/harper-core/dictionary.dict@@ -30673,7 +30673,7 @@ look/MDRSZGlookalike/MSlooker/Mlookout/MS-lookup+lookup/Sloom/MDSGloon/MSloonie/M@@ -49596,4 +49596,6 @@ Grammarly/MLanguageTool/MWebAssembly/MNeovim/M-quartile/s+quartile/S+autocomplete/MS+GPU/Sdiff --git a/harper-core/src/lexing/mod.rs b/harper-core/src/lexing/mod.rsindex e5d1d510f..48337a3d7 100644--- a/harper-core/src/lexing/mod.rs+++ b/harper-core/src/lexing/mod.rs@@ -117,6 +117,7 @@ fn lex_punctuation(source: &[char]) -> Option<FoundToken> {use Punctuation::*;let punct = match c {+ '@' => At,'~' => Tilde,'=' => Equal,'<' => LessThan,diff --git a/harper-core/src/linting/matcher.rs b/harper-core/src/linting/matcher.rsindex d0e9eee41..ee1d2864b 100644--- a/harper-core/src/linting/matcher.rs+++ b/harper-core/src/linting/matcher.rs@@ -93,6 +93,7 @@ pub struct Matcher {impl Matcher {pub fn new() -> Self {let mut triggers = pt! {+ "the","challenged" => "that challenged","stdin" => "standard input","stdout" => "standard output","your","doing" => "you're doing",diff --git a/harper-core/src/spell/dictionary.rs b/harper-core/src/spell/dictionary.rsindex 0abcd726d..ca6441f8c 100644--- a/harper-core/src/spell/dictionary.rs+++ b/harper-core/src/spell/dictionary.rs@@ -1,10 +1,12 @@+use std::borrow::Borrow;+use hashbrown::HashSet;use once_cell::sync::Lazy;use smallvec::SmallVec;use super::{hunspell::{parse_default_attribute_list, parse_default_word_list},- DictWord,+ seq_to_normalized, DictWord,};#[derive(Debug, Clone)]@@ -76,8 +78,9 @@ impl Dictionary {}pub fn contains_word(&self, word: &[char]) -> bool {- let lowercase: SmallVec<_> = word.iter().flat_map(|c| c.to_lowercase()).collect();+ let normalized = seq_to_normalized(word);+ let lowercase: SmallVec<_> = normalized.iter().flat_map(|c| c.to_lowercase()).collect();- self.word_set.contains(word) || self.word_set.contains(&lowercase)+ self.word_set.contains(normalized.as_ref()) || self.word_set.contains(&lowercase)}}diff --git a/harper-core/src/spell/mod.rs b/harper-core/src/spell/mod.rsindex 8000e41b0..d2f01a4db 100644--- a/harper-core/src/spell/mod.rs+++ b/harper-core/src/spell/mod.rs@@ -1,3 +1,5 @@+use std::borrow::Cow;+use itertools::{Itertools, MinMaxResult};use smallvec::SmallVec;@@ -18,6 +20,8 @@ pub fn suggest_correct_spelling<'a>(max_edit_dist: u8,dictionary: &'a Dictionary,) -> Vec<&'a [char]> {+ let misspelled_word = seq_to_normalized(misspelled_word);+let misspelled_lower: Vec<char> = misspelled_word.iter().flat_map(|v| v.to_lowercase())@@ -40,7 +44,7 @@ pub fn suggest_correct_spelling<'a>(.flat_map(|len| dictionary.words_with_len_iter(len));let pruned_words = words_to_search.filter_map(|word| {- let dist = edit_distance_min_alloc(misspelled_word, word, &mut buf_a, &mut buf_b);+ let dist = edit_distance_min_alloc(&misspelled_word, word, &mut buf_a, &mut buf_b);let dist_lower = edit_distance_min_alloc(&misspelled_lower, word, &mut buf_a, &mut buf_b);if dist.min(dist_lower) <= max_edit_dist {@@ -153,9 +157,26 @@ fn edit_distance(source: &[char], target: &[char]) -> u8 {edit_distance_min_alloc(source, target, &mut Vec::new(), &mut Vec::new())}+/// Convert a given character sequence to the standard character set+/// the dictionary is in.+fn seq_to_normalized(seq: &[char]) -> Cow<'_, [char]> {+ if seq.iter().any(|c| char_to_normalized(*c) != *c) {+ Cow::Owned(seq.iter().copied().map(char_to_normalized).collect())+ } else {+ Cow::Borrowed(seq)+ }+}++fn char_to_normalized(c: char) -> char {+ match c {+ '’' => '\'',+ _ => c,+ }+}+#[cfg(test)]mod tests {- use super::edit_distance;+ use super::{edit_distance, seq_to_normalized};fn assert_edit_dist(source: &str, target: &str, expected: u8) {let source: Vec<_> = source.chars().collect();@@ -165,6 +186,14 @@ mod tests {assert_eq!(dist, expected)}+ #[test]+ fn normalizes_weve() {+ let word = vec!['w', 'e', '’', 'v', 'e'];+ let norm = seq_to_normalized(&word);++ assert_eq!(norm.clone(), vec!['w', 'e', '\'', 'v', 'e'])+ }+#[test]fn simple1() {assert_edit_dist("kitten", "sitting", 3)diff --git a/harper-core/src/token.rs b/harper-core/src/token.rsindex 223ada310..eb98a2b7f 100644--- a/harper-core/src/token.rs+++ b/harper-core/src/token.rs@@ -71,50 +71,52 @@ impl TokenKind {#[derive(Debug, Is, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, PartialOrd)]#[serde(tag = "kind")]pub enum Punctuation {- /// .+ /// `.`Period,- /// !+ /// `!`Bang,- /// ?+ /// `?`Question,- /// :+ /// `:`Colon,- /// ;+ /// ``;``Semicolon,- /// "+ /// `"`Quote(Quote),- /// ,+ /// `,`Comma,- /// -+ /// `-`Hyphen,- /// [+ /// `[`OpenSquare,- /// ]+ /// `]`CloseSquare,- /// (+ /// `(`OpenRound,- /// )+ /// `)`CloseRound,- /// "+ /// `"`Hash,- /// '+ /// `'`Apostrophe,- /// %+ /// `%`Percent,- /// /+ /// `/`ForwardSlash,- /// \+ /// `\`Backslash,- /// <+ /// `<`LessThan,- /// >+ /// `>`GreaterThan,- /// =+ /// `=`Equal,- /// *+ /// `*`Star,- /// ~+ /// `~`Tilde,+ /// `@`+ At,}#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, PartialOrd)]diff --git a/web/src/routes/+page.svelte b/web/src/routes/+page.svelteindex ec7ae2940..3aaf88a7a 100644--- a/web/src/routes/+page.svelte+++ b/web/src/routes/+page.svelte@@ -13,7 +13,7 @@<div class="h-16" /><GutterCenter>- <h1 class="text-5xl font-bold text-center dark:text-white">Hi. I'm Harper.</h1>+ <h1 class="text-5xl font-bold text-center dark:text-white">Hi. I’m Harper.</h1><h2 class="text-3xl text-center dark:text-white">The Grammar Checker for Developers</h2><h2 class="text-2xl font-light italic text-center dark:text-white">Private. Free. Unobtrusive.