Skip to content

Offline, privacy-first grammar checker. Fast, open-source, Rust-powered

fix: false positive spell-checking with contractions with typographic apostrophes

Elijah Potter2c17de5dfa73
diff --git a/demo.md b/demo.md
index 83fbe3b49..3dc005f18 100644
--- a/demo.md
+++ b/demo.md
@@ -1,9 +1,8 @@
-Harper is a language checker for artists. it can detect
-improper capitalization and misspelled words. There are
-some cases, where the the standard grammar checkers
-don't cut it.
+There are some cases where the the standard grammar checkers
+don't cut it. That s where Harper comes in handy.
-That s where Harper comes in handy.
+Harper is a language checker for developers. it can detect
+improper capitalization and mispelled words.
Harper works everywhere, even offline. Since you r data
never leaves your device, you don't ned to worry aout us
diff --git a/harper-core/dictionary.dict b/harper-core/dictionary.dict
index 018c22c40..f1ca2371a 100644
--- a/harper-core/dictionary.dict
+++ b/harper-core/dictionary.dict
@@ -30673,7 +30673,7 @@ look/MDRSZG
lookalike/MS
looker/M
lookout/MS
-lookup
+lookup/S
loom/MDSG
loon/MS
loonie/M
@@ -49596,4 +49596,6 @@ Grammarly/M
LanguageTool/M
WebAssembly/M
Neovim/M
-quartile/s
+quartile/S
+autocomplete/MS
+GPU/S
diff --git a/harper-core/src/lexing/mod.rs b/harper-core/src/lexing/mod.rs
index e5d1d510f..48337a3d7 100644
--- a/harper-core/src/lexing/mod.rs
+++ b/harper-core/src/lexing/mod.rs
@@ -117,6 +117,7 @@ fn lex_punctuation(source: &[char]) -> Option<FoundToken> {
use Punctuation::*;
let punct = match c {
+ '@' => At,
'~' => Tilde,
'=' => Equal,
'<' => LessThan,
diff --git a/harper-core/src/linting/matcher.rs b/harper-core/src/linting/matcher.rs
index d0e9eee41..ee1d2864b 100644
--- a/harper-core/src/linting/matcher.rs
+++ b/harper-core/src/linting/matcher.rs
@@ -93,6 +93,7 @@ pub struct Matcher {
impl Matcher {
pub fn new() -> Self {
let mut triggers = pt! {
+ "the","challenged" => "that challenged",
"stdin" => "standard input",
"stdout" => "standard output",
"your","doing" => "you're doing",
diff --git a/harper-core/src/spell/dictionary.rs b/harper-core/src/spell/dictionary.rs
index 0abcd726d..ca6441f8c 100644
--- a/harper-core/src/spell/dictionary.rs
+++ b/harper-core/src/spell/dictionary.rs
@@ -1,10 +1,12 @@
+use std::borrow::Borrow;
+
use hashbrown::HashSet;
use once_cell::sync::Lazy;
use smallvec::SmallVec;
use super::{
hunspell::{parse_default_attribute_list, parse_default_word_list},
- DictWord,
+ seq_to_normalized, DictWord,
};
#[derive(Debug, Clone)]
@@ -76,8 +78,9 @@ impl Dictionary {
}
pub fn contains_word(&self, word: &[char]) -> bool {
- let lowercase: SmallVec<_> = word.iter().flat_map(|c| c.to_lowercase()).collect();
+ let normalized = seq_to_normalized(word);
+ let lowercase: SmallVec<_> = normalized.iter().flat_map(|c| c.to_lowercase()).collect();
- self.word_set.contains(word) || self.word_set.contains(&lowercase)
+ self.word_set.contains(normalized.as_ref()) || self.word_set.contains(&lowercase)
}
}
diff --git a/harper-core/src/spell/mod.rs b/harper-core/src/spell/mod.rs
index 8000e41b0..d2f01a4db 100644
--- a/harper-core/src/spell/mod.rs
+++ b/harper-core/src/spell/mod.rs
@@ -1,3 +1,5 @@
+use std::borrow::Cow;
+
use itertools::{Itertools, MinMaxResult};
use smallvec::SmallVec;
@@ -18,6 +20,8 @@ pub fn suggest_correct_spelling<'a>(
max_edit_dist: u8,
dictionary: &'a Dictionary,
) -> Vec<&'a [char]> {
+ let misspelled_word = seq_to_normalized(misspelled_word);
+
let misspelled_lower: Vec<char> = misspelled_word
.iter()
.flat_map(|v| v.to_lowercase())
@@ -40,7 +44,7 @@ pub fn suggest_correct_spelling<'a>(
.flat_map(|len| dictionary.words_with_len_iter(len));
let pruned_words = words_to_search.filter_map(|word| {
- let dist = edit_distance_min_alloc(misspelled_word, word, &mut buf_a, &mut buf_b);
+ let dist = edit_distance_min_alloc(&misspelled_word, word, &mut buf_a, &mut buf_b);
let dist_lower = edit_distance_min_alloc(&misspelled_lower, word, &mut buf_a, &mut buf_b);
if dist.min(dist_lower) <= max_edit_dist {
@@ -153,9 +157,26 @@ fn edit_distance(source: &[char], target: &[char]) -> u8 {
edit_distance_min_alloc(source, target, &mut Vec::new(), &mut Vec::new())
}
+/// Convert a given character sequence to the standard character set
+/// the dictionary is in.
+fn seq_to_normalized(seq: &[char]) -> Cow<'_, [char]> {
+ if seq.iter().any(|c| char_to_normalized(*c) != *c) {
+ Cow::Owned(seq.iter().copied().map(char_to_normalized).collect())
+ } else {
+ Cow::Borrowed(seq)
+ }
+}
+
+fn char_to_normalized(c: char) -> char {
+ match c {
+ '’' => '\'',
+ _ => c,
+ }
+}
+
#[cfg(test)]
mod tests {
- use super::edit_distance;
+ use super::{edit_distance, seq_to_normalized};
fn assert_edit_dist(source: &str, target: &str, expected: u8) {
let source: Vec<_> = source.chars().collect();
@@ -165,6 +186,14 @@ mod tests {
assert_eq!(dist, expected)
}
+ #[test]
+ fn normalizes_weve() {
+ let word = vec!['w', 'e', '’', 'v', 'e'];
+ let norm = seq_to_normalized(&word);
+
+ assert_eq!(norm.clone(), vec!['w', 'e', '\'', 'v', 'e'])
+ }
+
#[test]
fn simple1() {
assert_edit_dist("kitten", "sitting", 3)
diff --git a/harper-core/src/token.rs b/harper-core/src/token.rs
index 223ada310..eb98a2b7f 100644
--- a/harper-core/src/token.rs
+++ b/harper-core/src/token.rs
@@ -71,50 +71,52 @@ impl TokenKind {
#[derive(Debug, Is, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, PartialOrd)]
#[serde(tag = "kind")]
pub enum Punctuation {
- /// .
+ /// `.`
Period,
- /// !
+ /// `!`
Bang,
- /// ?
+ /// `?`
Question,
- /// :
+ /// `:`
Colon,
- /// ;
+ /// ``;``
Semicolon,
- /// "
+ /// `"`
Quote(Quote),
- /// ,
+ /// `,`
Comma,
- /// -
+ /// `-`
Hyphen,
- /// [
+ /// `[`
OpenSquare,
- /// ]
+ /// `]`
CloseSquare,
- /// (
+ /// `(`
OpenRound,
- /// )
+ /// `)`
CloseRound,
- /// "
+ /// `"`
Hash,
- /// '
+ /// `'`
Apostrophe,
- /// %
+ /// `%`
Percent,
- /// /
+ /// `/`
ForwardSlash,
- /// \
+ /// `\`
Backslash,
- /// <
+ /// `<`
LessThan,
- /// >
+ /// `>`
GreaterThan,
- /// =
+ /// `=`
Equal,
- /// *
+ /// `*`
Star,
- /// ~
+ /// `~`
Tilde,
+ /// `@`
+ At,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, PartialOrd)]
diff --git a/web/src/routes/+page.svelte b/web/src/routes/+page.svelte
index ec7ae2940..3aaf88a7a 100644
--- a/web/src/routes/+page.svelte
+++ b/web/src/routes/+page.svelte
@@ -13,7 +13,7 @@
<div class="h-16" />
<GutterCenter>
- <h1 class="text-5xl font-bold text-center dark:text-white">Hi. I'm Harper.</h1>
+ <h1 class="text-5xl font-bold text-center dark:text-white">Hi. I’m Harper.</h1>
<h2 class="text-3xl text-center dark:text-white">The Grammar Checker for Developers</h2>
<h2 class="text-2xl font-light italic text-center dark:text-white">
Private. Free. Unobtrusive.