Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
feat(core): created framework for correcting common phrases
Elijah Potterec39203a6a2b
diff --git a/harper-core/src/edit_distance.rs b/harper-core/src/edit_distance.rsnew file mode 100644index 000000000..746e420b7--- /dev/null+++ b/harper-core/src/edit_distance.rs@@ -0,0 +1,92 @@+// Computes the Levenshtein edit distance between two patterns.+// This is accomplished via a memory-optimized Wagner-Fischer algorithm+//+// This variant avoids allocation if you already have buffers.+#[inline]+pub fn edit_distance_min_alloc(+ source: &[char],+ target: &[char],+ previous_row: &mut Vec<u8>,+ current_row: &mut Vec<u8>,+) -> u8 {+ if cfg!(debug_assertions) {+ assert!(source.len() <= 255 && target.len() <= 255);+ }++ let row_width = source.len();+ let col_height = target.len();++ previous_row.clear();+ previous_row.extend(0u8..=row_width as u8);+ // Alright if not zeroed, since we overwrite it anyway.+ current_row.resize(row_width + 1, 0);++ for j in 1..=col_height {+ current_row[0] = j as u8;++ for i in 1..=row_width {+ let cost = if source[i - 1] == target[j - 1] { 0 } else { 1 };++ current_row[i] = (previous_row[i] + 1)+ .min(current_row[i - 1] + 1)+ .min(previous_row[i - 1] + cost);+ }++ std::mem::swap(previous_row, current_row);+ }++ previous_row[row_width]+}++pub fn edit_distance(source: &[char], target: &[char]) -> u8 {+ edit_distance_min_alloc(source, target, &mut Vec::new(), &mut Vec::new())+}++#[cfg(test)]+mod tests {+ use super::edit_distance;++ fn assert_edit_dist(source: &str, target: &str, expected: u8) {+ let source: Vec<_> = source.chars().collect();+ let target: Vec<_> = target.chars().collect();++ let dist = edit_distance(&source, &target);+ assert_eq!(dist, expected)+ }++ #[test]+ fn simple_edit_distance_1() {+ assert_edit_dist("kitten", "sitting", 3)+ }++ #[test]+ fn simple_edit_distance_2() {+ assert_edit_dist("saturday", "sunday", 3)+ }++ #[test]+ fn one_edit_distance() {+ let source: Vec<_> = "hello".chars().collect();+ let target: Vec<_> = "hellos".chars().collect();+ assert_eq!(edit_distance(&source, &target), 1);++ let target: Vec<_> = "hell".chars().collect();+ assert_eq!(edit_distance(&source, &target), 1);++ let target: Vec<_> = "hell".chars().collect();+ assert_eq!(edit_distance(&source, &target), 1);++ let target: Vec<_> = "hvllo".chars().collect();+ assert_eq!(edit_distance(&source, &target), 1);++ let target: Vec<_> = "Hello".chars().collect();+ assert_eq!(edit_distance(&source, &target), 1);+ }++ #[test]+ fn zero_edit_distance() {+ let source: Vec<_> = "hello".chars().collect();+ let target: Vec<_> = "hello".chars().collect();+ assert_eq!(edit_distance(&source, &target), 0);+ }+}diff --git a/harper-core/src/lib.rs b/harper-core/src/lib.rsindex 37801a975..19424b8d4 100644--- a/harper-core/src/lib.rs+++ b/harper-core/src/lib.rs@@ -5,6 +5,7 @@ mod char_ext;mod char_string;mod currency;mod document;+mod edit_distance;mod fat_token;pub mod language_detection;mod lexing;diff --git a/harper-core/src/linting/lint_group.rs b/harper-core/src/linting/lint_group.rsindex 4f2563c67..363727827 100644--- a/harper-core/src/linting/lint_group.rs+++ b/harper-core/src/linting/lint_group.rs@@ -16,6 +16,9 @@ use super::matcher::Matcher;use super::merge_words::MergeWords;use super::multiple_sequential_pronouns::MultipleSequentialPronouns;use super::number_suffix_capitalization::NumberSuffixCapitalization;+use super::phrase_corrections::{+ AndThis, Decision, HumanLife, NeedHelp, NoLonger, ThatChallenged, TurnItOff,+};use super::plural_conjugate::PluralConjugate;use super::pronoun_contraction::PronounContraction;use super::proper_noun_capitalization_linters::{@@ -195,6 +198,13 @@ create_lint_group_config!(SomewhatSomething => true,LetsConfusion => true,DespiteOf => true,+ AndThis => true,+ Decision => true,+ HumanLife => true,+ NeedHelp => true,+ NoLonger => true,+ ThatChallenged => true,+ TurnItOff => true);impl<T: Dictionary + Default> Default for LintGroup<T> {diff --git a/harper-core/src/linting/matcher.rs b/harper-core/src/linting/matcher.rsindex eb6d81119..dc59e4f74 100644--- a/harper-core/src/linting/matcher.rs+++ b/harper-core/src/linting/matcher.rs@@ -111,7 +111,6 @@ impl Matcher {"off","the","cuff" => "off-the-cuff","an","in" => "and in","my","self" => "myself",- "human","live" => "human life","eight","grade" => "eighth grade","and","also" => "and","todo" => "to-do",@@ -175,18 +174,12 @@ impl Matcher {"that","s" => "that is","That","s" => "that is","ms" => "milliseconds",- "the","hing" => "the thing",- "The","hing" => "The thing",- "need","helps" => "need help",- "an","this" => "and this","case", "sensitive" => "case-sensitive","Tree", "sitter" => "Tree-sitter","all", "of", "the" => "all the",- "not", "longer" => "no longer","to", "towards" => "towards","though", "process" => "thought process","the", "this" => "that this",- "take", "a", "decision" => "make a decision","same", "than" => "same as","Same", "than" => "same as","same", "then" => "same as",diff --git a/harper-core/src/linting/mod.rs b/harper-core/src/linting/mod.rsindex 6f4f66daf..a7a1aa18e 100644--- a/harper-core/src/linting/mod.rs+++ b/harper-core/src/linting/mod.rs@@ -24,6 +24,7 @@ mod multiple_sequential_pronouns;mod number_suffix_capitalization;mod oxford_comma;mod pattern_linter;+mod phrase_corrections;mod plural_conjugate;mod pronoun_contraction;mod proper_noun_capitalization_linters;@@ -59,6 +60,9 @@ pub use multiple_sequential_pronouns::MultipleSequentialPronouns;pub use number_suffix_capitalization::NumberSuffixCapitalization;pub use oxford_comma::OxfordComma;pub use pattern_linter::PatternLinter;+pub use phrase_corrections::{+ AndThis, Decision, HumanLife, NeedHelp, NoLonger, ThatChallenged, TurnItOff,+};pub use plural_conjugate::PluralConjugate;pub use pronoun_contraction::PronounContraction;pub use proper_noun_capitalization_linters::{diff --git a/harper-core/src/linting/phrase_corrections.rs b/harper-core/src/linting/phrase_corrections.rsnew file mode 100644index 000000000..b3ee9b323--- /dev/null+++ b/harper-core/src/linting/phrase_corrections.rs@@ -0,0 +1,112 @@+use super::{Lint, LintKind, PatternLinter};+use crate::linting::Suggestion;+use crate::patterns::{Pattern, SimilarToPhrase};+use crate::{Token, TokenStringExt};++/// Generate a linter that will look for a common phrase and correct mild errors that+/// are still composed of real words.+macro_rules! create_linter_for_phrase {+ ($name:ident, $correct_form:literal, $dist:literal) => {+ create_linter_for_phrase!(+ $name,+ SimilarToPhrase::from_phrase($correct_form, $dist),+ $correct_form,+ concat!("Did you mean the phrase `", $correct_form, "`?"),+ concat!(+ "Looks for slight improper modifications to the phrase `",+ $correct_form,+ "`."+ )+ );+ };+ ($name:ident, $pattern:expr, $correct_form:expr, $message:expr, $description:expr) => {+ #[doc = $description]+ pub struct $name {+ pattern: Box<dyn Pattern>,+ }++ impl $name {+ pub fn new() -> Self {+ Self {+ pattern: Box::new($pattern),+ }+ }+ }++ impl Default for $name {+ fn default() -> Self {+ Self::new()+ }+ }++ impl PatternLinter for $name {+ fn pattern(&self) -> &dyn Pattern {+ self.pattern.as_ref()+ }++ fn match_to_lint(&self, matched_tokens: &[Token], source: &[char]) -> Lint {+ let span = matched_tokens.span().unwrap();+ let matched_text = span.get_content(source);++ Lint {+ span,+ lint_kind: LintKind::Miscellaneous,+ suggestions: vec![Suggestion::replace_with_match_case(+ $correct_form.chars().collect(),+ matched_text,+ )],+ message: $message.to_string(),+ priority: 31,+ }+ }++ fn description(&self) -> &'static str {+ $description+ }+ }+ };+}++create_linter_for_phrase!(TurnItOff, "turn it off", 2);+create_linter_for_phrase!(HumanLife, "human life", 2);+create_linter_for_phrase!(ThatChallenged, "that challenged", 2);+create_linter_for_phrase!(NoLonger, "no longer", 1);+create_linter_for_phrase!(NeedHelp, "need help", 1);+create_linter_for_phrase!(AndThis, "and this", 1);+create_linter_for_phrase!(Decision, "make a decision", 1);++#[cfg(test)]+mod tests {+ use crate::linting::tests::{assert_lint_count, assert_suggestion_result};++ use super::{Decision, TurnItOff};++ #[test]+ fn turn_it_off_clean_lower() {+ assert_lint_count("turn it off", TurnItOff::default(), 0);+ }++ #[test]+ fn turn_it_off_clean_upper() {+ assert_lint_count("Turn it off", TurnItOff::default(), 0);+ }++ #[test]+ fn of_confusion() {+ assert_suggestion_result("Turn it of", TurnItOff::default(), "Turn it off");+ }++ #[test]+ fn i_and_of_confusion() {+ assert_suggestion_result("Turn i of", TurnItOff::default(), "Turn it off");+ }++ #[test]+ fn take_a_decision() {+ assert_suggestion_result(+ "we should take a decision on this",+ Decision::default(),+ "we should make a decision on this",+ );+ }+}diff --git a/harper-core/src/patterns/all.rs b/harper-core/src/patterns/all.rsnew file mode 100644index 000000000..a4ed834d0--- /dev/null+++ b/harper-core/src/patterns/all.rs@@ -0,0 +1,38 @@+use crate::Token;++use super::Pattern;++/// A [`Pattern`] that consumes a list of other patterns and only+/// matches if all the child patterns do.+///+/// It will match the length of the longest pattern.+#[derive(Default)]+pub struct All {+ children: Vec<Box<dyn Pattern>>,+}++impl All {+ pub fn add(&mut self, p: Box<dyn Pattern>) {+ self.children.push(p);+ }+}++impl Pattern for All {+ fn matches(&self, tokens: &[Token], source: &[char]) -> usize {+ let mut max = 0;++ for pattern in &self.children {+ let len = pattern.matches(tokens, source);++ if len == 0 {+ return 0;+ }++ if len > max {+ max = len;+ }+ }++ max+ }+}diff --git a/harper-core/src/patterns/any_capitalization.rs b/harper-core/src/patterns/any_capitalization.rsnew file mode 100644index 000000000..eab611fbd--- /dev/null+++ b/harper-core/src/patterns/any_capitalization.rs@@ -0,0 +1,49 @@+use crate::{CharString, Token};++use super::Pattern;++/// A [`Pattern`] that matches any capitalization of a provided word.+pub struct AnyCapitalization {+ word: CharString,+}++impl AnyCapitalization {+ pub fn new(word: CharString) -> Self {+ Self { word }+ }++ pub fn from_string(word: &str) -> Self {+ let chars = word.chars().collect();++ Self::new(chars)+ }+}++impl Pattern for AnyCapitalization {+ fn matches(&self, tokens: &[Token], source: &[char]) -> usize {+ let Some(tok) = tokens.first() else {+ return 0;+ };++ if !tok.kind.is_word() {+ return 0;+ }++ let tok_chars = tok.span.get_content(source);++ if tok_chars.len() != self.word.len() {+ return 0;+ }++ let partial_match = tok_chars+ .iter()+ .zip(&self.word)+ .all(|(a, b)| a.eq_ignore_ascii_case(b));++ if partial_match {+ 1+ } else {+ 0+ }+ }+}diff --git a/harper-core/src/patterns/mod.rs b/harper-core/src/patterns/mod.rsindex e71e70baf..6422a4166 100644--- a/harper-core/src/patterns/mod.rs+++ b/harper-core/src/patterns/mod.rs@@ -9,6 +9,8 @@ use std::collections::VecDeque;use crate::{Document, Span, Token, VecExt};+mod all;+mod any_capitalization;mod any_pattern;mod consumes_remaining_pattern;mod either_pattern;@@ -18,11 +20,15 @@ mod naive_pattern_group;mod noun_phrase;mod repeating_pattern;mod sequence_pattern;+mod similar_to_phrase;mod token_kind_pattern_group;mod whitespace_pattern;+mod within_edit_distance;mod word_pattern_group;mod word_set;+pub use all::All;+pub use any_capitalization::AnyCapitalization;pub use any_pattern::AnyPattern;use blanket::blanket;pub use consumes_remaining_pattern::ConsumesRemainingPattern;@@ -33,6 +39,7 @@ pub use naive_pattern_group::NaivePatternGroup;pub use noun_phrase::NounPhrase;pub use repeating_pattern::RepeatingPattern;pub use sequence_pattern::SequencePattern;+pub use similar_to_phrase::SimilarToPhrase;pub use token_kind_pattern_group::TokenKindPatternGroup;pub use whitespace_pattern::WhitespacePattern;pub use word_pattern_group::WordPatternGroup;diff --git a/harper-core/src/patterns/sequence_pattern.rs b/harper-core/src/patterns/sequence_pattern.rsindex ef5386a00..3413c9a4a 100644--- a/harper-core/src/patterns/sequence_pattern.rs+++ b/harper-core/src/patterns/sequence_pattern.rs@@ -2,8 +2,9 @@ use hashbrown::HashSet;use paste::paste;use super::whitespace_pattern::WhitespacePattern;-use super::{NounPhrase, Pattern, RepeatingPattern, WordSet};-use crate::{CharStringExt, Lrc, Token, TokenKind};+use super::{AnyCapitalization, NounPhrase, Pattern, RepeatingPattern, WordSet};+use crate::Lrc;+use crate::{CharStringExt, Token, TokenKind};/// A pattern that checks that a sequence of other patterns match./// There are specific extension methods available, but you can also use [`Self::then`] to add@@ -131,24 +132,7 @@ impl SequencePattern {/// Match examples of `word` that have any capitalization.pub fn then_any_capitalization_of(mut self, word: &'static str) -> Self {self.token_patterns- .push(Box::new(|tok: &Token, source: &[char]| {- if !tok.kind.is_word() {- return false;- }-- let tok_chars = tok.span.get_content(source);-- if tok_chars.len() != word.chars().count() {- return false;- }-- let partial_match = tok_chars- .iter()- .zip(word.chars())- .all(|(a, b)| a.eq_ignore_ascii_case(&b));-- partial_match- }));+ .push(Box::new(AnyCapitalization::from_string(word)));self}diff --git a/harper-core/src/patterns/similar_to_phrase.rs b/harper-core/src/patterns/similar_to_phrase.rsnew file mode 100644index 000000000..333d80969--- /dev/null+++ b/harper-core/src/patterns/similar_to_phrase.rs@@ -0,0 +1,76 @@+use crate::{Document, Token, TokenKind};++use super::{+ within_edit_distance::WithinEditDistance, AnyCapitalization, Pattern, SequencePattern,+};++pub struct SimilarToPhrase {+ phrase: SequencePattern,+ fuzzy_phrase: SequencePattern,+}++impl SimilarToPhrase {+ /// Create an error-tolerant SequencePattern that looks for phrases similar to (but not the same as) that contained+ /// in the provided text.+ ///+ /// This is an expensive operation, so try to only do it at startup and in tests.+ ///+ /// It will panic if your document is too complex, so only run this with curated phrases.+ pub fn from_phrase(text: &str, max_edit_dist: u8) -> Self {+ let document = Document::new_plain_english_curated(text);++ Self::from_doc(&document, max_edit_dist)+ }++ /// Create an error-tolerant SequencePattern that looks for phrases similar to (but not the same as) that contained+ /// in the provided document.+ ///+ /// This is an expensive operation, so try to only do it at startup and in tests.+ ///+ /// It will panic if your document contains certain token types, so only run this with curated phrases.+ pub fn from_doc(document: &Document, max_edit_dist: u8) -> Self {+ let mut phrase = SequencePattern::default();+ let mut fuzzy_phrase = SequencePattern::default();++ for token in document.fat_tokens() {+ match token.kind {+ TokenKind::Word(_word_metadata) => {+ phrase = phrase.then(Box::new(AnyCapitalization::new(+ token.content.as_slice().into(),+ )));+ fuzzy_phrase = fuzzy_phrase.then(Box::new(WithinEditDistance::new(+ token.content.into(),+ max_edit_dist,+ )));+ }+ TokenKind::Space(_) => {+ fuzzy_phrase = fuzzy_phrase.then_whitespace();+ phrase = phrase.then_whitespace();+ }+ TokenKind::ParagraphBreak => {+ fuzzy_phrase = fuzzy_phrase.then_whitespace();+ phrase = phrase.then_whitespace();+ }+ _ => panic!("Fell out of expected document formats."),+ }+ }++ Self {+ phrase,+ fuzzy_phrase,+ }+ }+}++impl Pattern for SimilarToPhrase {+ fn matches(&self, tokens: &[Token], source: &[char]) -> usize {+ let exact_match = self.phrase.matches(tokens, source);+ let fuzzy_match = self.fuzzy_phrase.matches(tokens, source);++ if (exact_match == 0) && fuzzy_match > 0 {+ exact_match.max(fuzzy_match)+ } else {+ 0+ }+ }+}diff --git a/harper-core/src/patterns/within_edit_distance.rs b/harper-core/src/patterns/within_edit_distance.rsnew file mode 100644index 000000000..9cc2c6690--- /dev/null+++ b/harper-core/src/patterns/within_edit_distance.rs@@ -0,0 +1,45 @@+use super::Pattern;+use crate::{CharString, Token, TokenKind};++use crate::edit_distance::edit_distance;++/// A [`Pattern`] that matches single words within a certain edit distance of a given word.+pub struct WithinEditDistance {+ word: CharString,+ max_edit_dist: u8,+}++impl WithinEditDistance {+ pub fn new(word: CharString, max_edit_dist: u8) -> Self {+ Self {+ word,+ max_edit_dist,+ }+ }++ pub fn from_str(word: &str, edit_dist: u8) -> Self {+ let chars = word.chars().collect();++ Self::new(chars, edit_dist)+ }+}++impl Pattern for WithinEditDistance {+ fn matches(&self, tokens: &[Token], source: &[char]) -> usize {+ let Some(first) = tokens.first() else {+ return 0;+ };++ let TokenKind::Word(_) = first.kind else {+ return 0;+ };++ let content = first.span.get_content(source);++ if edit_distance(content, &self.word) <= self.max_edit_dist {+ 1+ } else {+ 0+ }+ }+}diff --git a/harper-core/src/spell/full_dictionary.rs b/harper-core/src/spell/full_dictionary.rsindex cc20e1261..3f18c47d6 100644--- a/harper-core/src/spell/full_dictionary.rs+++ b/harper-core/src/spell/full_dictionary.rs@@ -1,8 +1,8 @@use super::{- edit_distance_min_alloc,hunspell::{parse_default_attribute_list, parse_default_word_list},seq_to_normalized,};+use crate::edit_distance::edit_distance_min_alloc;use hashbrown::HashMap;use itertools::Itertools;use lazy_static::lazy_static;diff --git a/harper-core/src/spell/mod.rs b/harper-core/src/spell/mod.rsindex bc21f4b8f..70c67eb0a 100644--- a/harper-core/src/spell/mod.rs+++ b/harper-core/src/spell/mod.rs@@ -122,50 +122,6 @@ fn char_to_normalized(c: char) -> char {}}-// Computes the Levenshtein edit distance between two patterns.-// This is accomplished via a memory-optimized Wagner-Fischer algorithm-//-// This variant avoids allocation if you already have buffers.-#[inline]-fn edit_distance_min_alloc(- source: &[char],- target: &[char],- previous_row: &mut Vec<u8>,- current_row: &mut Vec<u8>,-) -> u8 {- if cfg!(debug_assertions) {- assert!(source.len() <= 255 && target.len() <= 255);- }-- let row_width = source.len();- let col_height = target.len();-- previous_row.clear();- previous_row.extend(0u8..=row_width as u8);- // Alright if not zeroed, since we overwrite it anyway.- current_row.resize(row_width + 1, 0);-- for j in 1..=col_height {- current_row[0] = j as u8;-- for i in 1..=row_width {- let cost = if source[i - 1] == target[j - 1] { 0 } else { 1 };-- current_row[i] = (previous_row[i] + 1)- .min(current_row[i - 1] + 1)- .min(previous_row[i - 1] + cost);- }-- std::mem::swap(previous_row, current_row);- }-- previous_row[row_width]-}--fn edit_distance(source: &[char], target: &[char]) -> u8 {- edit_distance_min_alloc(source, target, &mut Vec::new(), &mut Vec::new())-}-#[cfg(test)]mod tests {use itertools::Itertools;@@ -173,21 +129,13 @@ mod tests {use crate::spell::FuzzyMatchResult;use super::{- edit_distance, order_suggestions, seq_to_normalized, suggest_correct_spelling_str,- Dictionary, FstDictionary, FullDictionary,+ order_suggestions, seq_to_normalized, suggest_correct_spelling_str, Dictionary,+ FstDictionary, FullDictionary,};const RESULT_LIMIT: usize = 60;const MAX_EDIT_DIST: u8 = 3;- fn assert_edit_dist(source: &str, target: &str, expected: u8) {- let source: Vec<_> = source.chars().collect();- let target: Vec<_> = target.chars().collect();-- let dist = edit_distance(&source, &target);- assert_eq!(dist, expected)- }-#[test]fn normalizes_weve() {let word = vec!['w', 'e', '’', 'v', 'e'];@@ -196,16 +144,6 @@ mod tests {assert_eq!(norm.clone(), vec!['w', 'e', '\'', 'v', 'e'])}- #[test]- fn simple_edit_distance_1() {- assert_edit_dist("kitten", "sitting", MAX_EDIT_DIST)- }-- #[test]- fn simple_edit_distance_2() {- assert_edit_dist("saturday", "sunday", MAX_EDIT_DIST)- }-#[test]fn produces_no_duplicates() {let results = suggest_correct_spelling_str(@@ -220,32 +158,6 @@ mod tests {assert_eq!(results.iter().unique().count(), results.len())}- #[test]- fn zero_edit_distance() {- let source: Vec<_> = "hello".chars().collect();- let target: Vec<_> = "hello".chars().collect();- assert_eq!(edit_distance(&source, &target), 0);- }-- #[test]- fn one_edit_distance() {- let source: Vec<_> = "hello".chars().collect();- let target: Vec<_> = "hellos".chars().collect();- assert_eq!(edit_distance(&source, &target), 1);-- let target: Vec<_> = "hell".chars().collect();- assert_eq!(edit_distance(&source, &target), 1);-- let target: Vec<_> = "hell".chars().collect();- assert_eq!(edit_distance(&source, &target), 1);-- let target: Vec<_> = "hvllo".chars().collect();- assert_eq!(edit_distance(&source, &target), 1);-- let target: Vec<_> = "Hello".chars().collect();- assert_eq!(edit_distance(&source, &target), 1);- }-/// Ensures that the suggestions are ordered taking into account commonality#[test]fn common_words_first() {