Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
feat: add spellcheck backoff to improve desperate situations
Elijah Potteraea25e43281a
diff --git a/README.md b/README.mdindex 49b60044c..8a3d33005 100644--- a/README.md+++ b/README.md@@ -12,7 +12,7 @@ Grammarly was too expensive and too overbearing.It's suggestions lacked context, and were often just plain _wrong_.Not to mention: it's a privacy nightmare.Everything you write with Grammarly is sent to their servers.-Their privacy policy claims they don't sell the data, but that doesn't mean they don't use it to train large language models god knows what else.+Their privacy policy claims they don't sell the data, but that doesn't mean they don't use it to train large language models and god knows what else.Not only that, but the round-trip-time of the network request makes revising your work all the more tedious.LanguageTool is great, if you have gigabytes of RAM to spare and are willing to download the ~16GB n-gram dataset.@@ -29,4 +29,3 @@ Harper is even small enough to load via [WebAssembly.](https://harper.elijahpottIf you want to use Harper on your machine, you will want to look at the [documentation for`harper-ls`](./harper-ls/README.md), the Language Server Protocol implementation.-diff --git a/harper-core/benches/parse_demo.rs b/harper-core/benches/parse_demo.rsindex f8da88dae..f049133bd 100644--- a/harper-core/benches/parse_demo.rs+++ b/harper-core/benches/parse_demo.rs@@ -33,6 +33,17 @@ fn lint_demo(bencher: Bencher) {});}+#[divan::bench]+fn lint_demo_uncached(bencher: Bencher) {+ let dictionary = FullDictionary::create_from_curated();+ bencher.bench_local(|| {+ let mut lint_set = LintSet::new().with_standard(dictionary.clone());+ let document = Document::new_markdown(black_box(DEMO));++ lint_set.lint(&document);+ });+}+fn main() {divan::main();}diff --git a/harper-core/dictionary.dict b/harper-core/dictionary.dictindex 81c842f2c..41652f3d2 100644--- a/harper-core/dictionary.dict+++ b/harper-core/dictionary.dict@@ -49605,3 +49605,4 @@ codebase/SLIDARLiDARLADAR+TODOdiff --git a/harper-core/src/document.rs b/harper-core/src/document.rsindex b9d249c1f..bda3393fe 100644--- a/harper-core/src/document.rs+++ b/harper-core/src/document.rs@@ -3,6 +3,7 @@ use std::fmt::Display;use itertools::Itertools;use crate::parsers::{Markdown, Parser, PlainEnglish};+use crate::TokenStringExt;use crate::{linting::Suggestion,span::Span,@@ -46,24 +47,116 @@ impl Document {/// Should be run after every change to the underlying [`Self::source`].fn parse(&mut self) {self.tokens = self.parser.parse(&self.source);-self.condense_contractions();- // Since quote matches depend on token indices.self.match_quotes();}- pub fn iter_quote_indices(&self) -> impl Iterator<Item = usize> + '_ {- self.tokens.iter().enumerate().filter_map(|(idx, token)| {- if let TokenKind::Punctuation(Punctuation::Quote(_)) = &token.kind {- Some(idx)+ /// Defensively attempt to grab a specific token.+ pub fn get_token(&self, index: usize) -> Option<Token> {+ self.tokens.get(index).copied()+ }++ pub fn tokens(&self) -> impl Iterator<Item = Token> + '_ {+ self.tokens.iter().copied()+ }++ pub fn fat_tokens(&self) -> impl Iterator<Item = FatToken> + '_ {+ self.tokens().map(|token| token.to_fat(&self.source))+ }++ /// Iterate over the locations of the sentence terminators in the document.+ fn sentence_terminators(&self) -> impl Iterator<Item = usize> + '_ {+ self.tokens.iter().enumerate().filter_map(|(index, token)| {+ if is_sentence_terminator(&token.kind) {+ return Some(index);+ }+ None+ })+ }++ /// Get the index of the last sentence terminator.+ fn last_sentence_terminator(&self) -> Option<usize> {+ self.tokens+ .iter()+ .enumerate()+ .rev()+ .find_map(|(index, token)| {+ if is_sentence_terminator(&token.kind) {+ return Some(index);+ }+ None+ })+ }++ pub fn sentences(&self) -> impl Iterator<Item = &'_ [Token]> + '_ {+ let first_sentence = self+ .sentence_terminators()+ .next()+ .map(|first_term| &self.tokens[0..=first_term]);++ let rest = self+ .sentence_terminators()+ .tuple_windows()+ .map(move |(a, b)| &self.tokens[a + 1..=b]);++ let last = if let Some(last_i) = self.last_sentence_terminator() {+ if last_i + 1 < self.tokens.len() {+ Some(&self.tokens[last_i + 1..])} else {None}+ } else {+ Some(self.tokens.as_slice())+ };++ first_sentence.into_iter().chain(rest).chain(last)+ }++ /** Returns all tokens whose `kind` is [`Punctuation::Word`] */+ pub fn words(&self) -> impl Iterator<Item = Token> + '_ {+ self.tokens+ .iter()+ .filter(|token| token.kind.is_word())+ .cloned()+ }++ pub fn get_span_content(&self, span: Span) -> &[char] {+ span.get_content(&self.source)+ }++ pub fn get_span_content_str(&self, span: Span) -> String {+ String::from_iter(self.get_span_content(span))+ }++ pub fn get_full_string(&self) -> String {+ self.get_span_content_str(Span {+ start: 0,+ end: self.source.len(),})}- pub fn iter_quotes(&self) -> impl Iterator<Item = Token> + '_ {- self.iter_quote_indices().map(|idx| self.tokens[idx])+ pub fn get_full_content(&self) -> &[char] {+ &self.source+ }++ pub fn apply_suggestion(&mut self, suggestion: &Suggestion, span: Span) {+ match suggestion {+ Suggestion::ReplaceWith(chars) => {+ // Avoid allocation if possible+ if chars.len() == span.len() {+ for (index, c) in chars.iter().enumerate() {+ self.source[index + span.start] = *c+ }+ } else {+ let popped = self.source.split_off(span.start);++ self.source.extend(chars);+ self.source.extend(popped.into_iter().skip(span.len()));+ }+ }+ }++ self.parse();}/// Searches for quotation marks and fills the [`Punctuation::Quote::twin_loc`] field.@@ -71,7 +164,7 @@ impl Document {////// Current algorithm is very basic and could use some work.fn match_quotes(&mut self) {- let quote_indices: Vec<usize> = self.iter_quote_indices().collect();+ let quote_indices: Vec<usize> = self.tokens.iter_quote_indices().collect();for i in 0..quote_indices.len() / 2 {let a_i = quote_indices[i * 2];@@ -89,7 +182,7 @@ impl Document {}}- /// Searches for contractions and condenses them down into single tokens+ /// Searches for contractions and condenses them down into single self.tokensfn condense_contractions(&mut self) {if self.tokens.len() < 3 {return;@@ -155,123 +248,65 @@ impl Document {.unwrap_or(replace_starts.len())..],)}+}- /// Defensively attempt to grab a specific token.- pub fn get_token(&self, index: usize) -> Option<Token> {- self.tokens.get(index).copied()- }+impl Display for Document {+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ for token in &self.tokens {+ write!(f, "{}", self.get_span_content_str(token.span))?;+ }- pub fn tokens(&self) -> impl Iterator<Item = Token> + '_ {- self.tokens.iter().copied()+ Ok(())}+}- pub fn fat_tokens(&self) -> impl Iterator<Item = FatToken> + '_ {- self.tokens().map(|token| token.to_fat(&self.source))+impl TokenStringExt for Document {+ fn first_word(&self) -> Option<Token> {+ self.tokens.first_word()}- /// Iterate over the locations of the sentence terminators in the document.- fn sentence_terminators(&self) -> impl Iterator<Item = usize> + '_ {- self.tokens.iter().enumerate().filter_map(|(index, token)| {- if is_sentence_terminator(&token.kind) {- return Some(index);- }- None- })+ fn first_sentence_word(&self) -> Option<Token> {+ self.tokens.first_sentence_word()}- /// Get the index of the last sentence terminator.- fn last_sentence_terminator(&self) -> Option<usize> {- self.tokens- .iter()- .enumerate()- .rev()- .find_map(|(index, token)| {- if is_sentence_terminator(&token.kind) {- return Some(index);- }- None- })+ fn first_non_whitespace(&self) -> Option<Token> {+ self.tokens.first_non_whitespace()}- pub fn sentences(&self) -> impl Iterator<Item = &'_ [Token]> + '_ {- let first_sentence = self- .sentence_terminators()- .next()- .map(|first_term| &self.tokens[0..=first_term]);-- let rest = self- .sentence_terminators()- .tuple_windows()- .map(move |(a, b)| &self.tokens[a + 1..=b]);-- let last = if let Some(last_i) = self.last_sentence_terminator() {- if last_i + 1 < self.tokens.len() {- Some(&self.tokens[last_i + 1..])- } else {- None- }- } else {- Some(self.tokens.as_slice())- };-- first_sentence.into_iter().chain(rest).chain(last)+ fn iter_word_indices(&self) -> impl Iterator<Item = usize> + '_ {+ self.tokens.iter_word_indices()}- /** Returns all tokens whose `kind` is [`Punctuation::Word`] */- pub fn words(&self) -> impl Iterator<Item = Token> + '_ {- self.tokens- .iter()- .filter(|token| token.kind.is_word())- .cloned()+ fn iter_words(&self) -> impl Iterator<Item = &Token> + '_ {+ self.tokens.iter_words()}- pub fn get_span_content(&self, span: Span) -> &[char] {- span.get_content(&self.source)+ fn iter_space_indices(&self) -> impl Iterator<Item = usize> + '_ {+ self.tokens.iter_space_indices()}- pub fn get_span_content_str(&self, span: Span) -> String {- String::from_iter(self.get_span_content(span))+ fn iter_spaces(&self) -> impl Iterator<Item = &Token> + '_ {+ self.tokens.iter_spaces()}- pub fn get_full_string(&self) -> String {- self.get_span_content_str(Span {- start: 0,- end: self.source.len(),- })+ fn iter_apostrophe_indices(&self) -> impl Iterator<Item = usize> + '_ {+ self.tokens.iter_apostrophe_indices()}- pub fn get_full_content(&self) -> &[char] {- &self.source+ fn iter_apostrophes(&self) -> impl Iterator<Item = &Token> + '_ {+ self.tokens.iter_apostrophes()}- pub fn apply_suggestion(&mut self, suggestion: &Suggestion, span: Span) {- match suggestion {- Suggestion::ReplaceWith(chars) => {- // Avoid allocation if possible- if chars.len() == span.len() {- for (index, c) in chars.iter().enumerate() {- self.source[index + span.start] = *c- }- } else {- let popped = self.source.split_off(span.start);-- self.source.extend(chars);- self.source.extend(popped.into_iter().skip(span.len()));- }- }- }-- self.parse();+ fn span(&self) -> Option<Span> {+ self.tokens.span()}-}-impl Display for Document {- fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {- for token in &self.tokens {- write!(f, "{}", self.get_span_content_str(token.span))?;- }+ fn iter_quote_indices(&self) -> impl Iterator<Item = usize> + '_ {+ self.tokens.iter_quote_indices()+ }- Ok(())+ fn iter_quotes(&self) -> impl Iterator<Item = Token> + '_ {+ self.tokens.iter_quotes()}}@@ -296,14 +331,35 @@ mod tests {token::TokenStringExt,};+ #[test]+ fn parses_sentences_correctly() {+ let text = "There were three little pigs. They built three little homes.";+ let document = Document::new(text, Box::new(PlainEnglish));++ let mut sentence_strs = vec![];++ for sentence in document.sentences() {+ if let Some(span) = sentence.span() {+ sentence_strs.push(document.get_span_content_str(span));+ }+ }++ assert_eq!(+ sentence_strs,+ vec![+ "There were three little pigs.",+ " They built three little homes."+ ]+ )+ }+fn assert_condensed_contractions(text: &str, final_tok_count: usize) {- let mut document = Document::new(text, Box::new(PlainEnglish));- document.condense_contractions();+ let document = Document::new(text, Box::new(PlainEnglish));assert_eq!(document.tokens.len(), final_tok_count);- let mut document = Document::new(text, Box::new(Markdown));- document.condense_contractions();+ let mut markdown_parser = Markdown;+ let document = Document::new(text, Box::new(markdown_parser));// We add one because the Markdown parser inserts a newline at end-of-input.assert_eq!(document.tokens.len(), final_tok_count + 1);@@ -333,26 +389,4 @@ mod tests {fn medium_contraction2() {assert_condensed_contractions("There's no way", 5);}-- #[test]- fn parses_sentences_correctly() {- let text = "There were three little pigs. They built three little homes.";- let document = Document::new(text, Box::new(PlainEnglish));-- let mut sentence_strs = vec![];-- for sentence in document.sentences() {- if let Some(span) = sentence.span() {- sentence_strs.push(document.get_span_content_str(span));- }- }-- assert_eq!(- sentence_strs,- vec![- "There were three little pigs.",- " They built three little homes."- ]- )- }}diff --git a/harper-core/src/linting/matcher.rs b/harper-core/src/linting/matcher.rsindex f734ffc77..b79ea82d5 100644--- a/harper-core/src/linting/matcher.rs+++ b/harper-core/src/linting/matcher.rs@@ -32,7 +32,6 @@ macro_rules! vecword {}macro_rules! pt {-($str:literal) => {PatternToken {kind: TokenKind::Word,@@ -102,6 +101,10 @@ impl Matcher {// This match list needs to be automatically expanded instead of explicitly defined// like it is now.let mut triggers = pt! {+ "and","also" => "and",+ "todo" => "to-do",+ "To-Do" => "To-do",+ "performing","this" => "perform this","united nations" => "United Nations","mins" => "minutes","min" => "minute",@@ -180,7 +183,9 @@ impl Matcher {"all","though" => "although","All","though" => "although","al","though" => "although",- "Al","though" => "although"+ "Al","though" => "although",+ "an","this" => "and this",+ "break","up" => "break-up"};// TODO: Improve the description for this lint specifically.@@ -192,21 +197,16 @@ impl Matcher {// Same goes for this En dashtriggers.push(Rule {- pattern: vec![pt!(Hyphen), pt!(Hyphen), pt!(Hyphen)],+ pattern: vec![pt!(Hyphen), pt!(Hyphen)],replace_with: vecword!("–"),});- // An this ellipsis+ // And this ellipsistriggers.push(Rule {pattern: vec![pt!(Period), pt!(Period), pt!(Period)],replace_with: vecword!("…"),});- triggers.push(Rule {- pattern: vec![pt!("break"), pt!(Hyphen), pt!("up")],- replace_with: vecword!("break-up"),- });-triggers.push(Rule {pattern: vec![pt!("L"), pt!(Period), pt!("L"), pt!(Period), pt!("M")],replace_with: vecword!("large language model"),@@ -238,9 +238,11 @@ impl Linter for Matcher {fn lint(&mut self, document: &Document) -> Vec<Lint> {let mut lints = Vec::new();+ let mut match_tokens = Vec::new();+for (index, _) in document.tokens().enumerate() {for trigger in &self.triggers {- let mut match_tokens = Vec::new();+ match_tokens.clear();for (p_index, pattern) in trigger.pattern.iter().enumerate() {let Some(token) = document.get_token(index + p_index) else {@@ -291,6 +293,14 @@ mod tests {let document = Document::new_plain_english("There fore.");let mut matcher = Matcher::new();let lints = matcher.lint(&document);- assert!(lints.len() == 1)+ assert!(lints.len() == 1);+ }++ #[test]+ fn matches_ellipsis() {+ let document = Document::new_plain_english("...");+ let mut matcher = Matcher::new();+ let lints = matcher.lint(&document);+ assert!(lints.len() == 1);}}diff --git a/harper-core/src/linting/spell_check.rs b/harper-core/src/linting/spell_check.rsindex 63f74dbb9..a72e05c26 100644--- a/harper-core/src/linting/spell_check.rs+++ b/harper-core/src/linting/spell_check.rs@@ -29,10 +29,20 @@ impl<T: Dictionary> SpellCheck<T> {self.word_cache.entry(word.clone()).or_insert_with(|| {- suggest_correct_spelling(&word, 100, 2, &self.dictionary)- .into_iter()- .map(|v| v.to_vec())- .collect()+ // Back off until we find a match.+ let mut suggestions = Vec::new();+ let mut dist = 2;++ while suggestions.is_empty() && dist < 5 {+ suggestions = suggest_correct_spelling(&word, 100, dist, &self.dictionary)+ .into_iter()+ .map(|v| v.to_vec())+ .collect();++ dist += 1;+ }++ suggestions}).clone()}diff --git a/harper-core/src/linting/wrong_quotes.rs b/harper-core/src/linting/wrong_quotes.rsindex 3ef8ac928..3e195d98c 100644--- a/harper-core/src/linting/wrong_quotes.rs+++ b/harper-core/src/linting/wrong_quotes.rs@@ -1,4 +1,4 @@-use crate::{document::Document, Suggestion, Token};+use crate::{document::Document, Suggestion, Token, TokenStringExt};use super::{Lint, Linter};diff --git a/harper-core/src/parsers/mod.rs b/harper-core/src/parsers/mod.rsindex 9e608c7cd..50354a188 100644--- a/harper-core/src/parsers/mod.rs+++ b/harper-core/src/parsers/mod.rs@@ -25,7 +25,7 @@ where#[cfg(test)]mod tests {- use super::{Markdown, Parser, PlainEnglish};+ use super::{Markdown, Parser, PlainEnglish, StrParser};use crate::{Punctuation,TokenKind::{self, *},diff --git a/harper-core/src/parsers/plain_english.rs b/harper-core/src/parsers/plain_english.rsindex c58e0cb19..332515dc4 100644--- a/harper-core/src/parsers/plain_english.rs+++ b/harper-core/src/parsers/plain_english.rs@@ -8,9 +8,13 @@ use crate::{/// without discrimination and until the end of input.pub struct PlainEnglish;+impl PlainEnglish {}+impl Parser for PlainEnglish {fn parse(&mut self, source: &[char]) -> Vec<Token> {let mut cursor = 0;++ // Lex tokenslet mut tokens = Vec::new();loop {diff --git a/harper-core/src/spell/mod.rs b/harper-core/src/spell/mod.rsindex 531eefabc..17a611e70 100644--- a/harper-core/src/spell/mod.rs+++ b/harper-core/src/spell/mod.rs@@ -122,6 +122,7 @@ pub fn suggest_correct_spelling_str(// This is accomplished via a memory-optimized Wagner-Fischer algorithm//// This variant avoids allocation if you already have buffers.+#[inline]fn edit_distance_min_alloc(source: &[char],target: &[char],diff --git a/harper-core/src/token.rs b/harper-core/src/token.rsindex a25f35f14..e3f75c5dc 100644--- a/harper-core/src/token.rs+++ b/harper-core/src/token.rs@@ -150,6 +150,9 @@ pub trait TokenStringExt {/// Grab the span that represents the beginning of the first element and the end of the last/// element.fn span(&self) -> Option<Span>;++ fn iter_quote_indices(&self) -> impl Iterator<Item = usize> + '_;+ fn iter_quotes(&self) -> impl Iterator<Item = Token> + '_;}impl TokenStringExt for [Token] {@@ -211,4 +214,18 @@ impl TokenStringExt for [Token] {fn span(&self) -> Option<Span> {Some(Span::new(self.first()?.span.start, self.last()?.span.end))}++ fn iter_quote_indices(&self) -> impl Iterator<Item = usize> + '_ {+ self.iter().enumerate().filter_map(|(idx, token)| {+ if let TokenKind::Punctuation(Punctuation::Quote(_)) = &token.kind {+ Some(idx)+ } else {+ None+ }+ })+ }++ fn iter_quotes(&self) -> impl Iterator<Item = Token> + '_ {+ self.iter_quote_indices().map(|idx| self[idx])+ }}