Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
Improved spelling suggestions by explicitly returning the longest and shortest words.
Elijah Potterf2b0b4899d9e
diff --git a/harper-core/Cargo.toml b/harper-core/Cargo.tomlindex cded36dfa..ec72a2548 100644--- a/harper-core/Cargo.toml+++ b/harper-core/Cargo.toml@@ -17,4 +17,3 @@ pulldown-cmark = "0.9.3"serde = { version = "1.0.190", features = ["derive"] }smallvec = "1.12.0"thiserror = "1.0.56"-diff --git a/harper-core/dictionary.dict b/harper-core/dictionary.dictindex c27857813..8eba8a792 100644--- a/harper-core/dictionary.dict+++ b/harper-core/dictionary.dict@@ -30417,7 +30417,7 @@ linolinoleum/Mlinseed/Mlint's-lint/CDG+lint/CDGRlintel/MSlintslinty/TR@@ -49583,3 +49583,5 @@ zymurgy/MStackOverflow/MCommonMark/Mlex/D+Levenshtein/M+QUERTYdiff --git a/harper-core/src/lexing/mod.rs b/harper-core/src/lexing/mod.rsindex 7ed16b598..e2946bf69 100644--- a/harper-core/src/lexing/mod.rs+++ b/harper-core/src/lexing/mod.rs@@ -117,6 +117,9 @@ fn lex_punctuation(source: &[char]) -> Option<FoundToken> {use Punctuation::*;let punct = match c {+ '=' => Equal,+ '<' => LessThan,+ '>' => GreaterThan,'/' => ForwardSlash,'\\' => Backslash,'%' => Percent,diff --git a/harper-core/src/linting/sentence_capitalization.rs b/harper-core/src/linting/sentence_capitalization.rsindex e578cc6a1..a140dc6d2 100644--- a/harper-core/src/linting/sentence_capitalization.rs+++ b/harper-core/src/linting/sentence_capitalization.rs@@ -14,7 +14,11 @@ impl Linter for SentenceCapitalization {let mut lints = Vec::new();for sentence in document.sentences() {- if let Some(first_word) = sentence.first_word() {+ if let Some(first_word) = sentence.first() {+ if !first_word.kind.is_word() {+ break;+ }+let letters = document.get_span_content(first_word.span);if let Some(first_letter) = letters.first() {diff --git a/harper-core/src/linting/spell_check.rs b/harper-core/src/linting/spell_check.rsindex a0e303262..a86882de0 100644--- a/harper-core/src/linting/spell_check.rs+++ b/harper-core/src/linting/spell_check.rs@@ -54,6 +54,15 @@ impl Linter for SpellCheck {possibilities.resize_with(3, || panic!());}+ // If the misspelled word is capitalized, capitalize the results too.+ if let Some(mis_f) = word_chars.first() {+ if mis_f.is_uppercase() {+ for sug_f in possibilities.iter_mut().filter_map(|w| w.first_mut()) {+ *sug_f = sug_f.to_uppercase().next().unwrap();+ }+ }+ }+let suggestions = possibilities.into_iter().map(|word| Suggestion::ReplaceWith(word.to_vec()));diff --git a/harper-core/src/parsers/markdown.rs b/harper-core/src/parsers/markdown.rsindex 08b7af3f0..2f3e38bd6 100644--- a/harper-core/src/parsers/markdown.rs+++ b/harper-core/src/parsers/markdown.rs@@ -1,5 +1,5 @@use super::{Parser, PlainEnglish, StrParser};-use crate::Token;+use crate::{Span, Token, TokenKind};/// A parser that wraps the [`PlainEnglish`] parser that allows one to parse CommonMark files.///@@ -36,6 +36,14 @@ impl Parser for Markdown {if let Some(tag) = stack.last() {use pulldown_cmark::Tag;+ if matches!(tag, Tag::CodeBlock(..)) {+ tokens.push(Token {+ span: Span::new(traversed_chars, text.chars().count()),+ kind: TokenKind::Unlintable,+ });+ continue;+ }+if !(matches!(tag, Tag::Paragraph)|| matches!(tag, Tag::Heading(_, _, _))|| matches!(tag, Tag::Item)diff --git a/harper-core/src/spell/mod.rs b/harper-core/src/spell/mod.rsindex 2ba609ace..4502b2772 100644--- a/harper-core/src/spell/mod.rs+++ b/harper-core/src/spell/mod.rs@@ -1,3 +1,4 @@+use itertools::{Itertools, MinMaxResult};use smallvec::SmallVec;pub use self::dictionary::Dictionary;@@ -8,14 +9,19 @@ mod hunspell;type DictWord = SmallVec<[char; 6]>;/// Suggest a correct spelling for a given misspelled word.-/// [misspelled_word] is assumed to be quite small (n < 100)-/// [max_edit_dist] relates to an optimization that allows the search algorithm to prune large portions of the search.+/// [`misspelled_word`] is assumed to be quite small (n < 100).+/// [`max_edit_dist`] relates to an optimization that allows the search algorithm to prune large portions of the search.pub fn suggest_correct_spelling<'a>(misspelled_word: &[char],result_limit: usize,max_edit_dist: u8,dictionary: &'a Dictionary,) -> Vec<&'a [char]> {+ let misspelled_lower: Vec<char> = misspelled_word+ .iter()+ .flat_map(|v| v.to_lowercase())+ .collect();+// 53 is the length of the longest word.let mut buf_a = Vec::with_capacity(53);let mut buf_b = Vec::with_capacity(53);@@ -27,19 +33,23 @@ pub fn suggest_correct_spelling<'a>(misspelled_word.len() - max_edit_dist as usize};+ // Note how we look at the biggest words firstlet words_to_search = (shortest_word_len..misspelled_word.len() + max_edit_dist as usize)+ .rev().flat_map(|len| dictionary.words_with_len_iter(len));let pruned_words = words_to_search.filter_map(|word| {let dist = edit_distance_min_alloc(misspelled_word, word, &mut buf_a, &mut buf_b);+ let dist_lower = edit_distance_min_alloc(&misspelled_lower, word, &mut buf_a, &mut buf_b);- if dist <= max_edit_dist {+ if dist.min(dist_lower) <= max_edit_dist {Some((word, dist))} else {None}});+ // Locate the words with the lowest edit distance.let mut found_dist: Vec<(&[char], u8)> = Vec::with_capacity(result_limit);for (word, dist) in pruned_words {@@ -55,42 +65,31 @@ pub fn suggest_correct_spelling<'a>(}}- // Remove edit dist- let mut found: Vec<&[char]> = found_dist.into_iter().map(|(word, _dist)| word).collect();-- found.sort_by_cached_key(|v| {- let mut key_dist = usize::MAX;+ // Create final, ordered list of suggestions.+ let mut found = Vec::with_capacity(found_dist.len());- // The error may be by omission at the end of the word.- if v.len() > misspelled_word.len() {- return edit_distance_min_alloc(v, misspelled_word, &mut buf_a, &mut buf_b) as usize;- }-- for (o, n) in v.iter().zip(misspelled_word.iter()) {- if o != n {- key_dist = key_distance(*o, *n)- .map(|v| v as usize)- .unwrap_or(usize::MAX);- break;+ // Often the longest and the shortest words are the most helpful, so lets push them first.+ let minmax = found_dist.iter().position_minmax_by_key(|a| a.0.len());+ if let MinMaxResult::MinMax(a, b) = minmax {+ if a == b {+ found.push(found_dist.remove(a).0);+ } else {+ found.push(found_dist[b].0);+ found.push(found_dist.remove(a).0);+ if a < b {+ found_dist.remove(b - 1);+ } else {+ found_dist.remove(b);}}+ }- // The error is likely by omission somewhere inside the word- if key_dist > 2 {- usize::MAX - v.len()- }- // The error is likely by replacement- else {- key_dist- }- });-- found.sort_by_key(|v| if dictionary.is_common_word(v) { 0 } else { 1 });+ found.extend(found_dist.into_iter().map(|v| v.0));found}-/// Convenience function over [suggest_correct_spelling] that does conversions for you.+/// Convenience function over [`suggest_correct_spelling`] that does conversions for you.pub fn suggest_correct_spelling_str(misspelled_word: impl AsRef<str>,result_limit: usize,@@ -124,7 +123,7 @@ fn edit_distance_min_alloc(previous_row.clear();previous_row.extend(0u8..=row_width as u8);- // Alright if not zeroed, since we overwrite it anyway+ // Alright if not zeroed, since we overwrite it anyway.current_row.resize(row_width + 1, 0);for j in 1..=col_height {@@ -148,40 +147,6 @@ fn edit_distance(source: &[char], target: &[char]) -> u8 {edit_distance_min_alloc(source, target, &mut Vec::new(), &mut Vec::new())}-/// Calculate the approximate distance between two letters on a querty keyboard-fn key_distance(key_a: char, key_b: char) -> Option<f32> {- let a = key_location(key_a)?;- let b = key_location(key_b)?;-- Some(((a.0 - b.0) * (a.1 - b.1)).sqrt())-}--/// Calculate the approximate position of a letter on a querty keyboard-fn key_location(key: char) -> Option<(f32, f32)> {- let keys = "1234567890qwertyuiopasdfghjklzxcvbnm";-- let idx = keys.find(key)?;-- // The starting index of each row of the keyboard- let mut resets = [0, 10, 20, 29].into_iter().enumerate().peekable();- // The amount each row is offset (on my keyboard at least)- let offsets = [0.0, 0.5, 0.75, 1.25];-- while let Some((r_idx, reset)) = resets.next() {- if idx >= reset {- if let Some((_, n_reset)) = resets.peek() {- if idx < *n_reset {- return Some(((idx - reset) as f32 + offsets[r_idx], r_idx as f32));- }- } else {- return Some(((idx - reset) as f32 + offsets[r_idx], r_idx as f32));- }- }- }-- None-}-#[cfg(test)]mod tests {use super::edit_distance;@@ -203,21 +168,4 @@ mod tests {fn simple2() {assert_edit_dist("saturday", "sunday", 3)}-- use super::key_location;-- #[test]- fn correct_q_pos() {- assert_eq!(key_location('q'), Some((0.5, 1.0)))- }-- #[test]- fn correct_a_pos() {- assert_eq!(key_location('a'), Some((0.75, 2.0)))- }-- #[test]- fn correct_g_pos() {- assert_eq!(key_location('g'), Some((4.75, 2.0)))- }}diff --git a/harper-core/src/token.rs b/harper-core/src/token.rsindex a5b15077e..c8054a043 100644--- a/harper-core/src/token.rs+++ b/harper-core/src/token.rs@@ -1,4 +1,5 @@use is_macro::Is;+use itertools::Itertools;use serde::{Deserialize, Serialize};use crate::span::Span;@@ -39,6 +40,9 @@ pub enum TokenKind {Space(usize),/// A sequence of "\n" newlinesNewline(usize),+ /// A special token used for things like inline code blocks that should be ignored by all+ /// linters.+ Unlintable,}impl TokenKind {@@ -92,6 +96,12 @@ pub enum Punctuation {ForwardSlash,/// \Backslash,+ /// <+ LessThan,+ /// >+ GreaterThan,+ /// Equal+ Equal,}#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]@@ -102,6 +112,10 @@ pub struct Quote {pub trait TokenStringExt {fn first_word(&self) -> Option<Token>;+ /// Grabs the first word in the sentence.+ /// Will also return [`None`] if there is an unlintable token in the position of the first+ /// word.+ fn first_sentence_word(&self) -> Option<Token>;fn iter_word_indices(&self) -> impl Iterator<Item = usize> + '_;fn iter_words(&self) -> impl Iterator<Item = &Token> + '_;fn iter_space_indices(&self) -> impl Iterator<Item = usize> + '_;@@ -118,6 +132,20 @@ impl TokenStringExt for [Token] {self.iter().find(|v| v.kind.is_word()).copied()}+ fn first_sentence_word(&self) -> Option<Token> {+ let (w_idx, word) = self.iter().find_position(|v| v.kind.is_word())?;++ let Some(u_idx) = self.iter().position(|v| v.kind.is_unlintable()) else {+ return Some(*word);+ };++ if w_idx > u_idx {+ Some(*word)+ } else {+ None+ }+ }+fn iter_word_indices(&self) -> impl Iterator<Item = usize> + '_ {self.iter().enumerate()diff --git a/harper-ls/src/backend.rs b/harper-ls/src/backend.rsindex 1e50edebc..11c7b52e4 100644--- a/harper-ls/src/backend.rs+++ b/harper-ls/src/backend.rs@@ -8,12 +8,13 @@ use tokio::sync::Mutex;use tower_lsp::{jsonrpc::Result,lsp_types::{- notification::PublishDiagnostics, CodeAction, CodeActionOrCommand, CodeActionParams,- CodeActionProviderCapability, CodeActionResponse, Diagnostic, DidChangeTextDocumentParams,- DidCloseTextDocumentParams, DidOpenTextDocumentParams, DidSaveTextDocumentParams,- InitializeParams, InitializeResult, InitializedParams, MessageType,- PublishDiagnosticsParams, Range, ServerCapabilities, TextDocumentSyncCapability,- TextDocumentSyncKind, TextDocumentSyncOptions, TextDocumentSyncSaveOptions, Url,+ notification::{PublishDiagnostics, ShowMessage},+ CodeAction, CodeActionOrCommand, CodeActionParams, CodeActionProviderCapability,+ CodeActionResponse, Diagnostic, DidChangeTextDocumentParams, DidCloseTextDocumentParams,+ DidOpenTextDocumentParams, DidSaveTextDocumentParams, InitializeParams, InitializeResult,+ InitializedParams, MessageType, PublishDiagnosticsParams, Range, ServerCapabilities,+ ShowMessageParams, TextDocumentSyncCapability, TextDocumentSyncKind,+ TextDocumentSyncOptions, TextDocumentSyncSaveOptions, Url,},Client, LanguageServer,};@@ -97,6 +98,17 @@ impl Backend {}async fn publish_diagnostics(&self, url: &Url) {+ let client = self.client.clone();++ tokio::spawn(async move {+ client+ .send_notification::<ShowMessage>(ShowMessageParams {+ typ: MessageType::INFO,+ message: "Linting...".to_string(),+ })+ .await+ });+let diagnostics = self.generate_diagnostics(url).await;let result = PublishDiagnosticsParams {