Skip to content

Offline, privacy-first grammar checker. Fast, open-source, Rust-powered

Improved spelling suggestions by explicitly returning the longest and shortest words.

Elijah Potterf2b0b4899d9e
diff --git a/harper-core/Cargo.toml b/harper-core/Cargo.toml
index cded36dfa..ec72a2548 100644
--- a/harper-core/Cargo.toml
+++ b/harper-core/Cargo.toml
@@ -17,4 +17,3 @@ pulldown-cmark = "0.9.3"
serde = { version = "1.0.190", features = ["derive"] }
smallvec = "1.12.0"
thiserror = "1.0.56"
-
diff --git a/harper-core/dictionary.dict b/harper-core/dictionary.dict
index c27857813..8eba8a792 100644
--- a/harper-core/dictionary.dict
+++ b/harper-core/dictionary.dict
@@ -30417,7 +30417,7 @@ lino
linoleum/M
linseed/M
lint's
-lint/CDG
+lint/CDGR
lintel/MS
lints
linty/TR
@@ -49583,3 +49583,5 @@ zymurgy/M
StackOverflow/M
CommonMark/M
lex/D
+Levenshtein/M
+QUERTY
diff --git a/harper-core/src/lexing/mod.rs b/harper-core/src/lexing/mod.rs
index 7ed16b598..e2946bf69 100644
--- a/harper-core/src/lexing/mod.rs
+++ b/harper-core/src/lexing/mod.rs
@@ -117,6 +117,9 @@ fn lex_punctuation(source: &[char]) -> Option<FoundToken> {
use Punctuation::*;
let punct = match c {
+ '=' => Equal,
+ '<' => LessThan,
+ '>' => GreaterThan,
'/' => ForwardSlash,
'\\' => Backslash,
'%' => Percent,
diff --git a/harper-core/src/linting/sentence_capitalization.rs b/harper-core/src/linting/sentence_capitalization.rs
index e578cc6a1..a140dc6d2 100644
--- a/harper-core/src/linting/sentence_capitalization.rs
+++ b/harper-core/src/linting/sentence_capitalization.rs
@@ -14,7 +14,11 @@ impl Linter for SentenceCapitalization {
let mut lints = Vec::new();
for sentence in document.sentences() {
- if let Some(first_word) = sentence.first_word() {
+ if let Some(first_word) = sentence.first() {
+ if !first_word.kind.is_word() {
+ break;
+ }
+
let letters = document.get_span_content(first_word.span);
if let Some(first_letter) = letters.first() {
diff --git a/harper-core/src/linting/spell_check.rs b/harper-core/src/linting/spell_check.rs
index a0e303262..a86882de0 100644
--- a/harper-core/src/linting/spell_check.rs
+++ b/harper-core/src/linting/spell_check.rs
@@ -54,6 +54,15 @@ impl Linter for SpellCheck {
possibilities.resize_with(3, || panic!());
}
+ // If the misspelled word is capitalized, capitalize the results too.
+ if let Some(mis_f) = word_chars.first() {
+ if mis_f.is_uppercase() {
+ for sug_f in possibilities.iter_mut().filter_map(|w| w.first_mut()) {
+ *sug_f = sug_f.to_uppercase().next().unwrap();
+ }
+ }
+ }
+
let suggestions = possibilities
.into_iter()
.map(|word| Suggestion::ReplaceWith(word.to_vec()));
diff --git a/harper-core/src/parsers/markdown.rs b/harper-core/src/parsers/markdown.rs
index 08b7af3f0..2f3e38bd6 100644
--- a/harper-core/src/parsers/markdown.rs
+++ b/harper-core/src/parsers/markdown.rs
@@ -1,5 +1,5 @@
use super::{Parser, PlainEnglish, StrParser};
-use crate::Token;
+use crate::{Span, Token, TokenKind};
/// A parser that wraps the [`PlainEnglish`] parser that allows one to parse CommonMark files.
///
@@ -36,6 +36,14 @@ impl Parser for Markdown {
if let Some(tag) = stack.last() {
use pulldown_cmark::Tag;
+ if matches!(tag, Tag::CodeBlock(..)) {
+ tokens.push(Token {
+ span: Span::new(traversed_chars, text.chars().count()),
+ kind: TokenKind::Unlintable,
+ });
+ continue;
+ }
+
if !(matches!(tag, Tag::Paragraph)
|| matches!(tag, Tag::Heading(_, _, _))
|| matches!(tag, Tag::Item)
diff --git a/harper-core/src/spell/mod.rs b/harper-core/src/spell/mod.rs
index 2ba609ace..4502b2772 100644
--- a/harper-core/src/spell/mod.rs
+++ b/harper-core/src/spell/mod.rs
@@ -1,3 +1,4 @@
+use itertools::{Itertools, MinMaxResult};
use smallvec::SmallVec;
pub use self::dictionary::Dictionary;
@@ -8,14 +9,19 @@ mod hunspell;
type DictWord = SmallVec<[char; 6]>;
/// Suggest a correct spelling for a given misspelled word.
-/// [misspelled_word] is assumed to be quite small (n < 100)
-/// [max_edit_dist] relates to an optimization that allows the search algorithm to prune large portions of the search.
+/// [`misspelled_word`] is assumed to be quite small (n < 100).
+/// [`max_edit_dist`] relates to an optimization that allows the search algorithm to prune large portions of the search.
pub fn suggest_correct_spelling<'a>(
misspelled_word: &[char],
result_limit: usize,
max_edit_dist: u8,
dictionary: &'a Dictionary,
) -> Vec<&'a [char]> {
+ let misspelled_lower: Vec<char> = misspelled_word
+ .iter()
+ .flat_map(|v| v.to_lowercase())
+ .collect();
+
// 53 is the length of the longest word.
let mut buf_a = Vec::with_capacity(53);
let mut buf_b = Vec::with_capacity(53);
@@ -27,19 +33,23 @@ pub fn suggest_correct_spelling<'a>(
misspelled_word.len() - max_edit_dist as usize
};
+ // Note how we look at the biggest words first
let words_to_search = (shortest_word_len..misspelled_word.len() + max_edit_dist as usize)
+ .rev()
.flat_map(|len| dictionary.words_with_len_iter(len));
let pruned_words = words_to_search.filter_map(|word| {
let dist = edit_distance_min_alloc(misspelled_word, word, &mut buf_a, &mut buf_b);
+ let dist_lower = edit_distance_min_alloc(&misspelled_lower, word, &mut buf_a, &mut buf_b);
- if dist <= max_edit_dist {
+ if dist.min(dist_lower) <= max_edit_dist {
Some((word, dist))
} else {
None
}
});
+ // Locate the words with the lowest edit distance.
let mut found_dist: Vec<(&[char], u8)> = Vec::with_capacity(result_limit);
for (word, dist) in pruned_words {
@@ -55,42 +65,31 @@ pub fn suggest_correct_spelling<'a>(
}
}
- // Remove edit dist
- let mut found: Vec<&[char]> = found_dist.into_iter().map(|(word, _dist)| word).collect();
-
- found.sort_by_cached_key(|v| {
- let mut key_dist = usize::MAX;
+ // Create final, ordered list of suggestions.
+ let mut found = Vec::with_capacity(found_dist.len());
- // The error may be by omission at the end of the word.
- if v.len() > misspelled_word.len() {
- return edit_distance_min_alloc(v, misspelled_word, &mut buf_a, &mut buf_b) as usize;
- }
-
- for (o, n) in v.iter().zip(misspelled_word.iter()) {
- if o != n {
- key_dist = key_distance(*o, *n)
- .map(|v| v as usize)
- .unwrap_or(usize::MAX);
- break;
+ // Often the longest and the shortest words are the most helpful, so lets push them first.
+ let minmax = found_dist.iter().position_minmax_by_key(|a| a.0.len());
+ if let MinMaxResult::MinMax(a, b) = minmax {
+ if a == b {
+ found.push(found_dist.remove(a).0);
+ } else {
+ found.push(found_dist[b].0);
+ found.push(found_dist.remove(a).0);
+ if a < b {
+ found_dist.remove(b - 1);
+ } else {
+ found_dist.remove(b);
}
}
+ }
- // The error is likely by omission somewhere inside the word
- if key_dist > 2 {
- usize::MAX - v.len()
- }
- // The error is likely by replacement
- else {
- key_dist
- }
- });
-
- found.sort_by_key(|v| if dictionary.is_common_word(v) { 0 } else { 1 });
+ found.extend(found_dist.into_iter().map(|v| v.0));
found
}
-/// Convenience function over [suggest_correct_spelling] that does conversions for you.
+/// Convenience function over [`suggest_correct_spelling`] that does conversions for you.
pub fn suggest_correct_spelling_str(
misspelled_word: impl AsRef<str>,
result_limit: usize,
@@ -124,7 +123,7 @@ fn edit_distance_min_alloc(
previous_row.clear();
previous_row.extend(0u8..=row_width as u8);
- // Alright if not zeroed, since we overwrite it anyway
+ // Alright if not zeroed, since we overwrite it anyway.
current_row.resize(row_width + 1, 0);
for j in 1..=col_height {
@@ -148,40 +147,6 @@ fn edit_distance(source: &[char], target: &[char]) -> u8 {
edit_distance_min_alloc(source, target, &mut Vec::new(), &mut Vec::new())
}
-/// Calculate the approximate distance between two letters on a querty keyboard
-fn key_distance(key_a: char, key_b: char) -> Option<f32> {
- let a = key_location(key_a)?;
- let b = key_location(key_b)?;
-
- Some(((a.0 - b.0) * (a.1 - b.1)).sqrt())
-}
-
-/// Calculate the approximate position of a letter on a querty keyboard
-fn key_location(key: char) -> Option<(f32, f32)> {
- let keys = "1234567890qwertyuiopasdfghjklzxcvbnm";
-
- let idx = keys.find(key)?;
-
- // The starting index of each row of the keyboard
- let mut resets = [0, 10, 20, 29].into_iter().enumerate().peekable();
- // The amount each row is offset (on my keyboard at least)
- let offsets = [0.0, 0.5, 0.75, 1.25];
-
- while let Some((r_idx, reset)) = resets.next() {
- if idx >= reset {
- if let Some((_, n_reset)) = resets.peek() {
- if idx < *n_reset {
- return Some(((idx - reset) as f32 + offsets[r_idx], r_idx as f32));
- }
- } else {
- return Some(((idx - reset) as f32 + offsets[r_idx], r_idx as f32));
- }
- }
- }
-
- None
-}
-
#[cfg(test)]
mod tests {
use super::edit_distance;
@@ -203,21 +168,4 @@ mod tests {
fn simple2() {
assert_edit_dist("saturday", "sunday", 3)
}
-
- use super::key_location;
-
- #[test]
- fn correct_q_pos() {
- assert_eq!(key_location('q'), Some((0.5, 1.0)))
- }
-
- #[test]
- fn correct_a_pos() {
- assert_eq!(key_location('a'), Some((0.75, 2.0)))
- }
-
- #[test]
- fn correct_g_pos() {
- assert_eq!(key_location('g'), Some((4.75, 2.0)))
- }
}
diff --git a/harper-core/src/token.rs b/harper-core/src/token.rs
index a5b15077e..c8054a043 100644
--- a/harper-core/src/token.rs
+++ b/harper-core/src/token.rs
@@ -1,4 +1,5 @@
use is_macro::Is;
+use itertools::Itertools;
use serde::{Deserialize, Serialize};
use crate::span::Span;
@@ -39,6 +40,9 @@ pub enum TokenKind {
Space(usize),
/// A sequence of "\n" newlines
Newline(usize),
+ /// A special token used for things like inline code blocks that should be ignored by all
+ /// linters.
+ Unlintable,
}
impl TokenKind {
@@ -92,6 +96,12 @@ pub enum Punctuation {
ForwardSlash,
/// \
Backslash,
+ /// <
+ LessThan,
+ /// >
+ GreaterThan,
+ /// Equal
+ Equal,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
@@ -102,6 +112,10 @@ pub struct Quote {
pub trait TokenStringExt {
fn first_word(&self) -> Option<Token>;
+ /// Grabs the first word in the sentence.
+ /// Will also return [`None`] if there is an unlintable token in the position of the first
+ /// word.
+ fn first_sentence_word(&self) -> Option<Token>;
fn iter_word_indices(&self) -> impl Iterator<Item = usize> + '_;
fn iter_words(&self) -> impl Iterator<Item = &Token> + '_;
fn iter_space_indices(&self) -> impl Iterator<Item = usize> + '_;
@@ -118,6 +132,20 @@ impl TokenStringExt for [Token] {
self.iter().find(|v| v.kind.is_word()).copied()
}
+ fn first_sentence_word(&self) -> Option<Token> {
+ let (w_idx, word) = self.iter().find_position(|v| v.kind.is_word())?;
+
+ let Some(u_idx) = self.iter().position(|v| v.kind.is_unlintable()) else {
+ return Some(*word);
+ };
+
+ if w_idx > u_idx {
+ Some(*word)
+ } else {
+ None
+ }
+ }
+
fn iter_word_indices(&self) -> impl Iterator<Item = usize> + '_ {
self.iter()
.enumerate()
diff --git a/harper-ls/src/backend.rs b/harper-ls/src/backend.rs
index 1e50edebc..11c7b52e4 100644
--- a/harper-ls/src/backend.rs
+++ b/harper-ls/src/backend.rs
@@ -8,12 +8,13 @@ use tokio::sync::Mutex;
use tower_lsp::{
jsonrpc::Result,
lsp_types::{
- notification::PublishDiagnostics, CodeAction, CodeActionOrCommand, CodeActionParams,
- CodeActionProviderCapability, CodeActionResponse, Diagnostic, DidChangeTextDocumentParams,
- DidCloseTextDocumentParams, DidOpenTextDocumentParams, DidSaveTextDocumentParams,
- InitializeParams, InitializeResult, InitializedParams, MessageType,
- PublishDiagnosticsParams, Range, ServerCapabilities, TextDocumentSyncCapability,
- TextDocumentSyncKind, TextDocumentSyncOptions, TextDocumentSyncSaveOptions, Url,
+ notification::{PublishDiagnostics, ShowMessage},
+ CodeAction, CodeActionOrCommand, CodeActionParams, CodeActionProviderCapability,
+ CodeActionResponse, Diagnostic, DidChangeTextDocumentParams, DidCloseTextDocumentParams,
+ DidOpenTextDocumentParams, DidSaveTextDocumentParams, InitializeParams, InitializeResult,
+ InitializedParams, MessageType, PublishDiagnosticsParams, Range, ServerCapabilities,
+ ShowMessageParams, TextDocumentSyncCapability, TextDocumentSyncKind,
+ TextDocumentSyncOptions, TextDocumentSyncSaveOptions, Url,
},
Client, LanguageServer,
};
@@ -97,6 +98,17 @@ impl Backend {
}
async fn publish_diagnostics(&self, url: &Url) {
+ let client = self.client.clone();
+
+ tokio::spawn(async move {
+ client
+ .send_notification::<ShowMessage>(ShowMessageParams {
+ typ: MessageType::INFO,
+ message: "Linting...".to_string(),
+ })
+ .await
+ });
+
let diagnostics = self.generate_diagnostics(url).await;
let result = PublishDiagnosticsParams {