Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
fix: non-English characters in words
Elijah Potter126aa37d0370
diff --git a/Cargo.lock b/Cargo.lockindex 5ba63be61..bd3b4fc84 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -547,6 +547,8 @@ dependencies = ["serde","smallvec","thiserror",+ "unicode-blocks",+ "unicode-width",][[package]]@@ -1661,6 +1663,12 @@ version = "0.3.15"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75"+[[package]]+name = "unicode-blocks"+version = "0.1.9"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "6b12e05d9e06373163a9bb6bb8c263c261b396643a99445fe6b9811fd376581b"+[[package]]name = "unicode-ident"version = "1.0.12"diff --git a/harper-core/Cargo.toml b/harper-core/Cargo.tomlindex fa693d5c7..b04d54c9c 100644--- a/harper-core/Cargo.toml+++ b/harper-core/Cargo.toml@@ -18,6 +18,8 @@ pulldown-cmark = "0.10.0"serde = { version = "1.0.197", features = ["derive"] }smallvec = { version = "1.13.1", features = ["serde"] }thiserror = "1.0.57"+unicode-blocks = "0.1.9"+unicode-width = "0.1.11"[dev-dependencies]divan = "0.1.14"diff --git a/harper-core/src/char_ext.rs b/harper-core/src/char_ext.rsnew file mode 100644index 000000000..1c4110456--- /dev/null+++ b/harper-core/src/char_ext.rs@@ -0,0 +1,41 @@+use unicode_width::UnicodeWidthChar;++use crate::Punctuation;++pub trait CharExt {+ /// Whether a character can be a component of a word.+ fn is_lingual(&self) -> bool;+ fn is_emoji(&self) -> bool;+ fn is_punctuation(&self) -> bool;+}++impl CharExt for char {+ fn is_lingual(&self) -> bool {+ !self.is_whitespace()+ && !self.is_numeric()+ && !self.is_emoji()+ && matches!(self.width(), Some(1..))+ && !self.is_punctuation()+ && (self.is_alphabetic() || matches!(self, '\u{00C0}'..='\u{00FF}'))+ }++ fn is_emoji(&self) -> bool {+ let Some(block) = unicode_blocks::find_unicode_block(*self) else {+ return false;+ };++ let blocks = [+ unicode_blocks::SPECIALS,+ unicode_blocks::EMOTICONS,+ unicode_blocks::MISCELLANEOUS_SYMBOLS,+ unicode_blocks::VARIATION_SELECTORS,+ unicode_blocks::SUPPLEMENTAL_SYMBOLS_AND_PICTOGRAPHS+ ];++ blocks.contains(&block)+ }++ fn is_punctuation(&self) -> bool {+ Punctuation::from_char(*self).is_some()+ }+}diff --git a/harper-core/src/document.rs b/harper-core/src/document.rsindex f2a52ae60..d2be66021 100644--- a/harper-core/src/document.rs+++ b/harper-core/src/document.rs@@ -5,8 +5,9 @@ use itertools::Itertools;use crate::linting::Suggestion;use crate::parsers::{Markdown, Parser, PlainEnglish};+use crate::punctuation::Punctuation;use crate::span::Span;-use crate::{FatToken, Punctuation, Token, TokenKind, TokenStringExt};+use crate::{FatToken, Token, TokenKind, TokenStringExt};pub struct Document {source: Vec<char>,@@ -57,6 +58,37 @@ impl Document {self.match_quotes();}+ /// Given a list of indices, this function removes the subsequent+ /// `stretch_len - 1` elements after each index.+ fn condense_indices(&mut self, indices: &[usize], stretch_len: usize) {+ // Trim+ let old = self.tokens.clone();+ self.tokens.clear();++ // Keep first chunk.+ self.tokens+ .extend_from_slice(&old[0..indices.first().copied().unwrap_or(indices.len())]);++ let mut iter = indices.iter().peekable();++ while let (Some(a_idx), b) = (iter.next(), iter.peek()) {+ self.tokens.push(old[*a_idx]);++ if let Some(b_idx) = b {+ self.tokens+ .extend_from_slice(&old[a_idx + stretch_len..**b_idx]);+ }+ }++ // Keep last chunk.+ self.tokens.extend_from_slice(+ &old[indices+ .last()+ .map(|v| v + stretch_len)+ .unwrap_or(indices.len())..]+ )+ }+pub fn get_token_at_char_index(&self, char_index: usize) -> Option<Token> {let index = self.tokens@@ -248,35 +280,7 @@ impl Document {}}- // Trim- let old = self.tokens.clone();- self.tokens.clear();-- // Keep first chunk.- self.tokens.extend_from_slice(- &old[0..replace_starts- .first()- .copied()- .unwrap_or(replace_starts.len())]- );-- let mut iter = replace_starts.iter().peekable();-- while let (Some(a_idx), b) = (iter.next(), iter.peek()) {- self.tokens.push(old[*a_idx]);-- if let Some(b_idx) = b {- self.tokens.extend_from_slice(&old[a_idx + 3..**b_idx]);- }- }-- // Keep last chunk.- self.tokens.extend_from_slice(- &old[replace_starts- .last()- .map(|v| v + 3)- .unwrap_or(replace_starts.len())..]- )+ self.condense_indices(&replace_starts, 3);}}diff --git a/harper-core/src/lexing/mod.rs b/harper-core/src/lexing/mod.rsindex 5e1e97620..9524e2b00 100644--- a/harper-core/src/lexing/mod.rs+++ b/harper-core/src/lexing/mod.rs@@ -5,7 +5,9 @@ mod url;use url::lex_url;use self::email_address::lex_email_address;-use crate::token::{Punctuation, Quote, TokenKind};+use crate::char_ext::CharExt;+use crate::punctuation::{Punctuation, Quote};+use crate::token::TokenKind;#[derive(Debug)]pub struct FoundToken {@@ -39,7 +41,7 @@ pub fn lex_token(source: &[char]) -> Option<FoundToken> {fn lex_word(source: &[char]) -> Option<FoundToken> {let end = source.iter()- .position(|c| !c.is_ascii_alphabetic())+ .position(|c| !c.is_lingual()).unwrap_or(source.len());if end == 0 {@@ -113,46 +115,7 @@ fn lex_punctuation(source: &[char]) -> Option<FoundToken> {}let c = source.first()?;-- use Punctuation::*;-- let punct = match c {- '@' => At,- '~' => Tilde,- '=' => Equal,- '<' => LessThan,- '>' => GreaterThan,- '/' => ForwardSlash,- '\\' => Backslash,- '%' => Percent,- '’' => Apostrophe,- '\'' => Apostrophe,- '.' => Period,- '!' => Bang,- '?' => Question,- ':' => Colon,- ';' => Semicolon,- ',' => Comma,- '-' => Hyphen,- '[' => OpenSquare,- ']' => CloseSquare,- '{' => OpenCurly,- '}' => CloseCurly,- '(' => OpenRound,- ')' => CloseRound,- '#' => Hash,- '*' => Star,- '&' => Ampersand,- '–' => EnDash,- '—' => EmDash,- '…' => Ellipsis,- '^' => Carrot,- '+' => Plus,- '$' => Dollar,- '|' => Pipe,- '_' => Underscore,- _ => return None- };+ let punct = Punctuation::from_char(*c)?;Some(FoundToken {next_index: 1,diff --git a/harper-core/src/lib.rs b/harper-core/src/lib.rsindex 94fc14c07..d85041fa1 100644--- a/harper-core/src/lib.rs+++ b/harper-core/src/lib.rs@@ -1,10 +1,12 @@#![allow(dead_code)]+mod char_ext;mod char_string;mod document;mod lexing;mod linting;pub mod parsers;+mod punctuation;mod span;mod spell;mod token;@@ -12,9 +14,10 @@ mod token;pub use char_string::{CharString, CharStringExt};pub use document::Document;pub use linting::{Lint, LintGroup, LintGroupConfig, LintKind, Linter, Suggestion};+pub use punctuation::{Punctuation, Quote};pub use span::Span;pub use spell::{Dictionary, FullDictionary, MergedDictionary};-pub use token::{FatToken, Punctuation, Token, TokenKind, TokenStringExt};+pub use token::{FatToken, Token, TokenKind, TokenStringExt};/// A utility function that removes overlapping lints in a vector,/// keeping the more important ones.diff --git a/harper-core/src/linting/unclosed_quotes.rs b/harper-core/src/linting/unclosed_quotes.rsindex a7083c496..e6b821394 100644--- a/harper-core/src/linting/unclosed_quotes.rs+++ b/harper-core/src/linting/unclosed_quotes.rs@@ -1,7 +1,6 @@use super::{Lint, LintKind, Linter};use crate::document::Document;-use crate::token::Quote;-use crate::{Punctuation, TokenKind};+use crate::{Punctuation, Quote, TokenKind};#[derive(Debug, Clone, Copy, Default)]pub struct UnclosedQuotes;diff --git a/harper-core/src/parsers/markdown.rs b/harper-core/src/parsers/markdown.rsindex cfc16aa3b..68ae8dac5 100644--- a/harper-core/src/parsers/markdown.rs+++ b/harper-core/src/parsers/markdown.rs@@ -39,10 +39,14 @@ impl Parser for Markdown {}pulldown_cmark::Event::Start(tag) => stack.push(tag),pulldown_cmark::Event::End(pulldown_cmark::TagEnd::Paragraph)- | pulldown_cmark::Event::End(pulldown_cmark::TagEnd::Item) => tokens.push(Token {- span: Span::new_with_len(traversed_chars, 1),- kind: TokenKind::Newline(2)- }),+ | pulldown_cmark::Event::End(pulldown_cmark::TagEnd::Item)+ | pulldown_cmark::Event::End(pulldown_cmark::TagEnd::TableCell) => {+ tokens.push(Token {+ span: Span::new_with_len(traversed_chars, 0),+ kind: TokenKind::Newline(2)+ });+ stack.pop();+ }pulldown_cmark::Event::End(_) => {stack.pop();}diff --git a/harper-core/src/parsers/mod.rs b/harper-core/src/parsers/mod.rsindex b19f2ce16..98ed7a586 100644--- a/harper-core/src/parsers/mod.rs+++ b/harper-core/src/parsers/mod.rs@@ -4,7 +4,7 @@ mod plain_english;pub use markdown::Markdown;pub use plain_english::PlainEnglish;-pub use crate::token::{Quote, Token, TokenKind, TokenStringExt};+pub use crate::token::{Token, TokenKind, TokenStringExt};pub trait Parser: Send + Sync {fn parse(&mut self, source: &[char]) -> Vec<Token>;@@ -110,4 +110,12 @@ mod tests {]);}++ /// Make sure that the English parser correctly identifies non-english+ /// characters as part of the same word.+ #[test]+ fn parses_non_english() {+ assert_tokens_eq_plain("Løvetann", &[Word]);+ assert_tokens_eq_plain("Naïve", &[Word]);+ }}diff --git a/harper-core/src/punctuation.rs b/harper-core/src/punctuation.rsnew file mode 100644index 000000000..c57e11ccc--- /dev/null+++ b/harper-core/src/punctuation.rs@@ -0,0 +1,125 @@+use is_macro::Is;+use serde::{Deserialize, Serialize};++#[derive(Debug, Is, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, PartialOrd)]+#[serde(tag = "kind")]+pub enum Punctuation {+ /// `…`+ Ellipsis,+ /// `–`+ EnDash,+ /// `—`+ EmDash,+ /// `&`+ Ampersand,+ /// `.`+ Period,+ /// `!`+ Bang,+ /// `?`+ Question,+ /// `:`+ Colon,+ /// ``;``+ Semicolon,+ /// `"`+ Quote(Quote),+ /// `,`+ Comma,+ /// `-`+ Hyphen,+ /// `[`+ OpenSquare,+ /// `]`+ CloseSquare,+ /// `(`+ OpenRound,+ /// `)`+ CloseRound,+ /// `{`+ OpenCurly,+ /// `}`+ CloseCurly,+ /// `"`+ Hash,+ /// `'`+ Apostrophe,+ /// `%`+ Percent,+ /// `/`+ ForwardSlash,+ /// `\`+ Backslash,+ /// `<`+ LessThan,+ /// `>`+ GreaterThan,+ /// `=`+ Equal,+ /// `*`+ Star,+ /// `~`+ Tilde,+ /// `@`+ At,+ /// `^`+ Carrot,+ /// `+`+ Plus,+ /// `$`+ Dollar,+ /// `|`+ Pipe,+ /// `_`+ Underscore+}++impl Punctuation {+ pub fn from_char(c: char) -> Option<Punctuation> {+ let punct = match c {+ '@' => Punctuation::At,+ '~' => Punctuation::Tilde,+ '=' => Punctuation::Equal,+ '<' => Punctuation::LessThan,+ '>' => Punctuation::GreaterThan,+ '/' => Punctuation::ForwardSlash,+ '\\' => Punctuation::Backslash,+ '%' => Punctuation::Percent,+ '’' => Punctuation::Apostrophe,+ '\'' => Punctuation::Apostrophe,+ '.' => Punctuation::Period,+ '!' => Punctuation::Bang,+ '?' => Punctuation::Question,+ ':' => Punctuation::Colon,+ ';' => Punctuation::Semicolon,+ ',' => Punctuation::Comma,+ '-' => Punctuation::Hyphen,+ '[' => Punctuation::OpenSquare,+ ']' => Punctuation::CloseSquare,+ '{' => Punctuation::OpenCurly,+ '}' => Punctuation::CloseCurly,+ '(' => Punctuation::OpenRound,+ ')' => Punctuation::CloseRound,+ '#' => Punctuation::Hash,+ '*' => Punctuation::Star,+ '&' => Punctuation::Ampersand,+ '–' => Punctuation::EnDash,+ '—' => Punctuation::EmDash,+ '…' => Punctuation::Ellipsis,+ '^' => Punctuation::Carrot,+ '+' => Punctuation::Plus,+ '$' => Punctuation::Dollar,+ '|' => Punctuation::Pipe,+ '_' => Punctuation::Underscore,+ _ => return None+ };++ Some(punct)+ }+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, PartialOrd)]+pub struct Quote {+ /// The location of the matching quote, if it exists.+ pub twin_loc: Option<usize>+}diff --git a/harper-core/src/token.rs b/harper-core/src/token.rsindex dfac8ba8e..b3a707152 100644--- a/harper-core/src/token.rs+++ b/harper-core/src/token.rs@@ -2,7 +2,9 @@ use is_macro::Is;use itertools::Itertools;use serde::{Deserialize, Serialize};+use crate::punctuation::Punctuation;use crate::span::Span;+use crate::Quote;#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Default)]pub struct Token {@@ -68,85 +70,6 @@ impl TokenKind {}}-#[derive(Debug, Is, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, PartialOrd)]-#[serde(tag = "kind")]-pub enum Punctuation {- /// `…`- Ellipsis,- /// `–`- EnDash,- /// `—`- EmDash,- /// `&`- Ampersand,- /// `.`- Period,- /// `!`- Bang,- /// `?`- Question,- /// `:`- Colon,- /// ``;``- Semicolon,- /// `"`- Quote(Quote),- /// `,`- Comma,- /// `-`- Hyphen,- /// `[`- OpenSquare,- /// `]`- CloseSquare,- /// `(`- OpenRound,- /// `)`- CloseRound,- /// `{`- OpenCurly,- /// `}`- CloseCurly,- /// `"`- Hash,- /// `'`- Apostrophe,- /// `%`- Percent,- /// `/`- ForwardSlash,- /// `\`- Backslash,- /// `<`- LessThan,- /// `>`- GreaterThan,- /// `=`- Equal,- /// `*`- Star,- /// `~`- Tilde,- /// `@`- At,- /// `^`- Carrot,- /// `+`- Plus,- /// `$`- Dollar,- /// `|`- Pipe,- /// `_`- Underscore-}--#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, PartialOrd)]-pub struct Quote {- /// The location of the matching quote, if it exists.- pub twin_loc: Option<usize>-}-pub trait TokenStringExt {fn first_word(&self) -> Option<Token>;/// Grabs the first word in the sentence.diff --git a/harper-ls/src/backend.rs b/harper-ls/src/backend.rsindex 6cc93420d..19070a0a0 100644--- a/harper-ls/src/backend.rs+++ b/harper-ls/src/backend.rs@@ -16,7 +16,7 @@ use harper_core::{use serde_json::Value;use tokio::sync::{Mutex, RwLock};use tower_lsp::jsonrpc::Result;-use tower_lsp::lsp_types::notification::{PublishDiagnostics, ShowMessage};+use tower_lsp::lsp_types::notification::PublishDiagnostics;use tower_lsp::lsp_types::{CodeActionOrCommand,CodeActionParams,@@ -38,7 +38,6 @@ use tower_lsp::lsp_types::{PublishDiagnosticsParams,Range,ServerCapabilities,- ShowMessageParams,TextDocumentSyncCapability,TextDocumentSyncKind,TextDocumentSyncOptions,@@ -120,8 +119,6 @@ impl Backend {#[instrument(skip(self, dict))]async fn save_file_dictionary(&self, url: &Url, dict: impl Dictionary) -> anyhow::Result<()> {- dbg!(self.get_file_dict_path(url).await);-Ok(save_dict(self.get_file_dict_path(url).await, dict).await?)}@@ -329,17 +326,6 @@ impl Backend {#[instrument(skip(self))]async fn publish_diagnostics(&self, url: &Url) {- let client = self.client.clone();-- tokio::spawn(async move {- client- .send_notification::<ShowMessage>(ShowMessageParams {- typ: MessageType::INFO,- message: "Linting...".to_string()- })- .await- });-let diagnostics = self.generate_diagnostics(url).await;let result = PublishDiagnosticsParams {@@ -524,8 +510,6 @@ impl LanguageServer for Backend {}};- dbg!(new_config.diagnostic_severity);-{let mut config = self.config.write().await;*config = new_config;