Skip to content

Offline, privacy-first grammar checker. Fast, open-source, Rust-powered

fix: non-English characters in words

Elijah Potter126aa37d0370
diff --git a/Cargo.lock b/Cargo.lock
index 5ba63be61..bd3b4fc84 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -547,6 +547,8 @@ dependencies = [
"serde",
"smallvec",
"thiserror",
+ "unicode-blocks",
+ "unicode-width",
]
[[package]]
@@ -1661,6 +1663,12 @@ version = "0.3.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75"
+[[package]]
+name = "unicode-blocks"
+version = "0.1.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6b12e05d9e06373163a9bb6bb8c263c261b396643a99445fe6b9811fd376581b"
+
[[package]]
name = "unicode-ident"
version = "1.0.12"
diff --git a/harper-core/Cargo.toml b/harper-core/Cargo.toml
index fa693d5c7..b04d54c9c 100644
--- a/harper-core/Cargo.toml
+++ b/harper-core/Cargo.toml
@@ -18,6 +18,8 @@ pulldown-cmark = "0.10.0"
serde = { version = "1.0.197", features = ["derive"] }
smallvec = { version = "1.13.1", features = ["serde"] }
thiserror = "1.0.57"
+unicode-blocks = "0.1.9"
+unicode-width = "0.1.11"
[dev-dependencies]
divan = "0.1.14"
diff --git a/harper-core/src/char_ext.rs b/harper-core/src/char_ext.rs
new file mode 100644
index 000000000..1c4110456
--- /dev/null
+++ b/harper-core/src/char_ext.rs
@@ -0,0 +1,41 @@
+use unicode_width::UnicodeWidthChar;
+
+use crate::Punctuation;
+
+pub trait CharExt {
+ /// Whether a character can be a component of a word.
+ fn is_lingual(&self) -> bool;
+ fn is_emoji(&self) -> bool;
+ fn is_punctuation(&self) -> bool;
+}
+
+impl CharExt for char {
+ fn is_lingual(&self) -> bool {
+ !self.is_whitespace()
+ && !self.is_numeric()
+ && !self.is_emoji()
+ && matches!(self.width(), Some(1..))
+ && !self.is_punctuation()
+ && (self.is_alphabetic() || matches!(self, '\u{00C0}'..='\u{00FF}'))
+ }
+
+ fn is_emoji(&self) -> bool {
+ let Some(block) = unicode_blocks::find_unicode_block(*self) else {
+ return false;
+ };
+
+ let blocks = [
+ unicode_blocks::SPECIALS,
+ unicode_blocks::EMOTICONS,
+ unicode_blocks::MISCELLANEOUS_SYMBOLS,
+ unicode_blocks::VARIATION_SELECTORS,
+ unicode_blocks::SUPPLEMENTAL_SYMBOLS_AND_PICTOGRAPHS
+ ];
+
+ blocks.contains(&block)
+ }
+
+ fn is_punctuation(&self) -> bool {
+ Punctuation::from_char(*self).is_some()
+ }
+}
diff --git a/harper-core/src/document.rs b/harper-core/src/document.rs
index f2a52ae60..d2be66021 100644
--- a/harper-core/src/document.rs
+++ b/harper-core/src/document.rs
@@ -5,8 +5,9 @@ use itertools::Itertools;
use crate::linting::Suggestion;
use crate::parsers::{Markdown, Parser, PlainEnglish};
+use crate::punctuation::Punctuation;
use crate::span::Span;
-use crate::{FatToken, Punctuation, Token, TokenKind, TokenStringExt};
+use crate::{FatToken, Token, TokenKind, TokenStringExt};
pub struct Document {
source: Vec<char>,
@@ -57,6 +58,37 @@ impl Document {
self.match_quotes();
}
+ /// Given a list of indices, this function removes the subsequent
+ /// `stretch_len - 1` elements after each index.
+ fn condense_indices(&mut self, indices: &[usize], stretch_len: usize) {
+ // Trim
+ let old = self.tokens.clone();
+ self.tokens.clear();
+
+ // Keep first chunk.
+ self.tokens
+ .extend_from_slice(&old[0..indices.first().copied().unwrap_or(indices.len())]);
+
+ let mut iter = indices.iter().peekable();
+
+ while let (Some(a_idx), b) = (iter.next(), iter.peek()) {
+ self.tokens.push(old[*a_idx]);
+
+ if let Some(b_idx) = b {
+ self.tokens
+ .extend_from_slice(&old[a_idx + stretch_len..**b_idx]);
+ }
+ }
+
+ // Keep last chunk.
+ self.tokens.extend_from_slice(
+ &old[indices
+ .last()
+ .map(|v| v + stretch_len)
+ .unwrap_or(indices.len())..]
+ )
+ }
+
pub fn get_token_at_char_index(&self, char_index: usize) -> Option<Token> {
let index = self
.tokens
@@ -248,35 +280,7 @@ impl Document {
}
}
- // Trim
- let old = self.tokens.clone();
- self.tokens.clear();
-
- // Keep first chunk.
- self.tokens.extend_from_slice(
- &old[0..replace_starts
- .first()
- .copied()
- .unwrap_or(replace_starts.len())]
- );
-
- let mut iter = replace_starts.iter().peekable();
-
- while let (Some(a_idx), b) = (iter.next(), iter.peek()) {
- self.tokens.push(old[*a_idx]);
-
- if let Some(b_idx) = b {
- self.tokens.extend_from_slice(&old[a_idx + 3..**b_idx]);
- }
- }
-
- // Keep last chunk.
- self.tokens.extend_from_slice(
- &old[replace_starts
- .last()
- .map(|v| v + 3)
- .unwrap_or(replace_starts.len())..]
- )
+ self.condense_indices(&replace_starts, 3);
}
}
diff --git a/harper-core/src/lexing/mod.rs b/harper-core/src/lexing/mod.rs
index 5e1e97620..9524e2b00 100644
--- a/harper-core/src/lexing/mod.rs
+++ b/harper-core/src/lexing/mod.rs
@@ -5,7 +5,9 @@ mod url;
use url::lex_url;
use self::email_address::lex_email_address;
-use crate::token::{Punctuation, Quote, TokenKind};
+use crate::char_ext::CharExt;
+use crate::punctuation::{Punctuation, Quote};
+use crate::token::TokenKind;
#[derive(Debug)]
pub struct FoundToken {
@@ -39,7 +41,7 @@ pub fn lex_token(source: &[char]) -> Option<FoundToken> {
fn lex_word(source: &[char]) -> Option<FoundToken> {
let end = source
.iter()
- .position(|c| !c.is_ascii_alphabetic())
+ .position(|c| !c.is_lingual())
.unwrap_or(source.len());
if end == 0 {
@@ -113,46 +115,7 @@ fn lex_punctuation(source: &[char]) -> Option<FoundToken> {
}
let c = source.first()?;
-
- use Punctuation::*;
-
- let punct = match c {
- '@' => At,
- '~' => Tilde,
- '=' => Equal,
- '<' => LessThan,
- '>' => GreaterThan,
- '/' => ForwardSlash,
- '\\' => Backslash,
- '%' => Percent,
- '’' => Apostrophe,
- '\'' => Apostrophe,
- '.' => Period,
- '!' => Bang,
- '?' => Question,
- ':' => Colon,
- ';' => Semicolon,
- ',' => Comma,
- '-' => Hyphen,
- '[' => OpenSquare,
- ']' => CloseSquare,
- '{' => OpenCurly,
- '}' => CloseCurly,
- '(' => OpenRound,
- ')' => CloseRound,
- '#' => Hash,
- '*' => Star,
- '&' => Ampersand,
- '–' => EnDash,
- '—' => EmDash,
- '…' => Ellipsis,
- '^' => Carrot,
- '+' => Plus,
- '$' => Dollar,
- '|' => Pipe,
- '_' => Underscore,
- _ => return None
- };
+ let punct = Punctuation::from_char(*c)?;
Some(FoundToken {
next_index: 1,
diff --git a/harper-core/src/lib.rs b/harper-core/src/lib.rs
index 94fc14c07..d85041fa1 100644
--- a/harper-core/src/lib.rs
+++ b/harper-core/src/lib.rs
@@ -1,10 +1,12 @@
#![allow(dead_code)]
+mod char_ext;
mod char_string;
mod document;
mod lexing;
mod linting;
pub mod parsers;
+mod punctuation;
mod span;
mod spell;
mod token;
@@ -12,9 +14,10 @@ mod token;
pub use char_string::{CharString, CharStringExt};
pub use document::Document;
pub use linting::{Lint, LintGroup, LintGroupConfig, LintKind, Linter, Suggestion};
+pub use punctuation::{Punctuation, Quote};
pub use span::Span;
pub use spell::{Dictionary, FullDictionary, MergedDictionary};
-pub use token::{FatToken, Punctuation, Token, TokenKind, TokenStringExt};
+pub use token::{FatToken, Token, TokenKind, TokenStringExt};
/// A utility function that removes overlapping lints in a vector,
/// keeping the more important ones.
diff --git a/harper-core/src/linting/unclosed_quotes.rs b/harper-core/src/linting/unclosed_quotes.rs
index a7083c496..e6b821394 100644
--- a/harper-core/src/linting/unclosed_quotes.rs
+++ b/harper-core/src/linting/unclosed_quotes.rs
@@ -1,7 +1,6 @@
use super::{Lint, LintKind, Linter};
use crate::document::Document;
-use crate::token::Quote;
-use crate::{Punctuation, TokenKind};
+use crate::{Punctuation, Quote, TokenKind};
#[derive(Debug, Clone, Copy, Default)]
pub struct UnclosedQuotes;
diff --git a/harper-core/src/parsers/markdown.rs b/harper-core/src/parsers/markdown.rs
index cfc16aa3b..68ae8dac5 100644
--- a/harper-core/src/parsers/markdown.rs
+++ b/harper-core/src/parsers/markdown.rs
@@ -39,10 +39,14 @@ impl Parser for Markdown {
}
pulldown_cmark::Event::Start(tag) => stack.push(tag),
pulldown_cmark::Event::End(pulldown_cmark::TagEnd::Paragraph)
- | pulldown_cmark::Event::End(pulldown_cmark::TagEnd::Item) => tokens.push(Token {
- span: Span::new_with_len(traversed_chars, 1),
- kind: TokenKind::Newline(2)
- }),
+ | pulldown_cmark::Event::End(pulldown_cmark::TagEnd::Item)
+ | pulldown_cmark::Event::End(pulldown_cmark::TagEnd::TableCell) => {
+ tokens.push(Token {
+ span: Span::new_with_len(traversed_chars, 0),
+ kind: TokenKind::Newline(2)
+ });
+ stack.pop();
+ }
pulldown_cmark::Event::End(_) => {
stack.pop();
}
diff --git a/harper-core/src/parsers/mod.rs b/harper-core/src/parsers/mod.rs
index b19f2ce16..98ed7a586 100644
--- a/harper-core/src/parsers/mod.rs
+++ b/harper-core/src/parsers/mod.rs
@@ -4,7 +4,7 @@ mod plain_english;
pub use markdown::Markdown;
pub use plain_english::PlainEnglish;
-pub use crate::token::{Quote, Token, TokenKind, TokenStringExt};
+pub use crate::token::{Token, TokenKind, TokenStringExt};
pub trait Parser: Send + Sync {
fn parse(&mut self, source: &[char]) -> Vec<Token>;
@@ -110,4 +110,12 @@ mod tests {
]
);
}
+
+ /// Make sure that the English parser correctly identifies non-english
+ /// characters as part of the same word.
+ #[test]
+ fn parses_non_english() {
+ assert_tokens_eq_plain("Løvetann", &[Word]);
+ assert_tokens_eq_plain("Naïve", &[Word]);
+ }
}
diff --git a/harper-core/src/punctuation.rs b/harper-core/src/punctuation.rs
new file mode 100644
index 000000000..c57e11ccc
--- /dev/null
+++ b/harper-core/src/punctuation.rs
@@ -0,0 +1,125 @@
+use is_macro::Is;
+use serde::{Deserialize, Serialize};
+
+#[derive(Debug, Is, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, PartialOrd)]
+#[serde(tag = "kind")]
+pub enum Punctuation {
+ /// `…`
+ Ellipsis,
+ /// `–`
+ EnDash,
+ /// `—`
+ EmDash,
+ /// `&`
+ Ampersand,
+ /// `.`
+ Period,
+ /// `!`
+ Bang,
+ /// `?`
+ Question,
+ /// `:`
+ Colon,
+ /// ``;``
+ Semicolon,
+ /// `"`
+ Quote(Quote),
+ /// `,`
+ Comma,
+ /// `-`
+ Hyphen,
+ /// `[`
+ OpenSquare,
+ /// `]`
+ CloseSquare,
+ /// `(`
+ OpenRound,
+ /// `)`
+ CloseRound,
+ /// `{`
+ OpenCurly,
+ /// `}`
+ CloseCurly,
+ /// `"`
+ Hash,
+ /// `'`
+ Apostrophe,
+ /// `%`
+ Percent,
+ /// `/`
+ ForwardSlash,
+ /// `\`
+ Backslash,
+ /// `<`
+ LessThan,
+ /// `>`
+ GreaterThan,
+ /// `=`
+ Equal,
+ /// `*`
+ Star,
+ /// `~`
+ Tilde,
+ /// `@`
+ At,
+ /// `^`
+ Carrot,
+ /// `+`
+ Plus,
+ /// `$`
+ Dollar,
+ /// `|`
+ Pipe,
+ /// `_`
+ Underscore
+}
+
+impl Punctuation {
+ pub fn from_char(c: char) -> Option<Punctuation> {
+ let punct = match c {
+ '@' => Punctuation::At,
+ '~' => Punctuation::Tilde,
+ '=' => Punctuation::Equal,
+ '<' => Punctuation::LessThan,
+ '>' => Punctuation::GreaterThan,
+ '/' => Punctuation::ForwardSlash,
+ '\\' => Punctuation::Backslash,
+ '%' => Punctuation::Percent,
+ '’' => Punctuation::Apostrophe,
+ '\'' => Punctuation::Apostrophe,
+ '.' => Punctuation::Period,
+ '!' => Punctuation::Bang,
+ '?' => Punctuation::Question,
+ ':' => Punctuation::Colon,
+ ';' => Punctuation::Semicolon,
+ ',' => Punctuation::Comma,
+ '-' => Punctuation::Hyphen,
+ '[' => Punctuation::OpenSquare,
+ ']' => Punctuation::CloseSquare,
+ '{' => Punctuation::OpenCurly,
+ '}' => Punctuation::CloseCurly,
+ '(' => Punctuation::OpenRound,
+ ')' => Punctuation::CloseRound,
+ '#' => Punctuation::Hash,
+ '*' => Punctuation::Star,
+ '&' => Punctuation::Ampersand,
+ '–' => Punctuation::EnDash,
+ '—' => Punctuation::EmDash,
+ '…' => Punctuation::Ellipsis,
+ '^' => Punctuation::Carrot,
+ '+' => Punctuation::Plus,
+ '$' => Punctuation::Dollar,
+ '|' => Punctuation::Pipe,
+ '_' => Punctuation::Underscore,
+ _ => return None
+ };
+
+ Some(punct)
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, PartialOrd)]
+pub struct Quote {
+ /// The location of the matching quote, if it exists.
+ pub twin_loc: Option<usize>
+}
diff --git a/harper-core/src/token.rs b/harper-core/src/token.rs
index dfac8ba8e..b3a707152 100644
--- a/harper-core/src/token.rs
+++ b/harper-core/src/token.rs
@@ -2,7 +2,9 @@ use is_macro::Is;
use itertools::Itertools;
use serde::{Deserialize, Serialize};
+use crate::punctuation::Punctuation;
use crate::span::Span;
+use crate::Quote;
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Default)]
pub struct Token {
@@ -68,85 +70,6 @@ impl TokenKind {
}
}
-#[derive(Debug, Is, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, PartialOrd)]
-#[serde(tag = "kind")]
-pub enum Punctuation {
- /// `…`
- Ellipsis,
- /// `–`
- EnDash,
- /// `—`
- EmDash,
- /// `&`
- Ampersand,
- /// `.`
- Period,
- /// `!`
- Bang,
- /// `?`
- Question,
- /// `:`
- Colon,
- /// ``;``
- Semicolon,
- /// `"`
- Quote(Quote),
- /// `,`
- Comma,
- /// `-`
- Hyphen,
- /// `[`
- OpenSquare,
- /// `]`
- CloseSquare,
- /// `(`
- OpenRound,
- /// `)`
- CloseRound,
- /// `{`
- OpenCurly,
- /// `}`
- CloseCurly,
- /// `"`
- Hash,
- /// `'`
- Apostrophe,
- /// `%`
- Percent,
- /// `/`
- ForwardSlash,
- /// `\`
- Backslash,
- /// `<`
- LessThan,
- /// `>`
- GreaterThan,
- /// `=`
- Equal,
- /// `*`
- Star,
- /// `~`
- Tilde,
- /// `@`
- At,
- /// `^`
- Carrot,
- /// `+`
- Plus,
- /// `$`
- Dollar,
- /// `|`
- Pipe,
- /// `_`
- Underscore
-}
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, PartialOrd)]
-pub struct Quote {
- /// The location of the matching quote, if it exists.
- pub twin_loc: Option<usize>
-}
-
pub trait TokenStringExt {
fn first_word(&self) -> Option<Token>;
/// Grabs the first word in the sentence.
diff --git a/harper-ls/src/backend.rs b/harper-ls/src/backend.rs
index 6cc93420d..19070a0a0 100644
--- a/harper-ls/src/backend.rs
+++ b/harper-ls/src/backend.rs
@@ -16,7 +16,7 @@ use harper_core::{
use serde_json::Value;
use tokio::sync::{Mutex, RwLock};
use tower_lsp::jsonrpc::Result;
-use tower_lsp::lsp_types::notification::{PublishDiagnostics, ShowMessage};
+use tower_lsp::lsp_types::notification::PublishDiagnostics;
use tower_lsp::lsp_types::{
CodeActionOrCommand,
CodeActionParams,
@@ -38,7 +38,6 @@ use tower_lsp::lsp_types::{
PublishDiagnosticsParams,
Range,
ServerCapabilities,
- ShowMessageParams,
TextDocumentSyncCapability,
TextDocumentSyncKind,
TextDocumentSyncOptions,
@@ -120,8 +119,6 @@ impl Backend {
#[instrument(skip(self, dict))]
async fn save_file_dictionary(&self, url: &Url, dict: impl Dictionary) -> anyhow::Result<()> {
- dbg!(self.get_file_dict_path(url).await);
-
Ok(save_dict(self.get_file_dict_path(url).await, dict).await?)
}
@@ -329,17 +326,6 @@ impl Backend {
#[instrument(skip(self))]
async fn publish_diagnostics(&self, url: &Url) {
- let client = self.client.clone();
-
- tokio::spawn(async move {
- client
- .send_notification::<ShowMessage>(ShowMessageParams {
- typ: MessageType::INFO,
- message: "Linting...".to_string()
- })
- .await
- });
-
let diagnostics = self.generate_diagnostics(url).await;
let result = PublishDiagnosticsParams {
@@ -524,8 +510,6 @@ impl LanguageServer for Backend {
}
};
- dbg!(new_config.diagnostic_severity);
-
{
let mut config = self.config.write().await;
*config = new_config;