Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
fix: Harper now ignores CJK characters
Elijah Potter50c8116a501b
diff --git a/harper-cli/src/main.rs b/harper-cli/src/main.rsindex ea8e4cbc1..fdd0ac804 100644--- a/harper-cli/src/main.rs+++ b/harper-cli/src/main.rs@@ -6,6 +6,7 @@ use anyhow::format_err;use ariadne::{Color, Label, Report, ReportKind, Source};use clap::Parser;use harper_comments::CommentParser;+use harper_core::parsers::Markdown;use harper_core::{remove_overlaps, Document, FullDictionary, LintGroup, LintGroupConfig, Linter};#[derive(Debug, Parser)]@@ -88,8 +89,16 @@ fn main() -> anyhow::Result<()> {fn load_file(file: &Path) -> anyhow::Result<(Document, String)> {let source = std::fs::read_to_string(file)?;- let parser = CommentParser::new_from_filename(file)- .ok_or(format_err!("Could not detect language ID."))?;+ let parser: Box<dyn harper_core::parsers::Parser> =+ if let Some("md") = file.extension().map(|v| v.to_str().unwrap()) {+ Box::new(Markdown)+ } else {+ Box::new(+ CommentParser::new_from_filename(file)+ .map(Box::new)+ .ok_or(format_err!("Could not detect language ID."))?+ )+ };Ok((Document::new(&source, Box::new(parser)), source))}diff --git a/harper-core/src/char_ext.rs b/harper-core/src/char_ext.rsindex 1c4110456..224d657cb 100644--- a/harper-core/src/char_ext.rs+++ b/harper-core/src/char_ext.rs@@ -3,20 +3,22 @@ use unicode_width::UnicodeWidthChar;use crate::Punctuation;pub trait CharExt {- /// Whether a character can be a component of a word.- fn is_lingual(&self) -> bool;+ fn is_cjk(&self) -> bool;+ /// Whether a character can be a component of an English word.+ fn is_english_lingual(&self) -> bool;fn is_emoji(&self) -> bool;fn is_punctuation(&self) -> bool;}impl CharExt for char {- fn is_lingual(&self) -> bool {+ fn is_english_lingual(&self) -> bool {!self.is_whitespace()&& !self.is_numeric()&& !self.is_emoji()&& matches!(self.width(), Some(1..))&& !self.is_punctuation()- && (self.is_alphabetic() || matches!(self, '\u{00C0}'..='\u{00FF}'))+ && self.is_alphabetic()+ && !self.is_cjk()}fn is_emoji(&self) -> bool {@@ -35,7 +37,38 @@ impl CharExt for char {blocks.contains(&block)}+ fn is_cjk(&self) -> bool {+ let Some(block) = unicode_blocks::find_unicode_block(*self) else {+ return false;+ };++ let blocks = [+ unicode_blocks::CJK_UNIFIED_IDEOGRAPHS,+ unicode_blocks::CJK_UNIFIED_IDEOGRAPHS_EXTENSION_A,+ unicode_blocks::CJK_UNIFIED_IDEOGRAPHS_EXTENSION_B,+ unicode_blocks::CJK_UNIFIED_IDEOGRAPHS_EXTENSION_C,+ unicode_blocks::CJK_UNIFIED_IDEOGRAPHS_EXTENSION_D,+ unicode_blocks::CJK_UNIFIED_IDEOGRAPHS_EXTENSION_E,+ unicode_blocks::CJK_UNIFIED_IDEOGRAPHS_EXTENSION_F,+ unicode_blocks::CJK_UNIFIED_IDEOGRAPHS_EXTENSION_G,+ unicode_blocks::CJK_UNIFIED_IDEOGRAPHS_EXTENSION_H,+ unicode_blocks::CJK_UNIFIED_IDEOGRAPHS_EXTENSION_I+ ];++ blocks.contains(&block)+ }+fn is_punctuation(&self) -> bool {Punctuation::from_char(*self).is_some()}}++#[cfg(test)]+mod tests {+ use super::CharExt;++ #[test]+ fn cjk_is_not_english_lingual() {+ assert!(!'世'.is_english_lingual())+ }+}diff --git a/harper-core/src/lexing/mod.rs b/harper-core/src/lexing/mod.rsindex 998b31807..a6b6d8ed2 100644--- a/harper-core/src/lexing/mod.rs+++ b/harper-core/src/lexing/mod.rs@@ -43,7 +43,7 @@ pub fn lex_token(source: &[char]) -> Option<FoundToken> {fn lex_word(source: &[char]) -> Option<FoundToken> {let end = source.iter()- .position(|c| !c.is_lingual())+ .position(|c| !c.is_english_lingual()).unwrap_or(source.len());if end == 0 {@@ -158,3 +158,14 @@ fn lex_catch(_source: &[char]) -> Option<FoundToken> {token: TokenKind::Unlintable})}++#[cfg(test)]+mod tests {+ use super::lex_word;++ #[test]+ fn lexes_cjk_as_unlintable() {+ let source: Vec<_> = "世".chars().collect();+ assert!(lex_word(&source).is_none());+ }+}diff --git a/harper-core/tests/run_tests.rs b/harper-core/tests/run_tests.rsindex 8a4408275..e4509c696 100644--- a/harper-core/tests/run_tests.rs+++ b/harper-core/tests/run_tests.rs@@ -31,3 +31,5 @@ macro_rules! create_test {create_test!(whack_bullets.md, 1);create_test!(preexisting.md, 0);+create_test!(issue_109.md, 0);+create_test!(chinese_lorem_ipsum.md, 2);diff --git a/harper-core/tests/test_sources/chinese_lorem_ipsum.md b/harper-core/tests/test_sources/chinese_lorem_ipsum.mdnew file mode 100644index 000000000..75c60f701--- /dev/null+++ b/harper-core/tests/test_sources/chinese_lorem_ipsum.md@@ -0,0 +1,17 @@+The following text was generated using [a Chinese lorem ipsum generator](https://pinkylam.me/generator/chinese-lorem-ipsum/).++食棵支每躲種。奶象打星爪子二細喜才記行在發像原斤!頁固點子衣點豆看身蝴看苗急午公何足,筆娘經色蝶行元香也要。麻了綠尼固世,色北書目登功;因告黑。++從下由服行巴登魚,棵苦亮珠春用訴星比都由發丟法主犬水現,能國書午錯馬吹隻幾外。筆泉可去牙用同貝步登娘毛完且哭,北巴種化吃石正由讀借示!幸造愛扒,才怕至成下。++亮您紅星浪中兩安乙菜上做玉做火愛玩植結面,蝴哭往兒完哥封波在,林春田早次王同月清丁筆道兩請玩燈們唱。++十兌雞做喜木發屋害要書目把麼早父方從以。冰正共片海告假衣夏冒吃戊很口松牠室肖。斗門看登給畫聽斤婆品歡個。++澡弓兩化員把戊各但他用親!給好校魚火包,根朵告幫着道具象海太或文苗。西首友雞頭、我跳花發,可功才門田訴去首或行。++吧都上午京主根相由或想、澡快送哭麻細麻白言原風童乍外、右更新一婆幸七片院。教刀麻哥幾在同四就遠新來長喜呢雪:兒時貓千十門功玩刀比。媽習出才,借食乞原成欠波米掃古校良具們昔!少學刀工火入。++兔追找古裏雞喜假經南瓜枝未發發几!菜書共媽申欠彩因到「金羊昌追秋才着海隻」雪像麻孝亮乞土點首事向往!知瓜祖。++因主很貫唱重目刃右,封飯房屋造土都犬帶刃抱他重他冬幸以坡十雲?跳肖頭反入小坐歡院帶士自央。diff --git a/harper-core/tests/test_sources/issue_109.md b/harper-core/tests/test_sources/issue_109.mdnew file mode 100644index 000000000..32340f872--- /dev/null+++ b/harper-core/tests/test_sources/issue_109.md@@ -0,0 +1 @@+本文档是使用 Microsoft Translate 翻译的。哈珀根本不应该抱怨。diff --git a/justfile b/justfileindex 13fc8a79a..dcaf1fccd 100644--- a/justfile+++ b/justfile@@ -99,3 +99,5 @@ test:parse file:cargo run --bin harper-cli -- parse {{file}}+lint file:+ cargo run --bin harper-cli -- lint {{file}}