Skip to content

Offline, privacy-first grammar checker. Fast, open-source, Rust-powered

fix: Harper now ignores CJK characters

Elijah Potter50c8116a501b
diff --git a/harper-cli/src/main.rs b/harper-cli/src/main.rs
index ea8e4cbc1..fdd0ac804 100644
--- a/harper-cli/src/main.rs
+++ b/harper-cli/src/main.rs
@@ -6,6 +6,7 @@ use anyhow::format_err;
use ariadne::{Color, Label, Report, ReportKind, Source};
use clap::Parser;
use harper_comments::CommentParser;
+use harper_core::parsers::Markdown;
use harper_core::{remove_overlaps, Document, FullDictionary, LintGroup, LintGroupConfig, Linter};
#[derive(Debug, Parser)]
@@ -88,8 +89,16 @@ fn main() -> anyhow::Result<()> {
fn load_file(file: &Path) -> anyhow::Result<(Document, String)> {
let source = std::fs::read_to_string(file)?;
- let parser = CommentParser::new_from_filename(file)
- .ok_or(format_err!("Could not detect language ID."))?;
+ let parser: Box<dyn harper_core::parsers::Parser> =
+ if let Some("md") = file.extension().map(|v| v.to_str().unwrap()) {
+ Box::new(Markdown)
+ } else {
+ Box::new(
+ CommentParser::new_from_filename(file)
+ .map(Box::new)
+ .ok_or(format_err!("Could not detect language ID."))?
+ )
+ };
Ok((Document::new(&source, Box::new(parser)), source))
}
diff --git a/harper-core/src/char_ext.rs b/harper-core/src/char_ext.rs
index 1c4110456..224d657cb 100644
--- a/harper-core/src/char_ext.rs
+++ b/harper-core/src/char_ext.rs
@@ -3,20 +3,22 @@ use unicode_width::UnicodeWidthChar;
use crate::Punctuation;
pub trait CharExt {
- /// Whether a character can be a component of a word.
- fn is_lingual(&self) -> bool;
+ fn is_cjk(&self) -> bool;
+ /// Whether a character can be a component of an English word.
+ fn is_english_lingual(&self) -> bool;
fn is_emoji(&self) -> bool;
fn is_punctuation(&self) -> bool;
}
impl CharExt for char {
- fn is_lingual(&self) -> bool {
+ fn is_english_lingual(&self) -> bool {
!self.is_whitespace()
&& !self.is_numeric()
&& !self.is_emoji()
&& matches!(self.width(), Some(1..))
&& !self.is_punctuation()
- && (self.is_alphabetic() || matches!(self, '\u{00C0}'..='\u{00FF}'))
+ && self.is_alphabetic()
+ && !self.is_cjk()
}
fn is_emoji(&self) -> bool {
@@ -35,7 +37,38 @@ impl CharExt for char {
blocks.contains(&block)
}
+ fn is_cjk(&self) -> bool {
+ let Some(block) = unicode_blocks::find_unicode_block(*self) else {
+ return false;
+ };
+
+ let blocks = [
+ unicode_blocks::CJK_UNIFIED_IDEOGRAPHS,
+ unicode_blocks::CJK_UNIFIED_IDEOGRAPHS_EXTENSION_A,
+ unicode_blocks::CJK_UNIFIED_IDEOGRAPHS_EXTENSION_B,
+ unicode_blocks::CJK_UNIFIED_IDEOGRAPHS_EXTENSION_C,
+ unicode_blocks::CJK_UNIFIED_IDEOGRAPHS_EXTENSION_D,
+ unicode_blocks::CJK_UNIFIED_IDEOGRAPHS_EXTENSION_E,
+ unicode_blocks::CJK_UNIFIED_IDEOGRAPHS_EXTENSION_F,
+ unicode_blocks::CJK_UNIFIED_IDEOGRAPHS_EXTENSION_G,
+ unicode_blocks::CJK_UNIFIED_IDEOGRAPHS_EXTENSION_H,
+ unicode_blocks::CJK_UNIFIED_IDEOGRAPHS_EXTENSION_I
+ ];
+
+ blocks.contains(&block)
+ }
+
fn is_punctuation(&self) -> bool {
Punctuation::from_char(*self).is_some()
}
}
+
+#[cfg(test)]
+mod tests {
+ use super::CharExt;
+
+ #[test]
+ fn cjk_is_not_english_lingual() {
+ assert!(!'世'.is_english_lingual())
+ }
+}
diff --git a/harper-core/src/lexing/mod.rs b/harper-core/src/lexing/mod.rs
index 998b31807..a6b6d8ed2 100644
--- a/harper-core/src/lexing/mod.rs
+++ b/harper-core/src/lexing/mod.rs
@@ -43,7 +43,7 @@ pub fn lex_token(source: &[char]) -> Option<FoundToken> {
fn lex_word(source: &[char]) -> Option<FoundToken> {
let end = source
.iter()
- .position(|c| !c.is_lingual())
+ .position(|c| !c.is_english_lingual())
.unwrap_or(source.len());
if end == 0 {
@@ -158,3 +158,14 @@ fn lex_catch(_source: &[char]) -> Option<FoundToken> {
token: TokenKind::Unlintable
})
}
+
+#[cfg(test)]
+mod tests {
+ use super::lex_word;
+
+ #[test]
+ fn lexes_cjk_as_unlintable() {
+ let source: Vec<_> = "世".chars().collect();
+ assert!(lex_word(&source).is_none());
+ }
+}
diff --git a/harper-core/tests/run_tests.rs b/harper-core/tests/run_tests.rs
index 8a4408275..e4509c696 100644
--- a/harper-core/tests/run_tests.rs
+++ b/harper-core/tests/run_tests.rs
@@ -31,3 +31,5 @@ macro_rules! create_test {
create_test!(whack_bullets.md, 1);
create_test!(preexisting.md, 0);
+create_test!(issue_109.md, 0);
+create_test!(chinese_lorem_ipsum.md, 2);
diff --git a/harper-core/tests/test_sources/chinese_lorem_ipsum.md b/harper-core/tests/test_sources/chinese_lorem_ipsum.md
new file mode 100644
index 000000000..75c60f701
--- /dev/null
+++ b/harper-core/tests/test_sources/chinese_lorem_ipsum.md
@@ -0,0 +1,17 @@
+The following text was generated using [a Chinese lorem ipsum generator](https://pinkylam.me/generator/chinese-lorem-ipsum/).
+
+食棵支每躲種。奶象打星爪子二細喜才記行在發像原斤!頁固點子衣點豆看身蝴看苗急午公何足,筆娘經色蝶行元香也要。麻了綠尼固世,色北書目登功;因告黑。
+
+從下由服行巴登魚,棵苦亮珠春用訴星比都由發丟法主犬水現,能國書午錯馬吹隻幾外。筆泉可去牙用同貝步登娘毛完且哭,北巴種化吃石正由讀借示!幸造愛扒,才怕至成下。
+
+亮您紅星浪中兩安乙菜上做玉做火愛玩植結面,蝴哭往兒完哥封波在,林春田早次王同月清丁筆道兩請玩燈們唱。
+
+十兌雞做喜木發屋害要書目把麼早父方從以。冰正共片海告假衣夏冒吃戊很口松牠室肖。斗門看登給畫聽斤婆品歡個。
+
+澡弓兩化員把戊各但他用親!給好校魚火包,根朵告幫着道具象海太或文苗。西首友雞頭、我跳花發,可功才門田訴去首或行。
+
+吧都上午京主根相由或想、澡快送哭麻細麻白言原風童乍外、右更新一婆幸七片院。教刀麻哥幾在同四就遠新來長喜呢雪:兒時貓千十門功玩刀比。媽習出才,借食乞原成欠波米掃古校良具們昔!少學刀工火入。
+
+兔追找古裏雞喜假經南瓜枝未發發几!菜書共媽申欠彩因到「金羊昌追秋才着海隻」雪像麻孝亮乞土點首事向往!知瓜祖。
+
+因主很貫唱重目刃右,封飯房屋造土都犬帶刃抱他重他冬幸以坡十雲?跳肖頭反入小坐歡院帶士自央。
diff --git a/harper-core/tests/test_sources/issue_109.md b/harper-core/tests/test_sources/issue_109.md
new file mode 100644
index 000000000..32340f872
--- /dev/null
+++ b/harper-core/tests/test_sources/issue_109.md
@@ -0,0 +1 @@
+本文档是使用 Microsoft Translate 翻译的。哈珀根本不应该抱怨。
diff --git a/justfile b/justfile
index 13fc8a79a..dcaf1fccd 100644
--- a/justfile
+++ b/justfile
@@ -99,3 +99,5 @@ test:
parse file:
cargo run --bin harper-cli -- parse {{file}}
+lint file:
+ cargo run --bin harper-cli -- lint {{file}}