Skip to content

Offline, privacy-first grammar checker. Fast, open-source, Rust-powered

fix(core): enhanced language detection to work on mixed languages better (Spanglish, for example)

Elijah Potter80aad994fecc
diff --git a/harper-core/src/document.rs b/harper-core/src/document.rs
index 58b9b59db..9ec2b3284 100644
--- a/harper-core/src/document.rs
+++ b/harper-core/src/document.rs
@@ -501,6 +501,7 @@ impl TokenStringExt for Document {
create_fns_on_doc!(unlintable);
create_fns_on_doc!(sentence_terminator);
create_fns_on_doc!(chunk_terminator);
+ create_fns_on_doc!(punctuation);
fn first_sentence_word(&self) -> Option<Token> {
self.tokens.first_sentence_word()
diff --git a/harper-core/src/language_detection.rs b/harper-core/src/language_detection.rs
index cd9a5bacb..ac037461e 100644
--- a/harper-core/src/language_detection.rs
+++ b/harper-core/src/language_detection.rs
@@ -1,11 +1,12 @@
use crate::{Dictionary, Document, Token, TokenKind};
+/// Check if the contents of the document are likely intended to represent
+/// English.
pub fn is_doc_likely_english(doc: &Document, dict: &impl Dictionary) -> bool {
is_likely_english(doc.get_tokens(), doc.get_source(), dict)
}
-/// Check if the contents of the document are likely intended to represent
-/// English.
+/// Check if given tokens are likely intended to represent English.
pub fn is_likely_english(toks: &[Token], source: &[char], dict: &impl Dictionary) -> bool {
let mut total_words = 0;
let mut valid_words = 0;
@@ -28,6 +29,10 @@ pub fn is_likely_english(toks: &[Token], source: &[char], dict: &impl Dictionary
}
}
+ if total_words <= 7 && total_words - valid_words > 0 {
+ return false;
+ }
+
if unlintable > valid_words {
return false;
}
@@ -36,7 +41,7 @@ pub fn is_likely_english(toks: &[Token], source: &[char], dict: &impl Dictionary
return false;
}
- if (valid_words as f64 / total_words as f64) < 0.4 {
+ if (valid_words as f64 / total_words as f64) < 0.7 {
return false;
}
@@ -113,4 +118,26 @@ def fibIter(n):
"#,
);
}
+
+ #[test]
+ fn mixed_french_english_park() {
+ assert_not_english("Je voudrais promener au the park a huit heures with ma voisine");
+ }
+
+ #[test]
+ fn mixed_french_english_drunk() {
+ assert_not_english("Je ne suis pas drunk, je suis only ivre by you");
+ }
+
+ #[test]
+ fn mixed_french_english_dress() {
+ assert_not_english(
+ "Je buy une robe nouveau chaque Tuesday, mais aujourd'hui, je don't have temps",
+ );
+ }
+
+ #[test]
+ fn english_motto() {
+ assert_english("I have a simple motto in life");
+ }
}
diff --git a/harper-core/src/parsers/isolate_english.rs b/harper-core/src/parsers/isolate_english.rs
index b6091bb17..70fcc69c1 100644
--- a/harper-core/src/parsers/isolate_english.rs
+++ b/harper-core/src/parsers/isolate_english.rs
@@ -24,12 +24,85 @@ impl<D: Dictionary> Parser for IsolateEnglish<D> {
let mut english_tokens: Vec<Token> = Vec::with_capacity(tokens.len());
- for sentence in tokens.iter_sentences() {
- if sentence.len() > 5 && is_likely_english(sentence, source, &self.dict) {
- english_tokens.extend(sentence);
+ for chunk in tokens.iter_chunks() {
+ if chunk.len() < 5 || is_likely_english(chunk, source, &self.dict) {
+ english_tokens.extend(chunk);
}
}
english_tokens
}
}
+
+#[cfg(test)]
+mod tests {
+ use crate::{parsers::PlainEnglish, Document, FullDictionary, TokenStringExt};
+
+ use super::IsolateEnglish;
+
+ /// Assert that the provided text contains _no_ chunks of valid English
+ fn assert_no_english(text: &str) {
+ let dict = FullDictionary::curated();
+
+ let document = Document::new(
+ text,
+ &mut IsolateEnglish::new(Box::new(PlainEnglish), dict.clone()),
+ &dict,
+ );
+
+ assert_eq!(document.iter_words().count(), 0);
+ assert_eq!(document.iter_punctuations().count(), 0);
+ }
+
+ /// Assert that, once stripped of non-English chunks, the resulting document looks like another
+ /// piece of text.
+ fn assert_stripped_english(source: &str, target: &str) {
+ let dict = FullDictionary::curated();
+
+ let document = Document::new(
+ source,
+ &mut IsolateEnglish::new(Box::new(PlainEnglish), dict.clone()),
+ &dict,
+ );
+
+ assert_eq!(document.to_string(), target);
+ }
+
+ #[test]
+ fn mixed_spanish_english_breakfast() {
+ assert_no_english(
+ "En la mañana, como a dish de los huevos, un poquito of tocino, y a lot of leche.",
+ );
+ }
+
+ #[test]
+ fn mixed_spanish_english_politics() {
+ assert_no_english(
+ "No estoy of acuerdo con the politics de Los estados unidos ahora; pienso que we need mas diversidad in el gobierno.",
+ );
+ }
+
+ #[test]
+ fn english_no_edit_motto() {
+ assert_stripped_english(
+ "I have a simple motto in life: ",
+ "I have a simple motto in life: ",
+ );
+ }
+
+ #[test]
+ fn chunked_trad_chinese_english() {
+ assert_stripped_english(
+ "I have a simple motto in life: 如果你渴了,就喝水。",
+ "I have a simple motto in life:",
+ );
+ }
+
+ #[test]
+ fn chunked_trad_polish_english() {
+ assert_stripped_english(
+ "I have a simple motto in life: jeśli jesteś spragniony, napij się wody.",
+ "I have a simple motto in life:",
+ );
+ }
+}
diff --git a/harper-core/src/token.rs b/harper-core/src/token.rs
index da33e7204..d6455f96d 100644
--- a/harper-core/src/token.rs
+++ b/harper-core/src/token.rs
@@ -104,7 +104,10 @@ impl TokenKind {
match self {
TokenKind::Punctuation(punct) => {
- matches!(punct, Punctuation::Comma | Punctuation::Quote { .. })
+ matches!(
+ punct,
+ Punctuation::Comma | Punctuation::Quote { .. } | Punctuation::Colon
+ )
}
_ => false,
}
@@ -373,6 +376,7 @@ pub trait TokenStringExt {
create_decl_for!(unlintable);
create_decl_for!(sentence_terminator);
create_decl_for!(chunk_terminator);
+ create_decl_for!(punctuation);
fn iter_linking_verb_indices(&self) -> impl Iterator<Item = usize> + '_;
fn iter_linking_verbs(&self) -> impl Iterator<Item = Token> + '_;
@@ -400,6 +404,7 @@ impl TokenStringExt for [Token] {
create_fns_for!(quote);
create_fns_for!(number);
create_fns_for!(at);
+ create_fns_for!(punctuation);
create_fns_for!(ellipsis);
create_fns_for!(unlintable);
create_fns_for!(sentence_terminator);
diff --git a/packages/web/src/routes/languagedetection/+page.svelte b/packages/web/src/routes/languagedetection/+page.svelte
index dc7a7f207..325822430 100644
--- a/packages/web/src/routes/languagedetection/+page.svelte
+++ b/packages/web/src/routes/languagedetection/+page.svelte
@@ -26,6 +26,14 @@
name: 'Chinese Lorem Ipsum',
value:
'食棵支每躲種。奶象打星爪子二細喜才記行在發像原斤!頁固點子衣點豆看身蝴看苗急午公何足,筆娘經色蝶行元香也要。麻了綠尼固世,色北書目登功;因告黑。'
+ },
+ {
+ name: 'Spanish English Mix',
+ value: 'En la mañana, como a dish de los huevos, un poquito of tocino, y a lot of leche.'
+ },
+ {
+ name: 'Polish English Chunked',
+ value: 'I have a simple motto in life: jeśli jesteś spragniony, napij się wody.'
}
];