Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
fix(core): enhanced language detection to work on mixed languages better (Spanglish, for example)
Elijah Potter80aad994fecc
diff --git a/harper-core/src/document.rs b/harper-core/src/document.rsindex 58b9b59db..9ec2b3284 100644--- a/harper-core/src/document.rs+++ b/harper-core/src/document.rs@@ -501,6 +501,7 @@ impl TokenStringExt for Document {create_fns_on_doc!(unlintable);create_fns_on_doc!(sentence_terminator);create_fns_on_doc!(chunk_terminator);+ create_fns_on_doc!(punctuation);fn first_sentence_word(&self) -> Option<Token> {self.tokens.first_sentence_word()diff --git a/harper-core/src/language_detection.rs b/harper-core/src/language_detection.rsindex cd9a5bacb..ac037461e 100644--- a/harper-core/src/language_detection.rs+++ b/harper-core/src/language_detection.rs@@ -1,11 +1,12 @@use crate::{Dictionary, Document, Token, TokenKind};+/// Check if the contents of the document are likely intended to represent+/// English.pub fn is_doc_likely_english(doc: &Document, dict: &impl Dictionary) -> bool {is_likely_english(doc.get_tokens(), doc.get_source(), dict)}-/// Check if the contents of the document are likely intended to represent-/// English.+/// Check if given tokens are likely intended to represent English.pub fn is_likely_english(toks: &[Token], source: &[char], dict: &impl Dictionary) -> bool {let mut total_words = 0;let mut valid_words = 0;@@ -28,6 +29,10 @@ pub fn is_likely_english(toks: &[Token], source: &[char], dict: &impl Dictionary}}+ if total_words <= 7 && total_words - valid_words > 0 {+ return false;+ }+if unlintable > valid_words {return false;}@@ -36,7 +41,7 @@ pub fn is_likely_english(toks: &[Token], source: &[char], dict: &impl Dictionaryreturn false;}- if (valid_words as f64 / total_words as f64) < 0.4 {+ if (valid_words as f64 / total_words as f64) < 0.7 {return false;}@@ -113,4 +118,26 @@ def fibIter(n):"#,);}++ #[test]+ fn mixed_french_english_park() {+ assert_not_english("Je voudrais promener au the park a huit heures with ma voisine");+ }++ #[test]+ fn mixed_french_english_drunk() {+ assert_not_english("Je ne suis pas drunk, je suis only ivre by you");+ }++ #[test]+ fn mixed_french_english_dress() {+ assert_not_english(+ "Je buy une robe nouveau chaque Tuesday, mais aujourd'hui, je don't have temps",+ );+ }++ #[test]+ fn english_motto() {+ assert_english("I have a simple motto in life");+ }}diff --git a/harper-core/src/parsers/isolate_english.rs b/harper-core/src/parsers/isolate_english.rsindex b6091bb17..70fcc69c1 100644--- a/harper-core/src/parsers/isolate_english.rs+++ b/harper-core/src/parsers/isolate_english.rs@@ -24,12 +24,85 @@ impl<D: Dictionary> Parser for IsolateEnglish<D> {let mut english_tokens: Vec<Token> = Vec::with_capacity(tokens.len());- for sentence in tokens.iter_sentences() {- if sentence.len() > 5 && is_likely_english(sentence, source, &self.dict) {- english_tokens.extend(sentence);+ for chunk in tokens.iter_chunks() {+ if chunk.len() < 5 || is_likely_english(chunk, source, &self.dict) {+ english_tokens.extend(chunk);}}english_tokens}}++#[cfg(test)]+mod tests {+ use crate::{parsers::PlainEnglish, Document, FullDictionary, TokenStringExt};++ use super::IsolateEnglish;++ /// Assert that the provided text contains _no_ chunks of valid English+ fn assert_no_english(text: &str) {+ let dict = FullDictionary::curated();++ let document = Document::new(+ text,+ &mut IsolateEnglish::new(Box::new(PlainEnglish), dict.clone()),+ &dict,+ );++ assert_eq!(document.iter_words().count(), 0);+ assert_eq!(document.iter_punctuations().count(), 0);+ }++ /// Assert that, once stripped of non-English chunks, the resulting document looks like another+ /// piece of text.+ fn assert_stripped_english(source: &str, target: &str) {+ let dict = FullDictionary::curated();++ let document = Document::new(+ source,+ &mut IsolateEnglish::new(Box::new(PlainEnglish), dict.clone()),+ &dict,+ );++ assert_eq!(document.to_string(), target);+ }++ #[test]+ fn mixed_spanish_english_breakfast() {+ assert_no_english(+ "En la mañana, como a dish de los huevos, un poquito of tocino, y a lot of leche.",+ );+ }++ #[test]+ fn mixed_spanish_english_politics() {+ assert_no_english(+ "No estoy of acuerdo con the politics de Los estados unidos ahora; pienso que we need mas diversidad in el gobierno.",+ );+ }++ #[test]+ fn english_no_edit_motto() {+ assert_stripped_english(+ "I have a simple motto in life: ",+ "I have a simple motto in life: ",+ );+ }++ #[test]+ fn chunked_trad_chinese_english() {+ assert_stripped_english(+ "I have a simple motto in life: 如果你渴了,就喝水。",+ "I have a simple motto in life:",+ );+ }++ #[test]+ fn chunked_trad_polish_english() {+ assert_stripped_english(+ "I have a simple motto in life: jeśli jesteś spragniony, napij się wody.",+ "I have a simple motto in life:",+ );+ }+}diff --git a/harper-core/src/token.rs b/harper-core/src/token.rsindex da33e7204..d6455f96d 100644--- a/harper-core/src/token.rs+++ b/harper-core/src/token.rs@@ -104,7 +104,10 @@ impl TokenKind {match self {TokenKind::Punctuation(punct) => {- matches!(punct, Punctuation::Comma | Punctuation::Quote { .. })+ matches!(+ punct,+ Punctuation::Comma | Punctuation::Quote { .. } | Punctuation::Colon+ )}_ => false,}@@ -373,6 +376,7 @@ pub trait TokenStringExt {create_decl_for!(unlintable);create_decl_for!(sentence_terminator);create_decl_for!(chunk_terminator);+ create_decl_for!(punctuation);fn iter_linking_verb_indices(&self) -> impl Iterator<Item = usize> + '_;fn iter_linking_verbs(&self) -> impl Iterator<Item = Token> + '_;@@ -400,6 +404,7 @@ impl TokenStringExt for [Token] {create_fns_for!(quote);create_fns_for!(number);create_fns_for!(at);+ create_fns_for!(punctuation);create_fns_for!(ellipsis);create_fns_for!(unlintable);create_fns_for!(sentence_terminator);diff --git a/packages/web/src/routes/languagedetection/+page.svelte b/packages/web/src/routes/languagedetection/+page.svelteindex dc7a7f207..325822430 100644--- a/packages/web/src/routes/languagedetection/+page.svelte+++ b/packages/web/src/routes/languagedetection/+page.svelte@@ -26,6 +26,14 @@name: 'Chinese Lorem Ipsum',value:'食棵支每躲種。奶象打星爪子二細喜才記行在發像原斤!頁固點子衣點豆看身蝴看苗急午公何足,筆娘經色蝶行元香也要。麻了綠尼固世,色北書目登功;因告黑。'+ },+ {+ name: 'Spanish English Mix',+ value: 'En la mañana, como a dish de los huevos, un poquito of tocino, y a lot of leche.'+ },+ {+ name: 'Polish English Chunked',+ value: 'I have a simple motto in life: jeśli jesteś spragniony, napij się wody.'}];