Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
fix(core): ignore top-level domains (#2750)
Elijah Potter6932518290e6
diff --git a/harper-core/src/document.rs b/harper-core/src/document.rsindex c3986efec..bf3073586 100644--- a/harper-core/src/document.rs+++ b/harper-core/src/document.rs@@ -147,6 +147,7 @@ impl Document {self.condense_number_suffixes();self.condense_ellipsis();self.condense_latin();+ self.condense_common_top_level_domains();self.condense_filename_extensions();self.condense_tldr();self.condense_ampersand_pairs();@@ -691,6 +692,51 @@ impl Document {self.tokens.remove_indices(to_remove);}+ /// Condenses common top-level domains (for example: `.blog`, `.com`) down to single tokens.+ fn condense_common_top_level_domains(&mut self) {+ const COMMON_TOP_LEVEL_DOMAINS: &[&str] = &[+ "ai", "app", "blog", "co", "com", "dev", "edu", "gov", "info", "io", "me", "mil",+ "net", "org", "shop", "tech", "uk", "us", "xyz",+ ];++ if self.tokens.len() < 2 {+ return;+ }++ let mut to_remove = VecDeque::new();+ for cursor in 1..self.tokens.len() {+ // left context, dot, tld, right context+ let l = self.get_token_offset(cursor, -2);+ let d = &self.tokens[cursor - 1];+ let tld = &self.tokens[cursor];+ let r = self.get_token_offset(cursor, 1);++ let is_tld_chunk = d.kind.is_period()+ && tld.kind.is_word()+ && tld+ .span+ .get_content(&self.source)+ .iter()+ .all(|c| c.is_ascii_alphabetic())+ && tld+ .span+ .get_content(&self.source)+ .eq_any_ignore_ascii_case_str(COMMON_TOP_LEVEL_DOMAINS)+ && ((l.is_none_or(|t| t.kind.is_whitespace())+ && r.is_none_or(|t| t.kind.is_whitespace()))+ || (l.is_some_and(|t| t.kind.is_open_round())+ && r.is_some_and(|t| t.kind.is_close_round())));++ if is_tld_chunk {+ self.tokens[cursor - 1].kind = TokenKind::Unlintable;+ self.tokens[cursor - 1].span.end = self.tokens[cursor].span.end;+ to_remove.push_back(cursor);+ }+ }++ self.tokens.remove_indices(to_remove);+ }+/// Condenses "tl;dr" down to a single word token.fn condense_tldr(&mut self) {if self.tokens.len() < 3 {@@ -1252,6 +1298,32 @@ mod tests {assert!(tldrs[0].span.get_content_string(&doc.source) == "TL;DRs");}+ #[test]+ fn condense_common_top_level_domains() {+ let doc = Document::new_plain_english_curated(".blog and .com and .NET");+ assert!(doc.tokens.len() == 9);+ assert!(doc.tokens[0].kind.is_unlintable());+ assert!(doc.tokens[4].kind.is_unlintable());+ assert!(doc.tokens[8].kind.is_unlintable());+ }++ #[test]+ fn condense_common_top_level_domains_in_parens() {+ let doc = Document::new_plain_english_curated("(.blog)");+ assert!(doc.tokens.len() == 3);+ assert!(doc.tokens[0].kind.is_open_round());+ assert!(doc.tokens[1].kind.is_unlintable());+ assert!(doc.tokens[2].kind.is_close_round());+ }++ #[test]+ fn doesnt_condense_unknown_top_level_domains() {+ let doc = Document::new_plain_english_curated(".harper");+ assert!(doc.tokens.len() == 2);+ assert!(doc.tokens[0].kind.is_punctuation());+ assert!(doc.tokens[1].kind.is_word());+ }+#[test]fn condense_r_and_d_caps() {let doc = Document::new_plain_english_curated("R&D");diff --git a/harper-core/tests/run_tests.rs b/harper-core/tests/run_tests.rsindex 38f7d0a3b..46569c2a7 100644--- a/harper-core/tests/run_tests.rs+++ b/harper-core/tests/run_tests.rs@@ -97,6 +97,7 @@ create_test!(title_case_errors.md, 2, Dialect::American);create_test!(title_case_clean.md, 0, Dialect::American);create_test!(issue_2233.md, 0, Dialect::American);create_test!(issue_2240.md, 0, Dialect::American);+create_test!(allows_domain_extensions.md, 0, Dialect::American);// It just matters that it is > 1create_test!(issue_2151.md, 4, Dialect::British);diff --git a/harper-core/tests/test_sources/allows_domain_extensions.md b/harper-core/tests/test_sources/allows_domain_extensions.mdnew file mode 100644index 000000000..ed1ce0964--- /dev/null+++ b/harper-core/tests/test_sources/allows_domain_extensions.md@@ -0,0 +1,19 @@+I would love a .ai domain.+I would love a .app domain.+I would love a .blog domain.+I would love a .co domain.+I would love a .com domain.+I would love a .dev domain.+I would love a .edu domain.+I would love a .gov domain.+I would love a .info domain.+I would love a .io domain.+I would love a .me domain.+I would love a .mil domain.+I would love a .net domain.+I would love a .org domain.+I would love a .shop domain.+I would love a .tech domain.+I would love a .uk domain.+I would love a .us domain.+I would love a .xyz domain.