Skip to content

Offline, privacy-first grammar checker. Fast, open-source, Rust-powered

fix(core): ignore top-level domains (#2750)

Elijah Potter6932518290e6
diff --git a/harper-core/src/document.rs b/harper-core/src/document.rs
index c3986efec..bf3073586 100644
--- a/harper-core/src/document.rs
+++ b/harper-core/src/document.rs
@@ -147,6 +147,7 @@ impl Document {
self.condense_number_suffixes();
self.condense_ellipsis();
self.condense_latin();
+ self.condense_common_top_level_domains();
self.condense_filename_extensions();
self.condense_tldr();
self.condense_ampersand_pairs();
@@ -691,6 +692,51 @@ impl Document {
self.tokens.remove_indices(to_remove);
}
+ /// Condenses common top-level domains (for example: `.blog`, `.com`) down to single tokens.
+ fn condense_common_top_level_domains(&mut self) {
+ const COMMON_TOP_LEVEL_DOMAINS: &[&str] = &[
+ "ai", "app", "blog", "co", "com", "dev", "edu", "gov", "info", "io", "me", "mil",
+ "net", "org", "shop", "tech", "uk", "us", "xyz",
+ ];
+
+ if self.tokens.len() < 2 {
+ return;
+ }
+
+ let mut to_remove = VecDeque::new();
+ for cursor in 1..self.tokens.len() {
+ // left context, dot, tld, right context
+ let l = self.get_token_offset(cursor, -2);
+ let d = &self.tokens[cursor - 1];
+ let tld = &self.tokens[cursor];
+ let r = self.get_token_offset(cursor, 1);
+
+ let is_tld_chunk = d.kind.is_period()
+ && tld.kind.is_word()
+ && tld
+ .span
+ .get_content(&self.source)
+ .iter()
+ .all(|c| c.is_ascii_alphabetic())
+ && tld
+ .span
+ .get_content(&self.source)
+ .eq_any_ignore_ascii_case_str(COMMON_TOP_LEVEL_DOMAINS)
+ && ((l.is_none_or(|t| t.kind.is_whitespace())
+ && r.is_none_or(|t| t.kind.is_whitespace()))
+ || (l.is_some_and(|t| t.kind.is_open_round())
+ && r.is_some_and(|t| t.kind.is_close_round())));
+
+ if is_tld_chunk {
+ self.tokens[cursor - 1].kind = TokenKind::Unlintable;
+ self.tokens[cursor - 1].span.end = self.tokens[cursor].span.end;
+ to_remove.push_back(cursor);
+ }
+ }
+
+ self.tokens.remove_indices(to_remove);
+ }
+
/// Condenses "tl;dr" down to a single word token.
fn condense_tldr(&mut self) {
if self.tokens.len() < 3 {
@@ -1252,6 +1298,32 @@ mod tests {
assert!(tldrs[0].span.get_content_string(&doc.source) == "TL;DRs");
}
+ #[test]
+ fn condense_common_top_level_domains() {
+ let doc = Document::new_plain_english_curated(".blog and .com and .NET");
+ assert!(doc.tokens.len() == 9);
+ assert!(doc.tokens[0].kind.is_unlintable());
+ assert!(doc.tokens[4].kind.is_unlintable());
+ assert!(doc.tokens[8].kind.is_unlintable());
+ }
+
+ #[test]
+ fn condense_common_top_level_domains_in_parens() {
+ let doc = Document::new_plain_english_curated("(.blog)");
+ assert!(doc.tokens.len() == 3);
+ assert!(doc.tokens[0].kind.is_open_round());
+ assert!(doc.tokens[1].kind.is_unlintable());
+ assert!(doc.tokens[2].kind.is_close_round());
+ }
+
+ #[test]
+ fn doesnt_condense_unknown_top_level_domains() {
+ let doc = Document::new_plain_english_curated(".harper");
+ assert!(doc.tokens.len() == 2);
+ assert!(doc.tokens[0].kind.is_punctuation());
+ assert!(doc.tokens[1].kind.is_word());
+ }
+
#[test]
fn condense_r_and_d_caps() {
let doc = Document::new_plain_english_curated("R&D");
diff --git a/harper-core/tests/run_tests.rs b/harper-core/tests/run_tests.rs
index 38f7d0a3b..46569c2a7 100644
--- a/harper-core/tests/run_tests.rs
+++ b/harper-core/tests/run_tests.rs
@@ -97,6 +97,7 @@ create_test!(title_case_errors.md, 2, Dialect::American);
create_test!(title_case_clean.md, 0, Dialect::American);
create_test!(issue_2233.md, 0, Dialect::American);
create_test!(issue_2240.md, 0, Dialect::American);
+create_test!(allows_domain_extensions.md, 0, Dialect::American);
// It just matters that it is > 1
create_test!(issue_2151.md, 4, Dialect::British);
diff --git a/harper-core/tests/test_sources/allows_domain_extensions.md b/harper-core/tests/test_sources/allows_domain_extensions.md
new file mode 100644
index 000000000..ed1ce0964
--- /dev/null
+++ b/harper-core/tests/test_sources/allows_domain_extensions.md
@@ -0,0 +1,19 @@
+I would love a .ai domain.
+I would love a .app domain.
+I would love a .blog domain.
+I would love a .co domain.
+I would love a .com domain.
+I would love a .dev domain.
+I would love a .edu domain.
+I would love a .gov domain.
+I would love a .info domain.
+I would love a .io domain.
+I would love a .me domain.
+I would love a .mil domain.
+I would love a .net domain.
+I would love a .org domain.
+I would love a .shop domain.
+I would love a .tech domain.
+I would love a .uk domain.
+I would love a .us domain.
+I would love a .xyz domain.