An open source, non-profit web search engine
Filter out more spam domains
Daoud Clarke78a9bfbb11d4
diff --git a/mwmbl/indexer/blacklist.py b/mwmbl/indexer/blacklist.pyindex cb480d6..e6e5adb 100644--- a/mwmbl/indexer/blacklist.py+++ b/mwmbl/indexer/blacklist.py@@ -22,11 +22,12 @@ def is_domain_blacklisted(domain: str, blacklist_domains: set[str]):# TODO: this is to filter out spammy domains that look like:# brofqpxj.uelinc.com- # gwaspsag.enflightmultisport.com- # fmcqgzvk.onlinejobs2day.com- # btmjmhyj.universityslandown.com- # djqfctsq.ropman.com+ # gzsmjc.fba01.com+ # 59648.etnomurcia.com+ ## Eventually we can figure out a better way to identify SEO spamdomain_parts = domain.split('.')- if len(domain_parts) == 3 and domain_parts[2] == "com" and len(domain_parts[0]) == 8:+ if (len(domain_parts) == 3 and domain_parts[2] == "com" and len(domain_parts[0]) in {6, 8}) or (+ set(domain_parts[0]) <= set("1234567890")+ ):return Truediff --git a/test/test_blacklist.py b/test/test_blacklist.pyindex b9f376a..b68c1ce 100644--- a/test/test_blacklist.py+++ b/test/test_blacklist.py@@ -4,10 +4,8 @@ from mwmbl.indexer.blacklist import is_domain_blacklisteddef test_blacklist_excludes_bad_pattern():bad_domains = ["brofqpxj.uelinc.com",- "gwaspsag.enflightmultisport.com",- "fmcqgzvk.onlinejobs2day.com",- "btmjmhyj.universityslandown.com",- "djqfctsq.ropman.com",+ "gzsmjc.fba01.com",+ "59648.etnomurcia.com",]for domain in bad_domains: