Official Crawl4AI Cloud: a hosted web scraping, extraction, and search API for AI agents. Free API key, 1,000 credits a month, no card. Fully open source: this repo is the Apache-2.0 crawler that powers it, and you can self-host it.
Python aiai-agentscrawl4ai-cloudcrawlerdata-extractionllmmarkdownmcpopen-sourceplaywrightpythonragscraperweb-crawlerweb-crawlingweb-scraping
fix(security): parse the real host in DomainFilter
ntohidi6557e967928f
diff --git a/crawl4ai/deep_crawling/filters.py b/crawl4ai/deep_crawling/filters.pyindex 2fb819ec..3562920a 100644--- a/crawl4ai/deep_crawling/filters.py+++ b/crawl4ai/deep_crawling/filters.py@@ -427,9 +427,6 @@ class DomainFilter(URLFilter):__slots__ = ("_allowed_domains", "_blocked_domains", "_domain_cache")- # Regex for fast domain extraction- _DOMAIN_REGEX = re.compile(r"://([^/]+)")-def __init__(self,allowed_domains: Union[str, List[str]] = None,@@ -450,12 +447,17 @@ class DomainFilter(URLFilter):)@staticmethod- def _normalize_domains(domains: Union[str, List[str]]) -> Set[str]:+ def _normalize_domain(domain: str) -> str:+ """Lowercase a domain and strip a trailing dot"""+ return domain.lower().rstrip(".")++ @classmethod+ def _normalize_domains(cls, domains: Union[str, List[str]]) -> Set[str]:"""Fast domain normalization"""if isinstance(domains, str):- return {domains.lower()}- return {d.lower() for d in domains}-+ return {cls._normalize_domain(domains)}+ return {cls._normalize_domain(d) for d in domains}+@staticmethoddef _is_subdomain(domain: str, parent_domain: str) -> bool:"""Check if domain is a subdomain of parent_domain"""@@ -464,9 +466,12 @@ class DomainFilter(URLFilter):@staticmethod@lru_cache(maxsize=10000)def _extract_domain(url: str) -> str:- """Ultra-fast domain extraction with regex and caching"""- match = DomainFilter._DOMAIN_REGEX.search(url)- return match.group(1).lower() if match else ""+ """Extract the host from a URL, dropping userinfo/port and trailing dot"""+ try:+ host = urlparse(url).hostname+ except ValueError:+ return ""+ return host.rstrip(".") if host else ""def apply(self, url: str) -> bool:"""Optimized domain checking with early returns"""diff --git a/tests/unit/test_domain_filter_host.py b/tests/unit/test_domain_filter_host.pynew file mode 100644index 00000000..e7d399ca--- /dev/null+++ b/tests/unit/test_domain_filter_host.py@@ -0,0 +1,60 @@+"""Tests for DomainFilter host extraction (userinfo/port/trailing-dot bypass)."""++from crawl4ai.deep_crawling.filters import DomainFilter+++def test_blocked_userinfo_bypass():+ f = DomainFilter(blocked_domains=["evil.com"])+ assert f.apply("https://[email protected]/") is False+++def test_blocked_trailing_dot_bypass():+ f = DomainFilter(blocked_domains=["evil.com"])+ assert f.apply("https://evil.com./") is False+++def test_blocked_port_bypass():+ f = DomainFilter(blocked_domains=["evil.com"])+ assert f.apply("https://evil.com:443/") is False+++def test_blocked_mixed_case():+ f = DomainFilter(blocked_domains=["evil.com"])+ assert f.apply("https://EVIL.COM/") is False+++def test_blocked_subdomain():+ f = DomainFilter(blocked_domains=["evil.com"])+ assert f.apply("https://a.evil.com/") is False+++def test_not_blocked_lookalike_domain():+ f = DomainFilter(blocked_domains=["evil.com"])+ assert f.apply("https://notevil.com/") is True+++def test_blocked_domain_configured_with_trailing_dot():+ f = DomainFilter(blocked_domains=["evil.com."])+ assert f.apply("https://evil.com/") is False+++def test_blocked_domain_configured_uppercase():+ f = DomainFilter(blocked_domains=["EVIL.COM"])+ assert f.apply("https://evil.com/") is False+++def test_allowed_with_port():+ f = DomainFilter(allowed_domains=["good.com"])+ assert f.apply("https://good.com:443/") is True+++def test_allowed_with_trailing_dot():+ f = DomainFilter(allowed_domains=["good.com"])+ assert f.apply("https://good.com./") is True+++def test_allowed_userinfo_spoof_not_allowed():+ # Userinfo "[email protected]" must not fool the allowlist into+ # treating the host as good.com; the real host is evil.com.+ f = DomainFilter(allowed_domains=["good.com"])+ assert f.apply("https://[email protected]/") is False