Skip to content

Official Crawl4AI Cloud: a hosted web scraping, extraction, and search API for AI agents. Free API key, 1,000 credits a month, no card. Fully open source: this repo is the Apache-2.0 crawler that powers it, and you can self-host it.

fix(security): parse the real host in DomainFilter

ntohidi6557e967928f
diff --git a/crawl4ai/deep_crawling/filters.py b/crawl4ai/deep_crawling/filters.py
index 2fb819ec..3562920a 100644
--- a/crawl4ai/deep_crawling/filters.py
+++ b/crawl4ai/deep_crawling/filters.py
@@ -427,9 +427,6 @@ class DomainFilter(URLFilter):
__slots__ = ("_allowed_domains", "_blocked_domains", "_domain_cache")
- # Regex for fast domain extraction
- _DOMAIN_REGEX = re.compile(r"://([^/]+)")
-
def __init__(
self,
allowed_domains: Union[str, List[str]] = None,
@@ -450,12 +447,17 @@ class DomainFilter(URLFilter):
)
@staticmethod
- def _normalize_domains(domains: Union[str, List[str]]) -> Set[str]:
+ def _normalize_domain(domain: str) -> str:
+ """Lowercase a domain and strip a trailing dot"""
+ return domain.lower().rstrip(".")
+
+ @classmethod
+ def _normalize_domains(cls, domains: Union[str, List[str]]) -> Set[str]:
"""Fast domain normalization"""
if isinstance(domains, str):
- return {domains.lower()}
- return {d.lower() for d in domains}
-
+ return {cls._normalize_domain(domains)}
+ return {cls._normalize_domain(d) for d in domains}
+
@staticmethod
def _is_subdomain(domain: str, parent_domain: str) -> bool:
"""Check if domain is a subdomain of parent_domain"""
@@ -464,9 +466,12 @@ class DomainFilter(URLFilter):
@staticmethod
@lru_cache(maxsize=10000)
def _extract_domain(url: str) -> str:
- """Ultra-fast domain extraction with regex and caching"""
- match = DomainFilter._DOMAIN_REGEX.search(url)
- return match.group(1).lower() if match else ""
+ """Extract the host from a URL, dropping userinfo/port and trailing dot"""
+ try:
+ host = urlparse(url).hostname
+ except ValueError:
+ return ""
+ return host.rstrip(".") if host else ""
def apply(self, url: str) -> bool:
"""Optimized domain checking with early returns"""
diff --git a/tests/unit/test_domain_filter_host.py b/tests/unit/test_domain_filter_host.py
new file mode 100644
index 00000000..e7d399ca
--- /dev/null
+++ b/tests/unit/test_domain_filter_host.py
@@ -0,0 +1,60 @@
+"""Tests for DomainFilter host extraction (userinfo/port/trailing-dot bypass)."""
+
+from crawl4ai.deep_crawling.filters import DomainFilter
+
+
+def test_blocked_userinfo_bypass():
+ f = DomainFilter(blocked_domains=["evil.com"])
+ assert f.apply("https://[email protected]/") is False
+
+
+def test_blocked_trailing_dot_bypass():
+ f = DomainFilter(blocked_domains=["evil.com"])
+ assert f.apply("https://evil.com./") is False
+
+
+def test_blocked_port_bypass():
+ f = DomainFilter(blocked_domains=["evil.com"])
+ assert f.apply("https://evil.com:443/") is False
+
+
+def test_blocked_mixed_case():
+ f = DomainFilter(blocked_domains=["evil.com"])
+ assert f.apply("https://EVIL.COM/") is False
+
+
+def test_blocked_subdomain():
+ f = DomainFilter(blocked_domains=["evil.com"])
+ assert f.apply("https://a.evil.com/") is False
+
+
+def test_not_blocked_lookalike_domain():
+ f = DomainFilter(blocked_domains=["evil.com"])
+ assert f.apply("https://notevil.com/") is True
+
+
+def test_blocked_domain_configured_with_trailing_dot():
+ f = DomainFilter(blocked_domains=["evil.com."])
+ assert f.apply("https://evil.com/") is False
+
+
+def test_blocked_domain_configured_uppercase():
+ f = DomainFilter(blocked_domains=["EVIL.COM"])
+ assert f.apply("https://evil.com/") is False
+
+
+def test_allowed_with_port():
+ f = DomainFilter(allowed_domains=["good.com"])
+ assert f.apply("https://good.com:443/") is True
+
+
+def test_allowed_with_trailing_dot():
+ f = DomainFilter(allowed_domains=["good.com"])
+ assert f.apply("https://good.com./") is True
+
+
+def test_allowed_userinfo_spoof_not_allowed():
+ # Userinfo "[email protected]" must not fool the allowlist into
+ # treating the host as good.com; the real host is evil.com.
+ f = DomainFilter(allowed_domains=["good.com"])
+ assert f.apply("https://[email protected]/") is False