Official Crawl4AI Cloud: a hosted web scraping, extraction, and search API for AI agents. Free API key, 1,000 credits a month, no card. Fully open source: this repo is the Apache-2.0 crawler that powers it, and you can self-host it.
Python aiai-agentscrawl4ai-cloudcrawlerdata-extractionllmmarkdownmcpopen-sourceplaywrightpythonragscraperweb-crawlerweb-crawlingweb-scraping
fix(security): route the cache freshness check through the egress proxy
ntohidi4acca113e2f1
diff --git a/SECURITY-CREDITS.md b/SECURITY-CREDITS.mdindex 781e13ce..b8ad3491 100644--- a/SECURITY-CREDITS.md+++ b/SECURITY-CREDITS.md@@ -25,3 +25,4 @@ We thank the following security researchers for their responsible disclosure:| arpe1618 | GitHub: [arpe1618](https://github.com/arpe1618) | Blind SSRF via the robots.txt fetch in RobotsParser.can_fetch bypassing the Docker egress controls (0.9.4) | 2026-09-04 || Ibrahim AlJaafreh - Cystack RedTeam | [LinkedIn](https://www.linkedin.com/in/ibrahim-aljaafreh-glitch/), [cystack.ps](https://cystack.ps) | SSRF with response disclosure via link_preview_config through the URL seeder (0.9.4) | 2026-09-04 || Adam Jordan | GitHub: [adamyordan](https://github.com/adamyordan) | Untrusted-config gate bypass via dict-wrapper laundering, leaking server env vars (0.9.4) | 2026-09-08 |+| arpe1618 | GitHub: [arpe1618](https://github.com/arpe1618) | Blind SSRF via the CacheValidator freshness check bypassing the Docker egress proxy (0.9.5) | 2026-09-29 |diff --git a/crawl4ai/cache_validator.py b/crawl4ai/cache_validator.pyindex b4b170ce..20684277 100644--- a/crawl4ai/cache_validator.py+++ b/crawl4ai/cache_validator.py@@ -18,6 +18,7 @@ from dataclasses import dataclassfrom typing import Optional, Tuplefrom enum import Enum+from .egress_policy import proxy_urlfrom .utils import compute_head_fingerprint@@ -72,9 +73,13 @@ class CacheValidator:async def _get_client(self) -> httpx.AsyncClient:"""Get or create the httpx client."""if self._client is None:+ # proxy=None unless an embedder installed one (the Docker server+ # does). The URL is the caller's crawl URL, so without it this+ # reached internal hosts directly and followed redirects into them.self._client = httpx.AsyncClient(http2=True,timeout=self.timeout,+ proxy=proxy_url(),follow_redirects=True,headers={"User-Agent": self.user_agent})diff --git a/deploy/docker/tests/test_security_ssrf_seeder.py b/deploy/docker/tests/test_security_ssrf_seeder.pyindex b0a83eff..86144b3c 100644--- a/deploy/docker/tests/test_security_ssrf_seeder.py+++ b/deploy/docker/tests/test_security_ssrf_seeder.py@@ -23,7 +23,8 @@ from egress_proxy import PinningProxyfrom crawl4ai import egress_policyfrom crawl4ai.async_url_seeder import AsyncUrlSeeder-from crawl4ai.utils import RobotsParser+from crawl4ai.cache_validator import CacheValidationResult, CacheValidator+from crawl4ai.utils import RobotsParser, compute_head_fingerprintpytestmark = pytest.mark.posture@@ -231,3 +232,79 @@ class TestRobotsEgress:await proxy.stop()await prunner.cleanup()await irunner.cleanup()+++class TestCacheValidatorEgress:+ """check_cache_freshness is untrusted-allowlisted, so validate() fetches the+ caller's crawl URL: HEAD with the stored validators, then a streamed GET."""++ async def test_validator_direct_loopback_is_blocked(self):+ hits = []++ async def internal(req):+ hits.append(req.path)+ return web.Response(text=INTERNAL_PAGE, content_type="text/html")++ runner, port = await _serve(internal)+ proxy = await _start_proxy()+ try:+ async with CacheValidator(timeout=5) as v:+ await v.validate(+ f"http://127.0.0.1:{port}/", stored_etag='"x"',+ stored_head_fingerprint="abc",+ )+ assert hits == [], "the internal service was reached"+ finally:+ await proxy.stop()+ await runner.cleanup()++ async def test_validator_redirect_into_internal_is_blocked(self, monkeypatch):+ hits = []++ async def internal(req):+ hits.append(req.path)+ return web.Response(text=INTERNAL_PAGE, content_type="text/html")++ irunner, iport = await _serve(internal)++ async def public(req):+ raise web.HTTPFound(f"http://127.0.0.1:{iport}/secret")++ prunner, pport = await _serve(public)+ _treat_port_as_public(monkeypatch, pport)++ proxy = await _start_proxy()+ try:+ async with CacheValidator(timeout=5) as v:+ await v.validate(+ f"http://127.0.0.1:{pport}/page", stored_etag='"x"',+ stored_head_fingerprint="abc",+ )+ assert hits == [], "the redirect reached the internal service"+ finally:+ await proxy.stop()+ await prunner.cleanup()+ await irunner.cleanup()++ async def test_validator_public_host_still_works(self, monkeypatch):+ """The fix must not simply block everything."""+ page = '<html><head><title>PUBLIC-OK</title></head><body>x</body></html>'++ async def public(req):+ return web.Response(text=page, content_type="text/html")++ runner, port = await _serve(public)+ _treat_port_as_public(monkeypatch, port)++ proxy = await _start_proxy()+ try:+ async with CacheValidator(timeout=5) as v:+ r = await v.validate(+ f"http://127.0.0.1:{port}/page",+ stored_head_fingerprint=compute_head_fingerprint(page),+ )+ assert r.status == CacheValidationResult.FRESH+ finally:+ await proxy.stop()+ await runner.cleanup()diff --git a/tests/unit/test_egress_policy.py b/tests/unit/test_egress_policy.pyindex 91202fe4..b2ae9303 100644--- a/tests/unit/test_egress_policy.py+++ b/tests/unit/test_egress_policy.py@@ -16,6 +16,7 @@ import pytestfrom crawl4ai import egress_policyfrom crawl4ai.async_url_seeder import AsyncUrlSeeder+from crawl4ai.cache_validator import CacheValidatorfrom crawl4ai.utils import RobotsParserPROXY = "http://127.0.0.1:9" # discard port; nothing here connects@@ -139,6 +140,35 @@ def test_robots_fetch_no_longer_disables_tls_verification(monkeypatch, tmp_path)assert calls[0][1].get("ssl", None) is None+# ── cache freshness check ──────────────────────────────────────────────+++def _validator_client():+ v = CacheValidator()+ return v, asyncio.run(v._get_client())+++def test_cache_validator_client_has_no_proxy_by_default():+ v, client = _validator_client()+ try:+ assert _proxy_transport(client) is None+ finally:+ asyncio.run(v.close())+++def test_cache_validator_client_uses_installed_proxy():+ """check_cache_freshness is untrusted-allowlisted, so this fetch is+ caller-controlled and must go through the egress proxy like the others."""+ egress_policy.set_egress_proxy(PROXY)+ v, client = _validator_client()+ try:+ transport = _proxy_transport(client)+ assert transport is not None+ assert transport._pool._http2 is True+ finally:+ asyncio.run(v.close())++# ── untrusted LinkPreviewConfig clamps ─────────────────────────────────