Skip to content

Official Crawl4AI Cloud: a hosted web scraping, extraction, and search API for AI agents. Free API key, 1,000 credits a month, no card. Fully open source: this repo is the Apache-2.0 crawler that powers it, and you can self-host it.

fix(security): route the cache freshness check through the egress proxy

ntohidi4acca113e2f1
diff --git a/SECURITY-CREDITS.md b/SECURITY-CREDITS.md
index 781e13ce..b8ad3491 100644
--- a/SECURITY-CREDITS.md
+++ b/SECURITY-CREDITS.md
@@ -25,3 +25,4 @@ We thank the following security researchers for their responsible disclosure:
| arpe1618 | GitHub: [arpe1618](https://github.com/arpe1618) | Blind SSRF via the robots.txt fetch in RobotsParser.can_fetch bypassing the Docker egress controls (0.9.4) | 2026-09-04 |
| Ibrahim AlJaafreh - Cystack RedTeam | [LinkedIn](https://www.linkedin.com/in/ibrahim-aljaafreh-glitch/), [cystack.ps](https://cystack.ps) | SSRF with response disclosure via link_preview_config through the URL seeder (0.9.4) | 2026-09-04 |
| Adam Jordan | GitHub: [adamyordan](https://github.com/adamyordan) | Untrusted-config gate bypass via dict-wrapper laundering, leaking server env vars (0.9.4) | 2026-09-08 |
+| arpe1618 | GitHub: [arpe1618](https://github.com/arpe1618) | Blind SSRF via the CacheValidator freshness check bypassing the Docker egress proxy (0.9.5) | 2026-09-29 |
diff --git a/crawl4ai/cache_validator.py b/crawl4ai/cache_validator.py
index b4b170ce..20684277 100644
--- a/crawl4ai/cache_validator.py
+++ b/crawl4ai/cache_validator.py
@@ -18,6 +18,7 @@ from dataclasses import dataclass
from typing import Optional, Tuple
from enum import Enum
+from .egress_policy import proxy_url
from .utils import compute_head_fingerprint
@@ -72,9 +73,13 @@ class CacheValidator:
async def _get_client(self) -> httpx.AsyncClient:
"""Get or create the httpx client."""
if self._client is None:
+ # proxy=None unless an embedder installed one (the Docker server
+ # does). The URL is the caller's crawl URL, so without it this
+ # reached internal hosts directly and followed redirects into them.
self._client = httpx.AsyncClient(
http2=True,
timeout=self.timeout,
+ proxy=proxy_url(),
follow_redirects=True,
headers={"User-Agent": self.user_agent}
)
diff --git a/deploy/docker/tests/test_security_ssrf_seeder.py b/deploy/docker/tests/test_security_ssrf_seeder.py
index b0a83eff..86144b3c 100644
--- a/deploy/docker/tests/test_security_ssrf_seeder.py
+++ b/deploy/docker/tests/test_security_ssrf_seeder.py
@@ -23,7 +23,8 @@ from egress_proxy import PinningProxy
from crawl4ai import egress_policy
from crawl4ai.async_url_seeder import AsyncUrlSeeder
-from crawl4ai.utils import RobotsParser
+from crawl4ai.cache_validator import CacheValidationResult, CacheValidator
+from crawl4ai.utils import RobotsParser, compute_head_fingerprint
pytestmark = pytest.mark.posture
@@ -231,3 +232,79 @@ class TestRobotsEgress:
await proxy.stop()
await prunner.cleanup()
await irunner.cleanup()
+
+
+class TestCacheValidatorEgress:
+ """check_cache_freshness is untrusted-allowlisted, so validate() fetches the
+ caller's crawl URL: HEAD with the stored validators, then a streamed GET."""
+
+ async def test_validator_direct_loopback_is_blocked(self):
+ hits = []
+
+ async def internal(req):
+ hits.append(req.path)
+ return web.Response(text=INTERNAL_PAGE, content_type="text/html")
+
+ runner, port = await _serve(internal)
+ proxy = await _start_proxy()
+ try:
+ async with CacheValidator(timeout=5) as v:
+ await v.validate(
+ f"http://127.0.0.1:{port}/", stored_etag='"x"',
+ stored_head_fingerprint="abc",
+ )
+ assert hits == [], "the internal service was reached"
+ finally:
+ await proxy.stop()
+ await runner.cleanup()
+
+ async def test_validator_redirect_into_internal_is_blocked(self, monkeypatch):
+ hits = []
+
+ async def internal(req):
+ hits.append(req.path)
+ return web.Response(text=INTERNAL_PAGE, content_type="text/html")
+
+ irunner, iport = await _serve(internal)
+
+ async def public(req):
+ raise web.HTTPFound(f"http://127.0.0.1:{iport}/secret")
+
+ prunner, pport = await _serve(public)
+ _treat_port_as_public(monkeypatch, pport)
+
+ proxy = await _start_proxy()
+ try:
+ async with CacheValidator(timeout=5) as v:
+ await v.validate(
+ f"http://127.0.0.1:{pport}/page", stored_etag='"x"',
+ stored_head_fingerprint="abc",
+ )
+ assert hits == [], "the redirect reached the internal service"
+ finally:
+ await proxy.stop()
+ await prunner.cleanup()
+ await irunner.cleanup()
+
+ async def test_validator_public_host_still_works(self, monkeypatch):
+ """The fix must not simply block everything."""
+ page = '<html><head><title>PUBLIC-OK</title></head><body>x</body></html>'
+
+ async def public(req):
+ return web.Response(text=page, content_type="text/html")
+
+ runner, port = await _serve(public)
+ _treat_port_as_public(monkeypatch, port)
+
+ proxy = await _start_proxy()
+ try:
+ async with CacheValidator(timeout=5) as v:
+ r = await v.validate(
+ f"http://127.0.0.1:{port}/page",
+ stored_head_fingerprint=compute_head_fingerprint(page),
+ )
+ assert r.status == CacheValidationResult.FRESH
+ finally:
+ await proxy.stop()
+ await runner.cleanup()
diff --git a/tests/unit/test_egress_policy.py b/tests/unit/test_egress_policy.py
index 91202fe4..b2ae9303 100644
--- a/tests/unit/test_egress_policy.py
+++ b/tests/unit/test_egress_policy.py
@@ -16,6 +16,7 @@ import pytest
from crawl4ai import egress_policy
from crawl4ai.async_url_seeder import AsyncUrlSeeder
+from crawl4ai.cache_validator import CacheValidator
from crawl4ai.utils import RobotsParser
PROXY = "http://127.0.0.1:9" # discard port; nothing here connects
@@ -139,6 +140,35 @@ def test_robots_fetch_no_longer_disables_tls_verification(monkeypatch, tmp_path)
assert calls[0][1].get("ssl", None) is None
+# ── cache freshness check ──────────────────────────────────────────────
+
+
+def _validator_client():
+ v = CacheValidator()
+ return v, asyncio.run(v._get_client())
+
+
+def test_cache_validator_client_has_no_proxy_by_default():
+ v, client = _validator_client()
+ try:
+ assert _proxy_transport(client) is None
+ finally:
+ asyncio.run(v.close())
+
+
+def test_cache_validator_client_uses_installed_proxy():
+ """check_cache_freshness is untrusted-allowlisted, so this fetch is
+ caller-controlled and must go through the egress proxy like the others."""
+ egress_policy.set_egress_proxy(PROXY)
+ v, client = _validator_client()
+ try:
+ transport = _proxy_transport(client)
+ assert transport is not None
+ assert transport._pool._http2 is True
+ finally:
+ asyncio.run(v.close())
+
+
# ── untrusted LinkPreviewConfig clamps ─────────────────────────────────