Official Crawl4AI Cloud: a hosted web scraping, extraction, and search API for AI agents. Free API key, 1,000 credits a month, no card. Fully open source: this repo is the Apache-2.0 crawler that powers it, and you can self-host it.
Python aiai-agentscrawl4ai-cloudcrawlerdata-extractionllmmarkdownmcpopen-sourceplaywrightpythonragscraperweb-crawlerweb-crawlingweb-scraping
fix(docker): chain egress proxy through upstream HTTP(S)_PROXY
Soham Kukreti890af3d4e6f8
diff --git a/deploy/docker/README.md b/deploy/docker/README.mdindex 7ce0dcd0..a57c857b 100644--- a/deploy/docker/README.md+++ b/deploy/docker/README.md@@ -116,6 +116,17 @@ EOL> The server will be available at `http://localhost:11235`. Visit `/playground` to access the interactive testing interface.+* **Behind a corporate proxy:** if the host reaches the internet only through+ an HTTP proxy, set the standard `HTTP_PROXY` / `HTTPS_PROXY` / `NO_PROXY`+ env vars (Docker's `proxies` config injects them automatically) — the+ server's egress proxy chains through it while keeping its SSRF protections+ (the upstream is asked to CONNECT to an already-validated, pinned IP).+ `CRAWL4AI_UPSTREAM_PROXY` overrides the env vars. Basic auth via+ `http://user:pass@proxy:port` is supported; for NTLM/Kerberos proxies,+ front them with a local translator (e.g. `cntlm`, `px`) and point+ `CRAWL4AI_UPSTREAM_PROXY` at it. Proxies that refuse CONNECT-to-an-IP, or+ containers with no DNS at all, are not yet supported.+#### 4. Stopping the Container```bashdiff --git a/deploy/docker/egress_proxy.py b/deploy/docker/egress_proxy.pyindex 4cdb56bd..1aa1bcc2 100644--- a/deploy/docker/egress_proxy.py+++ b/deploy/docker/egress_proxy.py@@ -13,13 +13,21 @@ TLS stays end-to-end (we tunnel ciphertext; Chromium verifies the cert/SNIagainst the real host - no MITM).Bound to 127.0.0.1 on an ephemeral port; started at server boot.++If HTTP_PROXY/HTTPS_PROXY (or CRAWL4AI_UPSTREAM_PROXY) is set, we still+resolve-and-pin locally but dial via the upstream proxy, asking it to CONNECT+to the PINNED IP — never the hostname — so the rebinding guarantee holds.+NO_PROXY bypasses it; with no proxy env set, behavior is unchanged."""from __future__ import annotationsimport asyncio+import base64+import ipaddressimport logging-from urllib.parse import urlsplit+import os+from urllib.parse import unquote, urlsplitfrom egress_broker import EgressBlocked, resolve_and_pin@@ -31,6 +39,63 @@ _BAD = b"HTTP/1.1 400 Bad Request\r\nContent-Length: 11\r\n\r\nBad Request"_MAX_HEADER_BYTES = 64 * 1024+def _env(*names: str) -> str:+ return next((os.environ[n] for n in names if os.environ.get(n)), "")+++def upstream_proxy(scheme: str = "https"):+ """(host, port, auth_header_bytes|None) of the upstream proxy, or None.++ Read per-call (not at import) so operators and tests see env changes.+ The target scheme picks HTTP(S)_PROXY per convention, falling back to+ the other pair when only one is set.+ """+ order = ("HTTP_PROXY", "http_proxy", "HTTPS_PROXY", "https_proxy") if scheme == "http" \+ else ("HTTPS_PROXY", "https_proxy", "HTTP_PROXY", "http_proxy")+ raw = _env("CRAWL4AI_UPSTREAM_PROXY", *order).strip()+ if not raw:+ return None+ sp = urlsplit(raw if "://" in raw else "http://" + raw)+ if not sp.hostname:+ return None+ auth = None+ if sp.username:+ cred = f"{unquote(sp.username)}:{unquote(sp.password or '')}".encode("utf-8")+ auth = b"Proxy-Authorization: Basic " + base64.b64encode(cred) + b"\r\n"+ return sp.hostname, sp.port or 80, auth+++def _no_proxy_match(host: str, ip: str) -> bool:+ """True if NO_PROXY says this target must bypass the upstream proxy."""+ entries = [e.strip() for e in _env("NO_PROXY", "no_proxy").split(",") if e.strip()]+ for entry in entries:+ if entry == "*":+ return True+ try:+ if ipaddress.ip_address(ip) in ipaddress.ip_network(entry, strict=False):+ return True+ continue+ except ValueError:+ pass+ suffix = entry.lower().lstrip(".")+ low = host.lower()+ if low == suffix or low.endswith("." + suffix):+ return True+ return False+++def _use_upstream(pin):+ """The upstream (host, port, auth) to route `pin` through, or None for direct."""+ up = upstream_proxy(pin.scheme)+ if up is None or _no_proxy_match(pin.host, pin.ip):+ return None+ return up+++def _bracket(ip: str) -> str:+ return f"[{ip}]" if ":" in ip else ip++class PinningProxy:"""Async HTTP forward-proxy that connects only to pinned, global IPs."""@@ -52,6 +117,12 @@ class PinningProxy:sock = self._server.sockets[0]self.bound_host, self.bound_port = sock.getsockname()[:2]logger.info("egress pinning proxy listening on %s", self.url)+ up = upstream_proxy()+ if up is not None:+ logger.info(+ "egress pinning proxy chaining through upstream proxy %s:%s",+ up[0], up[1],+ )return self.urlasync def stop(self) -> None:@@ -101,9 +172,7 @@ class PinningProxy:await self._drain_headers(client_reader)try:- up_reader, up_writer = await asyncio.wait_for(- asyncio.open_connection(pin.ip, int(port_s)), timeout=30- )+ up_reader, up_writer = await self._dial(pin, int(port_s))except Exception:await self._reply(client_writer, _BLOCKED)return@@ -129,15 +198,31 @@ class PinningProxy:path = sp.path or "/"if sp.query:path += "?" + sp.query+ upstream = _use_upstream(pin)+ dst = (upstream[0], upstream[1]) if upstream else (pin.ip, port)try:up_reader, up_writer = await asyncio.wait_for(- asyncio.open_connection(pin.ip, port), timeout=30+ asyncio.open_connection(*dst), timeout=30)except Exception:await self._reply(client_writer, _BLOCKED)return- # Re-issue in origin form with Host preserved.- out = f"{method} {path} HTTP/1.1\r\n".encode("latin-1")+ # Re-issue with Host preserved: origin form when dialing the pinned IP+ # directly, absolute form against the pinned IP when going through the+ # upstream proxy (which then needs no DNS lookup of its own).+ if upstream is None:+ out = f"{method} {path} HTTP/1.1\r\n".encode("latin-1")+ else:+ out = f"{method} http://{_bracket(pin.ip)}:{port}{path} HTTP/1.1\r\n".encode("latin-1")+ if upstream[2]:+ out += upstream[2]+ # One validated request per upstream connection: only this first+ # request is pinned/rewritten, so force close to keep a reused+ # client connection from smuggling unvalidated requests upstream.+ headers = b"".join(+ ln + b"\r\n" for ln in headers.split(b"\r\n")+ if ln and not ln.lower().startswith(b"connection:")+ ) + b"Connection: close\r\n"out += b"Host: " + sp.hostname.encode("latin-1")if sp.port:out += f":{sp.port}".encode("latin-1")@@ -147,6 +232,39 @@ class PinningProxy:await self._splice(client_reader, client_writer, up_reader, up_writer)# ─────────────────────────── helpers ───────────────────────────+ async def _dial(self, pin, port: int):+ """Open a byte pipe to the pinned IP: direct, or tunneled through the+ upstream proxy via CONNECT-to-the-pinned-IP (no upstream DNS lookup)."""+ upstream = _use_upstream(pin)+ if upstream is None:+ return await asyncio.wait_for(+ asyncio.open_connection(pin.ip, port), timeout=30+ )+ p_host, p_port, auth = upstream+ reader, writer = await asyncio.wait_for(+ asyncio.open_connection(p_host, p_port), timeout=30+ )+ try:+ dst = f"{_bracket(pin.ip)}:{port}"+ req = f"CONNECT {dst} HTTP/1.1\r\nHost: {dst}\r\n".encode("latin-1")+ if auth:+ req += auth+ req += b"\r\n"+ writer.write(req)+ await writer.drain()+ status = await asyncio.wait_for(reader.readline(), timeout=30)+ parts = status.split()+ if len(parts) < 2 or parts[1] != b"200":+ logger.warning("upstream proxy refused CONNECT: %r", status[:64])+ raise ConnectionError("upstream proxy refused CONNECT")+ # Drain the upstream's response headers so none of them leak into+ # the tunneled byte stream.+ await self._drain_headers(reader)+ except Exception:+ await self._safe_close(writer)+ raise+ return reader, writer+async def _drain_headers(self, reader):read = 0while True:diff --git a/deploy/docker/tests/test_security_egress_proxy.py b/deploy/docker/tests/test_security_egress_proxy.pyindex 03ebae1e..f53767f3 100644--- a/deploy/docker/tests/test_security_egress_proxy.py+++ b/deploy/docker/tests/test_security_egress_proxy.py@@ -20,6 +20,18 @@ from egress_proxy import PinningProxypytestmark = pytest.mark.posture+_PROXY_ENV = (+ "CRAWL4AI_UPSTREAM_PROXY", "HTTP_PROXY", "http_proxy",+ "HTTPS_PROXY", "https_proxy", "NO_PROXY", "no_proxy",+)++[email protected](autouse=True)+def _clear_proxy_env(monkeypatch):+ # Keep the suite deterministic on dev machines that sit behind a proxy.+ for name in _PROXY_ENV:+ monkeypatch.delenv(name, raising=False)+async def _fake_upstream():async def handle(reader, writer):@@ -121,6 +133,150 @@ class TestPinningProxy:await proxy.stop()+async def _fake_corporate_proxy(seen):+ """Minimal HTTP proxy: records the CONNECT request line, replies 200, then+ answers any tunneled bytes with TUNNEL-OK."""+ async def handle(reader, writer):+ line = await reader.readline()+ seen.append(line)+ while True: # drain CONNECT headers+ h = await reader.readline()+ if h in (b"\r\n", b"\n", b""):+ break+ writer.write(b"HTTP/1.1 200 Connection established\r\nVia: fake\r\n\r\n")+ await writer.drain()+ await reader.read(65536)+ writer.write(b"TUNNEL-OK")+ await writer.drain()+ writer.close()+ server = await asyncio.start_server(handle, "127.0.0.1", 0)+ return server, server.sockets[0].getsockname()[1]+++class TestUpstreamChaining:+ async def test_chained_connect_pins_ip_and_blocks_before_upstream(self, monkeypatch):+ """The chained-CONNECT security contract: the upstream receives the+ PINNED IP (never a hostname to resolve), its response headers do not+ leak into the tunnel, and a blocked target produces an opaque 403+ with zero upstream traffic."""+ seen = []+ corp, corp_port = await _fake_corporate_proxy(seen)+ monkeypatch.setenv("HTTPS_PROXY", f"http://127.0.0.1:{corp_port}")++ def fake_pin(url):+ if "internal.example" in url:+ raise EgressBlocked()+ return PinnedTarget("https", "good.example", 443, "203.0.113.7")+ monkeypatch.setattr(egress_proxy, "resolve_and_pin", fake_pin)++ proxy = PinningProxy()+ await proxy.start()+ try:+ r, w = await asyncio.open_connection(proxy.bound_host, proxy.bound_port)+ w.write(b"CONNECT good.example:443 HTTP/1.1\r\n\r\n")+ await w.drain()+ status = await asyncio.wait_for(r.readline(), timeout=5)+ assert b"200" in status+ await r.readline() # blank line after the 200+ w.write(b"hello")+ await w.drain()+ body = await asyncio.wait_for(r.read(100), timeout=5)+ # Upstream's Via header must NOT leak into the tunnel.+ assert body == b"TUNNEL-OK"+ w.close()++ r, w = await asyncio.open_connection(proxy.bound_host, proxy.bound_port)+ w.write(b"CONNECT internal.example:443 HTTP/1.1\r\n\r\n")+ await w.drain()+ status = await asyncio.wait_for(r.readline(), timeout=5)+ assert b"403" in status+ w.close()+ finally:+ await proxy.stop()+ corp.close()+ # The upstream saw ONLY the pinned IP of the allowed target.+ assert seen == [b"CONNECT 203.0.113.7:443 HTTP/1.1\r\n"]++ async def test_chained_plain_http_pinned_absolute_form_no_smuggling(self, monkeypatch):+ """Plain HTTP via upstream: the request is re-issued in absolute form+ against the PINNED IP (no name for the upstream to resolve), carries+ Connection: close, and a reused client connection cannot smuggle a+ second, unvalidated request upstream."""+ lines = []++ async def handle(reader, writer):+ req = b""+ while b"\r\n\r\n" not in req:+ req += await reader.read(4096)+ lines.append(req)+ writer.write(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nhi")+ await writer.drain()+ writer.close()+ corp = await asyncio.start_server(handle, "127.0.0.1", 0)+ corp_port = corp.sockets[0].getsockname()[1]+ monkeypatch.setenv("HTTP_PROXY", f"http://127.0.0.1:{corp_port}")++ def fake_pin(url):+ return PinnedTarget("http", "plain.example", 80, "203.0.113.7")+ monkeypatch.setattr(egress_proxy, "resolve_and_pin", fake_pin)++ proxy = PinningProxy()+ await proxy.start()+ try:+ r, w = await asyncio.open_connection(proxy.bound_host, proxy.bound_port)+ w.write(b"GET http://plain.example/ HTTP/1.1\r\n"+ b"Host: plain.example\r\nConnection: keep-alive\r\n\r\n")+ await w.drain()+ first = await asyncio.wait_for(r.read(200), timeout=5)+ assert b"200" in first+ # Attempt to smuggle an unvalidated request on the same connection.+ w.write(b"GET http://rebind.evil/ HTTP/1.1\r\nHost: rebind.evil\r\n\r\n")+ await w.drain()+ leftover = await asyncio.wait_for(r.read(200), timeout=5)+ assert leftover == b"" # upstream closed; nothing came back+ w.close()+ finally:+ await proxy.stop()+ corp.close()+ sent = b"".join(lines)+ assert sent.startswith(b"GET http://203.0.113.7:80/ HTTP/1.1\r\n")+ assert b"Connection: close" in sent+ assert b"keep-alive" not in sent+ assert b"rebind.evil" not in sent # the smuggled request never got upstream+++def test_upstream_proxy_env_parsing(monkeypatch):+ assert egress_proxy.upstream_proxy() is None+ monkeypatch.setenv("HTTP_PROXY", "http://192.168.180.254:56560")+ assert egress_proxy.upstream_proxy() == ("192.168.180.254", 56560, None)+ monkeypatch.setenv("HTTPS_PROXY", "http://user:p%[email protected]:8080")+ host, port, auth = egress_proxy.upstream_proxy()+ assert (host, port) == ("10.0.0.1", 8080)+ import base64+ assert base64.b64decode(auth.split(b" ")[-1].strip()) == b"user:p@ss"+ # scheme-aware selection: http targets prefer HTTP_PROXY+ assert egress_proxy.upstream_proxy("http") == ("192.168.180.254", 56560, None)+ monkeypatch.setenv("CRAWL4AI_UPSTREAM_PROXY", "proxy.corp:3128")+ assert egress_proxy.upstream_proxy() == ("proxy.corp", 3128, None)+ # whitespace-only env var means unset, not a proxy named " "+ monkeypatch.setenv("CRAWL4AI_UPSTREAM_PROXY", " ")+ monkeypatch.delenv("HTTP_PROXY")+ monkeypatch.delenv("HTTPS_PROXY")+ assert egress_proxy.upstream_proxy() is None+ # non-latin-1 credentials must not raise (encoded as UTF-8)+ monkeypatch.delenv("CRAWL4AI_UPSTREAM_PROXY")+ monkeypatch.setenv("HTTPS_PROXY", "http://u:%E5%AF%86%E7%A0%[email protected]:8080")+ assert egress_proxy.upstream_proxy()[2] is not None+ # NO_PROXY routing: suffix and CIDR entries force a direct dial+ pin = PinnedTarget("https", "site.corp.example", 443, "203.0.113.7")+ assert egress_proxy._use_upstream(pin) is not None+ monkeypatch.setenv("NO_PROXY", ".corp.example")+ assert egress_proxy._use_upstream(pin) is None+ monkeypatch.setenv("NO_PROXY", "203.0.113.0/24")+ assert egress_proxy._use_upstream(pin) is None++class TestEnforceEgressWiring:def test_enforce_egress_sets_proxy(self, monkeypatch):import egress_broker