Official Crawl4AI Cloud: a hosted web scraping, extraction, and search API for AI agents. Free API key, 1,000 credits a month, no card. Fully open source: this repo is the Apache-2.0 crawler that powers it, and you can self-host it.
Python aiai-agentscrawl4ai-cloudcrawlerdata-extractionllmmarkdownmcpopen-sourceplaywrightpythonragscraperweb-crawlerweb-crawlingweb-scraping
security: non-breaking hardening patch (0.8.8)
unclecodeaa81e8fe7d94
diff --git a/Dockerfile b/Dockerfileindex 86ede349..0c0830a0 100644--- a/Dockerfile+++ b/Dockerfile@@ -1,7 +1,7 @@FROM python:3.12-slim-bookworm AS build# C4ai version-ARG C4AI_VER=0.8.7+ARG C4AI_VER=0.8.8ENV C4AI_VERSION=$C4AI_VERLABEL c4ai.version=$C4AI_VERdiff --git a/crawl4ai/__version__.py b/crawl4ai/__version__.pyindex ef800fda..6784bb98 100644--- a/crawl4ai/__version__.py+++ b/crawl4ai/__version__.py@@ -1,7 +1,7 @@# crawl4ai/__version__.py# This is the version that will be used for stable releases-__version__ = "0.8.7"+__version__ = "0.8.8"# For nightly builds, this gets set during build process__nightly_version__ = Nonediff --git a/crawl4ai/async_configs.py b/crawl4ai/async_configs.pyindex 5938805f..92e5ac5f 100644--- a/crawl4ai/async_configs.py+++ b/crawl4ai/async_configs.py@@ -2022,6 +2022,30 @@ class CrawlerRunConfig():config_dict.update(kwargs)return CrawlerRunConfig.from_kwargs(config_dict)++# Environment variable names that `LLMConfig(api_token="env:NAME")` may NOT+# resolve. This blocks the credential-exfil gadget where an untrusted config+# body sets api_token="env:SECRET_KEY" and pairs it with a base_url to leak a+# server secret. Normal provider keys (OPENAI_API_KEY, ANTHROPIC_API_KEY,+# HF_TOKEN, ...) are unaffected.+_FORBIDDEN_ENV_SUBSTRINGS = ("SECRET", "PASSWORD", "PRIVATE", "PASSWD")+_FORBIDDEN_ENV_PREFIXES = ("CRAWL4AI", "AWS_SECRET")+_FORBIDDEN_ENV_EXACT = {"SECRET_KEY", "REDIS_PASSWORD", "TOKEN"}+++def _is_forbidden_env_name(name: str) -> bool:+ if not name:+ return True+ u = name.upper()+ if u in _FORBIDDEN_ENV_EXACT:+ return True+ if any(s in u for s in _FORBIDDEN_ENV_SUBSTRINGS):+ return True+ if any(u.startswith(p) for p in _FORBIDDEN_ENV_PREFIXES):+ return True+ return False++class LLMConfig:def __init__(self,@@ -2044,7 +2068,17 @@ class LLMConfig:if api_token and not api_token.startswith("env:"):self.api_token = api_tokenelif api_token and api_token.startswith("env:"):- self.api_token = os.getenv(api_token[4:])+ # `env:NAME` resolves an environment variable. Refuse to read names+ # that are clearly server secrets, so an untrusted config body cannot+ # exfiltrate them via `env:SECRET_KEY` and a paired base_url. Normal+ # provider keys (…_API_KEY / …_TOKEN) are unaffected.+ _env_name = api_token[4:]+ if _is_forbidden_env_name(_env_name):+ raise ValueError(+ f"LLMConfig.api_token may not reference the protected "+ f"environment variable {_env_name!r}"+ )+ self.api_token = os.getenv(_env_name)else:# Check if given provider starts with any of key in PROVIDER_MODELS_PREFIXES# If not, check if it is in PROVIDER_MODELSdiff --git a/deploy/docker/api.py b/deploy/docker/api.pyindex 80684bc0..42fcc11b 100644--- a/deploy/docker/api.py+++ b/deploy/docker/api.py@@ -129,7 +129,7 @@ async def handle_llm_qa(prompt_with_variables=prompt,api_token=get_llm_api_key(config, resolved_provider),temperature=temperature or get_llm_temperature(config, resolved_provider),- base_url=base_url or get_llm_base_url(config, resolved_provider),+ base_url=get_llm_base_url(config, resolved_provider), # ignore request base_url (key-exfil vector)base_delay=config["llm"].get("backoff_base_delay", 2),max_attempts=config["llm"].get("backoff_max_attempts", 3),exponential_factor=config["llm"].get("backoff_exponential_factor", 2)@@ -188,7 +188,7 @@ async def process_llm_extraction(provider=provider or config["llm"]["provider"],api_token=api_key,temperature=temperature or get_llm_temperature(config, provider),- base_url=base_url or get_llm_base_url(config, provider)+ base_url=get_llm_base_url(config, provider) # ignore request base_url (key-exfil vector)),instruction=instruction,schema=json.loads(schema) if schema else None,@@ -299,7 +299,7 @@ async def handle_markdown_request(provider=provider or config["llm"]["provider"],api_token=get_llm_api_key(config, provider), # Returns None to let litellm handle ittemperature=temperature or get_llm_temperature(config, provider),- base_url=base_url or get_llm_base_url(config, provider)+ base_url=get_llm_base_url(config, provider) # ignore request base_url (key-exfil vector)),instruction=query or "Extract main content")diff --git a/deploy/docker/schemas.py b/deploy/docker/schemas.pyindex c522ac18..dd561bb2 100644--- a/deploy/docker/schemas.py+++ b/deploy/docker/schemas.py@@ -121,6 +121,16 @@ class WebhookConfig(BaseModel):webhook_data_in_payload: bool = Falsewebhook_headers: Optional[Dict[str, str]] = None+ @field_validator("webhook_headers")+ @classmethod+ def _validate_headers(cls, v):+ # Reject unsafe outbound headers early (422). Mirrors+ # webhook.sanitize_webhook_headers; kept inline to avoid an import cycle.+ if not v:+ return v+ from webhook import sanitize_webhook_headers+ return sanitize_webhook_headers(v)+class WebhookPayload(BaseModel):"""Payload sent to webhook endpoints."""diff --git a/deploy/docker/server.py b/deploy/docker/server.pyindex e20e041e..e984ff62 100644--- a/deploy/docker/server.py+++ b/deploy/docker/server.py@@ -36,7 +36,7 @@ from schemas import (from utils import (FilterType, load_config, setup_logging, verify_email_domain,- validate_output_path, validate_webhook_url, validate_url_destination,+ validate_output_path, write_output_file, validate_webhook_url, validate_url_destination,)import osimport sys@@ -426,9 +426,7 @@ async def generate_screenshot(screenshot_data = results[0].screenshotif body.output_path:abs_path = validate_output_path(body.output_path)- os.makedirs(os.path.dirname(abs_path), exist_ok=True)- with open(abs_path, "wb") as f:- f.write(base64.b64decode(screenshot_data))+ write_output_file(abs_path, base64.b64decode(screenshot_data))return {"success": True, "path": abs_path}return {"success": True, "screenshot": screenshot_data}except Exception as e:@@ -464,9 +462,7 @@ async def generate_pdf(pdf_data = results[0].pdfif body.output_path:abs_path = validate_output_path(body.output_path)- os.makedirs(os.path.dirname(abs_path), exist_ok=True)- with open(abs_path, "wb") as f:- f.write(pdf_data)+ write_output_file(abs_path, pdf_data)return {"success": True, "path": abs_path}return {"success": True, "pdf": base64.b64encode(pdf_data).decode()}except Exception as e:diff --git a/deploy/docker/tests/test_security_0_8_8.py b/deploy/docker/tests/test_security_0_8_8.pynew file mode 100644index 00000000..468ac584--- /dev/null+++ b/deploy/docker/tests/test_security_0_8_8.py@@ -0,0 +1,158 @@+"""+Behavioral tests for the 0.8.8 non-breaking security patch.++Each fix here is backward-compatible: features keep working, only the unsafe+behavior is closed. (The full secure-by-default redesign is the later major.)++Covers:+ - SSRF blocklist gaps closed (NAT64 / 6to4 / :: / v4-mapped, not-is_global)+ + opaque error (no resolved IP leak)+ - output_path symlink/TOCTOU hardening (realpath containment + O_NOFOLLOW)+ with the feature kept+ - request-supplied LLM base_url ignored (key-exfil vector)+ - env:SECRET_KEY exfil gadget blocked in LLMConfig (provider keys still work)+ - CRLF-safe logging+ - webhook header sanitization+"""++import os+import socket+import sys++import pytest++DOCKER_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))+if DOCKER_DIR not in sys.path:+ sys.path.insert(0, DOCKER_DIR)+++def _patch_dns(monkeypatch, ip):+ def fake(host, port=None, *a, **k):+ return [(socket.AF_INET, socket.SOCK_STREAM, 6, "", (ip, port or 0))]+ monkeypatch.setattr(socket, "getaddrinfo", fake)+++class TestSsrfGapsClosed:+ @pytest.mark.parametrize("ip", [+ "169.254.169.254", # metadata+ "127.0.0.1", "10.0.0.5", "192.168.1.1", "100.64.0.1",+ "::1", "::", # v6 loopback + unspecified (was a gap)+ "::ffff:169.254.169.254", # v4-mapped metadata+ "64:ff9b::a9fe:a9fe", # NAT64 -> 169.254.169.254 (was a gap)+ "2002:a9fe:a9fe::1", # 6to4 embedding 169.254.169.254 (was a gap)+ ])+ def test_internal_blocked(self, monkeypatch, ip):+ import utils+ _patch_dns(monkeypatch, ip)+ with pytest.raises(ValueError):+ utils.validate_webhook_url("http://target.example/cb")++ @pytest.mark.parametrize("ip", ["8.8.8.8", "1.1.1.1"])+ def test_public_allowed(self, monkeypatch, ip):+ import utils+ _patch_dns(monkeypatch, ip)+ utils.validate_webhook_url("http://target.example/cb") # no raise++ def test_error_is_opaque(self, monkeypatch):+ import utils+ _patch_dns(monkeypatch, "169.254.169.254")+ with pytest.raises(ValueError) as e:+ utils.validate_webhook_url("http://target.example/")+ assert "169.254" not in str(e.value) # no resolved-IP leak+++class TestOutputPathHardening:+ def test_symlink_escape_rejected(self, monkeypatch, tmp_path):+ import utils+ allowed = tmp_path / "outputs"+ allowed.mkdir()+ monkeypatch.setattr(utils, "ALLOWED_OUTPUT_DIR", str(allowed))+ # Plant a symlinked subdir that points outside the allowed dir.+ outside = tmp_path / "outside"+ outside.mkdir()+ (allowed / "evil").symlink_to(outside)+ from fastapi import HTTPException+ with pytest.raises(HTTPException):+ utils.validate_output_path("evil/pwned.png") # realpath escapes++ def test_normal_path_ok(self, monkeypatch, tmp_path):+ import utils+ allowed = tmp_path / "outputs"+ allowed.mkdir()+ monkeypatch.setattr(utils, "ALLOWED_OUTPUT_DIR", str(allowed))+ p = utils.validate_output_path("sub/shot.png")+ assert p.startswith(str(allowed))++ def test_write_refuses_symlink_final_component(self, monkeypatch, tmp_path):+ import utils+ allowed = tmp_path / "outputs"+ allowed.mkdir()+ target = allowed / "shot.png"+ # Pre-plant a symlink at the final path pointing at a secret.+ secret = tmp_path / "secret"+ secret.write_text("SECRET")+ target.symlink_to(secret)+ with pytest.raises(OSError):+ utils.write_output_file(str(target), b"data") # O_NOFOLLOW refuses+++class TestLlmBaseUrlIgnored:+ def test_request_base_url_not_honored_in_source(self):+ # base_url from the request must never be passed to the LLM call.+ with open(os.path.join(DOCKER_DIR, "api.py")) as f:+ src = f.read()+ assert "base_url or get_llm_base_url" not in src+ assert "base_url=get_llm_base_url(config" in src+++class TestEnvSecretGuard:+ def test_env_secret_key_blocked(self):+ from crawl4ai.async_configs import LLMConfig+ with pytest.raises(ValueError):+ LLMConfig(api_token="env:SECRET_KEY")++ @pytest.mark.parametrize("name", ["REDIS_PASSWORD", "CRAWL4AI_API_TOKEN", "MY_PRIVATE_KEY"])+ def test_other_secrets_blocked(self, name):+ from crawl4ai.async_configs import LLMConfig+ with pytest.raises(ValueError):+ LLMConfig(api_token=f"env:{name}")++ def test_provider_key_still_works(self, monkeypatch):+ from crawl4ai.async_configs import LLMConfig+ monkeypatch.setenv("OPENAI_API_KEY", "sk-test-123")+ cfg = LLMConfig(provider="openai/gpt-4o-mini", api_token="env:OPENAI_API_KEY")+ assert cfg.api_token == "sk-test-123" # normal provider key unaffected+++class TestCRLFSafeLogging:+ def test_crlf_stripped(self):+ import logging+ from utils import CRLFSafeFilter+ rec = logging.LogRecord("t", logging.INFO, __file__, 1,+ "url=http://x/\r\nINJECTED login ok", None, None)+ CRLFSafeFilter().filter(rec)+ msg = rec.getMessage()+ assert "\r" not in msg and "\n" not in msg and "INJECTED" in msg+++class TestWebhookHeaderSanitization:+ def test_crlf_value_rejected(self):+ from webhook import sanitize_webhook_headers+ with pytest.raises(ValueError):+ sanitize_webhook_headers({"X-Foo": "bar\r\nInjected: 1"})++ @pytest.mark.parametrize("bad", ["Host", "Content-Length", "Authorization", "Cookie"])+ def test_hop_by_hop_denied(self, bad):+ from webhook import sanitize_webhook_headers+ with pytest.raises(ValueError):+ sanitize_webhook_headers({bad: "x"})++ def test_good_header_passes(self):+ from webhook import sanitize_webhook_headers+ assert sanitize_webhook_headers({"X-Trace-Id": "abc"}) == {"X-Trace-Id": "abc"}++ def test_schema_rejects_early(self):+ from schemas import WebhookConfig+ import pydantic+ with pytest.raises(pydantic.ValidationError):+ WebhookConfig(webhook_url="https://example.com/cb", webhook_headers={"Host": "evil"})diff --git a/deploy/docker/utils.py b/deploy/docker/utils.pyindex 881d6d2e..4e65af33 100644--- a/deploy/docker/utils.py+++ b/deploy/docker/utils.py@@ -149,12 +149,35 @@ def load_config() -> Dict:return config+class CRLFSafeFilter(logging.Filter):+ """Strip CR/LF/control chars from log records (log-injection / forging).++ A crawl URL or error reflected into a log line could otherwise inject+ newlines and forge additional log entries."""++ _BAD = {ord(c): None for c in "\r\n"} | {i: None for i in range(0, 32) if i != 9}++ def filter(self, record: "logging.LogRecord") -> bool:+ try:+ msg = record.getMessage()+ cleaned = msg.translate(self._BAD)+ if cleaned != msg:+ record.msg = cleaned+ record.args = ()+ except Exception:+ pass+ return True++def setup_logging(config: Dict) -> None:- """Configure application logging."""+ """Configure application logging with CRLF-safe records."""logging.basicConfig(level=config["logging"]["level"],format=config["logging"]["format"])+ crlf = CRLFSafeFilter()+ for handler in logging.getLogger().handlers:+ handler.addFilter(crlf)def get_base_url(request: Request) -> str:"""Get base URL including scheme and host."""@@ -299,17 +322,39 @@ ALLOWED_OUTPUT_DIR = os.environ.get("CRAWL4AI_OUTPUT_DIR", "/tmp/crawl4ai-outputdef validate_output_path(user_path: str) -> str:"""Resolve and restrict output_path to ALLOWED_OUTPUT_DIR.- Prevents arbitrary file write via path traversal."""++ Hardened against the symlink/TOCTOU bypass: in addition to the string+ containment check, the REAL path (symlinks resolved) of the target and its+ parent must also stay inside the allowed dir, so a symlink planted inside+ the dir cannot redirect the write elsewhere. The final write must also use+ O_NOFOLLOW (see write_output_file)."""from fastapi import HTTPExceptionsafe_path = os.path.normpath(user_path).lstrip(os.sep)abs_path = os.path.abspath(os.path.join(ALLOWED_OUTPUT_DIR, safe_path))abs_allowed = os.path.abspath(ALLOWED_OUTPUT_DIR) + os.sep- if not abs_path.startswith(abs_allowed):+ # Resolve symlinks on the parent dir (the file itself may not exist yet) and+ # re-check containment; this defeats a symlinked path component. The trailing+ # os.sep on both sides also rejects sibling-prefix names ("...-evil").+ real_parent = os.path.realpath(os.path.dirname(abs_path))+ real_path = os.path.join(real_parent, os.path.basename(abs_path))+ string_ok = abs_path.startswith(abs_allowed)+ real_ok = (real_path + os.sep).startswith(abs_allowed)+ if not (string_ok and real_ok):raise HTTPException(status_code=400,- detail=f"output_path must resolve within {ALLOWED_OUTPUT_DIR}"+ detail=f"output_path must resolve within {ALLOWED_OUTPUT_DIR}",)- return abs_path+ return real_path+++def write_output_file(abs_path: str, data: bytes) -> None:+ """Write bytes to a validated output path without following a symlink at the+ final component (defends the symlink-TOCTOU write)."""+ os.makedirs(os.path.dirname(abs_path), exist_ok=True)+ flags = os.O_WRONLY | os.O_CREAT | os.O_TRUNC | getattr(os, "O_NOFOLLOW", 0)+ fd = os.open(abs_path, flags, 0o600)+ with os.fdopen(fd, "wb") as f:+ f.write(data)import ipaddress@@ -355,42 +400,54 @@ def validate_url_destination(url: str) -> None:raise HTTPException(status_code=400, detail=f"URL blocked (SSRF protection): {e}")+_NAT64_NETWORK = ipaddress.ip_network("64:ff9b::/96")+_6TO4_NETWORK = ipaddress.ip_network("2002::/16")+_V4COMPAT_NETWORK = ipaddress.ip_network("::/96")++def _expand_ip_candidates(ip):- """Return [ip] plus any IPv4 form wrapped inside the IPv6 address.- SSRF guards must check the unwrapped form because ::ffff:127.0.0.1 and- ::127.0.0.1 route to 127.0.0.1 but would not match IPv4 blocklists directly."""+ """Return [ip] plus any IPv4 form embedded inside the IPv6 address.+ SSRF guards must check the unwrapped form because ::ffff:169.254.169.254,+ NAT64 64:ff9b::a9fe:a9fe, and 6to4 2002:a9fe:a9fe:: all route to an internal+ IPv4 that would not match an IPv6-only check."""candidates = [ip]if isinstance(ip, ipaddress.IPv6Address):- if ip.ipv4_mapped is not None:- candidates.append(ip.ipv4_mapped)- else:- as_int = int(ip)- if 0 < as_int < 2**32:- candidates.append(ipaddress.IPv4Address(as_int))+ mapped = ip.ipv4_mapped+ if mapped is not None:+ candidates.append(mapped)+ elif ip in _NAT64_NETWORK or ip in _V4COMPAT_NETWORK:+ candidates.append(ipaddress.IPv4Address(int(ip) & 0xFFFFFFFF))+ elif ip in _6TO4_NETWORK:+ candidates.append(ipaddress.IPv4Address((int(ip) >> 80) & 0xFFFFFFFF))return candidatesdef validate_webhook_url(url: str) -> None:- """Reject webhook URLs targeting internal/private/reserved networks (SSRF protection)."""+ """Reject webhook/crawl URLs targeting non-global (internal/private/reserved)+ networks (SSRF protection).++ One rule: reject any resolved IP where not ip.is_global, evaluated on every+ embedded IPv4 transition form (v4-mapped, NAT64, 6to4, v4-compat). This+ closes the gaps in the old explicit blocklist (the IPv6 unspecified ::,+ NAT64, 6to4). The error is opaque - it never echoes the resolved internal+ IP (which would be a DNS/oracle leak)."""parsed = urlparse(str(url))hostname = parsed.hostnameif not hostname:- raise ValueError("Webhook URL must have a valid hostname")+ raise ValueError("URL must have a valid hostname")hostname_lower = hostname.lower()- if hostname_lower in _BLOCKED_HOSTNAMES:- raise ValueError(f"Webhook URL hostname '{hostname}' is blocked")- if hostname_lower.startswith("host.docker.internal"):- raise ValueError(f"Webhook URL hostname '{hostname}' is blocked")+ if hostname_lower in _BLOCKED_HOSTNAMES or hostname_lower.startswith("host.docker.internal"):+ raise ValueError("URL host is blocked")try:resolved = socket.getaddrinfo(hostname, None)except socket.gaierror:- raise ValueError(f"Cannot resolve webhook hostname '{hostname}'")+ raise ValueError("Cannot resolve URL host")for _, _, _, _, sockaddr in resolved:ip = ipaddress.ip_address(sockaddr[0])for candidate in _expand_ip_candidates(ip):- for network in _BLOCKED_NETWORKS:- if candidate in network:- raise ValueError(f"Webhook URL resolves to blocked address: {ip}")+ if not candidate.is_global:+ # opaque: do not leak the resolved internal IP+ raise ValueError("URL resolves to a blocked address")def verify_email_domain(email: str) -> bool:diff --git a/deploy/docker/webhook.py b/deploy/docker/webhook.pyindex 4c409ae8..0dffd327 100644--- a/deploy/docker/webhook.py+++ b/deploy/docker/webhook.py@@ -6,11 +6,43 @@ This module provides webhook notification functionality with exponential backoffimport asyncioimport httpximport logging+import refrom typing import Dict, Optionalfrom datetime import datetime, timezonelogger = logging.getLogger(__name__)+# Webhook request-header policy: user-controlled outbound headers could inject+# hop-by-hop / smuggling headers or CRLF. Allow only well-formed names, reject+# control chars in values, and deny sensitive/hop-by-hop names.+_WEBHOOK_HEADER_NAME = re.compile(r"^[A-Za-z0-9-]{1,64}$")+_WEBHOOK_DENY_HEADERS = {+ "host", "content-length", "transfer-encoding", "connection",+ "content-type", "proxy-authorization", "authorization", "cookie",+ "expect", "upgrade", "te", "trailer",+}+_MAX_WEBHOOK_HEADERS = 20+_MAX_WEBHOOK_HEADER_VALUE = 2048+++def sanitize_webhook_headers(headers: Optional[Dict[str, str]]) -> Dict[str, str]:+ """Validate user-supplied webhook headers; raise ValueError on any bad one."""+ if not headers:+ return {}+ if len(headers) > _MAX_WEBHOOK_HEADERS:+ raise ValueError("too many webhook headers")+ clean: Dict[str, str] = {}+ for name, value in headers.items():+ if not isinstance(name, str) or not _WEBHOOK_HEADER_NAME.match(name):+ raise ValueError(f"invalid webhook header name: {name!r}")+ if name.lower() in _WEBHOOK_DENY_HEADERS:+ raise ValueError(f"webhook header not allowed: {name}")+ sval = str(value)+ if len(sval) > _MAX_WEBHOOK_HEADER_VALUE or any(c in sval for c in "\r\n\x00"):+ raise ValueError(f"invalid value for webhook header {name}")+ clean[name] = sval+ return clean+class WebhookDeliveryService:"""Handles webhook delivery with exponential backoff retry logic."""@@ -50,7 +82,12 @@ class WebhookDeliveryService:validate_webhook_url(webhook_url)default_headers = self.config.get("headers", {})- merged_headers = {**default_headers, **(headers or {})}+ try:+ safe_custom = sanitize_webhook_headers(headers)+ except ValueError as e:+ logger.warning(f"Dropping unsafe webhook headers: {e}")+ safe_custom = {}+ merged_headers = {**default_headers, **safe_custom}merged_headers["Content-Type"] = "application/json"async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=False) as client: