Official Crawl4AI Cloud: a hosted web scraping, extraction, and search API for AI agents. Free API key, 1,000 credits a month, no card. Fully open source: this repo is the Apache-2.0 crawler that powers it, and you can self-host it.
Python aiai-agentscrawl4ai-cloudcrawlerdata-extractionllmmarkdownmcpopen-sourceplaywrightpythonragscraperweb-crawlerweb-crawlingweb-scraping
fix(docker): declare output_path as deprecated no-op, harden legacy-compat tests
Soham Kukretia01442d3a5a6
diff --git a/deploy/docker/entrypoint.sh b/deploy/docker/entrypoint.shindex 19d6766f..3d003791 100644--- a/deploy/docker/entrypoint.sh+++ b/deploy/docker/entrypoint.sh@@ -32,7 +32,7 @@ elseGUNICORN_BIND="127.0.0.1:${PORT}"echo "entrypoint: no CRAWL4AI_API_TOKEN set; binding loopback only (${GUNICORN_BIND})." >&2echo "entrypoint: WARNING: this is the CONTAINER's loopback - published ports (-p ${PORT}:${PORT}) will NOT work; connections from the host will be reset." >&2- echo "entrypoint: to make the server reachable, set CRAWL4AI_API_TOKEN (docker run -e CRAWL4AI_API_TOKEN=..., or the .llm.env file with docker compose) and restart." >&2+ echo "entrypoint: to make the server reachable, set CRAWL4AI_API_TOKEN (docker run -e CRAWL4AI_API_TOKEN=..., or the .llm.env file with docker compose), then restart. (If you enabled security.jwt_enabled in a custom config.yml, set CRAWL4AI_JWT_ENABLED=true instead.)" >&2fiexport GUNICORN_BINDdiff --git a/deploy/docker/schemas.py b/deploy/docker/schemas.pyindex b1065b5a..978f2199 100644--- a/deploy/docker/schemas.py+++ b/deploy/docker/schemas.py@@ -46,7 +46,7 @@ class HookConfig(BaseModel):# exec()-based RCE surface). Captured here (instead of being dropped by# pydantic) solely so the server can tell the caller it was NOT executed;# it is never run.- code: Optional[Dict[str, str]] = Field(+ code: Optional[Any] = Field(default=None,description="REMOVED in 0.9.0: inline hook code is accepted for compatibility but never executed",)@@ -92,14 +92,29 @@ class ScreenshotRequest(BaseModel):url: strscreenshot_wait_for: Optional[float] = 2wait_for_images: Optional[bool] = False- # output_path removed: callers never name a filesystem path (it was an- # arbitrary-write -> RCE vector). The server writes to the sandboxed- # artifact store and returns an opaque artifact_id.+ # Deprecated no-op: caller paths were an arbitrary-write -> RCE vector.+ # Never written; the server stores results in the artifact store instead.+ output_path: Optional[str] = Field(+ default=None,+ deprecated=True,+ description=(+ "REMOVED in 0.9.0 and ignored - no file is written. Results are "+ "stored server-side; fetch via GET /artifacts/{artifact_id}."+ ),+ )class PDFRequest(BaseModel):url: str- # output_path removed (see ScreenshotRequest).+ # output_path deprecated no-op (see ScreenshotRequest).+ output_path: Optional[str] = Field(+ default=None,+ deprecated=True,+ description=(+ "REMOVED in 0.9.0 and ignored - no file is written. Results are "+ "stored server-side; fetch via GET /artifacts/{artifact_id}."+ ),+ )class JSEndpointRequest(BaseModel):diff --git a/deploy/docker/server.py b/deploy/docker/server.pyindex e6f63db2..eb0fddad 100644--- a/deploy/docker/server.py+++ b/deploy/docker/server.py@@ -701,6 +701,9 @@ async def generate_screenshot(sandboxed artifact store; the response includes an `artifact_id` and a `url` to fetch it."""validate_url_scheme(body.url)+ # model_dump, not attribute access: the field is marked deprecated and+ # reading the attribute emits DeprecationWarning on every request.+ legacy_output_path = body.model_dump(include={"output_path"}).get("output_path")crawler = Nonetry:cfg = CrawlerRunConfig(screenshot=True, screenshot_wait_for=body.screenshot_wait_for, wait_for_images=body.wait_for_images)@@ -711,9 +714,7 @@ async def generate_screenshot(screenshot_data = results[0].screenshotart = _store_artifact("png", base64.b64decode(screenshot_data))response = {"success": True, "screenshot": screenshot_data, **art}- # Legacy 0.8.x key, no longer a schema field: peek at the raw body- # (already parsed and cached by FastAPI) to warn that it was ignored.- if (await request.json()).get("output_path"):+ if legacy_output_path:response["warning"] = _OUTPUT_PATH_WARNINGreturn responseexcept HTTPException:@@ -741,6 +742,9 @@ async def generate_pdf(sandboxed artifact store; the response includes an `artifact_id` and a `url` to fetch it."""validate_url_scheme(body.url)+ # model_dump, not attribute access: the field is marked deprecated and+ # reading the attribute emits DeprecationWarning on every request.+ legacy_output_path = body.model_dump(include={"output_path"}).get("output_path")crawler = Nonetry:cfg = CrawlerRunConfig(pdf=True)@@ -751,9 +755,7 @@ async def generate_pdf(pdf_data = results[0].pdfart = _store_artifact("pdf", pdf_data)response = {"success": True, "pdf": base64.b64encode(pdf_data).decode(), **art}- # Legacy 0.8.x key, no longer a schema field: peek at the raw body- # (already parsed and cached by FastAPI) to warn that it was ignored.- if (await request.json()).get("output_path"):+ if legacy_output_path:response["warning"] = _OUTPUT_PATH_WARNINGreturn responseexcept HTTPException:@@ -943,7 +945,7 @@ async def crawl(crawler_configs=crawl_request.crawler_configs,)if crawl_request.hooks and crawl_request.hooks.code:- hooks_resp = results.setdefault("hooks", {"attached": []})+ hooks_resp = results.setdefault("hooks", {"status": "ignored", "attached": []})if not crawl_request.hooks.hooks:hooks_resp["status"] = "ignored"hooks_resp["warning"] = _HOOKS_CODE_WARNINGdiff --git a/deploy/docker/static/playground/index.html b/deploy/docker/static/playground/index.htmlindex cf72a303..ba368b76 100644--- a/deploy/docker/static/playground/index.html+++ b/deploy/docker/static/playground/index.html@@ -619,7 +619,15 @@// Build a useful error message from a failed HTTP responsefunction httpErrorMessage(response, data) {- let msg = (data && (data.detail || data.error)) || `HTTP ${response.status}`;+ let detail = data && (data.detail || data.error);+ // FastAPI 422s send detail as an array of error objects; other+ // non-strings would render as [object Object].+ if (Array.isArray(detail)) {+ detail = detail.map(e => e && e.msg ? `${(e.loc || []).join('.')}: ${e.msg}` : JSON.stringify(e)).join('; ');+ } else if (detail && typeof detail !== 'string') {+ detail = JSON.stringify(detail);+ }+ let msg = detail || `HTTP ${response.status}`;if (response.status === 401) {msg += getToken()? ' — token rejected; check the API token in the token bar (top right)'@@ -758,7 +766,7 @@method: 'GET',headers: { 'Accept': 'application/json' }});- responseData = await response.json();+ responseData = await response.json().catch(() => ({}));const time = Math.round(performance.now() - startTime);if (!response.ok) {updateStatus('error', time);@@ -821,7 +829,7 @@body: JSON.stringify(payload)});- responseData = await response.json();+ responseData = await response.json().catch(() => ({}));const time = Math.round(performance.now() - startTime);if (!response.ok) {diff --git a/deploy/docker/tests/requirements.txt b/deploy/docker/tests/requirements.txtindex 5f7a842f..b0206020 100644--- a/deploy/docker/tests/requirements.txt+++ b/deploy/docker/tests/requirements.txt@@ -1,2 +1,3 @@httpx>=0.25.0docker>=7.0.0+pyyaml>=6.0diff --git a/deploy/docker/tests/test_legacy_compat.py b/deploy/docker/tests/test_legacy_compat.pyindex 63a4e22f..2a76d6e2 100644--- a/deploy/docker/tests/test_legacy_compat.py+++ b/deploy/docker/tests/test_legacy_compat.py@@ -11,20 +11,17 @@ Behavioral tests for 0.9.x legacy-compatibility handling:never executed, and reported as status "ignored" with awarning when hooks are enabled; any hooks payload isstill refused (403) while hooks are disabled.- * compose file - the PID cap lives under deploy.resources.limits (not- pids_limit), which Compose v5 rejects alongside a- limits block.++(The compose PID-cap check lives in test_security_container_posture.py.)These exercise the running app via TestClient (no browser / Redis needed);crawl internals are stubbed where a handler would otherwise need a browser."""import base64-from pathlib import Pathfrom types import SimpleNamespaceimport pytest-import yamlfrom auth import create_access_token # noqa: E402@@ -46,7 +43,11 @@ class TestRootRedirect:assert r.headers["location"] == "/playground"def test_monitor_and_data_routes_stay_gated(self, stock_client):- assert stock_client.get("/monitor").status_code == 401+ # /monitor must not serve content without a token; a future+ # /monitor -> /dashboard redirect is fine (the target is UI-public),+ # so accept 401 or a redirect, never 200.+ r = stock_client.get("/monitor", follow_redirects=False)+ assert r.status_code in (401, 302, 307, 308)assert stock_client.get("/monitor/health").status_code == 401assert stock_client.post("/crawl", json={"urls": ["https://x"]}).status_code == 401@@ -100,6 +101,19 @@ class TestOutputPathWarning:assert r.status_code == 200assert "warning" not in r.json()+ @pytest.mark.parametrize("endpoint", ["/screenshot", "/pdf"])+ def test_output_path_is_never_written(self, stock_client, stub_crawler, endpoint, tmp_path):+ """Security tripwire for the 0.8.x arbitrary-write vuln: the handler+ runs for real here (only crawler/artifact store are stubbed), so any+ reintroduced write of body.output_path creates this file and fails."""+ target = tmp_path / "out.bin"+ r = stock_client.post(+ endpoint, json={"url": "https://example.com", "output_path": str(target)},+ headers=_bearer(),+ )+ assert r.status_code == 200+ assert not target.exists(), "output_path must never be written to disk"+# ───────────────────────── legacy hooks.code ─────────────────────────@@ -116,6 +130,19 @@ class TestLegacyHookCode:assert req.hooks.code == LEGACY_HOOKS["code"]assert req.hooks.hooks == []+ @pytest.mark.parametrize("code", [+ "def hook(): ...", # bare string+ {"before_goto": {"nested": "dict"}}, # non-string values+ ["a", "list"], # wrong container entirely+ ])+ def test_hook_code_accepts_any_legacy_shape(self, code):+ """The 0.8.x wire shape was never pinned; a 422 on a field we only+ report about would be worse than the silent drop it replaces."""+ from schemas import CrawlRequestWithHooks++ req = CrawlRequestWithHooks(urls=["https://x"], hooks={"code": code})+ assert req.hooks.code == code+# Any hooks payload is refused while hooks are disabled, but the detail# must not mislead legacy callers: enabling the flag would not run# hooks.code (removed in 0.9.0), so payloads carrying it get a removal@@ -191,27 +218,5 @@ class TestLegacyHookCode:assert "hooks" not in r.json()-# ───────────────────────── compose file ─────────────────────────---class TestComposeFile:- def test_pid_cap_lives_under_deploy_limits(self):- """Compose v5 rejects pids_limit next to deploy.resources.limits- ("can't set distinct values"); the cap must be expressed once, under- deploy.resources.limits.pids."""- import os-- override = os.environ.get("CRAWL4AI_COMPOSE_FILE")- if override:- compose_path = Path(override)- else:- here = Path(__file__).resolve()- if len(here.parents) < 4:- pytest.skip("not running from a repo checkout")- compose_path = here.parents[3] / "docker-compose.yml"- if not compose_path.exists():- pytest.skip(f"docker-compose.yml not found at {compose_path}")- doc = yaml.safe_load(compose_path.read_text())- base = doc["x-base-config"]- assert "pids_limit" not in base- assert base["deploy"]["resources"]["limits"]["pids"] == 512+# The compose PID-cap check lives in+# test_security_container_posture.py::test_pids_limit (YAML-parsed).diff --git a/deploy/docker/tests/test_security_2026_04.py b/deploy/docker/tests/test_security_2026_04.pyindex 4109de04..e015274b 100644--- a/deploy/docker/tests/test_security_2026_04.py+++ b/deploy/docker/tests/test_security_2026_04.py@@ -82,23 +82,29 @@ DEPLOY_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")# ============================================================================class TestOutputPathRemoved(unittest.TestCase):- """output_path is gone; the server owns paths via the artifact store.+ """output_path never reaches the filesystem; the server owns paths via the+ artifact store.The old string-only validate_output_path was bypassable (symlink/TOCTOU,sibling-prefix '...-evil') -> arbitrary write -> RCE. The fix is to never- accept a caller path at all. Behavioral artifact-store coverage (O_NOFOLLOW,- O_EXCL, hex id, TTL, quota) lives in test_security_artifact_store.py.+ use a caller path: output_path exists only as a deprecated no-op for 0.8.x+ compatibility and must stay that way. Artifact-store coverage lives in+ test_security_artifact_store.py; warning behavior in test_legacy_compat.py."""- def test_screenshot_request_has_no_output_path(self):+ def test_screenshot_output_path_is_deprecated_noop(self):sys.path.insert(0, DEPLOY_DIR)from schemas import ScreenshotRequest- self.assertNotIn("output_path", ScreenshotRequest.model_fields)+ field = ScreenshotRequest.model_fields["output_path"]+ self.assertTrue(field.deprecated, "output_path must be marked deprecated")+ self.assertIsNone(field.default)- def test_pdf_request_has_no_output_path(self):+ def test_pdf_output_path_is_deprecated_noop(self):sys.path.insert(0, DEPLOY_DIR)from schemas import PDFRequest- self.assertNotIn("output_path", PDFRequest.model_fields)+ field = PDFRequest.model_fields["output_path"]+ self.assertTrue(field.deprecated, "output_path must be marked deprecated")+ self.assertIsNone(field.default)def test_validate_output_path_deleted(self):sys.path.insert(0, DEPLOY_DIR)@@ -110,18 +116,21 @@ class TestOutputPathRemoved(unittest.TestCase):class TestPydanticPathValidator(unittest.TestCase):- """output_path (and its traversal validator) are gone entirely.+ """The traversal validator is gone entirely.Traversal rejection used to be the mitigation; the real fix is that no- caller path is accepted at all, so there is nothing to traverse. The- sandboxed artifact store owns all paths now.+ caller path is ever *used*, so there is nothing to traverse. output_path+ survives only as a deprecated no-op field (see TestOutputPathRemoved); the+ sandboxed artifact store owns all paths."""- def test_no_output_path_field_on_request_models(self):+ def test_output_path_is_inert_on_request_models(self):sys.path.insert(0, DEPLOY_DIR)from schemas import ScreenshotRequest, PDFRequest- self.assertNotIn("output_path", ScreenshotRequest.model_fields)- self.assertNotIn("output_path", PDFRequest.model_fields)+ for model in (ScreenshotRequest, PDFRequest):+ field = model.model_fields["output_path"]+ self.assertTrue(field.deprecated)+ self.assertIsNone(field.default)def test_traversal_validator_removed(self):# No reject_traversal validator should remain registered on the models.diff --git a/deploy/docker/tests/test_security_container_posture.py b/deploy/docker/tests/test_security_container_posture.pyindex 3399d6bd..eb5df615 100644--- a/deploy/docker/tests/test_security_container_posture.py+++ b/deploy/docker/tests/test_security_container_posture.py@@ -104,7 +104,14 @@ class TestCompose:assert "shm_size" in composedef test_pids_limit(self, compose):- assert "pids_limit" in compose+ # Parse, don't grep: a raw-text search matched the word "pids_limit"+ # inside a comment and guarded nothing. The cap lives under+ # deploy.resources.limits (not pids_limit) for Compose v5 compatibility.+ import yaml++ base = yaml.safe_load(compose)["x-base-config"]+ assert "pids_limit" not in base+ assert base["deploy"]["resources"]["limits"]["pids"] == 512def test_read_only_runtime_tmpfs_are_appuser_owned(self, compose):assert "/var/lib/redis:uid=999,gid=999,mode=0700" in compose