Skip to content

Official Crawl4AI Cloud: a hosted web scraping, extraction, and search API for AI agents. Free API key, 1,000 credits a month, no card. Fully open source: this repo is the Apache-2.0 crawler that powers it, and you can self-host it.

fix(docker): declare output_path as deprecated no-op, harden legacy-compat tests

Soham Kukretia01442d3a5a6
diff --git a/deploy/docker/entrypoint.sh b/deploy/docker/entrypoint.sh
index 19d6766f..3d003791 100644
--- a/deploy/docker/entrypoint.sh
+++ b/deploy/docker/entrypoint.sh
@@ -32,7 +32,7 @@ else
GUNICORN_BIND="127.0.0.1:${PORT}"
echo "entrypoint: no CRAWL4AI_API_TOKEN set; binding loopback only (${GUNICORN_BIND})." >&2
echo "entrypoint: WARNING: this is the CONTAINER's loopback - published ports (-p ${PORT}:${PORT}) will NOT work; connections from the host will be reset." >&2
- echo "entrypoint: to make the server reachable, set CRAWL4AI_API_TOKEN (docker run -e CRAWL4AI_API_TOKEN=..., or the .llm.env file with docker compose) and restart." >&2
+ echo "entrypoint: to make the server reachable, set CRAWL4AI_API_TOKEN (docker run -e CRAWL4AI_API_TOKEN=..., or the .llm.env file with docker compose), then restart. (If you enabled security.jwt_enabled in a custom config.yml, set CRAWL4AI_JWT_ENABLED=true instead.)" >&2
fi
export GUNICORN_BIND
diff --git a/deploy/docker/schemas.py b/deploy/docker/schemas.py
index b1065b5a..978f2199 100644
--- a/deploy/docker/schemas.py
+++ b/deploy/docker/schemas.py
@@ -46,7 +46,7 @@ class HookConfig(BaseModel):
# exec()-based RCE surface). Captured here (instead of being dropped by
# pydantic) solely so the server can tell the caller it was NOT executed;
# it is never run.
- code: Optional[Dict[str, str]] = Field(
+ code: Optional[Any] = Field(
default=None,
description="REMOVED in 0.9.0: inline hook code is accepted for compatibility but never executed",
)
@@ -92,14 +92,29 @@ class ScreenshotRequest(BaseModel):
url: str
screenshot_wait_for: Optional[float] = 2
wait_for_images: Optional[bool] = False
- # output_path removed: callers never name a filesystem path (it was an
- # arbitrary-write -> RCE vector). The server writes to the sandboxed
- # artifact store and returns an opaque artifact_id.
+ # Deprecated no-op: caller paths were an arbitrary-write -> RCE vector.
+ # Never written; the server stores results in the artifact store instead.
+ output_path: Optional[str] = Field(
+ default=None,
+ deprecated=True,
+ description=(
+ "REMOVED in 0.9.0 and ignored - no file is written. Results are "
+ "stored server-side; fetch via GET /artifacts/{artifact_id}."
+ ),
+ )
class PDFRequest(BaseModel):
url: str
- # output_path removed (see ScreenshotRequest).
+ # output_path deprecated no-op (see ScreenshotRequest).
+ output_path: Optional[str] = Field(
+ default=None,
+ deprecated=True,
+ description=(
+ "REMOVED in 0.9.0 and ignored - no file is written. Results are "
+ "stored server-side; fetch via GET /artifacts/{artifact_id}."
+ ),
+ )
class JSEndpointRequest(BaseModel):
diff --git a/deploy/docker/server.py b/deploy/docker/server.py
index e6f63db2..eb0fddad 100644
--- a/deploy/docker/server.py
+++ b/deploy/docker/server.py
@@ -701,6 +701,9 @@ async def generate_screenshot(
sandboxed artifact store; the response includes an `artifact_id` and a `url` to fetch it.
"""
validate_url_scheme(body.url)
+ # model_dump, not attribute access: the field is marked deprecated and
+ # reading the attribute emits DeprecationWarning on every request.
+ legacy_output_path = body.model_dump(include={"output_path"}).get("output_path")
crawler = None
try:
cfg = CrawlerRunConfig(screenshot=True, screenshot_wait_for=body.screenshot_wait_for, wait_for_images=body.wait_for_images)
@@ -711,9 +714,7 @@ async def generate_screenshot(
screenshot_data = results[0].screenshot
art = _store_artifact("png", base64.b64decode(screenshot_data))
response = {"success": True, "screenshot": screenshot_data, **art}
- # Legacy 0.8.x key, no longer a schema field: peek at the raw body
- # (already parsed and cached by FastAPI) to warn that it was ignored.
- if (await request.json()).get("output_path"):
+ if legacy_output_path:
response["warning"] = _OUTPUT_PATH_WARNING
return response
except HTTPException:
@@ -741,6 +742,9 @@ async def generate_pdf(
sandboxed artifact store; the response includes an `artifact_id` and a `url` to fetch it.
"""
validate_url_scheme(body.url)
+ # model_dump, not attribute access: the field is marked deprecated and
+ # reading the attribute emits DeprecationWarning on every request.
+ legacy_output_path = body.model_dump(include={"output_path"}).get("output_path")
crawler = None
try:
cfg = CrawlerRunConfig(pdf=True)
@@ -751,9 +755,7 @@ async def generate_pdf(
pdf_data = results[0].pdf
art = _store_artifact("pdf", pdf_data)
response = {"success": True, "pdf": base64.b64encode(pdf_data).decode(), **art}
- # Legacy 0.8.x key, no longer a schema field: peek at the raw body
- # (already parsed and cached by FastAPI) to warn that it was ignored.
- if (await request.json()).get("output_path"):
+ if legacy_output_path:
response["warning"] = _OUTPUT_PATH_WARNING
return response
except HTTPException:
@@ -943,7 +945,7 @@ async def crawl(
crawler_configs=crawl_request.crawler_configs,
)
if crawl_request.hooks and crawl_request.hooks.code:
- hooks_resp = results.setdefault("hooks", {"attached": []})
+ hooks_resp = results.setdefault("hooks", {"status": "ignored", "attached": []})
if not crawl_request.hooks.hooks:
hooks_resp["status"] = "ignored"
hooks_resp["warning"] = _HOOKS_CODE_WARNING
diff --git a/deploy/docker/static/playground/index.html b/deploy/docker/static/playground/index.html
index cf72a303..ba368b76 100644
--- a/deploy/docker/static/playground/index.html
+++ b/deploy/docker/static/playground/index.html
@@ -619,7 +619,15 @@
// Build a useful error message from a failed HTTP response
function httpErrorMessage(response, data) {
- let msg = (data && (data.detail || data.error)) || `HTTP ${response.status}`;
+ let detail = data && (data.detail || data.error);
+ // FastAPI 422s send detail as an array of error objects; other
+ // non-strings would render as [object Object].
+ if (Array.isArray(detail)) {
+ detail = detail.map(e => e && e.msg ? `${(e.loc || []).join('.')}: ${e.msg}` : JSON.stringify(e)).join('; ');
+ } else if (detail && typeof detail !== 'string') {
+ detail = JSON.stringify(detail);
+ }
+ let msg = detail || `HTTP ${response.status}`;
if (response.status === 401) {
msg += getToken()
? ' — token rejected; check the API token in the token bar (top right)'
@@ -758,7 +766,7 @@
method: 'GET',
headers: { 'Accept': 'application/json' }
});
- responseData = await response.json();
+ responseData = await response.json().catch(() => ({}));
const time = Math.round(performance.now() - startTime);
if (!response.ok) {
updateStatus('error', time);
@@ -821,7 +829,7 @@
body: JSON.stringify(payload)
});
- responseData = await response.json();
+ responseData = await response.json().catch(() => ({}));
const time = Math.round(performance.now() - startTime);
if (!response.ok) {
diff --git a/deploy/docker/tests/requirements.txt b/deploy/docker/tests/requirements.txt
index 5f7a842f..b0206020 100644
--- a/deploy/docker/tests/requirements.txt
+++ b/deploy/docker/tests/requirements.txt
@@ -1,2 +1,3 @@
httpx>=0.25.0
docker>=7.0.0
+pyyaml>=6.0
diff --git a/deploy/docker/tests/test_legacy_compat.py b/deploy/docker/tests/test_legacy_compat.py
index 63a4e22f..2a76d6e2 100644
--- a/deploy/docker/tests/test_legacy_compat.py
+++ b/deploy/docker/tests/test_legacy_compat.py
@@ -11,20 +11,17 @@ Behavioral tests for 0.9.x legacy-compatibility handling:
never executed, and reported as status "ignored" with a
warning when hooks are enabled; any hooks payload is
still refused (403) while hooks are disabled.
- * compose file - the PID cap lives under deploy.resources.limits (not
- pids_limit), which Compose v5 rejects alongside a
- limits block.
+
+(The compose PID-cap check lives in test_security_container_posture.py.)
These exercise the running app via TestClient (no browser / Redis needed);
crawl internals are stubbed where a handler would otherwise need a browser.
"""
import base64
-from pathlib import Path
from types import SimpleNamespace
import pytest
-import yaml
from auth import create_access_token # noqa: E402
@@ -46,7 +43,11 @@ class TestRootRedirect:
assert r.headers["location"] == "/playground"
def test_monitor_and_data_routes_stay_gated(self, stock_client):
- assert stock_client.get("/monitor").status_code == 401
+ # /monitor must not serve content without a token; a future
+ # /monitor -> /dashboard redirect is fine (the target is UI-public),
+ # so accept 401 or a redirect, never 200.
+ r = stock_client.get("/monitor", follow_redirects=False)
+ assert r.status_code in (401, 302, 307, 308)
assert stock_client.get("/monitor/health").status_code == 401
assert stock_client.post("/crawl", json={"urls": ["https://x"]}).status_code == 401
@@ -100,6 +101,19 @@ class TestOutputPathWarning:
assert r.status_code == 200
assert "warning" not in r.json()
+ @pytest.mark.parametrize("endpoint", ["/screenshot", "/pdf"])
+ def test_output_path_is_never_written(self, stock_client, stub_crawler, endpoint, tmp_path):
+ """Security tripwire for the 0.8.x arbitrary-write vuln: the handler
+ runs for real here (only crawler/artifact store are stubbed), so any
+ reintroduced write of body.output_path creates this file and fails."""
+ target = tmp_path / "out.bin"
+ r = stock_client.post(
+ endpoint, json={"url": "https://example.com", "output_path": str(target)},
+ headers=_bearer(),
+ )
+ assert r.status_code == 200
+ assert not target.exists(), "output_path must never be written to disk"
+
# ───────────────────────── legacy hooks.code ─────────────────────────
@@ -116,6 +130,19 @@ class TestLegacyHookCode:
assert req.hooks.code == LEGACY_HOOKS["code"]
assert req.hooks.hooks == []
+ @pytest.mark.parametrize("code", [
+ "def hook(): ...", # bare string
+ {"before_goto": {"nested": "dict"}}, # non-string values
+ ["a", "list"], # wrong container entirely
+ ])
+ def test_hook_code_accepts_any_legacy_shape(self, code):
+ """The 0.8.x wire shape was never pinned; a 422 on a field we only
+ report about would be worse than the silent drop it replaces."""
+ from schemas import CrawlRequestWithHooks
+
+ req = CrawlRequestWithHooks(urls=["https://x"], hooks={"code": code})
+ assert req.hooks.code == code
+
# Any hooks payload is refused while hooks are disabled, but the detail
# must not mislead legacy callers: enabling the flag would not run
# hooks.code (removed in 0.9.0), so payloads carrying it get a removal
@@ -191,27 +218,5 @@ class TestLegacyHookCode:
assert "hooks" not in r.json()
-# ───────────────────────── compose file ─────────────────────────
-
-
-class TestComposeFile:
- def test_pid_cap_lives_under_deploy_limits(self):
- """Compose v5 rejects pids_limit next to deploy.resources.limits
- ("can't set distinct values"); the cap must be expressed once, under
- deploy.resources.limits.pids."""
- import os
-
- override = os.environ.get("CRAWL4AI_COMPOSE_FILE")
- if override:
- compose_path = Path(override)
- else:
- here = Path(__file__).resolve()
- if len(here.parents) < 4:
- pytest.skip("not running from a repo checkout")
- compose_path = here.parents[3] / "docker-compose.yml"
- if not compose_path.exists():
- pytest.skip(f"docker-compose.yml not found at {compose_path}")
- doc = yaml.safe_load(compose_path.read_text())
- base = doc["x-base-config"]
- assert "pids_limit" not in base
- assert base["deploy"]["resources"]["limits"]["pids"] == 512
+# The compose PID-cap check lives in
+# test_security_container_posture.py::test_pids_limit (YAML-parsed).
diff --git a/deploy/docker/tests/test_security_2026_04.py b/deploy/docker/tests/test_security_2026_04.py
index 4109de04..e015274b 100644
--- a/deploy/docker/tests/test_security_2026_04.py
+++ b/deploy/docker/tests/test_security_2026_04.py
@@ -82,23 +82,29 @@ DEPLOY_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")
# ============================================================================
class TestOutputPathRemoved(unittest.TestCase):
- """output_path is gone; the server owns paths via the artifact store.
+ """output_path never reaches the filesystem; the server owns paths via the
+ artifact store.
The old string-only validate_output_path was bypassable (symlink/TOCTOU,
sibling-prefix '...-evil') -> arbitrary write -> RCE. The fix is to never
- accept a caller path at all. Behavioral artifact-store coverage (O_NOFOLLOW,
- O_EXCL, hex id, TTL, quota) lives in test_security_artifact_store.py.
+ use a caller path: output_path exists only as a deprecated no-op for 0.8.x
+ compatibility and must stay that way. Artifact-store coverage lives in
+ test_security_artifact_store.py; warning behavior in test_legacy_compat.py.
"""
- def test_screenshot_request_has_no_output_path(self):
+ def test_screenshot_output_path_is_deprecated_noop(self):
sys.path.insert(0, DEPLOY_DIR)
from schemas import ScreenshotRequest
- self.assertNotIn("output_path", ScreenshotRequest.model_fields)
+ field = ScreenshotRequest.model_fields["output_path"]
+ self.assertTrue(field.deprecated, "output_path must be marked deprecated")
+ self.assertIsNone(field.default)
- def test_pdf_request_has_no_output_path(self):
+ def test_pdf_output_path_is_deprecated_noop(self):
sys.path.insert(0, DEPLOY_DIR)
from schemas import PDFRequest
- self.assertNotIn("output_path", PDFRequest.model_fields)
+ field = PDFRequest.model_fields["output_path"]
+ self.assertTrue(field.deprecated, "output_path must be marked deprecated")
+ self.assertIsNone(field.default)
def test_validate_output_path_deleted(self):
sys.path.insert(0, DEPLOY_DIR)
@@ -110,18 +116,21 @@ class TestOutputPathRemoved(unittest.TestCase):
class TestPydanticPathValidator(unittest.TestCase):
- """output_path (and its traversal validator) are gone entirely.
+ """The traversal validator is gone entirely.
Traversal rejection used to be the mitigation; the real fix is that no
- caller path is accepted at all, so there is nothing to traverse. The
- sandboxed artifact store owns all paths now.
+ caller path is ever *used*, so there is nothing to traverse. output_path
+ survives only as a deprecated no-op field (see TestOutputPathRemoved); the
+ sandboxed artifact store owns all paths.
"""
- def test_no_output_path_field_on_request_models(self):
+ def test_output_path_is_inert_on_request_models(self):
sys.path.insert(0, DEPLOY_DIR)
from schemas import ScreenshotRequest, PDFRequest
- self.assertNotIn("output_path", ScreenshotRequest.model_fields)
- self.assertNotIn("output_path", PDFRequest.model_fields)
+ for model in (ScreenshotRequest, PDFRequest):
+ field = model.model_fields["output_path"]
+ self.assertTrue(field.deprecated)
+ self.assertIsNone(field.default)
def test_traversal_validator_removed(self):
# No reject_traversal validator should remain registered on the models.
diff --git a/deploy/docker/tests/test_security_container_posture.py b/deploy/docker/tests/test_security_container_posture.py
index 3399d6bd..eb5df615 100644
--- a/deploy/docker/tests/test_security_container_posture.py
+++ b/deploy/docker/tests/test_security_container_posture.py
@@ -104,7 +104,14 @@ class TestCompose:
assert "shm_size" in compose
def test_pids_limit(self, compose):
- assert "pids_limit" in compose
+ # Parse, don't grep: a raw-text search matched the word "pids_limit"
+ # inside a comment and guarded nothing. The cap lives under
+ # deploy.resources.limits (not pids_limit) for Compose v5 compatibility.
+ import yaml
+
+ base = yaml.safe_load(compose)["x-base-config"]
+ assert "pids_limit" not in base
+ assert base["deploy"]["resources"]["limits"]["pids"] == 512
def test_read_only_runtime_tmpfs_are_appuser_owned(self, compose):
assert "/var/lib/redis:uid=999,gid=999,mode=0700" in compose