Official Crawl4AI Cloud: a hosted web scraping, extraction, and search API for AI agents. Free API key, 1,000 credits a month, no card. Fully open source: this repo is the Apache-2.0 crawler that powers it, and you can self-host it.
Python aiai-agentscrawl4ai-cloudcrawlerdata-extractionllmmarkdownmcpopen-sourceplaywrightpythonragscraperweb-crawlerweb-crawlingweb-scraping
fix(docker): compose v5 compatibility, legacy-field warnings, playground error handling
Soham Kukreti8748b81bb64e
diff --git a/deploy/docker/entrypoint.sh b/deploy/docker/entrypoint.shindex b624a311..19d6766f 100644--- a/deploy/docker/entrypoint.sh+++ b/deploy/docker/entrypoint.sh@@ -31,6 +31,8 @@ else# No credential -> refuse to expose; serve loopback only.GUNICORN_BIND="127.0.0.1:${PORT}"echo "entrypoint: no CRAWL4AI_API_TOKEN set; binding loopback only (${GUNICORN_BIND})." >&2+ echo "entrypoint: WARNING: this is the CONTAINER's loopback - published ports (-p ${PORT}:${PORT}) will NOT work; connections from the host will be reset." >&2+ echo "entrypoint: to make the server reachable, set CRAWL4AI_API_TOKEN (docker run -e CRAWL4AI_API_TOKEN=..., or the .llm.env file with docker compose) and restart." >&2fiexport GUNICORN_BINDdiff --git a/deploy/docker/schemas.py b/deploy/docker/schemas.pyindex f066e5c8..b1065b5a 100644--- a/deploy/docker/schemas.py+++ b/deploy/docker/schemas.py@@ -42,6 +42,14 @@ class HookConfig(BaseModel):le=120,description="Timeout in seconds for each hook execution",)+ # Legacy 0.8.x field: inline-Python hook code, removed in 0.9.0 (it was an+ # exec()-based RCE surface). Captured here (instead of being dropped by+ # pydantic) solely so the server can tell the caller it was NOT executed;+ # it is never run.+ code: Optional[Dict[str, str]] = Field(+ default=None,+ description="REMOVED in 0.9.0: inline hook code is accepted for compatibility but never executed",+ )class Config:json_schema_extra = {diff --git a/deploy/docker/server.py b/deploy/docker/server.pyindex ce1403b8..86919ef0 100644--- a/deploy/docker/server.py+++ b/deploy/docker/server.py@@ -392,7 +392,7 @@ def _current_api_token() -> str:app.add_middleware(AuthGateMiddleware,token_provider=_current_api_token,- public_paths={HEALTH_PATH, "/token"},+ public_paths={HEALTH_PATH, "/token", "/"},public_prefixes=_UI_PREFIXES,)@@ -660,6 +660,17 @@ async def get_artifact(artifact_id: str, _td: Dict = Depends(token_dep)):# Screenshot endpoint+_OUTPUT_PATH_WARNING = (+ "output_path was removed in 0.9.0 and is ignored - no file was written. "+ "The result is stored server-side; fetch it with an authenticated "+ "GET /artifacts/{artifact_id}."+)++_HOOKS_CODE_WARNING = (+ "Inline hook code (hooks.code) was removed in 0.9.0 and was NOT executed. "+ "Use declarative hook actions instead (GET /hooks/info for the schema)."+)+@app.post("/screenshot")@limiter.limit(config["rate_limiting"]["default_limit"])@@ -684,7 +695,12 @@ async def generate_screenshot(raise HTTPException(500, detail=results[0].error_message or "Crawl failed")screenshot_data = results[0].screenshotart = _store_artifact("png", base64.b64decode(screenshot_data))- return {"success": True, "screenshot": screenshot_data, **art}+ response = {"success": True, "screenshot": screenshot_data, **art}+ # Legacy 0.8.x key, no longer a schema field: peek at the raw body+ # (already parsed and cached by FastAPI) to warn that it was ignored.+ if (await request.json()).get("output_path"):+ response["warning"] = _OUTPUT_PATH_WARNING+ return responseexcept HTTPException:raiseexcept Exception as e:@@ -719,7 +735,12 @@ async def generate_pdf(raise HTTPException(500, detail=results[0].error_message or "Crawl failed")pdf_data = results[0].pdfart = _store_artifact("pdf", pdf_data)- return {"success": True, "pdf": base64.b64encode(pdf_data).decode(), **art}+ response = {"success": True, "pdf": base64.b64encode(pdf_data).decode(), **art}+ # Legacy 0.8.x key, no longer a schema field: peek at the raw body+ # (already parsed and cached by FastAPI) to warn that it was ignored.+ if (await request.json()).get("output_path"):+ response["warning"] = _OUTPUT_PATH_WARNING+ return responseexcept HTTPException:raiseexcept Exception as e:@@ -889,7 +910,7 @@ async def crawl(raise HTTPException(400, f"Rejected config: {e}")if crawler_config.stream:return await stream_process(crawl_request=crawl_request)-+# Prepare hooks config if providedhooks_config = Noneif crawl_request.hooks:@@ -897,7 +918,7 @@ async def crawl('hooks': crawl_request.hooks.hooks,'timeout': crawl_request.hooks.timeout}-+results = await handle_crawl_request(urls=crawl_request.urls,browser_config=crawl_request.browser_config,@@ -906,6 +927,11 @@ async def crawl(hooks_config=hooks_config,crawler_configs=crawl_request.crawler_configs,)+ if crawl_request.hooks and crawl_request.hooks.code:+ hooks_resp = results.setdefault("hooks", {"attached": []})+ if not crawl_request.hooks.hooks:+ hooks_resp["status"] = "ignored"+ hooks_resp["warning"] = _HOOKS_CODE_WARNINGreturn JSONResponse(results)@@ -924,7 +950,7 @@ async def crawl_stream(return await stream_process(crawl_request=crawl_request)async def stream_process(crawl_request: CrawlRequestWithHooks):-+# Prepare hooks config if provided# Prepare hooks config if providedhooks_config = Noneif crawl_request.hooks:@@ -950,6 +976,8 @@ async def stream_process(crawl_request: CrawlRequestWithHooks):if hooks_info:import jsonheaders["X-Hooks-Status"] = json.dumps(hooks_info['status']['status'])+ if crawl_request.hooks and crawl_request.hooks.code:+ headers["X-Hooks-Warning"] = _HOOKS_CODE_WARNINGreturn StreamingResponse(stream_results(crawler, gen),diff --git a/deploy/docker/static/playground/index.html b/deploy/docker/static/playground/index.htmlindex e88c281a..cf72a303 100644--- a/deploy/docker/static/playground/index.html+++ b/deploy/docker/static/playground/index.html@@ -617,6 +617,17 @@}}+ // Build a useful error message from a failed HTTP response+ function httpErrorMessage(response, data) {+ let msg = (data && (data.detail || data.error)) || `HTTP ${response.status}`;+ if (response.status === 401) {+ msg += getToken()+ ? ' — token rejected; check the API token in the token bar (top right)'+ : ' — set your API token in the token bar (top right)';+ }+ return msg;+ }+// Generate code snippetsfunction generateSnippets(api, payload, method = 'POST') {// Python snippet@@ -751,7 +762,7 @@const time = Math.round(performance.now() - startTime);if (!response.ok) {updateStatus('error', time);- throw new Error(responseData.error || 'Request failed');+ throw new Error(httpErrorMessage(response, responseData));}updateStatus('success', time);document.querySelector('#response-content code').textContent = JSON.stringify(responseData, null, 2);@@ -765,6 +776,12 @@body: JSON.stringify(payload)});+ if (!response.ok) {+ const errData = await response.json().catch(() => ({}));+ updateStatus('error', Math.round(performance.now() - startTime));+ throw new Error(httpErrorMessage(response, errData));+ }+const reader = response.body.getReader();let text = '';let maxMemory = 0;@@ -809,7 +826,7 @@if (!response.ok) {updateStatus('error', time);- throw new Error(responseData.error || 'Request failed');+ throw new Error(httpErrorMessage(response, responseData));}updateStatus(diff --git a/deploy/docker/tests/test_legacy_compat.py b/deploy/docker/tests/test_legacy_compat.pynew file mode 100644index 00000000..b8b8402c--- /dev/null+++ b/deploy/docker/tests/test_legacy_compat.py@@ -0,0 +1,206 @@+"""+Behavioral tests for 0.9.x legacy-compatibility handling:++ * root redirect - "/" is public and redirects to /playground instead of+ dying in the auth gate with a bare 401; /monitor and the+ data routes stay gated.+ * output_path - /screenshot and /pdf still accept the 0.8.x output_path+ field but return a warning saying no file was written,+ instead of silently dropping it.+ * legacy hooks - hooks.code (removed 0.8.x inline Python) is captured,+ never executed, and reported as status "ignored" with a+ warning when hooks are enabled; any hooks payload is+ still refused (403) while hooks are disabled.+ * compose file - the PID cap lives under deploy.resources.limits (not+ pids_limit), which Compose v5 rejects alongside a+ limits block.++These exercise the running app via TestClient (no browser / Redis needed);+crawl internals are stubbed where a handler would otherwise need a browser.+"""++import base64+from pathlib import Path+from types import SimpleNamespace++import pytest+import yaml++from auth import create_access_token # noqa: E402+++def _bearer() -> dict:+ return {"Authorization": f"Bearer {create_access_token({'sub': '[email protected]'}, scope='data')}"}+++# ───────────────────────── root redirect ─────────────────────────+++class TestRootRedirect:+ def test_root_is_public_and_redirects_to_playground(self, stock_client):+ r = stock_client.get("/", follow_redirects=False)+ assert r.status_code in (302, 307), (+ f"GET / returned {r.status_code}; expected a redirect. The auth "+ f"gate must allow the exact path '/' so the redirect route runs."+ )+ assert r.headers["location"] == "/playground"++ def test_monitor_and_data_routes_stay_gated(self, stock_client):+ assert stock_client.get("/monitor").status_code == 401+ assert stock_client.get("/monitor/health").status_code == 401+ assert stock_client.post("/crawl", json={"urls": ["https://x"]}).status_code == 401+++# ───────────────────────── output_path warning ─────────────────────────+++def stub_crawler(server_module, monkeypatch):+ """Stub the crawler pool + artifact store so /screenshot and /pdf run+ without a browser. Returns the fake artifact dict for assertions."""+ art = {"artifact_id": "a1", "url": "/artifacts/a1", "mime": "x", "size": 1}+ png_b64 = base64.b64encode(b"fake-png").decode()++ fake_result = SimpleNamespace(success=True, screenshot=png_b64, pdf=b"fake-pdf")++ class _FakeCrawler:+ async def arun(self, url, config):+ return [fake_result]++ async def fake_get_crawler(cfg):+ return _FakeCrawler()++ async def fake_release_crawler(crawler):+ pass++ monkeypatch.setattr(server_module, "get_crawler", fake_get_crawler)+ monkeypatch.setattr(server_module, "release_crawler", fake_release_crawler)+ monkeypatch.setattr(server_module, "_store_artifact", lambda kind, data: dict(art))+ return art+++class TestOutputPathWarning:+ @pytest.mark.parametrize("endpoint,payload_key", [("/screenshot", "screenshot"), ("/pdf", "pdf")])+ def test_output_path_accepted_with_warning(self, stock_client, stub_crawler, endpoint, payload_key):+ r = stock_client.post(+ endpoint,+ json={"url": "https://example.com", "output_path": "/tmp/x.bin"},+ headers=_bearer(),+ )+ assert r.status_code == 200+ body = r.json()+ assert body["success"] is True+ assert "warning" in body, "output_path must not be silently dropped"+ assert "no file was written" in body["warning"]+ assert body["artifact_id"] == stub_crawler["artifact_id"]++ @pytest.mark.parametrize("endpoint", ["/screenshot", "/pdf"])+ def test_no_warning_without_output_path(self, stock_client, stub_crawler, endpoint):+ r = stock_client.post(endpoint, json={"url": "https://example.com"}, headers=_bearer())+ assert r.status_code == 200+ assert "warning" not in r.json()+++# ───────────────────────── legacy hooks.code ─────────────────────────++LEGACY_HOOKS = {"code": {"before_goto": "async def hook(p, c, u, **kw): return p"}}+DECLARATIVE_HOOKS = {"hooks": [{"action": "scroll_to_bottom", "params": {"max_steps": 2}}]}+++class TestLegacyHookCode:+ def test_hook_config_captures_code_field(self):+ """The legacy field must be parsed (not dropped) so it can be reported."""+ from schemas import CrawlRequestWithHooks++ req = CrawlRequestWithHooks(urls=["https://x"], hooks=LEGACY_HOOKS)+ assert req.hooks.code == LEGACY_HOOKS["code"]+ assert req.hooks.hooks == []++ @pytest.mark.parametrize("hooks_payload", [LEGACY_HOOKS, DECLARATIVE_HOOKS])+ def test_any_hooks_payload_403_when_disabled(self, server_module, monkeypatch, stock_client, hooks_payload):+ monkeypatch.setattr(server_module, "HOOKS_ENABLED", False)+ r = stock_client.post(+ "/crawl",+ json={"urls": ["https://example.com"], "hooks": hooks_payload},+ headers=_bearer(),+ )+ assert r.status_code == 403++ def _stub_crawl(self, server_module, monkeypatch, results):+ async def fake_handle_crawl_request(**kwargs):+ return dict(results)++ monkeypatch.setattr(server_module, "handle_crawl_request", fake_handle_crawl_request)++ def test_legacy_code_warned_and_ignored_when_enabled(self, server_module, monkeypatch, stock_client):+ monkeypatch.setattr(server_module, "HOOKS_ENABLED", True)+ # Empty declarative specs -> api.py reports a vacuous success+ self._stub_crawl(+ server_module, monkeypatch,+ {"success": True, "results": [{"success": True}],+ "hooks": {"status": "success", "attached": []}},+ )+ r = stock_client.post(+ "/crawl",+ json={"urls": ["https://example.com"], "hooks": LEGACY_HOOKS},+ headers=_bearer(),+ )+ assert r.status_code == 200+ hooks = r.json()["hooks"]+ assert hooks["status"] == "ignored", "vacuous 'success' must be rewritten"+ assert hooks["attached"] == []+ assert "NOT executed" in hooks["warning"]++ def test_mixed_request_keeps_declarative_status_and_warns(self, server_module, monkeypatch, stock_client):+ monkeypatch.setattr(server_module, "HOOKS_ENABLED", True)+ self._stub_crawl(+ server_module, monkeypatch,+ {"success": True, "results": [{"success": True}],+ "hooks": {"status": "success", "attached": ["before_retrieve_html"]}},+ )+ r = stock_client.post(+ "/crawl",+ json={"urls": ["https://example.com"],+ "hooks": {**DECLARATIVE_HOOKS, **LEGACY_HOOKS}},+ headers=_bearer(),+ )+ assert r.status_code == 200+ hooks = r.json()["hooks"]+ assert hooks["status"] == "success", "real declarative execution must not be relabeled"+ assert hooks["attached"] == ["before_retrieve_html"]+ assert "NOT executed" in hooks["warning"]++ def test_no_hooks_response_untouched(self, server_module, monkeypatch, stock_client):+ self._stub_crawl(+ server_module, monkeypatch,+ {"success": True, "results": [{"success": True}]},+ )+ r = stock_client.post("/crawl", json={"urls": ["https://example.com"]}, headers=_bearer())+ assert r.status_code == 200+ assert "hooks" not in r.json()+++# ───────────────────────── compose file ─────────────────────────+++class TestComposeFile:+ def test_pid_cap_lives_under_deploy_limits(self):+ """Compose v5 rejects pids_limit next to deploy.resources.limits+ ("can't set distinct values"); the cap must be expressed once, under+ deploy.resources.limits.pids."""+ import os++ override = os.environ.get("CRAWL4AI_COMPOSE_FILE")+ if override:+ compose_path = Path(override)+ else:+ here = Path(__file__).resolve()+ if len(here.parents) < 4:+ pytest.skip("not running from a repo checkout")+ compose_path = here.parents[3] / "docker-compose.yml"+ if not compose_path.exists():+ pytest.skip(f"docker-compose.yml not found at {compose_path}")+ doc = yaml.safe_load(compose_path.read_text())+ base = doc["x-base-config"]+ assert "pids_limit" not in base+ assert base["deploy"]["resources"]["limits"]["pids"] == 512diff --git a/docker-compose.yml b/docker-compose.ymlindex 1cd87ad9..5939b332 100644--- a/docker-compose.yml+++ b/docker-compose.yml@@ -24,7 +24,6 @@ x-base-config: &base-config- ALLsecurity_opt:- no-new-privileges:true- pids_limit: 512# Read-only root filesystem; only these paths are writable (tmpfs).read_only: truetmpfs:@@ -39,6 +38,9 @@ x-base-config: &base-configresources:limits:memory: 4G+ # PID cap; expressed here (not as pids_limit) so the file stays valid+ # on Compose v5+, which rejects pids_limit alongside a limits block.+ pids: 512reservations:memory: 1Grestart: unless-stopped