Official Crawl4AI Cloud: a hosted web scraping, extraction, and search API for AI agents. Free API key, 1,000 credits a month, no card. Fully open source: this repo is the Apache-2.0 crawler that powers it, and you can self-host it.
Python aiai-agentscrawl4ai-cloudcrawlerdata-extractionllmmarkdownmcpopen-sourceplaywrightpythonragscraperweb-crawlerweb-crawlingweb-scraping
fix(security): block PDF image-write fields from untrusted config bodies
ntohidi3273f756bbd0
diff --git a/crawl4ai/async_configs.py b/crawl4ai/async_configs.pyindex 3efb0d38..87500a90 100644--- a/crawl4ai/async_configs.py+++ b/crawl4ai/async_configs.py@@ -214,6 +214,29 @@ UNTRUSTED_FORBIDDEN_FIELDS = {"override_navigator", "magic", "process_in_browser", "shared_data","session_id",},+ # PDFContentScrapingStrategy has no scalar allowlist (it is kept for its+ # value fields), so its filesystem-write knobs must be blocked explicitly:+ # image_save_dir is an unconfined write destination and save_images_locally+ # turns the write on. extract_images stays allowed - it only returns the+ # image bytes base64-inline in the response, no disk write.+ "PDFContentScrapingStrategy": {+ "image_save_dir", "save_images_locally",+ },+}++# Field names that must NEVER be set from an untrusted body on ANY allowed+# type, checked regardless of per-type allowlist. This is the fail-closed+# backstop: the per-type maps above only cover BrowserConfig/CrawlerRunConfig,+# so a strategy that (now or in future) exposes a filesystem-write, code, or+# routing constructor arg is caught here even with no explicit allowlist.+# Presence => 400 (loud), matching js_code/extra_args behavior.+UNTRUSTED_GLOBAL_FORBIDDEN_FIELDS = {+ # filesystem write / read sinks+ "image_save_dir", "save_images_locally", "downloads_path", "user_data_dir",+ "output_path", "save_path", "file_path", "local_path", "storage_state",+ # code / command execution+ "js_code", "js_code_before_wait", "c4a_script", "init_scripts",+ "code", "command", "hook", "hooks",}# Scalar knobs an untrusted body MAY set, per class. A field not listed here is@@ -281,7 +304,7 @@ def _filter_untrusted_fields(type_name: str, params: dict) -> dict:allowlist = UNTRUSTED_FIELD_ALLOWLIST.get(type_name) # None => keep all non-forbiddenout = {}for key, value in params.items():- if key in forbidden:+ if key in UNTRUSTED_GLOBAL_FORBIDDEN_FIELDS or key in forbidden:raise UntrustedConfigError(f"field '{key}' is not permitted on {type_name} from an untrusted request")diff --git a/deploy/docker/tests/test_security_pdf_image_write.py b/deploy/docker/tests/test_security_pdf_image_write.pynew file mode 100644index 00000000..4f171ce4--- /dev/null+++ b/deploy/docker/tests/test_security_pdf_image_write.py@@ -0,0 +1,83 @@+#!/usr/bin/env python3+"""+Regression test for the PDF-image arbitrary-write via untrusted config body+(reported by sec-reex).++Root cause: PDFContentScrapingStrategy is an UNTRUSTED_ALLOWED_TYPE but had no+field allowlist, so _filter_untrusted_fields fell open and kept its+filesystem-write knobs (image_save_dir / save_images_locally). A request body+the API loads as UNTRUSTED could then steer the PDF image writer to an+attacker-chosen directory.++These tests hit the real deserialization path (from_serializable_dict /+CrawlerRunConfig.load with provenance=UNTRUSTED), not a copy.+"""++import os+import sys+import unittest++sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")))++from crawl4ai.async_configs import (+ CrawlerRunConfig,+ Provenance,+ UntrustedConfigError,+ from_serializable_dict,+)+++def _pdf_strategy(params: dict) -> dict:+ return {"type": "PDFContentScrapingStrategy", "params": params}+++class TestPdfImageWriteGate(unittest.TestCase):+ def test_untrusted_image_save_dir_is_rejected(self):+ """image_save_dir on an untrusted body must raise (loud 400), not pass."""+ body = _pdf_strategy({"save_images_locally": True, "image_save_dir": "/home/appuser/.ssh"})+ with self.assertRaises(UntrustedConfigError):+ from_serializable_dict(body, provenance=Provenance.UNTRUSTED)++ def test_untrusted_save_images_locally_is_rejected(self):+ """The boolean that turns the write on is forbidden on untrusted bodies too."""+ body = _pdf_strategy({"save_images_locally": True})+ with self.assertRaises(UntrustedConfigError):+ from_serializable_dict(body, provenance=Provenance.UNTRUSTED)++ def test_untrusted_rejected_when_nested_in_crawlerrunconfig(self):+ """The real server path: PDF strategy nested under CrawlerRunConfig.scraping_strategy."""+ body = {+ "type": "CrawlerRunConfig",+ "params": {+ "scraping_strategy": _pdf_strategy(+ {"save_images_locally": True, "image_save_dir": "/tmp/c4_oob"}+ )+ },+ }+ with self.assertRaises(UntrustedConfigError):+ CrawlerRunConfig.load(body, provenance=Provenance.UNTRUSTED)++ def test_untrusted_extract_images_still_allowed(self):+ """Negative control: extract_images returns bytes base64-inline, no disk+ write, so it must NOT be blocked (feature preserved over the API)."""+ body = _pdf_strategy({"extract_images": True})+ obj = from_serializable_dict(body, provenance=Provenance.UNTRUSTED)+ self.assertEqual(type(obj).__name__, "PDFContentScrapingStrategy")++ def test_trusted_path_unchanged(self):+ """No regression: the in-process SDK (TRUSTED) may still set image_save_dir."""+ body = _pdf_strategy({"save_images_locally": True, "image_save_dir": "/tmp/mine"})+ obj = from_serializable_dict(body, provenance=Provenance.TRUSTED)+ self.assertEqual(type(obj).__name__, "PDFContentScrapingStrategy")++ def test_global_backstop_covers_future_types(self):+ """The global forbidden set catches a write-arg regardless of per-type map.+ LXMLWebScrapingStrategy has no path arg today, but a smuggled downloads_path+ must still be rejected by the fail-closed backstop."""+ body = {"type": "LXMLWebScrapingStrategy", "params": {"downloads_path": "/etc"}}+ with self.assertRaises(UntrustedConfigError):+ from_serializable_dict(body, provenance=Provenance.UNTRUSTED)+++if __name__ == "__main__":+ unittest.main(verbosity=2)