Official Crawl4AI Cloud: a hosted web scraping, extraction, and search API for AI agents. Free API key, 1,000 credits a month, no card. Fully open source: this repo is the Apache-2.0 crawler that powers it, and you can self-host it.
Python aiai-agentscrawl4ai-cloudcrawlerdata-extractionllmmarkdownmcpopen-sourceplaywrightpythonragscraperweb-crawlerweb-crawlingweb-scraping
fix(security): stop {"type":"dict","value":X} laundering typed objects past the untrusted gate
ntohidice62ae674c34
diff --git a/crawl4ai/async_configs.py b/crawl4ai/async_configs.pyindex a6691e65..340b3f9b 100644--- a/crawl4ai/async_configs.py+++ b/crawl4ai/async_configs.py@@ -449,6 +449,21 @@ def to_serializable_dict(obj: Any, ignore_default_value : bool = False):return str(obj)+def _is_typed_shape(data: Any) -> bool:+ """True for the dict shapes to_serializable_dict() emits:+ {"type": "<ClassName>", "params": {...}} or {"type": "dict", "value": {...}}.++ Plain business dicts that happen to carry a "type" key (JSON-Schema+ fragments, JsonCss field specs like {"type": "text", "name": "..."}) have+ neither "params" nor "value" and are not typed shapes.+ """+ return (+ isinstance(data, dict)+ and "type" in data+ and ("params" in data or (data["type"] == "dict" and "value" in data))+ )++def from_serializable_dict(data: Any, provenance: "Provenance" = None) -> Any:"""Recursively convert a serializable dictionary back to an object instance.@@ -474,14 +489,23 @@ def from_serializable_dict(data: Any, provenance: "Provenance" = None) -> Any:# carry a "type" key (e.g. JSON-Schema fragments, JsonCss field specs like# {"type": "text", "name": "..."}) have neither "params" nor "value" and# must fall through to the raw-dict path below so they are passed as data.- if (- isinstance(data, dict)- and "type" in data- and ("params" in data or (data["type"] == "dict" and "value" in data))- ):+ if _is_typed_shape(data):# Handle plain dictionariesif data["type"] == "dict" and "value" in data:- return {k: from_serializable_dict(v, provenance) for k, v in data["value"].items()}+ unwrapped = {+ k: from_serializable_dict(v, provenance) for k, v in data["value"].items()+ }+ # The unwrap above recurses over .items(), so data["value"] is never+ # seen as a whole typed object and the type gate below never fires+ # for it. Without this check an untrusted body launders a forbidden+ # type past the gate by wrapping it:+ # {"type": "dict", "value": {"type": "LLMConfig", "params": {...}}}+ if provenance == Provenance.UNTRUSTED and _is_typed_shape(unwrapped):+ raise UntrustedConfigError(+ "an untrusted request may not wrap a typed object in "+ "{'type': 'dict', 'value': ...}"+ )+ return unwrapped# Security: only allow known-safe types to be deserialized.# Unknown types (e.g. logging.Logger serialized by older clients) are@@ -988,11 +1012,13 @@ class BrowserConfig:)@staticmethod- def from_kwargs(kwargs: dict) -> "BrowserConfig":+ def from_kwargs(kwargs: dict, provenance: "Provenance" = None) -> "BrowserConfig":# Auto-deserialize any dict values that use the {"type": ..., "params": ...}# serialization format (e.g. from JSON API requests or dump()/load() roundtrips).kwargs = {- k: from_serializable_dict(v) if isinstance(v, dict) and "type" in v else v+ k: from_serializable_dict(v, provenance)+ if isinstance(v, dict) and "type" in v+ else vfor k, v in kwargs.items()}# Only pass keys present in kwargs so that __init__ defaults (and@@ -1078,7 +1104,7 @@ class BrowserConfig:# so a body that sends bare kwargs (no {type,params}) cannot bypass it.if provenance == Provenance.UNTRUSTED and isinstance(config, dict):config = _enforce_untrusted("BrowserConfig", config)- return BrowserConfig.from_kwargs(config)+ return BrowserConfig.from_kwargs(config, provenance)def set_nstproxy(self,@@ -2119,12 +2145,14 @@ class CrawlerRunConfig():super().__setattr__(name, value)@staticmethod- def from_kwargs(kwargs: dict) -> "CrawlerRunConfig":+ def from_kwargs(kwargs: dict, provenance: "Provenance" = None) -> "CrawlerRunConfig":# Auto-deserialize any dict values that use the {"type": ..., "params": ...}# serialization format (e.g. from JSON API requests or dump()/load() roundtrips).# This covers markdown_generator, extraction_strategy, content_filter, etc.kwargs = {- k: from_serializable_dict(v) if isinstance(v, dict) and "type" in v else v+ k: from_serializable_dict(v, provenance)+ if isinstance(v, dict) and "type" in v+ else vfor k, v in kwargs.items()}# Only pass keys present in kwargs so that __init__ defaults (and@@ -2147,7 +2175,7 @@ class CrawlerRunConfig():return configif provenance == Provenance.UNTRUSTED and isinstance(config, dict):config = _enforce_untrusted("CrawlerRunConfig", config)- return CrawlerRunConfig.from_kwargs(config)+ return CrawlerRunConfig.from_kwargs(config, provenance)def to_dict(self):return {diff --git a/tests/unit/test_config_provenance.py b/tests/unit/test_config_provenance.pynew file mode 100644index 00000000..49246a8e--- /dev/null+++ b/tests/unit/test_config_provenance.py@@ -0,0 +1,118 @@+"""Untrusted config provenance gate.++Regression test for the {"type": "dict", "value": <typed object>} laundering+bypass: wrapping a forbidden typed object in a plain-dict envelope hid it from+the type gate, and from_kwargs then re-deserialized it as TRUSTED, which+resolves api_token="env:NAME" through os.getenv.+"""+import pytest++from crawl4ai.async_configs import (+ BrowserConfig,+ CrawlerRunConfig,+ Provenance,+ UntrustedConfigError,+)++UNTRUSTED = Provenance.UNTRUSTED+++def _llm_strategy(env_var):+ return {+ "type": "LLMExtractionStrategy",+ "params": {+ "llm_config": {+ "type": "dict",+ "value": {"type": "LLMConfig", "params": {"api_token": f"env:{env_var}"}},+ }+ },+ }+++def test_forbidden_type_refused_directly():+ data = {"type": "CrawlerRunConfig", "params": {"extraction_strategy": _llm_strategy("SECRET_KEY")}}+ with pytest.raises(UntrustedConfigError):+ CrawlerRunConfig.load(data, provenance=UNTRUSTED)+++def test_forbidden_type_refused_when_wrapped(monkeypatch):+ """The bypass: the whole strategy hidden behind {"type":"dict","value":...}."""+ monkeypatch.setenv("SECRET_KEY", "canary-secret-key-value")+ data = {+ "type": "CrawlerRunConfig",+ "extraction_strategy": {"type": "dict", "value": _llm_strategy("SECRET_KEY")},+ }+ with pytest.raises(UntrustedConfigError):+ CrawlerRunConfig.load(data, provenance=UNTRUSTED)+++def test_forbidden_type_refused_when_wrapped_in_params(monkeypatch):+ monkeypatch.setenv("OPENAI_API_KEY", "sk-canary-9f3a")+ data = {+ "type": "CrawlerRunConfig",+ "params": {"extraction_strategy": {"type": "dict", "value": _llm_strategy("OPENAI_API_KEY")}},+ }+ with pytest.raises(UntrustedConfigError):+ CrawlerRunConfig.load(data, provenance=UNTRUSTED)+++def test_browser_config_wrapped_proxy_refused():+ """Same envelope, other config class, other forbidden type."""+ data = {+ "type": "BrowserConfig",+ "params": {+ "proxy_config": {+ "type": "dict",+ "value": {"type": "ProxyConfig", "params": {"server": "http://evil:8080"}},+ }+ },+ }+ with pytest.raises(UntrustedConfigError):+ BrowserConfig.load(data, provenance=UNTRUSTED)+++def test_trusted_load_is_unchanged(monkeypatch):+ """TRUSTED (SDK) callers keep the old behavior: no raise, and the wrapped+ envelope still unwraps to the same plain dict it did before the fix."""+ monkeypatch.setenv("CRAWL4AI_TEST_TOKEN", "tok-123")+ cfg = CrawlerRunConfig.load(+ {"type": "CrawlerRunConfig", "params": {"extraction_strategy": _llm_strategy("CRAWL4AI_TEST_TOKEN")}}+ )+ assert cfg.extraction_strategy.llm_config == {+ "type": "LLMConfig",+ "params": {"api_token": "env:CRAWL4AI_TEST_TOKEN"},+ }+++def test_trusted_top_level_kwargs_still_deserialize(monkeypatch):+ """from_kwargs still builds typed objects for TRUSTED callers."""+ monkeypatch.setenv("CRAWL4AI_TEST_TOKEN", "tok-123")+ cfg = CrawlerRunConfig.load(+ {+ "type": "CrawlerRunConfig",+ "extraction_strategy": {"type": "dict", "value": _llm_strategy("CRAWL4AI_TEST_TOKEN")},+ }+ )+ assert cfg.extraction_strategy.llm_config.api_token == "tok-123"+++def test_plain_business_dict_with_type_key_still_passes():+ """A JsonCss schema carries "type" keys but is data, not a typed object."""+ schema = {+ "name": "Items",+ "baseSelector": ".item",+ "fields": [{"name": "title", "selector": "h1", "type": "text"}],+ }+ cfg = CrawlerRunConfig.load(+ {+ "type": "CrawlerRunConfig",+ "params": {+ "extraction_strategy": {+ "type": "JsonCssExtractionStrategy",+ "params": {"schema": {"type": "dict", "value": schema}},+ }+ },+ },+ provenance=UNTRUSTED,+ )+ assert cfg.extraction_strategy.schema["baseSelector"] == ".item"