Skip to content

Official Crawl4AI Cloud: a hosted web scraping, extraction, and search API for AI agents. Free API key, 1,000 credits a month, no card. Fully open source: this repo is the Apache-2.0 crawler that powers it, and you can self-host it.

fix(security): stop {"type":"dict","value":X} laundering typed objects past the untrusted gate

ntohidice62ae674c34
diff --git a/crawl4ai/async_configs.py b/crawl4ai/async_configs.py
index a6691e65..340b3f9b 100644
--- a/crawl4ai/async_configs.py
+++ b/crawl4ai/async_configs.py
@@ -449,6 +449,21 @@ def to_serializable_dict(obj: Any, ignore_default_value : bool = False):
return str(obj)
+def _is_typed_shape(data: Any) -> bool:
+ """True for the dict shapes to_serializable_dict() emits:
+ {"type": "<ClassName>", "params": {...}} or {"type": "dict", "value": {...}}.
+
+ Plain business dicts that happen to carry a "type" key (JSON-Schema
+ fragments, JsonCss field specs like {"type": "text", "name": "..."}) have
+ neither "params" nor "value" and are not typed shapes.
+ """
+ return (
+ isinstance(data, dict)
+ and "type" in data
+ and ("params" in data or (data["type"] == "dict" and "value" in data))
+ )
+
+
def from_serializable_dict(data: Any, provenance: "Provenance" = None) -> Any:
"""
Recursively convert a serializable dictionary back to an object instance.
@@ -474,14 +489,23 @@ def from_serializable_dict(data: Any, provenance: "Provenance" = None) -> Any:
# carry a "type" key (e.g. JSON-Schema fragments, JsonCss field specs like
# {"type": "text", "name": "..."}) have neither "params" nor "value" and
# must fall through to the raw-dict path below so they are passed as data.
- if (
- isinstance(data, dict)
- and "type" in data
- and ("params" in data or (data["type"] == "dict" and "value" in data))
- ):
+ if _is_typed_shape(data):
# Handle plain dictionaries
if data["type"] == "dict" and "value" in data:
- return {k: from_serializable_dict(v, provenance) for k, v in data["value"].items()}
+ unwrapped = {
+ k: from_serializable_dict(v, provenance) for k, v in data["value"].items()
+ }
+ # The unwrap above recurses over .items(), so data["value"] is never
+ # seen as a whole typed object and the type gate below never fires
+ # for it. Without this check an untrusted body launders a forbidden
+ # type past the gate by wrapping it:
+ # {"type": "dict", "value": {"type": "LLMConfig", "params": {...}}}
+ if provenance == Provenance.UNTRUSTED and _is_typed_shape(unwrapped):
+ raise UntrustedConfigError(
+ "an untrusted request may not wrap a typed object in "
+ "{'type': 'dict', 'value': ...}"
+ )
+ return unwrapped
# Security: only allow known-safe types to be deserialized.
# Unknown types (e.g. logging.Logger serialized by older clients) are
@@ -988,11 +1012,13 @@ class BrowserConfig:
)
@staticmethod
- def from_kwargs(kwargs: dict) -> "BrowserConfig":
+ def from_kwargs(kwargs: dict, provenance: "Provenance" = None) -> "BrowserConfig":
# Auto-deserialize any dict values that use the {"type": ..., "params": ...}
# serialization format (e.g. from JSON API requests or dump()/load() roundtrips).
kwargs = {
- k: from_serializable_dict(v) if isinstance(v, dict) and "type" in v else v
+ k: from_serializable_dict(v, provenance)
+ if isinstance(v, dict) and "type" in v
+ else v
for k, v in kwargs.items()
}
# Only pass keys present in kwargs so that __init__ defaults (and
@@ -1078,7 +1104,7 @@ class BrowserConfig:
# so a body that sends bare kwargs (no {type,params}) cannot bypass it.
if provenance == Provenance.UNTRUSTED and isinstance(config, dict):
config = _enforce_untrusted("BrowserConfig", config)
- return BrowserConfig.from_kwargs(config)
+ return BrowserConfig.from_kwargs(config, provenance)
def set_nstproxy(
self,
@@ -2119,12 +2145,14 @@ class CrawlerRunConfig():
super().__setattr__(name, value)
@staticmethod
- def from_kwargs(kwargs: dict) -> "CrawlerRunConfig":
+ def from_kwargs(kwargs: dict, provenance: "Provenance" = None) -> "CrawlerRunConfig":
# Auto-deserialize any dict values that use the {"type": ..., "params": ...}
# serialization format (e.g. from JSON API requests or dump()/load() roundtrips).
# This covers markdown_generator, extraction_strategy, content_filter, etc.
kwargs = {
- k: from_serializable_dict(v) if isinstance(v, dict) and "type" in v else v
+ k: from_serializable_dict(v, provenance)
+ if isinstance(v, dict) and "type" in v
+ else v
for k, v in kwargs.items()
}
# Only pass keys present in kwargs so that __init__ defaults (and
@@ -2147,7 +2175,7 @@ class CrawlerRunConfig():
return config
if provenance == Provenance.UNTRUSTED and isinstance(config, dict):
config = _enforce_untrusted("CrawlerRunConfig", config)
- return CrawlerRunConfig.from_kwargs(config)
+ return CrawlerRunConfig.from_kwargs(config, provenance)
def to_dict(self):
return {
diff --git a/tests/unit/test_config_provenance.py b/tests/unit/test_config_provenance.py
new file mode 100644
index 00000000..49246a8e
--- /dev/null
+++ b/tests/unit/test_config_provenance.py
@@ -0,0 +1,118 @@
+"""Untrusted config provenance gate.
+
+Regression test for the {"type": "dict", "value": <typed object>} laundering
+bypass: wrapping a forbidden typed object in a plain-dict envelope hid it from
+the type gate, and from_kwargs then re-deserialized it as TRUSTED, which
+resolves api_token="env:NAME" through os.getenv.
+"""
+import pytest
+
+from crawl4ai.async_configs import (
+ BrowserConfig,
+ CrawlerRunConfig,
+ Provenance,
+ UntrustedConfigError,
+)
+
+UNTRUSTED = Provenance.UNTRUSTED
+
+
+def _llm_strategy(env_var):
+ return {
+ "type": "LLMExtractionStrategy",
+ "params": {
+ "llm_config": {
+ "type": "dict",
+ "value": {"type": "LLMConfig", "params": {"api_token": f"env:{env_var}"}},
+ }
+ },
+ }
+
+
+def test_forbidden_type_refused_directly():
+ data = {"type": "CrawlerRunConfig", "params": {"extraction_strategy": _llm_strategy("SECRET_KEY")}}
+ with pytest.raises(UntrustedConfigError):
+ CrawlerRunConfig.load(data, provenance=UNTRUSTED)
+
+
+def test_forbidden_type_refused_when_wrapped(monkeypatch):
+ """The bypass: the whole strategy hidden behind {"type":"dict","value":...}."""
+ monkeypatch.setenv("SECRET_KEY", "canary-secret-key-value")
+ data = {
+ "type": "CrawlerRunConfig",
+ "extraction_strategy": {"type": "dict", "value": _llm_strategy("SECRET_KEY")},
+ }
+ with pytest.raises(UntrustedConfigError):
+ CrawlerRunConfig.load(data, provenance=UNTRUSTED)
+
+
+def test_forbidden_type_refused_when_wrapped_in_params(monkeypatch):
+ monkeypatch.setenv("OPENAI_API_KEY", "sk-canary-9f3a")
+ data = {
+ "type": "CrawlerRunConfig",
+ "params": {"extraction_strategy": {"type": "dict", "value": _llm_strategy("OPENAI_API_KEY")}},
+ }
+ with pytest.raises(UntrustedConfigError):
+ CrawlerRunConfig.load(data, provenance=UNTRUSTED)
+
+
+def test_browser_config_wrapped_proxy_refused():
+ """Same envelope, other config class, other forbidden type."""
+ data = {
+ "type": "BrowserConfig",
+ "params": {
+ "proxy_config": {
+ "type": "dict",
+ "value": {"type": "ProxyConfig", "params": {"server": "http://evil:8080"}},
+ }
+ },
+ }
+ with pytest.raises(UntrustedConfigError):
+ BrowserConfig.load(data, provenance=UNTRUSTED)
+
+
+def test_trusted_load_is_unchanged(monkeypatch):
+ """TRUSTED (SDK) callers keep the old behavior: no raise, and the wrapped
+ envelope still unwraps to the same plain dict it did before the fix."""
+ monkeypatch.setenv("CRAWL4AI_TEST_TOKEN", "tok-123")
+ cfg = CrawlerRunConfig.load(
+ {"type": "CrawlerRunConfig", "params": {"extraction_strategy": _llm_strategy("CRAWL4AI_TEST_TOKEN")}}
+ )
+ assert cfg.extraction_strategy.llm_config == {
+ "type": "LLMConfig",
+ "params": {"api_token": "env:CRAWL4AI_TEST_TOKEN"},
+ }
+
+
+def test_trusted_top_level_kwargs_still_deserialize(monkeypatch):
+ """from_kwargs still builds typed objects for TRUSTED callers."""
+ monkeypatch.setenv("CRAWL4AI_TEST_TOKEN", "tok-123")
+ cfg = CrawlerRunConfig.load(
+ {
+ "type": "CrawlerRunConfig",
+ "extraction_strategy": {"type": "dict", "value": _llm_strategy("CRAWL4AI_TEST_TOKEN")},
+ }
+ )
+ assert cfg.extraction_strategy.llm_config.api_token == "tok-123"
+
+
+def test_plain_business_dict_with_type_key_still_passes():
+ """A JsonCss schema carries "type" keys but is data, not a typed object."""
+ schema = {
+ "name": "Items",
+ "baseSelector": ".item",
+ "fields": [{"name": "title", "selector": "h1", "type": "text"}],
+ }
+ cfg = CrawlerRunConfig.load(
+ {
+ "type": "CrawlerRunConfig",
+ "params": {
+ "extraction_strategy": {
+ "type": "JsonCssExtractionStrategy",
+ "params": {"schema": {"type": "dict", "value": schema}},
+ }
+ },
+ },
+ provenance=UNTRUSTED,
+ )
+ assert cfg.extraction_strategy.schema["baseSelector"] == ".item"