Skip to content

Official Crawl4AI Cloud: a hosted web scraping, extraction, and search API for AI agents. Free API key, 1,000 credits a month, no card. Fully open source: this repo is the Apache-2.0 crawler that powers it, and you can self-host it.

fix(security): cap delay fields in untrusted CrawlerRunConfig bodies

ntohidid6fdff50cde4
diff --git a/crawl4ai/async_configs.py b/crawl4ai/async_configs.py
index 124e9d36..9ea6cf1f 100644
--- a/crawl4ai/async_configs.py
+++ b/crawl4ai/async_configs.py
@@ -319,6 +319,7 @@ UNTRUSTED_FIELD_ALLOWLIST = {
# Upper bounds applied to attacker-influenced quantities after filtering.
_DEFAULT_MAX_TIMEOUT_MS = 60_000
_MAX_SCROLL_STEPS = 1000
+_MAX_DELAY_SECONDS = 60 # ceiling for delay_before_return_html / mean_delay / max_range
_MAX_VIEWPORT = 4000
_MAX_PDF_BYTES = 100 * 1024 * 1024
_MAX_PDF_PAGES = 2000
@@ -390,10 +391,27 @@ def _clamp_untrusted(type_name: str, params: dict) -> dict:
return min(_DEFAULT_MAX_TIMEOUT_MS, ceiling)
return min(int(v), ceiling)
+ # delay_before_return_html is held with a browser page open; mean_delay and
+ # max_range seed the arun_many rate limiter. All three are seconds, not ms,
+ # so they get their own cap rather than reusing the ms timeout ceiling.
+ _delay_defaults = {
+ "delay_before_return_html": 0.1,
+ "mean_delay": 0.1,
+ "max_range": 0.3,
+ }
+
+ def _cap_delay(v, default):
+ if not isinstance(v, (int, float)) or isinstance(v, bool) or not v >= 0: # also NaN
+ return default
+ return min(v, _MAX_DELAY_SECONDS)
+
if type_name == "CrawlerRunConfig":
for f in ("page_timeout", "wait_for_timeout", "body_visibility_timeout"):
if f in params:
params[f] = _cap_timeout(params[f])
+ for f, default in _delay_defaults.items():
+ if f in params:
+ params[f] = _cap_delay(params[f], default)
if isinstance(params.get("max_scroll_steps"), int):
params["max_scroll_steps"] = min(params["max_scroll_steps"], _MAX_SCROLL_STEPS)
# smart_wait runs "js:...", function-shaped values, and any value that
diff --git a/tests/unit/test_config_provenance.py b/tests/unit/test_config_provenance.py
index 49246a8e..de693aa2 100644
--- a/tests/unit/test_config_provenance.py
+++ b/tests/unit/test_config_provenance.py
@@ -96,6 +96,60 @@ def test_trusted_top_level_kwargs_still_deserialize(monkeypatch):
assert cfg.extraction_strategy.llm_config.api_token == "tok-123"
+def test_untrusted_delay_fields_are_capped():
+ """delay_before_return_html / mean_delay / max_range are seconds held on a
+ live browser page or used as a rate-limiter base delay. Unclamped, an
+ untrusted body can hold a page (or a whole arun_many batch) open forever."""
+ cfg = CrawlerRunConfig.load(
+ {
+ "type": "CrawlerRunConfig",
+ "params": {
+ "delay_before_return_html": 1e9,
+ "mean_delay": 1e9,
+ "max_range": 1e9,
+ },
+ },
+ provenance=UNTRUSTED,
+ )
+ assert cfg.delay_before_return_html <= 60
+ assert cfg.mean_delay <= 60
+ assert cfg.max_range <= 60
+
+
+def test_untrusted_delay_fields_negative_or_non_numeric_fall_back_to_default():
+ cfg = CrawlerRunConfig.load(
+ {
+ "type": "CrawlerRunConfig",
+ "params": {
+ "delay_before_return_html": -5,
+ "mean_delay": "not-a-number",
+ "max_range": None,
+ },
+ },
+ provenance=UNTRUSTED,
+ )
+ assert cfg.delay_before_return_html == 0.1
+ assert cfg.mean_delay == 0.1
+ assert cfg.max_range == 0.3
+
+
+def test_trusted_delay_fields_are_not_clamped():
+ """TRUSTED (SDK) callers keep the old behavior: no cap applied."""
+ cfg = CrawlerRunConfig.load(
+ {
+ "type": "CrawlerRunConfig",
+ "params": {
+ "delay_before_return_html": 1e9,
+ "mean_delay": 1e9,
+ "max_range": 1e9,
+ },
+ }
+ )
+ assert cfg.delay_before_return_html == 1e9
+ assert cfg.mean_delay == 1e9
+ assert cfg.max_range == 1e9
+
+
def test_plain_business_dict_with_type_key_still_passes():
"""A JsonCss schema carries "type" keys but is data, not a typed object."""
schema = {