Official Crawl4AI Cloud: a hosted web scraping, extraction, and search API for AI agents. Free API key, 1,000 credits a month, no card. Fully open source: this repo is the Apache-2.0 crawler that powers it, and you can self-host it.
Python aiai-agentscrawl4ai-cloudcrawlerdata-extractionllmmarkdownmcpopen-sourceplaywrightpythonragscraperweb-crawlerweb-crawlingweb-scraping
fix(security): cap delay fields in untrusted CrawlerRunConfig bodies
ntohidid6fdff50cde4
diff --git a/crawl4ai/async_configs.py b/crawl4ai/async_configs.pyindex 124e9d36..9ea6cf1f 100644--- a/crawl4ai/async_configs.py+++ b/crawl4ai/async_configs.py@@ -319,6 +319,7 @@ UNTRUSTED_FIELD_ALLOWLIST = {# Upper bounds applied to attacker-influenced quantities after filtering._DEFAULT_MAX_TIMEOUT_MS = 60_000_MAX_SCROLL_STEPS = 1000+_MAX_DELAY_SECONDS = 60 # ceiling for delay_before_return_html / mean_delay / max_range_MAX_VIEWPORT = 4000_MAX_PDF_BYTES = 100 * 1024 * 1024_MAX_PDF_PAGES = 2000@@ -390,10 +391,27 @@ def _clamp_untrusted(type_name: str, params: dict) -> dict:return min(_DEFAULT_MAX_TIMEOUT_MS, ceiling)return min(int(v), ceiling)+ # delay_before_return_html is held with a browser page open; mean_delay and+ # max_range seed the arun_many rate limiter. All three are seconds, not ms,+ # so they get their own cap rather than reusing the ms timeout ceiling.+ _delay_defaults = {+ "delay_before_return_html": 0.1,+ "mean_delay": 0.1,+ "max_range": 0.3,+ }++ def _cap_delay(v, default):+ if not isinstance(v, (int, float)) or isinstance(v, bool) or not v >= 0: # also NaN+ return default+ return min(v, _MAX_DELAY_SECONDS)+if type_name == "CrawlerRunConfig":for f in ("page_timeout", "wait_for_timeout", "body_visibility_timeout"):if f in params:params[f] = _cap_timeout(params[f])+ for f, default in _delay_defaults.items():+ if f in params:+ params[f] = _cap_delay(params[f], default)if isinstance(params.get("max_scroll_steps"), int):params["max_scroll_steps"] = min(params["max_scroll_steps"], _MAX_SCROLL_STEPS)# smart_wait runs "js:...", function-shaped values, and any value thatdiff --git a/tests/unit/test_config_provenance.py b/tests/unit/test_config_provenance.pyindex 49246a8e..de693aa2 100644--- a/tests/unit/test_config_provenance.py+++ b/tests/unit/test_config_provenance.py@@ -96,6 +96,60 @@ def test_trusted_top_level_kwargs_still_deserialize(monkeypatch):assert cfg.extraction_strategy.llm_config.api_token == "tok-123"+def test_untrusted_delay_fields_are_capped():+ """delay_before_return_html / mean_delay / max_range are seconds held on a+ live browser page or used as a rate-limiter base delay. Unclamped, an+ untrusted body can hold a page (or a whole arun_many batch) open forever."""+ cfg = CrawlerRunConfig.load(+ {+ "type": "CrawlerRunConfig",+ "params": {+ "delay_before_return_html": 1e9,+ "mean_delay": 1e9,+ "max_range": 1e9,+ },+ },+ provenance=UNTRUSTED,+ )+ assert cfg.delay_before_return_html <= 60+ assert cfg.mean_delay <= 60+ assert cfg.max_range <= 60+++def test_untrusted_delay_fields_negative_or_non_numeric_fall_back_to_default():+ cfg = CrawlerRunConfig.load(+ {+ "type": "CrawlerRunConfig",+ "params": {+ "delay_before_return_html": -5,+ "mean_delay": "not-a-number",+ "max_range": None,+ },+ },+ provenance=UNTRUSTED,+ )+ assert cfg.delay_before_return_html == 0.1+ assert cfg.mean_delay == 0.1+ assert cfg.max_range == 0.3+++def test_trusted_delay_fields_are_not_clamped():+ """TRUSTED (SDK) callers keep the old behavior: no cap applied."""+ cfg = CrawlerRunConfig.load(+ {+ "type": "CrawlerRunConfig",+ "params": {+ "delay_before_return_html": 1e9,+ "mean_delay": 1e9,+ "max_range": 1e9,+ },+ }+ )+ assert cfg.delay_before_return_html == 1e9+ assert cfg.mean_delay == 1e9+ assert cfg.max_range == 1e9++def test_plain_business_dict_with_type_key_still_passes():"""A JsonCss schema carries "type" keys but is data, not a typed object."""schema = {