Official Crawl4AI Cloud: a hosted web scraping, extraction, and search API for AI agents. Free API key, 1,000 credits a month, no card. Fully open source: this repo is the Apache-2.0 crawler that powers it, and you can self-host it.
Python aiai-agentscrawl4ai-cloudcrawlerdata-extractionllmmarkdownmcpopen-sourceplaywrightpythonragscraperweb-crawlerweb-crawlingweb-scraping
fix(security): refuse caller-supplied regexes from untrusted bodies
ntohidi8aea2006e2cd
diff --git a/SECURITY-CREDITS.md b/SECURITY-CREDITS.mdindex b8ad3491..8791d0a3 100644--- a/SECURITY-CREDITS.md+++ b/SECURITY-CREDITS.md@@ -26,3 +26,4 @@ We thank the following security researchers for their responsible disclosure:| Ibrahim AlJaafreh - Cystack RedTeam | [LinkedIn](https://www.linkedin.com/in/ibrahim-aljaafreh-glitch/), [cystack.ps](https://cystack.ps) | SSRF with response disclosure via link_preview_config through the URL seeder (0.9.4) | 2026-09-04 || Adam Jordan | GitHub: [adamyordan](https://github.com/adamyordan) | Untrusted-config gate bypass via dict-wrapper laundering, leaking server env vars (0.9.4) | 2026-09-08 || arpe1618 | GitHub: [arpe1618](https://github.com/arpe1618) | Blind SSRF via the CacheValidator freshness check bypassing the Docker egress proxy (0.9.5) | 2026-09-29 |+| Zhixi "Jace" Sun | GitHub: [manus-use](https://github.com/manus-use) | Page JavaScript injection via css_selector and wait_for, HuggingFace model choice via CosineStrategy.model_name, and ReDoS via untrusted regex patterns on the Docker server (0.9.5) | 2026-10-01 |diff --git a/crawl4ai/async_configs.py b/crawl4ai/async_configs.pyindex 3478183b..124e9d36 100644--- a/crawl4ai/async_configs.py+++ b/crawl4ai/async_configs.py@@ -226,8 +226,28 @@ UNTRUSTED_FORBIDDEN_FIELDS = {# handed to from_pretrained: the caller would pick the weights the server# downloads and unpickles."CosineStrategy": {"model_name"},+ # Caller-supplied regexes. Python's re holds the GIL while it matches, so+ # one catastrophic-backtracking pattern freezes the whole server, and the+ # crawl deadline cannot interrupt it. The built-in patterns stay available.+ "RegexExtractionStrategy": {"custom"},+ "RegexChunking": {"patterns"},}+# Extraction schemas are free-form dicts, so their "regex" field type (a+# caller-supplied re.search pattern, same ReDoS as above) is checked by walking+# the schema rather than by field name.+_SCHEMA_STRATEGIES = {+ "JsonCssExtractionStrategy", "JsonXPathExtractionStrategy", "JsonLxmlExtractionStrategy",+}+++def _has_regex_field(node) -> bool:+ if isinstance(node, dict):+ return node.get("type") == "regex" or any(_has_regex_field(v) for v in node.values())+ if isinstance(node, list):+ return any(_has_regex_field(v) for v in node)+ return False+# Field names that must NEVER be set from an untrusted body on ANY allowed# type, checked regardless of per-type allowlist. This is the fail-closed# backstop: the per-type maps above only cover BrowserConfig/CrawlerRunConfig,@@ -404,6 +424,12 @@ def _clamp_untrusted(type_name: str, params: dict) -> dict:# Rasterizing every page is the most expensive thing this strategy can# do, and nothing about untrusted crawling needs it.params["extract_images"] = False+ elif type_name in _SCHEMA_STRATEGIES:+ if _has_regex_field(params.get("schema")):+ raise UntrustedConfigError(+ f"schema fields of type 'regex' are not permitted on {type_name} "+ "from an untrusted request"+ )elif type_name == "LinkPreviewConfig":# Also no field allowlist, and each link is a separate outbound fetch:# unclamped, one request body can order millions of them at anydiff --git a/tests/unit/test_untrusted_field_sinks.py b/tests/unit/test_untrusted_field_sinks.pyindex 73b6e80f..d4dc192a 100644--- a/tests/unit/test_untrusted_field_sinks.py+++ b/tests/unit/test_untrusted_field_sinks.py@@ -58,3 +58,69 @@ def test_untrusted_cosine_model_name_refused():"type": "CosineStrategy","params": {"model_name": "attacker-org/evil-model"},}})+++# ── caller-supplied regex ──────────────────────────────────────────────+++def test_untrusted_regex_strategy_custom_refused():+ with pytest.raises(UntrustedConfigError):+ _load({"extraction_strategy": {+ "type": "RegexExtractionStrategy",+ "params": {"custom": {"evil": "(a+)+$"}},+ }})+++def test_untrusted_regex_strategy_builtins_still_allowed():+ cfg = _load({"extraction_strategy": {+ "type": "RegexExtractionStrategy", "params": {"pattern": 1},+ }})+ assert type(cfg.extraction_strategy).__name__ == "RegexExtractionStrategy"+++def test_untrusted_regex_chunking_patterns_refused():+ with pytest.raises(UntrustedConfigError):+ _load({"chunking_strategy": {+ "type": "RegexChunking", "params": {"patterns": ["(a+)+$"]},+ }})+++def _schema(field):+ return {"name": "x", "baseSelector": "p", "fields": [field]}+++REGEX_FIELD = {"name": "f", "selector": "span", "type": "regex", "pattern": "(a+)+$"}+++ "schema",+ [+ _schema(REGEX_FIELD),+ # nested inside another field+ _schema({"name": "n", "selector": "div", "type": "nested", "fields": [REGEX_FIELD]}),+ # behind the {"type": "dict", "value": ...} envelope+ {"type": "dict", "value": _schema(REGEX_FIELD)},+ ],+)+ "strategy",+ ["JsonCssExtractionStrategy", "JsonXPathExtractionStrategy", "JsonLxmlExtractionStrategy"],+)+def test_untrusted_schema_regex_field_refused(strategy, schema):+ with pytest.raises(UntrustedConfigError):+ _load({"extraction_strategy": {"type": strategy, "params": {"schema": schema}}})+++def test_untrusted_schema_without_regex_still_allowed():+ cfg = _load({"extraction_strategy": {+ "type": "JsonCssExtractionStrategy",+ "params": {"schema": _schema({"name": "t", "selector": "h1", "type": "text"})},+ }})+ assert type(cfg.extraction_strategy).__name__ == "JsonCssExtractionStrategy"+++def test_trusted_regex_unchanged():+ cfg = _load({"chunking_strategy": {+ "type": "RegexChunking", "params": {"patterns": [r"\n"]},+ }}, Provenance.TRUSTED)+ assert cfg.chunking_strategy.patterns == [r"\n"]