Skip to content

Official Crawl4AI Cloud: a hosted web scraping, extraction, and search API for AI agents. Free API key, 1,000 credits a month, no card. Fully open source: this repo is the Apache-2.0 crawler that powers it, and you can self-host it.

fix(security): refuse caller-supplied regexes from untrusted bodies

ntohidi8aea2006e2cd
diff --git a/SECURITY-CREDITS.md b/SECURITY-CREDITS.md
index b8ad3491..8791d0a3 100644
--- a/SECURITY-CREDITS.md
+++ b/SECURITY-CREDITS.md
@@ -26,3 +26,4 @@ We thank the following security researchers for their responsible disclosure:
| Ibrahim AlJaafreh - Cystack RedTeam | [LinkedIn](https://www.linkedin.com/in/ibrahim-aljaafreh-glitch/), [cystack.ps](https://cystack.ps) | SSRF with response disclosure via link_preview_config through the URL seeder (0.9.4) | 2026-09-04 |
| Adam Jordan | GitHub: [adamyordan](https://github.com/adamyordan) | Untrusted-config gate bypass via dict-wrapper laundering, leaking server env vars (0.9.4) | 2026-09-08 |
| arpe1618 | GitHub: [arpe1618](https://github.com/arpe1618) | Blind SSRF via the CacheValidator freshness check bypassing the Docker egress proxy (0.9.5) | 2026-09-29 |
+| Zhixi "Jace" Sun | GitHub: [manus-use](https://github.com/manus-use) | Page JavaScript injection via css_selector and wait_for, HuggingFace model choice via CosineStrategy.model_name, and ReDoS via untrusted regex patterns on the Docker server (0.9.5) | 2026-10-01 |
diff --git a/crawl4ai/async_configs.py b/crawl4ai/async_configs.py
index 3478183b..124e9d36 100644
--- a/crawl4ai/async_configs.py
+++ b/crawl4ai/async_configs.py
@@ -226,8 +226,28 @@ UNTRUSTED_FORBIDDEN_FIELDS = {
# handed to from_pretrained: the caller would pick the weights the server
# downloads and unpickles.
"CosineStrategy": {"model_name"},
+ # Caller-supplied regexes. Python's re holds the GIL while it matches, so
+ # one catastrophic-backtracking pattern freezes the whole server, and the
+ # crawl deadline cannot interrupt it. The built-in patterns stay available.
+ "RegexExtractionStrategy": {"custom"},
+ "RegexChunking": {"patterns"},
}
+# Extraction schemas are free-form dicts, so their "regex" field type (a
+# caller-supplied re.search pattern, same ReDoS as above) is checked by walking
+# the schema rather than by field name.
+_SCHEMA_STRATEGIES = {
+ "JsonCssExtractionStrategy", "JsonXPathExtractionStrategy", "JsonLxmlExtractionStrategy",
+}
+
+
+def _has_regex_field(node) -> bool:
+ if isinstance(node, dict):
+ return node.get("type") == "regex" or any(_has_regex_field(v) for v in node.values())
+ if isinstance(node, list):
+ return any(_has_regex_field(v) for v in node)
+ return False
+
# Field names that must NEVER be set from an untrusted body on ANY allowed
# type, checked regardless of per-type allowlist. This is the fail-closed
# backstop: the per-type maps above only cover BrowserConfig/CrawlerRunConfig,
@@ -404,6 +424,12 @@ def _clamp_untrusted(type_name: str, params: dict) -> dict:
# Rasterizing every page is the most expensive thing this strategy can
# do, and nothing about untrusted crawling needs it.
params["extract_images"] = False
+ elif type_name in _SCHEMA_STRATEGIES:
+ if _has_regex_field(params.get("schema")):
+ raise UntrustedConfigError(
+ f"schema fields of type 'regex' are not permitted on {type_name} "
+ "from an untrusted request"
+ )
elif type_name == "LinkPreviewConfig":
# Also no field allowlist, and each link is a separate outbound fetch:
# unclamped, one request body can order millions of them at any
diff --git a/tests/unit/test_untrusted_field_sinks.py b/tests/unit/test_untrusted_field_sinks.py
index 73b6e80f..d4dc192a 100644
--- a/tests/unit/test_untrusted_field_sinks.py
+++ b/tests/unit/test_untrusted_field_sinks.py
@@ -58,3 +58,69 @@ def test_untrusted_cosine_model_name_refused():
"type": "CosineStrategy",
"params": {"model_name": "attacker-org/evil-model"},
}})
+
+
+# ── caller-supplied regex ──────────────────────────────────────────────
+
+
+def test_untrusted_regex_strategy_custom_refused():
+ with pytest.raises(UntrustedConfigError):
+ _load({"extraction_strategy": {
+ "type": "RegexExtractionStrategy",
+ "params": {"custom": {"evil": "(a+)+$"}},
+ }})
+
+
+def test_untrusted_regex_strategy_builtins_still_allowed():
+ cfg = _load({"extraction_strategy": {
+ "type": "RegexExtractionStrategy", "params": {"pattern": 1},
+ }})
+ assert type(cfg.extraction_strategy).__name__ == "RegexExtractionStrategy"
+
+
+def test_untrusted_regex_chunking_patterns_refused():
+ with pytest.raises(UntrustedConfigError):
+ _load({"chunking_strategy": {
+ "type": "RegexChunking", "params": {"patterns": ["(a+)+$"]},
+ }})
+
+
+def _schema(field):
+ return {"name": "x", "baseSelector": "p", "fields": [field]}
+
+
+REGEX_FIELD = {"name": "f", "selector": "span", "type": "regex", "pattern": "(a+)+$"}
+
+
+ "schema",
+ [
+ _schema(REGEX_FIELD),
+ # nested inside another field
+ _schema({"name": "n", "selector": "div", "type": "nested", "fields": [REGEX_FIELD]}),
+ # behind the {"type": "dict", "value": ...} envelope
+ {"type": "dict", "value": _schema(REGEX_FIELD)},
+ ],
+)
+ "strategy",
+ ["JsonCssExtractionStrategy", "JsonXPathExtractionStrategy", "JsonLxmlExtractionStrategy"],
+)
+def test_untrusted_schema_regex_field_refused(strategy, schema):
+ with pytest.raises(UntrustedConfigError):
+ _load({"extraction_strategy": {"type": strategy, "params": {"schema": schema}}})
+
+
+def test_untrusted_schema_without_regex_still_allowed():
+ cfg = _load({"extraction_strategy": {
+ "type": "JsonCssExtractionStrategy",
+ "params": {"schema": _schema({"name": "t", "selector": "h1", "type": "text"})},
+ }})
+ assert type(cfg.extraction_strategy).__name__ == "JsonCssExtractionStrategy"
+
+
+def test_trusted_regex_unchanged():
+ cfg = _load({"chunking_strategy": {
+ "type": "RegexChunking", "params": {"patterns": [r"\n"]},
+ }}, Provenance.TRUSTED)
+ assert cfg.chunking_strategy.patterns == [r"\n"]