Skip to content

Official Crawl4AI Cloud: a hosted web scraping, extraction, and search API for AI agents. Free API key, 1,000 credits a month, no card. Fully open source: this repo is the Apache-2.0 crawler that powers it, and you can self-host it.

feat(docker): total size budget for untrusted deep-crawl strategies

Soham Kukretid6f8a755d5a8
diff --git a/crawl4ai/async_configs.py b/crawl4ai/async_configs.py
index c131ff39..ad51cb17 100644
--- a/crawl4ai/async_configs.py
+++ b/crawl4ai/async_configs.py
@@ -316,6 +316,8 @@ _MAX_LINK_PREVIEW_LINKS = 100 # the class default
_MAX_LINK_PREVIEW_CONCURRENCY = 10 # the class default
_MAX_LINK_PREVIEW_TIMEOUT_S = 10 # seconds here, not ms
_MAX_DEEP_CRAWL_LIST = 100 # patterns, keywords, filters, scorers per object
+_MAX_DEEP_CRAWL_OBJECTS = 20 # filters and scorers in one strategy
+_MAX_DEEP_CRAWL_ITEMS = 200 # list and dict items (patterns, keywords, ...) in one strategy
def _max_timeout_ms() -> int:
@@ -358,6 +360,16 @@ def _deep_crawl_allowed() -> bool:
return os.getenv("CRAWL4AI_ALLOW_DEEP_CRAWL", "false").lower() == "true"
+def _all_nodes(node):
+ """Yield every value in a request tree. An object's params dict is entered but not yielded."""
+ yield node
+ if _is_typed_shape(node) and isinstance(node.get("params"), dict):
+ node = node["params"]
+ children = node.values() if isinstance(node, dict) else node if isinstance(node, list) else []
+ for child in children:
+ yield from _all_nodes(child)
+
+
def _filter_untrusted_fields(type_name: str, params: dict) -> dict:
"""Drop non-allowlisted fields and raise on forbidden (power) fields."""
forbidden = UNTRUSTED_FORBIDDEN_FIELDS.get(type_name, set())
@@ -380,6 +392,16 @@ def _filter_untrusted_fields(type_name: str, params: dict) -> dict:
raise UntrustedConfigError(
f"{type_name} list parameters are limited to {_MAX_DEEP_CRAWL_LIST} items"
)
+ if type_name in ("BFSDeepCrawlStrategy", "DFSDeepCrawlStrategy", "BestFirstCrawlingStrategy"):
+ # Filters and scorers run synchronously on the event loop for every discovered link.
+ nodes = [n for value in out.values() for n in _all_nodes(value)]
+ filters_and_scorers = sum(1 for n in nodes if isinstance(n, dict) and n.get("type") in UNTRUSTED_DEEP_CRAWL_TYPES)
+ list_items = sum(len(n) for n in nodes if isinstance(n, (list, dict)) and not _is_typed_shape(n))
+ if filters_and_scorers > _MAX_DEEP_CRAWL_OBJECTS or list_items > _MAX_DEEP_CRAWL_ITEMS:
+ raise UntrustedConfigError(
+ f"{type_name} is limited to {_MAX_DEEP_CRAWL_OBJECTS} filters and scorers "
+ f"and {_MAX_DEEP_CRAWL_ITEMS} list items in total"
+ )
if type_name == "URLPatternFilter":
# Regex and *.domain:// patterns compile unescaped, and globs not led by * search in
# quadratic time; allow only *-led globs, /prefix/* and *.ext.
@@ -396,6 +418,10 @@ def _filter_untrusted_fields(type_name: str, params: dict) -> dict:
"URLPatternFilter accepts only globs that start with '*' or end with a "
"single '/*' from an untrusted request"
)
+ if type_name == "CompositeScorer" and any(
+ isinstance(s, dict) and s.get("type") == "CompositeScorer" for s in out.get("scorers") or []
+ ):
+ raise UntrustedConfigError("CompositeScorer cannot be nested from an untrusted request")
if type_name == "CrawlerRunConfig" and out.get("deep_crawl_strategy") and out.get("fetch_ssl_certificate"):
# The certificate fetch dials each discovered link directly, outside the egress proxy.
raise UntrustedConfigError(
diff --git a/deploy/docker/README.md b/deploy/docker/README.md
index 1b8c5775..a5281601 100644
--- a/deploy/docker/README.md
+++ b/deploy/docker/README.md
@@ -612,10 +612,10 @@ variable itself, so a value in `.llm.env` is ignored.
Rules the server applies:
- **One start URL** per request, and not inside `crawler_configs`.
-- `max_pages` and `max_depth` are clamped to `limits.max_pages` and `limits.max_depth` in `config.yml` (default 100 and 5). The crawl stops after `limits.wall_clock_s`, also on `/crawl/stream`.
+- `max_pages` and `max_depth` are clamped to `limits.max_pages` and `limits.max_depth` in `config.yml` (default 100 and 5; `max_pages: 0` means no limit). The crawl stops after `limits.wall_clock_s`, also on `/crawl/stream`.
- **Allowed types:** `BFSDeepCrawlStrategy`, `DFSDeepCrawlStrategy`, `BestFirstCrawlingStrategy`; `FilterChain` with `URLPatternFilter`, `DomainFilter`, `ContentTypeFilter`; scorers `KeywordRelevanceScorer`, `CompositeScorer`, `DomainAuthorityScorer`, `FreshnessScorer`, `PathDepthScorer`. `SEOFilter` and `ContentRelevanceFilter` are refused (they fetch pages outside the egress controls).
- `URLPatternFilter` accepts globs that start with `*` (`*/blog/*`), `/prefix/*` or `*.ext`. Regex patterns are refused.
-- `resume_state` is refused, and so is `fetch_ssl_certificate` together with a deep crawl. Lists (patterns, keywords, filters, scorers) are limited to 100 items. Discovered links longer than 2048 characters are skipped.
+- `resume_state` is refused, and so is `fetch_ssl_certificate` together with a deep crawl. Lists (patterns, keywords, filters, scorers) are limited to 100 items. One strategy may have at most 20 filters and scorers and 200 list items in total, and a `CompositeScorer` cannot contain another `CompositeScorer`. Discovered links longer than 2048 characters are skipped.
Turn the flag on only when you accept that one request can fetch many pages.
diff --git a/deploy/docker/config.yml b/deploy/docker/config.yml
index c22bf381..8bf16a11 100644
--- a/deploy/docker/config.yml
+++ b/deploy/docker/config.yml
@@ -37,7 +37,7 @@ redis:
# (the pre-hardening behavior), so you can relax or disable each one.
limits:
max_body_bytes: 10485760 # 10 MiB request body cap (413 if exceeded); 0 = unbounded
- max_pages: 100 # deep-crawl page budget clamp (deep crawl needs CRAWL4AI_ALLOW_DEEP_CRAWL=true)
+ max_pages: 100 # deep-crawl page budget clamp (deep crawl needs CRAWL4AI_ALLOW_DEEP_CRAWL=true); 0 = no limit
max_depth: 5 # deep-crawl depth clamp
wall_clock_s: 300 # per-crawl deadline in seconds (504 on timeout); 0 = no deadline
queue: # background job queue for /crawl/job and /llm/job
diff --git a/deploy/docker/governor.py b/deploy/docker/governor.py
index 4cabe0ab..0c1f8d6a 100644
--- a/deploy/docker/governor.py
+++ b/deploy/docker/governor.py
@@ -90,7 +90,8 @@ def clamp_deep_crawl(crawler_config, *, max_pages: int = DEFAULT_MAX_PAGES,
return
# NaN fails every comparison, so test for "inside the range", not "above the cap".
mp = getattr(dc, "max_pages", None)
- if not isinstance(mp, (int, float)) or not 0 <= mp <= max_pages:
+ # max_pages=0 means no limit, like the other limits in config.yml.
+ if max_pages and (not isinstance(mp, (int, float)) or not 0 <= mp <= max_pages):
try:
dc.max_pages = max_pages
except Exception:
diff --git a/deploy/docker/tests/test_security_resource_caps.py b/deploy/docker/tests/test_security_resource_caps.py
index a60a3cc2..f20d3215 100644
--- a/deploy/docker/tests/test_security_resource_caps.py
+++ b/deploy/docker/tests/test_security_resource_caps.py
@@ -69,6 +69,14 @@ class TestDeepCrawlClamp:
assert not guard.apply("https://a.com/" + "x" * 3000)
assert guard.apply("https://a.com/blog/post")
+ def test_zero_max_pages_means_no_limit(self):
+ from governor import clamp_deep_crawl
+ from crawl4ai.deep_crawling import BFSDeepCrawlStrategy
+ from crawl4ai import CrawlerRunConfig
+ cfg = CrawlerRunConfig(deep_crawl_strategy=BFSDeepCrawlStrategy(max_depth=2))
+ clamp_deep_crawl(cfg, max_pages=0)
+ assert cfg.deep_crawl_strategy.max_pages == float("inf")
+
class TestUntrustedDeepCrawlStillForbidden:
def test_request_cannot_set_deep_crawl(self):
diff --git a/docs/md_v2/core/self-hosting.md b/docs/md_v2/core/self-hosting.md
index f5445c98..5f6ae1c4 100644
--- a/docs/md_v2/core/self-hosting.md
+++ b/docs/md_v2/core/self-hosting.md
@@ -1339,10 +1339,10 @@ the project root, or export it in your shell.
Rules the server applies:
- **One start URL** per request, and not inside `crawler_configs`.
-- `max_pages` and `max_depth` are clamped to `limits.max_pages` and `limits.max_depth` in `config.yml` (default 100 and 5). The crawl stops after `limits.wall_clock_s`, also on `/crawl/stream`.
+- `max_pages` and `max_depth` are clamped to `limits.max_pages` and `limits.max_depth` in `config.yml` (default 100 and 5; `max_pages: 0` means no limit). The crawl stops after `limits.wall_clock_s`, also on `/crawl/stream`.
- **Allowed types:** `BFSDeepCrawlStrategy`, `DFSDeepCrawlStrategy`, `BestFirstCrawlingStrategy`; `FilterChain` with `URLPatternFilter`, `DomainFilter`, `ContentTypeFilter`; scorers `KeywordRelevanceScorer`, `CompositeScorer`, `DomainAuthorityScorer`, `FreshnessScorer`, `PathDepthScorer`. `SEOFilter` and `ContentRelevanceFilter` are refused (they fetch pages outside the egress controls).
- `URLPatternFilter` accepts globs that start with `*` (`*/blog/*`), `/prefix/*` or `*.ext`. Regex patterns are refused.
-- `resume_state` is refused, and so is `fetch_ssl_certificate` together with a deep crawl. Lists (patterns, keywords, filters, scorers) are limited to 100 items. Discovered links longer than 2048 characters are skipped.
+- `resume_state` is refused, and so is `fetch_ssl_certificate` together with a deep crawl. Lists (patterns, keywords, filters, scorers) are limited to 100 items. One strategy may have at most 20 filters and scorers and 200 list items in total, and a `CompositeScorer` cannot contain another `CompositeScorer`. Discovered links longer than 2048 characters are skipped.
Turn the flag on only when you accept that one request can fetch many pages.
diff --git a/tests/unit/test_config_provenance.py b/tests/unit/test_config_provenance.py
index 883b7170..62f167a6 100644
--- a/tests/unit/test_config_provenance.py
+++ b/tests/unit/test_config_provenance.py
@@ -158,6 +158,33 @@ def test_deep_crawl_ssl_certificate_refused(monkeypatch):
CrawlerRunConfig.load(data, provenance=UNTRUSTED)
+def _pattern_chain(*sizes):
+ return {"type": "FilterChain", "params": {"filters": [
+ {"type": "URLPatternFilter", "params": {"patterns": [f"*/p{i}/*" for i in range(n)]}}
+ for n in sizes]}}
+
+
[email protected]("strategy, match", [
+ ({"filter_chain": _pattern_chain(101)}, "limited to 100 items"),
+ ({"filter_chain": _pattern_chain(*(1,) * 20)}, "in total"),
+ ({"filter_chain": _pattern_chain(100, 100, 1)}, "in total"),
+ ({"url_scorer": {"type": "KeywordRelevanceScorer", "params": {
+ "keywords": {"type": "dict", "value": {f"k{i}": 1 for i in range(201)}}}}}, "in total"),
+])
+def test_deep_crawl_size_budget_refused(monkeypatch, strategy, match):
+ monkeypatch.setenv("CRAWL4AI_ALLOW_DEEP_CRAWL", "true")
+ with pytest.raises(UntrustedConfigError, match=match):
+ CrawlerRunConfig.load(_deep_crawl(**strategy), provenance=UNTRUSTED)
+
+
+def test_deep_crawl_nested_composite_scorer_refused(monkeypatch):
+ monkeypatch.setenv("CRAWL4AI_ALLOW_DEEP_CRAWL", "true")
+ inner = {"type": "CompositeScorer", "params": {"scorers": [{"type": "PathDepthScorer", "params": {}}]}}
+ data = _deep_crawl(url_scorer={"type": "CompositeScorer", "params": {"scorers": [inner]}})
+ with pytest.raises(UntrustedConfigError, match="nested"):
+ CrawlerRunConfig.load(data, provenance=UNTRUSTED)
+
+
def test_url_pattern_star_led_glob_is_linear_and_unchanged():
import time
from crawl4ai.deep_crawling.filters import URLPatternFilter