Official Crawl4AI Cloud: a hosted web scraping, extraction, and search API for AI agents. Free API key, 1,000 credits a month, no card. Fully open source: this repo is the Apache-2.0 crawler that powers it, and you can self-host it.
Python aiai-agentscrawl4ai-cloudcrawlerdata-extractionllmmarkdownmcpopen-sourceplaywrightpythonragscraperweb-crawlerweb-crawlingweb-scraping
feat(docker): total size budget for untrusted deep-crawl strategies
Soham Kukretid6f8a755d5a8
diff --git a/crawl4ai/async_configs.py b/crawl4ai/async_configs.pyindex c131ff39..ad51cb17 100644--- a/crawl4ai/async_configs.py+++ b/crawl4ai/async_configs.py@@ -316,6 +316,8 @@ _MAX_LINK_PREVIEW_LINKS = 100 # the class default_MAX_LINK_PREVIEW_CONCURRENCY = 10 # the class default_MAX_LINK_PREVIEW_TIMEOUT_S = 10 # seconds here, not ms_MAX_DEEP_CRAWL_LIST = 100 # patterns, keywords, filters, scorers per object+_MAX_DEEP_CRAWL_OBJECTS = 20 # filters and scorers in one strategy+_MAX_DEEP_CRAWL_ITEMS = 200 # list and dict items (patterns, keywords, ...) in one strategydef _max_timeout_ms() -> int:@@ -358,6 +360,16 @@ def _deep_crawl_allowed() -> bool:return os.getenv("CRAWL4AI_ALLOW_DEEP_CRAWL", "false").lower() == "true"+def _all_nodes(node):+ """Yield every value in a request tree. An object's params dict is entered but not yielded."""+ yield node+ if _is_typed_shape(node) and isinstance(node.get("params"), dict):+ node = node["params"]+ children = node.values() if isinstance(node, dict) else node if isinstance(node, list) else []+ for child in children:+ yield from _all_nodes(child)++def _filter_untrusted_fields(type_name: str, params: dict) -> dict:"""Drop non-allowlisted fields and raise on forbidden (power) fields."""forbidden = UNTRUSTED_FORBIDDEN_FIELDS.get(type_name, set())@@ -380,6 +392,16 @@ def _filter_untrusted_fields(type_name: str, params: dict) -> dict:raise UntrustedConfigError(f"{type_name} list parameters are limited to {_MAX_DEEP_CRAWL_LIST} items")+ if type_name in ("BFSDeepCrawlStrategy", "DFSDeepCrawlStrategy", "BestFirstCrawlingStrategy"):+ # Filters and scorers run synchronously on the event loop for every discovered link.+ nodes = [n for value in out.values() for n in _all_nodes(value)]+ filters_and_scorers = sum(1 for n in nodes if isinstance(n, dict) and n.get("type") in UNTRUSTED_DEEP_CRAWL_TYPES)+ list_items = sum(len(n) for n in nodes if isinstance(n, (list, dict)) and not _is_typed_shape(n))+ if filters_and_scorers > _MAX_DEEP_CRAWL_OBJECTS or list_items > _MAX_DEEP_CRAWL_ITEMS:+ raise UntrustedConfigError(+ f"{type_name} is limited to {_MAX_DEEP_CRAWL_OBJECTS} filters and scorers "+ f"and {_MAX_DEEP_CRAWL_ITEMS} list items in total"+ )if type_name == "URLPatternFilter":# Regex and *.domain:// patterns compile unescaped, and globs not led by * search in# quadratic time; allow only *-led globs, /prefix/* and *.ext.@@ -396,6 +418,10 @@ def _filter_untrusted_fields(type_name: str, params: dict) -> dict:"URLPatternFilter accepts only globs that start with '*' or end with a ""single '/*' from an untrusted request")+ if type_name == "CompositeScorer" and any(+ isinstance(s, dict) and s.get("type") == "CompositeScorer" for s in out.get("scorers") or []+ ):+ raise UntrustedConfigError("CompositeScorer cannot be nested from an untrusted request")if type_name == "CrawlerRunConfig" and out.get("deep_crawl_strategy") and out.get("fetch_ssl_certificate"):# The certificate fetch dials each discovered link directly, outside the egress proxy.raise UntrustedConfigError(diff --git a/deploy/docker/README.md b/deploy/docker/README.mdindex 1b8c5775..a5281601 100644--- a/deploy/docker/README.md+++ b/deploy/docker/README.md@@ -612,10 +612,10 @@ variable itself, so a value in `.llm.env` is ignored.Rules the server applies:- **One start URL** per request, and not inside `crawler_configs`.-- `max_pages` and `max_depth` are clamped to `limits.max_pages` and `limits.max_depth` in `config.yml` (default 100 and 5). The crawl stops after `limits.wall_clock_s`, also on `/crawl/stream`.+- `max_pages` and `max_depth` are clamped to `limits.max_pages` and `limits.max_depth` in `config.yml` (default 100 and 5; `max_pages: 0` means no limit). The crawl stops after `limits.wall_clock_s`, also on `/crawl/stream`.- **Allowed types:** `BFSDeepCrawlStrategy`, `DFSDeepCrawlStrategy`, `BestFirstCrawlingStrategy`; `FilterChain` with `URLPatternFilter`, `DomainFilter`, `ContentTypeFilter`; scorers `KeywordRelevanceScorer`, `CompositeScorer`, `DomainAuthorityScorer`, `FreshnessScorer`, `PathDepthScorer`. `SEOFilter` and `ContentRelevanceFilter` are refused (they fetch pages outside the egress controls).- `URLPatternFilter` accepts globs that start with `*` (`*/blog/*`), `/prefix/*` or `*.ext`. Regex patterns are refused.-- `resume_state` is refused, and so is `fetch_ssl_certificate` together with a deep crawl. Lists (patterns, keywords, filters, scorers) are limited to 100 items. Discovered links longer than 2048 characters are skipped.+- `resume_state` is refused, and so is `fetch_ssl_certificate` together with a deep crawl. Lists (patterns, keywords, filters, scorers) are limited to 100 items. One strategy may have at most 20 filters and scorers and 200 list items in total, and a `CompositeScorer` cannot contain another `CompositeScorer`. Discovered links longer than 2048 characters are skipped.Turn the flag on only when you accept that one request can fetch many pages.diff --git a/deploy/docker/config.yml b/deploy/docker/config.ymlindex c22bf381..8bf16a11 100644--- a/deploy/docker/config.yml+++ b/deploy/docker/config.yml@@ -37,7 +37,7 @@ redis:# (the pre-hardening behavior), so you can relax or disable each one.limits:max_body_bytes: 10485760 # 10 MiB request body cap (413 if exceeded); 0 = unbounded- max_pages: 100 # deep-crawl page budget clamp (deep crawl needs CRAWL4AI_ALLOW_DEEP_CRAWL=true)+ max_pages: 100 # deep-crawl page budget clamp (deep crawl needs CRAWL4AI_ALLOW_DEEP_CRAWL=true); 0 = no limitmax_depth: 5 # deep-crawl depth clampwall_clock_s: 300 # per-crawl deadline in seconds (504 on timeout); 0 = no deadlinequeue: # background job queue for /crawl/job and /llm/jobdiff --git a/deploy/docker/governor.py b/deploy/docker/governor.pyindex 4cabe0ab..0c1f8d6a 100644--- a/deploy/docker/governor.py+++ b/deploy/docker/governor.py@@ -90,7 +90,8 @@ def clamp_deep_crawl(crawler_config, *, max_pages: int = DEFAULT_MAX_PAGES,return# NaN fails every comparison, so test for "inside the range", not "above the cap".mp = getattr(dc, "max_pages", None)- if not isinstance(mp, (int, float)) or not 0 <= mp <= max_pages:+ # max_pages=0 means no limit, like the other limits in config.yml.+ if max_pages and (not isinstance(mp, (int, float)) or not 0 <= mp <= max_pages):try:dc.max_pages = max_pagesexcept Exception:diff --git a/deploy/docker/tests/test_security_resource_caps.py b/deploy/docker/tests/test_security_resource_caps.pyindex a60a3cc2..f20d3215 100644--- a/deploy/docker/tests/test_security_resource_caps.py+++ b/deploy/docker/tests/test_security_resource_caps.py@@ -69,6 +69,14 @@ class TestDeepCrawlClamp:assert not guard.apply("https://a.com/" + "x" * 3000)assert guard.apply("https://a.com/blog/post")+ def test_zero_max_pages_means_no_limit(self):+ from governor import clamp_deep_crawl+ from crawl4ai.deep_crawling import BFSDeepCrawlStrategy+ from crawl4ai import CrawlerRunConfig+ cfg = CrawlerRunConfig(deep_crawl_strategy=BFSDeepCrawlStrategy(max_depth=2))+ clamp_deep_crawl(cfg, max_pages=0)+ assert cfg.deep_crawl_strategy.max_pages == float("inf")+class TestUntrustedDeepCrawlStillForbidden:def test_request_cannot_set_deep_crawl(self):diff --git a/docs/md_v2/core/self-hosting.md b/docs/md_v2/core/self-hosting.mdindex f5445c98..5f6ae1c4 100644--- a/docs/md_v2/core/self-hosting.md+++ b/docs/md_v2/core/self-hosting.md@@ -1339,10 +1339,10 @@ the project root, or export it in your shell.Rules the server applies:- **One start URL** per request, and not inside `crawler_configs`.-- `max_pages` and `max_depth` are clamped to `limits.max_pages` and `limits.max_depth` in `config.yml` (default 100 and 5). The crawl stops after `limits.wall_clock_s`, also on `/crawl/stream`.+- `max_pages` and `max_depth` are clamped to `limits.max_pages` and `limits.max_depth` in `config.yml` (default 100 and 5; `max_pages: 0` means no limit). The crawl stops after `limits.wall_clock_s`, also on `/crawl/stream`.- **Allowed types:** `BFSDeepCrawlStrategy`, `DFSDeepCrawlStrategy`, `BestFirstCrawlingStrategy`; `FilterChain` with `URLPatternFilter`, `DomainFilter`, `ContentTypeFilter`; scorers `KeywordRelevanceScorer`, `CompositeScorer`, `DomainAuthorityScorer`, `FreshnessScorer`, `PathDepthScorer`. `SEOFilter` and `ContentRelevanceFilter` are refused (they fetch pages outside the egress controls).- `URLPatternFilter` accepts globs that start with `*` (`*/blog/*`), `/prefix/*` or `*.ext`. Regex patterns are refused.-- `resume_state` is refused, and so is `fetch_ssl_certificate` together with a deep crawl. Lists (patterns, keywords, filters, scorers) are limited to 100 items. Discovered links longer than 2048 characters are skipped.+- `resume_state` is refused, and so is `fetch_ssl_certificate` together with a deep crawl. Lists (patterns, keywords, filters, scorers) are limited to 100 items. One strategy may have at most 20 filters and scorers and 200 list items in total, and a `CompositeScorer` cannot contain another `CompositeScorer`. Discovered links longer than 2048 characters are skipped.Turn the flag on only when you accept that one request can fetch many pages.diff --git a/tests/unit/test_config_provenance.py b/tests/unit/test_config_provenance.pyindex 883b7170..62f167a6 100644--- a/tests/unit/test_config_provenance.py+++ b/tests/unit/test_config_provenance.py@@ -158,6 +158,33 @@ def test_deep_crawl_ssl_certificate_refused(monkeypatch):CrawlerRunConfig.load(data, provenance=UNTRUSTED)+def _pattern_chain(*sizes):+ return {"type": "FilterChain", "params": {"filters": [+ {"type": "URLPatternFilter", "params": {"patterns": [f"*/p{i}/*" for i in range(n)]}}+ for n in sizes]}}++[email protected]("strategy, match", [+ ({"filter_chain": _pattern_chain(101)}, "limited to 100 items"),+ ({"filter_chain": _pattern_chain(*(1,) * 20)}, "in total"),+ ({"filter_chain": _pattern_chain(100, 100, 1)}, "in total"),+ ({"url_scorer": {"type": "KeywordRelevanceScorer", "params": {+ "keywords": {"type": "dict", "value": {f"k{i}": 1 for i in range(201)}}}}}, "in total"),+])+def test_deep_crawl_size_budget_refused(monkeypatch, strategy, match):+ monkeypatch.setenv("CRAWL4AI_ALLOW_DEEP_CRAWL", "true")+ with pytest.raises(UntrustedConfigError, match=match):+ CrawlerRunConfig.load(_deep_crawl(**strategy), provenance=UNTRUSTED)+++def test_deep_crawl_nested_composite_scorer_refused(monkeypatch):+ monkeypatch.setenv("CRAWL4AI_ALLOW_DEEP_CRAWL", "true")+ inner = {"type": "CompositeScorer", "params": {"scorers": [{"type": "PathDepthScorer", "params": {}}]}}+ data = _deep_crawl(url_scorer={"type": "CompositeScorer", "params": {"scorers": [inner]}})+ with pytest.raises(UntrustedConfigError, match="nested"):+ CrawlerRunConfig.load(data, provenance=UNTRUSTED)++def test_url_pattern_star_led_glob_is_linear_and_unchanged():import timefrom crawl4ai.deep_crawling.filters import URLPatternFilter