Official Crawl4AI Cloud: a hosted web scraping, extraction, and search API for AI agents. Free API key, 1,000 credits a month, no card. Fully open source: this repo is the Apache-2.0 crawler that powers it, and you can self-host it.
Python aiai-agentscrawl4ai-cloudcrawlerdata-extractionllmmarkdownmcpopen-sourceplaywrightpythonragscraperweb-crawlerweb-crawlingweb-scraping
fix(config): keep malformed timeouts at the default ceiling
ntohidi61eb573416fb
diff --git a/crawl4ai/async_configs.py b/crawl4ai/async_configs.pyindex 4e2037f4..a5c2b657 100644--- a/crawl4ai/async_configs.py+++ b/crawl4ai/async_configs.py@@ -357,8 +357,10 @@ def _clamp_untrusted(type_name: str, params: dict) -> dict:def _cap_timeout(v):# 0 historically meant "no timeout"; treat as the cap, never unbounded.+ # Malformed input is the one value an attacker gets for free, so it+ # falls back to the 60s default rather than to a raised ceiling.if not isinstance(v, (int, float)) or v <= 0:- return ceiling+ return min(_DEFAULT_MAX_TIMEOUT_MS, ceiling)return min(int(v), ceiling)if type_name == "CrawlerRunConfig":diff --git a/deploy/docker/MIGRATION.md b/deploy/docker/MIGRATION.mdindex c508e220..5123c950 100644--- a/deploy/docker/MIGRATION.md+++ b/deploy/docker/MIGRATION.md@@ -189,6 +189,16 @@ a smaller value tightens it. A value that is not a positive integer is refusedwith a warning and the 60000ms default kept, so a typo cannot silently widenthe bound.+Raising this ceiling alone is not enough. Two other deadlines cut a crawl+short first, and both are in `config.yml`:++- `limits.wall_clock_s` (default `300`) — the per-crawl deadline; the request+ gets a 504 at that point no matter what `page_timeout` says.+- `crawler.timeouts.batch_process` (default `300.0`) — the batch crawl budget.++So a 300000ms ceiling needs `wall_clock_s` and `batch_process` raised past 300+too, or the extra timeout can never be reached.+### Error responses are generic5xx responses return `{"error": "Internal server error", "correlation_id": "…"}`.diff --git a/tests/test_config_defaults.py b/tests/test_config_defaults.pyindex ad86f92f..72f98ed3 100644--- a/tests/test_config_defaults.py+++ b/tests/test_config_defaults.py@@ -361,7 +361,7 @@ class TestMaxTimeoutCeiling:# A typo must not silently widen a DoS bound, so the default is kept and# the operator is told rather than left to find out under load.- @pytest.mark.parametrize("value", ["", "abc", "0", "-1", "60_000", "1e5"])+ @pytest.mark.parametrize("value", ["", "abc", "0", "-1", "1e5"])def test_a_non_positive_integer_keeps_the_default(self, monkeypatch, value):monkeypatch.setenv("CRAWL4AI_MAX_TIMEOUT_MS", value)@@ -373,6 +373,38 @@ class TestMaxTimeoutCeiling:assert config.page_timeout == 60_000+ def test_an_underscored_integer_is_accepted(self, monkeypatch):+ # int("60_000") == 60000 in Python, so this is a valid ceiling, not a typo.+ monkeypatch.setenv("CRAWL4AI_MAX_TIMEOUT_MS", "120_000")++ config = CrawlerRunConfig.load(+ {"page_timeout": 500_000}, provenance=Provenance.UNTRUSTED+ )++ assert config.page_timeout == 120_000++ def test_malformed_timeout_falls_back_to_the_default_not_the_ceiling(+ self, monkeypatch+ ):+ # The one field an untrusted caller gets for free must not inherit a+ # raised ceiling just by being junk.+ monkeypatch.setenv("CRAWL4AI_MAX_TIMEOUT_MS", "300000")++ config = CrawlerRunConfig.load(+ {"page_timeout": 0}, provenance=Provenance.UNTRUSTED+ )++ assert config.page_timeout == 60_000++ def test_malformed_timeout_respects_a_tightened_ceiling(self, monkeypatch):+ monkeypatch.setenv("CRAWL4AI_MAX_TIMEOUT_MS", "5000")++ config = CrawlerRunConfig.load(+ {"page_timeout": "abc"}, provenance=Provenance.UNTRUSTED+ )++ assert config.page_timeout == 5_000+@pytest.mark.parametrize("value", ["abc", "0", "-1"])def test_a_bad_value_warns(self, monkeypatch, value):monkeypatch.setenv("CRAWL4AI_MAX_TIMEOUT_MS", value)