Skip to content

Official Crawl4AI Cloud: a hosted web scraping, extraction, and search API for AI agents. Free API key, 1,000 credits a month, no card. Fully open source: this repo is the Apache-2.0 crawler that powers it, and you can self-host it.

fix(config): keep malformed timeouts at the default ceiling

ntohidi61eb573416fb
diff --git a/crawl4ai/async_configs.py b/crawl4ai/async_configs.py
index 4e2037f4..a5c2b657 100644
--- a/crawl4ai/async_configs.py
+++ b/crawl4ai/async_configs.py
@@ -357,8 +357,10 @@ def _clamp_untrusted(type_name: str, params: dict) -> dict:
def _cap_timeout(v):
# 0 historically meant "no timeout"; treat as the cap, never unbounded.
+ # Malformed input is the one value an attacker gets for free, so it
+ # falls back to the 60s default rather than to a raised ceiling.
if not isinstance(v, (int, float)) or v <= 0:
- return ceiling
+ return min(_DEFAULT_MAX_TIMEOUT_MS, ceiling)
return min(int(v), ceiling)
if type_name == "CrawlerRunConfig":
diff --git a/deploy/docker/MIGRATION.md b/deploy/docker/MIGRATION.md
index c508e220..5123c950 100644
--- a/deploy/docker/MIGRATION.md
+++ b/deploy/docker/MIGRATION.md
@@ -189,6 +189,16 @@ a smaller value tightens it. A value that is not a positive integer is refused
with a warning and the 60000ms default kept, so a typo cannot silently widen
the bound.
+Raising this ceiling alone is not enough. Two other deadlines cut a crawl
+short first, and both are in `config.yml`:
+
+- `limits.wall_clock_s` (default `300`) — the per-crawl deadline; the request
+ gets a 504 at that point no matter what `page_timeout` says.
+- `crawler.timeouts.batch_process` (default `300.0`) — the batch crawl budget.
+
+So a 300000ms ceiling needs `wall_clock_s` and `batch_process` raised past 300
+too, or the extra timeout can never be reached.
+
### Error responses are generic
5xx responses return `{"error": "Internal server error", "correlation_id": "…"}`.
diff --git a/tests/test_config_defaults.py b/tests/test_config_defaults.py
index ad86f92f..72f98ed3 100644
--- a/tests/test_config_defaults.py
+++ b/tests/test_config_defaults.py
@@ -361,7 +361,7 @@ class TestMaxTimeoutCeiling:
# A typo must not silently widen a DoS bound, so the default is kept and
# the operator is told rather than left to find out under load.
- @pytest.mark.parametrize("value", ["", "abc", "0", "-1", "60_000", "1e5"])
+ @pytest.mark.parametrize("value", ["", "abc", "0", "-1", "1e5"])
def test_a_non_positive_integer_keeps_the_default(self, monkeypatch, value):
monkeypatch.setenv("CRAWL4AI_MAX_TIMEOUT_MS", value)
@@ -373,6 +373,38 @@ class TestMaxTimeoutCeiling:
assert config.page_timeout == 60_000
+ def test_an_underscored_integer_is_accepted(self, monkeypatch):
+ # int("60_000") == 60000 in Python, so this is a valid ceiling, not a typo.
+ monkeypatch.setenv("CRAWL4AI_MAX_TIMEOUT_MS", "120_000")
+
+ config = CrawlerRunConfig.load(
+ {"page_timeout": 500_000}, provenance=Provenance.UNTRUSTED
+ )
+
+ assert config.page_timeout == 120_000
+
+ def test_malformed_timeout_falls_back_to_the_default_not_the_ceiling(
+ self, monkeypatch
+ ):
+ # The one field an untrusted caller gets for free must not inherit a
+ # raised ceiling just by being junk.
+ monkeypatch.setenv("CRAWL4AI_MAX_TIMEOUT_MS", "300000")
+
+ config = CrawlerRunConfig.load(
+ {"page_timeout": 0}, provenance=Provenance.UNTRUSTED
+ )
+
+ assert config.page_timeout == 60_000
+
+ def test_malformed_timeout_respects_a_tightened_ceiling(self, monkeypatch):
+ monkeypatch.setenv("CRAWL4AI_MAX_TIMEOUT_MS", "5000")
+
+ config = CrawlerRunConfig.load(
+ {"page_timeout": "abc"}, provenance=Provenance.UNTRUSTED
+ )
+
+ assert config.page_timeout == 5_000
+
@pytest.mark.parametrize("value", ["abc", "0", "-1"])
def test_a_bad_value_warns(self, monkeypatch, value):
monkeypatch.setenv("CRAWL4AI_MAX_TIMEOUT_MS", value)