Skip to content

Official Crawl4AI Cloud: a hosted web scraping, extraction, and search API for AI agents. Free API key, 1,000 credits a month, no card. Fully open source: this repo is the Apache-2.0 crawler that powers it, and you can self-host it.

Merge pull request #2212 from damusix/fix/configurable-max-timeout

Nasrin8ccfbcd81ea2
diff --cc crawl4ai/async_configs.py
index 383079fe,92ae71ed..4e2037f4
--- a/crawl4ai/async_configs.py
+++ b/crawl4ai/async_configs.py
@@@ -293,13 -270,46 +293,48 @@@ UNTRUSTED_FIELD_ALLOWLIST =
}
# Upper bounds applied to attacker-influenced quantities after filtering.
- _MAX_TIMEOUT_MS = 60_000
+ _DEFAULT_MAX_TIMEOUT_MS = 60_000
_MAX_SCROLL_STEPS = 1000
_MAX_VIEWPORT = 4000
+_MAX_PDF_BYTES = 100 * 1024 * 1024
+_MAX_PDF_PAGES = 2000
+ def _max_timeout_ms() -> int:
+ """Ceiling for the untrusted timeout fields, in milliseconds.
+
+ 60s is the right bound for a server reachable by untrusted callers, and
+ stays the default. An operator whose deployment is not public — a crawler
+ on a private network fetching pages that legitimately take minutes — can
+ raise it with CRAWL4AI_MAX_TIMEOUT_MS, or lower it to tighten the bound.
+
+ Read per call rather than captured at import so the setting applies
+ wherever the process picked its environment up, and so a test can set it
+ without reloading the module. A value that is not a positive integer is
+ refused loudly and the default kept: a typo here would silently widen a
+ DoS bound, which is the one outcome worse than the timeout being fixed.
+ """
+ raw = os.getenv("CRAWL4AI_MAX_TIMEOUT_MS")
+
+ if raw is None or raw == "":
+ return _DEFAULT_MAX_TIMEOUT_MS
+
+ try:
+ ceiling = int(raw)
+ except ValueError:
+ ceiling = 0
+
+ if ceiling <= 0:
+ warnings.warn(
+ f"CRAWL4AI_MAX_TIMEOUT_MS={raw!r} is not a positive integer; "
+ f"keeping the {_DEFAULT_MAX_TIMEOUT_MS}ms default.",
+ stacklevel=2,
+ )
+ return _DEFAULT_MAX_TIMEOUT_MS
+
+ return ceiling
+
+
def _filter_untrusted_fields(type_name: str, params: dict) -> dict:
"""Drop non-allowlisted fields and raise on forbidden (power) fields."""
forbidden = UNTRUSTED_FORBIDDEN_FIELDS.get(type_name, set())