Official Crawl4AI Cloud: a hosted web scraping, extraction, and search API for AI agents. Free API key, 1,000 credits a month, no card. Fully open source: this repo is the Apache-2.0 crawler that powers it, and you can self-host it.
Python aiai-agentscrawl4ai-cloudcrawlerdata-extractionllmmarkdownmcpopen-sourceplaywrightpythonragscraperweb-crawlerweb-crawlingweb-scraping
Merge pull request #2212 from damusix/fix/configurable-max-timeout
Nasrin8ccfbcd81ea2
diff --cc crawl4ai/async_configs.pyindex 383079fe,92ae71ed..4e2037f4--- a/crawl4ai/async_configs.py+++ b/crawl4ai/async_configs.py@@@ -293,13 -270,46 +293,48 @@@ UNTRUSTED_FIELD_ALLOWLIST =}# Upper bounds applied to attacker-influenced quantities after filtering.- _MAX_TIMEOUT_MS = 60_000+ _DEFAULT_MAX_TIMEOUT_MS = 60_000_MAX_SCROLL_STEPS = 1000_MAX_VIEWPORT = 4000+_MAX_PDF_BYTES = 100 * 1024 * 1024+_MAX_PDF_PAGES = 2000+ def _max_timeout_ms() -> int:+ """Ceiling for the untrusted timeout fields, in milliseconds.++ 60s is the right bound for a server reachable by untrusted callers, and+ stays the default. An operator whose deployment is not public — a crawler+ on a private network fetching pages that legitimately take minutes — can+ raise it with CRAWL4AI_MAX_TIMEOUT_MS, or lower it to tighten the bound.++ Read per call rather than captured at import so the setting applies+ wherever the process picked its environment up, and so a test can set it+ without reloading the module. A value that is not a positive integer is+ refused loudly and the default kept: a typo here would silently widen a+ DoS bound, which is the one outcome worse than the timeout being fixed.+ """+ raw = os.getenv("CRAWL4AI_MAX_TIMEOUT_MS")++ if raw is None or raw == "":+ return _DEFAULT_MAX_TIMEOUT_MS++ try:+ ceiling = int(raw)+ except ValueError:+ ceiling = 0++ if ceiling <= 0:+ warnings.warn(+ f"CRAWL4AI_MAX_TIMEOUT_MS={raw!r} is not a positive integer; "+ f"keeping the {_DEFAULT_MAX_TIMEOUT_MS}ms default.",+ stacklevel=2,+ )+ return _DEFAULT_MAX_TIMEOUT_MS++ return ceiling++def _filter_untrusted_fields(type_name: str, params: dict) -> dict:"""Drop non-allowlisted fields and raise on forbidden (power) fields."""forbidden = UNTRUSTED_FORBIDDEN_FIELDS.get(type_name, set())