Official Crawl4AI Cloud: a hosted web scraping, extraction, and search API for AI agents. Free API key, 1,000 credits a month, no card. Fully open source: this repo is the Apache-2.0 crawler that powers it, and you can self-host it.
Python aiai-agentscrawl4ai-cloudcrawlerdata-extractionllmmarkdownmcpopen-sourceplaywrightpythonragscraperweb-crawlerweb-crawlingweb-scraping
fix(config): validate body visibility timeout
nightcityblade64bf678e1384
diff --git a/crawl4ai/async_configs.py b/crawl4ai/async_configs.pyindex 084060f9..3efb0d38 100644--- a/crawl4ai/async_configs.py+++ b/crawl4ai/async_configs.py@@ -300,7 +300,7 @@ def _clamp_untrusted(type_name: str, params: dict) -> dict:return min(int(v), _MAX_TIMEOUT_MS)if type_name == "CrawlerRunConfig":- for f in ("page_timeout", "wait_for_timeout"):+ for f in ("page_timeout", "wait_for_timeout", "body_visibility_timeout"):if f in params:params[f] = _cap_timeout(params[f])if isinstance(params.get("max_scroll_steps"), int):@@ -1774,6 +1774,12 @@ class CrawlerRunConfig():self.c4a_script = c4a_scriptself.js_only = js_onlyself.ignore_body_visibility = ignore_body_visibility+ if (+ not isinstance(body_visibility_timeout, (int, float))+ or isinstance(body_visibility_timeout, bool)+ or body_visibility_timeout <= 0+ ):+ raise ValueError("body_visibility_timeout must be a positive number")self.body_visibility_timeout = body_visibility_timeoutself.scan_full_page = scan_full_pageself.scroll_delay = scroll_delaydiff --git a/docs/md_v2/api/parameters.md b/docs/md_v2/api/parameters.mdindex 568e14c3..f9f759a5 100644--- a/docs/md_v2/api/parameters.md+++ b/docs/md_v2/api/parameters.md@@ -159,6 +159,7 @@ Use these for controlling whether you read or write from a local content cache.| **`c4a_script`** | `str or list[str]` (None) | C4A script that compiles to JavaScript. Alternative to writing raw JS. || **`js_only`** | `bool` (False) | If `True`, indicates we're reusing an existing session and only applying JS. No full reload. || **`ignore_body_visibility`** | `bool` (True) | Skip checking if `<body>` is visible. Usually best to keep `True`. |+| **`body_visibility_timeout`** | `int` (30000) | Maximum time in milliseconds to wait for `<body>` to become visible. Must be positive. || **`scan_full_page`** | `bool` (False) | If `True`, auto-scroll the page to load dynamic content (infinite scroll). || **`scroll_delay`** | `float` (0.2) | Delay between scroll steps when scanning the full page (`scan_full_page=True`) or capturing full-page screenshots. || **`max_scroll_steps`** | `int or None` (None) | Maximum number of scroll steps during full page scan. If None, scrolls until entire page is loaded. |diff --git a/docs/md_v2/complete-sdk-reference.md b/docs/md_v2/complete-sdk-reference.mdindex aa0517b2..1f329907 100644--- a/docs/md_v2/complete-sdk-reference.md+++ b/docs/md_v2/complete-sdk-reference.md@@ -1791,6 +1791,7 @@ run_cfg = CrawlerRunConfig(| **`js_code_before_wait`** | `str or list[str]` (None) | JavaScript to run **before** `wait_for`. Use for triggering loading that `wait_for` then checks. || **`js_only`** | `bool` (False) | If `True`, indicates we're reusing an existing session and only applying JS. No full reload. || **`ignore_body_visibility`** | `bool` (True) | Skip checking if `<body>` is visible. Usually best to keep `True`. |+| **`body_visibility_timeout`** | `int` (30000) | Maximum time in milliseconds to wait for `<body>` to become visible. Must be positive. || **`scan_full_page`** | `bool` (False) | If `True`, auto-scroll the page to load dynamic content (infinite scroll). || **`scroll_delay`** | `float` (0.2) | Delay between scroll steps when scanning the full page (`scan_full_page=True`) or capturing full-page screenshots. || **`process_iframes`** | `bool` (False) | Inlines iframe content for single-page extraction. |diff --git a/tests/test_config_defaults.py b/tests/test_config_defaults.pyindex 93a9484c..3a7fba64 100644--- a/tests/test_config_defaults.py+++ b/tests/test_config_defaults.py@@ -1,7 +1,12 @@"""Tests for BrowserConfig.set_defaults / CrawlerRunConfig.set_defaults."""+from types import SimpleNamespace+from unittest.mock import AsyncMock, MagicMock+import pytest+from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig+from crawl4ai.async_crawler_strategy import AsyncPlaywrightCrawlerStrategy@pytest.fixture(autouse=True)@@ -238,6 +243,46 @@ class TestDumpLoad:assert loaded.scan_full_page is Trueassert loaded.body_visibility_timeout == 2000+ @pytest.mark.parametrize("timeout", [None, 0, -1, "1000", True])+ def test_body_visibility_timeout_must_be_positive_number(self, timeout):+ with pytest.raises(ValueError, match="must be a positive number"):+ CrawlerRunConfig(body_visibility_timeout=timeout)++ def test_untrusted_body_visibility_timeout_is_clamped(self):+ from crawl4ai.async_configs import Provenance++ config = CrawlerRunConfig.load(+ {"body_visibility_timeout": 500_000}, provenance=Provenance.UNTRUSTED+ )+ assert config.body_visibility_timeout == 60_000++ @pytest.mark.asyncio+ async def test_body_visibility_timeout_reaches_wait(self):+ page = MagicMock()+ page.evaluate = AsyncMock()+ page.set_content = AsyncMock()+ page.wait_for_selector = AsyncMock()+ page.content = AsyncMock(return_value="<body>visible</body>")++ strategy = AsyncPlaywrightCrawlerStrategy.__new__(+ AsyncPlaywrightCrawlerStrategy+ )+ strategy.browser_config = SimpleNamespace(+ use_persistent_context=False, accept_downloads=False, text_mode=True+ )+ strategy.browser_manager = SimpleNamespace(+ get_page=AsyncMock(return_value=(page, MagicMock()))+ )+ strategy.execute_hook = AsyncMock()+ strategy.csp_compliant_wait = AsyncMock(return_value=True)++ config = CrawlerRunConfig(+ session_id="body-timeout-test", body_visibility_timeout=1234+ )+ await strategy._crawl_web("raw:<body>visible</body>", config)++ assert strategy.csp_compliant_wait.await_args.kwargs["timeout"] == 1234+def test_to_dict_includes_user_default_values(self):BrowserConfig.set_defaults(headless=False)cfg = BrowserConfig()