Skip to content

Official Crawl4AI Cloud: a hosted web scraping, extraction, and search API for AI agents. Free API key, 1,000 credits a month, no card. Fully open source: this repo is the Apache-2.0 crawler that powers it, and you can self-host it.

fix(config): validate body visibility timeout

nightcityblade64bf678e1384
diff --git a/crawl4ai/async_configs.py b/crawl4ai/async_configs.py
index 084060f9..3efb0d38 100644
--- a/crawl4ai/async_configs.py
+++ b/crawl4ai/async_configs.py
@@ -300,7 +300,7 @@ def _clamp_untrusted(type_name: str, params: dict) -> dict:
return min(int(v), _MAX_TIMEOUT_MS)
if type_name == "CrawlerRunConfig":
- for f in ("page_timeout", "wait_for_timeout"):
+ for f in ("page_timeout", "wait_for_timeout", "body_visibility_timeout"):
if f in params:
params[f] = _cap_timeout(params[f])
if isinstance(params.get("max_scroll_steps"), int):
@@ -1774,6 +1774,12 @@ class CrawlerRunConfig():
self.c4a_script = c4a_script
self.js_only = js_only
self.ignore_body_visibility = ignore_body_visibility
+ if (
+ not isinstance(body_visibility_timeout, (int, float))
+ or isinstance(body_visibility_timeout, bool)
+ or body_visibility_timeout <= 0
+ ):
+ raise ValueError("body_visibility_timeout must be a positive number")
self.body_visibility_timeout = body_visibility_timeout
self.scan_full_page = scan_full_page
self.scroll_delay = scroll_delay
diff --git a/docs/md_v2/api/parameters.md b/docs/md_v2/api/parameters.md
index 568e14c3..f9f759a5 100644
--- a/docs/md_v2/api/parameters.md
+++ b/docs/md_v2/api/parameters.md
@@ -159,6 +159,7 @@ Use these for controlling whether you read or write from a local content cache.
| **`c4a_script`** | `str or list[str]` (None) | C4A script that compiles to JavaScript. Alternative to writing raw JS. |
| **`js_only`** | `bool` (False) | If `True`, indicates we're reusing an existing session and only applying JS. No full reload. |
| **`ignore_body_visibility`** | `bool` (True) | Skip checking if `<body>` is visible. Usually best to keep `True`. |
+| **`body_visibility_timeout`** | `int` (30000) | Maximum time in milliseconds to wait for `<body>` to become visible. Must be positive. |
| **`scan_full_page`** | `bool` (False) | If `True`, auto-scroll the page to load dynamic content (infinite scroll). |
| **`scroll_delay`** | `float` (0.2) | Delay between scroll steps when scanning the full page (`scan_full_page=True`) or capturing full-page screenshots. |
| **`max_scroll_steps`** | `int or None` (None) | Maximum number of scroll steps during full page scan. If None, scrolls until entire page is loaded. |
diff --git a/docs/md_v2/complete-sdk-reference.md b/docs/md_v2/complete-sdk-reference.md
index aa0517b2..1f329907 100644
--- a/docs/md_v2/complete-sdk-reference.md
+++ b/docs/md_v2/complete-sdk-reference.md
@@ -1791,6 +1791,7 @@ run_cfg = CrawlerRunConfig(
| **`js_code_before_wait`** | `str or list[str]` (None) | JavaScript to run **before** `wait_for`. Use for triggering loading that `wait_for` then checks. |
| **`js_only`** | `bool` (False) | If `True`, indicates we're reusing an existing session and only applying JS. No full reload. |
| **`ignore_body_visibility`** | `bool` (True) | Skip checking if `<body>` is visible. Usually best to keep `True`. |
+| **`body_visibility_timeout`** | `int` (30000) | Maximum time in milliseconds to wait for `<body>` to become visible. Must be positive. |
| **`scan_full_page`** | `bool` (False) | If `True`, auto-scroll the page to load dynamic content (infinite scroll). |
| **`scroll_delay`** | `float` (0.2) | Delay between scroll steps when scanning the full page (`scan_full_page=True`) or capturing full-page screenshots. |
| **`process_iframes`** | `bool` (False) | Inlines iframe content for single-page extraction. |
diff --git a/tests/test_config_defaults.py b/tests/test_config_defaults.py
index 93a9484c..3a7fba64 100644
--- a/tests/test_config_defaults.py
+++ b/tests/test_config_defaults.py
@@ -1,7 +1,12 @@
"""Tests for BrowserConfig.set_defaults / CrawlerRunConfig.set_defaults."""
+from types import SimpleNamespace
+from unittest.mock import AsyncMock, MagicMock
+
import pytest
+
from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig
+from crawl4ai.async_crawler_strategy import AsyncPlaywrightCrawlerStrategy
@pytest.fixture(autouse=True)
@@ -238,6 +243,46 @@ class TestDumpLoad:
assert loaded.scan_full_page is True
assert loaded.body_visibility_timeout == 2000
+ @pytest.mark.parametrize("timeout", [None, 0, -1, "1000", True])
+ def test_body_visibility_timeout_must_be_positive_number(self, timeout):
+ with pytest.raises(ValueError, match="must be a positive number"):
+ CrawlerRunConfig(body_visibility_timeout=timeout)
+
+ def test_untrusted_body_visibility_timeout_is_clamped(self):
+ from crawl4ai.async_configs import Provenance
+
+ config = CrawlerRunConfig.load(
+ {"body_visibility_timeout": 500_000}, provenance=Provenance.UNTRUSTED
+ )
+ assert config.body_visibility_timeout == 60_000
+
+ @pytest.mark.asyncio
+ async def test_body_visibility_timeout_reaches_wait(self):
+ page = MagicMock()
+ page.evaluate = AsyncMock()
+ page.set_content = AsyncMock()
+ page.wait_for_selector = AsyncMock()
+ page.content = AsyncMock(return_value="<body>visible</body>")
+
+ strategy = AsyncPlaywrightCrawlerStrategy.__new__(
+ AsyncPlaywrightCrawlerStrategy
+ )
+ strategy.browser_config = SimpleNamespace(
+ use_persistent_context=False, accept_downloads=False, text_mode=True
+ )
+ strategy.browser_manager = SimpleNamespace(
+ get_page=AsyncMock(return_value=(page, MagicMock()))
+ )
+ strategy.execute_hook = AsyncMock()
+ strategy.csp_compliant_wait = AsyncMock(return_value=True)
+
+ config = CrawlerRunConfig(
+ session_id="body-timeout-test", body_visibility_timeout=1234
+ )
+ await strategy._crawl_web("raw:<body>visible</body>", config)
+
+ assert strategy.csp_compliant_wait.await_args.kwargs["timeout"] == 1234
+
def test_to_dict_includes_user_default_values(self):
BrowserConfig.set_defaults(headless=False)
cfg = BrowserConfig()