Skip to content

Official Crawl4AI Cloud: a hosted web scraping, extraction, and search API for AI agents. Free API key, 1,000 credits a month, no card. Fully open source: this repo is the Apache-2.0 crawler that powers it, and you can self-host it.

fix: remove unconditional setTimeout waits from overlay/consent removal scripts

Soham Kukretib536bf57faf5
diff --git a/crawl4ai/async_crawler_strategy.py b/crawl4ai/async_crawler_strategy.py
index 265c376e..843463aa 100644
--- a/crawl4ai/async_crawler_strategy.py
+++ b/crawl4ai/async_crawler_strategy.py
@@ -1534,7 +1534,7 @@ class AsyncPlaywrightCrawlerStrategy(AsyncCrawlerStrategy):
}})()
"""
)
- await page.wait_for_timeout(500) # Wait for any animations to complete
+ await page.wait_for_timeout(600) # Wait for any animations to complete
except Exception as e:
self.logger.warning(
message="Failed to remove overlay elements: {error}",
diff --git a/crawl4ai/js_snippet/remove_consent_popups.js b/crawl4ai/js_snippet/remove_consent_popups.js
index 9aac8d34..2f7eb0f3 100644
--- a/crawl4ai/js_snippet/remove_consent_popups.js
+++ b/crawl4ai/js_snippet/remove_consent_popups.js
@@ -292,6 +292,7 @@ async () => {
// =========================================================================
// Phase 2: Try CMP JavaScript APIs
// =========================================================================
+ let apiCalled = false;
// IAB TCF v2 API
if (typeof window.__tcfapi === 'function') {
@@ -304,6 +305,7 @@ async () => {
if (typeof window.Didomi !== 'undefined') {
try {
window.Didomi.setUserAgreeToAll();
+ apiCalled = true;
} catch (e) { /* continue */ }
}
@@ -311,6 +313,7 @@ async () => {
if (typeof window.Cookiebot !== 'undefined') {
try {
window.Cookiebot.submitCustomConsent(true, true, true);
+ apiCalled = true;
} catch (e) { /* continue */ }
}
@@ -318,6 +321,7 @@ async () => {
if (typeof window.Osano !== 'undefined') {
try {
window.Osano.cm.acceptAll();
+ apiCalled = true;
} catch (e) { /* continue */ }
}
@@ -325,11 +329,12 @@ async () => {
if (typeof window.klaro !== 'undefined') {
try {
window.klaro.getManager().acceptAll();
+ apiCalled = true;
} catch (e) { /* continue */ }
}
- // Wait for CMP animations/transitions
- await new Promise(r => setTimeout(r, 500));
+ // Wait for CMP animations/transitions - only when a consent action actually fired
+ if (accepted || apiCalled) await new Promise(r => setTimeout(r, 500));
// =========================================================================
// Phase 3: Remove known CMP containers by selector
diff --git a/crawl4ai/js_snippet/remove_overlay_elements.js b/crawl4ai/js_snippet/remove_overlay_elements.js
index a50d9427..36de9d97 100644
--- a/crawl4ai/js_snippet/remove_overlay_elements.js
+++ b/crawl4ai/js_snippet/remove_overlay_elements.js
@@ -114,7 +114,5 @@ async () => {
document.body.style.paddingRight = "0px";
document.body.style.overflow = "auto";
- // Wait a bit for any animations to complete
document.body.scrollIntoView(false);
- await new Promise((resolve) => setTimeout(resolve, 50));
};
diff --git a/tests/regression/test_reg_browser.py b/tests/regression/test_reg_browser.py
index ba901178..2d9f87fb 100644
--- a/tests/regression/test_reg_browser.py
+++ b/tests/regression/test_reg_browser.py
@@ -289,6 +289,26 @@ async def test_remove_overlay_elements(local_server):
assert len(result.html) > 0, "HTML should still be present after overlay removal"
+async def test_overlay_removal_on_csp_sandbox_page():
+ """raw.githubusercontent.com serves CSP `sandbox`, which disables page
+ timers; overlay/consent removal must not stall waiting on in-page
+ setTimeout (used to hang ~30s per page). URL is commit-pinned so the
+ response never changes."""
+ url = "https://raw.githubusercontent.com/unclecode/crawl4ai/055e2ecdc702228a80363e2c51ca79e473222072/README.md"
+ config = CrawlerRunConfig(
+ remove_overlay_elements=True, remove_consent_popups=True, verbose=False
+ )
+ async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+ start = time.perf_counter()
+ result = await crawler.arun(url=url, config=config)
+ elapsed = time.perf_counter() - start
+ assert result.success, f"Crawl failed on CSP-sandboxed page: {result.error_message}"
+ assert "Crawl4AI" in result.html, "Page content should be captured"
+ assert elapsed < 20, f"Overlay removal stalled on CSP-sandboxed page ({elapsed:.1f}s)"
+
+
# ---------------------------------------------------------------------------
# Stealth mode
# ---------------------------------------------------------------------------