Official Crawl4AI Cloud: a hosted web scraping, extraction, and search API for AI agents. Free API key, 1,000 credits a month, no card. Fully open source: this repo is the Apache-2.0 crawler that powers it, and you can self-host it.
Python aiai-agentscrawl4ai-cloudcrawlerdata-extractionllmmarkdownmcpopen-sourceplaywrightpythonragscraperweb-crawlerweb-crawlingweb-scraping
fix: remove unconditional setTimeout waits from overlay/consent removal scripts
Soham Kukretib536bf57faf5
diff --git a/crawl4ai/async_crawler_strategy.py b/crawl4ai/async_crawler_strategy.pyindex 265c376e..843463aa 100644--- a/crawl4ai/async_crawler_strategy.py+++ b/crawl4ai/async_crawler_strategy.py@@ -1534,7 +1534,7 @@ class AsyncPlaywrightCrawlerStrategy(AsyncCrawlerStrategy):}})()""")- await page.wait_for_timeout(500) # Wait for any animations to complete+ await page.wait_for_timeout(600) # Wait for any animations to completeexcept Exception as e:self.logger.warning(message="Failed to remove overlay elements: {error}",diff --git a/crawl4ai/js_snippet/remove_consent_popups.js b/crawl4ai/js_snippet/remove_consent_popups.jsindex 9aac8d34..2f7eb0f3 100644--- a/crawl4ai/js_snippet/remove_consent_popups.js+++ b/crawl4ai/js_snippet/remove_consent_popups.js@@ -292,6 +292,7 @@ async () => {// =========================================================================// Phase 2: Try CMP JavaScript APIs// =========================================================================+ let apiCalled = false;// IAB TCF v2 APIif (typeof window.__tcfapi === 'function') {@@ -304,6 +305,7 @@ async () => {if (typeof window.Didomi !== 'undefined') {try {window.Didomi.setUserAgreeToAll();+ apiCalled = true;} catch (e) { /* continue */ }}@@ -311,6 +313,7 @@ async () => {if (typeof window.Cookiebot !== 'undefined') {try {window.Cookiebot.submitCustomConsent(true, true, true);+ apiCalled = true;} catch (e) { /* continue */ }}@@ -318,6 +321,7 @@ async () => {if (typeof window.Osano !== 'undefined') {try {window.Osano.cm.acceptAll();+ apiCalled = true;} catch (e) { /* continue */ }}@@ -325,11 +329,12 @@ async () => {if (typeof window.klaro !== 'undefined') {try {window.klaro.getManager().acceptAll();+ apiCalled = true;} catch (e) { /* continue */ }}- // Wait for CMP animations/transitions- await new Promise(r => setTimeout(r, 500));+ // Wait for CMP animations/transitions - only when a consent action actually fired+ if (accepted || apiCalled) await new Promise(r => setTimeout(r, 500));// =========================================================================// Phase 3: Remove known CMP containers by selectordiff --git a/crawl4ai/js_snippet/remove_overlay_elements.js b/crawl4ai/js_snippet/remove_overlay_elements.jsindex a50d9427..36de9d97 100644--- a/crawl4ai/js_snippet/remove_overlay_elements.js+++ b/crawl4ai/js_snippet/remove_overlay_elements.js@@ -114,7 +114,5 @@ async () => {document.body.style.paddingRight = "0px";document.body.style.overflow = "auto";- // Wait a bit for any animations to completedocument.body.scrollIntoView(false);- await new Promise((resolve) => setTimeout(resolve, 50));};diff --git a/tests/regression/test_reg_browser.py b/tests/regression/test_reg_browser.pyindex ba901178..2d9f87fb 100644--- a/tests/regression/test_reg_browser.py+++ b/tests/regression/test_reg_browser.py@@ -289,6 +289,26 @@ async def test_remove_overlay_elements(local_server):assert len(result.html) > 0, "HTML should still be present after overlay removal"+async def test_overlay_removal_on_csp_sandbox_page():+ """raw.githubusercontent.com serves CSP `sandbox`, which disables page+ timers; overlay/consent removal must not stall waiting on in-page+ setTimeout (used to hang ~30s per page). URL is commit-pinned so the+ response never changes."""+ url = "https://raw.githubusercontent.com/unclecode/crawl4ai/055e2ecdc702228a80363e2c51ca79e473222072/README.md"+ config = CrawlerRunConfig(+ remove_overlay_elements=True, remove_consent_popups=True, verbose=False+ )+ async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:+ start = time.perf_counter()+ result = await crawler.arun(url=url, config=config)+ elapsed = time.perf_counter() - start+ assert result.success, f"Crawl failed on CSP-sandboxed page: {result.error_message}"+ assert "Crawl4AI" in result.html, "Page content should be captured"+ assert elapsed < 20, f"Overlay removal stalled on CSP-sandboxed page ({elapsed:.1f}s)"++# ---------------------------------------------------------------------------# Stealth mode# ---------------------------------------------------------------------------