Official Crawl4AI Cloud: a hosted web scraping, extraction, and search API for AI agents. Free API key, 1,000 credits a month, no card. Fully open source: this repo is the Apache-2.0 crawler that powers it, and you can self-host it.
Python aiai-agentscrawl4ai-cloudcrawlerdata-extractionllmmarkdownmcpopen-sourceplaywrightpythonragscraperweb-crawlerweb-crawlingweb-scraping
fix: remove_overlay_elements can remove body if it has a global popup class.
Krzysztof Olipraa574fbc5ae58
diff --git a/crawl4ai/js_snippet/remove_overlay_elements.js b/crawl4ai/js_snippet/remove_overlay_elements.jsindex a50d9427..2b274ac6 100644--- a/crawl4ai/js_snippet/remove_overlay_elements.js+++ b/crawl4ai/js_snippet/remove_overlay_elements.js@@ -5,6 +5,12 @@ async () => {return style.display !== "none" && style.visibility !== "hidden" && style.opacity !== "0";};+ // Never strip document structure <html>, <head>, or <body>; removing those tags empties the page.+ const isDocumentStructure = (elem) => {+ const tag = elem && elem.tagName;+ return tag === "HTML" || tag === "HEAD" || tag === "BODY";+ };+// Common selectors for popups and overlaysconst commonSelectors = [// Close buttons first@@ -54,6 +60,7 @@ async () => {// Find elements with high z-indexconst allElements = document.querySelectorAll("*");for (const elem of allElements) {+ if (isDocumentStructure(elem)) continue;const style = window.getComputedStyle(elem);const zIndex = parseInt(style.zIndex);const position = style.position;@@ -74,6 +81,7 @@ async () => {for (const selector of commonSelectors) {const elements = document.querySelectorAll(selector);elements.forEach((elem) => {+ if (isDocumentStructure(elem)) return;if (isVisible(elem)) {elem.remove();}@@ -88,6 +96,7 @@ async () => {const removeFixedElements = () => {const elements = document.querySelectorAll("*");elements.forEach((elem) => {+ if (isDocumentStructure(elem)) return;const style = window.getComputedStyle(elem);if ((style.position === "fixed" || style.position === "sticky") && isVisible(elem)) {elem.remove();@@ -110,11 +119,13 @@ async () => {};// Remove margin-right and padding-right from body (often added by modal scripts)- document.body.style.marginRight = "0px";- document.body.style.paddingRight = "0px";- document.body.style.overflow = "auto";+ if (document.body) {+ document.body.style.marginRight = "0px";+ document.body.style.paddingRight = "0px";+ document.body.style.overflow = "auto";- // Wait a bit for any animations to complete- document.body.scrollIntoView(false);+ // Wait a bit for any animations to complete+ document.body.scrollIntoView(false);+ }await new Promise((resolve) => setTimeout(resolve, 50));};diff --git a/tests/test_issue_2161_overlay_html_body.py b/tests/test_issue_2161_overlay_html_body.pynew file mode 100644index 00000000..7855a8e8--- /dev/null+++ b/tests/test_issue_2161_overlay_html_body.py@@ -0,0 +1,81 @@+"""Tests for issue #2161: remove_overlay_elements can remove body if it has a global popup class.++https://github.com/unclecode/crawl4ai/issues/2161++WordPress themes such as Qode/Bridge put classes like+`.qode_popup_menu_push_text_right` on <body>. The overlay snippet matches+`[class*="popup"]` and used to delete the entire document, leaving an empty page.+"""++import pytest++from crawl4ai import AsyncWebCrawler, CrawlerRunConfig+++QODE_BODY_HTML = """\+<!DOCTYPE html>+<html class="qode-theme popup-ready">+<head><title>Qode popup body class</title></head>+<body class="qode_popup_menu_push_text_right page-template">+ <main>+ <h1>Tunnel Girona</h1>+ <p>Portfolio page content that must survive overlay removal.</p>+ </main>+ <div class="popup-overlay" id="real-popup" style="position:fixed;z-index:9999;inset:0;background:rgba(0,0,0,0.5);">+ <p>Please subscribe</p>+ <button class="close">Close</button>+ </div>+</body>+</html>+"""++FIXED_BODY_HTML = """\+<!DOCTYPE html>+<html>+<head><title>Fixed body scroll lock</title></head>+<body class="modal-open" style="position:fixed;overflow:hidden;">+ <main>+ <h1>Locked body</h1>+ <p>Body is position:fixed while a modal is open; it must not be removed.</p>+ </main>+ <div class="modal overlay" style="position:fixed;z-index:10000;top:0;left:0;width:100%;height:100%;">+ Newsletter popup+ </div>+</body>+</html>+"""+++async def test_overlay_removal_keeps_body_with_popup_in_class():+ """#2161: Body/html with a class substring 'popup' must remain after overlay removal."""+ async with AsyncWebCrawler() as crawler:+ result = await crawler.arun(+ f"raw:{QODE_BODY_HTML}",+ config=CrawlerRunConfig(remove_overlay_elements=True, verbose=False),+ )++ assert result.success, result.error_message+ html = result.html.lower()+ assert "<body" in html, "body tag must not be removed"+ assert "<html" in html, "html tag must not be removed"+ assert "tunnel girona" in html+ assert "portfolio page content that must survive overlay removal" in html+ assert "please subscribe" not in html+ assert 'id="real-popup"' not in html and "id='real-popup'" not in html+++async def test_overlay_removal_keeps_position_fixed_body():+ """#2161: A scroll-locked (position:fixed) body must not be treated as an overlay."""+ async with AsyncWebCrawler() as crawler:+ result = await crawler.arun(+ f"raw:{FIXED_BODY_HTML}",+ config=CrawlerRunConfig(remove_overlay_elements=True, verbose=False),+ )++ assert result.success, result.error_message+ html = result.html.lower()+ assert "<body" in html, "position:fixed body must not be removed"+ assert "locked body" in html+ assert "body is position:fixed" in html