Skip to content

Official Crawl4AI Cloud: a hosted web scraping, extraction, and search API for AI agents. Free API key, 1,000 credits a month, no card. Fully open source: this repo is the Apache-2.0 crawler that powers it, and you can self-host it.

fix: remove_overlay_elements can remove body if it has a global popup class.

Krzysztof Olipraa574fbc5ae58
diff --git a/crawl4ai/js_snippet/remove_overlay_elements.js b/crawl4ai/js_snippet/remove_overlay_elements.js
index a50d9427..2b274ac6 100644
--- a/crawl4ai/js_snippet/remove_overlay_elements.js
+++ b/crawl4ai/js_snippet/remove_overlay_elements.js
@@ -5,6 +5,12 @@ async () => {
return style.display !== "none" && style.visibility !== "hidden" && style.opacity !== "0";
};
+ // Never strip document structure <html>, <head>, or <body>; removing those tags empties the page.
+ const isDocumentStructure = (elem) => {
+ const tag = elem && elem.tagName;
+ return tag === "HTML" || tag === "HEAD" || tag === "BODY";
+ };
+
// Common selectors for popups and overlays
const commonSelectors = [
// Close buttons first
@@ -54,6 +60,7 @@ async () => {
// Find elements with high z-index
const allElements = document.querySelectorAll("*");
for (const elem of allElements) {
+ if (isDocumentStructure(elem)) continue;
const style = window.getComputedStyle(elem);
const zIndex = parseInt(style.zIndex);
const position = style.position;
@@ -74,6 +81,7 @@ async () => {
for (const selector of commonSelectors) {
const elements = document.querySelectorAll(selector);
elements.forEach((elem) => {
+ if (isDocumentStructure(elem)) return;
if (isVisible(elem)) {
elem.remove();
}
@@ -88,6 +96,7 @@ async () => {
const removeFixedElements = () => {
const elements = document.querySelectorAll("*");
elements.forEach((elem) => {
+ if (isDocumentStructure(elem)) return;
const style = window.getComputedStyle(elem);
if ((style.position === "fixed" || style.position === "sticky") && isVisible(elem)) {
elem.remove();
@@ -110,11 +119,13 @@ async () => {
};
// Remove margin-right and padding-right from body (often added by modal scripts)
- document.body.style.marginRight = "0px";
- document.body.style.paddingRight = "0px";
- document.body.style.overflow = "auto";
+ if (document.body) {
+ document.body.style.marginRight = "0px";
+ document.body.style.paddingRight = "0px";
+ document.body.style.overflow = "auto";
- // Wait a bit for any animations to complete
- document.body.scrollIntoView(false);
+ // Wait a bit for any animations to complete
+ document.body.scrollIntoView(false);
+ }
await new Promise((resolve) => setTimeout(resolve, 50));
};
diff --git a/tests/test_issue_2161_overlay_html_body.py b/tests/test_issue_2161_overlay_html_body.py
new file mode 100644
index 00000000..7855a8e8
--- /dev/null
+++ b/tests/test_issue_2161_overlay_html_body.py
@@ -0,0 +1,81 @@
+"""Tests for issue #2161: remove_overlay_elements can remove body if it has a global popup class.
+
+https://github.com/unclecode/crawl4ai/issues/2161
+
+WordPress themes such as Qode/Bridge put classes like
+`.qode_popup_menu_push_text_right` on <body>. The overlay snippet matches
+`[class*="popup"]` and used to delete the entire document, leaving an empty page.
+"""
+
+import pytest
+
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
+
+
+QODE_BODY_HTML = """\
+<!DOCTYPE html>
+<html class="qode-theme popup-ready">
+<head><title>Qode popup body class</title></head>
+<body class="qode_popup_menu_push_text_right page-template">
+ <main>
+ <h1>Tunnel Girona</h1>
+ <p>Portfolio page content that must survive overlay removal.</p>
+ </main>
+ <div class="popup-overlay" id="real-popup" style="position:fixed;z-index:9999;inset:0;background:rgba(0,0,0,0.5);">
+ <p>Please subscribe</p>
+ <button class="close">Close</button>
+ </div>
+</body>
+</html>
+"""
+
+FIXED_BODY_HTML = """\
+<!DOCTYPE html>
+<html>
+<head><title>Fixed body scroll lock</title></head>
+<body class="modal-open" style="position:fixed;overflow:hidden;">
+ <main>
+ <h1>Locked body</h1>
+ <p>Body is position:fixed while a modal is open; it must not be removed.</p>
+ </main>
+ <div class="modal overlay" style="position:fixed;z-index:10000;top:0;left:0;width:100%;height:100%;">
+ Newsletter popup
+ </div>
+</body>
+</html>
+"""
+
+
+async def test_overlay_removal_keeps_body_with_popup_in_class():
+ """#2161: Body/html with a class substring 'popup' must remain after overlay removal."""
+ async with AsyncWebCrawler() as crawler:
+ result = await crawler.arun(
+ f"raw:{QODE_BODY_HTML}",
+ config=CrawlerRunConfig(remove_overlay_elements=True, verbose=False),
+ )
+
+ assert result.success, result.error_message
+ html = result.html.lower()
+ assert "<body" in html, "body tag must not be removed"
+ assert "<html" in html, "html tag must not be removed"
+ assert "tunnel girona" in html
+ assert "portfolio page content that must survive overlay removal" in html
+ assert "please subscribe" not in html
+ assert 'id="real-popup"' not in html and "id='real-popup'" not in html
+
+
+async def test_overlay_removal_keeps_position_fixed_body():
+ """#2161: A scroll-locked (position:fixed) body must not be treated as an overlay."""
+ async with AsyncWebCrawler() as crawler:
+ result = await crawler.arun(
+ f"raw:{FIXED_BODY_HTML}",
+ config=CrawlerRunConfig(remove_overlay_elements=True, verbose=False),
+ )
+
+ assert result.success, result.error_message
+ html = result.html.lower()
+ assert "<body" in html, "position:fixed body must not be removed"
+ assert "locked body" in html
+ assert "body is position:fixed" in html