Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

fix(ai): strip every noise tag, not just the ones before a nested one (#474)

Yiğit ERDOĞANe88029347afc
diff --git a/scrapling/core/shell.py b/scrapling/core/shell.py
index 1ee8189..40e90c5 100644
--- a/scrapling/core/shell.py
+++ b/scrapling/core/shell.py
@@ -637,7 +637,8 @@ class Convertor:
def _strip_noise_tags(cls, page: Selector) -> Selector:
"""Return a copy of the Selector with noise tags removed."""
clean_root = deepcopy(page._root)
- for element in clean_root.iter(*{"script", "style", "noscript", "svg"}):
+ # Collect first: dropping a match mid-walk sends the iterator into the detached subtree, where it ends
+ for element in list(clean_root.iter(*{"script", "style", "noscript", "svg"})):
element.drop_tree()
return Selector(root=clean_root, url=page.url)
diff --git a/tests/fetchers/test_response_markdown.py b/tests/fetchers/test_response_markdown.py
index 30adf27..9fe63f1 100644
--- a/tests/fetchers/test_response_markdown.py
+++ b/tests/fetchers/test_response_markdown.py
@@ -31,6 +31,24 @@ SECTIONS_HTML = """
"""
+NESTED_NOISE_HTML = """
+<html>
+ <head>
+ <noscript><style>.lazy { display: none; }</style></noscript>
+ <script>var token = "abc";</script>
+ </head>
+ <body>
+ <p>Article</p>
+ <svg><style>.a { fill: red; }</style><path d="M0"/></svg>
+ <p>More</p>
+ <noscript>Please enable JavaScript</noscript>
+ <svg><title>Logo</title></svg>
+ <script>window.dataLayer = [];</script>
+ </body>
+</html>
+"""
+
+
def _make_response(url: str = "https://example.com/", content: str = HTML) -> Response:
return Response(
url=url,
@@ -58,6 +76,15 @@ class TestResponseMarkdown:
assert "console.log" not in md
assert ".hint" not in md
+ def test_noise_tags_after_a_nested_noise_tag_are_stripped(self):
+ """Dropping a noise tag that contains another one must not end the scan early"""
+ md = _make_response(content=NESTED_NOISE_HTML).markdown()
+ assert "Article" in md
+ assert "More" in md
+ assert "Please enable JavaScript" not in md
+ assert "dataLayer" not in md
+ assert "Logo" not in md
+
def test_main_content_only_scopes_to_body(self):
md = _make_response().markdown(main_content_only=True)
assert "Docs Home" not in md