🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
fix(ai): strip every noise tag, not just the ones before a nested one (#474)
Yiğit ERDOĞANe88029347afc
diff --git a/scrapling/core/shell.py b/scrapling/core/shell.pyindex 1ee8189..40e90c5 100644--- a/scrapling/core/shell.py+++ b/scrapling/core/shell.py@@ -637,7 +637,8 @@ class Convertor:def _strip_noise_tags(cls, page: Selector) -> Selector:"""Return a copy of the Selector with noise tags removed."""clean_root = deepcopy(page._root)- for element in clean_root.iter(*{"script", "style", "noscript", "svg"}):+ # Collect first: dropping a match mid-walk sends the iterator into the detached subtree, where it ends+ for element in list(clean_root.iter(*{"script", "style", "noscript", "svg"})):element.drop_tree()return Selector(root=clean_root, url=page.url)diff --git a/tests/fetchers/test_response_markdown.py b/tests/fetchers/test_response_markdown.pyindex 30adf27..9fe63f1 100644--- a/tests/fetchers/test_response_markdown.py+++ b/tests/fetchers/test_response_markdown.py@@ -31,6 +31,24 @@ SECTIONS_HTML = """"""+NESTED_NOISE_HTML = """+<html>+ <head>+ <noscript><style>.lazy { display: none; }</style></noscript>+ <script>var token = "abc";</script>+ </head>+ <body>+ <p>Article</p>+ <svg><style>.a { fill: red; }</style><path d="M0"/></svg>+ <p>More</p>+ <noscript>Please enable JavaScript</noscript>+ <svg><title>Logo</title></svg>+ <script>window.dataLayer = [];</script>+ </body>+</html>+"""++def _make_response(url: str = "https://example.com/", content: str = HTML) -> Response:return Response(url=url,@@ -58,6 +76,15 @@ class TestResponseMarkdown:assert "console.log" not in mdassert ".hint" not in md+ def test_noise_tags_after_a_nested_noise_tag_are_stripped(self):+ """Dropping a noise tag that contains another one must not end the scan early"""+ md = _make_response(content=NESTED_NOISE_HTML).markdown()+ assert "Article" in md+ assert "More" in md+ assert "Please enable JavaScript" not in md+ assert "dataLayer" not in md+ assert "Logo" not in md+def test_main_content_only_scopes_to_body(self):md = _make_response().markdown(main_content_only=True)assert "Docs Home" not in md