🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
fix(mcp): make mcp use less tokens by striping useless tags
Karim shoairb50e8f050fd9
diff --git a/scrapling/core/shell.py b/scrapling/core/shell.pyindex 0bd6efc..80caa99 100644--- a/scrapling/core/shell.py+++ b/scrapling/core/shell.py@@ -1,5 +1,6 @@# -*- coding: utf-8 -*-from sys import stderr+from copy import deepcopyfrom functools import wrapsfrom re import sub as re_subfrom collections import namedtuple@@ -571,6 +572,14 @@ class Convertor:return markdownify(body)+ @classmethod+ def _strip_noise_tags(cls, page: Selector) -> Selector:+ """Return a copy of the Selector with noise tags removed."""+ clean_root = deepcopy(page._root)+ for element in clean_root.iter(*{"script", "style", "noscript", "svg"}):+ element.drop_tree()+ return Selector(root=clean_root, url=page.url)+@classmethoddef _extract_content(cls,@@ -587,6 +596,7 @@ class Convertor:else:if main_content_only:page = cast(Selector, page.css("body").first) or page+ page = cls._strip_noise_tags(page)pages = [page] if not css_selector else cast(Selectors, page.css(css_selector))for page in pages:@@ -596,7 +606,9 @@ class Convertor:case "html":yield page.html_contentcase "text":- txt_content = page.get_all_text(strip=True)+ txt_content = page.get_all_text(+ strip=True, ignore_tags=("script", "style", "noscript", "svg", "iframe")+ )for s in ("\n","\r",