Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

fix(mcp): make mcp use less tokens by striping useless tags

Karim shoairb50e8f050fd9
diff --git a/scrapling/core/shell.py b/scrapling/core/shell.py
index 0bd6efc..80caa99 100644
--- a/scrapling/core/shell.py
+++ b/scrapling/core/shell.py
@@ -1,5 +1,6 @@
# -*- coding: utf-8 -*-
from sys import stderr
+from copy import deepcopy
from functools import wraps
from re import sub as re_sub
from collections import namedtuple
@@ -571,6 +572,14 @@ class Convertor:
return markdownify(body)
+ @classmethod
+ def _strip_noise_tags(cls, page: Selector) -> Selector:
+ """Return a copy of the Selector with noise tags removed."""
+ clean_root = deepcopy(page._root)
+ for element in clean_root.iter(*{"script", "style", "noscript", "svg"}):
+ element.drop_tree()
+ return Selector(root=clean_root, url=page.url)
+
@classmethod
def _extract_content(
cls,
@@ -587,6 +596,7 @@ class Convertor:
else:
if main_content_only:
page = cast(Selector, page.css("body").first) or page
+ page = cls._strip_noise_tags(page)
pages = [page] if not css_selector else cast(Selectors, page.css(css_selector))
for page in pages:
@@ -596,7 +606,9 @@ class Convertor:
case "html":
yield page.html_content
case "text":
- txt_content = page.get_all_text(strip=True)
+ txt_content = page.get_all_text(
+ strip=True, ignore_tags=("script", "style", "noscript", "svg", "iframe")
+ )
for s in (
"\n",
"\r",