🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
fix(ai): strip control characters before extraction (#366)
yetval2548fbc79c85
diff --git a/scrapling/core/ai.py b/scrapling/core/ai.pyindex 82d9113..7f87a2c 100644--- a/scrapling/core/ai.py+++ b/scrapling/core/ai.py@@ -8,7 +8,7 @@ from mcp.server.fastmcp import FastMCP, Imagefrom mcp.types import ImageContent, TextContentfrom pydantic import BaseModel, Field-from scrapling.core.shell import Convertor+from scrapling.core.shell import Convertor, _CONTROL_CHARS_PATTERNfrom scrapling.engines.toolbelt.custom import Response as _ScraplingResponsefrom scrapling.engines.static import ImpersonateTypefrom scrapling.fetchers import (@@ -81,14 +81,15 @@ def _translate_response(main_content_only: bool,) -> ResponseModel:"""Extract content from a response and translate it to a ResponseModel."""- content = list(- Convertor._extract_content(+ content = [+ _CONTROL_CHARS_PATTERN.sub("", chunk)+ for chunk in Convertor._extract_content(page,css_selector=css_selector,extraction_type=extraction_type,main_content_only=main_content_only,)- )+ ]return ResponseModel(status=page.status, content=content, url=page.url)diff --git a/scrapling/core/shell.py b/scrapling/core/shell.pyindex cb805c6..762b9de 100644--- a/scrapling/core/shell.py+++ b/scrapling/core/shell.py@@ -80,6 +80,7 @@ _HIDDEN_XPATH = XPath(" | .//template")_ZWC_PATTERN = re_compile(r"[\u200b\u200c\u200d\ufeff\u2060\u180e]")+_CONTROL_CHARS_PATTERN = re_compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f]")# Suppress exit on error to handle parsing errors gracefully@@ -599,16 +600,17 @@ class Convertor:"""Strip hidden content that could be used for prompt injection.Removes CSS-hidden elements, aria-hidden elements, <template> tags,- HTML comments, and zero-width Unicode characters.+ HTML comments, zero-width Unicode characters, and XML-incompatible+ control characters."""clean_root = deepcopy(page._root)for element in cast(list, _HIDDEN_XPATH(clean_root)):element.drop_tree()for element in clean_root.iter():if element.text:- element.text = _ZWC_PATTERN.sub("", element.text)+ element.text = _CONTROL_CHARS_PATTERN.sub("", _ZWC_PATTERN.sub("", element.text))if element.tail:- element.tail = _ZWC_PATTERN.sub("", element.tail)+ element.tail = _CONTROL_CHARS_PATTERN.sub("", _ZWC_PATTERN.sub("", element.tail))return Selector(root=clean_root, url=page.url, keep_comments=False)@classmethoddiff --git a/tests/ai/test_ai_mcp.py b/tests/ai/test_ai_mcp.pyindex 3043a76..03395c5 100644--- a/tests/ai/test_ai_mcp.py+++ b/tests/ai/test_ai_mcp.py@@ -9,6 +9,7 @@ import pytestimport pytest_httpbinfrom mcp.types import ImageContent, TextContent+from scrapling.parser import Selectorfrom scrapling.core.ai import (ScraplingMCPServer,ResponseModel,@@ -16,7 +17,21 @@ from scrapling.core.ai import (SessionCreatedModel,SessionClosedModel,_normalize_credentials,+ _translate_response,)+++def test_translate_response_strips_control_characters():+ """Pages with control chars like U+0008 must not crash the get/fetch path (issue #366)"""+ html = "<html><body><p>Hello\x08World</p>\t\n<div>Foo\x0cbar</div></body></html>"+ page = Selector(html, url="https://jfinal.com/doc/1-5")+ page.status = 200++ result = _translate_response(page, "markdown", None, main_content_only=True)++ joined = "".join(result.content)+ assert "HelloWorld" in joined and "Foobar" in joined+ assert not any(ord(c) < 0x20 and c not in "\t\n\r" for c in joined)from scrapling.engines.toolbelt.custom import Response