Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

fix(ai): strip control characters before extraction (#366)

yetval2548fbc79c85
diff --git a/scrapling/core/ai.py b/scrapling/core/ai.py
index 82d9113..7f87a2c 100644
--- a/scrapling/core/ai.py
+++ b/scrapling/core/ai.py
@@ -8,7 +8,7 @@ from mcp.server.fastmcp import FastMCP, Image
from mcp.types import ImageContent, TextContent
from pydantic import BaseModel, Field
-from scrapling.core.shell import Convertor
+from scrapling.core.shell import Convertor, _CONTROL_CHARS_PATTERN
from scrapling.engines.toolbelt.custom import Response as _ScraplingResponse
from scrapling.engines.static import ImpersonateType
from scrapling.fetchers import (
@@ -81,14 +81,15 @@ def _translate_response(
main_content_only: bool,
) -> ResponseModel:
"""Extract content from a response and translate it to a ResponseModel."""
- content = list(
- Convertor._extract_content(
+ content = [
+ _CONTROL_CHARS_PATTERN.sub("", chunk)
+ for chunk in Convertor._extract_content(
page,
css_selector=css_selector,
extraction_type=extraction_type,
main_content_only=main_content_only,
)
- )
+ ]
return ResponseModel(status=page.status, content=content, url=page.url)
diff --git a/scrapling/core/shell.py b/scrapling/core/shell.py
index cb805c6..762b9de 100644
--- a/scrapling/core/shell.py
+++ b/scrapling/core/shell.py
@@ -80,6 +80,7 @@ _HIDDEN_XPATH = XPath(
" | .//template"
)
_ZWC_PATTERN = re_compile(r"[\u200b\u200c\u200d\ufeff\u2060\u180e]")
+_CONTROL_CHARS_PATTERN = re_compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f]")
# Suppress exit on error to handle parsing errors gracefully
@@ -599,16 +600,17 @@ class Convertor:
"""Strip hidden content that could be used for prompt injection.
Removes CSS-hidden elements, aria-hidden elements, <template> tags,
- HTML comments, and zero-width Unicode characters.
+ HTML comments, zero-width Unicode characters, and XML-incompatible
+ control characters.
"""
clean_root = deepcopy(page._root)
for element in cast(list, _HIDDEN_XPATH(clean_root)):
element.drop_tree()
for element in clean_root.iter():
if element.text:
- element.text = _ZWC_PATTERN.sub("", element.text)
+ element.text = _CONTROL_CHARS_PATTERN.sub("", _ZWC_PATTERN.sub("", element.text))
if element.tail:
- element.tail = _ZWC_PATTERN.sub("", element.tail)
+ element.tail = _CONTROL_CHARS_PATTERN.sub("", _ZWC_PATTERN.sub("", element.tail))
return Selector(root=clean_root, url=page.url, keep_comments=False)
@classmethod
diff --git a/tests/ai/test_ai_mcp.py b/tests/ai/test_ai_mcp.py
index 3043a76..03395c5 100644
--- a/tests/ai/test_ai_mcp.py
+++ b/tests/ai/test_ai_mcp.py
@@ -9,6 +9,7 @@ import pytest
import pytest_httpbin
from mcp.types import ImageContent, TextContent
+from scrapling.parser import Selector
from scrapling.core.ai import (
ScraplingMCPServer,
ResponseModel,
@@ -16,7 +17,21 @@ from scrapling.core.ai import (
SessionCreatedModel,
SessionClosedModel,
_normalize_credentials,
+ _translate_response,
)
+
+
+def test_translate_response_strips_control_characters():
+ """Pages with control chars like U+0008 must not crash the get/fetch path (issue #366)"""
+ html = "<html><body><p>Hello\x08World</p>\t\n<div>Foo\x0cbar</div></body></html>"
+ page = Selector(html, url="https://jfinal.com/doc/1-5")
+ page.status = 200
+
+ result = _translate_response(page, "markdown", None, main_content_only=True)
+
+ joined = "".join(result.content)
+ assert "HelloWorld" in joined and "Foobar" in joined
+ assert not any(ord(c) < 0x20 and c not in "\t\n\r" for c in joined)
from scrapling.engines.toolbelt.custom import Response