🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
fix:(mcp): upgrade mcp server to v2
Karim shoaire978f8616dc5
diff --git a/pyproject.toml b/pyproject.tomlindex 455ee28..3511a85 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -82,7 +82,7 @@ fetchers = ["protego>=0.6.2",]ai = [- "mcp>=1.27.0",+ "mcp>=2.0.0","markdownify>=1.2.0","scrapling[fetchers]",]diff --git a/scrapling/core/ai.py b/scrapling/core/ai.pyindex 81d2b32..0b7ed90 100644--- a/scrapling/core/ai.py+++ b/scrapling/core/ai.py@@ -5,7 +5,8 @@ from asyncio import gatherfrom datetime import datetime, timezonefrom dataclasses import dataclass, field-from mcp.server.fastmcp import FastMCP, Image+from mcp.server import MCPServer+from mcp.server.mcpserver import Imagefrom mcp.server.auth.provider import AccessToken, TokenVerifierfrom mcp.server.auth.settings import AuthSettingsfrom mcp.server.transport_security import TransportSecuritySettings@@ -924,22 +925,26 @@ class ScraplingMCPServer:return [_translate_response(page, extraction_type, css_selector, main_content_only) for page in responses]- def _build_server(self, host: str, port: int, allowed_hosts: Sequence[str] = ()) -> FastMCP:- """Build the FastMCP server with all tools registered and the optional security settings applied."""+ @staticmethod+ def _transport_security(allowed_hosts: Sequence[str]) -> Optional[TransportSecuritySettings]:+ """Build the DNS-rebinding protection settings for the streamable-http transport."""+ if not allowed_hosts:+ return None+ return TransportSecuritySettings(+ enable_dns_rebinding_protection=True,+ allowed_hosts=list(allowed_hosts),+ allowed_origins=[f"{scheme}://{host_}" for host_ in allowed_hosts for scheme in ("http", "https")],+ )++ def _build_server(self, host: str, port: int) -> MCPServer:+ """Build the MCPServer with all tools registered and the optional authentication settings applied."""settings: Dict[str, Any] = {}if self._auth_token:base_url = AnyHttpUrl(f"http://{host}:{port}")settings["token_verifier"] = _StaticTokenVerifier(self._auth_token)settings["auth"] = AuthSettings(issuer_url=base_url, resource_server_url=base_url)- if allowed_hosts:- settings["transport_security"] = TransportSecuritySettings(- enable_dns_rebinding_protection=True,- allowed_hosts=list(allowed_hosts),- allowed_origins=[f"{scheme}://{host_}" for host_ in allowed_hosts for scheme in ("http", "https")],- )-- server = FastMCP(name="Scrapling", host=host, port=port, **settings)+ server = MCPServer(name="Scrapling", **settings)# Session management toolsserver.add_tool(self.open_session, title="open_session", structured_output=True)server.add_tool(self.close_session, title="close_session", structured_output=True)@@ -966,7 +971,9 @@ class ScraplingMCPServer:structured_output=True,)# Screenshot tool (returns image + url content blocks, not structured JSON)- server.add_tool(self.screenshot, title="screenshot", description=self.screenshot.__doc__)+ server.add_tool(+ self.screenshot, title="screenshot", description=self.screenshot.__doc__, structured_output=False+ )return serverdef serve(self, http: bool, host: str, port: int, allowed_hosts: Sequence[str] = ()):@@ -982,4 +989,13 @@ class ScraplingMCPServer:"The authentication token only applies to the streamable-http transport, so it's ignored with stdio.")- self._build_server(host, port, allowed_hosts).run(transport="stdio" if not http else "streamable-http")+ server = self._build_server(host, port)+ if http:+ server.run(+ transport="streamable-http",+ host=host,+ port=port,+ transport_security=self._transport_security(allowed_hosts),+ )+ else:+ server.run()diff --git a/tests/ai/test_ai_mcp.py b/tests/ai/test_ai_mcp.pyindex afc9d36..541b717 100644--- a/tests/ai/test_ai_mcp.py+++ b/tests/ai/test_ai_mcp.py@@ -7,9 +7,10 @@ from threading import Threadimport pytestimport pytest_httpbin+from mcp.client import Clientfrom mcp.types import ImageContent, TextContent-from scrapling.parser import Selector+from scrapling.engines.toolbelt.custom import Responsefrom scrapling.core.ai import (MCP_AUTH_TOKEN_ENV,ScraplingMCPServer,@@ -26,15 +27,21 @@ from scrapling.core.ai import (def test_translate_response_strips_control_characters():"""Pages with control chars like U+0008 must not crash the get/fetch path (issue #366)"""html = "<html><body><p>Hello\x08World</p>\t\n<div>Foo\x0cbar</div></body></html>"- page = Selector(html, url="https://jfinal.com/doc/1-5")- page.status = 200+ page = Response(+ url="https://jfinal.com/doc/1-5",+ content=html,+ status=200,+ reason="OK",+ cookies={},+ headers={},+ request_headers={},+ )result = _translate_response(page, "markdown", None, main_content_only=True)joined = "".join(result.content)assert "HelloWorld" in joined and "Foobar" in joinedassert not any(ord(c) < 0x20 and c not in "\t\n\r" for c in joined)-from scrapling.engines.toolbelt.custom import Responseclass _FakeAsyncBrowserSession:@@ -368,7 +375,7 @@ class TestScreenshot:result = await server.screenshot(url=test_url, session_id=opened.session_id)assert isinstance(result, list) and len(result) == 2assert isinstance(result[0], ImageContent)- assert result[0].mimeType == "image/png"+ assert result[0].mime_type == "image/png"assert isinstance(result[1], TextContent)assert result[1].text == test_urlfinally:@@ -381,7 +388,7 @@ class TestScreenshot:try:result = await server.screenshot(url=test_url, session_id=opened.session_id, image_type="jpeg", quality=80)assert isinstance(result[0], ImageContent)- assert result[0].mimeType == "image/jpeg"+ assert result[0].mime_type == "image/jpeg"finally:await server.close_session(opened.session_id)@@ -392,7 +399,7 @@ class TestScreenshot:try:result = await server.screenshot(url=test_url, session_id=opened.session_id)assert isinstance(result[0], ImageContent)- assert result[0].mimeType == "image/png"+ assert result[0].mime_type == "image/png"finally:await server.close_session(opened.session_id)@@ -485,7 +492,7 @@ class TestStaticTokenVerifier:class TestMCPServerAuthentication:- """Test how the authentication token and transport security reach the FastMCP server"""+ """Test how the authentication token and transport security reach the MCP server"""def test_no_token_leaves_auth_disabled(self, monkeypatch):monkeypatch.delenv(MCP_AUTH_TOKEN_ENV, raising=False)@@ -510,20 +517,32 @@ class TestMCPServerAuthentication:assert ScraplingMCPServer(auth_token=explicit_key)._auth_token == explicit_keydef test_all_tools_are_registered_with_auth_enabled(self, monkeypatch):- """FastMCP raises when `auth` and `token_verifier` are mismatched, so building must stay valid"""+ """MCPServer raises when `auth` and `token_verifier` are mismatched, so building must stay valid"""monkeypatch.delenv(MCP_AUTH_TOKEN_ENV, raising=False)built = ScraplingMCPServer(auth_token=SHARED_KEY)._build_server("0.0.0.0", 8000)assert len(built._tool_manager.list_tools()) == 10- def test_allowed_hosts_enable_dns_rebinding_protection(self, monkeypatch):- monkeypatch.delenv(MCP_AUTH_TOKEN_ENV, raising=False)- server = ScraplingMCPServer()-- assert server._build_server("0.0.0.0", 8000).settings.transport_security is None+ def test_allowed_hosts_enable_dns_rebinding_protection(self):+ assert ScraplingMCPServer._transport_security(()) is None- security = server._build_server("0.0.0.0", 8000, ("mcp.example.com:8000",)).settings.transport_security+ security = ScraplingMCPServer._transport_security(("mcp.example.com:8000",))assert security is not Noneassert security.enable_dns_rebinding_protection is Trueassert security.allowed_hosts == ["mcp.example.com:8000"]assert security.allowed_origins == ["http://mcp.example.com:8000", "https://mcp.example.com:8000"]+++class TestServerToolRegistration:+ """Test the built server end-to-end through an in-memory MCP client"""++ @pytest.mark.asyncio+ async def test_tools_are_listed_with_expected_schemas(self):+ """All 10 tools are advertised, and only the screenshot tool skips the structured output schema"""+ server = ScraplingMCPServer()._build_server("127.0.0.1", 8000)+ async with Client(server) as client:+ tools = {tool.name: tool for tool in (await client.list_tools()).tools}++ assert len(tools) == 10+ assert tools["screenshot"].output_schema is None+ assert all(tool.output_schema is not None for name, tool in tools.items() if name != "screenshot")