🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
feat(mcp): add tool annotations, server card metadata, and a tools/list cache hint
Karim shoair0e16c4d471fc
diff --git a/scrapling/core/ai.py b/scrapling/core/ai.pyindex 91d03de..10b364e 100644--- a/scrapling/core/ai.py+++ b/scrapling/core/ai.py@@ -9,10 +9,12 @@ from mcp.server import MCPServerfrom mcp.server.mcpserver import Imagefrom mcp.server.auth.provider import AccessToken, TokenVerifierfrom mcp.server.auth.settings import AuthSettings+from mcp.server.caching import CacheHintfrom mcp.server.transport_security import TransportSecuritySettings-from mcp.types import ImageContent, TextContent+from mcp.types import Icon, ImageContent, TextContent, ToolAnnotationsfrom pydantic import AnyHttpUrl, BaseModel, Field+from scrapling import __version__from scrapling.core.utils import logfrom scrapling.core.shell import Convertor, _CONTROL_CHARS_PATTERNfrom scrapling.engines.toolbelt.custom import Response as _ScraplingResponse@@ -42,6 +44,9 @@ SessionType = Literal["dynamic", "stealthy"]ScreenshotType = Literal["png", "jpeg"]MCP_EXECUTABLE_PATH_ENV = "SCRAPLING_EXECUTABLE_PATH"MCP_AUTH_TOKEN_ENV = "SCRAPLING_MCP_AUTH_TOKEN" # nosec B105 - the name of the variable, not a token+_FETCH_TOOL_ANNOTATIONS = ToolAnnotations(read_only_hint=True, open_world_hint=True)+_SESSION_TOOL_ANNOTATIONS = ToolAnnotations(read_only_hint=False, destructive_hint=False, open_world_hint=True)+_LIST_TOOL_ANNOTATIONS = ToolAnnotations(read_only_hint=True, open_world_hint=False)class ResponseModel(BaseModel):@@ -906,6 +911,16 @@ class ScraplingMCPServer:def _build_server(self, host: str, port: int) -> MCPServer:"""Build the MCPServer with all tools registered and the optional authentication settings applied."""settings: Dict[str, Any] = {+ "title": "Scrapling",+ "version": __version__,+ "website_url": "https://scrapling.readthedocs.io/en/latest/",+ "icons": [+ Icon(+ src="https://raw.githubusercontent.com/D4Vinci/Scrapling/main/docs/assets/logo.png",+ mime_type="image/png",+ )+ ],+ "cache_hints": {"tools/list": CacheHint(ttl_ms=3_600_000, scope="public")},"instructions": """Follow these instructions precisely:1. When the `open_session` tool is used, make sure to close the session with `close_session` after you finish, and use `list_sessions` if you lose track of the open sessions.2. If the user didn't specify which tool to use, start with the `get` tool, then escalate. The `get` tool and the bulk version are only suitable for low-mid protection levels.@@ -919,7 +934,7 @@ class ScraplingMCPServer:8. If you are making multiple requests, use the bulk version of the tool to be more efficient.9. If you are crawling/browsing a website, be more efficient by using the `css_selector` parameter to only access the parts you are interested in and save money/time.10. The user can pass a CDP URL to connect to a remote browser session through the `open_session` tool, then use it in the rest of the tools.-"""+""",}if self._auth_token:base_url = AnyHttpUrl(f"http://{host}:{port}")@@ -928,16 +943,44 @@ class ScraplingMCPServer:server = MCPServer(name="Scrapling", **settings)# Session management tools- server.add_tool(self.open_session, title="open_session", structured_output=True)- server.add_tool(self.close_session, title="close_session", structured_output=True)- server.add_tool(self.list_sessions, title="list_sessions", structured_output=True)+ server.add_tool(+ self.open_session, title="open_session", structured_output=True, annotations=_SESSION_TOOL_ANNOTATIONS+ )+ server.add_tool(+ self.close_session, title="close_session", structured_output=True, annotations=_SESSION_TOOL_ANNOTATIONS+ )+ server.add_tool(+ self.list_sessions, title="list_sessions", structured_output=True, annotations=_LIST_TOOL_ANNOTATIONS+ )# HTTP tools- server.add_tool(self.get, title="get", description=self.get.__doc__, structured_output=True)- server.add_tool(self.bulk_get, title="bulk_get", description=self.bulk_get.__doc__, structured_output=True)+ server.add_tool(+ self.get,+ title="get",+ description=self.get.__doc__,+ structured_output=True,+ annotations=_FETCH_TOOL_ANNOTATIONS,+ )+ server.add_tool(+ self.bulk_get,+ title="bulk_get",+ description=self.bulk_get.__doc__,+ structured_output=True,+ annotations=_FETCH_TOOL_ANNOTATIONS,+ )# Dynamic browser tools- server.add_tool(self.fetch, title="fetch", description=self.fetch.__doc__, structured_output=True)server.add_tool(- self.bulk_fetch, title="bulk_fetch", description=self.bulk_fetch.__doc__, structured_output=True+ self.fetch,+ title="fetch",+ description=self.fetch.__doc__,+ structured_output=True,+ annotations=_FETCH_TOOL_ANNOTATIONS,+ )+ server.add_tool(+ self.bulk_fetch,+ title="bulk_fetch",+ description=self.bulk_fetch.__doc__,+ structured_output=True,+ annotations=_FETCH_TOOL_ANNOTATIONS,)# Stealthy browser toolsserver.add_tool(@@ -945,16 +988,22 @@ class ScraplingMCPServer:title="stealthy_fetch",description=self.stealthy_fetch.__doc__,structured_output=True,+ annotations=_FETCH_TOOL_ANNOTATIONS,)server.add_tool(self.bulk_stealthy_fetch,title="bulk_stealthy_fetch",description=self.bulk_stealthy_fetch.__doc__,structured_output=True,+ annotations=_FETCH_TOOL_ANNOTATIONS,)# Screenshot tool (returns image + url content blocks, not structured JSON)server.add_tool(- self.screenshot, title="screenshot", description=self.screenshot.__doc__, structured_output=False+ self.screenshot,+ title="screenshot",+ description=self.screenshot.__doc__,+ structured_output=False,+ annotations=_FETCH_TOOL_ANNOTATIONS,)return serverdiff --git a/tests/ai/test_ai_mcp.py b/tests/ai/test_ai_mcp.pyindex d45f7db..68b4894 100644--- a/tests/ai/test_ai_mcp.py+++ b/tests/ai/test_ai_mcp.py@@ -10,6 +10,7 @@ import pytest_httpbinfrom mcp.client import Clientfrom mcp.types import ImageContent, TextContent+from scrapling import __version__ as scrapling_versionfrom scrapling.engines.toolbelt.custom import Responsefrom scrapling.core.ai import (MCP_AUTH_TOKEN_ENV,@@ -547,3 +548,29 @@ class TestServerToolRegistration:assert len(tools) == 10assert tools["screenshot"].output_schema is Noneassert all(tool.output_schema is not None for name, tool in tools.items() if name != "screenshot")++ @pytest.mark.asyncio+ async def test_server_metadata_and_tool_annotations(self):+ """Server card metadata, cache hints, and tool annotations are advertised to clients"""+ server = ScraplingMCPServer()._build_server("127.0.0.1", 8000)+ async with Client(server) as client:+ info = client.server_info+ result = await client.list_tools()++ assert info is not None+ assert info.title == "Scrapling"+ assert info.version == scrapling_version+ assert info.website_url and info.icons+ assert result.ttl_ms == 3_600_000 and result.cache_scope == "public"++ annotations = {tool.name: tool.annotations for tool in result.tools if tool.annotations is not None}+ assert len(annotations) == 10+ for name in ("get", "bulk_get", "fetch", "bulk_fetch", "stealthy_fetch", "bulk_stealthy_fetch", "screenshot"):+ assert annotations[name].read_only_hint is True+ assert annotations[name].open_world_hint is True+ for name in ("open_session", "close_session"):+ assert annotations[name].read_only_hint is False+ assert annotations[name].destructive_hint is False+ assert annotations[name].open_world_hint is True+ assert annotations["list_sessions"].read_only_hint is True+ assert annotations["list_sessions"].open_world_hint is False