🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
fix(mcp): raise detailed mcp errors on newer versions
Karim shoair6f21ba42ce2c
diff --git a/scrapling/core/ai.py b/scrapling/core/ai.pyindex c57cecc..9bcea23 100644--- a/scrapling/core/ai.py+++ b/scrapling/core/ai.py@@ -4,19 +4,24 @@ from json import dumpsfrom time import monotonicfrom hmac import compare_digestfrom asyncio import gather+from functools import wrapsfrom contextlib import contextmanagerfrom datetime import datetime, timezonefrom dataclasses import dataclass, fieldfrom anyio import CancelScope+from curl_cffi.curl import CurlErrorfrom mcp.server import MCPServerfrom mcp.server.mcpserver import Image+from mcp.server.mcpserver.exceptions import ToolErrorfrom mcp.server.auth.provider import AccessToken, TokenVerifierfrom mcp.server.auth.settings import AuthSettingsfrom mcp.server.caching import CacheHintfrom mcp.server.transport_security import TransportSecuritySettingsfrom mcp.types import Icon, ImageContent, TextContent, ToolAnnotations+from msgspec import ValidationError as MsgspecValidationErrorfrom pydantic import AnyHttpUrl, BaseModel, Field+from patchright.async_api import Error as PatchrightErrorfrom scrapling import __version__from scrapling.core.utils import log@@ -41,6 +46,8 @@ from scrapling.core._types import (Dict,List,Any,+ Awaitable,+ Callable,Annotated,Set,Sequence,@@ -199,6 +206,19 @@ class _StaticTokenVerifier(TokenVerifier):return None+def _mcp_tool(tool: Callable[..., Awaitable[Any]]) -> Callable[..., Awaitable[Any]]:+ @wraps(tool)+ async def wrapped(*args: Any, **kwargs: Any) -> Any:+ try:+ return await tool(*args, **kwargs)+ except (ValueError, RuntimeError, TypeError, PatchrightError, CurlError) as error:+ if isinstance(error, TypeError) and not isinstance(error.__cause__, MsgspecValidationError):+ raise+ raise ToolError(str(error)) from error++ return wrapped++class ScraplingMCPServer:def __init__(self, executable_path: Optional[str] = None, auth_token: Optional[str] = None):"""Create a Scrapling MCP server.@@ -1064,44 +1084,53 @@ class ScraplingMCPServer:server = MCPServer(name="Scrapling", **settings)# Session management toolsserver.add_tool(- self.browser_open, title="Open browser", structured_output=True, annotations=_SESSION_TOOL_ANNOTATIONS+ _mcp_tool(self.browser_open),+ title="Open browser",+ structured_output=True,+ annotations=_SESSION_TOOL_ANNOTATIONS,)server.add_tool(- self.open_request_session,+ _mcp_tool(self.open_request_session),title="Open HTTP session",structured_output=True,annotations=_SESSION_TOOL_ANNOTATIONS,)server.add_tool(- self.close_session, title="Close session", structured_output=True, annotations=_SESSION_TOOL_ANNOTATIONS+ _mcp_tool(self.close_session),+ title="Close session",+ structured_output=True,+ annotations=_SESSION_TOOL_ANNOTATIONS,)server.add_tool(- self.list_sessions, title="List sessions", structured_output=True, annotations=_LIST_TOOL_ANNOTATIONS+ _mcp_tool(self.list_sessions),+ title="List sessions",+ structured_output=True,+ annotations=_LIST_TOOL_ANNOTATIONS,)# HTTP toolsserver.add_tool(- self.make_request,+ _mcp_tool(self.make_request),title="Send HTTP request",description=self.make_request.__doc__,structured_output=True,annotations=_FETCH_TOOL_ANNOTATIONS,)server.add_tool(- self.bulk_get,+ _mcp_tool(self.bulk_get),title="Get pages via HTTP requests",description=self.bulk_get.__doc__,structured_output=True,annotations=_FETCH_TOOL_ANNOTATIONS,)server.add_tool(- self.browser_fetch_once,+ _mcp_tool(self.browser_fetch_once),title="Fetch page in browser",description=self.browser_fetch_once.__doc__,structured_output=True,annotations=_FETCH_TOOL_ANNOTATIONS,)server.add_tool(- self.browser_fetch_many_once,+ _mcp_tool(self.browser_fetch_many_once),title="Fetch pages in browser",description=self.browser_fetch_many_once.__doc__,structured_output=True,@@ -1109,53 +1138,53 @@ class ScraplingMCPServer:)# Session-scoped fetch toolsserver.add_tool(- self.browser_fetch,+ _mcp_tool(self.browser_fetch),title="Fetch in browser session",description=self.browser_fetch.__doc__,structured_output=True,annotations=_FETCH_TOOL_ANNOTATIONS,)server.add_tool(- self.session_make_request,+ _mcp_tool(self.session_make_request),title="Send HTTP request in HTTP session",description=self.session_make_request.__doc__,structured_output=True,annotations=_FETCH_TOOL_ANNOTATIONS,)server.add_tool(- self.browser_snapshot,+ _mcp_tool(self.browser_snapshot),title="Browser page snapshot",description=self.browser_snapshot.__doc__,structured_output=False,annotations=_FETCH_TOOL_ANNOTATIONS,)server.add_tool(- self.browser_actions,+ _mcp_tool(self.browser_actions),title="Browser actions",description=self.browser_actions.__doc__,structured_output=False,annotations=_INPUT_TOOL_ANNOTATIONS,)server.add_tool(- self.browser_evaluate,+ _mcp_tool(self.browser_evaluate),title="Run JavaScript",description=self.browser_evaluate.__doc__,structured_output=False,annotations=_INPUT_TOOL_ANNOTATIONS,)server.add_tool(- self.browser_network_requests,+ _mcp_tool(self.browser_network_requests),title="List network requests",annotations=_LIST_TOOL_ANNOTATIONS,)server.add_tool(- self.browser_network_request,+ _mcp_tool(self.browser_network_request),title="Read network request",annotations=_LIST_TOOL_ANNOTATIONS,)# Screenshot tool (returns image + url content blocks, not structured JSON)server.add_tool(- self.browser_screenshot,+ _mcp_tool(self.browser_screenshot),title="Take browser session screenshot",description=self.browser_screenshot.__doc__,structured_output=False,diff --git a/tests/ai/test_browser_mouse.py b/tests/ai/test_browser_mouse.pyindex b1a4ee2..bd51bd4 100644--- a/tests/ai/test_browser_mouse.py+++ b/tests/ai/test_browser_mouse.py@@ -504,24 +504,27 @@ async def test_browser_mouse_cleanup_failure_preserves_click_error(page.is_closed.return_value = closedraise cleanup_error- page.locator.return_value.click.side_effect = press- page.mouse.up.side_effect = unpress- task = asyncio.create_task(- server.browser_actions(- "browser",- [{"type": "click", "selector": "#target", "button": "right"}, {"type": "wheel", "delta_y": 120}],- )- )- try:- await asyncio.wait_for(entered.wait(), 5)- task.cancel() if cancelled else release.set()+ async def click() -> None:with pytest.raises(asyncio.CancelledError if cancelled else RuntimeError) as error:- await asyncio.wait_for(task, 5)+ await server.browser_actions(+ "browser",+ [{"type": "click", "selector": "#target", "button": "right"}, {"type": "wheel", "delta_y": 120}],+ )original = error.value if cancelled else error.value.__cause__if not cancelled:assert str(error.value) == "Action 1 (click) failed: click timed out"assert original is timeoutassert original is not None and original.__cause__ is cleanup_error+ raise error.value++ page.locator.return_value.click.side_effect = press+ page.mouse.up.side_effect = unpress+ task = asyncio.create_task(click())+ try:+ await asyncio.wait_for(entered.wait(), 5)+ task.cancel() if cancelled else release.set()+ with pytest.raises(asyncio.CancelledError if cancelled else RuntimeError):+ await asyncio.wait_for(task, 5)finally:if not task.done():task.cancel()diff --git a/tests/ai/test_browser_screenshot.py b/tests/ai/test_browser_screenshot.pyindex df7306a..5c23ad1 100644--- a/tests/ai/test_browser_screenshot.py+++ b/tests/ai/test_browser_screenshot.py@@ -188,20 +188,25 @@ async def test_browser_screenshot_disposal_failure_preserves_capture_error_or_caraise native_errorreturn b"captured pixels"- element.screenshot.side_effect = capture- element.dispose.side_effect = cleanup_error- task = asyncio.create_task(server.browser_screenshot("browser", selector="#card"))- try:- await asyncio.wait_for(entered.wait(), 5)- task.cancel() if outcome == "cancel" else release.set()+ async def screenshot() -> None:with pytest.raises(asyncio.CancelledError if outcome == "cancel" else PatchrightError) as error:- await task+ await server.browser_screenshot("browser", selector="#card")if outcome == "success":assert error.value is cleanup_errorelse:if outcome == "error":assert error.value is native_errorassert error.value.__cause__ is cleanup_error+ raise error.value++ element.screenshot.side_effect = capture+ element.dispose.side_effect = cleanup_error+ task = asyncio.create_task(screenshot())+ try:+ await asyncio.wait_for(entered.wait(), 5)+ task.cancel() if outcome == "cancel" else release.set()+ with pytest.raises(asyncio.CancelledError if outcome == "cancel" else PatchrightError):+ await taskfinally:if not task.done():task.cancel()diff --git a/tests/ai/test_tool_errors.py b/tests/ai/test_tool_errors.pynew file mode 100644index 0000000..9b59b5f--- /dev/null+++ b/tests/ai/test_tool_errors.py@@ -0,0 +1,77 @@+from asyncio import CancelledError+from inspect import signature++import pytest+from curl_cffi.curl import CurlError+from mcp import MCPError+from mcp.client import Client+from mcp.server.mcpserver.exceptions import ToolError+from mcp.types import TextContent+from patchright.async_api import Error as PatchrightError++from scrapling.core.ai import ScraplingMCPServer, _mcp_tool+from scrapling.core._types import Any+++ "error",+ [+ ValueError("bad input"),+ RuntimeError("page busy"),+ PatchrightError("page closed"),+ CurlError("connection failed"),+ ],+)+async def test_expected_tool_errors_keep_their_message(error: Exception) -> None:+ async def tool() -> None:+ raise error++ with pytest.raises(ToolError) as result:+ await _mcp_tool(tool)()+ assert str(result.value) == str(error)+ assert result.value.__cause__ is error+++ "arguments, message",+ [+ ({"executable_path": "/missing/scrapling-browser-binary"}, "Browser executable path not found"),+ ({"cdp_url": "invalid://localhost:9222"}, "CDP URL must use"),+ ({"proxy": {"username": "test"}}, "server"),+ ],+)+async def test_browser_configuration_errors_reach_client(arguments: dict[str, Any], message: str) -> None:+ server = ScraplingMCPServer()+ async with Client(server._build_server("127.0.0.1", 8000)) as client:+ result = await client.call_tool("browser_open", arguments)+ assert result.is_error+ assert result.content and isinstance(result.content[0], TextContent)+ assert message in result.content[0].text+ assert await server.list_sessions() == []+++ "error",+ [KeyError("internal"), TypeError("internal"), CancelledError(), MCPError(code=-32603, message="protocol failure")],+)+async def test_other_tool_errors_pass_through(error: BaseException) -> None:+ async def tool() -> None:+ raise error++ with pytest.raises(type(error)) as result:+ await _mcp_tool(tool)()+ assert result.value is error+++async def test_tool_wrapper_preserves_signature_and_result() -> None:+ async def tool(value: int = 3) -> dict[str, int]:+ return {"value": value}++ wrapped = _mcp_tool(tool)+ assert wrapped.__name__ == tool.__name__+ assert signature(wrapped) == signature(tool)+ assert await wrapped(value=5) == {"value": 5}