Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

feat(mcp): Add new tools to move and click with the mouse

Karim shoair1af55df34ddf
diff --git a/scrapling/core/_types.py b/scrapling/core/_types.py
index fc8a075..cd026e7 100644
--- a/scrapling/core/_types.py
+++ b/scrapling/core/_types.py
@@ -8,6 +8,7 @@ from typing import (
cast,
overload,
Any,
+ Annotated,
Callable,
Dict,
Generator,
diff --git a/scrapling/core/ai.py b/scrapling/core/ai.py
index 8b769c5..933b8c2 100644
--- a/scrapling/core/ai.py
+++ b/scrapling/core/ai.py
@@ -1,10 +1,12 @@
from uuid import uuid4
from os import environ
from hmac import compare_digest
-from asyncio import gather
+from asyncio import CancelledError, gather
+from contextlib import contextmanager
from datetime import datetime, timezone
from dataclasses import dataclass, field
+from anyio import CancelScope
from mcp.server import MCPServer
from mcp.server.mcpserver import Image
from mcp.server.auth.provider import AccessToken, TokenVerifier
@@ -12,7 +14,9 @@ from mcp.server.auth.settings import AuthSettings
from mcp.server.caching import CacheHint
from mcp.server.transport_security import TransportSecuritySettings
from mcp.types import Icon, ImageContent, TextContent, ToolAnnotations
-from pydantic import AnyHttpUrl, BaseModel, Field
+from pydantic import AnyHttpUrl, BaseModel, Field, FiniteFloat, PositiveInt
+from playwright.async_api import TimeoutError as PlaywrightTimeoutError
+from patchright.async_api import TimeoutError as PatchrightTimeoutError
from scrapling import __version__
from scrapling.core.utils import log
@@ -34,8 +38,10 @@ from scrapling.core._types import (
Dict,
List,
Any,
+ Annotated,
Set,
Sequence,
+ Iterator,
SetCookieParam,
extraction_types,
SelectorWaitStates,
@@ -47,6 +53,9 @@ SessionType = Literal["dynamic", "stealthy", "static"]
BrowserSessionType = Literal["dynamic", "stealthy"]
SessionExtractionType = Literal[extraction_types, "snapshot"]
ScreenshotType = Literal["png", "jpeg"]
+MouseButton = Literal["left", "right", "middle"]
+NonNegativeFiniteFloat = Annotated[float, Field(ge=0, allow_inf_nan=False)]
+NonEmptyString = Annotated[str, Field(min_length=1)]
MCP_EXECUTABLE_PATH_ENV = "SCRAPLING_EXECUTABLE_PATH"
MCP_AUTH_TOKEN_ENV = "SCRAPLING_MCP_AUTH_TOKEN" # nosec B105 - the name of the variable, not a token
@@ -92,6 +101,9 @@ def _session_settings(session: Any) -> Dict[str, Any]:
_FETCH_TOOL_ANNOTATIONS = ToolAnnotations(read_only_hint=True, open_world_hint=True)
_SESSION_TOOL_ANNOTATIONS = ToolAnnotations(read_only_hint=False, destructive_hint=False, open_world_hint=True)
_LIST_TOOL_ANNOTATIONS = ToolAnnotations(read_only_hint=True, open_world_hint=False)
+_MOUSE_TOOL_ANNOTATIONS = ToolAnnotations(
+ read_only_hint=False, destructive_hint=True, idempotent_hint=False, open_world_hint=True
+)
class ResponseModel(BaseModel):
@@ -356,7 +368,7 @@ class ScraplingMCPServer:
self,
session_id: str,
depth: Optional[int] = None,
- boxes: bool = False,
+ boxes: bool = True,
) -> str:
"""Return the current page's AI ARIA snapshot with element references as plain text, without navigating.
@@ -371,14 +383,18 @@ class ScraplingMCPServer:
session_id: str,
css_selector: Optional[str] = None,
depth: Optional[int] = None,
- boxes: bool = False,
+ boxes: bool = True,
) -> str:
"""Reserve the current page and return its AI ARIA snapshot."""
- entry = self._get_session(session_id, expected_type=["dynamic", "stealthy"])
+ with self._browser_page(session_id) as (session, page):
+ return await session._snapshot(page, depth=depth, boxes=boxes, css_selector=css_selector)
+ @contextmanager
+ def _browser_page(self, session_id: str) -> Iterator[Tuple[Any, Any]]:
+ entry = self._get_session(session_id, expected_type=["dynamic", "stealthy"])
pool = entry.session.page_pool
if not pool.pages_count:
- raise ValueError(f"Session '{session_id}' has no page to snapshot. Use browser_fetch first.")
+ raise ValueError(f"Session '{session_id}' has no page. Use browser_fetch first.")
page_info = pool.get_ready_page()
if page_info is None:
raise RuntimeError(f"Session '{session_id}' has a busy page. Wait for the current request to finish.")
@@ -386,13 +402,74 @@ class ScraplingMCPServer:
try:
if page_info.page.is_closed():
raise RuntimeError(f"Session '{session_id}' has a closed page. Use browser_fetch to open a new one.")
- return await entry.session._snapshot(page_info.page, depth=depth, boxes=boxes, css_selector=css_selector)
+ yield entry.session, page_info.page
finally:
if page_info.page.is_closed():
pool.remove_page(page_info)
else:
page_info.mark_ready()
+ async def browser_mouse_move(
+ self,
+ session_id: str,
+ x: FiniteFloat,
+ y: FiniteFloat,
+ steps: PositiveInt = 1,
+ ) -> str:
+ """Move the native browser mouse on the current page; return plain text. Coordinates are CSS pixels from the main frame viewport's top-left.
+
+ :param session_id: ID from `browser_open`; call `browser_fetch` first.
+ :param x: Horizontal position.
+ :param y: Vertical position.
+ :param steps: Number of mousemove events.
+ """
+ with self._browser_page(session_id) as (_, page):
+ await page.mouse.move(x, y, steps=steps)
+ return f"Mouse moved to ({x}, {y})."
+
+ async def browser_click(
+ self,
+ session_id: str,
+ selector: Optional[NonEmptyString] = None,
+ ref: Optional[NonEmptyString] = None,
+ x: Optional[FiniteFloat] = None,
+ y: Optional[FiniteFloat] = None,
+ button: MouseButton = "left",
+ click_count: PositiveInt = 1,
+ delay: NonNegativeFiniteFloat = 0,
+ timeout: NonNegativeFiniteFloat = 30000,
+ ) -> str:
+ """Click using exactly one selector, snapshot ref, or (x, y) pair; return plain text.
+ Selector/ref clicks wait and scroll into view.
+ Coordinate clicks do not scroll or wait for navigation. Cancellation/timeouts release the button if the page stays open; completed actions remain.
+
+ :param session_id: ID from `browser_open`; call `browser_fetch` first.
+ :param selector: Playwright selector (e.g. CSS or XPath) matching exactly one element.
+ :param ref: Element reference from the current snapshot, e.g. "e2".
+ :param x: Horizontal CSS pixels from the main frame viewport's top-left.
+ :param y: Vertical CSS pixels from the same origin.
+ :param button: Mouse button.
+ :param click_count: Click count; use 2 for a double-click.
+ :param delay: Milliseconds between button press and release.
+ :param timeout: Selector/ref timeout in milliseconds; 0 disables it. Ignored for coordinates.
+ """
+ if sum(value is not None for value in (selector, ref, x)) != 1 or (x is None) != (y is None):
+ raise ValueError("Provide exactly one target: 'selector', 'ref', or both 'x' and 'y'.")
+ with self._browser_page(session_id) as (_, page):
+ try:
+ if selector is not None or ref is not None:
+ await page.locator(selector if selector is not None else f"aria-ref={ref}").click(
+ button=button, click_count=click_count, delay=delay, timeout=timeout
+ )
+ else:
+ await page.mouse.click(x, y, button=button, click_count=click_count, delay=delay)
+ except (CancelledError, PlaywrightTimeoutError, PatchrightTimeoutError):
+ with CancelScope(shield=True):
+ if not page.is_closed():
+ await page.mouse.up(button=button)
+ raise
+ return "Click sent."
+
async def browser_screenshot(
self,
url: str,
@@ -966,7 +1043,7 @@ class ScraplingMCPServer:
:param url: URL to fetch.
:param session_id: ID from `browser_open`.
- :param extraction_type: Content output format.
+ :param extraction_type: Content output format; snapshots include element bounding boxes.
:param css_selector: Select after `main_content_only` filtering; snapshots require exactly one match.
:param main_content_only: Sanitize <body> before selection; False uses the full document. Ignored for snapshots.
:param wait: Extra milliseconds after the page is ready, before returning.
@@ -1119,7 +1196,8 @@ class ScraplingMCPServer:
8. If you are making multiple parallel one-shot requests, use the bulk version of the tool to be more efficient.
9. If you are crawling/browsing a website, be more efficient by using the `css_selector` parameter to only access the parts you are interested in and save money/time. Example: use the `a` selector to extract the urls right away.
10. The user can pass a CDP URL to connect to a remote browser session through the `browser_open` tool, then use it with the session tools.
-11. Set `extraction_type="snapshot"` on `browser_fetch` to get an AI ARIA snapshot with element references in its content field. Use `css_selector` to snapshot one element, or omit it for the whole page. `main_content_only` and `pierce_shadow` do not filter snapshots. Use `browser_snapshot` to read the current page again without navigating; it returns plain text. Its `depth` and `boxes` options limit the tree or include element bounding boxes.
+11. Set `extraction_type="snapshot"` on `browser_fetch` to get an AI ARIA snapshot with element references and bounding boxes in its content field. Use `css_selector` to snapshot one element, or omit it for the whole page. `main_content_only` and `pierce_shadow` do not filter snapshots. Use `browser_snapshot` to read the current page without navigating; it returns plain text with boxes by default. Set `depth` to limit the tree or `boxes=False` to omit boxes.
+12. Use `browser_mouse_move` to move the mouse on the current page. Use `browser_click` with exactly one target: `selector`, `ref` from the current snapshot, or both `x` and `y` viewport CSS coordinates. Selector/ref clicks use Playwright's normal waiting and scrolling; coordinate clicks do not scroll or wait for navigation. Get coordinates from `browser_snapshot` and inspect the page afterward with `browser_snapshot`. Clicks can change website data. Do not repeat a click without checking the page state.
""",
}
if self._auth_token:
@@ -1211,6 +1289,20 @@ class ScraplingMCPServer:
structured_output=False,
annotations=_FETCH_TOOL_ANNOTATIONS,
)
+ server.add_tool(
+ self.browser_mouse_move,
+ title="Move mouse",
+ description=self.browser_mouse_move.__doc__,
+ structured_output=False,
+ annotations=_MOUSE_TOOL_ANNOTATIONS,
+ )
+ server.add_tool(
+ self.browser_click,
+ title="Click",
+ description=self.browser_click.__doc__,
+ structured_output=False,
+ annotations=_MOUSE_TOOL_ANNOTATIONS,
+ )
# Screenshot tool (returns image + url content blocks, not structured JSON)
server.add_tool(
self.browser_screenshot,
diff --git a/tests/ai/test_ai_mcp.py b/tests/ai/test_ai_mcp.py
index 4bee70d..5d75183 100644
--- a/tests/ai/test_ai_mcp.py
+++ b/tests/ai/test_ai_mcp.py
@@ -925,7 +925,7 @@ class TestMCPServerAuthentication:
monkeypatch.delenv(MCP_AUTH_TOKEN_ENV, raising=False)
built = ScraplingMCPServer(auth_token=SHARED_KEY)._build_server("0.0.0.0", 8000)
- assert len(built._tool_manager.list_tools()) == 14
+ assert len(built._tool_manager.list_tools()) == 16
def test_http_without_a_token_refuses_to_serve(self, monkeypatch):
"""The streamable-http transport requires authentication unless the caller explicitly opts out"""
@@ -982,23 +982,52 @@ class TestServerToolRegistration:
@pytest.mark.asyncio
async def test_tools_are_listed_with_expected_schemas(self):
- """All 14 tools are advertised, with plain content for screenshots and snapshots"""
+ """All 16 tools are advertised, with plain content for screenshots, snapshots, and mouse actions"""
server = ScraplingMCPServer()._build_server("127.0.0.1", 8000)
async with Client(server) as client:
assert client.instructions
tools = {tool.name: tool for tool in (await client.list_tools()).tools}
- for name in ("open_session", "session_fetch", "screenshot"):
+ for name in (
+ "open_session",
+ "session_fetch",
+ "screenshot",
+ "browser_mouse",
+ "browser_mouse_move_xy",
+ "fetch",
+ "bulk_fetch",
+ "stealthy_fetch",
+ "bulk_stealthy_fetch",
+ ):
result = await client.call_tool(name, {})
assert result.is_error
- assert len(tools) == 14
+ assert len(tools) == 16
assert tools["browser_screenshot"].output_schema is None
assert tools["browser_snapshot"].output_schema is None
- assert {"session_snapshot", "open_session", "session_fetch", "screenshot"}.isdisjoint(tools)
+ assert tools["browser_mouse_move"].output_schema is None
+ assert tools["browser_click"].output_schema is None
+ assert {
+ "session_snapshot",
+ "open_session",
+ "session_fetch",
+ "screenshot",
+ "browser_mouse",
+ "browser_mouse_move_xy",
+ "fetch",
+ "bulk_fetch",
+ "stealthy_fetch",
+ "bulk_stealthy_fetch",
+ }.isdisjoint(tools)
+ assert {
+ "browser_fetch_once",
+ "browser_fetch_many_once",
+ "browser_stealth_fetch_once",
+ "browser_stealth_fetch_many_once",
+ } <= tools.keys()
assert all(
tool.output_schema is not None
for name, tool in tools.items()
- if name not in ("browser_screenshot", "browser_snapshot")
+ if name not in ("browser_screenshot", "browser_snapshot", "browser_mouse_move", "browser_click")
)
@pytest.mark.asyncio
@@ -1063,7 +1092,7 @@ class TestServerToolRegistration:
assert result.ttl_ms == 3_600_000 and result.cache_scope == "public"
annotations = {tool.name: tool.annotations for tool in result.tools if tool.annotations is not None}
- assert len(annotations) == 14
+ assert len(annotations) == 16
for name in (
"make_request",
"bulk_get",
@@ -1084,6 +1113,11 @@ class TestServerToolRegistration:
assert annotations[name].open_world_hint is True
assert annotations["list_sessions"].read_only_hint is True
assert annotations["list_sessions"].open_world_hint is False
+ for name in ("browser_mouse_move", "browser_click"):
+ assert annotations[name].read_only_hint is False
+ assert annotations[name].destructive_hint is True
+ assert annotations[name].idempotent_hint is False
+ assert annotations[name].open_world_hint is True
@pytest.mark.asyncio
diff --git a/tests/ai/test_browser_mouse.py b/tests/ai/test_browser_mouse.py
new file mode 100644
index 0000000..853384c
--- /dev/null
+++ b/tests/ai/test_browser_mouse.py
@@ -0,0 +1,657 @@
+import asyncio
+from contextlib import suppress
+from json import loads
+from os import getenv
+from re import search
+from unittest.mock import AsyncMock, Mock
+
+import pytest
+from anyio import CancelScope
+from mcp.client import Client
+from mcp.types import TextContent
+from patchright.async_api import Error as PatchrightError
+from patchright.async_api import TimeoutError as PatchrightTimeoutError
+from playwright.async_api import Error as PlaywrightError
+from playwright.async_api import TimeoutError as PlaywrightTimeoutError
+from pydantic import ValidationError
+
+from scrapling.core.ai import ScraplingMCPServer, SessionType, _SessionEntry
+from scrapling.core._types import Any
+from scrapling.engines._browsers._base import AsyncSession
+
+
+HTML = """<!DOCTYPE html><html><head><style>
+#target { position: absolute; left: 80px; top: 80px; width: 120px; height: 60px; }
+#target:hover { background: rgb(0, 128, 0); }
+#next { position: absolute; left: 80px; top: 180px; width: 120px; height: 40px; }
+#below { position: absolute; left: 80px; top: 4000px; width: 120px; height: 60px; }
+</style></head><body><button id="target">Target</button><a id="next" href="/next">Next page</a>
+<button id="below">Below</button>
+<textarea id="events" hidden>[]</textarea><script>
+const events = document.querySelector('#events');
+for (const type of ['mousemove', 'mousedown', 'mouseup', 'click', 'dblclick', 'contextmenu', 'auxclick']) {
+ document.addEventListener(type, event => {
+ events.value = JSON.stringify([...JSON.parse(events.value), {type, x: event.clientX, y: event.clientY,
+ button: event.button, buttons: event.buttons, detail: event.detail, trusted: event.isTrusted,
+ target: event.target.id}]);
+ if (type === 'contextmenu') event.preventDefault();
+ });
+}
+</script></body></html>"""
+
+
+def _server(session_type: SessionType = "dynamic") -> tuple[ScraplingMCPServer, AsyncSession, Mock]:
+ server = ScraplingMCPServer()
+ session = AsyncSession()
+ session._is_alive = True
+ page = Mock()
+ page.is_closed.return_value = False
+ page.mouse.move = AsyncMock()
+ page.mouse.click = AsyncMock()
+ page.mouse.up = AsyncMock()
+ page.locator.return_value.click = AsyncMock()
+ session.page_pool.add_page(page).mark_ready()
+ server._sessions["browser"] = _SessionEntry(session, session_type)
+ return server, session, page
+
+
+async def test_browser_mouse_schema_and_annotations() -> None:
+ server = ScraplingMCPServer()
+ async with Client(server._build_server("127.0.0.1", 8000)) as client:
+ tools = {tool.name: tool for tool in (await client.list_tools()).tools}
+ removed = await client.call_tool("browser_mouse", {"session_id": "browser", "action": "move", "x": 1, "y": 2})
+ assert "browser_mouse" not in tools and removed.is_error
+ move, click = tools["browser_mouse_move"], tools["browser_click"]
+ assert set(move.input_schema["properties"]) == {"session_id", "x", "y", "steps"}
+ assert set(move.input_schema["required"]) == {"session_id", "x", "y"}
+ assert move.input_schema["properties"]["steps"]["exclusiveMinimum"] == 0
+ assert move.input_schema["properties"]["steps"]["default"] == 1
+ properties = click.input_schema["properties"]
+ assert set(properties) == {"session_id", "selector", "ref", "x", "y", "button", "click_count", "delay", "timeout"}
+ assert click.input_schema["required"] == ["session_id"]
+ assert set(properties["button"]["enum"]) == {"left", "right", "middle"}
+ assert properties["click_count"]["exclusiveMinimum"] == 0
+ assert properties["delay"]["minimum"] == properties["timeout"]["minimum"] == 0
+ assert {name: properties[name]["default"] for name in ("button", "click_count", "delay", "timeout")} == {
+ "button": "left",
+ "click_count": 1,
+ "delay": 0,
+ "timeout": 30000,
+ }
+ for tool in (move, click):
+ assert tool.output_schema is None
+ assert tool.annotations is not None
+ assert tool.annotations.read_only_hint is False
+ assert tool.annotations.destructive_hint is True
+ assert tool.annotations.idempotent_hint is False
+ assert tool.annotations.open_world_hint is True
+
+
[email protected]("session_type", ["dynamic", "stealthy"])
+ "tool, target, options, expected",
+ [
+ ("browser_mouse_move", {"x": -12.5, "y": 23.5}, {}, {"steps": 1}),
+ ("browser_mouse_move", {"x": -12.5, "y": 23.5}, {"steps": 5}, {"steps": 5}),
+ ("browser_click", {"x": -12.5, "y": 23.5}, {}, {"button": "left", "click_count": 1, "delay": 0}),
+ (
+ "browser_click",
+ {"x": -12.5, "y": 23.5},
+ {"button": "middle", "timeout": 400},
+ {"button": "middle", "click_count": 1, "delay": 0},
+ ),
+ (
+ "browser_click",
+ {"selector": "button"},
+ {},
+ {"button": "left", "click_count": 1, "delay": 0, "timeout": 30000},
+ ),
+ (
+ "browser_click",
+ {"ref": "e4"},
+ {"button": "right", "click_count": 2, "delay": 30, "timeout": 500},
+ {"button": "right", "click_count": 2, "delay": 30, "timeout": 500},
+ ),
+ ],
+)
+async def test_browser_mouse_forwards_native_actions(
+ session_type: SessionType, tool: str, target: dict[str, Any], options: dict[str, Any], expected: dict[str, Any]
+) -> None:
+ server, session, page = _server(session_type)
+ async with Client(server._build_server("127.0.0.1", 8000)) as client:
+ result = await client.call_tool(tool, {"session_id": "browser", **target, **options})
+ assert not result.is_error
+ assert result.structured_content is None
+ assert result.content == [
+ TextContent(
+ type="text", text="Mouse moved to (-12.5, 23.5)." if tool == "browser_mouse_move" else "Click sent."
+ )
+ ]
+ if tool == "browser_mouse_move":
+ page.mouse.move.assert_awaited_once_with(-12.5, 23.5, **expected)
+ page.mouse.click.assert_not_awaited()
+ page.locator.assert_not_called()
+ elif "x" in target:
+ page.mouse.click.assert_awaited_once_with(-12.5, 23.5, **expected)
+ page.mouse.move.assert_not_awaited()
+ page.locator.assert_not_called()
+ else:
+ page.locator.assert_called_once_with(target["selector"] if "selector" in target else "aria-ref=e4")
+ page.locator.return_value.click.assert_awaited_once_with(**expected)
+ page.mouse.move.assert_not_awaited()
+ page.mouse.click.assert_not_awaited()
+ assert session.page_pool.pages_count == 1
+ assert session.page_pool.pages[0].state == "ready"
+ for name in (
+ "goto",
+ "evaluate",
+ "add_init_script",
+ "set_extra_http_headers",
+ "unroute_all",
+ "set_default_timeout",
+ "set_default_navigation_timeout",
+ "wait_for_load_state",
+ ):
+ getattr(page, name).assert_not_called()
+
+
+ "tool, values",
+ [
+ ("browser_mouse_move", {"x": "invalid"}),
+ ("browser_mouse_move", {"y": None}),
+ ("browser_mouse_move", {"steps": 1.5}),
+ ("browser_mouse_move", {"steps": 0}),
+ ("browser_mouse_move", {"steps": -1}),
+ ("browser_click", {"button": "back"}),
+ ("browser_click", {"click_count": 1.5}),
+ ("browser_click", {"click_count": 0}),
+ ("browser_click", {"click_count": -1}),
+ ("browser_click", {"delay": -1}),
+ ("browser_click", {"delay": "invalid"}),
+ ("browser_click", {"timeout": -1}),
+ ("browser_click", {"selector": ""}),
+ ("browser_click", {"ref": ""}),
+ ],
+)
+async def test_browser_mouse_invalid_input_does_not_reserve_or_use_page(tool: str, values: dict[str, Any]) -> None:
+ server, session, page = _server()
+ async with Client(server._build_server("127.0.0.1", 8000)) as client:
+ result = await client.call_tool(tool, {"session_id": "browser", "x": 10, "y": 20, **values})
+ assert result.is_error
+ assert result.content and isinstance(result.content[0], TextContent)
+ page.mouse.move.assert_not_awaited()
+ page.mouse.click.assert_not_awaited()
+ page.is_closed.assert_not_called()
+ assert session.page_pool.pages[0].state == "ready"
+
+
+ "tool, fields", [("browser_mouse_move", ("x", "y")), ("browser_click", ("x", "y", "delay", "timeout"))]
+)
[email protected]("value", [float("nan"), float("inf"), float("-inf")])
+def test_browser_mouse_schema_rejects_nonfinite_values(tool: str, fields: tuple[str, ...], value: float) -> None:
+ server, session, page = _server()
+ registered = server._build_server("127.0.0.1", 8000)._tool_manager.get_tool(tool)
+ assert registered is not None
+ for field in fields:
+ with pytest.raises(ValidationError):
+ registered.fn_metadata.arg_model.model_validate({"session_id": "browser", "x": 10, "y": 20, field: value})
+ page.mouse.move.assert_not_awaited()
+ page.mouse.click.assert_not_awaited()
+ page.is_closed.assert_not_called()
+ assert session.page_pool.pages[0].state == "ready"
+
+
+ "target",
+ [
+ {},
+ {"x": 1},
+ {"y": 2},
+ {"selector": "button", "ref": "e2"},
+ {"selector": "button", "x": 1},
+ {"ref": "e2", "y": 2},
+ {"selector": "button", "x": 1, "y": 2},
+ {"ref": "e2", "x": 1, "y": 2},
+ ],
+)
+async def test_browser_click_requires_exactly_one_complete_target(target: dict[str, Any]) -> None:
+ server, session, page = _server()
+ async with Client(server._build_server("127.0.0.1", 8000)) as client:
+ result = await client.call_tool("browser_click", {"session_id": "browser", **target})
+ assert result.is_error
+ page.is_closed.assert_not_called()
+ page.mouse.click.assert_not_awaited()
+ page.locator.assert_not_called()
+ assert session.page_pool.pages[0].state == "ready"
+
+
[email protected]("tool", ["browser_mouse_move", "browser_click"])
+ "state, message",
+ [
+ ("missing", "not found"),
+ ("dead", "no longer alive"),
+ ("static", "'static'"),
+ ("empty", "browser_fetch"),
+ ("busy", "busy"),
+ ("closed", "closed"),
+ ],
+)
+async def test_browser_mouse_session_errors_reach_mcp(tool: str, state: str, message: str) -> None:
+ server, session, page = _server("static" if state == "static" else "dynamic")
+ if state == "missing":
+ server._sessions.clear()
+ elif state == "dead":
+ session._is_alive = False
+ elif state == "empty":
+ session.page_pool.clear()
+ elif state == "busy":
+ session.page_pool.pages[0].mark_busy()
+ elif state == "closed":
+ page.is_closed.return_value = True
+ async with Client(server._build_server("127.0.0.1", 8000)) as client:
+ result = await client.call_tool(tool, {"session_id": "browser", "x": 10, "y": 20})
+ assert result.is_error
+ assert result.content and isinstance(result.content[0], TextContent)
+ assert message in result.content[0].text
+ page.mouse.move.assert_not_awaited()
+ page.mouse.click.assert_not_awaited()
+ if state == "closed":
+ assert session.page_pool.pages_count == 0
+ elif state == "busy":
+ assert session.page_pool.pages[0].state == "busy"
+
+
[email protected]("closed", [False, True])
[email protected]("target", [{"x": 10, "y": 20}, {"selector": "button"}, {"ref": "e1"}])
+async def test_browser_mouse_failure_is_not_retried_and_releases_page(closed: bool, target: dict[str, Any]) -> None:
+ server, session, page = _server()
+
+ async def fail(*args: Any, **kwargs: Any) -> None:
+ page.is_closed.return_value = closed
+ raise RuntimeError("mouse failed")
+
+ click = page.mouse.click if "x" in target else page.locator.return_value.click
+ click.side_effect = fail
+ async with Client(server._build_server("127.0.0.1", 8000)) as client:
+ result = await client.call_tool("browser_click", {"session_id": "browser", **target})
+ assert result.is_error
+ assert result.content and isinstance(result.content[0], TextContent)
+ assert "mouse failed" in result.content[0].text
+ click.assert_awaited_once()
+ if closed:
+ assert session.page_pool.pages_count == 0
+ else:
+ assert session.page_pool.pages[0].state == "ready"
+ click.side_effect = None
+ await server.browser_click("browser", **target)
+ page.close.assert_not_called()
+ page.mouse.up.assert_not_awaited()
+
+
+async def test_browser_mouse_reserves_page_until_cancelled() -> None:
+ server, session, page = _server()
+ entered = asyncio.Event()
+ release = asyncio.Event()
+
+ async def move(*args: Any, **kwargs: Any) -> None:
+ entered.set()
+ await release.wait()
+
+ page.mouse.move.side_effect = move
+ task = asyncio.create_task(server.browser_mouse_move("browser", 10, 20))
+ try:
+ await asyncio.wait_for(entered.wait(), 5)
+ assert session.page_pool.pages[0].state == "busy"
+ assert session.page_pool.get_ready_page() is None
+ with pytest.raises(RuntimeError, match="busy"):
+ await server.browser_click("browser", x=10, y=20)
+ with pytest.raises(RuntimeError, match="busy"):
+ await server.browser_snapshot("browser")
+ finally:
+ task.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await task
+ assert session.page_pool.pages[0].state == "ready"
+ page.close.assert_not_called()
+
+
+ "session_type, error_type", [("dynamic", PlaywrightTimeoutError), ("stealthy", PatchrightTimeoutError)]
+)
+async def test_browser_click_timeout_holds_page_until_button_is_released(
+ session_type: SessionType, error_type: type[Exception]
+) -> None:
+ server, session, page = _server(session_type)
+ releasing = asyncio.Event()
+ release = asyncio.Event()
+
+ async def unpress(*args: Any, **kwargs: Any) -> None:
+ releasing.set()
+ await release.wait()
+
+ page.locator.return_value.click.side_effect = error_type("click timed out")
+ page.mouse.up.side_effect = unpress
+ task = asyncio.create_task(server.browser_click("browser", selector="button", button="right"))
+ try:
+ await asyncio.wait_for(releasing.wait(), 5)
+ assert session.page_pool.pages[0].state == "busy"
+ assert session.page_pool.get_ready_page() is None
+ with pytest.raises(RuntimeError, match="busy"):
+ await server.browser_mouse_move("browser", 30, 40)
+ release.set()
+ with pytest.raises(error_type, match="click timed out"):
+ await task
+ finally:
+ release.set()
+ with suppress(error_type):
+ await task
+ page.mouse.up.assert_awaited_once_with(button="right")
+ page.locator.return_value.click.assert_awaited_once()
+ assert session.page_pool.pages[0].state == "ready"
+ page.close.assert_not_called()
+
+
[email protected]("cancel_mode", ["task", "scope"])
[email protected]("target", [{"x": 10, "y": 20}, {"selector": "button"}, {"ref": "e1"}])
+async def test_browser_mouse_cancelled_click_holds_page_until_button_is_released(
+ cancel_mode: str, target: dict[str, Any]
+) -> None:
+ server, session, page = _server()
+ entered = asyncio.Event()
+ releasing = asyncio.Event()
+ release = asyncio.Event()
+ scope = CancelScope()
+
+ async def press(*args: Any, **kwargs: Any) -> None:
+ entered.set()
+ await asyncio.Event().wait()
+
+ async def unpress(*args: Any, **kwargs: Any) -> None:
+ releasing.set()
+ await release.wait()
+
+ async def click() -> None:
+ with scope:
+ await server.browser_click("browser", **target, button="right", delay=60000)
+
+ action = page.mouse.click if "x" in target else page.locator.return_value.click
+ action.side_effect = press
+ page.mouse.up.side_effect = unpress
+ task = asyncio.create_task(click())
+ try:
+ await asyncio.wait_for(entered.wait(), 5)
+ scope.cancel() if cancel_mode == "scope" else task.cancel()
+ await asyncio.wait_for(releasing.wait(), 5)
+ assert session.page_pool.pages[0].state == "busy"
+ assert session.page_pool.get_ready_page() is None
+ with pytest.raises(RuntimeError, match="busy"):
+ await server.browser_mouse_move("browser", 30, 40)
+ release.set()
+ if cancel_mode == "scope":
+ await asyncio.wait_for(task, 5)
+ assert scope.cancelled_caught
+ else:
+ with pytest.raises(asyncio.CancelledError):
+ await task
+ finally:
+ release.set()
+ if not task.done():
+ task.cancel()
+ with suppress(asyncio.CancelledError):
+ await task
+ page.mouse.up.assert_awaited_once_with(button="right")
+ action.assert_awaited_once()
+ assert session.page_pool.pages[0].state == "ready"
+ page.close.assert_not_called()
+
+
[email protected]("session_type", ["dynamic", "stealthy"])
+async def test_browser_mouse_live_native_events_and_navigation(session_type: SessionType) -> None:
+ server = ScraplingMCPServer(executable_path=getenv("SCRAPLING_EXECUTABLE_PATH"))
+ requests: list[str] = []
+
+ async def serve(route: Any) -> None:
+ if route.request.is_navigation_request():
+ requests.append(route.request.url)
+ await route.fulfill(
+ status=200,
+ content_type="text/html",
+ body="<html><body><h1>Arrived</h1></body></html>" if route.request.url.endswith("/next") else HTML,
+ )
+
+ async with Client(server._build_server("127.0.0.1", 8000)) as client:
+ opened = await client.call_tool("browser_open", {"session_type": session_type, "session_id": "browser"})
+ assert not opened.is_error
+ try:
+ session = server._sessions["browser"].session
+ await session.context.route("**/*", serve)
+ fetched = await client.call_tool(
+ "browser_fetch", {"session_id": "browser", "url": "https://mouse.test/", "google_search": False}
+ )
+ assert not fetched.is_error
+ page_info = session.page_pool.pages[0]
+ page = page_info.page
+ initial_dom = await page.content()
+ moved = await client.call_tool(
+ "browser_mouse_move", {"session_id": "browser", "x": 140, "y": 110, "steps": 4}
+ )
+ assert not moved.is_error
+ assert (
+ await page.locator("#target").evaluate("element => getComputedStyle(element).backgroundColor")
+ == "rgb(0, 128, 0)"
+ )
+ events = loads(await page.locator("#events").input_value())
+ assert events and events[-1]["type"] == "mousemove"
+ assert (events[-1]["x"], events[-1]["y"]) == (140, 110)
+ for options, event_type, button, detail in (
+ ({}, "click", 0, 1),
+ ({"click_count": 2, "delay": 10}, "dblclick", 0, 2),
+ ({"button": "right"}, "contextmenu", 2, 0),
+ ({"button": "middle"}, "auxclick", 1, 1),
+ ):
+ await page.locator("#events").evaluate("element => element.value = '[]'")
+ result = await client.call_tool(
+ "browser_click", {"session_id": "browser", "x": 140, "y": 110, **options}
+ )
+ assert not result.is_error
+ events = loads(await page.locator("#events").input_value())
+ assert all(event["trusted"] for event in events)
+ assert {"mousedown", "mouseup", event_type} <= {event["type"] for event in events}
+ match = next(event for event in events if event["type"] == event_type)
+ assert (match["button"], match["detail"]) == (button, detail)
+ assert session.page_pool.pages == [page_info] and page_info.state == "ready"
+ assert await page.content() == initial_dom
+ assert requests == ["https://mouse.test/"]
+ clicked = await client.call_tool("browser_click", {"session_id": "browser", "x": 140, "y": 200})
+ assert not clicked.is_error
+ await page.wait_for_url("https://mouse.test/next", timeout=5000)
+ snapshot = await client.call_tool("browser_snapshot", {"session_id": "browser"})
+ assert not snapshot.is_error
+ assert snapshot.content and isinstance(snapshot.content[0], TextContent)
+ assert "Arrived" in snapshot.content[0].text
+ assert requests == ["https://mouse.test/", "https://mouse.test/next"]
+ assert session.page_pool.pages == [page_info] and page_info.state == "ready"
+ finally:
+ closed = await client.call_tool("close_session", {"session_id": "browser"})
+ assert not closed.is_error
+
+
[email protected]("session_type", ["dynamic", "stealthy"])
[email protected]("cancel_mode", ["task", "scope"])
[email protected]("target_type", ["xy", "ref"])
+async def test_browser_mouse_live_cancelled_click_releases_native_button(
+ session_type: SessionType, cancel_mode: str, target_type: str
+) -> None:
+ server = ScraplingMCPServer(executable_path=getenv("SCRAPLING_EXECUTABLE_PATH"))
+ scope = CancelScope()
+ target: dict[str, Any] = {"x": 140, "y": 110}
+
+ async def serve(route: Any) -> None:
+ await route.fulfill(status=200, content_type="text/html", body=HTML)
+
+ async def click() -> None:
+ with scope:
+ await server.browser_click("browser", **target, delay=60000, timeout=0)
+
+ async with Client(server._build_server("127.0.0.1", 8000)) as client:
+ opened = await client.call_tool("browser_open", {"session_type": session_type, "session_id": "browser"})
+ assert not opened.is_error
+ task = None
+ try:
+ session = server._sessions["browser"].session
+ await session.context.route("**/*", serve)
+ fetched = await client.call_tool(
+ "browser_fetch", {"session_id": "browser", "url": "https://mouse.test/", "google_search": False}
+ )
+ assert not fetched.is_error
+ page_info = session.page_pool.pages[0]
+ page = page_info.page
+ if target_type == "ref":
+ match = search(r'button "Target".*?\[ref=([^\]]+)\]', await server.browser_snapshot("browser"))
+ assert match is not None
+ target = {"ref": match[1]}
+ task = asyncio.create_task(click())
+ await page.wait_for_function(
+ "JSON.parse(document.querySelector('#events').value).some(event => event.type === 'mousedown')",
+ timeout=5000,
+ )
+ assert page_info.state == "busy"
+ scope.cancel() if cancel_mode == "scope" else task.cancel()
+ if cancel_mode == "scope":
+ await asyncio.wait_for(task, 5)
+ assert scope.cancelled_caught
+ else:
+ with pytest.raises(asyncio.CancelledError):
+ await asyncio.wait_for(task, 5)
+ assert page_info.state == "ready"
+ moved = await client.call_tool("browser_mouse_move", {"session_id": "browser", "x": 150, "y": 120})
+ assert not moved.is_error
+ events = loads(await page.locator("#events").input_value())
+ assert all(event["trusted"] for event in events)
+ assert len([event for event in events if event["type"] == "mousedown"]) == 1
+ assert next(event for event in events if event["type"] == "mousedown")["buttons"] == 1
+ assert next(event for event in events if event["type"] == "mouseup")["buttons"] == 0
+ assert (events[-1]["type"], events[-1]["buttons"]) == ("mousemove", 0)
+ assert session.page_pool.pages == [page_info] and page_info.state == "ready"
+ finally:
+ if task is not None and not task.done():
+ task.cancel()
+ with suppress(asyncio.CancelledError):
+ await task
+ closed = await client.call_tool("close_session", {"session_id": "browser"})
+ assert not closed.is_error
+
+
[email protected]("session_type", ["dynamic", "stealthy"])
+async def test_browser_click_live_selectors_refs_and_scroll(session_type: SessionType) -> None:
+ server = ScraplingMCPServer(executable_path=getenv("SCRAPLING_EXECUTABLE_PATH"))
+
+ async def serve(route: Any) -> None:
+ await route.fulfill(status=200, content_type="text/html", body=HTML)
+
+ async with Client(server._build_server("127.0.0.1", 8000)) as client:
+ opened = await client.call_tool("browser_open", {"session_type": session_type, "session_id": "browser"})
+ assert not opened.is_error
+ try:
+ session = server._sessions["browser"].session
+ await session.context.route("**/*", serve)
+ fetched = await client.call_tool(
+ "browser_fetch", {"session_id": "browser", "url": "https://mouse.test/", "google_search": False}
+ )
+ assert not fetched.is_error
+ page_info = session.page_pool.pages[0]
+ page = page_info.page
+ match = search(r'button "Target".*?\[ref=([^\]]+)\]', await server.browser_snapshot("browser"))
+ assert match is not None
+ ref = match[1]
+ initial_dom = await page.content()
+ for target in (
+ {"selector": "#target"},
+ {"selector": 'xpath=//button[@id="target"]'},
+ {"ref": ref},
+ {"selector": "#below"},
+ ):
+ await page.locator("#events").evaluate("element => element.value = '[]'")
+ result = await client.call_tool("browser_click", {"session_id": "browser", **target})
+ assert not result.is_error
+ events = loads(await page.locator("#events").input_value())
+ clicks = [event for event in events if event["type"] == "click"]
+ assert len(clicks) == 1 and clicks[0]["trusted"]
+ assert clicks[0]["target"] == ("below" if target.get("selector") == "#below" else "target")
+ assert await page.evaluate("window.scrollY") > 0
+ assert await page.content() == initial_dom
+ await page.locator("#target").evaluate("element => element.remove()")
+ for target in ({"selector": "button, a"}, {"selector": "#missing"}, {"ref": ref}):
+ await page.locator("#events").evaluate("element => element.value = '[]'")
+ with pytest.raises((PlaywrightError, PatchrightError)) as error:
+ await server.browser_click(
+ "browser", selector=target.get("selector"), ref=target.get("ref"), timeout=100
+ )
+ events = loads(await page.locator("#events").input_value())
+ if target.get("selector") != "button, a":
+ assert isinstance(
+ error.value, PlaywrightTimeoutError if session_type == "dynamic" else PatchrightTimeoutError
+ )
+ assert not any(event["type"] in ("mousedown", "click") for event in events)
+ released = [event for event in events if event["type"] == "mouseup"]
+ assert len(released) == 1 and released[0]["buttons"] == 0
+ else:
+ assert "strict mode violation" in str(error.value)
+ assert not any(event["type"] in ("mousedown", "mouseup", "click") for event in events)
+ assert session.page_pool.pages == [page_info] and page_info.state == "ready"
+ finally:
+ closed = await client.call_tool("close_session", {"session_id": "browser"})
+ assert not closed.is_error
+
+
[email protected]("session_type", ["dynamic", "stealthy"])
+async def test_browser_click_live_timeout_releases_native_button(session_type: SessionType) -> None:
+ server = ScraplingMCPServer(executable_path=getenv("SCRAPLING_EXECUTABLE_PATH"))
+
+ async def serve(route: Any) -> None:
+ await route.fulfill(status=200, content_type="text/html", body=HTML)
+
+ async with Client(server._build_server("127.0.0.1", 8000)) as client:
+ opened = await client.call_tool("browser_open", {"session_type": session_type, "session_id": "browser"})
+ assert not opened.is_error
+ try:
+ session = server._sessions["browser"].session
+ await session.context.route("**/*", serve)
+ fetched = await client.call_tool(
+ "browser_fetch", {"session_id": "browser", "url": "https://mouse.test/", "google_search": False}
+ )
+ assert not fetched.is_error
+ page_info = session.page_pool.pages[0]
+ page = page_info.page
+ failed = await client.call_tool(
+ "browser_click", {"session_id": "browser", "selector": "#target", "delay": 1000, "timeout": 300}
+ )
+ assert failed.is_error
+ events = loads(await page.locator("#events").input_value())
+ assert next(event for event in events if event["type"] == "mousedown")["buttons"] == 1
+ moved = await client.call_tool("browser_mouse_move", {"session_id": "browser", "x": 150, "y": 120})
+ assert not moved.is_error
+ events = loads(await page.locator("#events").input_value())
+ assert (events[-1]["type"], events[-1]["buttons"]) == ("mousemove", 0)
+ assert next(event for event in events if event["type"] == "mouseup")["buttons"] == 0
+ assert all(event["trusted"] for event in events)
+ assert session.page_pool.pages == [page_info] and page_info.state == "ready"
+ finally:
+ closed = await client.call_tool("close_session", {"session_id": "browser"})
+ assert not closed.is_error
diff --git a/tests/ai/test_browser_snapshot.py b/tests/ai/test_browser_snapshot.py
index f162940..9ae306d 100644
--- a/tests/ai/test_browser_snapshot.py
+++ b/tests/ai/test_browser_snapshot.py
@@ -36,7 +36,8 @@ def _server(session_type: SessionType = "dynamic") -> tuple[ScraplingMCPServer,
@pytest.mark.asyncio
@pytest.mark.parametrize("session_type", ["dynamic", "stealthy"])
-async def test_browser_snapshot_returns_plain_mcp_text(session_type: SessionType) -> None:
[email protected]("boxes", [None, False, True])
+async def test_browser_snapshot_returns_plain_mcp_text(session_type: SessionType, boxes: bool | None) -> None:
server, session, page = _server(session_type)
async with Client(server._build_server("127.0.0.1", 8000)) as client:
tools = {tool.name: tool for tool in (await client.list_tools()).tools}
@@ -44,16 +45,20 @@ async def test_browser_snapshot_returns_plain_mcp_text(session_type: SessionType
tool = tools["browser_snapshot"]
assert tool.output_schema is None
assert set(tool.input_schema["properties"]) == {"session_id", "depth", "boxes"}
+ assert tool.input_schema["properties"]["boxes"]["default"] is True
assert tool.input_schema["required"] == ["session_id"]
fetch_props = tools["browser_fetch"].input_schema["properties"]
assert set(fetch_props["extraction_type"]["enum"]) == {"markdown", "html", "text", "snapshot"}
assert "depth" not in fetch_props and "boxes" not in fetch_props
assert "snapshot" not in tools["browser_fetch_once"].input_schema["properties"]["extraction_type"]["enum"]
- result = await client.call_tool("browser_snapshot", {"session_id": "browser", "depth": 3, "boxes": True})
+ args: dict[str, Any] = {"session_id": "browser", "depth": 3}
+ if boxes is not None:
+ args["boxes"] = boxes
+ result = await client.call_tool("browser_snapshot", args)
assert not result.is_error
assert result.structured_content is None
assert result.content == [TextContent(type="text", text=SNAPSHOT)]
- page.aria_snapshot.assert_awaited_once_with(mode="ai", depth=3, boxes=True)
+ page.aria_snapshot.assert_awaited_once_with(mode="ai", depth=3, boxes=True if boxes is None else boxes)
assert session.page_pool.pages_count == 1
assert session.page_pool.pages[0].state == "ready"
page.goto.assert_not_called()
@@ -151,6 +156,7 @@ async def test_browser_snapshot_live_mcp_round_trip(session_type: SessionType, c
assert fetched.structured_content["url"] == "https://snapshot.test/"
fetched_snapshot = fetched.structured_content["content"][0]
assert "after wait" in fetched_snapshot and "initial" not in fetched_snapshot
+ assert "[box=" in fetched_snapshot
assert ("Outside scope" in fetched_snapshot) is (css_selector is None)
assert requests == ["https://snapshot.test/"]
page_info = session.page_pool.pages[0]
@@ -163,7 +169,7 @@ async def test_browser_snapshot_live_mcp_round_trip(session_type: SessionType, c
navigations: list[str] = []
page.on("framenavigated", lambda frame: navigations.append(frame.url))
previous_requests = requests.copy()
- result = await client.call_tool("browser_snapshot", {"session_id": "browser", "boxes": True})
+ result = await client.call_tool("browser_snapshot", {"session_id": "browser"})
assert not result.is_error
assert result.structured_content is None
assert len(result.content) == 1 and isinstance(result.content[0], TextContent)
@@ -177,6 +183,11 @@ async def test_browser_snapshot_live_mcp_round_trip(session_type: SessionType, c
assert match is not None
await page.locator(f"aria-ref={match[1]}").click()
assert await page.locator("output").inner_text() == "live value"
+ without_boxes = await client.call_tool("browser_snapshot", {"session_id": "browser", "boxes": False})
+ assert not without_boxes.is_error
+ assert len(without_boxes.content) == 1 and isinstance(without_boxes.content[0], TextContent)
+ assert "[box=" not in without_boxes.content[0].text
+ assert "[ref=" in without_boxes.content[0].text
for selector in ("#missing", "button", "["):
previous_count = len(requests)
failed = await client.call_tool(
@@ -269,6 +280,7 @@ async def test_browser_fetch_snapshot_keeps_raw_content_and_response_metadata()
assert result.content == [snapshot]
assert result.status == 202
assert result.url == "https://snapshot.test/redirected"
+ page.aria_snapshot.assert_awaited_once_with(mode="ai", depth=None, boxes=True)
fetch.assert_awaited_once()
assert {"extraction_type", "css_selector", "depth", "boxes"}.isdisjoint(fetch.call_args.kwargs)