🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
style(mcp): move browser actions to a separate file
Karim shoair8366d0722dc0
diff --git a/scrapling/core/_browser_actions.py b/scrapling/core/_browser_actions.pynew file mode 100644index 0000000..8bdb54a--- /dev/null+++ b/scrapling/core/_browser_actions.py@@ -0,0 +1,218 @@+from random import uniform+from asyncio import CancelledError, sleep++from anyio import CancelScope+from pydantic import Field, FiniteFloat, PositiveInt+from patchright.async_api import TimeoutError as PatchrightTimeoutError++from scrapling.core._types import (+ Optional,+ Literal,+ Union,+ TypedDict,+ NotRequired,+ List,+ Any,+ Annotated,+ SelectorWaitStates,+)++MouseButton = Literal["left", "right", "middle"]+NonNegativeFiniteFloat = Annotated[float, Field(ge=0, allow_inf_nan=False)]+NonEmptyString = Annotated[str, Field(min_length=1)]+++class _MouseTarget(TypedDict, total=False):+ selector: Optional[NonEmptyString]+ ref: Optional[NonEmptyString]+ x: Optional[FiniteFloat]+ y: Optional[FiniteFloat]+ timeout: Annotated[+ NonNegativeFiniteFloat, Field(default=30000, description="Selector/ref timeout in ms; 0 disables it.")+ ]+++class _MouseMove(_MouseTarget):+ type: Literal["move"]+ steps: NotRequired[Annotated[PositiveInt, Field(default=1, description="Mousemove events for coordinates only.")]]+++class _MouseClick(_MouseTarget):+ type: Literal["click"]+ button: NotRequired[Annotated[MouseButton, Field(default="left")]]+ click_count: NotRequired[Annotated[PositiveInt, Field(default=1, description="2 for a double-click.")]]+ delay: NotRequired[+ Annotated[NonNegativeFiniteFloat, Field(default=0, description="Milliseconds between press and release.")]+ ]+++class _MouseWheel(TypedDict):+ type: Literal["wheel"]+ delta_x: NotRequired[+ Annotated[FiniteFloat, Field(default=0, description="Horizontal CSS pixels; positive scrolls right.")]+ ]+ delta_y: NotRequired[+ Annotated[FiniteFloat, Field(default=0, description="Vertical CSS pixels; positive scrolls down.")]+ ]+++class _TimeWait(TypedDict):+ type: Literal["wait_time"]+ milliseconds: NonNegativeFiniteFloat+++class _ConditionWait(TypedDict, total=False):+ timeout: Annotated[NonNegativeFiniteFloat, Field(default=30000, description="Wait limit in ms; 0 disables it.")]+++class _ElementWait(_ConditionWait):+ type: Literal["wait_element"]+ selector: Annotated[NonEmptyString, Field(description="Playwright selector for one element.")]+ state: NotRequired[Annotated[SelectorWaitStates, Field(default="visible", description="Hidden includes removal.")]]+++class _LoadWait(_ConditionWait):+ type: Literal["wait_load"]+ state: Annotated[+ Literal["domcontentloaded", "load", "networkidle"],+ Field(description="Current document readiness; networkidle waits for 500 ms without active connections."),+ ]+++class _KeyPress(TypedDict):+ type: Literal["press_key"]+ key: Annotated[+ NonEmptyString, Field(description="Key or shortcut at current focus, e.g. Enter or ControlOrMeta+A.")+ ]+++class _FormTarget(TypedDict, total=False):+ selector: Optional[NonEmptyString]+ ref: Optional[NonEmptyString]+ timeout: Annotated[+ NonNegativeFiniteFloat, Field(default=30000, description="Limit per native operation in ms; 0 disables it.")+ ]+++class _TextFormField(_FormTarget):+ type: Literal["textbox"]+ value: str+ clear: NotRequired[+ Annotated[bool, Field(default=True, description="False types at the current caret or selection.")]+ ]+++class _CheckboxFormField(_FormTarget):+ type: Literal["checkbox"]+ value: bool+++class _RadioFormField(_FormTarget):+ type: Literal["radio"]+ value: Literal[True]+++class _SelectFormField(_FormTarget):+ type: Literal["combobox"]+ value: Annotated[Union[str, List[str]], Field(description="Option label(s); [] clears selection.")]+++BrowserAction = Annotated[+ Union[+ _MouseMove,+ _MouseClick,+ _MouseWheel,+ _TextFormField,+ _CheckboxFormField,+ _RadioFormField,+ _SelectFormField,+ _KeyPress,+ _TimeWait,+ _ElementWait,+ _LoadWait,+ ],+ Field(discriminator="type"),+]+++def _validate_actions(actions: List[BrowserAction]) -> None:+ """Validate every action target before reserving the page."""+ for index, action in enumerate(actions, 1):+ if action["type"] in ("move", "click") and (+ sum(action.get(key) is not None for key in ("selector", "ref", "x")) != 1+ or (action.get("x") is None) != (action.get("y") is None)+ ):+ raise ValueError(+ f"Action {index} ({action['type']}) needs exactly one target: 'selector', 'ref', or both 'x' and 'y'."+ )+ if action["type"] in ("textbox", "checkbox", "radio", "combobox") and (+ (action.get("selector") is None) == (action.get("ref") is None)+ ):+ raise ValueError(f"Action {index} ({action['type']}) needs exactly one target: 'selector' or 'ref'.")+++async def _run_actions(page: Any, actions: List[BrowserAction], slowly: bool = False) -> None:+ """Run an ordered action chain on a reserved browser page."""+ for index, action in enumerate(actions, 1):+ try:+ if slowly and index > 1:+ await sleep(uniform(0.1, 0.3))+ if action["type"] == "wait_time":+ await page.wait_for_timeout(action["milliseconds"])+ elif action["type"] == "wait_element":+ await page.locator(action["selector"]).wait_for(+ state=action.get("state", "visible"), timeout=action.get("timeout", 30000)+ )+ elif action["type"] == "wait_load":+ await page.wait_for_load_state(action["state"], timeout=action.get("timeout", 30000))+ elif action["type"] == "press_key":+ await page.keyboard.press(action["key"])+ elif action["type"] == "wheel":+ await page.mouse.wheel(action.get("delta_x", 0), action.get("delta_y", 0))+ else:+ selector, ref = action.get("selector"), action.get("ref")+ timeout = action.get("timeout", 30000)+ if action["type"] == "move" or action["type"] == "click":+ locator = (+ page.locator(selector or f"aria-ref={ref}") if selector is not None or ref is not None else None+ )+ if action["type"] == "move":+ if locator is not None:+ await locator.hover(timeout=timeout)+ else:+ await page.mouse.move(action.get("x"), action.get("y"), steps=action.get("steps", 1))+ else:+ options = {+ "button": action.get("button", "left"),+ "click_count": action.get("click_count", 1),+ "delay": action.get("delay", 0),+ }+ try:+ if locator is not None:+ await locator.click(**options, timeout=timeout)+ else:+ await page.mouse.click(action.get("x"), action.get("y"), **options)+ except (CancelledError, PatchrightTimeoutError) as exc:+ try:+ with CancelScope(shield=True):+ if not page.is_closed():+ await page.mouse.up(button=options["button"])+ except Exception as cleanup_error:+ raise exc from cleanup_error+ raise+ else:+ locator = page.locator(selector or f"aria-ref={ref}")+ if action["type"] == "textbox":+ if action.get("clear", True):+ await locator.fill("" if slowly else action["value"], timeout=timeout)+ if slowly and action["value"]:+ for character in action["value"]:+ await locator.press_sequentially(character, delay=uniform(50, 150), timeout=timeout)+ elif not action.get("clear", True):+ await locator.press_sequentially(action["value"], timeout=timeout)+ elif action["type"] == "combobox":+ await locator.select_option(label=action["value"], timeout=timeout)+ else:+ await locator.set_checked(action["value"], timeout=timeout)+ except Exception as exc:+ raise RuntimeError(f"Action {index} ({action['type']}) failed: {exc}") from excdiff --git a/scrapling/core/ai.py b/scrapling/core/ai.pyindex f8cbf31..e05c1c3 100644--- a/scrapling/core/ai.py+++ b/scrapling/core/ai.py@@ -2,9 +2,8 @@ from uuid import uuid4from os import environfrom json import dumpsfrom time import monotonic-from random import uniformfrom hmac import compare_digest-from asyncio import CancelledError, gather, sleep+from asyncio import gatherfrom contextlib import contextmanagerfrom datetime import datetime, timezonefrom dataclasses import dataclass, field@@ -17,11 +16,17 @@ from mcp.server.auth.settings import AuthSettingsfrom mcp.server.caching import CacheHintfrom mcp.server.transport_security import TransportSecuritySettingsfrom mcp.types import Icon, ImageContent, TextContent, ToolAnnotations-from pydantic import AnyHttpUrl, BaseModel, Field, FiniteFloat, PositiveInt-from patchright.async_api import TimeoutError as PatchrightTimeoutError+from pydantic import AnyHttpUrl, BaseModel, Fieldfrom scrapling import __version__from scrapling.core.utils import log+from scrapling.core._browser_actions import (+ BrowserAction,+ NonEmptyString,+ NonNegativeFiniteFloat,+ _run_actions,+ _validate_actions,+)from scrapling.core.shell import Convertor, _CONTROL_CHARS_PATTERNfrom scrapling.engines.toolbelt.custom import Response as _ScraplingResponsefrom scrapling.engines.static import ImpersonateType@@ -30,12 +35,9 @@ from scrapling.engines._browsers._types import StealthFetchParamsfrom scrapling.core._types import (Optional,Literal,- Union,Tuple,Mapping,Dict,- TypedDict,- NotRequired,List,Any,Annotated,@@ -52,9 +54,6 @@ from scrapling.core._types import (SessionType = Literal["stealthy", "static"]SessionExtractionType = Literal[extraction_types, "snapshot"]ScreenshotType = Literal["png", "jpeg"]-MouseButton = Literal["left", "right", "middle"]-NonNegativeFiniteFloat = Annotated[float, Field(ge=0, allow_inf_nan=False)]-NonEmptyString = Annotated[str, Field(min_length=1)]MCP_EXECUTABLE_PATH_ENV = "SCRAPLING_EXECUTABLE_PATH"MCP_AUTH_TOKEN_ENV = "SCRAPLING_MCP_AUTH_TOKEN" # nosec B105 - the name of the variable, not a token@@ -104,119 +103,6 @@ _INPUT_TOOL_ANNOTATIONS = ToolAnnotations()-class _MouseTarget(TypedDict, total=False):- selector: Optional[NonEmptyString]- ref: Optional[NonEmptyString]- x: Optional[FiniteFloat]- y: Optional[FiniteFloat]- timeout: Annotated[- NonNegativeFiniteFloat, Field(default=30000, description="Selector/ref timeout in ms; 0 disables it.")- ]---class _MouseMove(_MouseTarget):- type: Literal["move"]- steps: NotRequired[Annotated[PositiveInt, Field(default=1, description="Mousemove events for coordinates only.")]]---class _MouseClick(_MouseTarget):- type: Literal["click"]- button: NotRequired[Annotated[MouseButton, Field(default="left")]]- click_count: NotRequired[Annotated[PositiveInt, Field(default=1, description="2 for a double-click.")]]- delay: NotRequired[- Annotated[NonNegativeFiniteFloat, Field(default=0, description="Milliseconds between press and release.")]- ]---class _MouseWheel(TypedDict):- type: Literal["wheel"]- delta_x: NotRequired[- Annotated[FiniteFloat, Field(default=0, description="Horizontal CSS pixels; positive scrolls right.")]- ]- delta_y: NotRequired[- Annotated[FiniteFloat, Field(default=0, description="Vertical CSS pixels; positive scrolls down.")]- ]---class _TimeWait(TypedDict):- type: Literal["wait_time"]- milliseconds: NonNegativeFiniteFloat---class _ConditionWait(TypedDict, total=False):- timeout: Annotated[NonNegativeFiniteFloat, Field(default=30000, description="Wait limit in ms; 0 disables it.")]---class _ElementWait(_ConditionWait):- type: Literal["wait_element"]- selector: Annotated[NonEmptyString, Field(description="Playwright selector for one element.")]- state: NotRequired[Annotated[SelectorWaitStates, Field(default="visible", description="Hidden includes removal.")]]---class _LoadWait(_ConditionWait):- type: Literal["wait_load"]- state: Annotated[- Literal["domcontentloaded", "load", "networkidle"],- Field(description="Current document readiness; networkidle waits for 500 ms without active connections."),- ]---class _KeyPress(TypedDict):- type: Literal["press_key"]- key: Annotated[- NonEmptyString, Field(description="Key or shortcut at current focus, e.g. Enter or ControlOrMeta+A.")- ]---class _FormTarget(TypedDict, total=False):- selector: Optional[NonEmptyString]- ref: Optional[NonEmptyString]- timeout: Annotated[- NonNegativeFiniteFloat, Field(default=30000, description="Limit per native operation in ms; 0 disables it.")- ]---class _TextFormField(_FormTarget):- type: Literal["textbox"]- value: str- clear: NotRequired[- Annotated[bool, Field(default=True, description="False types at the current caret or selection.")]- ]---class _CheckboxFormField(_FormTarget):- type: Literal["checkbox"]- value: bool---class _RadioFormField(_FormTarget):- type: Literal["radio"]- value: Literal[True]---class _SelectFormField(_FormTarget):- type: Literal["combobox"]- value: Annotated[Union[str, List[str]], Field(description="Option label(s); [] clears selection.")]---BrowserAction = Annotated[- Union[- _MouseMove,- _MouseClick,- _MouseWheel,- _TextFormField,- _CheckboxFormField,- _RadioFormField,- _SelectFormField,- _KeyPress,- _TimeWait,- _ElementWait,- _LoadWait,- ],- Field(discriminator="type"),-]--class ResponseModel(BaseModel):"""Request's response information structure."""@@ -523,88 +409,9 @@ class ScraplingMCPServer::param actions: Ordered actions; fields need one selector/ref, mouse targets one selector/ref or viewport (x, y).:param slowly: Fresh random delays: 50-150 ms per character, 100-300 ms between all actions, including waits."""- for index, action in enumerate(actions, 1):- if action["type"] in ("move", "click") and (- sum(action.get(key) is not None for key in ("selector", "ref", "x")) != 1- or (action.get("x") is None) != (action.get("y") is None)- ):- raise ValueError(- f"Action {index} ({action['type']}) needs exactly one target: 'selector', 'ref', or both 'x' and 'y'."- )- if action["type"] in ("textbox", "checkbox", "radio", "combobox") and (- (action.get("selector") is None) == (action.get("ref") is None)- ):- raise ValueError(f"Action {index} ({action['type']}) needs exactly one target: 'selector' or 'ref'.")+ _validate_actions(actions)with self._browser_page(session_id) as (_, page):- for index, action in enumerate(actions, 1):- try:- if slowly and index > 1:- await sleep(uniform(0.1, 0.3))- if action["type"] == "wait_time":- await page.wait_for_timeout(action["milliseconds"])- elif action["type"] == "wait_element":- await page.locator(action["selector"]).wait_for(- state=action.get("state", "visible"), timeout=action.get("timeout", 30000)- )- elif action["type"] == "wait_load":- await page.wait_for_load_state(action["state"], timeout=action.get("timeout", 30000))- elif action["type"] == "press_key":- await page.keyboard.press(action["key"])- elif action["type"] == "wheel":- await page.mouse.wheel(action.get("delta_x", 0), action.get("delta_y", 0))- else:- selector, ref = action.get("selector"), action.get("ref")- timeout = action.get("timeout", 30000)- if action["type"] == "move" or action["type"] == "click":- locator = (- page.locator(selector or f"aria-ref={ref}")- if selector is not None or ref is not None- else None- )- if action["type"] == "move":- if locator is not None:- await locator.hover(timeout=timeout)- else:- await page.mouse.move(- action.get("x"), action.get("y"), steps=action.get("steps", 1)- )- else:- options = {- "button": action.get("button", "left"),- "click_count": action.get("click_count", 1),- "delay": action.get("delay", 0),- }- try:- if locator is not None:- await locator.click(**options, timeout=timeout)- else:- await page.mouse.click(action.get("x"), action.get("y"), **options)- except (CancelledError, PatchrightTimeoutError) as exc:- try:- with CancelScope(shield=True):- if not page.is_closed():- await page.mouse.up(button=options["button"])- except Exception as cleanup_error:- raise exc from cleanup_error- raise- else:- locator = page.locator(selector or f"aria-ref={ref}")- if action["type"] == "textbox":- if action.get("clear", True):- await locator.fill("" if slowly else action["value"], timeout=timeout)- if slowly and action["value"]:- for character in action["value"]:- await locator.press_sequentially(- character, delay=uniform(50, 150), timeout=timeout- )- elif not action.get("clear", True):- await locator.press_sequentially(action["value"], timeout=timeout)- elif action["type"] == "combobox":- await locator.select_option(label=action["value"], timeout=timeout)- else:- await locator.set_checked(action["value"], timeout=timeout)- except Exception as exc:- raise RuntimeError(f"Action {index} ({action['type']}) failed: {exc}") from exc+ await _run_actions(page, actions, slowly)return "Actions completed."async def browser_evaluate(diff --git a/tests/ai/test_browser_actions.py b/tests/ai/test_browser_actions.pyindex 338152f..f9c7703 100644--- a/tests/ai/test_browser_actions.py+++ b/tests/ai/test_browser_actions.py@@ -36,8 +36,8 @@ async def test_mixed_actions_keep_order_and_sample_each_delay(slowly: bool, monkevents = Mock()pause = AsyncMock()random = Mock(side_effect=[55, 145, 0.11, 0.12, 0.13, 0.14, 0.15])- monkeypatch.setattr("scrapling.core.ai.uniform", random)- monkeypatch.setattr("scrapling.core.ai.sleep", pause)+ monkeypatch.setattr("scrapling.core._browser_actions.uniform", random)+ monkeypatch.setattr("scrapling.core._browser_actions.sleep", pause)async def reserved(*args: Any, **kwargs: Any) -> None:assert session.page_pool.pages[0].state == "busy"diff --git a/tests/ai/test_browser_fill_fields.py b/tests/ai/test_browser_fill_fields.pyindex 69644c0..b833b17 100644--- a/tests/ai/test_browser_fill_fields.py+++ b/tests/ai/test_browser_fill_fields.py@@ -110,7 +110,7 @@ async def test_browser_actions_fields_forwards_native_actions_in_order() -> None:server, session, page = _server()pause = AsyncMock()- monkeypatch.setattr("scrapling.core.ai.sleep", pause)+ monkeypatch.setattr("scrapling.core._browser_actions.sleep", pause)actions = [{**TEXT_FIELD, "ref": None},{"type": "textbox", "selector": None, "ref": "e2", "value": "", "clear": True},@@ -164,7 +164,7 @@ async def test_browser_actions_fields_types_without_clearing(slowly: bool, value: str, monkeypatch: pytest.MonkeyPatch) -> None:server, session, page = _server()- monkeypatch.setattr("scrapling.core.ai.uniform", Mock(side_effect=[60, 140]))+ monkeypatch.setattr("scrapling.core._browser_actions.uniform", Mock(side_effect=[60, 140]))async with Client(server._build_server("127.0.0.1", 8000)) as client:result = await client.call_tool("browser_actions",@@ -187,9 +187,9 @@ async def test_browser_actions_fields_slow_pacing_and_empty_text(monkeypatch: pycalls = Mock()calls.attach_mock(page.locator, "locator")calls.pause = AsyncMock()- monkeypatch.setattr("scrapling.core.ai.sleep", calls.pause)+ monkeypatch.setattr("scrapling.core._browser_actions.sleep", calls.pause)intervals = Mock(side_effect=[60, 140, 0.12, 0.27, 0.18, 0.22])- monkeypatch.setattr("scrapling.core.ai.uniform", intervals)+ monkeypatch.setattr("scrapling.core._browser_actions.uniform", intervals)actions = [{**TEXT_FIELD, "value": "ab"},{"type": "textbox", "ref": "e2", "value": ""},@@ -236,7 +236,7 @@ async def test_browser_actions_fields_slow_failure_stops_before_next_pause([None, RuntimeError("text failed")] if method == "press_sequentially" else RuntimeError("text failed"))pause = AsyncMock()- monkeypatch.setattr("scrapling.core.ai.sleep", pause)+ monkeypatch.setattr("scrapling.core._browser_actions.sleep", pause)async with Client(server._build_server("127.0.0.1", 8000)) as client:result = await client.call_tool("browser_actions", {"session_id": "browser", "actions": [TEXT_FIELD, TEXT_FIELD], "slowly": True}@@ -453,7 +453,7 @@ async def test_browser_actions_fields_slow_cancellation_releases_page(if stage == "typing":page.locator.return_value.press_sequentially.side_effect = pendingelse:- monkeypatch.setattr("scrapling.core.ai.sleep", pending)+ monkeypatch.setattr("scrapling.core._browser_actions.sleep", pending)task = asyncio.create_task(fill())try:await asyncio.wait_for(entered.wait(), 5)