Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
SKY-15254 Let Copilot work inside embedded pages (#8341)
Andrew Neilson2c1a97ac8a8e
diff --git a/skyvern-frontend/src/routes/workflows/copilot/copilotPhases.test.ts b/skyvern-frontend/src/routes/workflows/copilot/copilotPhases.test.tsindex 3508cee6b..86e926e09 100644--- a/skyvern-frontend/src/routes/workflows/copilot/copilotPhases.test.ts+++ b/skyvern-frontend/src/routes/workflows/copilot/copilotPhases.test.ts@@ -79,6 +79,15 @@ describe("toolActivityDisplayLabel — discovery tools (SKY-12385)", () => {expect(toolActivityDisplayLabel("inspect_page_for_composition")).toBe("Inspecting the page",);+ expect(toolActivityDisplayLabel("skyvern_frame_list")).toBe(+ "Finding embedded pages",+ );+ expect(toolActivityDisplayLabel("skyvern_frame_switch")).toBe(+ "Opening embedded page",+ );+ expect(toolActivityDisplayLabel("skyvern_frame_main")).toBe(+ "Returning to main page",+ );});it("labels fill_credential_field without naming any credential", () => {diff --git a/skyvern-frontend/src/routes/workflows/copilot/narrativeState.ts b/skyvern-frontend/src/routes/workflows/copilot/narrativeState.tsindex 231a1ac6f..73497b097 100644--- a/skyvern-frontend/src/routes/workflows/copilot/narrativeState.ts+++ b/skyvern-frontend/src/routes/workflows/copilot/narrativeState.ts@@ -599,6 +599,9 @@ const ACTIVITY_TOOL_DISPLAY_LABELS: Record<string, string> = {press_key: "Interacting with page",navigate_browser: "Opening page",wait_for_either_state: "Waiting for the page",+ skyvern_frame_list: "Finding embedded pages",+ skyvern_frame_switch: "Opening embedded page",+ skyvern_frame_main: "Returning to main page",get_block_schema: "Checking workflow block options",inspect_current_workflow: "Inspecting workflow",discover_workflow_entrypoint: "Finding the entry page",diff --git a/skyvern/forge/sdk/copilot/narration.py b/skyvern/forge/sdk/copilot/narration.pyindex 1e0eebe37..98955718d 100644--- a/skyvern/forge/sdk/copilot/narration.py+++ b/skyvern/forge/sdk/copilot/narration.py@@ -112,6 +112,9 @@ _TOOL_ACTIVITY_DISPLAY_LABELS = {"validate_block": "Checking the block","console_messages": "Reading the browser console","wait_for_either_state": "Waiting for the page",+ "skyvern_frame_list": "Finding embedded pages",+ "skyvern_frame_switch": "Opening embedded page",+ "skyvern_frame_main": "Returning to main page","fill_credential_field": "Entering saved credentials","edit_block": "Editing block","add_block": "Adding block",diff --git a/skyvern/forge/sdk/copilot/streaming_adapter.py b/skyvern/forge/sdk/copilot/streaming_adapter.pyindex 5b4a643e7..ee77f5cdb 100644--- a/skyvern/forge/sdk/copilot/streaming_adapter.py+++ b/skyvern/forge/sdk/copilot/streaming_adapter.py@@ -82,6 +82,9 @@ _OBSERVATION_TOOLS = {"select_option","press_key","wait_for_either_state",+ "skyvern_frame_list",+ "skyvern_frame_switch",+ "skyvern_frame_main",}_AUTHORING_TOOL_NAMES = frozenset({"update_and_run_blocks", "edit_block_and_run", "update_workflow"})diff --git a/skyvern/forge/sdk/copilot/tools/mcp_hooks.py b/skyvern/forge/sdk/copilot/tools/mcp_hooks.pyindex 7ca26653f..a8f737395 100644--- a/skyvern/forge/sdk/copilot/tools/mcp_hooks.py+++ b/skyvern/forge/sdk/copilot/tools/mcp_hooks.py@@ -1806,6 +1806,10 @@ def get_skyvern_mcp_alias_map() -> dict[str, str]:"select_option": "skyvern_select_option","press_key": "skyvern_press_key","wait_for_either_state": "skyvern_wait_for_either_state",+ # These frame controls already use their user-facing MCP names.+ "skyvern_frame_list": "skyvern_frame_list",+ "skyvern_frame_switch": "skyvern_frame_switch",+ "skyvern_frame_main": "skyvern_frame_main",}@@ -2025,4 +2029,25 @@ def _build_skyvern_mcp_overlays(pre_hook=_sensitive_origin_page_pre_hook,post_hook=_wait_for_either_state_post_hook,),+ "skyvern_frame_list": SchemaOverlay(+ hide_params=frozenset({"session_id", "cdp_url"}),+ copilot_params={BROWSER_TARGET_PARAM_NAME: BROWSER_TARGET_PARAM},+ requires_browser=True,+ pre_hook=_sensitive_origin_page_pre_hook,+ post_hook=_sensitive_origin_page_post_hook,+ ),+ "skyvern_frame_switch": SchemaOverlay(+ hide_params=frozenset({"session_id", "cdp_url"}),+ copilot_params={BROWSER_TARGET_PARAM_NAME: BROWSER_TARGET_PARAM},+ requires_browser=True,+ pre_hook=_sensitive_origin_page_pre_hook,+ post_hook=_sensitive_origin_page_post_hook,+ ),+ "skyvern_frame_main": SchemaOverlay(+ hide_params=frozenset({"session_id", "cdp_url"}),+ copilot_params={BROWSER_TARGET_PARAM_NAME: BROWSER_TARGET_PARAM},+ requires_browser=True,+ pre_hook=_sensitive_origin_page_pre_hook,+ post_hook=_sensitive_origin_page_post_hook,+ ),}diff --git a/skyvern/forge/sdk/copilot/unrecoverable_tool_error.py b/skyvern/forge/sdk/copilot/unrecoverable_tool_error.pyindex 3b350ad71..a35f8a4d3 100644--- a/skyvern/forge/sdk/copilot/unrecoverable_tool_error.py+++ b/skyvern/forge/sdk/copilot/unrecoverable_tool_error.py@@ -34,6 +34,9 @@ _BROWSER_SESSION_TOOL_NAMES = frozenset("select_option","press_key","wait_for_either_state",+ "skyvern_frame_list",+ "skyvern_frame_switch",+ "skyvern_frame_main",})_UNRECOVERABLE_TOOL_ERROR_CATEGORY = "UNRECOVERABLE_TOOL_ERROR"diff --git a/tests/unit/forge/sdk/copilot/test_browser_ablation_mode.py b/tests/unit/forge/sdk/copilot/test_browser_ablation_mode.pyindex c117e8d75..132664d3e 100644--- a/tests/unit/forge/sdk/copilot/test_browser_ablation_mode.py+++ b/tests/unit/forge/sdk/copilot/test_browser_ablation_mode.py@@ -6,6 +6,7 @@ from unittest.mock import AsyncMockimport pytestfrom mcp.types import Tool as MCPTool+from skyvern.cli.mcp_tools import mcpfrom skyvern.forge.sdk.copilot import screenshot_utilsfrom skyvern.forge.sdk.copilot.browser_ablation import (BROWSER_ABLATION_MCP_TOOL_EXCLUSIONS,@@ -21,6 +22,7 @@ from skyvern.forge.sdk.copilot.browser_ablation import ()from skyvern.forge.sdk.copilot.config import CopilotConfigfrom skyvern.forge.sdk.copilot.mcp_adapter import SchemaOverlay, SkyvernOverlayMCPServer+from skyvern.forge.sdk.copilot.tools import NATIVE_TOOLS, _build_skyvern_mcp_overlays, get_skyvern_mcp_alias_mapfrom skyvern.forge.sdk.schemas.workflow_copilot import (WorkflowCopilotBrowserAblationResponseUpdate,WorkflowCopilotStreamResponseUpdate,@@ -37,6 +39,9 @@ _EXISTING_BROWSER_ALIASES = ("select_option","press_key","wait_for_either_state",+ "skyvern_frame_list",+ "skyvern_frame_switch",+ "skyvern_frame_main",)_EXPECTED_BROWSER_ABLATION_MCP_TOOLS = (*_EXISTING_BROWSER_ALIASES,@@ -47,6 +52,7 @@ _EXPECTED_BROWSER_ABLATION_MCP_TOOLS = ("skyvern_tab_wait_for_new","skyvern_page",)+_FRAME_TOOLS = ("skyvern_frame_list", "skyvern_frame_switch", "skyvern_frame_main")def _catalogs() -> tuple[list[SimpleNamespace], dict[str, str], dict[str, SchemaOverlay]]:@@ -116,15 +122,33 @@ def test_browser_ablation_projects_registered_tab_and_page_tools_without_workfloassert "skyvern_workflow_run" not in surface.ordered_mcp_names-async def test_browser_ablation_projects_the_app_registry_contract() -> None:- from skyvern.cli.mcp_tools import mcp- from skyvern.forge.sdk.copilot.tools import (- NATIVE_TOOLS,- _build_skyvern_mcp_overlays,- get_skyvern_mcp_alias_map,+def test_normal_copilot_frame_contracts_are_shared_with_browser_ablation_and_hashed() -> None:+ aliases = get_skyvern_mcp_alias_map()+ overlays = _build_skyvern_mcp_overlays()+ normal = resolve_copilot_tool_surface(+ mode=None,+ native_tools=[],+ alias_map=aliases,+ overlays=overlays,)+ aliases_without_frames = {name: transport for name, transport in aliases.items() if name not in _FRAME_TOOLS}+ overlays_without_frames = {name: overlay for name, overlay in overlays.items() if name not in _FRAME_TOOLS}+ old_surface = resolve_copilot_tool_surface(+ mode=None,+ native_tools=[],+ alias_map=aliases_without_frames,+ overlays=overlays_without_frames,+ )++ assert all(aliases[name] == name for name in _FRAME_TOOLS)+ assert set(_FRAME_TOOLS) <= set(normal.ordered_mcp_names)+ assert all(overlays[name].requires_browser for name in _FRAME_TOOLS)+ assert all(overlays[name].hide_params == {"session_id", "cdp_url"} for name in _FRAME_TOOLS)+ assert normal.sha256 != old_surface.sha256++async def test_browser_ablation_projects_the_app_registry_contract() -> None:registered_tools = await mcp.list_tools(run_middleware=False)surface = resolve_copilot_tool_surface(mode=CopilotEvalMode.BROWSER_ABLATION,@@ -141,6 +165,7 @@ async def test_browser_ablation_projects_the_app_registry_contract() -> None:"skyvern_tab_wait_for_new","skyvern_tab_close","skyvern_page",+ *_FRAME_TOOLS,}.issubset(surface.ordered_mcp_names)assert surface.ordered_mcp_names == _EXPECTED_BROWSER_ABLATION_MCP_TOOLSassert BROWSER_ABLATION_MCP_TOOL_EXCLUSIONS.isdisjoint(surface.alias_map.values())diff --git a/tests/unit/test_copilot_hooks.py b/tests/unit/test_copilot_hooks.pyindex bcf65555f..636e1c2da 100644--- a/tests/unit/test_copilot_hooks.py+++ b/tests/unit/test_copilot_hooks.py@@ -768,6 +768,9 @@ class TestMCPToolOverlayCompleteness:"select_option","press_key","wait_for_either_state",+ "skyvern_frame_list",+ "skyvern_frame_switch",+ "skyvern_frame_main",}assert set(alias_map.keys()) == expected_aliasesassert all(v.startswith("skyvern_") for v in alias_map.values())@@ -848,6 +851,16 @@ class TestNewToolOverlayConfigs:assert overlay.pre_hook is mcp_hooks._sensitive_origin_page_pre_hookassert overlay.post_hook is mcp_hooks._sensitive_origin_page_post_hook+ def test_frame_control_overlays_refuse_sensitive_origin_pages(self) -> None:+ from skyvern.forge.sdk.copilot.tools import _build_skyvern_mcp_overlays++ overlays = _build_skyvern_mcp_overlays()++ for name in ("skyvern_frame_list", "skyvern_frame_switch", "skyvern_frame_main"):+ overlay = overlays[name]+ assert overlay.pre_hook is mcp_hooks._sensitive_origin_page_pre_hook+ assert overlay.post_hook is mcp_hooks._sensitive_origin_page_post_hook+def test_select_option_overlay(self) -> None:from skyvern.forge.sdk.copilot.tools import _build_skyvern_mcp_overlaysdiff --git a/tests/unit/test_copilot_mcp_adapter.py b/tests/unit/test_copilot_mcp_adapter.pyindex cacb2ab38..2a00ef44f 100644--- a/tests/unit/test_copilot_mcp_adapter.py+++ b/tests/unit/test_copilot_mcp_adapter.py@@ -788,6 +788,47 @@ class TestMCPToolTiming:assert ctx.flow_evidence == [existing_flow]assert ctx.scouted_output_covered_paths == {"output.existing"}+ @pytest.mark.asyncio+ @pytest.mark.parametrize(+ ("tool_name", "args"),+ [+ ("skyvern_frame_list", {}),+ ("skyvern_frame_switch", {"selector": "#payment-frame"}),+ ("skyvern_frame_main", {}),+ ],+ )+ async def test_frame_controls_refuse_tainted_browser_sessions_before_dispatch(+ self,+ _stub_browser_session: None,+ tool_name: str,+ args: dict[str, str],+ ) -> None:+ ctx = make_copilot_ctx(browser_session_id="pbs_1")+ ctx.sensitive_origin_browser_session_ids.add("pbs_1")+ dispatched: list[str] = []++ class _CapturingClient:+ async def call_tool(self, name: str, args: dict[str, Any], raise_on_error: bool = False) -> Any:+ dispatched.append(name)+ return SimpleNamespace(structured_content={"ok": True}, is_error=False, content=[])++ aliases = get_skyvern_mcp_alias_map()+ server = SkyvernOverlayMCPServer(+ transport=MagicMock(),+ overlays={tool_name: _build_skyvern_mcp_overlays()[tool_name]},+ alias_map={tool_name: aliases[tool_name]},+ allowlist=frozenset({aliases[tool_name]}),+ context_provider=lambda: ctx,+ )+ server._client = _CapturingClient()++ result = await server.call_tool(tool_name, args)++ surfaced = json.loads(result.content[0].text)+ assert surfaced["ok"] is False+ assert "specific named URL" in surfaced["error"]+ assert dispatched == []+@pytest.mark.asyncioasync def test_a_call_that_exceeds_its_ceiling_reports_its_wall_time(self, _fake_clock: list[float]) -> None:overlay = SchemaOverlay(requires_browser=True, timeout=30)@@ -1677,7 +1718,12 @@ def test_every_browser_overlay_offers_the_target_param() -> None:overlays = _build_skyvern_mcp_overlays(BlockAuthoringPolicy.CODE_ONLY_BROWSER)browser_overlays = {name: overlay for name, overlay in overlays.items() if overlay.requires_browser}- assert len(browser_overlays) == 10+ assert {+ "skyvern_frame_list",+ "skyvern_frame_switch",+ "skyvern_frame_main",+ } <= set(browser_overlays)+ assert len(browser_overlays) == 13assert all(BROWSER_TARGET_PARAM_NAME in overlay.copilot_params for overlay in browser_overlays.values())# inspect_page_for_composition owns target_url semantics; a second target would overload it.assert "inspect_page_for_composition" not in browser_overlaysdiff --git a/tests/unit/test_copilot_narrative_activity.py b/tests/unit/test_copilot_narrative_activity.pyindex cc0b68d70..17a2d94d2 100644--- a/tests/unit/test_copilot_narrative_activity.py+++ b/tests/unit/test_copilot_narrative_activity.py@@ -197,6 +197,9 @@ def test_record_activity_non_run_tool_result_routes_live_not_pinned() -> None:def test_tool_activity_display_label_covers_discovery_tools() -> None:assert tool_activity_display_label("discover_workflow_entrypoint") == "Finding the entry page"assert tool_activity_display_label("inspect_page_for_composition") == "Inspecting the page"+ assert tool_activity_display_label("skyvern_frame_list") == "Finding embedded pages"+ assert tool_activity_display_label("skyvern_frame_switch") == "Opening embedded page"+ assert tool_activity_display_label("skyvern_frame_main") == "Returning to main page"def test_build_narrative_payload_serializes_block_and_design_activity() -> None:diff --git a/tests/unit/test_iframe_mcp_e2e.py b/tests/unit/test_iframe_mcp_e2e.pyindex 79c732e28..6d04d0474 100644--- a/tests/unit/test_iframe_mcp_e2e.py+++ b/tests/unit/test_iframe_mcp_e2e.py@@ -10,24 +10,36 @@ Skipped in CI when Playwright browsers are not installed.from __future__ import annotationsimport asyncio+import json+from contextlib import asynccontextmanagerfrom pathlib import Pathfrom typing import Anyfrom unittest.mock import MagicMockimport pytestimport pytest_asyncio+from mcp.types import Tool as MCPToolfrom playwright.async_api import async_playwrightfrom skyvern.cli.core.result import BrowserContextfrom skyvern.cli.core.session_manager import SessionState, get_current_session, set_current_session+from skyvern.cli.mcp_tools import mcpfrom skyvern.cli.mcp_tools.browser import (+ skyvern_evaluate,skyvern_execute,skyvern_frame_list,skyvern_frame_main,skyvern_frame_switch,skyvern_observe,+ skyvern_type,)+from skyvern.forge.sdk.copilot import mcp_adapter+from skyvern.forge.sdk.copilot.browser_ablation import CopilotToolSurface+from skyvern.forge.sdk.copilot.mcp_adapter import SkyvernOverlayMCPServer+from skyvern.forge.sdk.copilot.runtime import AgentContext+from skyvern.forge.sdk.copilot.tools.mcp_hooks import _build_skyvern_mcp_overlays, get_skyvern_mcp_alias_mapfrom skyvern.library.skyvern_browser_page import SkyvernBrowserPage+from tests.unit.copilot_test_helpers import make_copilot_ctxdef _has_playwright_browser() -> bool:@@ -94,6 +106,38 @@ class _FakeBrowser:return SkyvernBrowserPage(MagicMock(), self._page)+class _LocalToolResult:+ def __init__(self, payload: dict[str, Any]) -> None:+ self.structured_content = payload+ self.is_error = payload.get("ok", True) is not True+ self.content: list[Any] = []+++class _LocalCopilotMCPClient:+ def __init__(self, tools: list[MCPTool]) -> None:+ self._tools = tools+ self.calls: list[tuple[str, dict[str, Any]]] = []+ self._dispatch = {+ "skyvern_frame_list": skyvern_frame_list,+ "skyvern_frame_switch": skyvern_frame_switch,+ "skyvern_frame_main": skyvern_frame_main,+ "skyvern_evaluate": skyvern_evaluate,+ "skyvern_type": skyvern_type,+ }++ async def list_tools(self) -> list[MCPTool]:+ return self._tools++ async def call_tool(self, name: str, args: dict[str, Any], raise_on_error: bool = False) -> _LocalToolResult:+ self.calls.append((name, dict(args)))+ return _LocalToolResult(await self._dispatch[name](**args))+++async def _copilot_payload(server: SkyvernOverlayMCPServer, name: str, args: dict[str, Any]) -> dict[str, Any]:+ result = await server._call_tool(name, args)+ return json.loads(result.content[0].text)++@pytest_asyncio.fixtureasync def mcp_session():"""Set up a real Playwright browser and wire it into SessionState."""@@ -135,6 +179,107 @@ async def test_mcp_frame_list_real_browser(mcp_session: SessionState) -> None:assert result["data"]["count"] >= 2+async def test_copilot_advertised_frame_tool_chain_real_browser(+ mcp_session: SessionState,+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ @asynccontextmanager+ async def _browser_scope(_ctx: AgentContext, *, session_id_override: str | None = None):+ yield++ async def _browser_available(_ctx: AgentContext) -> None:+ return None++ monkeypatch.setattr(mcp_adapter, "ensure_browser_session", _browser_available)+ monkeypatch.setattr(mcp_adapter, "mcp_browser_context", _browser_scope)++ surface_names = (+ "skyvern_frame_list",+ "skyvern_frame_switch",+ "skyvern_frame_main",+ "evaluate",+ "type_text",+ )+ canonical_aliases = get_skyvern_mcp_alias_map()+ canonical_overlays = _build_skyvern_mcp_overlays()+ aliases = {name: canonical_aliases[name] for name in surface_names}+ overlays = {name: canonical_overlays[name] for name in surface_names}+ registered = await mcp.list_tools(run_middleware=False)+ raw_names = frozenset(aliases.values())+ client = _LocalCopilotMCPClient(+ [+ MCPTool(name=tool.name, description=tool.description, inputSchema=tool.parameters)+ for tool in registered+ if tool.name in raw_names+ ]+ )+ copilot_ctx = make_copilot_ctx(browser_session_id=None)+ server = SkyvernOverlayMCPServer(+ transport=object(),+ overlays=overlays,+ alias_map=aliases,+ allowlist=raw_names,+ ordered_allowlist=tuple(aliases.values()),+ enforce_dispatch_allowlist=True,+ context_provider=lambda: copilot_ctx,+ )+ server._client = client # type: ignore[assignment]++ advertised = await server.list_tools()+ advertised_names = {tool.name for tool in advertised}+ assert set(surface_names) == advertised_names+ frame_contracts = [tool for tool in advertised if tool.name.startswith("skyvern_frame_")]+ assert all(tool.description for tool in frame_contracts)+ assert all("session_id" not in tool.inputSchema["properties"] for tool in frame_contracts)+ assert all("cdp_url" not in tool.inputSchema["properties"] for tool in frame_contracts)+ surface = CopilotToolSurface(+ native_tools=(),+ alias_map=aliases,+ overlays=overlays,+ ordered_native_names=(),+ ordered_mcp_names=surface_names,+ )+ without_frames = [tool for tool in advertised if not tool.name.startswith("skyvern_frame_")]+ old_surface = CopilotToolSurface(+ native_tools=(),+ alias_map={tool.name: aliases[tool.name] for tool in without_frames},+ overlays={tool.name: overlays[tool.name] for tool in without_frames},+ ordered_native_names=(),+ ordered_mcp_names=tuple(tool.name for tool in without_frames),+ )+ assert surface.advertised_sha256(advertised) != old_surface.advertised_sha256(without_frames)++ listed = await _copilot_payload(server, "skyvern_frame_list", {})+ assert listed["ok"] is True+ assert any(frame["name"] == "payment" for frame in listed["data"]["frames"])++ entered = await _copilot_payload(server, "skyvern_frame_switch", {"selector": "#pay-frame"})+ assert entered["ok"] is True+ assert entered["data"]["frame_name"] == "payment"++ card_value_expression = "document.querySelector('#pay-frame').contentDocument.querySelector('#card').value"+ before = await _copilot_payload(server, "evaluate", {"expression": card_value_expression})+ typed = await _copilot_payload(server, "type_text", {"selector": "#card", "text": "4242"})+ after = await _copilot_payload(server, "evaluate", {"expression": card_value_expression})+ assert before["ok"] is True, before+ assert after["ok"] is True, after+ assert before["data"]["result"] == ""+ assert typed["ok"] is True+ assert after["data"]["result"] == "4242"++ returned = await _copilot_payload(server, "skyvern_frame_main", {})+ continued = await _copilot_payload(server, "type_text", {"selector": "#main-input", "text": "continued"})+ main_value = await _copilot_payload(+ server, "evaluate", {"expression": "document.querySelector('#main-input').value"}+ )+ assert returned["ok"] is True+ assert continued["ok"] is True+ assert main_value["data"]["result"] == "continued"+ invented_coordinate_count = sum("x" in args or "y" in args for _, args in client.calls)+ assert invented_coordinate_count == 0++@pytest.mark.asyncioasync def test_mcp_frame_switch_by_selector(mcp_session: SessionState) -> None:result = await skyvern_frame_switch(selector="#pay-frame")