Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
SKY-15270: Show Copilot only the arguments each browser tool accepts (#8349)
Andrew Neilson9693986ecada
diff --git a/pyproject.toml b/pyproject.tomlindex 1a0d4735b..8d6c9b3f8 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -1,6 +1,6 @@[project]name = "skyvern"-version = "1.0.52"+version = "1.0.51"description = ""authors = [{ name = "Skyvern AI", email = "[email protected]" }]requires-python = ">=3.11,<3.15"diff --git a/skyvern/forge/sdk/copilot/mcp_adapter.py b/skyvern/forge/sdk/copilot/mcp_adapter.pyindex 93a59a736..74ac0f059 100644--- a/skyvern/forge/sdk/copilot/mcp_adapter.py+++ b/skyvern/forge/sdk/copilot/mcp_adapter.py@@ -1116,6 +1116,7 @@ def _apply_schema_overlay(required = overlay.required_overridesreturn {+ **input_schema,"type": input_schema.get("type", "object"),"properties": props,"required": required,diff --git a/tests/unit/forge/sdk/copilot/test_browser_ablation_mode.py b/tests/unit/forge/sdk/copilot/test_browser_ablation_mode.pyindex 132664d3e..69e36130c 100644--- a/tests/unit/forge/sdk/copilot/test_browser_ablation_mode.py+++ b/tests/unit/forge/sdk/copilot/test_browser_ablation_mode.py@@ -1,9 +1,12 @@from __future__ import annotationsfrom types import SimpleNamespace-from unittest.mock import AsyncMock+from typing import Any+from unittest.mock import AsyncMock, MagicMockimport pytest+from fastmcp.tools.tool import Tool as RegisteredTool+from jsonschema import Draft202012Validatorfrom mcp.types import Tool as MCPToolfrom skyvern.cli.mcp_tools import mcp@@ -21,7 +24,7 @@ from skyvern.forge.sdk.copilot.browser_ablation import (resolve_copilot_tool_surface,)from skyvern.forge.sdk.copilot.config import CopilotConfig-from skyvern.forge.sdk.copilot.mcp_adapter import SchemaOverlay, SkyvernOverlayMCPServer+from skyvern.forge.sdk.copilot.mcp_adapter import BROWSER_TARGET_PARAM_NAME, SchemaOverlay, SkyvernOverlayMCPServerfrom skyvern.forge.sdk.copilot.tools import NATIVE_TOOLS, _build_skyvern_mcp_overlays, get_skyvern_mcp_alias_mapfrom skyvern.forge.sdk.schemas.workflow_copilot import (WorkflowCopilotBrowserAblationResponseUpdate,@@ -174,6 +177,70 @@ async def test_browser_ablation_projects_the_app_registry_contract() -> None:assert registered_workflow_tools.isdisjoint(surface.alias_map.values())+async def _advertised_schemas(+ surface: CopilotToolSurface, registered: list[RegisteredTool]+) -> dict[str, dict[str, Any]]:+ server = SkyvernOverlayMCPServer(+ transport=MagicMock(),+ overlays=surface.overlays,+ alias_map=surface.alias_map,+ allowlist=frozenset(surface.alias_map.values()),+ context_provider=lambda: SimpleNamespace(organization_id="o_test"),+ )+ server._client = MagicMock()+ server._cached_raw_tools = [+ MCPTool(name=tool.name, description=tool.description, inputSchema=tool.parameters) for tool in registered+ ]+ return {tool.name: tool.inputSchema for tool in await server.list_tools()}+++def _production_surfaces(registered: list[RegisteredTool]) -> tuple[CopilotToolSurface, CopilotToolSurface]:+ aliases = get_skyvern_mcp_alias_map()+ overlays = _build_skyvern_mcp_overlays()+ normal = resolve_copilot_tool_surface(+ mode=None,+ native_tools=list(NATIVE_TOOLS),+ alias_map=aliases,+ overlays=overlays,+ )+ ablation = resolve_copilot_tool_surface(+ mode=CopilotEvalMode.BROWSER_ABLATION,+ native_tools=list(NATIVE_TOOLS),+ alias_map=aliases,+ overlays=overlays,+ registered_mcp_tools=registered,+ )+ return normal, ablation+++async def test_the_advertised_tab_contract_is_closed_around_the_fields_it_offers() -> None:+ registered = await mcp.list_tools(run_middleware=False)+ _, ablation = _production_surfaces(registered)++ tab_new = (await _advertised_schemas(ablation, registered))["skyvern_tab_new"]++ assert set(tab_new["properties"]) == {"url", BROWSER_TARGET_PARAM_NAME}+ assert tab_new["additionalProperties"] is False+ validator = Draft202012Validator(tab_new)+ assert validator.is_valid({"url": "https://example.com"})+ assert not validator.is_valid({"url": "https://example.com", "settle_seconds": 30})+++async def test_shared_browser_tools_advertise_one_contract_across_both_modes() -> None:+ registered = await mcp.list_tools(run_middleware=False)+ normal, ablation = _production_surfaces(registered)++ normal_schemas = await _advertised_schemas(normal, registered)+ ablation_schemas = await _advertised_schemas(ablation, registered)++ shared = set(normal_schemas) & set(ablation_schemas)+ assert set(_EXISTING_BROWSER_ALIASES) <= shared+ assert {name: normal_schemas[name] for name in shared} == {name: ablation_schemas[name] for name in shared}+ assert all(normal_schemas[name].get("additionalProperties") is False for name in shared)++def test_browser_ablation_preserves_existing_aliases_then_adds_required_registry_capabilities() -> None:native, aliases, overlays = _catalogs()diff --git a/tests/unit/test_copilot_mcp_adapter.py b/tests/unit/test_copilot_mcp_adapter.pyindex 2a00ef44f..2a178f4c0 100644--- a/tests/unit/test_copilot_mcp_adapter.py+++ b/tests/unit/test_copilot_mcp_adapter.py@@ -10,14 +10,17 @@ import pytestfrom mcp.types import CallToolResultfrom structlog.testing import capture_logs+from skyvern.cli.mcp_tools import mcpfrom skyvern.forge.sdk.cache.base import NoopLockfrom skyvern.forge.sdk.cache.local import LocalCachefrom skyvern.forge.sdk.copilot import mcp_adapter+from skyvern.forge.sdk.copilot.browser_ablation import CopilotEvalMode, resolve_copilot_tool_surfacefrom skyvern.forge.sdk.copilot.config import BlockAuthoringPolicyfrom skyvern.forge.sdk.copilot.mcp_adapter import (BROWSER_TARGET_PARAM_NAME,SchemaOverlay,SkyvernOverlayMCPServer,+ _apply_schema_overlay,_BrowserCallOutcome,_handle_browser_session_loss,_requested_output_path_choices,@@ -33,6 +36,7 @@ from skyvern.forge.sdk.copilot.runtime import (browser_page_custody_lock,mcp_to_copilot,)+from skyvern.forge.sdk.copilot.tools import NATIVE_TOOLSfrom skyvern.forge.sdk.copilot.tools.mcp_hooks import _build_skyvern_mcp_overlays, get_skyvern_mcp_alias_mapfrom skyvern.forge.sdk.copilot.turn_origin import TurnOriginfrom tests.unit.copilot_test_helpers import make_copilot_ctx@@ -72,6 +76,107 @@ class TestRequestedOutputPathChoices:def test_a_turn_owing_no_output_leaves_the_schema_alone(self) -> None:assert _requested_output_path_choices(_schema(), []) == _schema()+ def test_narrowing_the_path_choices_leaves_the_rest_of_the_contract_intact(self) -> None:+ source = {**_schema(), "additionalProperties": False}++ schema = _requested_output_path_choices(source, ["output.a"])++ assert schema["additionalProperties"] is False+ assert schema["properties"]["expression"] == source["properties"]["expression"]+ assert schema["required"] == source["required"]+++def _closed_source_schema() -> dict[str, Any]:+ return {+ "type": "object",+ "additionalProperties": False,+ "description": "Open a page.",+ "$defs": {"Where": {"type": "string", "enum": ["chat", "last_run"]}},+ "properties": {+ "session_id": {"type": "string", "description": "Browser session id."},+ "url": {"type": "string", "description": "URL to open."},+ "where": {"$ref": "#/$defs/Where"},+ },+ "required": ["url"],+ }+++async def _tab_new_overlay() -> SchemaOverlay:+ surface = resolve_copilot_tool_surface(+ mode=CopilotEvalMode.BROWSER_ABLATION,+ native_tools=list(NATIVE_TOOLS),+ alias_map=get_skyvern_mcp_alias_map(),+ overlays=_build_skyvern_mcp_overlays(),+ registered_mcp_tools=await mcp.list_tools(run_middleware=False),+ )+ return surface.overlays["skyvern_tab_new"]+++class TestSchemaOverlayPreservesTheSourceContract:+ def test_an_empty_overlay_leaves_the_accepted_argument_contract_untouched(self) -> None:+ source = _closed_source_schema()++ assert _apply_schema_overlay(source, SchemaOverlay()) == source++ @pytest.mark.parametrize(+ ("overlay", "expected_properties", "expected_required"),+ [+ (SchemaOverlay(hide_params=frozenset({"session_id"})), {"url", "where"}, ["url"]),+ (SchemaOverlay(forced_args={"session_id": "pbs_1"}), {"url", "where"}, ["url"]),+ (SchemaOverlay(arg_transforms={"target": "where"}), {"session_id", "url", "target"}, ["url"]),+ (+ SchemaOverlay(copilot_params={"target": {"type": "string"}}),+ {"session_id", "url", "where", "target"},+ ["url"],+ ),+ ],+ )+ def test_an_overlay_edits_the_fields_it_names_and_nothing_else(+ self,+ overlay: SchemaOverlay,+ expected_properties: set[str],+ expected_required: list[str],+ ) -> None:+ source = _closed_source_schema()++ schema = _apply_schema_overlay(source, overlay)++ assert set(schema["properties"]) == expected_properties+ assert schema["required"] == expected_required+ assert schema["additionalProperties"] is False+ assert schema["$defs"] == source["$defs"]+ assert schema["description"] == source["description"]++ def test_a_renamed_field_keeps_its_definition_reachable(self) -> None:+ schema = _apply_schema_overlay(_closed_source_schema(), SchemaOverlay(arg_transforms={"target": "where"}))++ assert schema["properties"]["target"] == {"$ref": "#/$defs/Where"}+ assert schema["$defs"]["Where"]["enum"] == ["chat", "last_run"]+++async def test_a_valid_tab_creation_dispatches_only_the_tools_own_arguments() -> None:+ overlay = await _tab_new_overlay()++ mcp_args = _transform_args({"url": "https://example.com", BROWSER_TARGET_PARAM_NAME: "last_run"}, overlay)++ assert mcp_args == {"url": "https://example.com"}+++async def test_an_argument_outside_the_contract_is_refused_rather_than_swallowed() -> None:+ overlay = await _tab_new_overlay()++ mcp_args = _transform_args({"url": "https://example.com", "settle_seconds": 30}, overlay)+ result = await mcp.call_tool("skyvern_tab_new", mcp_args)+ payload = result.structured_content++ assert mcp_args["settle_seconds"] == 30+ assert payload is not None+ assert payload["ok"] is False+ assert payload["error"]["message"] == "skyvern_tab_new was called with unsupported argument(s): settle_seconds."+ assert payload["error"]["details"]["unsupported_arguments"] == ["settle_seconds"]+def test_the_declared_path_says_the_expression_is_that_value() -> None:overlays = _build_skyvern_mcp_overlays(BlockAuthoringPolicy.CODE_ONLY_BROWSER)diff --git a/uv.lock b/uv.lockindex 05939543b..675e59e44 100644--- a/uv.lock+++ b/uv.lock@@ -6790,7 +6790,7 @@ wheels = [[[package]]name = "skyvern"-version = "1.0.52"+version = "1.0.51"source = { editable = "." }dependencies = [{ name = "cachetools" },