Skip to content

Automate browser based workflows with AI

SKY-15270: Show Copilot only the arguments each browser tool accepts (#8349)

Andrew Neilson9693986ecada
diff --git a/pyproject.toml b/pyproject.toml
index 1a0d4735b..8d6c9b3f8 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
[project]
name = "skyvern"
-version = "1.0.52"
+version = "1.0.51"
description = ""
authors = [{ name = "Skyvern AI", email = "[email protected]" }]
requires-python = ">=3.11,<3.15"
diff --git a/skyvern/forge/sdk/copilot/mcp_adapter.py b/skyvern/forge/sdk/copilot/mcp_adapter.py
index 93a59a736..74ac0f059 100644
--- a/skyvern/forge/sdk/copilot/mcp_adapter.py
+++ b/skyvern/forge/sdk/copilot/mcp_adapter.py
@@ -1116,6 +1116,7 @@ def _apply_schema_overlay(
required = overlay.required_overrides
return {
+ **input_schema,
"type": input_schema.get("type", "object"),
"properties": props,
"required": required,
diff --git a/tests/unit/forge/sdk/copilot/test_browser_ablation_mode.py b/tests/unit/forge/sdk/copilot/test_browser_ablation_mode.py
index 132664d3e..69e36130c 100644
--- a/tests/unit/forge/sdk/copilot/test_browser_ablation_mode.py
+++ b/tests/unit/forge/sdk/copilot/test_browser_ablation_mode.py
@@ -1,9 +1,12 @@
from __future__ import annotations
from types import SimpleNamespace
-from unittest.mock import AsyncMock
+from typing import Any
+from unittest.mock import AsyncMock, MagicMock
import pytest
+from fastmcp.tools.tool import Tool as RegisteredTool
+from jsonschema import Draft202012Validator
from mcp.types import Tool as MCPTool
from skyvern.cli.mcp_tools import mcp
@@ -21,7 +24,7 @@ from skyvern.forge.sdk.copilot.browser_ablation import (
resolve_copilot_tool_surface,
)
from skyvern.forge.sdk.copilot.config import CopilotConfig
-from skyvern.forge.sdk.copilot.mcp_adapter import SchemaOverlay, SkyvernOverlayMCPServer
+from skyvern.forge.sdk.copilot.mcp_adapter import BROWSER_TARGET_PARAM_NAME, SchemaOverlay, SkyvernOverlayMCPServer
from skyvern.forge.sdk.copilot.tools import NATIVE_TOOLS, _build_skyvern_mcp_overlays, get_skyvern_mcp_alias_map
from skyvern.forge.sdk.schemas.workflow_copilot import (
WorkflowCopilotBrowserAblationResponseUpdate,
@@ -174,6 +177,70 @@ async def test_browser_ablation_projects_the_app_registry_contract() -> None:
assert registered_workflow_tools.isdisjoint(surface.alias_map.values())
+async def _advertised_schemas(
+ surface: CopilotToolSurface, registered: list[RegisteredTool]
+) -> dict[str, dict[str, Any]]:
+ server = SkyvernOverlayMCPServer(
+ transport=MagicMock(),
+ overlays=surface.overlays,
+ alias_map=surface.alias_map,
+ allowlist=frozenset(surface.alias_map.values()),
+ context_provider=lambda: SimpleNamespace(organization_id="o_test"),
+ )
+ server._client = MagicMock()
+ server._cached_raw_tools = [
+ MCPTool(name=tool.name, description=tool.description, inputSchema=tool.parameters) for tool in registered
+ ]
+ return {tool.name: tool.inputSchema for tool in await server.list_tools()}
+
+
+def _production_surfaces(registered: list[RegisteredTool]) -> tuple[CopilotToolSurface, CopilotToolSurface]:
+ aliases = get_skyvern_mcp_alias_map()
+ overlays = _build_skyvern_mcp_overlays()
+ normal = resolve_copilot_tool_surface(
+ mode=None,
+ native_tools=list(NATIVE_TOOLS),
+ alias_map=aliases,
+ overlays=overlays,
+ )
+ ablation = resolve_copilot_tool_surface(
+ mode=CopilotEvalMode.BROWSER_ABLATION,
+ native_tools=list(NATIVE_TOOLS),
+ alias_map=aliases,
+ overlays=overlays,
+ registered_mcp_tools=registered,
+ )
+ return normal, ablation
+
+
+async def test_the_advertised_tab_contract_is_closed_around_the_fields_it_offers() -> None:
+ registered = await mcp.list_tools(run_middleware=False)
+ _, ablation = _production_surfaces(registered)
+
+ tab_new = (await _advertised_schemas(ablation, registered))["skyvern_tab_new"]
+
+ assert set(tab_new["properties"]) == {"url", BROWSER_TARGET_PARAM_NAME}
+ assert tab_new["additionalProperties"] is False
+ validator = Draft202012Validator(tab_new)
+ assert validator.is_valid({"url": "https://example.com"})
+ assert not validator.is_valid({"url": "https://example.com", "settle_seconds": 30})
+
+
+async def test_shared_browser_tools_advertise_one_contract_across_both_modes() -> None:
+ registered = await mcp.list_tools(run_middleware=False)
+ normal, ablation = _production_surfaces(registered)
+
+ normal_schemas = await _advertised_schemas(normal, registered)
+ ablation_schemas = await _advertised_schemas(ablation, registered)
+
+ shared = set(normal_schemas) & set(ablation_schemas)
+ assert set(_EXISTING_BROWSER_ALIASES) <= shared
+ assert {name: normal_schemas[name] for name in shared} == {name: ablation_schemas[name] for name in shared}
+ assert all(normal_schemas[name].get("additionalProperties") is False for name in shared)
+
+
def test_browser_ablation_preserves_existing_aliases_then_adds_required_registry_capabilities() -> None:
native, aliases, overlays = _catalogs()
diff --git a/tests/unit/test_copilot_mcp_adapter.py b/tests/unit/test_copilot_mcp_adapter.py
index 2a00ef44f..2a178f4c0 100644
--- a/tests/unit/test_copilot_mcp_adapter.py
+++ b/tests/unit/test_copilot_mcp_adapter.py
@@ -10,14 +10,17 @@ import pytest
from mcp.types import CallToolResult
from structlog.testing import capture_logs
+from skyvern.cli.mcp_tools import mcp
from skyvern.forge.sdk.cache.base import NoopLock
from skyvern.forge.sdk.cache.local import LocalCache
from skyvern.forge.sdk.copilot import mcp_adapter
+from skyvern.forge.sdk.copilot.browser_ablation import CopilotEvalMode, resolve_copilot_tool_surface
from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy
from skyvern.forge.sdk.copilot.mcp_adapter import (
BROWSER_TARGET_PARAM_NAME,
SchemaOverlay,
SkyvernOverlayMCPServer,
+ _apply_schema_overlay,
_BrowserCallOutcome,
_handle_browser_session_loss,
_requested_output_path_choices,
@@ -33,6 +36,7 @@ from skyvern.forge.sdk.copilot.runtime import (
browser_page_custody_lock,
mcp_to_copilot,
)
+from skyvern.forge.sdk.copilot.tools import NATIVE_TOOLS
from skyvern.forge.sdk.copilot.tools.mcp_hooks import _build_skyvern_mcp_overlays, get_skyvern_mcp_alias_map
from skyvern.forge.sdk.copilot.turn_origin import TurnOrigin
from tests.unit.copilot_test_helpers import make_copilot_ctx
@@ -72,6 +76,107 @@ class TestRequestedOutputPathChoices:
def test_a_turn_owing_no_output_leaves_the_schema_alone(self) -> None:
assert _requested_output_path_choices(_schema(), []) == _schema()
+ def test_narrowing_the_path_choices_leaves_the_rest_of_the_contract_intact(self) -> None:
+ source = {**_schema(), "additionalProperties": False}
+
+ schema = _requested_output_path_choices(source, ["output.a"])
+
+ assert schema["additionalProperties"] is False
+ assert schema["properties"]["expression"] == source["properties"]["expression"]
+ assert schema["required"] == source["required"]
+
+
+def _closed_source_schema() -> dict[str, Any]:
+ return {
+ "type": "object",
+ "additionalProperties": False,
+ "description": "Open a page.",
+ "$defs": {"Where": {"type": "string", "enum": ["chat", "last_run"]}},
+ "properties": {
+ "session_id": {"type": "string", "description": "Browser session id."},
+ "url": {"type": "string", "description": "URL to open."},
+ "where": {"$ref": "#/$defs/Where"},
+ },
+ "required": ["url"],
+ }
+
+
+async def _tab_new_overlay() -> SchemaOverlay:
+ surface = resolve_copilot_tool_surface(
+ mode=CopilotEvalMode.BROWSER_ABLATION,
+ native_tools=list(NATIVE_TOOLS),
+ alias_map=get_skyvern_mcp_alias_map(),
+ overlays=_build_skyvern_mcp_overlays(),
+ registered_mcp_tools=await mcp.list_tools(run_middleware=False),
+ )
+ return surface.overlays["skyvern_tab_new"]
+
+
+class TestSchemaOverlayPreservesTheSourceContract:
+ def test_an_empty_overlay_leaves_the_accepted_argument_contract_untouched(self) -> None:
+ source = _closed_source_schema()
+
+ assert _apply_schema_overlay(source, SchemaOverlay()) == source
+
+ @pytest.mark.parametrize(
+ ("overlay", "expected_properties", "expected_required"),
+ [
+ (SchemaOverlay(hide_params=frozenset({"session_id"})), {"url", "where"}, ["url"]),
+ (SchemaOverlay(forced_args={"session_id": "pbs_1"}), {"url", "where"}, ["url"]),
+ (SchemaOverlay(arg_transforms={"target": "where"}), {"session_id", "url", "target"}, ["url"]),
+ (
+ SchemaOverlay(copilot_params={"target": {"type": "string"}}),
+ {"session_id", "url", "where", "target"},
+ ["url"],
+ ),
+ ],
+ )
+ def test_an_overlay_edits_the_fields_it_names_and_nothing_else(
+ self,
+ overlay: SchemaOverlay,
+ expected_properties: set[str],
+ expected_required: list[str],
+ ) -> None:
+ source = _closed_source_schema()
+
+ schema = _apply_schema_overlay(source, overlay)
+
+ assert set(schema["properties"]) == expected_properties
+ assert schema["required"] == expected_required
+ assert schema["additionalProperties"] is False
+ assert schema["$defs"] == source["$defs"]
+ assert schema["description"] == source["description"]
+
+ def test_a_renamed_field_keeps_its_definition_reachable(self) -> None:
+ schema = _apply_schema_overlay(_closed_source_schema(), SchemaOverlay(arg_transforms={"target": "where"}))
+
+ assert schema["properties"]["target"] == {"$ref": "#/$defs/Where"}
+ assert schema["$defs"]["Where"]["enum"] == ["chat", "last_run"]
+
+
+async def test_a_valid_tab_creation_dispatches_only_the_tools_own_arguments() -> None:
+ overlay = await _tab_new_overlay()
+
+ mcp_args = _transform_args({"url": "https://example.com", BROWSER_TARGET_PARAM_NAME: "last_run"}, overlay)
+
+ assert mcp_args == {"url": "https://example.com"}
+
+
+async def test_an_argument_outside_the_contract_is_refused_rather_than_swallowed() -> None:
+ overlay = await _tab_new_overlay()
+
+ mcp_args = _transform_args({"url": "https://example.com", "settle_seconds": 30}, overlay)
+ result = await mcp.call_tool("skyvern_tab_new", mcp_args)
+ payload = result.structured_content
+
+ assert mcp_args["settle_seconds"] == 30
+ assert payload is not None
+ assert payload["ok"] is False
+ assert payload["error"]["message"] == "skyvern_tab_new was called with unsupported argument(s): settle_seconds."
+ assert payload["error"]["details"]["unsupported_arguments"] == ["settle_seconds"]
+
def test_the_declared_path_says_the_expression_is_that_value() -> None:
overlays = _build_skyvern_mcp_overlays(BlockAuthoringPolicy.CODE_ONLY_BROWSER)
diff --git a/uv.lock b/uv.lock
index 05939543b..675e59e44 100644
--- a/uv.lock
+++ b/uv.lock
@@ -6790,7 +6790,7 @@ wheels = [
[[package]]
name = "skyvern"
-version = "1.0.52"
+version = "1.0.51"
source = { editable = "." }
dependencies = [
{ name = "cachetools" },