Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
import asyncio
import base64
import copy
import io
import json
from collections.abc import AsyncIterator, Awaitable, Callable, Iterator
from contextlib import AbstractAsyncContextManager, asynccontextmanager
from dataclasses import replace
from functools import partial
from types import SimpleNamespace
from typing import Any
from unittest.mock import AsyncMock, MagicMock
from urllib.parse import quote, urlparse
import pytest
import yaml
from agents import FunctionTool
from agents.mcp.util import MCPUtil
from agents.tool_context import ToolContext
from fastmcp import FastMCP
from mcp import Tool as MCPTool
from mcp.types import CallToolResult
from PIL import Image
from playwright.async_api import Page, Request, Route
from structlog.testing import capture_logs
from skyvern.cli.core import client as client_module
from skyvern.cli.core.client import get_active_api_key
from skyvern.cli.core.guards import VALID_WAIT_UNTIL
from skyvern.cli.core.result import ErrorCode, make_error
from skyvern.cli.core.session_manager import RequestFailureLog, get_current_session
from skyvern.cli.mcp_tools import mcp
from skyvern.cli.mcp_tools._element_state import action_deadline_error, element_state_error
from skyvern.cli.mcp_tools.blocks import skyvern_block_validate, skyvern_workflow_knowledge
from skyvern.cli.mcp_tools.inspection import ensure_hooks_on_all_pages
from skyvern.cli.mcp_tools.workflow import skyvern_workflow_get
from skyvern.forge.sdk.cache.base import NoopLock
from skyvern.forge.sdk.cache.local import LocalCache
from skyvern.forge.sdk.copilot import agent as copilot_agent
from skyvern.forge.sdk.copilot import mcp_adapter
from skyvern.forge.sdk.copilot import runtime as copilot_runtime
from skyvern.forge.sdk.copilot.browser_ablation import CopilotEvalMode, resolve_copilot_tool_surface
from skyvern.forge.sdk.copilot.config import (
AGENT_BLOCKS_ONLY,
ALL_BLOCK_FAMILIES,
CODE_BLOCKS_ONLY,
AuthoringCapability,
BlockAuthoringPolicy,
CopilotConfig,
)
from skyvern.forge.sdk.copilot.context import USER_FACING_REASON_PARAM
from skyvern.forge.sdk.copilot.credential_fill_fields import CREDENTIAL_FILL_FIELD_NAMES
from skyvern.forge.sdk.copilot.enforcement import CopilotTotalTimeoutError
from skyvern.forge.sdk.copilot.mcp_adapter import (
BROWSER_TARGET_PARAM_NAME,
PageStateReader,
SchemaOverlay,
SkyvernOverlayMCPServer,
_apply_schema_overlay,
_BrowserCallOutcome,
_handle_browser_session_loss,
_requested_output_path_choices,
_transform_args,
resolve_browser_session_binding,
)
from skyvern.forge.sdk.copilot.model_input_capture import serialize_tool_surface
from skyvern.forge.sdk.copilot.output_utils import _RECENT_TOOL_OUTPUT_CHAR_CAP, MCP_RESULT_PROVENANCE_KEY
from skyvern.forge.sdk.copilot.request_policy import RequestPolicy
from skyvern.forge.sdk.copilot.runtime import (
AgentContext,
CopilotBrowserLivenessUndetermined,
CopilotBrowserSessionUnavailable,
DemonstratedAction,
OriginRunRedactionRegistry,
bound_call_browser_session,
browser_session_lock,
mcp_to_copilot,
staged_demonstrated_action,
)
from skyvern.forge.sdk.copilot.screenshot_utils import ScreenshotProvenance, enqueue_screenshot
from skyvern.forge.sdk.copilot.secret_scrub import (
clear_session_scrub_values,
register_secret_scrub_value,
register_secret_scrub_values_from_structure,
)
from skyvern.forge.sdk.copilot.tools import NATIVE_TOOLS, _with_action_reason, mcp_hooks, run_execution
from skyvern.forge.sdk.copilot.tools import scouting as scouting_module
from skyvern.forge.sdk.copilot.tools._shared import _composition_get_structured_evidence_result, on_working_page
from skyvern.forge.sdk.copilot.tools.banned_blocks import _block_authoring_violations
from skyvern.forge.sdk.copilot.tools.mcp_hooks import (
_FOR_LOOP_EXAMPLE,
_FOR_LOOP_GUIDANCE,
_FOR_LOOP_PROPERTY_DESCRIPTIONS,
_TASK_UNAVAILABLE_KNOWLEDGE_TOPIC,
_build_skyvern_mcp_overlays,
get_skyvern_mcp_alias_map,
)
from skyvern.forge.sdk.copilot.turn_origin import TurnOrigin
from skyvern.forge.sdk.workflow.models.workflow import WorkflowRunStatus
from skyvern.webeye.persistent_sessions_manager import (
BrowserOperation,
BrowserRetirement,
BrowserRetirementReason,
)
from tests.unit._mcp_browser_fakes import (
logged_request_failures,
make_request_failure,
make_session_state,
seed_request_failures,
)
from tests.unit._socket_closing_fixture import BODY_SENTINEL, socket_closing_fixture_page
from tests.unit.copilot_test_helpers import (
AUTO_RENDER_TURNSTILE_HTML,
CATALOG_PAGE_HTML,
FIXTURE_SITE_ORIGIN,
HIDDEN_TURNSTILE_HELPER_HTML,
FakeMCPServerManager,
FakeTabbedBrowserState,
challenge_browser_page,
make_copilot_ctx,
page_with_unprobeable_vendor_embedder,
patch_browser_tabs,
redact_parameter_values,
skip_no_browser,
)
from tests.unit.test_copilot_secret_scrub import _FakeClient, _FakeRawResult, _make_server
def _schema() -> dict:
return {
"type": "object",
"properties": {
"expression": {"type": "string"},
"output_path": {"type": "string", "description": "The requested output this read fills."},
},
"required": ["expression"],
}
def test_scrub_tool_result_redacts_encoded_matching_origin_values(monkeypatch: pytest.MonkeyPatch) -> None:
secret = "[email protected]?token=one&next=/done"
encoded = quote(secret, safe="")
seen_parameters: list[dict[str, Any]] = []
def redact(value: Any, parameters: dict[str, Any], **_budget: object) -> Any:
seen_parameters.append(parameters)
def walk(node: Any) -> Any:
if isinstance(node, str):
return node.replace(encoded, "[redacted]")
if isinstance(node, dict):
return {key: walk(item) for key, item in node.items()}
return node
return walk(value)
ctx = make_copilot_ctx()
ctx.last_run_blocks_workflow_run_id = "wr_sensitive"
ctx.origin_run_redaction_registry = OriginRunRedactionRegistry(
"wr_sensitive",
{"magic_link": secret},
contains_sensitive_values=True,
contains_all_sensitive_values=True,
)
monkeypatch.setattr(
mcp_adapter.app,
"AGENT_FUNCTION",
SimpleNamespace(redact_codeblock_parameter_values=redact),
)
scrubbed = mcp_adapter.scrub_model_facing_tool_result(
ctx,
{"ok": True, "data": {"url": f"https://example.test/callback?payload={encoded}"}},
)
assert scrubbed["ok"] is True
assert secret not in str(scrubbed)
assert encoded not in str(scrubbed)
assert "[redacted]" in str(scrubbed)
assert seen_parameters == [{"magic_link": secret}]
def test_a_driver_navigation_code_survives_a_scrubbed_value_that_spells_part_of_it() -> None:
"""Workflow parameter values are registered for scrubbing with no minimum length, so "net" is enough
to corrupt the token before it is lifted, and a proxy outage then reads as a target failure."""
ctx = make_copilot_ctx()
register_secret_scrub_value(ctx, "net")
raw = {
"ok": False,
"error": {
"code": "ACTION_FAILED",
"message": "Page.goto: net::ERR_TUNNEL_CONNECTION_FAILED at https://x.test/",
"hint": "",
"details": {"nav_error_code": "net::ERR_TUNNEL_CONNECTION_FAILED"},
},
}
flattened = mcp_to_copilot(mcp_adapter.scrub_model_facing_tool_result(ctx, raw, tool_name="skyvern_navigate"))
assert flattened["nav_error_code"] == "net::ERR_TUNNEL_CONNECTION_FAILED"
assert "net::" not in flattened["error"]
# The flattened shape is scrubbed again on its way to the model.
rescrubbed = mcp_adapter.scrub_model_facing_tool_result(ctx, flattened, tool_name="skyvern_navigate")
assert rescrubbed["nav_error_code"] == flattened["nav_error_code"]
register_secret_scrub_value(ctx, "target")
fail_closed = mcp_adapter.scrub_model_facing_tool_result(
ctx, {**flattened, "target": {"page": 0}}, tool_name="skyvern_navigate"
)
assert mcp_adapter.is_redaction_withheld(fail_closed) and fail_closed["ok"] is False
assert "nav_error_code" not in fail_closed and "ERR_TUNNEL" not in str(fail_closed)
@pytest.mark.parametrize("size_is_registered", [False, True])
def test_a_withheld_result_keeps_its_disposition_across_rescrubs_and_logs_once(size_is_registered: bool) -> None:
ctx = make_copilot_ctx()
register_secret_scrub_value(ctx, "data")
raw = {"ok": True, "data": {"result": "signed-in page text"}}
size = len(json.dumps(raw, ensure_ascii=False))
if size_is_registered:
register_secret_scrub_value(ctx, str(size))
outcome = mcp_adapter._browser_call_outcome_from_mapping(
raw_tool_name="skyvern_evaluate", source_browser_session_id=None, raw_result=raw
)
with capture_logs() as logs:
first = mcp_adapter.scrub_model_facing_tool_result(ctx, raw, tool_name="skyvern_evaluate")
withheld_outcome = outcome.with_raw_result(first)
projected = mcp_adapter._project_browser_call_outcome(withheld_outcome, display_tool_name="evaluate")
second = mcp_adapter.scrub_model_facing_tool_result(ctx, projected, tool_name="skyvern_evaluate")
assert first == second and mcp_adapter.is_redaction_withheld(second)
assert (withheld_outcome.ok, withheld_outcome.payload_omitted, withheld_outcome.error_kind) == (False, True, "tool")
assert "signed-in page text" not in json.dumps([first, projected, second])
assert (str(size) in json.dumps(second)) is not size_is_registered
assert [log["event"] for log in logs if log["event"] == "copilot_model_facing_result_withheld"] == [
"copilot_model_facing_result_withheld"
]
@pytest.mark.asyncio
@pytest.mark.usefixtures("_stub_browser_session")
async def test_structured_evidence_reports_a_withheld_read_by_its_reason() -> None:
ctx = make_copilot_ctx(browser_session_id="pbs_withheld_evidence")
register_secret_scrub_value(ctx, "data")
ctx.discovery_mcp_server = _make_server(
ctx,
{"ok": True, "data": {"result": json.dumps({"text": "signed-in page text"})}},
SchemaOverlay(requires_browser=True),
alias_map={"evaluate": "skyvern_evaluate"},
)
try:
evidence, error = await _composition_get_structured_evidence_result(
ctx, inspected_url="https://example.test/", current_url="https://example.test/"
)
finally:
clear_session_scrub_values(ctx.browser_session_id)
assert evidence is None
assert error is not None and error.startswith("structured page evidence was withheld: Result withheld")
assert "could not be redacted safely" in error and "too large" not in error
assert "returned an error" not in error and "signed-in" not in error
@pytest.mark.parametrize(
("code", "registered", "tool_name", "survives"),
[
# Every code the driver writes reaches the model intact, not only the ones ownership reads.
pytest.param("net::ERR_CONNECTION_REFUSED", "net", "skyvern_navigate", True, id="a_code_holding_a_value"),
pytest.param("net::ERR_TUNNEL_CONNECTION_FAILED", "net", "skyvern_navigate", True, id="a_proxy_code"),
# Only the navigation tool reports a code. Another tool echoing an argument into the field
# would otherwise have the whole token written back over the scrub.
pytest.param("net::ERR_HUNTER2", "HUNTER2", "skyvern_evaluate", False, id="another_tool_echoed_a_secret"),
pytest.param(
"net::ERR_CERT_HUNTER2", "net::ERR_CERT_HUNTER2", "skyvern_navigate", False, id="the_code_is_the_secret"
),
pytest.param("net::ERR_TUNNEL_CONNECTION_FAILED", "net", None, False, id="a_result_with_no_provenance"),
],
)
def test_the_model_facing_scrub_keeps_driver_codes_only_where_the_driver_reports_them(
code: str, registered: str, tool_name: str | None, survives: bool
) -> None:
ctx = make_copilot_ctx()
register_secret_scrub_value(ctx, registered)
scrubbed = mcp_adapter.scrub_model_facing_tool_result(
ctx, {"ok": False, "nav_error_code": code}, tool_name=tool_name
)
assert (scrubbed.get("nav_error_code") == code) is survives
assert (registered in str(scrubbed)) is (survives and registered in code)
@pytest.mark.parametrize(
("parameter_value", "survives"),
[
# The run's redaction parameters are a second vocabulary the scrub applies. A parameter that
# IS a code would be redacted by that pass and written back whole by the restore.
pytest.param("net::ERR_TUNNEL_CONNECTION_FAILED", False, id="a_parameter_that_is_the_code"),
# An ordinary parameter occurring inside a code must still not cost the model the code.
pytest.param("net", True, id="a_parameter_inside_the_code"),
],
)
def test_the_model_facing_scrub_reads_the_run_parameters_before_restoring_a_code(
monkeypatch: pytest.MonkeyPatch, parameter_value: str, survives: bool
) -> None:
"""A redaction parameter never registered with the exact-value scrubber is still redacted, so the
restore has to consult it too or it hands back what the parameter pass was hiding."""
def redact(value: object, parameters: dict[str, object], **_budget: object) -> object:
secret = str(next(iter(parameters.values())))
def walk(node: object) -> object:
if isinstance(node, str):
return node.replace(secret, "[redacted]")
if isinstance(node, dict):
return {key: walk(item) for key, item in node.items()}
if isinstance(node, list):
return [walk(item) for item in node]
return node
return walk(value)
monkeypatch.setattr(mcp_adapter.app.AGENT_FUNCTION, "redact_codeblock_parameter_values", redact)
code = "net::ERR_TUNNEL_CONNECTION_FAILED"
ctx = make_copilot_ctx()
ctx.codeblock_redaction_parameters = {"site": parameter_value}
scrubbed = mcp_adapter.scrub_model_facing_tool_result(
ctx, {"ok": False, "nav_error_code": code}, tool_name="skyvern_navigate"
)
assert (scrubbed.get("nav_error_code") == code) is survives
assert (parameter_value in str(scrubbed)) is survives
class TestRequestedOutputPathChoices:
def test_the_turn_s_requested_paths_become_the_choices(self) -> None:
# Live shape (SKY-13226): a free-form path let the model name its own purpose, so the read
# that saw the requested quantity was filed as exploration and never witnessed the path.
schema = _requested_output_path_choices(_schema(), ["output.azure_error_count"])
assert schema["properties"]["output_path"]["enum"] == ["output.azure_error_count"]
assert "output.azure_error_count" in schema["properties"]["output_path"]["description"]
def test_exploration_still_passes_by_omitting_the_path(self) -> None:
schema = _requested_output_path_choices(_schema(), ["output.azure_error_count"])
assert "output_path" not in schema["required"]
def test_every_requested_path_stays_available_for_a_reread(self) -> None:
schema = _requested_output_path_choices(_schema(), ["output.a", "output.b"])
assert schema["properties"]["output_path"]["enum"] == ["output.a", "output.b"]
def test_a_turn_owing_no_output_leaves_the_schema_alone(self) -> None:
assert _requested_output_path_choices(_schema(), []) == _schema()
def test_narrowing_the_path_choices_leaves_the_rest_of_the_contract_intact(self) -> None:
source = {**_schema(), "additionalProperties": False}
schema = _requested_output_path_choices(source, ["output.a"])
assert schema["additionalProperties"] is False
assert schema["properties"]["expression"] == source["properties"]["expression"]
assert schema["required"] == source["required"]
def _closed_source_schema() -> dict[str, Any]:
return {
"type": "object",
"additionalProperties": False,
"description": "Open a page.",
"$defs": {"Where": {"type": "string", "enum": ["chat", "last_run"]}},
"properties": {
"session_id": {"type": "string", "description": "Browser session id."},
"url": {"type": "string", "description": "URL to open."},
"where": {"$ref": "#/$defs/Where"},
},
"required": ["url"],
}
async def _tab_new_overlay() -> SchemaOverlay:
surface = resolve_copilot_tool_surface(
mode=CopilotEvalMode.BROWSER_ABLATION,
native_tools=list(NATIVE_TOOLS),
alias_map=get_skyvern_mcp_alias_map(),
overlays=_build_skyvern_mcp_overlays(),
registered_mcp_tools=await mcp.list_tools(run_middleware=False),
)
return surface.overlays["skyvern_tab_new"]
class TestSchemaOverlayPreservesTheSourceContract:
def test_an_empty_overlay_leaves_the_accepted_argument_contract_untouched(self) -> None:
source = _closed_source_schema()
assert _apply_schema_overlay(source, SchemaOverlay()) == source
@pytest.mark.parametrize(
("overlay", "expected_properties", "expected_required"),
[
(SchemaOverlay(hide_params=frozenset({"session_id"})), {"url", "where"}, ["url"]),
(SchemaOverlay(forced_args={"session_id": "pbs_1"}), {"url", "where"}, ["url"]),
(SchemaOverlay(arg_transforms={"target": "where"}), {"session_id", "url", "target"}, ["url"]),
(
SchemaOverlay(copilot_params={"target": {"type": "string"}}),
{"session_id", "url", "where", "target"},
["url"],
),
(
SchemaOverlay(param_patches={"url": {"enum": ["https://a.test"]}}),
{"session_id", "url", "where"},
["url"],
),
],
)
def test_an_overlay_edits_the_fields_it_names_and_nothing_else(
self,
overlay: SchemaOverlay,
expected_properties: set[str],
expected_required: list[str],
) -> None:
source = _closed_source_schema()
schema = _apply_schema_overlay(source, overlay)
assert set(schema["properties"]) == expected_properties
assert schema["required"] == expected_required
assert schema["additionalProperties"] is False
assert schema["$defs"] == source["$defs"]
assert schema["description"] == source["description"]
def test_a_renamed_field_keeps_its_definition_reachable(self) -> None:
schema = _apply_schema_overlay(_closed_source_schema(), SchemaOverlay(arg_transforms={"target": "where"}))
assert schema["properties"]["target"] == {"$ref": "#/$defs/Where"}
assert schema["$defs"]["Where"]["enum"] == ["chat", "last_run"]
def test_a_param_patch_overrides_the_keys_it_names_and_keeps_the_rest(self) -> None:
overlay = SchemaOverlay(param_patches={"url": {"enum": ["https://a.test"]}, "absent": {"type": "string"}})
schema = _apply_schema_overlay(_closed_source_schema(), overlay)
assert schema["properties"]["url"] == {
"type": "string",
"description": "URL to open.",
"enum": ["https://a.test"],
}
assert "absent" not in schema["properties"]
@pytest.mark.asyncio
async def test_a_patched_shared_param_exists_on_the_tool_and_still_reaches_it() -> None:
registered = {tool.name: tool for tool in await mcp.list_tools(run_middleware=False)}
alias_map = get_skyvern_mcp_alias_map()
patched = [(name, overlay) for name, overlay in _build_skyvern_mcp_overlays().items() if overlay.param_patches]
assert patched
for name, overlay in patched:
source = registered[alias_map[name]].parameters
for param in overlay.param_patches:
assert param in source["properties"], (name, param)
assert _transform_args({param: "value"}, overlay)[param] == "value"
@pytest.mark.asyncio
async def test_a_valid_tab_creation_dispatches_only_the_tools_own_arguments() -> None:
overlay = await _tab_new_overlay()
mcp_args = _transform_args({"url": "https://example.com", BROWSER_TARGET_PARAM_NAME: "last_run"}, overlay)
assert mcp_args == {"url": "https://example.com"}
@pytest.mark.asyncio
async def test_an_argument_outside_the_contract_is_refused_rather_than_swallowed() -> None:
overlay = await _tab_new_overlay()
mcp_args = _transform_args({"url": "https://example.com", "settle_seconds": 30}, overlay)
result = await mcp.call_tool("skyvern_tab_new", mcp_args)
payload = result.structured_content
assert mcp_args["settle_seconds"] == 30
assert payload is not None
assert payload["ok"] is False
assert payload["error"]["message"] == "skyvern_tab_new was called with unsupported argument(s): settle_seconds."
assert payload["error"]["details"]["unsupported_arguments"] == ["settle_seconds"]
def test_the_declared_path_says_the_expression_is_that_value() -> None:
overlays = _build_skyvern_mcp_overlays(BlockAuthoringPolicy.CODE_ONLY_BROWSER)
description = overlays["evaluate"].copilot_params["output_path"]["description"]
assert "evaluates to that one value" in description
assert "exploration" in description
_TIMING_EVENT = "MCP tool timing"
_PAYLOAD_KEYS_THAT_WOULD_LEAK = {"args", "merged_args", "arguments", "mcp_args", "data", "raw_mcp", "result"}
_BROWSER_BOOT_SECONDS = 2.0
_MCP_CALL_SECONDS = 3.0
_AFTER_CALL_SECONDS = 5.0
_WALL_MS = 5000
_SESSION_ONLY_MS = 2000
_MCP_CALL_MS = 3000
_CONTEXT_ENTER_SECONDS = 4.0
_CONTEXT_EXIT_SECONDS = 1.0
_CONTEXT_ENTER_MS = 4000
_CONTEXT_EXIT_MS = 1000
_GAP_WALL_MS = 10000
_ACTION_OVERLAY_CASES = ["click", "type_text", "select_option", "click_no_browser"]
_ACTION_ARGS = {
"click": {"selector": "#go"},
"type_text": {"selector": "#q", "text": "hello"},
"select_option": {"selector": "#s", "value": "a"},
}
_PHASE_KEYS = (
"phase_session_prepare_ms",
"phase_context_enter_ms",
"phase_dispatch_ms",
"phase_context_exit_ms",
"phase_residual_ms",
)
def _assert_every_millisecond_is_attributed(record: dict[str, Any]) -> None:
"""The residual is the remainder, so its bound is what proves the segments were charged correctly.
Summing the segments and the residual is an identity that holds for any values whatsoever.
"""
assert sum(record[key] for key in _PHASE_KEYS) == record["wall_clock_ms"]
assert 0 <= record["phase_residual_ms"] < len(mcp_adapter._MCP_CALL_SEGMENTS)
class _SharedLocalCache(LocalCache):
is_shared = True
def __init__(self) -> None:
super().__init__()
self.lock_names: list[str] = []
def get_lock(self, lock_name: str, blocking_timeout: int = 5, timeout: int = 10) -> NoopLock:
self.lock_names.append(lock_name)
return NoopLock(lock_name, blocking_timeout, timeout)
@pytest.fixture
def _fake_clock(monkeypatch: pytest.MonkeyPatch) -> Iterator[list[float]]:
now = [100.0]
monkeypatch.setattr(mcp_adapter, "time", SimpleNamespace(monotonic=lambda: now[0]))
yield now
@pytest.fixture
def _stub_browser_session(monkeypatch: pytest.MonkeyPatch, _fake_clock: list[float]) -> Iterator[None]:
async def _no_error(_ctx: AgentContext, **_kwargs: Any) -> None:
return None
@asynccontextmanager
async def _no_session_scope(_ctx: AgentContext, *, session_id_override: str | None = None) -> AsyncIterator[None]:
_fake_clock[0] += _BROWSER_BOOT_SECONDS
yield
monkeypatch.setattr(mcp_adapter, "ensure_browser_session", _no_error)
monkeypatch.setattr(mcp_adapter, "mcp_browser_context", _no_session_scope)
yield
def _install_timed_context(monkeypatch: pytest.MonkeyPatch, clock: list[float]) -> None:
@asynccontextmanager
async def _scope(_ctx: AgentContext, *, session_id_override: str | None = None) -> AsyncIterator[None]:
clock[0] += _BROWSER_BOOT_SECONDS + _CONTEXT_ENTER_SECONDS
try:
yield
finally:
clock[0] += _CONTEXT_EXIT_SECONDS
monkeypatch.setattr(mcp_adapter, "mcp_browser_context", _scope)
def _browser_server(on_call: Callable[[], None]) -> SkyvernOverlayMCPServer:
return _make_server(
make_copilot_ctx(browser_session_id="pbs_1"),
{"ok": True},
SchemaOverlay(requires_browser=True),
alias_map=get_skyvern_mcp_alias_map(),
on_call=on_call,
)
def _timing_records(captured: list[dict[str, Any]]) -> list[dict[str, Any]]:
return [record for record in captured if record.get("event") == _TIMING_EVENT]
def _browser_outcome_records(captured: list[dict[str, Any]]) -> list[dict[str, Any]]:
return [record for record in captured if record.get("event") == "copilot_browser_call_outcome"]
@pytest.mark.asyncio
async def test_generation_retirement_reports_replacement_without_expiring_the_session(
monkeypatch: pytest.MonkeyPatch,
) -> None:
ctx = make_copilot_ctx(browser_session_id="pbs_replaced")
async def _prepared(_ctx: AgentContext, **_kwargs: Any) -> tuple[None, None, None]:
return None, None, None
@asynccontextmanager
async def _retired(_ctx: AgentContext, **_kwargs: Any) -> AsyncIterator[None]:
raise mcp_adapter.CopilotBrowserGenerationRetired("pbs_replaced")
yield
monkeypatch.setattr(mcp_adapter, "_prepare_browser_session_for_dispatch", _prepared)
monkeypatch.setattr(mcp_adapter, "mcp_browser_context", _retired)
server = _make_server(ctx, {"ok": True}, SchemaOverlay(requires_browser=True))
result = await server.call_tool("evaluate", {})
surfaced = json.loads(result.content[0].text)
assert surfaced["error_code"] == "BROWSER_GENERATION_RETIRED"
assert surfaced["data"]["browser_session_continuity"] == {
"source": "direct_mcp",
"disposition": "generation_retired",
"cause": "connection_generation_retired",
"fresh_state_required": True,
"prior_action_effect": "unknown",
}
@pytest.mark.asyncio
async def test_terminal_retirement_uses_session_loss_recovery(
monkeypatch: pytest.MonkeyPatch,
) -> None:
ctx = make_copilot_ctx(browser_session_id="pbs_closed")
retired_error = mcp_adapter.CopilotBrowserGenerationRetired(
"pbs_closed",
BrowserRetirementReason.session_ending,
)
async def _prepared(_ctx: AgentContext, **_kwargs: Any) -> tuple[None, None, None]:
return None, None, None
@asynccontextmanager
async def _retired(_ctx: AgentContext, **_kwargs: Any) -> AsyncIterator[None]:
raise retired_error
yield
handle_loss = AsyncMock(return_value="failed")
monkeypatch.setattr(mcp_adapter, "_prepare_browser_session_for_dispatch", _prepared)
monkeypatch.setattr(mcp_adapter, "mcp_browser_context", _retired)
monkeypatch.setattr(mcp_adapter, "_handle_browser_session_loss", handle_loss)
server = _make_server(ctx, {"ok": True}, SchemaOverlay(requires_browser=True))
result = await server.call_tool("evaluate", {})
surfaced = json.loads(result.content[0].text)
assert surfaced["error_code"] == "SESSION_EXPIRED"
handle_loss.assert_awaited_once_with(
ctx,
tool_name="evaluate",
call_path="model",
lost_session_id="pbs_closed",
)
@pytest.mark.asyncio
async def test_browser_post_hook_finishes_inside_the_exact_generation(
monkeypatch: pytest.MonkeyPatch,
) -> None:
ctx = make_copilot_ctx(browser_session_id="pbs_post_hook")
scope_active = False
post_hook_saw_scope = False
async def _prepared(_ctx: AgentContext, **_kwargs: Any) -> tuple[None, None, None]:
return None, None, None
@asynccontextmanager
async def _scope(_ctx: AgentContext, **_kwargs: Any) -> AsyncIterator[None]:
nonlocal scope_active
scope_active = True
try:
yield
finally:
scope_active = False
async def _post_hook(
copilot_result: dict[str, Any], _raw_mcp: dict[str, Any], _ctx: AgentContext
) -> dict[str, Any]:
nonlocal post_hook_saw_scope
post_hook_saw_scope = scope_active
return copilot_result
monkeypatch.setattr(mcp_adapter, "_prepare_browser_session_for_dispatch", _prepared)
monkeypatch.setattr(mcp_adapter, "mcp_browser_context", _scope)
server = _make_server(
ctx,
{"ok": True},
SchemaOverlay(requires_browser=True, post_hook=_post_hook),
)
result = await server.call_tool("evaluate", {})
assert result.isError is False
assert post_hook_saw_scope is True
assert scope_active is False
@pytest.mark.asyncio
async def test_retirement_after_post_hook_rolls_back_context_and_defers_success_evidence(
monkeypatch: pytest.MonkeyPatch,
) -> None:
ctx = make_copilot_ctx(browser_session_id="pbs_retired_post_hook")
original_evidence = {"step": 1, "evidence": {"source_tool": "before"}}
ctx.flow_evidence = [original_evidence]
async def _prepared(_ctx: AgentContext, **_kwargs: Any) -> tuple[None, None, None]:
return None, None, None
@asynccontextmanager
async def _scope(_ctx: AgentContext, **_kwargs: Any) -> AsyncIterator[None]:
yield
raise mcp_adapter.CopilotBrowserGenerationRetired("pbs_retired_post_hook")
async def _post_hook(
copilot_result: dict[str, Any], _raw_mcp: dict[str, Any], hook_ctx: AgentContext
) -> dict[str, Any]:
hook_ctx.flow_evidence.append({"step": 2, "evidence": {"source_tool": "evaluate"}})
await asyncio.sleep(0)
return copilot_result
record_outcome = MagicMock()
monkeypatch.setattr(mcp_adapter, "_prepare_browser_session_for_dispatch", _prepared)
monkeypatch.setattr(mcp_adapter, "mcp_browser_context", _scope)
monkeypatch.setattr(mcp_adapter, "_record_browser_call_outcome", record_outcome)
server = _make_server(
ctx,
{"ok": True},
SchemaOverlay(requires_browser=True, post_hook=_post_hook),
alias_map={"evaluate": "skyvern_evaluate"},
)
result = await server.call_tool("evaluate", {})
assert result.isError is True
assert ctx.flow_evidence == [original_evidence]
assert record_outcome.call_count == 1
assert record_outcome.call_args.args[1].ok is False
@pytest.mark.asyncio
async def test_retirement_rollback_preserves_concurrent_sibling_evidence(
monkeypatch: pytest.MonkeyPatch,
) -> None:
ctx = make_copilot_ctx(browser_session_id="pbs_shared_context")
ctx.flow_evidence = []
allow_retiring_dispatch = asyncio.Event()
retiring_dispatch_started = asyncio.Event()
async def _prepared(_ctx: AgentContext, **_kwargs: Any) -> tuple[None, None, None]:
return None, None, None
@asynccontextmanager
async def _scope(_ctx: AgentContext, **_kwargs: Any) -> AsyncIterator[None]:
yield
if asyncio.current_task() is retiring_task:
raise mcp_adapter.CopilotBrowserGenerationRetired("pbs_shared_context")
class _ConcurrentClient:
async def call_tool(
self,
_name: str,
_args: dict[str, Any],
raise_on_error: bool = False,
) -> Any:
if asyncio.current_task() is retiring_task:
retiring_dispatch_started.set()
await allow_retiring_dispatch.wait()
tag = "retiring"
else:
tag = "sibling"
return SimpleNamespace(structured_content={"ok": True, "tag": tag}, is_error=False, content=[])
async def _post_hook(
copilot_result: dict[str, Any], raw_mcp: dict[str, Any], hook_ctx: AgentContext
) -> dict[str, Any]:
hook_ctx.flow_evidence.append({"source": raw_mcp["tag"]})
await asyncio.sleep(0)
return copilot_result
monkeypatch.setattr(mcp_adapter, "_prepare_browser_session_for_dispatch", _prepared)
monkeypatch.setattr(mcp_adapter, "mcp_browser_context", _scope)
server = _make_server(
ctx,
{"ok": True},
SchemaOverlay(requires_browser=True, post_hook=_post_hook),
)
server._client = _ConcurrentClient()
retiring_task = asyncio.create_task(server._call_tool("evaluate", {}))
await retiring_dispatch_started.wait()
sibling_task = asyncio.create_task(server._call_tool("evaluate", {}))
await asyncio.sleep(0)
allow_retiring_dispatch.set()
results = await asyncio.gather(retiring_task, sibling_task, return_exceptions=True)
assert isinstance(results[0], CallToolResult)
assert results[0].isError is True
assert not isinstance(results[1], BaseException)
assert ctx.flow_evidence == [{"source": "sibling"}]
def _attach_mocked_browser_session(
monkeypatch: pytest.MonkeyPatch, working_page: Page | None
) -> tuple[MagicMock, list[str]]:
browser = MagicMock()
browser.is_connected.return_value = True
browser_context = MagicMock()
browser_context.browser = browser
browser_context._impl_obj = SimpleNamespace(_close_was_called=False, _closed=False)
browser_state = MagicMock(browser_context=browser_context)
browser_state.get_working_page = AsyncMock(return_value=working_page)
operation_entries: list[str] = []
@asynccontextmanager
async def _operation(_session_id: str, resolved_state: Any) -> AsyncIterator[BrowserOperation]:
operation_entries.append(_session_id)
yield BrowserOperation(resolved_state, BrowserRetirement())
manager = MagicMock()
manager.get_browser_state = AsyncMock(return_value=browser_state)
manager.browser_operation = _operation
monkeypatch.setattr(copilot_runtime.app, "PERSISTENT_SESSIONS_MANAGER", manager)
monkeypatch.setattr(copilot_runtime, "get_skyvern", MagicMock(return_value=MagicMock()))
monkeypatch.setattr(copilot_runtime, "SkyvernBrowser", MagicMock(return_value=MagicMock()))
monkeypatch.setattr(copilot_runtime, "get_active_api_key", MagicMock(return_value="test-api-key"))
monkeypatch.setattr(copilot_runtime, "set_api_key_override", MagicMock(return_value=object()))
monkeypatch.setattr(copilot_runtime, "reset_api_key_override", MagicMock())
async def _prepared(_ctx: AgentContext, **_kwargs: Any) -> tuple[None, None, None]:
return None, None, None
monkeypatch.setattr(mcp_adapter, "_prepare_browser_session_for_dispatch", _prepared)
return manager, operation_entries
@pytest.mark.asyncio
async def test_browser_pre_hook_internal_read_reuses_the_outer_generation(
monkeypatch: pytest.MonkeyPatch,
) -> None:
ctx = make_copilot_ctx(browser_session_id="pbs_nested_hook")
ctx.api_key = "test-api-key"
manager, operation_entries = _attach_mocked_browser_session(monkeypatch, None)
register = MagicMock()
unregister = MagicMock()
monkeypatch.setattr(copilot_runtime, "register_copilot_session", register)
monkeypatch.setattr(copilot_runtime, "unregister_copilot_session", unregister)
server: SkyvernOverlayMCPServer
async def _pre_hook(_args: dict[str, Any], _ctx: AgentContext) -> None:
internal = await server.call_internal_tool("skyvern_evaluate", {"expression": "source()"})
assert internal["ok"] is True
server = _make_server(
ctx,
{"ok": True, "data": {"result": "done"}},
SchemaOverlay(requires_browser=True, pre_hook=_pre_hook),
alias_map={"evaluate": "skyvern_evaluate"},
)
result = await server.call_tool("evaluate", {"expression": "action()"})
assert result.isError is False
assert len(operation_entries) == 1
manager.get_browser_state.assert_awaited_once()
register.assert_called_once()
unregister.assert_called_once()
async def _call(server: SkyvernOverlayMCPServer, call_path: str) -> CallToolResult | dict[str, Any]:
if call_path == "model":
return await server.call_tool("evaluate", {"expression": "scan()"})
return await server.call_internal_tool("skyvern_evaluate", {"expression": "scan()"})
def _surfaced_error(result: CallToolResult | dict[str, Any], call_path: str) -> str:
if call_path == "model":
assert isinstance(result, CallToolResult)
return result.content[0].text
assert isinstance(result, dict)
return str(result.get("error"))
@pytest.mark.asyncio
async def test_a_browser_session_lock_does_not_block_an_unrelated_browser_session() -> None:
ctx = make_copilot_ctx(browser_session_id="pbs-debug")
run_lock = browser_session_lock(ctx, session_id="pbs-run")
debug_lock = browser_session_lock(ctx, session_id="pbs-debug")
await run_lock.acquire()
try:
await asyncio.wait_for(debug_lock.acquire(), timeout=0.1)
debug_lock.release()
assert browser_session_lock(ctx, session_id="pbs-run") is run_lock
assert browser_session_lock(ctx, session_id="pbs-debug") is debug_lock
finally:
run_lock.release()
def _server_whose_call_takes_time(
payload: dict[str, Any] | Exception,
overlay: SchemaOverlay,
clock: list[float],
alias_map: dict[str, str] | None = None,
is_error: bool = False,
browser_session_id: str | None = "pbs_1",
) -> SkyvernOverlayMCPServer:
def _advance() -> None:
clock[0] += _MCP_CALL_SECONDS
return _make_server(
make_copilot_ctx(browser_session_id=browser_session_id),
payload,
overlay,
alias_map=alias_map,
on_call=_advance,
is_error=is_error,
)
@pytest.mark.asyncio
async def test_internal_call_preserves_explicit_session_across_session_prepare(
monkeypatch: pytest.MonkeyPatch,
) -> None:
ctx = make_copilot_ctx(browser_session_id="pbs_snapshot")
server = _make_server(
ctx,
{"ok": True},
SchemaOverlay(requires_browser=True),
alias_map=get_skyvern_mcp_alias_map(),
)
dispatched: list[dict[str, Any]] = []
class _CapturingClient:
async def call_tool(self, name: str, args: dict[str, Any], raise_on_error: bool = False) -> Any:
dispatched.append(dict(args))
return SimpleNamespace(
structured_content={"ok": True},
is_error=False,
content=[],
)
async def _replace_ambient_session(_ctx: AgentContext, **_kwargs: Any) -> tuple[None, None, None]:
_ctx.browser_session_id = "pbs_replacement"
return None, None, None
@asynccontextmanager
async def _scope(_ctx: AgentContext, *, session_id_override: str | None = None) -> AsyncIterator[None]:
yield
server._client = _CapturingClient()
monkeypatch.setattr(mcp_adapter, "_prepare_browser_session_for_dispatch", _replace_ambient_session)
monkeypatch.setattr(mcp_adapter, "mcp_browser_context", _scope)
result = await server.call_internal_tool(
"skyvern_evaluate",
{"expression": "scan()", "session_id": "pbs_snapshot"},
)
assert result["ok"] is True
assert dispatched == [{"expression": "scan()", "session_id": "pbs_snapshot"}]
assert ctx.browser_session_id == "pbs_replacement"
def _whoami_server(ctx: AgentContext, seen: list[str | None]) -> SkyvernOverlayMCPServer:
probe = FastMCP("whoami-probe")
@probe.tool()
async def skyvern_whoami() -> dict[str, Any]:
seen.append(get_active_api_key())
return {"ok": True, "data": {}}
return SkyvernOverlayMCPServer(
transport=probe,
overlays={"whoami": SchemaOverlay()},
alias_map={"whoami": "skyvern_whoami"},
allowlist=frozenset({"skyvern_whoami"}),
context_provider=lambda: ctx,
)
@pytest.mark.asyncio
async def test_non_browser_tool_body_runs_with_the_org_api_key(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(client_module.settings, "SKYVERN_API_KEY", "sk-server-default")
seen: list[str | None] = []
server = _whoami_server(make_copilot_ctx(api_key="sk-copilot-org"), seen)
await server.connect()
try:
assert get_active_api_key() == "sk-server-default"
result = await server.call_tool("whoami", {})
finally:
await server.cleanup()
assert result.isError is False
assert seen == ["sk-copilot-org"]
assert get_active_api_key() == "sk-server-default"
@pytest.mark.asyncio
async def test_connect_without_an_api_key_never_reaches_a_tool(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(client_module.settings, "SKYVERN_API_KEY", "sk-server-default")
seen: list[str | None] = []
server = _whoami_server(make_copilot_ctx(api_key=None), seen)
with pytest.raises(RuntimeError, match="missing api_key"):
await server.connect()
assert server._client is None
assert seen == []
@pytest.mark.usefixtures("_stub_browser_session")
class TestSharedBrowserCallOutcome:
@staticmethod
def _server(
*,
model_tool_name: str,
raw_tool_name: str,
payload: dict[str, Any] | Exception,
is_error: bool = False,
) -> SkyvernOverlayMCPServer:
server = _make_server(
make_copilot_ctx(browser_session_id="pbs_1"),
payload,
SchemaOverlay(requires_browser=True),
alias_map={model_tool_name: raw_tool_name},
is_error=is_error,
)
server._overlays[model_tool_name] = SchemaOverlay(requires_browser=True)
return server
@pytest.mark.asyncio
@pytest.mark.parametrize(
("model_tool_name", "raw_tool_name", "payload", "is_error", "expected"),
[
pytest.param(
"evaluate",
"skyvern_evaluate",
{"ok": True, "data": {"result": {"count": 3}}},
False,
{
"dispatched": True,
"ok": True,
"error_kind": None,
"screenshot_present": False,
"screenshot_reference": None,
"response_truncated": False,
"payload_omitted": False,
},
id="evaluate-success",
),
pytest.param(
"evaluate",
"skyvern_evaluate",
{"ok": False, "error": {"code": "ACTION_FAILED", "message": "evaluation failed"}},
True,
{
"dispatched": True,
"ok": False,
"error_kind": "tool",
"screenshot_present": False,
"screenshot_reference": None,
"response_truncated": False,
"payload_omitted": False,
},
id="evaluate-tool-error",
),
pytest.param(
"get_browser_screenshot",
"skyvern_screenshot",
{
"dispatched": True,
"ok": True,
"data": {"path": "/tmp/frame.png", "data": "encoded-frame", "mime": "image/png"},
"artifacts": [{"kind": "screenshot", "path": "/tmp/frame.png", "mime": "image/png"}],
},
False,
{
"dispatched": True,
"ok": True,
"error_kind": None,
"screenshot_present": True,
"screenshot_reference": "/tmp/frame.png",
"response_truncated": False,
"payload_omitted": False,
},
id="screenshot-success",
),
pytest.param(
"get_browser_screenshot",
"skyvern_screenshot",
{
"ok": False,
"error": {"code": "RESPONSE_TOO_LARGE", "message": "inline screenshot omitted"},
"_truncated": True,
"_original_bytes": 200_000,
"_max_bytes": 100_000,
"artifacts": [{"kind": "screenshot", "path": "/tmp/oversized.png", "mime": "image/png"}],
},
False,
{
"ok": False,
"error_kind": "tool",
"screenshot_present": True,
"screenshot_reference": "/tmp/oversized.png",
"response_truncated": True,
"payload_omitted": True,
},
id="screenshot-oversized",
),
],
)
async def test_both_adapters_project_identical_common_facts_three_times(
self,
monkeypatch: pytest.MonkeyPatch,
model_tool_name: str,
raw_tool_name: str,
payload: dict[str, Any],
is_error: bool,
expected: dict[str, str | bool | None],
) -> None:
observed: list[_BrowserCallOutcome] = []
real_project = mcp_adapter._project_browser_call_outcome
def _capture(outcome: _BrowserCallOutcome, *, display_tool_name: str) -> dict[str, Any]:
observed.append(outcome)
return real_project(outcome, display_tool_name=display_tool_name)
monkeypatch.setattr(mcp_adapter, "_project_browser_call_outcome", _capture)
for _ in range(3):
server = self._server(
model_tool_name=model_tool_name,
raw_tool_name=raw_tool_name,
payload=payload,
is_error=is_error,
)
await server.call_tool(model_tool_name, {})
await server.call_internal_tool(raw_tool_name, {})
assert len(observed) == 6
for model_outcome, internal_outcome in zip(observed[::2], observed[1::2], strict=True):
assert model_outcome == internal_outcome
assert model_outcome.raw_tool_name == raw_tool_name
assert model_outcome.source_browser_session_id == "pbs_1"
for field_name, value in expected.items():
assert getattr(model_outcome, field_name) == value
@pytest.mark.asyncio
async def test_client_exception_identity_is_shared_beneath_adapter_wording(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
observed: list[_BrowserCallOutcome] = []
real_project = mcp_adapter._project_browser_call_outcome
def _capture(outcome: _BrowserCallOutcome, *, display_tool_name: str) -> dict[str, Any]:
observed.append(outcome)
return real_project(outcome, display_tool_name=display_tool_name)
monkeypatch.setattr(mcp_adapter, "_project_browser_call_outcome", _capture)
server = self._server(
model_tool_name="evaluate",
raw_tool_name="skyvern_evaluate",
payload=RuntimeError("transport unavailable"),
)
await server.call_tool("evaluate", {})
await server.call_internal_tool("skyvern_evaluate", {})
assert observed[0] == observed[1]
assert observed[0].error_kind == "protocol"
assert observed[0].protocol_error_detail == "transport unavailable"
def test_payload_custody_copies_containers_without_copying_inline_screenshot_bytes(self) -> None:
screenshot_base64 = "frame-" + "x" * 200_000
payload = {"ok": True, "data": {"data": screenshot_base64, "metadata": {"width": 1280}}}
outcome = mcp_adapter._browser_call_outcome_from_mapping(
raw_tool_name="skyvern_screenshot",
source_browser_session_id="pbs_1",
raw_result=payload,
)
first = outcome.raw_result()
second = outcome.raw_result()
assert first is not payload
assert first["data"] is not payload["data"]
assert first["data"]["data"] is screenshot_base64
assert second["data"]["data"] is screenshot_base64
first["data"]["metadata"]["width"] = 1
assert second["data"]["metadata"]["width"] == 1280
@pytest.mark.asyncio
async def test_typed_internal_accessor_preserves_legacy_dict(self) -> None:
payload = {"ok": True, "data": {"result": {"count": 3}}}
server = self._server(
model_tool_name="evaluate",
raw_tool_name="skyvern_evaluate",
payload=payload,
)
typed = await server.call_internal_browser_tool("skyvern_evaluate", {"expression": "scan()"})
legacy = await server.call_internal_tool("skyvern_evaluate", {"expression": "scan()"})
assert typed.result == legacy == mcp_to_copilot(payload)
assert typed.outcome.dispatched is True
assert typed.outcome.source_browser_session_generation == 0
@pytest.mark.asyncio
async def test_internal_not_connected_is_a_not_dispatched_outcome(self) -> None:
server = self._server(
model_tool_name="evaluate",
raw_tool_name="skyvern_evaluate",
payload={"ok": True},
)
server._client = None
with capture_logs() as captured:
call = await server.call_internal_browser_tool("skyvern_evaluate", {})
assert call.result["ok"] is False
assert call.outcome.dispatched is False
assert _browser_outcome_records(captured)[0]["dispatched"] is False
@pytest.mark.asyncio
async def test_model_connect_precondition_remains_before_attempt_admission(self) -> None:
server = self._server(
model_tool_name="evaluate",
raw_tool_name="skyvern_evaluate",
payload={"ok": True},
)
server._client = None
with capture_logs() as captured, pytest.raises(RuntimeError, match="Not connected"):
await server._call_tool("evaluate", {})
assert _browser_outcome_records(captured) == []
@pytest.mark.asyncio
@pytest.mark.parametrize("call_path", ["model", "internal"])
async def test_session_preparation_error_is_not_dispatched(
self, call_path: str, monkeypatch: pytest.MonkeyPatch
) -> None:
async def _session_error(_ctx: AgentContext, **_kwargs: Any) -> dict[str, Any]:
return {"ok": False, "error": "session setup failed"}
monkeypatch.setattr(mcp_adapter, "ensure_browser_session", _session_error)
server = self._server(
model_tool_name="evaluate",
raw_tool_name="skyvern_evaluate",
payload={"ok": True},
)
server._context_provider().browser_session_id = None
with capture_logs() as captured:
await _call(server, call_path)
records = _browser_outcome_records(captured)
assert len(records) == 1
assert records[0]["dispatched"] is False
@pytest.mark.asyncio
@pytest.mark.parametrize("call_path", ["model", "internal"])
async def test_pre_dispatch_replacement_retains_both_session_identities(
self, call_path: str, monkeypatch: pytest.MonkeyPatch
) -> None:
ctx = make_copilot_ctx(browser_session_id="pbs_original")
async def _replacement(
replacement_ctx: AgentContext,
**_kwargs: Any,
) -> tuple[None, dict[str, Any], mcp_adapter._BrowserSessionLossDisposition]:
replacement_ctx.browser_session_id = "pbs_replacement"
replacement_ctx.browser_session_continuity_generation = 1
replacement_ctx.browser_session_continuity_disposition = "reestablished"
return (
None,
mcp_adapter._browser_session_loss_result({}, disposition="reestablished"),
"reestablished",
)
monkeypatch.setattr(mcp_adapter, "_prepare_browser_session_for_dispatch", _replacement)
server = _make_server(
ctx,
{"ok": True},
SchemaOverlay(requires_browser=True),
alias_map={"evaluate": "skyvern_evaluate"},
)
with capture_logs() as captured:
await _call(server, call_path)
record = _browser_outcome_records(captured)[0]
assert record["dispatched"] is False
assert record["source_browser_session_id"] == "pbs_original"
assert record["replacement_browser_session_id"] == "pbs_replacement"
assert record["source_browser_session_generation"] == 0
assert record["completion_browser_session_generation"] == 1
@pytest.mark.asyncio
@pytest.mark.parametrize("call_path", ["model", "internal"])
async def test_context_entry_failure_is_not_dispatched(
self, call_path: str, monkeypatch: pytest.MonkeyPatch
) -> None:
@asynccontextmanager
async def _failed_context(_ctx: AgentContext, *, session_id_override: str | None = None) -> AsyncIterator[None]:
raise RuntimeError("context unavailable")
yield
monkeypatch.setattr(mcp_adapter, "mcp_browser_context", _failed_context)
server = self._server(
model_tool_name="evaluate",
raw_tool_name="skyvern_evaluate",
payload={"ok": True},
)
with capture_logs() as captured:
await _call(server, call_path)
records = _browser_outcome_records(captured)
assert len(records) == 1
assert records[0]["dispatched"] is False
@pytest.mark.asyncio
@pytest.mark.parametrize("call_path", ["model", "internal"])
async def test_cancellation_records_dispatch_then_reraises_same_exception(
self, call_path: str, monkeypatch: pytest.MonkeyPatch
) -> None:
cancellation = asyncio.CancelledError("caller deadline")
server = self._server(
model_tool_name="evaluate",
raw_tool_name="skyvern_evaluate",
payload={"ok": True},
)
async def _cancelled_call(*_args: Any, **_kwargs: Any) -> None:
raise cancellation
monkeypatch.setattr(server._client, "call_tool", _cancelled_call)
with capture_logs() as captured, pytest.raises(asyncio.CancelledError) as caught:
await _call(server, call_path)
assert caught.value is cancellation
records = _browser_outcome_records(captured)
assert len(records) == 1
assert records[0]["dispatched"] is True
assert records[0]["cancelled"] is True
@pytest.mark.asyncio
async def test_raw_secret_and_pre_hook_rejections_are_not_dispatched(self) -> None:
raw_secret_ctx = make_copilot_ctx(
browser_session_id="pbs_1",
request_policy=RequestPolicy(raw_secret_detected=True),
)
raw_secret_ctx.browser_session_continuity_disposition = "reestablished"
raw_secret_server = _make_server(
raw_secret_ctx,
{"ok": True},
SchemaOverlay(requires_browser=True),
alias_map={"evaluate": "skyvern_evaluate"},
)
async def _reject(_args: dict[str, Any], _ctx: AgentContext) -> dict[str, Any]:
return {"ok": False, "error": "screenshot unavailable"}
screenshot_server = _make_server(
make_copilot_ctx(browser_session_id="pbs_1"),
{"ok": True},
SchemaOverlay(requires_browser=True, pre_hook=_reject),
alias_map={"get_browser_screenshot": "skyvern_screenshot"},
)
screenshot_server._overlays["get_browser_screenshot"] = SchemaOverlay(
requires_browser=True,
pre_hook=_reject,
)
with capture_logs() as captured:
raw_secret_result = await raw_secret_server.call_tool("evaluate", {})
await screenshot_server.call_tool("get_browser_screenshot", {})
assert copilot_runtime.RAW_SECRET_BROWSER_ERROR in raw_secret_result.content[0].text
records = _browser_outcome_records(captured)
assert len(records) == 2
assert all(record["dispatched"] is False for record in records)
@pytest.mark.asyncio
async def test_raw_secret_refuses_an_internal_browser_call_on_an_already_open_session(self) -> None:
ctx = make_copilot_ctx(
browser_session_id="pbs_1",
request_policy=RequestPolicy(raw_secret_detected=True, raw_secret_handling="redacted_draft"),
)
dispatched = MagicMock()
server = _make_server(ctx, {"ok": True}, SchemaOverlay(requires_browser=True), on_call=dispatched)
with capture_logs() as captured:
result = await server.call_internal_tool("skyvern_navigate", {"url": "https://portal.example.com/login"})
assert result["ok"] is False
dispatched.assert_not_called()
timing = [entry for entry in captured if entry["event"] == "MCP tool timing"]
assert [entry["call_status"] for entry in timing] == ["raw_secret_denied"]
def test_a_protocol_failure_reports_the_model_surface_tool_and_both_session_identities(self) -> None:
"""A bare 'evaluate failed' leaves the model no way to tell where it now is, so the failed
call has to name a tool the model can actually call and both ends of the session identity."""
outcome = replace(
mcp_adapter._browser_protocol_exception_outcome(
raw_tool_name="skyvern_evaluate",
source_browser_session_id="pbs_source",
source_browser_session_generation=2,
dispatched=True,
exception=RuntimeError("transport closed"),
),
completion_browser_session_id="pbs_after",
completion_browser_session_generation=3,
)
projected = mcp_adapter._project_browser_call_outcome(outcome, display_tool_name="evaluate")
continuity = projected["data"]["browser_call_continuity"]
assert continuity["failed_tool"] == "evaluate"
assert continuity["browser_session_id"] == "pbs_source"
assert continuity["browser_session_generation"] == 2
assert continuity["browser_session_id_after"] == "pbs_after"
assert continuity["browser_session_generation_after"] == 3
def test_a_protocol_failure_states_no_liveness_or_read_fact_the_outcome_cannot_prove(self) -> None:
"""Dispatch is recorded before the await, so a dispatched call cannot tell pre- from
post-execution failure — the projection may report delivery and effect, never liveness."""
outcome = mcp_adapter._browser_protocol_exception_outcome(
raw_tool_name="skyvern_evaluate",
source_browser_session_id="pbs_source",
source_browser_session_generation=0,
dispatched=True,
exception=RuntimeError("transport closed"),
)
projected = mcp_adapter._project_browser_call_outcome(outcome, display_tool_name="evaluate")
continuity = projected["data"]["browser_call_continuity"]
assert continuity["dispatched"] is True
assert continuity["result_delivered"] is False
assert continuity["prior_action_effect"] == "unknown"
assert "session_state" not in continuity
assert "read_completed" not in continuity
assert "Observe the page again" in projected["error"]
@pytest.mark.asyncio
async def test_a_failure_before_dispatch_reports_no_effect_on_the_targeted_browser(self) -> None:
"""A pre-hook that dies never sends the call, so the block must say so instead of inviting a
re-observation, and a run-targeted call must name the run browser as where it ended, not the
chat's debug browser."""
async def _explode(_args: dict[str, Any], _ctx: AgentContext) -> dict[str, Any]:
raise RuntimeError("hook transport closed")
ctx = make_copilot_ctx(browser_session_id="pbs_debug")
ctx.last_run_blocks_browser_session_id = "pbs_run"
ctx.last_run_blocks_workflow_run_id = "wr_1"
ctx.browser_session_continuity_generation = 4
server = _make_server(
ctx,
{"ok": True},
SchemaOverlay(requires_browser=True, pre_hook=_explode),
alias_map={"evaluate": "skyvern_evaluate"},
)
with capture_logs() as captured:
projected = await server.call_tool("evaluate", {"target": "last_run"})
payload = json.loads(projected.content[0].text)
continuity = payload["data"]["browser_call_continuity"]
assert continuity["dispatched"] is False
assert continuity["prior_action_effect"] == "none"
assert continuity["browser_session_id"] == "pbs_run"
assert continuity["browser_session_id_after"] == "pbs_run"
assert continuity["browser_session_generation_after"] is None
assert "had no effect" in payload["error"]
assert "Observe the page again" not in payload["error"]
records = _browser_outcome_records(captured)
assert records[-1]["completion_browser_session_id"] == "pbs_run"
@pytest.mark.asyncio
async def test_a_failed_debug_call_states_the_last_run_browser_without_moving_to_it(self) -> None:
async def _explode(_args: dict[str, Any], _ctx: AgentContext) -> dict[str, Any]:
raise RuntimeError("Browser CDP connection failed during state lookup.")
ctx = make_copilot_ctx(browser_session_id="pbs_debug")
ctx.last_run_blocks_browser_session_id = "pbs_run"
ctx.last_run_blocks_workflow_run_id = "wr_1"
server = _make_server(
ctx,
{"ok": True},
SchemaOverlay(requires_browser=True, pre_hook=_explode),
alias_map={"evaluate": "skyvern_evaluate"},
)
projected = await server.call_tool("evaluate", {})
payload = json.loads(projected.content[0].text)
continuity = payload["data"]["browser_call_continuity"]
assert payload["ok"] is False
assert continuity["browser_session_id"] == "pbs_debug"
assert continuity["browser_session_id_after"] == "pbs_debug"
assert continuity["last_run_browser_session_id"] == "pbs_run"
assert continuity["last_run_workflow_run_id"] == "wr_1"
@pytest.mark.asyncio
async def test_a_failed_call_does_not_offer_its_own_browser_as_the_last_run_browser(self) -> None:
async def _explode(_args: dict[str, Any], _ctx: AgentContext) -> dict[str, Any]:
raise RuntimeError("Browser CDP connection failed during state lookup.")
ctx = make_copilot_ctx(browser_session_id="pbs_debug")
ctx.last_run_blocks_browser_session_id = "pbs_debug"
ctx.last_run_blocks_workflow_run_id = "wr_1"
server = _make_server(
ctx,
{"ok": True},
SchemaOverlay(requires_browser=True, pre_hook=_explode),
alias_map={"evaluate": "skyvern_evaluate"},
)
projected = await server.call_tool("evaluate", {})
continuity = json.loads(projected.content[0].text)["data"]["browser_call_continuity"]
assert continuity["browser_session_id"] == "pbs_debug"
assert "last_run_browser_session_id" not in continuity
@pytest.mark.asyncio
async def test_a_successful_call_carries_no_last_run_facts(self) -> None:
ctx = make_copilot_ctx(browser_session_id="pbs_debug")
ctx.last_run_blocks_browser_session_id = "pbs_run"
ctx.last_run_blocks_workflow_run_id = "wr_1"
server = _make_server(
ctx,
{"ok": True, "data": {"result": 1}},
SchemaOverlay(requires_browser=True),
alias_map={"evaluate": "skyvern_evaluate"},
)
projected = await server.call_tool("evaluate", {})
assert "pbs_run" not in projected.content[0].text
def test_a_protocol_failure_does_not_borrow_the_session_loss_continuity_block(self) -> None:
"""The session is not known to be lost here; reusing the loss block would tell the model a
replacement browser is ready when nothing established that."""
protocol = mcp_adapter._project_browser_call_outcome(
mcp_adapter._browser_protocol_exception_outcome(
raw_tool_name="skyvern_evaluate",
source_browser_session_id="pbs_source",
source_browser_session_generation=0,
dispatched=True,
exception=RuntimeError("transport closed"),
),
display_tool_name="evaluate",
)
loss = mcp_adapter._project_browser_call_outcome(
mcp_adapter._not_dispatched_browser_call_outcome(
raw_tool_name="skyvern_evaluate",
source_browser_session_id="pbs_lost",
source_browser_session_generation=0,
raw_result={},
ctx=make_copilot_ctx(browser_session_id="pbs_replacement"),
session_loss_disposition="reestablished",
),
display_tool_name="evaluate",
)
assert "browser_session_continuity" not in protocol["data"]
assert "error_code" not in protocol
assert loss["data"]["browser_session_continuity"]["disposition"] == "reestablished"
assert "browser_call_continuity" not in loss["data"]
def test_a_cancelled_call_still_names_the_last_run_browser(self) -> None:
"""A caller that gives up projects no continuity block of its own, and that abandonment is
the shape the other browser matters most on."""
ctx = make_copilot_ctx(browser_session_id="pbs_debug")
ctx.last_run_blocks_browser_session_id = "pbs_run"
ctx.last_run_blocks_workflow_run_id = "wr_1"
cancelled = mcp_adapter._cancelled_browser_call_outcome(
raw_tool_name="skyvern_evaluate",
source_browser_session_id="pbs_debug",
source_browser_session_generation=0,
dispatch_started=False,
ctx=ctx,
)
projected = mcp_adapter._project_browser_call_outcome(
mcp_adapter._scrub_browser_call_outcome(ctx, cancelled), display_tool_name="evaluate"
)
succeeded = mcp_adapter._project_browser_call_outcome(
mcp_adapter._scrub_browser_call_outcome(
ctx,
mcp_adapter._browser_call_outcome_from_mapping(
raw_tool_name="skyvern_evaluate",
source_browser_session_id="pbs_debug",
source_browser_session_generation=0,
dispatched=True,
raw_result={"ok": True, "data": {"result": 1}},
),
),
display_tool_name="evaluate",
)
continuity = projected["data"]["browser_call_continuity"]
assert continuity["last_run_browser_session_id"] == "pbs_run"
assert continuity["last_run_workflow_run_id"] == "wr_1"
assert "pbs_run" not in json.dumps(succeeded)
@pytest.mark.usefixtures("_stub_browser_session")
class TestMCPToolTiming:
@pytest.mark.asyncio
async def test_internal_call_logs_the_server_total_without_changing_the_result(
self, _fake_clock: list[float]
) -> None:
payload = {"ok": True, "data": {"x": 1}, "timing_ms": {"sdk": 812, "total": 815}}
server = _server_whose_call_takes_time(payload, SchemaOverlay(), _fake_clock)
with capture_logs() as captured:
result = await server.call_internal_tool("skyvern_evaluate", {"expression": "scan()"})
records = _timing_records(captured)
assert len(records) == 1
record = records[0]
assert record["tool_name"] == "skyvern_evaluate"
assert record["mcp_tool_name"] == "skyvern_evaluate"
assert record["call_path"] == "internal"
assert record["server_timing_ms"] == 815
assert record["wall_clock_ms"] == _WALL_MS
assert set(record) & _PAYLOAD_KEYS_THAT_WOULD_LEAK == set()
assert record["workflow_permanent_id"] == "wfp-1"
assert "turn_id" in record
assert "workflow_copilot_chat_id" in record
assert result == mcp_to_copilot(payload)
assert "timing_ms" not in result
@pytest.mark.asyncio
@pytest.mark.parametrize("timing_ms", [None, {}, "815ms", {"total": "815"}, {"total": True}])
async def test_a_result_without_a_server_timing_dict_still_logs(
self, timing_ms: str | dict[str, str | bool] | None, _fake_clock: list[float]
) -> None:
payload: dict[str, Any] = {"ok": True, "data": {"x": 1}}
if timing_ms is not None:
payload["timing_ms"] = timing_ms
server = _server_whose_call_takes_time(payload, SchemaOverlay(), _fake_clock)
with capture_logs() as captured:
await server.call_internal_tool("skyvern_evaluate", {"expression": "scan()"})
records = _timing_records(captured)
assert len(records) == 1
assert records[0]["server_timing_ms"] is None
assert records[0]["server_attach_ms"] is None
assert records[0]["tool_name"] == "skyvern_evaluate"
assert records[0]["mcp_tool_name"] == "skyvern_evaluate"
assert records[0]["call_path"] == "internal"
assert records[0]["wall_clock_ms"] == _WALL_MS
@pytest.mark.asyncio
async def test_the_browser_attach_is_named_and_folded_back_into_the_server_span_once(
self, _fake_clock: list[float]
) -> None:
payload = {"ok": True, "data": {"x": 1}, "timing_ms": {"attach": 300, "total": 515}}
server = _server_whose_call_takes_time(payload, SchemaOverlay(), _fake_clock)
with capture_logs() as captured:
await server.call_internal_tool("skyvern_evaluate", {"expression": "scan()"})
record = _timing_records(captured)[0]
assert record["server_attach_ms"] == 300
assert record["server_timing_ms"] == 815
assert record["phase_dispatch_untimed_ms"] == _MCP_CALL_MS - 815
@pytest.mark.asyncio
async def test_model_facing_call_logs_both_names_and_excludes_the_post_hook(self, _fake_clock: list[float]) -> None:
payload = {"ok": True, "data": {"result": 1}, "timing_ms": {"sdk": 812, "total": 815}}
async def _slow_post_hook(
copilot_result: dict[str, Any], raw_mcp: dict[str, Any], ctx: AgentContext
) -> dict[str, Any]:
_fake_clock[0] += _AFTER_CALL_SECONDS
return copilot_result
overlay = SchemaOverlay(requires_browser=True, post_hook=_slow_post_hook)
server = _server_whose_call_takes_time(payload, overlay, _fake_clock, alias_map=get_skyvern_mcp_alias_map())
with capture_logs() as captured:
result = await server.call_tool("evaluate", {"expression": "scan()"})
records = _timing_records(captured)
assert len(records) == 1
assert records[0]["tool_name"] == "evaluate"
assert records[0]["mcp_tool_name"] == "skyvern_evaluate"
assert records[0]["call_path"] == "model"
assert records[0]["server_timing_ms"] == 815
assert records[0]["wall_clock_ms"] == _WALL_MS
assert records[0]["call_status"] == "ok"
assert "timing_ms" not in result.content[0].text
@pytest.mark.asyncio
async def test_evaluate_redacts_matching_run_registry_before_disclosure(self, _stub_browser_session: None) -> None:
ctx = make_copilot_ctx(browser_session_id="pbs_1")
ctx.last_run_blocks_workflow_run_id = "wr_sensitive"
ctx.origin_run_redaction_registry = OriginRunRedactionRegistry(
"wr_sensitive",
{
"credential": {"username": "private-user", "password": "private-pass"},
"copilot_run_runtime_secret_values": ("654321",),
},
contains_sensitive_values=True,
contains_all_sensitive_values=True,
)
register_secret_scrub_values_from_structure(ctx, ctx.origin_run_redaction_registry.parameters)
overlay = _build_skyvern_mcp_overlays()["evaluate"]
server = _make_server(
ctx,
{
"ok": True,
"data": {
"result": {
"form": {"selector": "#otp-form", "field": "#totp"},
"echo": "private-user:private-pass:654321",
},
"url": "https://example.test/otp",
"title": "Verification for private-user",
},
},
overlay,
alias_map=get_skyvern_mcp_alias_map(),
)
result = await server.call_tool("evaluate", {"expression": "scan()"})
surfaced = json.loads(result.content[0].text)
assert surfaced["ok"] is True
assert surfaced["data"]["result"]["form"] == {"selector": "#otp-form", "field": "#totp"}
assert "private-user" not in result.content[0].text
assert "private-pass" not in result.content[0].text
assert "654321" not in result.content[0].text
assert "[REDACTED_SECRET]" in result.content[0].text
retained = json.dumps(
{
"composition_page_evidence": ctx.composition_page_evidence,
"flow_evidence": ctx.flow_evidence,
"scout_trajectory": ctx.scout_trajectory,
},
default=str,
)
assert "private-user" not in retained
assert "private-pass" not in retained
assert "654321" not in retained
@pytest.mark.asyncio
async def test_evaluate_is_withheld_until_the_run_hands_over_its_secrets(
self, _stub_browser_session: None, monkeypatch: pytest.MonkeyPatch
) -> None:
ctx = make_copilot_ctx(browser_session_id="pbs_1")
owed = OriginRunRedactionRegistry(
"wr_owed",
{"credential": {"password": "private-pass"}},
contains_sensitive_values=True,
contains_all_sensitive_values=False,
awaiting_runtime_secret_values=True,
)
ctx.origin_run_redaction_registry = owed
ctx.awaited_run_secret_handoffs[("pbs_1", "wr_owed")] = partial(
run_execution._settle_awaited_run_secrets, ctx, owed, "pbs_1"
)
monkeypatch.setattr(
run_execution, "consume_copilot_runtime_secret_values", AsyncMock(side_effect=[None, {"654321"}])
)
server = _make_server(
ctx,
{"ok": True, "data": {"result": "Your code is 654321", "url": "https://example.test/otp"}},
_build_skyvern_mcp_overlays()["evaluate"],
alias_map=get_skyvern_mcp_alias_map(),
)
withheld = await server.call_tool("evaluate", {"expression": "scan()"})
dispatched_while_owed = len(server._client.calls)
disclosed = await server.call_tool("evaluate", {"expression": "scan()"})
assert withheld.isError is True
assert "wr_owed" in json.loads(withheld.content[0].text)["error"]
assert dispatched_while_owed == 0
assert json.loads(disclosed.content[0].text)["ok"] is True
assert "654321" not in disclosed.content[0].text
@pytest.mark.asyncio
async def test_a_call_that_exceeds_its_ceiling_reports_its_wall_time(self, _fake_clock: list[float]) -> None:
overlay = SchemaOverlay(requires_browser=True, timeout=30)
server = _server_whose_call_takes_time(TimeoutError(), overlay, _fake_clock)
with capture_logs() as captured:
await server.call_tool("evaluate", {"expression": "scan()"})
records = _timing_records(captured)
assert len(records) == 1
assert records[0]["call_status"] == "timeout"
assert records[0]["server_timing_ms"] is None
assert records[0]["timing_server_overrun"] is None
assert records[0]["wall_clock_ms"] == _WALL_MS
@pytest.mark.asyncio
async def test_a_call_that_exceeds_its_ceiling_says_its_effect_is_unknown(self, _fake_clock: list[float]) -> None:
server = _server_whose_call_takes_time(TimeoutError(), SchemaOverlay(timeout=30), _fake_clock)
result = await server.call_tool("evaluate", {"expression": "scan()"})
surfaced = json.loads(result.content[0].text)
assert surfaced["ok"] is False
assert surfaced["error"] == (
f"evaluate did not answer within 30s and was cancelled. {mcp_adapter._EFFECT_UNKNOWN_GUIDANCE}"
)
@pytest.mark.asyncio
@pytest.mark.parametrize(
"payload",
[
{"ok": True, "data": {"x": 1}, "timing_ms": {"sdk": 812, "total": 815}},
RuntimeError("mcp exploded"),
],
ids=["ok", "error"],
)
async def test_an_internal_call_reports_the_model_facing_name_so_one_tool_is_one_facet(
self, payload: dict[str, Any] | Exception, _fake_clock: list[float]
) -> None:
server = _server_whose_call_takes_time(
payload, SchemaOverlay(), _fake_clock, alias_map={"evaluate": "skyvern_evaluate"}
)
with capture_logs() as captured:
await server.call_internal_tool("skyvern_evaluate", {"expression": "scan()"})
records = _timing_records(captured)
assert len(records) == 1
assert records[0]["tool_name"] == "evaluate"
assert records[0]["mcp_tool_name"] == "skyvern_evaluate"
assert records[0]["call_path"] == "internal"
@pytest.mark.asyncio
@pytest.mark.parametrize("call_path", ["model", "internal"])
async def test_a_call_cancelled_while_attaching_still_reports_the_budget_it_spent(
self, call_path: str, monkeypatch: pytest.MonkeyPatch, _fake_clock: list[float]
) -> None:
@asynccontextmanager
async def _cancelled(_ctx: AgentContext, *, session_id_override: str | None = None) -> AsyncIterator[None]:
_fake_clock[0] += _BROWSER_BOOT_SECONDS
raise asyncio.CancelledError
yield
monkeypatch.setattr(mcp_adapter, "mcp_browser_context", _cancelled)
server = _server_whose_call_takes_time({"ok": True}, SchemaOverlay(requires_browser=True), _fake_clock)
with capture_logs() as captured, pytest.raises(asyncio.CancelledError):
await _call(server, call_path)
records = _timing_records(captured)
assert len(records) == 1
assert records[0]["call_status"] == "cancelled"
assert records[0]["call_path"] == call_path
assert records[0]["wall_clock_ms"] == _SESSION_ONLY_MS
assert records[0]["server_timing_ms"] is None
assert records[0]["timing_phase"] == "context_enter"
assert records[0]["phase_context_enter_ms"] == _SESSION_ONLY_MS
assert records[0]["phase_dispatch_ms"] == 0
assert records[0]["phase_residual_ms"] >= 0
@pytest.mark.asyncio
@pytest.mark.parametrize("call_path", ["model", "internal"])
async def test_a_session_that_fails_to_resolve_still_reports_the_budget_it_spent(
self, call_path: str, monkeypatch: pytest.MonkeyPatch, _fake_clock: list[float]
) -> None:
async def _session_error(_ctx: AgentContext, **_kwargs: Any) -> dict[str, Any]:
_fake_clock[0] += _BROWSER_BOOT_SECONDS
return {"ok": False, "error": "no browser session"}
monkeypatch.setattr(mcp_adapter, "ensure_browser_session", _session_error)
server = _server_whose_call_takes_time(
{"ok": True}, SchemaOverlay(requires_browser=True), _fake_clock, browser_session_id=None
)
with capture_logs() as captured:
result = await _call(server, call_path)
records = _timing_records(captured)
assert len(records) == 1
assert records[0]["call_status"] == "session_error"
assert records[0]["call_path"] == call_path
assert records[0]["wall_clock_ms"] == _SESSION_ONLY_MS
assert records[0]["server_timing_ms"] is None
assert "no browser session" in _surfaced_error(result, call_path)
@pytest.mark.asyncio
@pytest.mark.parametrize("call_path", ["model", "internal"])
async def test_an_attach_that_raises_still_reports_its_budget(
self, call_path: str, monkeypatch: pytest.MonkeyPatch, _fake_clock: list[float]
) -> None:
@asynccontextmanager
async def _attach_raises(_ctx: AgentContext, *, session_id_override: str | None = None) -> AsyncIterator[None]:
_fake_clock[0] += _BROWSER_BOOT_SECONDS
raise RuntimeError("adoption failed")
yield
monkeypatch.setattr(mcp_adapter, "mcp_browser_context", _attach_raises)
server = _server_whose_call_takes_time({"ok": True}, SchemaOverlay(requires_browser=True), _fake_clock)
with capture_logs() as captured:
await _call(server, call_path)
records = _timing_records(captured)
assert len(records) == 1
assert records[0]["call_status"] == "error"
assert records[0]["call_path"] == call_path
assert records[0]["wall_clock_ms"] == _SESSION_ONLY_MS
@pytest.mark.asyncio
@pytest.mark.parametrize("call_path", ["model", "internal"])
async def test_a_tool_that_reports_failure_without_raising_is_not_recorded_as_ok(
self, call_path: str, _fake_clock: list[float]
) -> None:
payload = {"ok": False, "error": "selector not found", "timing_ms": {"total": 815}}
server = _server_whose_call_takes_time(
payload, SchemaOverlay(requires_browser=True), _fake_clock, alias_map=get_skyvern_mcp_alias_map()
)
with capture_logs() as captured:
await _call(server, call_path)
records = _timing_records(captured)
assert len(records) == 1
assert records[0]["call_status"] == "error"
assert records[0]["server_timing_ms"] == 815
@pytest.mark.asyncio
async def test_an_internal_call_without_a_connected_client_still_leaves_a_record(
self, _fake_clock: list[float]
) -> None:
server = _server_whose_call_takes_time({"ok": True}, SchemaOverlay(), _fake_clock)
server._client = None
with capture_logs() as captured:
await server.call_internal_tool("skyvern_evaluate", {"expression": "scan()"})
records = _timing_records(captured)
assert len(records) == 1
assert records[0]["call_status"] == "not_connected"
assert records[0]["call_path"] == "internal"
assert records[0]["server_timing_ms"] is None
@pytest.mark.asyncio
@pytest.mark.parametrize("call_path", ["model", "internal"])
async def test_a_call_whose_wall_clock_dwarfs_the_server_total_names_where_the_time_went(
self, call_path: str, monkeypatch: pytest.MonkeyPatch, _fake_clock: list[float]
) -> None:
_install_timed_context(monkeypatch, _fake_clock)
payload = {"ok": True, "data": {"x": 1}, "timing_ms": {"total": 815}}
server = _server_whose_call_takes_time(
payload, SchemaOverlay(requires_browser=True), _fake_clock, alias_map=get_skyvern_mcp_alias_map()
)
with capture_logs() as captured:
await _call(server, call_path)
record = _timing_records(captured)[0]
assert record["wall_clock_ms"] == _GAP_WALL_MS
assert record["server_timing_ms"] == 815
assert record["phase_session_prepare_ms"] == 0
assert record["phase_context_enter_ms"] == _SESSION_ONLY_MS + _CONTEXT_ENTER_MS
assert record["phase_dispatch_ms"] == _MCP_CALL_MS
assert record["phase_context_exit_ms"] == _CONTEXT_EXIT_MS
assert record["phase_residual_ms"] == 0
assert record["timing_phase"] is None
_assert_every_millisecond_is_attributed(record)
assert record["phase_dispatch_untimed_ms"] == _MCP_CALL_MS - 815
assert record["timing_server_overrun"] is False
outside_the_server = sum(record[key] for key in _PHASE_KEYS if key != "phase_dispatch_ms")
assert outside_the_server + record["phase_dispatch_untimed_ms"] == _GAP_WALL_MS - 815
@pytest.mark.asyncio
@pytest.mark.parametrize("call_path", ["model", "internal"])
async def test_a_tool_that_answered_with_an_error_names_no_segment_as_the_one_it_died_in(
self, call_path: str, monkeypatch: pytest.MonkeyPatch, _fake_clock: list[float]
) -> None:
_install_timed_context(monkeypatch, _fake_clock)
payload = {"ok": False, "error": "the page said no", "timing_ms": {"total": 815}}
server = _server_whose_call_takes_time(
payload, SchemaOverlay(requires_browser=True), _fake_clock, alias_map=get_skyvern_mcp_alias_map()
)
with capture_logs() as captured:
await _call(server, call_path)
record = _timing_records(captured)[0]
assert record["call_status"] == "error"
assert record["timing_phase"] is None
_assert_every_millisecond_is_attributed(record)
@pytest.mark.asyncio
@pytest.mark.parametrize(
("overlay_id", "payload"),
[
*(
pytest.param(
overlay_id,
{
"ok": True,
"data": {"selector": 'role=button[name="Go"]'},
"browser_context": {"url": "https://example.test/form", "title": "Form"},
},
id=f"answered-{overlay_id}",
)
for overlay_id in _ACTION_OVERLAY_CASES
),
pytest.param(
"click",
{
"ok": False,
"error": "Timeout 10000ms exceeded waiting for the element",
"browser_context": {"url": "https://example.test/form", "title": "Form"},
},
id="inner_timeout-click",
),
],
)
async def test_a_call_far_longer_than_any_plausible_ceiling_runs_to_completion(
self, overlay_id: str, payload: dict[str, Any], _stub_browser_session: None
) -> None:
tool_name, overlay = _real_action_overlay(overlay_id)
immediate_client = _HeldClient(payload)
immediate_client.release.set()
immediate = await _overlay_server(
make_copilot_ctx(browser_session_id="pbs_1"), immediate_client, tool_name, overlay
).call_tool(tool_name, _ACTION_ARGS[tool_name])
held_client = _HeldClient(payload)
held_server = _overlay_server(make_copilot_ctx(browser_session_id="pbs_1"), held_client, tool_name, overlay)
with capture_logs() as captured:
held = await _answer_after_loop_seconds(held_server, held_client, tool_name, 3600.0)
assert len(held_client.dispatched) == 1
assert held_client.cancelled is False
assert held.content[0].text == immediate.content[0].text
projected = json.loads(held.content[0].text)
assert (projected["ok"], projected.get("error")) == (payload["ok"], payload.get("error"))
assert _timing_records(captured)[0]["call_status"] == ("ok" if payload["ok"] else "error")
@pytest.mark.asyncio
@pytest.mark.parametrize("overlay_id", ["click", "click_no_browser"])
async def test_cancelling_an_action_in_flight_cancels_its_transport(
self, overlay_id: str, _stub_browser_session: None
) -> None:
tool_name, overlay = _real_action_overlay(overlay_id)
client = _HeldClient({"ok": True})
server = _overlay_server(make_copilot_ctx(browser_session_id="pbs_1"), client, tool_name, overlay)
pending = asyncio.create_task(server.call_tool(tool_name, _ACTION_ARGS[tool_name]))
await asyncio.wait_for(client.started.wait(), timeout=5)
pending.cancel()
with pytest.raises(asyncio.CancelledError):
await pending
assert client.cancelled is True
@pytest.mark.asyncio
@pytest.mark.parametrize("call_path", ["model", "internal"])
@pytest.mark.parametrize("cancel_at", ["context_enter", "dispatch", "context_exit"])
async def test_a_cancelled_call_names_the_segment_that_was_in_flight(
self, cancel_at: str, call_path: str, monkeypatch: pytest.MonkeyPatch, _fake_clock: list[float]
) -> None:
@asynccontextmanager
async def _scope(_ctx: AgentContext, *, session_id_override: str | None = None) -> AsyncIterator[None]:
_fake_clock[0] += _CONTEXT_ENTER_SECONDS
if cancel_at == "context_enter":
raise asyncio.CancelledError
try:
yield
finally:
_fake_clock[0] += _CONTEXT_EXIT_SECONDS
if cancel_at == "context_exit":
raise asyncio.CancelledError
monkeypatch.setattr(mcp_adapter, "mcp_browser_context", _scope)
def _advance() -> None:
_fake_clock[0] += _MCP_CALL_SECONDS
if cancel_at == "dispatch":
raise asyncio.CancelledError
server = _browser_server(_advance)
with capture_logs() as captured, pytest.raises(asyncio.CancelledError):
await _call(server, call_path)
record = _timing_records(captured)[0]
assert record["call_status"] == "cancelled"
assert record["call_path"] == call_path
assert record["timing_phase"] == cancel_at
assert record["server_timing_ms"] is None
assert record["phase_residual_ms"] >= 0
_assert_every_millisecond_is_attributed(record)
if cancel_at == "context_enter":
assert record["phase_dispatch_ms"] == 0
assert record["phase_context_exit_ms"] == 0
else:
assert record["phase_dispatch_ms"] == _MCP_CALL_MS
assert record["phase_context_exit_ms"] == _CONTEXT_EXIT_MS
@pytest.mark.asyncio
async def test_a_server_total_larger_than_the_dispatch_segment_is_not_subtracted(
self, monkeypatch: pytest.MonkeyPatch, _fake_clock: list[float]
) -> None:
_install_timed_context(monkeypatch, _fake_clock)
payload = {"ok": True, "data": {"x": 1}, "timing_ms": {"total": _MCP_CALL_MS + 1}}
server = _server_whose_call_takes_time(
payload, SchemaOverlay(requires_browser=True), _fake_clock, alias_map=get_skyvern_mcp_alias_map()
)
with capture_logs() as captured:
await server.call_tool("evaluate", {"expression": "scan()"})
record = _timing_records(captured)[0]
assert record["phase_dispatch_untimed_ms"] is None
assert record["timing_server_overrun"] is True
_assert_every_millisecond_is_attributed(record)
@pytest.mark.asyncio
async def test_a_timing_sink_that_fails_costs_the_record_and_not_the_tool_result(
self, monkeypatch: pytest.MonkeyPatch, _fake_clock: list[float]
) -> None:
payload = {"ok": True, "data": {"x": 1}, "timing_ms": {"total": 815}}
server = _server_whose_call_takes_time(payload, SchemaOverlay(), _fake_clock)
real_info = mcp_adapter.LOG.info
def _sink_down(event: str, **fields: Any) -> Any:
if event == "MCP tool timing":
raise RuntimeError("structlog sink unavailable")
return real_info(event, **fields)
monkeypatch.setattr(mcp_adapter.LOG, "info", _sink_down)
result = await server.call_internal_tool("skyvern_evaluate", {"expression": "scan()"})
assert result.get("data") == {"x": 1}
@pytest.mark.asyncio
async def test_a_tool_that_needs_no_browser_spends_its_whole_wall_clock_dispatching(
self, _fake_clock: list[float]
) -> None:
payload = {"ok": True, "data": {"x": 1}, "timing_ms": {"total": 815}}
server = _server_whose_call_takes_time(payload, SchemaOverlay(), _fake_clock)
with capture_logs() as captured:
await server.call_tool("evaluate", {"expression": "scan()"})
record = _timing_records(captured)[0]
assert record["wall_clock_ms"] == _MCP_CALL_MS
assert record["phase_dispatch_ms"] == _MCP_CALL_MS
assert record["phase_session_prepare_ms"] == 0
assert record["phase_context_enter_ms"] == 0
assert record["phase_context_exit_ms"] == 0
assert record["phase_residual_ms"] == 0
assert record["timing_phase"] is None
def _async_return(value: Any) -> Callable[..., Any]:
async def _call(*_args: Any, **_kwargs: Any) -> Any:
return value
return _call
class TestBrowserSessionContinuity:
@pytest.fixture(autouse=True)
def _isolated_coordination(self, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(mcp_adapter.app, "CACHE", LocalCache())
mcp_adapter._LOCAL_CONTINUITY_OUTCOMES.clear()
mcp_adapter._LOCAL_CONTINUITY_ROOTS.clear()
async def _close(_organization_id: str, _session_id: str) -> None:
return None
monkeypatch.setattr(mcp_adapter, "close_browser_session_quietly", _close)
@pytest.mark.asyncio
async def test_session_expiry_logs_and_reestablishes_once(
self, monkeypatch: pytest.MonkeyPatch, _fake_clock: list[float]
) -> None:
ctx = make_copilot_ctx(browser_session_id="pbs_lost", turn_id="turn_1", workflow_copilot_chat_id="chat_1")
calls: list[str | None] = []
async def _ensure(recovery_ctx: AgentContext, **_kwargs: Any) -> None:
calls.append(recovery_ctx.browser_session_id)
recovery_ctx.browser_session_id = "pbs_replacement"
monkeypatch.setattr(mcp_adapter, "ensure_browser_session", _ensure)
with capture_logs() as captured:
first = await _handle_browser_session_loss(
ctx,
tool_name="evaluate",
call_path="model",
lost_session_id="pbs_lost",
)
second = await _handle_browser_session_loss(
ctx,
tool_name="evaluate",
call_path="model",
lost_session_id="pbs_lost",
)
assert first == second == "reestablished"
assert calls == [None]
assert ctx.browser_session_id == "pbs_replacement"
records = [record for record in captured if record.get("event") == "copilot_browser_session_continuity_loss"]
assert records[0]["session_id"] == "pbs_lost"
assert all(record["error_code"] == "SESSION_EXPIRED" for record in records)
assert [record["continuity_disposition"] for record in records] == ["detected", "reestablished"]
assert all(record["turn_id"] == "turn_1" for record in records)
assert all(record["workflow_copilot_chat_id"] == "chat_1" for record in records)
@pytest.mark.asyncio
async def test_two_contexts_share_one_replacement_and_close_the_lost_session(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
shared_cache = _SharedLocalCache()
monkeypatch.setattr(mcp_adapter.app, "CACHE", shared_cache)
manager = MagicMock()
manager.get_browser_session_startup_timeout_seconds.return_value = 55.0
monkeypatch.setattr(mcp_adapter.app, "PERSISTENT_SESSIONS_MANAGER", manager)
first_ctx = make_copilot_ctx(browser_session_id="pbs_shared_lost")
second_ctx = make_copilot_ctx(browser_session_id="pbs_shared_lost")
allocations = 0
closed: list[str] = []
async def _ensure(recovery_ctx: AgentContext, **_kwargs: Any) -> None:
nonlocal allocations
allocations += 1
recovery_ctx.browser_session_id = f"pbs_shared_replacement_{allocations}"
async def _close(_organization_id: str, session_id: str) -> None:
closed.append(session_id)
monkeypatch.setattr(mcp_adapter, "ensure_browser_session", _ensure)
monkeypatch.setattr(mcp_adapter, "close_browser_session_quietly", _close)
dispositions = await asyncio.gather(
_handle_browser_session_loss(
first_ctx, tool_name="evaluate", call_path="model", lost_session_id="pbs_shared_lost"
),
_handle_browser_session_loss(
second_ctx, tool_name="evaluate", call_path="model", lost_session_id="pbs_shared_lost"
),
)
assert dispositions == ["reestablished", "reestablished"]
assert allocations == 1
assert first_ctx.browser_session_id == second_ctx.browser_session_id == "pbs_shared_replacement_1"
assert closed == ["pbs_shared_lost"]
assert shared_cache.lock_names
assert await shared_cache.get(mcp_adapter._continuity_outcome_key(first_ctx.organization_id, "pbs_shared_lost"))
@pytest.mark.asyncio
async def test_a_replacement_loss_does_not_allocate_a_second_replacement(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
ctx = make_copilot_ctx(browser_session_id="pbs_generation_0")
allocations = 0
async def _ensure(recovery_ctx: AgentContext, **_kwargs: Any) -> None:
nonlocal allocations
allocations += 1
recovery_ctx.browser_session_id = f"pbs_generation_{allocations}"
monkeypatch.setattr(mcp_adapter, "ensure_browser_session", _ensure)
first = await _handle_browser_session_loss(
ctx, tool_name="evaluate", call_path="model", lost_session_id="pbs_generation_0"
)
second = await _handle_browser_session_loss(
ctx, tool_name="evaluate", call_path="model", lost_session_id="pbs_generation_1"
)
assert first == "reestablished"
assert second == "failed"
assert allocations == 1
assert ctx.browser_session_id is None
assert ctx.blocker_signal is not None
@pytest.mark.asyncio
async def test_a_deadline_expiry_is_named_rather_than_reported_as_a_browser_failure(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
"""SKY-15044: a vendor session killed on its fixed deadline mid-turn reached the user as a
generic lost-session message, so a periodic, expected expiry read as a browser fault. The
finalized row says which happened, and the surfaced error has to say so too."""
ctx = make_copilot_ctx(browser_session_id="pbs_expired")
async def _ensure(recovery_ctx: AgentContext, **_kwargs: Any) -> None:
recovery_ctx.browser_session_id = "pbs_replacement"
monkeypatch.setattr(mcp_adapter, "ensure_browser_session", _ensure)
monkeypatch.setattr(
mcp_adapter,
"app",
SimpleNamespace(
PERSISTENT_SESSIONS_MANAGER=SimpleNamespace(seconds_until_fixed_deadline=_async_return(-3.0)),
CACHE=mcp_adapter.app.CACHE,
),
)
disposition = await _handle_browser_session_loss(
ctx, tool_name="evaluate", call_path="model", lost_session_id="pbs_expired"
)
assert disposition == "reestablished"
assert ctx.browser_session_continuity_deadline_expired is True
# The primary browser tools return only the projection of a shared outcome, so the cause
# has to survive that hop or evaluate and screenshot still report a generic failure.
projected = mcp_adapter._project_browser_call_outcome(
mcp_adapter._not_dispatched_browser_call_outcome(
raw_tool_name="skyvern_evaluate",
source_browser_session_id="pbs_expired",
source_browser_session_generation=0,
raw_result={},
ctx=ctx,
session_loss_disposition=disposition,
),
display_tool_name="evaluate",
)
assert "time limit" in projected["error"]
assert projected["data"]["browser_session_continuity"]["cause"] == "fixed_deadline_expiry"
@pytest.mark.asyncio
async def test_an_unexplained_loss_is_not_named_a_deadline_expiry(self, monkeypatch: pytest.MonkeyPatch) -> None:
"""The cause has to discriminate: a session that died well inside its deadline died for some
other reason, and reporting that as an expected expiry makes the signal meaningless."""
ctx = make_copilot_ctx(browser_session_id="pbs_dead")
async def _ensure(recovery_ctx: AgentContext, **_kwargs: Any) -> None:
recovery_ctx.browser_session_id = "pbs_replacement"
monkeypatch.setattr(mcp_adapter, "ensure_browser_session", _ensure)
monkeypatch.setattr(
mcp_adapter,
"app",
SimpleNamespace(
PERSISTENT_SESSIONS_MANAGER=SimpleNamespace(seconds_until_fixed_deadline=_async_return(742.0)),
CACHE=mcp_adapter.app.CACHE,
),
)
await _handle_browser_session_loss(ctx, tool_name="evaluate", call_path="model", lost_session_id="pbs_dead")
assert ctx.browser_session_continuity_deadline_expired is False
@pytest.mark.asyncio
async def test_observability_failure_does_not_abort_recovery(self, monkeypatch: pytest.MonkeyPatch) -> None:
ctx = make_copilot_ctx(browser_session_id="pbs_log_failure")
async def _ensure(recovery_ctx: AgentContext, **_kwargs: Any) -> None:
recovery_ctx.browser_session_id = "pbs_after_log_failure"
def _raise(*_args: Any, **_kwargs: Any) -> None:
raise RuntimeError("emitter unavailable")
monkeypatch.setattr(mcp_adapter, "ensure_browser_session", _ensure)
monkeypatch.setattr(mcp_adapter, "LOG", SimpleNamespace(warning=_raise, info=_raise))
disposition = await _handle_browser_session_loss(
ctx, tool_name="evaluate", call_path="model", lost_session_id="pbs_log_failure"
)
assert disposition == "reestablished"
assert ctx.browser_session_id == "pbs_after_log_failure"
@pytest.mark.asyncio
async def test_session_loss_adopts_concurrent_replacement(self, monkeypatch: pytest.MonkeyPatch) -> None:
ctx = make_copilot_ctx(browser_session_id="pbs_replacement")
seen_session_ids: list[str | None] = []
async def _ensure(recovery_ctx: AgentContext, **_kwargs: Any) -> None:
seen_session_ids.append(recovery_ctx.browser_session_id)
monkeypatch.setattr(mcp_adapter, "ensure_browser_session", _ensure)
disposition = await _handle_browser_session_loss(
ctx,
tool_name="evaluate",
call_path="model",
lost_session_id="pbs_stale",
)
assert disposition == "reestablished"
assert seen_session_ids == ["pbs_replacement"]
assert ctx.browser_session_id == "pbs_replacement"
assert ctx.browser_session_replacements == {"pbs_stale": "pbs_replacement"}
@pytest.mark.asyncio
async def test_failed_reestablish_sets_honest_terminal_signal(self, monkeypatch: pytest.MonkeyPatch) -> None:
ctx = make_copilot_ctx(browser_session_id="pbs_lost")
async def _ensure(_ctx: AgentContext, **_kwargs: Any) -> dict[str, Any]:
return {"ok": False, "error": "Failed to create browser session"}
monkeypatch.setattr(mcp_adapter, "ensure_browser_session", _ensure)
disposition = await _handle_browser_session_loss(
ctx,
tool_name="evaluate",
call_path="model",
lost_session_id="pbs_lost",
)
assert disposition == "failed"
assert ctx.blocker_signal is not None
assert ctx.blocker_signal.user_facing_reason == (
"The browser session was lost, and I couldn't re-establish it. Please retry this turn."
)
assert ctx.blocker_signal.internal_reason_code == "tool_error_browser_session_lost"
@pytest.mark.asyncio
async def test_session_lost_during_attach_reestablishes(self, monkeypatch: pytest.MonkeyPatch) -> None:
ctx = make_copilot_ctx(browser_session_id="pbs_lost")
async def _ensure(recovery_ctx: AgentContext, **_kwargs: Any) -> None:
if recovery_ctx.browser_session_id is None:
recovery_ctx.browser_session_id = "pbs_replacement"
@asynccontextmanager
async def _lost_scope(_ctx: AgentContext, *, session_id_override: str | None = None) -> AsyncIterator[None]:
raise CopilotBrowserSessionUnavailable("pbs_lost")
yield
monkeypatch.setattr(mcp_adapter, "ensure_browser_session", _ensure)
monkeypatch.setattr(mcp_adapter, "mcp_browser_context", _lost_scope)
server = _make_server(ctx, {"ok": True}, SchemaOverlay(requires_browser=True))
result = await server.call_tool("evaluate", {"expression": "scan()"})
assert "fresh browser session" in result.content[0].text
assert ctx.browser_session_id == "pbs_replacement"
@pytest.mark.asyncio
@pytest.mark.parametrize("call_path", ["model", "internal"])
async def test_attach_uncertainty_returns_a_typed_error_without_claiming_loss(
self, call_path: str, monkeypatch: pytest.MonkeyPatch
) -> None:
ctx = make_copilot_ctx(browser_session_id="pbs_lost")
async def _ensure(_ctx: AgentContext) -> None:
await asyncio.sleep(0)
@asynccontextmanager
async def _undetermined(_ctx: AgentContext, *, session_id_override: str | None = None) -> AsyncIterator[None]:
raise CopilotBrowserLivenessUndetermined()
yield
monkeypatch.setattr(mcp_adapter, "ensure_browser_session", _ensure)
monkeypatch.setattr(mcp_adapter, "mcp_browser_context", _undetermined)
server = _make_server(ctx, {"ok": True}, SchemaOverlay(requires_browser=True))
with capture_logs() as logs:
result = await _call(server, call_path)
assert ctx.blocker_signal is None
assert ctx.browser_session_id == "pbs_lost"
assert ctx.browser_session_replacements == {}
assert "could not be determined" in _surfaced_error(result, call_path)
continuity = [record for record in logs if record.get("event") == "copilot_browser_session_continuity_loss"]
assert continuity == []
timing = _timing_records(logs)
assert len(timing) == 1
assert timing[0]["call_status"] == "error"
@pytest.mark.asyncio
async def test_manager_reestablish_failure_terminalizes_session_loss(self, monkeypatch: pytest.MonkeyPatch) -> None:
ctx = make_copilot_ctx(browser_session_id="pbs_lost")
closed: list[str] = []
async def _manager_times_out(recovery_ctx: AgentContext, **_kwargs: Any) -> None:
recovery_ctx.browser_session_id = "pbs_partial"
raise TimeoutError
async def _close(_organization_id: str, session_id: str) -> None:
closed.append(session_id)
monkeypatch.setattr(mcp_adapter, "ensure_browser_session", _manager_times_out)
monkeypatch.setattr(mcp_adapter, "close_browser_session_quietly", _close)
disposition = await _handle_browser_session_loss(
ctx,
tool_name="evaluate",
call_path="model",
lost_session_id="pbs_lost",
)
assert disposition == "failed"
assert closed == ["pbs_lost", "pbs_partial"]
assert ctx.browser_session_id is None
assert ctx.blocker_signal is not None
assert ctx.blocker_signal.internal_reason_code == "tool_error_browser_session_lost"
@pytest.mark.asyncio
@pytest.mark.parametrize("call_path", ["model", "internal"])
async def test_tool_result_requires_fresh_perception_after_reestablish(
self, call_path: str, monkeypatch: pytest.MonkeyPatch, _fake_clock: list[float]
) -> None:
ctx = make_copilot_ctx(browser_session_id="pbs_lost")
observed: list[_BrowserCallOutcome] = []
real_project = mcp_adapter._project_browser_call_outcome
def _capture(outcome: _BrowserCallOutcome, *, display_tool_name: str) -> dict[str, Any]:
observed.append(outcome)
return real_project(outcome, display_tool_name=display_tool_name)
async def _ensure(recovery_ctx: AgentContext, **_kwargs: Any) -> None:
if recovery_ctx.browser_session_id is None:
recovery_ctx.browser_session_id = "pbs_replacement"
@asynccontextmanager
async def _scope(_ctx: AgentContext, *, session_id_override: str | None = None) -> AsyncIterator[None]:
yield
monkeypatch.setattr(mcp_adapter, "ensure_browser_session", _ensure)
monkeypatch.setattr(mcp_adapter, "mcp_browser_context", _scope)
monkeypatch.setattr(mcp_adapter, "_project_browser_call_outcome", _capture)
payload = {
"ok": False,
"error": {
"code": "SESSION_EXPIRED",
"message": "Browser session expired or closed.",
"hint": "Create a new browser session and retry this operation.",
},
}
server = _make_server(
ctx,
payload,
SchemaOverlay(requires_browser=True),
alias_map=get_skyvern_mcp_alias_map(),
)
result = await _call(server, call_path)
result_text = result.content[0].text if isinstance(result, CallToolResult) else json.dumps(result)
assert "SESSION_EXPIRED" in result_text
assert "fresh browser session" in result_text
assert "inspect the page" in result_text
assert ctx.browser_session_id == "pbs_replacement"
assert len(observed) == 1
assert observed[0].source_browser_session_id == "pbs_lost"
assert observed[0].session_loss_disposition == "reestablished"
assert observed[0].replacement_browser_session_id == "pbs_replacement"
@pytest.mark.asyncio
async def test_code_block_ai_fallback_does_not_enter_interactive_reestablish(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
ctx = make_copilot_ctx(browser_session_id="self-heal:wr_test", turn_origin=TurnOrigin.code_block_ai_fallback)
async def _ensure(_ctx: AgentContext, **_kwargs: Any) -> None:
return None
@asynccontextmanager
async def _scope(_ctx: AgentContext, *, session_id_override: str | None = None) -> AsyncIterator[None]:
yield
monkeypatch.setattr(mcp_adapter, "ensure_browser_session", _ensure)
monkeypatch.setattr(mcp_adapter, "mcp_browser_context", _scope)
server = _make_server(
ctx,
{
"ok": False,
"error": {"code": "SESSION_EXPIRED", "message": "Browser session expired or closed."},
},
SchemaOverlay(requires_browser=True),
)
result = await server.call_tool("evaluate", {"expression": "scan()"})
assert "fresh browser session" not in result.content[0].text
assert ctx.browser_session_replacements == {}
assert ctx.blocker_signal is None
def test_reestablish_lock_budget_is_derived_from_the_managers_startup_bound(monkeypatch: pytest.MonkeyPatch) -> None:
manager = MagicMock()
manager.get_browser_session_startup_timeout_seconds.return_value = 55.0
monkeypatch.setattr(mcp_adapter.app, "PERSISTENT_SESSIONS_MANAGER", manager)
assert mcp_adapter._reestablish_lock_seconds() == 85
assert not hasattr(mcp_adapter, "_SESSION_REESTABLISH_TIMEOUT_SECONDS")
def _binding_ctx(
*, debug: str | None, run: str | None, run_id: str | None = "wr_1", unavailable_reason: str | None = None
) -> SimpleNamespace:
return SimpleNamespace(
browser_session_id=debug,
last_run_blocks_browser_session_id=run,
last_run_blocks_workflow_run_id=run_id,
last_run_binding_unavailable_reason=unavailable_reason,
)
def test_default_target_binds_the_debug_browser() -> None:
binding = resolve_browser_session_binding(_binding_ctx(debug="pbs_debug", run="pbs_run"), {})
assert (binding.target, binding.session_id_override) == ("debug", None)
assert binding.provenance() == {"browser_target": "debug", "source_matches_target": True}
def test_last_run_target_binds_the_run_browser() -> None:
binding = resolve_browser_session_binding(_binding_ctx(debug="pbs_debug", run="pbs_run"), {"target": "last_run"})
assert binding.session_id_override == "pbs_run"
assert binding.provenance()["browser_target_workflow_run_id"] == "wr_1"
assert binding.source_matches_target is True
def test_last_run_without_a_recorded_run_session_is_disclosed_not_silently_served() -> None:
# last_run is a promise about which browser answered, so an unknown run session has to be
# reported rather than quietly satisfied from the debug browser.
binding = resolve_browser_session_binding(_binding_ctx(debug="pbs_debug", run=None), {"target": "last_run"})
assert binding.session_id_override is None
assert binding.source_matches_target is False
assert "No test run" in binding.provenance()["browser_target_unavailable"]
def test_a_refused_last_run_binding_states_why_rather_than_that_none_exists() -> None:
binding = resolve_browser_session_binding(
_binding_ctx(debug="pbs_debug", run=None, unavailable_reason="The last run this chat recorded is gone."),
{"target": "last_run"},
)
assert binding.session_id_override is None
assert binding.unavailable_reason == "The last run this chat recorded is gone."
def test_an_unknown_target_value_is_refused_rather_than_served_from_the_debug_browser() -> None:
# Coercing an off-enum target to the chat's browser would run a click or a fill in a browser the
# model did not name, and stamp the result as if it had landed where it was aimed.
binding = resolve_browser_session_binding(_binding_ctx(debug="pbs_debug", run="pbs_run"), {"target": "whatever"})
assert binding.session_id_override is None
assert binding.source_matches_target is False
assert binding.unavailable_reason is not None
assert "whatever" in binding.unavailable_reason
def test_a_run_that_shared_the_chats_browser_binds_no_override() -> None:
# There is nothing to redirect, and an override would pin the call to a recorded id that a later
# session re-establishment leaves behind.
binding = resolve_browser_session_binding(_binding_ctx(debug="pbs_same", run="pbs_same"), {"target": "last_run"})
assert (binding.target, binding.session_id_override) == ("last_run", None)
assert binding.unavailable_reason is None
def test_every_browser_overlay_offers_the_target_param() -> None:
overlays = _build_skyvern_mcp_overlays(BlockAuthoringPolicy.CODE_ONLY_BROWSER)
browser_overlays = {name: overlay for name, overlay in overlays.items() if overlay.requires_browser}
assert {
"skyvern_frame_list",
"skyvern_frame_switch",
"skyvern_frame_main",
"skyvern_tab_list",
"skyvern_tab_new",
"skyvern_tab_switch",
"skyvern_tab_close",
} <= set(browser_overlays)
assert len(browser_overlays) == 18
assert all(BROWSER_TARGET_PARAM_NAME in overlay.copilot_params for overlay in browser_overlays.values())
# inspect_page_for_composition owns target_url semantics; a second target would overload it.
assert "inspect_page_for_composition" not in browser_overlays
def test_the_target_param_never_reaches_the_underlying_tool() -> None:
overlay = _build_skyvern_mcp_overlays(BlockAuthoringPolicy.CODE_ONLY_BROWSER)["evaluate"]
mcp_args = _transform_args({"expression": "1+1", BROWSER_TARGET_PARAM_NAME: "last_run"}, overlay)
assert mcp_args == {"expression": "1+1"}
def test_a_dead_targeted_browser_does_not_tear_down_the_chats_session() -> None:
# Continuity recovery closes the lost browser and rebinds the chat to a replacement. Routing a
# targeted browser's death through it would destroy the page the model asked to look at and
# swap the chat's browser because a browser it does not own died.
ctx = SimpleNamespace(
organization_id="org",
browser_session_id="pbs_chat",
browser_session_replacements={},
)
async def _run() -> str:
with bound_call_browser_session("pbs_run"):
return await _handle_browser_session_loss(
ctx, # type: ignore[arg-type]
tool_name="evaluate",
call_path="model",
lost_session_id="pbs_run",
)
assert asyncio.run(_run()) == "failed"
assert ctx.browser_session_id == "pbs_chat"
assert ctx.browser_session_replacements == {}
@pytest.mark.asyncio
async def test_internal_probes_inside_a_targeted_call_read_the_targeted_browser(
monkeypatch: pytest.MonkeyPatch,
) -> None:
# A tool call is a dispatch wrapped in pre/post-hook probes. Sending the dispatch to the run's
# browser while the probes read the chat's would verify the action against a different page and
# stamp the result with the browser it did not observe.
ctx = make_copilot_ctx(browser_session_id="pbs_chat")
server = _make_server(
ctx,
{"ok": True},
SchemaOverlay(requires_browser=True),
alias_map=get_skyvern_mcp_alias_map(),
)
dispatched: list[dict[str, Any]] = []
scoped: list[str | None] = []
class _CapturingClient:
async def call_tool(self, name: str, args: dict[str, Any], raise_on_error: bool = False) -> Any:
dispatched.append(dict(args))
return SimpleNamespace(structured_content={"ok": True}, is_error=False, content=[])
@asynccontextmanager
async def _scope(_ctx: AgentContext, *, session_id_override: str | None = None) -> AsyncIterator[None]:
scoped.append(session_id_override)
yield
server._client = _CapturingClient()
monkeypatch.setattr(mcp_adapter, "mcp_browser_context", _scope)
with bound_call_browser_session("pbs_run"):
result = await server.call_internal_tool("skyvern_evaluate", {"expression": "scan()"})
assert result["ok"] is True
assert dispatched == [{"expression": "scan()", "session_id": "pbs_run"}]
assert scoped == ["pbs_run"]
assert ctx.browser_session_id == "pbs_chat"
@pytest.mark.asyncio
async def test_a_targeted_call_is_not_gated_on_the_chats_own_browser() -> None:
# The chat's continuity is not a precondition for a call acting in a browser the chat does not
# own; a dead debug browser must not refuse a look at the run's page.
ctx = make_copilot_ctx(browser_session_id="pbs_chat")
with bound_call_browser_session("pbs_run"):
prepared = await mcp_adapter._prepare_browser_session_for_dispatch(
ctx,
tool_name="evaluate",
call_path="model",
observed_generation=ctx.browser_session_continuity_generation,
)
assert prepared == (None, None, None)
assert ctx.browser_session_id == "pbs_chat"
@pytest.mark.asyncio
async def test_a_call_aimed_at_an_unavailable_browser_never_dispatches() -> None:
"""The no-silent-fallback property: a click or a fill aimed at the run's browser must not land
in the chat's, so the refusal has to happen before the underlying tool is reached."""
dispatched: list[bool] = []
# The api key matters: without one the call cannot dispatch anyway, and the dispatch assertion
# below would hold whether or not the refusal exists.
ctx = make_copilot_ctx(browser_session_id="pbs_chat", api_key="sk-test")
ctx.last_run_blocks_browser_session_id = None
ctx.last_run_blocks_workflow_run_id = None
server = _make_server(
ctx,
{"ok": True},
SchemaOverlay(requires_browser=True),
on_call=lambda: dispatched.append(True),
)
result = await server.call_tool("evaluate", {"expression": "1+1", BROWSER_TARGET_PARAM_NAME: "last_run"})
assert dispatched == []
payload = json.loads(result.content[0].text)
assert payload["ok"] is False
# The refusal names the target it could not honour; a later failure would not.
assert payload.get("browser_target") == "last_run"
assert "recorded a browser" in payload["error"]
def _overlay_server(
ctx: AgentContext,
client: _FakeClient,
tool_name: str,
overlay: SchemaOverlay,
page_state_reader: PageStateReader | None = None,
) -> SkyvernOverlayMCPServer:
aliases = get_skyvern_mcp_alias_map()
server = SkyvernOverlayMCPServer(
transport=MagicMock(),
overlays={tool_name: overlay},
alias_map={tool_name: aliases[tool_name]},
allowlist=frozenset({aliases[tool_name]}),
context_provider=lambda: ctx,
page_state_reader=page_state_reader,
)
server._client = client
return server
def _click_overlay_server(ctx: AgentContext, payload: dict[str, Any]) -> SkyvernOverlayMCPServer:
return _overlay_server(ctx, _FakeClient(payload), "click", _build_skyvern_mcp_overlays()["click"])
@pytest.mark.asyncio
async def test_staged_click_uses_normal_adapter_path_without_browser_dispatch(_stub_browser_session: None) -> None:
dispatched: list[bool] = []
ctx = make_copilot_ctx(browser_session_id="pbs_1")
server = _overlay_server(
ctx,
_FakeClient({"ok": False}, on_call=lambda: dispatched.append(True)),
"click",
_build_skyvern_mcp_overlays()["click"],
)
action = DemonstratedAction(
selector="#continue",
source_url="https://example.test/form",
packet={
"role_name": {"role": "button", "accessible_name": "Continue"},
"selector_match_count": 1,
"role_name_match_count": 1,
},
synthesized_result={
"ok": True,
"data": {"selector": "#continue"},
"browser_context": {"url": "https://example.test/next", "title": ""},
},
)
with staged_demonstrated_action(action):
result = await server.call_tool("click", {"selector": "#continue"})
assert result.isError is False
assert dispatched == []
interaction = ctx.scout_trajectory[-1]
assert interaction["actor"] == "user"
assert interaction["source_url"] == "https://example.test/form"
assert (interaction["role"], interaction["accessible_name"]) == ("button", "Continue")
@pytest.mark.asyncio
async def test_internal_browser_call_refuses_during_demonstration_replay() -> None:
ctx = make_copilot_ctx(browser_session_id="pbs_1")
server = _click_overlay_server(ctx, {"ok": True})
action = DemonstratedAction(
selector="#continue",
source_url="https://example.test/start",
packet=None,
synthesized_result={"ok": True},
)
with staged_demonstrated_action(action):
result = await server._call_internal_tool("skyvern_click", {"selector": "#continue"})
assert result.result == {"ok": False, "error": "Browser access is unavailable while replaying a demonstration."}
def _real_action_overlay(overlay_id: str) -> tuple[str, SchemaOverlay]:
if overlay_id == "click_no_browser":
return "click", replace(_build_skyvern_mcp_overlays()["click"], requires_browser=False)
return overlay_id, _build_skyvern_mcp_overlays()[overlay_id]
class _HeldClient(_FakeClient):
def __init__(self, payload: dict[str, Any]) -> None:
super().__init__(payload)
self.dispatched: list[str] = []
self.started = asyncio.Event()
self.release = asyncio.Event()
self.cancelled = False
async def call_tool(self, name: str, args: dict[str, Any], raise_on_error: bool = False) -> _FakeRawResult:
self.dispatched.append(name)
self.started.set()
try:
await self.release.wait()
except asyncio.CancelledError:
self.cancelled = True
raise
return await super().call_tool(name, args, raise_on_error)
async def _answer_after_loop_seconds(
server: SkyvernOverlayMCPServer, client: _HeldClient, tool_name: str, seconds: float
) -> CallToolResult:
loop = asyncio.get_running_loop()
pending = asyncio.create_task(server.call_tool(tool_name, _ACTION_ARGS[tool_name]))
await asyncio.wait_for(client.started.wait(), timeout=5)
real_time = loop.time
# Moves the event loop's own clock, which asyncio.wait_for schedules against, without a real wait.
with pytest.MonkeyPatch.context() as patch:
patch.setattr(loop, "time", lambda: real_time() + seconds)
for _ in range(3):
await asyncio.sleep(0)
client.release.set()
return await pending
def _sensitive_click_ctx() -> AgentContext:
ctx = make_copilot_ctx(browser_session_id="pbs_1")
clear_session_scrub_values("pbs_1")
ctx.last_run_blocks_workflow_run_id = "wr_sensitive"
ctx.origin_run_redaction_registry = OriginRunRedactionRegistry(
"wr_sensitive",
{"copilot_run_runtime_secret_values": ("654321",)},
contains_sensitive_values=True,
contains_all_sensitive_values=True,
)
register_secret_scrub_values_from_structure(ctx, ctx.origin_run_redaction_registry.parameters)
return ctx
class TestSensitiveOriginActionContinuation:
@pytest.mark.asyncio
async def test_click_discloses_a_scrubbed_result_after_a_terminal_matching_run(
self, _stub_browser_session: None
) -> None:
ctx = _sensitive_click_ctx()
server = _click_overlay_server(
ctx,
{
"ok": True,
"data": {
"selector": "a.nav--analytics",
"text_content": "Web analytics 654321",
"url": "https://example.test/app",
},
},
)
result = await server.call_tool("click", {"selector": "a.nav--analytics"})
surfaced = json.loads(result.content[0].text)
assert surfaced["ok"] is True
assert "654321" not in result.content[0].text
assert "[REDACTED_SECRET]" in result.content[0].text
_SCHEDULE_TOOLS = (
"list_workflow_schedules",
"get_workflow_schedule",
"create_workflow_schedule",
"update_workflow_schedule",
"enable_workflow_schedule",
"disable_workflow_schedule",
"cancel_workflow_schedule",
"delete_workflow_schedule",
)
def _schedule_server(
monkeypatch: pytest.MonkeyPatch,
ctx: AgentContext,
payload: dict[str, Any],
*,
definition: dict[str, Any] | None = None,
stored_parameters: dict[str, Any] | None = None,
) -> SkyvernOverlayMCPServer:
database = MagicMock()
database.workflows.get_workflow_by_permanent_id = AsyncMock(
return_value=SimpleNamespace(workflow_definition=definition or {"parameters": [], "blocks": []})
)
database.schedules.get_workflow_schedule_by_id = AsyncMock(
return_value=SimpleNamespace(parameters=stored_parameters)
)
database.workflow_params.get_workflow_copilot_chat_by_id = AsyncMock(return_value=None)
monkeypatch.setattr(mcp_hooks.app, "DATABASE", database)
server = _make_server(ctx, payload, SchemaOverlay(), alias_map=get_skyvern_mcp_alias_map())
server._overlays = _build_skyvern_mcp_overlays()
return server
async def _listed_tools(browser_tools_available: bool) -> tuple[dict[str, Any], dict[str, str]]:
surface = resolve_copilot_tool_surface(
mode=None,
native_tools=list(NATIVE_TOOLS),
alias_map=get_skyvern_mcp_alias_map(),
overlays=_build_skyvern_mcp_overlays(),
browser_tools_available=browser_tools_available,
)
server = SkyvernOverlayMCPServer(
transport=mcp,
overlays=surface.overlays,
alias_map=surface.alias_map,
allowlist=frozenset(surface.alias_map.values()),
context_provider=lambda: make_copilot_ctx(api_key="sk-copilot-org"),
)
await server.connect()
try:
return {tool.name: tool for tool in await server.list_tools()}, surface.alias_map
finally:
await server.cleanup()
@pytest.mark.asyncio
async def test_chat_surface_exposes_only_chat_bound_workflow_schedule_tools() -> None:
tools, alias_map = await _listed_tools(browser_tools_available=True)
assert sorted(name for name in tools if "schedule" in name) == sorted(_SCHEDULE_TOOLS)
assert "skyvern_schedule_list" not in alias_map.values()
for name in _SCHEDULE_TOOLS:
assert "workflow_permanent_id" not in tools[name].inputSchema["properties"]
assert "exact" not in tools[name].inputSchema["properties"]
assert "force" in tools["delete_workflow_schedule"].inputSchema["properties"]
for name in ("create_workflow_schedule", "update_workflow_schedule"):
assert {"cron_expression", "interval_seconds", "first_fire_at", "run_at"} <= set(
tools[name].inputSchema["properties"]
)
assert "cron_expression" not in tools["create_workflow_schedule"].inputSchema.get("required", [])
@pytest.mark.asyncio
async def test_browserless_surface_can_read_but_not_change_schedules() -> None:
tools, _ = await _listed_tools(browser_tools_available=False)
assert sorted(name for name in tools if "schedule" in name) == ["get_workflow_schedule", "list_workflow_schedules"]
@pytest.mark.asyncio
@pytest.mark.parametrize(
("browser_tools_available", "expected"),
[
(True, ["create_browser_profile", "get_browser_profile", "list_browser_profiles"]),
(False, ["get_browser_profile", "list_browser_profiles"]),
],
)
async def test_profile_create_needs_run_authority_and_takes_exactly_the_shared_sources(
browser_tools_available: bool, expected: list[str]
) -> None:
tools, _ = await _listed_tools(browser_tools_available=browser_tools_available)
assert sorted(name for name in tools if "profile" in name) == expected
if browser_tools_available:
schema = tools["create_browser_profile"].inputSchema
assert set(schema["properties"]) - {USER_FACING_REASON_PARAM} == {
"name",
"description",
"browser_session_id",
"workflow_run_id",
}
assert schema["required"] == ["name", USER_FACING_REASON_PARAM]
@pytest.mark.asyncio
@pytest.mark.parametrize("tool_name", _SCHEDULE_TOOLS)
async def test_schedule_tools_always_target_the_chat_workflow(tool_name: str, monkeypatch: pytest.MonkeyPatch) -> None:
ctx = make_copilot_ctx(workflow_permanent_id="wpid_chat")
server = _schedule_server(monkeypatch, ctx, {"ok": True, "data": {}})
await server.call_tool(tool_name, {"workflow_permanent_id": "wpid_foreign", "workflow_schedule_id": "wfs_1"})
assert server._client.calls == [
(
get_skyvern_mcp_alias_map()[tool_name],
{"workflow_schedule_id": "wfs_1", "workflow_permanent_id": "wpid_chat"},
)
]
@pytest.mark.asyncio
@pytest.mark.parametrize("browser_tools_available", [True, False])
async def test_run_list_is_offered_without_a_workflow_id_and_with_a_status_filter(
browser_tools_available: bool,
) -> None:
tools, _ = await _listed_tools(browser_tools_available=browser_tools_available)
properties = tools["list_workflow_runs"].inputSchema["properties"]
assert "workflow_id" not in properties
assert "search_key" not in properties
assert "error_code" not in properties
assert "status" in properties
assert "workflow_id" not in tools["list_workflow_runs"].inputSchema.get("required", [])
@pytest.mark.asyncio
async def test_run_list_always_targets_the_chat_workflow(monkeypatch: pytest.MonkeyPatch) -> None:
ctx = make_copilot_ctx(workflow_permanent_id="wpid_chat")
server = _schedule_server(monkeypatch, ctx, {"ok": True, "data": {"runs": []}})
await server.call_tool("list_workflow_runs", {"workflow_id": "wpid_foreign", "status": ["terminated"]})
assert server._client.calls == [
("skyvern_workflow_run_list", {"workflow_id": "wpid_chat", "status": ["terminated"]}),
]
@pytest.mark.asyncio
async def test_run_list_shows_the_model_only_the_fields_that_pick_a_run(monkeypatch: pytest.MonkeyPatch) -> None:
ctx = make_copilot_ctx(workflow_permanent_id="wpid_chat")
run = {
"run_id": "wr_1",
"status": "terminated",
"created_at": "2026-09-24T19:30:00Z",
"trigger_type": "scheduled",
"copilot_session_id": None,
"run_type": "workflow_run",
"failure_reason": "Login rejected for password hunter2-unseen",
"output_summary": {"present": True, "scalar_preview": {"api_token": "hunter2-unseen"}},
"artifact_summary": {"downloaded_file_names": ["hunter2-unseen.pdf"]},
"run_with": "agent",
}
server = _schedule_server(monkeypatch, ctx, {"ok": True, "data": {"runs": [run], "count": 1, "has_more": False}})
result = await server.call_tool("list_workflow_runs", {"status": ["terminated"]})
surfaced = json.loads(result.content[0].text)
assert surfaced["data"]["runs"] == [
{"run_id": "wr_1", "status": "terminated", "created_at": "2026-09-24T19:30:00Z", "trigger_type": "scheduled"}
]
assert "hunter2-unseen" not in result.content[0].text
@pytest.mark.asyncio
@pytest.mark.parametrize(
("tool_name", "args", "data"),
[
(
"list_workflow_runs",
{"status": ["terminated"]},
{"runs": [{"run_id": "wr_1", "failure_reason": "password hunter2-unseen"}]},
),
(
"get_workflow_schedule",
{"workflow_schedule_id": "wfs_1"},
{"workflow_schedule_id": "wfs_1", "parameters": {"password": "hunter2-unseen"}},
),
],
)
async def test_a_crashed_stored_value_filter_withholds_the_result(
tool_name: str, args: dict[str, Any], data: dict[str, Any], monkeypatch: pytest.MonkeyPatch
) -> None:
ctx = make_copilot_ctx(workflow_permanent_id="wpid_chat")
server = _schedule_server(monkeypatch, ctx, {"ok": True, "data": data})
server._overlays[tool_name] = replace(server._overlays[tool_name], post_hook=_crash)
result = await server.call_tool(tool_name, args)
assert result.isError is True
text = " ".join(content.text for content in result.content)
assert "withheld" in text
assert "hunter2-unseen" not in text
@pytest.mark.asyncio
async def test_schedule_create_is_refused_while_the_turn_holds_an_unsaved_proposal(
monkeypatch: pytest.MonkeyPatch,
) -> None:
ctx = make_copilot_ctx(workflow_permanent_id="wpid_chat", has_staged_proposal=True)
server = _schedule_server(monkeypatch, ctx, {"ok": True, "data": {"schedule": {"workflow_schedule_id": "wfs_1"}}})
args = {"cron_expression": "0 9 * * *", "timezone": "America/Los_Angeles"}
refused = await server.call_tool("create_workflow_schedule", args)
assert refused.isError is True
assert "wfs_" not in refused.content[0].text
assert server._client.calls == []
ctx.has_staged_proposal = False
created = await server.call_tool("create_workflow_schedule", args)
assert created.isError is False
assert [name for name, _ in server._client.calls] == ["skyvern_schedule_create"]
@pytest.mark.asyncio
@pytest.mark.parametrize("tool_name", ["create_workflow_schedule", "update_workflow_schedule"])
@pytest.mark.parametrize(
("unapproved_id", "approved_id", "asks_for_account"),
[("cred_never_cited", "cred_saved", False), ("goac_never_chosen", "goac_chosen", True)],
)
async def test_schedule_parameters_need_the_same_credential_approval_as_a_run(
tool_name: str,
unapproved_id: str,
approved_id: str,
asks_for_account: bool,
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(mcp_hooks, "_server_verified_google_account_choices", AsyncMock(return_value=None))
ctx = make_copilot_ctx(
workflow_permanent_id="wpid_chat",
request_policy=RequestPolicy(
persisted_workflow_credential_ids=["cred_saved"],
run_approved_google_connection_ids=["goac_chosen"],
),
)
server = _schedule_server(monkeypatch, ctx, {"ok": True, "data": {"schedule": {"workflow_schedule_id": "wfs_1"}}})
args = {"workflow_schedule_id": "wfs_1", "cron_expression": "0 9 * * *", "timezone": "America/Los_Angeles"}
refused = await server.call_tool(tool_name, {**args, "parameters": {"login": unapproved_id}})
assert refused.isError is True
assert server._client.calls == []
assert (ctx.blocker_signal is not None) is asks_for_account
admitted = await server.call_tool(tool_name, {**args, "parameters": {"login": approved_id}})
assert admitted.isError is False
assert [name for name, _ in server._client.calls] == [get_skyvern_mcp_alias_map()[tool_name]]
def _sheets_definition(credential_id: str) -> dict[str, Any]:
return {
"parameters": [],
"blocks": [{"label": "write", "block_type": "google_sheets_write", "credential_id": credential_id}],
}
@pytest.mark.asyncio
@pytest.mark.parametrize(
"tool_name", ["create_workflow_schedule", "update_workflow_schedule", "enable_workflow_schedule"]
)
async def test_schedule_needs_approval_for_google_references_in_the_saved_workflow(
tool_name: str,
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(mcp_hooks, "_server_verified_google_account_choices", AsyncMock(return_value=None))
ctx = make_copilot_ctx(workflow_permanent_id="wpid_chat", request_policy=RequestPolicy())
args = {"workflow_schedule_id": "wfs_1", "cron_expression": "0 9 * * *", "timezone": "America/Los_Angeles"}
payload = {"ok": True, "data": {"schedule": {"workflow_schedule_id": "wfs_1"}}}
server = _schedule_server(monkeypatch, ctx, payload, definition=_sheets_definition("{{ sheet_account }}"))
refused = await server.call_tool(tool_name, args)
assert refused.isError is True
assert server._client.calls == []
assert ctx.blocker_signal is not None
ctx = make_copilot_ctx(workflow_permanent_id="wpid_chat", request_policy=RequestPolicy())
server = _schedule_server(monkeypatch, ctx, payload)
admitted = await server.call_tool(tool_name, args)
assert admitted.isError is False
assert [name for name, _ in server._client.calls] == [get_skyvern_mcp_alias_map()[tool_name]]
@pytest.mark.asyncio
async def test_schedule_enable_rechecks_the_stored_parameters(monkeypatch: pytest.MonkeyPatch) -> None:
ctx = make_copilot_ctx(workflow_permanent_id="wpid_chat", request_policy=RequestPolicy())
server = _schedule_server(
monkeypatch,
ctx,
{"ok": True, "data": {}},
stored_parameters={"login": "cred_never_cited"},
)
refused = await server.call_tool("enable_workflow_schedule", {"workflow_schedule_id": "wfs_1"})
assert refused.isError is True
assert server._client.calls == []
_BUSY_PAGE_HTML = (
"<html><head><title>Busy</title></head><body><script>setTimeout(() => { for (;;) {} }, 50)</script></body></html>"
)
# A cross-site embedder is its own renderer under site isolation, so its busy loop stalls only the probe
# that measures the vendor frame it holds, not the top page's title.
_BUSY_EMBEDDER_ORIGIN = "https://busy-embedder.test"
_BUSY_VENDOR_EMBEDDER_HTML = (
'<iframe src="https://challenges.cloudflare.com/cdn-cgi/challenge-platform/turnstile/x" '
'style="width:300px;height:65px"></iframe>'
'<script>window.addEventListener("load", () => setTimeout(() => { for (;;) {} }, 100))</script>'
)
_ACCOUNT_TITLED_PAGE_HTML = "<html><head><title>Orders for acct-778</title></head><body></body></html>"
_CATALOG_PAGE_STATE = {"read": "ok", "url": None, "title": "Catalog", "challenge_vendor": None}
_UNREAD_PAGE_STATE = {"read": "failed", "url": None, "title": None, "challenge_vendor": None}
def _page_state(result: CallToolResult) -> dict[str, Any]:
payload = json.loads(result.content[0].text)
fields = [key for key in payload if key != MCP_RESULT_PROVENANCE_KEY]
assert fields[0] == "page_state", "page_state leads so a head-truncated result keeps it"
return payload["page_state"]
def _browser_evaluate_server(
ctx: AgentContext, payload: dict[str, Any] | Exception | None = None, overlay: SchemaOverlay | None = None
) -> SkyvernOverlayMCPServer:
return _make_server(
ctx,
payload or {"ok": True},
overlay or SchemaOverlay(requires_browser=True),
page_state_reader=scouting_module.read_page_state_on_page,
)
async def _crash(_result: dict[str, Any], _raw: dict[str, Any], _ctx: AgentContext) -> dict[str, Any]:
raise RuntimeError("post-hook crashed")
async def _passthrough(result: dict[str, Any], _raw: dict[str, Any], _ctx: AgentContext) -> dict[str, Any]:
return result
async def _refuse(_args: dict[str, Any], _ctx: AgentContext) -> dict[str, Any]:
return {"ok": False, "error": "refused before dispatch"}
async def _crash_before_dispatch(_args: dict[str, Any], _ctx: AgentContext) -> None:
raise RuntimeError("pre-hook crashed")
@pytest.mark.usefixtures("_stub_browser_session")
class TestPageStateOnBrowserResults:
@skip_no_browser
@pytest.mark.asyncio
async def test_a_rendered_turnstile_frame_is_named_and_a_plain_page_states_none(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
async with challenge_browser_page(AUTO_RENDER_TURNSTILE_HTML) as page:
patch_browser_tabs(monkeypatch, FakeTabbedBrowserState(page))
server = _browser_evaluate_server(make_copilot_ctx(browser_session_id="pbs_1"))
challenged = _page_state(await server.call_tool("evaluate", {"expression": "1"}))
async with challenge_browser_page(CATALOG_PAGE_HTML, expect_challenge_frame=False) as page:
patch_browser_tabs(monkeypatch, FakeTabbedBrowserState(page))
server = _browser_evaluate_server(make_copilot_ctx(browser_session_id="pbs_1"))
cleared = _page_state(await server.call_tool("evaluate", {"expression": "1"}))
async with challenge_browser_page(HIDDEN_TURNSTILE_HELPER_HTML) as page:
patch_browser_tabs(monkeypatch, FakeTabbedBrowserState(page))
server = _browser_evaluate_server(make_copilot_ctx(browser_session_id="pbs_1"))
hidden_helper = _page_state(await server.call_tool("evaluate", {"expression": "1"}))
assert challenged["read"] == "ok"
assert challenged["challenge_vendor"] == "challenges.cloudflare"
assert cleared == _CATALOG_PAGE_STATE
assert hidden_helper == _CATALOG_PAGE_STATE
@pytest.mark.asyncio
async def test_a_vendor_frame_under_an_unprobeable_embedder_is_reported_unread(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
patch_browser_tabs(monkeypatch, FakeTabbedBrowserState(page_with_unprobeable_vendor_embedder()))
server = _browser_evaluate_server(make_copilot_ctx(browser_session_id="pbs_1"))
result = await server.call_tool("evaluate", {"expression": "1"})
assert _page_state(result) == _UNREAD_PAGE_STATE
@skip_no_browser
@pytest.mark.asyncio
async def test_a_navigation_during_the_read_is_reported_unread(self, monkeypatch: pytest.MonkeyPatch) -> None:
async with challenge_browser_page(
site={"/": AUTO_RENDER_TURNSTILE_HTML, "/catalog": CATALOG_PAGE_HTML}
) as page:
read_title = page.title
async def title_after_redirect() -> str:
await page.goto(f"{FIXTURE_SITE_ORIGIN}/catalog", wait_until="load")
return await read_title()
monkeypatch.setattr(page, "title", title_after_redirect)
patch_browser_tabs(monkeypatch, FakeTabbedBrowserState(page))
server = _browser_evaluate_server(make_copilot_ctx(browser_session_id="pbs_1"))
result = await server.call_tool("evaluate", {"expression": "1"})
assert _page_state(result) == _UNREAD_PAGE_STATE
@skip_no_browser
@pytest.mark.asyncio
@pytest.mark.parametrize(
("payload", "overlay"),
[
pytest.param(RuntimeError("socket closed"), None, id="protocol-error"),
pytest.param(None, SchemaOverlay(requires_browser=True, post_hook=_crash), id="crashed-post-hook"),
pytest.param(None, SchemaOverlay(requires_browser=True, pre_hook=_refuse), id="refusing-pre-hook"),
pytest.param(
None, SchemaOverlay(requires_browser=True, pre_hook=_crash_before_dispatch), id="crashed-pre-hook"
),
],
)
async def test_every_exit_states_the_page(
self, monkeypatch: pytest.MonkeyPatch, payload: Exception | None, overlay: SchemaOverlay | None
) -> None:
async with challenge_browser_page(CATALOG_PAGE_HTML, expect_challenge_frame=False) as page:
patch_browser_tabs(monkeypatch, FakeTabbedBrowserState(page))
server = _browser_evaluate_server(make_copilot_ctx(browser_session_id="pbs_1"), payload, overlay)
result = await server.call_tool("evaluate", {"expression": "1"})
assert _page_state(result) == _CATALOG_PAGE_STATE
@skip_no_browser
@pytest.mark.asyncio
async def test_a_failed_click_states_the_page_without_its_query_or_fragment(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
async with challenge_browser_page(
site={"/catalog": CATALOG_PAGE_HTML},
path="/catalog?session=qv-5521#step-qv-5521",
expect_challenge_frame=False,
) as page:
patch_browser_tabs(monkeypatch, FakeTabbedBrowserState(page))
server = _overlay_server(
make_copilot_ctx(browser_session_id="pbs_1"),
_FakeClient({"ok": False, "error": "No element matches #go"}),
"click",
_build_skyvern_mcp_overlays()["click"],
page_state_reader=scouting_module.read_page_state_on_page,
)
result = await server.call_tool("click", {"selector": "#go"})
assert _page_state(result)["url"] == f"{FIXTURE_SITE_ORIGIN}/catalog"
assert "qv-5521" not in result.content[0].text
@skip_no_browser
@pytest.mark.asyncio
async def test_a_page_whose_renderer_never_yields_is_reported_unread(self, monkeypatch: pytest.MonkeyPatch) -> None:
async with challenge_browser_page(_BUSY_PAGE_HTML, expect_challenge_frame=False) as page:
await asyncio.sleep(0.2)
patch_browser_tabs(monkeypatch, FakeTabbedBrowserState(page))
server = _browser_evaluate_server(make_copilot_ctx(browser_session_id="pbs_1"))
result = await server.call_tool("evaluate", {"expression": "1"})
assert result.isError is False
assert _page_state(result) == _UNREAD_PAGE_STATE
@skip_no_browser
@pytest.mark.asyncio
async def test_a_vendor_frame_that_cannot_be_measured_is_not_reported_as_no_challenge(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
async def serve_busy_embedder(route: Route) -> None:
await route.fulfill(status=200, content_type="text/html", body=_BUSY_VENDOR_EMBEDDER_HTML)
async with challenge_browser_page(
site={"/": f'<title>Catalog</title><iframe src="{_BUSY_EMBEDDER_ORIGIN}/"></iframe>'},
expect_challenge_frame=False,
) as page:
await page.context.route(f"{_BUSY_EMBEDDER_ORIGIN}/**", serve_busy_embedder)
await page.reload(wait_until="commit")
try:
async with asyncio.timeout(10):
while not any(urlparse(frame.url).hostname == "challenges.cloudflare.com" for frame in page.frames):
await asyncio.sleep(0.05)
except TimeoutError:
pytest.fail("the vendor frame inside the busy embedder never committed")
await asyncio.sleep(0.5)
patch_browser_tabs(monkeypatch, FakeTabbedBrowserState(page))
server = _browser_evaluate_server(make_copilot_ctx(browser_session_id="pbs_1"))
result = await server.call_tool("evaluate", {"expression": "1"})
assert _page_state(result) == _UNREAD_PAGE_STATE
@skip_no_browser
@pytest.mark.asyncio
async def test_the_chat_mcp_server_states_the_page(self, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr("agents.mcp.MCPServerManager", FakeMCPServerManager)
monkeypatch.setattr(
"skyvern.forge.sdk.copilot.enforcement.run_with_enforcement",
AsyncMock(side_effect=CopilotTotalTimeoutError()),
)
ctx = make_copilot_ctx(browser_session_id="pbs_1")
with pytest.raises(CopilotTotalTimeoutError):
await copilot_agent._run_agent_loop_with_surface(
ctx=ctx,
stream=MagicMock(),
chat_id="chat-1",
initial_input="hello",
system_prompt="system prompt",
model_name="model-PRIMARY",
run_config=MagicMock(),
llm_key="PRIMARY",
copilot_config=CopilotConfig(),
native_tools=[],
alias_map={},
overlays={"evaluate": SchemaOverlay(requires_browser=True)},
output_guardrails=[],
)
server = ctx.discovery_mcp_server
server._client = _FakeClient({"ok": True})
async with challenge_browser_page(CATALOG_PAGE_HTML, expect_challenge_frame=False) as page:
patch_browser_tabs(monkeypatch, FakeTabbedBrowserState(page))
result = await server.call_tool("evaluate", {"expression": "1"})
assert _page_state(result) == _CATALOG_PAGE_STATE
@pytest.mark.asyncio
async def test_demonstration_handoff_precedes_the_first_model_call(self, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr("agents.mcp.MCPServerManager", FakeMCPServerManager)
history = [
{
"type": "function_call",
"call_id": "call_demo_turn-1_1",
"name": "navigate_browser",
"arguments": '{"url":"https://example.test"}',
},
{"type": "function_call_output", "call_id": "call_demo_turn-1_1", "output": '{"ok":true}'},
{"role": "user", "content": "[copilot:nudge] demonstration complete"},
]
order: list[str] = []
async def replay_demonstration(*args: Any, **kwargs: Any) -> list[dict[str, Any]]:
order.append("demonstration")
return history
async def run_with_enforcement(*args: Any, **kwargs: Any) -> None:
order.append("model")
assert kwargs["initial_input"] == [{"role": "user", "content": "opener"}, *history]
raise CopilotTotalTimeoutError()
monkeypatch.setattr(copilot_agent, "replay_demonstration", replay_demonstration)
monkeypatch.setattr("skyvern.forge.sdk.copilot.enforcement.run_with_enforcement", run_with_enforcement)
ctx = make_copilot_ctx(browser_session_id="pbs_1", turn_id="turn-1", demonstration_requested=True)
with pytest.raises(CopilotTotalTimeoutError):
await copilot_agent._run_agent_loop_with_surface(
ctx=ctx,
stream=MagicMock(),
chat_id="chat-1",
initial_input="opener",
system_prompt="system prompt",
model_name="model-PRIMARY",
run_config=MagicMock(),
llm_key="PRIMARY",
copilot_config=CopilotConfig(),
native_tools=[],
alias_map={},
overlays={},
output_guardrails=[],
)
assert order == ["demonstration", "model"]
assert ctx.demonstration_ran is True
@pytest.mark.asyncio
@pytest.mark.parametrize(
"payload",
[
pytest.param(
{"ok": False, "error": {"code": "SESSION_EXPIRED", "message": "Browser session expired"}},
id="expired-result",
),
pytest.param(CopilotBrowserSessionUnavailable("pbs_1"), id="unavailable-exception"),
pytest.param(
{"ok": False, "error": {"code": "browser_session_unavailable", "message": "The browser is gone"}},
id="unavailable-result",
),
],
)
async def test_a_lost_session_is_reported_unread_without_touching_a_browser(
self, monkeypatch: pytest.MonkeyPatch, payload: dict[str, Any] | Exception
) -> None:
live_page = AsyncMock()
monkeypatch.setattr(scouting_module, "live_working_page", live_page)
monkeypatch.setattr(mcp_adapter, "_handle_browser_session_loss", AsyncMock(return_value="failed"))
monkeypatch.setattr(mcp_adapter, "_browser_session_error_disposition", AsyncMock(return_value="failed"))
server = _browser_evaluate_server(make_copilot_ctx(browser_session_id="pbs_1"), payload)
result = await server.call_tool("evaluate", {"expression": "1"})
assert _page_state(result) == _UNREAD_PAGE_STATE
live_page.assert_not_awaited()
@pytest.mark.asyncio
async def test_a_timed_out_call_is_reported_unread_without_touching_a_browser(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
live_page = AsyncMock()
monkeypatch.setattr(scouting_module, "live_working_page", live_page)
server = _browser_evaluate_server(make_copilot_ctx(browser_session_id="pbs_1"), TimeoutError())
result = await server.call_tool("evaluate", {"expression": "1"})
assert _page_state(result) == _UNREAD_PAGE_STATE
live_page.assert_not_awaited()
@pytest.mark.asyncio
@pytest.mark.parametrize(
("ctx_overrides", "arguments", "prepared"),
[
pytest.param({}, {"expression": "1", "target": "elsewhere"}, None, id="unknown-target"),
pytest.param(
{"request_policy": RequestPolicy(raw_secret_detected=True)}, {"expression": "1"}, None, id="raw-secret"
),
pytest.param(
{},
{"expression": "1"},
(None, mcp_adapter._browser_session_loss_result({}, disposition="failed"), "failed"),
id="session-lost-before-dispatch",
),
pytest.param(
{},
{"expression": "1"},
({"ok": False, "error": "The browser session could not be prepared."}, None, None),
id="session-prepare-error",
),
],
)
async def test_a_call_refused_before_dispatch_is_reported_unread_without_touching_a_browser(
self,
monkeypatch: pytest.MonkeyPatch,
ctx_overrides: dict[str, Any],
arguments: dict[str, Any],
prepared: tuple[dict[str, Any] | None, dict[str, Any] | None, str | None] | None,
) -> None:
live_page = AsyncMock()
monkeypatch.setattr(scouting_module, "live_working_page", live_page)
if prepared is not None:
monkeypatch.setattr(mcp_adapter, "_prepare_browser_session_for_dispatch", AsyncMock(return_value=prepared))
server = _browser_evaluate_server(make_copilot_ctx(browser_session_id="pbs_1", **ctx_overrides))
result = await server.call_tool("evaluate", arguments)
assert result.isError is True
assert _page_state(result) == _UNREAD_PAGE_STATE
live_page.assert_not_awaited()
@skip_no_browser
@pytest.mark.asyncio
async def test_a_parameter_value_in_the_title_of_a_failed_call_is_redacted(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
ctx = make_copilot_ctx(browser_session_id="pbs_1")
redact_parameter_values(monkeypatch, ctx, {"account": "acct-778"})
async with challenge_browser_page(_ACCOUNT_TITLED_PAGE_HTML, expect_challenge_frame=False) as page:
patch_browser_tabs(monkeypatch, FakeTabbedBrowserState(page))
server = _browser_evaluate_server(ctx, RuntimeError("socket closed"))
result = await server.call_tool("evaluate", {"expression": "1"})
assert _page_state(result)["title"] == "Orders for [redacted]"
@skip_no_browser
@pytest.mark.asyncio
async def test_a_result_the_scrub_withholds_stays_an_error(self, monkeypatch: pytest.MonkeyPatch) -> None:
ctx = make_copilot_ctx(browser_session_id="pbs_scrub_emptied")
register_secret_scrub_value(ctx, "error")
try:
async with challenge_browser_page(CATALOG_PAGE_HTML, expect_challenge_frame=False) as page:
patch_browser_tabs(monkeypatch, FakeTabbedBrowserState(page))
server = _browser_evaluate_server(ctx, TimeoutError())
result = await server.call_tool("evaluate", {"expression": "1"})
finally:
clear_session_scrub_values(ctx.browser_session_id)
assert result.isError is True
assert "redaction_withheld" in result.content[0].text and "timed out" not in result.content[0].text
@pytest.mark.asyncio
async def test_actor_reason_actual_mcp_schema_alias_strips_only_metadata() -> None:
ctx = make_copilot_ctx(api_key="in-process-test-key")
aliases = get_skyvern_mcp_alias_map()
name = "get_block_schema"
server = SkyvernOverlayMCPServer(
transport=mcp,
overlays=_build_skyvern_mcp_overlays(),
alias_map={name: aliases[name]},
allowlist=frozenset({aliases[name]}),
context_provider=lambda: ctx,
)
await server.connect()
try:
advertised = await server.list_tools()
assert [tool.name for tool in advertised] == [name]
schema = advertised[0].inputSchema
assert schema["properties"]["user_facing_reason"]["type"] == ["string", "null"]
assert "user_facing_reason" in schema["required"]
tool = MCPUtil.to_function_tool(advertised[0], server, convert_schemas_to_strict=False)
assert serialize_tool_surface([tool]).payload["tools"][0]["params_json_schema"] == schema
tc = ToolContext(context=ctx, tool_name=name, tool_call_id="schema-call", tool_arguments="{}")
for reason in (None, "", " ", 17, {"bad": True}, "I will inspect the code block options."):
result = await tool.on_invoke_tool(tc, json.dumps({"block_type": "code", "user_facing_reason": reason}))
assert "input validation error" not in str(result).lower()
assert "code" in str(result)
omitted = await tool.on_invoke_tool(tc, json.dumps({"block_type": "code"}))
assert "input validation error" not in str(omitted).lower()
assert "code" in str(omitted)
original = _transform_args(
{"block_type": "code", "user_facing_reason": "Explain"}, _build_skyvern_mcp_overlays()[name]
)
assert original == {"block_type": "code"}
finally:
await server.cleanup()
@pytest.mark.asyncio
async def test_a_frame_staged_inside_a_tool_names_the_call_that_staged_it() -> None:
ctx = make_copilot_ctx()
def stage(shade: int) -> None:
frame = io.BytesIO()
Image.new("RGB", (30, 20), (shade, shade, shade)).save(frame, format="PNG")
assert enqueue_screenshot(
ctx,
base64.b64encode(frame.getvalue()).decode(),
provenance=ScreenshotProvenance.unknown(source_tool="click"),
)
async def click_post_hook(result: dict[str, Any], raw: Any, copilot_ctx: Any) -> dict[str, Any]:
stage(10)
return result
async def native_body(tool_ctx: Any, arguments: str) -> str:
stage(20)
return "ok"
server = SkyvernOverlayMCPServer(
transport=MagicMock(),
overlays={"click": SchemaOverlay(post_hook=click_post_hook)},
alias_map={},
allowlist=frozenset(),
context_provider=lambda: ctx,
)
server._client = SimpleNamespace(
call_tool=AsyncMock(
return_value=SimpleNamespace(structured_content={"ok": True, "data": {}}, is_error=False, content=[])
)
)
mcp_click = MCPUtil.to_function_tool(
MCPTool(name="click", inputSchema={"type": "object", "properties": {}}),
server,
convert_schemas_to_strict=False,
)
native = _with_action_reason(
FunctionTool(
name="inspect",
description="",
params_json_schema={"type": "object", "properties": {}},
on_invoke_tool=native_body,
)
)
def call(name: str, call_id: str) -> ToolContext[Any]:
return ToolContext(context=ctx, tool_name=name, tool_call_id=call_id, tool_arguments="{}")
await mcp_click.on_invoke_tool(call("click", "call-mcp"), "{}")
await native.on_invoke_tool(call("inspect", "call-native"), "{}")
stage(30)
assert [frame.tool_call_id for frame in ctx.pending_chat_screenshots] == ["call-mcp", "call-native", None]
_ModelCall = Callable[[str, dict[str, Any]], Awaitable[dict[str, Any]]]
@asynccontextmanager
async def _real_server_tools(
capability: AuthoringCapability, *names: str
) -> AsyncIterator[tuple[dict[str, MCPTool], _ModelCall]]:
ctx = make_copilot_ctx(api_key="in-process-test-key")
ctx.authoring_capability = capability
aliases = get_skyvern_mcp_alias_map()
server = SkyvernOverlayMCPServer(
transport=mcp,
overlays=_build_skyvern_mcp_overlays(capability),
alias_map={name: aliases[name] for name in names},
allowlist=frozenset(aliases[name] for name in names),
context_provider=lambda: ctx,
)
await server.connect()
try:
tools = {tool.name: tool for tool in await server.list_tools()}
async def call(name: str, arguments: dict[str, Any]) -> dict[str, Any]:
tool = MCPUtil.to_function_tool(tools[name], server, convert_schemas_to_strict=False)
tc = ToolContext(context=ctx, tool_name=name, tool_call_id=f"{name}-call", tool_arguments="{}")
output = await tool.on_invoke_tool(tc, json.dumps(arguments))
return json.loads(output["text"])
yield tools, call
finally:
await server.cleanup()
@pytest.mark.asyncio
async def test_for_loop_schema_and_knowledge_state_reference_precedence_without_refusing_both_inputs() -> None:
async with _real_server_tools(AGENT_BLOCKS_ONLY, "get_block_schema", "get_workflow_knowledge") as (_, call):
schema_data = (await call("get_block_schema", {"block_type": "for_loop"}))["data"]
knowledge = (await call("get_workflow_knowledge", {"topics": ["for_loop_block"]}))["data"]["sections"][
"for_loop_block"
]
schema = schema_data["schema"]
properties = schema["$defs"][schema["$ref"].removeprefix("#/$defs/")]["properties"]
assert {name: properties[name]["description"] for name in _FOR_LOOP_PROPERTY_DESCRIPTIONS} == (
_FOR_LOOP_PROPERTY_DESCRIPTIONS
)
assert schema_data["example"] == _FOR_LOOP_EXAMPLE
producer, loop = schema_data["example"]["blocks"]
assert loop["loop_over_parameter_key"] == f"{producer['label']}_output"
assert "loop_variable_reference" not in loop
assert knowledge["content"] == _FOR_LOOP_GUIDANCE
assert all(description in _FOR_LOOP_GUIDANCE for description in _FOR_LOOP_PROPERTY_DESCRIPTIONS.values())
assert yaml.safe_load(_FOR_LOOP_GUIDANCE.partition("the list):\n")[2]) == _FOR_LOOP_EXAMPLE
assert {"loop_over_parameter_key", "loop_variable_reference"} <= set(properties)
both_inputs = {
"block_type": "for_loop",
"label": "visit_each_row",
"loop_over_parameter_key": "parse_rows_output",
"loop_variable_reference": "current_value",
"loop_blocks": [{"block_type": "goto_url", "label": "open_row_url", "url": "{{ current_value.url }}"}],
}
submitted = copy.deepcopy(both_inputs)
assert _block_authoring_violations(submitted, AGENT_BLOCKS_ONLY) == []
assert submitted == both_inputs
def _enums_in(schema: object) -> list[list[str]]:
if isinstance(schema, dict):
own = [schema["enum"]] if "enum" in schema else []
return own + [found for value in schema.values() for found in _enums_in(value)]
if isinstance(schema, list):
return [found for value in schema for found in _enums_in(value)]
return []
@pytest.mark.asyncio
async def test_closed_value_parameters_carry_the_enforced_values_as_a_schema_enum() -> None:
overlay_tools = ("get_workflow_knowledge", "list_workflow_runs", "navigate_browser")
async with _real_server_tools(ALL_BLOCK_FAMILIES, *overlay_tools) as (tools, call):
served_topics = (await call("get_workflow_knowledge", {}))["data"]["topics"]
schemas = {name: tool.inputSchema for name, tool in tools.items()}
schemas |= {tool.name: tool.params_json_schema for tool in NATIVE_TOOLS if tool.name == "fill_credential_field"}
assert {
(name, param): _enums_in(schemas[name]["properties"][param])
for name, param in (
("get_workflow_knowledge", "topics"),
("list_workflow_runs", "status"),
("navigate_browser", "wait_until"),
("fill_credential_field", "field"),
)
} == {
("get_workflow_knowledge", "topics"): [served_topics],
("list_workflow_runs", "status"): [[status.value for status in WorkflowRunStatus]],
("navigate_browser", "wait_until"): [list(VALID_WAIT_UNTIL)],
("fill_credential_field", "field"): [list(CREDENTIAL_FILL_FIELD_NAMES)],
}
@pytest.mark.parametrize("capability", [ALL_BLOCK_FAMILIES, CODE_BLOCKS_ONLY, AGENT_BLOCKS_ONLY])
@pytest.mark.asyncio
async def test_the_task_unavailable_topic_never_reaches_the_model(capability: AuthoringCapability) -> None:
assert _TASK_UNAVAILABLE_KNOWLEDGE_TOPIC in (await skyvern_workflow_knowledge())["data"]["topics"]
async with _real_server_tools(capability, "get_workflow_knowledge") as (tools, call):
catalog = await call("get_workflow_knowledge", {})
beside_another = await call(
"get_workflow_knowledge", {"topics": [_TASK_UNAVAILABLE_KNOWLEDGE_TOPIC, "for_loop_block"]}
)
alone = await call("get_workflow_knowledge", {"topics": [_TASK_UNAVAILABLE_KNOWLEDGE_TOPIC.upper()]})
assert _TASK_UNAVAILABLE_KNOWLEDGE_TOPIC not in json.dumps(tools["get_workflow_knowledge"].inputSchema)
topics = catalog["data"]["topics"]
assert _TASK_UNAVAILABLE_KNOWLEDGE_TOPIC not in topics
assert "for_loop_block" in topics
assert catalog["data"]["count"] == len(topics)
assert list(beside_another["data"]["sections"]) == ["for_loop_block"]
assert alone == catalog
@pytest.mark.parametrize("capability", [ALL_BLOCK_FAMILIES, CODE_BLOCKS_ONLY, AGENT_BLOCKS_ONLY])
@pytest.mark.asyncio
async def test_validate_block_hides_code_only_and_never_calls_an_authorable_task_block_deprecated(
capability: AuthoringCapability,
) -> None:
task = json.dumps(
{"block_type": "task", "label": "open_report", "url": "https://example.com", "navigation_goal": "Open it."}
)
code = json.dumps({"block_type": "code", "label": "read_total", "code": "total = 1"})
assert (await skyvern_block_validate(block_json=task))["warnings"]
async with _real_server_tools(capability, "validate_block") as (tools, call):
assert "code_only" not in json.dumps(tools["validate_block"].inputSchema)
task_result = await call("validate_block", {"block_json": task, "code_only": True})
code_result = await call("validate_block", {"block_json": code})
assert "warnings" not in task_result
assert task_result["ok"] is capability.agent_blocks
if capability.agent_blocks:
assert task_result["data"]["valid"] is True
if capability.code_blocks:
assert code_result["data"]["valid"] is True
assert code_result["warnings"]
@pytest.mark.parametrize("capability", [ALL_BLOCK_FAMILIES, CODE_BLOCKS_ONLY, AGENT_BLOCKS_ONLY])
def test_every_aliased_tool_carries_a_copilot_description(capability: AuthoringCapability) -> None:
overlays = _build_skyvern_mcp_overlays(capability)
undescribed = [alias for alias in get_skyvern_mcp_alias_map() if not overlays[alias].description]
assert undescribed == []
async def _real_validate_block_failure() -> dict[str, Any]:
return await skyvern_block_validate(block_json='{"block_type": "text_prompt"}')
async def _real_get_workflow_failure() -> dict[str, Any]:
return await skyvern_workflow_get(workflow_id="not_a_wpid")
@pytest.mark.parametrize(
("tool_name", "arguments", "server_failure"),
[
("validate_block", {"block_json": '{"block_type": "text_prompt"}'}, _real_validate_block_failure),
("get_org_workflow", {"workflow_id": "wpid_missing"}, _real_get_workflow_failure),
],
)
@pytest.mark.asyncio
async def test_a_failed_server_call_reaches_the_model_without_the_servers_hint(
tool_name: str,
arguments: dict[str, Any],
server_failure: Callable[[], Awaitable[dict[str, Any]]],
) -> None:
payload = await server_failure()
assert payload["error"]["hint"]
aliases = get_skyvern_mcp_alias_map()
server = SkyvernOverlayMCPServer(
transport=MagicMock(),
overlays={tool_name: _build_skyvern_mcp_overlays()[tool_name]},
alias_map={tool_name: aliases[tool_name]},
allowlist=frozenset({aliases[tool_name]}),
context_provider=lambda: make_copilot_ctx(),
)
server._client = _FakeClient(payload)
result = await server.call_tool(tool_name, arguments)
surfaced = json.loads(result.content[0].text)
assert surfaced["ok"] is False
assert surfaced["error"] == payload["error"]["message"]
assert surfaced["error_code"] == payload["error"]["code"]
assert payload["error"]["hint"] not in result.content[0].text
_CLICK_ARGS = {"selector": "#go"}
@pytest.mark.usefixtures("_stub_browser_session")
@pytest.mark.parametrize(
("tool_name", "arguments", "server_error", "separator", "element_state"),
[
("click", _CLICK_ARGS, action_deadline_error(TimeoutError("Timeout 5000ms exceeded.")), " ", None),
(
"click",
_CLICK_ARGS,
action_deadline_error(TimeoutError("Timeout 5000ms exceeded."), mutating=False),
" ",
None,
),
("click", _CLICK_ARGS, make_error(ErrorCode.TIMEOUT, "The page stopped answering!", "Retry it."), " ", None),
(
"wait_for_either_state",
{"selector_a": "#signed-in", "selector_b": "#sign-in", "state": "visible"},
make_error(
ErrorCode.TIMEOUT,
"Neither '#signed-in' nor '#sign-in' reached 'visible' within 5000ms",
"Neither state was confirmed before the timeout elapsed",
),
". ",
None,
),
(
"click",
_CLICK_ARGS,
element_state_error("disabled", TimeoutError("Timeout 5000ms exceeded."), selector="#go", timeout_ms=5000),
None,
"disabled",
),
],
ids=["mutating_timeout", "read_timeout", "exclaimed_timeout", "wait_timeout", "element_state"],
)
@pytest.mark.asyncio
async def test_a_failed_browser_action_reports_typed_facts_instead_of_the_servers_hint(
tool_name: str,
arguments: dict[str, Any],
server_error: dict[str, Any],
separator: str | None,
element_state: str | None,
) -> None:
server = _overlay_server(
make_copilot_ctx(browser_session_id="pbs_1"),
_FakeClient({"ok": False, "error": server_error}),
tool_name,
_build_skyvern_mcp_overlays()[tool_name],
)
result = await server.call_tool(tool_name, arguments)
surfaced = json.loads(result.content[0].text)
assert surfaced["ok"] is False
assert surfaced["error"] == (
server_error["message"]
if separator is None
else f"{server_error['message']}{separator}{mcp_adapter._EFFECT_UNKNOWN_GUIDANCE}"
)
assert surfaced.get("element_state") == element_state
assert server_error["hint"] not in result.content[0].text
async def _rebuilding_post_hook(
copilot_result: dict[str, Any], _raw_mcp: dict[str, Any], _ctx: AgentContext
) -> dict[str, Any]:
return {"ok": copilot_result["ok"], "data": copilot_result["data"]}
def _tab(title: str) -> SimpleNamespace:
return SimpleNamespace(
url=f"https://app.test/{title.lower()}", frames=[], title=AsyncMock(return_value=title), is_closed=lambda: False
)
def _scope_exiting_with(exc: BaseException) -> Callable[..., AbstractAsyncContextManager[None]]:
@asynccontextmanager
async def scope(_ctx: AgentContext, *, session_id_override: str | None = None) -> AsyncIterator[None]:
yield
raise exc
return scope
async def _evaluate_result(server: SkyvernOverlayMCPServer) -> dict[str, Any]:
result = json.loads((await server.call_tool("evaluate", {"expression": "read()"})).content[0].text)
if "page_request_failures" in result:
keys = list(result)
assert keys.index("page_request_failures") == keys.index("page_state") + 1
return result
def _evaluate_browser_server(
ctx: AgentContext,
raw_result: dict[str, Any] | Exception,
on_call: Callable[[], None] | None = None,
overlay: SchemaOverlay | None = None,
) -> SkyvernOverlayMCPServer:
overlay, reader = (
overlay or SchemaOverlay(requires_browser=True, timeout=1),
scouting_module.read_page_state_on_page,
)
return _make_server(
ctx, raw_result, overlay, alias_map={"evaluate": "skyvern_evaluate"}, on_call=on_call, page_state_reader=reader
)
def _working_tab_with_failures(
monkeypatch: pytest.MonkeyPatch, ctx: AgentContext, *paths: str, closed: bool = False
) -> RequestFailureLog:
working_tab = _tab("Orders")
working_tab.is_closed = lambda: closed
patch_browser_tabs(monkeypatch, FakeTabbedBrowserState(working_tab))
return seed_request_failures(ctx, "pbs_failures", *[make_request_failure(working_tab, path) for path in paths])
class TestPageRequestFailures:
@pytest.fixture(autouse=True)
def _fresh_scrub_values(self) -> Iterator[None]:
clear_session_scrub_values("pbs_failures")
yield
clear_session_scrub_values("pbs_failures")
@skip_no_browser
@pytest.mark.asyncio
async def test_a_socket_closed_xhr_between_calls_surfaces_once_on_the_next_browser_result(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
ctx = make_copilot_ctx(browser_session_id="pbs_failures", api_key="test-api-key")
async with socket_closing_fixture_page() as (page, origin):
await page.goto(f"{origin}/")
_attach_mocked_browser_session(monkeypatch, page)
def _tool_body() -> None:
ensure_hooks_on_all_pages(get_current_session(), [page])
server = _make_server(
ctx,
{"ok": True, "data": {"result": "done"}},
SchemaOverlay(requires_browser=True, post_hook=_rebuilding_post_hook),
on_call=_tool_body,
page_state_reader=scouting_module.read_page_state_on_page,
)
earlier = [(await _evaluate_result(server)).get("page_request_failures") for _ in range(2)]
async with page.expect_event("requestfailed"):
await page.click("#view")
aborts_seen = asyncio.Event()
aborted = 0
def _count_abort(request: Request) -> None:
nonlocal aborted
aborted += 1
if aborted == 21:
aborts_seen.set()
page.on("requestfailed", _count_abort)
await page.evaluate(
"""async () => {
const controllers = Array.from({length: 21}, () => new AbortController());
const pending = controllers.map((c) => fetch('/slow', {signal: c.signal}).catch(() => null));
await new Promise((resolve) => setTimeout(resolve, 300));
controllers.forEach((c) => c.abort());
await Promise.all(pending);
}"""
)
await asyncio.wait_for(aborts_seen.wait(), timeout=10)
after_failure = await _evaluate_result(server)
next_call = await _evaluate_result(server)
assert earlier == [None, None]
reported = {
"method": "POST",
"resource_type": "xhr",
"failure_text": "net::ERR_EMPTY_RESPONSE",
"url": "/report",
}
# 21 aborts overflow both caps; the one different failure keeps its slot at each.
assert after_failure["page_request_failures"][0] == {**reported, "page_url": f"{origin}/"}
assert [(f["failure_text"], f.get("url")) for f in after_failure["page_request_failures"][1:]] == [
("net::ERR_ABORTED", "/slow")
] * 4
assert BODY_SENTINEL not in json.dumps(after_failure)
assert "page_request_failures" not in next_call
@pytest.mark.usefixtures("_stub_browser_session")
@pytest.mark.asyncio
async def test_model_results_drain_only_the_tab_they_read_and_internal_calls_drain_nothing(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
ctx = make_copilot_ctx(browser_session_id="pbs_failures")
working_tab, other_tab = _tab("Orders"), _tab("Reports")
browser = FakeTabbedBrowserState(working_tab, other_tab)
patch_browser_tabs(monkeypatch, browser)
working_failures = [make_request_failure(working_tab, f"/{n}") for n in range(7)]
noisy_failures = [make_request_failure(other_tab, f"/bg{n}", resource_type="fetch") for n in range(120)]
log = seed_request_failures(ctx, "pbs_failures", *working_failures, *noisy_failures)
recorded = logged_request_failures(log)
browser_server = _evaluate_browser_server(ctx, {"ok": True})
internal = await browser_server.call_internal_tool("skyvern_evaluate", {"expression": "probe()"})
assert internal["ok"] is True and "page_request_failures" not in internal
assert logged_request_failures(log) == recorded
refusing = SchemaOverlay(requires_browser=True, pre_hook=_refuse)
refused = await _evaluate_result(_evaluate_browser_server(ctx, {"ok": True}, overlay=refusing))
assert refused["ok"] is False and refused["page_state"]["title"] == "Orders"
assert [f["url"] for f in refused["page_request_failures"]] == [f"/{n}" for n in range(2, 7)]
assert logged_request_failures(log) == noisy_failures[-20:]
browser.active = other_tab
after_switch = (await _evaluate_result(browser_server))["page_request_failures"]
assert [(f["url"], f["resource_type"]) for f in after_switch] == [(f"/bg{n}", "fetch") for n in range(115, 120)]
assert logged_request_failures(log) == []
@pytest.mark.usefixtures("_stub_browser_session")
@pytest.mark.asyncio
@pytest.mark.parametrize(
("raw_result", "scope_exit_error", "reads_the_popup"),
[
pytest.param(TimeoutError(), None, False, id="timed-out-without-a-read"),
pytest.param(RuntimeError("client failed"), None, True, id="client-error"),
pytest.param({"ok": True}, RuntimeError("scope exit failed"), True, id="failed-after-projection"),
],
)
async def test_an_error_result_reports_only_the_failures_of_the_tab_its_page_state_read(
self,
monkeypatch: pytest.MonkeyPatch,
raw_result: dict[str, Any] | Exception,
scope_exit_error: Exception | None,
reads_the_popup: bool,
) -> None:
ctx = make_copilot_ctx(browser_session_id="pbs_failures")
register_secret_scrub_value(ctx, "acct-secret-991")
opener, popup = _tab("Opener"), _tab("Popup")
browser = FakeTabbedBrowserState(opener)
patch_browser_tabs(monkeypatch, browser)
if scope_exit_error is not None:
monkeypatch.setattr(mcp_adapter, "mcp_browser_context", _scope_exiting_with(scope_exit_error))
opener_failure = make_request_failure(opener, "/opener")
log = seed_request_failures(ctx, "pbs_failures", opener_failure)
popup_failure = make_request_failure(popup, "/accounts/acct-secret-991/report")
def _open_popup() -> None:
browser.tabs.append(popup)
browser.active = popup
log.append(popup_failure)
result = await _evaluate_result(_evaluate_browser_server(ctx, raw_result, on_call=_open_popup))
assert result["ok"] is False
assert "acct-secret-991" not in json.dumps(result)
assert result["page_state"].get("title") == ("Popup" if reads_the_popup else None)
assert len(result.get("page_request_failures", [])) == int(reads_the_popup)
assert logged_request_failures(log) == [opener_failure] + ([] if reads_the_popup else [popup_failure])
@pytest.mark.asyncio
async def test_a_native_action_that_is_the_first_browser_call_records_its_own_failures(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
ctx = make_copilot_ctx(browser_session_id="pbs_failures", api_key="test-api-key")
page = MagicMock(url="https://app.test/upload")
page.is_closed.return_value = False
_attach_mocked_browser_session(monkeypatch, page)
monkeypatch.setattr(
copilot_runtime,
"SkyvernBrowser",
MagicMock(return_value=MagicMock(_browser_context=MagicMock(pages=[page]))),
)
async def upload(acted_on: Page) -> dict[str, Any]:
listeners = [call.args[1] for call in page.on.call_args_list if call.args[0] == "requestfailed"]
for listener in listeners:
listener(
MagicMock(
url="https://app.test/upload/save",
method="POST",
resource_type="xhr",
failure="net::ERR_EMPTY_RESPONSE",
)
)
return {"ok": True}
result = await on_working_page(ctx, tool_name="upload_attached_file", no_page_error="no page", act=upload)
assert result == {"ok": True}
log = ctx.request_failure_logs_by_browser_session["pbs_failures"]
assert [(entry["failure_text"], entry["url"]) for entry in logged_request_failures(log)] == [
("net::ERR_EMPTY_RESPONSE", "https://app.test/upload/save")
]
@pytest.mark.usefixtures("_stub_browser_session")
@pytest.mark.asyncio
async def test_failures_a_result_cannot_show_go_back_to_the_log(self, monkeypatch: pytest.MonkeyPatch) -> None:
ctx = make_copilot_ctx(browser_session_id="pbs_failures")
log = _working_tab_with_failures(monkeypatch, ctx, "/0")
register_secret_scrub_value(ctx, "ok")
result = await _evaluate_result(_evaluate_browser_server(ctx, {"ok": True}))
assert "page_request_failures" not in result
assert len(logged_request_failures(log)) == 1
@pytest.mark.usefixtures("_stub_browser_session")
@pytest.mark.asyncio
@pytest.mark.parametrize(
("secret", "reported_path"),
[
("p@ss word", "/users/p@ss%20word/verify"),
("foo\\bar", "/users/foo/bar/verify"),
("abc?def", "/users/abc?def/verify"),
],
ids=["encoded-space", "backslash-as-slash", "spans-path-and-query"],
)
async def test_a_browser_a_secret_run_used_still_reports_its_failures_with_the_secret_scrubbed(
self, monkeypatch: pytest.MonkeyPatch, secret: str, reported_path: str
) -> None:
ctx = make_copilot_ctx(browser_session_id="pbs_failures")
ctx.secret_run_browser_session_ids.add("pbs_failures")
log = _working_tab_with_failures(monkeypatch, ctx, reported_path)
register_secret_scrub_value(ctx, secret)
result = await _evaluate_result(_evaluate_browser_server(ctx, {"ok": True}))
assert [failure["url"] for failure in result["page_request_failures"]] == ["/users/[REDACTED_SECRET]/verify"]
assert logged_request_failures(log) == []
@pytest.mark.usefixtures("_stub_browser_session")
@pytest.mark.asyncio
@pytest.mark.parametrize(
("secret", "request_url"),
[
("abc?token=def", "https://app.test/users/abc?token=def"),
("Tr0ub4dor&3#x", "https://app.test/users/Tr0ub4dor&3"),
("ABC?token=def", "https://abc/?token=def"),
("TenantKey9", "https://tenantkey9/report"),
("Straße9", "https://xn--strae9-eta/report"),
("Straße9", "https://xn--tenant-strae9-ldb.example/report"),
("PASS9", "https://pass9.example/report"),
("foo/../bar9", "https://app.test/users/bar9/verify"),
],
ids=[
"query-value-masked-at-record",
"fragment-never-sent",
"host-lowercased-cut",
"host-lowercased-whole",
"host-punycode",
"host-punycode-inside-a-label",
"host-compatibility-mapped",
"path-dot-segments-removed",
],
)
async def test_a_url_holding_a_secret_cut_at_its_query_or_fragment_is_withheld(
self, monkeypatch: pytest.MonkeyPatch, secret: str, request_url: str
) -> None:
ctx = make_copilot_ctx(browser_session_id="pbs_failures")
register_secret_scrub_value(ctx, secret)
tab = _tab("Users")
tab.on = MagicMock()
patch_browser_tabs(monkeypatch, FakeTabbedBrowserState(tab))
log = ctx.request_failure_logs_by_browser_session.setdefault("pbs_failures", RequestFailureLog())
ensure_hooks_on_all_pages(make_session_state(request_failures=log), [tab])
record = next(call.args[1] for call in tab.on.call_args_list if call.args[0] == "requestfailed")
record(MagicMock(url=request_url, method="POST", resource_type="xhr", failure="net::ERR_EMPTY_RESPONSE"))
result = await _evaluate_result(_evaluate_browser_server(ctx, {"ok": True}))
assert result["page_request_failures"] == [
{
"method": "POST",
"resource_type": "xhr",
"failure_text": "net::ERR_EMPTY_RESPONSE",
"page_url": "https://app.test/users",
}
]
@pytest.mark.usefixtures("_stub_browser_session")
@pytest.mark.asyncio
@pytest.mark.parametrize(
("tab_closed", "pre_hook", "kept"),
[(False, None, 1), (True, None, 0), (False, _refuse, 1)],
ids=["open-tab", "closed-tab", "refused-before-dispatch"],
)
async def test_a_cancelled_call_puts_its_failures_back_unless_their_tab_closed(
self, monkeypatch: pytest.MonkeyPatch, tab_closed: bool, pre_hook: mcp_adapter.PreHook | None, kept: int
) -> None:
ctx = make_copilot_ctx(browser_session_id="pbs_failures")
log = _working_tab_with_failures(monkeypatch, ctx, "/0", closed=tab_closed)
monkeypatch.setattr(mcp_adapter, "mcp_browser_context", _scope_exiting_with(asyncio.CancelledError()))
overlay = SchemaOverlay(requires_browser=True, timeout=1, pre_hook=pre_hook)
with pytest.raises(asyncio.CancelledError):
await _evaluate_result(_evaluate_browser_server(ctx, {"ok": True}, overlay=overlay))
assert len(logged_request_failures(log)) == kept
@pytest.mark.usefixtures("_stub_browser_session")
@pytest.mark.asyncio
async def test_a_navigate_result_drains_the_page_it_read(self, monkeypatch: pytest.MonkeyPatch) -> None:
ctx = make_copilot_ctx(browser_session_id="pbs_failures")
monkeypatch.setattr(mcp_hooks, "_bind_login_credential_for_observed_url", AsyncMock())
monkeypatch.setattr(mcp_hooks, "_capture_post_interaction_screenshot", AsyncMock(return_value=False))
log = _working_tab_with_failures(monkeypatch, ctx, "/0")
navigate = SchemaOverlay(requires_browser=True, timeout=1, post_hook=mcp_hooks._navigate_post_hook)
result = await _evaluate_result(
_evaluate_browser_server(ctx, {"ok": True, "data": {"url": "https://app.test/orders"}}, overlay=navigate)
)
assert result["page_state"]["title"] == "Orders"
assert [f["url"] for f in result["page_request_failures"]] == ["/0"]
assert logged_request_failures(log) == []
@pytest.mark.usefixtures("_stub_browser_session")
@pytest.mark.asyncio
async def test_failures_past_the_redaction_budget_go_back(self, monkeypatch: pytest.MonkeyPatch) -> None:
ctx = make_copilot_ctx(browser_session_id="pbs_failures")
ctx.codeblock_redaction_parameters = {"token": "unused-value"}
def redact_within_budget(value: Any, _parameters: dict[str, Any], **_budget: object) -> Any:
return value if len(json.dumps(value)) <= 1_500 else None
monkeypatch.setattr(mcp_adapter.app.AGENT_FUNCTION, "redact_codeblock_parameter_values", redact_within_budget)
log = _working_tab_with_failures(monkeypatch, ctx, *[f"/{n}" for n in range(5)])
result = await _evaluate_result(_evaluate_browser_server(ctx, {"ok": True, "data": {"result": "x" * 1_000}}))
assert result["ok"] is True and result["data"] == {"result": "x" * 1_000}
assert "page_request_failures" not in result
assert len(logged_request_failures(log)) == 5
@pytest.mark.usefixtures("_stub_browser_session")
@pytest.mark.asyncio
@pytest.mark.parametrize(
("result_chars", "reported"),
[(_RECENT_TOOL_OUTPUT_CHAR_CAP - 500, False), (_RECENT_TOOL_OUTPUT_CHAR_CAP, True)],
ids=["fits-without-them", "past-the-cap-anyway"],
)
async def test_failures_never_push_a_result_past_the_recent_output_cap(
self, monkeypatch: pytest.MonkeyPatch, result_chars: int, reported: bool
) -> None:
ctx = make_copilot_ctx(browser_session_id="pbs_failures")
log = _working_tab_with_failures(monkeypatch, ctx, *[f"/{n}" for n in range(5)])
result = await _evaluate_result(
_evaluate_browser_server(ctx, {"ok": True, "data": {"result": "x" * result_chars}})
)
assert ("page_request_failures" in result) is reported
assert len(logged_request_failures(log)) == (0 if reported else 5)
@pytest.mark.usefixtures("_stub_browser_session")
@pytest.mark.asyncio
async def test_a_url_that_scrubbing_grows_past_the_bound_is_left_out(self, monkeypatch: pytest.MonkeyPatch) -> None:
ctx = make_copilot_ctx(browser_session_id="pbs_failures")
register_secret_scrub_value(ctx, "7")
_working_tab_with_failures(monkeypatch, ctx, "/" + "7" * 280)
result = await _evaluate_result(_evaluate_browser_server(ctx, {"ok": True}))
(reported,) = result["page_request_failures"]
assert reported == {
"method": "POST",
"resource_type": "xhr",
"failure_text": "net::ERR_EMPTY_RESPONSE",
"page_url": "https://app.test/",
}
def test_the_model_sees_bounded_fields_and_only_the_host_of_a_cross_origin_request(self) -> None:
too_long = f"https://app.test/{'p' * 400}"
reported = mcp_adapter.reported_request_failures(
[
make_request_failure(
object(), url="https://app.test/api/r?id=7#top", page_url="https://app.test/h?s=1"
),
make_request_failure(object(), url="https://api.other.test/v1/accounts/123", method="X" * 9_999),
make_request_failure(object(), url=too_long, failure_text="F" * 9_999),
make_request_failure(object(), url="http://[redacted]/r", page_url=too_long),
]
)
assert [(r.get("url"), r.get("page_url"), r["method"], len(r["failure_text"])) for r in reported] == [
("/api/r", "https://app.test/h", "POST", 23),
("https://api.other.test", "https://app.test/", "OTHER", 23),
(None, "https://app.test/", "POST", 200),
(None, None, "POST", 23),
]