Skip to content

Automate browser based workflows with AI

91.3 KB
Raw Blame
"""Tests for skyvern_observe and skyvern_execute MCP tools."""

from __future__ import annotations

import asyncio
from types import SimpleNamespace
from typing import Any
from unittest.mock import AsyncMock, MagicMock, PropertyMock, call

import pytest

from skyvern.cli.core import browser_ops, session_manager
from skyvern.cli.core.browser_ops import (
    _DOMUTILS_INTERACTABILITY_READY_JS,
    _OBSERVE_INTERACTABLES_JS,
    ExecuteStep,
    NavigateResult,
    ObserveResult,
    _flatten_a11y_tree,
    do_execute,
    do_observe,
    ref_to_selector,
    serialize_elements,
)
from skyvern.cli.core.result import BrowserContext
from skyvern.cli.core.session_manager import scoped_session
from skyvern.cli.mcp_tools import browser as mcp_browser
from skyvern.cli.mcp_tools import mcp
from skyvern.cli.mcp_tools import tabs as mcp_tabs
from skyvern.client.errors import InternalServerError
from skyvern.exceptions import StaleFrameSelectionError
from skyvern.library.skyvern_browser_page import SkyvernBrowserPage
from skyvern.webeye.utils.page import JS_FUNCTION_DEFS
from tests.unit._mcp_browser_fakes import make_real_wait_for_timeout, make_session_state, make_skyvern_page

# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------


def _make_a11y_tree(**overrides: Any) -> dict[str, Any]:
    """Build a minimal a11y tree for testing."""
    tree: dict[str, Any] = {
        "role": "WebArea",
        "name": "",
        "children": overrides.get(
            "children",
            [
                {"role": "textbox", "name": "Email", "value": ""},
                {"role": "textbox", "name": "Password", "value": "secret123"},
                {"role": "button", "name": "Sign In"},
                {"role": "link", "name": "Forgot password?"},
                {"role": "heading", "name": "Login", "level": 1},
            ],
        ),
    }
    return tree


def _mock_dom_evaluate(result: list[dict[str, Any]], *, domutils_ready: bool = True) -> AsyncMock:
    async def evaluate(expression: str, arg: Any = None) -> Any:
        if expression == _DOMUTILS_INTERACTABILITY_READY_JS:
            return domutils_ready
        if expression == JS_FUNCTION_DEFS:
            return None
        return result

    return AsyncMock(side_effect=evaluate)


def _make_page(a11y_tree: dict[str, Any] | None = None) -> AsyncMock:
    """Create a mock page with accessibility.snapshot()."""
    page = AsyncMock()
    page._working_frame = None
    page.url = "https://example.com/login"
    page.title = AsyncMock(return_value="Login Page")

    tree = a11y_tree or _make_a11y_tree()
    page.accessibility = SimpleNamespace(snapshot=AsyncMock(return_value=tree))
    page.locator = AsyncMock()
    page.evaluate = _mock_dom_evaluate([])
    return page


def _make_values_page() -> AsyncMock:
    """Page with a non-password input (DOM value present) and a password input (DOM value absent)."""
    tree = _make_a11y_tree(
        children=[
            {"role": "textbox", "name": "Email", "value": "[email protected]"},
            {"role": "textbox", "name": "Current", "value": "snapshot-secret"},
        ]
    )
    page = _make_page(tree)
    page.evaluate = _mock_dom_evaluate(
        [
            {
                "role": "textbox",
                "name": "Email",
                "tag": "input",
                "selector": "#email",
                "value": "[email protected]",
            },
            {
                "role": "textbox",
                "name": "Current",
                "tag": "input",
                "selector": "#password",
            },
        ]
    )
    return page


# ---------------------------------------------------------------------------
# Unit tests: _flatten_a11y_tree
# ---------------------------------------------------------------------------


class TestFlattenA11yTree:
    def test_empty_tree(self) -> None:
        assert _flatten_a11y_tree(None) == []

    def test_skips_web_area_root(self) -> None:
        tree = {"role": "WebArea", "name": "", "children": []}
        assert _flatten_a11y_tree(tree) == []

    def test_flattens_nested(self) -> None:
        tree = {
            "role": "WebArea",
            "name": "",
            "children": [
                {
                    "role": "navigation",
                    "name": "Main",
                    "children": [
                        {"role": "link", "name": "Home"},
                        {"role": "link", "name": "About"},
                    ],
                },
                {"role": "button", "name": "Submit"},
            ],
        }
        flat = _flatten_a11y_tree(tree)
        roles = [e["role"] for e in flat]
        assert roles == ["navigation", "link", "link", "button"]

    def test_no_children_key(self) -> None:
        tree = {"role": "button", "name": "Click me"}
        flat = _flatten_a11y_tree(tree)
        assert len(flat) == 1
        assert flat[0]["name"] == "Click me"


# ---------------------------------------------------------------------------
# Unit tests: ref_to_selector
# ---------------------------------------------------------------------------


class TestRefToSelector:
    def test_with_name(self) -> None:
        assert ref_to_selector({"role": "button", "name": "Submit"}) == 'role=button[name="Submit"]'

    def test_without_name(self) -> None:
        assert ref_to_selector({"role": "textbox", "name": ""}) == "role=textbox"

    def test_name_with_quotes(self) -> None:
        result = ref_to_selector({"role": "button", "name": 'Click "here"'})
        assert result == 'role=button[name="Click \\"here\\""]'

    def test_match_index_zero_emits_nth(self) -> None:
        """Presence of match_index in the dict signals a duplicate group; emit nth even for 0."""
        elem = {"role": "combobox", "name": "", "match_index": 0}
        assert ref_to_selector(elem) == "role=combobox >> nth=0"

    def test_unnamed_duplicate_appends_nth(self) -> None:
        elem = {"role": "combobox", "name": "", "match_index": 2}
        assert ref_to_selector(elem) == "role=combobox >> nth=2"

    def test_named_duplicate_appends_nth(self) -> None:
        elem = {"role": "button", "name": "Edit", "match_index": 1}
        assert ref_to_selector(elem) == 'role=button[name="Edit"] >> nth=1'

    def test_missing_match_index_is_backward_compatible(self) -> None:
        assert ref_to_selector({"role": "textbox", "name": "Email"}) == 'role=textbox[name="Email"]'


# ---------------------------------------------------------------------------
# Unit tests: do_observe
# ---------------------------------------------------------------------------


class TestDoObserve:
    @pytest.mark.asyncio
    async def test_basic_observe(self) -> None:
        page = _make_page()
        result = await do_observe(page)

        assert isinstance(result, ObserveResult)
        assert result.url == "https://example.com/login"
        assert result.title == "Login Page"
        # Default interactive_only=True filters out heading
        assert result.element_count == 4
        assert result.total_on_page == 4

    @pytest.mark.asyncio
    async def test_ref_assignment(self) -> None:
        page = _make_page()
        result = await do_observe(page)
        refs = [e.ref for e in result.elements]
        assert refs == ["e0", "e1", "e2", "e3"]

    @pytest.mark.asyncio
    async def test_default_omits_values(self) -> None:
        page = _make_page()
        observed = serialize_elements((await do_observe(page)).elements)

        assert all("value" not in element for element in observed)

    @pytest.mark.asyncio
    async def test_include_values_keeps_non_password_dom_value_only(self) -> None:
        page = _make_values_page()

        observed = serialize_elements((await do_observe(page, include_values=True)).elements)

        email = next(element for element in observed if element["name"] == "Email")
        password = next(element for element in observed if element["name"] == "Current")
        assert email["value"] == "[email protected]"
        assert "value" not in password

    @pytest.mark.asyncio
    async def test_non_boolean_include_values_fails_closed(self) -> None:
        page = _make_values_page()

        await do_observe(page, include_values="false")  # type: ignore[arg-type]

        assert page.evaluate.await_args_list == [
            call(_DOMUTILS_INTERACTABILITY_READY_JS),
            call(
                _OBSERVE_INTERACTABLES_JS,
                {"scopeSelector": None, "includeValues": False},
            ),
        ]

    @pytest.mark.asyncio
    async def test_dom_scan_installs_production_predicate_when_missing(self) -> None:
        page = _make_page()
        page.evaluate = _mock_dom_evaluate([], domutils_ready=False)

        await do_observe(page)

        assert page.evaluate.await_args_list == [
            call(_DOMUTILS_INTERACTABILITY_READY_JS),
            call(JS_FUNCTION_DEFS),
            call(
                _OBSERVE_INTERACTABLES_JS,
                {"scopeSelector": None, "includeValues": False},
            ),
        ]

    @pytest.mark.asyncio
    async def test_dom_scan_times_out_fail_closed(self, monkeypatch: pytest.MonkeyPatch) -> None:
        page = _make_page(_make_a11y_tree(children=[{"role": "button", "name": "Go"}]))

        async def never_settles(expression: str, arg: Any = None) -> Any:
            await asyncio.sleep(3600)

        page.evaluate = AsyncMock(side_effect=never_settles)
        monkeypatch.setattr("skyvern.cli.core.browser_ops._DOM_SCAN_TIMEOUT_SECONDS", 0.05)

        result = await do_observe(page)

        assert [element.name for element in result.elements] == ["Go"]

    def test_dom_scan_keeps_sky_11953_password_guards(self) -> None:
        assert '(isPassword(element) ? "" : element.value)' in _OBSERVE_INTERACTABLES_JS
        assert "includeValues === true && !isPassword(element)" in _OBSERVE_INTERACTABLES_JS

    @pytest.mark.asyncio
    async def test_max_elements_cap(self) -> None:
        children = [{"role": "button", "name": f"Btn {i}"} for i in range(100)]
        page = _make_page(_make_a11y_tree(children=children))
        result = await do_observe(page, max_elements=10)
        assert result.element_count == 10
        assert result.total_on_page == 100

    @pytest.mark.asyncio
    async def test_interactive_only_false(self) -> None:
        page = _make_page()
        result = await do_observe(page, interactive_only=False)
        # Should include heading (non-interactive)
        assert result.element_count == 5

    @pytest.mark.asyncio
    async def test_selector_scoping(self) -> None:
        page = _make_page()
        mock_handle = AsyncMock()
        # locator() is synchronous in Playwright, returns a Locator
        locator_mock = MagicMock()
        locator_mock.first.element_handle = AsyncMock(return_value=mock_handle)
        page.locator = MagicMock(return_value=locator_mock)

        scoped_tree = {
            "role": "group",
            "name": "form",
            "children": [{"role": "textbox", "name": "Name"}],
        }
        page.accessibility.snapshot = AsyncMock(return_value=scoped_tree)

        result = await do_observe(page, selector="form#login")
        page.accessibility.snapshot.assert_awaited_once_with(root=mock_handle)
        assert result.element_count == 1

    @pytest.mark.asyncio
    @pytest.mark.parametrize("selector", [None, "#frame-form"])
    async def test_working_frame_uses_dom_only_and_reports_frame_url(
        self,
        selector: str | None,
    ) -> None:
        main_tree = _make_a11y_tree(children=[{"role": "button", "name": "Parent action"}])
        page = _make_page(main_tree)
        locator = MagicMock()
        locator.first.element_handle = AsyncMock(return_value=MagicMock())
        page.locator = MagicMock(return_value=locator)

        frame = SimpleNamespace(
            name="payment",
            url="https://example.com/payment-frame",
            page=page.page,
            title=AsyncMock(return_value="Payment Frame"),
            evaluate=_mock_dom_evaluate(
                [
                    {
                        "role": "button",
                        "name": "Frame action",
                        "tag": "button",
                        "selector": "#frame-action",
                    }
                ],
                domutils_ready=False,
            ),
        )
        page._working_frame = frame

        result = await do_observe(page, selector=selector)

        assert frame.evaluate.await_args_list == [
            call(_DOMUTILS_INTERACTABILITY_READY_JS),
            call(JS_FUNCTION_DEFS),
            call(
                _OBSERVE_INTERACTABLES_JS,
                {"scopeSelector": selector, "includeValues": False},
            ),
        ]
        page.evaluate.assert_not_awaited()
        page.accessibility.snapshot.assert_not_awaited()
        assert {element.name for element in result.elements} == {"Frame action"}
        assert result.url == frame.url
        assert result.title == "Payment Frame"
        frame.title.assert_awaited_once()
        page.title.assert_not_awaited()

    @pytest.mark.asyncio
    async def test_detached_working_frame_raises_before_evaluate(self) -> None:
        page = _make_page()
        frame = SimpleNamespace(
            name="payment",
            url="https://example.com/payment-frame",
            is_detached=lambda: True,
            evaluate=AsyncMock(),
        )
        page._working_frame = frame

        with pytest.raises(browser_ops.ObserveFrameError) as exc_info:
            await do_observe(page)

        assert exc_info.value.frame_name == frame.name
        assert exc_info.value.frame_url == frame.url
        frame.evaluate.assert_not_awaited()

    @pytest.mark.asyncio
    async def test_frame_switch_index_zero_uses_main_frame_accessibility_snapshot(self) -> None:
        main_frame = SimpleNamespace(name="", url="https://example.com/login")
        raw_page = SimpleNamespace(frames=[main_frame], main_frame=main_frame)
        snapshot = AsyncMock(return_value=_make_a11y_tree(children=[{"role": "button", "name": "Main"}]))
        page = SimpleNamespace(
            page=raw_page,
            _working_frame=SimpleNamespace(),
            url=main_frame.url,
            title=AsyncMock(return_value="Main Page"),
            accessibility=SimpleNamespace(snapshot=snapshot),
            evaluate=_mock_dom_evaluate([]),
        )

        switched = await SkyvernBrowserPage.frame_switch(page, index=0)
        observed = await do_observe(page)

        assert switched["url"] == main_frame.url
        assert page._working_frame is None
        snapshot.assert_awaited_once_with()
        assert {element.name for element in observed.elements} == {"Main"}

    @pytest.mark.asyncio
    async def test_working_frame_evaluate_failure_raises_typed_error(self) -> None:
        page = _make_page()
        page.evaluate = AsyncMock(return_value=[])
        page.accessibility = None
        page._working_frame = SimpleNamespace(
            name="payment",
            url="https://example.com/payment-frame",
            evaluate=AsyncMock(side_effect=RuntimeError("Execution context was destroyed")),
        )

        with pytest.raises(browser_ops.ObserveFrameError) as exc_info:
            await do_observe(page)

        error = str(exc_info.value)
        assert page._working_frame.name in error

    @pytest.mark.asyncio
    async def test_working_frame_on_different_page_raises_typed_error(self) -> None:
        page = _make_page()
        page.page = object()
        page._working_frame = SimpleNamespace(
            name="payment",
            url="https://example.com/payment-frame",
            page=object(),
            evaluate=AsyncMock(return_value=[]),
        )

        with pytest.raises(browser_ops.ObserveFrameError) as exc_info:
            await do_observe(page)

        assert "different page" in str(exc_info.value)

    @pytest.mark.asyncio
    async def test_main_frame_evaluate_failure_remains_best_effort(self) -> None:
        page = _make_page()
        page.evaluate = AsyncMock(side_effect=RuntimeError("Execution context was destroyed"))

        result = await do_observe(page)

        assert "Sign In" in {element.name for element in result.elements}

    @pytest.mark.asyncio
    async def test_combobox_options(self) -> None:
        tree = _make_a11y_tree(
            children=[
                {
                    "role": "combobox",
                    "name": "Country",
                    "children": [
                        {"role": "option", "name": "US"},
                        {"role": "option", "name": "UK"},
                        {"role": "option", "name": "CA"},
                    ],
                },
            ]
        )
        page = _make_page(tree)
        result = await do_observe(page)
        country = serialize_elements(result.elements)[0]
        assert country["options"] == ["US", "UK", "CA"]
        assert "value" not in country

    @pytest.mark.asyncio
    async def test_document_reload_during_observe_refuses_durability(self, monkeypatch: pytest.MonkeyPatch) -> None:
        monkeypatch.setenv("SKYVERN_MCP_OBSERVE_V2", "1")
        page = _make_page()
        document_ids = AsyncMock(side_effect=["cdp:doc-a", "cdp:doc-b"])
        monkeypatch.setattr(browser_ops, "get_observe_document_id", document_ids)

        result = await do_observe(page)

        # The marker bracket refuses durability instead of failing the observe: elements stay
        # usable via selectors, but no document identity is certified for durable refs.
        assert result.document_id is None
        assert document_ids.await_count == 2

    @pytest.mark.asyncio
    async def test_dom_interactables_are_merged_with_selectors(self) -> None:
        page = _make_page(_make_a11y_tree(children=[{"role": "textbox", "name": "City", "value": ""}]))
        page.evaluate = _mock_dom_evaluate(
            [
                {
                    "role": "option",
                    "name": "Lakewood",
                    "tag": "div",
                    "selector": "#city-list > div:nth-of-type(1)",
                }
            ]
        )

        result = await do_observe(page)
        serialized = serialize_elements(result.elements)

        city_option = next(element for element in serialized if element["name"] == "Lakewood")
        assert city_option["selector"] == "#city-list > div:nth-of-type(1)"
        assert ref_to_selector(city_option) == "#city-list > div:nth-of-type(1)"

    @pytest.mark.asyncio
    async def test_dom_interactables_work_without_accessibility_snapshot(self) -> None:
        page = SimpleNamespace(
            url="https://example.com/form",
            title=AsyncMock(return_value="Form"),
            evaluate=_mock_dom_evaluate(
                [
                    {
                        "role": "option",
                        "name": "Music",
                        "tag": "div",
                        "selector": "#category-options > div:nth-of-type(2)",
                    }
                ]
            ),
        )

        result = await do_observe(page)
        serialized = serialize_elements(result.elements)

        assert len(serialized) == 1
        assert serialized[0]["name"] == "Music"
        assert serialized[0]["selector"] == "#category-options > div:nth-of-type(2)"

    @pytest.mark.asyncio
    async def test_dom_selector_is_attached_to_matching_a11y_option(self) -> None:
        tree = _make_a11y_tree(
            children=[
                {
                    "role": "combobox",
                    "name": "Region",
                    "children": [
                        {"role": "option", "name": "North"},
                        {"role": "option", "name": "East"},
                    ],
                },
            ]
        )
        page = _make_page(tree)
        page.evaluate = _mock_dom_evaluate(
            [
                {
                    "role": "option",
                    "name": "East",
                    "tag": "option",
                    "selector": "#region > option:nth-of-type(2)",
                }
            ]
        )

        result = await do_observe(page)
        serialized = serialize_elements(result.elements)

        east_option = next(element for element in serialized if element["name"] == "East")
        assert east_option["selector"] == "#region > option:nth-of-type(2)"
        assert "value" not in east_option
        assert ref_to_selector(east_option) == "#region > option:nth-of-type(2)"

    @pytest.mark.asyncio
    async def test_duplicate_label_options_do_not_get_wrong_selector(self) -> None:
        # Two a11y options share a label and two DOM options share it too. The merge must NOT
        # attach a DOM selector to an a11y option (ambiguous) — that would risk a confidently
        # wrong deterministic action.
        tree = _make_a11y_tree(
            children=[
                {
                    "role": "combobox",
                    "name": "Region",
                    "children": [
                        {"role": "option", "name": "North"},
                        {"role": "option", "name": "North"},
                    ],
                },
            ]
        )
        page = _make_page(tree)
        page.evaluate = _mock_dom_evaluate(
            [
                {
                    "role": "option",
                    "name": "North",
                    "tag": "option",
                    "selector": "#r > option:nth-of-type(1)",
                },
                {
                    "role": "option",
                    "name": "North",
                    "tag": "option",
                    "selector": "#r > option:nth-of-type(2)",
                },
            ]
        )
        result = await do_observe(page)
        serialized = serialize_elements(result.elements)
        a11y_norths = [e for e in serialized if e["name"] == "North" and not e.get("selector")]
        assert len(a11y_norths) == 2

    @pytest.mark.asyncio
    async def test_match_index_stable_when_cap_reorders(self) -> None:
        # A selector-bearing duplicate reordered ahead of the cap must NOT shift the nth ordinal
        # of a selectorless duplicate sharing its (role, name). The a11y duplicate must keep its
        # original ordinal (0), so its `nth=N` fallback ref points at the right element.
        tree = _make_a11y_tree(
            children=[
                {"role": "combobox", "name": "Dup", "value": ""},
                {"role": "combobox", "name": "Dup", "value": ""},
            ]
        )
        page = _make_page(tree)
        page.evaluate = _mock_dom_evaluate(
            [
                {"role": "combobox", "name": "Dup", "tag": "select", "selector": "#dup3"},
            ]
        )
        result = await do_observe(page, max_elements=2)
        selectorless_dups = [e for e in result.elements if e.name == "Dup" and not e.selector]
        assert selectorless_dups, result.elements
        assert selectorless_dups[0].match_index == 0

    @pytest.mark.asyncio
    async def test_role_to_tag_mapping(self) -> None:
        page = _make_page()
        result = await do_observe(page)
        tags = {e.name: e.tag for e in result.elements}
        assert tags["Email"] == "input"
        assert tags["Sign In"] == "button"
        assert tags["Forgot password?"] == "a"

    @pytest.mark.asyncio
    async def test_unique_elements_have_match_index_zero(self) -> None:
        page = _make_page()
        result = await do_observe(page)
        assert all(e.match_index == 0 for e in result.elements)

    @pytest.mark.asyncio
    async def test_unnamed_duplicates_get_distinct_match_indices(self) -> None:
        """SKY-9701: multiple unnamed widgets of the same role must each get a unique index."""
        tree = _make_a11y_tree(
            children=[
                {"role": "combobox", "name": ""},
                {"role": "combobox", "name": ""},
                {"role": "combobox", "name": ""},
                {"role": "combobox", "name": ""},
            ]
        )
        page = _make_page(tree)
        result = await do_observe(page)
        assert [e.match_index for e in result.elements] == [0, 1, 2, 3]

    @pytest.mark.asyncio
    async def test_named_duplicates_get_distinct_match_indices(self) -> None:
        """Elements sharing both role and name also collide; disambiguate by index."""
        tree = _make_a11y_tree(
            children=[
                {"role": "button", "name": "Edit"},
                {"role": "button", "name": "Delete"},
                {"role": "button", "name": "Edit"},
            ]
        )
        page = _make_page(tree)
        result = await do_observe(page)
        grouped = [(e.role, e.name, e.match_index) for e in result.elements]
        assert grouped == [
            ("button", "Edit", 0),
            ("button", "Delete", 0),
            ("button", "Edit", 1),
        ]

    @pytest.mark.asyncio
    async def test_serialized_output_includes_match_index_for_every_member_of_duplicate_group(self) -> None:
        """Every member of a multi-element (role, name) group carries match_index, including the first."""
        tree = _make_a11y_tree(
            children=[
                {"role": "combobox", "name": ""},
                {"role": "combobox", "name": ""},
            ]
        )
        page = _make_page(tree)
        result = await do_observe(page)
        serialized = serialize_elements(result.elements)
        assert serialized[0]["match_index"] == 0
        assert serialized[1]["match_index"] == 1

    @pytest.mark.asyncio
    async def test_serialized_output_omits_match_index_for_unique_elements(self) -> None:
        page = _make_page()
        result = await do_observe(page)
        serialized = serialize_elements(result.elements)
        assert all("match_index" not in d for d in serialized)

    @pytest.mark.asyncio
    async def test_disambiguation_counts_collisions_beyond_max_elements_cap(self) -> None:
        """SKY-9701 follow-up: duplicates beyond the cap must still trigger disambiguation
        for kept refs, otherwise selector-only clicks on capped duplicates fall back to the
        ambiguous base selector."""
        children = [{"role": "combobox", "name": ""} for _ in range(6)]
        page = _make_page(_make_a11y_tree(children=children))
        result = await do_observe(page, max_elements=2)
        assert result.element_count == 2
        assert result.total_on_page == 6
        serialized = serialize_elements(result.elements)
        # Both kept refs must carry match_index so the selector emits nth=N,
        # even though only 2 of 6 are returned.
        assert serialized[0]["match_index"] == 0
        assert serialized[1]["match_index"] == 1
        selectors = [ref_to_selector(elem) for elem in serialized]
        assert selectors == ["role=combobox >> nth=0", "role=combobox >> nth=1"]

    @pytest.mark.asyncio
    async def test_unnamed_duplicate_resolves_to_distinct_selectors(self) -> None:
        """End-to-end: observe -> serialize -> ref_to_selector produces unique selectors for every ref, including e0."""
        tree = _make_a11y_tree(
            children=[
                {"role": "combobox", "name": ""},
                {"role": "combobox", "name": ""},
                {"role": "combobox", "name": ""},
                {"role": "combobox", "name": ""},
            ]
        )
        page = _make_page(tree)
        result = await do_observe(page)
        serialized = serialize_elements(result.elements)
        selectors = [ref_to_selector(elem) for elem in serialized]
        assert selectors == [
            "role=combobox >> nth=0",
            "role=combobox >> nth=1",
            "role=combobox >> nth=2",
            "role=combobox >> nth=3",
        ]


# ---------------------------------------------------------------------------
# Unit tests: do_execute
# ---------------------------------------------------------------------------


class TestDoExecute:
    @pytest.mark.asyncio
    async def test_basic_batch(self) -> None:
        call_log: list[str] = []

        async def dispatch(step: ExecuteStep, ref_map: dict) -> dict[str, Any] | None:
            call_log.append(step.tool)
            return {"status": "ok"}

        steps = [
            ExecuteStep(tool="navigate", params={"url": "https://example.com"}),
            ExecuteStep(tool="click", params={"selector": "#btn"}),
        ]
        result = await do_execute(dispatch, steps)

        assert result.steps_completed == 2
        assert result.steps_total == 2
        assert result.error_step is None
        assert call_log == ["navigate", "click"]

    @pytest.mark.asyncio
    async def test_failed_observe_clears_batch_local_refs(self) -> None:
        seen_ref_maps: list[dict[str, Any]] = []

        async def dispatch(step: ExecuteStep, ref_map: dict) -> dict[str, Any] | None:
            if step.tool == "click":
                seen_ref_maps.append(dict(ref_map))
                return None
            if step.params.get("fail"):
                raise RuntimeError("frame lost")
            return {"elements": [{"ref": "e0", "role": "button", "name": "X", "tag": "button", "selector": "#x"}]}

        steps = [
            ExecuteStep(tool="observe", params={}),
            ExecuteStep(tool="observe", params={"fail": True}),
            ExecuteStep(tool="click", params={"ref": "e0"}),
        ]
        result = await do_execute(dispatch, steps, stop_on_error=False)

        assert result.results[0].ok is True
        assert result.results[1].ok is False
        assert seen_ref_maps == [{}]

    @pytest.mark.asyncio
    async def test_stop_on_error_true(self) -> None:
        async def dispatch(step: ExecuteStep, ref_map: dict) -> dict[str, Any] | None:
            if step.tool == "click":
                raise RuntimeError("Element not found")
            return None

        steps = [
            ExecuteStep(tool="navigate", params={}),
            ExecuteStep(tool="click", params={}),
            ExecuteStep(tool="type", params={}),
        ]
        result = await do_execute(dispatch, steps, stop_on_error=True)

        assert result.steps_completed == 2
        assert result.error_step == 1
        assert result.results[1].ok is False
        assert "Element not found" in (result.results[1].error or "")

    @pytest.mark.asyncio
    async def test_stop_on_error_false_continues(self) -> None:
        async def dispatch(step: ExecuteStep, ref_map: dict) -> dict[str, Any] | None:
            if step.tool == "click":
                raise RuntimeError("fail")
            return None

        steps = [
            ExecuteStep(tool="click", params={}),
            ExecuteStep(tool="scroll", params={}),
        ]
        result = await do_execute(dispatch, steps, stop_on_error=False)

        assert result.steps_completed == 2
        assert result.results[0].ok is False
        assert result.results[1].ok is True

    @pytest.mark.asyncio
    async def test_design_3_blocks_sensitive_after_failed_nav(self) -> None:
        """DESIGN-3: type and evaluate are blocked after failed navigate."""

        async def dispatch(step: ExecuteStep, ref_map: dict) -> dict[str, Any] | None:
            if step.tool == "navigate":
                raise RuntimeError("DNS resolution failed")
            return None

        steps = [
            ExecuteStep(tool="navigate", params={}),
            ExecuteStep(tool="type", params={}),
            ExecuteStep(tool="click", params={}),
            ExecuteStep(tool="evaluate", params={}),
        ]
        result = await do_execute(dispatch, steps, stop_on_error=False)

        assert result.steps_completed == 4
        # navigate failed
        assert result.results[0].ok is False
        # "type" tool blocked (sensitive)
        assert result.results[1].ok is False
        assert "blocked_by_failed_navigate" in (result.results[1].error or "")
        # click allowed (non-sensitive)
        assert result.results[2].ok is True
        # evaluate blocked (sensitive)
        assert result.results[3].ok is False
        assert "blocked_by_failed_navigate" in (result.results[3].error or "")

    @pytest.mark.asyncio
    async def test_design_3_not_triggered_with_stop_on_error(self) -> None:
        """DESIGN-3 only applies when stop_on_error=false."""

        async def dispatch(step: ExecuteStep, ref_map: dict) -> dict[str, Any] | None:
            if step.tool == "navigate":
                raise RuntimeError("fail")
            return None

        steps = [
            ExecuteStep(tool="navigate", params={}),
            ExecuteStep(tool="type", params={}),
        ]
        result = await do_execute(dispatch, steps, stop_on_error=True)
        # Stops at navigate, never reaches type
        assert result.steps_completed == 1

    @pytest.mark.asyncio
    async def test_design_4_ref_map_replaced_on_observe(self) -> None:
        """DESIGN-4: Each observe replaces the entire ref_map."""
        ref_maps_seen: list[dict] = []

        async def dispatch(step: ExecuteStep, ref_map: dict) -> dict[str, Any] | None:
            ref_maps_seen.append(dict(ref_map))
            if step.tool == "observe":
                return {
                    "elements": [
                        {"ref": "e0", "role": "button", "name": f"Btn-{step.params.get('call', 0)}"},
                    ],
                    "element_count": 1,
                    "total_on_page": 1,
                }
            return None

        steps = [
            ExecuteStep(tool="observe", params={"call": 1}),
            ExecuteStep(tool="click", params={}),
            ExecuteStep(tool="observe", params={"call": 2}),
            ExecuteStep(tool="click", params={}),
        ]
        result = await do_execute(dispatch, steps)

        assert result.steps_completed == 4
        # After first observe, ref_map has Btn-1
        assert ref_maps_seen[1].get("e0", {}).get("name") == "Btn-1"
        # After second observe, ref_map replaced with Btn-2
        assert ref_maps_seen[3].get("e0", {}).get("name") == "Btn-2"

    @pytest.mark.asyncio
    async def test_empty_steps(self) -> None:
        async def dispatch(step: ExecuteStep, ref_map: dict) -> dict[str, Any] | None:
            return None

        result = await do_execute(dispatch, [])
        assert result.steps_completed == 0
        assert result.error_step is None


# ---------------------------------------------------------------------------
# MCP tool tests: skyvern_observe
# ---------------------------------------------------------------------------


@pytest.mark.asyncio
async def test_observe_execute_tool_contract_qualifies_ref_lifetime_by_transport() -> None:
    tools = {tool.name: tool for tool in await mcp.list_tools()}
    observe = tools["skyvern_observe"]
    execute = tools["skyvern_execute"]
    surfaces = {
        "observe description": observe.description,
        "execute description": execute.description,
        "execute steps schema": execute.parameters["properties"]["steps"]["description"],
        "server instructions": mcp.instructions,
    }

    for surface, text in surfaces.items():
        assert text is not None, f"Missing {surface}"
        normalized = text.lower()
        for term in ("stdio", "stateless http", "selector", "intent"):
            assert term in normalized, f"{surface} missing {term!r}"
        assert "across calls in this browser session" not in normalized
        assert "across calls in the same browser session" not in normalized
        assert "do not persist" not in normalized, f"{surface} uses persist wording for the hosted failure"
        if surface != "server instructions":
            assert "do not resolve" in normalized, f"{surface} missing resolve semantics"


class TestSkyvernObserveMCP:
    @pytest.mark.asyncio
    async def test_observe_returns_elements(self, monkeypatch: pytest.MonkeyPatch) -> None:
        page = _make_page()
        ctx = BrowserContext(mode="cloud_session", session_id="pbs_test")
        monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))

        result = await mcp_browser.skyvern_observe()

        assert result["ok"] is True
        assert len(result["data"]["elements"]) == 4
        assert result["data"]["element_count"] == 4

    @pytest.mark.asyncio
    async def test_observe_hint_qualifies_ref_lifetime_by_transport(
        self,
        monkeypatch: pytest.MonkeyPatch,
    ) -> None:
        page = _make_page()
        ctx = BrowserContext(mode="local")
        monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))

        result = await mcp_browser.skyvern_observe()

        hint = result["data"]["hint"].lower()
        for term in ("stdio", "stateless http", "selector", "intent"):
            assert term in hint
        assert "across calls in this browser session" not in hint

    @pytest.mark.asyncio
    async def test_default_observe_omits_values_from_output_and_registry(
        self,
        monkeypatch: pytest.MonkeyPatch,
    ) -> None:
        page = _make_page()
        ctx = BrowserContext(mode="local")
        state = make_session_state(context=ctx)
        monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))

        async with scoped_session(state):
            result = await mcp_browser.skyvern_observe()

        assert all("value" not in element for element in result["data"]["elements"])
        assert all("value" not in element for element in state._observed_refs["refs"].values())

    @pytest.mark.asyncio
    async def test_observe_include_values_keeps_values_out_of_registry(
        self,
        monkeypatch: pytest.MonkeyPatch,
    ) -> None:
        page = _make_values_page()
        ctx = BrowserContext(mode="local")
        state = make_session_state(context=ctx)
        monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))

        async with scoped_session(state):
            result = await mcp_browser.skyvern_observe(include_values=True)

        email = next(element for element in result["data"]["elements"] if element["name"] == "Email")
        assert email["value"] == "[email protected]"
        assert all("value" not in element for element in state._observed_refs["refs"].values())

    @pytest.mark.asyncio
    async def test_observe_include_values_keeps_non_password_dom_value(
        self,
        monkeypatch: pytest.MonkeyPatch,
    ) -> None:
        page = _make_values_page()
        ctx = BrowserContext(mode="local")
        monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))

        result = await mcp_browser.skyvern_observe(include_values=True)

        email = next(element for element in result["data"]["elements"] if element["name"] == "Email")
        password = next(element for element in result["data"]["elements"] if element["name"] == "Current")
        assert email["value"] == "[email protected]"
        assert "value" not in password

    @pytest.mark.asyncio
    async def test_observe_preserves_existing_positional_parameter_order(
        self,
        monkeypatch: pytest.MonkeyPatch,
    ) -> None:
        page = _make_page()
        ctx = BrowserContext(mode="local")
        observe = AsyncMock(
            return_value=ObserveResult(
                url=page.url,
                title="Login Page",
                elements=[],
                element_count=0,
                total_on_page=0,
            )
        )
        monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))
        monkeypatch.setattr(mcp_browser, "do_observe", observe)

        result = await mcp_browser.skyvern_observe(None, None, None, False, 7)

        assert result["ok"] is True
        observe.assert_awaited_once_with(
            page,
            selector=None,
            interactive_only=False,
            max_elements=7,
            include_values=False,
            propagate_page_change=True,
        )

    @pytest.mark.asyncio
    async def test_observe_no_browser_clears_existing_refs(self, monkeypatch: pytest.MonkeyPatch) -> None:
        from skyvern.cli.mcp_tools._session import BrowserNotAvailableError

        monkeypatch.setenv("SKYVERN_MCP_OBSERVE_V2", "1")
        ctx = BrowserContext(mode="local")
        state = make_session_state(context=ctx)
        state._observed_refs = {"e0": {"ref": "e0", "selector": "#stale"}}
        state._observe_v2_state.refs = dict(state._observed_refs)
        monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(side_effect=BrowserNotAvailableError("no browser")))

        async with scoped_session(state):
            result = await mcp_browser.skyvern_observe()

        assert result["ok"] is False
        assert state._observed_refs == {}
        assert state._observe_v2_state.refs == {}

    @pytest.mark.asyncio
    async def test_cancelled_observe_page_lookup_clears_existing_refs(self, monkeypatch: pytest.MonkeyPatch) -> None:
        monkeypatch.setenv("SKYVERN_MCP_OBSERVE_V2", "1")
        ctx = BrowserContext(mode="local")
        state = make_session_state(context=ctx)
        state._observed_refs = {"e0": {"ref": "e0", "selector": "#stale"}}
        state._observe_v2_state.refs = dict(state._observed_refs)
        monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(side_effect=asyncio.CancelledError))

        async with scoped_session(state):
            with pytest.raises(asyncio.CancelledError):
                await mcp_browser.skyvern_observe()

        assert state._observed_refs == {}
        assert state._observe_v2_state.refs == {}

    @pytest.mark.asyncio
    async def test_generic_observe_failure_clears_existing_refs(self, monkeypatch: pytest.MonkeyPatch) -> None:
        monkeypatch.setenv("SKYVERN_MCP_OBSERVE_V2", "1")
        page = _make_page()
        ctx = BrowserContext(mode="local")
        state = make_session_state(context=ctx)
        state._observed_refs = {"e0": {"ref": "e0", "selector": "#stale"}}
        state._observe_v2_state.refs = dict(state._observed_refs)
        monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))
        monkeypatch.setattr(mcp_browser, "do_observe", AsyncMock(side_effect=RuntimeError("snapshot failed")))

        async with scoped_session(state):
            result = await mcp_browser.skyvern_observe()

        assert result["ok"] is False
        assert state._observed_refs == {}
        assert state._observe_v2_state.refs == {}

    @pytest.mark.asyncio
    async def test_cancelled_observe_snapshot_clears_existing_refs(self, monkeypatch: pytest.MonkeyPatch) -> None:
        monkeypatch.setenv("SKYVERN_MCP_OBSERVE_V2", "1")
        page = _make_page()
        ctx = BrowserContext(mode="local")
        state = make_session_state(context=ctx)
        state._observed_refs = {"e0": {"ref": "e0", "selector": "#stale"}}
        state._observe_v2_state.refs = dict(state._observed_refs)
        monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))
        monkeypatch.setattr(mcp_browser, "do_observe", AsyncMock(side_effect=asyncio.CancelledError))

        async with scoped_session(state):
            with pytest.raises(asyncio.CancelledError):
                await mcp_browser.skyvern_observe()

        assert state._observed_refs == {}
        assert state._observe_v2_state.refs == {}

    @pytest.mark.asyncio
    async def test_cancelled_observe_publication_clears_existing_refs(
        self,
        monkeypatch: pytest.MonkeyPatch,
    ) -> None:
        monkeypatch.setenv("SKYVERN_MCP_OBSERVE_V2", "1")
        page = _make_page()
        ctx = BrowserContext(mode="local")
        state = make_session_state(context=ctx)
        state._observed_refs = {"e0": {"ref": "e0", "selector": "#stale"}}
        state._observe_v2_state.refs = dict(state._observed_refs)
        observe_result = ObserveResult(
            url=page.url,
            title="Login Page",
            elements=[],
            element_count=0,
            total_on_page=0,
            document_id="doc-1",
        )
        publish = AsyncMock(side_effect=asyncio.CancelledError)
        monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))
        monkeypatch.setattr(mcp_browser, "do_observe", AsyncMock(return_value=observe_result))
        monkeypatch.setattr(mcp_browser, "_publish_observe_v2_refs", publish)

        async with scoped_session(state):
            with pytest.raises(asyncio.CancelledError):
                await mcp_browser.skyvern_observe()

        publish.assert_awaited_once()
        assert state._observed_refs == {}
        assert state._observe_v2_state.refs == {}

    @pytest.mark.asyncio
    async def test_non_evaluable_working_frame_returns_structured_error(
        self,
        monkeypatch: pytest.MonkeyPatch,
    ) -> None:
        page = _make_page()
        page.evaluate = AsyncMock(return_value=[])
        page.accessibility = None
        frame = SimpleNamespace(
            name="payment",
            url="https://example.com/payment-frame",
            evaluate=AsyncMock(side_effect=RuntimeError("Execution context was destroyed")),
        )
        page._working_frame = frame
        ctx = BrowserContext(mode="local")
        state = make_session_state(context=ctx)
        state._observed_refs = {"page_key": (1, 2, "old", "old-frame"), "refs": {"e0": {"ref": "e0"}}}
        monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))

        async with scoped_session(state):
            result = await mcp_browser.skyvern_observe()

        assert result["ok"] is False
        error = result["error"]
        assert error["code"] == mcp_browser.ErrorCode.ACTION_FAILED
        assert error["details"]["frame_name"] == frame.name
        assert error["details"]["frame_url"] == frame.url
        assert frame.name in error["message"] or frame.url in error["message"]
        assert any(action in error["hint"] for action in ("skyvern_frame_main", "skyvern_frame_list", "selector"))
        assert state._observed_refs == {}


# ---------------------------------------------------------------------------
# MCP tool tests: skyvern_execute
# ---------------------------------------------------------------------------


class TestSkyvernExecuteMCP:
    @pytest.mark.asyncio
    async def test_execute_empty_steps(self, monkeypatch: pytest.MonkeyPatch) -> None:
        result = await mcp_browser.skyvern_execute(steps=[])
        assert result["ok"] is True
        assert result["data"]["steps_completed"] == 0

    @pytest.mark.asyncio
    async def test_execute_too_many_steps(self, monkeypatch: pytest.MonkeyPatch) -> None:
        steps = [{"tool": "click", "params": {}} for _ in range(25)]
        result = await mcp_browser.skyvern_execute(steps=steps)
        assert result["ok"] is False
        assert "Too many steps" in result["error"]["message"]

    @pytest.mark.asyncio
    async def test_execute_invalid_tool(self, monkeypatch: pytest.MonkeyPatch) -> None:
        result = await mcp_browser.skyvern_execute(steps=[{"tool": "act", "params": {}}])
        assert result["ok"] is False
        assert "unknown tool" in result["error"]["message"].lower()

    @pytest.mark.asyncio
    async def test_execute_missing_tool_field(self, monkeypatch: pytest.MonkeyPatch) -> None:
        result = await mcp_browser.skyvern_execute(steps=[{"params": {}}])
        assert result["ok"] is False
        assert "missing 'tool'" in result["error"]["message"].lower()

    @pytest.mark.asyncio
    async def test_execute_no_browser(self, monkeypatch: pytest.MonkeyPatch) -> None:
        from skyvern.cli.mcp_tools._session import BrowserNotAvailableError

        monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(side_effect=BrowserNotAvailableError("no browser")))

        result = await mcp_browser.skyvern_execute(steps=[{"tool": "click", "params": {"selector": "#btn"}}])
        assert result["ok"] is False

    @pytest.mark.asyncio
    async def test_execute_refuses_a_frame_owned_by_another_page(self, monkeypatch: pytest.MonkeyPatch) -> None:
        """Pins the stale-frame arm on skyvern_execute's observe-v2 document reads.

        Both reads sit in a try whose other handler is CancelledError, so without this arm the
        refusal escapes the tool instead of becoming a structured error.
        """
        page = make_skyvern_page(_make_page())
        type(page).locator_scope = PropertyMock(
            side_effect=StaleFrameSelectionError("payment", "https://frame.example.com/")
        )
        ctx = BrowserContext(mode="local")
        monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))
        monkeypatch.setattr(mcp_browser, "observe_v2_enabled", lambda: True)

        result = await mcp_browser.skyvern_execute(steps=[{"tool": "click", "params": {"selector": "#btn"}}])

        assert result["ok"] is False, result
        assert result["error"]["code"] == mcp_browser.ErrorCode.STALE_FRAME_SELECTION, result

    @pytest.mark.asyncio
    async def test_execute_dispatch_calls_tool(self, monkeypatch: pytest.MonkeyPatch) -> None:
        page = _make_page()
        ctx = BrowserContext(mode="local")
        monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))

        click_result = {"ok": True, "data": {"resolved_selector": "#btn"}}
        monkeypatch.setattr(mcp_browser, "skyvern_click", AsyncMock(return_value=click_result))

        result = await mcp_browser.skyvern_execute(steps=[{"tool": "click", "params": {"selector": "#btn"}}])
        assert result["ok"] is True
        assert result["data"]["steps_completed"] == 1
        mcp_browser.skyvern_click.assert_awaited_once()

    @pytest.mark.asyncio
    async def test_execute_observe_then_click_ref(self, monkeypatch: pytest.MonkeyPatch) -> None:
        """Integration: observe provides refs, click uses them."""
        page = _make_page()
        ctx = BrowserContext(mode="local")
        monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))

        click_result = {"ok": True, "data": None}
        monkeypatch.setattr(mcp_browser, "skyvern_click", AsyncMock(return_value=click_result))

        result = await mcp_browser.skyvern_execute(
            steps=[
                {"tool": "observe", "params": {}},
                {"tool": "click", "params": {"ref": "e2"}},  # e2 = Sign In button
            ]
        )
        assert result["ok"] is True
        assert result["data"]["steps_completed"] == 2

        # Verify click was called with selector resolved from ref
        click_call = mcp_browser.skyvern_click.call_args
        assert 'role=button[name="Sign In"]' in str(click_call)

    @pytest.mark.asyncio
    async def test_execute_observe_threads_include_values(self, monkeypatch: pytest.MonkeyPatch) -> None:
        page = _make_page()
        ctx = BrowserContext(mode="local")
        observe = AsyncMock(
            return_value=ObserveResult(
                url=page.url,
                title="Login Page",
                elements=[],
                element_count=0,
                total_on_page=0,
            )
        )
        monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))
        monkeypatch.setattr("skyvern.cli.core.browser_ops.do_observe", observe)

        result = await mcp_browser.skyvern_execute(steps=[{"tool": "observe", "params": {"include_values": True}}])

        assert result["ok"] is True
        observe.assert_awaited_once_with(page, include_values=True)

    @pytest.mark.asyncio
    async def test_execute_observe_returns_structured_frame_error_and_clears_refs(
        self,
        monkeypatch: pytest.MonkeyPatch,
    ) -> None:
        page = _make_page()
        frame = SimpleNamespace(
            name="payment",
            url="https://example.com/payment-frame",
            is_detached=lambda: True,
            evaluate=AsyncMock(),
        )
        page._working_frame = frame
        ctx = BrowserContext(mode="local")
        state = make_session_state(context=ctx)
        state._observed_refs = {"page_key": (1, 2, "old", "old-frame"), "refs": {"e0": {"ref": "e0"}}}
        monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))

        async with scoped_session(state):
            result = await mcp_browser.skyvern_execute(steps=[{"tool": "observe", "params": {}}])

        error = result["data"]["results"][0]["error"]
        assert result["ok"] is False
        assert error["code"] == mcp_browser.ErrorCode.ACTION_FAILED
        assert error["details"] == {"frame_name": frame.name, "frame_url": frame.url}
        assert "skyvern_frame_main" in error["hint"]
        assert state._observed_refs == {}
        frame.evaluate.assert_not_awaited()

    @pytest.mark.asyncio
    async def test_execute_resolves_ref_from_prior_observe(self, monkeypatch: pytest.MonkeyPatch) -> None:
        page = _make_page()
        ctx = BrowserContext(mode="cloud_session", session_id="pbs_test")
        monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))

        click_result = {"ok": True, "data": None}
        monkeypatch.setattr(mcp_browser, "skyvern_click", AsyncMock(return_value=click_result))

        async with scoped_session(make_session_state(context=ctx)):
            observe_result = await mcp_browser.skyvern_observe(session_id=ctx.session_id)
            ref = next(element["ref"] for element in observe_result["data"]["elements"] if element["name"] == "Sign In")

        async with scoped_session(make_session_state(context=ctx)):
            execute_result = await mcp_browser.skyvern_execute(
                steps=[{"tool": "click", "params": {"ref": ref}}],
                session_id=ctx.session_id,
            )

        assert execute_result["ok"] is True
        assert execute_result["data"]["steps_completed"] == 1
        assert mcp_browser.skyvern_click.call_args.kwargs["selector"] == 'role=button[name="Sign In"]'

    @pytest.mark.asyncio
    async def test_execute_observe_include_values_keeps_values_out_of_registry(
        self,
        monkeypatch: pytest.MonkeyPatch,
    ) -> None:
        page = _make_values_page()
        ctx = BrowserContext(mode="local")
        state = make_session_state(context=ctx)
        monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))

        async with scoped_session(state):
            result = await mcp_browser.skyvern_execute(steps=[{"tool": "observe", "params": {"include_values": True}}])

        elements = result["data"]["results"][0]["data"]["elements"]
        email = next(element for element in elements if element["name"] == "Email")
        assert email["value"] == "[email protected]"
        assert state._observed_refs["refs"]
        assert all("value" not in element for element in state._observed_refs["refs"].values())

    @pytest.mark.asyncio
    async def test_execute_resolves_ref_from_prior_observe_without_session_id(
        self,
        monkeypatch: pytest.MonkeyPatch,
    ) -> None:
        """The original stdio/local path keeps refs on the shared SessionState."""
        page = _make_page()
        ctx = BrowserContext(mode="local")
        state = make_session_state(context=ctx)
        monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))
        monkeypatch.setattr(mcp_browser, "skyvern_click", AsyncMock(return_value={"ok": True, "data": None}))

        async with scoped_session(state):
            observe_result = await mcp_browser.skyvern_observe()
            ref = next(element["ref"] for element in observe_result["data"]["elements"] if element["name"] == "Sign In")

        async with scoped_session(state):
            execute_result = await mcp_browser.skyvern_execute(steps=[{"tool": "click", "params": {"ref": ref}}])

        assert execute_result["ok"] is True
        assert state._observed_refs["refs"][ref]["name"] == "Sign In"
        assert mcp_browser.skyvern_click.call_args.kwargs["selector"] == 'role=button[name="Sign In"]'

    @pytest.mark.asyncio
    async def test_keyless_ref_is_stale_when_url_changes_during_observe(
        self,
        monkeypatch: pytest.MonkeyPatch,
    ) -> None:
        page = _make_page()

        async def replace_document_during_observe() -> str:
            page.url = "https://example.com/replaced-document"
            return "Replacement"

        page.title = AsyncMock(side_effect=replace_document_during_observe)
        ctx = BrowserContext(mode="local")
        state = make_session_state(context=ctx)
        monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))
        click = AsyncMock(return_value={"ok": True, "data": None})
        monkeypatch.setattr(mcp_browser, "skyvern_click", click)

        async with scoped_session(state):
            observe_result = await mcp_browser.skyvern_observe()
            ref = observe_result["data"]["elements"][0]["ref"]

        async with scoped_session(state):
            execute_result = await mcp_browser.skyvern_execute(steps=[{"tool": "click", "params": {"ref": ref}}])

        assert execute_result["data"]["results"][0]["error"] == (
            f"Unknown ref '{ref}' — call observe first or check ref exists"
        )
        click.assert_not_awaited()

    @pytest.mark.asyncio
    async def test_iframe_ref_is_stale_after_frame_navigation(self, monkeypatch: pytest.MonkeyPatch) -> None:
        page = _make_page()
        frame = SimpleNamespace(
            name="payment",
            url="https://example.com/payment-frame",
            page=page.page,
            title=AsyncMock(return_value="Payment"),
            evaluate=_mock_dom_evaluate(
                [
                    {
                        "role": "button",
                        "name": "Frame action",
                        "tag": "button",
                        "selector": "#frame-action",
                    }
                ]
            ),
        )

        async def goto(url: str) -> None:
            frame.url = url

        frame.goto = AsyncMock(side_effect=goto)
        page._working_frame = frame
        ctx = BrowserContext(mode="local")
        state = make_session_state(context=ctx)
        monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))
        click = AsyncMock(return_value={"ok": True, "data": None})
        monkeypatch.setattr(mcp_browser, "skyvern_click", click)

        async with scoped_session(state):
            observed = await mcp_browser.skyvern_observe()
            ref = observed["data"]["elements"][0]["ref"]
            await frame.goto("https://example.com/replacement-frame")
            result = await mcp_browser.skyvern_execute(steps=[{"tool": "click", "params": {"ref": ref}}])

        assert result["data"]["results"][0]["error"] == (
            f"Unknown ref '{ref}' — call observe first or check ref exists"
        )
        click.assert_not_awaited()

    @pytest.mark.asyncio
    async def test_execute_does_not_resolve_ref_from_different_session(self, monkeypatch: pytest.MonkeyPatch) -> None:
        page = _make_page()
        first_ctx = BrowserContext(mode="cloud_session", session_id="pbs_first")
        second_ctx = BrowserContext(mode="cloud_session", session_id="pbs_second")
        monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, first_ctx)))

        async with scoped_session(make_session_state(context=first_ctx)):
            observe_result = await mcp_browser.skyvern_observe(session_id=first_ctx.session_id)

        monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, second_ctx)))
        async with scoped_session(make_session_state(context=second_ctx)):
            execute_result = await mcp_browser.skyvern_execute(
                steps=[{"tool": "click", "params": {"ref": observe_result["data"]["elements"][0]["ref"]}}],
                session_id=second_ctx.session_id,
            )

        assert execute_result["data"]["results"][0]["error"] == (
            "Unknown ref 'e0' — call observe first or check ref exists"
        )

    @pytest.mark.asyncio
    async def test_execute_observe_step_persists_refs_for_next_call(self, monkeypatch: pytest.MonkeyPatch) -> None:
        page = _make_page()
        ctx = BrowserContext(mode="cloud_session", session_id="pbs_batch_observe")
        monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))

        click_result = {"ok": True, "data": None}
        monkeypatch.setattr(mcp_browser, "skyvern_click", AsyncMock(return_value=click_result))

        async with scoped_session(make_session_state(context=ctx)):
            observe_result = await mcp_browser.skyvern_execute(
                steps=[{"tool": "observe", "params": {}}],
                session_id=ctx.session_id,
            )
            click_result = await mcp_browser.skyvern_execute(
                steps=[{"tool": "click", "params": {"ref": "e2"}}],
                session_id=ctx.session_id,
            )

        assert observe_result["ok"] is True
        assert click_result["ok"] is True
        assert mcp_browser.skyvern_click.call_args.kwargs["selector"] == 'role=button[name="Sign In"]'

    @pytest.mark.asyncio
    async def test_second_observe_replaces_prior_session_refs(self, monkeypatch: pytest.MonkeyPatch) -> None:
        first_tree = _make_a11y_tree(
            children=[
                {"role": "button", "name": "First"},
                {"role": "button", "name": "Stale"},
            ]
        )
        second_tree = _make_a11y_tree(children=[{"role": "button", "name": "Current"}])
        page = _make_page(first_tree)
        page.accessibility.snapshot.side_effect = [first_tree, second_tree]
        ctx = BrowserContext(mode="cloud_session", session_id="pbs_replace_refs")
        monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))

        click_tool_result = {"ok": True, "data": None}
        monkeypatch.setattr(mcp_browser, "skyvern_click", AsyncMock(return_value=click_tool_result))

        async with scoped_session(make_session_state(context=ctx)):
            first_result = await mcp_browser.skyvern_observe(session_id=ctx.session_id)
            second_result = await mcp_browser.skyvern_observe(session_id=ctx.session_id)
            stale_result = await mcp_browser.skyvern_execute(
                steps=[{"tool": "click", "params": {"ref": "e1"}}],
                session_id=ctx.session_id,
            )
            current_result = await mcp_browser.skyvern_execute(
                steps=[{"tool": "click", "params": {"ref": "e0"}}],
                session_id=ctx.session_id,
            )

        assert [element["ref"] for element in first_result["data"]["elements"]] == ["e0", "e1"]
        assert [element["ref"] for element in second_result["data"]["elements"]] == ["e0"]
        assert stale_result["data"]["results"][0]["error"] == (
            "Unknown ref 'e1' — call observe first or check ref exists"
        )
        assert current_result["ok"] is True
        assert mcp_browser.skyvern_click.call_args.kwargs["selector"] == 'role=button[name="Current"]'

    @pytest.mark.asyncio
    @pytest.mark.usefixtures("public_dns")
    async def test_navigate_clears_session_refs(self, monkeypatch: pytest.MonkeyPatch) -> None:
        page = _make_page()
        ctx = BrowserContext(mode="cloud_session", session_id="pbs_navigate_refs")
        monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))

        async with scoped_session(make_session_state(context=ctx)):
            observe_result = await mcp_browser.skyvern_observe(session_id=ctx.session_id)
            navigate_result = await mcp_browser.skyvern_navigate(
                url="https://example.com/next",
                session_id=ctx.session_id,
            )
            execute_result = await mcp_browser.skyvern_execute(
                steps=[{"tool": "click", "params": {"ref": "e2"}}],
                session_id=ctx.session_id,
            )

        assert observe_result["ok"] is True
        assert navigate_result["ok"] is True
        assert execute_result["data"]["results"][0]["error"] == (
            "Unknown ref 'e2' — call observe first or check ref exists"
        )

    @pytest.mark.asyncio
    @pytest.mark.usefixtures("public_dns")
    async def test_navigate_clears_batch_refs(self, monkeypatch: pytest.MonkeyPatch) -> None:
        page = _make_page()
        ctx = BrowserContext(mode="cloud_session", session_id="pbs_batch_navigate_refs")
        monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))
        monkeypatch.setattr(mcp_browser, "skyvern_click", AsyncMock(return_value={"ok": True, "data": None}))

        async with scoped_session(make_session_state(context=ctx)):
            result = await mcp_browser.skyvern_execute(
                steps=[
                    {"tool": "observe", "params": {}},
                    {"tool": "navigate", "params": {"url": "https://example.com/next"}},
                    {"tool": "click", "params": {"ref": "e2"}},
                ],
                session_id=ctx.session_id,
            )

        assert result["ok"] is False
        assert result["data"]["error_step"] == 2
        assert result["data"]["results"][2]["error"] == ("Unknown ref 'e2' — call observe first or check ref exists")
        mcp_browser.skyvern_click.assert_not_awaited()

    @pytest.mark.asyncio
    async def test_tab_switch_invalidates_session_refs(self, monkeypatch: pytest.MonkeyPatch) -> None:
        page = _make_page()
        ctx = BrowserContext(mode="cloud_session", session_id="pbs_tab_refs")
        monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))

        tab_a = MagicMock()
        tab_a.url = "https://example.com/a"
        tab_a.title = AsyncMock(return_value="A")
        tab_a.is_closed = MagicMock(return_value=False)
        tab_b = MagicMock()
        tab_b.url = "https://example.com/b"
        tab_b.title = AsyncMock(return_value="B")
        tab_b.is_closed = MagicMock(return_value=False)
        tab_b.bring_to_front = AsyncMock()
        browser = MagicMock()
        browser._browser_context.pages = [tab_a, tab_b]
        monkeypatch.setattr(mcp_tabs, "resolve_browser", AsyncMock(return_value=(browser, ctx)))

        async with scoped_session(make_session_state(context=ctx, browser=browser)):
            observe_result = await mcp_browser.skyvern_observe(session_id=ctx.session_id)
            ref = observe_result["data"]["elements"][0]["ref"]
            switch_result = await mcp_tabs.skyvern_tab_switch(session_id=ctx.session_id, index=1)
            execute_result = await mcp_browser.skyvern_execute(
                steps=[{"tool": "click", "params": {"ref": ref}}],
                session_id=ctx.session_id,
            )

        assert switch_result["ok"] is True
        assert execute_result["data"]["results"][0]["error"] == (
            f"Unknown ref '{ref}' — call observe first or check ref exists"
        )

    @pytest.mark.asyncio
    async def test_frame_main_clears_session_refs(self, monkeypatch: pytest.MonkeyPatch) -> None:
        page = _make_page()
        page.frame_main = MagicMock()
        ctx = BrowserContext(mode="cloud_session", session_id="pbs_frame_refs")
        monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))

        async with scoped_session(make_session_state(context=ctx)):
            observe_result = await mcp_browser.skyvern_observe(session_id=ctx.session_id)
            ref = observe_result["data"]["elements"][0]["ref"]
            frame_result = await mcp_browser.skyvern_frame_main(session_id=ctx.session_id)
            execute_result = await mcp_browser.skyvern_execute(
                steps=[{"tool": "click", "params": {"ref": ref}}],
                session_id=ctx.session_id,
            )

        assert frame_result["ok"] is True
        assert execute_result["data"]["results"][0]["error"] == (
            f"Unknown ref '{ref}' — call observe first or check ref exists"
        )

    @pytest.mark.asyncio
    async def test_popup_page_change_invalidates_refs(self, monkeypatch: pytest.MonkeyPatch) -> None:
        """Implicit page transition (popup steals working page) must not resolve old refs."""
        page_a = _make_page()
        page_b = _make_page()
        ctx = BrowserContext(mode="cloud_session", session_id="pbs_popup_refs")

        async with scoped_session(make_session_state(context=ctx)):
            monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page_a, ctx)))
            observe_result = await mcp_browser.skyvern_observe(session_id=ctx.session_id)
            ref = observe_result["data"]["elements"][0]["ref"]

            monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page_b, ctx)))
            execute_result = await mcp_browser.skyvern_execute(
                steps=[{"tool": "click", "params": {"ref": ref}}],
                session_id=ctx.session_id,
            )

        assert execute_result["data"]["results"][0]["error"] == (
            f"Unknown ref '{ref}' — call observe first or check ref exists"
        )

    @pytest.mark.asyncio
    async def test_generation_eviction_does_not_resurrect_refs(self) -> None:
        """ABA guard: eviction of a cleared session's generation must not let a stale commit through."""
        ctx = BrowserContext(mode="cloud_session", session_id="pbs_aba_victim")
        element = {"ref": "e0", "role": "button", "name": "Sign In", "selector": "#signin"}

        async with scoped_session(make_session_state(context=ctx)):
            stale_generation = session_manager.session_ref_generation(session_id=ctx.session_id)
            session_manager.clear_session_ref_map(session_id=ctx.session_id)
            for i in range(session_manager._SESSION_REF_STORE_MAX + 5):
                session_manager.clear_session_ref_map(session_id=f"pbs_aba_churn_{i}")

            committed = session_manager.replace_session_ref_map(
                {"e0": element}, session_id=ctx.session_id, generation=stale_generation
            )
            assert committed is False
            assert session_manager.get_session_ref("e0", session_id=ctx.session_id) is None

    @pytest.mark.asyncio
    async def test_newer_publication_reservation_rejects_older_commit_and_clear(self) -> None:
        ctx = BrowserContext(mode="cloud_session", session_id="pbs_publication_cas")
        page_key = (1, None, "https://example.com", None)
        older = {"ref": "e0", "selector": "#older"}
        newer = {"ref": "e1", "selector": "#newer"}

        async with scoped_session(make_session_state(context=ctx)):
            older_generation = session_manager.begin_session_ref_publication(session_id=ctx.session_id)
            newer_generation = session_manager.begin_session_ref_publication(session_id=ctx.session_id)

            assert session_manager.replace_session_ref_map(
                {"e1": newer},
                session_id=ctx.session_id,
                generation=newer_generation,
                page_key=page_key,
            )
            committed_generation = session_manager.session_ref_generation(session_id=ctx.session_id)
            assert committed_generation > newer_generation
            assert not session_manager.clear_session_ref_map(
                session_id=ctx.session_id,
                generation=newer_generation,
            )
            assert not session_manager.replace_session_ref_map(
                {"e0": older},
                session_id=ctx.session_id,
                generation=older_generation,
                page_key=page_key,
            )
            assert not session_manager.clear_session_ref_map(
                session_id=ctx.session_id,
                generation=older_generation,
            )
            assert session_manager.get_session_ref("e1", session_id=ctx.session_id, page_key=page_key) == newer

    @pytest.mark.asyncio
    async def test_rejected_batch_observe_does_not_install_local_refs(self) -> None:
        """If session publication rejects an observe snapshot, the batch must not act on it either."""

        async def dispatch(step: ExecuteStep, ref_map: dict[str, dict[str, Any]]) -> dict[str, Any] | None:
            if step.tool == "observe":
                return {"elements": [{"ref": "e0", "role": "button", "name": "Sign In"}], "element_count": 1}
            if ref := step.params.get("ref"):
                if ref not in ref_map:
                    raise ValueError(f"Unknown ref '{ref}' — call observe first or check ref exists")
            return None

        result = await do_execute(
            dispatch,
            [ExecuteStep(tool="observe"), ExecuteStep(tool="click", params={"ref": "e0"})],
            on_ref_map_update=lambda ref_map: False,
        )

        assert result.error_step == 1
        assert result.results[0].ok is True
        assert "Unknown ref 'e0'" in str(result.results[1].error)
        assert len(result.results) == 2

    @pytest.mark.asyncio
    async def test_strict_rejected_batch_observe_fails_before_ref_step(self) -> None:
        async def dispatch(step: ExecuteStep, ref_map: dict[str, dict[str, Any]]) -> dict[str, Any] | None:
            if step.tool == "observe":
                return {"elements": [{"ref": "e0", "role": "button", "name": "Sign In"}], "element_count": 1}
            if ref := step.params.get("ref"):
                if ref not in ref_map:
                    raise ValueError(f"Unknown ref '{ref}' — call observe first or check ref exists")
            return None

        result = await do_execute(
            dispatch,
            [ExecuteStep(tool="observe"), ExecuteStep(tool="click", params={"ref": "e0"})],
            on_ref_map_update=lambda ref_map: False,
            fail_on_ref_map_rejection=True,
        )

        assert result.error_step == 0
        assert "observe_snapshot_superseded" in str(result.results[0].error)
        assert len(result.results) == 1

    @pytest.mark.asyncio
    async def test_batch_observe_awaits_async_ref_publication(self) -> None:
        async def dispatch(step: ExecuteStep, ref_map: dict[str, dict[str, Any]]) -> dict[str, Any] | None:
            if step.tool == "observe":
                return {"elements": [{"ref": "e0", "role": "button", "name": "Sign In"}], "element_count": 1}
            if step.params["ref"] not in ref_map:
                raise ValueError("missing accepted ref")
            return None

        publication_started = False

        async def publish(_ref_map: dict[str, dict[str, Any]]) -> bool:
            nonlocal publication_started
            publication_started = True
            return True

        result = await do_execute(
            dispatch,
            [ExecuteStep(tool="observe"), ExecuteStep(tool="click", params={"ref": "e0"})],
            on_ref_map_update=publish,
        )

        assert publication_started is True
        assert result.error_step is None

    @pytest.mark.asyncio
    @pytest.mark.usefixtures("public_dns")
    async def test_observe_started_during_navigation_is_discarded(self, monkeypatch: pytest.MonkeyPatch) -> None:
        """A generation captured while navigation is in flight must not commit after it completes."""
        page = _make_page()
        ctx = BrowserContext(mode="cloud_session", session_id="pbs_nav_race")
        monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))

        in_flight_generation: list[int] = []

        async def fake_navigate(*args: Any, **kwargs: Any) -> Any:
            in_flight_generation.append(session_manager.session_ref_generation(session_id=ctx.session_id))
            return NavigateResult(url="https://example.com/next", title="Next")

        monkeypatch.setattr(mcp_browser, "do_navigate", fake_navigate)

        async with scoped_session(make_session_state(context=ctx)):
            nav_result = await mcp_browser.skyvern_navigate("https://example.com/next", session_id=ctx.session_id)
            committed = session_manager.replace_session_ref_map(
                {"e0": {"ref": "e0", "selector": "#old"}},
                session_id=ctx.session_id,
                generation=in_flight_generation[0],
            )

        assert nav_result["ok"] is True
        assert committed is False
        assert session_manager.get_session_ref("e0", session_id=ctx.session_id) is None

    @pytest.mark.asyncio
    async def test_popup_mid_batch_invalidates_batch_refs(self, monkeypatch: pytest.MonkeyPatch) -> None:
        """A page change before ref publication rejects the unpublished observe snapshot."""
        monkeypatch.delenv("SKYVERN_MCP_OBSERVE_V2", raising=False)
        page_a = _make_page()
        page_b = _make_page()
        ctx = BrowserContext(mode="cloud_session", session_id="pbs_popup_batch")
        calls = 0

        async def get_page(**_kwargs: Any) -> tuple[Any, BrowserContext]:
            nonlocal calls
            calls += 1
            return (page_a, ctx) if calls <= 2 else (page_b, ctx)

        monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(side_effect=get_page))

        async with scoped_session(make_session_state(context=ctx)):
            result = await mcp_browser.skyvern_execute(
                steps=[
                    {"tool": "observe", "params": {}},
                    {"tool": "click", "params": {"ref": "e0"}},
                ],
                session_id=ctx.session_id,
            )

        assert result["ok"] is False
        assert len(result["data"]["results"]) == 2
        assert result["data"]["results"][0]["ok"] is True
        assert result["data"]["results"][1]["error"] == ("Unknown ref 'e0' — call observe first or check ref exists")

    @pytest.mark.asyncio
    async def test_popup_before_batch_observe_binds_refs_to_observed_page(
        self,
        monkeypatch: pytest.MonkeyPatch,
    ) -> None:
        """An observe after popup steal binds to the new page it actually inspected."""
        page_a = _make_page()
        page_b = _make_page()
        page_b.url = "https://example.com/popup"
        ctx = BrowserContext(mode="cloud_session", session_id="pbs_popup_before_observe")
        calls = 0

        async def get_page(**_kwargs: Any) -> tuple[Any, BrowserContext]:
            nonlocal calls
            calls += 1
            return (page_a, ctx) if calls == 1 else (page_b, ctx)

        monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(side_effect=get_page))
        monkeypatch.setattr(mcp_browser, "skyvern_evaluate", AsyncMock(return_value={"ok": True, "data": None}))
        click = AsyncMock(return_value={"ok": True, "data": None})
        monkeypatch.setattr(mcp_browser, "skyvern_click", click)

        async with scoped_session(make_session_state(context=ctx)):
            result = await mcp_browser.skyvern_execute(
                steps=[
                    {"tool": "evaluate", "params": {"expression": "window.open('about:blank')"}},
                    {"tool": "observe", "params": {}},
                    {"tool": "click", "params": {"ref": "e0"}},
                ],
                session_id=ctx.session_id,
            )

        assert result["ok"] is True
        click.assert_awaited_once()

    @pytest.mark.asyncio
    async def test_same_page_url_change_invalidates_batch_refs(self, monkeypatch: pytest.MonkeyPatch) -> None:
        """A same-tab document replacement keeps Page identity but invalidates refs by URL."""
        page = _make_page()
        ctx = BrowserContext(mode="cloud_session", session_id="pbs_click_nav_batch")
        monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))

        async def replace_document(**kwargs: Any) -> dict[str, Any]:
            page.url = "https://example.com/replaced-document"
            return {"ok": True, "data": None}

        monkeypatch.setattr(mcp_browser, "skyvern_evaluate", AsyncMock(side_effect=replace_document))
        click = AsyncMock(return_value={"ok": True, "data": None})
        monkeypatch.setattr(mcp_browser, "skyvern_click", click)

        async with scoped_session(make_session_state(context=ctx)):
            result = await mcp_browser.skyvern_execute(
                steps=[
                    {"tool": "observe", "params": {}},
                    {"tool": "evaluate", "params": {"expression": "location.href = '/replaced-document'"}},
                    {"tool": "click", "params": {"ref": "e0"}},
                ],
                session_id=ctx.session_id,
            )

        assert result["data"]["results"][2]["error"] == ("Unknown ref 'e0' — call observe first or check ref exists")
        click.assert_not_awaited()

    @pytest.mark.asyncio
    @pytest.mark.usefixtures("public_dns")
    async def test_failed_navigation_still_invalidates_midflight_observe(self, monkeypatch: pytest.MonkeyPatch) -> None:
        """A navigation that raises can still have replaced the document — mid-flight commits must die."""
        page = _make_page()
        ctx = BrowserContext(mode="cloud_session", session_id="pbs_failed_nav")
        monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))

        in_flight_generation: list[int] = []

        async def failing_navigate(*args: Any, **kwargs: Any) -> Any:
            in_flight_generation.append(session_manager.session_ref_generation(session_id=ctx.session_id))
            raise RuntimeError("net::ERR_ABORTED")

        monkeypatch.setattr(mcp_browser, "do_navigate", failing_navigate)

        async with scoped_session(make_session_state(context=ctx)):
            nav_result = await mcp_browser.skyvern_navigate("https://example.com/next", session_id=ctx.session_id)
            committed = session_manager.replace_session_ref_map(
                {"e0": {"ref": "e0", "selector": "#old"}},
                session_id=ctx.session_id,
                generation=in_flight_generation[0],
            )

        assert nav_result["ok"] is False
        assert committed is False
        assert session_manager.get_session_ref("e0", session_id=ctx.session_id) is None

    @pytest.mark.asyncio
    async def test_stale_observe_commit_is_discarded_after_clear(self) -> None:
        ctx = BrowserContext(mode="cloud_session", session_id="pbs_stale_gen")
        element = {"ref": "e0", "role": "button", "name": "Sign In", "selector": "#signin"}

        async with scoped_session(make_session_state(context=ctx)):
            stale_generation = session_manager.session_ref_generation(session_id=ctx.session_id)
            session_manager.clear_session_ref_map(session_id=ctx.session_id)
            session_manager.replace_session_ref_map(
                {"e0": element}, session_id=ctx.session_id, generation=stale_generation
            )
            assert session_manager.get_session_ref("e0", session_id=ctx.session_id) is None

            fresh_generation = session_manager.session_ref_generation(session_id=ctx.session_id)
            session_manager.replace_session_ref_map(
                {"e0": element}, session_id=ctx.session_id, generation=fresh_generation
            )
            assert session_manager.get_session_ref("e0", session_id=ctx.session_id) == element

    @pytest.mark.asyncio
    async def test_execute_observe_then_click_native_option_ref_selects_parent(
        self,
        monkeypatch: pytest.MonkeyPatch,
    ) -> None:
        option_selector = "#region > option:nth-of-type(4)"
        option_locator = SimpleNamespace(
            evaluate=AsyncMock(
                return_value={
                    "select_selector": "#region",
                    "value": "east",
                    "label": "East",
                }
            )
        )
        option_locator.first = option_locator
        select_option = AsyncMock()
        select_locator = SimpleNamespace(select_option=select_option)
        page = SimpleNamespace(
            url="https://example.com/form",
            title=AsyncMock(return_value="Form"),
            accessibility=None,
            evaluate=_mock_dom_evaluate(
                [
                    {
                        "role": "option",
                        "name": "East",
                        "tag": "option",
                        "selector": option_selector,
                    }
                ]
            ),
            locator=MagicMock(
                side_effect=lambda selector: option_locator if selector == option_selector else select_locator
            ),
            click=AsyncMock(return_value="#unexpected-click"),
        )
        page.page = page
        page.locator_scope = page
        page._working_frame = None
        ctx = BrowserContext(mode="local")
        monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))

        result = await mcp_browser.skyvern_execute(
            steps=[
                {"tool": "observe", "params": {}},
                {"tool": "click", "params": {"ref": "e0"}},
            ]
        )

        assert result["ok"] is True
        assert result["data"]["steps_completed"] == 2
        page.click.assert_not_awaited()
        select_option.assert_awaited_once_with(value="east", timeout=5000)
        assert result["data"]["results"][1]["data"]["resolved_selector"] == "#region"

    @pytest.mark.asyncio
    @pytest.mark.parametrize("stateless_http_mode", [False, True])
    async def test_execute_unknown_ref_error_is_transport_aware(
        self,
        monkeypatch: pytest.MonkeyPatch,
        stateless_http_mode: bool,
    ) -> None:
        page = _make_page()
        ctx = BrowserContext(mode="local")
        monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))

        previous_mode = session_manager.is_stateless_http_mode()
        try:
            session_manager.set_stateless_http_mode(stateless_http_mode)
            async with scoped_session(make_session_state(context=ctx)):
                result = await mcp_browser.skyvern_execute(steps=[{"tool": "click", "params": {"ref": "e99"}}])
        finally:
            session_manager.set_stateless_http_mode(previous_mode)

        assert result["ok"] is False
        error = result["data"]["results"][0]["error"]
        assert error.startswith("Unknown ref 'e99' — call observe first or check ref exists")
        if stateless_http_mode:
            assert "stateless HTTP mode" in error
            assert "do not resolve" in error
            assert "selector or intent" in error
        else:
            assert "stateless" not in error

    @pytest.mark.asyncio
    async def test_execute_preserves_structured_direct_failure_error(self, monkeypatch: pytest.MonkeyPatch) -> None:
        page = _make_page()
        ctx = BrowserContext(mode="local")
        monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))

        click_result = {
            "ok": False,
            "error": {
                "code": mcp_browser.ErrorCode.ACTION_FAILED,
                "message": "Selector matched an element that is not visible",
                "hint": "The element exists but is not visible.",
                "details": {"element_state": "hidden", "selector": "#field", "actionability_timeout_ms": 5000},
            },
        }
        monkeypatch.setattr(mcp_browser, "skyvern_click", AsyncMock(return_value=click_result))

        result = await mcp_browser.skyvern_execute(steps=[{"tool": "click", "params": {"selector": "#field"}}])

        assert result["ok"] is False
        step_error = result["data"]["results"][0]["error"]
        assert step_error["code"] == mcp_browser.ErrorCode.ACTION_FAILED
        assert step_error["details"]["element_state"] == "hidden"
        assert step_error["details"]["selector"] == "#field"

    @pytest.mark.asyncio
    async def test_execute_forwards_both_selectors_to_the_two_state_wait(self, monkeypatch: pytest.MonkeyPatch) -> None:
        """_TOOL_ACCEPTED_PARAMS drops unlisted params silently, degrading the wait to one selector."""
        page = _make_page()
        ctx = BrowserContext(mode="local")
        monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))
        wait = AsyncMock(return_value={"ok": True, "data": {}})
        monkeypatch.setattr(mcp_browser, "skyvern_wait_for_either_state", wait)

        await mcp_browser.skyvern_execute(
            steps=[
                {
                    "tool": "wait_for_either_state",
                    "params": {"selector_a": "#login", "selector_b": "#home", "timeout": 90000},
                }
            ]
        )

        assert wait.await_args.kwargs["selector_b"] == "#home"

    @pytest.mark.asyncio
    async def test_execute_wait_error_does_not_leak_headers(self, monkeypatch: pytest.MonkeyPatch) -> None:
        page = _make_page()
        page.validate = AsyncMock(
            side_effect=InternalServerError(
                body={"error": "Unexpected error: sk-BODY-SECRET"},
                headers={"authorization": "Bearer sk-SECRET"},
            )
        )
        page.wait_for_timeout = make_real_wait_for_timeout()
        ctx = BrowserContext(mode="local")
        monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))

        result = await mcp_browser.skyvern_execute(
            steps=[
                {
                    "tool": "wait",
                    "params": {
                        "intent": "the spinner disappears",
                        "timeout": 1000,
                        "poll_interval_ms": 500,
                    },
                }
            ]
        )

        assert result["ok"] is False
        step_error = result["data"]["results"][0]["error"]
        assert step_error["message"] == "HTTP 500: InternalServerError"
        assert step_error["details"] == {"exception_type": "InternalServerError", "status_code": 500}
        message = step_error["message"].lower()
        for leaked in ("authorization", "bearer", "sk-secret", "sk-body-secret", "headers", "body"):
            assert leaked not in message

    @pytest.mark.asyncio
    async def test_execute_ref_to_unnamed_duplicate_uses_nth(self, monkeypatch: pytest.MonkeyPatch) -> None:
        """SKY-9701: clicking ref=e2 on a page with 4 unnamed comboboxes resolves to the 3rd, not the 1st."""
        tree = _make_a11y_tree(
            children=[
                {"role": "combobox", "name": ""},
                {"role": "combobox", "name": ""},
                {"role": "combobox", "name": ""},
                {"role": "combobox", "name": ""},
            ]
        )
        page = _make_page(tree)
        ctx = BrowserContext(mode="local")
        monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))

        click_result = {"ok": True, "data": None}
        monkeypatch.setattr(mcp_browser, "skyvern_click", AsyncMock(return_value=click_result))

        result = await mcp_browser.skyvern_execute(
            steps=[
                {"tool": "observe", "params": {}},
                {"tool": "click", "params": {"ref": "e2"}},
            ]
        )
        assert result["ok"] is True
        click_kwargs = mcp_browser.skyvern_click.call_args.kwargs
        assert click_kwargs["selector"] == "role=combobox >> nth=2"

    @pytest.mark.asyncio
    async def test_execute_ref_e0_on_unnamed_duplicate_emits_nth_zero(self, monkeypatch: pytest.MonkeyPatch) -> None:
        """SKY-9701 follow-up: the first duplicate (e0) must also emit `nth=0` so Playwright
        strict mode does not raise and the click does not silently resolve to whichever
        element happens to be first."""
        tree = _make_a11y_tree(
            children=[
                {"role": "combobox", "name": ""},
                {"role": "combobox", "name": ""},
            ]
        )
        page = _make_page(tree)
        ctx = BrowserContext(mode="local")
        monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))

        click_result = {"ok": True, "data": None}
        monkeypatch.setattr(mcp_browser, "skyvern_click", AsyncMock(return_value=click_result))

        result = await mcp_browser.skyvern_execute(
            steps=[
                {"tool": "observe", "params": {}},
                {"tool": "click", "params": {"ref": "e0"}},
            ]
        )
        assert result["ok"] is True
        click_kwargs = mcp_browser.skyvern_click.call_args.kwargs
        assert click_kwargs["selector"] == "role=combobox >> nth=0"