Skip to content

Automate browser based workflows with AI

fix(SKY-12101): persist observe refs per browser session for cross-call execute (#7250)

Marc Kelechava962acb3cdd23
diff --git a/skyvern/cli/core/browser_ops.py b/skyvern/cli/core/browser_ops.py
index a502769fe..a0dbdb9d3 100644
--- a/skyvern/cli/core/browser_ops.py
+++ b/skyvern/cli/core/browser_ops.py
@@ -13,7 +13,7 @@ import time
from dataclasses import dataclass, field
from datetime import datetime, timezone
from pathlib import Path
-from typing import Any, Literal
+from typing import Any, Callable, Literal
from skyvern.forge.sdk.settings_manager import SettingsManager
from skyvern.webeye.utils.page import SkyvernFrame
@@ -1159,6 +1159,7 @@ async def do_execute(
dispatch_fn: Any,
steps: list[ExecuteStep],
stop_on_error: bool = True,
+ on_ref_map_update: Callable[[dict[str, dict[str, Any]]], bool] | None = None,
) -> ExecuteResult:
"""Execute a sequence of deterministic browser operations in one batch.
@@ -1169,6 +1170,9 @@ async def do_execute(
nav_failed = False
for i, step in enumerate(steps):
+ if step.tool == "navigate":
+ ref_map = {}
+
# DESIGN-3: Block sensitive ops after failed navigate
if nav_failed and not stop_on_error and step.tool in _SENSITIVE_TOOLS:
results.append(
@@ -1185,13 +1189,19 @@ async def do_execute(
t0 = time.monotonic()
try:
result = await dispatch_fn(step, ref_map)
+ # DESIGN-4: Each observe REPLACES the entire ref_map (not merges).
+ # If session publication rejects the snapshot (a concurrent navigation
+ # invalidated it), the batch must not act on it either.
+ if step.tool == "observe" and result and "elements" in result:
+ new_map = {elem["ref"]: elem for elem in result["elements"]}
+ if on_ref_map_update is None or on_ref_map_update(new_map):
+ ref_map = new_map
+ else:
+ ref_map = {}
+
wall_ms = int((time.monotonic() - t0) * 1000)
results.append(StepResult(step=i, tool=step.tool, ok=True, wall_ms=wall_ms, data=result))
- # DESIGN-4: Each observe REPLACES the entire ref_map (not merges)
- if step.tool == "observe" and result and "elements" in result:
- ref_map = {elem["ref"]: elem for elem in result["elements"]}
-
except Exception as e:
wall_ms = int((time.monotonic() - t0) * 1000)
error_payload: str | dict[str, Any] = e.error if isinstance(e, ToolStepError) else str(e)
diff --git a/skyvern/cli/core/session_manager.py b/skyvern/cli/core/session_manager.py
index 5cff98d7e..c73272b95 100644
--- a/skyvern/cli/core/session_manager.py
+++ b/skyvern/cli/core/session_manager.py
@@ -4,6 +4,7 @@ import asyncio
import itertools
import secrets
import time
+import weakref
from collections import deque
from contextlib import asynccontextmanager
from contextvars import ContextVar
@@ -62,6 +63,8 @@ class SessionState:
active_routes: dict[int, set[str]] = field(default_factory=dict)
# -- Iframe frame context --
_working_frame: Frame | None = None
+ _observed_refs: dict[str, dict[str, Any]] = field(default_factory=dict)
+ _observed_refs_generation: int = 0
def get_response_body(self, request_id: int) -> str | None:
"""Public accessor for cached response bodies (keyed by request_id)."""
@@ -76,6 +79,155 @@ _stateless_http_mode = False
# This bypasses ContextVar propagation issues when FastMCP runs tool handlers
# in a separate task whose context snapshot predates scoped_session().
_copilot_sessions: dict[str, SessionState] = {}
+# Ref snapshots and their invalidation generations, FIFO-capped like _body_store.
+# Capacity eviction can drop a live session's refs early; that fails safe (unknown
+# ref → the caller re-observes), which is why the tool contract doesn't mention it.
+# Generation tokens come from one process-wide counter and are allocated on first
+# touch, so an evicted entry re-materializes with a NEVER-seen token — a stale
+# observe holding an old token can't collide with it (no ABA reset to a default).
+_SESSION_REF_STORE_MAX = 64
+_session_ref_maps: dict[tuple[str, str, str | None], dict[str, Any]] = {}
+_session_ref_generations: dict[tuple[str, str, str | None], int] = {}
+_session_ref_generation_counter = itertools.count(1)
+# Identity tokens instead of id(): a GC'd page's id() can be reused by a new
+# page, making a stale snapshot look valid. Tokens are never reissued.
+_page_identity_tokens: weakref.WeakKeyDictionary[Page | Frame, int] = weakref.WeakKeyDictionary()
+
+
+def _identity_token(obj: Page | Frame) -> int:
+ try:
+ token = _page_identity_tokens.get(obj)
+ except TypeError:
+ # Not weak-referenceable (e.g. SimpleNamespace); real page/frame objects
+ # always are, so only they get the no-reuse guarantee.
+ return id(obj)
+ if token is None:
+ token = next(_session_ref_generation_counter)
+ _page_identity_tokens[obj] = token
+ return token
+
+
+def page_ref_key(page: SkyvernBrowserPage) -> tuple[int, int | None, str]:
+ """Identity of the document context a ref snapshot was taken from."""
+ return (
+ _identity_token(page.page),
+ _identity_token(page._working_frame) if page._working_frame is not None else None,
+ page.url,
+ )
+
+
+def _generation_for(key: tuple[str, str, str | None]) -> int:
+ generation = _session_ref_generations.get(key)
+ if generation is None:
+ generation = next(_session_ref_generation_counter)
+ _session_ref_generations[key] = generation
+ while len(_session_ref_generations) > _SESSION_REF_STORE_MAX:
+ _session_ref_generations.pop(next(iter(_session_ref_generations)))
+ return generation
+
+
+def _session_ref_key(
+ state: SessionState,
+ *,
+ session_id: str | None = None,
+ cdp_url: str | None = None,
+) -> tuple[str, str, str | None] | None:
+ context = state.context
+ resolved_session_id = session_id or (context.session_id if context else None)
+ resolved_cdp_url = cdp_url or (context.cdp_url if context else None)
+ # Prefer the hash stored at resolve_browser time — recomputing runs a
+ # deliberately slow PBKDF2 per call, and this sits on the per-ref hot path.
+ api_key_hash = state.api_key_hash or _api_key_hash(get_active_api_key())
+
+ if resolved_session_id:
+ return ("cloud_session", resolved_session_id, api_key_hash)
+ if resolved_cdp_url:
+ return ("cdp", resolved_cdp_url, api_key_hash)
+ return None
+
+
+def session_ref_generation(
+ *,
+ session_id: str | None = None,
+ cdp_url: str | None = None,
+) -> int:
+ state = get_current_session()
+ if key := _session_ref_key(state, session_id=session_id, cdp_url=cdp_url):
+ return _generation_for(key)
+ if state._observed_refs_generation == 0:
+ state._observed_refs_generation = next(_session_ref_generation_counter)
+ return state._observed_refs_generation
+
+
+def replace_session_ref_map(
+ ref_map: dict[str, dict[str, Any]],
+ *,
+ session_id: str | None = None,
+ cdp_url: str | None = None,
+ generation: int | None = None,
+ page_key: tuple[int, int | None, str] | None = None,
+) -> bool:
+ """Replace the session's ref snapshot (never merge). Returns False if discarded.
+
+ When ``generation`` is given, the snapshot is discarded if the registry was
+ cleared after that generation was captured — the observe raced a navigation
+ or context switch and describes a replaced document.
+ """
+ state = get_current_session()
+ snapshot: dict[str, Any] = {"page_key": page_key, "refs": dict(ref_map)}
+ if key := _session_ref_key(state, session_id=session_id, cdp_url=cdp_url):
+ if generation is not None and generation != _generation_for(key):
+ return False
+ _session_ref_maps[key] = snapshot
+ while len(_session_ref_maps) > _SESSION_REF_STORE_MAX:
+ _session_ref_maps.pop(next(iter(_session_ref_maps)))
+ else:
+ if generation is not None and generation != state._observed_refs_generation:
+ return False
+ state._observed_refs = snapshot
+ return True
+
+
+def get_session_ref(
+ ref: str,
+ *,
+ session_id: str | None = None,
+ cdp_url: str | None = None,
+ page_key: tuple[int, int | None, str] | None = None,
+) -> dict[str, Any] | None:
+ """Resolve a ref, failing closed when the snapshot was bound to a different
+ page/frame than the one the caller is on (popup steal, external tab close,
+ detached-frame fallback — transitions with no explicit clear hook)."""
+ state = get_current_session()
+ if key := _session_ref_key(state, session_id=session_id, cdp_url=cdp_url):
+ snapshot = _session_ref_maps.get(key)
+ if snapshot is not None:
+ # LRU touch: active sessions shouldn't be evicted by unrelated churn
+ _session_ref_maps[key] = _session_ref_maps.pop(key)
+ else:
+ snapshot = state._observed_refs or None
+ if not snapshot:
+ return None
+ if snapshot.get("page_key") != page_key:
+ return None
+ element: dict[str, Any] | None = snapshot["refs"].get(ref)
+ return element
+
+
+def clear_session_ref_map(
+ *,
+ session_id: str | None = None,
+ cdp_url: str | None = None,
+) -> None:
+ state = get_current_session()
+ if key := _session_ref_key(state, session_id=session_id, cdp_url=cdp_url):
+ _session_ref_maps.pop(key, None)
+ _session_ref_generations[key] = next(_session_ref_generation_counter)
+ while len(_session_ref_generations) > _SESSION_REF_STORE_MAX:
+ _session_ref_generations.pop(next(iter(_session_ref_generations)))
+ else:
+ state._observed_refs = {}
+ state._observed_refs_generation = next(_session_ref_generation_counter)
def register_copilot_session(session_id: str, state: SessionState) -> None:
@@ -306,6 +458,7 @@ async def close_current_session() -> None:
if current.browser is not None:
await current.browser.close()
finally:
+ clear_session_ref_map()
if current.context and current.context.session_id:
unregister_copilot_session(current.context.session_id)
set_current_session(SessionState())
@@ -446,6 +599,7 @@ async def browser_session(
await browser.close()
except Exception:
pass # Best effort cleanup
+ clear_session_ref_map()
set_current_session(SessionState())
diff --git a/skyvern/cli/mcp_tools/_session.py b/skyvern/cli/mcp_tools/_session.py
index 1f508ee17..bca564b83 100644
--- a/skyvern/cli/mcp_tools/_session.py
+++ b/skyvern/cli/mcp_tools/_session.py
@@ -10,11 +10,16 @@ from skyvern.cli.core.session_manager import (
BrowserNotAvailableError,
SessionState,
browser_session,
+ clear_session_ref_map,
close_current_session,
get_current_session,
get_page,
+ get_session_ref,
no_browser_error,
+ page_ref_key,
+ replace_session_ref_map,
resolve_browser,
+ session_ref_generation,
set_current_session,
)
@@ -22,11 +27,16 @@ __all__ = [
"BrowserNotAvailableError",
"SessionState",
"browser_session",
+ "clear_session_ref_map",
"close_current_session",
"get_current_session",
"get_page",
+ "get_session_ref",
"get_skyvern",
"no_browser_error",
+ "page_ref_key",
+ "replace_session_ref_map",
"resolve_browser",
+ "session_ref_generation",
"set_current_session",
]
diff --git a/skyvern/cli/mcp_tools/browser.py b/skyvern/cli/mcp_tools/browser.py
index 07fd2fb6c..a3ddca2f8 100644
--- a/skyvern/cli/mcp_tools/browser.py
+++ b/skyvern/cli/mcp_tools/browser.py
@@ -5,7 +5,7 @@ import base64
import json
import re
from datetime import datetime, timezone
-from typing import Annotated, Any, Literal
+from typing import Annotated, Any, Callable, Literal
import structlog
from playwright.async_api import TimeoutError as PlaywrightTimeoutError
@@ -65,7 +65,17 @@ from ._element_state import (
resolve_action_timeout_ms,
)
from ._localhost import is_localhost_url
-from ._session import BrowserNotAvailableError, get_current_session, get_page, no_browser_error
+from ._session import (
+ BrowserNotAvailableError,
+ clear_session_ref_map,
+ get_current_session,
+ get_page,
+ get_session_ref,
+ no_browser_error,
+ page_ref_key,
+ replace_session_ref_map,
+ session_ref_generation,
+)
LOG = structlog.get_logger(__name__)
@@ -240,6 +250,7 @@ async def skyvern_navigate(
# (even failed navigations can partially load and destroy existing frames)
state = get_current_session()
state._working_frame = None
+ clear_session_ref_map(session_id=ctx.session_id, cdp_url=ctx.cdp_url)
with Timer() as timer:
try:
@@ -261,6 +272,12 @@ async def skyvern_navigate(
timing_ms=timer.timing_ms,
error=make_error(ErrorCode.ACTION_FAILED, str(e), "Check that the URL is valid and accessible"),
)
+ finally:
+ # Clear again after the attempt (success OR failure — a failed goto can
+ # still partially replace the document): an observe that started while
+ # navigation was in flight captured a post-clear generation, so only a
+ # second bump can invalidate its snapshot of the old document.
+ clear_session_ref_map(session_id=ctx.session_id, cdp_url=ctx.cdp_url)
return make_result(
"skyvern_navigate",
@@ -1968,6 +1985,7 @@ async def skyvern_frame_switch(
# Persist frame on session state for subsequent MCP calls
state = get_current_session()
state._working_frame = page._working_frame
+ clear_session_ref_map(session_id=ctx.session_id, cdp_url=ctx.cdp_url)
except ValueError as e:
return make_result(
"skyvern_frame_switch",
@@ -2023,6 +2041,7 @@ async def skyvern_frame_main(
# Clear frame on session state
state = get_current_session()
state._working_frame = None
+ clear_session_ref_map(session_id=ctx.session_id, cdp_url=ctx.cdp_url)
return make_result(
"skyvern_frame_main",
@@ -2238,12 +2257,15 @@ async def skyvern_observe(
Field(description="Max elements to return. Default 50.", ge=1, le=200),
] = 50,
) -> dict[str, Any]:
- """Snapshot interactive page elements with stable refs (e0, e1, ...) for use in skyvern_execute batches."""
+ """Snapshot interactive elements with refs reusable in this browser session until the next observe or page/document context change (rarely earlier — on 'Unknown ref', re-observe)."""
try:
page, ctx = await get_page(session_id=session_id, cdp_url=cdp_url)
except BrowserNotAvailableError:
return make_result("skyvern_observe", ok=False, error=no_browser_error())
+ observe_page_key = page_ref_key(page)
+ generation = session_ref_generation(session_id=ctx.session_id, cdp_url=ctx.cdp_url)
+
with Timer() as timer:
try:
result = await do_observe(
@@ -2263,11 +2285,21 @@ async def skyvern_observe(
)
elements = serialize_elements(result.elements)
+ replace_session_ref_map(
+ {element["ref"]: element for element in elements},
+ session_id=ctx.session_id,
+ cdp_url=ctx.cdp_url,
+ generation=generation,
+ page_key=observe_page_key,
+ )
hint = (
f"Found {result.element_count} interactive elements"
f"{f' (of {result.total_on_page} total on page)' if result.total_on_page > result.element_count else ''}. "
"Use these refs in skyvern_execute steps, e.g.: "
- '{tool: "click", params: {ref: "e0"}}'
+ '{tool: "click", params: {ref: "e0"}}. '
+ "Refs remain valid across calls in this browser session until the next skyvern_observe, "
+ "skyvern_navigate, same-tab navigation, or tab/frame switch. Same-document DOM changes can also "
+ "invalidate ordinal refs; re-observe on 'Unknown ref' or unexpected failures."
)
return make_result(
"skyvern_observe",
@@ -2323,14 +2355,22 @@ async def _dispatch_step(
ref_map: dict[str, dict[str, Any]],
session_id: str | None,
cdp_url: str | None,
+ page_key: tuple[int, int | None, str] | None = None,
+ on_observe_page: Callable[[tuple[int, int | None, str]], None] | None = None,
) -> dict[str, Any] | None:
"""Route a step to the appropriate handler, resolving refs to selectors."""
params = dict(step.params)
- # Resolve ref to selector if present
+ # Resolve ref to selector if present. Refs bind to the page/frame they were
+ # observed on, so re-check identity against the page this step will actually
+ # run on — a popup or external tab change may have moved it mid-batch.
if ref := params.pop("ref", None):
- elem = ref_map.get(ref)
- if not elem:
+ current_page, _ = await get_page(session_id=session_id, cdp_url=cdp_url)
+ current_key = page_ref_key(current_page)
+ elem = ref_map.get(ref) if page_key is None or current_key == page_key else None
+ if elem is None:
+ elem = get_session_ref(ref, session_id=session_id, cdp_url=cdp_url, page_key=current_key)
+ if elem is None:
raise ValueError(f"Unknown ref '{ref}' — call observe first or check ref exists")
params["selector"] = ref_to_selector(elem)
@@ -2339,6 +2379,8 @@ async def _dispatch_step(
from skyvern.cli.core.browser_ops import do_observe as _do_observe
page, _ = await get_page(session_id=session_id, cdp_url=cdp_url)
+ if on_observe_page is not None:
+ on_observe_page(page_ref_key(page))
accepted = {"selector", "interactive_only", "max_elements"}
filtered = {k: v for k, v in params.items() if k in accepted}
result = await _do_observe(page, **filtered)
@@ -2378,7 +2420,11 @@ async def skyvern_execute(
Field(
description=(
"Array of {tool, params} step objects to execute sequentially. "
- f"Within params, refs from skyvern_observe are direct targets. {DIRECT_TARGET_DESCRIPTION}"
+ "Within params, refs from skyvern_observe are direct targets across calls in the same browser session. "
+ "The next skyvern_observe or page/document context change invalidates them; they can occasionally "
+ "expire early. Same-document DOM changes can also invalidate ordinal refs; on 'Unknown ref' or "
+ "unexpected failures, re-observe. "
+ f"{DIRECT_TARGET_DESCRIPTION}"
)
),
],
@@ -2389,7 +2435,7 @@ async def skyvern_execute(
Field(description="Stop at first failure (true) or continue past errors (false). Default true."),
] = True,
) -> dict[str, Any]:
- """Execute multiple browser operations in a single batch. Each step is {tool, params} using refs from skyvern_observe.
+ """Execute browser operations using current-session refs until the next observe or page/document context change.
Allowed tools: navigate, click, type, press_key, select_option, hover, scroll, wait, observe, screenshot, evaluate."""
if not steps:
return make_result(
@@ -2441,15 +2487,55 @@ async def skyvern_execute(
# Verify we can reach the browser before executing anything
try:
- await get_page(session_id=session_id, cdp_url=cdp_url)
+ page, ctx = await get_page(session_id=session_id, cdp_url=cdp_url)
except BrowserNotAvailableError:
return make_result("skyvern_execute", ok=False, error=no_browser_error())
+ batch_page_key = page_ref_key(page)
+
+ # Generation captured before each observe dispatch so a snapshot that raced
+ # a concurrent navigation/context switch is discarded, not committed.
+ observe_generation: dict[str, int] = {}
+ observe_page_key: tuple[int, int | None, str] | None = None
+
+ def capture_observe_page_key(page_key: tuple[int, int | None, str]) -> None:
+ nonlocal observe_page_key
+ observe_page_key = page_key
+
async def dispatch(step: ExecuteStep, ref_map: dict[str, dict[str, Any]]) -> dict[str, Any] | None:
- return await _dispatch_step(step, ref_map, session_id=session_id, cdp_url=cdp_url)
+ if step.tool == "observe":
+ observe_generation["value"] = session_ref_generation(session_id=ctx.session_id, cdp_url=ctx.cdp_url)
+ return await _dispatch_step(
+ step,
+ ref_map,
+ session_id=session_id,
+ cdp_url=cdp_url,
+ page_key=batch_page_key,
+ on_observe_page=capture_observe_page_key if step.tool == "observe" else None,
+ )
+
+ def publish_observe_refs(ref_map: dict[str, dict[str, Any]]) -> bool:
+ nonlocal batch_page_key
+ if observe_page_key is None:
+ return False
+ accepted = replace_session_ref_map(
+ ref_map,
+ session_id=ctx.session_id,
+ cdp_url=ctx.cdp_url,
+ generation=observe_generation.get("value"),
+ page_key=observe_page_key,
+ )
+ if accepted:
+ batch_page_key = observe_page_key
+ return accepted
with Timer() as timer:
- result = await do_execute(dispatch, parsed_steps, stop_on_error=stop_on_error)
+ result = await do_execute(
+ dispatch,
+ parsed_steps,
+ stop_on_error=stop_on_error,
+ on_ref_map_update=publish_observe_refs,
+ )
timer.mark("sdk")
step_results = []
diff --git a/skyvern/cli/mcp_tools/session.py b/skyvern/cli/mcp_tools/session.py
index 2ec8e3a3a..aab2ac6c0 100644
--- a/skyvern/cli/mcp_tools/session.py
+++ b/skyvern/cli/mcp_tools/session.py
@@ -21,6 +21,7 @@ from skyvern.schemas.runs import proxy_location_to_request
from ._common import BrowserContext, ErrorCode, Timer, make_error, make_result
from ._session import (
SessionState,
+ clear_session_ref_map,
close_current_session,
get_current_session,
get_skyvern,
@@ -284,6 +285,7 @@ async def skyvern_browser_session_close(
close_error: Exception | None = None
try:
result = await do_session_close(skyvern, session_id)
+ clear_session_ref_map(session_id=session_id)
except Exception as e:
close_error = e
diff --git a/skyvern/cli/mcp_tools/tabs.py b/skyvern/cli/mcp_tools/tabs.py
index 77c3cf69c..9f44f5c48 100644
--- a/skyvern/cli/mcp_tools/tabs.py
+++ b/skyvern/cli/mcp_tools/tabs.py
@@ -13,7 +13,13 @@ import structlog
from pydantic import BaseModel, Field
from ._common import ErrorCode, Timer, make_error, make_result
-from ._session import BrowserNotAvailableError, get_current_session, get_page, no_browser_error
+from ._session import (
+ BrowserNotAvailableError,
+ clear_session_ref_map,
+ get_current_session,
+ get_page,
+ no_browser_error,
+)
LOG = structlog.get_logger(__name__)
@@ -143,6 +149,7 @@ async def skyvern_tab_new(
state._active_page = new_page
# New tab has no iframes yet — clear stale frame reference
state._working_frame = None
+ clear_session_ref_map(session_id=ctx.session_id, cdp_url=ctx.cdp_url)
# Drain the event that _on_new_page() buffered for this explicitly
# created page, so tab_wait_for_new doesn't return it as a popup.
state._page_events = deque(
@@ -241,6 +248,7 @@ async def skyvern_tab_switch(
state._active_page = target
# Switching tabs invalidates any iframe frame reference from the old tab
state._working_frame = None
+ clear_session_ref_map(session_id=ctx.session_id, cdp_url=ctx.cdp_url)
# bring_to_front is a no-op in headless but helps in headed mode
try:
@@ -325,6 +333,7 @@ async def skyvern_tab_close(
state._active_page = None
# Closed tab's frame reference is no longer valid
state._working_frame = None
+ clear_session_ref_map(session_id=ctx.session_id, cdp_url=ctx.cdp_url)
# Clean up inspection hooks for the closed page
state._hooked_page_ids.discard(target_id)
diff --git a/tests/unit/test_mcp_observe_execute.py b/tests/unit/test_mcp_observe_execute.py
index 007f3fe57..bac855b89 100644
--- a/tests/unit/test_mcp_observe_execute.py
+++ b/tests/unit/test_mcp_observe_execute.py
@@ -2,12 +2,14 @@
from __future__ import annotations
+import gc
from types import SimpleNamespace
from typing import Any
from unittest.mock import AsyncMock, MagicMock
import pytest
+from skyvern.cli.core import session_manager
from skyvern.cli.core.browser_ops import (
ExecuteStep,
ObserveResult,
@@ -19,7 +21,10 @@ from skyvern.cli.core.browser_ops import (
serialize_elements,
)
from skyvern.cli.core.result import BrowserContext
+from skyvern.cli.core.session_manager import scoped_session
from skyvern.cli.mcp_tools import browser as mcp_browser
+from skyvern.cli.mcp_tools import tabs as mcp_tabs
+from tests.unit._mcp_browser_fakes import make_session_state
# ---------------------------------------------------------------------------
# Helpers
@@ -839,6 +844,488 @@ class TestSkyvernExecuteMCP:
click_call = mcp_browser.skyvern_click.call_args
assert 'role=button[name="Sign In"]' in str(click_call)
+ @pytest.mark.asyncio
+ async def test_execute_resolves_ref_from_prior_observe(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ page = _make_page()
+ ctx = BrowserContext(mode="cloud_session", session_id="pbs_test")
+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))
+
+ click_result = {"ok": True, "data": None}
+ monkeypatch.setattr(mcp_browser, "skyvern_click", AsyncMock(return_value=click_result))
+
+ async with scoped_session(make_session_state(context=ctx)):
+ observe_result = await mcp_browser.skyvern_observe(session_id=ctx.session_id)
+ ref = next(element["ref"] for element in observe_result["data"]["elements"] if element["name"] == "Sign In")
+
+ async with scoped_session(make_session_state(context=ctx)):
+ execute_result = await mcp_browser.skyvern_execute(
+ steps=[{"tool": "click", "params": {"ref": ref}}],
+ session_id=ctx.session_id,
+ )
+
+ assert execute_result["ok"] is True
+ assert execute_result["data"]["steps_completed"] == 1
+ assert mcp_browser.skyvern_click.call_args.kwargs["selector"] == 'role=button[name="Sign In"]'
+
+ @pytest.mark.asyncio
+ async def test_execute_resolves_ref_from_prior_observe_without_session_id(
+ self,
+ monkeypatch: pytest.MonkeyPatch,
+ ) -> None:
+ """The original stdio/local path keeps refs on the shared SessionState."""
+ page = _make_page()
+ ctx = BrowserContext(mode="local")
+ state = make_session_state(context=ctx)
+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))
+ monkeypatch.setattr(mcp_browser, "skyvern_click", AsyncMock(return_value={"ok": True, "data": None}))
+
+ async with scoped_session(state):
+ observe_result = await mcp_browser.skyvern_observe()
+ ref = next(element["ref"] for element in observe_result["data"]["elements"] if element["name"] == "Sign In")
+
+ async with scoped_session(state):
+ execute_result = await mcp_browser.skyvern_execute(steps=[{"tool": "click", "params": {"ref": ref}}])
+
+ assert execute_result["ok"] is True
+ assert state._observed_refs["refs"][ref]["name"] == "Sign In"
+ assert mcp_browser.skyvern_click.call_args.kwargs["selector"] == 'role=button[name="Sign In"]'
+
+ @pytest.mark.asyncio
+ async def test_keyless_ref_is_stale_when_url_changes_during_observe(
+ self,
+ monkeypatch: pytest.MonkeyPatch,
+ ) -> None:
+ page = _make_page()
+
+ async def replace_document_during_observe() -> str:
+ page.url = "https://example.com/replaced-document"
+ return "Replacement"
+
+ page.title = AsyncMock(side_effect=replace_document_during_observe)
+ ctx = BrowserContext(mode="local")
+ state = make_session_state(context=ctx)
+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))
+ click = AsyncMock(return_value={"ok": True, "data": None})
+ monkeypatch.setattr(mcp_browser, "skyvern_click", click)
+
+ async with scoped_session(state):
+ observe_result = await mcp_browser.skyvern_observe()
+ ref = observe_result["data"]["elements"][0]["ref"]
+
+ async with scoped_session(state):
+ execute_result = await mcp_browser.skyvern_execute(steps=[{"tool": "click", "params": {"ref": ref}}])
+
+ assert execute_result["data"]["results"][0]["error"] == (
+ f"Unknown ref '{ref}' — call observe first or check ref exists"
+ )
+ click.assert_not_awaited()
+
+ @pytest.mark.asyncio
+ async def test_execute_does_not_resolve_ref_from_different_session(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ page = _make_page()
+ first_ctx = BrowserContext(mode="cloud_session", session_id="pbs_first")
+ second_ctx = BrowserContext(mode="cloud_session", session_id="pbs_second")
+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, first_ctx)))
+
+ async with scoped_session(make_session_state(context=first_ctx)):
+ observe_result = await mcp_browser.skyvern_observe(session_id=first_ctx.session_id)
+
+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, second_ctx)))
+ async with scoped_session(make_session_state(context=second_ctx)):
+ execute_result = await mcp_browser.skyvern_execute(
+ steps=[{"tool": "click", "params": {"ref": observe_result["data"]["elements"][0]["ref"]}}],
+ session_id=second_ctx.session_id,
+ )
+
+ assert execute_result["data"]["results"][0]["error"] == (
+ "Unknown ref 'e0' — call observe first or check ref exists"
+ )
+
+ @pytest.mark.asyncio
+ async def test_execute_observe_step_persists_refs_for_next_call(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ page = _make_page()
+ ctx = BrowserContext(mode="cloud_session", session_id="pbs_batch_observe")
+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))
+
+ click_result = {"ok": True, "data": None}
+ monkeypatch.setattr(mcp_browser, "skyvern_click", AsyncMock(return_value=click_result))
+
+ async with scoped_session(make_session_state(context=ctx)):
+ observe_result = await mcp_browser.skyvern_execute(
+ steps=[{"tool": "observe", "params": {}}],
+ session_id=ctx.session_id,
+ )
+ click_result = await mcp_browser.skyvern_execute(
+ steps=[{"tool": "click", "params": {"ref": "e2"}}],
+ session_id=ctx.session_id,
+ )
+
+ assert observe_result["ok"] is True
+ assert click_result["ok"] is True
+ assert mcp_browser.skyvern_click.call_args.kwargs["selector"] == 'role=button[name="Sign In"]'
+
+ @pytest.mark.asyncio
+ async def test_second_observe_replaces_prior_session_refs(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ first_tree = _make_a11y_tree(
+ children=[
+ {"role": "button", "name": "First"},
+ {"role": "button", "name": "Stale"},
+ ]
+ )
+ second_tree = _make_a11y_tree(children=[{"role": "button", "name": "Current"}])
+ page = _make_page(first_tree)
+ page.accessibility.snapshot.side_effect = [first_tree, second_tree]
+ ctx = BrowserContext(mode="cloud_session", session_id="pbs_replace_refs")
+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))
+
+ click_tool_result = {"ok": True, "data": None}
+ monkeypatch.setattr(mcp_browser, "skyvern_click", AsyncMock(return_value=click_tool_result))
+
+ async with scoped_session(make_session_state(context=ctx)):
+ first_result = await mcp_browser.skyvern_observe(session_id=ctx.session_id)
+ second_result = await mcp_browser.skyvern_observe(session_id=ctx.session_id)
+ stale_result = await mcp_browser.skyvern_execute(
+ steps=[{"tool": "click", "params": {"ref": "e1"}}],
+ session_id=ctx.session_id,
+ )
+ current_result = await mcp_browser.skyvern_execute(
+ steps=[{"tool": "click", "params": {"ref": "e0"}}],
+ session_id=ctx.session_id,
+ )
+
+ assert [element["ref"] for element in first_result["data"]["elements"]] == ["e0", "e1"]
+ assert [element["ref"] for element in second_result["data"]["elements"]] == ["e0"]
+ assert stale_result["data"]["results"][0]["error"] == (
+ "Unknown ref 'e1' — call observe first or check ref exists"
+ )
+ assert current_result["ok"] is True
+ assert mcp_browser.skyvern_click.call_args.kwargs["selector"] == 'role=button[name="Current"]'
+
+ @pytest.mark.asyncio
+ async def test_navigate_clears_session_refs(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ page = _make_page()
+ ctx = BrowserContext(mode="cloud_session", session_id="pbs_navigate_refs")
+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))
+
+ async with scoped_session(make_session_state(context=ctx)):
+ observe_result = await mcp_browser.skyvern_observe(session_id=ctx.session_id)
+ navigate_result = await mcp_browser.skyvern_navigate(
+ url="https://example.com/next",
+ session_id=ctx.session_id,
+ )
+ execute_result = await mcp_browser.skyvern_execute(
+ steps=[{"tool": "click", "params": {"ref": "e2"}}],
+ session_id=ctx.session_id,
+ )
+
+ assert observe_result["ok"] is True
+ assert navigate_result["ok"] is True
+ assert execute_result["data"]["results"][0]["error"] == (
+ "Unknown ref 'e2' — call observe first or check ref exists"
+ )
+
+ @pytest.mark.asyncio
+ async def test_navigate_clears_batch_refs(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ page = _make_page()
+ ctx = BrowserContext(mode="cloud_session", session_id="pbs_batch_navigate_refs")
+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))
+ monkeypatch.setattr(mcp_browser, "skyvern_click", AsyncMock(return_value={"ok": True, "data": None}))
+
+ async with scoped_session(make_session_state(context=ctx)):
+ result = await mcp_browser.skyvern_execute(
+ steps=[
+ {"tool": "observe", "params": {}},
+ {"tool": "navigate", "params": {"url": "https://example.com/next"}},
+ {"tool": "click", "params": {"ref": "e2"}},
+ ],
+ session_id=ctx.session_id,
+ )
+
+ assert result["ok"] is False
+ assert result["data"]["error_step"] == 2
+ assert result["data"]["results"][2]["error"] == ("Unknown ref 'e2' — call observe first or check ref exists")
+ mcp_browser.skyvern_click.assert_not_awaited()
+
+ @pytest.mark.asyncio
+ async def test_tab_switch_invalidates_session_refs(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ page = _make_page()
+ ctx = BrowserContext(mode="cloud_session", session_id="pbs_tab_refs")
+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))
+ monkeypatch.setattr(mcp_tabs, "get_page", AsyncMock(return_value=(page, ctx)))
+
+ tab_a = MagicMock()
+ tab_a.url = "https://example.com/a"
+ tab_a.title = AsyncMock(return_value="A")
+ tab_a.is_closed = MagicMock(return_value=False)
+ tab_b = MagicMock()
+ tab_b.url = "https://example.com/b"
+ tab_b.title = AsyncMock(return_value="B")
+ tab_b.is_closed = MagicMock(return_value=False)
+ tab_b.bring_to_front = AsyncMock()
+ browser = MagicMock()
+ browser._browser_context.pages = [tab_a, tab_b]
+
+ async with scoped_session(make_session_state(context=ctx, browser=browser)):
+ observe_result = await mcp_browser.skyvern_observe(session_id=ctx.session_id)
+ ref = observe_result["data"]["elements"][0]["ref"]
+ switch_result = await mcp_tabs.skyvern_tab_switch(session_id=ctx.session_id, index=1)
+ execute_result = await mcp_browser.skyvern_execute(
+ steps=[{"tool": "click", "params": {"ref": ref}}],
+ session_id=ctx.session_id,
+ )
+
+ assert switch_result["ok"] is True
+ assert execute_result["data"]["results"][0]["error"] == (
+ f"Unknown ref '{ref}' — call observe first or check ref exists"
+ )
+
+ @pytest.mark.asyncio
+ async def test_frame_main_clears_session_refs(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ page = _make_page()
+ page.frame_main = MagicMock()
+ ctx = BrowserContext(mode="cloud_session", session_id="pbs_frame_refs")
+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))
+
+ async with scoped_session(make_session_state(context=ctx)):
+ observe_result = await mcp_browser.skyvern_observe(session_id=ctx.session_id)
+ ref = observe_result["data"]["elements"][0]["ref"]
+ frame_result = await mcp_browser.skyvern_frame_main(session_id=ctx.session_id)
+ execute_result = await mcp_browser.skyvern_execute(
+ steps=[{"tool": "click", "params": {"ref": ref}}],
+ session_id=ctx.session_id,
+ )
+
+ assert frame_result["ok"] is True
+ assert execute_result["data"]["results"][0]["error"] == (
+ f"Unknown ref '{ref}' — call observe first or check ref exists"
+ )
+
+ @pytest.mark.asyncio
+ async def test_popup_page_change_invalidates_refs(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ """Implicit page transition (popup steals working page) must not resolve old refs."""
+ page_a = _make_page()
+ page_b = _make_page()
+ ctx = BrowserContext(mode="cloud_session", session_id="pbs_popup_refs")
+
+ async with scoped_session(make_session_state(context=ctx)):
+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page_a, ctx)))
+ observe_result = await mcp_browser.skyvern_observe(session_id=ctx.session_id)
+ ref = observe_result["data"]["elements"][0]["ref"]
+
+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page_b, ctx)))
+ execute_result = await mcp_browser.skyvern_execute(
+ steps=[{"tool": "click", "params": {"ref": ref}}],
+ session_id=ctx.session_id,
+ )
+
+ assert execute_result["data"]["results"][0]["error"] == (
+ f"Unknown ref '{ref}' — call observe first or check ref exists"
+ )
+
+ @pytest.mark.asyncio
+ async def test_generation_eviction_does_not_resurrect_refs(self) -> None:
+ """ABA guard: eviction of a cleared session's generation must not let a stale commit through."""
+ ctx = BrowserContext(mode="cloud_session", session_id="pbs_aba_victim")
+ element = {"ref": "e0", "role": "button", "name": "Sign In", "selector": "#signin"}
+
+ async with scoped_session(make_session_state(context=ctx)):
+ stale_generation = session_manager.session_ref_generation(session_id=ctx.session_id)
+ session_manager.clear_session_ref_map(session_id=ctx.session_id)
+ for i in range(session_manager._SESSION_REF_STORE_MAX + 5):
+ session_manager.clear_session_ref_map(session_id=f"pbs_aba_churn_{i}")
+
+ committed = session_manager.replace_session_ref_map(
+ {"e0": element}, session_id=ctx.session_id, generation=stale_generation
+ )
+ assert committed is False
+ assert session_manager.get_session_ref("e0", session_id=ctx.session_id) is None
+
+ @pytest.mark.asyncio
+ async def test_rejected_batch_observe_does_not_install_local_refs(self) -> None:
+ """If session publication rejects an observe snapshot, the batch must not act on it either."""
+
+ async def dispatch(step: ExecuteStep, ref_map: dict[str, dict[str, Any]]) -> dict[str, Any] | None:
+ if step.tool == "observe":
+ return {"elements": [{"ref": "e0", "role": "button", "name": "Sign In"}], "element_count": 1}
+ if ref := step.params.get("ref"):
+ if ref not in ref_map:
+ raise ValueError(f"Unknown ref '{ref}' — call observe first or check ref exists")
+ return None
+
+ result = await do_execute(
+ dispatch,
+ [ExecuteStep(tool="observe"), ExecuteStep(tool="click", params={"ref": "e0"})],
+ on_ref_map_update=lambda ref_map: False,
+ )
+
+ assert result.error_step == 1
+ assert "Unknown ref 'e0'" in str(result.results[1].error)
+
+ @pytest.mark.asyncio
+ async def test_observe_started_during_navigation_is_discarded(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ """A generation captured while navigation is in flight must not commit after it completes."""
+ page = _make_page()
+ ctx = BrowserContext(mode="cloud_session", session_id="pbs_nav_race")
+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))
+
+ in_flight_generation: list[int] = []
+
+ async def fake_navigate(*args: Any, **kwargs: Any) -> Any:
+ in_flight_generation.append(session_manager.session_ref_generation(session_id=ctx.session_id))
+ return SimpleNamespace(url="https://example.com/next", title="Next")
+
+ monkeypatch.setattr(mcp_browser, "do_navigate", fake_navigate)
+
+ async with scoped_session(make_session_state(context=ctx)):
+ nav_result = await mcp_browser.skyvern_navigate("https://example.com/next", session_id=ctx.session_id)
+ committed = session_manager.replace_session_ref_map(
+ {"e0": {"ref": "e0", "selector": "#old"}},
+ session_id=ctx.session_id,
+ generation=in_flight_generation[0],
+ )
+
+ assert nav_result["ok"] is True
+ assert committed is False
+ assert session_manager.get_session_ref("e0", session_id=ctx.session_id) is None
+
+ @pytest.mark.asyncio
+ async def test_popup_mid_batch_invalidates_batch_refs(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ """A page change between an in-batch observe and a ref step must not resolve stale refs."""
+ page_a = _make_page()
+ page_b = _make_page()
+ ctx = BrowserContext(mode="cloud_session", session_id="pbs_popup_batch")
+ monkeypatch.setattr(
+ mcp_browser,
+ "get_page",
+ AsyncMock(side_effect=[(page_a, ctx), (page_a, ctx), (page_b, ctx)]),
+ )
+
+ async with scoped_session(make_session_state(context=ctx)):
+ result = await mcp_browser.skyvern_execute(
+ steps=[
+ {"tool": "observe", "params": {}},
+ {"tool": "click", "params": {"ref": "e0"}},
+ ],
+ session_id=ctx.session_id,
+ )
+
+ assert result["data"]["results"][1]["error"] == ("Unknown ref 'e0' — call observe first or check ref exists")
+
+ @pytest.mark.asyncio
+ async def test_popup_before_batch_observe_binds_refs_to_observed_page(
+ self,
+ monkeypatch: pytest.MonkeyPatch,
+ ) -> None:
+ """An observe after popup steal binds to the new page it actually inspected."""
+ page_a = _make_page()
+ page_b = _make_page()
+ page_b.url = "https://example.com/popup"
+ ctx = BrowserContext(mode="cloud_session", session_id="pbs_popup_before_observe")
+ monkeypatch.setattr(
+ mcp_browser,
+ "get_page",
+ AsyncMock(side_effect=[(page_a, ctx), (page_b, ctx), (page_b, ctx)]),
+ )
+ monkeypatch.setattr(mcp_browser, "skyvern_evaluate", AsyncMock(return_value={"ok": True, "data": None}))
+ click = AsyncMock(return_value={"ok": True, "data": None})
+ monkeypatch.setattr(mcp_browser, "skyvern_click", click)
+
+ async with scoped_session(make_session_state(context=ctx)):
+ result = await mcp_browser.skyvern_execute(
+ steps=[
+ {"tool": "evaluate", "params": {"expression": "window.open('about:blank')"}},
+ {"tool": "observe", "params": {}},
+ {"tool": "click", "params": {"ref": "e0"}},
+ ],
+ session_id=ctx.session_id,
+ )
+
+ assert result["ok"] is True
+ click.assert_awaited_once()
+
+ @pytest.mark.asyncio
+ async def test_same_page_url_change_invalidates_batch_refs(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ """A same-tab document replacement keeps Page identity but invalidates refs by URL."""
+ page = _make_page()
+ ctx = BrowserContext(mode="cloud_session", session_id="pbs_click_nav_batch")
+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))
+
+ async def replace_document(**kwargs: Any) -> dict[str, Any]:
+ page.url = "https://example.com/replaced-document"
+ return {"ok": True, "data": None}
+
+ monkeypatch.setattr(mcp_browser, "skyvern_evaluate", AsyncMock(side_effect=replace_document))
+ click = AsyncMock(return_value={"ok": True, "data": None})
+ monkeypatch.setattr(mcp_browser, "skyvern_click", click)
+
+ async with scoped_session(make_session_state(context=ctx)):
+ result = await mcp_browser.skyvern_execute(
+ steps=[
+ {"tool": "observe", "params": {}},
+ {"tool": "evaluate", "params": {"expression": "location.href = '/replaced-document'"}},
+ {"tool": "click", "params": {"ref": "e0"}},
+ ],
+ session_id=ctx.session_id,
+ )
+
+ assert result["data"]["results"][2]["error"] == ("Unknown ref 'e0' — call observe first or check ref exists")
+ click.assert_not_awaited()
+
+ @pytest.mark.asyncio
+ async def test_failed_navigation_still_invalidates_midflight_observe(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ """A navigation that raises can still have replaced the document — mid-flight commits must die."""
+ page = _make_page()
+ ctx = BrowserContext(mode="cloud_session", session_id="pbs_failed_nav")
+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))
+
+ in_flight_generation: list[int] = []
+
+ async def failing_navigate(*args: Any, **kwargs: Any) -> Any:
+ in_flight_generation.append(session_manager.session_ref_generation(session_id=ctx.session_id))
+ raise RuntimeError("net::ERR_ABORTED")
+
+ monkeypatch.setattr(mcp_browser, "do_navigate", failing_navigate)
+
+ async with scoped_session(make_session_state(context=ctx)):
+ nav_result = await mcp_browser.skyvern_navigate("https://example.com/next", session_id=ctx.session_id)
+ committed = session_manager.replace_session_ref_map(
+ {"e0": {"ref": "e0", "selector": "#old"}},
+ session_id=ctx.session_id,
+ generation=in_flight_generation[0],
+ )
+
+ assert nav_result["ok"] is False
+ assert committed is False
+ assert session_manager.get_session_ref("e0", session_id=ctx.session_id) is None
+
+ def test_page_identity_key_never_reused_after_gc(self) -> None:
+ """Identity tokens must not be reissued to a new page even if id() is reused."""
+ page = _make_page()
+ old_key = session_manager.page_ref_key(page)
+ del page
+ gc.collect()
+ new_page = _make_page()
+ assert session_manager.page_ref_key(new_page) != old_key
+
+ @pytest.mark.asyncio
+ async def test_stale_observe_commit_is_discarded_after_clear(self) -> None:
+ ctx = BrowserContext(mode="cloud_session", session_id="pbs_stale_gen")
+ element = {"ref": "e0", "role": "button", "name": "Sign In", "selector": "#signin"}
+
+ async with scoped_session(make_session_state(context=ctx)):
+ stale_generation = session_manager.session_ref_generation(session_id=ctx.session_id)
+ session_manager.clear_session_ref_map(session_id=ctx.session_id)
+ session_manager.replace_session_ref_map(
+ {"e0": element}, session_id=ctx.session_id, generation=stale_generation
+ )
+ assert session_manager.get_session_ref("e0", session_id=ctx.session_id) is None
+
+ fresh_generation = session_manager.session_ref_generation(session_id=ctx.session_id)
+ session_manager.replace_session_ref_map(
+ {"e0": element}, session_id=ctx.session_id, generation=fresh_generation
+ )
+ assert session_manager.get_session_ref("e0", session_id=ctx.session_id) == element
+
@pytest.mark.asyncio
async def test_execute_observe_then_click_native_option_ref_selects_parent(
self,
@@ -877,6 +1364,8 @@ class TestSkyvernExecuteMCP:
),
click=AsyncMock(return_value="#unexpected-click"),
)
+ page.page = page
+ page._working_frame = None
ctx = BrowserContext(mode="local")
monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))
@@ -899,9 +1388,11 @@ class TestSkyvernExecuteMCP:
ctx = BrowserContext(mode="local")
monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))
- result = await mcp_browser.skyvern_execute(steps=[{"tool": "click", "params": {"ref": "e99"}}])
+ async with scoped_session(make_session_state(context=ctx)):
+ result = await mcp_browser.skyvern_execute(steps=[{"tool": "click", "params": {"ref": "e99"}}])
+
assert result["ok"] is False
- assert "unknown ref" in result["data"]["results"][0]["error"].lower()
+ assert result["data"]["results"][0]["error"] == ("Unknown ref 'e99' — call observe first or check ref exists")
@pytest.mark.asyncio
async def test_execute_preserves_structured_direct_failure_error(self, monkeypatch: pytest.MonkeyPatch) -> None: