Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix(SKY-12101): persist observe refs per browser session for cross-call execute (#7250)
Marc Kelechava962acb3cdd23
diff --git a/skyvern/cli/core/browser_ops.py b/skyvern/cli/core/browser_ops.pyindex a502769fe..a0dbdb9d3 100644--- a/skyvern/cli/core/browser_ops.py+++ b/skyvern/cli/core/browser_ops.py@@ -13,7 +13,7 @@ import timefrom dataclasses import dataclass, fieldfrom datetime import datetime, timezonefrom pathlib import Path-from typing import Any, Literal+from typing import Any, Callable, Literalfrom skyvern.forge.sdk.settings_manager import SettingsManagerfrom skyvern.webeye.utils.page import SkyvernFrame@@ -1159,6 +1159,7 @@ async def do_execute(dispatch_fn: Any,steps: list[ExecuteStep],stop_on_error: bool = True,+ on_ref_map_update: Callable[[dict[str, dict[str, Any]]], bool] | None = None,) -> ExecuteResult:"""Execute a sequence of deterministic browser operations in one batch.@@ -1169,6 +1170,9 @@ async def do_execute(nav_failed = Falsefor i, step in enumerate(steps):+ if step.tool == "navigate":+ ref_map = {}+# DESIGN-3: Block sensitive ops after failed navigateif nav_failed and not stop_on_error and step.tool in _SENSITIVE_TOOLS:results.append(@@ -1185,13 +1189,19 @@ async def do_execute(t0 = time.monotonic()try:result = await dispatch_fn(step, ref_map)+ # DESIGN-4: Each observe REPLACES the entire ref_map (not merges).+ # If session publication rejects the snapshot (a concurrent navigation+ # invalidated it), the batch must not act on it either.+ if step.tool == "observe" and result and "elements" in result:+ new_map = {elem["ref"]: elem for elem in result["elements"]}+ if on_ref_map_update is None or on_ref_map_update(new_map):+ ref_map = new_map+ else:+ ref_map = {}+wall_ms = int((time.monotonic() - t0) * 1000)results.append(StepResult(step=i, tool=step.tool, ok=True, wall_ms=wall_ms, data=result))- # DESIGN-4: Each observe REPLACES the entire ref_map (not merges)- if step.tool == "observe" and result and "elements" in result:- ref_map = {elem["ref"]: elem for elem in result["elements"]}-except Exception as e:wall_ms = int((time.monotonic() - t0) * 1000)error_payload: str | dict[str, Any] = e.error if isinstance(e, ToolStepError) else str(e)diff --git a/skyvern/cli/core/session_manager.py b/skyvern/cli/core/session_manager.pyindex 5cff98d7e..c73272b95 100644--- a/skyvern/cli/core/session_manager.py+++ b/skyvern/cli/core/session_manager.py@@ -4,6 +4,7 @@ import asyncioimport itertoolsimport secretsimport time+import weakreffrom collections import dequefrom contextlib import asynccontextmanagerfrom contextvars import ContextVar@@ -62,6 +63,8 @@ class SessionState:active_routes: dict[int, set[str]] = field(default_factory=dict)# -- Iframe frame context --_working_frame: Frame | None = None+ _observed_refs: dict[str, dict[str, Any]] = field(default_factory=dict)+ _observed_refs_generation: int = 0def get_response_body(self, request_id: int) -> str | None:"""Public accessor for cached response bodies (keyed by request_id)."""@@ -76,6 +79,155 @@ _stateless_http_mode = False# This bypasses ContextVar propagation issues when FastMCP runs tool handlers# in a separate task whose context snapshot predates scoped_session()._copilot_sessions: dict[str, SessionState] = {}+# Ref snapshots and their invalidation generations, FIFO-capped like _body_store.+# Capacity eviction can drop a live session's refs early; that fails safe (unknown+# ref → the caller re-observes), which is why the tool contract doesn't mention it.+# Generation tokens come from one process-wide counter and are allocated on first+# touch, so an evicted entry re-materializes with a NEVER-seen token — a stale+# observe holding an old token can't collide with it (no ABA reset to a default).+_SESSION_REF_STORE_MAX = 64+_session_ref_maps: dict[tuple[str, str, str | None], dict[str, Any]] = {}+_session_ref_generations: dict[tuple[str, str, str | None], int] = {}+_session_ref_generation_counter = itertools.count(1)+# Identity tokens instead of id(): a GC'd page's id() can be reused by a new+# page, making a stale snapshot look valid. Tokens are never reissued.+_page_identity_tokens: weakref.WeakKeyDictionary[Page | Frame, int] = weakref.WeakKeyDictionary()+++def _identity_token(obj: Page | Frame) -> int:+ try:+ token = _page_identity_tokens.get(obj)+ except TypeError:+ # Not weak-referenceable (e.g. SimpleNamespace); real page/frame objects+ # always are, so only they get the no-reuse guarantee.+ return id(obj)+ if token is None:+ token = next(_session_ref_generation_counter)+ _page_identity_tokens[obj] = token+ return token+++def page_ref_key(page: SkyvernBrowserPage) -> tuple[int, int | None, str]:+ """Identity of the document context a ref snapshot was taken from."""+ return (+ _identity_token(page.page),+ _identity_token(page._working_frame) if page._working_frame is not None else None,+ page.url,+ )+++def _generation_for(key: tuple[str, str, str | None]) -> int:+ generation = _session_ref_generations.get(key)+ if generation is None:+ generation = next(_session_ref_generation_counter)+ _session_ref_generations[key] = generation+ while len(_session_ref_generations) > _SESSION_REF_STORE_MAX:+ _session_ref_generations.pop(next(iter(_session_ref_generations)))+ return generation+++def _session_ref_key(+ state: SessionState,+ *,+ session_id: str | None = None,+ cdp_url: str | None = None,+) -> tuple[str, str, str | None] | None:+ context = state.context+ resolved_session_id = session_id or (context.session_id if context else None)+ resolved_cdp_url = cdp_url or (context.cdp_url if context else None)+ # Prefer the hash stored at resolve_browser time — recomputing runs a+ # deliberately slow PBKDF2 per call, and this sits on the per-ref hot path.+ api_key_hash = state.api_key_hash or _api_key_hash(get_active_api_key())++ if resolved_session_id:+ return ("cloud_session", resolved_session_id, api_key_hash)+ if resolved_cdp_url:+ return ("cdp", resolved_cdp_url, api_key_hash)+ return None+++def session_ref_generation(+ *,+ session_id: str | None = None,+ cdp_url: str | None = None,+) -> int:+ state = get_current_session()+ if key := _session_ref_key(state, session_id=session_id, cdp_url=cdp_url):+ return _generation_for(key)+ if state._observed_refs_generation == 0:+ state._observed_refs_generation = next(_session_ref_generation_counter)+ return state._observed_refs_generation+++def replace_session_ref_map(+ ref_map: dict[str, dict[str, Any]],+ *,+ session_id: str | None = None,+ cdp_url: str | None = None,+ generation: int | None = None,+ page_key: tuple[int, int | None, str] | None = None,+) -> bool:+ """Replace the session's ref snapshot (never merge). Returns False if discarded.++ When ``generation`` is given, the snapshot is discarded if the registry was+ cleared after that generation was captured — the observe raced a navigation+ or context switch and describes a replaced document.+ """+ state = get_current_session()+ snapshot: dict[str, Any] = {"page_key": page_key, "refs": dict(ref_map)}+ if key := _session_ref_key(state, session_id=session_id, cdp_url=cdp_url):+ if generation is not None and generation != _generation_for(key):+ return False+ _session_ref_maps[key] = snapshot+ while len(_session_ref_maps) > _SESSION_REF_STORE_MAX:+ _session_ref_maps.pop(next(iter(_session_ref_maps)))+ else:+ if generation is not None and generation != state._observed_refs_generation:+ return False+ state._observed_refs = snapshot+ return True+++def get_session_ref(+ ref: str,+ *,+ session_id: str | None = None,+ cdp_url: str | None = None,+ page_key: tuple[int, int | None, str] | None = None,+) -> dict[str, Any] | None:+ """Resolve a ref, failing closed when the snapshot was bound to a different+ page/frame than the one the caller is on (popup steal, external tab close,+ detached-frame fallback — transitions with no explicit clear hook)."""+ state = get_current_session()+ if key := _session_ref_key(state, session_id=session_id, cdp_url=cdp_url):+ snapshot = _session_ref_maps.get(key)+ if snapshot is not None:+ # LRU touch: active sessions shouldn't be evicted by unrelated churn+ _session_ref_maps[key] = _session_ref_maps.pop(key)+ else:+ snapshot = state._observed_refs or None+ if not snapshot:+ return None+ if snapshot.get("page_key") != page_key:+ return None+ element: dict[str, Any] | None = snapshot["refs"].get(ref)+ return element+++def clear_session_ref_map(+ *,+ session_id: str | None = None,+ cdp_url: str | None = None,+) -> None:+ state = get_current_session()+ if key := _session_ref_key(state, session_id=session_id, cdp_url=cdp_url):+ _session_ref_maps.pop(key, None)+ _session_ref_generations[key] = next(_session_ref_generation_counter)+ while len(_session_ref_generations) > _SESSION_REF_STORE_MAX:+ _session_ref_generations.pop(next(iter(_session_ref_generations)))+ else:+ state._observed_refs = {}+ state._observed_refs_generation = next(_session_ref_generation_counter)def register_copilot_session(session_id: str, state: SessionState) -> None:@@ -306,6 +458,7 @@ async def close_current_session() -> None:if current.browser is not None:await current.browser.close()finally:+ clear_session_ref_map()if current.context and current.context.session_id:unregister_copilot_session(current.context.session_id)set_current_session(SessionState())@@ -446,6 +599,7 @@ async def browser_session(await browser.close()except Exception:pass # Best effort cleanup+ clear_session_ref_map()set_current_session(SessionState())diff --git a/skyvern/cli/mcp_tools/_session.py b/skyvern/cli/mcp_tools/_session.pyindex 1f508ee17..bca564b83 100644--- a/skyvern/cli/mcp_tools/_session.py+++ b/skyvern/cli/mcp_tools/_session.py@@ -10,11 +10,16 @@ from skyvern.cli.core.session_manager import (BrowserNotAvailableError,SessionState,browser_session,+ clear_session_ref_map,close_current_session,get_current_session,get_page,+ get_session_ref,no_browser_error,+ page_ref_key,+ replace_session_ref_map,resolve_browser,+ session_ref_generation,set_current_session,)@@ -22,11 +27,16 @@ __all__ = ["BrowserNotAvailableError","SessionState","browser_session",+ "clear_session_ref_map","close_current_session","get_current_session","get_page",+ "get_session_ref","get_skyvern","no_browser_error",+ "page_ref_key",+ "replace_session_ref_map","resolve_browser",+ "session_ref_generation","set_current_session",]diff --git a/skyvern/cli/mcp_tools/browser.py b/skyvern/cli/mcp_tools/browser.pyindex 07fd2fb6c..a3ddca2f8 100644--- a/skyvern/cli/mcp_tools/browser.py+++ b/skyvern/cli/mcp_tools/browser.py@@ -5,7 +5,7 @@ import base64import jsonimport refrom datetime import datetime, timezone-from typing import Annotated, Any, Literal+from typing import Annotated, Any, Callable, Literalimport structlogfrom playwright.async_api import TimeoutError as PlaywrightTimeoutError@@ -65,7 +65,17 @@ from ._element_state import (resolve_action_timeout_ms,)from ._localhost import is_localhost_url-from ._session import BrowserNotAvailableError, get_current_session, get_page, no_browser_error+from ._session import (+ BrowserNotAvailableError,+ clear_session_ref_map,+ get_current_session,+ get_page,+ get_session_ref,+ no_browser_error,+ page_ref_key,+ replace_session_ref_map,+ session_ref_generation,+)LOG = structlog.get_logger(__name__)@@ -240,6 +250,7 @@ async def skyvern_navigate(# (even failed navigations can partially load and destroy existing frames)state = get_current_session()state._working_frame = None+ clear_session_ref_map(session_id=ctx.session_id, cdp_url=ctx.cdp_url)with Timer() as timer:try:@@ -261,6 +272,12 @@ async def skyvern_navigate(timing_ms=timer.timing_ms,error=make_error(ErrorCode.ACTION_FAILED, str(e), "Check that the URL is valid and accessible"),)+ finally:+ # Clear again after the attempt (success OR failure — a failed goto can+ # still partially replace the document): an observe that started while+ # navigation was in flight captured a post-clear generation, so only a+ # second bump can invalidate its snapshot of the old document.+ clear_session_ref_map(session_id=ctx.session_id, cdp_url=ctx.cdp_url)return make_result("skyvern_navigate",@@ -1968,6 +1985,7 @@ async def skyvern_frame_switch(# Persist frame on session state for subsequent MCP callsstate = get_current_session()state._working_frame = page._working_frame+ clear_session_ref_map(session_id=ctx.session_id, cdp_url=ctx.cdp_url)except ValueError as e:return make_result("skyvern_frame_switch",@@ -2023,6 +2041,7 @@ async def skyvern_frame_main(# Clear frame on session statestate = get_current_session()state._working_frame = None+ clear_session_ref_map(session_id=ctx.session_id, cdp_url=ctx.cdp_url)return make_result("skyvern_frame_main",@@ -2238,12 +2257,15 @@ async def skyvern_observe(Field(description="Max elements to return. Default 50.", ge=1, le=200),] = 50,) -> dict[str, Any]:- """Snapshot interactive page elements with stable refs (e0, e1, ...) for use in skyvern_execute batches."""+ """Snapshot interactive elements with refs reusable in this browser session until the next observe or page/document context change (rarely earlier — on 'Unknown ref', re-observe)."""try:page, ctx = await get_page(session_id=session_id, cdp_url=cdp_url)except BrowserNotAvailableError:return make_result("skyvern_observe", ok=False, error=no_browser_error())+ observe_page_key = page_ref_key(page)+ generation = session_ref_generation(session_id=ctx.session_id, cdp_url=ctx.cdp_url)+with Timer() as timer:try:result = await do_observe(@@ -2263,11 +2285,21 @@ async def skyvern_observe()elements = serialize_elements(result.elements)+ replace_session_ref_map(+ {element["ref"]: element for element in elements},+ session_id=ctx.session_id,+ cdp_url=ctx.cdp_url,+ generation=generation,+ page_key=observe_page_key,+ )hint = (f"Found {result.element_count} interactive elements"f"{f' (of {result.total_on_page} total on page)' if result.total_on_page > result.element_count else ''}. ""Use these refs in skyvern_execute steps, e.g.: "- '{tool: "click", params: {ref: "e0"}}'+ '{tool: "click", params: {ref: "e0"}}. '+ "Refs remain valid across calls in this browser session until the next skyvern_observe, "+ "skyvern_navigate, same-tab navigation, or tab/frame switch. Same-document DOM changes can also "+ "invalidate ordinal refs; re-observe on 'Unknown ref' or unexpected failures.")return make_result("skyvern_observe",@@ -2323,14 +2355,22 @@ async def _dispatch_step(ref_map: dict[str, dict[str, Any]],session_id: str | None,cdp_url: str | None,+ page_key: tuple[int, int | None, str] | None = None,+ on_observe_page: Callable[[tuple[int, int | None, str]], None] | None = None,) -> dict[str, Any] | None:"""Route a step to the appropriate handler, resolving refs to selectors."""params = dict(step.params)- # Resolve ref to selector if present+ # Resolve ref to selector if present. Refs bind to the page/frame they were+ # observed on, so re-check identity against the page this step will actually+ # run on — a popup or external tab change may have moved it mid-batch.if ref := params.pop("ref", None):- elem = ref_map.get(ref)- if not elem:+ current_page, _ = await get_page(session_id=session_id, cdp_url=cdp_url)+ current_key = page_ref_key(current_page)+ elem = ref_map.get(ref) if page_key is None or current_key == page_key else None+ if elem is None:+ elem = get_session_ref(ref, session_id=session_id, cdp_url=cdp_url, page_key=current_key)+ if elem is None:raise ValueError(f"Unknown ref '{ref}' — call observe first or check ref exists")params["selector"] = ref_to_selector(elem)@@ -2339,6 +2379,8 @@ async def _dispatch_step(from skyvern.cli.core.browser_ops import do_observe as _do_observepage, _ = await get_page(session_id=session_id, cdp_url=cdp_url)+ if on_observe_page is not None:+ on_observe_page(page_ref_key(page))accepted = {"selector", "interactive_only", "max_elements"}filtered = {k: v for k, v in params.items() if k in accepted}result = await _do_observe(page, **filtered)@@ -2378,7 +2420,11 @@ async def skyvern_execute(Field(description=("Array of {tool, params} step objects to execute sequentially. "- f"Within params, refs from skyvern_observe are direct targets. {DIRECT_TARGET_DESCRIPTION}"+ "Within params, refs from skyvern_observe are direct targets across calls in the same browser session. "+ "The next skyvern_observe or page/document context change invalidates them; they can occasionally "+ "expire early. Same-document DOM changes can also invalidate ordinal refs; on 'Unknown ref' or "+ "unexpected failures, re-observe. "+ f"{DIRECT_TARGET_DESCRIPTION}")),],@@ -2389,7 +2435,7 @@ async def skyvern_execute(Field(description="Stop at first failure (true) or continue past errors (false). Default true."),] = True,) -> dict[str, Any]:- """Execute multiple browser operations in a single batch. Each step is {tool, params} using refs from skyvern_observe.+ """Execute browser operations using current-session refs until the next observe or page/document context change.Allowed tools: navigate, click, type, press_key, select_option, hover, scroll, wait, observe, screenshot, evaluate."""if not steps:return make_result(@@ -2441,15 +2487,55 @@ async def skyvern_execute(# Verify we can reach the browser before executing anythingtry:- await get_page(session_id=session_id, cdp_url=cdp_url)+ page, ctx = await get_page(session_id=session_id, cdp_url=cdp_url)except BrowserNotAvailableError:return make_result("skyvern_execute", ok=False, error=no_browser_error())+ batch_page_key = page_ref_key(page)++ # Generation captured before each observe dispatch so a snapshot that raced+ # a concurrent navigation/context switch is discarded, not committed.+ observe_generation: dict[str, int] = {}+ observe_page_key: tuple[int, int | None, str] | None = None++ def capture_observe_page_key(page_key: tuple[int, int | None, str]) -> None:+ nonlocal observe_page_key+ observe_page_key = page_key+async def dispatch(step: ExecuteStep, ref_map: dict[str, dict[str, Any]]) -> dict[str, Any] | None:- return await _dispatch_step(step, ref_map, session_id=session_id, cdp_url=cdp_url)+ if step.tool == "observe":+ observe_generation["value"] = session_ref_generation(session_id=ctx.session_id, cdp_url=ctx.cdp_url)+ return await _dispatch_step(+ step,+ ref_map,+ session_id=session_id,+ cdp_url=cdp_url,+ page_key=batch_page_key,+ on_observe_page=capture_observe_page_key if step.tool == "observe" else None,+ )++ def publish_observe_refs(ref_map: dict[str, dict[str, Any]]) -> bool:+ nonlocal batch_page_key+ if observe_page_key is None:+ return False+ accepted = replace_session_ref_map(+ ref_map,+ session_id=ctx.session_id,+ cdp_url=ctx.cdp_url,+ generation=observe_generation.get("value"),+ page_key=observe_page_key,+ )+ if accepted:+ batch_page_key = observe_page_key+ return acceptedwith Timer() as timer:- result = await do_execute(dispatch, parsed_steps, stop_on_error=stop_on_error)+ result = await do_execute(+ dispatch,+ parsed_steps,+ stop_on_error=stop_on_error,+ on_ref_map_update=publish_observe_refs,+ )timer.mark("sdk")step_results = []diff --git a/skyvern/cli/mcp_tools/session.py b/skyvern/cli/mcp_tools/session.pyindex 2ec8e3a3a..aab2ac6c0 100644--- a/skyvern/cli/mcp_tools/session.py+++ b/skyvern/cli/mcp_tools/session.py@@ -21,6 +21,7 @@ from skyvern.schemas.runs import proxy_location_to_requestfrom ._common import BrowserContext, ErrorCode, Timer, make_error, make_resultfrom ._session import (SessionState,+ clear_session_ref_map,close_current_session,get_current_session,get_skyvern,@@ -284,6 +285,7 @@ async def skyvern_browser_session_close(close_error: Exception | None = Nonetry:result = await do_session_close(skyvern, session_id)+ clear_session_ref_map(session_id=session_id)except Exception as e:close_error = ediff --git a/skyvern/cli/mcp_tools/tabs.py b/skyvern/cli/mcp_tools/tabs.pyindex 77c3cf69c..9f44f5c48 100644--- a/skyvern/cli/mcp_tools/tabs.py+++ b/skyvern/cli/mcp_tools/tabs.py@@ -13,7 +13,13 @@ import structlogfrom pydantic import BaseModel, Fieldfrom ._common import ErrorCode, Timer, make_error, make_result-from ._session import BrowserNotAvailableError, get_current_session, get_page, no_browser_error+from ._session import (+ BrowserNotAvailableError,+ clear_session_ref_map,+ get_current_session,+ get_page,+ no_browser_error,+)LOG = structlog.get_logger(__name__)@@ -143,6 +149,7 @@ async def skyvern_tab_new(state._active_page = new_page# New tab has no iframes yet — clear stale frame referencestate._working_frame = None+ clear_session_ref_map(session_id=ctx.session_id, cdp_url=ctx.cdp_url)# Drain the event that _on_new_page() buffered for this explicitly# created page, so tab_wait_for_new doesn't return it as a popup.state._page_events = deque(@@ -241,6 +248,7 @@ async def skyvern_tab_switch(state._active_page = target# Switching tabs invalidates any iframe frame reference from the old tabstate._working_frame = None+ clear_session_ref_map(session_id=ctx.session_id, cdp_url=ctx.cdp_url)# bring_to_front is a no-op in headless but helps in headed modetry:@@ -325,6 +333,7 @@ async def skyvern_tab_close(state._active_page = None# Closed tab's frame reference is no longer validstate._working_frame = None+ clear_session_ref_map(session_id=ctx.session_id, cdp_url=ctx.cdp_url)# Clean up inspection hooks for the closed pagestate._hooked_page_ids.discard(target_id)diff --git a/tests/unit/test_mcp_observe_execute.py b/tests/unit/test_mcp_observe_execute.pyindex 007f3fe57..bac855b89 100644--- a/tests/unit/test_mcp_observe_execute.py+++ b/tests/unit/test_mcp_observe_execute.py@@ -2,12 +2,14 @@from __future__ import annotations+import gcfrom types import SimpleNamespacefrom typing import Anyfrom unittest.mock import AsyncMock, MagicMockimport pytest+from skyvern.cli.core import session_managerfrom skyvern.cli.core.browser_ops import (ExecuteStep,ObserveResult,@@ -19,7 +21,10 @@ from skyvern.cli.core.browser_ops import (serialize_elements,)from skyvern.cli.core.result import BrowserContext+from skyvern.cli.core.session_manager import scoped_sessionfrom skyvern.cli.mcp_tools import browser as mcp_browser+from skyvern.cli.mcp_tools import tabs as mcp_tabs+from tests.unit._mcp_browser_fakes import make_session_state# ---------------------------------------------------------------------------# Helpers@@ -839,6 +844,488 @@ class TestSkyvernExecuteMCP:click_call = mcp_browser.skyvern_click.call_argsassert 'role=button[name="Sign In"]' in str(click_call)+ @pytest.mark.asyncio+ async def test_execute_resolves_ref_from_prior_observe(self, monkeypatch: pytest.MonkeyPatch) -> None:+ page = _make_page()+ ctx = BrowserContext(mode="cloud_session", session_id="pbs_test")+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))++ click_result = {"ok": True, "data": None}+ monkeypatch.setattr(mcp_browser, "skyvern_click", AsyncMock(return_value=click_result))++ async with scoped_session(make_session_state(context=ctx)):+ observe_result = await mcp_browser.skyvern_observe(session_id=ctx.session_id)+ ref = next(element["ref"] for element in observe_result["data"]["elements"] if element["name"] == "Sign In")++ async with scoped_session(make_session_state(context=ctx)):+ execute_result = await mcp_browser.skyvern_execute(+ steps=[{"tool": "click", "params": {"ref": ref}}],+ session_id=ctx.session_id,+ )++ assert execute_result["ok"] is True+ assert execute_result["data"]["steps_completed"] == 1+ assert mcp_browser.skyvern_click.call_args.kwargs["selector"] == 'role=button[name="Sign In"]'++ @pytest.mark.asyncio+ async def test_execute_resolves_ref_from_prior_observe_without_session_id(+ self,+ monkeypatch: pytest.MonkeyPatch,+ ) -> None:+ """The original stdio/local path keeps refs on the shared SessionState."""+ page = _make_page()+ ctx = BrowserContext(mode="local")+ state = make_session_state(context=ctx)+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))+ monkeypatch.setattr(mcp_browser, "skyvern_click", AsyncMock(return_value={"ok": True, "data": None}))++ async with scoped_session(state):+ observe_result = await mcp_browser.skyvern_observe()+ ref = next(element["ref"] for element in observe_result["data"]["elements"] if element["name"] == "Sign In")++ async with scoped_session(state):+ execute_result = await mcp_browser.skyvern_execute(steps=[{"tool": "click", "params": {"ref": ref}}])++ assert execute_result["ok"] is True+ assert state._observed_refs["refs"][ref]["name"] == "Sign In"+ assert mcp_browser.skyvern_click.call_args.kwargs["selector"] == 'role=button[name="Sign In"]'++ @pytest.mark.asyncio+ async def test_keyless_ref_is_stale_when_url_changes_during_observe(+ self,+ monkeypatch: pytest.MonkeyPatch,+ ) -> None:+ page = _make_page()++ async def replace_document_during_observe() -> str:+ page.url = "https://example.com/replaced-document"+ return "Replacement"++ page.title = AsyncMock(side_effect=replace_document_during_observe)+ ctx = BrowserContext(mode="local")+ state = make_session_state(context=ctx)+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))+ click = AsyncMock(return_value={"ok": True, "data": None})+ monkeypatch.setattr(mcp_browser, "skyvern_click", click)++ async with scoped_session(state):+ observe_result = await mcp_browser.skyvern_observe()+ ref = observe_result["data"]["elements"][0]["ref"]++ async with scoped_session(state):+ execute_result = await mcp_browser.skyvern_execute(steps=[{"tool": "click", "params": {"ref": ref}}])++ assert execute_result["data"]["results"][0]["error"] == (+ f"Unknown ref '{ref}' — call observe first or check ref exists"+ )+ click.assert_not_awaited()++ @pytest.mark.asyncio+ async def test_execute_does_not_resolve_ref_from_different_session(self, monkeypatch: pytest.MonkeyPatch) -> None:+ page = _make_page()+ first_ctx = BrowserContext(mode="cloud_session", session_id="pbs_first")+ second_ctx = BrowserContext(mode="cloud_session", session_id="pbs_second")+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, first_ctx)))++ async with scoped_session(make_session_state(context=first_ctx)):+ observe_result = await mcp_browser.skyvern_observe(session_id=first_ctx.session_id)++ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, second_ctx)))+ async with scoped_session(make_session_state(context=second_ctx)):+ execute_result = await mcp_browser.skyvern_execute(+ steps=[{"tool": "click", "params": {"ref": observe_result["data"]["elements"][0]["ref"]}}],+ session_id=second_ctx.session_id,+ )++ assert execute_result["data"]["results"][0]["error"] == (+ "Unknown ref 'e0' — call observe first or check ref exists"+ )++ @pytest.mark.asyncio+ async def test_execute_observe_step_persists_refs_for_next_call(self, monkeypatch: pytest.MonkeyPatch) -> None:+ page = _make_page()+ ctx = BrowserContext(mode="cloud_session", session_id="pbs_batch_observe")+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))++ click_result = {"ok": True, "data": None}+ monkeypatch.setattr(mcp_browser, "skyvern_click", AsyncMock(return_value=click_result))++ async with scoped_session(make_session_state(context=ctx)):+ observe_result = await mcp_browser.skyvern_execute(+ steps=[{"tool": "observe", "params": {}}],+ session_id=ctx.session_id,+ )+ click_result = await mcp_browser.skyvern_execute(+ steps=[{"tool": "click", "params": {"ref": "e2"}}],+ session_id=ctx.session_id,+ )++ assert observe_result["ok"] is True+ assert click_result["ok"] is True+ assert mcp_browser.skyvern_click.call_args.kwargs["selector"] == 'role=button[name="Sign In"]'++ @pytest.mark.asyncio+ async def test_second_observe_replaces_prior_session_refs(self, monkeypatch: pytest.MonkeyPatch) -> None:+ first_tree = _make_a11y_tree(+ children=[+ {"role": "button", "name": "First"},+ {"role": "button", "name": "Stale"},+ ]+ )+ second_tree = _make_a11y_tree(children=[{"role": "button", "name": "Current"}])+ page = _make_page(first_tree)+ page.accessibility.snapshot.side_effect = [first_tree, second_tree]+ ctx = BrowserContext(mode="cloud_session", session_id="pbs_replace_refs")+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))++ click_tool_result = {"ok": True, "data": None}+ monkeypatch.setattr(mcp_browser, "skyvern_click", AsyncMock(return_value=click_tool_result))++ async with scoped_session(make_session_state(context=ctx)):+ first_result = await mcp_browser.skyvern_observe(session_id=ctx.session_id)+ second_result = await mcp_browser.skyvern_observe(session_id=ctx.session_id)+ stale_result = await mcp_browser.skyvern_execute(+ steps=[{"tool": "click", "params": {"ref": "e1"}}],+ session_id=ctx.session_id,+ )+ current_result = await mcp_browser.skyvern_execute(+ steps=[{"tool": "click", "params": {"ref": "e0"}}],+ session_id=ctx.session_id,+ )++ assert [element["ref"] for element in first_result["data"]["elements"]] == ["e0", "e1"]+ assert [element["ref"] for element in second_result["data"]["elements"]] == ["e0"]+ assert stale_result["data"]["results"][0]["error"] == (+ "Unknown ref 'e1' — call observe first or check ref exists"+ )+ assert current_result["ok"] is True+ assert mcp_browser.skyvern_click.call_args.kwargs["selector"] == 'role=button[name="Current"]'++ @pytest.mark.asyncio+ async def test_navigate_clears_session_refs(self, monkeypatch: pytest.MonkeyPatch) -> None:+ page = _make_page()+ ctx = BrowserContext(mode="cloud_session", session_id="pbs_navigate_refs")+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))++ async with scoped_session(make_session_state(context=ctx)):+ observe_result = await mcp_browser.skyvern_observe(session_id=ctx.session_id)+ navigate_result = await mcp_browser.skyvern_navigate(+ url="https://example.com/next",+ session_id=ctx.session_id,+ )+ execute_result = await mcp_browser.skyvern_execute(+ steps=[{"tool": "click", "params": {"ref": "e2"}}],+ session_id=ctx.session_id,+ )++ assert observe_result["ok"] is True+ assert navigate_result["ok"] is True+ assert execute_result["data"]["results"][0]["error"] == (+ "Unknown ref 'e2' — call observe first or check ref exists"+ )++ @pytest.mark.asyncio+ async def test_navigate_clears_batch_refs(self, monkeypatch: pytest.MonkeyPatch) -> None:+ page = _make_page()+ ctx = BrowserContext(mode="cloud_session", session_id="pbs_batch_navigate_refs")+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))+ monkeypatch.setattr(mcp_browser, "skyvern_click", AsyncMock(return_value={"ok": True, "data": None}))++ async with scoped_session(make_session_state(context=ctx)):+ result = await mcp_browser.skyvern_execute(+ steps=[+ {"tool": "observe", "params": {}},+ {"tool": "navigate", "params": {"url": "https://example.com/next"}},+ {"tool": "click", "params": {"ref": "e2"}},+ ],+ session_id=ctx.session_id,+ )++ assert result["ok"] is False+ assert result["data"]["error_step"] == 2+ assert result["data"]["results"][2]["error"] == ("Unknown ref 'e2' — call observe first or check ref exists")+ mcp_browser.skyvern_click.assert_not_awaited()++ @pytest.mark.asyncio+ async def test_tab_switch_invalidates_session_refs(self, monkeypatch: pytest.MonkeyPatch) -> None:+ page = _make_page()+ ctx = BrowserContext(mode="cloud_session", session_id="pbs_tab_refs")+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))+ monkeypatch.setattr(mcp_tabs, "get_page", AsyncMock(return_value=(page, ctx)))++ tab_a = MagicMock()+ tab_a.url = "https://example.com/a"+ tab_a.title = AsyncMock(return_value="A")+ tab_a.is_closed = MagicMock(return_value=False)+ tab_b = MagicMock()+ tab_b.url = "https://example.com/b"+ tab_b.title = AsyncMock(return_value="B")+ tab_b.is_closed = MagicMock(return_value=False)+ tab_b.bring_to_front = AsyncMock()+ browser = MagicMock()+ browser._browser_context.pages = [tab_a, tab_b]++ async with scoped_session(make_session_state(context=ctx, browser=browser)):+ observe_result = await mcp_browser.skyvern_observe(session_id=ctx.session_id)+ ref = observe_result["data"]["elements"][0]["ref"]+ switch_result = await mcp_tabs.skyvern_tab_switch(session_id=ctx.session_id, index=1)+ execute_result = await mcp_browser.skyvern_execute(+ steps=[{"tool": "click", "params": {"ref": ref}}],+ session_id=ctx.session_id,+ )++ assert switch_result["ok"] is True+ assert execute_result["data"]["results"][0]["error"] == (+ f"Unknown ref '{ref}' — call observe first or check ref exists"+ )++ @pytest.mark.asyncio+ async def test_frame_main_clears_session_refs(self, monkeypatch: pytest.MonkeyPatch) -> None:+ page = _make_page()+ page.frame_main = MagicMock()+ ctx = BrowserContext(mode="cloud_session", session_id="pbs_frame_refs")+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))++ async with scoped_session(make_session_state(context=ctx)):+ observe_result = await mcp_browser.skyvern_observe(session_id=ctx.session_id)+ ref = observe_result["data"]["elements"][0]["ref"]+ frame_result = await mcp_browser.skyvern_frame_main(session_id=ctx.session_id)+ execute_result = await mcp_browser.skyvern_execute(+ steps=[{"tool": "click", "params": {"ref": ref}}],+ session_id=ctx.session_id,+ )++ assert frame_result["ok"] is True+ assert execute_result["data"]["results"][0]["error"] == (+ f"Unknown ref '{ref}' — call observe first or check ref exists"+ )++ @pytest.mark.asyncio+ async def test_popup_page_change_invalidates_refs(self, monkeypatch: pytest.MonkeyPatch) -> None:+ """Implicit page transition (popup steals working page) must not resolve old refs."""+ page_a = _make_page()+ page_b = _make_page()+ ctx = BrowserContext(mode="cloud_session", session_id="pbs_popup_refs")++ async with scoped_session(make_session_state(context=ctx)):+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page_a, ctx)))+ observe_result = await mcp_browser.skyvern_observe(session_id=ctx.session_id)+ ref = observe_result["data"]["elements"][0]["ref"]++ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page_b, ctx)))+ execute_result = await mcp_browser.skyvern_execute(+ steps=[{"tool": "click", "params": {"ref": ref}}],+ session_id=ctx.session_id,+ )++ assert execute_result["data"]["results"][0]["error"] == (+ f"Unknown ref '{ref}' — call observe first or check ref exists"+ )++ @pytest.mark.asyncio+ async def test_generation_eviction_does_not_resurrect_refs(self) -> None:+ """ABA guard: eviction of a cleared session's generation must not let a stale commit through."""+ ctx = BrowserContext(mode="cloud_session", session_id="pbs_aba_victim")+ element = {"ref": "e0", "role": "button", "name": "Sign In", "selector": "#signin"}++ async with scoped_session(make_session_state(context=ctx)):+ stale_generation = session_manager.session_ref_generation(session_id=ctx.session_id)+ session_manager.clear_session_ref_map(session_id=ctx.session_id)+ for i in range(session_manager._SESSION_REF_STORE_MAX + 5):+ session_manager.clear_session_ref_map(session_id=f"pbs_aba_churn_{i}")++ committed = session_manager.replace_session_ref_map(+ {"e0": element}, session_id=ctx.session_id, generation=stale_generation+ )+ assert committed is False+ assert session_manager.get_session_ref("e0", session_id=ctx.session_id) is None++ @pytest.mark.asyncio+ async def test_rejected_batch_observe_does_not_install_local_refs(self) -> None:+ """If session publication rejects an observe snapshot, the batch must not act on it either."""++ async def dispatch(step: ExecuteStep, ref_map: dict[str, dict[str, Any]]) -> dict[str, Any] | None:+ if step.tool == "observe":+ return {"elements": [{"ref": "e0", "role": "button", "name": "Sign In"}], "element_count": 1}+ if ref := step.params.get("ref"):+ if ref not in ref_map:+ raise ValueError(f"Unknown ref '{ref}' — call observe first or check ref exists")+ return None++ result = await do_execute(+ dispatch,+ [ExecuteStep(tool="observe"), ExecuteStep(tool="click", params={"ref": "e0"})],+ on_ref_map_update=lambda ref_map: False,+ )++ assert result.error_step == 1+ assert "Unknown ref 'e0'" in str(result.results[1].error)++ @pytest.mark.asyncio+ async def test_observe_started_during_navigation_is_discarded(self, monkeypatch: pytest.MonkeyPatch) -> None:+ """A generation captured while navigation is in flight must not commit after it completes."""+ page = _make_page()+ ctx = BrowserContext(mode="cloud_session", session_id="pbs_nav_race")+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))++ in_flight_generation: list[int] = []++ async def fake_navigate(*args: Any, **kwargs: Any) -> Any:+ in_flight_generation.append(session_manager.session_ref_generation(session_id=ctx.session_id))+ return SimpleNamespace(url="https://example.com/next", title="Next")++ monkeypatch.setattr(mcp_browser, "do_navigate", fake_navigate)++ async with scoped_session(make_session_state(context=ctx)):+ nav_result = await mcp_browser.skyvern_navigate("https://example.com/next", session_id=ctx.session_id)+ committed = session_manager.replace_session_ref_map(+ {"e0": {"ref": "e0", "selector": "#old"}},+ session_id=ctx.session_id,+ generation=in_flight_generation[0],+ )++ assert nav_result["ok"] is True+ assert committed is False+ assert session_manager.get_session_ref("e0", session_id=ctx.session_id) is None++ @pytest.mark.asyncio+ async def test_popup_mid_batch_invalidates_batch_refs(self, monkeypatch: pytest.MonkeyPatch) -> None:+ """A page change between an in-batch observe and a ref step must not resolve stale refs."""+ page_a = _make_page()+ page_b = _make_page()+ ctx = BrowserContext(mode="cloud_session", session_id="pbs_popup_batch")+ monkeypatch.setattr(+ mcp_browser,+ "get_page",+ AsyncMock(side_effect=[(page_a, ctx), (page_a, ctx), (page_b, ctx)]),+ )++ async with scoped_session(make_session_state(context=ctx)):+ result = await mcp_browser.skyvern_execute(+ steps=[+ {"tool": "observe", "params": {}},+ {"tool": "click", "params": {"ref": "e0"}},+ ],+ session_id=ctx.session_id,+ )++ assert result["data"]["results"][1]["error"] == ("Unknown ref 'e0' — call observe first or check ref exists")++ @pytest.mark.asyncio+ async def test_popup_before_batch_observe_binds_refs_to_observed_page(+ self,+ monkeypatch: pytest.MonkeyPatch,+ ) -> None:+ """An observe after popup steal binds to the new page it actually inspected."""+ page_a = _make_page()+ page_b = _make_page()+ page_b.url = "https://example.com/popup"+ ctx = BrowserContext(mode="cloud_session", session_id="pbs_popup_before_observe")+ monkeypatch.setattr(+ mcp_browser,+ "get_page",+ AsyncMock(side_effect=[(page_a, ctx), (page_b, ctx), (page_b, ctx)]),+ )+ monkeypatch.setattr(mcp_browser, "skyvern_evaluate", AsyncMock(return_value={"ok": True, "data": None}))+ click = AsyncMock(return_value={"ok": True, "data": None})+ monkeypatch.setattr(mcp_browser, "skyvern_click", click)++ async with scoped_session(make_session_state(context=ctx)):+ result = await mcp_browser.skyvern_execute(+ steps=[+ {"tool": "evaluate", "params": {"expression": "window.open('about:blank')"}},+ {"tool": "observe", "params": {}},+ {"tool": "click", "params": {"ref": "e0"}},+ ],+ session_id=ctx.session_id,+ )++ assert result["ok"] is True+ click.assert_awaited_once()++ @pytest.mark.asyncio+ async def test_same_page_url_change_invalidates_batch_refs(self, monkeypatch: pytest.MonkeyPatch) -> None:+ """A same-tab document replacement keeps Page identity but invalidates refs by URL."""+ page = _make_page()+ ctx = BrowserContext(mode="cloud_session", session_id="pbs_click_nav_batch")+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))++ async def replace_document(**kwargs: Any) -> dict[str, Any]:+ page.url = "https://example.com/replaced-document"+ return {"ok": True, "data": None}++ monkeypatch.setattr(mcp_browser, "skyvern_evaluate", AsyncMock(side_effect=replace_document))+ click = AsyncMock(return_value={"ok": True, "data": None})+ monkeypatch.setattr(mcp_browser, "skyvern_click", click)++ async with scoped_session(make_session_state(context=ctx)):+ result = await mcp_browser.skyvern_execute(+ steps=[+ {"tool": "observe", "params": {}},+ {"tool": "evaluate", "params": {"expression": "location.href = '/replaced-document'"}},+ {"tool": "click", "params": {"ref": "e0"}},+ ],+ session_id=ctx.session_id,+ )++ assert result["data"]["results"][2]["error"] == ("Unknown ref 'e0' — call observe first or check ref exists")+ click.assert_not_awaited()++ @pytest.mark.asyncio+ async def test_failed_navigation_still_invalidates_midflight_observe(self, monkeypatch: pytest.MonkeyPatch) -> None:+ """A navigation that raises can still have replaced the document — mid-flight commits must die."""+ page = _make_page()+ ctx = BrowserContext(mode="cloud_session", session_id="pbs_failed_nav")+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))++ in_flight_generation: list[int] = []++ async def failing_navigate(*args: Any, **kwargs: Any) -> Any:+ in_flight_generation.append(session_manager.session_ref_generation(session_id=ctx.session_id))+ raise RuntimeError("net::ERR_ABORTED")++ monkeypatch.setattr(mcp_browser, "do_navigate", failing_navigate)++ async with scoped_session(make_session_state(context=ctx)):+ nav_result = await mcp_browser.skyvern_navigate("https://example.com/next", session_id=ctx.session_id)+ committed = session_manager.replace_session_ref_map(+ {"e0": {"ref": "e0", "selector": "#old"}},+ session_id=ctx.session_id,+ generation=in_flight_generation[0],+ )++ assert nav_result["ok"] is False+ assert committed is False+ assert session_manager.get_session_ref("e0", session_id=ctx.session_id) is None++ def test_page_identity_key_never_reused_after_gc(self) -> None:+ """Identity tokens must not be reissued to a new page even if id() is reused."""+ page = _make_page()+ old_key = session_manager.page_ref_key(page)+ del page+ gc.collect()+ new_page = _make_page()+ assert session_manager.page_ref_key(new_page) != old_key++ @pytest.mark.asyncio+ async def test_stale_observe_commit_is_discarded_after_clear(self) -> None:+ ctx = BrowserContext(mode="cloud_session", session_id="pbs_stale_gen")+ element = {"ref": "e0", "role": "button", "name": "Sign In", "selector": "#signin"}++ async with scoped_session(make_session_state(context=ctx)):+ stale_generation = session_manager.session_ref_generation(session_id=ctx.session_id)+ session_manager.clear_session_ref_map(session_id=ctx.session_id)+ session_manager.replace_session_ref_map(+ {"e0": element}, session_id=ctx.session_id, generation=stale_generation+ )+ assert session_manager.get_session_ref("e0", session_id=ctx.session_id) is None++ fresh_generation = session_manager.session_ref_generation(session_id=ctx.session_id)+ session_manager.replace_session_ref_map(+ {"e0": element}, session_id=ctx.session_id, generation=fresh_generation+ )+ assert session_manager.get_session_ref("e0", session_id=ctx.session_id) == element+@pytest.mark.asyncioasync def test_execute_observe_then_click_native_option_ref_selects_parent(self,@@ -877,6 +1364,8 @@ class TestSkyvernExecuteMCP:),click=AsyncMock(return_value="#unexpected-click"),)+ page.page = page+ page._working_frame = Nonectx = BrowserContext(mode="local")monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))@@ -899,9 +1388,11 @@ class TestSkyvernExecuteMCP:ctx = BrowserContext(mode="local")monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))- result = await mcp_browser.skyvern_execute(steps=[{"tool": "click", "params": {"ref": "e99"}}])+ async with scoped_session(make_session_state(context=ctx)):+ result = await mcp_browser.skyvern_execute(steps=[{"tool": "click", "params": {"ref": "e99"}}])+assert result["ok"] is False- assert "unknown ref" in result["data"]["results"][0]["error"].lower()+ assert result["data"]["results"][0]["error"] == ("Unknown ref 'e99' — call observe first or check ref exists")@pytest.mark.asyncioasync def test_execute_preserves_structured_direct_failure_error(self, monkeypatch: pytest.MonkeyPatch) -> None: