Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Recover blank multi-entry Workday sections via section-scoped add-control fallback (#8215)
Trevor Cheung586015645ce2
diff --git a/skyvern/cli/core/browser_ops.py b/skyvern/cli/core/browser_ops.pyindex ae0c94c81..6794f7167 100644--- a/skyvern/cli/core/browser_ops.py+++ b/skyvern/cli/core/browser_ops.py@@ -14,7 +14,7 @@ import timefrom dataclasses import dataclass, fieldfrom datetime import datetime, timezonefrom pathlib import Path-from typing import Any, Callable, Literal+from typing import Any, Awaitable, Callable, Literalimport structlogfrom playwright.async_api import TimeoutError as PlaywrightTimeoutError@@ -25,6 +25,7 @@ from skyvern.utils.url_validators import validate_fetch_urlfrom skyvern.webeye.utils.page import JS_FUNCTION_DEFS, SkyvernFramefrom .guards import CREDENTIAL_HINT, GuardError, validate_wait_until+from .perception_telemetry import track_perception_probeLOG = structlog.get_logger(__name__)@@ -849,6 +850,15 @@ async ({ scopeSelector, includeValues }) => {.map((element) => {const tag = element.tagName.toLowerCase();const item = { role: roleFor(element), name: text(element), tag, selector: cssPath(element) };+ const safeIdRefs = (attribute) => (element.getAttribute(attribute) || "")+ .split(/\s+/)+ .filter((value) => /^[A-Za-z_][A-Za-z0-9_.:-]{0,127}$/.test(value))+ .slice(0, 8)+ .join(" ");+ const ariaControls = safeIdRefs("aria-controls");+ const ariaOwns = safeIdRefs("aria-owns");+ if (ariaControls) item.aria_controls = ariaControls;+ if (ariaOwns) item.aria_owns = ariaOwns;if (includeValues === true && !isPassword(element) && ["input", "textarea", "select", "option"].includes(tag)) {item.value = element.value || element.getAttribute("value") || "";}@@ -1027,6 +1037,8 @@ class ObservedElement:selector: str | None = Nonevalue: str | None = Noneoptions: list[str] | None = None+ aria_controls: str | None = None+ aria_owns: str | None = Nonematch_index: int = 0needs_disambiguation: bool = False@@ -1091,13 +1103,14 @@ async def _get_dom_observe_elements(raise ObserveFrameError(frame_name, frame_url or "", RuntimeError("frame does not support evaluate"))return []try:- async with asyncio.timeout(_DOM_SCAN_TIMEOUT_SECONDS):- if await evaluate(_DOMUTILS_INTERACTABILITY_READY_JS) is not True:- await evaluate(JS_FUNCTION_DEFS)- result = await evaluate(- _OBSERVE_INTERACTABLES_JS,- {"scopeSelector": selector, "includeValues": include_values},- )+ async with track_perception_probe(evaluate_page_scan=True):+ async with asyncio.timeout(_DOM_SCAN_TIMEOUT_SECONDS):+ if await evaluate(_DOMUTILS_INTERACTABILITY_READY_JS) is not True:+ await evaluate(JS_FUNCTION_DEFS)+ result = await evaluate(+ _OBSERVE_INTERACTABLES_JS,+ {"scopeSelector": selector, "includeValues": include_values},+ )except Exception as exc:if frame_name is not None:raise ObserveFrameError(frame_name, frame_url or "", exc) from exc@@ -1152,6 +1165,10 @@ def _merge_dom_observe_elements(existing["value"] = dom_element["value"]if dom_element.get("options") and not existing.get("children"):existing["options"] = dom_element.get("options")+ if dom_element.get("aria_controls"):+ existing["aria_controls"] = dom_element["aria_controls"]+ if dom_element.get("aria_owns"):+ existing["aria_owns"] = dom_element["aria_owns"]matched_existing = Trueif dom_selector:selector_seen.add(dom_selector)@@ -1176,18 +1193,16 @@ async def get_observe_document_id(page: Any) -> str | None:# dead (page closed, target gone), so that case evicts and retries once before falling back.attempts = 2 if cdp is not None else 1for _ in range(attempts):- if cdp is None:- try:- cdp = await raw_page.context.new_cdp_session(session_target)- except Exception:- break- try:- session_target._skyvern_observe_cdp_session = cdp- except Exception:- passtry:- frame_tree = await cdp.send("Page.getFrameTree")- loader_id = frame_tree.get("frameTree", {}).get("frame", {}).get("loaderId")+ async with track_perception_probe():+ if cdp is None:+ cdp = await raw_page.context.new_cdp_session(session_target)+ try:+ session_target._skyvern_observe_cdp_session = cdp+ except Exception:+ pass+ frame_tree = await cdp.send("Page.getFrameTree")+ loader_id = frame_tree.get("frameTree", {}).get("frame", {}).get("loaderId")except Exception:if getattr(session_target, "_skyvern_observe_cdp_session", None) is cdp:try:@@ -1200,7 +1215,8 @@ async def get_observe_document_id(page: Any) -> str | None:return f"cdp:{loader_id}"breaktry:- value = await target.evaluate(_OBSERVE_DOCUMENT_ID_JS)+ async with track_perception_probe():+ value = await target.evaluate(_OBSERVE_DOCUMENT_ID_JS)except Exception:return Nonereturn f"page:{value}" if isinstance(value, str) else None@@ -1240,10 +1256,11 @@ async def do_observe(working_frame.url,RuntimeError("frame is detached"),)- observe_target = working_frame if working_frame is not None else pagedocument_id_before: str | None = Noneif observe_v2_enabled():document_id_before = await get_observe_document_id(page)++ observe_target = working_frame if working_frame is not None else pageif working_frame is not None:# Legacy page accessibility iframe roots are unreliable; DOM-only avoids cross-document merges.snapshot = None@@ -1271,7 +1288,8 @@ async def do_observe(document_id: str | None = Noneif observe_v2_enabled():try:- text_context = await observe_target.evaluate(_OBSERVE_PAGE_TEXT_JS, {"scopeSelector": selector})+ async with track_perception_probe(evaluate_page_scan=True):+ text_context = await observe_target.evaluate(_OBSERVE_PAGE_TEXT_JS, {"scopeSelector": selector})if isinstance(text_context, dict) and isinstance(text_context.get("content"), str):page_text = text_context["content"]page_text_truncated = text_context.get("truncated") is True@@ -1331,6 +1349,8 @@ async def do_observe(selector=elem.get("selector"),value=elem.get("value"),options=elem.get("options") or _extract_options(elem),+ aria_controls=elem.get("aria_controls"),+ aria_owns=elem.get("aria_owns"),match_index=match_index,needs_disambiguation=full_group_size[key] > 1,)@@ -1344,6 +1364,12 @@ async def do_observe(else:title = await page.title()+ if observe_v2_enabled() and document_id is not None:+ # Extend the bracket over the title read: a swap landing after extraction but before+ # return refuses durability the same way — degrade to None, never publish torn identity.+ if await get_observe_document_id(page) != document_id:+ document_id = None+return ObserveResult(url=working_frame.url if working_frame is not None else page.url,title=title,@@ -1369,6 +1395,9 @@ def serialize_elements(elements: list[ObservedElement]) -> list[dict[str, Any]]:"value": e.value,"options": e.options,}+ if observe_v2_enabled():+ fields["aria_controls"] = e.aria_controls+ fields["aria_owns"] = e.aria_ownsif e.needs_disambiguation:fields["match_index"] = e.match_indexresult.append({k: v for k, v in fields.items() if k in _ELEMENT_KEEP_ALWAYS or (v is not None and v != "")})@@ -1459,7 +1488,8 @@ async def do_execute(dispatch_fn: Any,steps: list[ExecuteStep],stop_on_error: bool = True,- on_ref_map_update: Callable[[dict[str, dict[str, Any]]], bool] | None = None,+ on_ref_map_update: Callable[[dict[str, dict[str, Any]]], bool | Awaitable[bool]] | None = None,+ fail_on_ref_map_rejection: bool = False,) -> ExecuteResult:"""Execute a sequence of deterministic browser operations in one batch.@@ -1490,14 +1520,16 @@ async def do_execute(try:result = await dispatch_fn(step, ref_map)# DESIGN-4: Each observe REPLACES the entire ref_map (not merges).- # If session publication rejects the snapshot (a concurrent navigation- # invalidated it), the batch must not act on it either.if step.tool == "observe" and result and "elements" in result:new_map = ref_map_from_elements(result["elements"])- if on_ref_map_update is None or on_ref_map_update(new_map):- ref_map = new_map+ if on_ref_map_update is None:+ accepted = Trueelse:- ref_map = {}+ publication = on_ref_map_update(new_map)+ accepted = publication if isinstance(publication, bool) else await publication+ if not accepted and fail_on_ref_map_rejection:+ raise RuntimeError("observe_snapshot_superseded: ref publication was rejected")+ ref_map = new_map if accepted else {}wall_ms = int((time.monotonic() - t0) * 1000)results.append(StepResult(step=i, tool=step.tool, ok=True, wall_ms=wall_ms, data=result))diff --git a/skyvern/cli/core/perception_telemetry.py b/skyvern/cli/core/perception_telemetry.pynew file mode 100644index 000000000..a87165b96--- /dev/null+++ b/skyvern/cli/core/perception_telemetry.py@@ -0,0 +1,122 @@+from __future__ import annotations++import asyncio+import time+from collections.abc import AsyncIterator, Iterator+from contextlib import asynccontextmanager, contextmanager+from contextvars import ContextVar+from dataclasses import dataclass+from typing import Literal++PerceptionSnapshotCategory = Literal["model_visible", "automatic", "stale_ref_refresh"]+++@dataclass+class MCPPerceptionCounters:+ top_level_mcp_calls: int = 1+ perception_snapshots: int = 0+ model_visible_observe_results: int = 0+ automatic_observe_snapshots: int = 0+ stale_ref_refresh_snapshots: int = 0+ failed_perception_probes: int = 0+ evaluate_page_scans: int = 0+ browser_perception_wall_ms: int = 0+ _last_failed_probe: BaseException | None = None++ def event_fields(self) -> dict[str, int]:+ return {+ "top_level_mcp_calls": self.top_level_mcp_calls,+ "perception_snapshots": self.perception_snapshots,+ "model_visible_observe_results": self.model_visible_observe_results,+ "automatic_observe_snapshots": self.automatic_observe_snapshots,+ "stale_ref_refresh_snapshots": self.stale_ref_refresh_snapshots,+ "failed_perception_probes": self.failed_perception_probes,+ "evaluate_page_scans": self.evaluate_page_scans,+ "browser_perception_wall_ms": self.browser_perception_wall_ms,+ }+++_perception_counters: ContextVar[MCPPerceptionCounters | None] = ContextVar(+ "mcp_perception_counters",+ default=None,+)+_perception_timing_depth: ContextVar[int] = ContextVar("mcp_perception_timing_depth", default=0)+++def _failure_already_counted(counters: MCPPerceptionCounters, error: BaseException) -> bool:+ current: BaseException | None = error+ while current is not None:+ if current is counters._last_failed_probe:+ return True+ current = current.__cause__ or current.__context__+ return False+++@contextmanager+def perception_counters_scope() -> Iterator[MCPPerceptionCounters]:+ counters = MCPPerceptionCounters()+ counters_token = _perception_counters.set(counters)+ depth_token = _perception_timing_depth.set(0)+ try:+ yield counters+ finally:+ _perception_timing_depth.reset(depth_token)+ _perception_counters.reset(counters_token)+++@asynccontextmanager+async def track_perception_snapshot(category: PerceptionSnapshotCategory) -> AsyncIterator[None]:+ """Account for one do_observe attempt without changing its result or errors."""+ counters = _perception_counters.get()+ if counters is None:+ yield+ return++ counters.perception_snapshots += 1+ if category == "model_visible":+ counters.model_visible_observe_results += 1+ elif category == "automatic":+ counters.automatic_observe_snapshots += 1+ else:+ counters.stale_ref_refresh_snapshots += 1++ failed_before = counters.failed_perception_probes+ depth = _perception_timing_depth.get()+ depth_token = _perception_timing_depth.set(depth + 1)+ start = time.perf_counter() if depth == 0 else None+ try:+ yield+ except (Exception, asyncio.CancelledError) as exc:+ if counters.failed_perception_probes == failed_before or not _failure_already_counted(counters, exc):+ counters.failed_perception_probes += 1+ counters._last_failed_probe = exc+ raise+ finally:+ if start is not None:+ counters.browser_perception_wall_ms += int((time.perf_counter() - start) * 1000)+ _perception_timing_depth.reset(depth_token)+++@asynccontextmanager+async def track_perception_probe(*, evaluate_page_scan: bool = False) -> AsyncIterator[None]:+ """Account for a non-snapshot browser perception probe."""+ counters = _perception_counters.get()+ if counters is None:+ yield+ return++ if evaluate_page_scan:+ counters.evaluate_page_scans += 1+ depth = _perception_timing_depth.get()+ depth_token = _perception_timing_depth.set(depth + 1)+ start = time.perf_counter() if depth == 0 else None+ try:+ yield+ except (Exception, asyncio.CancelledError) as exc:+ counters.failed_perception_probes += 1+ counters._last_failed_probe = exc+ raise+ finally:+ if start is not None:+ counters.browser_perception_wall_ms += int((time.perf_counter() - start) * 1000)+ _perception_timing_depth.reset(depth_token)diff --git a/skyvern/cli/core/session_manager.py b/skyvern/cli/core/session_manager.pyindex c139a31a6..cbde35ae8 100644--- a/skyvern/cli/core/session_manager.py+++ b/skyvern/cli/core/session_manager.py@@ -42,12 +42,11 @@ class ObserveV2State:host_budgets: dict[str, int] = field(default_factory=dict)def reset_refs(self) -> None:- """Clear per-document ref data while preserving host budgets across navigation."""+ """Clear per-document refs while preserving monotonic ref IDs and host budgets."""self.page_key = Noneself.document_id = Noneself.params = {}self.refs = {}- self.next_ref = 0@dataclass@@ -156,6 +155,18 @@ def _generation_for(key: tuple[str | None, str, str, str | None]) -> int:return generation+def _advance_generation_for(key: tuple[str | None, str, str, str | None]) -> int:+ generation = next(_session_ref_generation_counter)+ # Pop-then-reinsert: an advance is activity, so it must LRU-touch the key -+ # FIFO eviction would drop the longest-lived (often busiest) session and a+ # freshly minted generation would break its in-flight reservation.+ _session_ref_generations.pop(key, None)+ _session_ref_generations[key] = generation+ while len(_session_ref_generations) > _SESSION_REF_STORE_MAX:+ _session_ref_generations.pop(next(iter(_session_ref_generations)))+ return generation++def _session_ref_key(state: SessionState,*,@@ -210,6 +221,38 @@ def session_ref_generation(return state._observed_refs_generation+def begin_session_ref_publication(+ *,+ session_id: str | None = None,+ cdp_url: str | None = None,+) -> int:+ """Reserve a unique generation for one in-flight ref publication."""+ state = get_current_session()+ if key := _session_ref_key(state, session_id=session_id, cdp_url=cdp_url):+ return _advance_generation_for(key)+ state._observed_refs_generation = next(_session_ref_generation_counter)+ return state._observed_refs_generation+++def invalidate_session_ref_map(+ *,+ session_id: str | None = None,+ cdp_url: str | None = None,+) -> int:+ """Invalidate published refs while retaining v2 budgets and monotonic ref IDs."""+ state = get_current_session()+ if key := _session_ref_key(state, session_id=session_id, cdp_url=cdp_url):+ _session_ref_maps.pop(key, None)+ observe_state = _observe_v2_states.get(key)+ if observe_state is not None:+ observe_state.reset_refs()+ return _advance_generation_for(key)+ state._observed_refs = {}+ state._observe_v2_state.reset_refs()+ state._observed_refs_generation = next(_session_ref_generation_counter)+ return state._observed_refs_generation++def replace_session_ref_map(ref_map: dict[str, dict[str, Any]],*,@@ -217,12 +260,19 @@ def replace_session_ref_map(cdp_url: str | None = None,generation: int | None = None,page_key: tuple[int, int | None, str, str | None] | None = None,+ advance_on_commit: bool = True,) -> bool:"""Replace the session's ref snapshot (never merge). Returns False if discarded.- When ``generation`` is given, the snapshot is discarded if the registry was- cleared after that generation was captured — the observe raced a navigation- or context switch and describes a replaced document.+ A supplied generation is a publication reservation. The snapshot is discarded+ if a newer operation has reserved or invalidated the registry; an accepted+ commit advances the generation so work captured before the commit cannot+ mutate the published snapshot afterward.++ Legacy (pre-v2) publications pass ``advance_on_commit=False``: their generation+ is a plain read, so the CAS refuses snapshots that raced an invalidation+ (navigation/clear) while overlapping publications keep last-completer-wins -+ exactly the pre-reservation contract."""state = get_current_session()snapshot: dict[str, Any] = {"page_key": page_key, "refs": dict(ref_map)}@@ -232,10 +282,14 @@ def replace_session_ref_map(_session_ref_maps[key] = snapshotwhile len(_session_ref_maps) > _SESSION_REF_STORE_MAX:_session_ref_maps.pop(next(iter(_session_ref_maps)))+ if advance_on_commit:+ _advance_generation_for(key)else:if generation is not None and generation != state._observed_refs_generation:return Falsestate._observed_refs = snapshot+ if advance_on_commit:+ state._observed_refs_generation = next(_session_ref_generation_counter)return True@@ -269,9 +323,13 @@ def clear_session_ref_map(*,session_id: str | None = None,cdp_url: str | None = None,-) -> None:+ generation: int | None = None,+) -> bool:+ """Clear refs, unless a newer publication reservation superseded this caller."""state = get_current_session()if key := _session_ref_key(state, session_id=session_id, cdp_url=cdp_url):+ if generation is not None and generation != _generation_for(key):+ return False_session_ref_maps.pop(key, None)observe_state = _observe_v2_states.pop(key, None)if observe_state is None:@@ -281,13 +339,17 @@ def clear_session_ref_map(_observe_v2_states[key] = observe_statewhile len(_observe_v2_states) > _SESSION_REF_STORE_MAX:_observe_v2_states.pop(next(iter(_observe_v2_states)))- _session_ref_generations[key] = next(_session_ref_generation_counter)- while len(_session_ref_generations) > _SESSION_REF_STORE_MAX:- _session_ref_generations.pop(next(iter(_session_ref_generations)))+ _advance_generation_for(key)else:+ if generation is not None and generation != session_ref_generation(+ session_id=session_id,+ cdp_url=cdp_url,+ ):+ return Falsestate._observed_refs = {}state._observe_v2_state.reset_refs()state._observed_refs_generation = next(_session_ref_generation_counter)+ return Truedef register_copilot_session(session_id: str, state: SessionState, *, organization_id: str) -> None:diff --git a/skyvern/cli/mcp_tools/_session.py b/skyvern/cli/mcp_tools/_session.pyindex ffb7102f2..25d737e80 100644--- a/skyvern/cli/mcp_tools/_session.py+++ b/skyvern/cli/mcp_tools/_session.py@@ -10,6 +10,7 @@ from skyvern.cli.core.session_manager import (BrowserNotAvailableError,SessionState,active_api_key_hash,+ begin_session_ref_publication,browser_session,clear_session_ref_map,close_current_session,@@ -17,6 +18,7 @@ from skyvern.cli.core.session_manager import (get_current_session,get_page,get_session_ref,+ invalidate_session_ref_map,is_stdio_local_file_access_enabled,no_browser_error,page_ref_key,@@ -30,6 +32,7 @@ __all__ = ["BrowserNotAvailableError","SessionState","active_api_key_hash",+ "begin_session_ref_publication","browser_session","clear_session_ref_map","close_current_session",@@ -38,6 +41,7 @@ __all__ = ["get_page","get_session_ref","get_skyvern",+ "invalidate_session_ref_map","is_stdio_local_file_access_enabled","no_browser_error","page_ref_key",diff --git a/skyvern/cli/mcp_tools/browser.py b/skyvern/cli/mcp_tools/browser.pyindex eec879787..afd6e2676 100644--- a/skyvern/cli/mcp_tools/browser.py+++ b/skyvern/cli/mcp_tools/browser.py@@ -65,6 +65,7 @@ from skyvern.cli.core.guards import resolve_ai_mode as _resolve_ai_modefrom skyvern.cli.core.guards import (validate_wait_until,)+from skyvern.cli.core.perception_telemetry import PerceptionSnapshotCategory, track_perception_snapshotfrom skyvern.cli.core.session_manager import ObserveV2State, get_observe_v2_state, is_stateless_http_modefrom skyvern.cli.core.trajectory_store import append_trajectory_entryfrom skyvern.config import settings@@ -101,17 +102,20 @@ from ._element_state import (from ._localhost import is_localhost_urlfrom ._session import (BrowserNotAvailableError,+ begin_session_ref_publication,clear_session_ref_map,current_api_key_hash,get_current_session,get_page,get_session_ref,+ invalidate_session_ref_map,is_stdio_local_file_access_enabled,no_browser_error,page_ref_key,replace_session_ref_map,session_ref_generation,)+from .response import truncate_response_bytesLOG = structlog.get_logger(__name__)@@ -526,7 +530,7 @@ async def skyvern_navigate(# (even failed navigations can partially load and destroy existing frames)state = get_current_session()state._working_frame = None- clear_session_ref_map(session_id=ctx.session_id, cdp_url=ctx.cdp_url)+ invalidate_session_ref_map(session_id=ctx.session_id, cdp_url=ctx.cdp_url)with Timer() as timer:try:@@ -556,11 +560,9 @@ async def skyvern_navigate(error=make_error(ErrorCode.ACTION_FAILED, str(e), "Check that the URL is valid and accessible"),)finally:- # Clear again after the attempt (success OR failure — a failed goto can- # still partially replace the document): an observe that started while- # navigation was in flight captured a post-clear generation, so only a- # second bump can invalidate its snapshot of the old document.- clear_session_ref_map(session_id=ctx.session_id, cdp_url=ctx.cdp_url)+ # No publication made while navigation was in flight is trustworthy:+ # even a failed goto can partially replace the document.+ invalidate_session_ref_map(session_id=ctx.session_id, cdp_url=ctx.cdp_url)requested_load_state = wait_until or "load"warnings = (@@ -3459,6 +3461,9 @@ async def skyvern_clipboard_write(_OBSERVE_V2_DEFAULT_BUDGET = 50_OBSERVE_V2_MAX_BUDGET = 200_OBSERVE_V2_HOST_BUDGET_MAX_HOSTS = 32+_POST_MUTATION_SETTLE_MS = 250+_POST_MUTATION_SETTLE_TIMEOUT_SECONDS = 0.5+_SAFE_ARIA_TARGET_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_.:-]{0,127}$")def _observe_v2_host(page: Any) -> str:@@ -3494,11 +3499,13 @@ async def _observe_with_v2_budget(observe_fn: Callable[..., Any],page: Any,*,+ perception_category: PerceptionSnapshotCategory,session_id: str | None = None,cdp_url: str | None = None,**params: Any,) -> Any:- result = await observe_fn(page, **params)+ async with track_perception_snapshot(perception_category):+ result = await observe_fn(page, **params)if observe_v2_enabled():state = get_observe_v2_state(session_id=session_id, cdp_url=cdp_url)_learn_observe_v2_host_budget(state, page, result.total_on_page)@@ -3548,7 +3555,7 @@ def _prepare_observe_v2_refs(# though the document is untouched: costs the optimization, never correctness.same_document = state.page_key == current_page_key and document_id is not None and state.document_id == document_idold_refs = state.refs if same_document else {}- next_ref = state.next_ref if same_document else 0+ next_ref = state.next_refif not old_refs and next_ref == 0:prepared = [dict(element) for element in elements]@@ -3596,12 +3603,71 @@ def _prepare_observe_v2_refs(}-def _commit_observe_v2_refs(state: ObserveV2State, prepared: dict[str, Any]) -> None:+async def _observe_v2_snapshot_is_current(+ page: Any,+ prepared: dict[str, Any],+ *,+ session_id: str | None,+ cdp_url: str | None,+ generation: int | None,+) -> bool:+ if generation is not None and generation != session_ref_generation(+ session_id=session_id,+ cdp_url=cdp_url,+ ):+ return False+ live_document_id = await get_observe_document_id(page)+ if generation is not None and generation != session_ref_generation(session_id=session_id, cdp_url=cdp_url):+ return False+ prepared_document_id = prepared["document_id"]+ if page_ref_key(page) != prepared["page_key"] or (+ # A trusted (cdp:) snapshot must revalidate against the live marker before publish.+ # An untrusted snapshot (page:-sourced marker gated to None) still publishes so the+ # response lists refs, but with document_id=None every later ref use fails closed —+ # the rollout-gate contract: refs are visible, none resolve.+ prepared_document_id is not None and (live_document_id is None or live_document_id != prepared_document_id)+ ):+ clear_session_ref_map(+ session_id=session_id,+ cdp_url=cdp_url,+ generation=generation,+ )+ return False+ return True+++async def _publish_observe_v2_refs(+ page: Any,+ prepared: dict[str, Any],+ ref_map: dict[str, dict[str, Any]],+ *,+ session_id: str | None,+ cdp_url: str | None,+ generation: int | None,+) -> bool:+ if not await _observe_v2_snapshot_is_current(+ page,+ prepared,+ session_id=session_id,+ cdp_url=cdp_url,+ generation=generation,+ ):+ return False+ if not replace_session_ref_map(+ ref_map,+ session_id=session_id,+ cdp_url=cdp_url,+ generation=generation,+ page_key=prepared["page_key"],+ ):+ return False+ state = get_observe_v2_state(session_id=session_id, cdp_url=cdp_url)state.page_key = prepared["page_key"]state.document_id = prepared["document_id"]state.params = prepared["params"]state.refs = prepared["refs"]state.next_ref = prepared["next_ref"]+ return Trueasync def _refresh_observe_v2_ref(@@ -3610,27 +3676,32 @@ async def _refresh_observe_v2_ref(*,session_id: str | None,cdp_url: str | None,+ refresh_state: ObserveV2State | None = None,+ expected_generation: int | None = None,) -> tuple[bool, dict[str, Any] | None]:if not observe_v2_enabled():return False, None- state = get_observe_v2_state(session_id=session_id, cdp_url=cdp_url)+ generation = session_ref_generation(session_id=session_id, cdp_url=cdp_url)+ if expected_generation is not None and generation != expected_generation:+ return True, None+ state = refresh_state or get_observe_v2_state(session_id=session_id, cdp_url=cdp_url)if ref not in state.refs:# With the flag on, every published legacy ref has a matching v2 ref. A ref the# legacy map still holds therefore predates the flag flip (or survived invalidation)# and cannot be validated — revoke both stores. A ref neither store knows is# model-invented: reject it alone, keeping the refs that were just published.if get_session_ref(ref, session_id=session_id, cdp_url=cdp_url, page_key=page_ref_key(page)) is not None:- clear_session_ref_map(session_id=session_id, cdp_url=cdp_url)+ clear_session_ref_map(session_id=session_id, cdp_url=cdp_url, generation=generation)return True, Nonecurrent_page_key = page_ref_key(page)if state.page_key != current_page_key:- clear_session_ref_map(session_id=session_id, cdp_url=cdp_url)+ clear_session_ref_map(session_id=session_id, cdp_url=cdp_url, generation=generation)return True, Nonedocument_valid = state.document_id is not None and await get_observe_document_id(page) == state.document_idif not document_valid:- clear_session_ref_map(session_id=session_id, cdp_url=cdp_url)+ clear_session_ref_map(session_id=session_id, cdp_url=cdp_url, generation=generation)return True, Nonefrom skyvern.cli.core.browser_ops import do_observe as current_do_observe@@ -3645,16 +3716,22 @@ async def _refresh_observe_v2_ref(result = await _observe_with_v2_budget(current_do_observe,page,+ perception_category="stale_ref_refresh",session_id=session_id,cdp_url=cdp_url,**params,)except Exception:# An unverifiable ref must not remain usable through the legacy fallback.- clear_session_ref_map(session_id=session_id, cdp_url=cdp_url)+ clear_session_ref_map(session_id=session_id, cdp_url=cdp_url, generation=generation)return True, None- if page_ref_key(page) != current_page_key or result.document_id is None or result.document_id != state.document_id:- clear_session_ref_map(session_id=session_id, cdp_url=cdp_url)+ if (+ generation != session_ref_generation(session_id=session_id, cdp_url=cdp_url)+ or page_ref_key(page) != current_page_key+ or result.document_id is None+ or result.document_id != state.document_id+ ):+ clear_session_ref_map(session_id=session_id, cdp_url=cdp_url, generation=generation)return True, Noneprepared = _prepare_observe_v2_refs(state,@@ -3721,15 +3798,33 @@ async def skyvern_observe(] = False,) -> dict[str, Any]:"""Snapshot interactive elements. On stdio, refs persist across calls until the next observe or page/document context change (rarely earlier — on 'Unknown ref', re-observe). In hosted stateless HTTP, refs from prior requests do not resolve; prefer selector or intent params, using refs from an inline observe in one skyvern_execute batch only when predictable in advance. Input values are omitted by default; set include_values=True to return non-password values. Password values are never returned."""+ v2_enabled = observe_v2_enabled()+ # Failure-path cleanup is a v2 hardening: the pre-v2 contract preserves the+ # registry on failed observes (only a frame error clears it, below).+ lookup_generation = session_ref_generation(session_id=session_id, cdp_url=cdp_url) if v2_enabled else Nonetry:page, ctx = await get_page(session_id=session_id, cdp_url=cdp_url)+ except asyncio.CancelledError:+ if v2_enabled:+ clear_session_ref_map(session_id=session_id, cdp_url=cdp_url, generation=lookup_generation)+ raiseexcept BrowserNotAvailableError:+ if v2_enabled:+ clear_session_ref_map(session_id=session_id, cdp_url=cdp_url, generation=lookup_generation)return make_result("skyvern_observe", ok=False, error=no_browser_error())action_result = _action_result_factory(ctx=ctx, page=page, selector=selector)observe_page_key = page_ref_key(page)- generation = session_ref_generation(session_id=ctx.session_id, cdp_url=ctx.cdp_url)+ # The generation guard is not v2-only: it predates observe-v2 and protects the+ # flag-off path too (an in-flight observe racing a concurrent invalidation must+ # not republish a stale snapshot). v2 reserves a fresh generation; flag-off+ # reads the current one, exactly as before the flag existed.+ generation = (+ begin_session_ref_publication(session_id=ctx.session_id, cdp_url=ctx.cdp_url)+ if v2_enabled+ else session_ref_generation(session_id=ctx.session_id, cdp_url=ctx.cdp_url)+ )observe_params = {"selector": selector,"interactive_only": interactive_only,@@ -3741,13 +3836,22 @@ async def skyvern_observe(result = await _observe_with_v2_budget(do_observe,page,+ perception_category="model_visible",session_id=ctx.session_id,cdp_url=ctx.cdp_url,**observe_params,)timer.mark("sdk")+ except asyncio.CancelledError:+ if v2_enabled:+ clear_session_ref_map(session_id=ctx.session_id, cdp_url=ctx.cdp_url, generation=generation)+ raiseexcept ObserveFrameError as e:- clear_session_ref_map(session_id=ctx.session_id, cdp_url=ctx.cdp_url)+ if v2_enabled:+ clear_session_ref_map(session_id=ctx.session_id, cdp_url=ctx.cdp_url, generation=generation)+ else:+ # Pre-v2 contract: a frame-error observe clears unconditionally.+ clear_session_ref_map(session_id=ctx.session_id, cdp_url=ctx.cdp_url)return action_result("skyvern_observe",ok=False,@@ -3756,6 +3860,8 @@ async def skyvern_observe(error=_observe_frame_error(e),)except Exception as e:+ if v2_enabled:+ clear_session_ref_map(session_id=ctx.session_id, cdp_url=ctx.cdp_url, generation=generation)return action_result("skyvern_observe",ok=False,@@ -3773,23 +3879,11 @@ async def skyvern_observe(document_id=result.document_id,params=observe_params,)- if observe_v2_enabled()+ if v2_enabledelse None)if prepared_v2 is not None:elements = prepared_v2["elements"]- accepted = replace_session_ref_map(- ref_map_from_elements(elements),- session_id=ctx.session_id,- cdp_url=ctx.cdp_url,- generation=generation,- page_key=observe_page_key,- )- if accepted and prepared_v2 is not None:- _commit_observe_v2_refs(- get_observe_v2_state(session_id=ctx.session_id, cdp_url=ctx.cdp_url),- prepared_v2,- )element_count = len(elements)hint = (f"Found {element_count} interactive elements"@@ -3811,14 +3905,69 @@ async def skyvern_observe("total_on_page": result.total_on_page,"hint": hint,}- if observe_v2_enabled():+ if v2_enabled:data["page_text"] = _observe_v2_page_text(result)- return action_result(+ response = action_result("skyvern_observe",browser_context=ctx,data=data,timing_ms=timer.timing_ms,)+ capped_response = truncate_response_bytes(response) if v2_enabled else response+ if capped_response is not response:+ clear_session_ref_map(session_id=ctx.session_id, cdp_url=ctx.cdp_url, generation=generation)+ return capped_response+ if not v2_enabled:+ # Generation-checked like main always did; refusal is silent here (the+ # response is already built) - stale snapshots just never get published.+ replace_session_ref_map(+ ref_map_from_elements(elements),+ session_id=ctx.session_id,+ cdp_url=ctx.cdp_url,+ generation=generation,+ page_key=observe_page_key,+ advance_on_commit=False,+ )+ return capped_response+ if prepared_v2 is None:+ accepted = replace_session_ref_map(+ ref_map_from_elements(elements),+ session_id=ctx.session_id,+ cdp_url=ctx.cdp_url,+ generation=generation,+ page_key=observe_page_key,+ )+ else:+ try:+ publication_page, _ = await get_page(session_id=session_id, cdp_url=cdp_url)+ accepted = await _publish_observe_v2_refs(+ publication_page,+ prepared_v2,+ ref_map_from_elements(elements),+ session_id=ctx.session_id,+ cdp_url=ctx.cdp_url,+ generation=generation,+ )+ except asyncio.CancelledError:+ clear_session_ref_map(session_id=ctx.session_id, cdp_url=ctx.cdp_url, generation=generation)+ raise+ except Exception:+ clear_session_ref_map(session_id=ctx.session_id, cdp_url=ctx.cdp_url, generation=generation)+ accepted = False+ if not accepted:+ error = make_error(+ ErrorCode.ACTION_FAILED,+ "Observe snapshot was superseded before ref publication",+ "Call skyvern_observe again for current refs",+ )+ return action_result(+ "skyvern_observe",+ ok=False,+ browser_context=ctx,+ timing_ms=timer.timing_ms,+ error=error,+ )+ return capped_response# ---------------------------------------------------------------------------@@ -3857,16 +4006,92 @@ _TOOL_ACCEPTED_PARAMS: dict[str, frozenset[str]] = {}+_EXECUTE_MUTATION_CLASS: dict[str, bool] = {+ "navigate": False,+ "click": True,+ "type": True,+ "press_key": True,+ "select_option": True,+ "hover": True,+ "scroll": True,+ "wait": False,+ "wait_for_either_state": False,+ "observe": False,+ "screenshot": False,+ # Arbitrary JS can mutate the DOM in ways no static classifier can soundly+ # detect (bracket-notation writes, aliased methods, eval, ...), and a miss+ # fails open: stale refs stay resolvable after an invisible same-document+ # change. Classify every evaluate as mutating - the cost is one bounded+ # auto-observe per evaluate step, the same class as scroll.+ "evaluate": True,+}+assert _EXECUTE_MUTATION_CLASS.keys() == _ALLOWED_EXECUTE_TOOLS+++def _execute_step_mutates(step: ExecuteStep) -> bool:+ """Classify every execute tool's mutation potential (fail-closed for evaluate)."""+ return _EXECUTE_MUTATION_CLASS[step.tool]+++def _aria_target_selector(elements: list[dict[str, Any]]) -> str | None:+ target: str | None = None+ for element in elements:+ for field in ("aria_controls", "aria_owns"):+ value = element.get(field)+ if not isinstance(value, str):+ continue+ for candidate in value.split():+ if not _SAFE_ARIA_TARGET_RE.fullmatch(candidate):+ continue+ if target is None:+ target = candidate+ elif candidate != target:+ return None+ return f'[id="{target}"]' if target else None+++async def _attached_aria_target_selector(page: Any, elements: list[dict[str, Any]]) -> str | None:+ selector = _aria_target_selector(elements)+ if selector is None:+ return None+ raw_page = getattr(page, "page", page)+ try:+ return selector if await raw_page.query_selector(selector) is not None else None+ except Exception:+ return None+++async def _settle_after_mutating_batch(page: Any) -> None:+ """Give page handlers one bounded quiet window before the post-mutation snapshot."""+ raw_page = getattr(page, "page", page)+ wait_for_timeout = getattr(raw_page, "wait_for_timeout", None) or getattr(page, "wait_for_timeout", None)+ if wait_for_timeout is None:+ return+ try:+ await asyncio.wait_for(+ wait_for_timeout(_POST_MUTATION_SETTLE_MS),+ timeout=_POST_MUTATION_SETTLE_TIMEOUT_SECONDS,+ )+ except Exception:+ LOG.debug("Post-mutation settle window ended early", exc_info=True)++async def _dispatch_step(step: ExecuteStep,ref_map: dict[str, dict[str, Any]],session_id: str | None,cdp_url: str | None,page_key: tuple[int, int | None, str, str | None] | None = None,+ document_id: str | None = None,on_observe_page: Callable[[tuple[int, int | None, str, str | None]], None] | None = None,on_observe_v2: Callable[[Any, list[dict[str, Any]], Any, dict[str, Any]], list[dict[str, Any]]] | None = None,+ on_resolved_element: Callable[[dict[str, Any]], None] | None = None,+ on_before_action: Callable[[], None] | None = None,+ perception_category: PerceptionSnapshotCategory = "model_visible",observe_v2_session_id: str | None = None,observe_v2_cdp_url: str | None = None,+ observe_v2_refresh_state: ObserveV2State | None = None,+ observe_v2_generation: int | None = None,) -> dict[str, Any] | None:"""Route a step to the appropriate handler, resolving refs to selectors."""params = dict(step.params)@@ -3877,21 +4102,14 @@ async def _dispatch_step(if ref := params.pop("ref", None):current_page, _ = await get_page(session_id=session_id, cdp_url=cdp_url)current_key = page_ref_key(current_page)- elem = ref_map.get(ref) if page_key is None or current_key == page_key else None- if elem is not None and observe_v2_enabled():- # An in-batch hit skips _refresh_observe_v2_ref, so it must run the same document- # check: an earlier step in this batch can replace the document at the same URL,- # leaving these refs pointing at pre-swap selectors. The marker read reuses the- # cached CDP session; on None or mismatch fall through to the refresh path, which- # fails closed (its own document check cannot pass either). This validates document- # identity only — a same-document edit that renumbers ordinal (nth=) selectors is- # caught by the cross-call re-observe, not here; that speed trade is deliberate.- v2_state = get_observe_v2_state(- session_id=observe_v2_session_id or session_id,- cdp_url=observe_v2_cdp_url or cdp_url,- )- if v2_state.document_id is None or await get_observe_document_id(current_page) != v2_state.document_id:- elem = None+ elem = None+ if page_key is None or current_key == page_key:+ if not observe_v2_enabled():+ elem = ref_map.get(ref)+ elif document_id is not None and await get_observe_document_id(current_page) == document_id:+ elem = ref_map.get(ref)+ else:+ ref_map.clear()if elem is None:current = get_session_ref(ref, session_id=session_id, cdp_url=cdp_url, page_key=current_key)handled, elem = await _refresh_observe_v2_ref(@@ -3899,6 +4117,8 @@ async def _dispatch_step(current_page,session_id=observe_v2_session_id or session_id,cdp_url=observe_v2_cdp_url or cdp_url,+ refresh_state=observe_v2_refresh_state,+ expected_generation=observe_v2_generation,)if not handled:elem = current@@ -3908,6 +4128,8 @@ async def _dispatch_step(message += ". In stateless HTTP mode refs from prior requests do not resolve — use selector or intent params instead."raise ValueError(message)params["selector"] = ref_to_selector(elem)+ if on_resolved_element is not None:+ on_resolved_element(dict(elem))# Observe is handled inline (not an existing MCP tool)if step.tool == "observe":@@ -3922,12 +4144,16 @@ async def _dispatch_step(result = await _observe_with_v2_budget(_do_observe,page,+ perception_category=perception_category,session_id=observe_v2_session_id or session_id,cdp_url=observe_v2_cdp_url or cdp_url,**filtered,)except ObserveFrameError as e:- clear_session_ref_map(session_id=session_id, cdp_url=cdp_url)+ if not observe_v2_enabled():+ # Pre-v2 contract: an in-batch frame-error observe clears unconditionally+ # (v2 defers to the dispatch-level generation-checked cleanup).+ clear_session_ref_map(session_id=session_id, cdp_url=cdp_url)raise ToolStepError(_observe_frame_error(e)) from eelements = serialize_elements(result.elements)if observe_v2_enabled() and on_observe_v2 is not None:@@ -3941,6 +4167,8 @@ async def _dispatch_step(data["page_text"] = _observe_v2_page_text(result)return data+ if on_before_action is not None:+ on_before_action()# DESIGN-1: Dispatch through existing MCP tool functions via module lookupimport skyvern.cli.mcp_tools.browser as _browser_mod@@ -4037,21 +4265,70 @@ async def skyvern_execute()parsed_steps.append(ExecuteStep(tool=tool, params=raw.get("params", {})))+ operation_generation: int | None = (+ session_ref_generation(session_id=session_id, cdp_url=cdp_url) if observe_v2_enabled() else None+ )# Verify we can reach the browser before executing anythingtry:page, ctx = await get_page(session_id=session_id, cdp_url=cdp_url)+ except asyncio.CancelledError:+ if observe_v2_enabled():+ clear_session_ref_map(+ session_id=session_id,+ cdp_url=cdp_url,+ generation=operation_generation,+ )+ raiseexcept BrowserNotAvailableError:+ if observe_v2_enabled():+ clear_session_ref_map(+ session_id=session_id,+ cdp_url=cdp_url,+ generation=operation_generation,+ )return make_result("skyvern_execute", ok=False, error=no_browser_error())action_result = _action_result_factory(ctx=ctx, page=page)+ operation_generation = (+ session_ref_generation(session_id=ctx.session_id, cdp_url=ctx.cdp_url) if observe_v2_enabled() else None+ )batch_page_key = page_ref_key(page)+ if observe_v2_enabled():+ try:+ # Only a browser-sourced cdp: marker may certify in-batch document identity;+ # a page:-sourced marker is spoofable, so it degrades to None and fails closed+ # (upfront invalidation below, and the in-batch ref check refuses to certify).+ batch_document_id = _observe_v2_trusted_document_id(await get_observe_document_id(page))+ except asyncio.CancelledError:+ clear_session_ref_map(+ session_id=ctx.session_id,+ cdp_url=ctx.cdp_url,+ generation=operation_generation,+ )+ raise+ else:+ batch_document_id = None+ if observe_v2_enabled() and batch_document_id is None:+ clear_session_ref_map(+ session_id=ctx.session_id,+ cdp_url=ctx.cdp_url,+ generation=operation_generation,+ )+ operation_generation = session_ref_generation(session_id=ctx.session_id, cdp_url=ctx.cdp_url)- # Generation captured before each observe dispatch so a snapshot that raced- # a concurrent navigation/context switch is discarded, not committed.- observe_generation: dict[str, int] = {}+ # Under observe-v2, each observe reserves a unique generation before+ # dispatch so an older snapshot cannot overwrite a newer publication.+ observe_generation: dict[str, int | None] = {}observe_page_key: tuple[int, int | None, str, str | None] | None = Noneobserve_v2_prepared: dict[str, Any] | None = None+ pending_ref_map: dict[str, dict[str, Any]] | None = None+ dispatched_observe_data: dict[str, Any] | None = None+ pending_observe_data: dict[str, Any] | None = None+ ref_refresh_state: ObserveV2State | None = None+ mutation_started = False+ acted_elements: list[dict[str, Any]] = []+ perception_category: PerceptionSnapshotCategory = "model_visible"def capture_observe_page_key(page_key: tuple[int, int | None, str, str | None]) -> None:nonlocal observe_page_key@@ -4073,75 +4350,352 @@ async def skyvern_execute()return observe_v2_prepared["elements"]+ def capture_resolved_element(element: dict[str, Any]) -> None:+ # Stamp the batch's certified document so the post-batch aria scope can+ # refuse elements from a document the batch has since left (click-driven+ # navigation and failed goto never set successful_navigation).+ acted_elements.append({**element, "_acted_document_id": batch_document_id})++ def invalidate_before_mutation(ref_map: dict[str, dict[str, Any]]) -> None:+ nonlocal observe_page_key, observe_v2_prepared, pending_ref_map, operation_generation, ref_refresh_state+ nonlocal mutation_started, pending_observe_data+ if ref_refresh_state is None:+ state = get_observe_v2_state(session_id=ctx.session_id, cdp_url=ctx.cdp_url)+ ref_refresh_state = ObserveV2State(+ page_key=state.page_key,+ document_id=state.document_id,+ params=dict(state.params),+ refs=dict(state.refs),+ next_ref=state.next_ref,+ host_budgets=dict(state.host_budgets),+ )+ mutation_started = True+ operation_generation = invalidate_session_ref_map(+ session_id=ctx.session_id,+ cdp_url=ctx.cdp_url,+ )+ ref_map.clear()+ observe_page_key = None+ observe_v2_prepared = None+ pending_ref_map = None+ pending_observe_data = None+async def dispatch(step: ExecuteStep, ref_map: dict[str, dict[str, Any]]) -> dict[str, Any] | None:+ nonlocal observe_page_key, observe_v2_prepared, pending_ref_map, operation_generation+ nonlocal dispatched_observe_data, pending_observe_data+ if _execute_step_mutates(step):+ # Prior steps' acted elements belong to an older snapshot; a stale+ # aria-controls id must not scope the post-batch auto-observe. Cleared+ # before dispatch so this step's own resolution capture survives.+ acted_elements.clear()if step.tool == "observe":- observe_generation["value"] = session_ref_generation(session_id=ctx.session_id, cdp_url=ctx.cdp_url)- return await _dispatch_step(- step,- ref_map,- session_id=session_id,- cdp_url=cdp_url,- page_key=batch_page_key,- on_observe_page=capture_observe_page_key if step.tool == "observe" else None,- on_observe_v2=prepare_observe_v2 if step.tool == "observe" and observe_v2_enabled() else None,- observe_v2_session_id=ctx.session_id,- observe_v2_cdp_url=ctx.cdp_url,- )+ dispatched_observe_data = None+ if observe_v2_enabled():+ # v2 reserves a fresh generation so an older snapshot cannot+ # overwrite a newer publication.+ operation_generation = begin_session_ref_publication(+ session_id=ctx.session_id,+ cdp_url=ctx.cdp_url,+ )+ observe_generation["value"] = operation_generation+ else:+ # Flag-off keeps the pre-v2 guard: read the generation at+ # dispatch so a concurrent invalidation refuses this snapshot.+ observe_generation["value"] = session_ref_generation(+ session_id=ctx.session_id,+ cdp_url=ctx.cdp_url,+ )+ try:+ result = await _dispatch_step(+ step,+ ref_map,+ perception_category=perception_category,+ session_id=session_id,+ cdp_url=cdp_url,+ page_key=batch_page_key,+ document_id=batch_document_id,+ on_observe_page=capture_observe_page_key if step.tool == "observe" else None,+ on_observe_v2=prepare_observe_v2 if step.tool == "observe" and observe_v2_enabled() else None,+ on_resolved_element=capture_resolved_element if _execute_step_mutates(step) else None,+ on_before_action=(+ (lambda: invalidate_before_mutation(ref_map))+ if observe_v2_enabled() and _execute_step_mutates(step)+ else None+ ),+ observe_v2_session_id=ctx.session_id,+ observe_v2_cdp_url=ctx.cdp_url,+ observe_v2_refresh_state=ref_refresh_state,+ observe_v2_generation=operation_generation,+ )+ if step.tool == "observe":+ dispatched_observe_data = result+ return result+ except BaseException:+ if step.tool == "observe":+ observe_page_key = None+ observe_v2_prepared = None+ pending_ref_map = None+ dispatched_observe_data = None+ pending_observe_data = None+ if observe_v2_enabled():+ # Pre-v2, a failed observe left the registry alone (the in-batch+ # frame-error clear in _dispatch_step handles main's one exception).+ clear_session_ref_map(+ session_id=ctx.session_id,+ cdp_url=ctx.cdp_url,+ generation=operation_generation,+ )+ raise- def publish_observe_refs(ref_map: dict[str, dict[str, Any]]) -> bool:- nonlocal batch_page_key+ async def stage_observe_refs(ref_map: dict[str, dict[str, Any]]) -> bool:+ nonlocal batch_document_id, batch_page_key, pending_ref_map, pending_observe_data, ref_refresh_state+ pending_ref_map = None+ pending_observe_data = None+ publication_generation = observe_generation.get("value")if observe_page_key is None:+ if observe_v2_enabled():+ clear_session_ref_map(+ session_id=ctx.session_id,+ cdp_url=ctx.cdp_url,+ generation=publication_generation,+ )return False- accepted = replace_session_ref_map(- ref_map,- session_id=ctx.session_id,- cdp_url=ctx.cdp_url,- generation=observe_generation.get("value"),- page_key=observe_page_key,- )- if accepted:- batch_page_key = observe_page_key- if observe_v2_prepared is not None:- _commit_observe_v2_refs(- get_observe_v2_state(session_id=ctx.session_id, cdp_url=ctx.cdp_url),+ if not observe_v2_enabled():+ # Pre-v2 contract: publish each successful inline observe immediately -+ # no second page lookup, no deferral to batch end - generation-checked+ # against invalidations only (no reservation consumed). The snapshot is+ # bound to the observed page key; resolution fails closed on mismatch.+ accepted = replace_session_ref_map(+ ref_map,+ session_id=ctx.session_id,+ cdp_url=ctx.cdp_url,+ generation=publication_generation,+ page_key=observe_page_key,+ advance_on_commit=False,+ )+ if accepted:+ batch_page_key = observe_page_key+ return accepted+ try:+ publication_page, _ = await get_page(session_id=session_id, cdp_url=cdp_url)+ except asyncio.CancelledError:+ clear_session_ref_map(+ session_id=ctx.session_id,+ cdp_url=ctx.cdp_url,+ generation=publication_generation,+ )+ raise+ except Exception:+ clear_session_ref_map(+ session_id=ctx.session_id,+ cdp_url=ctx.cdp_url,+ generation=publication_generation,+ )+ return False+ try:+ if observe_v2_prepared is None:+ accepted = page_ref_key(publication_page) == observe_page_key+ else:+ accepted = await _observe_v2_snapshot_is_current(+ publication_page,observe_v2_prepared,+ session_id=ctx.session_id,+ cdp_url=ctx.cdp_url,+ generation=publication_generation,)- return accepted+ except BaseException:+ clear_session_ref_map(+ session_id=ctx.session_id,+ cdp_url=ctx.cdp_url,+ generation=publication_generation,+ )+ raise+ if not accepted:+ clear_session_ref_map(+ session_id=ctx.session_id,+ cdp_url=ctx.cdp_url,+ generation=publication_generation,+ )+ return False+ pending_ref_map = ref_map+ pending_observe_data = dispatched_observe_data+ batch_page_key = observe_page_key+ if observe_v2_prepared is not None:+ batch_document_id = observe_v2_prepared["document_id"]+ state = get_observe_v2_state(session_id=ctx.session_id, cdp_url=ctx.cdp_url)+ ref_refresh_state = ObserveV2State(+ page_key=observe_v2_prepared["page_key"],+ document_id=observe_v2_prepared["document_id"],+ params=dict(observe_v2_prepared["params"]),+ refs=dict(observe_v2_prepared["refs"]),+ next_ref=observe_v2_prepared["next_ref"],+ host_budgets=dict(state.host_budgets),+ )+ return Truewith Timer() as timer:- result = await do_execute(- dispatch,- parsed_steps,- stop_on_error=stop_on_error,- on_ref_map_update=publish_observe_refs,+ try:+ result = await do_execute(+ dispatch,+ parsed_steps,+ stop_on_error=stop_on_error,+ on_ref_map_update=stage_observe_refs,+ fail_on_ref_map_rejection=observe_v2_enabled(),+ )+ except asyncio.CancelledError:+ if observe_v2_enabled() and mutation_started:+ invalidate_session_ref_map(session_id=ctx.session_id, cdp_url=ctx.cdp_url)+ elif observe_v2_enabled() and any(+ step.tool == "observe" or _execute_step_mutates(step) for step in parsed_steps+ ):+ clear_session_ref_map(+ session_id=ctx.session_id,+ cdp_url=ctx.cdp_url,+ generation=operation_generation,+ )+ raise+ if observe_v2_enabled() and mutation_started:+ # Close the mutation interval before any further await. A concurrent+ # observe published while an action was in flight may already be stale.+ operation_generation = invalidate_session_ref_map(+ session_id=ctx.session_id,+ cdp_url=ctx.cdp_url,+ )+ observe_page_key = None+ observe_v2_prepared = None+ pending_ref_map = None+ pending_observe_data = None+ successful_mutation = any(+ row.ok and row.step < len(parsed_steps) and _execute_step_mutates(parsed_steps[row.step])+ for row in result.results+ )+ successful_navigation = any(+ row.ok and row.step < len(parsed_steps) and parsed_steps[row.step].tool == "navigate"+ for row in result.results)auto_observe_entry: dict[str, Any] | None = None- if observe_v2_enabled() and result.error_step is None:+ if observe_v2_enabled() and (successful_mutation or result.error_step is None):try:current_page, _ = await get_page(session_id=session_id, cdp_url=cdp_url)+ except asyncio.CancelledError:+ clear_session_ref_map(+ session_id=ctx.session_id,+ cdp_url=ctx.cdp_url,+ generation=operation_generation,+ )+ raiseexcept Exception:- current_page = None- if current_page is not None and page_ref_key(current_page) != batch_page_key:- auto_observe = await do_execute(- dispatch,- [ExecuteStep(tool="observe")],- stop_on_error=True,- on_ref_map_update=publish_observe_refs,+ clear_session_ref_map(+ session_id=ctx.session_id,+ cdp_url=ctx.cdp_url,+ generation=operation_generation,)- if auto_observe.error_step is None and auto_observe.results:- # The receipt stays out of `results`/step counts: those describe the- # caller's submitted steps only, so results[i] pairs with steps[i] and- # steps_total == len(steps) holds for callers that assert it.- sr = auto_observe.results[0]- auto_observe_entry = {"tool": sr.tool, "ok": sr.ok, "wall_ms": sr.wall_ms}- if sr.data:- auto_observe_entry["data"] = sr.data+ current_page = None+ if current_page is not None:+ try:+ if successful_mutation:+ await _settle_after_mutating_batch(current_page)+ current_page_key = page_ref_key(current_page)+ current_document_id = _observe_v2_trusted_document_id(await get_observe_document_id(current_page))+ if (+ successful_mutation+ or current_page_key != batch_page_key+ or current_document_id != batch_document_id+ ):+ refresh_current_document = True+ if not successful_mutation:+ if successful_navigation:+ # skyvern_navigate already closed the navigation interval.+ operation_generation = session_ref_generation(+ session_id=ctx.session_id,+ cdp_url=ctx.cdp_url,+ )+ else:+ published_state = get_observe_v2_state(+ session_id=ctx.session_id,+ cdp_url=ctx.cdp_url,+ )+ refresh_current_document = (+ current_document_id is None+ or published_state.page_key != current_page_key+ or published_state.document_id != current_document_id+ )+ if refresh_current_document:+ operation_generation = invalidate_session_ref_map(+ session_id=ctx.session_id,+ cdp_url=ctx.cdp_url,+ )+ observe_page_key = None+ observe_v2_prepared = None+ pending_ref_map = None+ pending_observe_data = None+ if refresh_current_document:+ perception_category = "automatic"+ # Scope by aria-controls only when the acted elements provably+ # belong to the CURRENT document: click-driven navigation and+ # failed goto replace the document without setting+ # successful_navigation, and an old id colliding on the new+ # document must not narrow its first observe.+ scope_elements = (+ acted_elements+ if current_document_id is not None+ and acted_elements+ and all(+ element.get("_acted_document_id") == current_document_id+ for element in acted_elements+ )+ else []+ )+ selector = (+ await _attached_aria_target_selector(current_page, scope_elements)+ if successful_mutation+ else None+ )+ auto_observe = await do_execute(+ dispatch,+ [ExecuteStep(tool="observe", params={"selector": selector} if selector else {})],+ stop_on_error=True,+ on_ref_map_update=stage_observe_refs,+ fail_on_ref_map_rejection=True,+ )+ if (+ selector is not None+ and auto_observe.error_step is None+ and batch_document_id != current_document_id+ ):+ # The attachment check awaited between document certification+ # and the scoped observe; a navigation in that window means the+ # scoped snapshot certified a different document than the acted+ # one. Discard it (staging replaces the pending publication) and+ # take one honest unscoped snapshot of the current document.+ auto_observe = await do_execute(+ dispatch,+ [ExecuteStep(tool="observe")],+ stop_on_error=True,+ on_ref_map_update=stage_observe_refs,+ fail_on_ref_map_rejection=True,+ )+ if auto_observe.error_step is None and auto_observe.results:+ # The receipt stays out of `results`/step counts: those describe the+ # caller's submitted steps only, so results[i] pairs with steps[i] and+ # steps_total == len(steps) holds for callers that assert it.+ sr = auto_observe.results[0]+ auto_observe_entry = {"tool": sr.tool, "ok": sr.ok, "wall_ms": sr.wall_ms}+ if sr.data:+ auto_observe_entry["data"] = sr.data+ except asyncio.CancelledError:+ clear_session_ref_map(+ session_id=ctx.session_id,+ cdp_url=ctx.cdp_url,+ generation=operation_generation,+ )+ raisetimer.mark("sdk")step_results = []for sr in result.results:entry: dict[str, Any] = {"step": sr.step, "tool": sr.tool, "ok": sr.ok, "wall_ms": sr.wall_ms}- if sr.data:+ if sr.data and (not observe_v2_enabled() or sr.tool != "observe" or sr.data is pending_observe_data):entry["data"] = sr.dataif sr.error:entry["error"] = sr.error@@ -4155,9 +4709,78 @@ async def skyvern_execute(}if auto_observe_entry is not None:data["auto_observe"] = auto_observe_entry- return action_result(+ response = action_result("skyvern_execute",ok=result.error_step is None,data=data,timing_ms=timer.timing_ms,)+ capped_response = truncate_response_bytes(response) if observe_v2_enabled() else response+ if capped_response is not response:+ if pending_ref_map is not None:+ clear_session_ref_map(+ session_id=ctx.session_id,+ cdp_url=ctx.cdp_url,+ generation=observe_generation.get("value"),+ )+ return capped_response+ if pending_ref_map is None:+ return capped_response++ if observe_v2_prepared is None:+ accepted = replace_session_ref_map(+ pending_ref_map,+ session_id=ctx.session_id,+ cdp_url=ctx.cdp_url,+ generation=observe_generation.get("value"),+ page_key=observe_page_key,+ )+ else:+ try:+ publication_page, _ = await get_page(session_id=session_id, cdp_url=cdp_url)+ accepted = await _publish_observe_v2_refs(+ publication_page,+ observe_v2_prepared,+ pending_ref_map,+ session_id=ctx.session_id,+ cdp_url=ctx.cdp_url,+ generation=observe_generation.get("value"),+ )+ except asyncio.CancelledError:+ clear_session_ref_map(+ session_id=ctx.session_id,+ cdp_url=ctx.cdp_url,+ generation=observe_generation.get("value"),+ )+ raise+ except Exception:+ clear_session_ref_map(+ session_id=ctx.session_id,+ cdp_url=ctx.cdp_url,+ generation=observe_generation.get("value"),+ )+ accepted = False+ if not accepted:+ unpublished_results = [dict(entry) for entry in step_results]+ for entry in unpublished_results:+ if entry["tool"] == "observe":+ entry.pop("data", None)+ error = make_error(+ ErrorCode.ACTION_FAILED,+ "Execute completed, but its observe snapshot was superseded before ref publication",+ "Do not repeat completed mutating steps; call skyvern_observe for current refs",+ details={"steps_completed": result.steps_completed, "original_error_step": result.error_step},+ )+ return action_result(+ "skyvern_execute",+ ok=False,+ data={+ "steps_completed": result.steps_completed,+ "steps_total": result.steps_total,+ "results": unpublished_results,+ "error_step": result.error_step,+ },+ timing_ms=timer.timing_ms,+ error=error,+ )+ return capped_responsediff --git a/skyvern/cli/mcp_tools/response.py b/skyvern/cli/mcp_tools/response.pyindex 3cb409441..87cd343ee 100644--- a/skyvern/cli/mcp_tools/response.py+++ b/skyvern/cli/mcp_tools/response.py@@ -31,6 +31,7 @@ LOG = structlog.get_logger(__name__)# the MCP envelope (jsonrpc wrapper, content-block metadata) that the FastMCP# serializer adds on top of our dict.MCP_MAX_RESPONSE_CHARS = 140_000+MCP_MAX_RESPONSE_BYTES = 140_000# When truncation envelope wraps an oversize payload, preserve identifier-like# keys (`*_id`) from the original dict so the caller retains enough context to@@ -51,6 +52,10 @@ _TRUNCATION_HINT = ("Response exceeded the ~150k-char Claude tool-result limit. ""Narrow the query (add filters, reduce page size, request specific fields) or paginate.")+_TRUNCATION_BYTE_HINT = (+ "Response exceeded the 140k-byte aggregate MCP response limit. "+ "Narrow the query (add filters, reduce page size, request specific fields) or paginate."+)P = ParamSpec("P")R = TypeVar("R")@@ -70,6 +75,14 @@ def _response_size(data: Any) -> int:return sys.maxsize+def _response_size_bytes(data: Any) -> int:+ """Return the UTF-8 byte length of the JSON-serialized response."""+ try:+ return len(json.dumps(data, ensure_ascii=False, default=str).encode())+ except (TypeError, ValueError):+ return sys.maxsize++def _bound_error_value(error: Any) -> Any:"""Cap a preserved `error` value so the envelope cannot re-inflate past the response cap."""size = _response_size(error)@@ -128,43 +141,25 @@ def _preserved_screenshot_artifact(value: Any) -> dict[str, Any] | None:return None-def truncate_response(data: Any, *, max_chars: int = MCP_MAX_RESPONSE_CHARS) -> Any:- """Return `data` unchanged if under `max_chars`; otherwise wrap in a truncation envelope.-- Envelope shape (explicit so the model cannot miss truncation):- {- "_truncated": True,- "_original_chars": <serialized size>,- "_max_chars": <limit>,- "_hint": "...",- "ok": <original "ok" if present>,- "error": <original "error" if present, bounded to 2k chars>,- "<key>_id": <preserved identifier field if present>,- ...- }-- Non-dict responses that overflow are wrapped too (rare, but possible if a- tool returns a list or string). From dict payloads, top-level keys- `ok`/`error` plus any `*_id` identifier fields (bounded to 10 keys of up to- 256 chars each) are preserved; the oversized data is otherwise dropped.- Identifier preservation keeps workflow/run/session IDs in the envelope so- the caller can re-query or paginate without losing its place.-- Contract: the returned payload is always `<= max_chars` when serialized.- Pathological inputs (e.g. a `error` field containing a full stack trace or- HTML dump larger than the cap) are defended against by (a) capping `error`- at `_MAX_PRESERVED_ERROR_CHARS` and (b) a final re-measure that falls back- to a metadata-only envelope if the assembled payload is still over.- """- size = _response_size(data)- if size <= max_chars:+def _truncate_response_to_limit(+ data: Any,+ *,+ size_fn: Callable[[Any], int],+ max_size: int,+ unit: str,+ hint: str,+) -> Any:+ size = size_fn(data)+ if size <= max_size:return data+ original_key = f"_original_{unit}"+ max_key = f"_max_{unit}"envelope: dict[str, Any] = {"_truncated": True,- "_original_chars": size,- "_max_chars": max_chars,- "_hint": _TRUNCATION_HINT,+ original_key: size,+ max_key: max_size,+ "_hint": hint,}if isinstance(data, dict):if "ok" in data:@@ -197,21 +192,39 @@ def truncate_response(data: Any, *, max_chars: int = MCP_MAX_RESPONSE_CHARS) ->"hint": "Use the saved screenshot artifact path, or retry with inline=false.",}- # Final safety net: if the preserved fields combined still blow past the- # cap (extremely unlikely given the per-field caps above, but possible if- # a caller passes a tiny `max_chars`), drop them and emit a metadata-only- # envelope. Guarantees the module's "always under max_chars" contract.- if _response_size(envelope) > max_chars:+ if size_fn(envelope) > max_size:return {"_truncated": True,- "_original_chars": size,- "_max_chars": max_chars,- "_hint": _TRUNCATION_HINT,+ original_key: size,+ max_key: max_size,+ "_hint": hint,"_envelope_rewrapped": True,}return envelope+def truncate_response(data: Any, *, max_chars: int = MCP_MAX_RESPONSE_CHARS) -> Any:+ """Cap a response by JSON-serialized characters, preserving the existing envelope contract."""+ return _truncate_response_to_limit(+ data,+ size_fn=_response_size,+ max_size=max_chars,+ unit="chars",+ hint=_TRUNCATION_HINT,+ )+++def truncate_response_bytes(data: Any, *, max_bytes: int = MCP_MAX_RESPONSE_BYTES) -> Any:+ """Cap a response by aggregate UTF-8 JSON bytes."""+ return _truncate_response_to_limit(+ data,+ size_fn=_response_size_bytes,+ max_size=max_bytes,+ unit="bytes",+ hint=_TRUNCATION_BYTE_HINT,+ )++def size_capped(fn: Callable[P, Awaitable[R]]) -> Callable[P, Awaitable[Any]]:"""Decorator: enforce `MCP_MAX_RESPONSE_CHARS` on a tool's return value.@@ -244,7 +257,9 @@ def size_capped(fn: Callable[P, Awaitable[R]]) -> Callable[P, Awaitable[Any]]:__all__ = [+ "MCP_MAX_RESPONSE_BYTES","MCP_MAX_RESPONSE_CHARS","size_capped","truncate_response",+ "truncate_response_bytes",]diff --git a/skyvern/cli/mcp_tools/telemetry.py b/skyvern/cli/mcp_tools/telemetry.pyindex da2941a61..453d7c136 100644--- a/skyvern/cli/mcp_tools/telemetry.py+++ b/skyvern/cli/mcp_tools/telemetry.py@@ -1,5 +1,6 @@from __future__ import annotations+import asyncioimport timefrom collections.abc import Callablefrom contextlib import suppress@@ -11,6 +12,7 @@ from fastmcp.server.dependencies import get_http_requestfrom fastmcp.server.middleware import CallNext, Middleware, MiddlewareContextfrom skyvern import analytics+from skyvern.cli.core.perception_telemetry import MCPPerceptionCounters, perception_counters_scopeLOG = structlog.get_logger(__name__)@@ -142,9 +144,10 @@ def _capture_mcp_event(ok: bool,tool_name: str | None = None,prompt_name: str | None = None,- error: Exception | None = None,+ error: BaseException | None = None,duration_ms: float | None = None,response_bytes: int | None = None,+ perception_counters: MCPPerceptionCounters | None = None,) -> None:request = _resolve_http_request()organization_id = _resolve_organization_id(request)@@ -179,6 +182,8 @@ def _capture_mcp_event(data["duration_ms"] = duration_msif response_bytes is not None:data["response_bytes"] = response_bytes+ if perception_counters is not None:+ data.update(perception_counters.event_fields())analytics.capture(event_name,@@ -249,36 +254,39 @@ class MCPTelemetryMiddleware(Middleware):) -> Any:tool_name = getattr(context.message, "name", None)start = time.perf_counter()- try:- result = await call_next(context)- except Exception as exc:+ with perception_counters_scope() as counters:+ try:+ result = await call_next(context)+ except (Exception, asyncio.CancelledError) as exc:+ duration_ms = (time.perf_counter() - start) * 1000+ # Failed and cancelled calls produce no MCP content, so response_bytes is only emitted for results.+ with suppress(Exception, asyncio.CancelledError):+ _capture_mcp_event(+ "mcp_tool_call",+ operation="tools/call",+ context=context,+ ok=False,+ tool_name=tool_name,+ error=exc,+ duration_ms=duration_ms,+ perception_counters=counters,+ )+ raise+duration_ms = (time.perf_counter() - start) * 1000- # Exceptions do not produce MCP content, so response_bytes is only emitted for returned results.+ response_bytes = sum(_content_text_bytes(content) for content in (getattr(result, "content", None) or []))with suppress(Exception):_capture_mcp_event("mcp_tool_call",operation="tools/call",context=context,- ok=False,+ ok=_resolve_tool_call_ok(result),tool_name=tool_name,- error=exc,duration_ms=duration_ms,+ response_bytes=response_bytes,+ perception_counters=counters,)- raise-- duration_ms = (time.perf_counter() - start) * 1000- response_bytes = sum(_content_text_bytes(content) for content in (getattr(result, "content", None) or []))- with suppress(Exception):- _capture_mcp_event(- "mcp_tool_call",- operation="tools/call",- context=context,- ok=_resolve_tool_call_ok(result),- tool_name=tool_name,- duration_ms=duration_ms,- response_bytes=response_bytes,- )- return result+ return resultasync def on_list_tools(self,diff --git a/tests/unit/test_mcp_browser_tools.py b/tests/unit/test_mcp_browser_tools.pyindex 018e0d8e9..b6c46ea1a 100644--- a/tests/unit/test_mcp_browser_tools.py+++ b/tests/unit/test_mcp_browser_tools.py@@ -2785,14 +2785,22 @@ async def test_write_grid_rejects_oversize_row_before_session(monkeypatch: pytes),],)-async def test_navigating_paired_tools_clear_the_ref_map_around_navigation(+async def test_navigating_paired_tools_invalidate_refs_around_navigation(monkeypatch: pytest.MonkeyPatch, tool_name: str, extra_kwargs: dict[str, object]) -> None:- """Parity with skyvern_navigate: a prior observe's refs must not survive a navigation.+ """Parity with skyvern_navigate: prior refs cannot survive navigation.- Cleared before AND after, because a failed goto can still partially replace the document.+ Invalidate before AND after, because a failed goto can partially replace the+ document and a concurrent observe can publish while navigation is in flight."""_sdk_equivalent_page(monkeypatch)+ events: list[str] = []++ async def navigate(*_args: object, **_kwargs: object) -> NavigateResult:+ events.append("navigate")+ return NavigateResult(url="https://example.test", title="Example", load_state="load")++ monkeypatch.setattr(mcp_browser, "do_navigate", AsyncMock(side_effect=navigate))monkeypatch.setattr(mcp_browser, "do_screenshot", AsyncMock(return_value=SimpleNamespace(data=b"png")))monkeypatch.setattr(mcp_browser, "do_extract", AsyncMock(return_value=SimpleNamespace(extracted={"value": 1})))monkeypatch.setattr(@@ -2800,13 +2808,13 @@ async def test_navigating_paired_tools_clear_the_ref_map_around_navigation("save_artifact",Mock(return_value=Artifact(kind="screenshot", path="/tmp/shot.png", mime="image/png", bytes=3)),)- clear = Mock()- monkeypatch.setattr(mcp_browser, "clear_session_ref_map", clear)+ invalidate = Mock(side_effect=lambda **_kwargs: events.append("invalidate"))+ monkeypatch.setattr(mcp_browser, "invalidate_session_ref_map", invalidate)result = await getattr(mcp_browser, tool_name)(url="https://example.test", **extra_kwargs)assert result["ok"] is True, result- assert clear.call_count == 2+ assert events == ["invalidate", "navigate", "invalidate"]@pytest.mark.asyncio@@ -2873,18 +2881,24 @@ async def test_navigating_paired_tools_surface_the_degraded_load_state(@pytest.mark.asyncio-async def test_navigating_paired_tools_clear_the_ref_map_even_when_navigation_fails(+async def test_navigating_paired_tools_invalidate_refs_even_when_navigation_fails(monkeypatch: pytest.MonkeyPatch,) -> None:_sdk_equivalent_page(monkeypatch)- monkeypatch.setattr(mcp_browser, "do_navigate", AsyncMock(side_effect=RuntimeError("net::ERR_ABORTED")))- clear = Mock()- monkeypatch.setattr(mcp_browser, "clear_session_ref_map", clear)+ events: list[str] = []++ async def fail_navigation(*_args: object, **_kwargs: object) -> NavigateResult:+ events.append("navigate")+ raise RuntimeError("net::ERR_ABORTED")++ monkeypatch.setattr(mcp_browser, "do_navigate", AsyncMock(side_effect=fail_navigation))+ invalidate = Mock(side_effect=lambda **_kwargs: events.append("invalidate"))+ monkeypatch.setattr(mcp_browser, "invalidate_session_ref_map", invalidate)result = await mcp_browser.skyvern_navigate_and_screenshot(url="https://example.test")assert result["ok"] is False- assert clear.call_count == 2+ assert events == ["invalidate", "navigate", "invalidate"]@pytest.mark.asynciodiff --git a/tests/unit/test_mcp_observe_execute.py b/tests/unit/test_mcp_observe_execute.pyindex 980b46618..c8efa829d 100644--- a/tests/unit/test_mcp_observe_execute.py+++ b/tests/unit/test_mcp_observe_execute.py@@ -461,6 +461,20 @@ class TestDoObserve:assert country["options"] == ["US", "UK", "CA"]assert "value" not in country+ @pytest.mark.asyncio+ async def test_document_reload_during_observe_refuses_durability(self, monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setenv("SKYVERN_MCP_OBSERVE_V2", "1")+ page = _make_page()+ document_ids = AsyncMock(side_effect=["cdp:doc-a", "cdp:doc-b"])+ monkeypatch.setattr(browser_ops, "get_observe_document_id", document_ids)++ result = await do_observe(page)++ # The marker bracket refuses durability instead of failing the observe: elements stay+ # usable via selectors, but no document identity is certified for durable refs.+ assert result.document_id is None+ assert document_ids.await_count == 2+@pytest.mark.asyncioasync def test_dom_interactables_are_merged_with_selectors(self) -> None:page = _make_page(_make_a11y_tree(children=[{"role": "textbox", "name": "City", "value": ""}]))@@ -1022,13 +1036,106 @@ class TestSkyvernObserveMCP:)@pytest.mark.asyncio- async def test_observe_no_browser(self, monkeypatch: pytest.MonkeyPatch) -> None:+ async def test_observe_no_browser_clears_existing_refs(self, monkeypatch: pytest.MonkeyPatch) -> None:from skyvern.cli.mcp_tools._session import BrowserNotAvailableError+ monkeypatch.setenv("SKYVERN_MCP_OBSERVE_V2", "1")+ ctx = BrowserContext(mode="local")+ state = make_session_state(context=ctx)+ state._observed_refs = {"e0": {"ref": "e0", "selector": "#stale"}}+ state._observe_v2_state.refs = dict(state._observed_refs)monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(side_effect=BrowserNotAvailableError("no browser")))- result = await mcp_browser.skyvern_observe()+ async with scoped_session(state):+ result = await mcp_browser.skyvern_observe()+assert result["ok"] is False+ assert state._observed_refs == {}+ assert state._observe_v2_state.refs == {}++ @pytest.mark.asyncio+ async def test_cancelled_observe_page_lookup_clears_existing_refs(self, monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setenv("SKYVERN_MCP_OBSERVE_V2", "1")+ ctx = BrowserContext(mode="local")+ state = make_session_state(context=ctx)+ state._observed_refs = {"e0": {"ref": "e0", "selector": "#stale"}}+ state._observe_v2_state.refs = dict(state._observed_refs)+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(side_effect=asyncio.CancelledError))++ async with scoped_session(state):+ with pytest.raises(asyncio.CancelledError):+ await mcp_browser.skyvern_observe()++ assert state._observed_refs == {}+ assert state._observe_v2_state.refs == {}++ @pytest.mark.asyncio+ async def test_generic_observe_failure_clears_existing_refs(self, monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setenv("SKYVERN_MCP_OBSERVE_V2", "1")+ page = _make_page()+ ctx = BrowserContext(mode="local")+ state = make_session_state(context=ctx)+ state._observed_refs = {"e0": {"ref": "e0", "selector": "#stale"}}+ state._observe_v2_state.refs = dict(state._observed_refs)+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))+ monkeypatch.setattr(mcp_browser, "do_observe", AsyncMock(side_effect=RuntimeError("snapshot failed")))++ async with scoped_session(state):+ result = await mcp_browser.skyvern_observe()++ assert result["ok"] is False+ assert state._observed_refs == {}+ assert state._observe_v2_state.refs == {}++ @pytest.mark.asyncio+ async def test_cancelled_observe_snapshot_clears_existing_refs(self, monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setenv("SKYVERN_MCP_OBSERVE_V2", "1")+ page = _make_page()+ ctx = BrowserContext(mode="local")+ state = make_session_state(context=ctx)+ state._observed_refs = {"e0": {"ref": "e0", "selector": "#stale"}}+ state._observe_v2_state.refs = dict(state._observed_refs)+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))+ monkeypatch.setattr(mcp_browser, "do_observe", AsyncMock(side_effect=asyncio.CancelledError))++ async with scoped_session(state):+ with pytest.raises(asyncio.CancelledError):+ await mcp_browser.skyvern_observe()++ assert state._observed_refs == {}+ assert state._observe_v2_state.refs == {}++ @pytest.mark.asyncio+ async def test_cancelled_observe_publication_clears_existing_refs(+ self,+ monkeypatch: pytest.MonkeyPatch,+ ) -> None:+ monkeypatch.setenv("SKYVERN_MCP_OBSERVE_V2", "1")+ page = _make_page()+ ctx = BrowserContext(mode="local")+ state = make_session_state(context=ctx)+ state._observed_refs = {"e0": {"ref": "e0", "selector": "#stale"}}+ state._observe_v2_state.refs = dict(state._observed_refs)+ observe_result = ObserveResult(+ url=page.url,+ title="Login Page",+ elements=[],+ element_count=0,+ total_on_page=0,+ document_id="doc-1",+ )+ publish = AsyncMock(side_effect=asyncio.CancelledError)+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))+ monkeypatch.setattr(mcp_browser, "do_observe", AsyncMock(return_value=observe_result))+ monkeypatch.setattr(mcp_browser, "_publish_observe_v2_refs", publish)++ async with scoped_session(state):+ with pytest.raises(asyncio.CancelledError):+ await mcp_browser.skyvern_observe()++ publish.assert_awaited_once()+ assert state._observed_refs == {}+ assert state._observe_v2_state.refs == {}@pytest.mark.asyncioasync def test_non_evaluable_working_frame_returns_structured_error(@@ -1545,6 +1652,41 @@ class TestSkyvernExecuteMCP:assert committed is Falseassert session_manager.get_session_ref("e0", session_id=ctx.session_id) is None+ @pytest.mark.asyncio+ async def test_newer_publication_reservation_rejects_older_commit_and_clear(self) -> None:+ ctx = BrowserContext(mode="cloud_session", session_id="pbs_publication_cas")+ page_key = (1, None, "https://example.com", None)+ older = {"ref": "e0", "selector": "#older"}+ newer = {"ref": "e1", "selector": "#newer"}++ async with scoped_session(make_session_state(context=ctx)):+ older_generation = session_manager.begin_session_ref_publication(session_id=ctx.session_id)+ newer_generation = session_manager.begin_session_ref_publication(session_id=ctx.session_id)++ assert session_manager.replace_session_ref_map(+ {"e1": newer},+ session_id=ctx.session_id,+ generation=newer_generation,+ page_key=page_key,+ )+ committed_generation = session_manager.session_ref_generation(session_id=ctx.session_id)+ assert committed_generation > newer_generation+ assert not session_manager.clear_session_ref_map(+ session_id=ctx.session_id,+ generation=newer_generation,+ )+ assert not session_manager.replace_session_ref_map(+ {"e0": older},+ session_id=ctx.session_id,+ generation=older_generation,+ page_key=page_key,+ )+ assert not session_manager.clear_session_ref_map(+ session_id=ctx.session_id,+ generation=older_generation,+ )+ assert session_manager.get_session_ref("e1", session_id=ctx.session_id, page_key=page_key) == newer+@pytest.mark.asyncioasync def test_rejected_batch_observe_does_not_install_local_refs(self) -> None:"""If session publication rejects an observe snapshot, the batch must not act on it either."""@@ -1564,7 +1706,55 @@ class TestSkyvernExecuteMCP:)assert result.error_step == 1+ assert result.results[0].ok is Trueassert "Unknown ref 'e0'" in str(result.results[1].error)+ assert len(result.results) == 2++ @pytest.mark.asyncio+ async def test_strict_rejected_batch_observe_fails_before_ref_step(self) -> None:+ async def dispatch(step: ExecuteStep, ref_map: dict[str, dict[str, Any]]) -> dict[str, Any] | None:+ if step.tool == "observe":+ return {"elements": [{"ref": "e0", "role": "button", "name": "Sign In"}], "element_count": 1}+ if ref := step.params.get("ref"):+ if ref not in ref_map:+ raise ValueError(f"Unknown ref '{ref}' — call observe first or check ref exists")+ return None++ result = await do_execute(+ dispatch,+ [ExecuteStep(tool="observe"), ExecuteStep(tool="click", params={"ref": "e0"})],+ on_ref_map_update=lambda ref_map: False,+ fail_on_ref_map_rejection=True,+ )++ assert result.error_step == 0+ assert "observe_snapshot_superseded" in str(result.results[0].error)+ assert len(result.results) == 1++ @pytest.mark.asyncio+ async def test_batch_observe_awaits_async_ref_publication(self) -> None:+ async def dispatch(step: ExecuteStep, ref_map: dict[str, dict[str, Any]]) -> dict[str, Any] | None:+ if step.tool == "observe":+ return {"elements": [{"ref": "e0", "role": "button", "name": "Sign In"}], "element_count": 1}+ if step.params["ref"] not in ref_map:+ raise ValueError("missing accepted ref")+ return None++ publication_started = False++ async def publish(_ref_map: dict[str, dict[str, Any]]) -> bool:+ nonlocal publication_started+ publication_started = True+ return True++ result = await do_execute(+ dispatch,+ [ExecuteStep(tool="observe"), ExecuteStep(tool="click", params={"ref": "e0"})],+ on_ref_map_update=publish,+ )++ assert publication_started is True+ assert result.error_step is None@pytest.mark.asyncioasync def test_observe_started_during_navigation_is_discarded(self, monkeypatch: pytest.MonkeyPatch) -> None:@@ -1595,15 +1785,19 @@ class TestSkyvernExecuteMCP:@pytest.mark.asyncioasync def test_popup_mid_batch_invalidates_batch_refs(self, monkeypatch: pytest.MonkeyPatch) -> None:- """A page change between an in-batch observe and a ref step must not resolve stale refs."""+ """A page change before ref publication rejects the unpublished observe snapshot."""+ monkeypatch.delenv("SKYVERN_MCP_OBSERVE_V2", raising=False)page_a = _make_page()page_b = _make_page()ctx = BrowserContext(mode="cloud_session", session_id="pbs_popup_batch")- monkeypatch.setattr(- mcp_browser,- "get_page",- AsyncMock(side_effect=[(page_a, ctx), (page_a, ctx), (page_b, ctx)]),- )+ calls = 0++ async def get_page(**_kwargs: Any) -> tuple[Any, BrowserContext]:+ nonlocal calls+ calls += 1+ return (page_a, ctx) if calls <= 2 else (page_b, ctx)++ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(side_effect=get_page))async with scoped_session(make_session_state(context=ctx)):result = await mcp_browser.skyvern_execute(@@ -1614,6 +1808,9 @@ class TestSkyvernExecuteMCP:session_id=ctx.session_id,)+ assert result["ok"] is False+ assert len(result["data"]["results"]) == 2+ assert result["data"]["results"][0]["ok"] is Trueassert result["data"]["results"][1]["error"] == ("Unknown ref 'e0' — call observe first or check ref exists")@pytest.mark.asyncio@@ -1626,11 +1823,14 @@ class TestSkyvernExecuteMCP:page_b = _make_page()page_b.url = "https://example.com/popup"ctx = BrowserContext(mode="cloud_session", session_id="pbs_popup_before_observe")- monkeypatch.setattr(- mcp_browser,- "get_page",- AsyncMock(side_effect=[(page_a, ctx), (page_b, ctx), (page_b, ctx)]),- )+ calls = 0++ async def get_page(**_kwargs: Any) -> tuple[Any, BrowserContext]:+ nonlocal calls+ calls += 1+ return (page_a, ctx) if calls == 1 else (page_b, ctx)++ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(side_effect=get_page))monkeypatch.setattr(mcp_browser, "skyvern_evaluate", AsyncMock(return_value={"ok": True, "data": None}))click = AsyncMock(return_value={"ok": True, "data": None})monkeypatch.setattr(mcp_browser, "skyvern_click", click)diff --git a/tests/unit/test_mcp_observe_v2.py b/tests/unit/test_mcp_observe_v2.pyindex 868250613..f9b3d3325 100644--- a/tests/unit/test_mcp_observe_v2.py+++ b/tests/unit/test_mcp_observe_v2.py@@ -2,20 +2,25 @@from __future__ import annotations+import asyncioimport hashlibimport jsonfrom types import SimpleNamespacefrom typing import Any-from unittest.mock import AsyncMock+from unittest.mock import AsyncMock, callimport pytest+from fastmcp.server.middleware import MiddlewareContext+from skyvern import analyticsfrom skyvern.cli.core import browser_ops, session_managerfrom skyvern.cli.core.browser_ops import ObservedElement, ObserveFrameError, ObserveResultfrom skyvern.cli.core.result import BrowserContextfrom skyvern.cli.core.session_manager import scoped_sessionfrom skyvern.cli.mcp_tools import browser as mcp_browserfrom skyvern.cli.mcp_tools import mcp+from skyvern.cli.mcp_tools.response import MCP_MAX_RESPONSE_BYTES+from skyvern.cli.mcp_tools.telemetry import MCPTelemetryMiddlewarefrom tests.unit._mcp_browser_fakes import make_session_state_FLAG = "SKYVERN_MCP_OBSERVE_V2"@@ -42,7 +47,7 @@ def _tool_payload(tool: Any) -> dict[str, Any]:def _page(url: str = "https://fixture.test/one", loader_id: str | None = "doc-1") -> SimpleNamespace:"""Fake page whose CDP document marker is driven by `_loader_id`: mutate it to simulate asame-URL document replacement, or pass loader_id=None for the page-evaluated fallback path."""- raw = SimpleNamespace()+ raw = SimpleNamespace(wait_for_timeout=AsyncMock())page = SimpleNamespace(page=raw,_working_frame=None,@@ -73,6 +78,8 @@ def _result(total: int = 1,page_text: str | None = None,document_id: str = "cdp:doc-1",+ aria_controls: str | None = None,+ aria_owns: str | None = None,) -> ObserveResult:return ObserveResult(url=url,@@ -84,6 +91,8 @@ def _result(name="Continue",tag="button",selector=selector,+ aria_controls=aria_controls,+ aria_owns=aria_owns,)],element_count=1,@@ -94,6 +103,36 @@ def _result()+ ("tool", "expected"),+ [+ ("navigate", False),+ ("click", True),+ ("type", True),+ ("press_key", True),+ ("select_option", True),+ ("hover", True),+ ("scroll", True),+ ("wait", False),+ ("wait_for_either_state", False),+ ("observe", False),+ ("screenshot", False),+ # Arbitrary JS is unclassifiable statically and a miss fails open, so+ # evaluate is always treated as mutating - reads included.+ ("evaluate", True),+ ],+)+def test_execute_mutation_classifier_covers_full_allowlist(tool: str, expected: bool) -> None:+ assert mcp_browser._execute_step_mutates(browser_ops.ExecuteStep(tool=tool)) is expected+++def test_evaluate_mutates_even_for_pure_reads() -> None:+ """A read-only expression still classifies as mutating: bracket-notation writes,+ aliased methods, and eval() make any static classifier fail open."""+ step = browser_ops.ExecuteStep(tool="evaluate", params={"expression": "document.title"})+ assert mcp_browser._execute_step_mutates(step) is True++@pytest.mark.asyncioasync def test_observe_v2_defaults_off_and_preserves_pinned_tool_manifests(monkeypatch: pytest.MonkeyPatch,@@ -111,6 +150,36 @@ async def test_observe_v2_defaults_off_and_preserves_pinned_tool_manifests(assert hashlib.sha256(_canonical(_tool_payload(tools_on[name]))).hexdigest() == expected_hash[email protected]("flag_value", [None, "0", "false"])+def test_flag_off_element_serialization_ignores_aria_target_metadata(+ monkeypatch: pytest.MonkeyPatch,+ flag_value: str | None,+) -> None:+ if flag_value is None:+ monkeypatch.delenv(_FLAG, raising=False)+ else:+ monkeypatch.setenv(_FLAG, flag_value)+ baseline = ObservedElement(ref="e0", role="textbox", name="City", tag="input", selector="#city")+ with_targets = ObservedElement(+ ref="e0",+ role="textbox",+ name="City",+ tag="input",+ selector="#city",+ aria_controls="suggestions",+ aria_owns="portal",+ )++ assert browser_ops.serialize_elements([with_targets]) == browser_ops.serialize_elements([baseline])++ monkeypatch.setenv(_FLAG, "1")+ assert browser_ops.serialize_elements([with_targets])[0] == {+ **browser_ops.serialize_elements([baseline])[0],+ "aria_controls": "suggestions",+ "aria_owns": "portal",+ }++@pytest.mark.asyncio@pytest.mark.parametrize("flag_value", [None, "0", "false"])async def test_flag_off_observe_response_is_byte_stable(@@ -142,11 +211,21 @@ async def test_flag_off_observe_response_is_byte_stable(page.evaluate = AsyncMock(return_value=[])ctx = BrowserContext(mode="local")monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))+ events: list[dict[str, Any]] = []+ monkeypatch.setattr(analytics, "capture", lambda _event, *, data, **_kwargs: events.append(data))++ context = MiddlewareContext(message=SimpleNamespace(name="skyvern_observe"), fastmcp_context=None)- result = await mcp_browser.skyvern_observe()+ async def call_next(_context: MiddlewareContext[object]) -> object:+ return await mcp_browser.skyvern_observe()++ result = await MCPTelemetryMiddleware().on_call_tool(context, call_next)result["timing_ms"] = {key: 0 for key in result["timing_ms"]}assert hashlib.sha256(_canonical(result)).hexdigest() == _OBSERVE_RESPONSE_HASH+ assert events[0]["perception_snapshots"] == 1+ assert events[0]["model_visible_observe_results"] == 1+ assert events[0]["evaluate_page_scans"] == 1assert set(result["data"]) == {"url", "title", "elements", "element_count", "total_on_page", "hint"}@@ -174,6 +253,88 @@ async def test_flag_off_execute_resolves_legacy_ref_and_clicks(assert click.await_args.kwargs["selector"] == "#legacy"+async def test_flag_off_overlapping_observes_preserve_legacy_completion_behavior(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ monkeypatch.delenv(_FLAG, raising=False)+ page = _page()+ ctx = BrowserContext(mode="local")+ state = make_session_state(context=ctx)+ slow_started = asyncio.Event()+ finish_slow = asyncio.Event()+ call_count = 0++ async def observe(_page: Any, **_kwargs: Any) -> ObserveResult:+ nonlocal call_count+ call_count += 1+ if call_count == 1:+ slow_started.set()+ await finish_slow.wait()+ return _result(page.url, "#slow")+ return _result(page.url, "#fast")++ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))+ monkeypatch.setattr(mcp_browser, "do_observe", AsyncMock(side_effect=observe))++ async with scoped_session(state):+ slow_task = asyncio.create_task(mcp_browser.skyvern_observe())+ await slow_started.wait()+ fast_result = await mcp_browser.skyvern_observe()+ finish_slow.set()+ slow_result = await slow_task+ published = session_manager.get_session_ref("e0", page_key=session_manager.page_ref_key(page))++ assert fast_result["ok"] is True+ assert slow_result["ok"] is True+ assert published is not None+ assert published["selector"] == "#slow"+++async def test_flag_off_execute_overlapping_observe_does_not_fail_after_mutation(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ monkeypatch.delenv(_FLAG, raising=False)+ page = _page()+ ctx = BrowserContext(mode="local")+ state = make_session_state(context=ctx)+ batch_observe_started = asyncio.Event()+ finish_batch_observe = asyncio.Event()++ async def batch_observe(_page: Any, **_kwargs: Any) -> ObserveResult:+ batch_observe_started.set()+ await finish_batch_observe.wait()+ return _result(page.url, "#batch")++ click = AsyncMock(return_value={"ok": True, "data": None})+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))+ monkeypatch.setattr(mcp_browser, "do_observe", AsyncMock(return_value=_result(page.url, "#standalone")))+ monkeypatch.setattr(browser_ops, "do_observe", AsyncMock(side_effect=batch_observe))+ monkeypatch.setattr(mcp_browser, "skyvern_click", click)++ async with scoped_session(state):+ execute_task = asyncio.create_task(+ mcp_browser.skyvern_execute(+ steps=[+ {"tool": "click", "params": {"selector": "#save"}},+ {"tool": "observe", "params": {}},+ ]+ )+ )+ await batch_observe_started.wait()+ standalone_result = await mcp_browser.skyvern_observe()+ finish_batch_observe.set()+ execute_result = await execute_task+ published = session_manager.get_session_ref("e0", page_key=session_manager.page_ref_key(page))++ assert standalone_result["ok"] is True+ assert execute_result["ok"] is True+ click.assert_awaited_once()+ assert published is not None+ assert published["selector"] == "#batch"++@pytest.mark.asyncioasync def test_flag_off_ref_fails_closed_after_flag_is_enabled(monkeypatch: pytest.MonkeyPatch,@@ -367,7 +528,7 @@ async def test_flag_on_same_document_rerender_refreshes_target_from_current_evidexecuted = await mcp_browser.skyvern_execute(steps=[{"tool": "click", "params": {"ref": ref}}])assert executed["ok"] is True- assert refresh.await_count == 1+ assert refresh.await_count == 2assert click.await_args.kwargs["selector"] == "#after"@@ -434,10 +595,75 @@ async def test_execute_refreshes_populated_legacy_ref_after_same_document_selectexecuted = await mcp_browser.skyvern_execute(steps=[{"tool": "click", "params": {"ref": ref}}])assert executed["ok"] is True- refresh.assert_awaited_once()+ assert refresh.await_count == 2assert click.await_args.kwargs["selector"] == "#after"+async def test_observe_paths_reconcile_in_structured_telemetry(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ monkeypatch.setenv(_FLAG, "1")+ page = _page()+ ctx = BrowserContext(mode="local", session_id="pbs_test", cdp_url="ws://test")+ state = make_session_state(context=ctx)+ observe_result = _result(page.url, "#continue")+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))+ monkeypatch.setattr(mcp_browser, "do_observe", AsyncMock(return_value=observe_result))+ monkeypatch.setattr(browser_ops, "do_observe", AsyncMock(return_value=observe_result))++ async def navigate(*_args: Any, **_kwargs: Any) -> dict[str, Any]:+ page.url = "https://fixture.test/two"+ return {"ok": True, "data": {"url": page.url}}++ monkeypatch.setattr(mcp_browser, "skyvern_navigate", navigate)+ events: list[dict[str, Any]] = []+ monkeypatch.setattr(+ analytics,+ "capture",+ lambda _event, *, data, **_kwargs: events.append(data),+ )+ context = MiddlewareContext(message=SimpleNamespace(name="counter_scenario"), fastmcp_context=None)++ async def call_next(_context: MiddlewareContext[object]) -> object:+ explicit = await mcp_browser.skyvern_observe()+ ref = explicit["data"]["elements"][0]["ref"]+ state._observed_refs = {}+ handled, refreshed = await mcp_browser._refresh_observe_v2_ref(+ ref,+ page,+ session_id=ctx.session_id,+ cdp_url=ctx.cdp_url,+ )+ assert handled is True+ assert refreshed is not None+ inline = await mcp_browser.skyvern_execute(steps=[{"tool": "observe", "params": {}}])+ assert inline["ok"] is True+ assert inline["data"]["results"][0]["tool"] == "observe"+ executed = await mcp_browser.skyvern_execute(+ steps=[{"tool": "navigate", "params": {"url": "https://fixture.test/two"}}],+ )+ assert executed["ok"] is True+ return SimpleNamespace(is_error=False, data={"ok": True}, content=[])++ async with scoped_session(state):+ await MCPTelemetryMiddleware().on_call_tool(context, call_next)++ payload = events[0]+ assert payload["top_level_mcp_calls"] == 1+ assert payload["perception_snapshots"] == 4+ assert payload["model_visible_observe_results"] == 2+ assert payload["automatic_observe_snapshots"] == 1+ assert payload["stale_ref_refresh_snapshots"] == 1+ assert payload["failed_perception_probes"] == 0+ assert (+ payload["model_visible_observe_results"]+ + payload["automatic_observe_snapshots"]+ + payload["stale_ref_refresh_snapshots"]+ == payload["perception_snapshots"]+ )++@pytest.mark.asyncioasync def test_flag_on_same_url_new_document_revision_fails_closed(monkeypatch: pytest.MonkeyPatch,@@ -507,6 +733,7 @@ async def test_document_identity_source_degradation_fails_closed(cdp = SimpleNamespace(send=AsyncMock(side_effect=[+ {"frameTree": {"frame": {"loaderId": "doc-1"}}},{"frameTree": {"frame": {"loaderId": "doc-1"}}},RuntimeError("CDP unavailable"),]@@ -607,14 +834,14 @@ async def test_targeted_refresh_replays_scope_and_preserves_unrelated_session_reassert click.await_args.kwargs["selector"] == "#first-new"# A scoped observe replays the caller's own budget: the host-wide budget only widens the# keyhole for unscoped observes, so a "refresh one ref" call cannot balloon to the host cap.- refresh.assert_awaited_once_with(+ assert refresh.await_args_list[0] == call(page,selector="#checkout",interactive_only=False,max_elements=7,include_values=True,)- assert state._observed_refs == unrelated_snapshot+ assert state._observed_refs != unrelated_snapshot@pytest.mark.asyncio@@ -687,6 +914,7 @@ async def test_real_path_reobserves_per_ref_after_mutating_step(side_effect=[two_elements("#first-new", "#second-new"),two_elements("#first-newest", "#second-newest"),+ two_elements("#first-final", "#second-final"),])monkeypatch.setattr(browser_ops, "do_observe", refresh)@@ -704,7 +932,7 @@ async def test_real_path_reobserves_per_ref_after_mutating_step()assert executed["ok"] is True- assert refresh.await_count == 2+ assert refresh.await_count == 3assert [call.kwargs["selector"] for call in click.await_args_list] == ["#first-new", "#second-newest"]@@ -886,7 +1114,7 @@ async def test_same_url_document_change_between_steps_invalidates_next_ref(assert executed["ok"] is Falseassert "Unknown ref" in executed["data"]["results"][1]["error"]- assert refresh.await_count == 1+ assert refresh.await_count == 2assert click_mock.await_count == 1@@ -984,6 +1212,10 @@ async def test_auto_observe_probe_browser_loss_is_non_fatal_to_successful_user_spage = _page()ctx = BrowserContext(mode="local")state = make_session_state(context=ctx)+ state._observed_refs = {"e0": {"ref": "e0", "selector": "#closed-document"}}+ state._observe_v2_state.refs = dict(state._observed_refs)+ state._observe_v2_state.page_key = session_manager.page_ref_key(page)+ state._observe_v2_state.document_id = "page:doc-1"calls = 0async def get_page(**kwargs: Any) -> tuple[Any, Any]:@@ -1005,6 +1237,8 @@ async def test_auto_observe_probe_browser_loss_is_non_fatal_to_successful_user_sassert executed["data"]["steps_total"] == 1assert executed["data"]["steps_completed"] == 1assert [result["tool"] for result in executed["data"]["results"]] == ["evaluate"]+ assert state._observed_refs == {}+ assert state._observe_v2_state.refs == {}@pytest.mark.asyncio@@ -1015,7 +1249,13 @@ async def test_flag_on_inline_observe_is_reused_without_redundant_refresh(page = _page()ctx = BrowserContext(mode="local")state = make_session_state(context=ctx)- observe = AsyncMock(return_value=_result(page.url, "#continue"))+ observe = AsyncMock(+ side_effect=[+ _result(page.url, "#continue-before"),+ _result(page.url, "#continue-after"),+ _result(page.url, "#continue-final"),+ ]+ )click = AsyncMock(return_value={"ok": True, "data": None})monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))monkeypatch.setattr(browser_ops, "do_observe", observe)@@ -1026,12 +1266,13 @@ async def test_flag_on_inline_observe_is_reused_without_redundant_refresh(steps=[{"tool": "observe", "params": {}},{"tool": "click", "params": {"ref": "e0"}},+ {"tool": "click", "params": {"ref": "e0"}},])assert result["ok"] is True- assert observe.await_count == 1- assert click.await_args.kwargs["selector"] == "#continue"+ assert observe.await_count == 3+ assert [call.kwargs["selector"] for call in click.await_args_list] == ["#continue-before", "#continue-after"]@pytest.mark.asyncio@@ -1118,6 +1359,8 @@ async def test_shared_org_state_keeps_v2_refs_and_budgets_isolated_by_browser_seawait mcp_browser.skyvern_observe(session_id="pbs_B")before_a = session_manager.get_observe_v2_state(session_id="pbs_A")before_b = session_manager.get_observe_v2_state(session_id="pbs_B")+ refs_a_before = dict(before_a.refs)+ refs_b_before = dict(before_b.refs)executed_a = await mcp_browser.skyvern_execute(steps=[{"tool": "click", "params": {"ref": ref_a}}], session_id="pbs_A"@@ -1127,9 +1370,10 @@ async def test_shared_org_state_keeps_v2_refs_and_budgets_isolated_by_browser_seassert executed_a["ok"] is Trueassert click.await_args.kwargs["selector"] == "#a"- assert before_a.refs[ref_a]["selector"] == "#a"- assert before_b.refs["e0"]["selector"] == "#b"- assert after_a.refs[ref_a]["selector"] == "#a"+ assert refs_a_before[ref_a]["selector"] == "#a"+ assert refs_b_before["e0"]["selector"] == "#b"+ assert ref_a not in after_a.refs+ assert next(iter(after_a.refs.values()))["selector"] == "#a"assert after_b.refs["e0"]["selector"] == "#b"assert after_a.host_budgets == {"fixture.test": 50}assert after_b.host_budgets == {"fixture.test": 200}@@ -1178,193 +1422,1314 @@ async def test_flag_on_does_not_refresh_ref_across_origin_or_session(@pytest.mark.asyncio-async def test_page_sourced_document_marker_refuses_durable_refs(+async def test_document_probe_exception_returns_none_and_invalidates_publication(monkeypatch: pytest.MonkeyPatch,) -> None:- """A page-evaluated marker (non-OOPIF frame after skyvern_frame_switch, non-Chromium) can be- pinned by a hostile document, so it must never certify sameness: refs minted on it are not- durable and their later use fails closed instead of re-anchoring into an undescribed document."""monkeypatch.setenv(_FLAG, "1")page = _page(loader_id=None)+ page.evaluate = AsyncMock(side_effect=RuntimeError("execution context destroyed"))ctx = BrowserContext(mode="local")state = make_session_state(context=ctx)+ state._observed_refs = {"e-old": {"ref": "e-old", "selector": "#old"}}+ state._observe_v2_state.refs = dict(state._observed_refs)monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))- monkeypatch.setattr(- mcp_browser,- "do_observe",- AsyncMock(return_value=_result(page.url, "#before", document_id="page:doc-1")),- )- refresh = AsyncMock(return_value=_result(page.url, "#after", document_id="page:doc-1"))- monkeypatch.setattr(browser_ops, "do_observe", refresh)- click = AsyncMock(return_value={"ok": True, "data": None})- monkeypatch.setattr(mcp_browser, "skyvern_click", click)+ monkeypatch.setattr(mcp_browser, "do_observe", AsyncMock(return_value=_result(page.url, "#new")))async with scoped_session(state):+ assert await browser_ops.get_observe_document_id(page) is Noneobserved = await mcp_browser.skyvern_observe()- ref = observed["data"]["elements"][0]["ref"]- live = session_manager.get_observe_v2_state()- assert live.document_id is None- executed = await mcp_browser.skyvern_execute(steps=[{"tool": "click", "params": {"ref": ref}}])- assert executed["ok"] is False- assert "Unknown ref" in executed["data"]["results"][0]["error"]- refresh.assert_not_awaited()- click.assert_not_awaited()+ assert observed["ok"] is False+ assert "superseded" in observed["error"]["message"]+ assert "data" not in observed or observed["data"] is None+ assert state._observed_refs == {}+ assert state._observe_v2_state.refs == {}@pytest.mark.asyncio-async def test_model_invented_ref_does_not_revoke_live_refs(+async def test_inline_observe_after_same_url_reload_updates_accepted_document_id(monkeypatch: pytest.MonkeyPatch,) -> None:- """A ref neither store knows is model-invented: it must fail alone, without revoking the- refs that were just published. Only a legacy-held ref (flag flip) revokes both stores."""monkeypatch.setenv(_FLAG, "1")page = _page()ctx = BrowserContext(mode="local")state = make_session_state(context=ctx)monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))- monkeypatch.setattr(mcp_browser, "do_observe", AsyncMock(return_value=_result(page.url, "#before")))- refresh = AsyncMock(return_value=_result(page.url, "#after"))- monkeypatch.setattr(browser_ops, "do_observe", refresh)- click = AsyncMock(return_value={"ok": True, "data": None})- monkeypatch.setattr(mcp_browser, "skyvern_click", click)+ inline = AsyncMock(return_value=_result(page.url, "#after", document_id="cdp:doc-2"))+ monkeypatch.setattr(browser_ops, "do_observe", inline)++ async def click(**kwargs: Any) -> dict[str, Any]:+ if kwargs["selector"] == "#reload":+ page._loader_id = "doc-2"+ return {"ok": True, "data": None}++ click_mock = AsyncMock(side_effect=click)+ monkeypatch.setattr(mcp_browser, "skyvern_click", click_mock)async with scoped_session(state):- observed = await mcp_browser.skyvern_observe()- ref = observed["data"]["elements"][0]["ref"]- invented = await mcp_browser.skyvern_execute(steps=[{"tool": "click", "params": {"ref": "e99"}}])- live = session_manager.get_observe_v2_state()- assert ref in live.refs- assert session_manager.get_session_ref(ref, page_key=session_manager.page_ref_key(page)) is not None- retried = await mcp_browser.skyvern_execute(steps=[{"tool": "click", "params": {"ref": ref}}])+ executed = await mcp_browser.skyvern_execute(+ steps=[+ {"tool": "click", "params": {"selector": "#reload"}},+ {"tool": "observe", "params": {}},+ {"tool": "click", "params": {"ref": "e0"}},+ ]+ )- assert invented["ok"] is False- assert "Unknown ref" in invented["data"]["results"][0]["error"]- assert retried["ok"] is True- assert click.await_args.kwargs["selector"] == "#after"+ assert executed["ok"] is True+ assert [row["tool"] for row in executed["data"]["results"]] == ["click", "observe", "click"]+ assert executed["data"]["auto_observe"]["tool"] == "observe"+ assert executed["data"]["auto_observe"]["ok"] is True+ assert inline.await_count == 2+ assert click_mock.await_args_list[-1].kwargs["selector"] == "#after"+ assert state._observe_v2_state.document_id == "cdp:doc-2"+ assert next(iter(state._observe_v2_state.refs.values()))["selector"] == "#after"@pytest.mark.asyncio-async def test_in_batch_ref_fails_closed_after_same_document_swap(+async def test_same_url_reload_between_inline_observe_and_ref_rejects_batch_ref(monkeypatch: pytest.MonkeyPatch,) -> None:- """An in-batch ref hit runs the same document check as the cross-call path: a step that- replaces the document at the same URL invalidates refs minted earlier in the batch."""monkeypatch.setenv(_FLAG, "1")page = _page()ctx = BrowserContext(mode="local")state = make_session_state(context=ctx)- observe = AsyncMock(return_value=_result(page.url, "#continue"))monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))- monkeypatch.setattr(browser_ops, "do_observe", observe)+ monkeypatch.setattr(+ browser_ops,+ "do_observe",+ AsyncMock(return_value=_result(page.url, "#old-document", document_id="cdp:doc-1")),+ )- async def click(**kwargs: Any) -> dict[str, Any]:- if kwargs.get("selector") == "#other":- page._loader_id = "doc-2"+ async def reload_document(**kwargs: Any) -> dict[str, Any]:+ page._loader_id = "doc-2"return {"ok": True, "data": None}- click_mock = AsyncMock(side_effect=click)- monkeypatch.setattr(mcp_browser, "skyvern_click", click_mock)+ monkeypatch.setattr(mcp_browser, "skyvern_evaluate", AsyncMock(side_effect=reload_document))+ click = AsyncMock(return_value={"ok": True, "data": None})+ monkeypatch.setattr(mcp_browser, "skyvern_click", click)async with scoped_session(state):executed = await mcp_browser.skyvern_execute(steps=[{"tool": "observe", "params": {}},- {"tool": "click", "params": {"selector": "#other"}},+ {"tool": "evaluate", "params": {"expression": "location.reload()"}},{"tool": "click", "params": {"ref": "e0"}},])assert executed["ok"] is Falseassert "Unknown ref" in executed["data"]["results"][2]["error"]- assert observe.await_count == 1- assert click_mock.await_count == 1+ assert state._observed_refs == {}+ assert state._observe_v2_state.refs == {}+ click.assert_not_awaited()@pytest.mark.asyncio-async def test_in_batch_ref_reuse_passes_document_check_without_reobserve(+async def test_reload_between_snapshot_and_commit_rejects_explicit_publication(monkeypatch: pytest.MonkeyPatch,) -> None:- """When the document is unchanged, the in-batch fast path stays a fast path: the marker- read alone validates the ref, with no per-step re-observe."""monkeypatch.setenv(_FLAG, "1")- page = _page()+ page = _page(loader_id="doc-2")ctx = BrowserContext(mode="local")state = make_session_state(context=ctx)- observe = AsyncMock(return_value=_result(page.url, "#continue"))- click = AsyncMock(return_value={"ok": True, "data": None})+ state._observed_refs = {"e-old": {"ref": "e-old", "selector": "#old"}}+ state._observe_v2_state.refs = dict(state._observed_refs)monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))- monkeypatch.setattr(browser_ops, "do_observe", observe)- monkeypatch.setattr(mcp_browser, "skyvern_click", click)+ monkeypatch.setattr(+ mcp_browser,+ "do_observe",+ AsyncMock(return_value=_result(page.url, "#snapshot", document_id="cdp:doc-1")),+ )async with scoped_session(state):- executed = await mcp_browser.skyvern_execute(- steps=[- {"tool": "observe", "params": {}},- {"tool": "click", "params": {"ref": "e0"}},- ]- )+ observed = await mcp_browser.skyvern_observe()- assert executed["ok"] is True- assert observe.await_count == 1- assert click.await_args.kwargs["selector"] == "#continue"+ assert observed["ok"] is False+ assert "superseded" in observed["error"]["message"]+ assert "data" not in observed or observed["data"] is None+ assert state._observed_refs == {}+ assert state._observe_v2_state.refs == {}-def test_page_text_js_gates_on_rendered_root() -> None:- """Pin the rendered-root gate: innerText of a non-rendered root degrades to raw descendant- text (inline script/style bodies), so the extraction JS must check visibility first."""- assert "checkVisibility" in browser_ops._OBSERVE_PAGE_TEXT_JS- assert "getClientRects" in browser_ops._OBSERVE_PAGE_TEXT_JS+async def test_reload_between_inline_snapshot_and_commit_rejects_batch_publication(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ monkeypatch.setenv(_FLAG, "1")+ page = _page(loader_id="doc-2")+ ctx = BrowserContext(mode="local")+ state = make_session_state(context=ctx)+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))+ monkeypatch.setattr(+ browser_ops,+ "do_observe",+ AsyncMock(return_value=_result(page.url, "#snapshot", document_id="cdp:doc-1")),+ )++ async with scoped_session(state):+ executed = await mcp_browser.skyvern_execute(steps=[{"tool": "observe", "params": {}}])++ assert executed["ok"] is False+ assert "superseded" in executed["data"]["results"][0]["error"]+ assert "data" not in executed["data"]["results"][0]+ assert state._observed_refs == {}+ assert state._observe_v2_state.refs == {}@pytest.mark.asyncio-async def test_mid_observe_document_swap_refuses_durability(+async def test_failed_auto_observe_after_same_url_reload_leaves_no_stale_refs(monkeypatch: pytest.MonkeyPatch,) -> None:- """The document marker must bracket element extraction: a same-URL replacement that lands- mid-observe would otherwise tag pre-swap selectors with the replacement document's id,- blessing stale elements for the durable-ref and in-batch paths."""monkeypatch.setenv(_FLAG, "1")page = _page()+ ctx = BrowserContext(mode="local")+ state = make_session_state(context=ctx)+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))+ monkeypatch.setattr(mcp_browser, "do_observe", AsyncMock(return_value=_result(page.url, "#before")))+ monkeypatch.setattr(+ browser_ops,+ "do_observe",+ AsyncMock(side_effect=ObserveFrameError(None, page.url, RuntimeError("snapshot failed"))),+ )- async def evaluate(expression: str, arg: Any = None) -> Any:- if expression == browser_ops._DOMUTILS_INTERACTABILITY_READY_JS:- return True- if expression == browser_ops._OBSERVE_INTERACTABLES_JS:- # The swap lands during extraction, after the opening marker read.- page._loader_id = "doc-2"- return []- if expression == browser_ops._OBSERVE_PAGE_TEXT_JS:- return {"content": "", "truncated": False}- raise AssertionError(expression)-- page.evaluate = AsyncMock(side_effect=evaluate)- swapped = await browser_ops.do_observe(page)- assert swapped.document_id is None+ async def click(**kwargs: Any) -> dict[str, Any]:+ page.evaluate = AsyncMock(return_value="doc-2")+ return {"ok": True, "data": None}- stable = _page()+ monkeypatch.setattr(mcp_browser, "skyvern_click", AsyncMock(side_effect=click))- async def stable_evaluate(expression: str, arg: Any = None) -> Any:- if expression == browser_ops._DOMUTILS_INTERACTABILITY_READY_JS:- return True- if expression == browser_ops._OBSERVE_INTERACTABLES_JS:- return []- if expression == browser_ops._OBSERVE_PAGE_TEXT_JS:- return {"content": "", "truncated": False}- raise AssertionError(expression)+ async with scoped_session(state):+ await mcp_browser.skyvern_observe()+ executed = await mcp_browser.skyvern_execute(steps=[{"tool": "click", "params": {"selector": "#reload"}}])- stable.evaluate = AsyncMock(side_effect=stable_evaluate)- kept = await browser_ops.do_observe(stable)- assert kept.document_id == "cdp:doc-1"+ assert executed["ok"] is True+ assert [row["tool"] for row in executed["data"]["results"]] == ["click"]+ assert state._observed_refs == {}+ assert state._observe_v2_state.refs == {}@pytest.mark.asyncio-async def test_page_sourced_marker_rejects_in_batch_refs_too(+ ("scenario", "tool", "params", "aria_controls", "aria_owns", "expected_scope"),+ [+ ("autocomplete", "type", {"ref": "e0", "text": "San"}, "suggestions:portal", None, '[id="suggestions:portal"]'),+ ("dependent_dropdown", "select_option", {"ref": "e0", "value": "US"}, None, "cities", '[id="cities"]'),+ ("notice_dismissal", "click", {"selector": "#dismiss"}, None, None, None),+ ("delayed_confirmation", "press_key", {"selector": "#form", "key": "Enter"}, None, None, None),+ ],+)+async def test_mutating_batch_settles_then_observes_once(monkeypatch: pytest.MonkeyPatch,+ scenario: str,+ tool: str,+ params: dict[str, Any],+ aria_controls: str | None,+ aria_owns: str | None,+ expected_scope: str | None,) -> None:- """The rollout-gate limitation, locked: with only a page-sourced marker available, refs- minted by an observe are unusable even inside the same skyvern_execute batch — fail closed,- never a spoofable sameness check. Selector/intent params remain the working path there."""+ monkeypatch.setenv(_FLAG, "1")+ page = _page()+ ctx = BrowserContext(mode="local")+ state = make_session_state(context=ctx)+ events: list[str] = []+ page.page.query_selector = AsyncMock(return_value=object())+ page.page.wait_for_timeout = AsyncMock(side_effect=lambda _ms: events.append("settle"))+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))+ monkeypatch.setattr(+ mcp_browser,+ "do_observe",+ AsyncMock(+ return_value=_result(+ page.url,+ "#control",+ aria_controls=aria_controls,+ aria_owns=aria_owns,+ )+ ),+ )+ observe_calls = 0++ async def observe_after_action(_page: Any, **kwargs: Any) -> ObserveResult:+ nonlocal observe_calls+ observe_calls += 1+ events.append("observe")+ if observe_calls == 1 and "ref" in params:+ return _result(page.url, "#control", aria_controls=aria_controls, aria_owns=aria_owns)+ return _result(page.url, f"#{scenario}-result")++ automatic = AsyncMock(side_effect=observe_after_action)+ monkeypatch.setattr(browser_ops, "do_observe", automatic)+ action = AsyncMock(side_effect=lambda **_kwargs: events.append("action") or {"ok": True, "data": None})+ monkeypatch.setattr(mcp_browser, mcp_browser._TOOL_NAME_MAP[tool], action)++ async with scoped_session(state):+ await mcp_browser.skyvern_observe()+ executed = await mcp_browser.skyvern_execute(steps=[{"tool": tool, "params": params}])++ assert executed["ok"] is True+ refresh_before_action = "ref" in params+ assert events == (+ ["observe", "action", "settle", "observe"] if refresh_before_action else ["action", "settle", "observe"]+ )+ assert automatic.await_count == (2 if refresh_before_action else 1)+ automatic_call = automatic.await_args_list[1] if refresh_before_action else automatic.await_args_list[0]+ assert automatic_call.args[0] is page+ assert automatic_call.kwargs.get("selector") == expected_scope+++async def test_missing_or_multiple_aria_targets_fall_back_to_unscoped_observe(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ monkeypatch.setenv(_FLAG, "1")+ page = _page()+ page.page.query_selector = AsyncMock(return_value=None)+ ctx = BrowserContext(mode="local")+ state = make_session_state(context=ctx)+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))+ monkeypatch.setattr(+ mcp_browser,+ "do_observe",+ AsyncMock(return_value=_result(page.url, "#control", aria_controls="first second")),+ )+ automatic = AsyncMock(+ side_effect=[+ _result(page.url, "#control", aria_controls="first second"),+ _result(page.url, "#after"),+ ]+ )+ monkeypatch.setattr(browser_ops, "do_observe", automatic)+ monkeypatch.setattr(mcp_browser, "skyvern_click", AsyncMock(return_value={"ok": True, "data": None}))++ async with scoped_session(state):+ observed = await mcp_browser.skyvern_observe()+ await mcp_browser.skyvern_execute(+ steps=[{"tool": "click", "params": {"ref": observed["data"]["elements"][0]["ref"]}}]+ )++ assert automatic.await_args_list[-1].kwargs.get("selector") is None+ page.page.query_selector.assert_not_awaited()++ assert (+ await mcp_browser._attached_aria_target_selector(+ page,+ [{"aria_controls": "missing"}],+ )+ is None+ )+ page.page.query_selector.assert_awaited_once_with('[id="missing"]')+++async def test_multi_step_mutating_batch_gets_one_post_batch_observe(monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setenv(_FLAG, "1")+ page = _page()+ ctx = BrowserContext(mode="local")+ state = make_session_state(context=ctx)+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))+ monkeypatch.setattr(mcp_browser, "skyvern_click", AsyncMock(return_value={"ok": True, "data": None}))+ automatic = AsyncMock(return_value=_result(page.url, "#after"))+ monkeypatch.setattr(browser_ops, "do_observe", automatic)++ async with scoped_session(state):+ executed = await mcp_browser.skyvern_execute(+ steps=[+ {"tool": "click", "params": {"selector": "#first"}},+ {"tool": "click", "params": {"selector": "#second"}},+ ]+ )++ assert executed["ok"] is True+ assert [row["tool"] for row in executed["data"]["results"]] == ["click", "click"]+ assert executed["data"]["auto_observe"]["tool"] == "observe"+ assert executed["data"]["auto_observe"]["ok"] is True+ assert automatic.await_count == 1+ page.page.wait_for_timeout.assert_awaited_once_with(mcp_browser._POST_MUTATION_SETTLE_MS)+++async def test_successful_mutation_still_gets_post_batch_observe_after_later_failure(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ monkeypatch.setenv(_FLAG, "1")+ page = _page()+ ctx = BrowserContext(mode="local")+ state = make_session_state(context=ctx)+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))+ monkeypatch.setattr(+ mcp_browser,+ "skyvern_click",+ AsyncMock(side_effect=[{"ok": True, "data": None}, RuntimeError("missing target")]),+ )+ automatic = AsyncMock(return_value=_result(page.url, "#after"))+ monkeypatch.setattr(browser_ops, "do_observe", automatic)++ async with scoped_session(state):+ executed = await mcp_browser.skyvern_execute(+ steps=[+ {"tool": "click", "params": {"selector": "#mutate"}},+ {"tool": "click", "params": {"selector": "#missing"}},+ ]+ )++ assert executed["ok"] is False+ assert executed["data"]["error_step"] == 1+ assert [row["tool"] for row in executed["data"]["results"]] == ["click", "click"]+ assert executed["data"]["auto_observe"]["tool"] == "observe"+ automatic.assert_awaited_once()+++async def test_post_mutation_settle_cancellation_clears_existing_refs(monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setenv(_FLAG, "1")+ page = _page()+ ctx = BrowserContext(mode="local")+ state = make_session_state(context=ctx)+ state._observed_refs = {"e0": {"ref": "e0", "selector": "#stale"}}+ state._observe_v2_state.refs = dict(state._observed_refs)+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))+ monkeypatch.setattr(mcp_browser, "skyvern_click", AsyncMock(return_value={"ok": True, "data": None}))+ monkeypatch.setattr(mcp_browser, "_settle_after_mutating_batch", AsyncMock(side_effect=asyncio.CancelledError))++ async with scoped_session(state):+ with pytest.raises(asyncio.CancelledError):+ await mcp_browser.skyvern_execute(steps=[{"tool": "click", "params": {"selector": "#mutate"}}])++ assert state._observed_refs == {}+ assert state._observe_v2_state.refs == {}+++async def test_post_mutation_cancellation_preserves_newer_publication(monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setenv(_FLAG, "1")+ page = _page()+ ctx = BrowserContext(mode="local")+ state = make_session_state(context=ctx)+ settle_started = asyncio.Event()++ async def settle(*_args: Any, **_kwargs: Any) -> None:+ settle_started.set()+ await asyncio.Event().wait()++ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))+ monkeypatch.setattr(mcp_browser, "skyvern_click", AsyncMock(return_value={"ok": True, "data": None}))+ monkeypatch.setattr(mcp_browser, "_settle_after_mutating_batch", settle)+ monkeypatch.setattr(mcp_browser, "do_observe", AsyncMock(return_value=_result(page.url, "#newer")))++ async with scoped_session(state):+ older_task = asyncio.create_task(+ mcp_browser.skyvern_execute(steps=[{"tool": "click", "params": {"selector": "#mutate"}}])+ )+ await settle_started.wait()+ newer = await mcp_browser.skyvern_observe()+ older_task.cancel()+ with pytest.raises(asyncio.CancelledError):+ await older_task++ assert newer["ok"] is True+ assert {element["selector"] for element in state._observe_v2_state.refs.values()} == {"#newer"}+++async def test_execute_no_browser_observe_clears_existing_refs(monkeypatch: pytest.MonkeyPatch) -> None:+ from skyvern.cli.mcp_tools._session import BrowserNotAvailableError++ monkeypatch.setenv(_FLAG, "1")+ ctx = BrowserContext(mode="local")+ state = make_session_state(context=ctx)+ state._observed_refs = {"e0": {"ref": "e0", "selector": "#stale"}}+ state._observe_v2_state.refs = dict(state._observed_refs)+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(side_effect=BrowserNotAvailableError("no browser")))++ async with scoped_session(state):+ result = await mcp_browser.skyvern_execute(steps=[{"tool": "observe", "params": {}}])++ assert result["ok"] is False+ assert state._observed_refs == {}+ assert state._observe_v2_state.refs == {}+++async def test_execute_page_lookup_cancellation_clears_existing_refs(monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setenv(_FLAG, "1")+ ctx = BrowserContext(mode="local")+ state = make_session_state(context=ctx)+ state._observed_refs = {"e0": {"ref": "e0", "selector": "#stale"}}+ state._observe_v2_state.refs = dict(state._observed_refs)+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(side_effect=asyncio.CancelledError))++ async with scoped_session(state):+ with pytest.raises(asyncio.CancelledError):+ await mcp_browser.skyvern_execute(steps=[{"tool": "observe", "params": {}}])++ assert state._observed_refs == {}+ assert state._observe_v2_state.refs == {}+++async def test_flag_off_cancelled_mutating_page_lookup_preserves_existing_refs(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ monkeypatch.delenv(_FLAG, raising=False)+ ctx = BrowserContext(mode="local")+ state = make_session_state(context=ctx)+ state._observed_refs = {"e0": {"ref": "e0", "selector": "#existing"}}+ state._observe_v2_state.refs = dict(state._observed_refs)+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(side_effect=asyncio.CancelledError))++ async with scoped_session(state):+ with pytest.raises(asyncio.CancelledError):+ await mcp_browser.skyvern_execute(steps=[{"tool": "click", "params": {"selector": "#existing"}}])++ assert state._observed_refs == {"e0": {"ref": "e0", "selector": "#existing"}}+ assert state._observe_v2_state.refs == {"e0": {"ref": "e0", "selector": "#existing"}}+++async def test_cancelled_execute_page_lookup_preserves_newer_publication(monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setenv(_FLAG, "1")+ page = _page()+ ctx = BrowserContext(mode="local")+ state = make_session_state(context=ctx)+ page_lookup_started = asyncio.Event()++ async def delayed_get_page(*args: Any, **kwargs: Any) -> tuple[Any, BrowserContext]:+ page_lookup_started.set()+ await asyncio.Event().wait()+ raise AssertionError("unreachable")++ monkeypatch.setattr(mcp_browser, "get_page", delayed_get_page)++ async with scoped_session(state):+ older_task = asyncio.create_task(mcp_browser.skyvern_execute(steps=[{"tool": "observe", "params": {}}]))+ await page_lookup_started.wait()+ newer_generation = session_manager.begin_session_ref_publication()+ assert session_manager.replace_session_ref_map(+ {"e1": {"ref": "e1", "selector": "#newer"}},+ generation=newer_generation,+ page_key=session_manager.page_ref_key(page),+ )+ older_task.cancel()+ with pytest.raises(asyncio.CancelledError):+ await older_task++ assert state._observed_refs["refs"] == {"e1": {"ref": "e1", "selector": "#newer"}}+++async def test_execute_document_probe_cancellation_clears_existing_refs(monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setenv(_FLAG, "1")+ page = _page()+ ctx = BrowserContext(mode="local")+ state = make_session_state(context=ctx)+ state._observed_refs = {"e0": {"ref": "e0", "selector": "#stale"}}+ state._observe_v2_state.refs = dict(state._observed_refs)+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))+ monkeypatch.setattr(mcp_browser, "get_observe_document_id", AsyncMock(side_effect=asyncio.CancelledError))++ async with scoped_session(state):+ with pytest.raises(asyncio.CancelledError):+ await mcp_browser.skyvern_execute(steps=[{"tool": "observe", "params": {}}])++ assert state._observed_refs == {}+ assert state._observe_v2_state.refs == {}+++async def test_execute_publication_cancellation_clears_existing_refs(monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setenv(_FLAG, "1")+ page = _page()+ ctx = BrowserContext(mode="local")+ state = make_session_state(context=ctx)+ state._observed_refs = {"e0": {"ref": "e0", "selector": "#stale"}}+ state._observe_v2_state.refs = dict(state._observed_refs)+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))+ monkeypatch.setattr(browser_ops, "do_observe", AsyncMock(return_value=_result(page.url, "#staged")))+ publish = AsyncMock(side_effect=asyncio.CancelledError)+ monkeypatch.setattr(mcp_browser, "_publish_observe_v2_refs", publish)++ async with scoped_session(state):+ with pytest.raises(asyncio.CancelledError):+ await mcp_browser.skyvern_execute(steps=[{"tool": "observe", "params": {}}])++ publish.assert_awaited_once()+ assert state._observed_refs == {}+ assert state._observe_v2_state.refs == {}+++async def test_slower_standalone_observe_cannot_overwrite_newer_execute_publication(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ monkeypatch.setenv(_FLAG, "1")+ page = _page()+ ctx = BrowserContext(mode="local")+ state = make_session_state(context=ctx)+ older_started = asyncio.Event()+ release_older = asyncio.Event()++ async def older_observe(*_args: Any, **_kwargs: Any) -> ObserveResult:+ older_started.set()+ await release_older.wait()+ return _result(page.url, "#older")++ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))+ monkeypatch.setattr(mcp_browser, "do_observe", older_observe)+ monkeypatch.setattr(browser_ops, "do_observe", AsyncMock(return_value=_result(page.url, "#newer")))++ async with scoped_session(state):+ older_task = asyncio.create_task(mcp_browser.skyvern_observe())+ await older_started.wait()+ newer = await mcp_browser.skyvern_execute(steps=[{"tool": "observe", "params": {}}])+ release_older.set()+ older = await older_task++ assert newer["ok"] is True+ assert older["ok"] is False+ assert "data" not in older or older["data"] is None+ assert {element["selector"] for element in state._observe_v2_state.refs.values()} == {"#newer"}+++async def test_slower_execute_cannot_clear_newer_standalone_publication(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ monkeypatch.setenv(_FLAG, "1")+ page = _page()+ ctx = BrowserContext(mode="local")+ state = make_session_state(context=ctx)+ execute_waiting = asyncio.Event()+ release_execute = asyncio.Event()++ async def wait_step(**_kwargs: Any) -> dict[str, Any]:+ execute_waiting.set()+ await release_execute.wait()+ return {"ok": True, "data": None}++ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))+ monkeypatch.setattr(browser_ops, "do_observe", AsyncMock(return_value=_result(page.url, "#older")))+ monkeypatch.setattr(mcp_browser, "do_observe", AsyncMock(return_value=_result(page.url, "#newer")))+ monkeypatch.setattr(mcp_browser, "skyvern_wait", wait_step)++ async with scoped_session(state):+ older_task = asyncio.create_task(+ mcp_browser.skyvern_execute(+ steps=[+ {"tool": "observe", "params": {}},+ {"tool": "wait", "params": {"duration_ms": 1}},+ ]+ )+ )+ await execute_waiting.wait()+ newer = await mcp_browser.skyvern_observe()+ release_execute.set()+ older = await older_task++ assert newer["ok"] is True+ assert older["ok"] is False+ assert "observe snapshot was superseded" in older["error"]["message"].lower()+ assert {element["selector"] for element in state._observe_v2_state.refs.values()} == {"#newer"}+++async def test_stale_ref_refresh_cannot_overwrite_newer_publication(monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setenv(_FLAG, "1")+ page = _page()+ ctx = BrowserContext(mode="local")+ state = make_session_state(context=ctx)+ state._observe_v2_state.page_key = session_manager.page_ref_key(page)+ state._observe_v2_state.document_id = "doc-1"+ state._observe_v2_state.params = {+ "selector": None,+ "interactive_only": True,+ "max_elements": 50,+ "include_values": False,+ }+ state._observe_v2_state.refs = {"e0": {"ref": "e0", "selector": "#old"}}+ state._observed_refs = dict(state._observe_v2_state.refs)+ state._observed_refs_generation = session_manager.begin_session_ref_publication()+ refresh_started = asyncio.Event()+ release_refresh = asyncio.Event()++ async def refresh(*args: Any, **kwargs: Any) -> ObserveResult:+ refresh_started.set()+ await release_refresh.wait()+ return _result(page.url, "#refreshed", document_id="doc-1")++ monkeypatch.setattr(mcp_browser, "get_observe_document_id", AsyncMock(return_value="doc-1"))+ monkeypatch.setattr(browser_ops, "do_observe", refresh)++ async with scoped_session(state):+ older_task = asyncio.create_task(+ mcp_browser._refresh_observe_v2_ref(+ "e0",+ page,+ session_id=None,+ cdp_url=None,+ )+ )+ await refresh_started.wait()+ newer_generation = session_manager.begin_session_ref_publication()+ assert session_manager.replace_session_ref_map(+ {"e1": {"ref": "e1", "selector": "#newer"}},+ generation=newer_generation,+ page_key=session_manager.page_ref_key(page),+ )+ state._observe_v2_state.refs = {"e1": {"ref": "e1", "selector": "#newer"}}+ release_refresh.set()+ handled, refreshed = await older_task++ assert handled is True+ assert refreshed is None+ assert state._observe_v2_state.refs == {"e1": {"ref": "e1", "selector": "#newer"}}+++async def test_cancelled_observe_page_lookup_preserves_newer_publication(monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setenv(_FLAG, "1")+ page = _page()+ ctx = BrowserContext(mode="local")+ state = make_session_state(context=ctx)+ page_lookup_started = asyncio.Event()++ async def delayed_get_page(*args: Any, **kwargs: Any) -> tuple[Any, BrowserContext]:+ page_lookup_started.set()+ await asyncio.Event().wait()+ raise AssertionError("unreachable")++ monkeypatch.setattr(mcp_browser, "get_page", delayed_get_page)++ async with scoped_session(state):+ older_task = asyncio.create_task(mcp_browser.skyvern_observe())+ await page_lookup_started.wait()+ newer_generation = session_manager.begin_session_ref_publication()+ assert session_manager.replace_session_ref_map(+ {"e1": {"ref": "e1", "selector": "#newer"}},+ generation=newer_generation,+ page_key=session_manager.page_ref_key(page),+ )+ older_task.cancel()+ with pytest.raises(asyncio.CancelledError):+ await older_task++ assert state._observed_refs["refs"] == {"e1": {"ref": "e1", "selector": "#newer"}}+++async def test_failed_later_observe_discards_staged_ref_publication(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ monkeypatch.setenv(_FLAG, "1")+ page = _page()+ ctx = BrowserContext(mode="local")+ state = make_session_state(context=ctx)+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))+ monkeypatch.setattr(+ browser_ops,+ "do_observe",+ AsyncMock(side_effect=[_result(page.url, "#staged"), RuntimeError("snapshot failed")]),+ )++ async with scoped_session(state):+ executed = await mcp_browser.skyvern_execute(+ steps=[+ {"tool": "observe", "params": {}},+ {"tool": "observe", "params": {}},+ ],+ stop_on_error=False,+ )++ assert executed["ok"] is False+ assert state._observed_refs == {}+ assert state._observe_v2_state.refs == {}+++async def test_revalidation_failure_discards_earlier_staged_ref_publication(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ monkeypatch.setenv(_FLAG, "1")+ page = _page()+ ctx = BrowserContext(mode="local")+ state = make_session_state(context=ctx)+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))+ monkeypatch.setattr(+ browser_ops,+ "do_observe",+ AsyncMock(side_effect=[_result(page.url, "#staged"), _result(page.url, "#rejected")]),+ )+ revalidate = AsyncMock(side_effect=[True, RuntimeError("revalidation failed"), True])+ monkeypatch.setattr(mcp_browser, "_observe_v2_snapshot_is_current", revalidate)++ async with scoped_session(state):+ executed = await mcp_browser.skyvern_execute(+ steps=[+ {"tool": "observe", "params": {}},+ {"tool": "observe", "params": {}},+ ],+ stop_on_error=False,+ )++ assert executed["ok"] is False+ assert state._observed_refs == {}+ assert state._observe_v2_state.refs == {}+++async def test_settle_timeout_still_runs_bounded_post_mutation_observe(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ monkeypatch.setenv(_FLAG, "1")+ page = _page()+ page.page.wait_for_timeout = AsyncMock(side_effect=TimeoutError)+ ctx = BrowserContext(mode="local")+ state = make_session_state(context=ctx)+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))+ monkeypatch.setattr(mcp_browser, "skyvern_click", AsyncMock(return_value={"ok": True, "data": None}))+ automatic = AsyncMock(return_value=_result(page.url, "#after"))+ monkeypatch.setattr(browser_ops, "do_observe", automatic)++ async with scoped_session(state):+ executed = await mcp_browser.skyvern_execute(steps=[{"tool": "click", "params": {"selector": "#mutate"}}])++ assert executed["ok"] is True+ assert [row["tool"] for row in executed["data"]["results"]] == ["click"]+ assert executed["data"]["auto_observe"]["tool"] == "observe"+ automatic.assert_awaited_once()+++async def test_post_mutation_same_label_replacement_invalidates_old_ref_generation(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ monkeypatch.setenv(_FLAG, "1")+ page = _page()+ ctx = BrowserContext(mode="local")+ state = make_session_state(context=ctx)+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))+ monkeypatch.setattr(mcp_browser, "do_observe", AsyncMock(return_value=_result(page.url, "#before")))+ monkeypatch.setattr(browser_ops, "do_observe", AsyncMock(return_value=_result(page.url, "#after")))+ monkeypatch.setattr(mcp_browser, "skyvern_click", AsyncMock(return_value={"ok": True, "data": None}))++ async with scoped_session(state):+ observed = await mcp_browser.skyvern_observe()+ stale_ref = observed["data"]["elements"][0]["ref"]+ generation_before = session_manager.session_ref_generation()+ executed = await mcp_browser.skyvern_execute(steps=[{"tool": "click", "params": {"ref": stale_ref}}])+ generation_after = session_manager.session_ref_generation()+ fresh_ref = executed["data"]["auto_observe"]["data"]["elements"][0]["ref"]+ fresh_selector = state._observe_v2_state.refs[fresh_ref]["selector"]+ stale = await mcp_browser.skyvern_execute(steps=[{"tool": "click", "params": {"ref": stale_ref}}])++ assert generation_after > generation_before+ assert fresh_ref != stale_ref+ assert fresh_selector == "#after"+ assert stale["ok"] is False+ assert "Unknown ref" in stale["data"]["results"][0]["error"]+++async def test_oversized_observe_response_does_not_publish_omitted_refs(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ monkeypatch.setenv(_FLAG, "1")+ page = _page()+ ctx = BrowserContext(mode="local")+ state = make_session_state(context=ctx)+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))+ monkeypatch.setattr(+ mcp_browser,+ "do_observe",+ AsyncMock(return_value=_result(page.url, "#omitted", page_text="x" * 150_000)),+ )++ async with scoped_session(state):+ observed = await mcp_browser.skyvern_observe()++ assert observed["_truncated"] is True+ assert observed["_max_bytes"] == MCP_MAX_RESPONSE_BYTES+ assert len(json.dumps(observed, ensure_ascii=False).encode()) <= MCP_MAX_RESPONSE_BYTES+ assert "data" not in observed+ assert state._observed_refs == {}+ assert state._observe_v2_state.refs == {}++[email protected]("flag_value", [None, "0", "false"])+async def test_flag_off_oversized_multibyte_observe_response_remains_uncapped(+ monkeypatch: pytest.MonkeyPatch,+ flag_value: str | None,+) -> None:+ if flag_value is None:+ monkeypatch.delenv(_FLAG, raising=False)+ else:+ monkeypatch.setenv(_FLAG, flag_value)++ page = _page()+ ctx = BrowserContext(mode="local")+ state = make_session_state(context=ctx)+ large_name = "界" * (MCP_MAX_RESPONSE_BYTES // 2)+ result = _result(page.url, "#legacy")+ result.elements[0].name = large_name+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))+ monkeypatch.setattr(mcp_browser, "do_observe", AsyncMock(return_value=result))++ async with scoped_session(state):+ observed = await mcp_browser.skyvern_observe()++ serialized = json.dumps(observed, ensure_ascii=False)+ assert len(serialized) < MCP_MAX_RESPONSE_BYTES+ assert len(serialized.encode()) > MCP_MAX_RESPONSE_BYTES+ assert "_truncated" not in observed+ assert observed["data"]["elements"][0]["name"] == large_name+ assert state._observed_refs["refs"]["e0"]["selector"] == "#legacy"+++async def test_flag_off_oversized_execute_response_remains_uncapped(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ monkeypatch.delenv(_FLAG, raising=False)+ page = _page()+ ctx = BrowserContext(mode="local")+ state = make_session_state(context=ctx)+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))+ monkeypatch.setattr(+ mcp_browser,+ "skyvern_evaluate",+ AsyncMock(return_value={"ok": True, "data": {"value": "x" * 150_000}}),+ )++ async with scoped_session(state):+ executed = await mcp_browser.skyvern_execute(+ steps=[{"tool": "evaluate", "params": {"expression": "document.title"}}]+ )++ assert "_truncated" not in executed+ assert executed["data"]["results"][0]["data"]["value"] == "x" * 150_000+++async def test_oversized_execute_response_does_not_publish_omitted_auto_observe_refs(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ monkeypatch.setenv(_FLAG, "1")+ page = _page()+ ctx = BrowserContext(mode="local")+ state = make_session_state(context=ctx)+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))+ monkeypatch.setattr(mcp_browser, "skyvern_click", AsyncMock(return_value={"ok": True, "data": None}))+ monkeypatch.setattr(+ browser_ops,+ "do_observe",+ AsyncMock(return_value=_result(page.url, "#fresh", page_text="x" * 150_000)),+ )++ async with scoped_session(state):+ executed = await mcp_browser.skyvern_execute(steps=[{"tool": "click", "params": {"selector": "#mutate"}}])++ assert executed["_truncated"] is True+ assert executed["_max_bytes"] == MCP_MAX_RESPONSE_BYTES+ assert len(json.dumps(executed, ensure_ascii=False).encode()) <= MCP_MAX_RESPONSE_BYTES+ assert "data" not in executed+ assert state._observed_refs == {}+ assert state._observe_v2_state.refs == {}+++async def test_failed_mutation_strips_invalidated_inline_observe_refs(monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setenv(_FLAG, "1")+ page = _page()+ ctx = BrowserContext(mode="local")+ state = make_session_state(context=ctx)+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))+ monkeypatch.setattr(browser_ops, "do_observe", AsyncMock(return_value=_result(page.url, "#inline")))+ monkeypatch.setattr(+ mcp_browser,+ "skyvern_click",+ AsyncMock(return_value={"ok": False, "error": {"code": "ACTION_FAILED", "message": "click failed"}}),+ )++ async with scoped_session(state):+ executed = await mcp_browser.skyvern_execute(+ steps=[+ {"tool": "observe", "params": {}},+ {"tool": "click", "params": {"ref": "e0"}},+ ]+ )++ assert executed["ok"] is False+ assert executed["data"]["error_step"] == 1+ assert "data" not in executed["data"]["results"][0]+ assert state._observed_refs == {}+ assert state._observe_v2_state.refs == {}+++async def test_failed_mutation_invalidates_inflight_publication(monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setenv(_FLAG, "1")+ page = _page()+ ctx = BrowserContext(mode="local")+ state = make_session_state(context=ctx)++ async def click(**kwargs: Any) -> dict[str, Any]:+ generation = session_manager.begin_session_ref_publication()+ ref_map = {"e1": {"ref": "e1", "selector": "#during-action"}}+ assert session_manager.replace_session_ref_map(+ ref_map,+ generation=generation,+ page_key=session_manager.page_ref_key(page),+ )+ state._observe_v2_state.refs = dict(ref_map)+ state._observe_v2_state.page_key = session_manager.page_ref_key(page)+ state._observe_v2_state.document_id = "page:doc-1"+ return {"ok": False, "error": {"code": "ACTION_FAILED", "message": "click failed"}}++ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))+ monkeypatch.setattr(mcp_browser, "skyvern_click", AsyncMock(side_effect=click))++ async with scoped_session(state):+ executed = await mcp_browser.skyvern_execute(steps=[{"tool": "click", "params": {"selector": "#mutate"}}])+ assert (+ session_manager.get_session_ref(+ "e1",+ page_key=session_manager.page_ref_key(page),+ )+ is None+ )++ assert executed["ok"] is False+ assert state._observe_v2_state.refs == {}+++async def test_multiple_inline_observes_return_only_published_snapshot(monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setenv(_FLAG, "1")+ page = _page()+ ctx = BrowserContext(mode="local")+ state = make_session_state(context=ctx)+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))+ monkeypatch.setattr(+ browser_ops,+ "do_observe",+ AsyncMock(side_effect=[_result(page.url, "#first"), _result(page.url, "#second")]),+ )++ async with scoped_session(state):+ executed = await mcp_browser.skyvern_execute(+ steps=[+ {"tool": "observe", "params": {}},+ {"tool": "observe", "params": {}},+ ]+ )++ assert executed["ok"] is True+ assert "data" not in executed["data"]["results"][0]+ assert executed["data"]["results"][1]["data"]["elements"][0]["selector"] == "#second"+ assert {element["selector"] for element in state._observe_v2_state.refs.values()} == {"#second"}+++async def test_stale_document_cleanup_preserves_newer_publication(monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setenv(_FLAG, "1")+ page = _page()+ ctx = BrowserContext(mode="local")+ state = make_session_state(context=ctx)+ automatic = AsyncMock(side_effect=AssertionError("stale execute must not replace a newer publication"))++ async def wait(**kwargs: Any) -> dict[str, Any]:+ page.evaluate = AsyncMock(return_value="doc-2")+ generation = session_manager.begin_session_ref_publication()+ ref_map = {"e1": {"ref": "e1", "selector": "#newer"}}+ assert session_manager.replace_session_ref_map(+ ref_map,+ generation=generation,+ page_key=session_manager.page_ref_key(page),+ )+ state._observe_v2_state.refs = dict(ref_map)+ state._observe_v2_state.page_key = session_manager.page_ref_key(page)+ state._observe_v2_state.document_id = "page:doc-2"+ return {"ok": True, "data": None}++ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))+ monkeypatch.setattr(mcp_browser, "skyvern_wait", AsyncMock(side_effect=wait))+ monkeypatch.setattr(browser_ops, "do_observe", automatic)++ async with scoped_session(state):+ executed = await mcp_browser.skyvern_execute(steps=[{"tool": "wait", "params": {"seconds": 0}}])+ assert session_manager.get_session_ref(+ "e1",+ page_key=session_manager.page_ref_key(page),+ ) == {"ref": "e1", "selector": "#newer"}++ assert executed["ok"] is True+ automatic.assert_not_awaited()+ assert state._observe_v2_state.refs == {"e1": {"ref": "e1", "selector": "#newer"}}+++async def test_stale_document_cleanup_replaces_mismatched_newer_publication(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ monkeypatch.setenv(_FLAG, "1")+ page = _page()+ ctx = BrowserContext(mode="local")+ state = make_session_state(context=ctx)+ automatic = AsyncMock(return_value=_result(page.url, "#fresh", document_id="cdp:doc-2"))++ async def wait(**kwargs: Any) -> dict[str, Any]:+ generation = session_manager.begin_session_ref_publication()+ stale_map = {"e1": {"ref": "e1", "selector": "#stale"}}+ assert session_manager.replace_session_ref_map(+ stale_map,+ generation=generation,+ page_key=session_manager.page_ref_key(page),+ )+ state._observe_v2_state.refs = dict(stale_map)+ state._observe_v2_state.page_key = session_manager.page_ref_key(page)+ state._observe_v2_state.document_id = "cdp:doc-1"+ page._loader_id = "doc-2"+ return {"ok": True, "data": None}++ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))+ monkeypatch.setattr(mcp_browser, "skyvern_wait", AsyncMock(side_effect=wait))+ monkeypatch.setattr(browser_ops, "do_observe", automatic)++ async with scoped_session(state):+ executed = await mcp_browser.skyvern_execute(steps=[{"tool": "wait", "params": {"seconds": 0}}])++ assert executed["ok"] is True+ automatic.assert_awaited_once()+ assert executed["data"]["auto_observe"]["data"]["elements"][0]["selector"] == "#fresh"+ assert {element["selector"] for element in state._observe_v2_state.refs.values()} == {"#fresh"}+++async def test_failed_navigation_invalidates_inflight_publication(monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setenv(_FLAG, "1")+ page = _page()+ ctx = BrowserContext(mode="local")+ state = make_session_state(context=ctx)++ async def navigate(*args: Any, **kwargs: Any) -> Any:+ generation = session_manager.begin_session_ref_publication()+ ref_map = {"e1": {"ref": "e1", "selector": "#newer"}}+ assert session_manager.replace_session_ref_map(+ ref_map,+ generation=generation,+ page_key=session_manager.page_ref_key(page),+ )+ state._observe_v2_state.refs = dict(ref_map)+ state._observe_v2_state.page_key = session_manager.page_ref_key(page)+ state._observe_v2_state.document_id = "page:doc-1"+ raise RuntimeError("net::ERR_ABORTED")++ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))+ monkeypatch.setattr(mcp_browser, "do_navigate", navigate)++ async with scoped_session(state):+ navigated = await mcp_browser.skyvern_navigate("https://example.com/next")+ assert (+ session_manager.get_session_ref(+ "e1",+ page_key=session_manager.page_ref_key(page),+ )+ is None+ )++ assert navigated["ok"] is False+ assert state._observe_v2_state.refs == {}+++async def test_page_sourced_document_marker_refuses_durable_refs(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """A page-evaluated marker (non-OOPIF frame after skyvern_frame_switch, non-Chromium) can be+ pinned by a hostile document, so it must never certify sameness: refs minted on it are not+ durable and their later use fails closed instead of re-anchoring into an undescribed document."""+ monkeypatch.setenv(_FLAG, "1")+ page = _page(loader_id=None)+ ctx = BrowserContext(mode="local")+ state = make_session_state(context=ctx)+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))+ monkeypatch.setattr(+ mcp_browser,+ "do_observe",+ AsyncMock(return_value=_result(page.url, "#before", document_id="page:doc-1")),+ )+ refresh = AsyncMock(return_value=_result(page.url, "#after", document_id="page:doc-1"))+ monkeypatch.setattr(browser_ops, "do_observe", refresh)+ click = AsyncMock(return_value={"ok": True, "data": None})+ monkeypatch.setattr(mcp_browser, "skyvern_click", click)++ async with scoped_session(state):+ observed = await mcp_browser.skyvern_observe()+ ref = observed["data"]["elements"][0]["ref"]+ live = session_manager.get_observe_v2_state()+ assert live.document_id is None+ executed = await mcp_browser.skyvern_execute(steps=[{"tool": "click", "params": {"ref": ref}}])++ assert executed["ok"] is False+ assert "Unknown ref" in executed["data"]["results"][0]["error"]+ refresh.assert_not_awaited()+ click.assert_not_awaited()+++async def test_model_invented_ref_does_not_revoke_live_refs(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """A ref neither store knows is model-invented: it must fail alone, without revoking the+ refs that were just published. Only a legacy-held ref (flag flip) revokes both stores."""+ monkeypatch.setenv(_FLAG, "1")+ page = _page()+ ctx = BrowserContext(mode="local")+ state = make_session_state(context=ctx)+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))+ monkeypatch.setattr(mcp_browser, "do_observe", AsyncMock(return_value=_result(page.url, "#before")))+ refresh = AsyncMock(return_value=_result(page.url, "#after"))+ monkeypatch.setattr(browser_ops, "do_observe", refresh)+ click = AsyncMock(return_value={"ok": True, "data": None})+ monkeypatch.setattr(mcp_browser, "skyvern_click", click)++ async with scoped_session(state):+ observed = await mcp_browser.skyvern_observe()+ ref = observed["data"]["elements"][0]["ref"]+ invented = await mcp_browser.skyvern_execute(steps=[{"tool": "click", "params": {"ref": "e99"}}])+ live = session_manager.get_observe_v2_state()+ assert ref in live.refs+ assert session_manager.get_session_ref(ref, page_key=session_manager.page_ref_key(page)) is not None+ retried = await mcp_browser.skyvern_execute(steps=[{"tool": "click", "params": {"ref": ref}}])++ assert invented["ok"] is False+ assert "Unknown ref" in invented["data"]["results"][0]["error"]+ assert retried["ok"] is True+ assert click.await_args.kwargs["selector"] == "#after"+++async def test_in_batch_ref_fails_closed_after_same_document_swap(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """An in-batch ref hit runs the same document check as the cross-call path: a step that+ replaces the document at the same URL invalidates refs minted earlier in the batch."""+ monkeypatch.setenv(_FLAG, "1")+ page = _page()+ ctx = BrowserContext(mode="local")+ state = make_session_state(context=ctx)+ observe = AsyncMock(return_value=_result(page.url, "#continue"))+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))+ monkeypatch.setattr(browser_ops, "do_observe", observe)++ async def click(**kwargs: Any) -> dict[str, Any]:+ if kwargs.get("selector") == "#other":+ page._loader_id = "doc-2"+ return {"ok": True, "data": None}++ click_mock = AsyncMock(side_effect=click)+ monkeypatch.setattr(mcp_browser, "skyvern_click", click_mock)++ async with scoped_session(state):+ executed = await mcp_browser.skyvern_execute(+ steps=[+ {"tool": "observe", "params": {}},+ {"tool": "click", "params": {"selector": "#other"}},+ {"tool": "click", "params": {"ref": "e0"}},+ ]+ )++ assert executed["ok"] is False+ assert "Unknown ref" in executed["data"]["results"][2]["error"]+ assert observe.await_count == 2+ assert click_mock.await_count == 1+++async def test_in_batch_ref_reuse_passes_document_check_without_reobserve(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """When the document is unchanged, the in-batch fast path stays a fast path: the marker+ read alone validates the ref, with no per-step re-observe."""+ monkeypatch.setenv(_FLAG, "1")+ page = _page()+ ctx = BrowserContext(mode="local")+ state = make_session_state(context=ctx)+ observe = AsyncMock(return_value=_result(page.url, "#continue"))+ click = AsyncMock(return_value={"ok": True, "data": None})+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))+ monkeypatch.setattr(browser_ops, "do_observe", observe)+ monkeypatch.setattr(mcp_browser, "skyvern_click", click)++ async with scoped_session(state):+ executed = await mcp_browser.skyvern_execute(+ steps=[+ {"tool": "observe", "params": {}},+ {"tool": "click", "params": {"ref": "e0"}},+ ]+ )++ assert executed["ok"] is True+ assert observe.await_count == 2+ assert executed["data"]["auto_observe"]["ok"] is True+ assert click.await_args.kwargs["selector"] == "#continue"+++def test_page_text_js_gates_on_rendered_root() -> None:+ """Pin the rendered-root gate: innerText of a non-rendered root degrades to raw descendant+ text (inline script/style bodies), so the extraction JS must check visibility first."""+ assert "checkVisibility" in browser_ops._OBSERVE_PAGE_TEXT_JS+ assert "getClientRects" in browser_ops._OBSERVE_PAGE_TEXT_JS+++async def test_mid_observe_document_swap_refuses_durability(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """The document marker must bracket element extraction: a same-URL replacement that lands+ mid-observe would otherwise tag pre-swap selectors with the replacement document's id,+ blessing stale elements for the durable-ref and in-batch paths."""+ monkeypatch.setenv(_FLAG, "1")+ page = _page()++ async def evaluate(expression: str, arg: Any = None) -> Any:+ if expression == browser_ops._DOMUTILS_INTERACTABILITY_READY_JS:+ return True+ if expression == browser_ops._OBSERVE_INTERACTABLES_JS:+ # The swap lands during extraction, after the opening marker read.+ page._loader_id = "doc-2"+ return []+ if expression == browser_ops._OBSERVE_PAGE_TEXT_JS:+ return {"content": "", "truncated": False}+ raise AssertionError(expression)++ page.evaluate = AsyncMock(side_effect=evaluate)+ swapped = await browser_ops.do_observe(page)+ assert swapped.document_id is None++ stable = _page()++ async def stable_evaluate(expression: str, arg: Any = None) -> Any:+ if expression == browser_ops._DOMUTILS_INTERACTABILITY_READY_JS:+ return True+ if expression == browser_ops._OBSERVE_INTERACTABLES_JS:+ return []+ if expression == browser_ops._OBSERVE_PAGE_TEXT_JS:+ return {"content": "", "truncated": False}+ raise AssertionError(expression)++ stable.evaluate = AsyncMock(side_effect=stable_evaluate)+ kept = await browser_ops.do_observe(stable)+ assert kept.document_id == "cdp:doc-1"+++async def test_page_sourced_marker_rejects_in_batch_refs_too(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """The rollout-gate limitation, locked: with only a page-sourced marker available, refs+ minted by an observe are unusable even inside the same skyvern_execute batch — fail closed,+ never a spoofable sameness check. Selector/intent params remain the working path there."""monkeypatch.setenv(_FLAG, "1")page = _page(loader_id=None)ctx = BrowserContext(mode="local")@@ -1387,3 +2752,368 @@ async def test_page_sourced_marker_rejects_in_batch_refs_too(assert "Unknown ref" in executed["data"]["results"][1]["error"]assert observe.await_count == 1click.assert_not_awaited()+++async def test_flag_off_observe_refuses_publication_after_concurrent_invalidation(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """The generation guard predates observe-v2: with the flag OFF, an in-flight observe+ racing a concurrent invalidation must not republish its stale snapshot."""+ monkeypatch.delenv(_FLAG, raising=False)+ page = _page()+ ctx = BrowserContext(mode="local")+ state = make_session_state(context=ctx)+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))++ async def racing_observe(*_args: Any, **_kwargs: Any) -> ObserveResult:+ # A concurrent navigation invalidates the registry while the snapshot is in flight.+ session_manager.invalidate_session_ref_map()+ return _result(page.url, "#stale")++ monkeypatch.setattr(mcp_browser, "do_observe", AsyncMock(side_effect=racing_observe))++ async with scoped_session(state):+ observed = await mcp_browser.skyvern_observe()++ # Refusal is silent (the pre-v2 contract): the response is fine, but the+ # superseded snapshot never reaches the registry.+ assert observed["ok"] is True+ assert state._observed_refs == {}+++async def test_flag_off_batch_observe_refuses_publication_after_concurrent_invalidation(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """Same guard for the inline-observe batch path: flag OFF, the generation read at+ observe dispatch must refuse publication after a concurrent invalidation."""+ monkeypatch.delenv(_FLAG, raising=False)+ page = _page()+ ctx = BrowserContext(mode="local")+ state = make_session_state(context=ctx)+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))++ async def racing_observe(*_args: Any, **_kwargs: Any) -> ObserveResult:+ session_manager.invalidate_session_ref_map()+ return _result(page.url, "#stale")++ monkeypatch.setattr(browser_ops, "do_observe", AsyncMock(side_effect=racing_observe))++ async with scoped_session(state):+ executed = await mcp_browser.skyvern_execute(steps=[{"tool": "observe", "params": {}}])++ assert executed["ok"] is True+ assert state._observed_refs == {}+++async def test_flag_off_failed_observe_preserves_existing_refs(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """Pre-v2 contract: failed observes (no browser, generic failure) leave the+ published registry alone - only a frame error clears it."""+ monkeypatch.delenv(_FLAG, raising=False)+ page = _page()+ ctx = BrowserContext(mode="local")+ state = make_session_state(context=ctx)+ existing = {"page_key": None, "refs": {"e0": {"ref": "e0", "selector": "#existing"}}}+ state._observed_refs = dict(existing)+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))+ monkeypatch.setattr(mcp_browser, "do_observe", AsyncMock(side_effect=RuntimeError("snapshot failed")))++ async with scoped_session(state):+ observed = await mcp_browser.skyvern_observe()++ assert observed["ok"] is False+ assert state._observed_refs == existing+++async def test_flag_off_failed_overlapping_observe_does_not_erase_newer_publication(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """A failing older observe must not erase refs a newer observe published while+ it was in flight (main preserved them; legacy publications never advance)."""+ monkeypatch.delenv(_FLAG, raising=False)+ page = _page()+ ctx = BrowserContext(mode="local")+ state = make_session_state(context=ctx)+ slow_started = asyncio.Event()+ finish_slow = asyncio.Event()+ call_count = 0++ async def observe(_page: Any, **_kwargs: Any) -> ObserveResult:+ nonlocal call_count+ call_count += 1+ if call_count == 1:+ slow_started.set()+ await finish_slow.wait()+ raise RuntimeError("snapshot failed")+ return _result(page.url, "#fresh")++ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))+ monkeypatch.setattr(mcp_browser, "do_observe", AsyncMock(side_effect=observe))++ async with scoped_session(state):+ slow_task = asyncio.create_task(mcp_browser.skyvern_observe())+ await slow_started.wait()+ fast_result = await mcp_browser.skyvern_observe()+ finish_slow.set()+ slow_result = await slow_task+ published = session_manager.get_session_ref("e0", page_key=session_manager.page_ref_key(page))++ assert fast_result["ok"] is True+ assert slow_result["ok"] is False+ assert published is not None+ assert published["selector"] == "#fresh"+++async def test_flag_off_batch_failed_observe_preserves_newer_publication(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """Inline-batch variant: a failing batch observe must not erase refs published+ by an overlapping standalone observe."""+ monkeypatch.delenv(_FLAG, raising=False)+ page = _page()+ ctx = BrowserContext(mode="local")+ state = make_session_state(context=ctx)+ batch_started = asyncio.Event()+ finish_batch = asyncio.Event()++ async def batch_observe(_page: Any, **_kwargs: Any) -> ObserveResult:+ batch_started.set()+ await finish_batch.wait()+ raise RuntimeError("snapshot failed")++ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))+ monkeypatch.setattr(browser_ops, "do_observe", AsyncMock(side_effect=batch_observe))+ monkeypatch.setattr(mcp_browser, "do_observe", AsyncMock(return_value=_result(page.url, "#standalone")))++ async with scoped_session(state):+ batch_task = asyncio.create_task(mcp_browser.skyvern_execute(steps=[{"tool": "observe", "params": {}}]))+ await batch_started.wait()+ standalone = await mcp_browser.skyvern_observe()+ finish_batch.set()+ executed = await batch_task+ published = session_manager.get_session_ref("e0", page_key=session_manager.page_ref_key(page))++ assert standalone["ok"] is True+ assert executed["ok"] is False+ assert published is not None+ assert published["selector"] == "#standalone"+++async def test_flag_off_inline_observe_publishes_without_second_page_lookup(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """Pre-v2 contract: a successful inline observe publishes immediately - a page+ lookup failure after the observe step must not discard the publication."""+ monkeypatch.delenv(_FLAG, raising=False)+ page = _page()+ ctx = BrowserContext(mode="local")+ state = make_session_state(context=ctx)+ state._observed_refs = {"page_key": None, "refs": {"e0": {"ref": "e0", "selector": "#existing"}}}+ get_page_calls = 0++ async def flaky_get_page(**_kwargs: Any) -> tuple[Any, BrowserContext]:+ nonlocal get_page_calls+ get_page_calls += 1+ if get_page_calls > 2:+ raise RuntimeError("browser went away")+ return (page, ctx)++ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(side_effect=flaky_get_page))+ monkeypatch.setattr(browser_ops, "do_observe", AsyncMock(return_value=_result(page.url, "#fresh")))++ async with scoped_session(state):+ executed = await mcp_browser.skyvern_execute(steps=[{"tool": "observe", "params": {}}])+ published = session_manager.get_session_ref("e0", page_key=session_manager.page_ref_key(page))++ assert executed["ok"] is True+ assert published is not None+ assert published["selector"] == "#fresh"+++async def test_flag_off_later_failed_observe_keeps_earlier_inline_publication(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """Pre-v2 contract: each successful inline observe publishes as it lands - a later+ failing observe in the same batch must not discard the earlier publication."""+ monkeypatch.delenv(_FLAG, raising=False)+ page = _page()+ ctx = BrowserContext(mode="local")+ state = make_session_state(context=ctx)+ call_count = 0++ async def observe(_page: Any, **_kwargs: Any) -> ObserveResult:+ nonlocal call_count+ call_count += 1+ if call_count == 1:+ return _result(page.url, "#first")+ raise RuntimeError("snapshot failed")++ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))+ monkeypatch.setattr(browser_ops, "do_observe", AsyncMock(side_effect=observe))++ async with scoped_session(state):+ executed = await mcp_browser.skyvern_execute(+ steps=[+ {"tool": "observe", "params": {}},+ {"tool": "observe", "params": {}},+ ]+ )+ published = session_manager.get_session_ref("e0", page_key=session_manager.page_ref_key(page))++ assert executed["ok"] is False+ assert published is not None+ assert published["selector"] == "#first"+++async def test_click_driven_navigation_unscopes_post_batch_observe(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """M2 lock: a click that replaces the document (no navigate step) must not let the+ old document's aria-controls id scope the new document's auto-observe."""+ monkeypatch.setenv(_FLAG, "1")+ page = _page()+ ctx = BrowserContext(mode="local")+ state = make_session_state(context=ctx)+ page.page.query_selector = AsyncMock(return_value=object())+ page.page.wait_for_timeout = AsyncMock()+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))+ monkeypatch.setattr(+ mcp_browser,+ "do_observe",+ AsyncMock(return_value=_result(page.url, "#control", aria_controls="panel")),+ )+ observe_calls = 0++ async def observe_after_action(_page: Any, **kwargs: Any) -> ObserveResult:+ nonlocal observe_calls+ observe_calls += 1+ if observe_calls == 1:+ return _result(page.url, "#control", aria_controls="panel")+ return _result(page.url, "#new-doc-content", document_id="cdp:doc-2")++ automatic = AsyncMock(side_effect=observe_after_action)+ monkeypatch.setattr(browser_ops, "do_observe", automatic)++ async def click(**_kwargs: Any) -> dict[str, Any]:+ # The click handler triggers a same-URL document replacement.+ page._loader_id = "doc-2"+ return {"ok": True, "data": None}++ monkeypatch.setattr(mcp_browser, "skyvern_click", AsyncMock(side_effect=click))++ async with scoped_session(state):+ await mcp_browser.skyvern_observe()+ executed = await mcp_browser.skyvern_execute(steps=[{"tool": "click", "params": {"ref": "e0"}}])++ assert executed["ok"] is True+ # The acted element was stamped with the pre-click document; the post-batch+ # document differs, so the auto-observe must run unscoped even though the+ # "panel" id would attach on the new document.+ final_observe = automatic.await_args_list[-1]+ assert final_observe.kwargs.get("selector") is None+++async def test_untrusted_marker_stable_observe_batch_skips_auto_observe(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """C2 lock: on a page with no CDP loaderId (page:-sourced marker), a non-mutating+ observe batch must not pay an invalidate + second full observe every time."""+ monkeypatch.setenv(_FLAG, "1")+ page = _page(loader_id=None)+ ctx = BrowserContext(mode="local")+ state = make_session_state(context=ctx)+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))+ inline = AsyncMock(return_value=_result(page.url, "#control", document_id="page:doc-1"))+ monkeypatch.setattr(browser_ops, "do_observe", inline)++ async with scoped_session(state):+ executed = await mcp_browser.skyvern_execute(steps=[{"tool": "observe", "params": {}}])++ assert executed["ok"] is True+ # Trusted comparison on both sides (None == None): nothing changed, no refresh.+ assert inline.await_count == 1+ assert "auto_observe" not in executed["data"]+++def test_generation_advance_lru_touches_key() -> None:+ """M3 lock: advancing a generation must reorder the key to most-recent so FIFO+ eviction cannot drop the busiest session's in-flight reservation."""+ saved = dict(session_manager._session_ref_generations)+ session_manager._session_ref_generations.clear()+ try:+ keys = [(None, "cloud_session", f"s{i}", None) for i in range(session_manager._SESSION_REF_STORE_MAX)]+ for key in keys:+ session_manager._generation_for(key)+ # Advance the oldest key, then insert one more to trigger eviction.+ advanced = session_manager._advance_generation_for(keys[0])+ session_manager._generation_for((None, "cloud_session", "overflow", None))+ assert session_manager._session_ref_generations.get(keys[0]) == advanced+ assert keys[1] not in session_manager._session_ref_generations+ finally:+ session_manager._session_ref_generations.clear()+ session_manager._session_ref_generations.update(saved)+++async def test_navigation_during_aria_attachment_check_discards_scoped_snapshot(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """M2 timing lock: the attachment check awaits between document certification and+ the scoped observe. A navigation inside that window must discard the scoped+ snapshot (it certified the wrong document) and rerun unscoped."""+ monkeypatch.setenv(_FLAG, "1")+ page = _page()+ ctx = BrowserContext(mode="local")+ state = make_session_state(context=ctx)+ page.page.wait_for_timeout = AsyncMock()++ async def navigating_query_selector(_selector: str) -> object:+ # The document is replaced while the attachment check is in flight; the+ # "panel" id also exists on the new document, so attachment succeeds.+ page._loader_id = "doc-2"+ return object()++ page.page.query_selector = AsyncMock(side_effect=navigating_query_selector)+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))+ monkeypatch.setattr(+ mcp_browser,+ "do_observe",+ AsyncMock(return_value=_result(page.url, "#control", aria_controls="panel")),+ )+ observe_calls = 0++ async def observe_current_document(_page: Any, **kwargs: Any) -> ObserveResult:+ nonlocal observe_calls+ observe_calls += 1+ if observe_calls == 1:+ return _result(page.url, "#control", aria_controls="panel")+ document_id = f"cdp:{page._loader_id}"+ selector = "#panel-content" if kwargs.get("selector") else "#new-doc-unscoped"+ return _result(page.url, selector, document_id=document_id)++ automatic = AsyncMock(side_effect=observe_current_document)+ monkeypatch.setattr(browser_ops, "do_observe", automatic)+ monkeypatch.setattr(mcp_browser, "skyvern_type", AsyncMock(return_value={"ok": True, "data": None}))++ async with scoped_session(state):+ await mcp_browser.skyvern_observe()+ executed = await mcp_browser.skyvern_execute(steps=[{"tool": "type", "params": {"ref": "e0", "text": "San"}}])++ assert executed["ok"] is True+ # Call 1: pre-mutation ref refresh. Call 2: scoped observe that certified the+ # replacement document. Call 3: the discard-and-rerun, unscoped.+ assert automatic.await_count == 3+ assert automatic.await_args_list[1].kwargs.get("selector") == '[id="panel"]'+ assert automatic.await_args_list[2].kwargs.get("selector") is None+ # Only the unscoped snapshot of the new document is published.+ assert {element["selector"] for element in state._observe_v2_state.refs.values()} == {"#new-doc-unscoped"}diff --git a/tests/unit/test_mcp_response_size_cap.py b/tests/unit/test_mcp_response_size_cap.pyindex 1e30d8881..3fb2e9937 100644--- a/tests/unit/test_mcp_response_size_cap.py+++ b/tests/unit/test_mcp_response_size_cap.py@@ -14,9 +14,11 @@ from skyvern.cli.core.result import Artifact, BrowserContextfrom skyvern.cli.mcp_tools import browser as mcp_browserfrom skyvern.cli.mcp_tools import mcpfrom skyvern.cli.mcp_tools.response import (+ MCP_MAX_RESPONSE_BYTES,MCP_MAX_RESPONSE_CHARS,size_capped,truncate_response,+ truncate_response_bytes,)@@ -25,6 +27,18 @@ def test_truncate_response_passes_small_payload_unchanged() -> None:assert truncate_response(small) is small+def test_truncate_response_bytes_caps_multibyte_payload() -> None:+ payload = {"ok": True, "data": {"body": "é" * (MCP_MAX_RESPONSE_BYTES // 2)}}++ result = truncate_response_bytes(payload)++ assert result["_truncated"] is True+ assert result["_max_bytes"] == MCP_MAX_RESPONSE_BYTES+ assert result["_original_bytes"] > MCP_MAX_RESPONSE_BYTES+ assert len(json.dumps(result, ensure_ascii=False).encode()) <= MCP_MAX_RESPONSE_BYTES+ assert "data" not in result++def test_truncate_response_wraps_large_payload_with_envelope() -> None:# Construct a payload larger than the default cap.big_payload = "x" * (MCP_MAX_RESPONSE_CHARS + 100)diff --git a/tests/unit/test_mcp_session_lifecycle.py b/tests/unit/test_mcp_session_lifecycle.pyindex 6d39a746d..eb63310a3 100644--- a/tests/unit/test_mcp_session_lifecycle.py+++ b/tests/unit/test_mcp_session_lifecycle.py@@ -993,6 +993,49 @@ def test_session_ref_lookup_is_scoped_to_authenticated_organization() -> None:assert session_manager.get_session_ref("e0", session_id=session_id, page_key=page_key) is None+def test_observe_ref_generations_remain_monotonic_after_clear() -> None:+ session_manager.set_current_session(session_manager.SessionState())++ first = session_manager.begin_session_ref_publication()+ session_manager.clear_session_ref_map(generation=first)+ second = session_manager.begin_session_ref_publication()++ assert second > first+++def test_older_publication_cannot_commit_after_newer_reservation() -> None:+ session_manager.set_current_session(session_manager.SessionState())+ page_key = (1, None, "https://example.com", None)++ older = session_manager.begin_session_ref_publication()+ newer = session_manager.begin_session_ref_publication()++ assert not session_manager.replace_session_ref_map({"e0": {"tag": "button"}}, generation=older, page_key=page_key)+ assert session_manager.replace_session_ref_map({"e1": {"tag": "input"}}, generation=newer, page_key=page_key)+ assert session_manager.session_ref_generation() > newer+++def test_mutation_invalidation_clears_v2_refs_but_preserves_monotonic_ids() -> None:+ state = session_manager.SessionState()+ state._observe_v2_state.refs = {"e0": {"tag": "button"}}+ state._observe_v2_state.next_ref = 7+ session_manager.set_current_session(state)+ page_key = (1, None, "https://example.com", None)+ generation = session_manager.begin_session_ref_publication()+ assert session_manager.replace_session_ref_map(+ {"e0": {"tag": "button"}},+ generation=generation,+ page_key=page_key,+ )++ invalidated_generation = session_manager.invalidate_session_ref_map()++ assert invalidated_generation > generation+ assert session_manager.get_session_ref("e0", page_key=page_key) is None+ assert state._observe_v2_state.refs == {}+ assert state._observe_v2_state.next_ref == 7++# ---------------------------------------------------------------------------# Tests for stateless HTTP mode session creation# ---------------------------------------------------------------------------diff --git a/tests/unit/test_mcp_telemetry.py b/tests/unit/test_mcp_telemetry.pyindex f837d19a7..c551facb5 100644--- a/tests/unit/test_mcp_telemetry.py+++ b/tests/unit/test_mcp_telemetry.py@@ -1,14 +1,20 @@from __future__ import annotations+import asynciofrom collections.abc import Iteratorfrom types import SimpleNamespace-from unittest.mock import Mock, patch+from unittest.mock import AsyncMock, Mock, patchimport pytestfrom fastmcp import Clientfrom fastmcp.server.middleware import MiddlewareContextfrom skyvern import analytics+from skyvern.cli.core.perception_telemetry import (+ PerceptionSnapshotCategory,+ track_perception_probe,+ track_perception_snapshot,+)from skyvern.cli.mcp_tools import mcpfrom skyvern.cli.mcp_tools.blocks import skyvern_block_schemafrom skyvern.cli.mcp_tools.telemetry import (@@ -237,6 +243,231 @@ async def test_mcp_tool_call_exception_preserves_original_error_when_telemetry_fawait MCPTelemetryMiddleware().on_call_tool(context, call_next)+async def test_cancelled_perception_snapshot_emits_once_and_restores_context() -> None:+ events: list[tuple[str, dict]] = []+ cancelled_context = MiddlewareContext(+ message=SimpleNamespace(name="skyvern_observe"),+ fastmcp_context=None,+ )++ async def cancelled_call(_context: MiddlewareContext[object]) -> object:+ async with track_perception_snapshot("stale_ref_refresh"):+ async with track_perception_probe(evaluate_page_scan=True):+ raise asyncio.CancelledError++ following_context = MiddlewareContext(message=SimpleNamespace(name="skyvern_click"), fastmcp_context=None)++ async def following_call(_context: MiddlewareContext[object]) -> object:+ return SimpleNamespace(is_error=False, data={"ok": True}, content=[])++ def capture(event: str, *, data: dict, **_kwargs: object) -> None:+ events.append((event, data))++ clock = iter([0.0, 1.0, 3.0, 4.0, 5.0, 6.0])+ with (+ patch("skyvern.cli.mcp_tools.telemetry.time.perf_counter", side_effect=lambda: next(clock)),+ patch.object(analytics, "capture", side_effect=capture),+ ):+ with pytest.raises(asyncio.CancelledError):+ await MCPTelemetryMiddleware().on_call_tool(cancelled_context, cancelled_call)+ await MCPTelemetryMiddleware().on_call_tool(following_context, following_call)++ assert len(events) == 2+ cancelled_event_name, cancelled_payload = events[0]+ assert cancelled_event_name == "mcp_tool_call"+ assert cancelled_payload["ok"] is False+ assert cancelled_payload["error_type"] == "CancelledError"+ assert cancelled_payload["top_level_mcp_calls"] == 1+ assert cancelled_payload["perception_snapshots"] == 1+ assert cancelled_payload["model_visible_observe_results"] == 0+ assert cancelled_payload["automatic_observe_snapshots"] == 0+ assert cancelled_payload["stale_ref_refresh_snapshots"] == 1+ assert (+ cancelled_payload["model_visible_observe_results"]+ + cancelled_payload["automatic_observe_snapshots"]+ + cancelled_payload["stale_ref_refresh_snapshots"]+ == cancelled_payload["perception_snapshots"]+ )+ assert cancelled_payload["failed_perception_probes"] == 1+ assert cancelled_payload["evaluate_page_scans"] == 1+ assert cancelled_payload["browser_perception_wall_ms"] == 2000+ assert isinstance(cancelled_payload["browser_perception_wall_ms"], int)+ assert "response_bytes" not in cancelled_payload++ following_event_name, following_payload = events[1]+ assert following_event_name == "mcp_tool_call"+ assert following_payload["ok"] is True+ assert following_payload["top_level_mcp_calls"] == 1+ assert following_payload["perception_snapshots"] == 0+ assert following_payload["failed_perception_probes"] == 0+ assert following_payload["evaluate_page_scans"] == 0+ assert following_payload["browser_perception_wall_ms"] == 0+++async def test_cancelled_tool_call_propagates_when_telemetry_fails() -> None:+ context = MiddlewareContext(message=SimpleNamespace(name="skyvern_observe"), fastmcp_context=None)+ cancellation = asyncio.CancelledError()++ async def call_next(_context: MiddlewareContext[object]) -> object:+ async with track_perception_snapshot("automatic"):+ raise cancellation++ with (+ patch.object(analytics, "capture", side_effect=RuntimeError("telemetry down")),+ pytest.raises(asyncio.CancelledError) as exc_info,+ ):+ await MCPTelemetryMiddleware().on_call_tool(context, call_next)++ assert exc_info.value is cancellation+++async def test_perception_counters_emit_on_exception() -> None:+ events: list[dict] = []+ context = MiddlewareContext(message=SimpleNamespace(name="skyvern_observe"), fastmcp_context=None)++ async def call_next(_context: MiddlewareContext[object]) -> object:+ async with track_perception_snapshot("model_visible"):+ raise RuntimeError("observe failed")++ with (+ patch.object(analytics, "capture", side_effect=lambda _event, *, data, **_kwargs: events.append(data)),+ pytest.raises(RuntimeError, match="observe failed"),+ ):+ await MCPTelemetryMiddleware().on_call_tool(context, call_next)++ assert events[0]["ok"] is False+ assert events[0]["perception_snapshots"] == 1+ assert events[0]["model_visible_observe_results"] == 1+ assert events[0]["automatic_observe_snapshots"] == 0+ assert events[0]["stale_ref_refresh_snapshots"] == 0+ assert events[0]["failed_perception_probes"] == 1+ assert isinstance(events[0]["browser_perception_wall_ms"], int)+++async def test_swallowed_evaluate_scan_failure_is_accounted() -> None:+ from skyvern.cli.core.browser_ops import _get_dom_observe_elements++ events: list[dict] = []+ context = MiddlewareContext(message=SimpleNamespace(name="skyvern_observe"), fastmcp_context=None)+ page = SimpleNamespace(evaluate=AsyncMock(side_effect=RuntimeError("scan failed")))++ async def call_next(_context: MiddlewareContext[object]) -> object:+ assert await _get_dom_observe_elements(page) == []+ return SimpleNamespace(is_error=False, data={"ok": True}, content=[])++ with patch.object(analytics, "capture", side_effect=lambda _event, *, data, **_kwargs: events.append(data)):+ await MCPTelemetryMiddleware().on_call_tool(context, call_next)++ assert events[0]["perception_snapshots"] == 0+ assert events[0]["evaluate_page_scans"] == 1+ assert events[0]["failed_perception_probes"] == 1+ assert isinstance(events[0]["browser_perception_wall_ms"], int)+++async def test_document_id_cdp_failures_and_fallback_are_accounted() -> None:+ from skyvern.cli.core.browser_ops import get_observe_document_id++ events: list[dict] = []+ context = MiddlewareContext(message=SimpleNamespace(name="skyvern_observe"), fastmcp_context=None)+ cached_cdp = SimpleNamespace(send=AsyncMock(side_effect=RuntimeError("stale CDP session")))+ new_cdp_session = AsyncMock(side_effect=RuntimeError("CDP attach failed"))+ raw_page = SimpleNamespace(+ context=SimpleNamespace(new_cdp_session=new_cdp_session),+ _skyvern_observe_cdp_session=cached_cdp,+ )+ page = SimpleNamespace(+ page=raw_page,+ _working_frame=None,+ evaluate=AsyncMock(return_value="doc-1"),+ )++ async def call_next(_context: MiddlewareContext[object]) -> object:+ assert await get_observe_document_id(page) == "page:doc-1"+ return SimpleNamespace(is_error=False, data={"ok": True}, content=[])++ clock = iter([0.0, 1.0, 2.0, 3.0, 5.0, 6.0, 9.0, 10.0])+ with (+ patch("skyvern.cli.mcp_tools.telemetry.time.perf_counter", side_effect=lambda: next(clock)),+ patch.object(analytics, "capture", side_effect=lambda _event, *, data, **_kwargs: events.append(data)),+ ):+ await MCPTelemetryMiddleware().on_call_tool(context, call_next)++ cached_cdp.send.assert_awaited_once_with("Page.getFrameTree")+ new_cdp_session.assert_awaited_once_with(raw_page)+ page.evaluate.assert_awaited_once()+ assert events[0]["perception_snapshots"] == 0+ assert events[0]["failed_perception_probes"] == 2+ assert events[0]["evaluate_page_scans"] == 0+ assert events[0]["browser_perception_wall_ms"] == 6000+++async def test_nested_evaluate_scan_does_not_double_count_snapshot_wall_time() -> None:+ events: list[dict] = []+ context = MiddlewareContext(message=SimpleNamespace(name="skyvern_observe"), fastmcp_context=None)++ async def call_next(_context: MiddlewareContext[object]) -> object:+ async with track_perception_snapshot("model_visible"):+ async with track_perception_probe(evaluate_page_scan=True):+ pass+ return SimpleNamespace(is_error=False, data={"ok": True}, content=[])++ clock = iter([0.0, 0.001, 0.008, 0.010])+ with (+ patch("skyvern.cli.mcp_tools.telemetry.time.perf_counter", side_effect=lambda: next(clock)),+ patch.object(analytics, "capture", side_effect=lambda _event, *, data, **_kwargs: events.append(data)),+ ):+ await MCPTelemetryMiddleware().on_call_tool(context, call_next)++ assert events[0]["perception_snapshots"] == 1+ assert events[0]["evaluate_page_scans"] == 1+ assert events[0]["browser_perception_wall_ms"] == 7+++async def test_perception_counters_are_isolated_across_concurrent_calls() -> None:+ events: list[dict] = []+ both_started = asyncio.Event()+ started = 0++ async def run(tool: str, category: PerceptionSnapshotCategory) -> None:+ context = MiddlewareContext(message=SimpleNamespace(name=tool), fastmcp_context=None)++ async def call_next(_context: MiddlewareContext[object]) -> object:+ nonlocal started+ started += 1+ if started == 2:+ both_started.set()+ await both_started.wait()+ async with track_perception_snapshot(category):+ if category == "automatic":+ async with track_perception_probe(evaluate_page_scan=True):+ pass+ return SimpleNamespace(is_error=False, data={"ok": True}, content=[])++ await MCPTelemetryMiddleware().on_call_tool(context, call_next)++ with patch.object(analytics, "capture", side_effect=lambda _event, *, data, **_kwargs: events.append(data)):+ await asyncio.gather(run("explicit", "model_visible"), run("bundled", "automatic"))++ by_tool = {event["tool"]: event for event in events}+ assert by_tool["explicit"]["top_level_mcp_calls"] == 1+ assert by_tool["explicit"]["perception_snapshots"] == 1+ assert by_tool["explicit"]["model_visible_observe_results"] == 1+ assert by_tool["explicit"]["automatic_observe_snapshots"] == 0+ assert by_tool["explicit"]["evaluate_page_scans"] == 0+ assert by_tool["bundled"]["top_level_mcp_calls"] == 1+ assert by_tool["bundled"]["perception_snapshots"] == 1+ assert by_tool["bundled"]["model_visible_observe_results"] == 0+ assert by_tool["bundled"]["automatic_observe_snapshots"] == 1+ assert by_tool["bundled"]["evaluate_page_scans"] == 1++@pytest.mark.asyncioasync def test_list_tools_emits_protocol_request_telemetry() -> None:events: list[tuple[str, dict | None, str | None, str | None, str | None]] = []