Skip to content

Automate browser based workflows with AI

Recover blank multi-entry Workday sections via section-scoped add-control fallback (#8215)

Trevor Cheung586015645ce2
diff --git a/skyvern/cli/core/browser_ops.py b/skyvern/cli/core/browser_ops.py
index ae0c94c81..6794f7167 100644
--- a/skyvern/cli/core/browser_ops.py
+++ b/skyvern/cli/core/browser_ops.py
@@ -14,7 +14,7 @@ import time
from dataclasses import dataclass, field
from datetime import datetime, timezone
from pathlib import Path
-from typing import Any, Callable, Literal
+from typing import Any, Awaitable, Callable, Literal
import structlog
from playwright.async_api import TimeoutError as PlaywrightTimeoutError
@@ -25,6 +25,7 @@ from skyvern.utils.url_validators import validate_fetch_url
from skyvern.webeye.utils.page import JS_FUNCTION_DEFS, SkyvernFrame
from .guards import CREDENTIAL_HINT, GuardError, validate_wait_until
+from .perception_telemetry import track_perception_probe
LOG = structlog.get_logger(__name__)
@@ -849,6 +850,15 @@ async ({ scopeSelector, includeValues }) => {
.map((element) => {
const tag = element.tagName.toLowerCase();
const item = { role: roleFor(element), name: text(element), tag, selector: cssPath(element) };
+ const safeIdRefs = (attribute) => (element.getAttribute(attribute) || "")
+ .split(/\s+/)
+ .filter((value) => /^[A-Za-z_][A-Za-z0-9_.:-]{0,127}$/.test(value))
+ .slice(0, 8)
+ .join(" ");
+ const ariaControls = safeIdRefs("aria-controls");
+ const ariaOwns = safeIdRefs("aria-owns");
+ if (ariaControls) item.aria_controls = ariaControls;
+ if (ariaOwns) item.aria_owns = ariaOwns;
if (includeValues === true && !isPassword(element) && ["input", "textarea", "select", "option"].includes(tag)) {
item.value = element.value || element.getAttribute("value") || "";
}
@@ -1027,6 +1037,8 @@ class ObservedElement:
selector: str | None = None
value: str | None = None
options: list[str] | None = None
+ aria_controls: str | None = None
+ aria_owns: str | None = None
match_index: int = 0
needs_disambiguation: bool = False
@@ -1091,13 +1103,14 @@ async def _get_dom_observe_elements(
raise ObserveFrameError(frame_name, frame_url or "", RuntimeError("frame does not support evaluate"))
return []
try:
- async with asyncio.timeout(_DOM_SCAN_TIMEOUT_SECONDS):
- if await evaluate(_DOMUTILS_INTERACTABILITY_READY_JS) is not True:
- await evaluate(JS_FUNCTION_DEFS)
- result = await evaluate(
- _OBSERVE_INTERACTABLES_JS,
- {"scopeSelector": selector, "includeValues": include_values},
- )
+ async with track_perception_probe(evaluate_page_scan=True):
+ async with asyncio.timeout(_DOM_SCAN_TIMEOUT_SECONDS):
+ if await evaluate(_DOMUTILS_INTERACTABILITY_READY_JS) is not True:
+ await evaluate(JS_FUNCTION_DEFS)
+ result = await evaluate(
+ _OBSERVE_INTERACTABLES_JS,
+ {"scopeSelector": selector, "includeValues": include_values},
+ )
except Exception as exc:
if frame_name is not None:
raise ObserveFrameError(frame_name, frame_url or "", exc) from exc
@@ -1152,6 +1165,10 @@ def _merge_dom_observe_elements(
existing["value"] = dom_element["value"]
if dom_element.get("options") and not existing.get("children"):
existing["options"] = dom_element.get("options")
+ if dom_element.get("aria_controls"):
+ existing["aria_controls"] = dom_element["aria_controls"]
+ if dom_element.get("aria_owns"):
+ existing["aria_owns"] = dom_element["aria_owns"]
matched_existing = True
if dom_selector:
selector_seen.add(dom_selector)
@@ -1176,18 +1193,16 @@ async def get_observe_document_id(page: Any) -> str | None:
# dead (page closed, target gone), so that case evicts and retries once before falling back.
attempts = 2 if cdp is not None else 1
for _ in range(attempts):
- if cdp is None:
- try:
- cdp = await raw_page.context.new_cdp_session(session_target)
- except Exception:
- break
- try:
- session_target._skyvern_observe_cdp_session = cdp
- except Exception:
- pass
try:
- frame_tree = await cdp.send("Page.getFrameTree")
- loader_id = frame_tree.get("frameTree", {}).get("frame", {}).get("loaderId")
+ async with track_perception_probe():
+ if cdp is None:
+ cdp = await raw_page.context.new_cdp_session(session_target)
+ try:
+ session_target._skyvern_observe_cdp_session = cdp
+ except Exception:
+ pass
+ frame_tree = await cdp.send("Page.getFrameTree")
+ loader_id = frame_tree.get("frameTree", {}).get("frame", {}).get("loaderId")
except Exception:
if getattr(session_target, "_skyvern_observe_cdp_session", None) is cdp:
try:
@@ -1200,7 +1215,8 @@ async def get_observe_document_id(page: Any) -> str | None:
return f"cdp:{loader_id}"
break
try:
- value = await target.evaluate(_OBSERVE_DOCUMENT_ID_JS)
+ async with track_perception_probe():
+ value = await target.evaluate(_OBSERVE_DOCUMENT_ID_JS)
except Exception:
return None
return f"page:{value}" if isinstance(value, str) else None
@@ -1240,10 +1256,11 @@ async def do_observe(
working_frame.url,
RuntimeError("frame is detached"),
)
- observe_target = working_frame if working_frame is not None else page
document_id_before: str | None = None
if observe_v2_enabled():
document_id_before = await get_observe_document_id(page)
+
+ observe_target = working_frame if working_frame is not None else page
if working_frame is not None:
# Legacy page accessibility iframe roots are unreliable; DOM-only avoids cross-document merges.
snapshot = None
@@ -1271,7 +1288,8 @@ async def do_observe(
document_id: str | None = None
if observe_v2_enabled():
try:
- text_context = await observe_target.evaluate(_OBSERVE_PAGE_TEXT_JS, {"scopeSelector": selector})
+ async with track_perception_probe(evaluate_page_scan=True):
+ text_context = await observe_target.evaluate(_OBSERVE_PAGE_TEXT_JS, {"scopeSelector": selector})
if isinstance(text_context, dict) and isinstance(text_context.get("content"), str):
page_text = text_context["content"]
page_text_truncated = text_context.get("truncated") is True
@@ -1331,6 +1349,8 @@ async def do_observe(
selector=elem.get("selector"),
value=elem.get("value"),
options=elem.get("options") or _extract_options(elem),
+ aria_controls=elem.get("aria_controls"),
+ aria_owns=elem.get("aria_owns"),
match_index=match_index,
needs_disambiguation=full_group_size[key] > 1,
)
@@ -1344,6 +1364,12 @@ async def do_observe(
else:
title = await page.title()
+ if observe_v2_enabled() and document_id is not None:
+ # Extend the bracket over the title read: a swap landing after extraction but before
+ # return refuses durability the same way — degrade to None, never publish torn identity.
+ if await get_observe_document_id(page) != document_id:
+ document_id = None
+
return ObserveResult(
url=working_frame.url if working_frame is not None else page.url,
title=title,
@@ -1369,6 +1395,9 @@ def serialize_elements(elements: list[ObservedElement]) -> list[dict[str, Any]]:
"value": e.value,
"options": e.options,
}
+ if observe_v2_enabled():
+ fields["aria_controls"] = e.aria_controls
+ fields["aria_owns"] = e.aria_owns
if e.needs_disambiguation:
fields["match_index"] = e.match_index
result.append({k: v for k, v in fields.items() if k in _ELEMENT_KEEP_ALWAYS or (v is not None and v != "")})
@@ -1459,7 +1488,8 @@ async def do_execute(
dispatch_fn: Any,
steps: list[ExecuteStep],
stop_on_error: bool = True,
- on_ref_map_update: Callable[[dict[str, dict[str, Any]]], bool] | None = None,
+ on_ref_map_update: Callable[[dict[str, dict[str, Any]]], bool | Awaitable[bool]] | None = None,
+ fail_on_ref_map_rejection: bool = False,
) -> ExecuteResult:
"""Execute a sequence of deterministic browser operations in one batch.
@@ -1490,14 +1520,16 @@ async def do_execute(
try:
result = await dispatch_fn(step, ref_map)
# DESIGN-4: Each observe REPLACES the entire ref_map (not merges).
- # If session publication rejects the snapshot (a concurrent navigation
- # invalidated it), the batch must not act on it either.
if step.tool == "observe" and result and "elements" in result:
new_map = ref_map_from_elements(result["elements"])
- if on_ref_map_update is None or on_ref_map_update(new_map):
- ref_map = new_map
+ if on_ref_map_update is None:
+ accepted = True
else:
- ref_map = {}
+ publication = on_ref_map_update(new_map)
+ accepted = publication if isinstance(publication, bool) else await publication
+ if not accepted and fail_on_ref_map_rejection:
+ raise RuntimeError("observe_snapshot_superseded: ref publication was rejected")
+ ref_map = new_map if accepted else {}
wall_ms = int((time.monotonic() - t0) * 1000)
results.append(StepResult(step=i, tool=step.tool, ok=True, wall_ms=wall_ms, data=result))
diff --git a/skyvern/cli/core/perception_telemetry.py b/skyvern/cli/core/perception_telemetry.py
new file mode 100644
index 000000000..a87165b96
--- /dev/null
+++ b/skyvern/cli/core/perception_telemetry.py
@@ -0,0 +1,122 @@
+from __future__ import annotations
+
+import asyncio
+import time
+from collections.abc import AsyncIterator, Iterator
+from contextlib import asynccontextmanager, contextmanager
+from contextvars import ContextVar
+from dataclasses import dataclass
+from typing import Literal
+
+PerceptionSnapshotCategory = Literal["model_visible", "automatic", "stale_ref_refresh"]
+
+
+@dataclass
+class MCPPerceptionCounters:
+ top_level_mcp_calls: int = 1
+ perception_snapshots: int = 0
+ model_visible_observe_results: int = 0
+ automatic_observe_snapshots: int = 0
+ stale_ref_refresh_snapshots: int = 0
+ failed_perception_probes: int = 0
+ evaluate_page_scans: int = 0
+ browser_perception_wall_ms: int = 0
+ _last_failed_probe: BaseException | None = None
+
+ def event_fields(self) -> dict[str, int]:
+ return {
+ "top_level_mcp_calls": self.top_level_mcp_calls,
+ "perception_snapshots": self.perception_snapshots,
+ "model_visible_observe_results": self.model_visible_observe_results,
+ "automatic_observe_snapshots": self.automatic_observe_snapshots,
+ "stale_ref_refresh_snapshots": self.stale_ref_refresh_snapshots,
+ "failed_perception_probes": self.failed_perception_probes,
+ "evaluate_page_scans": self.evaluate_page_scans,
+ "browser_perception_wall_ms": self.browser_perception_wall_ms,
+ }
+
+
+_perception_counters: ContextVar[MCPPerceptionCounters | None] = ContextVar(
+ "mcp_perception_counters",
+ default=None,
+)
+_perception_timing_depth: ContextVar[int] = ContextVar("mcp_perception_timing_depth", default=0)
+
+
+def _failure_already_counted(counters: MCPPerceptionCounters, error: BaseException) -> bool:
+ current: BaseException | None = error
+ while current is not None:
+ if current is counters._last_failed_probe:
+ return True
+ current = current.__cause__ or current.__context__
+ return False
+
+
+@contextmanager
+def perception_counters_scope() -> Iterator[MCPPerceptionCounters]:
+ counters = MCPPerceptionCounters()
+ counters_token = _perception_counters.set(counters)
+ depth_token = _perception_timing_depth.set(0)
+ try:
+ yield counters
+ finally:
+ _perception_timing_depth.reset(depth_token)
+ _perception_counters.reset(counters_token)
+
+
+@asynccontextmanager
+async def track_perception_snapshot(category: PerceptionSnapshotCategory) -> AsyncIterator[None]:
+ """Account for one do_observe attempt without changing its result or errors."""
+ counters = _perception_counters.get()
+ if counters is None:
+ yield
+ return
+
+ counters.perception_snapshots += 1
+ if category == "model_visible":
+ counters.model_visible_observe_results += 1
+ elif category == "automatic":
+ counters.automatic_observe_snapshots += 1
+ else:
+ counters.stale_ref_refresh_snapshots += 1
+
+ failed_before = counters.failed_perception_probes
+ depth = _perception_timing_depth.get()
+ depth_token = _perception_timing_depth.set(depth + 1)
+ start = time.perf_counter() if depth == 0 else None
+ try:
+ yield
+ except (Exception, asyncio.CancelledError) as exc:
+ if counters.failed_perception_probes == failed_before or not _failure_already_counted(counters, exc):
+ counters.failed_perception_probes += 1
+ counters._last_failed_probe = exc
+ raise
+ finally:
+ if start is not None:
+ counters.browser_perception_wall_ms += int((time.perf_counter() - start) * 1000)
+ _perception_timing_depth.reset(depth_token)
+
+
+@asynccontextmanager
+async def track_perception_probe(*, evaluate_page_scan: bool = False) -> AsyncIterator[None]:
+ """Account for a non-snapshot browser perception probe."""
+ counters = _perception_counters.get()
+ if counters is None:
+ yield
+ return
+
+ if evaluate_page_scan:
+ counters.evaluate_page_scans += 1
+ depth = _perception_timing_depth.get()
+ depth_token = _perception_timing_depth.set(depth + 1)
+ start = time.perf_counter() if depth == 0 else None
+ try:
+ yield
+ except (Exception, asyncio.CancelledError) as exc:
+ counters.failed_perception_probes += 1
+ counters._last_failed_probe = exc
+ raise
+ finally:
+ if start is not None:
+ counters.browser_perception_wall_ms += int((time.perf_counter() - start) * 1000)
+ _perception_timing_depth.reset(depth_token)
diff --git a/skyvern/cli/core/session_manager.py b/skyvern/cli/core/session_manager.py
index c139a31a6..cbde35ae8 100644
--- a/skyvern/cli/core/session_manager.py
+++ b/skyvern/cli/core/session_manager.py
@@ -42,12 +42,11 @@ class ObserveV2State:
host_budgets: dict[str, int] = field(default_factory=dict)
def reset_refs(self) -> None:
- """Clear per-document ref data while preserving host budgets across navigation."""
+ """Clear per-document refs while preserving monotonic ref IDs and host budgets."""
self.page_key = None
self.document_id = None
self.params = {}
self.refs = {}
- self.next_ref = 0
@dataclass
@@ -156,6 +155,18 @@ def _generation_for(key: tuple[str | None, str, str, str | None]) -> int:
return generation
+def _advance_generation_for(key: tuple[str | None, str, str, str | None]) -> int:
+ generation = next(_session_ref_generation_counter)
+ # Pop-then-reinsert: an advance is activity, so it must LRU-touch the key -
+ # FIFO eviction would drop the longest-lived (often busiest) session and a
+ # freshly minted generation would break its in-flight reservation.
+ _session_ref_generations.pop(key, None)
+ _session_ref_generations[key] = generation
+ while len(_session_ref_generations) > _SESSION_REF_STORE_MAX:
+ _session_ref_generations.pop(next(iter(_session_ref_generations)))
+ return generation
+
+
def _session_ref_key(
state: SessionState,
*,
@@ -210,6 +221,38 @@ def session_ref_generation(
return state._observed_refs_generation
+def begin_session_ref_publication(
+ *,
+ session_id: str | None = None,
+ cdp_url: str | None = None,
+) -> int:
+ """Reserve a unique generation for one in-flight ref publication."""
+ state = get_current_session()
+ if key := _session_ref_key(state, session_id=session_id, cdp_url=cdp_url):
+ return _advance_generation_for(key)
+ state._observed_refs_generation = next(_session_ref_generation_counter)
+ return state._observed_refs_generation
+
+
+def invalidate_session_ref_map(
+ *,
+ session_id: str | None = None,
+ cdp_url: str | None = None,
+) -> int:
+ """Invalidate published refs while retaining v2 budgets and monotonic ref IDs."""
+ state = get_current_session()
+ if key := _session_ref_key(state, session_id=session_id, cdp_url=cdp_url):
+ _session_ref_maps.pop(key, None)
+ observe_state = _observe_v2_states.get(key)
+ if observe_state is not None:
+ observe_state.reset_refs()
+ return _advance_generation_for(key)
+ state._observed_refs = {}
+ state._observe_v2_state.reset_refs()
+ state._observed_refs_generation = next(_session_ref_generation_counter)
+ return state._observed_refs_generation
+
+
def replace_session_ref_map(
ref_map: dict[str, dict[str, Any]],
*,
@@ -217,12 +260,19 @@ def replace_session_ref_map(
cdp_url: str | None = None,
generation: int | None = None,
page_key: tuple[int, int | None, str, str | None] | None = None,
+ advance_on_commit: bool = True,
) -> bool:
"""Replace the session's ref snapshot (never merge). Returns False if discarded.
- When ``generation`` is given, the snapshot is discarded if the registry was
- cleared after that generation was captured — the observe raced a navigation
- or context switch and describes a replaced document.
+ A supplied generation is a publication reservation. The snapshot is discarded
+ if a newer operation has reserved or invalidated the registry; an accepted
+ commit advances the generation so work captured before the commit cannot
+ mutate the published snapshot afterward.
+
+ Legacy (pre-v2) publications pass ``advance_on_commit=False``: their generation
+ is a plain read, so the CAS refuses snapshots that raced an invalidation
+ (navigation/clear) while overlapping publications keep last-completer-wins -
+ exactly the pre-reservation contract.
"""
state = get_current_session()
snapshot: dict[str, Any] = {"page_key": page_key, "refs": dict(ref_map)}
@@ -232,10 +282,14 @@ def replace_session_ref_map(
_session_ref_maps[key] = snapshot
while len(_session_ref_maps) > _SESSION_REF_STORE_MAX:
_session_ref_maps.pop(next(iter(_session_ref_maps)))
+ if advance_on_commit:
+ _advance_generation_for(key)
else:
if generation is not None and generation != state._observed_refs_generation:
return False
state._observed_refs = snapshot
+ if advance_on_commit:
+ state._observed_refs_generation = next(_session_ref_generation_counter)
return True
@@ -269,9 +323,13 @@ def clear_session_ref_map(
*,
session_id: str | None = None,
cdp_url: str | None = None,
-) -> None:
+ generation: int | None = None,
+) -> bool:
+ """Clear refs, unless a newer publication reservation superseded this caller."""
state = get_current_session()
if key := _session_ref_key(state, session_id=session_id, cdp_url=cdp_url):
+ if generation is not None and generation != _generation_for(key):
+ return False
_session_ref_maps.pop(key, None)
observe_state = _observe_v2_states.pop(key, None)
if observe_state is None:
@@ -281,13 +339,17 @@ def clear_session_ref_map(
_observe_v2_states[key] = observe_state
while len(_observe_v2_states) > _SESSION_REF_STORE_MAX:
_observe_v2_states.pop(next(iter(_observe_v2_states)))
- _session_ref_generations[key] = next(_session_ref_generation_counter)
- while len(_session_ref_generations) > _SESSION_REF_STORE_MAX:
- _session_ref_generations.pop(next(iter(_session_ref_generations)))
+ _advance_generation_for(key)
else:
+ if generation is not None and generation != session_ref_generation(
+ session_id=session_id,
+ cdp_url=cdp_url,
+ ):
+ return False
state._observed_refs = {}
state._observe_v2_state.reset_refs()
state._observed_refs_generation = next(_session_ref_generation_counter)
+ return True
def register_copilot_session(session_id: str, state: SessionState, *, organization_id: str) -> None:
diff --git a/skyvern/cli/mcp_tools/_session.py b/skyvern/cli/mcp_tools/_session.py
index ffb7102f2..25d737e80 100644
--- a/skyvern/cli/mcp_tools/_session.py
+++ b/skyvern/cli/mcp_tools/_session.py
@@ -10,6 +10,7 @@ from skyvern.cli.core.session_manager import (
BrowserNotAvailableError,
SessionState,
active_api_key_hash,
+ begin_session_ref_publication,
browser_session,
clear_session_ref_map,
close_current_session,
@@ -17,6 +18,7 @@ from skyvern.cli.core.session_manager import (
get_current_session,
get_page,
get_session_ref,
+ invalidate_session_ref_map,
is_stdio_local_file_access_enabled,
no_browser_error,
page_ref_key,
@@ -30,6 +32,7 @@ __all__ = [
"BrowserNotAvailableError",
"SessionState",
"active_api_key_hash",
+ "begin_session_ref_publication",
"browser_session",
"clear_session_ref_map",
"close_current_session",
@@ -38,6 +41,7 @@ __all__ = [
"get_page",
"get_session_ref",
"get_skyvern",
+ "invalidate_session_ref_map",
"is_stdio_local_file_access_enabled",
"no_browser_error",
"page_ref_key",
diff --git a/skyvern/cli/mcp_tools/browser.py b/skyvern/cli/mcp_tools/browser.py
index eec879787..afd6e2676 100644
--- a/skyvern/cli/mcp_tools/browser.py
+++ b/skyvern/cli/mcp_tools/browser.py
@@ -65,6 +65,7 @@ from skyvern.cli.core.guards import resolve_ai_mode as _resolve_ai_mode
from skyvern.cli.core.guards import (
validate_wait_until,
)
+from skyvern.cli.core.perception_telemetry import PerceptionSnapshotCategory, track_perception_snapshot
from skyvern.cli.core.session_manager import ObserveV2State, get_observe_v2_state, is_stateless_http_mode
from skyvern.cli.core.trajectory_store import append_trajectory_entry
from skyvern.config import settings
@@ -101,17 +102,20 @@ from ._element_state import (
from ._localhost import is_localhost_url
from ._session import (
BrowserNotAvailableError,
+ begin_session_ref_publication,
clear_session_ref_map,
current_api_key_hash,
get_current_session,
get_page,
get_session_ref,
+ invalidate_session_ref_map,
is_stdio_local_file_access_enabled,
no_browser_error,
page_ref_key,
replace_session_ref_map,
session_ref_generation,
)
+from .response import truncate_response_bytes
LOG = structlog.get_logger(__name__)
@@ -526,7 +530,7 @@ async def skyvern_navigate(
# (even failed navigations can partially load and destroy existing frames)
state = get_current_session()
state._working_frame = None
- clear_session_ref_map(session_id=ctx.session_id, cdp_url=ctx.cdp_url)
+ invalidate_session_ref_map(session_id=ctx.session_id, cdp_url=ctx.cdp_url)
with Timer() as timer:
try:
@@ -556,11 +560,9 @@ async def skyvern_navigate(
error=make_error(ErrorCode.ACTION_FAILED, str(e), "Check that the URL is valid and accessible"),
)
finally:
- # Clear again after the attempt (success OR failure — a failed goto can
- # still partially replace the document): an observe that started while
- # navigation was in flight captured a post-clear generation, so only a
- # second bump can invalidate its snapshot of the old document.
- clear_session_ref_map(session_id=ctx.session_id, cdp_url=ctx.cdp_url)
+ # No publication made while navigation was in flight is trustworthy:
+ # even a failed goto can partially replace the document.
+ invalidate_session_ref_map(session_id=ctx.session_id, cdp_url=ctx.cdp_url)
requested_load_state = wait_until or "load"
warnings = (
@@ -3459,6 +3461,9 @@ async def skyvern_clipboard_write(
_OBSERVE_V2_DEFAULT_BUDGET = 50
_OBSERVE_V2_MAX_BUDGET = 200
_OBSERVE_V2_HOST_BUDGET_MAX_HOSTS = 32
+_POST_MUTATION_SETTLE_MS = 250
+_POST_MUTATION_SETTLE_TIMEOUT_SECONDS = 0.5
+_SAFE_ARIA_TARGET_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_.:-]{0,127}$")
def _observe_v2_host(page: Any) -> str:
@@ -3494,11 +3499,13 @@ async def _observe_with_v2_budget(
observe_fn: Callable[..., Any],
page: Any,
*,
+ perception_category: PerceptionSnapshotCategory,
session_id: str | None = None,
cdp_url: str | None = None,
**params: Any,
) -> Any:
- result = await observe_fn(page, **params)
+ async with track_perception_snapshot(perception_category):
+ result = await observe_fn(page, **params)
if observe_v2_enabled():
state = get_observe_v2_state(session_id=session_id, cdp_url=cdp_url)
_learn_observe_v2_host_budget(state, page, result.total_on_page)
@@ -3548,7 +3555,7 @@ def _prepare_observe_v2_refs(
# though the document is untouched: costs the optimization, never correctness.
same_document = state.page_key == current_page_key and document_id is not None and state.document_id == document_id
old_refs = state.refs if same_document else {}
- next_ref = state.next_ref if same_document else 0
+ next_ref = state.next_ref
if not old_refs and next_ref == 0:
prepared = [dict(element) for element in elements]
@@ -3596,12 +3603,71 @@ def _prepare_observe_v2_refs(
}
-def _commit_observe_v2_refs(state: ObserveV2State, prepared: dict[str, Any]) -> None:
+async def _observe_v2_snapshot_is_current(
+ page: Any,
+ prepared: dict[str, Any],
+ *,
+ session_id: str | None,
+ cdp_url: str | None,
+ generation: int | None,
+) -> bool:
+ if generation is not None and generation != session_ref_generation(
+ session_id=session_id,
+ cdp_url=cdp_url,
+ ):
+ return False
+ live_document_id = await get_observe_document_id(page)
+ if generation is not None and generation != session_ref_generation(session_id=session_id, cdp_url=cdp_url):
+ return False
+ prepared_document_id = prepared["document_id"]
+ if page_ref_key(page) != prepared["page_key"] or (
+ # A trusted (cdp:) snapshot must revalidate against the live marker before publish.
+ # An untrusted snapshot (page:-sourced marker gated to None) still publishes so the
+ # response lists refs, but with document_id=None every later ref use fails closed —
+ # the rollout-gate contract: refs are visible, none resolve.
+ prepared_document_id is not None and (live_document_id is None or live_document_id != prepared_document_id)
+ ):
+ clear_session_ref_map(
+ session_id=session_id,
+ cdp_url=cdp_url,
+ generation=generation,
+ )
+ return False
+ return True
+
+
+async def _publish_observe_v2_refs(
+ page: Any,
+ prepared: dict[str, Any],
+ ref_map: dict[str, dict[str, Any]],
+ *,
+ session_id: str | None,
+ cdp_url: str | None,
+ generation: int | None,
+) -> bool:
+ if not await _observe_v2_snapshot_is_current(
+ page,
+ prepared,
+ session_id=session_id,
+ cdp_url=cdp_url,
+ generation=generation,
+ ):
+ return False
+ if not replace_session_ref_map(
+ ref_map,
+ session_id=session_id,
+ cdp_url=cdp_url,
+ generation=generation,
+ page_key=prepared["page_key"],
+ ):
+ return False
+ state = get_observe_v2_state(session_id=session_id, cdp_url=cdp_url)
state.page_key = prepared["page_key"]
state.document_id = prepared["document_id"]
state.params = prepared["params"]
state.refs = prepared["refs"]
state.next_ref = prepared["next_ref"]
+ return True
async def _refresh_observe_v2_ref(
@@ -3610,27 +3676,32 @@ async def _refresh_observe_v2_ref(
*,
session_id: str | None,
cdp_url: str | None,
+ refresh_state: ObserveV2State | None = None,
+ expected_generation: int | None = None,
) -> tuple[bool, dict[str, Any] | None]:
if not observe_v2_enabled():
return False, None
- state = get_observe_v2_state(session_id=session_id, cdp_url=cdp_url)
+ generation = session_ref_generation(session_id=session_id, cdp_url=cdp_url)
+ if expected_generation is not None and generation != expected_generation:
+ return True, None
+ state = refresh_state or get_observe_v2_state(session_id=session_id, cdp_url=cdp_url)
if ref not in state.refs:
# With the flag on, every published legacy ref has a matching v2 ref. A ref the
# legacy map still holds therefore predates the flag flip (or survived invalidation)
# and cannot be validated — revoke both stores. A ref neither store knows is
# model-invented: reject it alone, keeping the refs that were just published.
if get_session_ref(ref, session_id=session_id, cdp_url=cdp_url, page_key=page_ref_key(page)) is not None:
- clear_session_ref_map(session_id=session_id, cdp_url=cdp_url)
+ clear_session_ref_map(session_id=session_id, cdp_url=cdp_url, generation=generation)
return True, None
current_page_key = page_ref_key(page)
if state.page_key != current_page_key:
- clear_session_ref_map(session_id=session_id, cdp_url=cdp_url)
+ clear_session_ref_map(session_id=session_id, cdp_url=cdp_url, generation=generation)
return True, None
document_valid = state.document_id is not None and await get_observe_document_id(page) == state.document_id
if not document_valid:
- clear_session_ref_map(session_id=session_id, cdp_url=cdp_url)
+ clear_session_ref_map(session_id=session_id, cdp_url=cdp_url, generation=generation)
return True, None
from skyvern.cli.core.browser_ops import do_observe as current_do_observe
@@ -3645,16 +3716,22 @@ async def _refresh_observe_v2_ref(
result = await _observe_with_v2_budget(
current_do_observe,
page,
+ perception_category="stale_ref_refresh",
session_id=session_id,
cdp_url=cdp_url,
**params,
)
except Exception:
# An unverifiable ref must not remain usable through the legacy fallback.
- clear_session_ref_map(session_id=session_id, cdp_url=cdp_url)
+ clear_session_ref_map(session_id=session_id, cdp_url=cdp_url, generation=generation)
return True, None
- if page_ref_key(page) != current_page_key or result.document_id is None or result.document_id != state.document_id:
- clear_session_ref_map(session_id=session_id, cdp_url=cdp_url)
+ if (
+ generation != session_ref_generation(session_id=session_id, cdp_url=cdp_url)
+ or page_ref_key(page) != current_page_key
+ or result.document_id is None
+ or result.document_id != state.document_id
+ ):
+ clear_session_ref_map(session_id=session_id, cdp_url=cdp_url, generation=generation)
return True, None
prepared = _prepare_observe_v2_refs(
state,
@@ -3721,15 +3798,33 @@ async def skyvern_observe(
] = False,
) -> dict[str, Any]:
"""Snapshot interactive elements. On stdio, refs persist across calls until the next observe or page/document context change (rarely earlier — on 'Unknown ref', re-observe). In hosted stateless HTTP, refs from prior requests do not resolve; prefer selector or intent params, using refs from an inline observe in one skyvern_execute batch only when predictable in advance. Input values are omitted by default; set include_values=True to return non-password values. Password values are never returned."""
+ v2_enabled = observe_v2_enabled()
+ # Failure-path cleanup is a v2 hardening: the pre-v2 contract preserves the
+ # registry on failed observes (only a frame error clears it, below).
+ lookup_generation = session_ref_generation(session_id=session_id, cdp_url=cdp_url) if v2_enabled else None
try:
page, ctx = await get_page(session_id=session_id, cdp_url=cdp_url)
+ except asyncio.CancelledError:
+ if v2_enabled:
+ clear_session_ref_map(session_id=session_id, cdp_url=cdp_url, generation=lookup_generation)
+ raise
except BrowserNotAvailableError:
+ if v2_enabled:
+ clear_session_ref_map(session_id=session_id, cdp_url=cdp_url, generation=lookup_generation)
return make_result("skyvern_observe", ok=False, error=no_browser_error())
action_result = _action_result_factory(ctx=ctx, page=page, selector=selector)
observe_page_key = page_ref_key(page)
- generation = session_ref_generation(session_id=ctx.session_id, cdp_url=ctx.cdp_url)
+ # The generation guard is not v2-only: it predates observe-v2 and protects the
+ # flag-off path too (an in-flight observe racing a concurrent invalidation must
+ # not republish a stale snapshot). v2 reserves a fresh generation; flag-off
+ # reads the current one, exactly as before the flag existed.
+ generation = (
+ begin_session_ref_publication(session_id=ctx.session_id, cdp_url=ctx.cdp_url)
+ if v2_enabled
+ else session_ref_generation(session_id=ctx.session_id, cdp_url=ctx.cdp_url)
+ )
observe_params = {
"selector": selector,
"interactive_only": interactive_only,
@@ -3741,13 +3836,22 @@ async def skyvern_observe(
result = await _observe_with_v2_budget(
do_observe,
page,
+ perception_category="model_visible",
session_id=ctx.session_id,
cdp_url=ctx.cdp_url,
**observe_params,
)
timer.mark("sdk")
+ except asyncio.CancelledError:
+ if v2_enabled:
+ clear_session_ref_map(session_id=ctx.session_id, cdp_url=ctx.cdp_url, generation=generation)
+ raise
except ObserveFrameError as e:
- clear_session_ref_map(session_id=ctx.session_id, cdp_url=ctx.cdp_url)
+ if v2_enabled:
+ clear_session_ref_map(session_id=ctx.session_id, cdp_url=ctx.cdp_url, generation=generation)
+ else:
+ # Pre-v2 contract: a frame-error observe clears unconditionally.
+ clear_session_ref_map(session_id=ctx.session_id, cdp_url=ctx.cdp_url)
return action_result(
"skyvern_observe",
ok=False,
@@ -3756,6 +3860,8 @@ async def skyvern_observe(
error=_observe_frame_error(e),
)
except Exception as e:
+ if v2_enabled:
+ clear_session_ref_map(session_id=ctx.session_id, cdp_url=ctx.cdp_url, generation=generation)
return action_result(
"skyvern_observe",
ok=False,
@@ -3773,23 +3879,11 @@ async def skyvern_observe(
document_id=result.document_id,
params=observe_params,
)
- if observe_v2_enabled()
+ if v2_enabled
else None
)
if prepared_v2 is not None:
elements = prepared_v2["elements"]
- accepted = replace_session_ref_map(
- ref_map_from_elements(elements),
- session_id=ctx.session_id,
- cdp_url=ctx.cdp_url,
- generation=generation,
- page_key=observe_page_key,
- )
- if accepted and prepared_v2 is not None:
- _commit_observe_v2_refs(
- get_observe_v2_state(session_id=ctx.session_id, cdp_url=ctx.cdp_url),
- prepared_v2,
- )
element_count = len(elements)
hint = (
f"Found {element_count} interactive elements"
@@ -3811,14 +3905,69 @@ async def skyvern_observe(
"total_on_page": result.total_on_page,
"hint": hint,
}
- if observe_v2_enabled():
+ if v2_enabled:
data["page_text"] = _observe_v2_page_text(result)
- return action_result(
+ response = action_result(
"skyvern_observe",
browser_context=ctx,
data=data,
timing_ms=timer.timing_ms,
)
+ capped_response = truncate_response_bytes(response) if v2_enabled else response
+ if capped_response is not response:
+ clear_session_ref_map(session_id=ctx.session_id, cdp_url=ctx.cdp_url, generation=generation)
+ return capped_response
+ if not v2_enabled:
+ # Generation-checked like main always did; refusal is silent here (the
+ # response is already built) - stale snapshots just never get published.
+ replace_session_ref_map(
+ ref_map_from_elements(elements),
+ session_id=ctx.session_id,
+ cdp_url=ctx.cdp_url,
+ generation=generation,
+ page_key=observe_page_key,
+ advance_on_commit=False,
+ )
+ return capped_response
+ if prepared_v2 is None:
+ accepted = replace_session_ref_map(
+ ref_map_from_elements(elements),
+ session_id=ctx.session_id,
+ cdp_url=ctx.cdp_url,
+ generation=generation,
+ page_key=observe_page_key,
+ )
+ else:
+ try:
+ publication_page, _ = await get_page(session_id=session_id, cdp_url=cdp_url)
+ accepted = await _publish_observe_v2_refs(
+ publication_page,
+ prepared_v2,
+ ref_map_from_elements(elements),
+ session_id=ctx.session_id,
+ cdp_url=ctx.cdp_url,
+ generation=generation,
+ )
+ except asyncio.CancelledError:
+ clear_session_ref_map(session_id=ctx.session_id, cdp_url=ctx.cdp_url, generation=generation)
+ raise
+ except Exception:
+ clear_session_ref_map(session_id=ctx.session_id, cdp_url=ctx.cdp_url, generation=generation)
+ accepted = False
+ if not accepted:
+ error = make_error(
+ ErrorCode.ACTION_FAILED,
+ "Observe snapshot was superseded before ref publication",
+ "Call skyvern_observe again for current refs",
+ )
+ return action_result(
+ "skyvern_observe",
+ ok=False,
+ browser_context=ctx,
+ timing_ms=timer.timing_ms,
+ error=error,
+ )
+ return capped_response
# ---------------------------------------------------------------------------
@@ -3857,16 +4006,92 @@ _TOOL_ACCEPTED_PARAMS: dict[str, frozenset[str]] = {
}
+_EXECUTE_MUTATION_CLASS: dict[str, bool] = {
+ "navigate": False,
+ "click": True,
+ "type": True,
+ "press_key": True,
+ "select_option": True,
+ "hover": True,
+ "scroll": True,
+ "wait": False,
+ "wait_for_either_state": False,
+ "observe": False,
+ "screenshot": False,
+ # Arbitrary JS can mutate the DOM in ways no static classifier can soundly
+ # detect (bracket-notation writes, aliased methods, eval, ...), and a miss
+ # fails open: stale refs stay resolvable after an invisible same-document
+ # change. Classify every evaluate as mutating - the cost is one bounded
+ # auto-observe per evaluate step, the same class as scroll.
+ "evaluate": True,
+}
+assert _EXECUTE_MUTATION_CLASS.keys() == _ALLOWED_EXECUTE_TOOLS
+
+
+def _execute_step_mutates(step: ExecuteStep) -> bool:
+ """Classify every execute tool's mutation potential (fail-closed for evaluate)."""
+ return _EXECUTE_MUTATION_CLASS[step.tool]
+
+
+def _aria_target_selector(elements: list[dict[str, Any]]) -> str | None:
+ target: str | None = None
+ for element in elements:
+ for field in ("aria_controls", "aria_owns"):
+ value = element.get(field)
+ if not isinstance(value, str):
+ continue
+ for candidate in value.split():
+ if not _SAFE_ARIA_TARGET_RE.fullmatch(candidate):
+ continue
+ if target is None:
+ target = candidate
+ elif candidate != target:
+ return None
+ return f'[id="{target}"]' if target else None
+
+
+async def _attached_aria_target_selector(page: Any, elements: list[dict[str, Any]]) -> str | None:
+ selector = _aria_target_selector(elements)
+ if selector is None:
+ return None
+ raw_page = getattr(page, "page", page)
+ try:
+ return selector if await raw_page.query_selector(selector) is not None else None
+ except Exception:
+ return None
+
+
+async def _settle_after_mutating_batch(page: Any) -> None:
+ """Give page handlers one bounded quiet window before the post-mutation snapshot."""
+ raw_page = getattr(page, "page", page)
+ wait_for_timeout = getattr(raw_page, "wait_for_timeout", None) or getattr(page, "wait_for_timeout", None)
+ if wait_for_timeout is None:
+ return
+ try:
+ await asyncio.wait_for(
+ wait_for_timeout(_POST_MUTATION_SETTLE_MS),
+ timeout=_POST_MUTATION_SETTLE_TIMEOUT_SECONDS,
+ )
+ except Exception:
+ LOG.debug("Post-mutation settle window ended early", exc_info=True)
+
+
async def _dispatch_step(
step: ExecuteStep,
ref_map: dict[str, dict[str, Any]],
session_id: str | None,
cdp_url: str | None,
page_key: tuple[int, int | None, str, str | None] | None = None,
+ document_id: str | None = None,
on_observe_page: Callable[[tuple[int, int | None, str, str | None]], None] | None = None,
on_observe_v2: Callable[[Any, list[dict[str, Any]], Any, dict[str, Any]], list[dict[str, Any]]] | None = None,
+ on_resolved_element: Callable[[dict[str, Any]], None] | None = None,
+ on_before_action: Callable[[], None] | None = None,
+ perception_category: PerceptionSnapshotCategory = "model_visible",
observe_v2_session_id: str | None = None,
observe_v2_cdp_url: str | None = None,
+ observe_v2_refresh_state: ObserveV2State | None = None,
+ observe_v2_generation: int | None = None,
) -> dict[str, Any] | None:
"""Route a step to the appropriate handler, resolving refs to selectors."""
params = dict(step.params)
@@ -3877,21 +4102,14 @@ async def _dispatch_step(
if ref := params.pop("ref", None):
current_page, _ = await get_page(session_id=session_id, cdp_url=cdp_url)
current_key = page_ref_key(current_page)
- elem = ref_map.get(ref) if page_key is None or current_key == page_key else None
- if elem is not None and observe_v2_enabled():
- # An in-batch hit skips _refresh_observe_v2_ref, so it must run the same document
- # check: an earlier step in this batch can replace the document at the same URL,
- # leaving these refs pointing at pre-swap selectors. The marker read reuses the
- # cached CDP session; on None or mismatch fall through to the refresh path, which
- # fails closed (its own document check cannot pass either). This validates document
- # identity only — a same-document edit that renumbers ordinal (nth=) selectors is
- # caught by the cross-call re-observe, not here; that speed trade is deliberate.
- v2_state = get_observe_v2_state(
- session_id=observe_v2_session_id or session_id,
- cdp_url=observe_v2_cdp_url or cdp_url,
- )
- if v2_state.document_id is None or await get_observe_document_id(current_page) != v2_state.document_id:
- elem = None
+ elem = None
+ if page_key is None or current_key == page_key:
+ if not observe_v2_enabled():
+ elem = ref_map.get(ref)
+ elif document_id is not None and await get_observe_document_id(current_page) == document_id:
+ elem = ref_map.get(ref)
+ else:
+ ref_map.clear()
if elem is None:
current = get_session_ref(ref, session_id=session_id, cdp_url=cdp_url, page_key=current_key)
handled, elem = await _refresh_observe_v2_ref(
@@ -3899,6 +4117,8 @@ async def _dispatch_step(
current_page,
session_id=observe_v2_session_id or session_id,
cdp_url=observe_v2_cdp_url or cdp_url,
+ refresh_state=observe_v2_refresh_state,
+ expected_generation=observe_v2_generation,
)
if not handled:
elem = current
@@ -3908,6 +4128,8 @@ async def _dispatch_step(
message += ". In stateless HTTP mode refs from prior requests do not resolve — use selector or intent params instead."
raise ValueError(message)
params["selector"] = ref_to_selector(elem)
+ if on_resolved_element is not None:
+ on_resolved_element(dict(elem))
# Observe is handled inline (not an existing MCP tool)
if step.tool == "observe":
@@ -3922,12 +4144,16 @@ async def _dispatch_step(
result = await _observe_with_v2_budget(
_do_observe,
page,
+ perception_category=perception_category,
session_id=observe_v2_session_id or session_id,
cdp_url=observe_v2_cdp_url or cdp_url,
**filtered,
)
except ObserveFrameError as e:
- clear_session_ref_map(session_id=session_id, cdp_url=cdp_url)
+ if not observe_v2_enabled():
+ # Pre-v2 contract: an in-batch frame-error observe clears unconditionally
+ # (v2 defers to the dispatch-level generation-checked cleanup).
+ clear_session_ref_map(session_id=session_id, cdp_url=cdp_url)
raise ToolStepError(_observe_frame_error(e)) from e
elements = serialize_elements(result.elements)
if observe_v2_enabled() and on_observe_v2 is not None:
@@ -3941,6 +4167,8 @@ async def _dispatch_step(
data["page_text"] = _observe_v2_page_text(result)
return data
+ if on_before_action is not None:
+ on_before_action()
# DESIGN-1: Dispatch through existing MCP tool functions via module lookup
import skyvern.cli.mcp_tools.browser as _browser_mod
@@ -4037,21 +4265,70 @@ async def skyvern_execute(
)
parsed_steps.append(ExecuteStep(tool=tool, params=raw.get("params", {})))
+ operation_generation: int | None = (
+ session_ref_generation(session_id=session_id, cdp_url=cdp_url) if observe_v2_enabled() else None
+ )
# Verify we can reach the browser before executing anything
try:
page, ctx = await get_page(session_id=session_id, cdp_url=cdp_url)
+ except asyncio.CancelledError:
+ if observe_v2_enabled():
+ clear_session_ref_map(
+ session_id=session_id,
+ cdp_url=cdp_url,
+ generation=operation_generation,
+ )
+ raise
except BrowserNotAvailableError:
+ if observe_v2_enabled():
+ clear_session_ref_map(
+ session_id=session_id,
+ cdp_url=cdp_url,
+ generation=operation_generation,
+ )
return make_result("skyvern_execute", ok=False, error=no_browser_error())
action_result = _action_result_factory(ctx=ctx, page=page)
+ operation_generation = (
+ session_ref_generation(session_id=ctx.session_id, cdp_url=ctx.cdp_url) if observe_v2_enabled() else None
+ )
batch_page_key = page_ref_key(page)
+ if observe_v2_enabled():
+ try:
+ # Only a browser-sourced cdp: marker may certify in-batch document identity;
+ # a page:-sourced marker is spoofable, so it degrades to None and fails closed
+ # (upfront invalidation below, and the in-batch ref check refuses to certify).
+ batch_document_id = _observe_v2_trusted_document_id(await get_observe_document_id(page))
+ except asyncio.CancelledError:
+ clear_session_ref_map(
+ session_id=ctx.session_id,
+ cdp_url=ctx.cdp_url,
+ generation=operation_generation,
+ )
+ raise
+ else:
+ batch_document_id = None
+ if observe_v2_enabled() and batch_document_id is None:
+ clear_session_ref_map(
+ session_id=ctx.session_id,
+ cdp_url=ctx.cdp_url,
+ generation=operation_generation,
+ )
+ operation_generation = session_ref_generation(session_id=ctx.session_id, cdp_url=ctx.cdp_url)
- # Generation captured before each observe dispatch so a snapshot that raced
- # a concurrent navigation/context switch is discarded, not committed.
- observe_generation: dict[str, int] = {}
+ # Under observe-v2, each observe reserves a unique generation before
+ # dispatch so an older snapshot cannot overwrite a newer publication.
+ observe_generation: dict[str, int | None] = {}
observe_page_key: tuple[int, int | None, str, str | None] | None = None
observe_v2_prepared: dict[str, Any] | None = None
+ pending_ref_map: dict[str, dict[str, Any]] | None = None
+ dispatched_observe_data: dict[str, Any] | None = None
+ pending_observe_data: dict[str, Any] | None = None
+ ref_refresh_state: ObserveV2State | None = None
+ mutation_started = False
+ acted_elements: list[dict[str, Any]] = []
+ perception_category: PerceptionSnapshotCategory = "model_visible"
def capture_observe_page_key(page_key: tuple[int, int | None, str, str | None]) -> None:
nonlocal observe_page_key
@@ -4073,75 +4350,352 @@ async def skyvern_execute(
)
return observe_v2_prepared["elements"]
+ def capture_resolved_element(element: dict[str, Any]) -> None:
+ # Stamp the batch's certified document so the post-batch aria scope can
+ # refuse elements from a document the batch has since left (click-driven
+ # navigation and failed goto never set successful_navigation).
+ acted_elements.append({**element, "_acted_document_id": batch_document_id})
+
+ def invalidate_before_mutation(ref_map: dict[str, dict[str, Any]]) -> None:
+ nonlocal observe_page_key, observe_v2_prepared, pending_ref_map, operation_generation, ref_refresh_state
+ nonlocal mutation_started, pending_observe_data
+ if ref_refresh_state is None:
+ state = get_observe_v2_state(session_id=ctx.session_id, cdp_url=ctx.cdp_url)
+ ref_refresh_state = ObserveV2State(
+ page_key=state.page_key,
+ document_id=state.document_id,
+ params=dict(state.params),
+ refs=dict(state.refs),
+ next_ref=state.next_ref,
+ host_budgets=dict(state.host_budgets),
+ )
+ mutation_started = True
+ operation_generation = invalidate_session_ref_map(
+ session_id=ctx.session_id,
+ cdp_url=ctx.cdp_url,
+ )
+ ref_map.clear()
+ observe_page_key = None
+ observe_v2_prepared = None
+ pending_ref_map = None
+ pending_observe_data = None
+
async def dispatch(step: ExecuteStep, ref_map: dict[str, dict[str, Any]]) -> dict[str, Any] | None:
+ nonlocal observe_page_key, observe_v2_prepared, pending_ref_map, operation_generation
+ nonlocal dispatched_observe_data, pending_observe_data
+ if _execute_step_mutates(step):
+ # Prior steps' acted elements belong to an older snapshot; a stale
+ # aria-controls id must not scope the post-batch auto-observe. Cleared
+ # before dispatch so this step's own resolution capture survives.
+ acted_elements.clear()
if step.tool == "observe":
- observe_generation["value"] = session_ref_generation(session_id=ctx.session_id, cdp_url=ctx.cdp_url)
- return await _dispatch_step(
- step,
- ref_map,
- session_id=session_id,
- cdp_url=cdp_url,
- page_key=batch_page_key,
- on_observe_page=capture_observe_page_key if step.tool == "observe" else None,
- on_observe_v2=prepare_observe_v2 if step.tool == "observe" and observe_v2_enabled() else None,
- observe_v2_session_id=ctx.session_id,
- observe_v2_cdp_url=ctx.cdp_url,
- )
+ dispatched_observe_data = None
+ if observe_v2_enabled():
+ # v2 reserves a fresh generation so an older snapshot cannot
+ # overwrite a newer publication.
+ operation_generation = begin_session_ref_publication(
+ session_id=ctx.session_id,
+ cdp_url=ctx.cdp_url,
+ )
+ observe_generation["value"] = operation_generation
+ else:
+ # Flag-off keeps the pre-v2 guard: read the generation at
+ # dispatch so a concurrent invalidation refuses this snapshot.
+ observe_generation["value"] = session_ref_generation(
+ session_id=ctx.session_id,
+ cdp_url=ctx.cdp_url,
+ )
+ try:
+ result = await _dispatch_step(
+ step,
+ ref_map,
+ perception_category=perception_category,
+ session_id=session_id,
+ cdp_url=cdp_url,
+ page_key=batch_page_key,
+ document_id=batch_document_id,
+ on_observe_page=capture_observe_page_key if step.tool == "observe" else None,
+ on_observe_v2=prepare_observe_v2 if step.tool == "observe" and observe_v2_enabled() else None,
+ on_resolved_element=capture_resolved_element if _execute_step_mutates(step) else None,
+ on_before_action=(
+ (lambda: invalidate_before_mutation(ref_map))
+ if observe_v2_enabled() and _execute_step_mutates(step)
+ else None
+ ),
+ observe_v2_session_id=ctx.session_id,
+ observe_v2_cdp_url=ctx.cdp_url,
+ observe_v2_refresh_state=ref_refresh_state,
+ observe_v2_generation=operation_generation,
+ )
+ if step.tool == "observe":
+ dispatched_observe_data = result
+ return result
+ except BaseException:
+ if step.tool == "observe":
+ observe_page_key = None
+ observe_v2_prepared = None
+ pending_ref_map = None
+ dispatched_observe_data = None
+ pending_observe_data = None
+ if observe_v2_enabled():
+ # Pre-v2, a failed observe left the registry alone (the in-batch
+ # frame-error clear in _dispatch_step handles main's one exception).
+ clear_session_ref_map(
+ session_id=ctx.session_id,
+ cdp_url=ctx.cdp_url,
+ generation=operation_generation,
+ )
+ raise
- def publish_observe_refs(ref_map: dict[str, dict[str, Any]]) -> bool:
- nonlocal batch_page_key
+ async def stage_observe_refs(ref_map: dict[str, dict[str, Any]]) -> bool:
+ nonlocal batch_document_id, batch_page_key, pending_ref_map, pending_observe_data, ref_refresh_state
+ pending_ref_map = None
+ pending_observe_data = None
+ publication_generation = observe_generation.get("value")
if observe_page_key is None:
+ if observe_v2_enabled():
+ clear_session_ref_map(
+ session_id=ctx.session_id,
+ cdp_url=ctx.cdp_url,
+ generation=publication_generation,
+ )
return False
- accepted = replace_session_ref_map(
- ref_map,
- session_id=ctx.session_id,
- cdp_url=ctx.cdp_url,
- generation=observe_generation.get("value"),
- page_key=observe_page_key,
- )
- if accepted:
- batch_page_key = observe_page_key
- if observe_v2_prepared is not None:
- _commit_observe_v2_refs(
- get_observe_v2_state(session_id=ctx.session_id, cdp_url=ctx.cdp_url),
+ if not observe_v2_enabled():
+ # Pre-v2 contract: publish each successful inline observe immediately -
+ # no second page lookup, no deferral to batch end - generation-checked
+ # against invalidations only (no reservation consumed). The snapshot is
+ # bound to the observed page key; resolution fails closed on mismatch.
+ accepted = replace_session_ref_map(
+ ref_map,
+ session_id=ctx.session_id,
+ cdp_url=ctx.cdp_url,
+ generation=publication_generation,
+ page_key=observe_page_key,
+ advance_on_commit=False,
+ )
+ if accepted:
+ batch_page_key = observe_page_key
+ return accepted
+ try:
+ publication_page, _ = await get_page(session_id=session_id, cdp_url=cdp_url)
+ except asyncio.CancelledError:
+ clear_session_ref_map(
+ session_id=ctx.session_id,
+ cdp_url=ctx.cdp_url,
+ generation=publication_generation,
+ )
+ raise
+ except Exception:
+ clear_session_ref_map(
+ session_id=ctx.session_id,
+ cdp_url=ctx.cdp_url,
+ generation=publication_generation,
+ )
+ return False
+ try:
+ if observe_v2_prepared is None:
+ accepted = page_ref_key(publication_page) == observe_page_key
+ else:
+ accepted = await _observe_v2_snapshot_is_current(
+ publication_page,
observe_v2_prepared,
+ session_id=ctx.session_id,
+ cdp_url=ctx.cdp_url,
+ generation=publication_generation,
)
- return accepted
+ except BaseException:
+ clear_session_ref_map(
+ session_id=ctx.session_id,
+ cdp_url=ctx.cdp_url,
+ generation=publication_generation,
+ )
+ raise
+ if not accepted:
+ clear_session_ref_map(
+ session_id=ctx.session_id,
+ cdp_url=ctx.cdp_url,
+ generation=publication_generation,
+ )
+ return False
+ pending_ref_map = ref_map
+ pending_observe_data = dispatched_observe_data
+ batch_page_key = observe_page_key
+ if observe_v2_prepared is not None:
+ batch_document_id = observe_v2_prepared["document_id"]
+ state = get_observe_v2_state(session_id=ctx.session_id, cdp_url=ctx.cdp_url)
+ ref_refresh_state = ObserveV2State(
+ page_key=observe_v2_prepared["page_key"],
+ document_id=observe_v2_prepared["document_id"],
+ params=dict(observe_v2_prepared["params"]),
+ refs=dict(observe_v2_prepared["refs"]),
+ next_ref=observe_v2_prepared["next_ref"],
+ host_budgets=dict(state.host_budgets),
+ )
+ return True
with Timer() as timer:
- result = await do_execute(
- dispatch,
- parsed_steps,
- stop_on_error=stop_on_error,
- on_ref_map_update=publish_observe_refs,
+ try:
+ result = await do_execute(
+ dispatch,
+ parsed_steps,
+ stop_on_error=stop_on_error,
+ on_ref_map_update=stage_observe_refs,
+ fail_on_ref_map_rejection=observe_v2_enabled(),
+ )
+ except asyncio.CancelledError:
+ if observe_v2_enabled() and mutation_started:
+ invalidate_session_ref_map(session_id=ctx.session_id, cdp_url=ctx.cdp_url)
+ elif observe_v2_enabled() and any(
+ step.tool == "observe" or _execute_step_mutates(step) for step in parsed_steps
+ ):
+ clear_session_ref_map(
+ session_id=ctx.session_id,
+ cdp_url=ctx.cdp_url,
+ generation=operation_generation,
+ )
+ raise
+ if observe_v2_enabled() and mutation_started:
+ # Close the mutation interval before any further await. A concurrent
+ # observe published while an action was in flight may already be stale.
+ operation_generation = invalidate_session_ref_map(
+ session_id=ctx.session_id,
+ cdp_url=ctx.cdp_url,
+ )
+ observe_page_key = None
+ observe_v2_prepared = None
+ pending_ref_map = None
+ pending_observe_data = None
+ successful_mutation = any(
+ row.ok and row.step < len(parsed_steps) and _execute_step_mutates(parsed_steps[row.step])
+ for row in result.results
+ )
+ successful_navigation = any(
+ row.ok and row.step < len(parsed_steps) and parsed_steps[row.step].tool == "navigate"
+ for row in result.results
)
auto_observe_entry: dict[str, Any] | None = None
- if observe_v2_enabled() and result.error_step is None:
+ if observe_v2_enabled() and (successful_mutation or result.error_step is None):
try:
current_page, _ = await get_page(session_id=session_id, cdp_url=cdp_url)
+ except asyncio.CancelledError:
+ clear_session_ref_map(
+ session_id=ctx.session_id,
+ cdp_url=ctx.cdp_url,
+ generation=operation_generation,
+ )
+ raise
except Exception:
- current_page = None
- if current_page is not None and page_ref_key(current_page) != batch_page_key:
- auto_observe = await do_execute(
- dispatch,
- [ExecuteStep(tool="observe")],
- stop_on_error=True,
- on_ref_map_update=publish_observe_refs,
+ clear_session_ref_map(
+ session_id=ctx.session_id,
+ cdp_url=ctx.cdp_url,
+ generation=operation_generation,
)
- if auto_observe.error_step is None and auto_observe.results:
- # The receipt stays out of `results`/step counts: those describe the
- # caller's submitted steps only, so results[i] pairs with steps[i] and
- # steps_total == len(steps) holds for callers that assert it.
- sr = auto_observe.results[0]
- auto_observe_entry = {"tool": sr.tool, "ok": sr.ok, "wall_ms": sr.wall_ms}
- if sr.data:
- auto_observe_entry["data"] = sr.data
+ current_page = None
+ if current_page is not None:
+ try:
+ if successful_mutation:
+ await _settle_after_mutating_batch(current_page)
+ current_page_key = page_ref_key(current_page)
+ current_document_id = _observe_v2_trusted_document_id(await get_observe_document_id(current_page))
+ if (
+ successful_mutation
+ or current_page_key != batch_page_key
+ or current_document_id != batch_document_id
+ ):
+ refresh_current_document = True
+ if not successful_mutation:
+ if successful_navigation:
+ # skyvern_navigate already closed the navigation interval.
+ operation_generation = session_ref_generation(
+ session_id=ctx.session_id,
+ cdp_url=ctx.cdp_url,
+ )
+ else:
+ published_state = get_observe_v2_state(
+ session_id=ctx.session_id,
+ cdp_url=ctx.cdp_url,
+ )
+ refresh_current_document = (
+ current_document_id is None
+ or published_state.page_key != current_page_key
+ or published_state.document_id != current_document_id
+ )
+ if refresh_current_document:
+ operation_generation = invalidate_session_ref_map(
+ session_id=ctx.session_id,
+ cdp_url=ctx.cdp_url,
+ )
+ observe_page_key = None
+ observe_v2_prepared = None
+ pending_ref_map = None
+ pending_observe_data = None
+ if refresh_current_document:
+ perception_category = "automatic"
+ # Scope by aria-controls only when the acted elements provably
+ # belong to the CURRENT document: click-driven navigation and
+ # failed goto replace the document without setting
+ # successful_navigation, and an old id colliding on the new
+ # document must not narrow its first observe.
+ scope_elements = (
+ acted_elements
+ if current_document_id is not None
+ and acted_elements
+ and all(
+ element.get("_acted_document_id") == current_document_id
+ for element in acted_elements
+ )
+ else []
+ )
+ selector = (
+ await _attached_aria_target_selector(current_page, scope_elements)
+ if successful_mutation
+ else None
+ )
+ auto_observe = await do_execute(
+ dispatch,
+ [ExecuteStep(tool="observe", params={"selector": selector} if selector else {})],
+ stop_on_error=True,
+ on_ref_map_update=stage_observe_refs,
+ fail_on_ref_map_rejection=True,
+ )
+ if (
+ selector is not None
+ and auto_observe.error_step is None
+ and batch_document_id != current_document_id
+ ):
+ # The attachment check awaited between document certification
+ # and the scoped observe; a navigation in that window means the
+ # scoped snapshot certified a different document than the acted
+ # one. Discard it (staging replaces the pending publication) and
+ # take one honest unscoped snapshot of the current document.
+ auto_observe = await do_execute(
+ dispatch,
+ [ExecuteStep(tool="observe")],
+ stop_on_error=True,
+ on_ref_map_update=stage_observe_refs,
+ fail_on_ref_map_rejection=True,
+ )
+ if auto_observe.error_step is None and auto_observe.results:
+ # The receipt stays out of `results`/step counts: those describe the
+ # caller's submitted steps only, so results[i] pairs with steps[i] and
+ # steps_total == len(steps) holds for callers that assert it.
+ sr = auto_observe.results[0]
+ auto_observe_entry = {"tool": sr.tool, "ok": sr.ok, "wall_ms": sr.wall_ms}
+ if sr.data:
+ auto_observe_entry["data"] = sr.data
+ except asyncio.CancelledError:
+ clear_session_ref_map(
+ session_id=ctx.session_id,
+ cdp_url=ctx.cdp_url,
+ generation=operation_generation,
+ )
+ raise
timer.mark("sdk")
step_results = []
for sr in result.results:
entry: dict[str, Any] = {"step": sr.step, "tool": sr.tool, "ok": sr.ok, "wall_ms": sr.wall_ms}
- if sr.data:
+ if sr.data and (not observe_v2_enabled() or sr.tool != "observe" or sr.data is pending_observe_data):
entry["data"] = sr.data
if sr.error:
entry["error"] = sr.error
@@ -4155,9 +4709,78 @@ async def skyvern_execute(
}
if auto_observe_entry is not None:
data["auto_observe"] = auto_observe_entry
- return action_result(
+ response = action_result(
"skyvern_execute",
ok=result.error_step is None,
data=data,
timing_ms=timer.timing_ms,
)
+ capped_response = truncate_response_bytes(response) if observe_v2_enabled() else response
+ if capped_response is not response:
+ if pending_ref_map is not None:
+ clear_session_ref_map(
+ session_id=ctx.session_id,
+ cdp_url=ctx.cdp_url,
+ generation=observe_generation.get("value"),
+ )
+ return capped_response
+ if pending_ref_map is None:
+ return capped_response
+
+ if observe_v2_prepared is None:
+ accepted = replace_session_ref_map(
+ pending_ref_map,
+ session_id=ctx.session_id,
+ cdp_url=ctx.cdp_url,
+ generation=observe_generation.get("value"),
+ page_key=observe_page_key,
+ )
+ else:
+ try:
+ publication_page, _ = await get_page(session_id=session_id, cdp_url=cdp_url)
+ accepted = await _publish_observe_v2_refs(
+ publication_page,
+ observe_v2_prepared,
+ pending_ref_map,
+ session_id=ctx.session_id,
+ cdp_url=ctx.cdp_url,
+ generation=observe_generation.get("value"),
+ )
+ except asyncio.CancelledError:
+ clear_session_ref_map(
+ session_id=ctx.session_id,
+ cdp_url=ctx.cdp_url,
+ generation=observe_generation.get("value"),
+ )
+ raise
+ except Exception:
+ clear_session_ref_map(
+ session_id=ctx.session_id,
+ cdp_url=ctx.cdp_url,
+ generation=observe_generation.get("value"),
+ )
+ accepted = False
+ if not accepted:
+ unpublished_results = [dict(entry) for entry in step_results]
+ for entry in unpublished_results:
+ if entry["tool"] == "observe":
+ entry.pop("data", None)
+ error = make_error(
+ ErrorCode.ACTION_FAILED,
+ "Execute completed, but its observe snapshot was superseded before ref publication",
+ "Do not repeat completed mutating steps; call skyvern_observe for current refs",
+ details={"steps_completed": result.steps_completed, "original_error_step": result.error_step},
+ )
+ return action_result(
+ "skyvern_execute",
+ ok=False,
+ data={
+ "steps_completed": result.steps_completed,
+ "steps_total": result.steps_total,
+ "results": unpublished_results,
+ "error_step": result.error_step,
+ },
+ timing_ms=timer.timing_ms,
+ error=error,
+ )
+ return capped_response
diff --git a/skyvern/cli/mcp_tools/response.py b/skyvern/cli/mcp_tools/response.py
index 3cb409441..87cd343ee 100644
--- a/skyvern/cli/mcp_tools/response.py
+++ b/skyvern/cli/mcp_tools/response.py
@@ -31,6 +31,7 @@ LOG = structlog.get_logger(__name__)
# the MCP envelope (jsonrpc wrapper, content-block metadata) that the FastMCP
# serializer adds on top of our dict.
MCP_MAX_RESPONSE_CHARS = 140_000
+MCP_MAX_RESPONSE_BYTES = 140_000
# When truncation envelope wraps an oversize payload, preserve identifier-like
# keys (`*_id`) from the original dict so the caller retains enough context to
@@ -51,6 +52,10 @@ _TRUNCATION_HINT = (
"Response exceeded the ~150k-char Claude tool-result limit. "
"Narrow the query (add filters, reduce page size, request specific fields) or paginate."
)
+_TRUNCATION_BYTE_HINT = (
+ "Response exceeded the 140k-byte aggregate MCP response limit. "
+ "Narrow the query (add filters, reduce page size, request specific fields) or paginate."
+)
P = ParamSpec("P")
R = TypeVar("R")
@@ -70,6 +75,14 @@ def _response_size(data: Any) -> int:
return sys.maxsize
+def _response_size_bytes(data: Any) -> int:
+ """Return the UTF-8 byte length of the JSON-serialized response."""
+ try:
+ return len(json.dumps(data, ensure_ascii=False, default=str).encode())
+ except (TypeError, ValueError):
+ return sys.maxsize
+
+
def _bound_error_value(error: Any) -> Any:
"""Cap a preserved `error` value so the envelope cannot re-inflate past the response cap."""
size = _response_size(error)
@@ -128,43 +141,25 @@ def _preserved_screenshot_artifact(value: Any) -> dict[str, Any] | None:
return None
-def truncate_response(data: Any, *, max_chars: int = MCP_MAX_RESPONSE_CHARS) -> Any:
- """Return `data` unchanged if under `max_chars`; otherwise wrap in a truncation envelope.
-
- Envelope shape (explicit so the model cannot miss truncation):
- {
- "_truncated": True,
- "_original_chars": <serialized size>,
- "_max_chars": <limit>,
- "_hint": "...",
- "ok": <original "ok" if present>,
- "error": <original "error" if present, bounded to 2k chars>,
- "<key>_id": <preserved identifier field if present>,
- ...
- }
-
- Non-dict responses that overflow are wrapped too (rare, but possible if a
- tool returns a list or string). From dict payloads, top-level keys
- `ok`/`error` plus any `*_id` identifier fields (bounded to 10 keys of up to
- 256 chars each) are preserved; the oversized data is otherwise dropped.
- Identifier preservation keeps workflow/run/session IDs in the envelope so
- the caller can re-query or paginate without losing its place.
-
- Contract: the returned payload is always `<= max_chars` when serialized.
- Pathological inputs (e.g. a `error` field containing a full stack trace or
- HTML dump larger than the cap) are defended against by (a) capping `error`
- at `_MAX_PRESERVED_ERROR_CHARS` and (b) a final re-measure that falls back
- to a metadata-only envelope if the assembled payload is still over.
- """
- size = _response_size(data)
- if size <= max_chars:
+def _truncate_response_to_limit(
+ data: Any,
+ *,
+ size_fn: Callable[[Any], int],
+ max_size: int,
+ unit: str,
+ hint: str,
+) -> Any:
+ size = size_fn(data)
+ if size <= max_size:
return data
+ original_key = f"_original_{unit}"
+ max_key = f"_max_{unit}"
envelope: dict[str, Any] = {
"_truncated": True,
- "_original_chars": size,
- "_max_chars": max_chars,
- "_hint": _TRUNCATION_HINT,
+ original_key: size,
+ max_key: max_size,
+ "_hint": hint,
}
if isinstance(data, dict):
if "ok" in data:
@@ -197,21 +192,39 @@ def truncate_response(data: Any, *, max_chars: int = MCP_MAX_RESPONSE_CHARS) ->
"hint": "Use the saved screenshot artifact path, or retry with inline=false.",
}
- # Final safety net: if the preserved fields combined still blow past the
- # cap (extremely unlikely given the per-field caps above, but possible if
- # a caller passes a tiny `max_chars`), drop them and emit a metadata-only
- # envelope. Guarantees the module's "always under max_chars" contract.
- if _response_size(envelope) > max_chars:
+ if size_fn(envelope) > max_size:
return {
"_truncated": True,
- "_original_chars": size,
- "_max_chars": max_chars,
- "_hint": _TRUNCATION_HINT,
+ original_key: size,
+ max_key: max_size,
+ "_hint": hint,
"_envelope_rewrapped": True,
}
return envelope
+def truncate_response(data: Any, *, max_chars: int = MCP_MAX_RESPONSE_CHARS) -> Any:
+ """Cap a response by JSON-serialized characters, preserving the existing envelope contract."""
+ return _truncate_response_to_limit(
+ data,
+ size_fn=_response_size,
+ max_size=max_chars,
+ unit="chars",
+ hint=_TRUNCATION_HINT,
+ )
+
+
+def truncate_response_bytes(data: Any, *, max_bytes: int = MCP_MAX_RESPONSE_BYTES) -> Any:
+ """Cap a response by aggregate UTF-8 JSON bytes."""
+ return _truncate_response_to_limit(
+ data,
+ size_fn=_response_size_bytes,
+ max_size=max_bytes,
+ unit="bytes",
+ hint=_TRUNCATION_BYTE_HINT,
+ )
+
+
def size_capped(fn: Callable[P, Awaitable[R]]) -> Callable[P, Awaitable[Any]]:
"""Decorator: enforce `MCP_MAX_RESPONSE_CHARS` on a tool's return value.
@@ -244,7 +257,9 @@ def size_capped(fn: Callable[P, Awaitable[R]]) -> Callable[P, Awaitable[Any]]:
__all__ = [
+ "MCP_MAX_RESPONSE_BYTES",
"MCP_MAX_RESPONSE_CHARS",
"size_capped",
"truncate_response",
+ "truncate_response_bytes",
]
diff --git a/skyvern/cli/mcp_tools/telemetry.py b/skyvern/cli/mcp_tools/telemetry.py
index da2941a61..453d7c136 100644
--- a/skyvern/cli/mcp_tools/telemetry.py
+++ b/skyvern/cli/mcp_tools/telemetry.py
@@ -1,5 +1,6 @@
from __future__ import annotations
+import asyncio
import time
from collections.abc import Callable
from contextlib import suppress
@@ -11,6 +12,7 @@ from fastmcp.server.dependencies import get_http_request
from fastmcp.server.middleware import CallNext, Middleware, MiddlewareContext
from skyvern import analytics
+from skyvern.cli.core.perception_telemetry import MCPPerceptionCounters, perception_counters_scope
LOG = structlog.get_logger(__name__)
@@ -142,9 +144,10 @@ def _capture_mcp_event(
ok: bool,
tool_name: str | None = None,
prompt_name: str | None = None,
- error: Exception | None = None,
+ error: BaseException | None = None,
duration_ms: float | None = None,
response_bytes: int | None = None,
+ perception_counters: MCPPerceptionCounters | None = None,
) -> None:
request = _resolve_http_request()
organization_id = _resolve_organization_id(request)
@@ -179,6 +182,8 @@ def _capture_mcp_event(
data["duration_ms"] = duration_ms
if response_bytes is not None:
data["response_bytes"] = response_bytes
+ if perception_counters is not None:
+ data.update(perception_counters.event_fields())
analytics.capture(
event_name,
@@ -249,36 +254,39 @@ class MCPTelemetryMiddleware(Middleware):
) -> Any:
tool_name = getattr(context.message, "name", None)
start = time.perf_counter()
- try:
- result = await call_next(context)
- except Exception as exc:
+ with perception_counters_scope() as counters:
+ try:
+ result = await call_next(context)
+ except (Exception, asyncio.CancelledError) as exc:
+ duration_ms = (time.perf_counter() - start) * 1000
+ # Failed and cancelled calls produce no MCP content, so response_bytes is only emitted for results.
+ with suppress(Exception, asyncio.CancelledError):
+ _capture_mcp_event(
+ "mcp_tool_call",
+ operation="tools/call",
+ context=context,
+ ok=False,
+ tool_name=tool_name,
+ error=exc,
+ duration_ms=duration_ms,
+ perception_counters=counters,
+ )
+ raise
+
duration_ms = (time.perf_counter() - start) * 1000
- # Exceptions do not produce MCP content, so response_bytes is only emitted for returned results.
+ response_bytes = sum(_content_text_bytes(content) for content in (getattr(result, "content", None) or []))
with suppress(Exception):
_capture_mcp_event(
"mcp_tool_call",
operation="tools/call",
context=context,
- ok=False,
+ ok=_resolve_tool_call_ok(result),
tool_name=tool_name,
- error=exc,
duration_ms=duration_ms,
+ response_bytes=response_bytes,
+ perception_counters=counters,
)
- raise
-
- duration_ms = (time.perf_counter() - start) * 1000
- response_bytes = sum(_content_text_bytes(content) for content in (getattr(result, "content", None) or []))
- with suppress(Exception):
- _capture_mcp_event(
- "mcp_tool_call",
- operation="tools/call",
- context=context,
- ok=_resolve_tool_call_ok(result),
- tool_name=tool_name,
- duration_ms=duration_ms,
- response_bytes=response_bytes,
- )
- return result
+ return result
async def on_list_tools(
self,
diff --git a/tests/unit/test_mcp_browser_tools.py b/tests/unit/test_mcp_browser_tools.py
index 018e0d8e9..b6c46ea1a 100644
--- a/tests/unit/test_mcp_browser_tools.py
+++ b/tests/unit/test_mcp_browser_tools.py
@@ -2785,14 +2785,22 @@ async def test_write_grid_rejects_oversize_row_before_session(monkeypatch: pytes
),
],
)
-async def test_navigating_paired_tools_clear_the_ref_map_around_navigation(
+async def test_navigating_paired_tools_invalidate_refs_around_navigation(
monkeypatch: pytest.MonkeyPatch, tool_name: str, extra_kwargs: dict[str, object]
) -> None:
- """Parity with skyvern_navigate: a prior observe's refs must not survive a navigation.
+ """Parity with skyvern_navigate: prior refs cannot survive navigation.
- Cleared before AND after, because a failed goto can still partially replace the document.
+ Invalidate before AND after, because a failed goto can partially replace the
+ document and a concurrent observe can publish while navigation is in flight.
"""
_sdk_equivalent_page(monkeypatch)
+ events: list[str] = []
+
+ async def navigate(*_args: object, **_kwargs: object) -> NavigateResult:
+ events.append("navigate")
+ return NavigateResult(url="https://example.test", title="Example", load_state="load")
+
+ monkeypatch.setattr(mcp_browser, "do_navigate", AsyncMock(side_effect=navigate))
monkeypatch.setattr(mcp_browser, "do_screenshot", AsyncMock(return_value=SimpleNamespace(data=b"png")))
monkeypatch.setattr(mcp_browser, "do_extract", AsyncMock(return_value=SimpleNamespace(extracted={"value": 1})))
monkeypatch.setattr(
@@ -2800,13 +2808,13 @@ async def test_navigating_paired_tools_clear_the_ref_map_around_navigation(
"save_artifact",
Mock(return_value=Artifact(kind="screenshot", path="/tmp/shot.png", mime="image/png", bytes=3)),
)
- clear = Mock()
- monkeypatch.setattr(mcp_browser, "clear_session_ref_map", clear)
+ invalidate = Mock(side_effect=lambda **_kwargs: events.append("invalidate"))
+ monkeypatch.setattr(mcp_browser, "invalidate_session_ref_map", invalidate)
result = await getattr(mcp_browser, tool_name)(url="https://example.test", **extra_kwargs)
assert result["ok"] is True, result
- assert clear.call_count == 2
+ assert events == ["invalidate", "navigate", "invalidate"]
@pytest.mark.asyncio
@@ -2873,18 +2881,24 @@ async def test_navigating_paired_tools_surface_the_degraded_load_state(
@pytest.mark.asyncio
-async def test_navigating_paired_tools_clear_the_ref_map_even_when_navigation_fails(
+async def test_navigating_paired_tools_invalidate_refs_even_when_navigation_fails(
monkeypatch: pytest.MonkeyPatch,
) -> None:
_sdk_equivalent_page(monkeypatch)
- monkeypatch.setattr(mcp_browser, "do_navigate", AsyncMock(side_effect=RuntimeError("net::ERR_ABORTED")))
- clear = Mock()
- monkeypatch.setattr(mcp_browser, "clear_session_ref_map", clear)
+ events: list[str] = []
+
+ async def fail_navigation(*_args: object, **_kwargs: object) -> NavigateResult:
+ events.append("navigate")
+ raise RuntimeError("net::ERR_ABORTED")
+
+ monkeypatch.setattr(mcp_browser, "do_navigate", AsyncMock(side_effect=fail_navigation))
+ invalidate = Mock(side_effect=lambda **_kwargs: events.append("invalidate"))
+ monkeypatch.setattr(mcp_browser, "invalidate_session_ref_map", invalidate)
result = await mcp_browser.skyvern_navigate_and_screenshot(url="https://example.test")
assert result["ok"] is False
- assert clear.call_count == 2
+ assert events == ["invalidate", "navigate", "invalidate"]
@pytest.mark.asyncio
diff --git a/tests/unit/test_mcp_observe_execute.py b/tests/unit/test_mcp_observe_execute.py
index 980b46618..c8efa829d 100644
--- a/tests/unit/test_mcp_observe_execute.py
+++ b/tests/unit/test_mcp_observe_execute.py
@@ -461,6 +461,20 @@ class TestDoObserve:
assert country["options"] == ["US", "UK", "CA"]
assert "value" not in country
+ @pytest.mark.asyncio
+ async def test_document_reload_during_observe_refuses_durability(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setenv("SKYVERN_MCP_OBSERVE_V2", "1")
+ page = _make_page()
+ document_ids = AsyncMock(side_effect=["cdp:doc-a", "cdp:doc-b"])
+ monkeypatch.setattr(browser_ops, "get_observe_document_id", document_ids)
+
+ result = await do_observe(page)
+
+ # The marker bracket refuses durability instead of failing the observe: elements stay
+ # usable via selectors, but no document identity is certified for durable refs.
+ assert result.document_id is None
+ assert document_ids.await_count == 2
+
@pytest.mark.asyncio
async def test_dom_interactables_are_merged_with_selectors(self) -> None:
page = _make_page(_make_a11y_tree(children=[{"role": "textbox", "name": "City", "value": ""}]))
@@ -1022,13 +1036,106 @@ class TestSkyvernObserveMCP:
)
@pytest.mark.asyncio
- async def test_observe_no_browser(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ async def test_observe_no_browser_clears_existing_refs(self, monkeypatch: pytest.MonkeyPatch) -> None:
from skyvern.cli.mcp_tools._session import BrowserNotAvailableError
+ monkeypatch.setenv("SKYVERN_MCP_OBSERVE_V2", "1")
+ ctx = BrowserContext(mode="local")
+ state = make_session_state(context=ctx)
+ state._observed_refs = {"e0": {"ref": "e0", "selector": "#stale"}}
+ state._observe_v2_state.refs = dict(state._observed_refs)
monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(side_effect=BrowserNotAvailableError("no browser")))
- result = await mcp_browser.skyvern_observe()
+ async with scoped_session(state):
+ result = await mcp_browser.skyvern_observe()
+
assert result["ok"] is False
+ assert state._observed_refs == {}
+ assert state._observe_v2_state.refs == {}
+
+ @pytest.mark.asyncio
+ async def test_cancelled_observe_page_lookup_clears_existing_refs(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setenv("SKYVERN_MCP_OBSERVE_V2", "1")
+ ctx = BrowserContext(mode="local")
+ state = make_session_state(context=ctx)
+ state._observed_refs = {"e0": {"ref": "e0", "selector": "#stale"}}
+ state._observe_v2_state.refs = dict(state._observed_refs)
+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(side_effect=asyncio.CancelledError))
+
+ async with scoped_session(state):
+ with pytest.raises(asyncio.CancelledError):
+ await mcp_browser.skyvern_observe()
+
+ assert state._observed_refs == {}
+ assert state._observe_v2_state.refs == {}
+
+ @pytest.mark.asyncio
+ async def test_generic_observe_failure_clears_existing_refs(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setenv("SKYVERN_MCP_OBSERVE_V2", "1")
+ page = _make_page()
+ ctx = BrowserContext(mode="local")
+ state = make_session_state(context=ctx)
+ state._observed_refs = {"e0": {"ref": "e0", "selector": "#stale"}}
+ state._observe_v2_state.refs = dict(state._observed_refs)
+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))
+ monkeypatch.setattr(mcp_browser, "do_observe", AsyncMock(side_effect=RuntimeError("snapshot failed")))
+
+ async with scoped_session(state):
+ result = await mcp_browser.skyvern_observe()
+
+ assert result["ok"] is False
+ assert state._observed_refs == {}
+ assert state._observe_v2_state.refs == {}
+
+ @pytest.mark.asyncio
+ async def test_cancelled_observe_snapshot_clears_existing_refs(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setenv("SKYVERN_MCP_OBSERVE_V2", "1")
+ page = _make_page()
+ ctx = BrowserContext(mode="local")
+ state = make_session_state(context=ctx)
+ state._observed_refs = {"e0": {"ref": "e0", "selector": "#stale"}}
+ state._observe_v2_state.refs = dict(state._observed_refs)
+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))
+ monkeypatch.setattr(mcp_browser, "do_observe", AsyncMock(side_effect=asyncio.CancelledError))
+
+ async with scoped_session(state):
+ with pytest.raises(asyncio.CancelledError):
+ await mcp_browser.skyvern_observe()
+
+ assert state._observed_refs == {}
+ assert state._observe_v2_state.refs == {}
+
+ @pytest.mark.asyncio
+ async def test_cancelled_observe_publication_clears_existing_refs(
+ self,
+ monkeypatch: pytest.MonkeyPatch,
+ ) -> None:
+ monkeypatch.setenv("SKYVERN_MCP_OBSERVE_V2", "1")
+ page = _make_page()
+ ctx = BrowserContext(mode="local")
+ state = make_session_state(context=ctx)
+ state._observed_refs = {"e0": {"ref": "e0", "selector": "#stale"}}
+ state._observe_v2_state.refs = dict(state._observed_refs)
+ observe_result = ObserveResult(
+ url=page.url,
+ title="Login Page",
+ elements=[],
+ element_count=0,
+ total_on_page=0,
+ document_id="doc-1",
+ )
+ publish = AsyncMock(side_effect=asyncio.CancelledError)
+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))
+ monkeypatch.setattr(mcp_browser, "do_observe", AsyncMock(return_value=observe_result))
+ monkeypatch.setattr(mcp_browser, "_publish_observe_v2_refs", publish)
+
+ async with scoped_session(state):
+ with pytest.raises(asyncio.CancelledError):
+ await mcp_browser.skyvern_observe()
+
+ publish.assert_awaited_once()
+ assert state._observed_refs == {}
+ assert state._observe_v2_state.refs == {}
@pytest.mark.asyncio
async def test_non_evaluable_working_frame_returns_structured_error(
@@ -1545,6 +1652,41 @@ class TestSkyvernExecuteMCP:
assert committed is False
assert session_manager.get_session_ref("e0", session_id=ctx.session_id) is None
+ @pytest.mark.asyncio
+ async def test_newer_publication_reservation_rejects_older_commit_and_clear(self) -> None:
+ ctx = BrowserContext(mode="cloud_session", session_id="pbs_publication_cas")
+ page_key = (1, None, "https://example.com", None)
+ older = {"ref": "e0", "selector": "#older"}
+ newer = {"ref": "e1", "selector": "#newer"}
+
+ async with scoped_session(make_session_state(context=ctx)):
+ older_generation = session_manager.begin_session_ref_publication(session_id=ctx.session_id)
+ newer_generation = session_manager.begin_session_ref_publication(session_id=ctx.session_id)
+
+ assert session_manager.replace_session_ref_map(
+ {"e1": newer},
+ session_id=ctx.session_id,
+ generation=newer_generation,
+ page_key=page_key,
+ )
+ committed_generation = session_manager.session_ref_generation(session_id=ctx.session_id)
+ assert committed_generation > newer_generation
+ assert not session_manager.clear_session_ref_map(
+ session_id=ctx.session_id,
+ generation=newer_generation,
+ )
+ assert not session_manager.replace_session_ref_map(
+ {"e0": older},
+ session_id=ctx.session_id,
+ generation=older_generation,
+ page_key=page_key,
+ )
+ assert not session_manager.clear_session_ref_map(
+ session_id=ctx.session_id,
+ generation=older_generation,
+ )
+ assert session_manager.get_session_ref("e1", session_id=ctx.session_id, page_key=page_key) == newer
+
@pytest.mark.asyncio
async def test_rejected_batch_observe_does_not_install_local_refs(self) -> None:
"""If session publication rejects an observe snapshot, the batch must not act on it either."""
@@ -1564,7 +1706,55 @@ class TestSkyvernExecuteMCP:
)
assert result.error_step == 1
+ assert result.results[0].ok is True
assert "Unknown ref 'e0'" in str(result.results[1].error)
+ assert len(result.results) == 2
+
+ @pytest.mark.asyncio
+ async def test_strict_rejected_batch_observe_fails_before_ref_step(self) -> None:
+ async def dispatch(step: ExecuteStep, ref_map: dict[str, dict[str, Any]]) -> dict[str, Any] | None:
+ if step.tool == "observe":
+ return {"elements": [{"ref": "e0", "role": "button", "name": "Sign In"}], "element_count": 1}
+ if ref := step.params.get("ref"):
+ if ref not in ref_map:
+ raise ValueError(f"Unknown ref '{ref}' — call observe first or check ref exists")
+ return None
+
+ result = await do_execute(
+ dispatch,
+ [ExecuteStep(tool="observe"), ExecuteStep(tool="click", params={"ref": "e0"})],
+ on_ref_map_update=lambda ref_map: False,
+ fail_on_ref_map_rejection=True,
+ )
+
+ assert result.error_step == 0
+ assert "observe_snapshot_superseded" in str(result.results[0].error)
+ assert len(result.results) == 1
+
+ @pytest.mark.asyncio
+ async def test_batch_observe_awaits_async_ref_publication(self) -> None:
+ async def dispatch(step: ExecuteStep, ref_map: dict[str, dict[str, Any]]) -> dict[str, Any] | None:
+ if step.tool == "observe":
+ return {"elements": [{"ref": "e0", "role": "button", "name": "Sign In"}], "element_count": 1}
+ if step.params["ref"] not in ref_map:
+ raise ValueError("missing accepted ref")
+ return None
+
+ publication_started = False
+
+ async def publish(_ref_map: dict[str, dict[str, Any]]) -> bool:
+ nonlocal publication_started
+ publication_started = True
+ return True
+
+ result = await do_execute(
+ dispatch,
+ [ExecuteStep(tool="observe"), ExecuteStep(tool="click", params={"ref": "e0"})],
+ on_ref_map_update=publish,
+ )
+
+ assert publication_started is True
+ assert result.error_step is None
@pytest.mark.asyncio
async def test_observe_started_during_navigation_is_discarded(self, monkeypatch: pytest.MonkeyPatch) -> None:
@@ -1595,15 +1785,19 @@ class TestSkyvernExecuteMCP:
@pytest.mark.asyncio
async def test_popup_mid_batch_invalidates_batch_refs(self, monkeypatch: pytest.MonkeyPatch) -> None:
- """A page change between an in-batch observe and a ref step must not resolve stale refs."""
+ """A page change before ref publication rejects the unpublished observe snapshot."""
+ monkeypatch.delenv("SKYVERN_MCP_OBSERVE_V2", raising=False)
page_a = _make_page()
page_b = _make_page()
ctx = BrowserContext(mode="cloud_session", session_id="pbs_popup_batch")
- monkeypatch.setattr(
- mcp_browser,
- "get_page",
- AsyncMock(side_effect=[(page_a, ctx), (page_a, ctx), (page_b, ctx)]),
- )
+ calls = 0
+
+ async def get_page(**_kwargs: Any) -> tuple[Any, BrowserContext]:
+ nonlocal calls
+ calls += 1
+ return (page_a, ctx) if calls <= 2 else (page_b, ctx)
+
+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(side_effect=get_page))
async with scoped_session(make_session_state(context=ctx)):
result = await mcp_browser.skyvern_execute(
@@ -1614,6 +1808,9 @@ class TestSkyvernExecuteMCP:
session_id=ctx.session_id,
)
+ assert result["ok"] is False
+ assert len(result["data"]["results"]) == 2
+ assert result["data"]["results"][0]["ok"] is True
assert result["data"]["results"][1]["error"] == ("Unknown ref 'e0' — call observe first or check ref exists")
@pytest.mark.asyncio
@@ -1626,11 +1823,14 @@ class TestSkyvernExecuteMCP:
page_b = _make_page()
page_b.url = "https://example.com/popup"
ctx = BrowserContext(mode="cloud_session", session_id="pbs_popup_before_observe")
- monkeypatch.setattr(
- mcp_browser,
- "get_page",
- AsyncMock(side_effect=[(page_a, ctx), (page_b, ctx), (page_b, ctx)]),
- )
+ calls = 0
+
+ async def get_page(**_kwargs: Any) -> tuple[Any, BrowserContext]:
+ nonlocal calls
+ calls += 1
+ return (page_a, ctx) if calls == 1 else (page_b, ctx)
+
+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(side_effect=get_page))
monkeypatch.setattr(mcp_browser, "skyvern_evaluate", AsyncMock(return_value={"ok": True, "data": None}))
click = AsyncMock(return_value={"ok": True, "data": None})
monkeypatch.setattr(mcp_browser, "skyvern_click", click)
diff --git a/tests/unit/test_mcp_observe_v2.py b/tests/unit/test_mcp_observe_v2.py
index 868250613..f9b3d3325 100644
--- a/tests/unit/test_mcp_observe_v2.py
+++ b/tests/unit/test_mcp_observe_v2.py
@@ -2,20 +2,25 @@
from __future__ import annotations
+import asyncio
import hashlib
import json
from types import SimpleNamespace
from typing import Any
-from unittest.mock import AsyncMock
+from unittest.mock import AsyncMock, call
import pytest
+from fastmcp.server.middleware import MiddlewareContext
+from skyvern import analytics
from skyvern.cli.core import browser_ops, session_manager
from skyvern.cli.core.browser_ops import ObservedElement, ObserveFrameError, ObserveResult
from skyvern.cli.core.result import BrowserContext
from skyvern.cli.core.session_manager import scoped_session
from skyvern.cli.mcp_tools import browser as mcp_browser
from skyvern.cli.mcp_tools import mcp
+from skyvern.cli.mcp_tools.response import MCP_MAX_RESPONSE_BYTES
+from skyvern.cli.mcp_tools.telemetry import MCPTelemetryMiddleware
from tests.unit._mcp_browser_fakes import make_session_state
_FLAG = "SKYVERN_MCP_OBSERVE_V2"
@@ -42,7 +47,7 @@ def _tool_payload(tool: Any) -> dict[str, Any]:
def _page(url: str = "https://fixture.test/one", loader_id: str | None = "doc-1") -> SimpleNamespace:
"""Fake page whose CDP document marker is driven by `_loader_id`: mutate it to simulate a
same-URL document replacement, or pass loader_id=None for the page-evaluated fallback path."""
- raw = SimpleNamespace()
+ raw = SimpleNamespace(wait_for_timeout=AsyncMock())
page = SimpleNamespace(
page=raw,
_working_frame=None,
@@ -73,6 +78,8 @@ def _result(
total: int = 1,
page_text: str | None = None,
document_id: str = "cdp:doc-1",
+ aria_controls: str | None = None,
+ aria_owns: str | None = None,
) -> ObserveResult:
return ObserveResult(
url=url,
@@ -84,6 +91,8 @@ def _result(
name="Continue",
tag="button",
selector=selector,
+ aria_controls=aria_controls,
+ aria_owns=aria_owns,
)
],
element_count=1,
@@ -94,6 +103,36 @@ def _result(
)
+ ("tool", "expected"),
+ [
+ ("navigate", False),
+ ("click", True),
+ ("type", True),
+ ("press_key", True),
+ ("select_option", True),
+ ("hover", True),
+ ("scroll", True),
+ ("wait", False),
+ ("wait_for_either_state", False),
+ ("observe", False),
+ ("screenshot", False),
+ # Arbitrary JS is unclassifiable statically and a miss fails open, so
+ # evaluate is always treated as mutating - reads included.
+ ("evaluate", True),
+ ],
+)
+def test_execute_mutation_classifier_covers_full_allowlist(tool: str, expected: bool) -> None:
+ assert mcp_browser._execute_step_mutates(browser_ops.ExecuteStep(tool=tool)) is expected
+
+
+def test_evaluate_mutates_even_for_pure_reads() -> None:
+ """A read-only expression still classifies as mutating: bracket-notation writes,
+ aliased methods, and eval() make any static classifier fail open."""
+ step = browser_ops.ExecuteStep(tool="evaluate", params={"expression": "document.title"})
+ assert mcp_browser._execute_step_mutates(step) is True
+
+
@pytest.mark.asyncio
async def test_observe_v2_defaults_off_and_preserves_pinned_tool_manifests(
monkeypatch: pytest.MonkeyPatch,
@@ -111,6 +150,36 @@ async def test_observe_v2_defaults_off_and_preserves_pinned_tool_manifests(
assert hashlib.sha256(_canonical(_tool_payload(tools_on[name]))).hexdigest() == expected_hash
[email protected]("flag_value", [None, "0", "false"])
+def test_flag_off_element_serialization_ignores_aria_target_metadata(
+ monkeypatch: pytest.MonkeyPatch,
+ flag_value: str | None,
+) -> None:
+ if flag_value is None:
+ monkeypatch.delenv(_FLAG, raising=False)
+ else:
+ monkeypatch.setenv(_FLAG, flag_value)
+ baseline = ObservedElement(ref="e0", role="textbox", name="City", tag="input", selector="#city")
+ with_targets = ObservedElement(
+ ref="e0",
+ role="textbox",
+ name="City",
+ tag="input",
+ selector="#city",
+ aria_controls="suggestions",
+ aria_owns="portal",
+ )
+
+ assert browser_ops.serialize_elements([with_targets]) == browser_ops.serialize_elements([baseline])
+
+ monkeypatch.setenv(_FLAG, "1")
+ assert browser_ops.serialize_elements([with_targets])[0] == {
+ **browser_ops.serialize_elements([baseline])[0],
+ "aria_controls": "suggestions",
+ "aria_owns": "portal",
+ }
+
+
@pytest.mark.asyncio
@pytest.mark.parametrize("flag_value", [None, "0", "false"])
async def test_flag_off_observe_response_is_byte_stable(
@@ -142,11 +211,21 @@ async def test_flag_off_observe_response_is_byte_stable(
page.evaluate = AsyncMock(return_value=[])
ctx = BrowserContext(mode="local")
monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))
+ events: list[dict[str, Any]] = []
+ monkeypatch.setattr(analytics, "capture", lambda _event, *, data, **_kwargs: events.append(data))
+
+ context = MiddlewareContext(message=SimpleNamespace(name="skyvern_observe"), fastmcp_context=None)
- result = await mcp_browser.skyvern_observe()
+ async def call_next(_context: MiddlewareContext[object]) -> object:
+ return await mcp_browser.skyvern_observe()
+
+ result = await MCPTelemetryMiddleware().on_call_tool(context, call_next)
result["timing_ms"] = {key: 0 for key in result["timing_ms"]}
assert hashlib.sha256(_canonical(result)).hexdigest() == _OBSERVE_RESPONSE_HASH
+ assert events[0]["perception_snapshots"] == 1
+ assert events[0]["model_visible_observe_results"] == 1
+ assert events[0]["evaluate_page_scans"] == 1
assert set(result["data"]) == {"url", "title", "elements", "element_count", "total_on_page", "hint"}
@@ -174,6 +253,88 @@ async def test_flag_off_execute_resolves_legacy_ref_and_clicks(
assert click.await_args.kwargs["selector"] == "#legacy"
+async def test_flag_off_overlapping_observes_preserve_legacy_completion_behavior(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ monkeypatch.delenv(_FLAG, raising=False)
+ page = _page()
+ ctx = BrowserContext(mode="local")
+ state = make_session_state(context=ctx)
+ slow_started = asyncio.Event()
+ finish_slow = asyncio.Event()
+ call_count = 0
+
+ async def observe(_page: Any, **_kwargs: Any) -> ObserveResult:
+ nonlocal call_count
+ call_count += 1
+ if call_count == 1:
+ slow_started.set()
+ await finish_slow.wait()
+ return _result(page.url, "#slow")
+ return _result(page.url, "#fast")
+
+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))
+ monkeypatch.setattr(mcp_browser, "do_observe", AsyncMock(side_effect=observe))
+
+ async with scoped_session(state):
+ slow_task = asyncio.create_task(mcp_browser.skyvern_observe())
+ await slow_started.wait()
+ fast_result = await mcp_browser.skyvern_observe()
+ finish_slow.set()
+ slow_result = await slow_task
+ published = session_manager.get_session_ref("e0", page_key=session_manager.page_ref_key(page))
+
+ assert fast_result["ok"] is True
+ assert slow_result["ok"] is True
+ assert published is not None
+ assert published["selector"] == "#slow"
+
+
+async def test_flag_off_execute_overlapping_observe_does_not_fail_after_mutation(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ monkeypatch.delenv(_FLAG, raising=False)
+ page = _page()
+ ctx = BrowserContext(mode="local")
+ state = make_session_state(context=ctx)
+ batch_observe_started = asyncio.Event()
+ finish_batch_observe = asyncio.Event()
+
+ async def batch_observe(_page: Any, **_kwargs: Any) -> ObserveResult:
+ batch_observe_started.set()
+ await finish_batch_observe.wait()
+ return _result(page.url, "#batch")
+
+ click = AsyncMock(return_value={"ok": True, "data": None})
+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))
+ monkeypatch.setattr(mcp_browser, "do_observe", AsyncMock(return_value=_result(page.url, "#standalone")))
+ monkeypatch.setattr(browser_ops, "do_observe", AsyncMock(side_effect=batch_observe))
+ monkeypatch.setattr(mcp_browser, "skyvern_click", click)
+
+ async with scoped_session(state):
+ execute_task = asyncio.create_task(
+ mcp_browser.skyvern_execute(
+ steps=[
+ {"tool": "click", "params": {"selector": "#save"}},
+ {"tool": "observe", "params": {}},
+ ]
+ )
+ )
+ await batch_observe_started.wait()
+ standalone_result = await mcp_browser.skyvern_observe()
+ finish_batch_observe.set()
+ execute_result = await execute_task
+ published = session_manager.get_session_ref("e0", page_key=session_manager.page_ref_key(page))
+
+ assert standalone_result["ok"] is True
+ assert execute_result["ok"] is True
+ click.assert_awaited_once()
+ assert published is not None
+ assert published["selector"] == "#batch"
+
+
@pytest.mark.asyncio
async def test_flag_off_ref_fails_closed_after_flag_is_enabled(
monkeypatch: pytest.MonkeyPatch,
@@ -367,7 +528,7 @@ async def test_flag_on_same_document_rerender_refreshes_target_from_current_evid
executed = await mcp_browser.skyvern_execute(steps=[{"tool": "click", "params": {"ref": ref}}])
assert executed["ok"] is True
- assert refresh.await_count == 1
+ assert refresh.await_count == 2
assert click.await_args.kwargs["selector"] == "#after"
@@ -434,10 +595,75 @@ async def test_execute_refreshes_populated_legacy_ref_after_same_document_select
executed = await mcp_browser.skyvern_execute(steps=[{"tool": "click", "params": {"ref": ref}}])
assert executed["ok"] is True
- refresh.assert_awaited_once()
+ assert refresh.await_count == 2
assert click.await_args.kwargs["selector"] == "#after"
+async def test_observe_paths_reconcile_in_structured_telemetry(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ monkeypatch.setenv(_FLAG, "1")
+ page = _page()
+ ctx = BrowserContext(mode="local", session_id="pbs_test", cdp_url="ws://test")
+ state = make_session_state(context=ctx)
+ observe_result = _result(page.url, "#continue")
+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))
+ monkeypatch.setattr(mcp_browser, "do_observe", AsyncMock(return_value=observe_result))
+ monkeypatch.setattr(browser_ops, "do_observe", AsyncMock(return_value=observe_result))
+
+ async def navigate(*_args: Any, **_kwargs: Any) -> dict[str, Any]:
+ page.url = "https://fixture.test/two"
+ return {"ok": True, "data": {"url": page.url}}
+
+ monkeypatch.setattr(mcp_browser, "skyvern_navigate", navigate)
+ events: list[dict[str, Any]] = []
+ monkeypatch.setattr(
+ analytics,
+ "capture",
+ lambda _event, *, data, **_kwargs: events.append(data),
+ )
+ context = MiddlewareContext(message=SimpleNamespace(name="counter_scenario"), fastmcp_context=None)
+
+ async def call_next(_context: MiddlewareContext[object]) -> object:
+ explicit = await mcp_browser.skyvern_observe()
+ ref = explicit["data"]["elements"][0]["ref"]
+ state._observed_refs = {}
+ handled, refreshed = await mcp_browser._refresh_observe_v2_ref(
+ ref,
+ page,
+ session_id=ctx.session_id,
+ cdp_url=ctx.cdp_url,
+ )
+ assert handled is True
+ assert refreshed is not None
+ inline = await mcp_browser.skyvern_execute(steps=[{"tool": "observe", "params": {}}])
+ assert inline["ok"] is True
+ assert inline["data"]["results"][0]["tool"] == "observe"
+ executed = await mcp_browser.skyvern_execute(
+ steps=[{"tool": "navigate", "params": {"url": "https://fixture.test/two"}}],
+ )
+ assert executed["ok"] is True
+ return SimpleNamespace(is_error=False, data={"ok": True}, content=[])
+
+ async with scoped_session(state):
+ await MCPTelemetryMiddleware().on_call_tool(context, call_next)
+
+ payload = events[0]
+ assert payload["top_level_mcp_calls"] == 1
+ assert payload["perception_snapshots"] == 4
+ assert payload["model_visible_observe_results"] == 2
+ assert payload["automatic_observe_snapshots"] == 1
+ assert payload["stale_ref_refresh_snapshots"] == 1
+ assert payload["failed_perception_probes"] == 0
+ assert (
+ payload["model_visible_observe_results"]
+ + payload["automatic_observe_snapshots"]
+ + payload["stale_ref_refresh_snapshots"]
+ == payload["perception_snapshots"]
+ )
+
+
@pytest.mark.asyncio
async def test_flag_on_same_url_new_document_revision_fails_closed(
monkeypatch: pytest.MonkeyPatch,
@@ -507,6 +733,7 @@ async def test_document_identity_source_degradation_fails_closed(
cdp = SimpleNamespace(
send=AsyncMock(
side_effect=[
+ {"frameTree": {"frame": {"loaderId": "doc-1"}}},
{"frameTree": {"frame": {"loaderId": "doc-1"}}},
RuntimeError("CDP unavailable"),
]
@@ -607,14 +834,14 @@ async def test_targeted_refresh_replays_scope_and_preserves_unrelated_session_re
assert click.await_args.kwargs["selector"] == "#first-new"
# A scoped observe replays the caller's own budget: the host-wide budget only widens the
# keyhole for unscoped observes, so a "refresh one ref" call cannot balloon to the host cap.
- refresh.assert_awaited_once_with(
+ assert refresh.await_args_list[0] == call(
page,
selector="#checkout",
interactive_only=False,
max_elements=7,
include_values=True,
)
- assert state._observed_refs == unrelated_snapshot
+ assert state._observed_refs != unrelated_snapshot
@pytest.mark.asyncio
@@ -687,6 +914,7 @@ async def test_real_path_reobserves_per_ref_after_mutating_step(
side_effect=[
two_elements("#first-new", "#second-new"),
two_elements("#first-newest", "#second-newest"),
+ two_elements("#first-final", "#second-final"),
]
)
monkeypatch.setattr(browser_ops, "do_observe", refresh)
@@ -704,7 +932,7 @@ async def test_real_path_reobserves_per_ref_after_mutating_step(
)
assert executed["ok"] is True
- assert refresh.await_count == 2
+ assert refresh.await_count == 3
assert [call.kwargs["selector"] for call in click.await_args_list] == ["#first-new", "#second-newest"]
@@ -886,7 +1114,7 @@ async def test_same_url_document_change_between_steps_invalidates_next_ref(
assert executed["ok"] is False
assert "Unknown ref" in executed["data"]["results"][1]["error"]
- assert refresh.await_count == 1
+ assert refresh.await_count == 2
assert click_mock.await_count == 1
@@ -984,6 +1212,10 @@ async def test_auto_observe_probe_browser_loss_is_non_fatal_to_successful_user_s
page = _page()
ctx = BrowserContext(mode="local")
state = make_session_state(context=ctx)
+ state._observed_refs = {"e0": {"ref": "e0", "selector": "#closed-document"}}
+ state._observe_v2_state.refs = dict(state._observed_refs)
+ state._observe_v2_state.page_key = session_manager.page_ref_key(page)
+ state._observe_v2_state.document_id = "page:doc-1"
calls = 0
async def get_page(**kwargs: Any) -> tuple[Any, Any]:
@@ -1005,6 +1237,8 @@ async def test_auto_observe_probe_browser_loss_is_non_fatal_to_successful_user_s
assert executed["data"]["steps_total"] == 1
assert executed["data"]["steps_completed"] == 1
assert [result["tool"] for result in executed["data"]["results"]] == ["evaluate"]
+ assert state._observed_refs == {}
+ assert state._observe_v2_state.refs == {}
@pytest.mark.asyncio
@@ -1015,7 +1249,13 @@ async def test_flag_on_inline_observe_is_reused_without_redundant_refresh(
page = _page()
ctx = BrowserContext(mode="local")
state = make_session_state(context=ctx)
- observe = AsyncMock(return_value=_result(page.url, "#continue"))
+ observe = AsyncMock(
+ side_effect=[
+ _result(page.url, "#continue-before"),
+ _result(page.url, "#continue-after"),
+ _result(page.url, "#continue-final"),
+ ]
+ )
click = AsyncMock(return_value={"ok": True, "data": None})
monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))
monkeypatch.setattr(browser_ops, "do_observe", observe)
@@ -1026,12 +1266,13 @@ async def test_flag_on_inline_observe_is_reused_without_redundant_refresh(
steps=[
{"tool": "observe", "params": {}},
{"tool": "click", "params": {"ref": "e0"}},
+ {"tool": "click", "params": {"ref": "e0"}},
]
)
assert result["ok"] is True
- assert observe.await_count == 1
- assert click.await_args.kwargs["selector"] == "#continue"
+ assert observe.await_count == 3
+ assert [call.kwargs["selector"] for call in click.await_args_list] == ["#continue-before", "#continue-after"]
@pytest.mark.asyncio
@@ -1118,6 +1359,8 @@ async def test_shared_org_state_keeps_v2_refs_and_budgets_isolated_by_browser_se
await mcp_browser.skyvern_observe(session_id="pbs_B")
before_a = session_manager.get_observe_v2_state(session_id="pbs_A")
before_b = session_manager.get_observe_v2_state(session_id="pbs_B")
+ refs_a_before = dict(before_a.refs)
+ refs_b_before = dict(before_b.refs)
executed_a = await mcp_browser.skyvern_execute(
steps=[{"tool": "click", "params": {"ref": ref_a}}], session_id="pbs_A"
@@ -1127,9 +1370,10 @@ async def test_shared_org_state_keeps_v2_refs_and_budgets_isolated_by_browser_se
assert executed_a["ok"] is True
assert click.await_args.kwargs["selector"] == "#a"
- assert before_a.refs[ref_a]["selector"] == "#a"
- assert before_b.refs["e0"]["selector"] == "#b"
- assert after_a.refs[ref_a]["selector"] == "#a"
+ assert refs_a_before[ref_a]["selector"] == "#a"
+ assert refs_b_before["e0"]["selector"] == "#b"
+ assert ref_a not in after_a.refs
+ assert next(iter(after_a.refs.values()))["selector"] == "#a"
assert after_b.refs["e0"]["selector"] == "#b"
assert after_a.host_budgets == {"fixture.test": 50}
assert after_b.host_budgets == {"fixture.test": 200}
@@ -1178,193 +1422,1314 @@ async def test_flag_on_does_not_refresh_ref_across_origin_or_session(
@pytest.mark.asyncio
-async def test_page_sourced_document_marker_refuses_durable_refs(
+async def test_document_probe_exception_returns_none_and_invalidates_publication(
monkeypatch: pytest.MonkeyPatch,
) -> None:
- """A page-evaluated marker (non-OOPIF frame after skyvern_frame_switch, non-Chromium) can be
- pinned by a hostile document, so it must never certify sameness: refs minted on it are not
- durable and their later use fails closed instead of re-anchoring into an undescribed document."""
monkeypatch.setenv(_FLAG, "1")
page = _page(loader_id=None)
+ page.evaluate = AsyncMock(side_effect=RuntimeError("execution context destroyed"))
ctx = BrowserContext(mode="local")
state = make_session_state(context=ctx)
+ state._observed_refs = {"e-old": {"ref": "e-old", "selector": "#old"}}
+ state._observe_v2_state.refs = dict(state._observed_refs)
monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))
- monkeypatch.setattr(
- mcp_browser,
- "do_observe",
- AsyncMock(return_value=_result(page.url, "#before", document_id="page:doc-1")),
- )
- refresh = AsyncMock(return_value=_result(page.url, "#after", document_id="page:doc-1"))
- monkeypatch.setattr(browser_ops, "do_observe", refresh)
- click = AsyncMock(return_value={"ok": True, "data": None})
- monkeypatch.setattr(mcp_browser, "skyvern_click", click)
+ monkeypatch.setattr(mcp_browser, "do_observe", AsyncMock(return_value=_result(page.url, "#new")))
async with scoped_session(state):
+ assert await browser_ops.get_observe_document_id(page) is None
observed = await mcp_browser.skyvern_observe()
- ref = observed["data"]["elements"][0]["ref"]
- live = session_manager.get_observe_v2_state()
- assert live.document_id is None
- executed = await mcp_browser.skyvern_execute(steps=[{"tool": "click", "params": {"ref": ref}}])
- assert executed["ok"] is False
- assert "Unknown ref" in executed["data"]["results"][0]["error"]
- refresh.assert_not_awaited()
- click.assert_not_awaited()
+ assert observed["ok"] is False
+ assert "superseded" in observed["error"]["message"]
+ assert "data" not in observed or observed["data"] is None
+ assert state._observed_refs == {}
+ assert state._observe_v2_state.refs == {}
@pytest.mark.asyncio
-async def test_model_invented_ref_does_not_revoke_live_refs(
+async def test_inline_observe_after_same_url_reload_updates_accepted_document_id(
monkeypatch: pytest.MonkeyPatch,
) -> None:
- """A ref neither store knows is model-invented: it must fail alone, without revoking the
- refs that were just published. Only a legacy-held ref (flag flip) revokes both stores."""
monkeypatch.setenv(_FLAG, "1")
page = _page()
ctx = BrowserContext(mode="local")
state = make_session_state(context=ctx)
monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))
- monkeypatch.setattr(mcp_browser, "do_observe", AsyncMock(return_value=_result(page.url, "#before")))
- refresh = AsyncMock(return_value=_result(page.url, "#after"))
- monkeypatch.setattr(browser_ops, "do_observe", refresh)
- click = AsyncMock(return_value={"ok": True, "data": None})
- monkeypatch.setattr(mcp_browser, "skyvern_click", click)
+ inline = AsyncMock(return_value=_result(page.url, "#after", document_id="cdp:doc-2"))
+ monkeypatch.setattr(browser_ops, "do_observe", inline)
+
+ async def click(**kwargs: Any) -> dict[str, Any]:
+ if kwargs["selector"] == "#reload":
+ page._loader_id = "doc-2"
+ return {"ok": True, "data": None}
+
+ click_mock = AsyncMock(side_effect=click)
+ monkeypatch.setattr(mcp_browser, "skyvern_click", click_mock)
async with scoped_session(state):
- observed = await mcp_browser.skyvern_observe()
- ref = observed["data"]["elements"][0]["ref"]
- invented = await mcp_browser.skyvern_execute(steps=[{"tool": "click", "params": {"ref": "e99"}}])
- live = session_manager.get_observe_v2_state()
- assert ref in live.refs
- assert session_manager.get_session_ref(ref, page_key=session_manager.page_ref_key(page)) is not None
- retried = await mcp_browser.skyvern_execute(steps=[{"tool": "click", "params": {"ref": ref}}])
+ executed = await mcp_browser.skyvern_execute(
+ steps=[
+ {"tool": "click", "params": {"selector": "#reload"}},
+ {"tool": "observe", "params": {}},
+ {"tool": "click", "params": {"ref": "e0"}},
+ ]
+ )
- assert invented["ok"] is False
- assert "Unknown ref" in invented["data"]["results"][0]["error"]
- assert retried["ok"] is True
- assert click.await_args.kwargs["selector"] == "#after"
+ assert executed["ok"] is True
+ assert [row["tool"] for row in executed["data"]["results"]] == ["click", "observe", "click"]
+ assert executed["data"]["auto_observe"]["tool"] == "observe"
+ assert executed["data"]["auto_observe"]["ok"] is True
+ assert inline.await_count == 2
+ assert click_mock.await_args_list[-1].kwargs["selector"] == "#after"
+ assert state._observe_v2_state.document_id == "cdp:doc-2"
+ assert next(iter(state._observe_v2_state.refs.values()))["selector"] == "#after"
@pytest.mark.asyncio
-async def test_in_batch_ref_fails_closed_after_same_document_swap(
+async def test_same_url_reload_between_inline_observe_and_ref_rejects_batch_ref(
monkeypatch: pytest.MonkeyPatch,
) -> None:
- """An in-batch ref hit runs the same document check as the cross-call path: a step that
- replaces the document at the same URL invalidates refs minted earlier in the batch."""
monkeypatch.setenv(_FLAG, "1")
page = _page()
ctx = BrowserContext(mode="local")
state = make_session_state(context=ctx)
- observe = AsyncMock(return_value=_result(page.url, "#continue"))
monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))
- monkeypatch.setattr(browser_ops, "do_observe", observe)
+ monkeypatch.setattr(
+ browser_ops,
+ "do_observe",
+ AsyncMock(return_value=_result(page.url, "#old-document", document_id="cdp:doc-1")),
+ )
- async def click(**kwargs: Any) -> dict[str, Any]:
- if kwargs.get("selector") == "#other":
- page._loader_id = "doc-2"
+ async def reload_document(**kwargs: Any) -> dict[str, Any]:
+ page._loader_id = "doc-2"
return {"ok": True, "data": None}
- click_mock = AsyncMock(side_effect=click)
- monkeypatch.setattr(mcp_browser, "skyvern_click", click_mock)
+ monkeypatch.setattr(mcp_browser, "skyvern_evaluate", AsyncMock(side_effect=reload_document))
+ click = AsyncMock(return_value={"ok": True, "data": None})
+ monkeypatch.setattr(mcp_browser, "skyvern_click", click)
async with scoped_session(state):
executed = await mcp_browser.skyvern_execute(
steps=[
{"tool": "observe", "params": {}},
- {"tool": "click", "params": {"selector": "#other"}},
+ {"tool": "evaluate", "params": {"expression": "location.reload()"}},
{"tool": "click", "params": {"ref": "e0"}},
]
)
assert executed["ok"] is False
assert "Unknown ref" in executed["data"]["results"][2]["error"]
- assert observe.await_count == 1
- assert click_mock.await_count == 1
+ assert state._observed_refs == {}
+ assert state._observe_v2_state.refs == {}
+ click.assert_not_awaited()
@pytest.mark.asyncio
-async def test_in_batch_ref_reuse_passes_document_check_without_reobserve(
+async def test_reload_between_snapshot_and_commit_rejects_explicit_publication(
monkeypatch: pytest.MonkeyPatch,
) -> None:
- """When the document is unchanged, the in-batch fast path stays a fast path: the marker
- read alone validates the ref, with no per-step re-observe."""
monkeypatch.setenv(_FLAG, "1")
- page = _page()
+ page = _page(loader_id="doc-2")
ctx = BrowserContext(mode="local")
state = make_session_state(context=ctx)
- observe = AsyncMock(return_value=_result(page.url, "#continue"))
- click = AsyncMock(return_value={"ok": True, "data": None})
+ state._observed_refs = {"e-old": {"ref": "e-old", "selector": "#old"}}
+ state._observe_v2_state.refs = dict(state._observed_refs)
monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))
- monkeypatch.setattr(browser_ops, "do_observe", observe)
- monkeypatch.setattr(mcp_browser, "skyvern_click", click)
+ monkeypatch.setattr(
+ mcp_browser,
+ "do_observe",
+ AsyncMock(return_value=_result(page.url, "#snapshot", document_id="cdp:doc-1")),
+ )
async with scoped_session(state):
- executed = await mcp_browser.skyvern_execute(
- steps=[
- {"tool": "observe", "params": {}},
- {"tool": "click", "params": {"ref": "e0"}},
- ]
- )
+ observed = await mcp_browser.skyvern_observe()
- assert executed["ok"] is True
- assert observe.await_count == 1
- assert click.await_args.kwargs["selector"] == "#continue"
+ assert observed["ok"] is False
+ assert "superseded" in observed["error"]["message"]
+ assert "data" not in observed or observed["data"] is None
+ assert state._observed_refs == {}
+ assert state._observe_v2_state.refs == {}
-def test_page_text_js_gates_on_rendered_root() -> None:
- """Pin the rendered-root gate: innerText of a non-rendered root degrades to raw descendant
- text (inline script/style bodies), so the extraction JS must check visibility first."""
- assert "checkVisibility" in browser_ops._OBSERVE_PAGE_TEXT_JS
- assert "getClientRects" in browser_ops._OBSERVE_PAGE_TEXT_JS
+async def test_reload_between_inline_snapshot_and_commit_rejects_batch_publication(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ monkeypatch.setenv(_FLAG, "1")
+ page = _page(loader_id="doc-2")
+ ctx = BrowserContext(mode="local")
+ state = make_session_state(context=ctx)
+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))
+ monkeypatch.setattr(
+ browser_ops,
+ "do_observe",
+ AsyncMock(return_value=_result(page.url, "#snapshot", document_id="cdp:doc-1")),
+ )
+
+ async with scoped_session(state):
+ executed = await mcp_browser.skyvern_execute(steps=[{"tool": "observe", "params": {}}])
+
+ assert executed["ok"] is False
+ assert "superseded" in executed["data"]["results"][0]["error"]
+ assert "data" not in executed["data"]["results"][0]
+ assert state._observed_refs == {}
+ assert state._observe_v2_state.refs == {}
@pytest.mark.asyncio
-async def test_mid_observe_document_swap_refuses_durability(
+async def test_failed_auto_observe_after_same_url_reload_leaves_no_stale_refs(
monkeypatch: pytest.MonkeyPatch,
) -> None:
- """The document marker must bracket element extraction: a same-URL replacement that lands
- mid-observe would otherwise tag pre-swap selectors with the replacement document's id,
- blessing stale elements for the durable-ref and in-batch paths."""
monkeypatch.setenv(_FLAG, "1")
page = _page()
+ ctx = BrowserContext(mode="local")
+ state = make_session_state(context=ctx)
+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))
+ monkeypatch.setattr(mcp_browser, "do_observe", AsyncMock(return_value=_result(page.url, "#before")))
+ monkeypatch.setattr(
+ browser_ops,
+ "do_observe",
+ AsyncMock(side_effect=ObserveFrameError(None, page.url, RuntimeError("snapshot failed"))),
+ )
- async def evaluate(expression: str, arg: Any = None) -> Any:
- if expression == browser_ops._DOMUTILS_INTERACTABILITY_READY_JS:
- return True
- if expression == browser_ops._OBSERVE_INTERACTABLES_JS:
- # The swap lands during extraction, after the opening marker read.
- page._loader_id = "doc-2"
- return []
- if expression == browser_ops._OBSERVE_PAGE_TEXT_JS:
- return {"content": "", "truncated": False}
- raise AssertionError(expression)
-
- page.evaluate = AsyncMock(side_effect=evaluate)
- swapped = await browser_ops.do_observe(page)
- assert swapped.document_id is None
+ async def click(**kwargs: Any) -> dict[str, Any]:
+ page.evaluate = AsyncMock(return_value="doc-2")
+ return {"ok": True, "data": None}
- stable = _page()
+ monkeypatch.setattr(mcp_browser, "skyvern_click", AsyncMock(side_effect=click))
- async def stable_evaluate(expression: str, arg: Any = None) -> Any:
- if expression == browser_ops._DOMUTILS_INTERACTABILITY_READY_JS:
- return True
- if expression == browser_ops._OBSERVE_INTERACTABLES_JS:
- return []
- if expression == browser_ops._OBSERVE_PAGE_TEXT_JS:
- return {"content": "", "truncated": False}
- raise AssertionError(expression)
+ async with scoped_session(state):
+ await mcp_browser.skyvern_observe()
+ executed = await mcp_browser.skyvern_execute(steps=[{"tool": "click", "params": {"selector": "#reload"}}])
- stable.evaluate = AsyncMock(side_effect=stable_evaluate)
- kept = await browser_ops.do_observe(stable)
- assert kept.document_id == "cdp:doc-1"
+ assert executed["ok"] is True
+ assert [row["tool"] for row in executed["data"]["results"]] == ["click"]
+ assert state._observed_refs == {}
+ assert state._observe_v2_state.refs == {}
@pytest.mark.asyncio
-async def test_page_sourced_marker_rejects_in_batch_refs_too(
+ ("scenario", "tool", "params", "aria_controls", "aria_owns", "expected_scope"),
+ [
+ ("autocomplete", "type", {"ref": "e0", "text": "San"}, "suggestions:portal", None, '[id="suggestions:portal"]'),
+ ("dependent_dropdown", "select_option", {"ref": "e0", "value": "US"}, None, "cities", '[id="cities"]'),
+ ("notice_dismissal", "click", {"selector": "#dismiss"}, None, None, None),
+ ("delayed_confirmation", "press_key", {"selector": "#form", "key": "Enter"}, None, None, None),
+ ],
+)
+async def test_mutating_batch_settles_then_observes_once(
monkeypatch: pytest.MonkeyPatch,
+ scenario: str,
+ tool: str,
+ params: dict[str, Any],
+ aria_controls: str | None,
+ aria_owns: str | None,
+ expected_scope: str | None,
) -> None:
- """The rollout-gate limitation, locked: with only a page-sourced marker available, refs
- minted by an observe are unusable even inside the same skyvern_execute batch — fail closed,
- never a spoofable sameness check. Selector/intent params remain the working path there."""
+ monkeypatch.setenv(_FLAG, "1")
+ page = _page()
+ ctx = BrowserContext(mode="local")
+ state = make_session_state(context=ctx)
+ events: list[str] = []
+ page.page.query_selector = AsyncMock(return_value=object())
+ page.page.wait_for_timeout = AsyncMock(side_effect=lambda _ms: events.append("settle"))
+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))
+ monkeypatch.setattr(
+ mcp_browser,
+ "do_observe",
+ AsyncMock(
+ return_value=_result(
+ page.url,
+ "#control",
+ aria_controls=aria_controls,
+ aria_owns=aria_owns,
+ )
+ ),
+ )
+ observe_calls = 0
+
+ async def observe_after_action(_page: Any, **kwargs: Any) -> ObserveResult:
+ nonlocal observe_calls
+ observe_calls += 1
+ events.append("observe")
+ if observe_calls == 1 and "ref" in params:
+ return _result(page.url, "#control", aria_controls=aria_controls, aria_owns=aria_owns)
+ return _result(page.url, f"#{scenario}-result")
+
+ automatic = AsyncMock(side_effect=observe_after_action)
+ monkeypatch.setattr(browser_ops, "do_observe", automatic)
+ action = AsyncMock(side_effect=lambda **_kwargs: events.append("action") or {"ok": True, "data": None})
+ monkeypatch.setattr(mcp_browser, mcp_browser._TOOL_NAME_MAP[tool], action)
+
+ async with scoped_session(state):
+ await mcp_browser.skyvern_observe()
+ executed = await mcp_browser.skyvern_execute(steps=[{"tool": tool, "params": params}])
+
+ assert executed["ok"] is True
+ refresh_before_action = "ref" in params
+ assert events == (
+ ["observe", "action", "settle", "observe"] if refresh_before_action else ["action", "settle", "observe"]
+ )
+ assert automatic.await_count == (2 if refresh_before_action else 1)
+ automatic_call = automatic.await_args_list[1] if refresh_before_action else automatic.await_args_list[0]
+ assert automatic_call.args[0] is page
+ assert automatic_call.kwargs.get("selector") == expected_scope
+
+
+async def test_missing_or_multiple_aria_targets_fall_back_to_unscoped_observe(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ monkeypatch.setenv(_FLAG, "1")
+ page = _page()
+ page.page.query_selector = AsyncMock(return_value=None)
+ ctx = BrowserContext(mode="local")
+ state = make_session_state(context=ctx)
+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))
+ monkeypatch.setattr(
+ mcp_browser,
+ "do_observe",
+ AsyncMock(return_value=_result(page.url, "#control", aria_controls="first second")),
+ )
+ automatic = AsyncMock(
+ side_effect=[
+ _result(page.url, "#control", aria_controls="first second"),
+ _result(page.url, "#after"),
+ ]
+ )
+ monkeypatch.setattr(browser_ops, "do_observe", automatic)
+ monkeypatch.setattr(mcp_browser, "skyvern_click", AsyncMock(return_value={"ok": True, "data": None}))
+
+ async with scoped_session(state):
+ observed = await mcp_browser.skyvern_observe()
+ await mcp_browser.skyvern_execute(
+ steps=[{"tool": "click", "params": {"ref": observed["data"]["elements"][0]["ref"]}}]
+ )
+
+ assert automatic.await_args_list[-1].kwargs.get("selector") is None
+ page.page.query_selector.assert_not_awaited()
+
+ assert (
+ await mcp_browser._attached_aria_target_selector(
+ page,
+ [{"aria_controls": "missing"}],
+ )
+ is None
+ )
+ page.page.query_selector.assert_awaited_once_with('[id="missing"]')
+
+
+async def test_multi_step_mutating_batch_gets_one_post_batch_observe(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setenv(_FLAG, "1")
+ page = _page()
+ ctx = BrowserContext(mode="local")
+ state = make_session_state(context=ctx)
+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))
+ monkeypatch.setattr(mcp_browser, "skyvern_click", AsyncMock(return_value={"ok": True, "data": None}))
+ automatic = AsyncMock(return_value=_result(page.url, "#after"))
+ monkeypatch.setattr(browser_ops, "do_observe", automatic)
+
+ async with scoped_session(state):
+ executed = await mcp_browser.skyvern_execute(
+ steps=[
+ {"tool": "click", "params": {"selector": "#first"}},
+ {"tool": "click", "params": {"selector": "#second"}},
+ ]
+ )
+
+ assert executed["ok"] is True
+ assert [row["tool"] for row in executed["data"]["results"]] == ["click", "click"]
+ assert executed["data"]["auto_observe"]["tool"] == "observe"
+ assert executed["data"]["auto_observe"]["ok"] is True
+ assert automatic.await_count == 1
+ page.page.wait_for_timeout.assert_awaited_once_with(mcp_browser._POST_MUTATION_SETTLE_MS)
+
+
+async def test_successful_mutation_still_gets_post_batch_observe_after_later_failure(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ monkeypatch.setenv(_FLAG, "1")
+ page = _page()
+ ctx = BrowserContext(mode="local")
+ state = make_session_state(context=ctx)
+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))
+ monkeypatch.setattr(
+ mcp_browser,
+ "skyvern_click",
+ AsyncMock(side_effect=[{"ok": True, "data": None}, RuntimeError("missing target")]),
+ )
+ automatic = AsyncMock(return_value=_result(page.url, "#after"))
+ monkeypatch.setattr(browser_ops, "do_observe", automatic)
+
+ async with scoped_session(state):
+ executed = await mcp_browser.skyvern_execute(
+ steps=[
+ {"tool": "click", "params": {"selector": "#mutate"}},
+ {"tool": "click", "params": {"selector": "#missing"}},
+ ]
+ )
+
+ assert executed["ok"] is False
+ assert executed["data"]["error_step"] == 1
+ assert [row["tool"] for row in executed["data"]["results"]] == ["click", "click"]
+ assert executed["data"]["auto_observe"]["tool"] == "observe"
+ automatic.assert_awaited_once()
+
+
+async def test_post_mutation_settle_cancellation_clears_existing_refs(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setenv(_FLAG, "1")
+ page = _page()
+ ctx = BrowserContext(mode="local")
+ state = make_session_state(context=ctx)
+ state._observed_refs = {"e0": {"ref": "e0", "selector": "#stale"}}
+ state._observe_v2_state.refs = dict(state._observed_refs)
+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))
+ monkeypatch.setattr(mcp_browser, "skyvern_click", AsyncMock(return_value={"ok": True, "data": None}))
+ monkeypatch.setattr(mcp_browser, "_settle_after_mutating_batch", AsyncMock(side_effect=asyncio.CancelledError))
+
+ async with scoped_session(state):
+ with pytest.raises(asyncio.CancelledError):
+ await mcp_browser.skyvern_execute(steps=[{"tool": "click", "params": {"selector": "#mutate"}}])
+
+ assert state._observed_refs == {}
+ assert state._observe_v2_state.refs == {}
+
+
+async def test_post_mutation_cancellation_preserves_newer_publication(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setenv(_FLAG, "1")
+ page = _page()
+ ctx = BrowserContext(mode="local")
+ state = make_session_state(context=ctx)
+ settle_started = asyncio.Event()
+
+ async def settle(*_args: Any, **_kwargs: Any) -> None:
+ settle_started.set()
+ await asyncio.Event().wait()
+
+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))
+ monkeypatch.setattr(mcp_browser, "skyvern_click", AsyncMock(return_value={"ok": True, "data": None}))
+ monkeypatch.setattr(mcp_browser, "_settle_after_mutating_batch", settle)
+ monkeypatch.setattr(mcp_browser, "do_observe", AsyncMock(return_value=_result(page.url, "#newer")))
+
+ async with scoped_session(state):
+ older_task = asyncio.create_task(
+ mcp_browser.skyvern_execute(steps=[{"tool": "click", "params": {"selector": "#mutate"}}])
+ )
+ await settle_started.wait()
+ newer = await mcp_browser.skyvern_observe()
+ older_task.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await older_task
+
+ assert newer["ok"] is True
+ assert {element["selector"] for element in state._observe_v2_state.refs.values()} == {"#newer"}
+
+
+async def test_execute_no_browser_observe_clears_existing_refs(monkeypatch: pytest.MonkeyPatch) -> None:
+ from skyvern.cli.mcp_tools._session import BrowserNotAvailableError
+
+ monkeypatch.setenv(_FLAG, "1")
+ ctx = BrowserContext(mode="local")
+ state = make_session_state(context=ctx)
+ state._observed_refs = {"e0": {"ref": "e0", "selector": "#stale"}}
+ state._observe_v2_state.refs = dict(state._observed_refs)
+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(side_effect=BrowserNotAvailableError("no browser")))
+
+ async with scoped_session(state):
+ result = await mcp_browser.skyvern_execute(steps=[{"tool": "observe", "params": {}}])
+
+ assert result["ok"] is False
+ assert state._observed_refs == {}
+ assert state._observe_v2_state.refs == {}
+
+
+async def test_execute_page_lookup_cancellation_clears_existing_refs(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setenv(_FLAG, "1")
+ ctx = BrowserContext(mode="local")
+ state = make_session_state(context=ctx)
+ state._observed_refs = {"e0": {"ref": "e0", "selector": "#stale"}}
+ state._observe_v2_state.refs = dict(state._observed_refs)
+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(side_effect=asyncio.CancelledError))
+
+ async with scoped_session(state):
+ with pytest.raises(asyncio.CancelledError):
+ await mcp_browser.skyvern_execute(steps=[{"tool": "observe", "params": {}}])
+
+ assert state._observed_refs == {}
+ assert state._observe_v2_state.refs == {}
+
+
+async def test_flag_off_cancelled_mutating_page_lookup_preserves_existing_refs(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ monkeypatch.delenv(_FLAG, raising=False)
+ ctx = BrowserContext(mode="local")
+ state = make_session_state(context=ctx)
+ state._observed_refs = {"e0": {"ref": "e0", "selector": "#existing"}}
+ state._observe_v2_state.refs = dict(state._observed_refs)
+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(side_effect=asyncio.CancelledError))
+
+ async with scoped_session(state):
+ with pytest.raises(asyncio.CancelledError):
+ await mcp_browser.skyvern_execute(steps=[{"tool": "click", "params": {"selector": "#existing"}}])
+
+ assert state._observed_refs == {"e0": {"ref": "e0", "selector": "#existing"}}
+ assert state._observe_v2_state.refs == {"e0": {"ref": "e0", "selector": "#existing"}}
+
+
+async def test_cancelled_execute_page_lookup_preserves_newer_publication(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setenv(_FLAG, "1")
+ page = _page()
+ ctx = BrowserContext(mode="local")
+ state = make_session_state(context=ctx)
+ page_lookup_started = asyncio.Event()
+
+ async def delayed_get_page(*args: Any, **kwargs: Any) -> tuple[Any, BrowserContext]:
+ page_lookup_started.set()
+ await asyncio.Event().wait()
+ raise AssertionError("unreachable")
+
+ monkeypatch.setattr(mcp_browser, "get_page", delayed_get_page)
+
+ async with scoped_session(state):
+ older_task = asyncio.create_task(mcp_browser.skyvern_execute(steps=[{"tool": "observe", "params": {}}]))
+ await page_lookup_started.wait()
+ newer_generation = session_manager.begin_session_ref_publication()
+ assert session_manager.replace_session_ref_map(
+ {"e1": {"ref": "e1", "selector": "#newer"}},
+ generation=newer_generation,
+ page_key=session_manager.page_ref_key(page),
+ )
+ older_task.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await older_task
+
+ assert state._observed_refs["refs"] == {"e1": {"ref": "e1", "selector": "#newer"}}
+
+
+async def test_execute_document_probe_cancellation_clears_existing_refs(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setenv(_FLAG, "1")
+ page = _page()
+ ctx = BrowserContext(mode="local")
+ state = make_session_state(context=ctx)
+ state._observed_refs = {"e0": {"ref": "e0", "selector": "#stale"}}
+ state._observe_v2_state.refs = dict(state._observed_refs)
+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))
+ monkeypatch.setattr(mcp_browser, "get_observe_document_id", AsyncMock(side_effect=asyncio.CancelledError))
+
+ async with scoped_session(state):
+ with pytest.raises(asyncio.CancelledError):
+ await mcp_browser.skyvern_execute(steps=[{"tool": "observe", "params": {}}])
+
+ assert state._observed_refs == {}
+ assert state._observe_v2_state.refs == {}
+
+
+async def test_execute_publication_cancellation_clears_existing_refs(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setenv(_FLAG, "1")
+ page = _page()
+ ctx = BrowserContext(mode="local")
+ state = make_session_state(context=ctx)
+ state._observed_refs = {"e0": {"ref": "e0", "selector": "#stale"}}
+ state._observe_v2_state.refs = dict(state._observed_refs)
+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))
+ monkeypatch.setattr(browser_ops, "do_observe", AsyncMock(return_value=_result(page.url, "#staged")))
+ publish = AsyncMock(side_effect=asyncio.CancelledError)
+ monkeypatch.setattr(mcp_browser, "_publish_observe_v2_refs", publish)
+
+ async with scoped_session(state):
+ with pytest.raises(asyncio.CancelledError):
+ await mcp_browser.skyvern_execute(steps=[{"tool": "observe", "params": {}}])
+
+ publish.assert_awaited_once()
+ assert state._observed_refs == {}
+ assert state._observe_v2_state.refs == {}
+
+
+async def test_slower_standalone_observe_cannot_overwrite_newer_execute_publication(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ monkeypatch.setenv(_FLAG, "1")
+ page = _page()
+ ctx = BrowserContext(mode="local")
+ state = make_session_state(context=ctx)
+ older_started = asyncio.Event()
+ release_older = asyncio.Event()
+
+ async def older_observe(*_args: Any, **_kwargs: Any) -> ObserveResult:
+ older_started.set()
+ await release_older.wait()
+ return _result(page.url, "#older")
+
+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))
+ monkeypatch.setattr(mcp_browser, "do_observe", older_observe)
+ monkeypatch.setattr(browser_ops, "do_observe", AsyncMock(return_value=_result(page.url, "#newer")))
+
+ async with scoped_session(state):
+ older_task = asyncio.create_task(mcp_browser.skyvern_observe())
+ await older_started.wait()
+ newer = await mcp_browser.skyvern_execute(steps=[{"tool": "observe", "params": {}}])
+ release_older.set()
+ older = await older_task
+
+ assert newer["ok"] is True
+ assert older["ok"] is False
+ assert "data" not in older or older["data"] is None
+ assert {element["selector"] for element in state._observe_v2_state.refs.values()} == {"#newer"}
+
+
+async def test_slower_execute_cannot_clear_newer_standalone_publication(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ monkeypatch.setenv(_FLAG, "1")
+ page = _page()
+ ctx = BrowserContext(mode="local")
+ state = make_session_state(context=ctx)
+ execute_waiting = asyncio.Event()
+ release_execute = asyncio.Event()
+
+ async def wait_step(**_kwargs: Any) -> dict[str, Any]:
+ execute_waiting.set()
+ await release_execute.wait()
+ return {"ok": True, "data": None}
+
+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))
+ monkeypatch.setattr(browser_ops, "do_observe", AsyncMock(return_value=_result(page.url, "#older")))
+ monkeypatch.setattr(mcp_browser, "do_observe", AsyncMock(return_value=_result(page.url, "#newer")))
+ monkeypatch.setattr(mcp_browser, "skyvern_wait", wait_step)
+
+ async with scoped_session(state):
+ older_task = asyncio.create_task(
+ mcp_browser.skyvern_execute(
+ steps=[
+ {"tool": "observe", "params": {}},
+ {"tool": "wait", "params": {"duration_ms": 1}},
+ ]
+ )
+ )
+ await execute_waiting.wait()
+ newer = await mcp_browser.skyvern_observe()
+ release_execute.set()
+ older = await older_task
+
+ assert newer["ok"] is True
+ assert older["ok"] is False
+ assert "observe snapshot was superseded" in older["error"]["message"].lower()
+ assert {element["selector"] for element in state._observe_v2_state.refs.values()} == {"#newer"}
+
+
+async def test_stale_ref_refresh_cannot_overwrite_newer_publication(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setenv(_FLAG, "1")
+ page = _page()
+ ctx = BrowserContext(mode="local")
+ state = make_session_state(context=ctx)
+ state._observe_v2_state.page_key = session_manager.page_ref_key(page)
+ state._observe_v2_state.document_id = "doc-1"
+ state._observe_v2_state.params = {
+ "selector": None,
+ "interactive_only": True,
+ "max_elements": 50,
+ "include_values": False,
+ }
+ state._observe_v2_state.refs = {"e0": {"ref": "e0", "selector": "#old"}}
+ state._observed_refs = dict(state._observe_v2_state.refs)
+ state._observed_refs_generation = session_manager.begin_session_ref_publication()
+ refresh_started = asyncio.Event()
+ release_refresh = asyncio.Event()
+
+ async def refresh(*args: Any, **kwargs: Any) -> ObserveResult:
+ refresh_started.set()
+ await release_refresh.wait()
+ return _result(page.url, "#refreshed", document_id="doc-1")
+
+ monkeypatch.setattr(mcp_browser, "get_observe_document_id", AsyncMock(return_value="doc-1"))
+ monkeypatch.setattr(browser_ops, "do_observe", refresh)
+
+ async with scoped_session(state):
+ older_task = asyncio.create_task(
+ mcp_browser._refresh_observe_v2_ref(
+ "e0",
+ page,
+ session_id=None,
+ cdp_url=None,
+ )
+ )
+ await refresh_started.wait()
+ newer_generation = session_manager.begin_session_ref_publication()
+ assert session_manager.replace_session_ref_map(
+ {"e1": {"ref": "e1", "selector": "#newer"}},
+ generation=newer_generation,
+ page_key=session_manager.page_ref_key(page),
+ )
+ state._observe_v2_state.refs = {"e1": {"ref": "e1", "selector": "#newer"}}
+ release_refresh.set()
+ handled, refreshed = await older_task
+
+ assert handled is True
+ assert refreshed is None
+ assert state._observe_v2_state.refs == {"e1": {"ref": "e1", "selector": "#newer"}}
+
+
+async def test_cancelled_observe_page_lookup_preserves_newer_publication(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setenv(_FLAG, "1")
+ page = _page()
+ ctx = BrowserContext(mode="local")
+ state = make_session_state(context=ctx)
+ page_lookup_started = asyncio.Event()
+
+ async def delayed_get_page(*args: Any, **kwargs: Any) -> tuple[Any, BrowserContext]:
+ page_lookup_started.set()
+ await asyncio.Event().wait()
+ raise AssertionError("unreachable")
+
+ monkeypatch.setattr(mcp_browser, "get_page", delayed_get_page)
+
+ async with scoped_session(state):
+ older_task = asyncio.create_task(mcp_browser.skyvern_observe())
+ await page_lookup_started.wait()
+ newer_generation = session_manager.begin_session_ref_publication()
+ assert session_manager.replace_session_ref_map(
+ {"e1": {"ref": "e1", "selector": "#newer"}},
+ generation=newer_generation,
+ page_key=session_manager.page_ref_key(page),
+ )
+ older_task.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await older_task
+
+ assert state._observed_refs["refs"] == {"e1": {"ref": "e1", "selector": "#newer"}}
+
+
+async def test_failed_later_observe_discards_staged_ref_publication(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ monkeypatch.setenv(_FLAG, "1")
+ page = _page()
+ ctx = BrowserContext(mode="local")
+ state = make_session_state(context=ctx)
+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))
+ monkeypatch.setattr(
+ browser_ops,
+ "do_observe",
+ AsyncMock(side_effect=[_result(page.url, "#staged"), RuntimeError("snapshot failed")]),
+ )
+
+ async with scoped_session(state):
+ executed = await mcp_browser.skyvern_execute(
+ steps=[
+ {"tool": "observe", "params": {}},
+ {"tool": "observe", "params": {}},
+ ],
+ stop_on_error=False,
+ )
+
+ assert executed["ok"] is False
+ assert state._observed_refs == {}
+ assert state._observe_v2_state.refs == {}
+
+
+async def test_revalidation_failure_discards_earlier_staged_ref_publication(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ monkeypatch.setenv(_FLAG, "1")
+ page = _page()
+ ctx = BrowserContext(mode="local")
+ state = make_session_state(context=ctx)
+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))
+ monkeypatch.setattr(
+ browser_ops,
+ "do_observe",
+ AsyncMock(side_effect=[_result(page.url, "#staged"), _result(page.url, "#rejected")]),
+ )
+ revalidate = AsyncMock(side_effect=[True, RuntimeError("revalidation failed"), True])
+ monkeypatch.setattr(mcp_browser, "_observe_v2_snapshot_is_current", revalidate)
+
+ async with scoped_session(state):
+ executed = await mcp_browser.skyvern_execute(
+ steps=[
+ {"tool": "observe", "params": {}},
+ {"tool": "observe", "params": {}},
+ ],
+ stop_on_error=False,
+ )
+
+ assert executed["ok"] is False
+ assert state._observed_refs == {}
+ assert state._observe_v2_state.refs == {}
+
+
+async def test_settle_timeout_still_runs_bounded_post_mutation_observe(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ monkeypatch.setenv(_FLAG, "1")
+ page = _page()
+ page.page.wait_for_timeout = AsyncMock(side_effect=TimeoutError)
+ ctx = BrowserContext(mode="local")
+ state = make_session_state(context=ctx)
+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))
+ monkeypatch.setattr(mcp_browser, "skyvern_click", AsyncMock(return_value={"ok": True, "data": None}))
+ automatic = AsyncMock(return_value=_result(page.url, "#after"))
+ monkeypatch.setattr(browser_ops, "do_observe", automatic)
+
+ async with scoped_session(state):
+ executed = await mcp_browser.skyvern_execute(steps=[{"tool": "click", "params": {"selector": "#mutate"}}])
+
+ assert executed["ok"] is True
+ assert [row["tool"] for row in executed["data"]["results"]] == ["click"]
+ assert executed["data"]["auto_observe"]["tool"] == "observe"
+ automatic.assert_awaited_once()
+
+
+async def test_post_mutation_same_label_replacement_invalidates_old_ref_generation(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ monkeypatch.setenv(_FLAG, "1")
+ page = _page()
+ ctx = BrowserContext(mode="local")
+ state = make_session_state(context=ctx)
+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))
+ monkeypatch.setattr(mcp_browser, "do_observe", AsyncMock(return_value=_result(page.url, "#before")))
+ monkeypatch.setattr(browser_ops, "do_observe", AsyncMock(return_value=_result(page.url, "#after")))
+ monkeypatch.setattr(mcp_browser, "skyvern_click", AsyncMock(return_value={"ok": True, "data": None}))
+
+ async with scoped_session(state):
+ observed = await mcp_browser.skyvern_observe()
+ stale_ref = observed["data"]["elements"][0]["ref"]
+ generation_before = session_manager.session_ref_generation()
+ executed = await mcp_browser.skyvern_execute(steps=[{"tool": "click", "params": {"ref": stale_ref}}])
+ generation_after = session_manager.session_ref_generation()
+ fresh_ref = executed["data"]["auto_observe"]["data"]["elements"][0]["ref"]
+ fresh_selector = state._observe_v2_state.refs[fresh_ref]["selector"]
+ stale = await mcp_browser.skyvern_execute(steps=[{"tool": "click", "params": {"ref": stale_ref}}])
+
+ assert generation_after > generation_before
+ assert fresh_ref != stale_ref
+ assert fresh_selector == "#after"
+ assert stale["ok"] is False
+ assert "Unknown ref" in stale["data"]["results"][0]["error"]
+
+
+async def test_oversized_observe_response_does_not_publish_omitted_refs(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ monkeypatch.setenv(_FLAG, "1")
+ page = _page()
+ ctx = BrowserContext(mode="local")
+ state = make_session_state(context=ctx)
+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))
+ monkeypatch.setattr(
+ mcp_browser,
+ "do_observe",
+ AsyncMock(return_value=_result(page.url, "#omitted", page_text="x" * 150_000)),
+ )
+
+ async with scoped_session(state):
+ observed = await mcp_browser.skyvern_observe()
+
+ assert observed["_truncated"] is True
+ assert observed["_max_bytes"] == MCP_MAX_RESPONSE_BYTES
+ assert len(json.dumps(observed, ensure_ascii=False).encode()) <= MCP_MAX_RESPONSE_BYTES
+ assert "data" not in observed
+ assert state._observed_refs == {}
+ assert state._observe_v2_state.refs == {}
+
+
[email protected]("flag_value", [None, "0", "false"])
+async def test_flag_off_oversized_multibyte_observe_response_remains_uncapped(
+ monkeypatch: pytest.MonkeyPatch,
+ flag_value: str | None,
+) -> None:
+ if flag_value is None:
+ monkeypatch.delenv(_FLAG, raising=False)
+ else:
+ monkeypatch.setenv(_FLAG, flag_value)
+
+ page = _page()
+ ctx = BrowserContext(mode="local")
+ state = make_session_state(context=ctx)
+ large_name = "界" * (MCP_MAX_RESPONSE_BYTES // 2)
+ result = _result(page.url, "#legacy")
+ result.elements[0].name = large_name
+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))
+ monkeypatch.setattr(mcp_browser, "do_observe", AsyncMock(return_value=result))
+
+ async with scoped_session(state):
+ observed = await mcp_browser.skyvern_observe()
+
+ serialized = json.dumps(observed, ensure_ascii=False)
+ assert len(serialized) < MCP_MAX_RESPONSE_BYTES
+ assert len(serialized.encode()) > MCP_MAX_RESPONSE_BYTES
+ assert "_truncated" not in observed
+ assert observed["data"]["elements"][0]["name"] == large_name
+ assert state._observed_refs["refs"]["e0"]["selector"] == "#legacy"
+
+
+async def test_flag_off_oversized_execute_response_remains_uncapped(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ monkeypatch.delenv(_FLAG, raising=False)
+ page = _page()
+ ctx = BrowserContext(mode="local")
+ state = make_session_state(context=ctx)
+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))
+ monkeypatch.setattr(
+ mcp_browser,
+ "skyvern_evaluate",
+ AsyncMock(return_value={"ok": True, "data": {"value": "x" * 150_000}}),
+ )
+
+ async with scoped_session(state):
+ executed = await mcp_browser.skyvern_execute(
+ steps=[{"tool": "evaluate", "params": {"expression": "document.title"}}]
+ )
+
+ assert "_truncated" not in executed
+ assert executed["data"]["results"][0]["data"]["value"] == "x" * 150_000
+
+
+async def test_oversized_execute_response_does_not_publish_omitted_auto_observe_refs(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ monkeypatch.setenv(_FLAG, "1")
+ page = _page()
+ ctx = BrowserContext(mode="local")
+ state = make_session_state(context=ctx)
+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))
+ monkeypatch.setattr(mcp_browser, "skyvern_click", AsyncMock(return_value={"ok": True, "data": None}))
+ monkeypatch.setattr(
+ browser_ops,
+ "do_observe",
+ AsyncMock(return_value=_result(page.url, "#fresh", page_text="x" * 150_000)),
+ )
+
+ async with scoped_session(state):
+ executed = await mcp_browser.skyvern_execute(steps=[{"tool": "click", "params": {"selector": "#mutate"}}])
+
+ assert executed["_truncated"] is True
+ assert executed["_max_bytes"] == MCP_MAX_RESPONSE_BYTES
+ assert len(json.dumps(executed, ensure_ascii=False).encode()) <= MCP_MAX_RESPONSE_BYTES
+ assert "data" not in executed
+ assert state._observed_refs == {}
+ assert state._observe_v2_state.refs == {}
+
+
+async def test_failed_mutation_strips_invalidated_inline_observe_refs(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setenv(_FLAG, "1")
+ page = _page()
+ ctx = BrowserContext(mode="local")
+ state = make_session_state(context=ctx)
+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))
+ monkeypatch.setattr(browser_ops, "do_observe", AsyncMock(return_value=_result(page.url, "#inline")))
+ monkeypatch.setattr(
+ mcp_browser,
+ "skyvern_click",
+ AsyncMock(return_value={"ok": False, "error": {"code": "ACTION_FAILED", "message": "click failed"}}),
+ )
+
+ async with scoped_session(state):
+ executed = await mcp_browser.skyvern_execute(
+ steps=[
+ {"tool": "observe", "params": {}},
+ {"tool": "click", "params": {"ref": "e0"}},
+ ]
+ )
+
+ assert executed["ok"] is False
+ assert executed["data"]["error_step"] == 1
+ assert "data" not in executed["data"]["results"][0]
+ assert state._observed_refs == {}
+ assert state._observe_v2_state.refs == {}
+
+
+async def test_failed_mutation_invalidates_inflight_publication(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setenv(_FLAG, "1")
+ page = _page()
+ ctx = BrowserContext(mode="local")
+ state = make_session_state(context=ctx)
+
+ async def click(**kwargs: Any) -> dict[str, Any]:
+ generation = session_manager.begin_session_ref_publication()
+ ref_map = {"e1": {"ref": "e1", "selector": "#during-action"}}
+ assert session_manager.replace_session_ref_map(
+ ref_map,
+ generation=generation,
+ page_key=session_manager.page_ref_key(page),
+ )
+ state._observe_v2_state.refs = dict(ref_map)
+ state._observe_v2_state.page_key = session_manager.page_ref_key(page)
+ state._observe_v2_state.document_id = "page:doc-1"
+ return {"ok": False, "error": {"code": "ACTION_FAILED", "message": "click failed"}}
+
+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))
+ monkeypatch.setattr(mcp_browser, "skyvern_click", AsyncMock(side_effect=click))
+
+ async with scoped_session(state):
+ executed = await mcp_browser.skyvern_execute(steps=[{"tool": "click", "params": {"selector": "#mutate"}}])
+ assert (
+ session_manager.get_session_ref(
+ "e1",
+ page_key=session_manager.page_ref_key(page),
+ )
+ is None
+ )
+
+ assert executed["ok"] is False
+ assert state._observe_v2_state.refs == {}
+
+
+async def test_multiple_inline_observes_return_only_published_snapshot(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setenv(_FLAG, "1")
+ page = _page()
+ ctx = BrowserContext(mode="local")
+ state = make_session_state(context=ctx)
+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))
+ monkeypatch.setattr(
+ browser_ops,
+ "do_observe",
+ AsyncMock(side_effect=[_result(page.url, "#first"), _result(page.url, "#second")]),
+ )
+
+ async with scoped_session(state):
+ executed = await mcp_browser.skyvern_execute(
+ steps=[
+ {"tool": "observe", "params": {}},
+ {"tool": "observe", "params": {}},
+ ]
+ )
+
+ assert executed["ok"] is True
+ assert "data" not in executed["data"]["results"][0]
+ assert executed["data"]["results"][1]["data"]["elements"][0]["selector"] == "#second"
+ assert {element["selector"] for element in state._observe_v2_state.refs.values()} == {"#second"}
+
+
+async def test_stale_document_cleanup_preserves_newer_publication(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setenv(_FLAG, "1")
+ page = _page()
+ ctx = BrowserContext(mode="local")
+ state = make_session_state(context=ctx)
+ automatic = AsyncMock(side_effect=AssertionError("stale execute must not replace a newer publication"))
+
+ async def wait(**kwargs: Any) -> dict[str, Any]:
+ page.evaluate = AsyncMock(return_value="doc-2")
+ generation = session_manager.begin_session_ref_publication()
+ ref_map = {"e1": {"ref": "e1", "selector": "#newer"}}
+ assert session_manager.replace_session_ref_map(
+ ref_map,
+ generation=generation,
+ page_key=session_manager.page_ref_key(page),
+ )
+ state._observe_v2_state.refs = dict(ref_map)
+ state._observe_v2_state.page_key = session_manager.page_ref_key(page)
+ state._observe_v2_state.document_id = "page:doc-2"
+ return {"ok": True, "data": None}
+
+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))
+ monkeypatch.setattr(mcp_browser, "skyvern_wait", AsyncMock(side_effect=wait))
+ monkeypatch.setattr(browser_ops, "do_observe", automatic)
+
+ async with scoped_session(state):
+ executed = await mcp_browser.skyvern_execute(steps=[{"tool": "wait", "params": {"seconds": 0}}])
+ assert session_manager.get_session_ref(
+ "e1",
+ page_key=session_manager.page_ref_key(page),
+ ) == {"ref": "e1", "selector": "#newer"}
+
+ assert executed["ok"] is True
+ automatic.assert_not_awaited()
+ assert state._observe_v2_state.refs == {"e1": {"ref": "e1", "selector": "#newer"}}
+
+
+async def test_stale_document_cleanup_replaces_mismatched_newer_publication(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ monkeypatch.setenv(_FLAG, "1")
+ page = _page()
+ ctx = BrowserContext(mode="local")
+ state = make_session_state(context=ctx)
+ automatic = AsyncMock(return_value=_result(page.url, "#fresh", document_id="cdp:doc-2"))
+
+ async def wait(**kwargs: Any) -> dict[str, Any]:
+ generation = session_manager.begin_session_ref_publication()
+ stale_map = {"e1": {"ref": "e1", "selector": "#stale"}}
+ assert session_manager.replace_session_ref_map(
+ stale_map,
+ generation=generation,
+ page_key=session_manager.page_ref_key(page),
+ )
+ state._observe_v2_state.refs = dict(stale_map)
+ state._observe_v2_state.page_key = session_manager.page_ref_key(page)
+ state._observe_v2_state.document_id = "cdp:doc-1"
+ page._loader_id = "doc-2"
+ return {"ok": True, "data": None}
+
+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))
+ monkeypatch.setattr(mcp_browser, "skyvern_wait", AsyncMock(side_effect=wait))
+ monkeypatch.setattr(browser_ops, "do_observe", automatic)
+
+ async with scoped_session(state):
+ executed = await mcp_browser.skyvern_execute(steps=[{"tool": "wait", "params": {"seconds": 0}}])
+
+ assert executed["ok"] is True
+ automatic.assert_awaited_once()
+ assert executed["data"]["auto_observe"]["data"]["elements"][0]["selector"] == "#fresh"
+ assert {element["selector"] for element in state._observe_v2_state.refs.values()} == {"#fresh"}
+
+
+async def test_failed_navigation_invalidates_inflight_publication(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setenv(_FLAG, "1")
+ page = _page()
+ ctx = BrowserContext(mode="local")
+ state = make_session_state(context=ctx)
+
+ async def navigate(*args: Any, **kwargs: Any) -> Any:
+ generation = session_manager.begin_session_ref_publication()
+ ref_map = {"e1": {"ref": "e1", "selector": "#newer"}}
+ assert session_manager.replace_session_ref_map(
+ ref_map,
+ generation=generation,
+ page_key=session_manager.page_ref_key(page),
+ )
+ state._observe_v2_state.refs = dict(ref_map)
+ state._observe_v2_state.page_key = session_manager.page_ref_key(page)
+ state._observe_v2_state.document_id = "page:doc-1"
+ raise RuntimeError("net::ERR_ABORTED")
+
+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))
+ monkeypatch.setattr(mcp_browser, "do_navigate", navigate)
+
+ async with scoped_session(state):
+ navigated = await mcp_browser.skyvern_navigate("https://example.com/next")
+ assert (
+ session_manager.get_session_ref(
+ "e1",
+ page_key=session_manager.page_ref_key(page),
+ )
+ is None
+ )
+
+ assert navigated["ok"] is False
+ assert state._observe_v2_state.refs == {}
+
+
+async def test_page_sourced_document_marker_refuses_durable_refs(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """A page-evaluated marker (non-OOPIF frame after skyvern_frame_switch, non-Chromium) can be
+ pinned by a hostile document, so it must never certify sameness: refs minted on it are not
+ durable and their later use fails closed instead of re-anchoring into an undescribed document."""
+ monkeypatch.setenv(_FLAG, "1")
+ page = _page(loader_id=None)
+ ctx = BrowserContext(mode="local")
+ state = make_session_state(context=ctx)
+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))
+ monkeypatch.setattr(
+ mcp_browser,
+ "do_observe",
+ AsyncMock(return_value=_result(page.url, "#before", document_id="page:doc-1")),
+ )
+ refresh = AsyncMock(return_value=_result(page.url, "#after", document_id="page:doc-1"))
+ monkeypatch.setattr(browser_ops, "do_observe", refresh)
+ click = AsyncMock(return_value={"ok": True, "data": None})
+ monkeypatch.setattr(mcp_browser, "skyvern_click", click)
+
+ async with scoped_session(state):
+ observed = await mcp_browser.skyvern_observe()
+ ref = observed["data"]["elements"][0]["ref"]
+ live = session_manager.get_observe_v2_state()
+ assert live.document_id is None
+ executed = await mcp_browser.skyvern_execute(steps=[{"tool": "click", "params": {"ref": ref}}])
+
+ assert executed["ok"] is False
+ assert "Unknown ref" in executed["data"]["results"][0]["error"]
+ refresh.assert_not_awaited()
+ click.assert_not_awaited()
+
+
+async def test_model_invented_ref_does_not_revoke_live_refs(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """A ref neither store knows is model-invented: it must fail alone, without revoking the
+ refs that were just published. Only a legacy-held ref (flag flip) revokes both stores."""
+ monkeypatch.setenv(_FLAG, "1")
+ page = _page()
+ ctx = BrowserContext(mode="local")
+ state = make_session_state(context=ctx)
+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))
+ monkeypatch.setattr(mcp_browser, "do_observe", AsyncMock(return_value=_result(page.url, "#before")))
+ refresh = AsyncMock(return_value=_result(page.url, "#after"))
+ monkeypatch.setattr(browser_ops, "do_observe", refresh)
+ click = AsyncMock(return_value={"ok": True, "data": None})
+ monkeypatch.setattr(mcp_browser, "skyvern_click", click)
+
+ async with scoped_session(state):
+ observed = await mcp_browser.skyvern_observe()
+ ref = observed["data"]["elements"][0]["ref"]
+ invented = await mcp_browser.skyvern_execute(steps=[{"tool": "click", "params": {"ref": "e99"}}])
+ live = session_manager.get_observe_v2_state()
+ assert ref in live.refs
+ assert session_manager.get_session_ref(ref, page_key=session_manager.page_ref_key(page)) is not None
+ retried = await mcp_browser.skyvern_execute(steps=[{"tool": "click", "params": {"ref": ref}}])
+
+ assert invented["ok"] is False
+ assert "Unknown ref" in invented["data"]["results"][0]["error"]
+ assert retried["ok"] is True
+ assert click.await_args.kwargs["selector"] == "#after"
+
+
+async def test_in_batch_ref_fails_closed_after_same_document_swap(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """An in-batch ref hit runs the same document check as the cross-call path: a step that
+ replaces the document at the same URL invalidates refs minted earlier in the batch."""
+ monkeypatch.setenv(_FLAG, "1")
+ page = _page()
+ ctx = BrowserContext(mode="local")
+ state = make_session_state(context=ctx)
+ observe = AsyncMock(return_value=_result(page.url, "#continue"))
+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))
+ monkeypatch.setattr(browser_ops, "do_observe", observe)
+
+ async def click(**kwargs: Any) -> dict[str, Any]:
+ if kwargs.get("selector") == "#other":
+ page._loader_id = "doc-2"
+ return {"ok": True, "data": None}
+
+ click_mock = AsyncMock(side_effect=click)
+ monkeypatch.setattr(mcp_browser, "skyvern_click", click_mock)
+
+ async with scoped_session(state):
+ executed = await mcp_browser.skyvern_execute(
+ steps=[
+ {"tool": "observe", "params": {}},
+ {"tool": "click", "params": {"selector": "#other"}},
+ {"tool": "click", "params": {"ref": "e0"}},
+ ]
+ )
+
+ assert executed["ok"] is False
+ assert "Unknown ref" in executed["data"]["results"][2]["error"]
+ assert observe.await_count == 2
+ assert click_mock.await_count == 1
+
+
+async def test_in_batch_ref_reuse_passes_document_check_without_reobserve(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """When the document is unchanged, the in-batch fast path stays a fast path: the marker
+ read alone validates the ref, with no per-step re-observe."""
+ monkeypatch.setenv(_FLAG, "1")
+ page = _page()
+ ctx = BrowserContext(mode="local")
+ state = make_session_state(context=ctx)
+ observe = AsyncMock(return_value=_result(page.url, "#continue"))
+ click = AsyncMock(return_value={"ok": True, "data": None})
+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))
+ monkeypatch.setattr(browser_ops, "do_observe", observe)
+ monkeypatch.setattr(mcp_browser, "skyvern_click", click)
+
+ async with scoped_session(state):
+ executed = await mcp_browser.skyvern_execute(
+ steps=[
+ {"tool": "observe", "params": {}},
+ {"tool": "click", "params": {"ref": "e0"}},
+ ]
+ )
+
+ assert executed["ok"] is True
+ assert observe.await_count == 2
+ assert executed["data"]["auto_observe"]["ok"] is True
+ assert click.await_args.kwargs["selector"] == "#continue"
+
+
+def test_page_text_js_gates_on_rendered_root() -> None:
+ """Pin the rendered-root gate: innerText of a non-rendered root degrades to raw descendant
+ text (inline script/style bodies), so the extraction JS must check visibility first."""
+ assert "checkVisibility" in browser_ops._OBSERVE_PAGE_TEXT_JS
+ assert "getClientRects" in browser_ops._OBSERVE_PAGE_TEXT_JS
+
+
+async def test_mid_observe_document_swap_refuses_durability(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """The document marker must bracket element extraction: a same-URL replacement that lands
+ mid-observe would otherwise tag pre-swap selectors with the replacement document's id,
+ blessing stale elements for the durable-ref and in-batch paths."""
+ monkeypatch.setenv(_FLAG, "1")
+ page = _page()
+
+ async def evaluate(expression: str, arg: Any = None) -> Any:
+ if expression == browser_ops._DOMUTILS_INTERACTABILITY_READY_JS:
+ return True
+ if expression == browser_ops._OBSERVE_INTERACTABLES_JS:
+ # The swap lands during extraction, after the opening marker read.
+ page._loader_id = "doc-2"
+ return []
+ if expression == browser_ops._OBSERVE_PAGE_TEXT_JS:
+ return {"content": "", "truncated": False}
+ raise AssertionError(expression)
+
+ page.evaluate = AsyncMock(side_effect=evaluate)
+ swapped = await browser_ops.do_observe(page)
+ assert swapped.document_id is None
+
+ stable = _page()
+
+ async def stable_evaluate(expression: str, arg: Any = None) -> Any:
+ if expression == browser_ops._DOMUTILS_INTERACTABILITY_READY_JS:
+ return True
+ if expression == browser_ops._OBSERVE_INTERACTABLES_JS:
+ return []
+ if expression == browser_ops._OBSERVE_PAGE_TEXT_JS:
+ return {"content": "", "truncated": False}
+ raise AssertionError(expression)
+
+ stable.evaluate = AsyncMock(side_effect=stable_evaluate)
+ kept = await browser_ops.do_observe(stable)
+ assert kept.document_id == "cdp:doc-1"
+
+
+async def test_page_sourced_marker_rejects_in_batch_refs_too(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """The rollout-gate limitation, locked: with only a page-sourced marker available, refs
+ minted by an observe are unusable even inside the same skyvern_execute batch — fail closed,
+ never a spoofable sameness check. Selector/intent params remain the working path there."""
monkeypatch.setenv(_FLAG, "1")
page = _page(loader_id=None)
ctx = BrowserContext(mode="local")
@@ -1387,3 +2752,368 @@ async def test_page_sourced_marker_rejects_in_batch_refs_too(
assert "Unknown ref" in executed["data"]["results"][1]["error"]
assert observe.await_count == 1
click.assert_not_awaited()
+
+
+async def test_flag_off_observe_refuses_publication_after_concurrent_invalidation(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """The generation guard predates observe-v2: with the flag OFF, an in-flight observe
+ racing a concurrent invalidation must not republish its stale snapshot."""
+ monkeypatch.delenv(_FLAG, raising=False)
+ page = _page()
+ ctx = BrowserContext(mode="local")
+ state = make_session_state(context=ctx)
+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))
+
+ async def racing_observe(*_args: Any, **_kwargs: Any) -> ObserveResult:
+ # A concurrent navigation invalidates the registry while the snapshot is in flight.
+ session_manager.invalidate_session_ref_map()
+ return _result(page.url, "#stale")
+
+ monkeypatch.setattr(mcp_browser, "do_observe", AsyncMock(side_effect=racing_observe))
+
+ async with scoped_session(state):
+ observed = await mcp_browser.skyvern_observe()
+
+ # Refusal is silent (the pre-v2 contract): the response is fine, but the
+ # superseded snapshot never reaches the registry.
+ assert observed["ok"] is True
+ assert state._observed_refs == {}
+
+
+async def test_flag_off_batch_observe_refuses_publication_after_concurrent_invalidation(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """Same guard for the inline-observe batch path: flag OFF, the generation read at
+ observe dispatch must refuse publication after a concurrent invalidation."""
+ monkeypatch.delenv(_FLAG, raising=False)
+ page = _page()
+ ctx = BrowserContext(mode="local")
+ state = make_session_state(context=ctx)
+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))
+
+ async def racing_observe(*_args: Any, **_kwargs: Any) -> ObserveResult:
+ session_manager.invalidate_session_ref_map()
+ return _result(page.url, "#stale")
+
+ monkeypatch.setattr(browser_ops, "do_observe", AsyncMock(side_effect=racing_observe))
+
+ async with scoped_session(state):
+ executed = await mcp_browser.skyvern_execute(steps=[{"tool": "observe", "params": {}}])
+
+ assert executed["ok"] is True
+ assert state._observed_refs == {}
+
+
+async def test_flag_off_failed_observe_preserves_existing_refs(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """Pre-v2 contract: failed observes (no browser, generic failure) leave the
+ published registry alone - only a frame error clears it."""
+ monkeypatch.delenv(_FLAG, raising=False)
+ page = _page()
+ ctx = BrowserContext(mode="local")
+ state = make_session_state(context=ctx)
+ existing = {"page_key": None, "refs": {"e0": {"ref": "e0", "selector": "#existing"}}}
+ state._observed_refs = dict(existing)
+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))
+ monkeypatch.setattr(mcp_browser, "do_observe", AsyncMock(side_effect=RuntimeError("snapshot failed")))
+
+ async with scoped_session(state):
+ observed = await mcp_browser.skyvern_observe()
+
+ assert observed["ok"] is False
+ assert state._observed_refs == existing
+
+
+async def test_flag_off_failed_overlapping_observe_does_not_erase_newer_publication(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """A failing older observe must not erase refs a newer observe published while
+ it was in flight (main preserved them; legacy publications never advance)."""
+ monkeypatch.delenv(_FLAG, raising=False)
+ page = _page()
+ ctx = BrowserContext(mode="local")
+ state = make_session_state(context=ctx)
+ slow_started = asyncio.Event()
+ finish_slow = asyncio.Event()
+ call_count = 0
+
+ async def observe(_page: Any, **_kwargs: Any) -> ObserveResult:
+ nonlocal call_count
+ call_count += 1
+ if call_count == 1:
+ slow_started.set()
+ await finish_slow.wait()
+ raise RuntimeError("snapshot failed")
+ return _result(page.url, "#fresh")
+
+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))
+ monkeypatch.setattr(mcp_browser, "do_observe", AsyncMock(side_effect=observe))
+
+ async with scoped_session(state):
+ slow_task = asyncio.create_task(mcp_browser.skyvern_observe())
+ await slow_started.wait()
+ fast_result = await mcp_browser.skyvern_observe()
+ finish_slow.set()
+ slow_result = await slow_task
+ published = session_manager.get_session_ref("e0", page_key=session_manager.page_ref_key(page))
+
+ assert fast_result["ok"] is True
+ assert slow_result["ok"] is False
+ assert published is not None
+ assert published["selector"] == "#fresh"
+
+
+async def test_flag_off_batch_failed_observe_preserves_newer_publication(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """Inline-batch variant: a failing batch observe must not erase refs published
+ by an overlapping standalone observe."""
+ monkeypatch.delenv(_FLAG, raising=False)
+ page = _page()
+ ctx = BrowserContext(mode="local")
+ state = make_session_state(context=ctx)
+ batch_started = asyncio.Event()
+ finish_batch = asyncio.Event()
+
+ async def batch_observe(_page: Any, **_kwargs: Any) -> ObserveResult:
+ batch_started.set()
+ await finish_batch.wait()
+ raise RuntimeError("snapshot failed")
+
+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))
+ monkeypatch.setattr(browser_ops, "do_observe", AsyncMock(side_effect=batch_observe))
+ monkeypatch.setattr(mcp_browser, "do_observe", AsyncMock(return_value=_result(page.url, "#standalone")))
+
+ async with scoped_session(state):
+ batch_task = asyncio.create_task(mcp_browser.skyvern_execute(steps=[{"tool": "observe", "params": {}}]))
+ await batch_started.wait()
+ standalone = await mcp_browser.skyvern_observe()
+ finish_batch.set()
+ executed = await batch_task
+ published = session_manager.get_session_ref("e0", page_key=session_manager.page_ref_key(page))
+
+ assert standalone["ok"] is True
+ assert executed["ok"] is False
+ assert published is not None
+ assert published["selector"] == "#standalone"
+
+
+async def test_flag_off_inline_observe_publishes_without_second_page_lookup(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """Pre-v2 contract: a successful inline observe publishes immediately - a page
+ lookup failure after the observe step must not discard the publication."""
+ monkeypatch.delenv(_FLAG, raising=False)
+ page = _page()
+ ctx = BrowserContext(mode="local")
+ state = make_session_state(context=ctx)
+ state._observed_refs = {"page_key": None, "refs": {"e0": {"ref": "e0", "selector": "#existing"}}}
+ get_page_calls = 0
+
+ async def flaky_get_page(**_kwargs: Any) -> tuple[Any, BrowserContext]:
+ nonlocal get_page_calls
+ get_page_calls += 1
+ if get_page_calls > 2:
+ raise RuntimeError("browser went away")
+ return (page, ctx)
+
+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(side_effect=flaky_get_page))
+ monkeypatch.setattr(browser_ops, "do_observe", AsyncMock(return_value=_result(page.url, "#fresh")))
+
+ async with scoped_session(state):
+ executed = await mcp_browser.skyvern_execute(steps=[{"tool": "observe", "params": {}}])
+ published = session_manager.get_session_ref("e0", page_key=session_manager.page_ref_key(page))
+
+ assert executed["ok"] is True
+ assert published is not None
+ assert published["selector"] == "#fresh"
+
+
+async def test_flag_off_later_failed_observe_keeps_earlier_inline_publication(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """Pre-v2 contract: each successful inline observe publishes as it lands - a later
+ failing observe in the same batch must not discard the earlier publication."""
+ monkeypatch.delenv(_FLAG, raising=False)
+ page = _page()
+ ctx = BrowserContext(mode="local")
+ state = make_session_state(context=ctx)
+ call_count = 0
+
+ async def observe(_page: Any, **_kwargs: Any) -> ObserveResult:
+ nonlocal call_count
+ call_count += 1
+ if call_count == 1:
+ return _result(page.url, "#first")
+ raise RuntimeError("snapshot failed")
+
+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))
+ monkeypatch.setattr(browser_ops, "do_observe", AsyncMock(side_effect=observe))
+
+ async with scoped_session(state):
+ executed = await mcp_browser.skyvern_execute(
+ steps=[
+ {"tool": "observe", "params": {}},
+ {"tool": "observe", "params": {}},
+ ]
+ )
+ published = session_manager.get_session_ref("e0", page_key=session_manager.page_ref_key(page))
+
+ assert executed["ok"] is False
+ assert published is not None
+ assert published["selector"] == "#first"
+
+
+async def test_click_driven_navigation_unscopes_post_batch_observe(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """M2 lock: a click that replaces the document (no navigate step) must not let the
+ old document's aria-controls id scope the new document's auto-observe."""
+ monkeypatch.setenv(_FLAG, "1")
+ page = _page()
+ ctx = BrowserContext(mode="local")
+ state = make_session_state(context=ctx)
+ page.page.query_selector = AsyncMock(return_value=object())
+ page.page.wait_for_timeout = AsyncMock()
+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))
+ monkeypatch.setattr(
+ mcp_browser,
+ "do_observe",
+ AsyncMock(return_value=_result(page.url, "#control", aria_controls="panel")),
+ )
+ observe_calls = 0
+
+ async def observe_after_action(_page: Any, **kwargs: Any) -> ObserveResult:
+ nonlocal observe_calls
+ observe_calls += 1
+ if observe_calls == 1:
+ return _result(page.url, "#control", aria_controls="panel")
+ return _result(page.url, "#new-doc-content", document_id="cdp:doc-2")
+
+ automatic = AsyncMock(side_effect=observe_after_action)
+ monkeypatch.setattr(browser_ops, "do_observe", automatic)
+
+ async def click(**_kwargs: Any) -> dict[str, Any]:
+ # The click handler triggers a same-URL document replacement.
+ page._loader_id = "doc-2"
+ return {"ok": True, "data": None}
+
+ monkeypatch.setattr(mcp_browser, "skyvern_click", AsyncMock(side_effect=click))
+
+ async with scoped_session(state):
+ await mcp_browser.skyvern_observe()
+ executed = await mcp_browser.skyvern_execute(steps=[{"tool": "click", "params": {"ref": "e0"}}])
+
+ assert executed["ok"] is True
+ # The acted element was stamped with the pre-click document; the post-batch
+ # document differs, so the auto-observe must run unscoped even though the
+ # "panel" id would attach on the new document.
+ final_observe = automatic.await_args_list[-1]
+ assert final_observe.kwargs.get("selector") is None
+
+
+async def test_untrusted_marker_stable_observe_batch_skips_auto_observe(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """C2 lock: on a page with no CDP loaderId (page:-sourced marker), a non-mutating
+ observe batch must not pay an invalidate + second full observe every time."""
+ monkeypatch.setenv(_FLAG, "1")
+ page = _page(loader_id=None)
+ ctx = BrowserContext(mode="local")
+ state = make_session_state(context=ctx)
+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))
+ inline = AsyncMock(return_value=_result(page.url, "#control", document_id="page:doc-1"))
+ monkeypatch.setattr(browser_ops, "do_observe", inline)
+
+ async with scoped_session(state):
+ executed = await mcp_browser.skyvern_execute(steps=[{"tool": "observe", "params": {}}])
+
+ assert executed["ok"] is True
+ # Trusted comparison on both sides (None == None): nothing changed, no refresh.
+ assert inline.await_count == 1
+ assert "auto_observe" not in executed["data"]
+
+
+def test_generation_advance_lru_touches_key() -> None:
+ """M3 lock: advancing a generation must reorder the key to most-recent so FIFO
+ eviction cannot drop the busiest session's in-flight reservation."""
+ saved = dict(session_manager._session_ref_generations)
+ session_manager._session_ref_generations.clear()
+ try:
+ keys = [(None, "cloud_session", f"s{i}", None) for i in range(session_manager._SESSION_REF_STORE_MAX)]
+ for key in keys:
+ session_manager._generation_for(key)
+ # Advance the oldest key, then insert one more to trigger eviction.
+ advanced = session_manager._advance_generation_for(keys[0])
+ session_manager._generation_for((None, "cloud_session", "overflow", None))
+ assert session_manager._session_ref_generations.get(keys[0]) == advanced
+ assert keys[1] not in session_manager._session_ref_generations
+ finally:
+ session_manager._session_ref_generations.clear()
+ session_manager._session_ref_generations.update(saved)
+
+
+async def test_navigation_during_aria_attachment_check_discards_scoped_snapshot(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """M2 timing lock: the attachment check awaits between document certification and
+ the scoped observe. A navigation inside that window must discard the scoped
+ snapshot (it certified the wrong document) and rerun unscoped."""
+ monkeypatch.setenv(_FLAG, "1")
+ page = _page()
+ ctx = BrowserContext(mode="local")
+ state = make_session_state(context=ctx)
+ page.page.wait_for_timeout = AsyncMock()
+
+ async def navigating_query_selector(_selector: str) -> object:
+ # The document is replaced while the attachment check is in flight; the
+ # "panel" id also exists on the new document, so attachment succeeds.
+ page._loader_id = "doc-2"
+ return object()
+
+ page.page.query_selector = AsyncMock(side_effect=navigating_query_selector)
+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))
+ monkeypatch.setattr(
+ mcp_browser,
+ "do_observe",
+ AsyncMock(return_value=_result(page.url, "#control", aria_controls="panel")),
+ )
+ observe_calls = 0
+
+ async def observe_current_document(_page: Any, **kwargs: Any) -> ObserveResult:
+ nonlocal observe_calls
+ observe_calls += 1
+ if observe_calls == 1:
+ return _result(page.url, "#control", aria_controls="panel")
+ document_id = f"cdp:{page._loader_id}"
+ selector = "#panel-content" if kwargs.get("selector") else "#new-doc-unscoped"
+ return _result(page.url, selector, document_id=document_id)
+
+ automatic = AsyncMock(side_effect=observe_current_document)
+ monkeypatch.setattr(browser_ops, "do_observe", automatic)
+ monkeypatch.setattr(mcp_browser, "skyvern_type", AsyncMock(return_value={"ok": True, "data": None}))
+
+ async with scoped_session(state):
+ await mcp_browser.skyvern_observe()
+ executed = await mcp_browser.skyvern_execute(steps=[{"tool": "type", "params": {"ref": "e0", "text": "San"}}])
+
+ assert executed["ok"] is True
+ # Call 1: pre-mutation ref refresh. Call 2: scoped observe that certified the
+ # replacement document. Call 3: the discard-and-rerun, unscoped.
+ assert automatic.await_count == 3
+ assert automatic.await_args_list[1].kwargs.get("selector") == '[id="panel"]'
+ assert automatic.await_args_list[2].kwargs.get("selector") is None
+ # Only the unscoped snapshot of the new document is published.
+ assert {element["selector"] for element in state._observe_v2_state.refs.values()} == {"#new-doc-unscoped"}
diff --git a/tests/unit/test_mcp_response_size_cap.py b/tests/unit/test_mcp_response_size_cap.py
index 1e30d8881..3fb2e9937 100644
--- a/tests/unit/test_mcp_response_size_cap.py
+++ b/tests/unit/test_mcp_response_size_cap.py
@@ -14,9 +14,11 @@ from skyvern.cli.core.result import Artifact, BrowserContext
from skyvern.cli.mcp_tools import browser as mcp_browser
from skyvern.cli.mcp_tools import mcp
from skyvern.cli.mcp_tools.response import (
+ MCP_MAX_RESPONSE_BYTES,
MCP_MAX_RESPONSE_CHARS,
size_capped,
truncate_response,
+ truncate_response_bytes,
)
@@ -25,6 +27,18 @@ def test_truncate_response_passes_small_payload_unchanged() -> None:
assert truncate_response(small) is small
+def test_truncate_response_bytes_caps_multibyte_payload() -> None:
+ payload = {"ok": True, "data": {"body": "é" * (MCP_MAX_RESPONSE_BYTES // 2)}}
+
+ result = truncate_response_bytes(payload)
+
+ assert result["_truncated"] is True
+ assert result["_max_bytes"] == MCP_MAX_RESPONSE_BYTES
+ assert result["_original_bytes"] > MCP_MAX_RESPONSE_BYTES
+ assert len(json.dumps(result, ensure_ascii=False).encode()) <= MCP_MAX_RESPONSE_BYTES
+ assert "data" not in result
+
+
def test_truncate_response_wraps_large_payload_with_envelope() -> None:
# Construct a payload larger than the default cap.
big_payload = "x" * (MCP_MAX_RESPONSE_CHARS + 100)
diff --git a/tests/unit/test_mcp_session_lifecycle.py b/tests/unit/test_mcp_session_lifecycle.py
index 6d39a746d..eb63310a3 100644
--- a/tests/unit/test_mcp_session_lifecycle.py
+++ b/tests/unit/test_mcp_session_lifecycle.py
@@ -993,6 +993,49 @@ def test_session_ref_lookup_is_scoped_to_authenticated_organization() -> None:
assert session_manager.get_session_ref("e0", session_id=session_id, page_key=page_key) is None
+def test_observe_ref_generations_remain_monotonic_after_clear() -> None:
+ session_manager.set_current_session(session_manager.SessionState())
+
+ first = session_manager.begin_session_ref_publication()
+ session_manager.clear_session_ref_map(generation=first)
+ second = session_manager.begin_session_ref_publication()
+
+ assert second > first
+
+
+def test_older_publication_cannot_commit_after_newer_reservation() -> None:
+ session_manager.set_current_session(session_manager.SessionState())
+ page_key = (1, None, "https://example.com", None)
+
+ older = session_manager.begin_session_ref_publication()
+ newer = session_manager.begin_session_ref_publication()
+
+ assert not session_manager.replace_session_ref_map({"e0": {"tag": "button"}}, generation=older, page_key=page_key)
+ assert session_manager.replace_session_ref_map({"e1": {"tag": "input"}}, generation=newer, page_key=page_key)
+ assert session_manager.session_ref_generation() > newer
+
+
+def test_mutation_invalidation_clears_v2_refs_but_preserves_monotonic_ids() -> None:
+ state = session_manager.SessionState()
+ state._observe_v2_state.refs = {"e0": {"tag": "button"}}
+ state._observe_v2_state.next_ref = 7
+ session_manager.set_current_session(state)
+ page_key = (1, None, "https://example.com", None)
+ generation = session_manager.begin_session_ref_publication()
+ assert session_manager.replace_session_ref_map(
+ {"e0": {"tag": "button"}},
+ generation=generation,
+ page_key=page_key,
+ )
+
+ invalidated_generation = session_manager.invalidate_session_ref_map()
+
+ assert invalidated_generation > generation
+ assert session_manager.get_session_ref("e0", page_key=page_key) is None
+ assert state._observe_v2_state.refs == {}
+ assert state._observe_v2_state.next_ref == 7
+
+
# ---------------------------------------------------------------------------
# Tests for stateless HTTP mode session creation
# ---------------------------------------------------------------------------
diff --git a/tests/unit/test_mcp_telemetry.py b/tests/unit/test_mcp_telemetry.py
index f837d19a7..c551facb5 100644
--- a/tests/unit/test_mcp_telemetry.py
+++ b/tests/unit/test_mcp_telemetry.py
@@ -1,14 +1,20 @@
from __future__ import annotations
+import asyncio
from collections.abc import Iterator
from types import SimpleNamespace
-from unittest.mock import Mock, patch
+from unittest.mock import AsyncMock, Mock, patch
import pytest
from fastmcp import Client
from fastmcp.server.middleware import MiddlewareContext
from skyvern import analytics
+from skyvern.cli.core.perception_telemetry import (
+ PerceptionSnapshotCategory,
+ track_perception_probe,
+ track_perception_snapshot,
+)
from skyvern.cli.mcp_tools import mcp
from skyvern.cli.mcp_tools.blocks import skyvern_block_schema
from skyvern.cli.mcp_tools.telemetry import (
@@ -237,6 +243,231 @@ async def test_mcp_tool_call_exception_preserves_original_error_when_telemetry_f
await MCPTelemetryMiddleware().on_call_tool(context, call_next)
+async def test_cancelled_perception_snapshot_emits_once_and_restores_context() -> None:
+ events: list[tuple[str, dict]] = []
+ cancelled_context = MiddlewareContext(
+ message=SimpleNamespace(name="skyvern_observe"),
+ fastmcp_context=None,
+ )
+
+ async def cancelled_call(_context: MiddlewareContext[object]) -> object:
+ async with track_perception_snapshot("stale_ref_refresh"):
+ async with track_perception_probe(evaluate_page_scan=True):
+ raise asyncio.CancelledError
+
+ following_context = MiddlewareContext(message=SimpleNamespace(name="skyvern_click"), fastmcp_context=None)
+
+ async def following_call(_context: MiddlewareContext[object]) -> object:
+ return SimpleNamespace(is_error=False, data={"ok": True}, content=[])
+
+ def capture(event: str, *, data: dict, **_kwargs: object) -> None:
+ events.append((event, data))
+
+ clock = iter([0.0, 1.0, 3.0, 4.0, 5.0, 6.0])
+ with (
+ patch("skyvern.cli.mcp_tools.telemetry.time.perf_counter", side_effect=lambda: next(clock)),
+ patch.object(analytics, "capture", side_effect=capture),
+ ):
+ with pytest.raises(asyncio.CancelledError):
+ await MCPTelemetryMiddleware().on_call_tool(cancelled_context, cancelled_call)
+ await MCPTelemetryMiddleware().on_call_tool(following_context, following_call)
+
+ assert len(events) == 2
+ cancelled_event_name, cancelled_payload = events[0]
+ assert cancelled_event_name == "mcp_tool_call"
+ assert cancelled_payload["ok"] is False
+ assert cancelled_payload["error_type"] == "CancelledError"
+ assert cancelled_payload["top_level_mcp_calls"] == 1
+ assert cancelled_payload["perception_snapshots"] == 1
+ assert cancelled_payload["model_visible_observe_results"] == 0
+ assert cancelled_payload["automatic_observe_snapshots"] == 0
+ assert cancelled_payload["stale_ref_refresh_snapshots"] == 1
+ assert (
+ cancelled_payload["model_visible_observe_results"]
+ + cancelled_payload["automatic_observe_snapshots"]
+ + cancelled_payload["stale_ref_refresh_snapshots"]
+ == cancelled_payload["perception_snapshots"]
+ )
+ assert cancelled_payload["failed_perception_probes"] == 1
+ assert cancelled_payload["evaluate_page_scans"] == 1
+ assert cancelled_payload["browser_perception_wall_ms"] == 2000
+ assert isinstance(cancelled_payload["browser_perception_wall_ms"], int)
+ assert "response_bytes" not in cancelled_payload
+
+ following_event_name, following_payload = events[1]
+ assert following_event_name == "mcp_tool_call"
+ assert following_payload["ok"] is True
+ assert following_payload["top_level_mcp_calls"] == 1
+ assert following_payload["perception_snapshots"] == 0
+ assert following_payload["failed_perception_probes"] == 0
+ assert following_payload["evaluate_page_scans"] == 0
+ assert following_payload["browser_perception_wall_ms"] == 0
+
+
+async def test_cancelled_tool_call_propagates_when_telemetry_fails() -> None:
+ context = MiddlewareContext(message=SimpleNamespace(name="skyvern_observe"), fastmcp_context=None)
+ cancellation = asyncio.CancelledError()
+
+ async def call_next(_context: MiddlewareContext[object]) -> object:
+ async with track_perception_snapshot("automatic"):
+ raise cancellation
+
+ with (
+ patch.object(analytics, "capture", side_effect=RuntimeError("telemetry down")),
+ pytest.raises(asyncio.CancelledError) as exc_info,
+ ):
+ await MCPTelemetryMiddleware().on_call_tool(context, call_next)
+
+ assert exc_info.value is cancellation
+
+
+async def test_perception_counters_emit_on_exception() -> None:
+ events: list[dict] = []
+ context = MiddlewareContext(message=SimpleNamespace(name="skyvern_observe"), fastmcp_context=None)
+
+ async def call_next(_context: MiddlewareContext[object]) -> object:
+ async with track_perception_snapshot("model_visible"):
+ raise RuntimeError("observe failed")
+
+ with (
+ patch.object(analytics, "capture", side_effect=lambda _event, *, data, **_kwargs: events.append(data)),
+ pytest.raises(RuntimeError, match="observe failed"),
+ ):
+ await MCPTelemetryMiddleware().on_call_tool(context, call_next)
+
+ assert events[0]["ok"] is False
+ assert events[0]["perception_snapshots"] == 1
+ assert events[0]["model_visible_observe_results"] == 1
+ assert events[0]["automatic_observe_snapshots"] == 0
+ assert events[0]["stale_ref_refresh_snapshots"] == 0
+ assert events[0]["failed_perception_probes"] == 1
+ assert isinstance(events[0]["browser_perception_wall_ms"], int)
+
+
+async def test_swallowed_evaluate_scan_failure_is_accounted() -> None:
+ from skyvern.cli.core.browser_ops import _get_dom_observe_elements
+
+ events: list[dict] = []
+ context = MiddlewareContext(message=SimpleNamespace(name="skyvern_observe"), fastmcp_context=None)
+ page = SimpleNamespace(evaluate=AsyncMock(side_effect=RuntimeError("scan failed")))
+
+ async def call_next(_context: MiddlewareContext[object]) -> object:
+ assert await _get_dom_observe_elements(page) == []
+ return SimpleNamespace(is_error=False, data={"ok": True}, content=[])
+
+ with patch.object(analytics, "capture", side_effect=lambda _event, *, data, **_kwargs: events.append(data)):
+ await MCPTelemetryMiddleware().on_call_tool(context, call_next)
+
+ assert events[0]["perception_snapshots"] == 0
+ assert events[0]["evaluate_page_scans"] == 1
+ assert events[0]["failed_perception_probes"] == 1
+ assert isinstance(events[0]["browser_perception_wall_ms"], int)
+
+
+async def test_document_id_cdp_failures_and_fallback_are_accounted() -> None:
+ from skyvern.cli.core.browser_ops import get_observe_document_id
+
+ events: list[dict] = []
+ context = MiddlewareContext(message=SimpleNamespace(name="skyvern_observe"), fastmcp_context=None)
+ cached_cdp = SimpleNamespace(send=AsyncMock(side_effect=RuntimeError("stale CDP session")))
+ new_cdp_session = AsyncMock(side_effect=RuntimeError("CDP attach failed"))
+ raw_page = SimpleNamespace(
+ context=SimpleNamespace(new_cdp_session=new_cdp_session),
+ _skyvern_observe_cdp_session=cached_cdp,
+ )
+ page = SimpleNamespace(
+ page=raw_page,
+ _working_frame=None,
+ evaluate=AsyncMock(return_value="doc-1"),
+ )
+
+ async def call_next(_context: MiddlewareContext[object]) -> object:
+ assert await get_observe_document_id(page) == "page:doc-1"
+ return SimpleNamespace(is_error=False, data={"ok": True}, content=[])
+
+ clock = iter([0.0, 1.0, 2.0, 3.0, 5.0, 6.0, 9.0, 10.0])
+ with (
+ patch("skyvern.cli.mcp_tools.telemetry.time.perf_counter", side_effect=lambda: next(clock)),
+ patch.object(analytics, "capture", side_effect=lambda _event, *, data, **_kwargs: events.append(data)),
+ ):
+ await MCPTelemetryMiddleware().on_call_tool(context, call_next)
+
+ cached_cdp.send.assert_awaited_once_with("Page.getFrameTree")
+ new_cdp_session.assert_awaited_once_with(raw_page)
+ page.evaluate.assert_awaited_once()
+ assert events[0]["perception_snapshots"] == 0
+ assert events[0]["failed_perception_probes"] == 2
+ assert events[0]["evaluate_page_scans"] == 0
+ assert events[0]["browser_perception_wall_ms"] == 6000
+
+
+async def test_nested_evaluate_scan_does_not_double_count_snapshot_wall_time() -> None:
+ events: list[dict] = []
+ context = MiddlewareContext(message=SimpleNamespace(name="skyvern_observe"), fastmcp_context=None)
+
+ async def call_next(_context: MiddlewareContext[object]) -> object:
+ async with track_perception_snapshot("model_visible"):
+ async with track_perception_probe(evaluate_page_scan=True):
+ pass
+ return SimpleNamespace(is_error=False, data={"ok": True}, content=[])
+
+ clock = iter([0.0, 0.001, 0.008, 0.010])
+ with (
+ patch("skyvern.cli.mcp_tools.telemetry.time.perf_counter", side_effect=lambda: next(clock)),
+ patch.object(analytics, "capture", side_effect=lambda _event, *, data, **_kwargs: events.append(data)),
+ ):
+ await MCPTelemetryMiddleware().on_call_tool(context, call_next)
+
+ assert events[0]["perception_snapshots"] == 1
+ assert events[0]["evaluate_page_scans"] == 1
+ assert events[0]["browser_perception_wall_ms"] == 7
+
+
+async def test_perception_counters_are_isolated_across_concurrent_calls() -> None:
+ events: list[dict] = []
+ both_started = asyncio.Event()
+ started = 0
+
+ async def run(tool: str, category: PerceptionSnapshotCategory) -> None:
+ context = MiddlewareContext(message=SimpleNamespace(name=tool), fastmcp_context=None)
+
+ async def call_next(_context: MiddlewareContext[object]) -> object:
+ nonlocal started
+ started += 1
+ if started == 2:
+ both_started.set()
+ await both_started.wait()
+ async with track_perception_snapshot(category):
+ if category == "automatic":
+ async with track_perception_probe(evaluate_page_scan=True):
+ pass
+ return SimpleNamespace(is_error=False, data={"ok": True}, content=[])
+
+ await MCPTelemetryMiddleware().on_call_tool(context, call_next)
+
+ with patch.object(analytics, "capture", side_effect=lambda _event, *, data, **_kwargs: events.append(data)):
+ await asyncio.gather(run("explicit", "model_visible"), run("bundled", "automatic"))
+
+ by_tool = {event["tool"]: event for event in events}
+ assert by_tool["explicit"]["top_level_mcp_calls"] == 1
+ assert by_tool["explicit"]["perception_snapshots"] == 1
+ assert by_tool["explicit"]["model_visible_observe_results"] == 1
+ assert by_tool["explicit"]["automatic_observe_snapshots"] == 0
+ assert by_tool["explicit"]["evaluate_page_scans"] == 0
+ assert by_tool["bundled"]["top_level_mcp_calls"] == 1
+ assert by_tool["bundled"]["perception_snapshots"] == 1
+ assert by_tool["bundled"]["model_visible_observe_results"] == 0
+ assert by_tool["bundled"]["automatic_observe_snapshots"] == 1
+ assert by_tool["bundled"]["evaluate_page_scans"] == 1
+
+
@pytest.mark.asyncio
async def test_list_tools_emits_protocol_request_telemetry() -> None:
events: list[tuple[str, dict | None, str | None, str | None, str | None]] = []